fix: slow and control Graphs backfill

src/controllers/webhook.controller.ts

@@ -5,6 +5,48 @@ import axios from 'axios';
 import fs from 'fs';
 import path from 'path';
 
+type CachedOrderDetails = {
+    expiresAt: number;
+    fullOrderDetails: any;
+    statusProcessamento: string;
+    whatsappVendedor: string;
+};
+
+const orderDetailsCache = new Map<string, CachedOrderDetails>();
+const configuredCacheTtlMs = Number(process.env.TINY_ORDER_DETAILS_CACHE_TTL_MS || 120000);
+const ORDER_DETAILS_CACHE_TTL_MS = Number.isFinite(configuredCacheTtlMs) ? configuredCacheTtlMs : 120000;
+
+function pruneExpiredOrderDetailsCache() {
+    const now = Date.now();
+    for (const [orderId, cached] of orderDetailsCache.entries()) {
+        if (now > cached.expiresAt) {
+            orderDetailsCache.delete(orderId);
+        }
+    }
+}
+
+function getCachedOrderDetails(orderId: string) {
+    const cached = orderDetailsCache.get(orderId);
+    if (!cached) return null;
+
+    if (Date.now() > cached.expiresAt) {
+        orderDetailsCache.delete(orderId);
+        return null;
+    }
+
+    return cached;
+}
+
+function setCachedOrderDetails(orderId: string, cached: Omit<CachedOrderDetails, 'expiresAt'>) {
+    if (ORDER_DETAILS_CACHE_TTL_MS <= 0) return;
+
+    pruneExpiredOrderDetailsCache();
+    orderDetailsCache.set(orderId, {
+        ...cached,
+        expiresAt: Date.now() + ORDER_DETAILS_CACHE_TTL_MS
+    });
+}
+
 export const handleTinyOrderUpdate = async (req: Request, res: Response): Promise<void> => {
   try {
     // 1. Security Check: Verify token from Tiny
@@ -42,7 +84,13 @@ export const handleTinyOrderUpdate = async (req: Request, res: Response): Promis
     let statusProcessamento = "";
     let whatsappVendedor = "";
 
-    if (tinyApiToken) {
+    const cachedDetails = getCachedOrderDetails(String(orderId));
+    if (cachedDetails) {
+        fullOrderDetails = cachedDetails.fullOrderDetails;
+        statusProcessamento = cachedDetails.statusProcessamento;
+        whatsappVendedor = cachedDetails.whatsappVendedor;
+        console.log(`Using cached order details for Order ID: ${orderId}.`);
+    } else if (tinyApiToken) {
         try {
             console.log(`Fetching full details for Order ID: ${orderId} from Tiny API...`);
             const params = new URLSearchParams();
@@ -61,7 +109,7 @@ export const handleTinyOrderUpdate = async (req: Request, res: Response): Promis
 
                 // OPTION B: Fetch Vendor's WhatsApp
                 const idVendedor = fullOrderDetails.id_vendedor;
-                if (idVendedor && idVendedor !== "0") {
+                if (process.env.TINY_FETCH_SELLER_DETAILS === 'true' && idVendedor && idVendedor !== "0") {
                     console.log(`Fetching seller details for Seller ID: ${idVendedor}...`);
                     const vendorParams = new URLSearchParams();
                     vendorParams.append('token', tinyApiToken);
@@ -80,6 +128,12 @@ export const handleTinyOrderUpdate = async (req: Request, res: Response): Promis
                         console.error('Failed to fetch seller details:', JSON.stringify(vendorResponse.data?.retorno?.erros || 'Unknown API error'));
                     }
                 }
+
+                setCachedOrderDetails(String(orderId), {
+                    fullOrderDetails,
+                    statusProcessamento,
+                    whatsappVendedor
+                });
             } else {
                 console.error('Tiny API returned an error:', apiResponse.data?.retorno?.erros || 'Unknown error');
             }

src/index.ts

@@ -3,6 +3,7 @@ import dotenv from 'dotenv';
 import webhookRoutes from './routes/webhook.route';
 import { spawn } from 'child_process';
 import path from 'path';
+import fs from 'fs';
 
 dotenv.config();
 
@@ -50,6 +51,7 @@ app.get('/api/trigger-graphs-backfill', (req: Request, res: Response) => {
 
   const scriptPath = path.join(__dirname, 'scripts', 'backfill-graphs.js');
   const env = { ...process.env };
+  const stopFile = env.GRAPHS_BACKFILL_STOP_FILE || path.join(process.cwd(), 'graphs_backfill_stop.json');
 
   const optionalEnvMap: Record<string, string> = {
     start: 'GRAPHS_BACKFILL_START_DATE',
@@ -57,7 +59,10 @@ app.get('/api/trigger-graphs-backfill', (req: Request, res: Response) => {
     dryRun: 'GRAPHS_BACKFILL_DRY_RUN',
     maxOrders: 'GRAPHS_BACKFILL_MAX_ORDERS',
     maxDays: 'GRAPHS_BACKFILL_MAX_DAYS',
-    contactFallback: 'GRAPHS_BACKFILL_CONTACT_FALLBACK'
+    contactFallback: 'GRAPHS_BACKFILL_CONTACT_FALLBACK',
+    orderDelayMs: 'GRAPHS_BACKFILL_ORDER_DELAY_MS',
+    blockDelayMs: 'GRAPHS_BACKFILL_TINY_BLOCK_DELAY_MS',
+    maxTinyBlockRetries: 'GRAPHS_BACKFILL_MAX_TINY_BLOCK_RETRIES'
   };
 
   for (const [queryKey, envKey] of Object.entries(optionalEnvMap)) {
@@ -67,6 +72,10 @@ app.get('/api/trigger-graphs-backfill', (req: Request, res: Response) => {
     }
   }
 
+  if (fs.existsSync(stopFile)) {
+    fs.rmSync(stopFile, { force: true });
+  }
+
   const child = spawn('node', [scriptPath], {
     detached: true,
     stdio: 'inherit',
@@ -85,11 +94,61 @@ app.get('/api/trigger-graphs-backfill', (req: Request, res: Response) => {
       dryRun: env.GRAPHS_BACKFILL_DRY_RUN || 'false',
       maxOrders: env.GRAPHS_BACKFILL_MAX_ORDERS || null,
       maxDays: env.GRAPHS_BACKFILL_MAX_DAYS || null,
-      contactFallback: env.GRAPHS_BACKFILL_CONTACT_FALLBACK || 'true'
+      contactFallback: env.GRAPHS_BACKFILL_CONTACT_FALLBACK || 'true',
+      orderDelayMs: env.GRAPHS_BACKFILL_ORDER_DELAY_MS || '6000',
+      blockDelayMs: env.GRAPHS_BACKFILL_TINY_BLOCK_DELAY_MS || '600000',
+      maxTinyBlockRetries: env.GRAPHS_BACKFILL_MAX_TINY_BLOCK_RETRIES || null,
+      graphsApiUrlConfigured: Boolean(env.GRAPHS_API_URL || env.NEXSTAR_GRAPHS_API_URL),
+      graphsApiKeyConfigured: Boolean(env.GRAPHS_API_KEY || env.NEXSTAR_GRAPHS_API_KEY)
     }
   });
 });
 
+app.get('/api/graphs-backfill-status', (req: Request, res: Response) => {
+  const expectedToken = process.env.TINY_WEBHOOK_SECRET;
+  if (expectedToken && req.query.token !== expectedToken) {
+    res.status(401).json({ error: 'Unauthorized' });
+    return;
+  }
+
+  const stateFile = process.env.GRAPHS_BACKFILL_STATE_FILE || path.join(process.cwd(), 'graphs_backfill_state.json');
+  const stopFile = process.env.GRAPHS_BACKFILL_STOP_FILE || path.join(process.cwd(), 'graphs_backfill_stop.json');
+  let state = null;
+
+  if (fs.existsSync(stateFile)) {
+    try {
+      state = JSON.parse(fs.readFileSync(stateFile, 'utf-8'));
+    } catch (error: any) {
+      res.status(500).json({ error: `Could not read backfill state: ${error.message}` });
+      return;
+    }
+  }
+
+  res.status(200).json({
+    status: state?.status || 'unknown',
+    stopRequested: fs.existsSync(stopFile),
+    state
+  });
+});
+
+app.get('/api/stop-graphs-backfill', (req: Request, res: Response) => {
+  const expectedToken = process.env.TINY_WEBHOOK_SECRET;
+  if (expectedToken && req.query.token !== expectedToken) {
+    res.status(401).json({ error: 'Unauthorized' });
+    return;
+  }
+
+  const stopFile = process.env.GRAPHS_BACKFILL_STOP_FILE || path.join(process.cwd(), 'graphs_backfill_stop.json');
+  fs.writeFileSync(stopFile, JSON.stringify({
+    requestedAt: new Date().toISOString()
+  }, null, 2));
+
+  res.status(200).json({
+    status: 'STOP_REQUESTED',
+    message: 'Graphs backfill will stop after the current wait/request finishes.'
+  });
+});
+
 // Hidden endpoint to download the stock CSV log directly from the browser
 app.get('/api/stock-logs/download', (req: Request, res: Response) => {
   const expectedToken = process.env.TINY_WEBHOOK_SECRET;

src/scripts/backfill-graphs.ts

@@ -44,6 +44,12 @@ type ContactInfo = {
 type BackfillState = {
     nextDate: string;
     lastCompletedDate?: string;
+    status?: 'running' | 'stopped' | 'completed';
+    currentDate?: string;
+    lastOrderId?: string;
+    processedDays?: number;
+    processedOrders?: number;
+    lastMessage?: string;
     updatedAt?: string;
 };
 
@@ -55,17 +61,31 @@ const CONTACT_FALLBACK = process.env.GRAPHS_BACKFILL_CONTACT_FALLBACK !== 'false
 const NORMALIZE_PHONE = process.env.GRAPHS_BACKFILL_NORMALIZE_PHONE !== 'false';
 const START_DATE = process.env.GRAPHS_BACKFILL_START_DATE || '13/02/2025';
 const END_DATE = process.env.GRAPHS_BACKFILL_END_DATE || getTodayDate();
-const MAX_DAYS = Number(process.env.GRAPHS_BACKFILL_MAX_DAYS || 0);
-const MAX_ORDERS = Number(process.env.GRAPHS_BACKFILL_MAX_ORDERS || 0);
-const TINY_SEARCH_DELAY_MS = Number(process.env.TINY_SEARCH_DELAY_MS || 1000);
-const TINY_DETAIL_DELAY_MS = Number(process.env.TINY_DETAIL_DELAY_MS || 1500);
-const TINY_CONTACT_DELAY_MS = Number(process.env.TINY_CONTACT_DELAY_MS || TINY_DETAIL_DELAY_MS);
+const numberEnv = (name: string, fallback: number) => {
+    const value = Number(process.env[name] ?? fallback);
+    return Number.isFinite(value) && value >= 0 ? value : fallback;
+};
+const MAX_DAYS = numberEnv('GRAPHS_BACKFILL_MAX_DAYS', 0);
+const MAX_ORDERS = numberEnv('GRAPHS_BACKFILL_MAX_ORDERS', 0);
+const TINY_SEARCH_DELAY_MS = numberEnv('TINY_SEARCH_DELAY_MS', 1000);
+const TINY_DETAIL_DELAY_MS = numberEnv('TINY_DETAIL_DELAY_MS', 1500);
+const TINY_CONTACT_DELAY_MS = numberEnv('TINY_CONTACT_DELAY_MS', TINY_DETAIL_DELAY_MS);
+const ORDER_DELAY_MS = numberEnv('GRAPHS_BACKFILL_ORDER_DELAY_MS', 6000);
+const TINY_BLOCK_DELAY_MS = numberEnv('GRAPHS_BACKFILL_TINY_BLOCK_DELAY_MS', 600000);
+const MAX_TINY_BLOCK_RETRIES = numberEnv('GRAPHS_BACKFILL_MAX_TINY_BLOCK_RETRIES', 0);
 const STATE_FILE = process.env.GRAPHS_BACKFILL_STATE_FILE || path.join(process.cwd(), 'graphs_backfill_state.json');
+const STOP_FILE = process.env.GRAPHS_BACKFILL_STOP_FILE || path.join(process.cwd(), 'graphs_backfill_stop.json');
 
 const contactCache = new Map<string, ContactInfo | null>();
 
 const sleep = (ms: number) => new Promise(resolve => setTimeout(resolve, ms));
 
+class BackfillStoppedError extends Error {
+    constructor() {
+        super('Graphs backfill stopped by request.');
+    }
+}
+
 function getTodayDate() {
     const date = new Date();
     return formatDate(date);
@@ -130,6 +150,26 @@ function saveState(state: BackfillState) {
     }, null, 2));
 }
 
+function assertNotStopped() {
+    if (fs.existsSync(STOP_FILE)) {
+        throw new BackfillStoppedError();
+    }
+}
+
+async function sleepWithStop(ms: number, reason: string) {
+    if (ms <= 0) return;
+
+    const startedAt = Date.now();
+    let remaining = ms;
+    console.log(`[wait] Waiting ${ms}ms before ${reason}.`);
+
+    while (remaining > 0) {
+        assertNotStopped();
+        await sleep(Math.min(remaining, 1000));
+        remaining = ms - (Date.now() - startedAt);
+    }
+}
+
 function normalizeNumber(value: unknown) {
     if (typeof value === 'number') return Number.isFinite(value) ? value : 0;
     if (value === undefined || value === null) return 0;
@@ -167,12 +207,46 @@ function pickFirst(...values: unknown[]) {
     return '';
 }
 
-async function tinyPost(servicePhp: string, params: URLSearchParams) {
-    return axios.post(`https://api.tiny.com.br/api2/${servicePhp}`, params, {
-        headers: { 'Content-Type': 'application/x-www-form-urlencoded' }
+function getTinyErrors(data: any) {
+    const errors = data?.retorno?.erros;
+    if (!errors) return [];
+    return Array.isArray(errors) ? errors : [errors];
+}
+
+function isTinyApiBlocked(data: any) {
+    return getTinyErrors(data).some((entry: any) => {
+        const message = String(entry?.erro || entry || '').toLowerCase();
+        return message.includes('api bloqueada') || message.includes('excedido o número de acessos');
     });
 }
 
+async function tinyPost(servicePhp: string, params: URLSearchParams) {
+    let blockedAttempts = 0;
+
+    while (true) {
+        assertNotStopped();
+
+        const response = await axios.post(`https://api.tiny.com.br/api2/${servicePhp}`, params, {
+            headers: { 'Content-Type': 'application/x-www-form-urlencoded' }
+        });
+
+        if (!isTinyApiBlocked(response.data)) {
+            return response;
+        }
+
+        blockedAttempts += 1;
+        if (MAX_TINY_BLOCK_RETRIES && blockedAttempts > MAX_TINY_BLOCK_RETRIES) {
+            return response;
+        }
+
+        const retryLabel = MAX_TINY_BLOCK_RETRIES
+            ? `${blockedAttempts}/${MAX_TINY_BLOCK_RETRIES}`
+            : String(blockedAttempts);
+        console.warn(`[Tiny] API blocked on ${servicePhp}. Pausing backfill for ${TINY_BLOCK_DELAY_MS}ms before retry ${retryLabel}.`);
+        await sleepWithStop(TINY_BLOCK_DELAY_MS, 'retrying Tiny API after block');
+    }
+}
+
 async function fetchOrdersForDate(dateStr: string, page = 1): Promise<TinyOrderSummary[]> {
     console.log(`[Tiny] Searching orders for ${dateStr}, page ${page}`);
 
@@ -184,7 +258,7 @@ async function fetchOrdersForDate(dateStr: string, page = 1): Promise<TinyOrderS
     params.append('pagina', String(page));
 
     const response = await tinyPost('pedidos.pesquisa.php', params);
-    await sleep(TINY_SEARCH_DELAY_MS);
+    await sleepWithStop(TINY_SEARCH_DELAY_MS, 'next Tiny search request');
 
     const retorno = response.data?.retorno;
     if (retorno?.status === 'OK') {
@@ -214,7 +288,7 @@ async function fetchOrderDetails(orderId: string): Promise<TinyOrderDetails | nu
 
     try {
         const response = await tinyPost('pedido.obter.php', params);
-        await sleep(TINY_DETAIL_DELAY_MS);
+        await sleepWithStop(TINY_DETAIL_DELAY_MS, 'next Tiny order detail request');
 
         if (response.data?.retorno?.status === 'OK') {
             return {
@@ -226,8 +300,9 @@ async function fetchOrderDetails(orderId: string): Promise<TinyOrderDetails | nu
         console.error(`[Tiny] Failed to fetch order ${orderId}:`, JSON.stringify(response.data?.retorno?.erros || 'Unknown error'));
         return null;
     } catch (error: any) {
+        if (error instanceof BackfillStoppedError) throw error;
         console.error(`[Tiny] Request failed for order ${orderId}: ${error.message}`);
-        await sleep(5000);
+        await sleepWithStop(5000, 'continuing after Tiny request failure');
         return null;
     }
 }
@@ -250,7 +325,7 @@ async function fetchContactInfo(cliente: any): Promise<ContactInfo | null> {
         if (cpfCnpj) searchParams.append('cpf_cnpj', cpfCnpj);
 
         const searchResponse = await tinyPost('contatos.pesquisa.php', searchParams);
-        await sleep(TINY_CONTACT_DELAY_MS);
+        await sleepWithStop(TINY_CONTACT_DELAY_MS, 'next Tiny contact request');
 
         const contatoResumo = searchResponse.data?.retorno?.contatos?.[0]?.contato;
         const contatoId = contatoResumo?.id;
@@ -265,7 +340,7 @@ async function fetchContactInfo(cliente: any): Promise<ContactInfo | null> {
         detailParams.append('id', String(contatoId));
 
         const detailResponse = await tinyPost('contato.obter.php', detailParams);
-        await sleep(TINY_CONTACT_DELAY_MS);
+        await sleepWithStop(TINY_CONTACT_DELAY_MS, 'next Tiny contact request');
 
         if (detailResponse.data?.retorno?.status !== 'OK') {
             contactCache.set(cacheKey, null);
@@ -281,6 +356,7 @@ async function fetchContactInfo(cliente: any): Promise<ContactInfo | null> {
         contactCache.set(cacheKey, info);
         return info;
     } catch (error: any) {
+        if (error instanceof BackfillStoppedError) throw error;
         console.error(`[Tiny] Contact fallback failed for ${name || cpfCnpj}: ${error.message}`);
         contactCache.set(cacheKey, null);
         return null;
@@ -372,16 +448,32 @@ async function runBackfill() {
 
     const state = loadState();
     let currentDate = state.nextDate;
+    let lastCompletedDate = state.lastCompletedDate;
     let processedDays = 0;
     let processedOrders = 0;
+    const orderPace = ORDER_DELAY_MS ? `~${Math.floor(60000 / ORDER_DELAY_MS)}/minute` : 'unlimited';
 
     console.log(`[config] Start date: ${currentDate}`);
     console.log(`[config] End date: ${END_DATE}`);
     console.log(`[config] Dry run: ${DRY_RUN ? 'yes' : 'no'}`);
     console.log(`[config] Contact fallback: ${CONTACT_FALLBACK ? 'yes' : 'no'}`);
+    console.log(`[config] Order pace: 1 order every ${ORDER_DELAY_MS}ms (${orderPace})`);
+    console.log(`[config] Tiny block pause: ${TINY_BLOCK_DELAY_MS}ms`);
     console.log(`[config] State file: ${STATE_FILE}`);
+    console.log(`[config] Stop file: ${STOP_FILE}`);
+
+    saveState({
+        ...state,
+        status: 'running',
+        currentDate,
+        processedDays,
+        processedOrders,
+        lastMessage: 'Backfill started'
+    });
 
     while (!isAfter(currentDate, END_DATE)) {
+        assertNotStopped();
+
         if (MAX_DAYS && processedDays >= MAX_DAYS) {
             console.log(`[limit] Stopped after ${MAX_DAYS} day(s).`);
             break;
@@ -392,8 +484,19 @@ async function runBackfill() {
         console.log(`[Tiny] Found ${summaries.length} order(s) for ${currentDate}`);
 
         for (const summary of summaries) {
+            assertNotStopped();
+
             if (MAX_ORDERS && processedOrders >= MAX_ORDERS) {
                 console.log(`[limit] Stopped after ${MAX_ORDERS} order(s).`);
+                saveState({
+                    nextDate: currentDate,
+                    lastCompletedDate,
+                    status: 'completed',
+                    currentDate,
+                    processedDays,
+                    processedOrders,
+                    lastMessage: `Stopped after ${MAX_ORDERS} order(s)`
+                });
                 return;
             }
 
@@ -409,22 +512,63 @@ async function runBackfill() {
             const rows = await buildGraphsRows(details, summary);
             await sendRowsToGraphs(rows, orderId);
             processedOrders += 1;
+
+            saveState({
+                nextDate: currentDate,
+                lastCompletedDate,
+                status: 'running',
+                currentDate,
+                lastOrderId: orderId,
+                processedDays,
+                processedOrders,
+                lastMessage: `Processed order ${orderId}`
+            });
+
+            if (!MAX_ORDERS || processedOrders < MAX_ORDERS) {
+                await sleepWithStop(ORDER_DELAY_MS, 'next order');
+            }
         }
 
         const completedDate = currentDate;
         currentDate = getNextDate(currentDate);
+        lastCompletedDate = completedDate;
         processedDays += 1;
         saveState({
             nextDate: currentDate,
-            lastCompletedDate: completedDate
+            lastCompletedDate,
+            status: 'running',
+            currentDate,
+            processedDays,
+            processedOrders,
+            lastMessage: `Completed ${completedDate}`
         });
         console.log(`[state] Completed ${completedDate}. Next date: ${currentDate}`);
     }
 
+    saveState({
+        nextDate: currentDate,
+        lastCompletedDate,
+        status: 'completed',
+        currentDate,
+        processedDays,
+        processedOrders,
+        lastMessage: 'Backfill completed'
+    });
     console.log(`\nDone. Processed ${processedOrders} order(s) across ${processedDays} day(s).`);
 }
 
 runBackfill().catch((error: any) => {
+    if (error instanceof BackfillStoppedError) {
+        console.log('Backfill stopped by request.');
+        const state = loadState();
+        saveState({
+            ...state,
+            status: 'stopped',
+            lastMessage: 'Backfill stopped by request'
+        });
+        process.exit(0);
+    }
+
     console.error(`Backfill failed: ${error.message}`);
     process.exit(1);
 });