Improve graphs backfill resilience

src/index.ts

@@ -35,6 +35,14 @@ app.get('/api/trigger-backfill', (req: Request, res: Response) => {
     stdio: 'inherit' // Inherit allows the logs to show up in Portainer's log viewer!
   });
 
+  child.on('error', (error) => {
+    console.error(`[server]: Backfill process failed to start: ${error.message}`);
+  });
+
+  child.on('exit', (code, signal) => {
+    console.log(`[server]: Backfill process exited. code=${code ?? 'null'} signal=${signal ?? 'null'}`);
+  });
+  
   child.unref();
 
   console.log('[server]: Backfill script manually triggered via HTTP endpoint.');
@@ -60,6 +68,7 @@ app.get('/api/trigger-graphs-backfill', (req: Request, res: Response) => {
     maxOrders: 'GRAPHS_BACKFILL_MAX_ORDERS',
     maxDays: 'GRAPHS_BACKFILL_MAX_DAYS',
     contactFallback: 'GRAPHS_BACKFILL_CONTACT_FALLBACK',
+    sellerFallback: 'GRAPHS_BACKFILL_SELLER_FALLBACK',
     tinyRequestDelayMs: 'GRAPHS_BACKFILL_TINY_REQUEST_DELAY_MS',
     orderDelayMs: 'GRAPHS_BACKFILL_ORDER_DELAY_MS',
     blockDelayMs: 'GRAPHS_BACKFILL_TINY_BLOCK_DELAY_MS',
@@ -83,6 +92,14 @@ app.get('/api/trigger-graphs-backfill', (req: Request, res: Response) => {
     env
   });
 
+  child.on('error', (error) => {
+    console.error(`[server]: Graphs backfill process failed to start: ${error.message}`);
+  });
+
+  child.on('exit', (code, signal) => {
+    console.log(`[server]: Graphs backfill process exited. code=${code ?? 'null'} signal=${signal ?? 'null'}`);
+  });
+
   child.unref();
 
   console.log('[server]: Graphs backfill script manually triggered via HTTP endpoint.');
@@ -96,6 +113,7 @@ app.get('/api/trigger-graphs-backfill', (req: Request, res: Response) => {
       maxOrders: env.GRAPHS_BACKFILL_MAX_ORDERS || null,
       maxDays: env.GRAPHS_BACKFILL_MAX_DAYS || null,
       contactFallback: env.GRAPHS_BACKFILL_CONTACT_FALLBACK || 'true',
+      sellerFallback: env.GRAPHS_BACKFILL_SELLER_FALLBACK || 'true',
       tinyRequestDelayMs: env.GRAPHS_BACKFILL_TINY_REQUEST_DELAY_MS || '6000',
       orderDelayMs: env.GRAPHS_BACKFILL_ORDER_DELAY_MS || '0',
       blockDelayMs: env.GRAPHS_BACKFILL_TINY_BLOCK_DELAY_MS || '600000',

src/scripts/backfill-graphs.ts

@@ -41,14 +41,20 @@ type ContactInfo = {
     phone: string;
 };
 
+type SellerInfo = {
+    name: string;
+};
+
 type BackfillState = {
     nextDate: string;
     lastCompletedDate?: string;
     status?: 'running' | 'stopped' | 'completed';
     currentDate?: string;
     lastOrderId?: string;
+    lastFailedOrderId?: string;
     processedDays?: number;
     processedOrders?: number;
+    failedOrders?: number;
     lastMessage?: string;
     updatedAt?: string;
 };
@@ -58,6 +64,7 @@ const GRAPHS_API_URL = process.env.GRAPHS_API_URL || process.env.NEXSTAR_GRAPHS_
 const GRAPHS_API_KEY = process.env.GRAPHS_API_KEY || process.env.NEXSTAR_GRAPHS_API_KEY;
 const DRY_RUN = process.env.GRAPHS_BACKFILL_DRY_RUN === 'true';
 const CONTACT_FALLBACK = process.env.GRAPHS_BACKFILL_CONTACT_FALLBACK !== 'false';
+const SELLER_FALLBACK = process.env.GRAPHS_BACKFILL_SELLER_FALLBACK !== 'false';
 const NORMALIZE_PHONE = process.env.GRAPHS_BACKFILL_NORMALIZE_PHONE !== 'false';
 const START_DATE = process.env.GRAPHS_BACKFILL_START_DATE || '13/02/2025';
 const END_DATE = process.env.GRAPHS_BACKFILL_END_DATE || getTodayDate();
@@ -77,10 +84,13 @@ const TINY_CONTACT_DELAY_MS = numberEnv('TINY_CONTACT_DELAY_MS', TINY_DETAIL_DEL
 const ORDER_DELAY_MS = numberEnv('GRAPHS_BACKFILL_ORDER_DELAY_MS', 0);
 const TINY_BLOCK_DELAY_MS = numberEnv('GRAPHS_BACKFILL_TINY_BLOCK_DELAY_MS', 600000);
 const MAX_TINY_BLOCK_RETRIES = numberEnv('GRAPHS_BACKFILL_MAX_TINY_BLOCK_RETRIES', 0);
+const GRAPHS_API_MAX_RETRIES = numberEnv('GRAPHS_BACKFILL_GRAPHS_API_MAX_RETRIES', 3);
+const GRAPHS_API_RETRY_DELAY_MS = numberEnv('GRAPHS_BACKFILL_GRAPHS_API_RETRY_DELAY_MS', 30000);
 const STATE_FILE = process.env.GRAPHS_BACKFILL_STATE_FILE || path.join(process.cwd(), 'graphs_backfill_state.json');
 const STOP_FILE = process.env.GRAPHS_BACKFILL_STOP_FILE || path.join(process.cwd(), 'graphs_backfill_stop.json');
 
 const contactCache = new Map<string, ContactInfo | null>();
+const sellerCache = new Map<string, SellerInfo | null>();
 let lastTinyRequestAt = 0;
 
 const sleep = (ms: number) => new Promise(resolve => setTimeout(resolve, ms));
@@ -218,6 +228,16 @@ function getTinyErrors(data: any) {
     return Array.isArray(errors) ? errors : [errors];
 }
 
+function describeRequestError(error: any) {
+    if (error?.response) {
+        const body = JSON.stringify(error.response.data ?? '');
+        const truncatedBody = body.length > 500 ? `${body.slice(0, 500)}...` : body;
+        return `HTTP ${error.response.status}: ${truncatedBody}`;
+    }
+
+    return error?.message || String(error);
+}
+
 function isTinyApiBlocked(data: any) {
     return getTinyErrors(data).some((entry: any) => {
         const message = String(entry?.erro || entry || '').toLowerCase();
@@ -239,9 +259,24 @@ async function tinyPost(servicePhp: string, params: URLSearchParams) {
         assertNotStopped();
         await waitForTinyRequestSlot(servicePhp);
 
-        const response = await axios.post(`https://api.tiny.com.br/api2/${servicePhp}`, params, {
-            headers: { 'Content-Type': 'application/x-www-form-urlencoded' }
-        });
+        let response;
+        try {
+            response = await axios.post(`https://api.tiny.com.br/api2/${servicePhp}`, params, {
+                headers: { 'Content-Type': 'application/x-www-form-urlencoded' }
+            });
+        } catch (error: any) {
+            blockedAttempts += 1;
+            if (MAX_TINY_BLOCK_RETRIES && blockedAttempts > MAX_TINY_BLOCK_RETRIES) {
+                throw error;
+            }
+
+            const retryLabel = MAX_TINY_BLOCK_RETRIES
+                ? `${blockedAttempts}/${MAX_TINY_BLOCK_RETRIES}`
+                : String(blockedAttempts);
+            console.warn(`[Tiny] Request failed on ${servicePhp}: ${describeRequestError(error)}. Pausing ${TINY_BLOCK_DELAY_MS}ms before retry ${retryLabel}.`);
+            await sleepWithStop(TINY_BLOCK_DELAY_MS, 'retrying Tiny API after request failure');
+            continue;
+        }
 
         if (!isTinyApiBlocked(response.data)) {
             return response;
@@ -376,6 +411,41 @@ async function fetchContactInfo(cliente: any): Promise<ContactInfo | null> {
     }
 }
 
+async function fetchSellerInfo(sellerId: string): Promise<SellerInfo | null> {
+    const normalizedSellerId = String(sellerId || '').trim();
+    if (!SELLER_FALLBACK || !normalizedSellerId || normalizedSellerId === '0') return null;
+    if (sellerCache.has(normalizedSellerId)) return sellerCache.get(normalizedSellerId) || null;
+
+    try {
+        const detailParams = new URLSearchParams();
+        detailParams.append('token', TINY_API_TOKEN!);
+        detailParams.append('formato', 'JSON');
+        detailParams.append('id', normalizedSellerId);
+
+        const detailResponse = await tinyPost('contato.obter.php', detailParams);
+        await sleepWithStop(TINY_CONTACT_DELAY_MS, 'next Tiny seller contact request');
+
+        if (detailResponse.data?.retorno?.status !== 'OK') {
+            console.warn(`[Tiny] Seller lookup failed for ${normalizedSellerId}: ${JSON.stringify(detailResponse.data?.retorno?.erros || 'Unknown error')}`);
+            sellerCache.set(normalizedSellerId, null);
+            return null;
+        }
+
+        const contato = detailResponse.data.retorno.contato || {};
+        const info = {
+            name: pickFirst(contato.nome, contato.razao_social, contato.fantasia, contato.nome_fantasia)
+        };
+
+        sellerCache.set(normalizedSellerId, info.name ? info : null);
+        return info.name ? info : null;
+    } catch (error: any) {
+        if (error instanceof BackfillStoppedError) throw error;
+        console.error(`[Tiny] Seller lookup failed for ${normalizedSellerId}: ${describeRequestError(error)}`);
+        sellerCache.set(normalizedSellerId, null);
+        return null;
+    }
+}
+
 async function buildGraphsRows(details: TinyOrderDetails, summary: TinyOrderSummary): Promise<GraphsOrderRow[]> {
     const pedido = details.pedido || {};
     const cliente = pedido.cliente || {};
@@ -398,6 +468,12 @@ async function buildGraphsRows(details: TinyOrderDetails, summary: TinyOrderSumm
     const customerPhone = normalizePhone(pickFirst(orderCustomerPhone, contactInfo?.phone));
     const customerFantasyName = pickFirst(orderCustomerFantasyName, contactInfo?.fantasia);
     const ecommerce = pedido.ecommerce || {};
+    const sellerId = pickFirst(pedido.id_vendedor);
+    const orderSellerName = pickFirst(pedido.nome_vendedor);
+    const sellerInfo = sellerId && sellerId !== '0' && !orderSellerName
+        ? await fetchSellerInfo(sellerId)
+        : null;
+    const sellerName = pickFirst(orderSellerName, sellerInfo?.name);
 
     return itens
         .map((entry: any) => entry?.item || entry)
@@ -413,8 +489,8 @@ async function buildGraphsRows(details: TinyOrderDetails, summary: TinyOrderSumm
             ID_Pedido: orderId,
             Fone_Cliente: customerPhone,
             cliente_nome_fantasia: customerFantasyName,
-            id_vendedor: pickFirst(pedido.id_vendedor),
-            nome_vendedor: pickFirst(pedido.nome_vendedor),
+            id_vendedor: sellerId,
+            nome_vendedor: sellerName,
             marketplace: pickFirst(ecommerce.nomeEcommerce, pedido.nome_ecommerce),
             canal_venda: pickFirst(ecommerce.canalVenda, pedido.canal_venda),
             numero_ecommerce: pickFirst(
@@ -428,23 +504,42 @@ async function buildGraphsRows(details: TinyOrderDetails, summary: TinyOrderSumm
 async function sendRowsToGraphs(rows: GraphsOrderRow[], orderId: string) {
     if (!rows.length) {
         console.warn(`[Graphs] Order ${orderId} has no items. Skipping.`);
-        return;
+        return true;
     }
 
     if (DRY_RUN) {
         console.log(`[dry-run] Would send ${rows.length} row(s) for order ${orderId}`);
         console.log(JSON.stringify(rows.slice(0, 3), null, 2));
-        return;
+        return true;
     }
 
-    await axios.post(GRAPHS_API_URL!, rows, {
-        headers: {
-            'Content-Type': 'application/json',
-            'x-api-key': GRAPHS_API_KEY!
-        }
-    });
+    let attempt = 0;
+    while (true) {
+        try {
+            await axios.post(GRAPHS_API_URL!, rows, {
+                headers: {
+                    'Content-Type': 'application/json',
+                    'x-api-key': GRAPHS_API_KEY!
+                }
+            });
 
-    console.log(`[Graphs] Sent ${rows.length} row(s) for order ${orderId}`);
+            console.log(`[Graphs] Sent ${rows.length} row(s) for order ${orderId}`);
+            return true;
+        } catch (error: any) {
+            attempt += 1;
+            const status = error?.response?.status;
+            const shouldRetry = !status || status >= 500;
+
+            console.error(`[Graphs] Failed to send order ${orderId} on attempt ${attempt}: ${describeRequestError(error)}`);
+
+            if (!shouldRetry || (GRAPHS_API_MAX_RETRIES && attempt >= GRAPHS_API_MAX_RETRIES)) {
+                console.error(`[Graphs] Giving up on order ${orderId}; backfill will continue with the next order.`);
+                return false;
+            }
+
+            await sleepWithStop(GRAPHS_API_RETRY_DELAY_MS, `retrying Graphs API for order ${orderId}`);
+        }
+    }
 }
 
 async function runBackfill() {
@@ -464,6 +559,7 @@ async function runBackfill() {
     let lastCompletedDate = state.lastCompletedDate;
     let processedDays = 0;
     let processedOrders = 0;
+    let failedOrders = state.failedOrders || 0;
     const orderPace = ORDER_DELAY_MS ? `~${Math.floor(60000 / ORDER_DELAY_MS)}/minute` : 'unlimited';
     const tinyRequestPace = TINY_REQUEST_DELAY_MS ? `~${Math.floor(60000 / TINY_REQUEST_DELAY_MS)}/minute` : 'unlimited';
 
@@ -474,6 +570,7 @@ async function runBackfill() {
     console.log(`[config] Tiny request pace: 1 request every ${TINY_REQUEST_DELAY_MS}ms (${tinyRequestPace})`);
     console.log(`[config] Order pace: 1 order every ${ORDER_DELAY_MS}ms (${orderPace})`);
     console.log(`[config] Tiny block pause: ${TINY_BLOCK_DELAY_MS}ms`);
+    console.log(`[config] Graphs API retries: ${GRAPHS_API_MAX_RETRIES || 'unlimited'}`);
     console.log(`[config] State file: ${STATE_FILE}`);
     console.log(`[config] Stop file: ${STOP_FILE}`);
 
@@ -483,6 +580,7 @@ async function runBackfill() {
         currentDate,
         processedDays,
         processedOrders,
+        failedOrders,
         lastMessage: 'Backfill started'
     });
 
@@ -521,23 +619,61 @@ async function runBackfill() {
                 continue;
             }
 
-            const details = await fetchOrderDetails(orderId);
-            if (!details) continue;
+            try {
+                const details = await fetchOrderDetails(orderId);
+                if (!details) {
+                    failedOrders += 1;
+                    saveState({
+                        nextDate: currentDate,
+                        lastCompletedDate,
+                        status: 'running',
+                        currentDate,
+                        lastFailedOrderId: orderId,
+                        processedDays,
+                        processedOrders,
+                        failedOrders,
+                        lastMessage: `Could not fetch Tiny details for order ${orderId}`
+                    });
+                    continue;
+                }
 
-            const rows = await buildGraphsRows(details, summary);
-            await sendRowsToGraphs(rows, orderId);
-            processedOrders += 1;
+                const rows = await buildGraphsRows(details, summary);
+                const sent = await sendRowsToGraphs(rows, orderId);
+                if (sent) {
+                    processedOrders += 1;
+                } else {
+                    failedOrders += 1;
+                }
 
-            saveState({
-                nextDate: currentDate,
-                lastCompletedDate,
-                status: 'running',
-                currentDate,
-                lastOrderId: orderId,
-                processedDays,
-                processedOrders,
-                lastMessage: `Processed order ${orderId}`
-            });
+                saveState({
+                    nextDate: currentDate,
+                    lastCompletedDate,
+                    status: 'running',
+                    currentDate,
+                    lastOrderId: sent ? orderId : undefined,
+                    lastFailedOrderId: sent ? undefined : orderId,
+                    processedDays,
+                    processedOrders,
+                    failedOrders,
+                    lastMessage: sent ? `Processed order ${orderId}` : `Failed order ${orderId}; continuing`
+                });
+            } catch (error: any) {
+                if (error instanceof BackfillStoppedError) throw error;
+
+                failedOrders += 1;
+                console.error(`[backfill] Unexpected failure for order ${orderId}: ${describeRequestError(error)}. Continuing with next order.`);
+                saveState({
+                    nextDate: currentDate,
+                    lastCompletedDate,
+                    status: 'running',
+                    currentDate,
+                    lastFailedOrderId: orderId,
+                    processedDays,
+                    processedOrders,
+                    failedOrders,
+                    lastMessage: `Unexpected failure for order ${orderId}; continuing`
+                });
+            }
 
             if (!MAX_ORDERS || processedOrders < MAX_ORDERS) {
                 await sleepWithStop(ORDER_DELAY_MS, 'next order');
@@ -555,6 +691,7 @@ async function runBackfill() {
             currentDate,
             processedDays,
             processedOrders,
+            failedOrders,
             lastMessage: `Completed ${completedDate}`
         });
         console.log(`[state] Completed ${completedDate}. Next date: ${currentDate}`);
@@ -567,9 +704,10 @@ async function runBackfill() {
         currentDate,
         processedDays,
         processedOrders,
-        lastMessage: 'Backfill completed'
+        failedOrders,
+        lastMessage: failedOrders ? `Backfill completed with ${failedOrders} failed order(s)` : 'Backfill completed'
     });
-    console.log(`\nDone. Processed ${processedOrders} order(s) across ${processedDays} day(s).`);
+    console.log(`\nDone. Processed ${processedOrders} order(s) across ${processedDays} day(s). Failed orders: ${failedOrders}.`);
 }
 
 runBackfill().catch((error: any) => {