From 831f82ee57b853bd4b22f4c005c1f588d3f106b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cau=C3=AA=20Faleiros?= Date: Fri, 3 Jul 2026 12:19:43 -0300 Subject: [PATCH] Improve graphs backfill resilience --- src/index.ts | 18 +++ src/scripts/backfill-graphs.ts | 200 ++++++++++++++++++++++++++++----- 2 files changed, 187 insertions(+), 31 deletions(-) diff --git a/src/index.ts b/src/index.ts index bbda48f..76f68c5 100644 --- a/src/index.ts +++ b/src/index.ts @@ -34,6 +34,14 @@ app.get('/api/trigger-backfill', (req: Request, res: Response) => { detached: true, stdio: 'inherit' // Inherit allows the logs to show up in Portainer's log viewer! }); + + child.on('error', (error) => { + console.error(`[server]: Backfill process failed to start: ${error.message}`); + }); + + child.on('exit', (code, signal) => { + console.log(`[server]: Backfill process exited. code=${code ?? 'null'} signal=${signal ?? 'null'}`); + }); child.unref(); @@ -60,6 +68,7 @@ app.get('/api/trigger-graphs-backfill', (req: Request, res: Response) => { maxOrders: 'GRAPHS_BACKFILL_MAX_ORDERS', maxDays: 'GRAPHS_BACKFILL_MAX_DAYS', contactFallback: 'GRAPHS_BACKFILL_CONTACT_FALLBACK', + sellerFallback: 'GRAPHS_BACKFILL_SELLER_FALLBACK', tinyRequestDelayMs: 'GRAPHS_BACKFILL_TINY_REQUEST_DELAY_MS', orderDelayMs: 'GRAPHS_BACKFILL_ORDER_DELAY_MS', blockDelayMs: 'GRAPHS_BACKFILL_TINY_BLOCK_DELAY_MS', @@ -83,6 +92,14 @@ app.get('/api/trigger-graphs-backfill', (req: Request, res: Response) => { env }); + child.on('error', (error) => { + console.error(`[server]: Graphs backfill process failed to start: ${error.message}`); + }); + + child.on('exit', (code, signal) => { + console.log(`[server]: Graphs backfill process exited. code=${code ?? 'null'} signal=${signal ?? 'null'}`); + }); + child.unref(); console.log('[server]: Graphs backfill script manually triggered via HTTP endpoint.'); @@ -96,6 +113,7 @@ app.get('/api/trigger-graphs-backfill', (req: Request, res: Response) => { maxOrders: env.GRAPHS_BACKFILL_MAX_ORDERS || null, maxDays: env.GRAPHS_BACKFILL_MAX_DAYS || null, contactFallback: env.GRAPHS_BACKFILL_CONTACT_FALLBACK || 'true', + sellerFallback: env.GRAPHS_BACKFILL_SELLER_FALLBACK || 'true', tinyRequestDelayMs: env.GRAPHS_BACKFILL_TINY_REQUEST_DELAY_MS || '6000', orderDelayMs: env.GRAPHS_BACKFILL_ORDER_DELAY_MS || '0', blockDelayMs: env.GRAPHS_BACKFILL_TINY_BLOCK_DELAY_MS || '600000', diff --git a/src/scripts/backfill-graphs.ts b/src/scripts/backfill-graphs.ts index 7db405a..227d9b6 100644 --- a/src/scripts/backfill-graphs.ts +++ b/src/scripts/backfill-graphs.ts @@ -41,14 +41,20 @@ type ContactInfo = { phone: string; }; +type SellerInfo = { + name: string; +}; + type BackfillState = { nextDate: string; lastCompletedDate?: string; status?: 'running' | 'stopped' | 'completed'; currentDate?: string; lastOrderId?: string; + lastFailedOrderId?: string; processedDays?: number; processedOrders?: number; + failedOrders?: number; lastMessage?: string; updatedAt?: string; }; @@ -58,6 +64,7 @@ const GRAPHS_API_URL = process.env.GRAPHS_API_URL || process.env.NEXSTAR_GRAPHS_ const GRAPHS_API_KEY = process.env.GRAPHS_API_KEY || process.env.NEXSTAR_GRAPHS_API_KEY; const DRY_RUN = process.env.GRAPHS_BACKFILL_DRY_RUN === 'true'; const CONTACT_FALLBACK = process.env.GRAPHS_BACKFILL_CONTACT_FALLBACK !== 'false'; +const SELLER_FALLBACK = process.env.GRAPHS_BACKFILL_SELLER_FALLBACK !== 'false'; const NORMALIZE_PHONE = process.env.GRAPHS_BACKFILL_NORMALIZE_PHONE !== 'false'; const START_DATE = process.env.GRAPHS_BACKFILL_START_DATE || '13/02/2025'; const END_DATE = process.env.GRAPHS_BACKFILL_END_DATE || getTodayDate(); @@ -77,10 +84,13 @@ const TINY_CONTACT_DELAY_MS = numberEnv('TINY_CONTACT_DELAY_MS', TINY_DETAIL_DEL const ORDER_DELAY_MS = numberEnv('GRAPHS_BACKFILL_ORDER_DELAY_MS', 0); const TINY_BLOCK_DELAY_MS = numberEnv('GRAPHS_BACKFILL_TINY_BLOCK_DELAY_MS', 600000); const MAX_TINY_BLOCK_RETRIES = numberEnv('GRAPHS_BACKFILL_MAX_TINY_BLOCK_RETRIES', 0); +const GRAPHS_API_MAX_RETRIES = numberEnv('GRAPHS_BACKFILL_GRAPHS_API_MAX_RETRIES', 3); +const GRAPHS_API_RETRY_DELAY_MS = numberEnv('GRAPHS_BACKFILL_GRAPHS_API_RETRY_DELAY_MS', 30000); const STATE_FILE = process.env.GRAPHS_BACKFILL_STATE_FILE || path.join(process.cwd(), 'graphs_backfill_state.json'); const STOP_FILE = process.env.GRAPHS_BACKFILL_STOP_FILE || path.join(process.cwd(), 'graphs_backfill_stop.json'); const contactCache = new Map(); +const sellerCache = new Map(); let lastTinyRequestAt = 0; const sleep = (ms: number) => new Promise(resolve => setTimeout(resolve, ms)); @@ -218,6 +228,16 @@ function getTinyErrors(data: any) { return Array.isArray(errors) ? errors : [errors]; } +function describeRequestError(error: any) { + if (error?.response) { + const body = JSON.stringify(error.response.data ?? ''); + const truncatedBody = body.length > 500 ? `${body.slice(0, 500)}...` : body; + return `HTTP ${error.response.status}: ${truncatedBody}`; + } + + return error?.message || String(error); +} + function isTinyApiBlocked(data: any) { return getTinyErrors(data).some((entry: any) => { const message = String(entry?.erro || entry || '').toLowerCase(); @@ -239,9 +259,24 @@ async function tinyPost(servicePhp: string, params: URLSearchParams) { assertNotStopped(); await waitForTinyRequestSlot(servicePhp); - const response = await axios.post(`https://api.tiny.com.br/api2/${servicePhp}`, params, { - headers: { 'Content-Type': 'application/x-www-form-urlencoded' } - }); + let response; + try { + response = await axios.post(`https://api.tiny.com.br/api2/${servicePhp}`, params, { + headers: { 'Content-Type': 'application/x-www-form-urlencoded' } + }); + } catch (error: any) { + blockedAttempts += 1; + if (MAX_TINY_BLOCK_RETRIES && blockedAttempts > MAX_TINY_BLOCK_RETRIES) { + throw error; + } + + const retryLabel = MAX_TINY_BLOCK_RETRIES + ? `${blockedAttempts}/${MAX_TINY_BLOCK_RETRIES}` + : String(blockedAttempts); + console.warn(`[Tiny] Request failed on ${servicePhp}: ${describeRequestError(error)}. Pausing ${TINY_BLOCK_DELAY_MS}ms before retry ${retryLabel}.`); + await sleepWithStop(TINY_BLOCK_DELAY_MS, 'retrying Tiny API after request failure'); + continue; + } if (!isTinyApiBlocked(response.data)) { return response; @@ -376,6 +411,41 @@ async function fetchContactInfo(cliente: any): Promise { } } +async function fetchSellerInfo(sellerId: string): Promise { + const normalizedSellerId = String(sellerId || '').trim(); + if (!SELLER_FALLBACK || !normalizedSellerId || normalizedSellerId === '0') return null; + if (sellerCache.has(normalizedSellerId)) return sellerCache.get(normalizedSellerId) || null; + + try { + const detailParams = new URLSearchParams(); + detailParams.append('token', TINY_API_TOKEN!); + detailParams.append('formato', 'JSON'); + detailParams.append('id', normalizedSellerId); + + const detailResponse = await tinyPost('contato.obter.php', detailParams); + await sleepWithStop(TINY_CONTACT_DELAY_MS, 'next Tiny seller contact request'); + + if (detailResponse.data?.retorno?.status !== 'OK') { + console.warn(`[Tiny] Seller lookup failed for ${normalizedSellerId}: ${JSON.stringify(detailResponse.data?.retorno?.erros || 'Unknown error')}`); + sellerCache.set(normalizedSellerId, null); + return null; + } + + const contato = detailResponse.data.retorno.contato || {}; + const info = { + name: pickFirst(contato.nome, contato.razao_social, contato.fantasia, contato.nome_fantasia) + }; + + sellerCache.set(normalizedSellerId, info.name ? info : null); + return info.name ? info : null; + } catch (error: any) { + if (error instanceof BackfillStoppedError) throw error; + console.error(`[Tiny] Seller lookup failed for ${normalizedSellerId}: ${describeRequestError(error)}`); + sellerCache.set(normalizedSellerId, null); + return null; + } +} + async function buildGraphsRows(details: TinyOrderDetails, summary: TinyOrderSummary): Promise { const pedido = details.pedido || {}; const cliente = pedido.cliente || {}; @@ -398,6 +468,12 @@ async function buildGraphsRows(details: TinyOrderDetails, summary: TinyOrderSumm const customerPhone = normalizePhone(pickFirst(orderCustomerPhone, contactInfo?.phone)); const customerFantasyName = pickFirst(orderCustomerFantasyName, contactInfo?.fantasia); const ecommerce = pedido.ecommerce || {}; + const sellerId = pickFirst(pedido.id_vendedor); + const orderSellerName = pickFirst(pedido.nome_vendedor); + const sellerInfo = sellerId && sellerId !== '0' && !orderSellerName + ? await fetchSellerInfo(sellerId) + : null; + const sellerName = pickFirst(orderSellerName, sellerInfo?.name); return itens .map((entry: any) => entry?.item || entry) @@ -413,8 +489,8 @@ async function buildGraphsRows(details: TinyOrderDetails, summary: TinyOrderSumm ID_Pedido: orderId, Fone_Cliente: customerPhone, cliente_nome_fantasia: customerFantasyName, - id_vendedor: pickFirst(pedido.id_vendedor), - nome_vendedor: pickFirst(pedido.nome_vendedor), + id_vendedor: sellerId, + nome_vendedor: sellerName, marketplace: pickFirst(ecommerce.nomeEcommerce, pedido.nome_ecommerce), canal_venda: pickFirst(ecommerce.canalVenda, pedido.canal_venda), numero_ecommerce: pickFirst( @@ -428,23 +504,42 @@ async function buildGraphsRows(details: TinyOrderDetails, summary: TinyOrderSumm async function sendRowsToGraphs(rows: GraphsOrderRow[], orderId: string) { if (!rows.length) { console.warn(`[Graphs] Order ${orderId} has no items. Skipping.`); - return; + return true; } if (DRY_RUN) { console.log(`[dry-run] Would send ${rows.length} row(s) for order ${orderId}`); console.log(JSON.stringify(rows.slice(0, 3), null, 2)); - return; + return true; } - await axios.post(GRAPHS_API_URL!, rows, { - headers: { - 'Content-Type': 'application/json', - 'x-api-key': GRAPHS_API_KEY! - } - }); + let attempt = 0; + while (true) { + try { + await axios.post(GRAPHS_API_URL!, rows, { + headers: { + 'Content-Type': 'application/json', + 'x-api-key': GRAPHS_API_KEY! + } + }); - console.log(`[Graphs] Sent ${rows.length} row(s) for order ${orderId}`); + console.log(`[Graphs] Sent ${rows.length} row(s) for order ${orderId}`); + return true; + } catch (error: any) { + attempt += 1; + const status = error?.response?.status; + const shouldRetry = !status || status >= 500; + + console.error(`[Graphs] Failed to send order ${orderId} on attempt ${attempt}: ${describeRequestError(error)}`); + + if (!shouldRetry || (GRAPHS_API_MAX_RETRIES && attempt >= GRAPHS_API_MAX_RETRIES)) { + console.error(`[Graphs] Giving up on order ${orderId}; backfill will continue with the next order.`); + return false; + } + + await sleepWithStop(GRAPHS_API_RETRY_DELAY_MS, `retrying Graphs API for order ${orderId}`); + } + } } async function runBackfill() { @@ -464,6 +559,7 @@ async function runBackfill() { let lastCompletedDate = state.lastCompletedDate; let processedDays = 0; let processedOrders = 0; + let failedOrders = state.failedOrders || 0; const orderPace = ORDER_DELAY_MS ? `~${Math.floor(60000 / ORDER_DELAY_MS)}/minute` : 'unlimited'; const tinyRequestPace = TINY_REQUEST_DELAY_MS ? `~${Math.floor(60000 / TINY_REQUEST_DELAY_MS)}/minute` : 'unlimited'; @@ -474,6 +570,7 @@ async function runBackfill() { console.log(`[config] Tiny request pace: 1 request every ${TINY_REQUEST_DELAY_MS}ms (${tinyRequestPace})`); console.log(`[config] Order pace: 1 order every ${ORDER_DELAY_MS}ms (${orderPace})`); console.log(`[config] Tiny block pause: ${TINY_BLOCK_DELAY_MS}ms`); + console.log(`[config] Graphs API retries: ${GRAPHS_API_MAX_RETRIES || 'unlimited'}`); console.log(`[config] State file: ${STATE_FILE}`); console.log(`[config] Stop file: ${STOP_FILE}`); @@ -483,6 +580,7 @@ async function runBackfill() { currentDate, processedDays, processedOrders, + failedOrders, lastMessage: 'Backfill started' }); @@ -521,23 +619,61 @@ async function runBackfill() { continue; } - const details = await fetchOrderDetails(orderId); - if (!details) continue; + try { + const details = await fetchOrderDetails(orderId); + if (!details) { + failedOrders += 1; + saveState({ + nextDate: currentDate, + lastCompletedDate, + status: 'running', + currentDate, + lastFailedOrderId: orderId, + processedDays, + processedOrders, + failedOrders, + lastMessage: `Could not fetch Tiny details for order ${orderId}` + }); + continue; + } - const rows = await buildGraphsRows(details, summary); - await sendRowsToGraphs(rows, orderId); - processedOrders += 1; + const rows = await buildGraphsRows(details, summary); + const sent = await sendRowsToGraphs(rows, orderId); + if (sent) { + processedOrders += 1; + } else { + failedOrders += 1; + } - saveState({ - nextDate: currentDate, - lastCompletedDate, - status: 'running', - currentDate, - lastOrderId: orderId, - processedDays, - processedOrders, - lastMessage: `Processed order ${orderId}` - }); + saveState({ + nextDate: currentDate, + lastCompletedDate, + status: 'running', + currentDate, + lastOrderId: sent ? orderId : undefined, + lastFailedOrderId: sent ? undefined : orderId, + processedDays, + processedOrders, + failedOrders, + lastMessage: sent ? `Processed order ${orderId}` : `Failed order ${orderId}; continuing` + }); + } catch (error: any) { + if (error instanceof BackfillStoppedError) throw error; + + failedOrders += 1; + console.error(`[backfill] Unexpected failure for order ${orderId}: ${describeRequestError(error)}. Continuing with next order.`); + saveState({ + nextDate: currentDate, + lastCompletedDate, + status: 'running', + currentDate, + lastFailedOrderId: orderId, + processedDays, + processedOrders, + failedOrders, + lastMessage: `Unexpected failure for order ${orderId}; continuing` + }); + } if (!MAX_ORDERS || processedOrders < MAX_ORDERS) { await sleepWithStop(ORDER_DELAY_MS, 'next order'); @@ -555,6 +691,7 @@ async function runBackfill() { currentDate, processedDays, processedOrders, + failedOrders, lastMessage: `Completed ${completedDate}` }); console.log(`[state] Completed ${completedDate}. Next date: ${currentDate}`); @@ -567,9 +704,10 @@ async function runBackfill() { currentDate, processedDays, processedOrders, - lastMessage: 'Backfill completed' + failedOrders, + lastMessage: failedOrders ? `Backfill completed with ${failedOrders} failed order(s)` : 'Backfill completed' }); - console.log(`\nDone. Processed ${processedOrders} order(s) across ${processedDays} day(s).`); + console.log(`\nDone. Processed ${processedOrders} order(s) across ${processedDays} day(s). Failed orders: ${failedOrders}.`); } runBackfill().catch((error: any) => {