From 4d7daff496917ae9edf7ba1c8782475dd5a5492b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cau=C3=AA=20Faleiros?= Date: Tue, 23 Jun 2026 10:03:22 -0300 Subject: [PATCH] fix: throttle Graphs backfill by Tiny request --- docker-compose.yml | 1 + src/controllers/webhook.controller.ts | 142 +++++++++++++++++--------- src/index.ts | 4 +- src/scripts/backfill-graphs.ts | 18 +++- 4 files changed, 110 insertions(+), 55 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index b2cf8e3..2a6f97b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -19,6 +19,7 @@ services: - GRAPHS_API_URL=${GRAPHS_API_URL} - GRAPHS_API_KEY=${GRAPHS_API_KEY} - GRAPHS_BACKFILL_CONTACT_FALLBACK=${GRAPHS_BACKFILL_CONTACT_FALLBACK} + - GRAPHS_BACKFILL_TINY_REQUEST_DELAY_MS=${GRAPHS_BACKFILL_TINY_REQUEST_DELAY_MS} - GRAPHS_BACKFILL_ORDER_DELAY_MS=${GRAPHS_BACKFILL_ORDER_DELAY_MS} - GRAPHS_BACKFILL_TINY_BLOCK_DELAY_MS=${GRAPHS_BACKFILL_TINY_BLOCK_DELAY_MS} - GRAPHS_BACKFILL_MAX_TINY_BLOCK_RETRIES=${GRAPHS_BACKFILL_MAX_TINY_BLOCK_RETRIES} diff --git a/src/controllers/webhook.controller.ts b/src/controllers/webhook.controller.ts index 8355fef..5ea6120 100644 --- a/src/controllers/webhook.controller.ts +++ b/src/controllers/webhook.controller.ts @@ -12,7 +12,10 @@ type CachedOrderDetails = { whatsappVendedor: string; }; +type OrderDetailsResult = Omit; + const orderDetailsCache = new Map(); +const inFlightOrderDetails = new Map>(); const configuredCacheTtlMs = Number(process.env.TINY_ORDER_DETAILS_CACHE_TTL_MS || 120000); const ORDER_DETAILS_CACHE_TTL_MS = Number.isFinite(configuredCacheTtlMs) ? configuredCacheTtlMs : 120000; @@ -47,6 +50,88 @@ function setCachedOrderDetails(orderId: string, cached: Omit { + console.log(`Fetching full details for Order ID: ${orderId} from Tiny API...`); + const params = new URLSearchParams(); + params.append('token', tinyApiToken); + params.append('id', orderId); + params.append('formato', 'JSON'); + + const apiResponse = await axios.post('https://api.tiny.com.br/api2/pedido.obter.php', params, { + headers: { 'Content-Type': 'application/x-www-form-urlencoded' } + }); + + if (apiResponse.data?.retorno?.status !== 'OK') { + console.error('Tiny API returned an error:', apiResponse.data?.retorno?.erros || 'Unknown error'); + return null; + } + + const fullOrderDetails = apiResponse.data.retorno.pedido; + const statusProcessamento = apiResponse.data.retorno.status_processamento || ""; + let whatsappVendedor = ""; + console.log(`Successfully fetched order details! Found phone: ${fullOrderDetails.cliente?.celular || fullOrderDetails.cliente?.fone || 'None'}`); + + // Seller details are off by default because this is an extra Tiny API call per order. + const idVendedor = fullOrderDetails.id_vendedor; + if (process.env.TINY_FETCH_SELLER_DETAILS === 'true' && idVendedor && idVendedor !== "0") { + console.log(`Fetching seller details for Seller ID: ${idVendedor}...`); + const vendorParams = new URLSearchParams(); + vendorParams.append('token', tinyApiToken); + vendorParams.append('id', idVendedor); + vendorParams.append('formato', 'JSON'); + + const vendorResponse = await axios.post('https://api.tiny.com.br/api2/contato.obter.php', vendorParams, { + headers: { 'Content-Type': 'application/x-www-form-urlencoded' } + }); + + if (vendorResponse.data?.retorno?.status === 'OK') { + const contato = vendorResponse.data.retorno.contato; + whatsappVendedor = contato?.celular || contato?.fone || contato?.telefone || ""; + console.log(`Successfully fetched seller WhatsApp: ${whatsappVendedor || 'None'}`); + } else { + console.error('Failed to fetch seller details:', JSON.stringify(vendorResponse.data?.retorno?.erros || 'Unknown API error')); + } + } + + return { + fullOrderDetails, + statusProcessamento, + whatsappVendedor + }; +} + +async function getOrFetchOrderDetails(orderId: string, tinyApiToken: string): Promise { + const cachedDetails = getCachedOrderDetails(orderId); + if (cachedDetails) { + console.log(`Using cached order details for Order ID: ${orderId}.`); + return { + fullOrderDetails: cachedDetails.fullOrderDetails, + statusProcessamento: cachedDetails.statusProcessamento, + whatsappVendedor: cachedDetails.whatsappVendedor + }; + } + + const existingFetch = inFlightOrderDetails.get(orderId); + if (existingFetch) { + console.log(`Waiting for in-flight Tiny fetch for Order ID: ${orderId}.`); + return existingFetch; + } + + const fetchPromise = fetchOrderDetailsFromTiny(orderId, tinyApiToken) + .then(result => { + if (result) { + setCachedOrderDetails(orderId, result); + } + return result; + }) + .finally(() => { + inFlightOrderDetails.delete(orderId); + }); + + inFlightOrderDetails.set(orderId, fetchPromise); + return fetchPromise; +} + export const handleTinyOrderUpdate = async (req: Request, res: Response): Promise => { try { // 1. Security Check: Verify token from Tiny @@ -84,58 +169,13 @@ export const handleTinyOrderUpdate = async (req: Request, res: Response): Promis let statusProcessamento = ""; let whatsappVendedor = ""; - const cachedDetails = getCachedOrderDetails(String(orderId)); - if (cachedDetails) { - fullOrderDetails = cachedDetails.fullOrderDetails; - statusProcessamento = cachedDetails.statusProcessamento; - whatsappVendedor = cachedDetails.whatsappVendedor; - console.log(`Using cached order details for Order ID: ${orderId}.`); - } else if (tinyApiToken) { + if (tinyApiToken) { try { - console.log(`Fetching full details for Order ID: ${orderId} from Tiny API...`); - const params = new URLSearchParams(); - params.append('token', tinyApiToken); - params.append('id', orderId); - params.append('formato', 'JSON'); - - const apiResponse = await axios.post('https://api.tiny.com.br/api2/pedido.obter.php', params, { - headers: { 'Content-Type': 'application/x-www-form-urlencoded' } - }); - - if (apiResponse.data?.retorno?.status === 'OK') { - fullOrderDetails = apiResponse.data.retorno.pedido; - statusProcessamento = apiResponse.data.retorno.status_processamento || ""; - console.log(`Successfully fetched order details! Found phone: ${fullOrderDetails.cliente?.celular || fullOrderDetails.cliente?.fone || 'None'}`); - - // OPTION B: Fetch Vendor's WhatsApp - const idVendedor = fullOrderDetails.id_vendedor; - if (process.env.TINY_FETCH_SELLER_DETAILS === 'true' && idVendedor && idVendedor !== "0") { - console.log(`Fetching seller details for Seller ID: ${idVendedor}...`); - const vendorParams = new URLSearchParams(); - vendorParams.append('token', tinyApiToken); - vendorParams.append('id', idVendedor); - vendorParams.append('formato', 'JSON'); - - const vendorResponse = await axios.post('https://api.tiny.com.br/api2/contato.obter.php', vendorParams, { - headers: { 'Content-Type': 'application/x-www-form-urlencoded' } - }); - - if (vendorResponse.data?.retorno?.status === 'OK') { - const contato = vendorResponse.data.retorno.contato; - whatsappVendedor = contato?.celular || contato?.fone || contato?.telefone || ""; - console.log(`Successfully fetched seller WhatsApp: ${whatsappVendedor || 'None'}`); - } else { - console.error('Failed to fetch seller details:', JSON.stringify(vendorResponse.data?.retorno?.erros || 'Unknown API error')); - } - } - - setCachedOrderDetails(String(orderId), { - fullOrderDetails, - statusProcessamento, - whatsappVendedor - }); - } else { - console.error('Tiny API returned an error:', apiResponse.data?.retorno?.erros || 'Unknown error'); + const details = await getOrFetchOrderDetails(String(orderId), tinyApiToken); + if (details) { + fullOrderDetails = details.fullOrderDetails; + statusProcessamento = details.statusProcessamento; + whatsappVendedor = details.whatsappVendedor; } } catch (apiError: any) { console.error('Failed to fetch from Tiny API:', apiError.message); diff --git a/src/index.ts b/src/index.ts index 8d5469f..bbda48f 100644 --- a/src/index.ts +++ b/src/index.ts @@ -60,6 +60,7 @@ app.get('/api/trigger-graphs-backfill', (req: Request, res: Response) => { maxOrders: 'GRAPHS_BACKFILL_MAX_ORDERS', maxDays: 'GRAPHS_BACKFILL_MAX_DAYS', contactFallback: 'GRAPHS_BACKFILL_CONTACT_FALLBACK', + tinyRequestDelayMs: 'GRAPHS_BACKFILL_TINY_REQUEST_DELAY_MS', orderDelayMs: 'GRAPHS_BACKFILL_ORDER_DELAY_MS', blockDelayMs: 'GRAPHS_BACKFILL_TINY_BLOCK_DELAY_MS', maxTinyBlockRetries: 'GRAPHS_BACKFILL_MAX_TINY_BLOCK_RETRIES' @@ -95,7 +96,8 @@ app.get('/api/trigger-graphs-backfill', (req: Request, res: Response) => { maxOrders: env.GRAPHS_BACKFILL_MAX_ORDERS || null, maxDays: env.GRAPHS_BACKFILL_MAX_DAYS || null, contactFallback: env.GRAPHS_BACKFILL_CONTACT_FALLBACK || 'true', - orderDelayMs: env.GRAPHS_BACKFILL_ORDER_DELAY_MS || '6000', + tinyRequestDelayMs: env.GRAPHS_BACKFILL_TINY_REQUEST_DELAY_MS || '6000', + orderDelayMs: env.GRAPHS_BACKFILL_ORDER_DELAY_MS || '0', blockDelayMs: env.GRAPHS_BACKFILL_TINY_BLOCK_DELAY_MS || '600000', maxTinyBlockRetries: env.GRAPHS_BACKFILL_MAX_TINY_BLOCK_RETRIES || null, graphsApiUrlConfigured: Boolean(env.GRAPHS_API_URL || env.NEXSTAR_GRAPHS_API_URL), diff --git a/src/scripts/backfill-graphs.ts b/src/scripts/backfill-graphs.ts index 95c7a7c..6b20cc3 100644 --- a/src/scripts/backfill-graphs.ts +++ b/src/scripts/backfill-graphs.ts @@ -67,16 +67,18 @@ const numberEnv = (name: string, fallback: number) => { }; const MAX_DAYS = numberEnv('GRAPHS_BACKFILL_MAX_DAYS', 0); const MAX_ORDERS = numberEnv('GRAPHS_BACKFILL_MAX_ORDERS', 0); -const TINY_SEARCH_DELAY_MS = numberEnv('TINY_SEARCH_DELAY_MS', 1000); -const TINY_DETAIL_DELAY_MS = numberEnv('TINY_DETAIL_DELAY_MS', 1500); +const TINY_REQUEST_DELAY_MS = numberEnv('GRAPHS_BACKFILL_TINY_REQUEST_DELAY_MS', 6000); +const TINY_SEARCH_DELAY_MS = numberEnv('TINY_SEARCH_DELAY_MS', 0); +const TINY_DETAIL_DELAY_MS = numberEnv('TINY_DETAIL_DELAY_MS', 0); const TINY_CONTACT_DELAY_MS = numberEnv('TINY_CONTACT_DELAY_MS', TINY_DETAIL_DELAY_MS); -const ORDER_DELAY_MS = numberEnv('GRAPHS_BACKFILL_ORDER_DELAY_MS', 6000); +const ORDER_DELAY_MS = numberEnv('GRAPHS_BACKFILL_ORDER_DELAY_MS', 0); const TINY_BLOCK_DELAY_MS = numberEnv('GRAPHS_BACKFILL_TINY_BLOCK_DELAY_MS', 600000); const MAX_TINY_BLOCK_RETRIES = numberEnv('GRAPHS_BACKFILL_MAX_TINY_BLOCK_RETRIES', 0); const STATE_FILE = process.env.GRAPHS_BACKFILL_STATE_FILE || path.join(process.cwd(), 'graphs_backfill_state.json'); const STOP_FILE = process.env.GRAPHS_BACKFILL_STOP_FILE || path.join(process.cwd(), 'graphs_backfill_stop.json'); const contactCache = new Map(); +let lastTinyRequestAt = 0; const sleep = (ms: number) => new Promise(resolve => setTimeout(resolve, ms)); @@ -220,11 +222,19 @@ function isTinyApiBlocked(data: any) { }); } +async function waitForTinyRequestSlot(servicePhp: string) { + const elapsed = Date.now() - lastTinyRequestAt; + const waitMs = Math.max(0, TINY_REQUEST_DELAY_MS - elapsed); + await sleepWithStop(waitMs, `Tiny ${servicePhp} request slot`); + lastTinyRequestAt = Date.now(); +} + async function tinyPost(servicePhp: string, params: URLSearchParams) { let blockedAttempts = 0; while (true) { assertNotStopped(); + await waitForTinyRequestSlot(servicePhp); const response = await axios.post(`https://api.tiny.com.br/api2/${servicePhp}`, params, { headers: { 'Content-Type': 'application/x-www-form-urlencoded' } @@ -452,11 +462,13 @@ async function runBackfill() { let processedDays = 0; let processedOrders = 0; const orderPace = ORDER_DELAY_MS ? `~${Math.floor(60000 / ORDER_DELAY_MS)}/minute` : 'unlimited'; + const tinyRequestPace = TINY_REQUEST_DELAY_MS ? `~${Math.floor(60000 / TINY_REQUEST_DELAY_MS)}/minute` : 'unlimited'; console.log(`[config] Start date: ${currentDate}`); console.log(`[config] End date: ${END_DATE}`); console.log(`[config] Dry run: ${DRY_RUN ? 'yes' : 'no'}`); console.log(`[config] Contact fallback: ${CONTACT_FALLBACK ? 'yes' : 'no'}`); + console.log(`[config] Tiny request pace: 1 request every ${TINY_REQUEST_DELAY_MS}ms (${tinyRequestPace})`); console.log(`[config] Order pace: 1 order every ${ORDER_DELAY_MS}ms (${orderPace})`); console.log(`[config] Tiny block pause: ${TINY_BLOCK_DELAY_MS}ms`); console.log(`[config] State file: ${STATE_FILE}`);