From 44cfdbb4c94881a66f6bd611c2eaf856b09b3981 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cau=C3=AA=20Faleiros?= Date: Tue, 23 Jun 2026 09:32:08 -0300 Subject: [PATCH] fix: slow and control Graphs backfill --- .gitignore | 1 + docker-compose.yml | 8 ++ src/controllers/webhook.controller.ts | 58 ++++++++- src/index.ts | 63 +++++++++- src/scripts/backfill-graphs.ts | 172 +++++++++++++++++++++++--- 5 files changed, 284 insertions(+), 18 deletions(-) diff --git a/.gitignore b/.gitignore index a2a8a40..4dfe2a2 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ node_modules dist .env graphs_backfill_state.json +graphs_backfill_stop.json diff --git a/docker-compose.yml b/docker-compose.yml index ae06932..b2cf8e3 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -16,6 +16,14 @@ services: - TINY_WEBHOOK_SECRET=${TINY_WEBHOOK_SECRET} - N8N_AUTH_TOKEN=${N8N_AUTH_TOKEN} - TINY_API_TOKEN=${TINY_API_TOKEN} + - GRAPHS_API_URL=${GRAPHS_API_URL} + - GRAPHS_API_KEY=${GRAPHS_API_KEY} + - GRAPHS_BACKFILL_CONTACT_FALLBACK=${GRAPHS_BACKFILL_CONTACT_FALLBACK} + - GRAPHS_BACKFILL_ORDER_DELAY_MS=${GRAPHS_BACKFILL_ORDER_DELAY_MS} + - GRAPHS_BACKFILL_TINY_BLOCK_DELAY_MS=${GRAPHS_BACKFILL_TINY_BLOCK_DELAY_MS} + - GRAPHS_BACKFILL_MAX_TINY_BLOCK_RETRIES=${GRAPHS_BACKFILL_MAX_TINY_BLOCK_RETRIES} + - TINY_FETCH_SELLER_DETAILS=${TINY_FETCH_SELLER_DETAILS} + - TINY_ORDER_DETAILS_CACHE_TTL_MS=${TINY_ORDER_DETAILS_CACHE_TTL_MS} # Optional: If you use a shared Docker network for Nginx Proxy Manager, you can attach it here: # networks: # - nginx-proxy-manager-network diff --git a/src/controllers/webhook.controller.ts b/src/controllers/webhook.controller.ts index df7d3c7..8355fef 100644 --- a/src/controllers/webhook.controller.ts +++ b/src/controllers/webhook.controller.ts @@ -5,6 +5,48 @@ import axios from 'axios'; import fs from 'fs'; import path from 'path'; +type CachedOrderDetails = { + expiresAt: number; + fullOrderDetails: any; + statusProcessamento: string; + whatsappVendedor: string; +}; + +const orderDetailsCache = new Map(); +const configuredCacheTtlMs = Number(process.env.TINY_ORDER_DETAILS_CACHE_TTL_MS || 120000); +const ORDER_DETAILS_CACHE_TTL_MS = Number.isFinite(configuredCacheTtlMs) ? configuredCacheTtlMs : 120000; + +function pruneExpiredOrderDetailsCache() { + const now = Date.now(); + for (const [orderId, cached] of orderDetailsCache.entries()) { + if (now > cached.expiresAt) { + orderDetailsCache.delete(orderId); + } + } +} + +function getCachedOrderDetails(orderId: string) { + const cached = orderDetailsCache.get(orderId); + if (!cached) return null; + + if (Date.now() > cached.expiresAt) { + orderDetailsCache.delete(orderId); + return null; + } + + return cached; +} + +function setCachedOrderDetails(orderId: string, cached: Omit) { + if (ORDER_DETAILS_CACHE_TTL_MS <= 0) return; + + pruneExpiredOrderDetailsCache(); + orderDetailsCache.set(orderId, { + ...cached, + expiresAt: Date.now() + ORDER_DETAILS_CACHE_TTL_MS + }); +} + export const handleTinyOrderUpdate = async (req: Request, res: Response): Promise => { try { // 1. Security Check: Verify token from Tiny @@ -42,7 +84,13 @@ export const handleTinyOrderUpdate = async (req: Request, res: Response): Promis let statusProcessamento = ""; let whatsappVendedor = ""; - if (tinyApiToken) { + const cachedDetails = getCachedOrderDetails(String(orderId)); + if (cachedDetails) { + fullOrderDetails = cachedDetails.fullOrderDetails; + statusProcessamento = cachedDetails.statusProcessamento; + whatsappVendedor = cachedDetails.whatsappVendedor; + console.log(`Using cached order details for Order ID: ${orderId}.`); + } else if (tinyApiToken) { try { console.log(`Fetching full details for Order ID: ${orderId} from Tiny API...`); const params = new URLSearchParams(); @@ -61,7 +109,7 @@ export const handleTinyOrderUpdate = async (req: Request, res: Response): Promis // OPTION B: Fetch Vendor's WhatsApp const idVendedor = fullOrderDetails.id_vendedor; - if (idVendedor && idVendedor !== "0") { + if (process.env.TINY_FETCH_SELLER_DETAILS === 'true' && idVendedor && idVendedor !== "0") { console.log(`Fetching seller details for Seller ID: ${idVendedor}...`); const vendorParams = new URLSearchParams(); vendorParams.append('token', tinyApiToken); @@ -80,6 +128,12 @@ export const handleTinyOrderUpdate = async (req: Request, res: Response): Promis console.error('Failed to fetch seller details:', JSON.stringify(vendorResponse.data?.retorno?.erros || 'Unknown API error')); } } + + setCachedOrderDetails(String(orderId), { + fullOrderDetails, + statusProcessamento, + whatsappVendedor + }); } else { console.error('Tiny API returned an error:', apiResponse.data?.retorno?.erros || 'Unknown error'); } diff --git a/src/index.ts b/src/index.ts index e7643fd..8d5469f 100644 --- a/src/index.ts +++ b/src/index.ts @@ -3,6 +3,7 @@ import dotenv from 'dotenv'; import webhookRoutes from './routes/webhook.route'; import { spawn } from 'child_process'; import path from 'path'; +import fs from 'fs'; dotenv.config(); @@ -50,6 +51,7 @@ app.get('/api/trigger-graphs-backfill', (req: Request, res: Response) => { const scriptPath = path.join(__dirname, 'scripts', 'backfill-graphs.js'); const env = { ...process.env }; + const stopFile = env.GRAPHS_BACKFILL_STOP_FILE || path.join(process.cwd(), 'graphs_backfill_stop.json'); const optionalEnvMap: Record = { start: 'GRAPHS_BACKFILL_START_DATE', @@ -57,7 +59,10 @@ app.get('/api/trigger-graphs-backfill', (req: Request, res: Response) => { dryRun: 'GRAPHS_BACKFILL_DRY_RUN', maxOrders: 'GRAPHS_BACKFILL_MAX_ORDERS', maxDays: 'GRAPHS_BACKFILL_MAX_DAYS', - contactFallback: 'GRAPHS_BACKFILL_CONTACT_FALLBACK' + contactFallback: 'GRAPHS_BACKFILL_CONTACT_FALLBACK', + orderDelayMs: 'GRAPHS_BACKFILL_ORDER_DELAY_MS', + blockDelayMs: 'GRAPHS_BACKFILL_TINY_BLOCK_DELAY_MS', + maxTinyBlockRetries: 'GRAPHS_BACKFILL_MAX_TINY_BLOCK_RETRIES' }; for (const [queryKey, envKey] of Object.entries(optionalEnvMap)) { @@ -67,6 +72,10 @@ app.get('/api/trigger-graphs-backfill', (req: Request, res: Response) => { } } + if (fs.existsSync(stopFile)) { + fs.rmSync(stopFile, { force: true }); + } + const child = spawn('node', [scriptPath], { detached: true, stdio: 'inherit', @@ -85,11 +94,61 @@ app.get('/api/trigger-graphs-backfill', (req: Request, res: Response) => { dryRun: env.GRAPHS_BACKFILL_DRY_RUN || 'false', maxOrders: env.GRAPHS_BACKFILL_MAX_ORDERS || null, maxDays: env.GRAPHS_BACKFILL_MAX_DAYS || null, - contactFallback: env.GRAPHS_BACKFILL_CONTACT_FALLBACK || 'true' + contactFallback: env.GRAPHS_BACKFILL_CONTACT_FALLBACK || 'true', + orderDelayMs: env.GRAPHS_BACKFILL_ORDER_DELAY_MS || '6000', + blockDelayMs: env.GRAPHS_BACKFILL_TINY_BLOCK_DELAY_MS || '600000', + maxTinyBlockRetries: env.GRAPHS_BACKFILL_MAX_TINY_BLOCK_RETRIES || null, + graphsApiUrlConfigured: Boolean(env.GRAPHS_API_URL || env.NEXSTAR_GRAPHS_API_URL), + graphsApiKeyConfigured: Boolean(env.GRAPHS_API_KEY || env.NEXSTAR_GRAPHS_API_KEY) } }); }); +app.get('/api/graphs-backfill-status', (req: Request, res: Response) => { + const expectedToken = process.env.TINY_WEBHOOK_SECRET; + if (expectedToken && req.query.token !== expectedToken) { + res.status(401).json({ error: 'Unauthorized' }); + return; + } + + const stateFile = process.env.GRAPHS_BACKFILL_STATE_FILE || path.join(process.cwd(), 'graphs_backfill_state.json'); + const stopFile = process.env.GRAPHS_BACKFILL_STOP_FILE || path.join(process.cwd(), 'graphs_backfill_stop.json'); + let state = null; + + if (fs.existsSync(stateFile)) { + try { + state = JSON.parse(fs.readFileSync(stateFile, 'utf-8')); + } catch (error: any) { + res.status(500).json({ error: `Could not read backfill state: ${error.message}` }); + return; + } + } + + res.status(200).json({ + status: state?.status || 'unknown', + stopRequested: fs.existsSync(stopFile), + state + }); +}); + +app.get('/api/stop-graphs-backfill', (req: Request, res: Response) => { + const expectedToken = process.env.TINY_WEBHOOK_SECRET; + if (expectedToken && req.query.token !== expectedToken) { + res.status(401).json({ error: 'Unauthorized' }); + return; + } + + const stopFile = process.env.GRAPHS_BACKFILL_STOP_FILE || path.join(process.cwd(), 'graphs_backfill_stop.json'); + fs.writeFileSync(stopFile, JSON.stringify({ + requestedAt: new Date().toISOString() + }, null, 2)); + + res.status(200).json({ + status: 'STOP_REQUESTED', + message: 'Graphs backfill will stop after the current wait/request finishes.' + }); +}); + // Hidden endpoint to download the stock CSV log directly from the browser app.get('/api/stock-logs/download', (req: Request, res: Response) => { const expectedToken = process.env.TINY_WEBHOOK_SECRET; diff --git a/src/scripts/backfill-graphs.ts b/src/scripts/backfill-graphs.ts index 1719288..95c7a7c 100644 --- a/src/scripts/backfill-graphs.ts +++ b/src/scripts/backfill-graphs.ts @@ -44,6 +44,12 @@ type ContactInfo = { type BackfillState = { nextDate: string; lastCompletedDate?: string; + status?: 'running' | 'stopped' | 'completed'; + currentDate?: string; + lastOrderId?: string; + processedDays?: number; + processedOrders?: number; + lastMessage?: string; updatedAt?: string; }; @@ -55,17 +61,31 @@ const CONTACT_FALLBACK = process.env.GRAPHS_BACKFILL_CONTACT_FALLBACK !== 'false const NORMALIZE_PHONE = process.env.GRAPHS_BACKFILL_NORMALIZE_PHONE !== 'false'; const START_DATE = process.env.GRAPHS_BACKFILL_START_DATE || '13/02/2025'; const END_DATE = process.env.GRAPHS_BACKFILL_END_DATE || getTodayDate(); -const MAX_DAYS = Number(process.env.GRAPHS_BACKFILL_MAX_DAYS || 0); -const MAX_ORDERS = Number(process.env.GRAPHS_BACKFILL_MAX_ORDERS || 0); -const TINY_SEARCH_DELAY_MS = Number(process.env.TINY_SEARCH_DELAY_MS || 1000); -const TINY_DETAIL_DELAY_MS = Number(process.env.TINY_DETAIL_DELAY_MS || 1500); -const TINY_CONTACT_DELAY_MS = Number(process.env.TINY_CONTACT_DELAY_MS || TINY_DETAIL_DELAY_MS); +const numberEnv = (name: string, fallback: number) => { + const value = Number(process.env[name] ?? fallback); + return Number.isFinite(value) && value >= 0 ? value : fallback; +}; +const MAX_DAYS = numberEnv('GRAPHS_BACKFILL_MAX_DAYS', 0); +const MAX_ORDERS = numberEnv('GRAPHS_BACKFILL_MAX_ORDERS', 0); +const TINY_SEARCH_DELAY_MS = numberEnv('TINY_SEARCH_DELAY_MS', 1000); +const TINY_DETAIL_DELAY_MS = numberEnv('TINY_DETAIL_DELAY_MS', 1500); +const TINY_CONTACT_DELAY_MS = numberEnv('TINY_CONTACT_DELAY_MS', TINY_DETAIL_DELAY_MS); +const ORDER_DELAY_MS = numberEnv('GRAPHS_BACKFILL_ORDER_DELAY_MS', 6000); +const TINY_BLOCK_DELAY_MS = numberEnv('GRAPHS_BACKFILL_TINY_BLOCK_DELAY_MS', 600000); +const MAX_TINY_BLOCK_RETRIES = numberEnv('GRAPHS_BACKFILL_MAX_TINY_BLOCK_RETRIES', 0); const STATE_FILE = process.env.GRAPHS_BACKFILL_STATE_FILE || path.join(process.cwd(), 'graphs_backfill_state.json'); +const STOP_FILE = process.env.GRAPHS_BACKFILL_STOP_FILE || path.join(process.cwd(), 'graphs_backfill_stop.json'); const contactCache = new Map(); const sleep = (ms: number) => new Promise(resolve => setTimeout(resolve, ms)); +class BackfillStoppedError extends Error { + constructor() { + super('Graphs backfill stopped by request.'); + } +} + function getTodayDate() { const date = new Date(); return formatDate(date); @@ -130,6 +150,26 @@ function saveState(state: BackfillState) { }, null, 2)); } +function assertNotStopped() { + if (fs.existsSync(STOP_FILE)) { + throw new BackfillStoppedError(); + } +} + +async function sleepWithStop(ms: number, reason: string) { + if (ms <= 0) return; + + const startedAt = Date.now(); + let remaining = ms; + console.log(`[wait] Waiting ${ms}ms before ${reason}.`); + + while (remaining > 0) { + assertNotStopped(); + await sleep(Math.min(remaining, 1000)); + remaining = ms - (Date.now() - startedAt); + } +} + function normalizeNumber(value: unknown) { if (typeof value === 'number') return Number.isFinite(value) ? value : 0; if (value === undefined || value === null) return 0; @@ -167,12 +207,46 @@ function pickFirst(...values: unknown[]) { return ''; } -async function tinyPost(servicePhp: string, params: URLSearchParams) { - return axios.post(`https://api.tiny.com.br/api2/${servicePhp}`, params, { - headers: { 'Content-Type': 'application/x-www-form-urlencoded' } +function getTinyErrors(data: any) { + const errors = data?.retorno?.erros; + if (!errors) return []; + return Array.isArray(errors) ? errors : [errors]; +} + +function isTinyApiBlocked(data: any) { + return getTinyErrors(data).some((entry: any) => { + const message = String(entry?.erro || entry || '').toLowerCase(); + return message.includes('api bloqueada') || message.includes('excedido o número de acessos'); }); } +async function tinyPost(servicePhp: string, params: URLSearchParams) { + let blockedAttempts = 0; + + while (true) { + assertNotStopped(); + + const response = await axios.post(`https://api.tiny.com.br/api2/${servicePhp}`, params, { + headers: { 'Content-Type': 'application/x-www-form-urlencoded' } + }); + + if (!isTinyApiBlocked(response.data)) { + return response; + } + + blockedAttempts += 1; + if (MAX_TINY_BLOCK_RETRIES && blockedAttempts > MAX_TINY_BLOCK_RETRIES) { + return response; + } + + const retryLabel = MAX_TINY_BLOCK_RETRIES + ? `${blockedAttempts}/${MAX_TINY_BLOCK_RETRIES}` + : String(blockedAttempts); + console.warn(`[Tiny] API blocked on ${servicePhp}. Pausing backfill for ${TINY_BLOCK_DELAY_MS}ms before retry ${retryLabel}.`); + await sleepWithStop(TINY_BLOCK_DELAY_MS, 'retrying Tiny API after block'); + } +} + async function fetchOrdersForDate(dateStr: string, page = 1): Promise { console.log(`[Tiny] Searching orders for ${dateStr}, page ${page}`); @@ -184,7 +258,7 @@ async function fetchOrdersForDate(dateStr: string, page = 1): Promise { if (cpfCnpj) searchParams.append('cpf_cnpj', cpfCnpj); const searchResponse = await tinyPost('contatos.pesquisa.php', searchParams); - await sleep(TINY_CONTACT_DELAY_MS); + await sleepWithStop(TINY_CONTACT_DELAY_MS, 'next Tiny contact request'); const contatoResumo = searchResponse.data?.retorno?.contatos?.[0]?.contato; const contatoId = contatoResumo?.id; @@ -265,7 +340,7 @@ async function fetchContactInfo(cliente: any): Promise { detailParams.append('id', String(contatoId)); const detailResponse = await tinyPost('contato.obter.php', detailParams); - await sleep(TINY_CONTACT_DELAY_MS); + await sleepWithStop(TINY_CONTACT_DELAY_MS, 'next Tiny contact request'); if (detailResponse.data?.retorno?.status !== 'OK') { contactCache.set(cacheKey, null); @@ -281,6 +356,7 @@ async function fetchContactInfo(cliente: any): Promise { contactCache.set(cacheKey, info); return info; } catch (error: any) { + if (error instanceof BackfillStoppedError) throw error; console.error(`[Tiny] Contact fallback failed for ${name || cpfCnpj}: ${error.message}`); contactCache.set(cacheKey, null); return null; @@ -372,16 +448,32 @@ async function runBackfill() { const state = loadState(); let currentDate = state.nextDate; + let lastCompletedDate = state.lastCompletedDate; let processedDays = 0; let processedOrders = 0; + const orderPace = ORDER_DELAY_MS ? `~${Math.floor(60000 / ORDER_DELAY_MS)}/minute` : 'unlimited'; console.log(`[config] Start date: ${currentDate}`); console.log(`[config] End date: ${END_DATE}`); console.log(`[config] Dry run: ${DRY_RUN ? 'yes' : 'no'}`); console.log(`[config] Contact fallback: ${CONTACT_FALLBACK ? 'yes' : 'no'}`); + console.log(`[config] Order pace: 1 order every ${ORDER_DELAY_MS}ms (${orderPace})`); + console.log(`[config] Tiny block pause: ${TINY_BLOCK_DELAY_MS}ms`); console.log(`[config] State file: ${STATE_FILE}`); + console.log(`[config] Stop file: ${STOP_FILE}`); + + saveState({ + ...state, + status: 'running', + currentDate, + processedDays, + processedOrders, + lastMessage: 'Backfill started' + }); while (!isAfter(currentDate, END_DATE)) { + assertNotStopped(); + if (MAX_DAYS && processedDays >= MAX_DAYS) { console.log(`[limit] Stopped after ${MAX_DAYS} day(s).`); break; @@ -392,8 +484,19 @@ async function runBackfill() { console.log(`[Tiny] Found ${summaries.length} order(s) for ${currentDate}`); for (const summary of summaries) { + assertNotStopped(); + if (MAX_ORDERS && processedOrders >= MAX_ORDERS) { console.log(`[limit] Stopped after ${MAX_ORDERS} order(s).`); + saveState({ + nextDate: currentDate, + lastCompletedDate, + status: 'completed', + currentDate, + processedDays, + processedOrders, + lastMessage: `Stopped after ${MAX_ORDERS} order(s)` + }); return; } @@ -409,22 +512,63 @@ async function runBackfill() { const rows = await buildGraphsRows(details, summary); await sendRowsToGraphs(rows, orderId); processedOrders += 1; + + saveState({ + nextDate: currentDate, + lastCompletedDate, + status: 'running', + currentDate, + lastOrderId: orderId, + processedDays, + processedOrders, + lastMessage: `Processed order ${orderId}` + }); + + if (!MAX_ORDERS || processedOrders < MAX_ORDERS) { + await sleepWithStop(ORDER_DELAY_MS, 'next order'); + } } const completedDate = currentDate; currentDate = getNextDate(currentDate); + lastCompletedDate = completedDate; processedDays += 1; saveState({ nextDate: currentDate, - lastCompletedDate: completedDate + lastCompletedDate, + status: 'running', + currentDate, + processedDays, + processedOrders, + lastMessage: `Completed ${completedDate}` }); console.log(`[state] Completed ${completedDate}. Next date: ${currentDate}`); } + saveState({ + nextDate: currentDate, + lastCompletedDate, + status: 'completed', + currentDate, + processedDays, + processedOrders, + lastMessage: 'Backfill completed' + }); console.log(`\nDone. Processed ${processedOrders} order(s) across ${processedDays} day(s).`); } runBackfill().catch((error: any) => { + if (error instanceof BackfillStoppedError) { + console.log('Backfill stopped by request.'); + const state = loadState(); + saveState({ + ...state, + status: 'stopped', + lastMessage: 'Backfill stopped by request' + }); + process.exit(0); + } + console.error(`Backfill failed: ${error.message}`); process.exit(1); });