fix: throttle Graphs backfill by Tiny request

2026-06-23 10:03:22 -03:00
parent 44cfdbb4c9
commit 4d7daff496
4 changed files with 110 additions and 55 deletions
--- a/src/controllers/webhook.controller.ts
+++ b/src/controllers/webhook.controller.ts
@@ -12,7 +12,10 @@ type CachedOrderDetails = {
    whatsappVendedor: string;
 };

+type OrderDetailsResult = Omit<CachedOrderDetails, 'expiresAt'>;
+
 const orderDetailsCache = new Map<string, CachedOrderDetails>();
+const inFlightOrderDetails = new Map<string, Promise<OrderDetailsResult | null>>();
 const configuredCacheTtlMs = Number(process.env.TINY_ORDER_DETAILS_CACHE_TTL_MS || 120000);
 const ORDER_DETAILS_CACHE_TTL_MS = Number.isFinite(configuredCacheTtlMs) ? configuredCacheTtlMs : 120000;

@@ -47,6 +50,88 @@ function setCachedOrderDetails(orderId: string, cached: Omit<CachedOrderDetails,
    });
 }

+async function fetchOrderDetailsFromTiny(orderId: string, tinyApiToken: string): Promise<OrderDetailsResult | null> {
+    console.log(`Fetching full details for Order ID: ${orderId} from Tiny API...`);
+    const params = new URLSearchParams();
+    params.append('token', tinyApiToken);
+    params.append('id', orderId);
+    params.append('formato', 'JSON');
+
+    const apiResponse = await axios.post('https://api.tiny.com.br/api2/pedido.obter.php', params, {
+        headers: { 'Content-Type': 'application/x-www-form-urlencoded' }
+    });
+
+    if (apiResponse.data?.retorno?.status !== 'OK') {
+        console.error('Tiny API returned an error:', apiResponse.data?.retorno?.erros || 'Unknown error');
+        return null;
+    }
+
+    const fullOrderDetails = apiResponse.data.retorno.pedido;
+    const statusProcessamento = apiResponse.data.retorno.status_processamento || "";
+    let whatsappVendedor = "";
+    console.log(`Successfully fetched order details! Found phone: ${fullOrderDetails.cliente?.celular || fullOrderDetails.cliente?.fone || 'None'}`);
+
+    // Seller details are off by default because this is an extra Tiny API call per order.
+    const idVendedor = fullOrderDetails.id_vendedor;
+    if (process.env.TINY_FETCH_SELLER_DETAILS === 'true' && idVendedor && idVendedor !== "0") {
+        console.log(`Fetching seller details for Seller ID: ${idVendedor}...`);
+        const vendorParams = new URLSearchParams();
+        vendorParams.append('token', tinyApiToken);
+        vendorParams.append('id', idVendedor);
+        vendorParams.append('formato', 'JSON');
+
+        const vendorResponse = await axios.post('https://api.tiny.com.br/api2/contato.obter.php', vendorParams, {
+            headers: { 'Content-Type': 'application/x-www-form-urlencoded' }
+        });
+
+        if (vendorResponse.data?.retorno?.status === 'OK') {
+            const contato = vendorResponse.data.retorno.contato;
+            whatsappVendedor = contato?.celular || contato?.fone || contato?.telefone || "";
+            console.log(`Successfully fetched seller WhatsApp: ${whatsappVendedor || 'None'}`);
+        } else {
+            console.error('Failed to fetch seller details:', JSON.stringify(vendorResponse.data?.retorno?.erros || 'Unknown API error'));
+        }
+    }
+
+    return {
+        fullOrderDetails,
+        statusProcessamento,
+        whatsappVendedor
+    };
+}
+
+async function getOrFetchOrderDetails(orderId: string, tinyApiToken: string): Promise<OrderDetailsResult | null> {
+    const cachedDetails = getCachedOrderDetails(orderId);
+    if (cachedDetails) {
+        console.log(`Using cached order details for Order ID: ${orderId}.`);
+        return {
+            fullOrderDetails: cachedDetails.fullOrderDetails,
+            statusProcessamento: cachedDetails.statusProcessamento,
+            whatsappVendedor: cachedDetails.whatsappVendedor
+        };
+    }
+
+    const existingFetch = inFlightOrderDetails.get(orderId);
+    if (existingFetch) {
+        console.log(`Waiting for in-flight Tiny fetch for Order ID: ${orderId}.`);
+        return existingFetch;
+    }
+
+    const fetchPromise = fetchOrderDetailsFromTiny(orderId, tinyApiToken)
+        .then(result => {
+            if (result) {
+                setCachedOrderDetails(orderId, result);
+            }
+            return result;
+        })
+        .finally(() => {
+            inFlightOrderDetails.delete(orderId);
+        });
+
+    inFlightOrderDetails.set(orderId, fetchPromise);
+    return fetchPromise;
+}
+
 export const handleTinyOrderUpdate = async (req: Request, res: Response): Promise<void> => {
  try {
    // 1. Security Check: Verify token from Tiny
@@ -84,58 +169,13 @@ export const handleTinyOrderUpdate = async (req: Request, res: Response): Promis
    let statusProcessamento = "";
    let whatsappVendedor = "";

-    const cachedDetails = getCachedOrderDetails(String(orderId));
-    if (cachedDetails) {
-        fullOrderDetails = cachedDetails.fullOrderDetails;
-        statusProcessamento = cachedDetails.statusProcessamento;
-        whatsappVendedor = cachedDetails.whatsappVendedor;
-        console.log(`Using cached order details for Order ID: ${orderId}.`);
-    } else if (tinyApiToken) {
+    if (tinyApiToken) {
        try {
-            console.log(`Fetching full details for Order ID: ${orderId} from Tiny API...`);
-            const params = new URLSearchParams();
-            params.append('token', tinyApiToken);
-            params.append('id', orderId);
-            params.append('formato', 'JSON');
-
-            const apiResponse = await axios.post('https://api.tiny.com.br/api2/pedido.obter.php', params, {
-                headers: { 'Content-Type': 'application/x-www-form-urlencoded' }
-            });
-
-            if (apiResponse.data?.retorno?.status === 'OK') {
-                fullOrderDetails = apiResponse.data.retorno.pedido;
-                statusProcessamento = apiResponse.data.retorno.status_processamento || "";
-                console.log(`Successfully fetched order details! Found phone: ${fullOrderDetails.cliente?.celular || fullOrderDetails.cliente?.fone || 'None'}`);
-
-                // OPTION B: Fetch Vendor's WhatsApp
-                const idVendedor = fullOrderDetails.id_vendedor;
-                if (process.env.TINY_FETCH_SELLER_DETAILS === 'true' && idVendedor && idVendedor !== "0") {
-                    console.log(`Fetching seller details for Seller ID: ${idVendedor}...`);
-                    const vendorParams = new URLSearchParams();
-                    vendorParams.append('token', tinyApiToken);
-                    vendorParams.append('id', idVendedor);
-                    vendorParams.append('formato', 'JSON');
-
-                    const vendorResponse = await axios.post('https://api.tiny.com.br/api2/contato.obter.php', vendorParams, {
-                        headers: { 'Content-Type': 'application/x-www-form-urlencoded' }
-                    });
-
-                    if (vendorResponse.data?.retorno?.status === 'OK') {
-                        const contato = vendorResponse.data.retorno.contato;
-                        whatsappVendedor = contato?.celular || contato?.fone || contato?.telefone || "";
-                        console.log(`Successfully fetched seller WhatsApp: ${whatsappVendedor || 'None'}`);
-                    } else {
-                        console.error('Failed to fetch seller details:', JSON.stringify(vendorResponse.data?.retorno?.erros || 'Unknown API error'));
-                    }
-                }
-
-                setCachedOrderDetails(String(orderId), {
-                    fullOrderDetails,
-                    statusProcessamento,
-                    whatsappVendedor
-                });
-            } else {
-                console.error('Tiny API returned an error:', apiResponse.data?.retorno?.erros || 'Unknown error');
+            const details = await getOrFetchOrderDetails(String(orderId), tinyApiToken);
+            if (details) {
+                fullOrderDetails = details.fullOrderDetails;
+                statusProcessamento = details.statusProcessamento;
+                whatsappVendedor = details.whatsappVendedor;
            }
        } catch (apiError: any) {
            console.error('Failed to fetch from Tiny API:', apiError.message);
--- a/src/index.ts
+++ b/src/index.ts
@@ -60,6 +60,7 @@ app.get('/api/trigger-graphs-backfill', (req: Request, res: Response) => {
    maxOrders: 'GRAPHS_BACKFILL_MAX_ORDERS',
    maxDays: 'GRAPHS_BACKFILL_MAX_DAYS',
    contactFallback: 'GRAPHS_BACKFILL_CONTACT_FALLBACK',
+    tinyRequestDelayMs: 'GRAPHS_BACKFILL_TINY_REQUEST_DELAY_MS',
    orderDelayMs: 'GRAPHS_BACKFILL_ORDER_DELAY_MS',
    blockDelayMs: 'GRAPHS_BACKFILL_TINY_BLOCK_DELAY_MS',
    maxTinyBlockRetries: 'GRAPHS_BACKFILL_MAX_TINY_BLOCK_RETRIES'
@@ -95,7 +96,8 @@ app.get('/api/trigger-graphs-backfill', (req: Request, res: Response) => {
      maxOrders: env.GRAPHS_BACKFILL_MAX_ORDERS || null,
      maxDays: env.GRAPHS_BACKFILL_MAX_DAYS || null,
      contactFallback: env.GRAPHS_BACKFILL_CONTACT_FALLBACK || 'true',
-      orderDelayMs: env.GRAPHS_BACKFILL_ORDER_DELAY_MS || '6000',
+      tinyRequestDelayMs: env.GRAPHS_BACKFILL_TINY_REQUEST_DELAY_MS || '6000',
+      orderDelayMs: env.GRAPHS_BACKFILL_ORDER_DELAY_MS || '0',
      blockDelayMs: env.GRAPHS_BACKFILL_TINY_BLOCK_DELAY_MS || '600000',
      maxTinyBlockRetries: env.GRAPHS_BACKFILL_MAX_TINY_BLOCK_RETRIES || null,
      graphsApiUrlConfigured: Boolean(env.GRAPHS_API_URL || env.NEXSTAR_GRAPHS_API_URL),
--- a/src/scripts/backfill-graphs.ts
+++ b/src/scripts/backfill-graphs.ts
@@ -67,16 +67,18 @@ const numberEnv = (name: string, fallback: number) => {
 };
 const MAX_DAYS = numberEnv('GRAPHS_BACKFILL_MAX_DAYS', 0);
 const MAX_ORDERS = numberEnv('GRAPHS_BACKFILL_MAX_ORDERS', 0);
-const TINY_SEARCH_DELAY_MS = numberEnv('TINY_SEARCH_DELAY_MS', 1000);
-const TINY_DETAIL_DELAY_MS = numberEnv('TINY_DETAIL_DELAY_MS', 1500);
+const TINY_REQUEST_DELAY_MS = numberEnv('GRAPHS_BACKFILL_TINY_REQUEST_DELAY_MS', 6000);
+const TINY_SEARCH_DELAY_MS = numberEnv('TINY_SEARCH_DELAY_MS', 0);
+const TINY_DETAIL_DELAY_MS = numberEnv('TINY_DETAIL_DELAY_MS', 0);
 const TINY_CONTACT_DELAY_MS = numberEnv('TINY_CONTACT_DELAY_MS', TINY_DETAIL_DELAY_MS);
-const ORDER_DELAY_MS = numberEnv('GRAPHS_BACKFILL_ORDER_DELAY_MS', 6000);
+const ORDER_DELAY_MS = numberEnv('GRAPHS_BACKFILL_ORDER_DELAY_MS', 0);
 const TINY_BLOCK_DELAY_MS = numberEnv('GRAPHS_BACKFILL_TINY_BLOCK_DELAY_MS', 600000);
 const MAX_TINY_BLOCK_RETRIES = numberEnv('GRAPHS_BACKFILL_MAX_TINY_BLOCK_RETRIES', 0);
 const STATE_FILE = process.env.GRAPHS_BACKFILL_STATE_FILE || path.join(process.cwd(), 'graphs_backfill_state.json');
 const STOP_FILE = process.env.GRAPHS_BACKFILL_STOP_FILE || path.join(process.cwd(), 'graphs_backfill_stop.json');

 const contactCache = new Map<string, ContactInfo | null>();
+let lastTinyRequestAt = 0;

 const sleep = (ms: number) => new Promise(resolve => setTimeout(resolve, ms));

@@ -220,11 +222,19 @@ function isTinyApiBlocked(data: any) {
    });
 }

+async function waitForTinyRequestSlot(servicePhp: string) {
+    const elapsed = Date.now() - lastTinyRequestAt;
+    const waitMs = Math.max(0, TINY_REQUEST_DELAY_MS - elapsed);
+    await sleepWithStop(waitMs, `Tiny ${servicePhp} request slot`);
+    lastTinyRequestAt = Date.now();
+}
+
 async function tinyPost(servicePhp: string, params: URLSearchParams) {
    let blockedAttempts = 0;

    while (true) {
        assertNotStopped();
+        await waitForTinyRequestSlot(servicePhp);

        const response = await axios.post(`https://api.tiny.com.br/api2/${servicePhp}`, params, {
            headers: { 'Content-Type': 'application/x-www-form-urlencoded' }
@@ -452,11 +462,13 @@ async function runBackfill() {
    let processedDays = 0;
    let processedOrders = 0;
    const orderPace = ORDER_DELAY_MS ? `~${Math.floor(60000 / ORDER_DELAY_MS)}/minute` : 'unlimited';
+    const tinyRequestPace = TINY_REQUEST_DELAY_MS ? `~${Math.floor(60000 / TINY_REQUEST_DELAY_MS)}/minute` : 'unlimited';

    console.log(`[config] Start date: ${currentDate}`);
    console.log(`[config] End date: ${END_DATE}`);
    console.log(`[config] Dry run: ${DRY_RUN ? 'yes' : 'no'}`);
    console.log(`[config] Contact fallback: ${CONTACT_FALLBACK ? 'yes' : 'no'}`);
+    console.log(`[config] Tiny request pace: 1 request every ${TINY_REQUEST_DELAY_MS}ms (${tinyRequestPace})`);
    console.log(`[config] Order pace: 1 order every ${ORDER_DELAY_MS}ms (${orderPace})`);
    console.log(`[config] Tiny block pause: ${TINY_BLOCK_DELAY_MS}ms`);
    console.log(`[config] State file: ${STATE_FILE}`);