fix: throttle Graphs backfill by Tiny request

docker-compose.yml

@@ -19,6 +19,7 @@ services:
       - GRAPHS_API_URL=${GRAPHS_API_URL}
       - GRAPHS_API_KEY=${GRAPHS_API_KEY}
       - GRAPHS_BACKFILL_CONTACT_FALLBACK=${GRAPHS_BACKFILL_CONTACT_FALLBACK}
+      - GRAPHS_BACKFILL_TINY_REQUEST_DELAY_MS=${GRAPHS_BACKFILL_TINY_REQUEST_DELAY_MS}
       - GRAPHS_BACKFILL_ORDER_DELAY_MS=${GRAPHS_BACKFILL_ORDER_DELAY_MS}
       - GRAPHS_BACKFILL_TINY_BLOCK_DELAY_MS=${GRAPHS_BACKFILL_TINY_BLOCK_DELAY_MS}
       - GRAPHS_BACKFILL_MAX_TINY_BLOCK_RETRIES=${GRAPHS_BACKFILL_MAX_TINY_BLOCK_RETRIES}

src/controllers/webhook.controller.ts

@@ -12,7 +12,10 @@ type CachedOrderDetails = {
     whatsappVendedor: string;
 };
 
+type OrderDetailsResult = Omit<CachedOrderDetails, 'expiresAt'>;
+
 const orderDetailsCache = new Map<string, CachedOrderDetails>();
+const inFlightOrderDetails = new Map<string, Promise<OrderDetailsResult | null>>();
 const configuredCacheTtlMs = Number(process.env.TINY_ORDER_DETAILS_CACHE_TTL_MS || 120000);
 const ORDER_DETAILS_CACHE_TTL_MS = Number.isFinite(configuredCacheTtlMs) ? configuredCacheTtlMs : 120000;
 
@@ -47,6 +50,88 @@ function setCachedOrderDetails(orderId: string, cached: Omit<CachedOrderDetails,
     });
 }
 
+async function fetchOrderDetailsFromTiny(orderId: string, tinyApiToken: string): Promise<OrderDetailsResult | null> {
+    console.log(`Fetching full details for Order ID: ${orderId} from Tiny API...`);
+    const params = new URLSearchParams();
+    params.append('token', tinyApiToken);
+    params.append('id', orderId);
+    params.append('formato', 'JSON');
+
+    const apiResponse = await axios.post('https://api.tiny.com.br/api2/pedido.obter.php', params, {
+        headers: { 'Content-Type': 'application/x-www-form-urlencoded' }
+    });
+
+    if (apiResponse.data?.retorno?.status !== 'OK') {
+        console.error('Tiny API returned an error:', apiResponse.data?.retorno?.erros || 'Unknown error');
+        return null;
+    }
+
+    const fullOrderDetails = apiResponse.data.retorno.pedido;
+    const statusProcessamento = apiResponse.data.retorno.status_processamento || "";
+    let whatsappVendedor = "";
+    console.log(`Successfully fetched order details! Found phone: ${fullOrderDetails.cliente?.celular || fullOrderDetails.cliente?.fone || 'None'}`);
+
+    // Seller details are off by default because this is an extra Tiny API call per order.
+    const idVendedor = fullOrderDetails.id_vendedor;
+    if (process.env.TINY_FETCH_SELLER_DETAILS === 'true' && idVendedor && idVendedor !== "0") {
+        console.log(`Fetching seller details for Seller ID: ${idVendedor}...`);
+        const vendorParams = new URLSearchParams();
+        vendorParams.append('token', tinyApiToken);
+        vendorParams.append('id', idVendedor);
+        vendorParams.append('formato', 'JSON');
+
+        const vendorResponse = await axios.post('https://api.tiny.com.br/api2/contato.obter.php', vendorParams, {
+            headers: { 'Content-Type': 'application/x-www-form-urlencoded' }
+        });
+
+        if (vendorResponse.data?.retorno?.status === 'OK') {
+            const contato = vendorResponse.data.retorno.contato;
+            whatsappVendedor = contato?.celular || contato?.fone || contato?.telefone || "";
+            console.log(`Successfully fetched seller WhatsApp: ${whatsappVendedor || 'None'}`);
+        } else {
+            console.error('Failed to fetch seller details:', JSON.stringify(vendorResponse.data?.retorno?.erros || 'Unknown API error'));
+        }
+    }
+
+    return {
+        fullOrderDetails,
+        statusProcessamento,
+        whatsappVendedor
+    };
+}
+
+async function getOrFetchOrderDetails(orderId: string, tinyApiToken: string): Promise<OrderDetailsResult | null> {
+    const cachedDetails = getCachedOrderDetails(orderId);
+    if (cachedDetails) {
+        console.log(`Using cached order details for Order ID: ${orderId}.`);
+        return {
+            fullOrderDetails: cachedDetails.fullOrderDetails,
+            statusProcessamento: cachedDetails.statusProcessamento,
+            whatsappVendedor: cachedDetails.whatsappVendedor
+        };
+    }
+
+    const existingFetch = inFlightOrderDetails.get(orderId);
+    if (existingFetch) {
+        console.log(`Waiting for in-flight Tiny fetch for Order ID: ${orderId}.`);
+        return existingFetch;
+    }
+
+    const fetchPromise = fetchOrderDetailsFromTiny(orderId, tinyApiToken)
+        .then(result => {
+            if (result) {
+                setCachedOrderDetails(orderId, result);
+            }
+            return result;
+        })
+        .finally(() => {
+            inFlightOrderDetails.delete(orderId);
+        });
+
+    inFlightOrderDetails.set(orderId, fetchPromise);
+    return fetchPromise;
+}
+
 export const handleTinyOrderUpdate = async (req: Request, res: Response): Promise<void> => {
   try {
     // 1. Security Check: Verify token from Tiny
@@ -84,58 +169,13 @@ export const handleTinyOrderUpdate = async (req: Request, res: Response): Promis
     let statusProcessamento = "";
     let whatsappVendedor = "";
 
-    const cachedDetails = getCachedOrderDetails(String(orderId));
+    if (tinyApiToken) {
-    if (cachedDetails) {
-        fullOrderDetails = cachedDetails.fullOrderDetails;
-        statusProcessamento = cachedDetails.statusProcessamento;
-        whatsappVendedor = cachedDetails.whatsappVendedor;
-        console.log(`Using cached order details for Order ID: ${orderId}.`);
-    } else if (tinyApiToken) {
         try {
-            console.log(`Fetching full details for Order ID: ${orderId} from Tiny API...`);
+            const details = await getOrFetchOrderDetails(String(orderId), tinyApiToken);
-            const params = new URLSearchParams();
+            if (details) {
-            params.append('token', tinyApiToken);
+                fullOrderDetails = details.fullOrderDetails;
-            params.append('id', orderId);
+                statusProcessamento = details.statusProcessamento;
-            params.append('formato', 'JSON');
+                whatsappVendedor = details.whatsappVendedor;
-
-            const apiResponse = await axios.post('https://api.tiny.com.br/api2/pedido.obter.php', params, {
-                headers: { 'Content-Type': 'application/x-www-form-urlencoded' }
-            });
-
-            if (apiResponse.data?.retorno?.status === 'OK') {
-                fullOrderDetails = apiResponse.data.retorno.pedido;
-                statusProcessamento = apiResponse.data.retorno.status_processamento || "";
-                console.log(`Successfully fetched order details! Found phone: ${fullOrderDetails.cliente?.celular || fullOrderDetails.cliente?.fone || 'None'}`);
-
-                // OPTION B: Fetch Vendor's WhatsApp
-                const idVendedor = fullOrderDetails.id_vendedor;
-                if (process.env.TINY_FETCH_SELLER_DETAILS === 'true' && idVendedor && idVendedor !== "0") {
-                    console.log(`Fetching seller details for Seller ID: ${idVendedor}...`);
-                    const vendorParams = new URLSearchParams();
-                    vendorParams.append('token', tinyApiToken);
-                    vendorParams.append('id', idVendedor);
-                    vendorParams.append('formato', 'JSON');
-
-                    const vendorResponse = await axios.post('https://api.tiny.com.br/api2/contato.obter.php', vendorParams, {
-                        headers: { 'Content-Type': 'application/x-www-form-urlencoded' }
-                    });
-
-                    if (vendorResponse.data?.retorno?.status === 'OK') {
-                        const contato = vendorResponse.data.retorno.contato;
-                        whatsappVendedor = contato?.celular || contato?.fone || contato?.telefone || "";
-                        console.log(`Successfully fetched seller WhatsApp: ${whatsappVendedor || 'None'}`);
-                    } else {
-                        console.error('Failed to fetch seller details:', JSON.stringify(vendorResponse.data?.retorno?.erros || 'Unknown API error'));
-                    }
-                }
-
-                setCachedOrderDetails(String(orderId), {
-                    fullOrderDetails,
-                    statusProcessamento,
-                    whatsappVendedor
-                });
-            } else {
-                console.error('Tiny API returned an error:', apiResponse.data?.retorno?.erros || 'Unknown error');
             }
         } catch (apiError: any) {
             console.error('Failed to fetch from Tiny API:', apiError.message);

src/index.ts

@@ -60,6 +60,7 @@ app.get('/api/trigger-graphs-backfill', (req: Request, res: Response) => {
     maxOrders: 'GRAPHS_BACKFILL_MAX_ORDERS',
     maxDays: 'GRAPHS_BACKFILL_MAX_DAYS',
     contactFallback: 'GRAPHS_BACKFILL_CONTACT_FALLBACK',
+    tinyRequestDelayMs: 'GRAPHS_BACKFILL_TINY_REQUEST_DELAY_MS',
     orderDelayMs: 'GRAPHS_BACKFILL_ORDER_DELAY_MS',
     blockDelayMs: 'GRAPHS_BACKFILL_TINY_BLOCK_DELAY_MS',
     maxTinyBlockRetries: 'GRAPHS_BACKFILL_MAX_TINY_BLOCK_RETRIES'
@@ -95,7 +96,8 @@ app.get('/api/trigger-graphs-backfill', (req: Request, res: Response) => {
       maxOrders: env.GRAPHS_BACKFILL_MAX_ORDERS || null,
       maxDays: env.GRAPHS_BACKFILL_MAX_DAYS || null,
       contactFallback: env.GRAPHS_BACKFILL_CONTACT_FALLBACK || 'true',
-      orderDelayMs: env.GRAPHS_BACKFILL_ORDER_DELAY_MS || '6000',
+      tinyRequestDelayMs: env.GRAPHS_BACKFILL_TINY_REQUEST_DELAY_MS || '6000',
+      orderDelayMs: env.GRAPHS_BACKFILL_ORDER_DELAY_MS || '0',
       blockDelayMs: env.GRAPHS_BACKFILL_TINY_BLOCK_DELAY_MS || '600000',
       maxTinyBlockRetries: env.GRAPHS_BACKFILL_MAX_TINY_BLOCK_RETRIES || null,
       graphsApiUrlConfigured: Boolean(env.GRAPHS_API_URL || env.NEXSTAR_GRAPHS_API_URL),

src/scripts/backfill-graphs.ts

@@ -67,16 +67,18 @@ const numberEnv = (name: string, fallback: number) => {
 };
 const MAX_DAYS = numberEnv('GRAPHS_BACKFILL_MAX_DAYS', 0);
 const MAX_ORDERS = numberEnv('GRAPHS_BACKFILL_MAX_ORDERS', 0);
-const TINY_SEARCH_DELAY_MS = numberEnv('TINY_SEARCH_DELAY_MS', 1000);
+const TINY_REQUEST_DELAY_MS = numberEnv('GRAPHS_BACKFILL_TINY_REQUEST_DELAY_MS', 6000);
-const TINY_DETAIL_DELAY_MS = numberEnv('TINY_DETAIL_DELAY_MS', 1500);
+const TINY_SEARCH_DELAY_MS = numberEnv('TINY_SEARCH_DELAY_MS', 0);
+const TINY_DETAIL_DELAY_MS = numberEnv('TINY_DETAIL_DELAY_MS', 0);
 const TINY_CONTACT_DELAY_MS = numberEnv('TINY_CONTACT_DELAY_MS', TINY_DETAIL_DELAY_MS);
-const ORDER_DELAY_MS = numberEnv('GRAPHS_BACKFILL_ORDER_DELAY_MS', 6000);
+const ORDER_DELAY_MS = numberEnv('GRAPHS_BACKFILL_ORDER_DELAY_MS', 0);
 const TINY_BLOCK_DELAY_MS = numberEnv('GRAPHS_BACKFILL_TINY_BLOCK_DELAY_MS', 600000);
 const MAX_TINY_BLOCK_RETRIES = numberEnv('GRAPHS_BACKFILL_MAX_TINY_BLOCK_RETRIES', 0);
 const STATE_FILE = process.env.GRAPHS_BACKFILL_STATE_FILE || path.join(process.cwd(), 'graphs_backfill_state.json');
 const STOP_FILE = process.env.GRAPHS_BACKFILL_STOP_FILE || path.join(process.cwd(), 'graphs_backfill_stop.json');
 
 const contactCache = new Map<string, ContactInfo | null>();
+let lastTinyRequestAt = 0;
 
 const sleep = (ms: number) => new Promise(resolve => setTimeout(resolve, ms));
 
@@ -220,11 +222,19 @@ function isTinyApiBlocked(data: any) {
     });
 }
 
+async function waitForTinyRequestSlot(servicePhp: string) {
+    const elapsed = Date.now() - lastTinyRequestAt;
+    const waitMs = Math.max(0, TINY_REQUEST_DELAY_MS - elapsed);
+    await sleepWithStop(waitMs, `Tiny ${servicePhp} request slot`);
+    lastTinyRequestAt = Date.now();
+}
+
 async function tinyPost(servicePhp: string, params: URLSearchParams) {
     let blockedAttempts = 0;
 
     while (true) {
         assertNotStopped();
+        await waitForTinyRequestSlot(servicePhp);
 
         const response = await axios.post(`https://api.tiny.com.br/api2/${servicePhp}`, params, {
             headers: { 'Content-Type': 'application/x-www-form-urlencoded' }
@@ -452,11 +462,13 @@ async function runBackfill() {
     let processedDays = 0;
     let processedOrders = 0;
     const orderPace = ORDER_DELAY_MS ? `~${Math.floor(60000 / ORDER_DELAY_MS)}/minute` : 'unlimited';
+    const tinyRequestPace = TINY_REQUEST_DELAY_MS ? `~${Math.floor(60000 / TINY_REQUEST_DELAY_MS)}/minute` : 'unlimited';
 
     console.log(`[config] Start date: ${currentDate}`);
     console.log(`[config] End date: ${END_DATE}`);
     console.log(`[config] Dry run: ${DRY_RUN ? 'yes' : 'no'}`);
     console.log(`[config] Contact fallback: ${CONTACT_FALLBACK ? 'yes' : 'no'}`);
+    console.log(`[config] Tiny request pace: 1 request every ${TINY_REQUEST_DELAY_MS}ms (${tinyRequestPace})`);
     console.log(`[config] Order pace: 1 order every ${ORDER_DELAY_MS}ms (${orderPace})`);
     console.log(`[config] Tiny block pause: ${TINY_BLOCK_DELAY_MS}ms`);
     console.log(`[config] State file: ${STATE_FILE}`);