fix: throttle Tiny API usage across backfills

docker-compose.yml

@@ -16,6 +16,13 @@ services:
       - TINY_WEBHOOK_SECRET=${TINY_WEBHOOK_SECRET}
       - N8N_AUTH_TOKEN=${N8N_AUTH_TOKEN}
       - TINY_API_TOKEN=${TINY_API_TOKEN}
+      - GRAPHS_API_URL=${GRAPHS_API_URL}
+      - GRAPHS_API_KEY=${GRAPHS_API_KEY}
+      - GRAPHS_BACKFILL_CONTACT_FALLBACK=${GRAPHS_BACKFILL_CONTACT_FALLBACK}
+      - TINY_MIN_REQUEST_INTERVAL_MS=${TINY_MIN_REQUEST_INTERVAL_MS}
+      - TINY_BLOCK_RETRY_DELAY_MS=${TINY_BLOCK_RETRY_DELAY_MS}
+      - TINY_BLOCK_MAX_RETRIES=${TINY_BLOCK_MAX_RETRIES}
+      - TINY_FETCH_SELLER_DETAILS=${TINY_FETCH_SELLER_DETAILS}
     # Optional: If you use a shared Docker network for Nginx Proxy Manager, you can attach it here:
     # networks:
     #   - nginx-proxy-manager-network

src/controllers/webhook.controller.ts

@@ -1,7 +1,7 @@
 import { Request, Response } from 'express';
 import { sendToN8n } from '../services/n8n.service';
 import { sendTinyOrderToGraphs } from '../services/graphs.service';
-import axios from 'axios';
+import { tinyPost } from '../services/tiny-api.service';
 import fs from 'fs';
 import path from 'path';
 
@@ -50,9 +50,7 @@ export const handleTinyOrderUpdate = async (req: Request, res: Response): Promis
             params.append('id', orderId);
             params.append('formato', 'JSON');
 
-            const apiResponse = await axios.post('https://api.tiny.com.br/api2/pedido.obter.php', params, {
-                headers: { 'Content-Type': 'application/x-www-form-urlencoded' }
-            });
+            const apiResponse = await tinyPost('pedido.obter.php', params);
 
             if (apiResponse.data?.retorno?.status === 'OK') {
                 fullOrderDetails = apiResponse.data.retorno.pedido;
@@ -61,16 +59,14 @@ export const handleTinyOrderUpdate = async (req: Request, res: Response): Promis
 
                 // OPTION B: Fetch Vendor's WhatsApp
                 const idVendedor = fullOrderDetails.id_vendedor;
-                if (idVendedor && idVendedor !== "0") {
+                if (process.env.TINY_FETCH_SELLER_DETAILS !== 'false' && idVendedor && idVendedor !== "0") {
                     console.log(`Fetching seller details for Seller ID: ${idVendedor}...`);
                     const vendorParams = new URLSearchParams();
                     vendorParams.append('token', tinyApiToken);
                     vendorParams.append('id', idVendedor);
                     vendorParams.append('formato', 'JSON');
 
-                    const vendorResponse = await axios.post('https://api.tiny.com.br/api2/contato.obter.php', vendorParams, {
-                        headers: { 'Content-Type': 'application/x-www-form-urlencoded' }
-                    });
+                    const vendorResponse = await tinyPost('contato.obter.php', vendorParams);
 
                     if (vendorResponse.data?.retorno?.status === 'OK') {
                         const contato = vendorResponse.data.retorno.contato;

src/index.ts

@@ -85,7 +85,11 @@ app.get('/api/trigger-graphs-backfill', (req: Request, res: Response) => {
       dryRun: env.GRAPHS_BACKFILL_DRY_RUN || 'false',
       maxOrders: env.GRAPHS_BACKFILL_MAX_ORDERS || null,
       maxDays: env.GRAPHS_BACKFILL_MAX_DAYS || null,
-      contactFallback: env.GRAPHS_BACKFILL_CONTACT_FALLBACK || 'true'
+      contactFallback: env.GRAPHS_BACKFILL_CONTACT_FALLBACK || 'true',
+      graphsApiUrlConfigured: Boolean(env.GRAPHS_API_URL || env.NEXSTAR_GRAPHS_API_URL),
+      graphsApiKeyConfigured: Boolean(env.GRAPHS_API_KEY || env.NEXSTAR_GRAPHS_API_KEY),
+      tinyMinRequestIntervalMs: env.TINY_MIN_REQUEST_INTERVAL_MS || null,
+      tinyFetchSellerDetails: env.TINY_FETCH_SELLER_DETAILS || 'true'
     }
   });
 });

src/scripts/backfill-graphs.ts

@@ -2,6 +2,7 @@ import axios from 'axios';
 import dotenv from 'dotenv';
 import fs from 'fs';
 import path from 'path';
+import { tinyPost } from '../services/tiny-api.service';
 
 dotenv.config();
 
@@ -57,9 +58,11 @@ const START_DATE = process.env.GRAPHS_BACKFILL_START_DATE || '13/02/2025';
 const END_DATE = process.env.GRAPHS_BACKFILL_END_DATE || getTodayDate();
 const MAX_DAYS = Number(process.env.GRAPHS_BACKFILL_MAX_DAYS || 0);
 const MAX_ORDERS = Number(process.env.GRAPHS_BACKFILL_MAX_ORDERS || 0);
-const TINY_SEARCH_DELAY_MS = Number(process.env.TINY_SEARCH_DELAY_MS || 1000);
-const TINY_DETAIL_DELAY_MS = Number(process.env.TINY_DETAIL_DELAY_MS || 1500);
-const TINY_CONTACT_DELAY_MS = Number(process.env.TINY_CONTACT_DELAY_MS || TINY_DETAIL_DELAY_MS);
+const TINY_MIN_REQUEST_INTERVAL_MS = Number(process.env.TINY_MIN_REQUEST_INTERVAL_MS || 0);
+const USE_SHARED_RATE_LIMIT = TINY_MIN_REQUEST_INTERVAL_MS > 0;
+const TINY_SEARCH_DELAY_MS = USE_SHARED_RATE_LIMIT ? 0 : Number(process.env.TINY_SEARCH_DELAY_MS || 1000);
+const TINY_DETAIL_DELAY_MS = USE_SHARED_RATE_LIMIT ? 0 : Number(process.env.TINY_DETAIL_DELAY_MS || 1500);
+const TINY_CONTACT_DELAY_MS = USE_SHARED_RATE_LIMIT ? 0 : Number(process.env.TINY_CONTACT_DELAY_MS || TINY_DETAIL_DELAY_MS);
 const STATE_FILE = process.env.GRAPHS_BACKFILL_STATE_FILE || path.join(process.cwd(), 'graphs_backfill_state.json');
 
 const contactCache = new Map<string, ContactInfo | null>();
@@ -167,12 +170,6 @@ function pickFirst(...values: unknown[]) {
     return '';
 }
 
-async function tinyPost(servicePhp: string, params: URLSearchParams) {
-    return axios.post(`https://api.tiny.com.br/api2/${servicePhp}`, params, {
-        headers: { 'Content-Type': 'application/x-www-form-urlencoded' }
-    });
-}
-
 async function fetchOrdersForDate(dateStr: string, page = 1): Promise<TinyOrderSummary[]> {
     console.log(`[Tiny] Searching orders for ${dateStr}, page ${page}`);
 
@@ -379,6 +376,12 @@ async function runBackfill() {
     console.log(`[config] End date: ${END_DATE}`);
     console.log(`[config] Dry run: ${DRY_RUN ? 'yes' : 'no'}`);
     console.log(`[config] Contact fallback: ${CONTACT_FALLBACK ? 'yes' : 'no'}`);
+    if (USE_SHARED_RATE_LIMIT) {
+        const requestsPerMinute = Math.floor(60000 / TINY_MIN_REQUEST_INTERVAL_MS);
+        console.log(`[config] Shared Tiny rate limit: 1 request every ${TINY_MIN_REQUEST_INTERVAL_MS}ms (~${requestsPerMinute}/minute)`);
+    } else {
+        console.log('[config] Shared Tiny rate limit: disabled');
+    }
     console.log(`[config] State file: ${STATE_FILE}`);
 
     while (!isAfter(currentDate, END_DATE)) {

src/scripts/backfill.ts

@@ -2,6 +2,7 @@ import axios from 'axios';
 import dotenv from 'dotenv';
 import fs from 'fs';
 import path from 'path';
+import { tinyPost } from '../services/tiny-api.service';
 
 dotenv.config();
 
@@ -50,9 +51,7 @@ const fetchOrdersForDate = async (dateStr: string, page: number = 1): Promise<an
     params.append('pagina', page.toString());
 
     try {
-        const response = await axios.post('https://api.tiny.com.br/api2/pedidos.pesquisa.php', params, {
-            headers: { 'Content-Type': 'application/x-www-form-urlencoded' }
-        });
+        const response = await tinyPost('pedidos.pesquisa.php', params);
         
         await sleep(1000); // 1 second delay
 
@@ -92,9 +91,7 @@ const fetchOrderDetails = async (orderId: string) => {
     params.append('formato', 'JSON');
 
     try {
-        const response = await axios.post('https://api.tiny.com.br/api2/pedido.obter.php', params, {
-            headers: { 'Content-Type': 'application/x-www-form-urlencoded' }
-        });
+        const response = await tinyPost('pedido.obter.php', params);
         
         await sleep(1500); // 1.5 second delay. This is the heavy part.
 
@@ -184,4 +181,4 @@ const runBackfill = async () => {
     console.log('\n✅ BACKFILL COMPLETE! Caught up to today.');
 };
 
-runBackfill();
+runBackfill();

src/services/tiny-api.service.ts

@@ -0,0 +1,127 @@
+import axios from 'axios';
+import fs from 'fs';
+import os from 'os';
+import path from 'path';
+
+const DEFAULT_LOCK_DIR = path.join(os.tmpdir(), 'api-tiny-n8n-rate-limit.lock');
+const DEFAULT_STATE_FILE = path.join(os.tmpdir(), 'api-tiny-n8n-rate-limit.json');
+const LOCK_STALE_MS = 30000;
+const DEFAULT_BLOCK_RETRY_DELAY_MS = 120000;
+const DEFAULT_BLOCK_MAX_RETRIES = 30;
+
+const sleep = (ms: number) => new Promise(resolve => setTimeout(resolve, ms));
+
+const getPositiveNumberEnv = (name: string, fallback: number) => {
+    const configured = Number(process.env[name] || fallback);
+    return Number.isFinite(configured) && configured > 0 ? configured : 0;
+};
+
+const getMinIntervalMs = () => getPositiveNumberEnv('TINY_MIN_REQUEST_INTERVAL_MS', 0);
+const getBlockRetryDelayMs = () => getPositiveNumberEnv('TINY_BLOCK_RETRY_DELAY_MS', DEFAULT_BLOCK_RETRY_DELAY_MS);
+const getBlockMaxRetries = () => getPositiveNumberEnv('TINY_BLOCK_MAX_RETRIES', DEFAULT_BLOCK_MAX_RETRIES);
+const getLockDir = () => process.env.TINY_RATE_LIMIT_LOCK_DIR || DEFAULT_LOCK_DIR;
+const getStateFile = () => process.env.TINY_RATE_LIMIT_STATE_FILE || DEFAULT_STATE_FILE;
+
+const readLastRequestAt = () => {
+    try {
+        const parsed = JSON.parse(fs.readFileSync(getStateFile(), 'utf-8'));
+        return Number(parsed.lastRequestAt || 0);
+    } catch {
+        return 0;
+    }
+};
+
+const writeLastRequestAt = (timestamp: number) => {
+    fs.writeFileSync(getStateFile(), JSON.stringify({ lastRequestAt: timestamp }, null, 2));
+};
+
+const removeStaleLock = (lockDir: string) => {
+    try {
+        const stats = fs.statSync(lockDir);
+        if (Date.now() - stats.mtimeMs > LOCK_STALE_MS) {
+            fs.rmSync(lockDir, { recursive: true, force: true });
+        }
+    } catch {
+        // Lock does not exist or cannot be inspected. The acquire loop will retry.
+    }
+};
+
+const acquireLock = async () => {
+    const lockDir = getLockDir();
+
+    while (true) {
+        try {
+            fs.mkdirSync(lockDir);
+            return lockDir;
+        } catch (error: any) {
+            if (error?.code !== 'EEXIST') throw error;
+            removeStaleLock(lockDir);
+            await sleep(100);
+        }
+    }
+};
+
+const releaseLock = (lockDir: string) => {
+    fs.rmSync(lockDir, { recursive: true, force: true });
+};
+
+const waitForTinySlot = async () => {
+    const minIntervalMs = getMinIntervalMs();
+    if (!minIntervalMs) return;
+
+    const lockDir = await acquireLock();
+
+    try {
+        const lastRequestAt = readLastRequestAt();
+        const elapsed = Date.now() - lastRequestAt;
+        const waitMs = Math.max(0, minIntervalMs - elapsed);
+
+        if (waitMs > 0) {
+            console.log(`[Tiny Rate Limit] Waiting ${waitMs}ms before next Tiny API request.`);
+            await sleep(waitMs);
+        }
+
+        writeLastRequestAt(Date.now());
+    } finally {
+        releaseLock(lockDir);
+    }
+};
+
+const getTinyErrors = (data: any) => {
+    const erros = data?.retorno?.erros;
+    if (!erros) return [];
+    return Array.isArray(erros) ? erros : [erros];
+};
+
+const isTinyApiBlocked = (data: any) => {
+    return getTinyErrors(data).some((entry: any) => {
+        const message = String(entry?.erro || entry || '').toLowerCase();
+        return message.includes('api bloqueada') || message.includes('excedido o número de acessos');
+    });
+};
+
+export const tinyPost = async (servicePhp: string, params: URLSearchParams) => {
+    let attempt = 0;
+
+    while (true) {
+        await waitForTinySlot();
+
+        const response = await axios.post(`https://api.tiny.com.br/api2/${servicePhp}`, params, {
+            headers: { 'Content-Type': 'application/x-www-form-urlencoded' }
+        });
+
+        if (!isTinyApiBlocked(response.data)) {
+            return response;
+        }
+
+        attempt += 1;
+        const maxRetries = getBlockMaxRetries();
+        if (attempt > maxRetries) {
+            return response;
+        }
+
+        const delayMs = getBlockRetryDelayMs();
+        console.warn(`[Tiny Rate Limit] Tiny blocked ${servicePhp}. Waiting ${delayMs}ms before retry ${attempt}/${maxRetries}.`);
+        await sleep(delayMs);
+    }
+};