From 10d7048497170daea17680edb90158c7baa5e58b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cau=C3=AA=20Faleiros?= Date: Mon, 22 Jun 2026 16:17:40 -0300 Subject: [PATCH] fix: throttle Tiny API usage across backfills --- docker-compose.yml | 7 ++ src/controllers/webhook.controller.ts | 12 +-- src/index.ts | 6 +- src/scripts/backfill-graphs.ts | 21 +++-- src/scripts/backfill.ts | 11 +-- src/services/tiny-api.service.ts | 127 ++++++++++++++++++++++++++ 6 files changed, 159 insertions(+), 25 deletions(-) create mode 100644 src/services/tiny-api.service.ts diff --git a/docker-compose.yml b/docker-compose.yml index ae06932..d011261 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -16,6 +16,13 @@ services: - TINY_WEBHOOK_SECRET=${TINY_WEBHOOK_SECRET} - N8N_AUTH_TOKEN=${N8N_AUTH_TOKEN} - TINY_API_TOKEN=${TINY_API_TOKEN} + - GRAPHS_API_URL=${GRAPHS_API_URL} + - GRAPHS_API_KEY=${GRAPHS_API_KEY} + - GRAPHS_BACKFILL_CONTACT_FALLBACK=${GRAPHS_BACKFILL_CONTACT_FALLBACK} + - TINY_MIN_REQUEST_INTERVAL_MS=${TINY_MIN_REQUEST_INTERVAL_MS} + - TINY_BLOCK_RETRY_DELAY_MS=${TINY_BLOCK_RETRY_DELAY_MS} + - TINY_BLOCK_MAX_RETRIES=${TINY_BLOCK_MAX_RETRIES} + - TINY_FETCH_SELLER_DETAILS=${TINY_FETCH_SELLER_DETAILS} # Optional: If you use a shared Docker network for Nginx Proxy Manager, you can attach it here: # networks: # - nginx-proxy-manager-network diff --git a/src/controllers/webhook.controller.ts b/src/controllers/webhook.controller.ts index df7d3c7..7b08e02 100644 --- a/src/controllers/webhook.controller.ts +++ b/src/controllers/webhook.controller.ts @@ -1,7 +1,7 @@ import { Request, Response } from 'express'; import { sendToN8n } from '../services/n8n.service'; import { sendTinyOrderToGraphs } from '../services/graphs.service'; -import axios from 'axios'; +import { tinyPost } from '../services/tiny-api.service'; import fs from 'fs'; import path from 'path'; @@ -50,9 +50,7 @@ export const handleTinyOrderUpdate = async (req: Request, res: Response): Promis params.append('id', orderId); params.append('formato', 'JSON'); - const apiResponse = await axios.post('https://api.tiny.com.br/api2/pedido.obter.php', params, { - headers: { 'Content-Type': 'application/x-www-form-urlencoded' } - }); + const apiResponse = await tinyPost('pedido.obter.php', params); if (apiResponse.data?.retorno?.status === 'OK') { fullOrderDetails = apiResponse.data.retorno.pedido; @@ -61,16 +59,14 @@ export const handleTinyOrderUpdate = async (req: Request, res: Response): Promis // OPTION B: Fetch Vendor's WhatsApp const idVendedor = fullOrderDetails.id_vendedor; - if (idVendedor && idVendedor !== "0") { + if (process.env.TINY_FETCH_SELLER_DETAILS !== 'false' && idVendedor && idVendedor !== "0") { console.log(`Fetching seller details for Seller ID: ${idVendedor}...`); const vendorParams = new URLSearchParams(); vendorParams.append('token', tinyApiToken); vendorParams.append('id', idVendedor); vendorParams.append('formato', 'JSON'); - const vendorResponse = await axios.post('https://api.tiny.com.br/api2/contato.obter.php', vendorParams, { - headers: { 'Content-Type': 'application/x-www-form-urlencoded' } - }); + const vendorResponse = await tinyPost('contato.obter.php', vendorParams); if (vendorResponse.data?.retorno?.status === 'OK') { const contato = vendorResponse.data.retorno.contato; diff --git a/src/index.ts b/src/index.ts index e7643fd..a0603a1 100644 --- a/src/index.ts +++ b/src/index.ts @@ -85,7 +85,11 @@ app.get('/api/trigger-graphs-backfill', (req: Request, res: Response) => { dryRun: env.GRAPHS_BACKFILL_DRY_RUN || 'false', maxOrders: env.GRAPHS_BACKFILL_MAX_ORDERS || null, maxDays: env.GRAPHS_BACKFILL_MAX_DAYS || null, - contactFallback: env.GRAPHS_BACKFILL_CONTACT_FALLBACK || 'true' + contactFallback: env.GRAPHS_BACKFILL_CONTACT_FALLBACK || 'true', + graphsApiUrlConfigured: Boolean(env.GRAPHS_API_URL || env.NEXSTAR_GRAPHS_API_URL), + graphsApiKeyConfigured: Boolean(env.GRAPHS_API_KEY || env.NEXSTAR_GRAPHS_API_KEY), + tinyMinRequestIntervalMs: env.TINY_MIN_REQUEST_INTERVAL_MS || null, + tinyFetchSellerDetails: env.TINY_FETCH_SELLER_DETAILS || 'true' } }); }); diff --git a/src/scripts/backfill-graphs.ts b/src/scripts/backfill-graphs.ts index 1719288..0b93834 100644 --- a/src/scripts/backfill-graphs.ts +++ b/src/scripts/backfill-graphs.ts @@ -2,6 +2,7 @@ import axios from 'axios'; import dotenv from 'dotenv'; import fs from 'fs'; import path from 'path'; +import { tinyPost } from '../services/tiny-api.service'; dotenv.config(); @@ -57,9 +58,11 @@ const START_DATE = process.env.GRAPHS_BACKFILL_START_DATE || '13/02/2025'; const END_DATE = process.env.GRAPHS_BACKFILL_END_DATE || getTodayDate(); const MAX_DAYS = Number(process.env.GRAPHS_BACKFILL_MAX_DAYS || 0); const MAX_ORDERS = Number(process.env.GRAPHS_BACKFILL_MAX_ORDERS || 0); -const TINY_SEARCH_DELAY_MS = Number(process.env.TINY_SEARCH_DELAY_MS || 1000); -const TINY_DETAIL_DELAY_MS = Number(process.env.TINY_DETAIL_DELAY_MS || 1500); -const TINY_CONTACT_DELAY_MS = Number(process.env.TINY_CONTACT_DELAY_MS || TINY_DETAIL_DELAY_MS); +const TINY_MIN_REQUEST_INTERVAL_MS = Number(process.env.TINY_MIN_REQUEST_INTERVAL_MS || 0); +const USE_SHARED_RATE_LIMIT = TINY_MIN_REQUEST_INTERVAL_MS > 0; +const TINY_SEARCH_DELAY_MS = USE_SHARED_RATE_LIMIT ? 0 : Number(process.env.TINY_SEARCH_DELAY_MS || 1000); +const TINY_DETAIL_DELAY_MS = USE_SHARED_RATE_LIMIT ? 0 : Number(process.env.TINY_DETAIL_DELAY_MS || 1500); +const TINY_CONTACT_DELAY_MS = USE_SHARED_RATE_LIMIT ? 0 : Number(process.env.TINY_CONTACT_DELAY_MS || TINY_DETAIL_DELAY_MS); const STATE_FILE = process.env.GRAPHS_BACKFILL_STATE_FILE || path.join(process.cwd(), 'graphs_backfill_state.json'); const contactCache = new Map(); @@ -167,12 +170,6 @@ function pickFirst(...values: unknown[]) { return ''; } -async function tinyPost(servicePhp: string, params: URLSearchParams) { - return axios.post(`https://api.tiny.com.br/api2/${servicePhp}`, params, { - headers: { 'Content-Type': 'application/x-www-form-urlencoded' } - }); -} - async function fetchOrdersForDate(dateStr: string, page = 1): Promise { console.log(`[Tiny] Searching orders for ${dateStr}, page ${page}`); @@ -379,6 +376,12 @@ async function runBackfill() { console.log(`[config] End date: ${END_DATE}`); console.log(`[config] Dry run: ${DRY_RUN ? 'yes' : 'no'}`); console.log(`[config] Contact fallback: ${CONTACT_FALLBACK ? 'yes' : 'no'}`); + if (USE_SHARED_RATE_LIMIT) { + const requestsPerMinute = Math.floor(60000 / TINY_MIN_REQUEST_INTERVAL_MS); + console.log(`[config] Shared Tiny rate limit: 1 request every ${TINY_MIN_REQUEST_INTERVAL_MS}ms (~${requestsPerMinute}/minute)`); + } else { + console.log('[config] Shared Tiny rate limit: disabled'); + } console.log(`[config] State file: ${STATE_FILE}`); while (!isAfter(currentDate, END_DATE)) { diff --git a/src/scripts/backfill.ts b/src/scripts/backfill.ts index aaf0147..d749ef5 100644 --- a/src/scripts/backfill.ts +++ b/src/scripts/backfill.ts @@ -2,6 +2,7 @@ import axios from 'axios'; import dotenv from 'dotenv'; import fs from 'fs'; import path from 'path'; +import { tinyPost } from '../services/tiny-api.service'; dotenv.config(); @@ -50,9 +51,7 @@ const fetchOrdersForDate = async (dateStr: string, page: number = 1): Promise { params.append('formato', 'JSON'); try { - const response = await axios.post('https://api.tiny.com.br/api2/pedido.obter.php', params, { - headers: { 'Content-Type': 'application/x-www-form-urlencoded' } - }); + const response = await tinyPost('pedido.obter.php', params); await sleep(1500); // 1.5 second delay. This is the heavy part. @@ -184,4 +181,4 @@ const runBackfill = async () => { console.log('\n✅ BACKFILL COMPLETE! Caught up to today.'); }; -runBackfill(); \ No newline at end of file +runBackfill(); diff --git a/src/services/tiny-api.service.ts b/src/services/tiny-api.service.ts new file mode 100644 index 0000000..9adbe72 --- /dev/null +++ b/src/services/tiny-api.service.ts @@ -0,0 +1,127 @@ +import axios from 'axios'; +import fs from 'fs'; +import os from 'os'; +import path from 'path'; + +const DEFAULT_LOCK_DIR = path.join(os.tmpdir(), 'api-tiny-n8n-rate-limit.lock'); +const DEFAULT_STATE_FILE = path.join(os.tmpdir(), 'api-tiny-n8n-rate-limit.json'); +const LOCK_STALE_MS = 30000; +const DEFAULT_BLOCK_RETRY_DELAY_MS = 120000; +const DEFAULT_BLOCK_MAX_RETRIES = 30; + +const sleep = (ms: number) => new Promise(resolve => setTimeout(resolve, ms)); + +const getPositiveNumberEnv = (name: string, fallback: number) => { + const configured = Number(process.env[name] || fallback); + return Number.isFinite(configured) && configured > 0 ? configured : 0; +}; + +const getMinIntervalMs = () => getPositiveNumberEnv('TINY_MIN_REQUEST_INTERVAL_MS', 0); +const getBlockRetryDelayMs = () => getPositiveNumberEnv('TINY_BLOCK_RETRY_DELAY_MS', DEFAULT_BLOCK_RETRY_DELAY_MS); +const getBlockMaxRetries = () => getPositiveNumberEnv('TINY_BLOCK_MAX_RETRIES', DEFAULT_BLOCK_MAX_RETRIES); +const getLockDir = () => process.env.TINY_RATE_LIMIT_LOCK_DIR || DEFAULT_LOCK_DIR; +const getStateFile = () => process.env.TINY_RATE_LIMIT_STATE_FILE || DEFAULT_STATE_FILE; + +const readLastRequestAt = () => { + try { + const parsed = JSON.parse(fs.readFileSync(getStateFile(), 'utf-8')); + return Number(parsed.lastRequestAt || 0); + } catch { + return 0; + } +}; + +const writeLastRequestAt = (timestamp: number) => { + fs.writeFileSync(getStateFile(), JSON.stringify({ lastRequestAt: timestamp }, null, 2)); +}; + +const removeStaleLock = (lockDir: string) => { + try { + const stats = fs.statSync(lockDir); + if (Date.now() - stats.mtimeMs > LOCK_STALE_MS) { + fs.rmSync(lockDir, { recursive: true, force: true }); + } + } catch { + // Lock does not exist or cannot be inspected. The acquire loop will retry. + } +}; + +const acquireLock = async () => { + const lockDir = getLockDir(); + + while (true) { + try { + fs.mkdirSync(lockDir); + return lockDir; + } catch (error: any) { + if (error?.code !== 'EEXIST') throw error; + removeStaleLock(lockDir); + await sleep(100); + } + } +}; + +const releaseLock = (lockDir: string) => { + fs.rmSync(lockDir, { recursive: true, force: true }); +}; + +const waitForTinySlot = async () => { + const minIntervalMs = getMinIntervalMs(); + if (!minIntervalMs) return; + + const lockDir = await acquireLock(); + + try { + const lastRequestAt = readLastRequestAt(); + const elapsed = Date.now() - lastRequestAt; + const waitMs = Math.max(0, minIntervalMs - elapsed); + + if (waitMs > 0) { + console.log(`[Tiny Rate Limit] Waiting ${waitMs}ms before next Tiny API request.`); + await sleep(waitMs); + } + + writeLastRequestAt(Date.now()); + } finally { + releaseLock(lockDir); + } +}; + +const getTinyErrors = (data: any) => { + const erros = data?.retorno?.erros; + if (!erros) return []; + return Array.isArray(erros) ? erros : [erros]; +}; + +const isTinyApiBlocked = (data: any) => { + return getTinyErrors(data).some((entry: any) => { + const message = String(entry?.erro || entry || '').toLowerCase(); + return message.includes('api bloqueada') || message.includes('excedido o número de acessos'); + }); +}; + +export const tinyPost = async (servicePhp: string, params: URLSearchParams) => { + let attempt = 0; + + while (true) { + await waitForTinySlot(); + + const response = await axios.post(`https://api.tiny.com.br/api2/${servicePhp}`, params, { + headers: { 'Content-Type': 'application/x-www-form-urlencoded' } + }); + + if (!isTinyApiBlocked(response.data)) { + return response; + } + + attempt += 1; + const maxRetries = getBlockMaxRetries(); + if (attempt > maxRetries) { + return response; + } + + const delayMs = getBlockRetryDelayMs(); + console.warn(`[Tiny Rate Limit] Tiny blocked ${servicePhp}. Waiting ${delayMs}ms before retry ${attempt}/${maxRetries}.`); + await sleep(delayMs); + } +};