blyzer/tiny-webhook
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Revert "fix: throttle Tiny API usage across backfills"

Browse Source 
This reverts commit 10d7048497.
This commit is contained in:
Cauê Faleiros
2026-06-22 16:31:36 -03:00
parent ae41ae4a55
commit 1d24831aa7
6 changed files with 25 additions and 159 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
docker-compose.yml
View File

@@ -16,13 +16,6 @@ services:
- TINY_WEBHOOK_SECRET=${TINY_WEBHOOK_SECRET} - TINY_WEBHOOK_SECRET=${TINY_WEBHOOK_SECRET}
- N8N_AUTH_TOKEN=${N8N_AUTH_TOKEN} - N8N_AUTH_TOKEN=${N8N_AUTH_TOKEN}
- TINY_API_TOKEN=${TINY_API_TOKEN} - TINY_API_TOKEN=${TINY_API_TOKEN}
- GRAPHS_API_URL=${GRAPHS_API_URL}
- GRAPHS_API_KEY=${GRAPHS_API_KEY}
- GRAPHS_BACKFILL_CONTACT_FALLBACK=${GRAPHS_BACKFILL_CONTACT_FALLBACK}
- TINY_MIN_REQUEST_INTERVAL_MS=${TINY_MIN_REQUEST_INTERVAL_MS}
- TINY_BLOCK_RETRY_DELAY_MS=${TINY_BLOCK_RETRY_DELAY_MS}
- TINY_BLOCK_MAX_RETRIES=${TINY_BLOCK_MAX_RETRIES}
- TINY_FETCH_SELLER_DETAILS=${TINY_FETCH_SELLER_DETAILS}
# Optional: If you use a shared Docker network for Nginx Proxy Manager, you can attach it here: # Optional: If you use a shared Docker network for Nginx Proxy Manager, you can attach it here:
# networks: # networks:
# - nginx-proxy-manager-network # - nginx-proxy-manager-network

12
src/controllers/webhook.controller.ts
View File

@@ -1,7 +1,7 @@
import { Request, Response } from 'express'; import { Request, Response } from 'express';
import { sendToN8n } from '../services/n8n.service'; import { sendToN8n } from '../services/n8n.service';
import { sendTinyOrderToGraphs } from '../services/graphs.service'; import { sendTinyOrderToGraphs } from '../services/graphs.service';
import { tinyPost } from '../services/tiny-api.service'; import axios from 'axios';
import fs from 'fs'; import fs from 'fs';
import path from 'path'; import path from 'path';
@@ -50,7 +50,9 @@ export const handleTinyOrderUpdate = async (req: Request, res: Response): Promis
params.append('id', orderId); params.append('id', orderId);
params.append('formato', 'JSON'); params.append('formato', 'JSON');
const apiResponse = await tinyPost('pedido.obter.php', params); const apiResponse = await axios.post('https://api.tiny.com.br/api2/pedido.obter.php', params, {
headers: { 'Content-Type': 'application/x-www-form-urlencoded' }
});
if (apiResponse.data?.retorno?.status === 'OK') { if (apiResponse.data?.retorno?.status === 'OK') {
fullOrderDetails = apiResponse.data.retorno.pedido; fullOrderDetails = apiResponse.data.retorno.pedido;
@@ -59,14 +61,16 @@ export const handleTinyOrderUpdate = async (req: Request, res: Response): Promis
// OPTION B: Fetch Vendor's WhatsApp // OPTION B: Fetch Vendor's WhatsApp
const idVendedor = fullOrderDetails.id_vendedor; const idVendedor = fullOrderDetails.id_vendedor;
if (process.env.TINY_FETCH_SELLER_DETAILS !== 'false' && idVendedor && idVendedor !== "0") { if (idVendedor && idVendedor !== "0") {
console.log(`Fetching seller details for Seller ID: ${idVendedor}...`); console.log(`Fetching seller details for Seller ID: ${idVendedor}...`);
const vendorParams = new URLSearchParams(); const vendorParams = new URLSearchParams();
vendorParams.append('token', tinyApiToken); vendorParams.append('token', tinyApiToken);
vendorParams.append('id', idVendedor); vendorParams.append('id', idVendedor);
vendorParams.append('formato', 'JSON'); vendorParams.append('formato', 'JSON');
const vendorResponse = await tinyPost('contato.obter.php', vendorParams); const vendorResponse = await axios.post('https://api.tiny.com.br/api2/contato.obter.php', vendorParams, {
headers: { 'Content-Type': 'application/x-www-form-urlencoded' }
});
if (vendorResponse.data?.retorno?.status === 'OK') { if (vendorResponse.data?.retorno?.status === 'OK') {
const contato = vendorResponse.data.retorno.contato; const contato = vendorResponse.data.retorno.contato;

6
src/index.ts
View File

@@ -85,11 +85,7 @@ app.get('/api/trigger-graphs-backfill', (req: Request, res: Response) => {
dryRun: env.GRAPHS_BACKFILL_DRY_RUN || 'false', dryRun: env.GRAPHS_BACKFILL_DRY_RUN || 'false',
maxOrders: env.GRAPHS_BACKFILL_MAX_ORDERS || null, maxOrders: env.GRAPHS_BACKFILL_MAX_ORDERS || null,
maxDays: env.GRAPHS_BACKFILL_MAX_DAYS || null, maxDays: env.GRAPHS_BACKFILL_MAX_DAYS || null,
contactFallback: env.GRAPHS_BACKFILL_CONTACT_FALLBACK || 'true', contactFallback: env.GRAPHS_BACKFILL_CONTACT_FALLBACK || 'true'
graphsApiUrlConfigured: Boolean(env.GRAPHS_API_URL || env.NEXSTAR_GRAPHS_API_URL),
graphsApiKeyConfigured: Boolean(env.GRAPHS_API_KEY || env.NEXSTAR_GRAPHS_API_KEY),
tinyMinRequestIntervalMs: env.TINY_MIN_REQUEST_INTERVAL_MS || null,
tinyFetchSellerDetails: env.TINY_FETCH_SELLER_DETAILS || 'true'
} }
}); });
}); });

21
src/scripts/backfill-graphs.ts
View File

@@ -2,7 +2,6 @@ import axios from 'axios';
import dotenv from 'dotenv'; import dotenv from 'dotenv';
import fs from 'fs'; import fs from 'fs';
import path from 'path'; import path from 'path';
import { tinyPost } from '../services/tiny-api.service';
dotenv.config(); dotenv.config();
@@ -58,11 +57,9 @@ const START_DATE = process.env.GRAPHS_BACKFILL_START_DATE || '13/02/2025';
const END_DATE = process.env.GRAPHS_BACKFILL_END_DATE || getTodayDate(); const END_DATE = process.env.GRAPHS_BACKFILL_END_DATE || getTodayDate();
const MAX_DAYS = Number(process.env.GRAPHS_BACKFILL_MAX_DAYS || 0); const MAX_DAYS = Number(process.env.GRAPHS_BACKFILL_MAX_DAYS || 0);
const MAX_ORDERS = Number(process.env.GRAPHS_BACKFILL_MAX_ORDERS || 0); const MAX_ORDERS = Number(process.env.GRAPHS_BACKFILL_MAX_ORDERS || 0);
const TINY_MIN_REQUEST_INTERVAL_MS = Number(process.env.TINY_MIN_REQUEST_INTERVAL_MS || 0); const TINY_SEARCH_DELAY_MS = Number(process.env.TINY_SEARCH_DELAY_MS || 1000);
const USE_SHARED_RATE_LIMIT = TINY_MIN_REQUEST_INTERVAL_MS > 0; const TINY_DETAIL_DELAY_MS = Number(process.env.TINY_DETAIL_DELAY_MS || 1500);
const TINY_SEARCH_DELAY_MS = USE_SHARED_RATE_LIMIT ? 0 : Number(process.env.TINY_SEARCH_DELAY_MS || 1000); const TINY_CONTACT_DELAY_MS = Number(process.env.TINY_CONTACT_DELAY_MS || TINY_DETAIL_DELAY_MS);
const TINY_DETAIL_DELAY_MS = USE_SHARED_RATE_LIMIT ? 0 : Number(process.env.TINY_DETAIL_DELAY_MS || 1500);
const TINY_CONTACT_DELAY_MS = USE_SHARED_RATE_LIMIT ? 0 : Number(process.env.TINY_CONTACT_DELAY_MS || TINY_DETAIL_DELAY_MS);
const STATE_FILE = process.env.GRAPHS_BACKFILL_STATE_FILE || path.join(process.cwd(), 'graphs_backfill_state.json'); const STATE_FILE = process.env.GRAPHS_BACKFILL_STATE_FILE || path.join(process.cwd(), 'graphs_backfill_state.json');
const contactCache = new Map<string, ContactInfo | null>(); const contactCache = new Map<string, ContactInfo | null>();
@@ -170,6 +167,12 @@ function pickFirst(...values: unknown[]) {
return ''; return '';
} }
async function tinyPost(servicePhp: string, params: URLSearchParams) {
return axios.post(`https://api.tiny.com.br/api2/${servicePhp}`, params, {
headers: { 'Content-Type': 'application/x-www-form-urlencoded' }
});
}
async function fetchOrdersForDate(dateStr: string, page = 1): Promise<TinyOrderSummary[]> { async function fetchOrdersForDate(dateStr: string, page = 1): Promise<TinyOrderSummary[]> {
console.log(`[Tiny] Searching orders for ${dateStr}, page ${page}`); console.log(`[Tiny] Searching orders for ${dateStr}, page ${page}`);
@@ -376,12 +379,6 @@ async function runBackfill() {
console.log(`[config] End date: ${END_DATE}`); console.log(`[config] End date: ${END_DATE}`);
console.log(`[config] Dry run: ${DRY_RUN ? 'yes' : 'no'}`); console.log(`[config] Dry run: ${DRY_RUN ? 'yes' : 'no'}`);
console.log(`[config] Contact fallback: ${CONTACT_FALLBACK ? 'yes' : 'no'}`); console.log(`[config] Contact fallback: ${CONTACT_FALLBACK ? 'yes' : 'no'}`);
if (USE_SHARED_RATE_LIMIT) {
const requestsPerMinute = Math.floor(60000 / TINY_MIN_REQUEST_INTERVAL_MS);
console.log(`[config] Shared Tiny rate limit: 1 request every ${TINY_MIN_REQUEST_INTERVAL_MS}ms (~${requestsPerMinute}/minute)`);
} else {
console.log('[config] Shared Tiny rate limit: disabled');
}
console.log(`[config] State file: ${STATE_FILE}`); console.log(`[config] State file: ${STATE_FILE}`);
while (!isAfter(currentDate, END_DATE)) { while (!isAfter(currentDate, END_DATE)) {

9
src/scripts/backfill.ts
View File

@@ -2,7 +2,6 @@ import axios from 'axios';
import dotenv from 'dotenv'; import dotenv from 'dotenv';
import fs from 'fs'; import fs from 'fs';
import path from 'path'; import path from 'path';
import { tinyPost } from '../services/tiny-api.service';
dotenv.config(); dotenv.config();
@@ -51,7 +50,9 @@ const fetchOrdersForDate = async (dateStr: string, page: number = 1): Promise<an
params.append('pagina', page.toString()); params.append('pagina', page.toString());
try { try {
const response = await tinyPost('pedidos.pesquisa.php', params); const response = await axios.post('https://api.tiny.com.br/api2/pedidos.pesquisa.php', params, {
headers: { 'Content-Type': 'application/x-www-form-urlencoded' }
});
await sleep(1000); // 1 second delay await sleep(1000); // 1 second delay
@@ -91,7 +92,9 @@ const fetchOrderDetails = async (orderId: string) => {
params.append('formato', 'JSON'); params.append('formato', 'JSON');
try { try {
const response = await tinyPost('pedido.obter.php', params); const response = await axios.post('https://api.tiny.com.br/api2/pedido.obter.php', params, {
headers: { 'Content-Type': 'application/x-www-form-urlencoded' }
});
await sleep(1500); // 1.5 second delay. This is the heavy part. await sleep(1500); // 1.5 second delay. This is the heavy part.

127
src/services/tiny-api.service.ts
View File

@@ -1,127 +0,0 @@
import axios from 'axios';
import fs from 'fs';
import os from 'os';
import path from 'path';
const DEFAULT_LOCK_DIR = path.join(os.tmpdir(), 'api-tiny-n8n-rate-limit.lock');
const DEFAULT_STATE_FILE = path.join(os.tmpdir(), 'api-tiny-n8n-rate-limit.json');
const LOCK_STALE_MS = 30000;
const DEFAULT_BLOCK_RETRY_DELAY_MS = 120000;
const DEFAULT_BLOCK_MAX_RETRIES = 30;
const sleep = (ms: number) => new Promise(resolve => setTimeout(resolve, ms));
const getPositiveNumberEnv = (name: string, fallback: number) => {
const configured = Number(process.env[name] || fallback);
return Number.isFinite(configured) && configured > 0 ? configured : 0;
};
const getMinIntervalMs = () => getPositiveNumberEnv('TINY_MIN_REQUEST_INTERVAL_MS', 0);
const getBlockRetryDelayMs = () => getPositiveNumberEnv('TINY_BLOCK_RETRY_DELAY_MS', DEFAULT_BLOCK_RETRY_DELAY_MS);
const getBlockMaxRetries = () => getPositiveNumberEnv('TINY_BLOCK_MAX_RETRIES', DEFAULT_BLOCK_MAX_RETRIES);
const getLockDir = () => process.env.TINY_RATE_LIMIT_LOCK_DIR || DEFAULT_LOCK_DIR;
const getStateFile = () => process.env.TINY_RATE_LIMIT_STATE_FILE || DEFAULT_STATE_FILE;
const readLastRequestAt = () => {
try {
const parsed = JSON.parse(fs.readFileSync(getStateFile(), 'utf-8'));
return Number(parsed.lastRequestAt || 0);
} catch {
return 0;
}
};
const writeLastRequestAt = (timestamp: number) => {
fs.writeFileSync(getStateFile(), JSON.stringify({ lastRequestAt: timestamp }, null, 2));
};
const removeStaleLock = (lockDir: string) => {
try {
const stats = fs.statSync(lockDir);
if (Date.now() - stats.mtimeMs > LOCK_STALE_MS) {
fs.rmSync(lockDir, { recursive: true, force: true });
}
} catch {
// Lock does not exist or cannot be inspected. The acquire loop will retry.
}
};
const acquireLock = async () => {
const lockDir = getLockDir();
while (true) {
try {
fs.mkdirSync(lockDir);
return lockDir;
} catch (error: any) {
if (error?.code !== 'EEXIST') throw error;
removeStaleLock(lockDir);
await sleep(100);
}
}
};
const releaseLock = (lockDir: string) => {
fs.rmSync(lockDir, { recursive: true, force: true });
};
const waitForTinySlot = async () => {
const minIntervalMs = getMinIntervalMs();
if (!minIntervalMs) return;
const lockDir = await acquireLock();
try {
const lastRequestAt = readLastRequestAt();
const elapsed = Date.now() - lastRequestAt;
const waitMs = Math.max(0, minIntervalMs - elapsed);
if (waitMs > 0) {
console.log(`[Tiny Rate Limit] Waiting ${waitMs}ms before next Tiny API request.`);
await sleep(waitMs);
}
writeLastRequestAt(Date.now());
} finally {
releaseLock(lockDir);
}
};
const getTinyErrors = (data: any) => {
const erros = data?.retorno?.erros;
if (!erros) return [];
return Array.isArray(erros) ? erros : [erros];
};
const isTinyApiBlocked = (data: any) => {
return getTinyErrors(data).some((entry: any) => {
const message = String(entry?.erro || entry || '').toLowerCase();
return message.includes('api bloqueada') || message.includes('excedido o número de acessos');
});
};
export const tinyPost = async (servicePhp: string, params: URLSearchParams) => {
let attempt = 0;
while (true) {
await waitForTinySlot();
const response = await axios.post(`https://api.tiny.com.br/api2/${servicePhp}`, params, {
headers: { 'Content-Type': 'application/x-www-form-urlencoded' }
});
if (!isTinyApiBlocked(response.data)) {
return response;
}
attempt += 1;
const maxRetries = getBlockMaxRetries();
if (attempt > maxRetries) {
return response;
}
const delayMs = getBlockRetryDelayMs();
console.warn(`[Tiny Rate Limit] Tiny blocked ${servicePhp}. Waiting ${delayMs}ms before retry ${attempt}/${maxRetries}.`);
await sleep(delayMs);
}
};