blyzer/tiny-webhook
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: slow and control Graphs backfill
All checks were successful
Build and Deploy / build-and-push (push) Successful in 2m18s
Details

Browse Source
This commit is contained in:
Cauê Faleiros
2026-06-23 09:32:08 -03:00
parent 1d24831aa7
commit 44cfdbb4c9
5 changed files with 284 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files

172
src/scripts/backfill-graphs.ts
View File

@@ -44,6 +44,12 @@ type ContactInfo = {
type BackfillState = {
nextDate: string;
lastCompletedDate?: string;
status?: 'running' | 'stopped' | 'completed';
currentDate?: string;
lastOrderId?: string;
processedDays?: number;
processedOrders?: number;
lastMessage?: string;
updatedAt?: string;
};
@@ -55,17 +61,31 @@ const CONTACT_FALLBACK = process.env.GRAPHS_BACKFILL_CONTACT_FALLBACK !== 'false
const NORMALIZE_PHONE = process.env.GRAPHS_BACKFILL_NORMALIZE_PHONE !== 'false';
const START_DATE = process.env.GRAPHS_BACKFILL_START_DATE || '13/02/2025';
const END_DATE = process.env.GRAPHS_BACKFILL_END_DATE || getTodayDate();
const MAX_DAYS = Number(process.env.GRAPHS_BACKFILL_MAX_DAYS || 0);
const MAX_ORDERS = Number(process.env.GRAPHS_BACKFILL_MAX_ORDERS || 0);
const TINY_SEARCH_DELAY_MS = Number(process.env.TINY_SEARCH_DELAY_MS || 1000);
const TINY_DETAIL_DELAY_MS = Number(process.env.TINY_DETAIL_DELAY_MS || 1500);
const TINY_CONTACT_DELAY_MS = Number(process.env.TINY_CONTACT_DELAY_MS || TINY_DETAIL_DELAY_MS);
const numberEnv = (name: string, fallback: number) => {
const value = Number(process.env[name] ?? fallback);
return Number.isFinite(value) && value >= 0 ? value : fallback;
};
const MAX_DAYS = numberEnv('GRAPHS_BACKFILL_MAX_DAYS', 0);
const MAX_ORDERS = numberEnv('GRAPHS_BACKFILL_MAX_ORDERS', 0);
const TINY_SEARCH_DELAY_MS = numberEnv('TINY_SEARCH_DELAY_MS', 1000);
const TINY_DETAIL_DELAY_MS = numberEnv('TINY_DETAIL_DELAY_MS', 1500);
const TINY_CONTACT_DELAY_MS = numberEnv('TINY_CONTACT_DELAY_MS', TINY_DETAIL_DELAY_MS);
const ORDER_DELAY_MS = numberEnv('GRAPHS_BACKFILL_ORDER_DELAY_MS', 6000);
const TINY_BLOCK_DELAY_MS = numberEnv('GRAPHS_BACKFILL_TINY_BLOCK_DELAY_MS', 600000);
const MAX_TINY_BLOCK_RETRIES = numberEnv('GRAPHS_BACKFILL_MAX_TINY_BLOCK_RETRIES', 0);
const STATE_FILE = process.env.GRAPHS_BACKFILL_STATE_FILE || path.join(process.cwd(), 'graphs_backfill_state.json');
const STOP_FILE = process.env.GRAPHS_BACKFILL_STOP_FILE || path.join(process.cwd(), 'graphs_backfill_stop.json');
const contactCache = new Map<string, ContactInfo | null>();
const sleep = (ms: number) => new Promise(resolve => setTimeout(resolve, ms));
class BackfillStoppedError extends Error {
constructor() {
super('Graphs backfill stopped by request.');
}
}
function getTodayDate() {
const date = new Date();
return formatDate(date);
@@ -130,6 +150,26 @@ function saveState(state: BackfillState) {
}, null, 2));
}
function assertNotStopped() {
if (fs.existsSync(STOP_FILE)) {
throw new BackfillStoppedError();
}
}
async function sleepWithStop(ms: number, reason: string) {
if (ms <= 0) return;
const startedAt = Date.now();
let remaining = ms;
console.log(`[wait] Waiting ${ms}ms before ${reason}.`);
while (remaining > 0) {
assertNotStopped();
await sleep(Math.min(remaining, 1000));
remaining = ms - (Date.now() - startedAt);
}
}
function normalizeNumber(value: unknown) {
if (typeof value === 'number') return Number.isFinite(value) ? value : 0;
if (value === undefined || value === null) return 0;
@@ -167,12 +207,46 @@ function pickFirst(...values: unknown[]) {
return '';
}
async function tinyPost(servicePhp: string, params: URLSearchParams) {
return axios.post(`https://api.tiny.com.br/api2/${servicePhp}`, params, {
headers: { 'Content-Type': 'application/x-www-form-urlencoded' }
function getTinyErrors(data: any) {
const errors = data?.retorno?.erros;
if (!errors) return [];
return Array.isArray(errors) ? errors : [errors];
}
function isTinyApiBlocked(data: any) {
return getTinyErrors(data).some((entry: any) => {
const message = String(entry?.erro || entry || '').toLowerCase();
return message.includes('api bloqueada') || message.includes('excedido o número de acessos');
});
}
async function tinyPost(servicePhp: string, params: URLSearchParams) {
let blockedAttempts = 0;
while (true) {
assertNotStopped();
const response = await axios.post(`https://api.tiny.com.br/api2/${servicePhp}`, params, {
headers: { 'Content-Type': 'application/x-www-form-urlencoded' }
});
if (!isTinyApiBlocked(response.data)) {
return response;
}
blockedAttempts += 1;
if (MAX_TINY_BLOCK_RETRIES && blockedAttempts > MAX_TINY_BLOCK_RETRIES) {
return response;
}
const retryLabel = MAX_TINY_BLOCK_RETRIES
? `${blockedAttempts}/${MAX_TINY_BLOCK_RETRIES}`
: String(blockedAttempts);
console.warn(`[Tiny] API blocked on ${servicePhp}. Pausing backfill for ${TINY_BLOCK_DELAY_MS}ms before retry ${retryLabel}.`);
await sleepWithStop(TINY_BLOCK_DELAY_MS, 'retrying Tiny API after block');
}
}
async function fetchOrdersForDate(dateStr: string, page = 1): Promise<TinyOrderSummary[]> {
console.log(`[Tiny] Searching orders for ${dateStr}, page ${page}`);
@@ -184,7 +258,7 @@ async function fetchOrdersForDate(dateStr: string, page = 1): Promise<TinyOrderS
params.append('pagina', String(page));
const response = await tinyPost('pedidos.pesquisa.php', params);
await sleep(TINY_SEARCH_DELAY_MS);
await sleepWithStop(TINY_SEARCH_DELAY_MS, 'next Tiny search request');
const retorno = response.data?.retorno;
if (retorno?.status === 'OK') {
@@ -214,7 +288,7 @@ async function fetchOrderDetails(orderId: string): Promise<TinyOrderDetails | nu
try {
const response = await tinyPost('pedido.obter.php', params);
await sleep(TINY_DETAIL_DELAY_MS);
await sleepWithStop(TINY_DETAIL_DELAY_MS, 'next Tiny order detail request');
if (response.data?.retorno?.status === 'OK') {
return {
@@ -226,8 +300,9 @@ async function fetchOrderDetails(orderId: string): Promise<TinyOrderDetails | nu
console.error(`[Tiny] Failed to fetch order ${orderId}:`, JSON.stringify(response.data?.retorno?.erros || 'Unknown error'));
return null;
} catch (error: any) {
if (error instanceof BackfillStoppedError) throw error;
console.error(`[Tiny] Request failed for order ${orderId}: ${error.message}`);
await sleep(5000);
await sleepWithStop(5000, 'continuing after Tiny request failure');
return null;
}
}
@@ -250,7 +325,7 @@ async function fetchContactInfo(cliente: any): Promise<ContactInfo | null> {
if (cpfCnpj) searchParams.append('cpf_cnpj', cpfCnpj);
const searchResponse = await tinyPost('contatos.pesquisa.php', searchParams);
await sleep(TINY_CONTACT_DELAY_MS);
await sleepWithStop(TINY_CONTACT_DELAY_MS, 'next Tiny contact request');
const contatoResumo = searchResponse.data?.retorno?.contatos?.[0]?.contato;
const contatoId = contatoResumo?.id;
@@ -265,7 +340,7 @@ async function fetchContactInfo(cliente: any): Promise<ContactInfo | null> {
detailParams.append('id', String(contatoId));
const detailResponse = await tinyPost('contato.obter.php', detailParams);
await sleep(TINY_CONTACT_DELAY_MS);
await sleepWithStop(TINY_CONTACT_DELAY_MS, 'next Tiny contact request');
if (detailResponse.data?.retorno?.status !== 'OK') {
contactCache.set(cacheKey, null);
@@ -281,6 +356,7 @@ async function fetchContactInfo(cliente: any): Promise<ContactInfo | null> {
contactCache.set(cacheKey, info);
return info;
} catch (error: any) {
if (error instanceof BackfillStoppedError) throw error;
console.error(`[Tiny] Contact fallback failed for ${name || cpfCnpj}: ${error.message}`);
contactCache.set(cacheKey, null);
return null;
@@ -372,16 +448,32 @@ async function runBackfill() {
const state = loadState();
let currentDate = state.nextDate;
let lastCompletedDate = state.lastCompletedDate;
let processedDays = 0;
let processedOrders = 0;
const orderPace = ORDER_DELAY_MS ? `~${Math.floor(60000 / ORDER_DELAY_MS)}/minute` : 'unlimited';
console.log(`[config] Start date: ${currentDate}`);
console.log(`[config] End date: ${END_DATE}`);
console.log(`[config] Dry run: ${DRY_RUN ? 'yes' : 'no'}`);
console.log(`[config] Contact fallback: ${CONTACT_FALLBACK ? 'yes' : 'no'}`);
console.log(`[config] Order pace: 1 order every ${ORDER_DELAY_MS}ms (${orderPace})`);
console.log(`[config] Tiny block pause: ${TINY_BLOCK_DELAY_MS}ms`);
console.log(`[config] State file: ${STATE_FILE}`);
console.log(`[config] Stop file: ${STOP_FILE}`);
saveState({
...state,
status: 'running',
currentDate,
processedDays,
processedOrders,
lastMessage: 'Backfill started'
});
while (!isAfter(currentDate, END_DATE)) {
assertNotStopped();
if (MAX_DAYS && processedDays >= MAX_DAYS) {
console.log(`[limit] Stopped after ${MAX_DAYS} day(s).`);
break;
@@ -392,8 +484,19 @@ async function runBackfill() {
console.log(`[Tiny] Found ${summaries.length} order(s) for ${currentDate}`);
for (const summary of summaries) {
assertNotStopped();
if (MAX_ORDERS && processedOrders >= MAX_ORDERS) {
console.log(`[limit] Stopped after ${MAX_ORDERS} order(s).`);
saveState({
nextDate: currentDate,
lastCompletedDate,
status: 'completed',
currentDate,
processedDays,
processedOrders,
lastMessage: `Stopped after ${MAX_ORDERS} order(s)`
});
return;
}
@@ -409,22 +512,63 @@ async function runBackfill() {
const rows = await buildGraphsRows(details, summary);
await sendRowsToGraphs(rows, orderId);
processedOrders += 1;
saveState({
nextDate: currentDate,
lastCompletedDate,
status: 'running',
currentDate,
lastOrderId: orderId,
processedDays,
processedOrders,
lastMessage: `Processed order ${orderId}`
});
if (!MAX_ORDERS || processedOrders < MAX_ORDERS) {
await sleepWithStop(ORDER_DELAY_MS, 'next order');
}
}
const completedDate = currentDate;
currentDate = getNextDate(currentDate);
lastCompletedDate = completedDate;
processedDays += 1;
saveState({
nextDate: currentDate,
lastCompletedDate: completedDate
lastCompletedDate,
status: 'running',
currentDate,
processedDays,
processedOrders,
lastMessage: `Completed ${completedDate}`
});
console.log(`[state] Completed ${completedDate}. Next date: ${currentDate}`);
}
saveState({
nextDate: currentDate,
lastCompletedDate,
status: 'completed',
currentDate,
processedDays,
processedOrders,
lastMessage: 'Backfill completed'
});
console.log(`\nDone. Processed ${processedOrders} order(s) across ${processedDays} day(s).`);
}
runBackfill().catch((error: any) => {
if (error instanceof BackfillStoppedError) {
console.log('Backfill stopped by request.');
const state = loadState();
saveState({
...state,
status: 'stopped',
lastMessage: 'Backfill stopped by request'
});
process.exit(0);
}
console.error(`Backfill failed: ${error.message}`);
process.exit(1);
});