blyzer/tiny-webhook
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Improve graphs backfill resilience
All checks were successful
Build and Deploy / build-and-push (push) Successful in 2m56s
Details

Browse Source
This commit is contained in:
Cauê Faleiros
2026-07-03 12:19:43 -03:00
parent 83f165381d
commit 831f82ee57
2 changed files with 187 additions and 31 deletions
Download Patch File Download Diff File Expand all files Collapse all files

200
src/scripts/backfill-graphs.ts
View File

@@ -41,14 +41,20 @@ type ContactInfo = {
phone: string;
};
type SellerInfo = {
name: string;
};
type BackfillState = {
nextDate: string;
lastCompletedDate?: string;
status?: 'running' | 'stopped' | 'completed';
currentDate?: string;
lastOrderId?: string;
lastFailedOrderId?: string;
processedDays?: number;
processedOrders?: number;
failedOrders?: number;
lastMessage?: string;
updatedAt?: string;
};
@@ -58,6 +64,7 @@ const GRAPHS_API_URL = process.env.GRAPHS_API_URL || process.env.NEXSTAR_GRAPHS_
const GRAPHS_API_KEY = process.env.GRAPHS_API_KEY || process.env.NEXSTAR_GRAPHS_API_KEY;
const DRY_RUN = process.env.GRAPHS_BACKFILL_DRY_RUN === 'true';
const CONTACT_FALLBACK = process.env.GRAPHS_BACKFILL_CONTACT_FALLBACK !== 'false';
const SELLER_FALLBACK = process.env.GRAPHS_BACKFILL_SELLER_FALLBACK !== 'false';
const NORMALIZE_PHONE = process.env.GRAPHS_BACKFILL_NORMALIZE_PHONE !== 'false';
const START_DATE = process.env.GRAPHS_BACKFILL_START_DATE || '13/02/2025';
const END_DATE = process.env.GRAPHS_BACKFILL_END_DATE || getTodayDate();
@@ -77,10 +84,13 @@ const TINY_CONTACT_DELAY_MS = numberEnv('TINY_CONTACT_DELAY_MS', TINY_DETAIL_DEL
const ORDER_DELAY_MS = numberEnv('GRAPHS_BACKFILL_ORDER_DELAY_MS', 0);
const TINY_BLOCK_DELAY_MS = numberEnv('GRAPHS_BACKFILL_TINY_BLOCK_DELAY_MS', 600000);
const MAX_TINY_BLOCK_RETRIES = numberEnv('GRAPHS_BACKFILL_MAX_TINY_BLOCK_RETRIES', 0);
const GRAPHS_API_MAX_RETRIES = numberEnv('GRAPHS_BACKFILL_GRAPHS_API_MAX_RETRIES', 3);
const GRAPHS_API_RETRY_DELAY_MS = numberEnv('GRAPHS_BACKFILL_GRAPHS_API_RETRY_DELAY_MS', 30000);
const STATE_FILE = process.env.GRAPHS_BACKFILL_STATE_FILE || path.join(process.cwd(), 'graphs_backfill_state.json');
const STOP_FILE = process.env.GRAPHS_BACKFILL_STOP_FILE || path.join(process.cwd(), 'graphs_backfill_stop.json');
const contactCache = new Map<string, ContactInfo | null>();
const sellerCache = new Map<string, SellerInfo | null>();
let lastTinyRequestAt = 0;
const sleep = (ms: number) => new Promise(resolve => setTimeout(resolve, ms));
@@ -218,6 +228,16 @@ function getTinyErrors(data: any) {
return Array.isArray(errors) ? errors : [errors];
}
function describeRequestError(error: any) {
if (error?.response) {
const body = JSON.stringify(error.response.data ?? '');
const truncatedBody = body.length > 500 ? `${body.slice(0, 500)}...` : body;
return `HTTP ${error.response.status}: ${truncatedBody}`;
}
return error?.message || String(error);
}
function isTinyApiBlocked(data: any) {
return getTinyErrors(data).some((entry: any) => {
const message = String(entry?.erro || entry || '').toLowerCase();
@@ -239,9 +259,24 @@ async function tinyPost(servicePhp: string, params: URLSearchParams) {
assertNotStopped();
await waitForTinyRequestSlot(servicePhp);
const response = await axios.post(`https://api.tiny.com.br/api2/${servicePhp}`, params, {
headers: { 'Content-Type': 'application/x-www-form-urlencoded' }
});
let response;
try {
response = await axios.post(`https://api.tiny.com.br/api2/${servicePhp}`, params, {
headers: { 'Content-Type': 'application/x-www-form-urlencoded' }
});
} catch (error: any) {
blockedAttempts += 1;
if (MAX_TINY_BLOCK_RETRIES && blockedAttempts > MAX_TINY_BLOCK_RETRIES) {
throw error;
}
const retryLabel = MAX_TINY_BLOCK_RETRIES
? `${blockedAttempts}/${MAX_TINY_BLOCK_RETRIES}`
: String(blockedAttempts);
console.warn(`[Tiny] Request failed on ${servicePhp}: ${describeRequestError(error)}. Pausing ${TINY_BLOCK_DELAY_MS}ms before retry ${retryLabel}.`);
await sleepWithStop(TINY_BLOCK_DELAY_MS, 'retrying Tiny API after request failure');
continue;
}
if (!isTinyApiBlocked(response.data)) {
return response;
@@ -376,6 +411,41 @@ async function fetchContactInfo(cliente: any): Promise<ContactInfo | null> {
}
}
async function fetchSellerInfo(sellerId: string): Promise<SellerInfo | null> {
const normalizedSellerId = String(sellerId || '').trim();
if (!SELLER_FALLBACK || !normalizedSellerId || normalizedSellerId === '0') return null;
if (sellerCache.has(normalizedSellerId)) return sellerCache.get(normalizedSellerId) || null;
try {
const detailParams = new URLSearchParams();
detailParams.append('token', TINY_API_TOKEN!);
detailParams.append('formato', 'JSON');
detailParams.append('id', normalizedSellerId);
const detailResponse = await tinyPost('contato.obter.php', detailParams);
await sleepWithStop(TINY_CONTACT_DELAY_MS, 'next Tiny seller contact request');
if (detailResponse.data?.retorno?.status !== 'OK') {
console.warn(`[Tiny] Seller lookup failed for ${normalizedSellerId}: ${JSON.stringify(detailResponse.data?.retorno?.erros || 'Unknown error')}`);
sellerCache.set(normalizedSellerId, null);
return null;
}
const contato = detailResponse.data.retorno.contato || {};
const info = {
name: pickFirst(contato.nome, contato.razao_social, contato.fantasia, contato.nome_fantasia)
};
sellerCache.set(normalizedSellerId, info.name ? info : null);
return info.name ? info : null;
} catch (error: any) {
if (error instanceof BackfillStoppedError) throw error;
console.error(`[Tiny] Seller lookup failed for ${normalizedSellerId}: ${describeRequestError(error)}`);
sellerCache.set(normalizedSellerId, null);
return null;
}
}
async function buildGraphsRows(details: TinyOrderDetails, summary: TinyOrderSummary): Promise<GraphsOrderRow[]> {
const pedido = details.pedido || {};
const cliente = pedido.cliente || {};
@@ -398,6 +468,12 @@ async function buildGraphsRows(details: TinyOrderDetails, summary: TinyOrderSumm
const customerPhone = normalizePhone(pickFirst(orderCustomerPhone, contactInfo?.phone));
const customerFantasyName = pickFirst(orderCustomerFantasyName, contactInfo?.fantasia);
const ecommerce = pedido.ecommerce || {};
const sellerId = pickFirst(pedido.id_vendedor);
const orderSellerName = pickFirst(pedido.nome_vendedor);
const sellerInfo = sellerId && sellerId !== '0' && !orderSellerName
? await fetchSellerInfo(sellerId)
: null;
const sellerName = pickFirst(orderSellerName, sellerInfo?.name);
return itens
.map((entry: any) => entry?.item || entry)
@@ -413,8 +489,8 @@ async function buildGraphsRows(details: TinyOrderDetails, summary: TinyOrderSumm
ID_Pedido: orderId,
Fone_Cliente: customerPhone,
cliente_nome_fantasia: customerFantasyName,
id_vendedor: pickFirst(pedido.id_vendedor),
nome_vendedor: pickFirst(pedido.nome_vendedor),
id_vendedor: sellerId,
nome_vendedor: sellerName,
marketplace: pickFirst(ecommerce.nomeEcommerce, pedido.nome_ecommerce),
canal_venda: pickFirst(ecommerce.canalVenda, pedido.canal_venda),
numero_ecommerce: pickFirst(
@@ -428,23 +504,42 @@ async function buildGraphsRows(details: TinyOrderDetails, summary: TinyOrderSumm
async function sendRowsToGraphs(rows: GraphsOrderRow[], orderId: string) {
if (!rows.length) {
console.warn(`[Graphs] Order ${orderId} has no items. Skipping.`);
return;
return true;
}
if (DRY_RUN) {
console.log(`[dry-run] Would send ${rows.length} row(s) for order ${orderId}`);
console.log(JSON.stringify(rows.slice(0, 3), null, 2));
return;
return true;
}
await axios.post(GRAPHS_API_URL!, rows, {
headers: {
'Content-Type': 'application/json',
'x-api-key': GRAPHS_API_KEY!
}
});
let attempt = 0;
while (true) {
try {
await axios.post(GRAPHS_API_URL!, rows, {
headers: {
'Content-Type': 'application/json',
'x-api-key': GRAPHS_API_KEY!
}
});
console.log(`[Graphs] Sent ${rows.length} row(s) for order ${orderId}`);
console.log(`[Graphs] Sent ${rows.length} row(s) for order ${orderId}`);
return true;
} catch (error: any) {
attempt += 1;
const status = error?.response?.status;
const shouldRetry = !status || status >= 500;
console.error(`[Graphs] Failed to send order ${orderId} on attempt ${attempt}: ${describeRequestError(error)}`);
if (!shouldRetry || (GRAPHS_API_MAX_RETRIES && attempt >= GRAPHS_API_MAX_RETRIES)) {
console.error(`[Graphs] Giving up on order ${orderId}; backfill will continue with the next order.`);
return false;
}
await sleepWithStop(GRAPHS_API_RETRY_DELAY_MS, `retrying Graphs API for order ${orderId}`);
}
}
}
async function runBackfill() {
@@ -464,6 +559,7 @@ async function runBackfill() {
let lastCompletedDate = state.lastCompletedDate;
let processedDays = 0;
let processedOrders = 0;
let failedOrders = state.failedOrders || 0;
const orderPace = ORDER_DELAY_MS ? `~${Math.floor(60000 / ORDER_DELAY_MS)}/minute` : 'unlimited';
const tinyRequestPace = TINY_REQUEST_DELAY_MS ? `~${Math.floor(60000 / TINY_REQUEST_DELAY_MS)}/minute` : 'unlimited';
@@ -474,6 +570,7 @@ async function runBackfill() {
console.log(`[config] Tiny request pace: 1 request every ${TINY_REQUEST_DELAY_MS}ms (${tinyRequestPace})`);
console.log(`[config] Order pace: 1 order every ${ORDER_DELAY_MS}ms (${orderPace})`);
console.log(`[config] Tiny block pause: ${TINY_BLOCK_DELAY_MS}ms`);
console.log(`[config] Graphs API retries: ${GRAPHS_API_MAX_RETRIES || 'unlimited'}`);
console.log(`[config] State file: ${STATE_FILE}`);
console.log(`[config] Stop file: ${STOP_FILE}`);
@@ -483,6 +580,7 @@ async function runBackfill() {
currentDate,
processedDays,
processedOrders,
failedOrders,
lastMessage: 'Backfill started'
});
@@ -521,23 +619,61 @@ async function runBackfill() {
continue;
}
const details = await fetchOrderDetails(orderId);
if (!details) continue;
try {
const details = await fetchOrderDetails(orderId);
if (!details) {
failedOrders += 1;
saveState({
nextDate: currentDate,
lastCompletedDate,
status: 'running',
currentDate,
lastFailedOrderId: orderId,
processedDays,
processedOrders,
failedOrders,
lastMessage: `Could not fetch Tiny details for order ${orderId}`
});
continue;
}
const rows = await buildGraphsRows(details, summary);
await sendRowsToGraphs(rows, orderId);
processedOrders += 1;
const rows = await buildGraphsRows(details, summary);
const sent = await sendRowsToGraphs(rows, orderId);
if (sent) {
processedOrders += 1;
} else {
failedOrders += 1;
}
saveState({
nextDate: currentDate,
lastCompletedDate,
status: 'running',
currentDate,
lastOrderId: orderId,
processedDays,
processedOrders,
lastMessage: `Processed order ${orderId}`
});
saveState({
nextDate: currentDate,
lastCompletedDate,
status: 'running',
currentDate,
lastOrderId: sent ? orderId : undefined,
lastFailedOrderId: sent ? undefined : orderId,
processedDays,
processedOrders,
failedOrders,
lastMessage: sent ? `Processed order ${orderId}` : `Failed order ${orderId}; continuing`
});
} catch (error: any) {
if (error instanceof BackfillStoppedError) throw error;
failedOrders += 1;
console.error(`[backfill] Unexpected failure for order ${orderId}: ${describeRequestError(error)}. Continuing with next order.`);
saveState({
nextDate: currentDate,
lastCompletedDate,
status: 'running',
currentDate,
lastFailedOrderId: orderId,
processedDays,
processedOrders,
failedOrders,
lastMessage: `Unexpected failure for order ${orderId}; continuing`
});
}
if (!MAX_ORDERS || processedOrders < MAX_ORDERS) {
await sleepWithStop(ORDER_DELAY_MS, 'next order');
@@ -555,6 +691,7 @@ async function runBackfill() {
currentDate,
processedDays,
processedOrders,
failedOrders,
lastMessage: `Completed ${completedDate}`
});
console.log(`[state] Completed ${completedDate}. Next date: ${currentDate}`);
@@ -567,9 +704,10 @@ async function runBackfill() {
currentDate,
processedDays,
processedOrders,
lastMessage: 'Backfill completed'
failedOrders,
lastMessage: failedOrders ? `Backfill completed with ${failedOrders} failed order(s)` : 'Backfill completed'
});
console.log(`\nDone. Processed ${processedOrders} order(s) across ${processedDays} day(s).`);
console.log(`\nDone. Processed ${processedOrders} order(s) across ${processedDays} day(s). Failed orders: ${failedOrders}.`);
}
runBackfill().catch((error: any) => {