blyzer/tiny-webhook
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add Olist V3 product structure sync

Browse Source
This commit is contained in:
Cauê Faleiros
2026-07-28 14:09:48 -03:00
parent 4547642411
commit b1067325b9
7 changed files with 841 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files

104
src/scripts/sync-product-structures.ts
View File

@@ -1,4 +1,5 @@
import axios from 'axios';
import crypto from 'crypto';
import dotenv from 'dotenv';
import fs from 'fs';
import path from 'path';
@@ -47,6 +48,9 @@ type ProductStructurePayload = {
type SyncState = {
status?: 'running' | 'stopped' | 'completed';
sourceSignature?: string;
totalProducts?: number;
nextIndex?: number;
processedProducts?: number;
skippedProducts?: number;
failedProducts?: number;
@@ -120,9 +124,11 @@ const REQUEST_DELAY_MS = numberEnv('PRODUCT_STRUCTURES_REQUEST_DELAY_MS', 2500);
const GRAPHS_RETRY_DELAY_MS = numberEnv('PRODUCT_STRUCTURES_GRAPHS_RETRY_DELAY_MS', 30000);
const GRAPHS_MAX_RETRIES = numberEnv('PRODUCT_STRUCTURES_GRAPHS_MAX_RETRIES', 3);
const MAX_PRODUCTS = numberEnv('PRODUCT_STRUCTURES_MAX_PRODUCTS', 0);
const RESUME = process.env.PRODUCT_STRUCTURES_RESUME !== 'false';
let nextRequestAt = 0;
const productDetailCache = new Map<string, TinyProduct | null>();
let latestState: SyncState | null = null;
class SyncStoppedError extends Error {
constructor() {
@@ -159,10 +165,23 @@ async function waitForRequestSlot(reason: string) {
}
function saveState(state: SyncState) {
fs.writeFileSync(STATE_FILE, JSON.stringify({
latestState = {
...state,
updatedAt: new Date().toISOString()
}, null, 2));
};
fs.writeFileSync(STATE_FILE, JSON.stringify(latestState, null, 2));
}
function readState(): SyncState | null {
if (!fs.existsSync(STATE_FILE)) return null;
try {
const parsed = JSON.parse(fs.readFileSync(STATE_FILE, 'utf-8'));
return parsed && typeof parsed === 'object' ? parsed : null;
} catch (error: any) {
console.warn(`[state] Could not read previous state; starting from the beginning: ${error.message}`);
return null;
}
}
function describeRequestError(error: any) {
@@ -457,29 +476,58 @@ async function runSync() {
console.log(`[config] Graphs URL: ${GRAPHS_PRODUCTION_ORDERS_URL}`);
console.log(`[config] Dry run: ${DRY_RUN ? 'yes' : 'no'}`);
console.log(`[config] Request pace: 1 request every ${REQUEST_DELAY_MS}ms`);
console.log(`[config] Resume: ${RESUME ? 'yes' : 'no'}`);
console.log(`[config] State file: ${STATE_FILE}`);
console.log(`[config] Stop file: ${STOP_FILE}`);
console.log(`[source] Found ${requestedProducts.length} requested product(s).`);
const sourceSignature = crypto
.createHash('sha256')
.update(requestedProducts.map(product => `${product.id}|${product.sku}`).join('\n'))
.digest('hex');
const previousState = readState();
const canResume = RESUME
&& !SKUS.length
&& !PRODUCT_IDS.length
&& previousState?.status !== 'completed'
&& previousState?.sourceSignature === sourceSignature
&& Number.isInteger(previousState?.nextIndex)
&& (previousState?.nextIndex || 0) > 0
&& (previousState?.nextIndex || 0) < requestedProducts.length;
const startIndex = canResume ? previousState!.nextIndex! : 0;
let processedProducts = canResume ? previousState?.processedProducts || 0 : 0;
let skippedProducts = canResume ? previousState?.skippedProducts || 0 : 0;
let failedProducts = canResume ? previousState?.failedProducts || 0 : 0;
let handledThisRun = 0;
let nextIndex = startIndex;
let reachedLimit = false;
if (canResume) {
console.log(`[state] Resuming at ${startIndex + 1}/${requestedProducts.length} after ${previousState?.lastSku || 'the last checkpoint'}.`);
}
saveState({
status: 'running',
processedProducts: 0,
skippedProducts: 0,
failedProducts: 0,
lastMessage: 'Product structures sync started'
sourceSignature,
totalProducts: requestedProducts.length,
nextIndex: startIndex,
processedProducts,
skippedProducts,
failedProducts,
lastSku: canResume ? previousState?.lastSku : undefined,
lastMessage: canResume ? 'Product structures sync resumed' : 'Product structures sync started'
});
let processedProducts = 0;
let skippedProducts = 0;
let failedProducts = 0;
for (const requested of requestedProducts) {
for (let index = startIndex; index < requestedProducts.length; index += 1) {
assertNotStopped();
if (MAX_PRODUCTS && processedProducts + skippedProducts + failedProducts >= MAX_PRODUCTS) {
if (MAX_PRODUCTS && handledThisRun >= MAX_PRODUCTS) {
console.log(`[limit] Stopped after ${MAX_PRODUCTS} product(s).`);
reachedLimit = true;
break;
}
nextIndex = index + 1;
const requested = requestedProducts[index];
const label = requested.sku || requested.id;
try {
const product = await resolveProduct(requested);
@@ -488,12 +536,16 @@ async function runSync() {
failedProducts += 1;
saveState({
status: 'running',
sourceSignature,
totalProducts: requestedProducts.length,
nextIndex,
processedProducts,
skippedProducts,
failedProducts,
lastFailedSku: label,
lastMessage: `Product not found for ${label}`
});
handledThisRun += 1;
continue;
}
@@ -502,12 +554,16 @@ async function runSync() {
failedProducts += 1;
saveState({
status: 'running',
sourceSignature,
totalProducts: requestedProducts.length,
nextIndex,
processedProducts,
skippedProducts,
failedProducts,
lastFailedSku: product.codigo || product.id,
lastMessage: `Could not get structure for ${product.codigo || product.id}`
});
handledThisRun += 1;
continue;
}
@@ -516,12 +572,16 @@ async function runSync() {
console.log(`[Tiny] No structure components for ${payload.order.productSku}; skipping Graphs send.`);
saveState({
status: 'running',
sourceSignature,
totalProducts: requestedProducts.length,
nextIndex,
processedProducts,
skippedProducts,
failedProducts,
lastSku: payload.order.productSku,
lastMessage: `Skipped ${payload.order.productSku}; no structure components`
});
handledThisRun += 1;
continue;
}
@@ -534,6 +594,9 @@ async function runSync() {
saveState({
status: 'running',
sourceSignature,
totalProducts: requestedProducts.length,
nextIndex,
processedProducts,
skippedProducts,
failedProducts,
@@ -541,35 +604,46 @@ async function runSync() {
lastFailedSku: sent ? undefined : payload.order.productSku,
lastMessage: sent ? `Synced structure ${payload.order.productSku}` : `Failed structure ${payload.order.productSku}; continuing`
});
handledThisRun += 1;
} catch (error: any) {
if (error instanceof SyncStoppedError) throw error;
failedProducts += 1;
console.error(`[sync] Failed product ${label}: ${describeRequestError(error)}`);
saveState({
status: 'running',
sourceSignature,
totalProducts: requestedProducts.length,
nextIndex,
processedProducts,
skippedProducts,
failedProducts,
lastFailedSku: label,
lastMessage: `Failed product ${label}; continuing`
});
handledThisRun += 1;
}
}
saveState({
status: 'completed',
status: reachedLimit ? 'stopped' : 'completed',
sourceSignature,
totalProducts: requestedProducts.length,
nextIndex,
processedProducts,
skippedProducts,
failedProducts,
lastMessage: failedProducts ? `Product structures sync completed with ${failedProducts} failed product(s)` : 'Product structures sync completed'
lastMessage: reachedLimit
? `Product structures sync paused after ${handledThisRun} product(s)`
: (failedProducts ? `Product structures sync completed with ${failedProducts} failed product(s)` : 'Product structures sync completed')
});
console.log(`Done. Synced ${processedProducts} product structure(s). Skipped products: ${skippedProducts}. Failed products: ${failedProducts}.`);
console.log(`${reachedLimit ? 'Paused' : 'Done'}. Synced ${processedProducts} product structure(s). Skipped products: ${skippedProducts}. Failed products: ${failedProducts}.`);
}
runSync().catch((error: any) => {
if (error instanceof SyncStoppedError) {
console.log('Product structures sync stopped by request.');
saveState({
...latestState,
status: 'stopped',
lastMessage: 'Product structures sync stopped by request'
});