blyzer/tiny-webhook
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add Olist V3 product structure sync

Browse Source
This commit is contained in:
Cauê Faleiros
2026-07-28 14:09:48 -03:00
parent 4547642411
commit b1067325b9
7 changed files with 841 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
.gitignore vendored
View File

@@ -5,5 +5,8 @@ graphs_backfill_state.json
graphs_backfill_stop.json
product_structures_sync_state.json
product_structures_sync_stop.json
product_structures_v3_sync_state.json
product_structures_v3_sync_stop.json
data-runtime/
production_orders_sync_state.json
production_orders_sync_stop.json

13
docker-compose.yml
View File

@@ -27,6 +27,14 @@ services:
- PRODUCT_STRUCTURES_GRAPHS_API_KEY=${PRODUCT_STRUCTURES_GRAPHS_API_KEY}
- PRODUCT_STRUCTURES_REQUEST_DELAY_MS=${PRODUCT_STRUCTURES_REQUEST_DELAY_MS:-2500}
- PRODUCT_STRUCTURES_FETCH_COMPONENT_UNITS=${PRODUCT_STRUCTURES_FETCH_COMPONENT_UNITS:-true}
- OLIST_V3_CLIENT_ID=${OLIST_V3_CLIENT_ID}
- OLIST_V3_CLIENT_SECRET=${OLIST_V3_CLIENT_SECRET}
- OLIST_V3_REDIRECT_URI=${OLIST_V3_REDIRECT_URI}
- OLIST_V3_TOKEN_FILE=${OLIST_V3_TOKEN_FILE:-/app/data-runtime/olist_v3_tokens.json}
- OLIST_V3_OAUTH_STATE_FILE=${OLIST_V3_OAUTH_STATE_FILE:-/app/data-runtime/olist_v3_oauth_state.json}
- PRODUCT_STRUCTURES_V3_GRAPHS_API_URL=${PRODUCT_STRUCTURES_V3_GRAPHS_API_URL}
- PRODUCT_STRUCTURES_V3_GRAPHS_API_KEY=${PRODUCT_STRUCTURES_V3_GRAPHS_API_KEY}
- PRODUCT_STRUCTURES_V3_REQUEST_DELAY_MS=${PRODUCT_STRUCTURES_V3_REQUEST_DELAY_MS:-1000}
- PRODUCTION_ORDERS_TINY_ERP_COOKIE=${PRODUCTION_ORDERS_TINY_ERP_COOKIE}
- PRODUCTION_ORDERS_GRAPHS_API_URL=${PRODUCTION_ORDERS_GRAPHS_API_URL}
- PRODUCTION_ORDERS_GRAPHS_API_KEY=${PRODUCTION_ORDERS_GRAPHS_API_KEY}
@@ -40,7 +48,12 @@ services:
# Optional: If you use a shared Docker network for Nginx Proxy Manager, you can attach it here:
# networks:
# - nginx-proxy-manager-network
volumes:
- middleware_data:/app/data-runtime
# networks:
# nginx-proxy-manager-network:
# external: true
volumes:
middleware_data:

1
package.json
View File

@@ -9,6 +9,7 @@
"start": "node dist/index.js",
"backfill:graphs": "tsx src/scripts/backfill-graphs.ts",
"sync:product-structures": "tsx src/scripts/sync-product-structures.ts",
"sync:product-structures:v3": "tsx src/scripts/sync-product-structures-v3.ts",
"sync:production-orders": "tsx src/scripts/sync-production-orders.ts",
"test": "echo \"Error: no test specified\" && exit 1"
},

162
src/index.ts
View File

@@ -4,6 +4,12 @@ import webhookRoutes from './routes/webhook.route';
import { spawn } from 'child_process';
import path from 'path';
import fs from 'fs';
import {
exchangeOlistV3AuthorizationCode,
getOlistV3AuthorizationUrl,
getOlistV3AuthStatus,
isOlistV3Configured
} from './services/olist-v3.service';
dotenv.config();
@@ -31,6 +37,56 @@ app.get('/health', (req: Request, res: Response) => {
res.status(200).json({ status: 'OK', message: 'Tiny-n8n middleware is running.' });
});
// Starts the official Olist V3 OAuth flow. The callback validates a one-time state value.
app.get('/api/olist-v3/authorize', (req: Request, res: Response) => {
const expectedToken = process.env.TINY_WEBHOOK_SECRET;
if (expectedToken && req.query.token !== expectedToken) {
res.status(401).json({ error: 'Unauthorized' });
return;
}
if (!isOlistV3Configured()) {
res.status(400).json({ error: 'Missing OLIST_V3_CLIENT_ID, OLIST_V3_CLIENT_SECRET, or OLIST_V3_REDIRECT_URI.' });
return;
}
res.redirect(getOlistV3AuthorizationUrl());
});
app.get('/api/olist-v3/callback', async (req: Request, res: Response) => {
const providerError = typeof req.query.error === 'string' ? req.query.error : '';
const code = typeof req.query.code === 'string' ? req.query.code : '';
const state = typeof req.query.state === 'string' ? req.query.state : '';
if (providerError) {
res.status(400).send('Olist V3 authorization was denied or failed. Return to the middleware and start authorization again.');
return;
}
if (!code || !state) {
res.status(400).send('Missing OAuth code or state. Start authorization again.');
return;
}
try {
await exchangeOlistV3AuthorizationCode(code, state);
res.status(200).send('Olist V3 authorized successfully. You can close this page and trigger the V3 product structure test.');
} catch (error: any) {
console.error(`[server]: Olist V3 OAuth callback failed: ${error.message}`);
res.status(400).send('Olist V3 authorization could not be completed. Check Portainer logs and start authorization again.');
}
});
app.get('/api/olist-v3/status', (req: Request, res: Response) => {
const expectedToken = process.env.TINY_WEBHOOK_SECRET;
if (expectedToken && req.query.token !== expectedToken) {
res.status(401).json({ error: 'Unauthorized' });
return;
}
res.status(200).json(getOlistV3AuthStatus());
});
// Hidden endpoint to trigger the backfill script without terminal access
app.get('/api/trigger-backfill', (req: Request, res: Response) => {
const expectedToken = process.env.TINY_WEBHOOK_SECRET;
@@ -296,6 +352,112 @@ app.get('/api/stop-product-structures-sync', (req: Request, res: Response) => {
});
});
// Hidden endpoint to sync manufactured-product compositions from the official Olist V3 API.
app.get('/api/trigger-product-structures-v3-sync', (req: Request, res: Response) => {
const expectedToken = process.env.TINY_WEBHOOK_SECRET;
if (expectedToken && req.query.token !== expectedToken) {
res.status(401).json({ error: 'Unauthorized' });
return;
}
const scriptPath = path.join(__dirname, 'scripts', 'sync-product-structures-v3.js');
const env = { ...process.env };
const stopFile = env.PRODUCT_STRUCTURES_V3_STOP_FILE || path.join(process.cwd(), 'data-runtime', 'product_structures_v3_sync_stop.json');
const optionalEnvMap: Record<string, string> = {
productIds: 'PRODUCT_STRUCTURES_V3_PRODUCT_IDS',
inputFile: 'PRODUCT_STRUCTURES_V3_INPUT_FILE',
graphsUrl: 'PRODUCT_STRUCTURES_V3_GRAPHS_API_URL',
graphsApiKey: 'PRODUCT_STRUCTURES_V3_GRAPHS_API_KEY',
dryRun: 'PRODUCT_STRUCTURES_V3_DRY_RUN',
maxProducts: 'PRODUCT_STRUCTURES_V3_MAX_PRODUCTS',
requestDelayMs: 'PRODUCT_STRUCTURES_V3_REQUEST_DELAY_MS',
graphsRetryDelayMs: 'PRODUCT_STRUCTURES_V3_GRAPHS_RETRY_DELAY_MS',
graphsMaxRetries: 'PRODUCT_STRUCTURES_V3_GRAPHS_MAX_RETRIES'
};
for (const [queryKey, envKey] of Object.entries(optionalEnvMap)) {
const value = req.query[queryKey];
if (typeof value === 'string' && value.trim() !== '') {
env[envKey] = value.trim();
}
}
if (!env.OLIST_V3_CLIENT_ID || !env.OLIST_V3_CLIENT_SECRET || !env.OLIST_V3_REDIRECT_URI) {
res.status(400).json({ error: 'Olist V3 is not configured. Set the client ID, client secret, and redirect URI first.' });
return;
}
if (!getOlistV3AuthStatus().authorized) {
res.status(409).json({ error: 'Olist V3 is not authorized. Open /api/olist-v3/authorize first.' });
return;
}
if (fs.existsSync(stopFile)) {
fs.rmSync(stopFile, { force: true });
}
const child = spawn('node', [scriptPath], { detached: true, stdio: 'inherit', env });
child.on('error', (error) => {
console.error(`[server]: Olist V3 product structures sync failed to start: ${error.message}`);
});
child.on('exit', (code, signal) => {
console.log(`[server]: Olist V3 product structures sync exited. code=${code ?? 'null'} signal=${signal ?? 'null'}`);
});
child.unref();
console.log('[server]: Olist V3 product structures sync manually triggered via HTTP endpoint.');
res.status(200).json({
status: 'STARTED',
message: 'Olist V3 product structures sync is running in the background. Check Portainer logs for progress.',
options: {
productIds: env.PRODUCT_STRUCTURES_V3_PRODUCT_IDS || null,
inputFile: env.PRODUCT_STRUCTURES_V3_INPUT_FILE || (env.PRODUCT_STRUCTURES_V3_PRODUCT_IDS ? null : '/app/data/product-structure-products.csv'),
graphsUrl: env.PRODUCT_STRUCTURES_V3_GRAPHS_API_URL || deriveProductionOrdersUrl(env),
dryRun: env.PRODUCT_STRUCTURES_V3_DRY_RUN || 'false',
maxProducts: env.PRODUCT_STRUCTURES_V3_MAX_PRODUCTS || null,
requestDelayMs: env.PRODUCT_STRUCTURES_V3_REQUEST_DELAY_MS || '1000',
olistV3Configured: true,
olistV3Authorized: getOlistV3AuthStatus().authorized,
graphsApiKeyConfigured: Boolean(env.PRODUCT_STRUCTURES_V3_GRAPHS_API_KEY || env.GRAPHS_API_KEY || env.NEXSTAR_GRAPHS_API_KEY)
}
});
});
app.get('/api/product-structures-v3-sync-status', (req: Request, res: Response) => {
const expectedToken = process.env.TINY_WEBHOOK_SECRET;
if (expectedToken && req.query.token !== expectedToken) {
res.status(401).json({ error: 'Unauthorized' });
return;
}
const stateFile = process.env.PRODUCT_STRUCTURES_V3_STATE_FILE || path.join(process.cwd(), 'data-runtime', 'product_structures_v3_sync_state.json');
const stopFile = process.env.PRODUCT_STRUCTURES_V3_STOP_FILE || path.join(process.cwd(), 'data-runtime', 'product_structures_v3_sync_stop.json');
let state = null;
if (fs.existsSync(stateFile)) {
try {
state = JSON.parse(fs.readFileSync(stateFile, 'utf-8'));
} catch (error: any) {
res.status(500).json({ error: `Could not read Olist V3 product structures state: ${error.message}` });
return;
}
}
res.status(200).json({ status: state?.status || 'unknown', stopRequested: fs.existsSync(stopFile), state });
});
app.get('/api/stop-product-structures-v3-sync', (req: Request, res: Response) => {
const expectedToken = process.env.TINY_WEBHOOK_SECRET;
if (expectedToken && req.query.token !== expectedToken) {
res.status(401).json({ error: 'Unauthorized' });
return;
}
const stopFile = process.env.PRODUCT_STRUCTURES_V3_STOP_FILE || path.join(process.cwd(), 'data-runtime', 'product_structures_v3_sync_stop.json');
fs.mkdirSync(path.dirname(stopFile), { recursive: true });
fs.writeFileSync(stopFile, JSON.stringify({ requestedAt: new Date().toISOString() }, null, 2));
res.status(200).json({ status: 'STOP_REQUESTED', message: 'Olist V3 product structures sync will stop after the current request.' });
});
// Hidden endpoint to sync Tiny/Olist production order details into Graphs
app.get('/api/trigger-production-orders-sync', (req: Request, res: Response) => {
const expectedToken = process.env.TINY_WEBHOOK_SECRET;

399
src/scripts/sync-product-structures-v3.ts Normal file
View File

@@ -0,0 +1,399 @@
import axios from 'axios';
import crypto from 'crypto';
import dotenv from 'dotenv';
import fs from 'fs';
import path from 'path';
import { getOlistV3AccessToken, isOlistV3Configured } from '../services/olist-v3.service';
dotenv.config();
type RequestedProduct = { id: string };
type V3Product = { id: string; sku: string; descricao: string; unidade: string };
type V3StructureItem = {
produto?: { id?: number | string; sku?: string; descricao?: string };
quantidade?: number | string;
};
type V3FabricatedProduct = { produtos?: V3StructureItem[]; etapas?: string[] };
type StructurePayload = {
order: {
tinyId: string;
number: string;
status: string;
productSku: string;
productDescription: string;
quantity: string;
unit: string;
issueDate: null;
expectedDate: null;
supplier: string;
lotCode: string;
notes: string;
};
components: Array<{
componentTinyId: string;
componentName: string;
componentSku: string;
quantityPerUnit: string;
totalQuantity: string;
unit: string;
}>;
steps: Array<{
stepNumber: number;
name: string;
startDate: null;
endDate: null;
status: string;
}>;
};
type SyncState = {
status?: 'running' | 'stopped' | 'completed';
sourceSignature?: string;
totalProducts?: number;
nextIndex?: number;
processedProducts?: number;
skippedProducts?: number;
failedProducts?: number;
lastProductId?: string;
lastFailedProductId?: string;
lastMessage?: string;
updatedAt?: string;
};
const V3_API_BASE_URL = (process.env.OLIST_V3_API_BASE_URL || 'https://api.tiny.com.br/public-api/v3').replace(/\/+$/, '');
const DEFAULT_INPUT_FILE = path.join(process.cwd(), 'data', 'product-structure-products.csv');
const INPUT_FILE = process.env.PRODUCT_STRUCTURES_V3_INPUT_FILE || DEFAULT_INPUT_FILE;
const PRODUCT_IDS = String(process.env.PRODUCT_STRUCTURES_V3_PRODUCT_IDS || '')
.split(',')
.map(value => value.trim())
.filter(Boolean);
const DRY_RUN = process.env.PRODUCT_STRUCTURES_V3_DRY_RUN === 'true';
const RESUME = process.env.PRODUCT_STRUCTURES_V3_RESUME !== 'false';
const RUNTIME_DIRECTORY = path.join(process.cwd(), 'data-runtime');
const STATE_FILE = process.env.PRODUCT_STRUCTURES_V3_STATE_FILE || path.join(RUNTIME_DIRECTORY, 'product_structures_v3_sync_state.json');
const STOP_FILE = process.env.PRODUCT_STRUCTURES_V3_STOP_FILE || path.join(RUNTIME_DIRECTORY, 'product_structures_v3_sync_stop.json');
function deriveGraphsUrl() {
const raw = process.env.PRODUCT_STRUCTURES_V3_GRAPHS_API_URL
|| process.env.PRODUCT_STRUCTURES_GRAPHS_API_URL
|| process.env.PRODUCTION_ORDERS_GRAPHS_API_URL
|| process.env.GRAPHS_API_URL
|| '';
if (!raw) return 'http://localhost:3004/api/production-orders/tiny-sync';
try {
const parsed = new URL(raw);
return parsed.pathname.includes('/api/data')
? `${parsed.origin}/api/production-orders/tiny-sync`
: raw;
} catch {
return raw.replace(/\/api\/data.*$/, '/api/production-orders/tiny-sync');
}
}
const GRAPHS_URL = deriveGraphsUrl();
const GRAPHS_API_KEY = process.env.PRODUCT_STRUCTURES_V3_GRAPHS_API_KEY
|| process.env.PRODUCT_STRUCTURES_GRAPHS_API_KEY
|| process.env.PRODUCTION_ORDERS_GRAPHS_API_KEY
|| process.env.GRAPHS_API_KEY
|| process.env.NEXSTAR_GRAPHS_API_KEY
|| process.env.API_KEY
|| 'nexstar_secret_key_123';
function numberEnv(name: string, fallback: number) {
const value = Number(process.env[name]);
return Number.isFinite(value) && value >= 0 ? value : fallback;
}
const REQUEST_DELAY_MS = numberEnv('PRODUCT_STRUCTURES_V3_REQUEST_DELAY_MS', 1000);
const MAX_PRODUCTS = numberEnv('PRODUCT_STRUCTURES_V3_MAX_PRODUCTS', 0);
const GRAPHS_MAX_RETRIES = numberEnv('PRODUCT_STRUCTURES_V3_GRAPHS_MAX_RETRIES', 3);
const GRAPHS_RETRY_DELAY_MS = numberEnv('PRODUCT_STRUCTURES_V3_GRAPHS_RETRY_DELAY_MS', 30000);
let nextRequestAt = 0;
let latestState: SyncState | null = null;
const componentProductCache = new Map<string, V3Product | null>();
class SyncStoppedError extends Error {
constructor() {
super('V3 product structure sync stopped by request.');
}
}
const sleep = (ms: number) => new Promise(resolve => setTimeout(resolve, ms));
const text = (value: unknown) => String(value ?? '').trim();
function readProductsFromCsv(filePath: string): RequestedProduct[] {
const rows = fs.readFileSync(filePath, 'utf-8').replace(/^\uFEFF/, '').split(/\r?\n/).filter(Boolean);
const headers = rows.shift()?.split(',').map(header => header.trim().toLowerCase()) || [];
const idIndex = headers.findIndex(header => ['id', 'productid', 'tinyid', 'id produto'].includes(header));
if (idIndex < 0) throw new Error(`No product id column found in ${filePath}.`);
return rows
.map(row => row.split(',')[idIndex]?.trim())
.filter(Boolean)
.map(id => ({ id }));
}
function loadRequestedProducts() {
const fromFile = PRODUCT_IDS.length ? [] : readProductsFromCsv(INPUT_FILE);
const requested = fromFile.concat(PRODUCT_IDS.map(id => ({ id })));
return requested.filter((item, index, all) => all.findIndex(other => other.id === item.id) === index);
}
function saveState(state: SyncState) {
latestState = { ...state, updatedAt: new Date().toISOString() };
fs.mkdirSync(path.dirname(STATE_FILE), { recursive: true });
fs.writeFileSync(STATE_FILE, JSON.stringify(latestState, null, 2));
}
function readState() {
if (!fs.existsSync(STATE_FILE)) return null;
try {
return JSON.parse(fs.readFileSync(STATE_FILE, 'utf-8')) as SyncState;
} catch (error: any) {
console.warn(`[state] Could not read previous state: ${error.message}`);
return null;
}
}
function assertNotStopped() {
if (fs.existsSync(STOP_FILE)) throw new SyncStoppedError();
}
async function waitForSlot(reason: string) {
const waitMs = Math.max(0, nextRequestAt - Date.now());
if (waitMs > 0) console.log(`[wait] Waiting ${waitMs}ms before ${reason}.`);
let remaining = waitMs;
while (remaining > 0) {
assertNotStopped();
await sleep(Math.min(remaining, 1000));
remaining = Math.max(0, nextRequestAt - Date.now());
}
nextRequestAt = Date.now() + REQUEST_DELAY_MS;
}
function describeError(error: any) {
if (error?.response) {
const body = JSON.stringify(error.response.data ?? '');
return `HTTP ${error.response.status}: ${body.slice(0, 500)}`;
}
return error?.message || String(error);
}
async function v3Get<T>(pathname: string): Promise<T> {
await waitForSlot(`Olist V3 ${pathname}`);
const request = async (forceRefresh: boolean) => axios.get<T>(`${V3_API_BASE_URL}${pathname}`, {
headers: { Authorization: `Bearer ${await getOlistV3AccessToken(forceRefresh)}` },
timeout: 30000
});
try {
return (await request(false)).data;
} catch (error: any) {
if (error?.response?.status !== 401) throw error;
await waitForSlot(`Olist V3 ${pathname} retry after token refresh`);
return (await request(true)).data;
}
}
async function getProduct(id: string): Promise<V3Product> {
const raw = await v3Get<any>(`/produtos/${encodeURIComponent(id)}`);
return {
id: text(raw?.id || id),
sku: text(raw?.sku),
descricao: text(raw?.descricao),
unidade: text(raw?.unidade)
};
}
async function getComponentProduct(id: string) {
if (componentProductCache.has(id)) return componentProductCache.get(id) || null;
try {
const product = await getProduct(id);
componentProductCache.set(id, product);
return product;
} catch (error: any) {
console.warn(`[Olist V3] Could not get component ${id}: ${describeError(error)}`);
componentProductCache.set(id, null);
return null;
}
}
async function getFabricatedProduct(id: string): Promise<V3FabricatedProduct | null> {
try {
return await v3Get<V3FabricatedProduct>(`/produtos/${encodeURIComponent(id)}/fabricado`);
} catch (error: any) {
if ([400, 404].includes(error?.response?.status)) return null;
throw error;
}
}
function formatDecimal(value: unknown) {
const parsed = Number(value);
return Number.isFinite(parsed) ? String(parsed) : '0';
}
async function buildPayload(productId: string): Promise<StructurePayload | null> {
const product = await getProduct(productId);
const manufactured = await getFabricatedProduct(product.id);
const rawComponents = Array.isArray(manufactured?.produtos) ? manufactured.produtos : [];
if (!manufactured || !rawComponents.length) return null;
const components = [];
for (const item of rawComponents) {
const componentId = text(item.produto?.id);
const componentDetail = componentId ? await getComponentProduct(componentId) : null;
const quantity = formatDecimal(item.quantidade);
components.push({
componentTinyId: componentId,
componentName: text(item.produto?.descricao || componentDetail?.descricao),
componentSku: text(item.produto?.sku || componentDetail?.sku),
quantityPerUnit: quantity,
totalQuantity: quantity,
unit: text(componentDetail?.unidade)
});
}
return {
order: {
tinyId: `STRUCTURE-V3-${product.id}`,
number: `STRUCTURE-V3-${product.sku || product.id}`,
status: 'completed',
productSku: product.sku,
productDescription: product.descricao,
quantity: '1',
unit: product.unidade || 'UN',
issueDate: null,
expectedDate: null,
supplier: '',
lotCode: '',
notes: `Composição sincronizada da API pública Olist V3 para produto ${product.id}.`
},
components: components.filter(component => component.componentSku || component.componentName),
steps: (manufactured?.etapas || []).map((name, index) => ({
stepNumber: index + 1,
name: text(name),
startDate: null,
endDate: null,
status: 'pendente'
})).filter(step => step.name)
};
}
async function sendToGraphs(payload: StructurePayload) {
if (DRY_RUN) {
console.log(`[dry-run] ${JSON.stringify(payload)}`);
return true;
}
for (let attempt = 1; ; attempt += 1) {
try {
await axios.post(GRAPHS_URL, payload, {
headers: { 'Content-Type': 'application/json', 'x-api-key': GRAPHS_API_KEY },
timeout: 30000
});
console.log(`[Graphs V3] Sent ${payload.order.productSku} with ${payload.components.length} component(s).`);
return true;
} catch (error: any) {
const retryable = !error?.response?.status || error.response.status >= 500;
console.error(`[Graphs V3] Send failed for ${payload.order.productSku}, attempt ${attempt}: ${describeError(error)}`);
if (!retryable || (GRAPHS_MAX_RETRIES && attempt >= GRAPHS_MAX_RETRIES)) return false;
await sleep(GRAPHS_RETRY_DELAY_MS);
}
}
}
async function run() {
if (!isOlistV3Configured()) {
throw new Error('Olist V3 is not configured. Set OLIST_V3_CLIENT_ID, OLIST_V3_CLIENT_SECRET, and OLIST_V3_REDIRECT_URI.');
}
const products = loadRequestedProducts();
if (!products.length) throw new Error('No V3 products configured.');
await getOlistV3AccessToken();
const sourceSignature = crypto.createHash('sha256').update(products.map(product => product.id).join('\n')).digest('hex');
const previous = readState();
const canResume = RESUME
&& !PRODUCT_IDS.length
&& previous?.status !== 'completed'
&& previous?.sourceSignature === sourceSignature
&& Number.isInteger(previous?.nextIndex)
&& (previous?.nextIndex || 0) > 0
&& (previous?.nextIndex || 0) < products.length;
let nextIndex = canResume ? previous!.nextIndex! : 0;
let processedProducts = canResume ? previous?.processedProducts || 0 : 0;
let skippedProducts = canResume ? previous?.skippedProducts || 0 : 0;
let failedProducts = canResume ? previous?.failedProducts || 0 : 0;
let handledThisRun = 0;
let reachedLimit = false;
console.log(`[config] Olist V3 product structures: ${products.length} product(s), ${REQUEST_DELAY_MS}ms request pace.`);
if (canResume) console.log(`[state] Resuming at ${nextIndex + 1}/${products.length}.`);
const checkpoint = (state: Partial<SyncState>) => saveState({
status: 'running',
sourceSignature,
totalProducts: products.length,
nextIndex,
processedProducts,
skippedProducts,
failedProducts,
...state
});
checkpoint({ lastMessage: canResume ? 'V3 structure sync resumed' : 'V3 structure sync started' });
for (let index = nextIndex; index < products.length; index += 1) {
assertNotStopped();
if (MAX_PRODUCTS && handledThisRun >= MAX_PRODUCTS) {
reachedLimit = true;
break;
}
const productId = products[index].id;
nextIndex = index + 1;
try {
const payload = await buildPayload(productId);
if (!payload?.components.length) {
skippedProducts += 1;
console.log(`[Olist V3] No manufactured-product composition for ${productId}; skipping Graphs send.`);
checkpoint({ lastProductId: productId, lastMessage: `Skipped ${productId}; no composition` });
} else if (await sendToGraphs(payload)) {
processedProducts += 1;
checkpoint({ lastProductId: productId, lastMessage: `Synced ${payload.order.productSku}` });
} else {
failedProducts += 1;
checkpoint({ lastFailedProductId: productId, lastMessage: `Graphs send failed for ${productId}` });
}
} catch (error: any) {
if (error instanceof SyncStoppedError) throw error;
failedProducts += 1;
console.error(`[sync] Failed product ${productId}: ${describeError(error)}`);
checkpoint({ lastFailedProductId: productId, lastMessage: `Failed ${productId}; continuing` });
}
handledThisRun += 1;
}
saveState({
status: reachedLimit ? 'stopped' : 'completed',
sourceSignature,
totalProducts: products.length,
nextIndex,
processedProducts,
skippedProducts,
failedProducts,
lastMessage: reachedLimit ? `V3 structure sync paused after ${handledThisRun} product(s)` : 'V3 structure sync completed'
});
console.log(`Done. V3 synced ${processedProducts}; skipped ${skippedProducts}; failed ${failedProducts}.`);
}
run().catch((error: any) => {
if (error instanceof SyncStoppedError) {
saveState({ ...latestState, status: 'stopped', lastMessage: 'V3 structure sync stopped by request' });
console.log('V3 product structure sync stopped by request.');
process.exit(0);
}
console.error(`[sync] V3 product structure sync failed: ${describeError(error)}`);
saveState({ ...latestState, status: 'stopped', lastMessage: describeError(error) });
process.exit(1);
});

104
src/scripts/sync-product-structures.ts
View File

@@ -1,4 +1,5 @@
import axios from 'axios';
import crypto from 'crypto';
import dotenv from 'dotenv';
import fs from 'fs';
import path from 'path';
@@ -47,6 +48,9 @@ type ProductStructurePayload = {
type SyncState = {
status?: 'running' | 'stopped' | 'completed';
sourceSignature?: string;
totalProducts?: number;
nextIndex?: number;
processedProducts?: number;
skippedProducts?: number;
failedProducts?: number;
@@ -120,9 +124,11 @@ const REQUEST_DELAY_MS = numberEnv('PRODUCT_STRUCTURES_REQUEST_DELAY_MS', 2500);
const GRAPHS_RETRY_DELAY_MS = numberEnv('PRODUCT_STRUCTURES_GRAPHS_RETRY_DELAY_MS', 30000);
const GRAPHS_MAX_RETRIES = numberEnv('PRODUCT_STRUCTURES_GRAPHS_MAX_RETRIES', 3);
const MAX_PRODUCTS = numberEnv('PRODUCT_STRUCTURES_MAX_PRODUCTS', 0);
const RESUME = process.env.PRODUCT_STRUCTURES_RESUME !== 'false';
let nextRequestAt = 0;
const productDetailCache = new Map<string, TinyProduct | null>();
let latestState: SyncState | null = null;
class SyncStoppedError extends Error {
constructor() {
@@ -159,10 +165,23 @@ async function waitForRequestSlot(reason: string) {
}
function saveState(state: SyncState) {
fs.writeFileSync(STATE_FILE, JSON.stringify({
latestState = {
...state,
updatedAt: new Date().toISOString()
}, null, 2));
};
fs.writeFileSync(STATE_FILE, JSON.stringify(latestState, null, 2));
}
function readState(): SyncState | null {
if (!fs.existsSync(STATE_FILE)) return null;
try {
const parsed = JSON.parse(fs.readFileSync(STATE_FILE, 'utf-8'));
return parsed && typeof parsed === 'object' ? parsed : null;
} catch (error: any) {
console.warn(`[state] Could not read previous state; starting from the beginning: ${error.message}`);
return null;
}
}
function describeRequestError(error: any) {
@@ -457,29 +476,58 @@ async function runSync() {
console.log(`[config] Graphs URL: ${GRAPHS_PRODUCTION_ORDERS_URL}`);
console.log(`[config] Dry run: ${DRY_RUN ? 'yes' : 'no'}`);
console.log(`[config] Request pace: 1 request every ${REQUEST_DELAY_MS}ms`);
console.log(`[config] Resume: ${RESUME ? 'yes' : 'no'}`);
console.log(`[config] State file: ${STATE_FILE}`);
console.log(`[config] Stop file: ${STOP_FILE}`);
console.log(`[source] Found ${requestedProducts.length} requested product(s).`);
const sourceSignature = crypto
.createHash('sha256')
.update(requestedProducts.map(product => `${product.id}|${product.sku}`).join('\n'))
.digest('hex');
const previousState = readState();
const canResume = RESUME
&& !SKUS.length
&& !PRODUCT_IDS.length
&& previousState?.status !== 'completed'
&& previousState?.sourceSignature === sourceSignature
&& Number.isInteger(previousState?.nextIndex)
&& (previousState?.nextIndex || 0) > 0
&& (previousState?.nextIndex || 0) < requestedProducts.length;
const startIndex = canResume ? previousState!.nextIndex! : 0;
let processedProducts = canResume ? previousState?.processedProducts || 0 : 0;
let skippedProducts = canResume ? previousState?.skippedProducts || 0 : 0;
let failedProducts = canResume ? previousState?.failedProducts || 0 : 0;
let handledThisRun = 0;
let nextIndex = startIndex;
let reachedLimit = false;
if (canResume) {
console.log(`[state] Resuming at ${startIndex + 1}/${requestedProducts.length} after ${previousState?.lastSku || 'the last checkpoint'}.`);
}
saveState({
status: 'running',
processedProducts: 0,
skippedProducts: 0,
failedProducts: 0,
lastMessage: 'Product structures sync started'
sourceSignature,
totalProducts: requestedProducts.length,
nextIndex: startIndex,
processedProducts,
skippedProducts,
failedProducts,
lastSku: canResume ? previousState?.lastSku : undefined,
lastMessage: canResume ? 'Product structures sync resumed' : 'Product structures sync started'
});
let processedProducts = 0;
let skippedProducts = 0;
let failedProducts = 0;
for (const requested of requestedProducts) {
for (let index = startIndex; index < requestedProducts.length; index += 1) {
assertNotStopped();
if (MAX_PRODUCTS && processedProducts + skippedProducts + failedProducts >= MAX_PRODUCTS) {
if (MAX_PRODUCTS && handledThisRun >= MAX_PRODUCTS) {
console.log(`[limit] Stopped after ${MAX_PRODUCTS} product(s).`);
reachedLimit = true;
break;
}
nextIndex = index + 1;
const requested = requestedProducts[index];
const label = requested.sku || requested.id;
try {
const product = await resolveProduct(requested);
@@ -488,12 +536,16 @@ async function runSync() {
failedProducts += 1;
saveState({
status: 'running',
sourceSignature,
totalProducts: requestedProducts.length,
nextIndex,
processedProducts,
skippedProducts,
failedProducts,
lastFailedSku: label,
lastMessage: `Product not found for ${label}`
});
handledThisRun += 1;
continue;
}
@@ -502,12 +554,16 @@ async function runSync() {
failedProducts += 1;
saveState({
status: 'running',
sourceSignature,
totalProducts: requestedProducts.length,
nextIndex,
processedProducts,
skippedProducts,
failedProducts,
lastFailedSku: product.codigo || product.id,
lastMessage: `Could not get structure for ${product.codigo || product.id}`
});
handledThisRun += 1;
continue;
}
@@ -516,12 +572,16 @@ async function runSync() {
console.log(`[Tiny] No structure components for ${payload.order.productSku}; skipping Graphs send.`);
saveState({
status: 'running',
sourceSignature,
totalProducts: requestedProducts.length,
nextIndex,
processedProducts,
skippedProducts,
failedProducts,
lastSku: payload.order.productSku,
lastMessage: `Skipped ${payload.order.productSku}; no structure components`
});
handledThisRun += 1;
continue;
}
@@ -534,6 +594,9 @@ async function runSync() {
saveState({
status: 'running',
sourceSignature,
totalProducts: requestedProducts.length,
nextIndex,
processedProducts,
skippedProducts,
failedProducts,
@@ -541,35 +604,46 @@ async function runSync() {
lastFailedSku: sent ? undefined : payload.order.productSku,
lastMessage: sent ? `Synced structure ${payload.order.productSku}` : `Failed structure ${payload.order.productSku}; continuing`
});
handledThisRun += 1;
} catch (error: any) {
if (error instanceof SyncStoppedError) throw error;
failedProducts += 1;
console.error(`[sync] Failed product ${label}: ${describeRequestError(error)}`);
saveState({
status: 'running',
sourceSignature,
totalProducts: requestedProducts.length,
nextIndex,
processedProducts,
skippedProducts,
failedProducts,
lastFailedSku: label,
lastMessage: `Failed product ${label}; continuing`
});
handledThisRun += 1;
}
}
saveState({
status: 'completed',
status: reachedLimit ? 'stopped' : 'completed',
sourceSignature,
totalProducts: requestedProducts.length,
nextIndex,
processedProducts,
skippedProducts,
failedProducts,
lastMessage: failedProducts ? `Product structures sync completed with ${failedProducts} failed product(s)` : 'Product structures sync completed'
lastMessage: reachedLimit
? `Product structures sync paused after ${handledThisRun} product(s)`
: (failedProducts ? `Product structures sync completed with ${failedProducts} failed product(s)` : 'Product structures sync completed')
});
console.log(`Done. Synced ${processedProducts} product structure(s). Skipped products: ${skippedProducts}. Failed products: ${failedProducts}.`);
console.log(`${reachedLimit ? 'Paused' : 'Done'}. Synced ${processedProducts} product structure(s). Skipped products: ${skippedProducts}. Failed products: ${failedProducts}.`);
}
runSync().catch((error: any) => {
if (error instanceof SyncStoppedError) {
console.log('Product structures sync stopped by request.');
saveState({
...latestState,
status: 'stopped',
lastMessage: 'Product structures sync stopped by request'
});

174
src/services/olist-v3.service.ts Normal file
View File

@@ -0,0 +1,174 @@
import axios from 'axios';
import crypto from 'crypto';
import fs from 'fs';
import path from 'path';
type OlistV3Tokens = {
access_token: string;
refresh_token?: string;
expires_in?: number;
expires_at?: string;
};
type OlistV3AuthorizationState = {
state: string;
createdAt: string;
};
const OAUTH_BASE_URL = 'https://accounts.tiny.com.br/realms/tiny/protocol/openid-connect';
const TOKEN_URL = `${OAUTH_BASE_URL}/token`;
const AUTHORIZE_URL = `${OAUTH_BASE_URL}/auth`;
const DEFAULT_TOKEN_FILE = path.join(process.cwd(), 'data-runtime', 'olist_v3_tokens.json');
const DEFAULT_STATE_FILE = path.join(process.cwd(), 'data-runtime', 'olist_v3_oauth_state.json');
function requiredEnv(name: string) {
const value = process.env[name]?.trim();
if (!value) throw new Error(`Missing ${name}.`);
return value;
}
function getTokenFile() {
return process.env.OLIST_V3_TOKEN_FILE || DEFAULT_TOKEN_FILE;
}
function getStateFile() {
return process.env.OLIST_V3_OAUTH_STATE_FILE || DEFAULT_STATE_FILE;
}
function ensureParentDirectory(filePath: string) {
fs.mkdirSync(path.dirname(filePath), { recursive: true });
}
function writeJson(filePath: string, value: unknown) {
ensureParentDirectory(filePath);
fs.writeFileSync(filePath, JSON.stringify(value, null, 2), { mode: 0o600 });
}
function readJson<T>(filePath: string): T | null {
if (!fs.existsSync(filePath)) return null;
return JSON.parse(fs.readFileSync(filePath, 'utf-8')) as T;
}
function getClientConfig() {
return {
clientId: requiredEnv('OLIST_V3_CLIENT_ID'),
clientSecret: requiredEnv('OLIST_V3_CLIENT_SECRET'),
redirectUri: requiredEnv('OLIST_V3_REDIRECT_URI')
};
}
function saveTokens(tokens: OlistV3Tokens) {
const expiresIn = Number(tokens.expires_in || 0);
const expiresAt = Number.isFinite(expiresIn) && expiresIn > 0
? new Date(Date.now() + Math.max(0, expiresIn - 60) * 1000).toISOString()
: undefined;
const previous = readJson<OlistV3Tokens>(getTokenFile());
writeJson(getTokenFile(), {
...previous,
...tokens,
refresh_token: tokens.refresh_token || previous?.refresh_token,
expires_at: expiresAt || previous?.expires_at
});
}
function tokenIsUsable(tokens: OlistV3Tokens) {
if (!tokens.access_token) return false;
if (!tokens.expires_at) return true;
return new Date(tokens.expires_at).getTime() > Date.now();
}
async function requestTokens(params: URLSearchParams) {
const response = await axios.post<OlistV3Tokens>(TOKEN_URL, params, {
headers: { 'Content-Type': 'application/x-www-form-urlencoded' },
timeout: 30000
});
if (!response.data?.access_token) {
throw new Error('Olist V3 OAuth did not return an access token.');
}
saveTokens(response.data);
return response.data;
}
export function isOlistV3Configured() {
return Boolean(
process.env.OLIST_V3_CLIENT_ID?.trim()
&& process.env.OLIST_V3_CLIENT_SECRET?.trim()
&& process.env.OLIST_V3_REDIRECT_URI?.trim()
);
}
export function getOlistV3AuthorizationUrl() {
const { clientId, redirectUri } = getClientConfig();
const state = crypto.randomBytes(32).toString('hex');
writeJson(getStateFile(), { state, createdAt: new Date().toISOString() } satisfies OlistV3AuthorizationState);
const params = new URLSearchParams({
client_id: clientId,
redirect_uri: redirectUri,
scope: 'openid',
response_type: 'code',
state
});
return `${AUTHORIZE_URL}?${params.toString()}`;
}
export async function exchangeOlistV3AuthorizationCode(code: string, state: string) {
const expected = readJson<OlistV3AuthorizationState>(getStateFile());
const expectedState = Buffer.from(expected?.state || '');
const receivedState = Buffer.from(state || '');
if (!expected?.state || expectedState.length !== receivedState.length || !crypto.timingSafeEqual(expectedState, receivedState)) {
throw new Error('Invalid OAuth state. Start the authorization flow again.');
}
const createdAt = new Date(expected.createdAt).getTime();
if (!Number.isFinite(createdAt) || Date.now() - createdAt > 10 * 60 * 1000) {
throw new Error('OAuth state expired. Start the authorization flow again.');
}
const { clientId, clientSecret, redirectUri } = getClientConfig();
const params = new URLSearchParams({
grant_type: 'authorization_code',
client_id: clientId,
client_secret: clientSecret,
redirect_uri: redirectUri,
code
});
const tokens = await requestTokens(params);
fs.rmSync(getStateFile(), { force: true });
return tokens;
}
export async function getOlistV3AccessToken(forceRefresh = false) {
const current = readJson<OlistV3Tokens>(getTokenFile());
if (current && !forceRefresh && tokenIsUsable(current)) {
return current.access_token;
}
if (!current?.refresh_token) {
throw new Error('Olist V3 is not authorized. Open /api/olist-v3/authorize first.');
}
const { clientId, clientSecret } = getClientConfig();
const params = new URLSearchParams({
grant_type: 'refresh_token',
client_id: clientId,
client_secret: clientSecret,
refresh_token: current.refresh_token
});
const tokens = await requestTokens(params);
return tokens.access_token;
}
export function getOlistV3AuthStatus() {
const tokens = readJson<OlistV3Tokens>(getTokenFile());
return {
configured: isOlistV3Configured(),
authorized: Boolean(tokens?.access_token || tokens?.refresh_token),
accessTokenUsable: Boolean(tokens && tokenIsUsable(tokens)),
tokenExpiresAt: tokens?.expires_at || null,
tokenFile: getTokenFile()
};
}