diff --git a/.gitignore b/.gitignore index c45b1f8..fb538b0 100644 --- a/.gitignore +++ b/.gitignore @@ -5,5 +5,8 @@ graphs_backfill_state.json graphs_backfill_stop.json product_structures_sync_state.json product_structures_sync_stop.json +product_structures_v3_sync_state.json +product_structures_v3_sync_stop.json +data-runtime/ production_orders_sync_state.json production_orders_sync_stop.json diff --git a/docker-compose.yml b/docker-compose.yml index 330c8fa..b0049da 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -27,6 +27,14 @@ services: - PRODUCT_STRUCTURES_GRAPHS_API_KEY=${PRODUCT_STRUCTURES_GRAPHS_API_KEY} - PRODUCT_STRUCTURES_REQUEST_DELAY_MS=${PRODUCT_STRUCTURES_REQUEST_DELAY_MS:-2500} - PRODUCT_STRUCTURES_FETCH_COMPONENT_UNITS=${PRODUCT_STRUCTURES_FETCH_COMPONENT_UNITS:-true} + - OLIST_V3_CLIENT_ID=${OLIST_V3_CLIENT_ID} + - OLIST_V3_CLIENT_SECRET=${OLIST_V3_CLIENT_SECRET} + - OLIST_V3_REDIRECT_URI=${OLIST_V3_REDIRECT_URI} + - OLIST_V3_TOKEN_FILE=${OLIST_V3_TOKEN_FILE:-/app/data-runtime/olist_v3_tokens.json} + - OLIST_V3_OAUTH_STATE_FILE=${OLIST_V3_OAUTH_STATE_FILE:-/app/data-runtime/olist_v3_oauth_state.json} + - PRODUCT_STRUCTURES_V3_GRAPHS_API_URL=${PRODUCT_STRUCTURES_V3_GRAPHS_API_URL} + - PRODUCT_STRUCTURES_V3_GRAPHS_API_KEY=${PRODUCT_STRUCTURES_V3_GRAPHS_API_KEY} + - PRODUCT_STRUCTURES_V3_REQUEST_DELAY_MS=${PRODUCT_STRUCTURES_V3_REQUEST_DELAY_MS:-1000} - PRODUCTION_ORDERS_TINY_ERP_COOKIE=${PRODUCTION_ORDERS_TINY_ERP_COOKIE} - PRODUCTION_ORDERS_GRAPHS_API_URL=${PRODUCTION_ORDERS_GRAPHS_API_URL} - PRODUCTION_ORDERS_GRAPHS_API_KEY=${PRODUCTION_ORDERS_GRAPHS_API_KEY} @@ -40,7 +48,12 @@ services: # Optional: If you use a shared Docker network for Nginx Proxy Manager, you can attach it here: # networks: # - nginx-proxy-manager-network + volumes: + - middleware_data:/app/data-runtime # networks: # nginx-proxy-manager-network: # external: true + +volumes: + middleware_data: diff --git a/package.json b/package.json index 3d9955f..4d7b67b 100644 --- a/package.json +++ b/package.json @@ -9,6 +9,7 @@ "start": "node dist/index.js", "backfill:graphs": "tsx src/scripts/backfill-graphs.ts", "sync:product-structures": "tsx src/scripts/sync-product-structures.ts", + "sync:product-structures:v3": "tsx src/scripts/sync-product-structures-v3.ts", "sync:production-orders": "tsx src/scripts/sync-production-orders.ts", "test": "echo \"Error: no test specified\" && exit 1" }, diff --git a/src/index.ts b/src/index.ts index 7662861..3974c36 100644 --- a/src/index.ts +++ b/src/index.ts @@ -4,6 +4,12 @@ import webhookRoutes from './routes/webhook.route'; import { spawn } from 'child_process'; import path from 'path'; import fs from 'fs'; +import { + exchangeOlistV3AuthorizationCode, + getOlistV3AuthorizationUrl, + getOlistV3AuthStatus, + isOlistV3Configured +} from './services/olist-v3.service'; dotenv.config(); @@ -31,6 +37,56 @@ app.get('/health', (req: Request, res: Response) => { res.status(200).json({ status: 'OK', message: 'Tiny-n8n middleware is running.' }); }); +// Starts the official Olist V3 OAuth flow. The callback validates a one-time state value. +app.get('/api/olist-v3/authorize', (req: Request, res: Response) => { + const expectedToken = process.env.TINY_WEBHOOK_SECRET; + if (expectedToken && req.query.token !== expectedToken) { + res.status(401).json({ error: 'Unauthorized' }); + return; + } + + if (!isOlistV3Configured()) { + res.status(400).json({ error: 'Missing OLIST_V3_CLIENT_ID, OLIST_V3_CLIENT_SECRET, or OLIST_V3_REDIRECT_URI.' }); + return; + } + + res.redirect(getOlistV3AuthorizationUrl()); +}); + +app.get('/api/olist-v3/callback', async (req: Request, res: Response) => { + const providerError = typeof req.query.error === 'string' ? req.query.error : ''; + const code = typeof req.query.code === 'string' ? req.query.code : ''; + const state = typeof req.query.state === 'string' ? req.query.state : ''; + + if (providerError) { + res.status(400).send('Olist V3 authorization was denied or failed. Return to the middleware and start authorization again.'); + return; + } + + if (!code || !state) { + res.status(400).send('Missing OAuth code or state. Start authorization again.'); + return; + } + + try { + await exchangeOlistV3AuthorizationCode(code, state); + res.status(200).send('Olist V3 authorized successfully. You can close this page and trigger the V3 product structure test.'); + } catch (error: any) { + console.error(`[server]: Olist V3 OAuth callback failed: ${error.message}`); + res.status(400).send('Olist V3 authorization could not be completed. Check Portainer logs and start authorization again.'); + } +}); + +app.get('/api/olist-v3/status', (req: Request, res: Response) => { + const expectedToken = process.env.TINY_WEBHOOK_SECRET; + if (expectedToken && req.query.token !== expectedToken) { + res.status(401).json({ error: 'Unauthorized' }); + return; + } + + res.status(200).json(getOlistV3AuthStatus()); +}); + // Hidden endpoint to trigger the backfill script without terminal access app.get('/api/trigger-backfill', (req: Request, res: Response) => { const expectedToken = process.env.TINY_WEBHOOK_SECRET; @@ -296,6 +352,112 @@ app.get('/api/stop-product-structures-sync', (req: Request, res: Response) => { }); }); +// Hidden endpoint to sync manufactured-product compositions from the official Olist V3 API. +app.get('/api/trigger-product-structures-v3-sync', (req: Request, res: Response) => { + const expectedToken = process.env.TINY_WEBHOOK_SECRET; + if (expectedToken && req.query.token !== expectedToken) { + res.status(401).json({ error: 'Unauthorized' }); + return; + } + + const scriptPath = path.join(__dirname, 'scripts', 'sync-product-structures-v3.js'); + const env = { ...process.env }; + const stopFile = env.PRODUCT_STRUCTURES_V3_STOP_FILE || path.join(process.cwd(), 'data-runtime', 'product_structures_v3_sync_stop.json'); + const optionalEnvMap: Record = { + productIds: 'PRODUCT_STRUCTURES_V3_PRODUCT_IDS', + inputFile: 'PRODUCT_STRUCTURES_V3_INPUT_FILE', + graphsUrl: 'PRODUCT_STRUCTURES_V3_GRAPHS_API_URL', + graphsApiKey: 'PRODUCT_STRUCTURES_V3_GRAPHS_API_KEY', + dryRun: 'PRODUCT_STRUCTURES_V3_DRY_RUN', + maxProducts: 'PRODUCT_STRUCTURES_V3_MAX_PRODUCTS', + requestDelayMs: 'PRODUCT_STRUCTURES_V3_REQUEST_DELAY_MS', + graphsRetryDelayMs: 'PRODUCT_STRUCTURES_V3_GRAPHS_RETRY_DELAY_MS', + graphsMaxRetries: 'PRODUCT_STRUCTURES_V3_GRAPHS_MAX_RETRIES' + }; + + for (const [queryKey, envKey] of Object.entries(optionalEnvMap)) { + const value = req.query[queryKey]; + if (typeof value === 'string' && value.trim() !== '') { + env[envKey] = value.trim(); + } + } + + if (!env.OLIST_V3_CLIENT_ID || !env.OLIST_V3_CLIENT_SECRET || !env.OLIST_V3_REDIRECT_URI) { + res.status(400).json({ error: 'Olist V3 is not configured. Set the client ID, client secret, and redirect URI first.' }); + return; + } + + if (!getOlistV3AuthStatus().authorized) { + res.status(409).json({ error: 'Olist V3 is not authorized. Open /api/olist-v3/authorize first.' }); + return; + } + + if (fs.existsSync(stopFile)) { + fs.rmSync(stopFile, { force: true }); + } + + const child = spawn('node', [scriptPath], { detached: true, stdio: 'inherit', env }); + child.on('error', (error) => { + console.error(`[server]: Olist V3 product structures sync failed to start: ${error.message}`); + }); + child.on('exit', (code, signal) => { + console.log(`[server]: Olist V3 product structures sync exited. code=${code ?? 'null'} signal=${signal ?? 'null'}`); + }); + child.unref(); + + console.log('[server]: Olist V3 product structures sync manually triggered via HTTP endpoint.'); + res.status(200).json({ + status: 'STARTED', + message: 'Olist V3 product structures sync is running in the background. Check Portainer logs for progress.', + options: { + productIds: env.PRODUCT_STRUCTURES_V3_PRODUCT_IDS || null, + inputFile: env.PRODUCT_STRUCTURES_V3_INPUT_FILE || (env.PRODUCT_STRUCTURES_V3_PRODUCT_IDS ? null : '/app/data/product-structure-products.csv'), + graphsUrl: env.PRODUCT_STRUCTURES_V3_GRAPHS_API_URL || deriveProductionOrdersUrl(env), + dryRun: env.PRODUCT_STRUCTURES_V3_DRY_RUN || 'false', + maxProducts: env.PRODUCT_STRUCTURES_V3_MAX_PRODUCTS || null, + requestDelayMs: env.PRODUCT_STRUCTURES_V3_REQUEST_DELAY_MS || '1000', + olistV3Configured: true, + olistV3Authorized: getOlistV3AuthStatus().authorized, + graphsApiKeyConfigured: Boolean(env.PRODUCT_STRUCTURES_V3_GRAPHS_API_KEY || env.GRAPHS_API_KEY || env.NEXSTAR_GRAPHS_API_KEY) + } + }); +}); + +app.get('/api/product-structures-v3-sync-status', (req: Request, res: Response) => { + const expectedToken = process.env.TINY_WEBHOOK_SECRET; + if (expectedToken && req.query.token !== expectedToken) { + res.status(401).json({ error: 'Unauthorized' }); + return; + } + + const stateFile = process.env.PRODUCT_STRUCTURES_V3_STATE_FILE || path.join(process.cwd(), 'data-runtime', 'product_structures_v3_sync_state.json'); + const stopFile = process.env.PRODUCT_STRUCTURES_V3_STOP_FILE || path.join(process.cwd(), 'data-runtime', 'product_structures_v3_sync_stop.json'); + let state = null; + if (fs.existsSync(stateFile)) { + try { + state = JSON.parse(fs.readFileSync(stateFile, 'utf-8')); + } catch (error: any) { + res.status(500).json({ error: `Could not read Olist V3 product structures state: ${error.message}` }); + return; + } + } + + res.status(200).json({ status: state?.status || 'unknown', stopRequested: fs.existsSync(stopFile), state }); +}); + +app.get('/api/stop-product-structures-v3-sync', (req: Request, res: Response) => { + const expectedToken = process.env.TINY_WEBHOOK_SECRET; + if (expectedToken && req.query.token !== expectedToken) { + res.status(401).json({ error: 'Unauthorized' }); + return; + } + + const stopFile = process.env.PRODUCT_STRUCTURES_V3_STOP_FILE || path.join(process.cwd(), 'data-runtime', 'product_structures_v3_sync_stop.json'); + fs.mkdirSync(path.dirname(stopFile), { recursive: true }); + fs.writeFileSync(stopFile, JSON.stringify({ requestedAt: new Date().toISOString() }, null, 2)); + res.status(200).json({ status: 'STOP_REQUESTED', message: 'Olist V3 product structures sync will stop after the current request.' }); +}); + // Hidden endpoint to sync Tiny/Olist production order details into Graphs app.get('/api/trigger-production-orders-sync', (req: Request, res: Response) => { const expectedToken = process.env.TINY_WEBHOOK_SECRET; diff --git a/src/scripts/sync-product-structures-v3.ts b/src/scripts/sync-product-structures-v3.ts new file mode 100644 index 0000000..e779d8e --- /dev/null +++ b/src/scripts/sync-product-structures-v3.ts @@ -0,0 +1,399 @@ +import axios from 'axios'; +import crypto from 'crypto'; +import dotenv from 'dotenv'; +import fs from 'fs'; +import path from 'path'; +import { getOlistV3AccessToken, isOlistV3Configured } from '../services/olist-v3.service'; + +dotenv.config(); + +type RequestedProduct = { id: string }; +type V3Product = { id: string; sku: string; descricao: string; unidade: string }; +type V3StructureItem = { + produto?: { id?: number | string; sku?: string; descricao?: string }; + quantidade?: number | string; +}; +type V3FabricatedProduct = { produtos?: V3StructureItem[]; etapas?: string[] }; +type StructurePayload = { + order: { + tinyId: string; + number: string; + status: string; + productSku: string; + productDescription: string; + quantity: string; + unit: string; + issueDate: null; + expectedDate: null; + supplier: string; + lotCode: string; + notes: string; + }; + components: Array<{ + componentTinyId: string; + componentName: string; + componentSku: string; + quantityPerUnit: string; + totalQuantity: string; + unit: string; + }>; + steps: Array<{ + stepNumber: number; + name: string; + startDate: null; + endDate: null; + status: string; + }>; +}; +type SyncState = { + status?: 'running' | 'stopped' | 'completed'; + sourceSignature?: string; + totalProducts?: number; + nextIndex?: number; + processedProducts?: number; + skippedProducts?: number; + failedProducts?: number; + lastProductId?: string; + lastFailedProductId?: string; + lastMessage?: string; + updatedAt?: string; +}; + +const V3_API_BASE_URL = (process.env.OLIST_V3_API_BASE_URL || 'https://api.tiny.com.br/public-api/v3').replace(/\/+$/, ''); +const DEFAULT_INPUT_FILE = path.join(process.cwd(), 'data', 'product-structure-products.csv'); +const INPUT_FILE = process.env.PRODUCT_STRUCTURES_V3_INPUT_FILE || DEFAULT_INPUT_FILE; +const PRODUCT_IDS = String(process.env.PRODUCT_STRUCTURES_V3_PRODUCT_IDS || '') + .split(',') + .map(value => value.trim()) + .filter(Boolean); +const DRY_RUN = process.env.PRODUCT_STRUCTURES_V3_DRY_RUN === 'true'; +const RESUME = process.env.PRODUCT_STRUCTURES_V3_RESUME !== 'false'; +const RUNTIME_DIRECTORY = path.join(process.cwd(), 'data-runtime'); +const STATE_FILE = process.env.PRODUCT_STRUCTURES_V3_STATE_FILE || path.join(RUNTIME_DIRECTORY, 'product_structures_v3_sync_state.json'); +const STOP_FILE = process.env.PRODUCT_STRUCTURES_V3_STOP_FILE || path.join(RUNTIME_DIRECTORY, 'product_structures_v3_sync_stop.json'); + +function deriveGraphsUrl() { + const raw = process.env.PRODUCT_STRUCTURES_V3_GRAPHS_API_URL + || process.env.PRODUCT_STRUCTURES_GRAPHS_API_URL + || process.env.PRODUCTION_ORDERS_GRAPHS_API_URL + || process.env.GRAPHS_API_URL + || ''; + if (!raw) return 'http://localhost:3004/api/production-orders/tiny-sync'; + + try { + const parsed = new URL(raw); + return parsed.pathname.includes('/api/data') + ? `${parsed.origin}/api/production-orders/tiny-sync` + : raw; + } catch { + return raw.replace(/\/api\/data.*$/, '/api/production-orders/tiny-sync'); + } +} + +const GRAPHS_URL = deriveGraphsUrl(); +const GRAPHS_API_KEY = process.env.PRODUCT_STRUCTURES_V3_GRAPHS_API_KEY + || process.env.PRODUCT_STRUCTURES_GRAPHS_API_KEY + || process.env.PRODUCTION_ORDERS_GRAPHS_API_KEY + || process.env.GRAPHS_API_KEY + || process.env.NEXSTAR_GRAPHS_API_KEY + || process.env.API_KEY + || 'nexstar_secret_key_123'; + +function numberEnv(name: string, fallback: number) { + const value = Number(process.env[name]); + return Number.isFinite(value) && value >= 0 ? value : fallback; +} + +const REQUEST_DELAY_MS = numberEnv('PRODUCT_STRUCTURES_V3_REQUEST_DELAY_MS', 1000); +const MAX_PRODUCTS = numberEnv('PRODUCT_STRUCTURES_V3_MAX_PRODUCTS', 0); +const GRAPHS_MAX_RETRIES = numberEnv('PRODUCT_STRUCTURES_V3_GRAPHS_MAX_RETRIES', 3); +const GRAPHS_RETRY_DELAY_MS = numberEnv('PRODUCT_STRUCTURES_V3_GRAPHS_RETRY_DELAY_MS', 30000); + +let nextRequestAt = 0; +let latestState: SyncState | null = null; +const componentProductCache = new Map(); + +class SyncStoppedError extends Error { + constructor() { + super('V3 product structure sync stopped by request.'); + } +} + +const sleep = (ms: number) => new Promise(resolve => setTimeout(resolve, ms)); +const text = (value: unknown) => String(value ?? '').trim(); + +function readProductsFromCsv(filePath: string): RequestedProduct[] { + const rows = fs.readFileSync(filePath, 'utf-8').replace(/^\uFEFF/, '').split(/\r?\n/).filter(Boolean); + const headers = rows.shift()?.split(',').map(header => header.trim().toLowerCase()) || []; + const idIndex = headers.findIndex(header => ['id', 'productid', 'tinyid', 'id produto'].includes(header)); + if (idIndex < 0) throw new Error(`No product id column found in ${filePath}.`); + + return rows + .map(row => row.split(',')[idIndex]?.trim()) + .filter(Boolean) + .map(id => ({ id })); +} + +function loadRequestedProducts() { + const fromFile = PRODUCT_IDS.length ? [] : readProductsFromCsv(INPUT_FILE); + const requested = fromFile.concat(PRODUCT_IDS.map(id => ({ id }))); + return requested.filter((item, index, all) => all.findIndex(other => other.id === item.id) === index); +} + +function saveState(state: SyncState) { + latestState = { ...state, updatedAt: new Date().toISOString() }; + fs.mkdirSync(path.dirname(STATE_FILE), { recursive: true }); + fs.writeFileSync(STATE_FILE, JSON.stringify(latestState, null, 2)); +} + +function readState() { + if (!fs.existsSync(STATE_FILE)) return null; + try { + return JSON.parse(fs.readFileSync(STATE_FILE, 'utf-8')) as SyncState; + } catch (error: any) { + console.warn(`[state] Could not read previous state: ${error.message}`); + return null; + } +} + +function assertNotStopped() { + if (fs.existsSync(STOP_FILE)) throw new SyncStoppedError(); +} + +async function waitForSlot(reason: string) { + const waitMs = Math.max(0, nextRequestAt - Date.now()); + if (waitMs > 0) console.log(`[wait] Waiting ${waitMs}ms before ${reason}.`); + let remaining = waitMs; + while (remaining > 0) { + assertNotStopped(); + await sleep(Math.min(remaining, 1000)); + remaining = Math.max(0, nextRequestAt - Date.now()); + } + nextRequestAt = Date.now() + REQUEST_DELAY_MS; +} + +function describeError(error: any) { + if (error?.response) { + const body = JSON.stringify(error.response.data ?? ''); + return `HTTP ${error.response.status}: ${body.slice(0, 500)}`; + } + return error?.message || String(error); +} + +async function v3Get(pathname: string): Promise { + await waitForSlot(`Olist V3 ${pathname}`); + const request = async (forceRefresh: boolean) => axios.get(`${V3_API_BASE_URL}${pathname}`, { + headers: { Authorization: `Bearer ${await getOlistV3AccessToken(forceRefresh)}` }, + timeout: 30000 + }); + + try { + return (await request(false)).data; + } catch (error: any) { + if (error?.response?.status !== 401) throw error; + await waitForSlot(`Olist V3 ${pathname} retry after token refresh`); + return (await request(true)).data; + } +} + +async function getProduct(id: string): Promise { + const raw = await v3Get(`/produtos/${encodeURIComponent(id)}`); + return { + id: text(raw?.id || id), + sku: text(raw?.sku), + descricao: text(raw?.descricao), + unidade: text(raw?.unidade) + }; +} + +async function getComponentProduct(id: string) { + if (componentProductCache.has(id)) return componentProductCache.get(id) || null; + try { + const product = await getProduct(id); + componentProductCache.set(id, product); + return product; + } catch (error: any) { + console.warn(`[Olist V3] Could not get component ${id}: ${describeError(error)}`); + componentProductCache.set(id, null); + return null; + } +} + +async function getFabricatedProduct(id: string): Promise { + try { + return await v3Get(`/produtos/${encodeURIComponent(id)}/fabricado`); + } catch (error: any) { + if ([400, 404].includes(error?.response?.status)) return null; + throw error; + } +} + +function formatDecimal(value: unknown) { + const parsed = Number(value); + return Number.isFinite(parsed) ? String(parsed) : '0'; +} + +async function buildPayload(productId: string): Promise { + const product = await getProduct(productId); + const manufactured = await getFabricatedProduct(product.id); + const rawComponents = Array.isArray(manufactured?.produtos) ? manufactured.produtos : []; + if (!manufactured || !rawComponents.length) return null; + + const components = []; + for (const item of rawComponents) { + const componentId = text(item.produto?.id); + const componentDetail = componentId ? await getComponentProduct(componentId) : null; + const quantity = formatDecimal(item.quantidade); + components.push({ + componentTinyId: componentId, + componentName: text(item.produto?.descricao || componentDetail?.descricao), + componentSku: text(item.produto?.sku || componentDetail?.sku), + quantityPerUnit: quantity, + totalQuantity: quantity, + unit: text(componentDetail?.unidade) + }); + } + + return { + order: { + tinyId: `STRUCTURE-V3-${product.id}`, + number: `STRUCTURE-V3-${product.sku || product.id}`, + status: 'completed', + productSku: product.sku, + productDescription: product.descricao, + quantity: '1', + unit: product.unidade || 'UN', + issueDate: null, + expectedDate: null, + supplier: '', + lotCode: '', + notes: `Composição sincronizada da API pública Olist V3 para produto ${product.id}.` + }, + components: components.filter(component => component.componentSku || component.componentName), + steps: (manufactured?.etapas || []).map((name, index) => ({ + stepNumber: index + 1, + name: text(name), + startDate: null, + endDate: null, + status: 'pendente' + })).filter(step => step.name) + }; +} + +async function sendToGraphs(payload: StructurePayload) { + if (DRY_RUN) { + console.log(`[dry-run] ${JSON.stringify(payload)}`); + return true; + } + + for (let attempt = 1; ; attempt += 1) { + try { + await axios.post(GRAPHS_URL, payload, { + headers: { 'Content-Type': 'application/json', 'x-api-key': GRAPHS_API_KEY }, + timeout: 30000 + }); + console.log(`[Graphs V3] Sent ${payload.order.productSku} with ${payload.components.length} component(s).`); + return true; + } catch (error: any) { + const retryable = !error?.response?.status || error.response.status >= 500; + console.error(`[Graphs V3] Send failed for ${payload.order.productSku}, attempt ${attempt}: ${describeError(error)}`); + if (!retryable || (GRAPHS_MAX_RETRIES && attempt >= GRAPHS_MAX_RETRIES)) return false; + await sleep(GRAPHS_RETRY_DELAY_MS); + } + } +} + +async function run() { + if (!isOlistV3Configured()) { + throw new Error('Olist V3 is not configured. Set OLIST_V3_CLIENT_ID, OLIST_V3_CLIENT_SECRET, and OLIST_V3_REDIRECT_URI.'); + } + + const products = loadRequestedProducts(); + if (!products.length) throw new Error('No V3 products configured.'); + await getOlistV3AccessToken(); + + const sourceSignature = crypto.createHash('sha256').update(products.map(product => product.id).join('\n')).digest('hex'); + const previous = readState(); + const canResume = RESUME + && !PRODUCT_IDS.length + && previous?.status !== 'completed' + && previous?.sourceSignature === sourceSignature + && Number.isInteger(previous?.nextIndex) + && (previous?.nextIndex || 0) > 0 + && (previous?.nextIndex || 0) < products.length; + let nextIndex = canResume ? previous!.nextIndex! : 0; + let processedProducts = canResume ? previous?.processedProducts || 0 : 0; + let skippedProducts = canResume ? previous?.skippedProducts || 0 : 0; + let failedProducts = canResume ? previous?.failedProducts || 0 : 0; + let handledThisRun = 0; + let reachedLimit = false; + + console.log(`[config] Olist V3 product structures: ${products.length} product(s), ${REQUEST_DELAY_MS}ms request pace.`); + if (canResume) console.log(`[state] Resuming at ${nextIndex + 1}/${products.length}.`); + + const checkpoint = (state: Partial) => saveState({ + status: 'running', + sourceSignature, + totalProducts: products.length, + nextIndex, + processedProducts, + skippedProducts, + failedProducts, + ...state + }); + checkpoint({ lastMessage: canResume ? 'V3 structure sync resumed' : 'V3 structure sync started' }); + + for (let index = nextIndex; index < products.length; index += 1) { + assertNotStopped(); + if (MAX_PRODUCTS && handledThisRun >= MAX_PRODUCTS) { + reachedLimit = true; + break; + } + + const productId = products[index].id; + nextIndex = index + 1; + try { + const payload = await buildPayload(productId); + if (!payload?.components.length) { + skippedProducts += 1; + console.log(`[Olist V3] No manufactured-product composition for ${productId}; skipping Graphs send.`); + checkpoint({ lastProductId: productId, lastMessage: `Skipped ${productId}; no composition` }); + } else if (await sendToGraphs(payload)) { + processedProducts += 1; + checkpoint({ lastProductId: productId, lastMessage: `Synced ${payload.order.productSku}` }); + } else { + failedProducts += 1; + checkpoint({ lastFailedProductId: productId, lastMessage: `Graphs send failed for ${productId}` }); + } + } catch (error: any) { + if (error instanceof SyncStoppedError) throw error; + failedProducts += 1; + console.error(`[sync] Failed product ${productId}: ${describeError(error)}`); + checkpoint({ lastFailedProductId: productId, lastMessage: `Failed ${productId}; continuing` }); + } + handledThisRun += 1; + } + + saveState({ + status: reachedLimit ? 'stopped' : 'completed', + sourceSignature, + totalProducts: products.length, + nextIndex, + processedProducts, + skippedProducts, + failedProducts, + lastMessage: reachedLimit ? `V3 structure sync paused after ${handledThisRun} product(s)` : 'V3 structure sync completed' + }); + console.log(`Done. V3 synced ${processedProducts}; skipped ${skippedProducts}; failed ${failedProducts}.`); +} + +run().catch((error: any) => { + if (error instanceof SyncStoppedError) { + saveState({ ...latestState, status: 'stopped', lastMessage: 'V3 structure sync stopped by request' }); + console.log('V3 product structure sync stopped by request.'); + process.exit(0); + } + console.error(`[sync] V3 product structure sync failed: ${describeError(error)}`); + saveState({ ...latestState, status: 'stopped', lastMessage: describeError(error) }); + process.exit(1); +}); diff --git a/src/scripts/sync-product-structures.ts b/src/scripts/sync-product-structures.ts index cc45d88..c1ab1b2 100644 --- a/src/scripts/sync-product-structures.ts +++ b/src/scripts/sync-product-structures.ts @@ -1,4 +1,5 @@ import axios from 'axios'; +import crypto from 'crypto'; import dotenv from 'dotenv'; import fs from 'fs'; import path from 'path'; @@ -47,6 +48,9 @@ type ProductStructurePayload = { type SyncState = { status?: 'running' | 'stopped' | 'completed'; + sourceSignature?: string; + totalProducts?: number; + nextIndex?: number; processedProducts?: number; skippedProducts?: number; failedProducts?: number; @@ -120,9 +124,11 @@ const REQUEST_DELAY_MS = numberEnv('PRODUCT_STRUCTURES_REQUEST_DELAY_MS', 2500); const GRAPHS_RETRY_DELAY_MS = numberEnv('PRODUCT_STRUCTURES_GRAPHS_RETRY_DELAY_MS', 30000); const GRAPHS_MAX_RETRIES = numberEnv('PRODUCT_STRUCTURES_GRAPHS_MAX_RETRIES', 3); const MAX_PRODUCTS = numberEnv('PRODUCT_STRUCTURES_MAX_PRODUCTS', 0); +const RESUME = process.env.PRODUCT_STRUCTURES_RESUME !== 'false'; let nextRequestAt = 0; const productDetailCache = new Map(); +let latestState: SyncState | null = null; class SyncStoppedError extends Error { constructor() { @@ -159,10 +165,23 @@ async function waitForRequestSlot(reason: string) { } function saveState(state: SyncState) { - fs.writeFileSync(STATE_FILE, JSON.stringify({ + latestState = { ...state, updatedAt: new Date().toISOString() - }, null, 2)); + }; + fs.writeFileSync(STATE_FILE, JSON.stringify(latestState, null, 2)); +} + +function readState(): SyncState | null { + if (!fs.existsSync(STATE_FILE)) return null; + + try { + const parsed = JSON.parse(fs.readFileSync(STATE_FILE, 'utf-8')); + return parsed && typeof parsed === 'object' ? parsed : null; + } catch (error: any) { + console.warn(`[state] Could not read previous state; starting from the beginning: ${error.message}`); + return null; + } } function describeRequestError(error: any) { @@ -457,29 +476,58 @@ async function runSync() { console.log(`[config] Graphs URL: ${GRAPHS_PRODUCTION_ORDERS_URL}`); console.log(`[config] Dry run: ${DRY_RUN ? 'yes' : 'no'}`); console.log(`[config] Request pace: 1 request every ${REQUEST_DELAY_MS}ms`); + console.log(`[config] Resume: ${RESUME ? 'yes' : 'no'}`); console.log(`[config] State file: ${STATE_FILE}`); console.log(`[config] Stop file: ${STOP_FILE}`); console.log(`[source] Found ${requestedProducts.length} requested product(s).`); + const sourceSignature = crypto + .createHash('sha256') + .update(requestedProducts.map(product => `${product.id}|${product.sku}`).join('\n')) + .digest('hex'); + const previousState = readState(); + const canResume = RESUME + && !SKUS.length + && !PRODUCT_IDS.length + && previousState?.status !== 'completed' + && previousState?.sourceSignature === sourceSignature + && Number.isInteger(previousState?.nextIndex) + && (previousState?.nextIndex || 0) > 0 + && (previousState?.nextIndex || 0) < requestedProducts.length; + const startIndex = canResume ? previousState!.nextIndex! : 0; + let processedProducts = canResume ? previousState?.processedProducts || 0 : 0; + let skippedProducts = canResume ? previousState?.skippedProducts || 0 : 0; + let failedProducts = canResume ? previousState?.failedProducts || 0 : 0; + let handledThisRun = 0; + let nextIndex = startIndex; + let reachedLimit = false; + + if (canResume) { + console.log(`[state] Resuming at ${startIndex + 1}/${requestedProducts.length} after ${previousState?.lastSku || 'the last checkpoint'}.`); + } + saveState({ status: 'running', - processedProducts: 0, - skippedProducts: 0, - failedProducts: 0, - lastMessage: 'Product structures sync started' + sourceSignature, + totalProducts: requestedProducts.length, + nextIndex: startIndex, + processedProducts, + skippedProducts, + failedProducts, + lastSku: canResume ? previousState?.lastSku : undefined, + lastMessage: canResume ? 'Product structures sync resumed' : 'Product structures sync started' }); - let processedProducts = 0; - let skippedProducts = 0; - let failedProducts = 0; - - for (const requested of requestedProducts) { + for (let index = startIndex; index < requestedProducts.length; index += 1) { assertNotStopped(); - if (MAX_PRODUCTS && processedProducts + skippedProducts + failedProducts >= MAX_PRODUCTS) { + if (MAX_PRODUCTS && handledThisRun >= MAX_PRODUCTS) { console.log(`[limit] Stopped after ${MAX_PRODUCTS} product(s).`); + reachedLimit = true; break; } + nextIndex = index + 1; + const requested = requestedProducts[index]; const label = requested.sku || requested.id; try { const product = await resolveProduct(requested); @@ -488,12 +536,16 @@ async function runSync() { failedProducts += 1; saveState({ status: 'running', + sourceSignature, + totalProducts: requestedProducts.length, + nextIndex, processedProducts, skippedProducts, failedProducts, lastFailedSku: label, lastMessage: `Product not found for ${label}` }); + handledThisRun += 1; continue; } @@ -502,12 +554,16 @@ async function runSync() { failedProducts += 1; saveState({ status: 'running', + sourceSignature, + totalProducts: requestedProducts.length, + nextIndex, processedProducts, skippedProducts, failedProducts, lastFailedSku: product.codigo || product.id, lastMessage: `Could not get structure for ${product.codigo || product.id}` }); + handledThisRun += 1; continue; } @@ -516,12 +572,16 @@ async function runSync() { console.log(`[Tiny] No structure components for ${payload.order.productSku}; skipping Graphs send.`); saveState({ status: 'running', + sourceSignature, + totalProducts: requestedProducts.length, + nextIndex, processedProducts, skippedProducts, failedProducts, lastSku: payload.order.productSku, lastMessage: `Skipped ${payload.order.productSku}; no structure components` }); + handledThisRun += 1; continue; } @@ -534,6 +594,9 @@ async function runSync() { saveState({ status: 'running', + sourceSignature, + totalProducts: requestedProducts.length, + nextIndex, processedProducts, skippedProducts, failedProducts, @@ -541,35 +604,46 @@ async function runSync() { lastFailedSku: sent ? undefined : payload.order.productSku, lastMessage: sent ? `Synced structure ${payload.order.productSku}` : `Failed structure ${payload.order.productSku}; continuing` }); + handledThisRun += 1; } catch (error: any) { if (error instanceof SyncStoppedError) throw error; failedProducts += 1; console.error(`[sync] Failed product ${label}: ${describeRequestError(error)}`); saveState({ status: 'running', + sourceSignature, + totalProducts: requestedProducts.length, + nextIndex, processedProducts, skippedProducts, failedProducts, lastFailedSku: label, lastMessage: `Failed product ${label}; continuing` }); + handledThisRun += 1; } } saveState({ - status: 'completed', + status: reachedLimit ? 'stopped' : 'completed', + sourceSignature, + totalProducts: requestedProducts.length, + nextIndex, processedProducts, skippedProducts, failedProducts, - lastMessage: failedProducts ? `Product structures sync completed with ${failedProducts} failed product(s)` : 'Product structures sync completed' + lastMessage: reachedLimit + ? `Product structures sync paused after ${handledThisRun} product(s)` + : (failedProducts ? `Product structures sync completed with ${failedProducts} failed product(s)` : 'Product structures sync completed') }); - console.log(`Done. Synced ${processedProducts} product structure(s). Skipped products: ${skippedProducts}. Failed products: ${failedProducts}.`); + console.log(`${reachedLimit ? 'Paused' : 'Done'}. Synced ${processedProducts} product structure(s). Skipped products: ${skippedProducts}. Failed products: ${failedProducts}.`); } runSync().catch((error: any) => { if (error instanceof SyncStoppedError) { console.log('Product structures sync stopped by request.'); saveState({ + ...latestState, status: 'stopped', lastMessage: 'Product structures sync stopped by request' }); diff --git a/src/services/olist-v3.service.ts b/src/services/olist-v3.service.ts new file mode 100644 index 0000000..ed7859b --- /dev/null +++ b/src/services/olist-v3.service.ts @@ -0,0 +1,174 @@ +import axios from 'axios'; +import crypto from 'crypto'; +import fs from 'fs'; +import path from 'path'; + +type OlistV3Tokens = { + access_token: string; + refresh_token?: string; + expires_in?: number; + expires_at?: string; +}; + +type OlistV3AuthorizationState = { + state: string; + createdAt: string; +}; + +const OAUTH_BASE_URL = 'https://accounts.tiny.com.br/realms/tiny/protocol/openid-connect'; +const TOKEN_URL = `${OAUTH_BASE_URL}/token`; +const AUTHORIZE_URL = `${OAUTH_BASE_URL}/auth`; +const DEFAULT_TOKEN_FILE = path.join(process.cwd(), 'data-runtime', 'olist_v3_tokens.json'); +const DEFAULT_STATE_FILE = path.join(process.cwd(), 'data-runtime', 'olist_v3_oauth_state.json'); + +function requiredEnv(name: string) { + const value = process.env[name]?.trim(); + if (!value) throw new Error(`Missing ${name}.`); + return value; +} + +function getTokenFile() { + return process.env.OLIST_V3_TOKEN_FILE || DEFAULT_TOKEN_FILE; +} + +function getStateFile() { + return process.env.OLIST_V3_OAUTH_STATE_FILE || DEFAULT_STATE_FILE; +} + +function ensureParentDirectory(filePath: string) { + fs.mkdirSync(path.dirname(filePath), { recursive: true }); +} + +function writeJson(filePath: string, value: unknown) { + ensureParentDirectory(filePath); + fs.writeFileSync(filePath, JSON.stringify(value, null, 2), { mode: 0o600 }); +} + +function readJson(filePath: string): T | null { + if (!fs.existsSync(filePath)) return null; + return JSON.parse(fs.readFileSync(filePath, 'utf-8')) as T; +} + +function getClientConfig() { + return { + clientId: requiredEnv('OLIST_V3_CLIENT_ID'), + clientSecret: requiredEnv('OLIST_V3_CLIENT_SECRET'), + redirectUri: requiredEnv('OLIST_V3_REDIRECT_URI') + }; +} + +function saveTokens(tokens: OlistV3Tokens) { + const expiresIn = Number(tokens.expires_in || 0); + const expiresAt = Number.isFinite(expiresIn) && expiresIn > 0 + ? new Date(Date.now() + Math.max(0, expiresIn - 60) * 1000).toISOString() + : undefined; + const previous = readJson(getTokenFile()); + + writeJson(getTokenFile(), { + ...previous, + ...tokens, + refresh_token: tokens.refresh_token || previous?.refresh_token, + expires_at: expiresAt || previous?.expires_at + }); +} + +function tokenIsUsable(tokens: OlistV3Tokens) { + if (!tokens.access_token) return false; + if (!tokens.expires_at) return true; + return new Date(tokens.expires_at).getTime() > Date.now(); +} + +async function requestTokens(params: URLSearchParams) { + const response = await axios.post(TOKEN_URL, params, { + headers: { 'Content-Type': 'application/x-www-form-urlencoded' }, + timeout: 30000 + }); + if (!response.data?.access_token) { + throw new Error('Olist V3 OAuth did not return an access token.'); + } + + saveTokens(response.data); + return response.data; +} + +export function isOlistV3Configured() { + return Boolean( + process.env.OLIST_V3_CLIENT_ID?.trim() + && process.env.OLIST_V3_CLIENT_SECRET?.trim() + && process.env.OLIST_V3_REDIRECT_URI?.trim() + ); +} + +export function getOlistV3AuthorizationUrl() { + const { clientId, redirectUri } = getClientConfig(); + const state = crypto.randomBytes(32).toString('hex'); + writeJson(getStateFile(), { state, createdAt: new Date().toISOString() } satisfies OlistV3AuthorizationState); + + const params = new URLSearchParams({ + client_id: clientId, + redirect_uri: redirectUri, + scope: 'openid', + response_type: 'code', + state + }); + return `${AUTHORIZE_URL}?${params.toString()}`; +} + +export async function exchangeOlistV3AuthorizationCode(code: string, state: string) { + const expected = readJson(getStateFile()); + const expectedState = Buffer.from(expected?.state || ''); + const receivedState = Buffer.from(state || ''); + if (!expected?.state || expectedState.length !== receivedState.length || !crypto.timingSafeEqual(expectedState, receivedState)) { + throw new Error('Invalid OAuth state. Start the authorization flow again.'); + } + + const createdAt = new Date(expected.createdAt).getTime(); + if (!Number.isFinite(createdAt) || Date.now() - createdAt > 10 * 60 * 1000) { + throw new Error('OAuth state expired. Start the authorization flow again.'); + } + + const { clientId, clientSecret, redirectUri } = getClientConfig(); + const params = new URLSearchParams({ + grant_type: 'authorization_code', + client_id: clientId, + client_secret: clientSecret, + redirect_uri: redirectUri, + code + }); + + const tokens = await requestTokens(params); + fs.rmSync(getStateFile(), { force: true }); + return tokens; +} + +export async function getOlistV3AccessToken(forceRefresh = false) { + const current = readJson(getTokenFile()); + if (current && !forceRefresh && tokenIsUsable(current)) { + return current.access_token; + } + + if (!current?.refresh_token) { + throw new Error('Olist V3 is not authorized. Open /api/olist-v3/authorize first.'); + } + + const { clientId, clientSecret } = getClientConfig(); + const params = new URLSearchParams({ + grant_type: 'refresh_token', + client_id: clientId, + client_secret: clientSecret, + refresh_token: current.refresh_token + }); + const tokens = await requestTokens(params); + return tokens.access_token; +} + +export function getOlistV3AuthStatus() { + const tokens = readJson(getTokenFile()); + return { + configured: isOlistV3Configured(), + authorized: Boolean(tokens?.access_token || tokens?.refresh_token), + accessTokenUsable: Boolean(tokens && tokenIsUsable(tokens)), + tokenExpiresAt: tokens?.expires_at || null, + tokenFile: getTokenFile() + }; +}