diff --git a/package.json b/package.json index df20fac..d22a63b 100644 --- a/package.json +++ b/package.json @@ -8,6 +8,7 @@ "build": "tsc", "start": "node dist/index.js", "backfill:graphs": "tsx src/scripts/backfill-graphs.ts", + "sync:product-structures": "tsx src/scripts/sync-product-structures.ts", "test": "echo \"Error: no test specified\" && exit 1" }, "keywords": [], diff --git a/src/index.ts b/src/index.ts index 76f68c5..75e4dda 100644 --- a/src/index.ts +++ b/src/index.ts @@ -169,6 +169,121 @@ app.get('/api/stop-graphs-backfill', (req: Request, res: Response) => { }); }); +// Hidden endpoint to sync Tiny product structures/composition into Graphs consumption references +app.get('/api/trigger-product-structures-sync', (req: Request, res: Response) => { + const expectedToken = process.env.TINY_WEBHOOK_SECRET; + if (expectedToken && req.query.token !== expectedToken) { + res.status(401).json({ error: 'Unauthorized' }); + return; + } + + const scriptPath = path.join(__dirname, 'scripts', 'sync-product-structures.js'); + const env = { ...process.env }; + const stopFile = env.PRODUCT_STRUCTURES_STOP_FILE || path.join(process.cwd(), 'product_structures_sync_stop.json'); + + const optionalEnvMap: Record = { + skus: 'PRODUCT_STRUCTURE_SKUS', + productIds: 'PRODUCT_STRUCTURE_PRODUCT_IDS', + inputFile: 'PRODUCT_STRUCTURES_INPUT_FILE', + graphsUrl: 'PRODUCT_STRUCTURES_GRAPHS_API_URL', + graphsApiKey: 'PRODUCT_STRUCTURES_GRAPHS_API_KEY', + dryRun: 'PRODUCT_STRUCTURES_DRY_RUN', + maxProducts: 'PRODUCT_STRUCTURES_MAX_PRODUCTS', + requestDelayMs: 'PRODUCT_STRUCTURES_REQUEST_DELAY_MS', + graphsRetryDelayMs: 'PRODUCT_STRUCTURES_GRAPHS_RETRY_DELAY_MS', + graphsMaxRetries: 'PRODUCT_STRUCTURES_GRAPHS_MAX_RETRIES' + }; + + for (const [queryKey, envKey] of Object.entries(optionalEnvMap)) { + const value = req.query[queryKey]; + if (typeof value === 'string' && value.trim() !== '') { + env[envKey] = value.trim(); + } + } + + if (fs.existsSync(stopFile)) { + fs.rmSync(stopFile, { force: true }); + } + + const child = spawn('node', [scriptPath], { + detached: true, + stdio: 'inherit', + env + }); + + child.on('error', (error) => { + console.error(`[server]: Product structures sync process failed to start: ${error.message}`); + }); + + child.on('exit', (code, signal) => { + console.log(`[server]: Product structures sync process exited. code=${code ?? 'null'} signal=${signal ?? 'null'}`); + }); + + child.unref(); + + console.log('[server]: Product structures sync script manually triggered via HTTP endpoint.'); + res.status(200).json({ + status: 'STARTED', + message: 'Product structures sync is running in the background. Check Portainer logs for progress.', + options: { + skus: env.PRODUCT_STRUCTURE_SKUS || null, + productIds: env.PRODUCT_STRUCTURE_PRODUCT_IDS || null, + inputFile: env.PRODUCT_STRUCTURES_INPUT_FILE || null, + graphsUrl: env.PRODUCT_STRUCTURES_GRAPHS_API_URL || env.PRODUCTION_ORDERS_GRAPHS_API_URL || env.GRAPHS_PRODUCTION_ORDERS_API_URL || null, + dryRun: env.PRODUCT_STRUCTURES_DRY_RUN || 'false', + maxProducts: env.PRODUCT_STRUCTURES_MAX_PRODUCTS || null, + requestDelayMs: env.PRODUCT_STRUCTURES_REQUEST_DELAY_MS || '2500', + graphsMaxRetries: env.PRODUCT_STRUCTURES_GRAPHS_MAX_RETRIES || '3', + graphsApiKeyConfigured: Boolean(env.PRODUCT_STRUCTURES_GRAPHS_API_KEY || env.PRODUCTION_ORDERS_GRAPHS_API_KEY || env.GRAPHS_PRODUCTION_ORDERS_API_KEY || env.GRAPHS_API_KEY || env.NEXSTAR_GRAPHS_API_KEY || env.API_KEY) + } + }); +}); + +app.get('/api/product-structures-sync-status', (req: Request, res: Response) => { + const expectedToken = process.env.TINY_WEBHOOK_SECRET; + if (expectedToken && req.query.token !== expectedToken) { + res.status(401).json({ error: 'Unauthorized' }); + return; + } + + const stateFile = process.env.PRODUCT_STRUCTURES_STATE_FILE || path.join(process.cwd(), 'product_structures_sync_state.json'); + const stopFile = process.env.PRODUCT_STRUCTURES_STOP_FILE || path.join(process.cwd(), 'product_structures_sync_stop.json'); + let state = null; + + if (fs.existsSync(stateFile)) { + try { + state = JSON.parse(fs.readFileSync(stateFile, 'utf-8')); + } catch (error: any) { + res.status(500).json({ error: `Could not read product structures sync state: ${error.message}` }); + return; + } + } + + res.status(200).json({ + status: state?.status || 'unknown', + stopRequested: fs.existsSync(stopFile), + state + }); +}); + +app.get('/api/stop-product-structures-sync', (req: Request, res: Response) => { + const expectedToken = process.env.TINY_WEBHOOK_SECRET; + if (expectedToken && req.query.token !== expectedToken) { + res.status(401).json({ error: 'Unauthorized' }); + return; + } + + const stopFile = process.env.PRODUCT_STRUCTURES_STOP_FILE || path.join(process.cwd(), 'product_structures_sync_stop.json'); + fs.writeFileSync(stopFile, JSON.stringify({ + requestedAt: new Date().toISOString() + }, null, 2)); + + res.status(200).json({ + status: 'STOP_REQUESTED', + message: 'Product structures sync will stop after the current wait/request finishes.' + }); +}); + // Hidden endpoint to download the stock CSV log directly from the browser app.get('/api/stock-logs/download', (req: Request, res: Response) => { const expectedToken = process.env.TINY_WEBHOOK_SECRET; diff --git a/src/scripts/sync-product-structures.ts b/src/scripts/sync-product-structures.ts new file mode 100644 index 0000000..5db2385 --- /dev/null +++ b/src/scripts/sync-product-structures.ts @@ -0,0 +1,547 @@ +import axios from 'axios'; +import dotenv from 'dotenv'; +import fs from 'fs'; +import path from 'path'; + +dotenv.config(); + +type TinyProduct = { + id: string; + codigo: string; + nome: string; + unidade?: string; +}; + +type TinyStructureComponent = { + id_componente?: string | number; + codigo?: string | number; + nome?: string; + quantidade?: string | number; +}; + +type ProductStructurePayload = { + order: { + tinyId: string; + number: string; + status: string; + productSku: string; + productDescription: string; + quantity: string; + unit: string; + issueDate: string | null; + expectedDate: string | null; + supplier: string; + lotCode: string; + notes: string; + }; + components: Array<{ + componentTinyId: string; + componentName: string; + componentSku: string; + quantityPerUnit: string; + totalQuantity: string; + unit: string; + }>; + steps: never[]; +}; + +type SyncState = { + status?: 'running' | 'stopped' | 'completed'; + processedProducts?: number; + failedProducts?: number; + lastSku?: string; + lastFailedSku?: string; + lastMessage?: string; + updatedAt?: string; +}; + +type RequestedProduct = { + id: string; + sku: string; +}; + +const TINY_API_TOKEN = process.env.TINY_API_TOKEN; +const GRAPHS_PRODUCTION_ORDERS_URL = ( + process.env.PRODUCT_STRUCTURES_GRAPHS_API_URL || + process.env.PRODUCTION_ORDERS_GRAPHS_API_URL || + process.env.GRAPHS_PRODUCTION_ORDERS_API_URL || + (process.env.GRAPHS_API_BASE_URL ? `${process.env.GRAPHS_API_BASE_URL.replace(/\/+$/, '')}/api/production-orders/tiny-sync` : '') || + 'http://localhost:3004/api/production-orders/tiny-sync' +); +const GRAPHS_PRODUCTION_ORDERS_API_KEY = ( + process.env.PRODUCT_STRUCTURES_GRAPHS_API_KEY || + process.env.PRODUCTION_ORDERS_GRAPHS_API_KEY || + process.env.GRAPHS_PRODUCTION_ORDERS_API_KEY || + process.env.PRODUCTION_ORDERS_API_KEY || + process.env.GRAPHS_API_KEY || + process.env.NEXSTAR_GRAPHS_API_KEY || + process.env.API_KEY || + 'nexstar_secret_key_123' +); +const INPUT_FILE = process.env.PRODUCT_STRUCTURES_INPUT_FILE || ''; +const SKUS = String(process.env.PRODUCT_STRUCTURE_SKUS || '') + .split(',') + .map(value => value.trim()) + .filter(Boolean); +const PRODUCT_IDS = String(process.env.PRODUCT_STRUCTURE_PRODUCT_IDS || '') + .split(',') + .map(value => value.trim()) + .filter(Boolean); +const DRY_RUN = process.env.PRODUCT_STRUCTURES_DRY_RUN === 'true'; +const FETCH_COMPONENT_UNITS = process.env.PRODUCT_STRUCTURES_FETCH_COMPONENT_UNITS !== 'false'; +const STATE_FILE = process.env.PRODUCT_STRUCTURES_STATE_FILE || path.join(process.cwd(), 'product_structures_sync_state.json'); +const STOP_FILE = process.env.PRODUCT_STRUCTURES_STOP_FILE || path.join(process.cwd(), 'product_structures_sync_stop.json'); + +const numberEnv = (name: string, fallback: number) => { + const raw = process.env[name]; + if (raw === undefined || raw.trim() === '') return fallback; + + const value = Number(raw); + return Number.isFinite(value) && value >= 0 ? value : fallback; +}; +const REQUEST_DELAY_MS = numberEnv('PRODUCT_STRUCTURES_REQUEST_DELAY_MS', 2500); +const GRAPHS_RETRY_DELAY_MS = numberEnv('PRODUCT_STRUCTURES_GRAPHS_RETRY_DELAY_MS', 30000); +const GRAPHS_MAX_RETRIES = numberEnv('PRODUCT_STRUCTURES_GRAPHS_MAX_RETRIES', 3); +const MAX_PRODUCTS = numberEnv('PRODUCT_STRUCTURES_MAX_PRODUCTS', 0); + +let nextRequestAt = 0; +const productDetailCache = new Map(); + +class SyncStoppedError extends Error { + constructor() { + super('Product structures sync stopped by request.'); + } +} + +const sleep = (ms: number) => new Promise(resolve => setTimeout(resolve, ms)); + +function assertNotStopped() { + if (fs.existsSync(STOP_FILE)) { + throw new SyncStoppedError(); + } +} + +async function sleepWithStop(ms: number, reason: string) { + if (ms <= 0) return; + + const startedAt = Date.now(); + let remaining = ms; + console.log(`[wait] Waiting ${ms}ms before ${reason}.`); + + while (remaining > 0) { + assertNotStopped(); + await sleep(Math.min(remaining, 1000)); + remaining = ms - (Date.now() - startedAt); + } +} + +async function waitForRequestSlot(reason: string) { + const waitMs = Math.max(0, nextRequestAt - Date.now()); + await sleepWithStop(waitMs, reason); + nextRequestAt = Date.now() + REQUEST_DELAY_MS; +} + +function saveState(state: SyncState) { + fs.writeFileSync(STATE_FILE, JSON.stringify({ + ...state, + updatedAt: new Date().toISOString() + }, null, 2)); +} + +function describeRequestError(error: any) { + if (error?.response) { + const body = JSON.stringify(error.response.data ?? ''); + const truncatedBody = body.length > 500 ? `${body.slice(0, 500)}...` : body; + return `HTTP ${error.response.status}: ${truncatedBody}`; + } + + return error?.message || String(error); +} + +function normalizeText(value: unknown) { + return String(value ?? '').trim(); +} + +function normalizeNumber(value: unknown) { + if (typeof value === 'number') return Number.isFinite(value) ? value : 0; + const raw = normalizeText(value); + if (!raw) return 0; + const normalized = raw.includes(',') + ? raw.replace(/\./g, '').replace(',', '.') + : raw; + const parsed = Number(normalized); + return Number.isFinite(parsed) ? parsed : 0; +} + +function formatTinyDecimal(value: number) { + return Number.isFinite(value) ? String(value) : '0'; +} + +function parseDelimitedLine(line: string, delimiter: string) { + const values: string[] = []; + let current = ''; + let quoted = false; + + for (let index = 0; index < line.length; index += 1) { + const char = line[index]; + const nextChar = line[index + 1]; + + if (char === '"' && quoted && nextChar === '"') { + current += '"'; + index += 1; + continue; + } + + if (char === '"') { + quoted = !quoted; + continue; + } + + if (char === delimiter && !quoted) { + values.push(current); + current = ''; + continue; + } + + current += char; + } + + values.push(current); + return values; +} + +function parseDelimitedFile(raw: string, delimiter: string) { + const lines = raw + .replace(/^\uFEFF/, '') + .split(/\r?\n/) + .filter(line => line.trim() !== ''); + const headers = parseDelimitedLine(lines[0] || '', delimiter).map(header => header.trim()); + + return lines.slice(1).map(line => { + const values = parseDelimitedLine(line, delimiter); + return headers.reduce>((row, header, index) => { + row[header] = values[index] || ''; + return row; + }, {}); + }); +} + +function readInputFile() { + const filePath = path.resolve(INPUT_FILE); + const raw = fs.readFileSync(filePath, 'utf-8'); + const extension = path.extname(filePath).toLowerCase(); + + if (extension === '.csv') { + return parseDelimitedFile(raw, ','); + } + + if (extension === '.tsv') { + return parseDelimitedFile(raw, '\t'); + } + + const parsed = JSON.parse(raw); + if (Array.isArray(parsed)) return parsed; + return parsed?.skus || parsed?.products || parsed?.items || []; +} + +function loadRequestedProducts() { + const productsFromFile: RequestedProduct[] = INPUT_FILE + ? readInputFile().map((entry: any) => ({ + id: normalizeText(entry.id || entry.tinyId || entry.productId || entry['ID Produto'] || entry.ID_Produto), + sku: normalizeText(entry.sku || entry.codigo || entry.productSku || entry.SKU || entry.Código || entry.Codigo) + })) + : []; + + return productsFromFile + .concat(SKUS.map(sku => ({ id: '', sku }))) + .concat(PRODUCT_IDS.map(id => ({ id, sku: '' }))) + .filter((entry: RequestedProduct, index: number, all: RequestedProduct[]) => { + const key = entry.id ? `id:${entry.id}` : `sku:${entry.sku.toUpperCase()}`; + return (entry.id || entry.sku) && all.findIndex((other: RequestedProduct) => { + const otherKey = other.id ? `id:${other.id}` : `sku:${other.sku.toUpperCase()}`; + return otherKey === key; + }) === index; + }); +} + +async function tinyPost(servicePhp: string, params: URLSearchParams) { + if (!TINY_API_TOKEN) { + throw new Error('Missing TINY_API_TOKEN.'); + } + + await waitForRequestSlot(`Tiny ${servicePhp} request slot`); + return axios.post(`https://api.tiny.com.br/api2/${servicePhp}`, params, { + headers: { 'Content-Type': 'application/x-www-form-urlencoded' } + }); +} + +async function searchProductBySku(sku: string): Promise { + const params = new URLSearchParams(); + params.append('token', TINY_API_TOKEN!); + params.append('formato', 'JSON'); + params.append('pesquisa', sku); + params.append('pagina', '1'); + + const response = await tinyPost('produtos.pesquisa.php', params); + const retorno = response.data?.retorno; + if (retorno?.status !== 'OK') { + console.warn(`[Tiny] Product search failed for ${sku}: ${JSON.stringify(retorno?.erros || retorno || 'Unknown error')}`); + return null; + } + + const products = (retorno.produtos || []) + .map((entry: any) => entry?.produto || entry) + .filter(Boolean); + const exact = products.find((product: any) => normalizeText(product.codigo).toUpperCase() === sku.toUpperCase()); + const selected = exact || products[0]; + if (!selected) return null; + + return { + id: normalizeText(selected.id), + codigo: normalizeText(selected.codigo), + nome: normalizeText(selected.nome), + unidade: normalizeText(selected.unidade) + }; +} + +async function getProductById(id: string): Promise { + if (productDetailCache.has(id)) return productDetailCache.get(id) || null; + + const params = new URLSearchParams(); + params.append('token', TINY_API_TOKEN!); + params.append('formato', 'JSON'); + params.append('id', id); + + const response = await tinyPost('produto.obter.php', params); + const produto = response.data?.retorno?.produto; + if (!produto) { + console.warn(`[Tiny] Product get failed for ${id}: ${JSON.stringify(response.data?.retorno?.erros || response.data?.retorno || 'Unknown error')}`); + productDetailCache.set(id, null); + return null; + } + + const detail = { + id: normalizeText(produto.id || id), + codigo: normalizeText(produto.codigo), + nome: normalizeText(produto.nome), + unidade: normalizeText(produto.unidade) + }; + productDetailCache.set(id, detail); + return detail; +} + +async function getProductStructure(product: TinyProduct): Promise { + const params = new URLSearchParams(); + params.append('token', TINY_API_TOKEN!); + params.append('formato', 'JSON'); + params.append('id', product.id); + + const response = await tinyPost('produto.obter.estrutura.php', params); + const retorno = response.data?.retorno; + if (retorno?.status !== 'OK') { + console.warn(`[Tiny] Product structure failed for ${product.codigo || product.id}: ${JSON.stringify(retorno?.erros || retorno || 'Unknown error')}`); + return null; + } + + const produto = retorno.produto || {}; + const finishedSku = normalizeText(produto.codigo || product.codigo); + const finishedName = normalizeText(produto.nome || product.nome); + const rawComponents = (Array.isArray(produto.estrutura) ? produto.estrutura : []) + .map((entry: any) => entry?.item || entry) + .filter(Boolean); + const components = []; + + for (const component of rawComponents as TinyStructureComponent[]) { + const quantityPerUnit = normalizeNumber(component.quantidade); + const componentTinyId = normalizeText(component.id_componente); + const componentDetail = FETCH_COMPONENT_UNITS && componentTinyId + ? await getProductById(componentTinyId) + : null; + + components.push({ + componentTinyId, + componentName: normalizeText(component.nome || componentDetail?.nome), + componentSku: normalizeText(component.codigo || componentDetail?.codigo), + quantityPerUnit: formatTinyDecimal(quantityPerUnit), + totalQuantity: formatTinyDecimal(quantityPerUnit), + unit: normalizeText(componentDetail?.unidade) + }); + } + + const filteredComponents = components + .filter((component: ProductStructurePayload['components'][number]) => component.componentSku || component.componentName); + + return { + order: { + tinyId: `STRUCTURE-${product.id}`, + number: `STRUCTURE-${finishedSku || product.id}`, + status: 'completed', + productSku: finishedSku, + productDescription: finishedName, + quantity: '1', + unit: product.unidade || 'UN', + issueDate: null, + expectedDate: null, + supplier: '', + lotCode: '', + notes: `Composição sincronizada do Tiny produto.obter.estrutura.php para produto ${product.id}.` + }, + components: filteredComponents, + steps: [] + }; +} + +async function resolveProduct(requested: RequestedProduct) { + if (requested.id) return getProductById(requested.id); + return searchProductBySku(requested.sku); +} + +async function sendStructure(payload: ProductStructurePayload) { + if (DRY_RUN) { + console.log(`[dry-run] Would send structure ${payload.order.productSku}`); + console.log(JSON.stringify(payload, null, 2)); + return true; + } + + let attempt = 0; + while (true) { + try { + await waitForRequestSlot(`Graphs product structure ${payload.order.productSku}`); + await axios.post(GRAPHS_PRODUCTION_ORDERS_URL, payload, { + headers: { + 'Content-Type': 'application/json', + 'x-api-key': GRAPHS_PRODUCTION_ORDERS_API_KEY + } + }); + console.log(`[Graphs Structure] Sent ${payload.order.productSku} with ${payload.components.length} component(s).`); + return true; + } catch (error: any) { + attempt += 1; + const status = error?.response?.status; + const shouldRetry = !status || status >= 500; + console.error(`[Graphs Structure] Failed to send ${payload.order.productSku} on attempt ${attempt}: ${describeRequestError(error)}`); + + if (!shouldRetry || (GRAPHS_MAX_RETRIES && attempt >= GRAPHS_MAX_RETRIES)) { + console.error(`[Graphs Structure] Giving up on ${payload.order.productSku}; sync will continue.`); + return false; + } + + await sleepWithStop(GRAPHS_RETRY_DELAY_MS, `retrying Graphs structure ${payload.order.productSku}`); + } + } +} + +async function runSync() { + const requestedProducts = loadRequestedProducts(); + if (!requestedProducts.length) { + throw new Error('No products configured. Set PRODUCT_STRUCTURE_SKUS, PRODUCT_STRUCTURE_PRODUCT_IDS, or PRODUCT_STRUCTURES_INPUT_FILE.'); + } + + console.log(`[config] Graphs URL: ${GRAPHS_PRODUCTION_ORDERS_URL}`); + console.log(`[config] Dry run: ${DRY_RUN ? 'yes' : 'no'}`); + console.log(`[config] Request pace: 1 request every ${REQUEST_DELAY_MS}ms`); + console.log(`[config] State file: ${STATE_FILE}`); + console.log(`[config] Stop file: ${STOP_FILE}`); + console.log(`[source] Found ${requestedProducts.length} requested product(s).`); + + saveState({ + status: 'running', + processedProducts: 0, + failedProducts: 0, + lastMessage: 'Product structures sync started' + }); + + let processedProducts = 0; + let failedProducts = 0; + + for (const requested of requestedProducts) { + assertNotStopped(); + if (MAX_PRODUCTS && processedProducts + failedProducts >= MAX_PRODUCTS) { + console.log(`[limit] Stopped after ${MAX_PRODUCTS} product(s).`); + break; + } + + const label = requested.sku || requested.id; + try { + const product = await resolveProduct(requested); + if (!product?.id) { + console.warn(`[Tiny] Product not found for ${label}.`); + failedProducts += 1; + saveState({ + status: 'running', + processedProducts, + failedProducts, + lastFailedSku: label, + lastMessage: `Product not found for ${label}` + }); + continue; + } + + const payload = await getProductStructure(product); + if (!payload) { + failedProducts += 1; + saveState({ + status: 'running', + processedProducts, + failedProducts, + lastFailedSku: product.codigo || product.id, + lastMessage: `Could not get structure for ${product.codigo || product.id}` + }); + continue; + } + + const sent = await sendStructure(payload); + if (sent) { + processedProducts += 1; + } else { + failedProducts += 1; + } + + saveState({ + status: 'running', + processedProducts, + failedProducts, + lastSku: sent ? payload.order.productSku : undefined, + lastFailedSku: sent ? undefined : payload.order.productSku, + lastMessage: sent ? `Synced structure ${payload.order.productSku}` : `Failed structure ${payload.order.productSku}; continuing` + }); + } catch (error: any) { + if (error instanceof SyncStoppedError) throw error; + failedProducts += 1; + console.error(`[sync] Failed product ${label}: ${describeRequestError(error)}`); + saveState({ + status: 'running', + processedProducts, + failedProducts, + lastFailedSku: label, + lastMessage: `Failed product ${label}; continuing` + }); + } + } + + saveState({ + status: 'completed', + processedProducts, + failedProducts, + lastMessage: failedProducts ? `Product structures sync completed with ${failedProducts} failed product(s)` : 'Product structures sync completed' + }); + console.log(`Done. Synced ${processedProducts} product structure(s). Failed products: ${failedProducts}.`); +} + +runSync().catch((error: any) => { + if (error instanceof SyncStoppedError) { + console.log('Product structures sync stopped by request.'); + saveState({ + status: 'stopped', + lastMessage: 'Product structures sync stopped by request' + }); + process.exit(0); + } + + console.error(`Product structures sync failed: ${describeRequestError(error)}`); + saveState({ + status: 'stopped', + lastMessage: `Product structures sync failed: ${describeRequestError(error)}` + }); + process.exit(1); +});