Compare commits

..

8 Commits

Author SHA1 Message Date
Cauê Faleiros
4659222a41 Fix Swarm deployment template defaults
All checks were successful
Build and Deploy / build-and-push (push) Successful in 52s
2026-07-29 14:58:02 -03:00
Cauê Faleiros
06b59d8d37 Sync only V3 product compositions
All checks were successful
Build and Deploy / build-and-push (push) Successful in 1m46s
2026-07-29 14:46:25 -03:00
Cauê Faleiros
b1067325b9 Add Olist V3 product structure sync 2026-07-28 14:09:48 -03:00
Cauê Faleiros
4547642411 Add Tiny Olist production order sync
All checks were successful
Build and Deploy / build-and-push (push) Successful in 49s
2026-07-23 14:36:45 -03:00
Cauê Faleiros
e7ea6b4276 Let product structure tests override default CSV
All checks were successful
Build and Deploy / build-and-push (push) Successful in 53s
2026-07-23 14:15:05 -03:00
Cauê Faleiros
12aa5c7f83 Make product structure sync use product export
All checks were successful
Build and Deploy / build-and-push (push) Successful in 1m23s
2026-07-23 14:03:43 -03:00
Cauê Faleiros
bda1e22be3 Add Tiny product structure sync 2026-07-23 13:57:47 -03:00
Cauê Faleiros
831f82ee57 Improve graphs backfill resilience
All checks were successful
Build and Deploy / build-and-push (push) Successful in 2m56s
2026-07-03 12:19:43 -03:00
11 changed files with 10042 additions and 31 deletions

7
.gitignore vendored
View File

@@ -3,3 +3,10 @@ dist
.env .env
graphs_backfill_state.json graphs_backfill_state.json
graphs_backfill_stop.json graphs_backfill_stop.json
product_structures_sync_state.json
product_structures_sync_stop.json
product_structures_v3_sync_state.json
product_structures_v3_sync_stop.json
data-runtime/
production_orders_sync_state.json
production_orders_sync_stop.json

View File

@@ -22,6 +22,7 @@ RUN npm install --omit=dev
# Copy built code from the builder stage # Copy built code from the builder stage
COPY --from=builder /app/dist ./dist COPY --from=builder /app/dist ./dist
COPY --from=builder /app/data ./data
# Expose the port the app runs on # Expose the port the app runs on
EXPOSE 3000 EXPOSE 3000

File diff suppressed because it is too large Load Diff

View File

@@ -23,6 +23,22 @@ services:
- GRAPHS_BACKFILL_ORDER_DELAY_MS=${GRAPHS_BACKFILL_ORDER_DELAY_MS:-0} - GRAPHS_BACKFILL_ORDER_DELAY_MS=${GRAPHS_BACKFILL_ORDER_DELAY_MS:-0}
- GRAPHS_BACKFILL_TINY_BLOCK_DELAY_MS=${GRAPHS_BACKFILL_TINY_BLOCK_DELAY_MS:-600000} - GRAPHS_BACKFILL_TINY_BLOCK_DELAY_MS=${GRAPHS_BACKFILL_TINY_BLOCK_DELAY_MS:-600000}
- GRAPHS_BACKFILL_MAX_TINY_BLOCK_RETRIES=${GRAPHS_BACKFILL_MAX_TINY_BLOCK_RETRIES:-0} - GRAPHS_BACKFILL_MAX_TINY_BLOCK_RETRIES=${GRAPHS_BACKFILL_MAX_TINY_BLOCK_RETRIES:-0}
- PRODUCT_STRUCTURES_GRAPHS_API_URL=${PRODUCT_STRUCTURES_GRAPHS_API_URL}
- PRODUCT_STRUCTURES_GRAPHS_API_KEY=${PRODUCT_STRUCTURES_GRAPHS_API_KEY}
- PRODUCT_STRUCTURES_REQUEST_DELAY_MS=${PRODUCT_STRUCTURES_REQUEST_DELAY_MS:-2500}
- PRODUCT_STRUCTURES_FETCH_COMPONENT_UNITS=${PRODUCT_STRUCTURES_FETCH_COMPONENT_UNITS:-true}
- OLIST_V3_CLIENT_ID=${OLIST_V3_CLIENT_ID}
- OLIST_V3_CLIENT_SECRET=${OLIST_V3_CLIENT_SECRET}
- OLIST_V3_REDIRECT_URI=${OLIST_V3_REDIRECT_URI}
- OLIST_V3_TOKEN_FILE=${OLIST_V3_TOKEN_FILE:-/app/data-runtime/olist_v3_tokens.json}
- OLIST_V3_OAUTH_STATE_FILE=${OLIST_V3_OAUTH_STATE_FILE:-/app/data-runtime/olist_v3_oauth_state.json}
- PRODUCT_STRUCTURES_V3_GRAPHS_API_URL=${PRODUCT_STRUCTURES_V3_GRAPHS_API_URL}
- PRODUCT_STRUCTURES_V3_GRAPHS_API_KEY=${PRODUCT_STRUCTURES_V3_GRAPHS_API_KEY}
- PRODUCT_STRUCTURES_V3_REQUEST_DELAY_MS=${PRODUCT_STRUCTURES_V3_REQUEST_DELAY_MS:-1000}
- PRODUCTION_ORDERS_TINY_ERP_COOKIE=${PRODUCTION_ORDERS_TINY_ERP_COOKIE}
- PRODUCTION_ORDERS_GRAPHS_API_URL=${PRODUCTION_ORDERS_GRAPHS_API_URL}
- PRODUCTION_ORDERS_GRAPHS_API_KEY=${PRODUCTION_ORDERS_GRAPHS_API_KEY}
- PRODUCTION_ORDERS_REQUEST_DELAY_MS=${PRODUCTION_ORDERS_REQUEST_DELAY_MS:-2500}
- TINY_FETCH_SELLER_DETAILS=${TINY_FETCH_SELLER_DETAILS:-false} - TINY_FETCH_SELLER_DETAILS=${TINY_FETCH_SELLER_DETAILS:-false}
- TINY_ORDER_DETAILS_CACHE_TTL_MS=${TINY_ORDER_DETAILS_CACHE_TTL_MS:-120000} - TINY_ORDER_DETAILS_CACHE_TTL_MS=${TINY_ORDER_DETAILS_CACHE_TTL_MS:-120000}
- TINY_LIVE_REQUEST_DELAY_MS=${TINY_LIVE_REQUEST_DELAY_MS:-6000} - TINY_LIVE_REQUEST_DELAY_MS=${TINY_LIVE_REQUEST_DELAY_MS:-6000}
@@ -30,7 +46,12 @@ services:
# Optional: If you use a shared Docker network for Nginx Proxy Manager, you can attach it here: # Optional: If you use a shared Docker network for Nginx Proxy Manager, you can attach it here:
# networks: # networks:
# - nginx-proxy-manager-network # - nginx-proxy-manager-network
volumes:
- middleware_data:/app/data-runtime
# networks: # networks:
# nginx-proxy-manager-network: # nginx-proxy-manager-network:
# external: true # external: true
volumes:
middleware_data:

View File

@@ -8,6 +8,9 @@
"build": "tsc", "build": "tsc",
"start": "node dist/index.js", "start": "node dist/index.js",
"backfill:graphs": "tsx src/scripts/backfill-graphs.ts", "backfill:graphs": "tsx src/scripts/backfill-graphs.ts",
"sync:product-structures": "tsx src/scripts/sync-product-structures.ts",
"sync:product-structures:v3": "tsx src/scripts/sync-product-structures-v3.ts",
"sync:production-orders": "tsx src/scripts/sync-production-orders.ts",
"test": "echo \"Error: no test specified\" && exit 1" "test": "echo \"Error: no test specified\" && exit 1"
}, },
"keywords": [], "keywords": [],

View File

@@ -4,12 +4,30 @@ import webhookRoutes from './routes/webhook.route';
import { spawn } from 'child_process'; import { spawn } from 'child_process';
import path from 'path'; import path from 'path';
import fs from 'fs'; import fs from 'fs';
import {
exchangeOlistV3AuthorizationCode,
getOlistV3AuthorizationUrl,
getOlistV3AuthStatus,
isOlistV3Configured
} from './services/olist-v3.service';
dotenv.config(); dotenv.config();
const app: Express = express(); const app: Express = express();
const port = process.env.PORT || 3000; const port = process.env.PORT || 3000;
function deriveProductionOrdersUrl(env: NodeJS.ProcessEnv) {
const raw = env.GRAPHS_API_URL || env.NEXSTAR_GRAPHS_API_URL || '';
if (!raw) return null;
try {
const parsed = new URL(raw);
return `${parsed.origin}/api/production-orders/tiny-sync`;
} catch {
return raw.replace(/\/api\/data.*$/, '/api/production-orders/tiny-sync');
}
}
app.use(express.json()); app.use(express.json());
app.use(express.urlencoded({ extended: true })); app.use(express.urlencoded({ extended: true }));
@@ -19,6 +37,56 @@ app.get('/health', (req: Request, res: Response) => {
res.status(200).json({ status: 'OK', message: 'Tiny-n8n middleware is running.' }); res.status(200).json({ status: 'OK', message: 'Tiny-n8n middleware is running.' });
}); });
// Starts the official Olist V3 OAuth flow. The callback validates a one-time state value.
app.get('/api/olist-v3/authorize', (req: Request, res: Response) => {
const expectedToken = process.env.TINY_WEBHOOK_SECRET;
if (expectedToken && req.query.token !== expectedToken) {
res.status(401).json({ error: 'Unauthorized' });
return;
}
if (!isOlistV3Configured()) {
res.status(400).json({ error: 'Missing OLIST_V3_CLIENT_ID, OLIST_V3_CLIENT_SECRET, or OLIST_V3_REDIRECT_URI.' });
return;
}
res.redirect(getOlistV3AuthorizationUrl());
});
app.get('/api/olist-v3/callback', async (req: Request, res: Response) => {
const providerError = typeof req.query.error === 'string' ? req.query.error : '';
const code = typeof req.query.code === 'string' ? req.query.code : '';
const state = typeof req.query.state === 'string' ? req.query.state : '';
if (providerError) {
res.status(400).send('Olist V3 authorization was denied or failed. Return to the middleware and start authorization again.');
return;
}
if (!code || !state) {
res.status(400).send('Missing OAuth code or state. Start authorization again.');
return;
}
try {
await exchangeOlistV3AuthorizationCode(code, state);
res.status(200).send('Olist V3 authorized successfully. You can close this page and trigger the V3 product structure test.');
} catch (error: any) {
console.error(`[server]: Olist V3 OAuth callback failed: ${error.message}`);
res.status(400).send('Olist V3 authorization could not be completed. Check Portainer logs and start authorization again.');
}
});
app.get('/api/olist-v3/status', (req: Request, res: Response) => {
const expectedToken = process.env.TINY_WEBHOOK_SECRET;
if (expectedToken && req.query.token !== expectedToken) {
res.status(401).json({ error: 'Unauthorized' });
return;
}
res.status(200).json(getOlistV3AuthStatus());
});
// Hidden endpoint to trigger the backfill script without terminal access // Hidden endpoint to trigger the backfill script without terminal access
app.get('/api/trigger-backfill', (req: Request, res: Response) => { app.get('/api/trigger-backfill', (req: Request, res: Response) => {
const expectedToken = process.env.TINY_WEBHOOK_SECRET; const expectedToken = process.env.TINY_WEBHOOK_SECRET;
@@ -35,6 +103,14 @@ app.get('/api/trigger-backfill', (req: Request, res: Response) => {
stdio: 'inherit' // Inherit allows the logs to show up in Portainer's log viewer! stdio: 'inherit' // Inherit allows the logs to show up in Portainer's log viewer!
}); });
child.on('error', (error) => {
console.error(`[server]: Backfill process failed to start: ${error.message}`);
});
child.on('exit', (code, signal) => {
console.log(`[server]: Backfill process exited. code=${code ?? 'null'} signal=${signal ?? 'null'}`);
});
child.unref(); child.unref();
console.log('[server]: Backfill script manually triggered via HTTP endpoint.'); console.log('[server]: Backfill script manually triggered via HTTP endpoint.');
@@ -60,6 +136,7 @@ app.get('/api/trigger-graphs-backfill', (req: Request, res: Response) => {
maxOrders: 'GRAPHS_BACKFILL_MAX_ORDERS', maxOrders: 'GRAPHS_BACKFILL_MAX_ORDERS',
maxDays: 'GRAPHS_BACKFILL_MAX_DAYS', maxDays: 'GRAPHS_BACKFILL_MAX_DAYS',
contactFallback: 'GRAPHS_BACKFILL_CONTACT_FALLBACK', contactFallback: 'GRAPHS_BACKFILL_CONTACT_FALLBACK',
sellerFallback: 'GRAPHS_BACKFILL_SELLER_FALLBACK',
tinyRequestDelayMs: 'GRAPHS_BACKFILL_TINY_REQUEST_DELAY_MS', tinyRequestDelayMs: 'GRAPHS_BACKFILL_TINY_REQUEST_DELAY_MS',
orderDelayMs: 'GRAPHS_BACKFILL_ORDER_DELAY_MS', orderDelayMs: 'GRAPHS_BACKFILL_ORDER_DELAY_MS',
blockDelayMs: 'GRAPHS_BACKFILL_TINY_BLOCK_DELAY_MS', blockDelayMs: 'GRAPHS_BACKFILL_TINY_BLOCK_DELAY_MS',
@@ -83,6 +160,14 @@ app.get('/api/trigger-graphs-backfill', (req: Request, res: Response) => {
env env
}); });
child.on('error', (error) => {
console.error(`[server]: Graphs backfill process failed to start: ${error.message}`);
});
child.on('exit', (code, signal) => {
console.log(`[server]: Graphs backfill process exited. code=${code ?? 'null'} signal=${signal ?? 'null'}`);
});
child.unref(); child.unref();
console.log('[server]: Graphs backfill script manually triggered via HTTP endpoint.'); console.log('[server]: Graphs backfill script manually triggered via HTTP endpoint.');
@@ -96,6 +181,7 @@ app.get('/api/trigger-graphs-backfill', (req: Request, res: Response) => {
maxOrders: env.GRAPHS_BACKFILL_MAX_ORDERS || null, maxOrders: env.GRAPHS_BACKFILL_MAX_ORDERS || null,
maxDays: env.GRAPHS_BACKFILL_MAX_DAYS || null, maxDays: env.GRAPHS_BACKFILL_MAX_DAYS || null,
contactFallback: env.GRAPHS_BACKFILL_CONTACT_FALLBACK || 'true', contactFallback: env.GRAPHS_BACKFILL_CONTACT_FALLBACK || 'true',
sellerFallback: env.GRAPHS_BACKFILL_SELLER_FALLBACK || 'true',
tinyRequestDelayMs: env.GRAPHS_BACKFILL_TINY_REQUEST_DELAY_MS || '6000', tinyRequestDelayMs: env.GRAPHS_BACKFILL_TINY_REQUEST_DELAY_MS || '6000',
orderDelayMs: env.GRAPHS_BACKFILL_ORDER_DELAY_MS || '0', orderDelayMs: env.GRAPHS_BACKFILL_ORDER_DELAY_MS || '0',
blockDelayMs: env.GRAPHS_BACKFILL_TINY_BLOCK_DELAY_MS || '600000', blockDelayMs: env.GRAPHS_BACKFILL_TINY_BLOCK_DELAY_MS || '600000',
@@ -151,6 +237,346 @@ app.get('/api/stop-graphs-backfill', (req: Request, res: Response) => {
}); });
}); });
// Hidden endpoint to sync Tiny product structures/composition into Graphs consumption references
app.get('/api/trigger-product-structures-sync', (req: Request, res: Response) => {
const expectedToken = process.env.TINY_WEBHOOK_SECRET;
if (expectedToken && req.query.token !== expectedToken) {
res.status(401).json({ error: 'Unauthorized' });
return;
}
const scriptPath = path.join(__dirname, 'scripts', 'sync-product-structures.js');
const env = { ...process.env };
const stopFile = env.PRODUCT_STRUCTURES_STOP_FILE || path.join(process.cwd(), 'product_structures_sync_stop.json');
const optionalEnvMap: Record<string, string> = {
skus: 'PRODUCT_STRUCTURE_SKUS',
productIds: 'PRODUCT_STRUCTURE_PRODUCT_IDS',
inputFile: 'PRODUCT_STRUCTURES_INPUT_FILE',
graphsUrl: 'PRODUCT_STRUCTURES_GRAPHS_API_URL',
graphsApiKey: 'PRODUCT_STRUCTURES_GRAPHS_API_KEY',
dryRun: 'PRODUCT_STRUCTURES_DRY_RUN',
maxProducts: 'PRODUCT_STRUCTURES_MAX_PRODUCTS',
requestDelayMs: 'PRODUCT_STRUCTURES_REQUEST_DELAY_MS',
graphsRetryDelayMs: 'PRODUCT_STRUCTURES_GRAPHS_RETRY_DELAY_MS',
graphsMaxRetries: 'PRODUCT_STRUCTURES_GRAPHS_MAX_RETRIES'
};
for (const [queryKey, envKey] of Object.entries(optionalEnvMap)) {
const value = req.query[queryKey];
if (typeof value === 'string' && value.trim() !== '') {
env[envKey] = value.trim();
}
}
if (fs.existsSync(stopFile)) {
fs.rmSync(stopFile, { force: true });
}
const child = spawn('node', [scriptPath], {
detached: true,
stdio: 'inherit',
env
});
child.on('error', (error) => {
console.error(`[server]: Product structures sync process failed to start: ${error.message}`);
});
child.on('exit', (code, signal) => {
console.log(`[server]: Product structures sync process exited. code=${code ?? 'null'} signal=${signal ?? 'null'}`);
});
child.unref();
console.log('[server]: Product structures sync script manually triggered via HTTP endpoint.');
res.status(200).json({
status: 'STARTED',
message: 'Product structures sync is running in the background. Check Portainer logs for progress.',
options: {
skus: env.PRODUCT_STRUCTURE_SKUS || null,
productIds: env.PRODUCT_STRUCTURE_PRODUCT_IDS || null,
inputFile: env.PRODUCT_STRUCTURES_INPUT_FILE || ((env.PRODUCT_STRUCTURE_SKUS || env.PRODUCT_STRUCTURE_PRODUCT_IDS) ? null : '/app/data/product-structure-products.csv'),
graphsUrl: env.PRODUCT_STRUCTURES_GRAPHS_API_URL || env.PRODUCTION_ORDERS_GRAPHS_API_URL || env.GRAPHS_PRODUCTION_ORDERS_API_URL || deriveProductionOrdersUrl(env),
dryRun: env.PRODUCT_STRUCTURES_DRY_RUN || 'false',
maxProducts: env.PRODUCT_STRUCTURES_MAX_PRODUCTS || null,
requestDelayMs: env.PRODUCT_STRUCTURES_REQUEST_DELAY_MS || '2500',
graphsMaxRetries: env.PRODUCT_STRUCTURES_GRAPHS_MAX_RETRIES || '3',
graphsApiKeyConfigured: Boolean(env.PRODUCT_STRUCTURES_GRAPHS_API_KEY || env.PRODUCTION_ORDERS_GRAPHS_API_KEY || env.GRAPHS_PRODUCTION_ORDERS_API_KEY || env.GRAPHS_API_KEY || env.NEXSTAR_GRAPHS_API_KEY || env.API_KEY)
}
});
});
app.get('/api/product-structures-sync-status', (req: Request, res: Response) => {
const expectedToken = process.env.TINY_WEBHOOK_SECRET;
if (expectedToken && req.query.token !== expectedToken) {
res.status(401).json({ error: 'Unauthorized' });
return;
}
const stateFile = process.env.PRODUCT_STRUCTURES_STATE_FILE || path.join(process.cwd(), 'product_structures_sync_state.json');
const stopFile = process.env.PRODUCT_STRUCTURES_STOP_FILE || path.join(process.cwd(), 'product_structures_sync_stop.json');
let state = null;
if (fs.existsSync(stateFile)) {
try {
state = JSON.parse(fs.readFileSync(stateFile, 'utf-8'));
} catch (error: any) {
res.status(500).json({ error: `Could not read product structures sync state: ${error.message}` });
return;
}
}
res.status(200).json({
status: state?.status || 'unknown',
stopRequested: fs.existsSync(stopFile),
state
});
});
app.get('/api/stop-product-structures-sync', (req: Request, res: Response) => {
const expectedToken = process.env.TINY_WEBHOOK_SECRET;
if (expectedToken && req.query.token !== expectedToken) {
res.status(401).json({ error: 'Unauthorized' });
return;
}
const stopFile = process.env.PRODUCT_STRUCTURES_STOP_FILE || path.join(process.cwd(), 'product_structures_sync_stop.json');
fs.writeFileSync(stopFile, JSON.stringify({
requestedAt: new Date().toISOString()
}, null, 2));
res.status(200).json({
status: 'STOP_REQUESTED',
message: 'Product structures sync will stop after the current wait/request finishes.'
});
});
// Hidden endpoint to sync manufactured-product compositions from the official Olist V3 API.
app.get('/api/trigger-product-structures-v3-sync', (req: Request, res: Response) => {
const expectedToken = process.env.TINY_WEBHOOK_SECRET;
if (expectedToken && req.query.token !== expectedToken) {
res.status(401).json({ error: 'Unauthorized' });
return;
}
const scriptPath = path.join(__dirname, 'scripts', 'sync-product-structures-v3.js');
const env = { ...process.env };
const stopFile = env.PRODUCT_STRUCTURES_V3_STOP_FILE || path.join(process.cwd(), 'data-runtime', 'product_structures_v3_sync_stop.json');
const optionalEnvMap: Record<string, string> = {
productIds: 'PRODUCT_STRUCTURES_V3_PRODUCT_IDS',
inputFile: 'PRODUCT_STRUCTURES_V3_INPUT_FILE',
graphsUrl: 'PRODUCT_STRUCTURES_V3_GRAPHS_API_URL',
graphsApiKey: 'PRODUCT_STRUCTURES_V3_GRAPHS_API_KEY',
dryRun: 'PRODUCT_STRUCTURES_V3_DRY_RUN',
maxProducts: 'PRODUCT_STRUCTURES_V3_MAX_PRODUCTS',
requestDelayMs: 'PRODUCT_STRUCTURES_V3_REQUEST_DELAY_MS',
graphsRetryDelayMs: 'PRODUCT_STRUCTURES_V3_GRAPHS_RETRY_DELAY_MS',
graphsMaxRetries: 'PRODUCT_STRUCTURES_V3_GRAPHS_MAX_RETRIES'
};
for (const [queryKey, envKey] of Object.entries(optionalEnvMap)) {
const value = req.query[queryKey];
if (typeof value === 'string' && value.trim() !== '') {
env[envKey] = value.trim();
}
}
if (!env.OLIST_V3_CLIENT_ID || !env.OLIST_V3_CLIENT_SECRET || !env.OLIST_V3_REDIRECT_URI) {
res.status(400).json({ error: 'Olist V3 is not configured. Set the client ID, client secret, and redirect URI first.' });
return;
}
if (!getOlistV3AuthStatus().authorized) {
res.status(409).json({ error: 'Olist V3 is not authorized. Open /api/olist-v3/authorize first.' });
return;
}
if (fs.existsSync(stopFile)) {
fs.rmSync(stopFile, { force: true });
}
const child = spawn('node', [scriptPath], { detached: true, stdio: 'inherit', env });
child.on('error', (error) => {
console.error(`[server]: Olist V3 product structures sync failed to start: ${error.message}`);
});
child.on('exit', (code, signal) => {
console.log(`[server]: Olist V3 product structures sync exited. code=${code ?? 'null'} signal=${signal ?? 'null'}`);
});
child.unref();
console.log('[server]: Olist V3 product structures sync manually triggered via HTTP endpoint.');
res.status(200).json({
status: 'STARTED',
message: 'Olist V3 product structures sync is running in the background. Check Portainer logs for progress.',
options: {
productIds: env.PRODUCT_STRUCTURES_V3_PRODUCT_IDS || null,
inputFile: env.PRODUCT_STRUCTURES_V3_INPUT_FILE || (env.PRODUCT_STRUCTURES_V3_PRODUCT_IDS ? null : '/app/data/product-structure-products.csv'),
graphsUrl: env.PRODUCT_STRUCTURES_V3_GRAPHS_API_URL || deriveProductionOrdersUrl(env),
dryRun: env.PRODUCT_STRUCTURES_V3_DRY_RUN || 'false',
maxProducts: env.PRODUCT_STRUCTURES_V3_MAX_PRODUCTS || null,
requestDelayMs: env.PRODUCT_STRUCTURES_V3_REQUEST_DELAY_MS || '1000',
olistV3Configured: true,
olistV3Authorized: getOlistV3AuthStatus().authorized,
graphsApiKeyConfigured: Boolean(env.PRODUCT_STRUCTURES_V3_GRAPHS_API_KEY || env.GRAPHS_API_KEY || env.NEXSTAR_GRAPHS_API_KEY)
}
});
});
app.get('/api/product-structures-v3-sync-status', (req: Request, res: Response) => {
const expectedToken = process.env.TINY_WEBHOOK_SECRET;
if (expectedToken && req.query.token !== expectedToken) {
res.status(401).json({ error: 'Unauthorized' });
return;
}
const stateFile = process.env.PRODUCT_STRUCTURES_V3_STATE_FILE || path.join(process.cwd(), 'data-runtime', 'product_structures_v3_sync_state.json');
const stopFile = process.env.PRODUCT_STRUCTURES_V3_STOP_FILE || path.join(process.cwd(), 'data-runtime', 'product_structures_v3_sync_stop.json');
let state = null;
if (fs.existsSync(stateFile)) {
try {
state = JSON.parse(fs.readFileSync(stateFile, 'utf-8'));
} catch (error: any) {
res.status(500).json({ error: `Could not read Olist V3 product structures state: ${error.message}` });
return;
}
}
res.status(200).json({ status: state?.status || 'unknown', stopRequested: fs.existsSync(stopFile), state });
});
app.get('/api/stop-product-structures-v3-sync', (req: Request, res: Response) => {
const expectedToken = process.env.TINY_WEBHOOK_SECRET;
if (expectedToken && req.query.token !== expectedToken) {
res.status(401).json({ error: 'Unauthorized' });
return;
}
const stopFile = process.env.PRODUCT_STRUCTURES_V3_STOP_FILE || path.join(process.cwd(), 'data-runtime', 'product_structures_v3_sync_stop.json');
fs.mkdirSync(path.dirname(stopFile), { recursive: true });
fs.writeFileSync(stopFile, JSON.stringify({ requestedAt: new Date().toISOString() }, null, 2));
res.status(200).json({ status: 'STOP_REQUESTED', message: 'Olist V3 product structures sync will stop after the current request.' });
});
// Hidden endpoint to sync Tiny/Olist production order details into Graphs
app.get('/api/trigger-production-orders-sync', (req: Request, res: Response) => {
const expectedToken = process.env.TINY_WEBHOOK_SECRET;
if (expectedToken && req.query.token !== expectedToken) {
res.status(401).json({ error: 'Unauthorized' });
return;
}
const scriptPath = path.join(__dirname, 'scripts', 'sync-production-orders.js');
const env = { ...process.env };
const stopFile = env.PRODUCTION_ORDERS_STOP_FILE || path.join(process.cwd(), 'production_orders_sync_stop.json');
const optionalEnvMap: Record<string, string> = {
orderIds: 'PRODUCTION_ORDER_IDS',
detailUrl: 'PRODUCTION_ORDERS_DETAIL_URL',
productDataUrl: 'PRODUCTION_ORDERS_PRODUCT_DATA_URL',
listUrl: 'PRODUCTION_ORDERS_LIST_URL',
listBody: 'PRODUCTION_ORDERS_LIST_BODY',
detailBodyTemplate: 'PRODUCTION_ORDERS_DETAIL_BODY_TEMPLATE',
productDataBodyTemplate: 'PRODUCTION_ORDERS_PRODUCT_DATA_BODY_TEMPLATE',
graphsUrl: 'PRODUCTION_ORDERS_GRAPHS_API_URL',
graphsApiKey: 'PRODUCTION_ORDERS_GRAPHS_API_KEY',
dryRun: 'PRODUCTION_ORDERS_DRY_RUN',
maxOrders: 'PRODUCTION_ORDERS_MAX_ORDERS',
requestDelayMs: 'PRODUCTION_ORDERS_REQUEST_DELAY_MS',
graphsRetryDelayMs: 'PRODUCTION_ORDERS_GRAPHS_RETRY_DELAY_MS',
graphsMaxRetries: 'PRODUCTION_ORDERS_GRAPHS_MAX_RETRIES'
};
for (const [queryKey, envKey] of Object.entries(optionalEnvMap)) {
const value = req.query[queryKey];
if (typeof value === 'string' && value.trim() !== '') {
env[envKey] = value.trim();
}
}
if (fs.existsSync(stopFile)) {
fs.rmSync(stopFile, { force: true });
}
const child = spawn('node', [scriptPath], {
detached: true,
stdio: 'inherit',
env
});
child.on('error', (error) => {
console.error(`[server]: Production orders sync process failed to start: ${error.message}`);
});
child.on('exit', (code, signal) => {
console.log(`[server]: Production orders sync process exited. code=${code ?? 'null'} signal=${signal ?? 'null'}`);
});
child.unref();
console.log('[server]: Production orders sync script manually triggered via HTTP endpoint.');
res.status(200).json({
status: 'STARTED',
message: 'Production orders sync is running in the background. Check Portainer logs for progress.',
options: {
orderIds: env.PRODUCTION_ORDER_IDS || null,
detailUrl: env.PRODUCTION_ORDERS_DETAIL_URL || 'https://erp.olist.com/services/ordem.producao.server/1/obterOrdemProducao',
productDataUrl: env.PRODUCTION_ORDERS_PRODUCT_DATA_URL || 'https://erp.olist.com/services/ordem.producao.server/1/buscarDadosProduto',
graphsUrl: env.PRODUCTION_ORDERS_GRAPHS_API_URL || env.GRAPHS_PRODUCTION_ORDERS_API_URL || deriveProductionOrdersUrl(env),
dryRun: env.PRODUCTION_ORDERS_DRY_RUN || 'false',
maxOrders: env.PRODUCTION_ORDERS_MAX_ORDERS || null,
requestDelayMs: env.PRODUCTION_ORDERS_REQUEST_DELAY_MS || '2500',
tinyErpAuthConfigured: Boolean(env.PRODUCTION_ORDERS_TINY_ERP_COOKIE || env.PRODUCTION_ORDERS_TINY_ERP_AUTHORIZATION),
graphsApiKeyConfigured: Boolean(env.PRODUCTION_ORDERS_GRAPHS_API_KEY || env.GRAPHS_PRODUCTION_ORDERS_API_KEY || env.GRAPHS_API_KEY || env.NEXSTAR_GRAPHS_API_KEY || env.API_KEY)
}
});
});
app.get('/api/production-orders-sync-status', (req: Request, res: Response) => {
const expectedToken = process.env.TINY_WEBHOOK_SECRET;
if (expectedToken && req.query.token !== expectedToken) {
res.status(401).json({ error: 'Unauthorized' });
return;
}
const stateFile = process.env.PRODUCTION_ORDERS_STATE_FILE || path.join(process.cwd(), 'production_orders_sync_state.json');
const stopFile = process.env.PRODUCTION_ORDERS_STOP_FILE || path.join(process.cwd(), 'production_orders_sync_stop.json');
let state = null;
if (fs.existsSync(stateFile)) {
try {
state = JSON.parse(fs.readFileSync(stateFile, 'utf-8'));
} catch (error: any) {
res.status(500).json({ error: `Could not read production orders sync state: ${error.message}` });
return;
}
}
res.status(200).json({
status: state?.status || 'unknown',
stopRequested: fs.existsSync(stopFile),
state
});
});
app.get('/api/stop-production-orders-sync', (req: Request, res: Response) => {
const expectedToken = process.env.TINY_WEBHOOK_SECRET;
if (expectedToken && req.query.token !== expectedToken) {
res.status(401).json({ error: 'Unauthorized' });
return;
}
const stopFile = process.env.PRODUCTION_ORDERS_STOP_FILE || path.join(process.cwd(), 'production_orders_sync_stop.json');
fs.writeFileSync(stopFile, JSON.stringify({
requestedAt: new Date().toISOString()
}, null, 2));
res.status(200).json({
status: 'STOP_REQUESTED',
message: 'Production orders sync will stop after the current wait/request finishes.'
});
});
// Hidden endpoint to download the stock CSV log directly from the browser // Hidden endpoint to download the stock CSV log directly from the browser
app.get('/api/stock-logs/download', (req: Request, res: Response) => { app.get('/api/stock-logs/download', (req: Request, res: Response) => {
const expectedToken = process.env.TINY_WEBHOOK_SECRET; const expectedToken = process.env.TINY_WEBHOOK_SECRET;

View File

@@ -41,14 +41,20 @@ type ContactInfo = {
phone: string; phone: string;
}; };
type SellerInfo = {
name: string;
};
type BackfillState = { type BackfillState = {
nextDate: string; nextDate: string;
lastCompletedDate?: string; lastCompletedDate?: string;
status?: 'running' | 'stopped' | 'completed'; status?: 'running' | 'stopped' | 'completed';
currentDate?: string; currentDate?: string;
lastOrderId?: string; lastOrderId?: string;
lastFailedOrderId?: string;
processedDays?: number; processedDays?: number;
processedOrders?: number; processedOrders?: number;
failedOrders?: number;
lastMessage?: string; lastMessage?: string;
updatedAt?: string; updatedAt?: string;
}; };
@@ -58,6 +64,7 @@ const GRAPHS_API_URL = process.env.GRAPHS_API_URL || process.env.NEXSTAR_GRAPHS_
const GRAPHS_API_KEY = process.env.GRAPHS_API_KEY || process.env.NEXSTAR_GRAPHS_API_KEY; const GRAPHS_API_KEY = process.env.GRAPHS_API_KEY || process.env.NEXSTAR_GRAPHS_API_KEY;
const DRY_RUN = process.env.GRAPHS_BACKFILL_DRY_RUN === 'true'; const DRY_RUN = process.env.GRAPHS_BACKFILL_DRY_RUN === 'true';
const CONTACT_FALLBACK = process.env.GRAPHS_BACKFILL_CONTACT_FALLBACK !== 'false'; const CONTACT_FALLBACK = process.env.GRAPHS_BACKFILL_CONTACT_FALLBACK !== 'false';
const SELLER_FALLBACK = process.env.GRAPHS_BACKFILL_SELLER_FALLBACK !== 'false';
const NORMALIZE_PHONE = process.env.GRAPHS_BACKFILL_NORMALIZE_PHONE !== 'false'; const NORMALIZE_PHONE = process.env.GRAPHS_BACKFILL_NORMALIZE_PHONE !== 'false';
const START_DATE = process.env.GRAPHS_BACKFILL_START_DATE || '13/02/2025'; const START_DATE = process.env.GRAPHS_BACKFILL_START_DATE || '13/02/2025';
const END_DATE = process.env.GRAPHS_BACKFILL_END_DATE || getTodayDate(); const END_DATE = process.env.GRAPHS_BACKFILL_END_DATE || getTodayDate();
@@ -77,10 +84,13 @@ const TINY_CONTACT_DELAY_MS = numberEnv('TINY_CONTACT_DELAY_MS', TINY_DETAIL_DEL
const ORDER_DELAY_MS = numberEnv('GRAPHS_BACKFILL_ORDER_DELAY_MS', 0); const ORDER_DELAY_MS = numberEnv('GRAPHS_BACKFILL_ORDER_DELAY_MS', 0);
const TINY_BLOCK_DELAY_MS = numberEnv('GRAPHS_BACKFILL_TINY_BLOCK_DELAY_MS', 600000); const TINY_BLOCK_DELAY_MS = numberEnv('GRAPHS_BACKFILL_TINY_BLOCK_DELAY_MS', 600000);
const MAX_TINY_BLOCK_RETRIES = numberEnv('GRAPHS_BACKFILL_MAX_TINY_BLOCK_RETRIES', 0); const MAX_TINY_BLOCK_RETRIES = numberEnv('GRAPHS_BACKFILL_MAX_TINY_BLOCK_RETRIES', 0);
const GRAPHS_API_MAX_RETRIES = numberEnv('GRAPHS_BACKFILL_GRAPHS_API_MAX_RETRIES', 3);
const GRAPHS_API_RETRY_DELAY_MS = numberEnv('GRAPHS_BACKFILL_GRAPHS_API_RETRY_DELAY_MS', 30000);
const STATE_FILE = process.env.GRAPHS_BACKFILL_STATE_FILE || path.join(process.cwd(), 'graphs_backfill_state.json'); const STATE_FILE = process.env.GRAPHS_BACKFILL_STATE_FILE || path.join(process.cwd(), 'graphs_backfill_state.json');
const STOP_FILE = process.env.GRAPHS_BACKFILL_STOP_FILE || path.join(process.cwd(), 'graphs_backfill_stop.json'); const STOP_FILE = process.env.GRAPHS_BACKFILL_STOP_FILE || path.join(process.cwd(), 'graphs_backfill_stop.json');
const contactCache = new Map<string, ContactInfo | null>(); const contactCache = new Map<string, ContactInfo | null>();
const sellerCache = new Map<string, SellerInfo | null>();
let lastTinyRequestAt = 0; let lastTinyRequestAt = 0;
const sleep = (ms: number) => new Promise(resolve => setTimeout(resolve, ms)); const sleep = (ms: number) => new Promise(resolve => setTimeout(resolve, ms));
@@ -218,6 +228,16 @@ function getTinyErrors(data: any) {
return Array.isArray(errors) ? errors : [errors]; return Array.isArray(errors) ? errors : [errors];
} }
function describeRequestError(error: any) {
if (error?.response) {
const body = JSON.stringify(error.response.data ?? '');
const truncatedBody = body.length > 500 ? `${body.slice(0, 500)}...` : body;
return `HTTP ${error.response.status}: ${truncatedBody}`;
}
return error?.message || String(error);
}
function isTinyApiBlocked(data: any) { function isTinyApiBlocked(data: any) {
return getTinyErrors(data).some((entry: any) => { return getTinyErrors(data).some((entry: any) => {
const message = String(entry?.erro || entry || '').toLowerCase(); const message = String(entry?.erro || entry || '').toLowerCase();
@@ -239,9 +259,24 @@ async function tinyPost(servicePhp: string, params: URLSearchParams) {
assertNotStopped(); assertNotStopped();
await waitForTinyRequestSlot(servicePhp); await waitForTinyRequestSlot(servicePhp);
const response = await axios.post(`https://api.tiny.com.br/api2/${servicePhp}`, params, { let response;
try {
response = await axios.post(`https://api.tiny.com.br/api2/${servicePhp}`, params, {
headers: { 'Content-Type': 'application/x-www-form-urlencoded' } headers: { 'Content-Type': 'application/x-www-form-urlencoded' }
}); });
} catch (error: any) {
blockedAttempts += 1;
if (MAX_TINY_BLOCK_RETRIES && blockedAttempts > MAX_TINY_BLOCK_RETRIES) {
throw error;
}
const retryLabel = MAX_TINY_BLOCK_RETRIES
? `${blockedAttempts}/${MAX_TINY_BLOCK_RETRIES}`
: String(blockedAttempts);
console.warn(`[Tiny] Request failed on ${servicePhp}: ${describeRequestError(error)}. Pausing ${TINY_BLOCK_DELAY_MS}ms before retry ${retryLabel}.`);
await sleepWithStop(TINY_BLOCK_DELAY_MS, 'retrying Tiny API after request failure');
continue;
}
if (!isTinyApiBlocked(response.data)) { if (!isTinyApiBlocked(response.data)) {
return response; return response;
@@ -376,6 +411,41 @@ async function fetchContactInfo(cliente: any): Promise<ContactInfo | null> {
} }
} }
async function fetchSellerInfo(sellerId: string): Promise<SellerInfo | null> {
const normalizedSellerId = String(sellerId || '').trim();
if (!SELLER_FALLBACK || !normalizedSellerId || normalizedSellerId === '0') return null;
if (sellerCache.has(normalizedSellerId)) return sellerCache.get(normalizedSellerId) || null;
try {
const detailParams = new URLSearchParams();
detailParams.append('token', TINY_API_TOKEN!);
detailParams.append('formato', 'JSON');
detailParams.append('id', normalizedSellerId);
const detailResponse = await tinyPost('contato.obter.php', detailParams);
await sleepWithStop(TINY_CONTACT_DELAY_MS, 'next Tiny seller contact request');
if (detailResponse.data?.retorno?.status !== 'OK') {
console.warn(`[Tiny] Seller lookup failed for ${normalizedSellerId}: ${JSON.stringify(detailResponse.data?.retorno?.erros || 'Unknown error')}`);
sellerCache.set(normalizedSellerId, null);
return null;
}
const contato = detailResponse.data.retorno.contato || {};
const info = {
name: pickFirst(contato.nome, contato.razao_social, contato.fantasia, contato.nome_fantasia)
};
sellerCache.set(normalizedSellerId, info.name ? info : null);
return info.name ? info : null;
} catch (error: any) {
if (error instanceof BackfillStoppedError) throw error;
console.error(`[Tiny] Seller lookup failed for ${normalizedSellerId}: ${describeRequestError(error)}`);
sellerCache.set(normalizedSellerId, null);
return null;
}
}
async function buildGraphsRows(details: TinyOrderDetails, summary: TinyOrderSummary): Promise<GraphsOrderRow[]> { async function buildGraphsRows(details: TinyOrderDetails, summary: TinyOrderSummary): Promise<GraphsOrderRow[]> {
const pedido = details.pedido || {}; const pedido = details.pedido || {};
const cliente = pedido.cliente || {}; const cliente = pedido.cliente || {};
@@ -398,6 +468,12 @@ async function buildGraphsRows(details: TinyOrderDetails, summary: TinyOrderSumm
const customerPhone = normalizePhone(pickFirst(orderCustomerPhone, contactInfo?.phone)); const customerPhone = normalizePhone(pickFirst(orderCustomerPhone, contactInfo?.phone));
const customerFantasyName = pickFirst(orderCustomerFantasyName, contactInfo?.fantasia); const customerFantasyName = pickFirst(orderCustomerFantasyName, contactInfo?.fantasia);
const ecommerce = pedido.ecommerce || {}; const ecommerce = pedido.ecommerce || {};
const sellerId = pickFirst(pedido.id_vendedor);
const orderSellerName = pickFirst(pedido.nome_vendedor);
const sellerInfo = sellerId && sellerId !== '0' && !orderSellerName
? await fetchSellerInfo(sellerId)
: null;
const sellerName = pickFirst(orderSellerName, sellerInfo?.name);
return itens return itens
.map((entry: any) => entry?.item || entry) .map((entry: any) => entry?.item || entry)
@@ -413,8 +489,8 @@ async function buildGraphsRows(details: TinyOrderDetails, summary: TinyOrderSumm
ID_Pedido: orderId, ID_Pedido: orderId,
Fone_Cliente: customerPhone, Fone_Cliente: customerPhone,
cliente_nome_fantasia: customerFantasyName, cliente_nome_fantasia: customerFantasyName,
id_vendedor: pickFirst(pedido.id_vendedor), id_vendedor: sellerId,
nome_vendedor: pickFirst(pedido.nome_vendedor), nome_vendedor: sellerName,
marketplace: pickFirst(ecommerce.nomeEcommerce, pedido.nome_ecommerce), marketplace: pickFirst(ecommerce.nomeEcommerce, pedido.nome_ecommerce),
canal_venda: pickFirst(ecommerce.canalVenda, pedido.canal_venda), canal_venda: pickFirst(ecommerce.canalVenda, pedido.canal_venda),
numero_ecommerce: pickFirst( numero_ecommerce: pickFirst(
@@ -428,15 +504,18 @@ async function buildGraphsRows(details: TinyOrderDetails, summary: TinyOrderSumm
async function sendRowsToGraphs(rows: GraphsOrderRow[], orderId: string) { async function sendRowsToGraphs(rows: GraphsOrderRow[], orderId: string) {
if (!rows.length) { if (!rows.length) {
console.warn(`[Graphs] Order ${orderId} has no items. Skipping.`); console.warn(`[Graphs] Order ${orderId} has no items. Skipping.`);
return; return true;
} }
if (DRY_RUN) { if (DRY_RUN) {
console.log(`[dry-run] Would send ${rows.length} row(s) for order ${orderId}`); console.log(`[dry-run] Would send ${rows.length} row(s) for order ${orderId}`);
console.log(JSON.stringify(rows.slice(0, 3), null, 2)); console.log(JSON.stringify(rows.slice(0, 3), null, 2));
return; return true;
} }
let attempt = 0;
while (true) {
try {
await axios.post(GRAPHS_API_URL!, rows, { await axios.post(GRAPHS_API_URL!, rows, {
headers: { headers: {
'Content-Type': 'application/json', 'Content-Type': 'application/json',
@@ -445,6 +524,22 @@ async function sendRowsToGraphs(rows: GraphsOrderRow[], orderId: string) {
}); });
console.log(`[Graphs] Sent ${rows.length} row(s) for order ${orderId}`); console.log(`[Graphs] Sent ${rows.length} row(s) for order ${orderId}`);
return true;
} catch (error: any) {
attempt += 1;
const status = error?.response?.status;
const shouldRetry = !status || status >= 500;
console.error(`[Graphs] Failed to send order ${orderId} on attempt ${attempt}: ${describeRequestError(error)}`);
if (!shouldRetry || (GRAPHS_API_MAX_RETRIES && attempt >= GRAPHS_API_MAX_RETRIES)) {
console.error(`[Graphs] Giving up on order ${orderId}; backfill will continue with the next order.`);
return false;
}
await sleepWithStop(GRAPHS_API_RETRY_DELAY_MS, `retrying Graphs API for order ${orderId}`);
}
}
} }
async function runBackfill() { async function runBackfill() {
@@ -464,6 +559,7 @@ async function runBackfill() {
let lastCompletedDate = state.lastCompletedDate; let lastCompletedDate = state.lastCompletedDate;
let processedDays = 0; let processedDays = 0;
let processedOrders = 0; let processedOrders = 0;
let failedOrders = state.failedOrders || 0;
const orderPace = ORDER_DELAY_MS ? `~${Math.floor(60000 / ORDER_DELAY_MS)}/minute` : 'unlimited'; const orderPace = ORDER_DELAY_MS ? `~${Math.floor(60000 / ORDER_DELAY_MS)}/minute` : 'unlimited';
const tinyRequestPace = TINY_REQUEST_DELAY_MS ? `~${Math.floor(60000 / TINY_REQUEST_DELAY_MS)}/minute` : 'unlimited'; const tinyRequestPace = TINY_REQUEST_DELAY_MS ? `~${Math.floor(60000 / TINY_REQUEST_DELAY_MS)}/minute` : 'unlimited';
@@ -474,6 +570,7 @@ async function runBackfill() {
console.log(`[config] Tiny request pace: 1 request every ${TINY_REQUEST_DELAY_MS}ms (${tinyRequestPace})`); console.log(`[config] Tiny request pace: 1 request every ${TINY_REQUEST_DELAY_MS}ms (${tinyRequestPace})`);
console.log(`[config] Order pace: 1 order every ${ORDER_DELAY_MS}ms (${orderPace})`); console.log(`[config] Order pace: 1 order every ${ORDER_DELAY_MS}ms (${orderPace})`);
console.log(`[config] Tiny block pause: ${TINY_BLOCK_DELAY_MS}ms`); console.log(`[config] Tiny block pause: ${TINY_BLOCK_DELAY_MS}ms`);
console.log(`[config] Graphs API retries: ${GRAPHS_API_MAX_RETRIES || 'unlimited'}`);
console.log(`[config] State file: ${STATE_FILE}`); console.log(`[config] State file: ${STATE_FILE}`);
console.log(`[config] Stop file: ${STOP_FILE}`); console.log(`[config] Stop file: ${STOP_FILE}`);
@@ -483,6 +580,7 @@ async function runBackfill() {
currentDate, currentDate,
processedDays, processedDays,
processedOrders, processedOrders,
failedOrders,
lastMessage: 'Backfill started' lastMessage: 'Backfill started'
}); });
@@ -521,23 +619,61 @@ async function runBackfill() {
continue; continue;
} }
try {
const details = await fetchOrderDetails(orderId); const details = await fetchOrderDetails(orderId);
if (!details) continue; if (!details) {
failedOrders += 1;
saveState({
nextDate: currentDate,
lastCompletedDate,
status: 'running',
currentDate,
lastFailedOrderId: orderId,
processedDays,
processedOrders,
failedOrders,
lastMessage: `Could not fetch Tiny details for order ${orderId}`
});
continue;
}
const rows = await buildGraphsRows(details, summary); const rows = await buildGraphsRows(details, summary);
await sendRowsToGraphs(rows, orderId); const sent = await sendRowsToGraphs(rows, orderId);
if (sent) {
processedOrders += 1; processedOrders += 1;
} else {
failedOrders += 1;
}
saveState({ saveState({
nextDate: currentDate, nextDate: currentDate,
lastCompletedDate, lastCompletedDate,
status: 'running', status: 'running',
currentDate, currentDate,
lastOrderId: orderId, lastOrderId: sent ? orderId : undefined,
lastFailedOrderId: sent ? undefined : orderId,
processedDays, processedDays,
processedOrders, processedOrders,
lastMessage: `Processed order ${orderId}` failedOrders,
lastMessage: sent ? `Processed order ${orderId}` : `Failed order ${orderId}; continuing`
}); });
} catch (error: any) {
if (error instanceof BackfillStoppedError) throw error;
failedOrders += 1;
console.error(`[backfill] Unexpected failure for order ${orderId}: ${describeRequestError(error)}. Continuing with next order.`);
saveState({
nextDate: currentDate,
lastCompletedDate,
status: 'running',
currentDate,
lastFailedOrderId: orderId,
processedDays,
processedOrders,
failedOrders,
lastMessage: `Unexpected failure for order ${orderId}; continuing`
});
}
if (!MAX_ORDERS || processedOrders < MAX_ORDERS) { if (!MAX_ORDERS || processedOrders < MAX_ORDERS) {
await sleepWithStop(ORDER_DELAY_MS, 'next order'); await sleepWithStop(ORDER_DELAY_MS, 'next order');
@@ -555,6 +691,7 @@ async function runBackfill() {
currentDate, currentDate,
processedDays, processedDays,
processedOrders, processedOrders,
failedOrders,
lastMessage: `Completed ${completedDate}` lastMessage: `Completed ${completedDate}`
}); });
console.log(`[state] Completed ${completedDate}. Next date: ${currentDate}`); console.log(`[state] Completed ${completedDate}. Next date: ${currentDate}`);
@@ -567,9 +704,10 @@ async function runBackfill() {
currentDate, currentDate,
processedDays, processedDays,
processedOrders, processedOrders,
lastMessage: 'Backfill completed' failedOrders,
lastMessage: failedOrders ? `Backfill completed with ${failedOrders} failed order(s)` : 'Backfill completed'
}); });
console.log(`\nDone. Processed ${processedOrders} order(s) across ${processedDays} day(s).`); console.log(`\nDone. Processed ${processedOrders} order(s) across ${processedDays} day(s). Failed orders: ${failedOrders}.`);
} }
runBackfill().catch((error: any) => { runBackfill().catch((error: any) => {

View File

@@ -0,0 +1,387 @@
import axios from 'axios';
import crypto from 'crypto';
import dotenv from 'dotenv';
import fs from 'fs';
import path from 'path';
import { getOlistV3AccessToken, isOlistV3Configured } from '../services/olist-v3.service';
dotenv.config();
type RequestedProduct = { id: string };
type V3Product = { id: string; sku: string; descricao: string; unidade: string };
type V3StructureItem = {
produto?: { id?: number | string; sku?: string; descricao?: string };
quantidade?: number | string;
};
type V3FabricatedProduct = { produtos?: V3StructureItem[]; etapas?: string[] };
type StructurePayload = {
order: {
tinyId: string;
number: string;
status: string;
productSku: string;
productDescription: string;
quantity: string;
unit: string;
issueDate: null;
expectedDate: null;
supplier: string;
lotCode: string;
notes: string;
};
components: Array<{
componentTinyId: string;
componentName: string;
componentSku: string;
quantityPerUnit: string;
totalQuantity: string;
unit: string;
}>;
steps: never[];
};
type SyncState = {
status?: 'running' | 'stopped' | 'completed';
sourceSignature?: string;
totalProducts?: number;
nextIndex?: number;
processedProducts?: number;
skippedProducts?: number;
failedProducts?: number;
lastProductId?: string;
lastFailedProductId?: string;
lastMessage?: string;
updatedAt?: string;
};
const V3_API_BASE_URL = (process.env.OLIST_V3_API_BASE_URL || 'https://api.tiny.com.br/public-api/v3').replace(/\/+$/, '');
const DEFAULT_INPUT_FILE = path.join(process.cwd(), 'data', 'product-structure-products.csv');
const INPUT_FILE = process.env.PRODUCT_STRUCTURES_V3_INPUT_FILE || DEFAULT_INPUT_FILE;
const PRODUCT_IDS = String(process.env.PRODUCT_STRUCTURES_V3_PRODUCT_IDS || '')
.split(',')
.map(value => value.trim())
.filter(Boolean);
const DRY_RUN = process.env.PRODUCT_STRUCTURES_V3_DRY_RUN === 'true';
const RESUME = process.env.PRODUCT_STRUCTURES_V3_RESUME !== 'false';
const RUNTIME_DIRECTORY = path.join(process.cwd(), 'data-runtime');
const STATE_FILE = process.env.PRODUCT_STRUCTURES_V3_STATE_FILE || path.join(RUNTIME_DIRECTORY, 'product_structures_v3_sync_state.json');
const STOP_FILE = process.env.PRODUCT_STRUCTURES_V3_STOP_FILE || path.join(RUNTIME_DIRECTORY, 'product_structures_v3_sync_stop.json');
function deriveGraphsUrl() {
const raw = process.env.PRODUCT_STRUCTURES_V3_GRAPHS_API_URL
|| process.env.PRODUCT_STRUCTURES_GRAPHS_API_URL
|| process.env.PRODUCTION_ORDERS_GRAPHS_API_URL
|| process.env.GRAPHS_API_URL
|| '';
if (!raw) return 'http://localhost:3004/api/production-orders/tiny-sync';
try {
const parsed = new URL(raw);
return parsed.pathname.includes('/api/data')
? `${parsed.origin}/api/production-orders/tiny-sync`
: raw;
} catch {
return raw.replace(/\/api\/data.*$/, '/api/production-orders/tiny-sync');
}
}
const GRAPHS_URL = deriveGraphsUrl();
const GRAPHS_API_KEY = process.env.PRODUCT_STRUCTURES_V3_GRAPHS_API_KEY
|| process.env.PRODUCT_STRUCTURES_GRAPHS_API_KEY
|| process.env.PRODUCTION_ORDERS_GRAPHS_API_KEY
|| process.env.GRAPHS_API_KEY
|| process.env.NEXSTAR_GRAPHS_API_KEY
|| process.env.API_KEY
|| 'nexstar_secret_key_123';
function numberEnv(name: string, fallback: number) {
const value = Number(process.env[name]);
return Number.isFinite(value) && value >= 0 ? value : fallback;
}
const REQUEST_DELAY_MS = numberEnv('PRODUCT_STRUCTURES_V3_REQUEST_DELAY_MS', 1000);
const MAX_PRODUCTS = numberEnv('PRODUCT_STRUCTURES_V3_MAX_PRODUCTS', 0);
const GRAPHS_MAX_RETRIES = numberEnv('PRODUCT_STRUCTURES_V3_GRAPHS_MAX_RETRIES', 3);
const GRAPHS_RETRY_DELAY_MS = numberEnv('PRODUCT_STRUCTURES_V3_GRAPHS_RETRY_DELAY_MS', 30000);
let nextRequestAt = 0;
let latestState: SyncState | null = null;
const componentProductCache = new Map<string, V3Product | null>();
class SyncStoppedError extends Error {
constructor() {
super('V3 product structure sync stopped by request.');
}
}
const sleep = (ms: number) => new Promise(resolve => setTimeout(resolve, ms));
const text = (value: unknown) => String(value ?? '').trim();
function readProductsFromCsv(filePath: string): RequestedProduct[] {
const rows = fs.readFileSync(filePath, 'utf-8').replace(/^\uFEFF/, '').split(/\r?\n/).filter(Boolean);
const headers = rows.shift()?.split(',').map(header => header.trim().toLowerCase()) || [];
const idIndex = headers.findIndex(header => ['id', 'productid', 'tinyid', 'id produto'].includes(header));
if (idIndex < 0) throw new Error(`No product id column found in ${filePath}.`);
return rows
.map(row => row.split(',')[idIndex]?.trim())
.filter(Boolean)
.map(id => ({ id }));
}
function loadRequestedProducts() {
const fromFile = PRODUCT_IDS.length ? [] : readProductsFromCsv(INPUT_FILE);
const requested = fromFile.concat(PRODUCT_IDS.map(id => ({ id })));
return requested.filter((item, index, all) => all.findIndex(other => other.id === item.id) === index);
}
function saveState(state: SyncState) {
latestState = { ...state, updatedAt: new Date().toISOString() };
fs.mkdirSync(path.dirname(STATE_FILE), { recursive: true });
fs.writeFileSync(STATE_FILE, JSON.stringify(latestState, null, 2));
}
function readState() {
if (!fs.existsSync(STATE_FILE)) return null;
try {
return JSON.parse(fs.readFileSync(STATE_FILE, 'utf-8')) as SyncState;
} catch (error: any) {
console.warn(`[state] Could not read previous state: ${error.message}`);
return null;
}
}
function assertNotStopped() {
if (fs.existsSync(STOP_FILE)) throw new SyncStoppedError();
}
async function waitForSlot(reason: string) {
const waitMs = Math.max(0, nextRequestAt - Date.now());
if (waitMs > 0) console.log(`[wait] Waiting ${waitMs}ms before ${reason}.`);
let remaining = waitMs;
while (remaining > 0) {
assertNotStopped();
await sleep(Math.min(remaining, 1000));
remaining = Math.max(0, nextRequestAt - Date.now());
}
nextRequestAt = Date.now() + REQUEST_DELAY_MS;
}
function describeError(error: any) {
if (error?.response) {
const body = JSON.stringify(error.response.data ?? '');
return `HTTP ${error.response.status}: ${body.slice(0, 500)}`;
}
return error?.message || String(error);
}
async function v3Get<T>(pathname: string): Promise<T> {
await waitForSlot(`Olist V3 ${pathname}`);
const request = async (forceRefresh: boolean) => axios.get<T>(`${V3_API_BASE_URL}${pathname}`, {
headers: { Authorization: `Bearer ${await getOlistV3AccessToken(forceRefresh)}` },
timeout: 30000
});
try {
return (await request(false)).data;
} catch (error: any) {
if (error?.response?.status !== 401) throw error;
await waitForSlot(`Olist V3 ${pathname} retry after token refresh`);
return (await request(true)).data;
}
}
async function getProduct(id: string): Promise<V3Product> {
const raw = await v3Get<any>(`/produtos/${encodeURIComponent(id)}`);
return {
id: text(raw?.id || id),
sku: text(raw?.sku),
descricao: text(raw?.descricao),
unidade: text(raw?.unidade)
};
}
async function getComponentProduct(id: string) {
if (componentProductCache.has(id)) return componentProductCache.get(id) || null;
try {
const product = await getProduct(id);
componentProductCache.set(id, product);
return product;
} catch (error: any) {
console.warn(`[Olist V3] Could not get component ${id}: ${describeError(error)}`);
componentProductCache.set(id, null);
return null;
}
}
async function getFabricatedProduct(id: string): Promise<V3FabricatedProduct | null> {
try {
return await v3Get<V3FabricatedProduct>(`/produtos/${encodeURIComponent(id)}/fabricado`);
} catch (error: any) {
if ([400, 404].includes(error?.response?.status)) return null;
throw error;
}
}
function formatDecimal(value: unknown) {
const parsed = Number(value);
return Number.isFinite(parsed) ? String(parsed) : '0';
}
async function buildPayload(productId: string): Promise<StructurePayload | null> {
const product = await getProduct(productId);
const manufactured = await getFabricatedProduct(product.id);
const rawComponents = Array.isArray(manufactured?.produtos) ? manufactured.produtos : [];
if (!manufactured || !rawComponents.length) return null;
const components = [];
for (const item of rawComponents) {
const componentId = text(item.produto?.id);
const componentDetail = componentId ? await getComponentProduct(componentId) : null;
const quantity = formatDecimal(item.quantidade);
components.push({
componentTinyId: componentId,
componentName: text(item.produto?.descricao || componentDetail?.descricao),
componentSku: text(item.produto?.sku || componentDetail?.sku),
quantityPerUnit: quantity,
totalQuantity: quantity,
unit: text(componentDetail?.unidade)
});
}
return {
order: {
tinyId: `STRUCTURE-V3-${product.id}`,
number: `STRUCTURE-V3-${product.sku || product.id}`,
status: 'completed',
productSku: product.sku,
productDescription: product.descricao,
quantity: '1',
unit: product.unidade || 'UN',
issueDate: null,
expectedDate: null,
supplier: '',
lotCode: '',
notes: `Composição sincronizada da API pública Olist V3 para produto ${product.id}.`
},
components: components.filter(component => component.componentSku || component.componentName),
steps: []
};
}
async function sendToGraphs(payload: StructurePayload) {
if (DRY_RUN) {
console.log(`[dry-run] ${JSON.stringify(payload)}`);
return true;
}
for (let attempt = 1; ; attempt += 1) {
try {
await axios.post(GRAPHS_URL, payload, {
headers: { 'Content-Type': 'application/json', 'x-api-key': GRAPHS_API_KEY },
timeout: 30000
});
console.log(`[Graphs V3] Sent ${payload.order.productSku} with ${payload.components.length} component(s).`);
return true;
} catch (error: any) {
const retryable = !error?.response?.status || error.response.status >= 500;
console.error(`[Graphs V3] Send failed for ${payload.order.productSku}, attempt ${attempt}: ${describeError(error)}`);
if (!retryable || (GRAPHS_MAX_RETRIES && attempt >= GRAPHS_MAX_RETRIES)) return false;
await sleep(GRAPHS_RETRY_DELAY_MS);
}
}
}
async function run() {
if (!isOlistV3Configured()) {
throw new Error('Olist V3 is not configured. Set OLIST_V3_CLIENT_ID, OLIST_V3_CLIENT_SECRET, and OLIST_V3_REDIRECT_URI.');
}
const products = loadRequestedProducts();
if (!products.length) throw new Error('No V3 products configured.');
await getOlistV3AccessToken();
const sourceSignature = crypto.createHash('sha256').update(products.map(product => product.id).join('\n')).digest('hex');
const previous = readState();
const canResume = RESUME
&& !PRODUCT_IDS.length
&& previous?.status !== 'completed'
&& previous?.sourceSignature === sourceSignature
&& Number.isInteger(previous?.nextIndex)
&& (previous?.nextIndex || 0) > 0
&& (previous?.nextIndex || 0) < products.length;
let nextIndex = canResume ? previous!.nextIndex! : 0;
let processedProducts = canResume ? previous?.processedProducts || 0 : 0;
let skippedProducts = canResume ? previous?.skippedProducts || 0 : 0;
let failedProducts = canResume ? previous?.failedProducts || 0 : 0;
let handledThisRun = 0;
let reachedLimit = false;
console.log(`[config] Olist V3 product structures: ${products.length} product(s), ${REQUEST_DELAY_MS}ms request pace.`);
if (canResume) console.log(`[state] Resuming at ${nextIndex + 1}/${products.length}.`);
const checkpoint = (state: Partial<SyncState>) => saveState({
status: 'running',
sourceSignature,
totalProducts: products.length,
nextIndex,
processedProducts,
skippedProducts,
failedProducts,
...state
});
checkpoint({ lastMessage: canResume ? 'V3 structure sync resumed' : 'V3 structure sync started' });
for (let index = nextIndex; index < products.length; index += 1) {
assertNotStopped();
if (MAX_PRODUCTS && handledThisRun >= MAX_PRODUCTS) {
reachedLimit = true;
break;
}
const productId = products[index].id;
nextIndex = index + 1;
try {
const payload = await buildPayload(productId);
if (!payload?.components.length) {
skippedProducts += 1;
console.log(`[Olist V3] No manufactured-product composition for ${productId}; skipping Graphs send.`);
checkpoint({ lastProductId: productId, lastMessage: `Skipped ${productId}; no composition` });
} else if (await sendToGraphs(payload)) {
processedProducts += 1;
checkpoint({ lastProductId: productId, lastMessage: `Synced ${payload.order.productSku}` });
} else {
failedProducts += 1;
checkpoint({ lastFailedProductId: productId, lastMessage: `Graphs send failed for ${productId}` });
}
} catch (error: any) {
if (error instanceof SyncStoppedError) throw error;
failedProducts += 1;
console.error(`[sync] Failed product ${productId}: ${describeError(error)}`);
checkpoint({ lastFailedProductId: productId, lastMessage: `Failed ${productId}; continuing` });
}
handledThisRun += 1;
}
saveState({
status: reachedLimit ? 'stopped' : 'completed',
sourceSignature,
totalProducts: products.length,
nextIndex,
processedProducts,
skippedProducts,
failedProducts,
lastMessage: reachedLimit ? `V3 structure sync paused after ${handledThisRun} product(s)` : 'V3 structure sync completed'
});
console.log(`Done. V3 synced ${processedProducts}; skipped ${skippedProducts}; failed ${failedProducts}.`);
}
run().catch((error: any) => {
if (error instanceof SyncStoppedError) {
saveState({ ...latestState, status: 'stopped', lastMessage: 'V3 structure sync stopped by request' });
console.log('V3 product structure sync stopped by request.');
process.exit(0);
}
console.error(`[sync] V3 product structure sync failed: ${describeError(error)}`);
saveState({ ...latestState, status: 'stopped', lastMessage: describeError(error) });
process.exit(1);
});

View File

@@ -0,0 +1,659 @@
import axios from 'axios';
import crypto from 'crypto';
import dotenv from 'dotenv';
import fs from 'fs';
import path from 'path';
dotenv.config();
type TinyProduct = {
id: string;
codigo: string;
nome: string;
unidade?: string;
};
type TinyStructureComponent = {
id_componente?: string | number;
codigo?: string | number;
nome?: string;
quantidade?: string | number;
};
type ProductStructurePayload = {
order: {
tinyId: string;
number: string;
status: string;
productSku: string;
productDescription: string;
quantity: string;
unit: string;
issueDate: string | null;
expectedDate: string | null;
supplier: string;
lotCode: string;
notes: string;
};
components: Array<{
componentTinyId: string;
componentName: string;
componentSku: string;
quantityPerUnit: string;
totalQuantity: string;
unit: string;
}>;
steps: never[];
};
type SyncState = {
status?: 'running' | 'stopped' | 'completed';
sourceSignature?: string;
totalProducts?: number;
nextIndex?: number;
processedProducts?: number;
skippedProducts?: number;
failedProducts?: number;
lastSku?: string;
lastFailedSku?: string;
lastMessage?: string;
updatedAt?: string;
};
type RequestedProduct = {
id: string;
sku: string;
};
function deriveProductionOrdersUrl() {
const raw = process.env.GRAPHS_API_URL || process.env.NEXSTAR_GRAPHS_API_URL || '';
if (!raw) return '';
try {
const parsed = new URL(raw);
return `${parsed.origin}/api/production-orders/tiny-sync`;
} catch {
return raw.replace(/\/api\/data.*$/, '/api/production-orders/tiny-sync');
}
}
const TINY_API_TOKEN = process.env.TINY_API_TOKEN;
const GRAPHS_PRODUCTION_ORDERS_URL = (
process.env.PRODUCT_STRUCTURES_GRAPHS_API_URL ||
process.env.PRODUCTION_ORDERS_GRAPHS_API_URL ||
process.env.GRAPHS_PRODUCTION_ORDERS_API_URL ||
(process.env.GRAPHS_API_BASE_URL ? `${process.env.GRAPHS_API_BASE_URL.replace(/\/+$/, '')}/api/production-orders/tiny-sync` : '') ||
deriveProductionOrdersUrl() ||
'http://localhost:3004/api/production-orders/tiny-sync'
);
const GRAPHS_PRODUCTION_ORDERS_API_KEY = (
process.env.PRODUCT_STRUCTURES_GRAPHS_API_KEY ||
process.env.PRODUCTION_ORDERS_GRAPHS_API_KEY ||
process.env.GRAPHS_PRODUCTION_ORDERS_API_KEY ||
process.env.PRODUCTION_ORDERS_API_KEY ||
process.env.GRAPHS_API_KEY ||
process.env.NEXSTAR_GRAPHS_API_KEY ||
process.env.API_KEY ||
'nexstar_secret_key_123'
);
const SKUS = String(process.env.PRODUCT_STRUCTURE_SKUS || '')
.split(',')
.map(value => value.trim())
.filter(Boolean);
const PRODUCT_IDS = String(process.env.PRODUCT_STRUCTURE_PRODUCT_IDS || '')
.split(',')
.map(value => value.trim())
.filter(Boolean);
const DEFAULT_INPUT_FILE = path.join(process.cwd(), 'data', 'product-structure-products.csv');
const EXPLICIT_INPUT_FILE = process.env.PRODUCT_STRUCTURES_INPUT_FILE || '';
const SHOULD_USE_DEFAULT_INPUT_FILE = !EXPLICIT_INPUT_FILE && !SKUS.length && !PRODUCT_IDS.length && fs.existsSync(DEFAULT_INPUT_FILE);
const INPUT_FILE = EXPLICIT_INPUT_FILE || (SHOULD_USE_DEFAULT_INPUT_FILE ? DEFAULT_INPUT_FILE : '');
const DRY_RUN = process.env.PRODUCT_STRUCTURES_DRY_RUN === 'true';
const FETCH_COMPONENT_UNITS = process.env.PRODUCT_STRUCTURES_FETCH_COMPONENT_UNITS !== 'false';
const STATE_FILE = process.env.PRODUCT_STRUCTURES_STATE_FILE || path.join(process.cwd(), 'product_structures_sync_state.json');
const STOP_FILE = process.env.PRODUCT_STRUCTURES_STOP_FILE || path.join(process.cwd(), 'product_structures_sync_stop.json');
const numberEnv = (name: string, fallback: number) => {
const raw = process.env[name];
if (raw === undefined || raw.trim() === '') return fallback;
const value = Number(raw);
return Number.isFinite(value) && value >= 0 ? value : fallback;
};
const REQUEST_DELAY_MS = numberEnv('PRODUCT_STRUCTURES_REQUEST_DELAY_MS', 2500);
const GRAPHS_RETRY_DELAY_MS = numberEnv('PRODUCT_STRUCTURES_GRAPHS_RETRY_DELAY_MS', 30000);
const GRAPHS_MAX_RETRIES = numberEnv('PRODUCT_STRUCTURES_GRAPHS_MAX_RETRIES', 3);
const MAX_PRODUCTS = numberEnv('PRODUCT_STRUCTURES_MAX_PRODUCTS', 0);
const RESUME = process.env.PRODUCT_STRUCTURES_RESUME !== 'false';
let nextRequestAt = 0;
const productDetailCache = new Map<string, TinyProduct | null>();
let latestState: SyncState | null = null;
class SyncStoppedError extends Error {
constructor() {
super('Product structures sync stopped by request.');
}
}
const sleep = (ms: number) => new Promise(resolve => setTimeout(resolve, ms));
function assertNotStopped() {
if (fs.existsSync(STOP_FILE)) {
throw new SyncStoppedError();
}
}
async function sleepWithStop(ms: number, reason: string) {
if (ms <= 0) return;
const startedAt = Date.now();
let remaining = ms;
console.log(`[wait] Waiting ${ms}ms before ${reason}.`);
while (remaining > 0) {
assertNotStopped();
await sleep(Math.min(remaining, 1000));
remaining = ms - (Date.now() - startedAt);
}
}
async function waitForRequestSlot(reason: string) {
const waitMs = Math.max(0, nextRequestAt - Date.now());
await sleepWithStop(waitMs, reason);
nextRequestAt = Date.now() + REQUEST_DELAY_MS;
}
function saveState(state: SyncState) {
latestState = {
...state,
updatedAt: new Date().toISOString()
};
fs.writeFileSync(STATE_FILE, JSON.stringify(latestState, null, 2));
}
function readState(): SyncState | null {
if (!fs.existsSync(STATE_FILE)) return null;
try {
const parsed = JSON.parse(fs.readFileSync(STATE_FILE, 'utf-8'));
return parsed && typeof parsed === 'object' ? parsed : null;
} catch (error: any) {
console.warn(`[state] Could not read previous state; starting from the beginning: ${error.message}`);
return null;
}
}
function describeRequestError(error: any) {
if (error?.response) {
const body = JSON.stringify(error.response.data ?? '');
const truncatedBody = body.length > 500 ? `${body.slice(0, 500)}...` : body;
return `HTTP ${error.response.status}: ${truncatedBody}`;
}
return error?.message || String(error);
}
function normalizeText(value: unknown) {
return String(value ?? '').trim();
}
function normalizeNumber(value: unknown) {
if (typeof value === 'number') return Number.isFinite(value) ? value : 0;
const raw = normalizeText(value);
if (!raw) return 0;
const normalized = raw.includes(',')
? raw.replace(/\./g, '').replace(',', '.')
: raw;
const parsed = Number(normalized);
return Number.isFinite(parsed) ? parsed : 0;
}
function formatTinyDecimal(value: number) {
return Number.isFinite(value) ? String(value) : '0';
}
function parseDelimitedLine(line: string, delimiter: string) {
const values: string[] = [];
let current = '';
let quoted = false;
for (let index = 0; index < line.length; index += 1) {
const char = line[index];
const nextChar = line[index + 1];
if (char === '"' && quoted && nextChar === '"') {
current += '"';
index += 1;
continue;
}
if (char === '"') {
quoted = !quoted;
continue;
}
if (char === delimiter && !quoted) {
values.push(current);
current = '';
continue;
}
current += char;
}
values.push(current);
return values;
}
function parseDelimitedFile(raw: string, delimiter: string) {
const lines = raw
.replace(/^\uFEFF/, '')
.split(/\r?\n/)
.filter(line => line.trim() !== '');
const headers = parseDelimitedLine(lines[0] || '', delimiter).map(header => header.trim());
return lines.slice(1).map(line => {
const values = parseDelimitedLine(line, delimiter);
return headers.reduce<Record<string, string>>((row, header, index) => {
row[header] = values[index] || '';
return row;
}, {});
});
}
function readInputFile() {
const filePath = path.resolve(INPUT_FILE);
const raw = fs.readFileSync(filePath, 'utf-8');
const extension = path.extname(filePath).toLowerCase();
if (extension === '.csv') {
return parseDelimitedFile(raw, ',');
}
if (extension === '.tsv') {
return parseDelimitedFile(raw, '\t');
}
const parsed = JSON.parse(raw);
if (Array.isArray(parsed)) return parsed;
return parsed?.skus || parsed?.products || parsed?.items || [];
}
function loadRequestedProducts() {
const productsFromFile: RequestedProduct[] = INPUT_FILE
? readInputFile().map((entry: any) => ({
id: normalizeText(entry.id || entry.tinyId || entry.productId || entry['ID Produto'] || entry.ID_Produto),
sku: normalizeText(entry.sku || entry.codigo || entry.productSku || entry.SKU || entry.Código || entry.Codigo)
}))
: [];
return productsFromFile
.concat(SKUS.map(sku => ({ id: '', sku })))
.concat(PRODUCT_IDS.map(id => ({ id, sku: '' })))
.filter((entry: RequestedProduct, index: number, all: RequestedProduct[]) => {
const key = entry.id ? `id:${entry.id}` : `sku:${entry.sku.toUpperCase()}`;
return (entry.id || entry.sku) && all.findIndex((other: RequestedProduct) => {
const otherKey = other.id ? `id:${other.id}` : `sku:${other.sku.toUpperCase()}`;
return otherKey === key;
}) === index;
});
}
async function tinyPost(servicePhp: string, params: URLSearchParams) {
if (!TINY_API_TOKEN) {
throw new Error('Missing TINY_API_TOKEN.');
}
await waitForRequestSlot(`Tiny ${servicePhp} request slot`);
return axios.post(`https://api.tiny.com.br/api2/${servicePhp}`, params, {
headers: { 'Content-Type': 'application/x-www-form-urlencoded' }
});
}
async function searchProductBySku(sku: string): Promise<TinyProduct | null> {
const params = new URLSearchParams();
params.append('token', TINY_API_TOKEN!);
params.append('formato', 'JSON');
params.append('pesquisa', sku);
params.append('pagina', '1');
const response = await tinyPost('produtos.pesquisa.php', params);
const retorno = response.data?.retorno;
if (retorno?.status !== 'OK') {
console.warn(`[Tiny] Product search failed for ${sku}: ${JSON.stringify(retorno?.erros || retorno || 'Unknown error')}`);
return null;
}
const products = (retorno.produtos || [])
.map((entry: any) => entry?.produto || entry)
.filter(Boolean);
const exact = products.find((product: any) => normalizeText(product.codigo).toUpperCase() === sku.toUpperCase());
const selected = exact || products[0];
if (!selected) return null;
return {
id: normalizeText(selected.id),
codigo: normalizeText(selected.codigo),
nome: normalizeText(selected.nome),
unidade: normalizeText(selected.unidade)
};
}
async function getProductById(id: string): Promise<TinyProduct | null> {
if (productDetailCache.has(id)) return productDetailCache.get(id) || null;
const params = new URLSearchParams();
params.append('token', TINY_API_TOKEN!);
params.append('formato', 'JSON');
params.append('id', id);
const response = await tinyPost('produto.obter.php', params);
const produto = response.data?.retorno?.produto;
if (!produto) {
console.warn(`[Tiny] Product get failed for ${id}: ${JSON.stringify(response.data?.retorno?.erros || response.data?.retorno || 'Unknown error')}`);
productDetailCache.set(id, null);
return null;
}
const detail = {
id: normalizeText(produto.id || id),
codigo: normalizeText(produto.codigo),
nome: normalizeText(produto.nome),
unidade: normalizeText(produto.unidade)
};
productDetailCache.set(id, detail);
return detail;
}
async function getProductStructure(product: TinyProduct): Promise<ProductStructurePayload | null> {
const params = new URLSearchParams();
params.append('token', TINY_API_TOKEN!);
params.append('formato', 'JSON');
params.append('id', product.id);
const response = await tinyPost('produto.obter.estrutura.php', params);
const retorno = response.data?.retorno;
if (retorno?.status !== 'OK') {
console.warn(`[Tiny] Product structure failed for ${product.codigo || product.id}: ${JSON.stringify(retorno?.erros || retorno || 'Unknown error')}`);
return null;
}
const produto = retorno.produto || {};
const finishedSku = normalizeText(produto.codigo || product.codigo);
const finishedName = normalizeText(produto.nome || product.nome);
const rawComponents = (Array.isArray(produto.estrutura) ? produto.estrutura : [])
.map((entry: any) => entry?.item || entry)
.filter(Boolean);
const components = [];
for (const component of rawComponents as TinyStructureComponent[]) {
const quantityPerUnit = normalizeNumber(component.quantidade);
const componentTinyId = normalizeText(component.id_componente);
const componentDetail = FETCH_COMPONENT_UNITS && componentTinyId
? await getProductById(componentTinyId)
: null;
components.push({
componentTinyId,
componentName: normalizeText(component.nome || componentDetail?.nome),
componentSku: normalizeText(component.codigo || componentDetail?.codigo),
quantityPerUnit: formatTinyDecimal(quantityPerUnit),
totalQuantity: formatTinyDecimal(quantityPerUnit),
unit: normalizeText(componentDetail?.unidade)
});
}
const filteredComponents = components
.filter((component: ProductStructurePayload['components'][number]) => component.componentSku || component.componentName);
return {
order: {
tinyId: `STRUCTURE-${product.id}`,
number: `STRUCTURE-${finishedSku || product.id}`,
status: 'completed',
productSku: finishedSku,
productDescription: finishedName,
quantity: '1',
unit: product.unidade || 'UN',
issueDate: null,
expectedDate: null,
supplier: '',
lotCode: '',
notes: `Composição sincronizada do Tiny produto.obter.estrutura.php para produto ${product.id}.`
},
components: filteredComponents,
steps: []
};
}
async function resolveProduct(requested: RequestedProduct) {
if (requested.id) return getProductById(requested.id);
return searchProductBySku(requested.sku);
}
async function sendStructure(payload: ProductStructurePayload) {
if (DRY_RUN) {
console.log(`[dry-run] Would send structure ${payload.order.productSku}`);
console.log(JSON.stringify(payload, null, 2));
return true;
}
let attempt = 0;
while (true) {
try {
await waitForRequestSlot(`Graphs product structure ${payload.order.productSku}`);
await axios.post(GRAPHS_PRODUCTION_ORDERS_URL, payload, {
headers: {
'Content-Type': 'application/json',
'x-api-key': GRAPHS_PRODUCTION_ORDERS_API_KEY
}
});
console.log(`[Graphs Structure] Sent ${payload.order.productSku} with ${payload.components.length} component(s).`);
return true;
} catch (error: any) {
attempt += 1;
const status = error?.response?.status;
const shouldRetry = !status || status >= 500;
console.error(`[Graphs Structure] Failed to send ${payload.order.productSku} on attempt ${attempt}: ${describeRequestError(error)}`);
if (!shouldRetry || (GRAPHS_MAX_RETRIES && attempt >= GRAPHS_MAX_RETRIES)) {
console.error(`[Graphs Structure] Giving up on ${payload.order.productSku}; sync will continue.`);
return false;
}
await sleepWithStop(GRAPHS_RETRY_DELAY_MS, `retrying Graphs structure ${payload.order.productSku}`);
}
}
}
async function runSync() {
const requestedProducts = loadRequestedProducts();
if (!requestedProducts.length) {
throw new Error('No products configured. Set PRODUCT_STRUCTURE_SKUS, PRODUCT_STRUCTURE_PRODUCT_IDS, or PRODUCT_STRUCTURES_INPUT_FILE.');
}
console.log(`[config] Graphs URL: ${GRAPHS_PRODUCTION_ORDERS_URL}`);
console.log(`[config] Dry run: ${DRY_RUN ? 'yes' : 'no'}`);
console.log(`[config] Request pace: 1 request every ${REQUEST_DELAY_MS}ms`);
console.log(`[config] Resume: ${RESUME ? 'yes' : 'no'}`);
console.log(`[config] State file: ${STATE_FILE}`);
console.log(`[config] Stop file: ${STOP_FILE}`);
console.log(`[source] Found ${requestedProducts.length} requested product(s).`);
const sourceSignature = crypto
.createHash('sha256')
.update(requestedProducts.map(product => `${product.id}|${product.sku}`).join('\n'))
.digest('hex');
const previousState = readState();
const canResume = RESUME
&& !SKUS.length
&& !PRODUCT_IDS.length
&& previousState?.status !== 'completed'
&& previousState?.sourceSignature === sourceSignature
&& Number.isInteger(previousState?.nextIndex)
&& (previousState?.nextIndex || 0) > 0
&& (previousState?.nextIndex || 0) < requestedProducts.length;
const startIndex = canResume ? previousState!.nextIndex! : 0;
let processedProducts = canResume ? previousState?.processedProducts || 0 : 0;
let skippedProducts = canResume ? previousState?.skippedProducts || 0 : 0;
let failedProducts = canResume ? previousState?.failedProducts || 0 : 0;
let handledThisRun = 0;
let nextIndex = startIndex;
let reachedLimit = false;
if (canResume) {
console.log(`[state] Resuming at ${startIndex + 1}/${requestedProducts.length} after ${previousState?.lastSku || 'the last checkpoint'}.`);
}
saveState({
status: 'running',
sourceSignature,
totalProducts: requestedProducts.length,
nextIndex: startIndex,
processedProducts,
skippedProducts,
failedProducts,
lastSku: canResume ? previousState?.lastSku : undefined,
lastMessage: canResume ? 'Product structures sync resumed' : 'Product structures sync started'
});
for (let index = startIndex; index < requestedProducts.length; index += 1) {
assertNotStopped();
if (MAX_PRODUCTS && handledThisRun >= MAX_PRODUCTS) {
console.log(`[limit] Stopped after ${MAX_PRODUCTS} product(s).`);
reachedLimit = true;
break;
}
nextIndex = index + 1;
const requested = requestedProducts[index];
const label = requested.sku || requested.id;
try {
const product = await resolveProduct(requested);
if (!product?.id) {
console.warn(`[Tiny] Product not found for ${label}.`);
failedProducts += 1;
saveState({
status: 'running',
sourceSignature,
totalProducts: requestedProducts.length,
nextIndex,
processedProducts,
skippedProducts,
failedProducts,
lastFailedSku: label,
lastMessage: `Product not found for ${label}`
});
handledThisRun += 1;
continue;
}
const payload = await getProductStructure(product);
if (!payload) {
failedProducts += 1;
saveState({
status: 'running',
sourceSignature,
totalProducts: requestedProducts.length,
nextIndex,
processedProducts,
skippedProducts,
failedProducts,
lastFailedSku: product.codigo || product.id,
lastMessage: `Could not get structure for ${product.codigo || product.id}`
});
handledThisRun += 1;
continue;
}
if (!payload.components.length) {
skippedProducts += 1;
console.log(`[Tiny] No structure components for ${payload.order.productSku}; skipping Graphs send.`);
saveState({
status: 'running',
sourceSignature,
totalProducts: requestedProducts.length,
nextIndex,
processedProducts,
skippedProducts,
failedProducts,
lastSku: payload.order.productSku,
lastMessage: `Skipped ${payload.order.productSku}; no structure components`
});
handledThisRun += 1;
continue;
}
const sent = await sendStructure(payload);
if (sent) {
processedProducts += 1;
} else {
failedProducts += 1;
}
saveState({
status: 'running',
sourceSignature,
totalProducts: requestedProducts.length,
nextIndex,
processedProducts,
skippedProducts,
failedProducts,
lastSku: sent ? payload.order.productSku : undefined,
lastFailedSku: sent ? undefined : payload.order.productSku,
lastMessage: sent ? `Synced structure ${payload.order.productSku}` : `Failed structure ${payload.order.productSku}; continuing`
});
handledThisRun += 1;
} catch (error: any) {
if (error instanceof SyncStoppedError) throw error;
failedProducts += 1;
console.error(`[sync] Failed product ${label}: ${describeRequestError(error)}`);
saveState({
status: 'running',
sourceSignature,
totalProducts: requestedProducts.length,
nextIndex,
processedProducts,
skippedProducts,
failedProducts,
lastFailedSku: label,
lastMessage: `Failed product ${label}; continuing`
});
handledThisRun += 1;
}
}
saveState({
status: reachedLimit ? 'stopped' : 'completed',
sourceSignature,
totalProducts: requestedProducts.length,
nextIndex,
processedProducts,
skippedProducts,
failedProducts,
lastMessage: reachedLimit
? `Product structures sync paused after ${handledThisRun} product(s)`
: (failedProducts ? `Product structures sync completed with ${failedProducts} failed product(s)` : 'Product structures sync completed')
});
console.log(`${reachedLimit ? 'Paused' : 'Done'}. Synced ${processedProducts} product structure(s). Skipped products: ${skippedProducts}. Failed products: ${failedProducts}.`);
}
runSync().catch((error: any) => {
if (error instanceof SyncStoppedError) {
console.log('Product structures sync stopped by request.');
saveState({
...latestState,
status: 'stopped',
lastMessage: 'Product structures sync stopped by request'
});
process.exit(0);
}
console.error(`Product structures sync failed: ${describeRequestError(error)}`);
saveState({
status: 'stopped',
lastMessage: `Product structures sync failed: ${describeRequestError(error)}`
});
process.exit(1);
});

View File

@@ -0,0 +1,574 @@
import axios from 'axios';
import dotenv from 'dotenv';
import fs from 'fs';
import path from 'path';
dotenv.config();
type ProductionOrderPayload = {
order: {
tinyId: string;
number: string;
status: string;
productSku: string;
productDescription: string;
quantity: string;
unit: string;
issueDate: string | null;
expectedDate: string | null;
supplier: string;
lotCode: string;
notes: string;
};
components: Array<{
componentName: string;
componentSku: string;
quantityPerUnit: string;
totalQuantity: string;
unit: string;
}>;
steps: Array<{
stepNumber: number;
name: string;
startDate: string | null;
endDate: string | null;
status: string;
}>;
};
type XajaxCommand = {
cmd?: string;
elm?: string;
prop?: string;
val?: string;
src?: string;
fn?: string;
args?: any[];
};
type SyncState = {
status?: 'running' | 'stopped' | 'completed';
processedOrders?: number;
failedOrders?: number;
lastTinyId?: string;
lastFailedTinyId?: string;
lastMessage?: string;
updatedAt?: string;
};
function deriveProductionOrdersUrl() {
const raw = process.env.GRAPHS_API_URL || process.env.NEXSTAR_GRAPHS_API_URL || '';
if (!raw) return '';
try {
const parsed = new URL(raw);
return `${parsed.origin}/api/production-orders/tiny-sync`;
} catch {
return raw.replace(/\/api\/data.*$/, '/api/production-orders/tiny-sync');
}
}
const TINY_ERP_BASE_URL = (process.env.PRODUCTION_ORDERS_TINY_ERP_BASE_URL || 'https://erp.olist.com').replace(/\/+$/, '');
const TINY_ERP_COOKIE = process.env.PRODUCTION_ORDERS_TINY_ERP_COOKIE || '';
const TINY_ERP_AUTHORIZATION = process.env.PRODUCTION_ORDERS_TINY_ERP_AUTHORIZATION || '';
const DETAIL_URL = process.env.PRODUCTION_ORDERS_DETAIL_URL || `${TINY_ERP_BASE_URL}/services/ordem.producao.server/1/obterOrdemProducao`;
const PRODUCT_DATA_URL = process.env.PRODUCTION_ORDERS_PRODUCT_DATA_URL || `${TINY_ERP_BASE_URL}/services/ordem.producao.server/1/buscarDadosProduto`;
const LIST_URL = process.env.PRODUCTION_ORDERS_LIST_URL || `${TINY_ERP_BASE_URL}/services/ordem.producao.server/1/listarOrdensProducao`;
const DETAIL_BODY_TEMPLATE = process.env.PRODUCTION_ORDERS_DETAIL_BODY_TEMPLATE || 'id={{id}}';
const PRODUCT_DATA_BODY_TEMPLATE = process.env.PRODUCTION_ORDERS_PRODUCT_DATA_BODY_TEMPLATE || 'idProduto={{productId}}&quantidade={{quantity}}&gerarOP=N';
const LIST_BODY = process.env.PRODUCTION_ORDERS_LIST_BODY || '';
const GRAPHS_PRODUCTION_ORDERS_URL = (
process.env.PRODUCTION_ORDERS_GRAPHS_API_URL ||
process.env.GRAPHS_PRODUCTION_ORDERS_API_URL ||
(process.env.GRAPHS_API_BASE_URL ? `${process.env.GRAPHS_API_BASE_URL.replace(/\/+$/, '')}/api/production-orders/tiny-sync` : '') ||
deriveProductionOrdersUrl() ||
'http://localhost:3004/api/production-orders/tiny-sync'
);
const GRAPHS_PRODUCTION_ORDERS_API_KEY = (
process.env.PRODUCTION_ORDERS_GRAPHS_API_KEY ||
process.env.GRAPHS_PRODUCTION_ORDERS_API_KEY ||
process.env.PRODUCTION_ORDERS_API_KEY ||
process.env.GRAPHS_API_KEY ||
process.env.NEXSTAR_GRAPHS_API_KEY ||
process.env.API_KEY ||
'nexstar_secret_key_123'
);
const TINY_API_TOKEN = process.env.TINY_API_TOKEN || '';
const ORDER_IDS = String(process.env.PRODUCTION_ORDER_IDS || '')
.split(',')
.map(id => id.trim())
.filter(Boolean);
const DRY_RUN = process.env.PRODUCTION_ORDERS_DRY_RUN === 'true';
const STATE_FILE = process.env.PRODUCTION_ORDERS_STATE_FILE || path.join(process.cwd(), 'production_orders_sync_state.json');
const STOP_FILE = process.env.PRODUCTION_ORDERS_STOP_FILE || path.join(process.cwd(), 'production_orders_sync_stop.json');
const numberEnv = (name: string, fallback: number) => {
const raw = process.env[name];
if (raw === undefined || raw.trim() === '') return fallback;
const value = Number(raw);
return Number.isFinite(value) && value >= 0 ? value : fallback;
};
const REQUEST_DELAY_MS = numberEnv('PRODUCTION_ORDERS_REQUEST_DELAY_MS', 2500);
const GRAPHS_RETRY_DELAY_MS = numberEnv('PRODUCTION_ORDERS_GRAPHS_RETRY_DELAY_MS', 30000);
const GRAPHS_MAX_RETRIES = numberEnv('PRODUCTION_ORDERS_GRAPHS_MAX_RETRIES', 3);
const MAX_ORDERS = numberEnv('PRODUCTION_ORDERS_MAX_ORDERS', 0);
let nextRequestAt = 0;
const productCache = new Map<string, { sku: string; description: string; unit: string } | null>();
class SyncStoppedError extends Error {
constructor() {
super('Production orders sync stopped by request.');
}
}
const sleep = (ms: number) => new Promise(resolve => setTimeout(resolve, ms));
function assertNotStopped() {
if (fs.existsSync(STOP_FILE)) {
throw new SyncStoppedError();
}
}
async function sleepWithStop(ms: number, reason: string) {
if (ms <= 0) return;
const startedAt = Date.now();
let remaining = ms;
console.log(`[wait] Waiting ${ms}ms before ${reason}.`);
while (remaining > 0) {
assertNotStopped();
await sleep(Math.min(remaining, 1000));
remaining = ms - (Date.now() - startedAt);
}
}
async function waitForRequestSlot(reason: string) {
const waitMs = Math.max(0, nextRequestAt - Date.now());
await sleepWithStop(waitMs, reason);
nextRequestAt = Date.now() + REQUEST_DELAY_MS;
}
function saveState(state: SyncState) {
fs.writeFileSync(STATE_FILE, JSON.stringify({
...state,
updatedAt: new Date().toISOString()
}, null, 2));
}
function describeRequestError(error: any) {
if (error?.response) {
const body = JSON.stringify(error.response.data ?? '');
const truncatedBody = body.length > 500 ? `${body.slice(0, 500)}...` : body;
return `HTTP ${error.response.status}: ${truncatedBody}`;
}
return error?.message || String(error);
}
function normalizeText(value: unknown) {
return String(value ?? '').trim();
}
function pickFirst(...values: unknown[]) {
for (const value of values) {
const normalized = normalizeText(value);
if (normalized) return normalized;
}
return '';
}
function pickFirstNullable(...values: unknown[]) {
const value = pickFirst(...values);
return value || null;
}
function fillTemplate(template: string, values: Record<string, string>) {
return template.replace(/\{\{(\w+)\}\}/g, (_, key) => encodeURIComponent(values[key] || ''));
}
function parseResponse(data: any) {
if (typeof data === 'object' && data !== null) return data;
const raw = String(data ?? '').trim();
if (!raw) return {};
try {
return JSON.parse(raw);
} catch {
const jsonStart = raw.indexOf('{');
const jsonEnd = raw.lastIndexOf('}');
if (jsonStart >= 0 && jsonEnd > jsonStart) {
return JSON.parse(raw.slice(jsonStart, jsonEnd + 1));
}
}
throw new Error('Could not parse Tiny/Olist internal response as JSON.');
}
function getCommands(data: any): XajaxCommand[] {
const parsed = parseResponse(data);
return Array.isArray(parsed?.response) ? parsed.response : [];
}
function getAssignedValue(commands: XajaxCommand[], element: string) {
const command = commands.find(entry => entry.cmd === 'as' && entry.elm === element);
return normalizeText(command?.val);
}
function getScript(commands: XajaxCommand[], pattern: RegExp) {
return commands.find(entry => entry.cmd === 'sc' && pattern.test(String(entry.src || '')))?.src || '';
}
function decodeHtml(value: string) {
return value
.replace(/&nbsp;/g, ' ')
.replace(/&amp;/g, '&')
.replace(/&lt;/g, '<')
.replace(/&gt;/g, '>')
.replace(/&quot;/g, '"')
.replace(/&#39;/g, "'");
}
function stripHtml(value: string) {
return decodeHtml(value.replace(/<[^>]+>/g, ' ')).replace(/\s+/g, ' ').trim();
}
function parseQuantityAndUnit(value: string) {
const normalized = stripHtml(value);
const match = normalized.match(/^(.+?)\s+([A-Za-z]{1,5})$/);
if (!match) return { quantity: normalized, unit: '' };
return { quantity: match[1].trim(), unit: match[2].trim() };
}
function parseCompositionHtml(html: string): ProductionOrderPayload['components'] {
const rows = html.match(/<tr\b[^>]*>[\s\S]*?<\/tr>/gi) || [];
return rows.flatMap(row => {
const cells = row.match(/<td\b[^>]*>[\s\S]*?<\/td>/gi) || [];
if (cells.length < 4) return [];
const componentName = stripHtml(cells[0] || '');
const componentSku = stripHtml(cells[1] || '');
const perUnit = parseQuantityAndUnit(cells[2] || '');
const total = parseQuantityAndUnit(cells[3] || '');
return [{
componentName,
componentSku,
quantityPerUnit: perUnit.quantity,
totalQuantity: total.quantity,
unit: perUnit.unit || total.unit
}];
}).filter(component => component.componentName || component.componentSku);
}
function parseSteps(commands: XajaxCommand[]): ProductionOrderPayload['steps'] {
const script = getScript(commands, /addDetails\(/);
const match = script.match(/addDetails\(([\s\S]*)\)/);
if (!match) return [];
try {
const steps = JSON.parse(match[1]);
return (Array.isArray(steps) ? steps : []).map((step, index) => ({
stepNumber: index + 1,
name: pickFirst(step?.tarefa, step?.name, 'Etapa'),
startDate: pickFirstNullable(step?.dataInicio),
endDate: pickFirstNullable(step?.dataTermino),
status: /Pendente/i.test(String(step?.situacao || '')) ? 'pendente' : pickFirst(step?.situacao, 'pendente')
}));
} catch (error: any) {
console.warn(`[parse] Could not parse OP steps: ${error.message}`);
return [];
}
}
function parseStatus(commands: XajaxCommand[]) {
const html = getAssignedValue(commands, 'view-info-ordem-producao');
if (!html) return '';
return stripHtml(html).replace(/^[-\s]+/, '');
}
function parseNotesMeta(notes: string) {
return {
supplier: notes.match(/FONE?CEDOR\s*:\s*([^\n]+)/i)?.[1]?.trim() || '',
lotCode: notes.match(/LOTE\s*:\s*([^\n]+)/i)?.[1]?.trim() || ''
};
}
function parseBuscarDadosProdutoCall(commands: XajaxCommand[]) {
const script = getScript(commands, /buscarDadosProduto\(/);
const match = script.match(/buscarDadosProduto\(([^,]+),\s*'([^']*)',\s*'([^']*)'\)/);
if (!match) return null;
return {
productId: normalizeText(match[1]),
quantity: normalizeText(match[2]),
flag: normalizeText(match[3])
};
}
async function tinyErpPost(url: string, body: string, reason: string) {
await waitForRequestSlot(reason);
const headers: Record<string, string> = {
Accept: 'application/json, text/javascript, */*; q=0.01',
'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8',
Origin: TINY_ERP_BASE_URL,
Referer: `${TINY_ERP_BASE_URL}/ordens_producao`,
'X-Custom-Request-For': 'XAJAX',
'X-Requested-With': 'XMLHttpRequest',
'X-User-Agent': 'Mozilla/5.0'
};
if (TINY_ERP_COOKIE) headers.Cookie = TINY_ERP_COOKIE;
if (TINY_ERP_AUTHORIZATION) headers.Authorization = TINY_ERP_AUTHORIZATION;
return axios.post(url, body, { headers });
}
async function fetchTinyProduct(productId: string) {
if (!TINY_API_TOKEN || !productId) return null;
if (productCache.has(productId)) return productCache.get(productId) || null;
await waitForRequestSlot(`Tiny produto.obter.php ${productId}`);
const params = new URLSearchParams();
params.append('token', TINY_API_TOKEN);
params.append('formato', 'JSON');
params.append('id', productId);
try {
const response = await axios.post('https://api.tiny.com.br/api2/produto.obter.php', params, {
headers: { 'Content-Type': 'application/x-www-form-urlencoded' }
});
const produto = response.data?.retorno?.produto;
const normalized = produto ? {
sku: normalizeText(produto.codigo),
description: normalizeText(produto.nome),
unit: normalizeText(produto.unidade)
} : null;
productCache.set(productId, normalized);
return normalized;
} catch (error: any) {
console.warn(`[Tiny] Could not fetch product ${productId}: ${describeRequestError(error)}`);
productCache.set(productId, null);
return null;
}
}
async function fetchOpDetail(orderId: string) {
const body = fillTemplate(DETAIL_BODY_TEMPLATE, { id: orderId });
const response = await tinyErpPost(DETAIL_URL, body, `Tiny/Olist OP detail ${orderId}`);
return getCommands(response.data);
}
async function fetchComposition(productId: string, quantity: string, flag: string) {
const body = fillTemplate(PRODUCT_DATA_BODY_TEMPLATE, { productId, quantity, flag });
const response = await tinyErpPost(PRODUCT_DATA_URL, body, `Tiny/Olist product composition ${productId}`);
const commands = getCommands(response.data);
const compositionHtml = getAssignedValue(commands, 'div-composicao');
return parseCompositionHtml(compositionHtml);
}
function normalizeDetail(
orderId: string,
commands: XajaxCommand[],
components: ProductionOrderPayload['components'],
finishedProduct: { sku: string; description: string; unit: string } | null
): ProductionOrderPayload | null {
const notes = getAssignedValue(commands, 'observacoes');
const notesMeta = parseNotesMeta(notes);
const productData = parseBuscarDadosProdutoCall(commands);
const productId = getAssignedValue(commands, 'idProduto');
const productSku = getAssignedValue(commands, 'codigo') || getAssignedValue(commands, 'sku') || finishedProduct?.sku || productId;
const payload: ProductionOrderPayload = {
order: {
tinyId: getAssignedValue(commands, 'id') || orderId,
number: getAssignedValue(commands, 'numero') || orderId,
status: parseStatus(commands) || getAssignedValue(commands, 'situacao'),
productSku,
productDescription: getAssignedValue(commands, 'produto') || finishedProduct?.description || '',
quantity: getAssignedValue(commands, 'quantidade') || productData?.quantity || '',
unit: stripHtml(getAssignedValue(commands, 'un-produto')) || finishedProduct?.unit || 'UN',
issueDate: pickFirstNullable(getAssignedValue(commands, 'dataInicio')),
expectedDate: pickFirstNullable(getAssignedValue(commands, 'dataPrevista')),
supplier: notesMeta.supplier,
lotCode: notesMeta.lotCode,
notes
},
components,
steps: parseSteps(commands)
};
if (!payload.order.tinyId || !payload.order.productDescription || !payload.order.quantity || !payload.components.length) {
console.warn(`[normalize] OP ${orderId} missing required data. tinyId=${payload.order.tinyId || 'none'} product=${payload.order.productDescription || 'none'} quantity=${payload.order.quantity || 'none'} components=${payload.components.length}`);
return null;
}
return payload;
}
async function loadOrderIdsFromList() {
if (!LIST_BODY) return [];
const response = await tinyErpPost(LIST_URL, LIST_BODY, 'Tiny/Olist OP list');
const commands = getCommands(response.data);
const serialized = JSON.stringify(commands);
return Array.from(new Set(Array.from(serialized.matchAll(/(?:idOrdemProducao|id)["':\\]+(\d{6,})/g)).map(match => match[1])));
}
async function loadOrderIds() {
if (ORDER_IDS.length) return ORDER_IDS;
const listedIds = await loadOrderIdsFromList();
if (listedIds.length) return listedIds;
throw new Error('No OP source configured. Set PRODUCTION_ORDER_IDS first. Later we can use PRODUCTION_ORDERS_LIST_BODY once the listarOrdensProducao Form Data is captured.');
}
async function sendProductionOrder(payload: ProductionOrderPayload) {
if (DRY_RUN) {
console.log(`[dry-run] Would send OP ${payload.order.tinyId || payload.order.number}`);
console.log(JSON.stringify(payload, null, 2));
return true;
}
let attempt = 0;
while (true) {
try {
await waitForRequestSlot(`Graphs production order ${payload.order.tinyId || payload.order.number}`);
await axios.post(GRAPHS_PRODUCTION_ORDERS_URL, payload, {
headers: {
'Content-Type': 'application/json',
'x-api-key': GRAPHS_PRODUCTION_ORDERS_API_KEY
}
});
console.log(`[Graphs OP] Sent OP ${payload.order.tinyId || payload.order.number} with ${payload.components.length} component(s).`);
return true;
} catch (error: any) {
attempt += 1;
const status = error?.response?.status;
const shouldRetry = !status || status >= 500;
console.error(`[Graphs OP] Failed to send OP ${payload.order.tinyId || payload.order.number} on attempt ${attempt}: ${describeRequestError(error)}`);
if (!shouldRetry || (GRAPHS_MAX_RETRIES && attempt >= GRAPHS_MAX_RETRIES)) {
console.error(`[Graphs OP] Giving up on OP ${payload.order.tinyId || payload.order.number}; sync will continue.`);
return false;
}
await sleepWithStop(GRAPHS_RETRY_DELAY_MS, `retrying Graphs OP ${payload.order.tinyId || payload.order.number}`);
}
}
}
async function runSync() {
if (!TINY_ERP_COOKIE && !TINY_ERP_AUTHORIZATION) {
throw new Error('Missing Tiny/Olist browser auth. Set PRODUCTION_ORDERS_TINY_ERP_COOKIE from the browser session cookie.');
}
console.log(`[config] Detail URL: ${DETAIL_URL}`);
console.log(`[config] Product data URL: ${PRODUCT_DATA_URL}`);
console.log(`[config] Graphs OP URL: ${GRAPHS_PRODUCTION_ORDERS_URL}`);
console.log(`[config] Dry run: ${DRY_RUN ? 'yes' : 'no'}`);
console.log(`[config] Request pace: 1 request every ${REQUEST_DELAY_MS}ms`);
console.log(`[config] State file: ${STATE_FILE}`);
console.log(`[config] Stop file: ${STOP_FILE}`);
saveState({
status: 'running',
processedOrders: 0,
failedOrders: 0,
lastMessage: 'Production orders sync started'
});
const orderIds = await loadOrderIds();
console.log(`[source] Found ${orderIds.length} OP id(s).`);
let processedOrders = 0;
let failedOrders = 0;
for (const orderId of orderIds) {
assertNotStopped();
if (MAX_ORDERS && processedOrders + failedOrders >= MAX_ORDERS) {
console.log(`[limit] Stopped after ${MAX_ORDERS} OP(s).`);
break;
}
try {
const detailCommands = await fetchOpDetail(orderId);
const productData = parseBuscarDadosProdutoCall(detailCommands);
if (!productData?.productId) {
throw new Error(`Could not find buscarDadosProduto call for OP ${orderId}.`);
}
const components = await fetchComposition(productData.productId, productData.quantity, productData.flag);
const finishedProduct = await fetchTinyProduct(productData.productId);
const payload = normalizeDetail(orderId, detailCommands, components, finishedProduct);
if (!payload) {
failedOrders += 1;
saveState({
status: 'running',
processedOrders,
failedOrders,
lastFailedTinyId: orderId,
lastMessage: `Could not normalize OP ${orderId}`
});
continue;
}
const sent = await sendProductionOrder(payload);
if (sent) {
processedOrders += 1;
} else {
failedOrders += 1;
}
saveState({
status: 'running',
processedOrders,
failedOrders,
lastTinyId: sent ? payload.order.tinyId : undefined,
lastFailedTinyId: sent ? undefined : payload.order.tinyId,
lastMessage: sent ? `Synced OP ${payload.order.tinyId}` : `Failed OP ${payload.order.tinyId}; continuing`
});
} catch (error: any) {
failedOrders += 1;
console.error(`[Tiny/Olist OP] Failed OP ${orderId}: ${describeRequestError(error)}`);
saveState({
status: 'running',
processedOrders,
failedOrders,
lastFailedTinyId: orderId,
lastMessage: `Failed OP ${orderId}; continuing`
});
}
}
saveState({
status: 'completed',
processedOrders,
failedOrders,
lastMessage: failedOrders ? `Production orders sync completed with ${failedOrders} failed OP(s)` : 'Production orders sync completed'
});
console.log(`Done. Synced ${processedOrders} OP(s). Failed OPs: ${failedOrders}.`);
}
runSync().catch((error: any) => {
if (error instanceof SyncStoppedError) {
console.log('Production orders sync stopped by request.');
saveState({
status: 'stopped',
lastMessage: 'Production orders sync stopped by request'
});
process.exit(0);
}
console.error(`Production orders sync failed: ${describeRequestError(error)}`);
saveState({
status: 'stopped',
lastMessage: `Production orders sync failed: ${describeRequestError(error)}`
});
process.exit(1);
});

View File

@@ -0,0 +1,174 @@
import axios from 'axios';
import crypto from 'crypto';
import fs from 'fs';
import path from 'path';
type OlistV3Tokens = {
access_token: string;
refresh_token?: string;
expires_in?: number;
expires_at?: string;
};
type OlistV3AuthorizationState = {
state: string;
createdAt: string;
};
const OAUTH_BASE_URL = 'https://accounts.tiny.com.br/realms/tiny/protocol/openid-connect';
const TOKEN_URL = `${OAUTH_BASE_URL}/token`;
const AUTHORIZE_URL = `${OAUTH_BASE_URL}/auth`;
const DEFAULT_TOKEN_FILE = path.join(process.cwd(), 'data-runtime', 'olist_v3_tokens.json');
const DEFAULT_STATE_FILE = path.join(process.cwd(), 'data-runtime', 'olist_v3_oauth_state.json');
function requiredEnv(name: string) {
const value = process.env[name]?.trim();
if (!value) throw new Error(`Missing ${name}.`);
return value;
}
function getTokenFile() {
return process.env.OLIST_V3_TOKEN_FILE || DEFAULT_TOKEN_FILE;
}
function getStateFile() {
return process.env.OLIST_V3_OAUTH_STATE_FILE || DEFAULT_STATE_FILE;
}
function ensureParentDirectory(filePath: string) {
fs.mkdirSync(path.dirname(filePath), { recursive: true });
}
function writeJson(filePath: string, value: unknown) {
ensureParentDirectory(filePath);
fs.writeFileSync(filePath, JSON.stringify(value, null, 2), { mode: 0o600 });
}
function readJson<T>(filePath: string): T | null {
if (!fs.existsSync(filePath)) return null;
return JSON.parse(fs.readFileSync(filePath, 'utf-8')) as T;
}
function getClientConfig() {
return {
clientId: requiredEnv('OLIST_V3_CLIENT_ID'),
clientSecret: requiredEnv('OLIST_V3_CLIENT_SECRET'),
redirectUri: requiredEnv('OLIST_V3_REDIRECT_URI')
};
}
function saveTokens(tokens: OlistV3Tokens) {
const expiresIn = Number(tokens.expires_in || 0);
const expiresAt = Number.isFinite(expiresIn) && expiresIn > 0
? new Date(Date.now() + Math.max(0, expiresIn - 60) * 1000).toISOString()
: undefined;
const previous = readJson<OlistV3Tokens>(getTokenFile());
writeJson(getTokenFile(), {
...previous,
...tokens,
refresh_token: tokens.refresh_token || previous?.refresh_token,
expires_at: expiresAt || previous?.expires_at
});
}
function tokenIsUsable(tokens: OlistV3Tokens) {
if (!tokens.access_token) return false;
if (!tokens.expires_at) return true;
return new Date(tokens.expires_at).getTime() > Date.now();
}
async function requestTokens(params: URLSearchParams) {
const response = await axios.post<OlistV3Tokens>(TOKEN_URL, params, {
headers: { 'Content-Type': 'application/x-www-form-urlencoded' },
timeout: 30000
});
if (!response.data?.access_token) {
throw new Error('Olist V3 OAuth did not return an access token.');
}
saveTokens(response.data);
return response.data;
}
export function isOlistV3Configured() {
return Boolean(
process.env.OLIST_V3_CLIENT_ID?.trim()
&& process.env.OLIST_V3_CLIENT_SECRET?.trim()
&& process.env.OLIST_V3_REDIRECT_URI?.trim()
);
}
export function getOlistV3AuthorizationUrl() {
const { clientId, redirectUri } = getClientConfig();
const state = crypto.randomBytes(32).toString('hex');
writeJson(getStateFile(), { state, createdAt: new Date().toISOString() } satisfies OlistV3AuthorizationState);
const params = new URLSearchParams({
client_id: clientId,
redirect_uri: redirectUri,
scope: 'openid',
response_type: 'code',
state
});
return `${AUTHORIZE_URL}?${params.toString()}`;
}
export async function exchangeOlistV3AuthorizationCode(code: string, state: string) {
const expected = readJson<OlistV3AuthorizationState>(getStateFile());
const expectedState = Buffer.from(expected?.state || '');
const receivedState = Buffer.from(state || '');
if (!expected?.state || expectedState.length !== receivedState.length || !crypto.timingSafeEqual(expectedState, receivedState)) {
throw new Error('Invalid OAuth state. Start the authorization flow again.');
}
const createdAt = new Date(expected.createdAt).getTime();
if (!Number.isFinite(createdAt) || Date.now() - createdAt > 10 * 60 * 1000) {
throw new Error('OAuth state expired. Start the authorization flow again.');
}
const { clientId, clientSecret, redirectUri } = getClientConfig();
const params = new URLSearchParams({
grant_type: 'authorization_code',
client_id: clientId,
client_secret: clientSecret,
redirect_uri: redirectUri,
code
});
const tokens = await requestTokens(params);
fs.rmSync(getStateFile(), { force: true });
return tokens;
}
export async function getOlistV3AccessToken(forceRefresh = false) {
const current = readJson<OlistV3Tokens>(getTokenFile());
if (current && !forceRefresh && tokenIsUsable(current)) {
return current.access_token;
}
if (!current?.refresh_token) {
throw new Error('Olist V3 is not authorized. Open /api/olist-v3/authorize first.');
}
const { clientId, clientSecret } = getClientConfig();
const params = new URLSearchParams({
grant_type: 'refresh_token',
client_id: clientId,
client_secret: clientSecret,
refresh_token: current.refresh_token
});
const tokens = await requestTokens(params);
return tokens.access_token;
}
export function getOlistV3AuthStatus() {
const tokens = readJson<OlistV3Tokens>(getTokenFile());
return {
configured: isOlistV3Configured(),
authorized: Boolean(tokens?.access_token || tokens?.refresh_token),
accessTokenUsable: Boolean(tokens && tokenIsUsable(tokens)),
tokenExpiresAt: tokens?.expires_at || null,
tokenFile: getTokenFile()
};
}