Import finalized Olist production orders

This commit is contained in:
Cauê Faleiros
2026-08-12 10:57:18 -03:00
parent 42d64447d5
commit 6d3a1d6b1a
10 changed files with 494 additions and 7 deletions

View File

@@ -454,6 +454,41 @@ const initDB = async () => {
ADD COLUMN IF NOT EXISTS yield_pieces_per_kg NUMERIC(14, 4);
`).catch(() => {});
await pool.query(`
ALTER TABLE production_orders
ADD COLUMN IF NOT EXISTS completed_date DATE;
`).catch(() => {});
await pool.query(`
CREATE TABLE IF NOT EXISTS production_order_import_runs (
id SERIAL PRIMARY KEY,
filename TEXT NOT NULL,
status VARCHAR(40) NOT NULL DEFAULT 'running',
total_rows INTEGER NOT NULL DEFAULT 0,
imported_count INTEGER NOT NULL DEFAULT 0,
updated_count INTEGER NOT NULL DEFAULT 0,
skipped_count INTEGER NOT NULL DEFAULT 0,
failed_count INTEGER NOT NULL DEFAULT 0,
error_message TEXT,
started_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
completed_at TIMESTAMPTZ
);
`);
await pool.query(`
CREATE TABLE IF NOT EXISTS production_yield_recommendations (
family_key VARCHAR(20) PRIMARY KEY,
sample_count INTEGER NOT NULL DEFAULT 0,
yield_sample_count INTEGER NOT NULL DEFAULT 0,
roll_sample_count INTEGER NOT NULL DEFAULT 0,
avg_yield_pieces_per_kg NUMERIC(14, 4),
avg_kg_per_roll NUMERIC(14, 4),
recommended_units_per_roll NUMERIC(14, 4),
last_completed_date DATE,
updated_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP
);
`);
await pool.query(`
ALTER TABLE production_orders
ALTER COLUMN created_at TYPE TIMESTAMPTZ USING created_at AT TIME ZONE 'America/Sao_Paulo',
@@ -627,6 +662,8 @@ const initDB = async () => {
await pool.query(`CREATE INDEX IF NOT EXISTS idx_production_orders_status ON production_orders (status);`);
await pool.query(`CREATE INDEX IF NOT EXISTS idx_production_orders_issue_date ON production_orders (issue_date DESC);`);
await pool.query(`CREATE INDEX IF NOT EXISTS idx_production_orders_expected_date ON production_orders (expected_date DESC);`);
await pool.query(`CREATE INDEX IF NOT EXISTS idx_production_orders_completed_date ON production_orders (completed_date DESC);`);
await pool.query(`CREATE INDEX IF NOT EXISTS idx_production_order_import_runs_started_at ON production_order_import_runs (started_at DESC);`);
await pool.query(`CREATE INDEX IF NOT EXISTS idx_production_order_markers_order_id ON production_order_markers (production_order_id);`);
await pool.query(`CREATE INDEX IF NOT EXISTS idx_production_order_components_order_id ON production_order_components (production_order_id);`);
await pool.query(`CREATE INDEX IF NOT EXISTS idx_production_order_components_sku ON production_order_components (component_sku);`);

View File

@@ -9,6 +9,7 @@ const {
requestOlistSyncStop,
startOlistSync
} = require('../services/olistService');
const { importFinalizedProductionOrderCsv, listProductionOrderImportData } = require('../services/productionOrderCsvImportService');
const router = express.Router();
@@ -48,6 +49,22 @@ router.post('/olist/sync/stop', verifySuperAdmin, async (req, res, next) => {
}
});
router.get('/olist/production-order-imports', verifySuperAdmin, async (req, res, next) => {
try {
res.json(await listProductionOrderImportData());
} catch (error) {
next(error);
}
});
router.post('/olist/production-order-imports', verifySuperAdmin, async (req, res, next) => {
try {
res.status(201).json(await importFinalizedProductionOrderCsv(req.body || {}));
} catch (error) {
next(error);
}
});
router.get('/olist/runs/:runId', verifySuperAdmin, async (req, res, next) => {
try {
res.json(await getOlistRunDetails(req.params.runId, {

View File

@@ -19,7 +19,7 @@ const createApp = () => {
const app = express();
app.use(cors());
app.use(bodyParser.json({ limit: '10mb' }));
app.use(bodyParser.json({ limit: '25mb' }));
app.use('/api', authRoutes);
app.use('/api', dataRoutes);

View File

@@ -0,0 +1,268 @@
const { pool } = require('../db');
const FAMILY_KEYS = new Set(['BLCS', 'BLOS', 'BLMC', 'BLPM']);
const REQUIRED_HEADERS = ['id_op', 'numero_op', 'sku_produto', 'descricao_produto', 'quantidade_produzida', 'data_emissao', 'data_conclusao', 'situacao', 'observacoes'];
const normalizeText = (value) => String(value || '').replace(/\s+/g, ' ').trim();
const normalizeNumber = (value) => {
const normalized = String(value || '').trim().replace(/\s/g, '');
if (!normalized) return null;
const decimal = normalized.includes(',')
? normalized.replace(/\./g, '').replace(',', '.')
: normalized;
const number = Number(decimal);
return Number.isFinite(number) && number > 0 ? number : null;
};
const normalizeDate = (value) => {
const raw = String(value || '').trim();
const iso = raw.match(/^(\d{4})-(\d{2})-(\d{2})/);
if (iso) return `${iso[1]}-${iso[2]}-${iso[3]}`;
const br = raw.match(/^(\d{1,2})\/(\d{1,2})\/(\d{4})/);
return br ? `${br[3]}-${br[2].padStart(2, '0')}-${br[1].padStart(2, '0')}` : null;
};
const normalizeHeader = (header) => String(header || '')
.replace(/^\uFEFF/, '')
.normalize('NFD')
.replace(/\p{Diacritic}/gu, '')
.trim()
.toLowerCase()
.replace(/\s+/g, '_');
const parseCsv = (csv) => {
const rows = [];
let row = [];
let field = '';
let quoted = false;
for (let index = 0; index < csv.length; index += 1) {
const character = csv[index];
const next = csv[index + 1];
if (character === '"') {
if (quoted && next === '"') {
field += '"';
index += 1;
} else {
quoted = !quoted;
}
} else if (character === ',' && !quoted) {
row.push(field);
field = '';
} else if ((character === '\n' || character === '\r') && !quoted) {
if (character === '\r' && next === '\n') index += 1;
row.push(field);
if (row.some(value => value.length > 0)) rows.push(row);
row = [];
field = '';
} else {
field += character;
}
}
row.push(field);
if (row.some(value => value.length > 0)) rows.push(row);
if (quoted) throw new Error('CSV possui aspas não finalizadas.');
return rows;
};
const extractObservationFields = (notes) => {
const raw = String(notes || '');
const lineValue = (pattern) => raw.match(pattern)?.[1]?.trim() || '';
const supplier = lineValue(/(?:^|\n)\s*FORNECEDOR\s*:\s*([^\r\n]+)/i);
const lotCode = lineValue(/(?:^|\n)\s*LOTE\s*:\s*([^\r\n]+)/i);
const yieldPiecesPerKg = normalizeNumber(raw.match(/RENDIMENTO\s*:\s*([\d.,]+)/i)?.[1]);
const explicitRolls = normalizeNumber(raw.match(/(?:QUANTIDADE\s+DE\s+ROLOS|ROLOS|BOBINAS)\s*:\s*([\d.,]+)/i)?.[1]);
const fabricKg = normalizeNumber(raw.match(/(?:QUILOS?\s+DE\s+MALHA|QUILOS?\s+MALHA)\s*:\s*([\d.,]+)(?:\s*KG)?/i)?.[1])
|| normalizeNumber(raw.match(/(?:ROLOS|BOBINAS)\s*:\s*[\d.,]+\s*[-–]\s*([\d.,]+)\s*KG/i)?.[1]);
const ribKg = normalizeNumber(raw.match(/(?:QUILOS?\s+DE\s+(?:GOLA|RIBANA|VIEZ(?:\s*\([^)]*\))?))\s*:\s*([\d.,]+)(?:\s*KG)?/i)?.[1]);
return { supplier, lotCode, yieldPiecesPerKg, rollQuantity: explicitRolls, fabricKg, ribKg };
};
const parseFinalizedProductionOrderCsv = (csv) => {
if (typeof csv !== 'string' || !csv.trim()) {
const error = new Error('Selecione um arquivo CSV de OPs finalizadas.');
error.statusCode = 400;
throw error;
}
const rows = parseCsv(csv);
if (rows.length < 2) {
const error = new Error('O CSV não possui OPs para importar.');
error.statusCode = 400;
throw error;
}
const headers = rows[0].map(normalizeHeader);
const missingHeaders = REQUIRED_HEADERS.filter(header => !headers.includes(header));
if (missingHeaders.length) {
const error = new Error(`CSV sem colunas obrigatórias: ${missingHeaders.join(', ')}.`);
error.statusCode = 400;
throw error;
}
const headerIndex = Object.fromEntries(headers.map((header, index) => [header, index]));
const records = [];
const issues = [];
rows.slice(1).forEach((row, rowOffset) => {
const line = rowOffset + 2;
const value = (header) => row[headerIndex[header]] || '';
const tinyId = normalizeText(value('id_op'));
const number = normalizeText(value('numero_op'));
const productSku = normalizeText(value('sku_produto')).toUpperCase();
const productDescription = normalizeText(value('descricao_produto'));
const quantity = normalizeNumber(value('quantidade_produzida'));
const status = normalizeText(value('situacao')).toLowerCase();
const notes = String(value('observacoes') || '').trim();
if (!tinyId || !number || !productSku || !productDescription || !quantity) {
issues.push({ line, message: 'OP sem identificação, SKU, produto ou quantidade válida.' });
return;
}
if (!/^finalizad[ao]$/.test(status)) {
issues.push({ line, number, message: 'Ignorada: a OP não está Finalizada.' });
return;
}
const familyKey = productSku.split('.')[0];
const extracted = extractObservationFields(notes);
records.push({
tinyId,
number,
productSku,
productDescription,
quantity,
issueDate: normalizeDate(value('data_emissao')),
completedDate: normalizeDate(value('data_conclusao')),
orderReference: normalizeText(value('pedido_relacionado')),
notes,
familyKey: FAMILY_KEYS.has(familyKey) ? familyKey : null,
...extracted
});
});
return { records, issues, totalRows: rows.length - 1 };
};
const refreshYieldRecommendations = async (client) => {
await client.query('DELETE FROM production_yield_recommendations;');
await client.query(`
INSERT INTO production_yield_recommendations (
family_key, sample_count, yield_sample_count, roll_sample_count,
avg_yield_pieces_per_kg, avg_kg_per_roll, recommended_units_per_roll,
last_completed_date, updated_at
)
SELECT
split_part(product_sku, '.', 1) AS family_key,
COUNT(*)::int,
COUNT(yield_pieces_per_kg)::int,
COUNT(*) FILTER (WHERE fabric_kg > 0 AND roll_quantity > 0)::int,
AVG(yield_pieces_per_kg),
AVG(fabric_kg / roll_quantity) FILTER (WHERE fabric_kg > 0 AND roll_quantity > 0),
AVG(yield_pieces_per_kg) FILTER (WHERE fabric_kg > 0 AND roll_quantity > 0 AND yield_pieces_per_kg > 0)
* AVG(fabric_kg / roll_quantity) FILTER (WHERE fabric_kg > 0 AND roll_quantity > 0 AND yield_pieces_per_kg > 0),
MAX(completed_date),
CURRENT_TIMESTAMP
FROM production_orders
WHERE status = 'finished'
AND split_part(product_sku, '.', 1) IN ('BLCS', 'BLOS', 'BLMC', 'BLPM')
GROUP BY split_part(product_sku, '.', 1);
`);
};
const listProductionOrderImportData = async () => {
const [runs, recommendations] = await Promise.all([
pool.query(`SELECT id, filename, status, total_rows, imported_count, updated_count, skipped_count, failed_count, error_message, started_at, completed_at FROM production_order_import_runs ORDER BY started_at DESC LIMIT 10;`),
pool.query(`SELECT family_key, sample_count, yield_sample_count, roll_sample_count, avg_yield_pieces_per_kg, avg_kg_per_roll, recommended_units_per_roll, last_completed_date, updated_at FROM production_yield_recommendations ORDER BY family_key;`)
]);
return {
runs: runs.rows.map(row => ({
id: row.id, filename: row.filename, status: row.status, totalRows: row.total_rows,
imported: row.imported_count, updated: row.updated_count, skipped: row.skipped_count,
failed: row.failed_count, errorMessage: row.error_message || '', startedAt: row.started_at,
completedAt: row.completed_at
})),
recommendations: recommendations.rows.map(row => ({
familyKey: row.family_key, sampleCount: Number(row.sample_count), yieldSampleCount: Number(row.yield_sample_count),
rollSampleCount: Number(row.roll_sample_count), avgYieldPiecesPerKg: row.avg_yield_pieces_per_kg === null ? null : Number(row.avg_yield_pieces_per_kg),
avgKgPerRoll: row.avg_kg_per_roll === null ? null : Number(row.avg_kg_per_roll),
recommendedUnitsPerRoll: row.recommended_units_per_roll === null ? null : Number(row.recommended_units_per_roll),
lastCompletedDate: row.last_completed_date || null, updatedAt: row.updated_at
}))
};
};
const importFinalizedProductionOrderCsv = async ({ filename, csv }) => {
const parsed = parseFinalizedProductionOrderCsv(csv);
const client = await pool.connect();
const safeFilename = normalizeText(filename) || 'ops-finalizadas.csv';
let runId;
let imported = 0;
let updated = 0;
let failed = 0;
try {
await client.query('BEGIN');
const run = await client.query(`
INSERT INTO production_order_import_runs (filename, status, total_rows, skipped_count)
VALUES ($1, 'running', $2, $3)
RETURNING id;
`, [safeFilename, parsed.totalRows, parsed.issues.length]);
runId = run.rows[0].id;
for (const record of parsed.records) {
try {
await client.query('SAVEPOINT production_order_import_record;');
const existing = await client.query('SELECT id FROM production_orders WHERE tiny_id = $1 OR number = $2 LIMIT 1;', [record.tinyId, record.number]);
const payload = JSON.stringify({ source: 'olist_finalized_op_csv', importedAt: new Date().toISOString(), record });
if (existing.rows[0]) {
await client.query(`
UPDATE production_orders SET tiny_id = $1, number = $2, status = 'finished', order_reference = NULLIF($3, ''), issue_date = $4,
completed_date = $5, product_sku = $6, product_description = $7, quantity = $8, unit = 'UN', integration_status = 'Olist CSV',
notes = NULLIF($9, ''), supplier = NULLIF($10, ''), lot_code = NULLIF($11, ''), roll_quantity = $12, fabric_kg = $13, rib_kg = $14,
yield_pieces_per_kg = $15, tiny_payload = $16::jsonb, updated_at = CURRENT_TIMESTAMP
WHERE id = $17;
`, [record.tinyId, record.number, record.orderReference, record.issueDate, record.completedDate, record.productSku, record.productDescription, record.quantity, record.notes, record.supplier, record.lotCode, record.rollQuantity, record.fabricKg, record.ribKg, record.yieldPiecesPerKg, payload, existing.rows[0].id]);
updated += 1;
} else {
await client.query(`
INSERT INTO production_orders (tiny_id, number, status, order_reference, issue_date, completed_date, product_sku, product_description, quantity, unit, integration_status, notes, supplier, lot_code, roll_quantity, fabric_kg, rib_kg, yield_pieces_per_kg, tiny_payload)
VALUES ($1, $2, 'finished', NULLIF($3, ''), $4, $5, $6, $7, $8, 'UN', 'Olist CSV', NULLIF($9, ''), NULLIF($10, ''), NULLIF($11, ''), $12, $13, $14, $15, $16::jsonb);
`, [record.tinyId, record.number, record.orderReference, record.issueDate, record.completedDate, record.productSku, record.productDescription, record.quantity, record.notes, record.supplier, record.lotCode, record.rollQuantity, record.fabricKg, record.ribKg, record.yieldPiecesPerKg, payload]);
imported += 1;
}
await client.query('RELEASE SAVEPOINT production_order_import_record;');
} catch (error) {
await client.query('ROLLBACK TO SAVEPOINT production_order_import_record;');
failed += 1;
parsed.issues.push({ number: record.number, message: error.message });
}
}
await refreshYieldRecommendations(client);
await client.query(`
UPDATE production_order_import_runs
SET status = $1, imported_count = $2, updated_count = $3, skipped_count = $4, failed_count = $5, error_message = $6, completed_at = CURRENT_TIMESTAMP
WHERE id = $7;
`, [failed ? 'completed_with_errors' : 'completed', imported, updated, parsed.issues.length, failed, parsed.issues.slice(0, 20).map(issue => `OP ${issue.number || `linha ${issue.line}`}: ${issue.message}`).join('\n') || null, runId]);
await client.query('COMMIT');
} catch (error) {
await client.query('ROLLBACK');
throw error;
} finally {
client.release();
}
return { runId, totalRows: parsed.totalRows, imported, updated, skipped: parsed.issues.length, failed, issues: parsed.issues.slice(0, 20), ...(await listProductionOrderImportData()) };
};
module.exports = {
extractObservationFields,
importFinalizedProductionOrderCsv,
listProductionOrderImportData,
parseCsv,
parseFinalizedProductionOrderCsv
};

View File

@@ -254,6 +254,7 @@ const mapProductionOrderRow = (row) => {
orderReference: row.order_reference || '',
issueDate: formatDate(row.issue_date),
expectedDate: formatDate(row.expected_date),
completedDate: formatDate(row.completed_date),
productSku: row.product_sku || '',
productDescription: row.product_description || '',
quantity: Number(row.quantity || 0),
@@ -284,6 +285,7 @@ const getOrderById = async (id, client = pool) => {
po.order_reference,
po.issue_date,
po.expected_date,
po.completed_date,
po.product_sku,
po.product_description,
po.quantity,
@@ -366,6 +368,7 @@ const baseProductionOrderSelect = `
po.order_reference,
po.issue_date,
po.expected_date,
po.completed_date,
po.product_sku,
po.product_description,
po.quantity,

View File

@@ -0,0 +1,30 @@
const assert = require('node:assert/strict');
const test = require('node:test');
const { extractObservationFields, parseFinalizedProductionOrderCsv } = require('../services/productionOrderCsvImportService');
test('extractObservationFields reads Brazilian production notes without using the OP quantity as rolls', () => {
const result = extractObservationFields(`MALHA\nFORNECEDOR: TECEBEM\nBOBINAS: 10 - 209,94KG\nRENDIMENTO: 3,6\nLOTE: 0245163`);
assert.deepEqual(result, {
supplier: 'TECEBEM',
lotCode: '0245163',
yieldPiecesPerKg: 3.6,
rollQuantity: 10,
fabricKg: 209.94,
ribKg: null
});
});
test('parseFinalizedProductionOrderCsv supports multiline observations and skips non-finalized OPs', () => {
const csv = `id_op,numero_op,sku_produto,descricao_produto,quantidade_produzida,data_emissao,data_conclusao,situacao,observacoes,pedido_relacionado\n1,10,BLCS.PTO.G,Camiseta preta G,100,2026-08-01T00:00:00.000Z,2026-08-02T00:00:00.000Z,Finalizada,"FORNECEDOR: NEXSTAR\nRENDIMENTO: 5,44\nQUANTIDADE DE ROLOS: 30\nQUILOS DE MALHA: 577,90",PLANO-1\n2,11,BLCS.PTO.M,Camiseta preta M,100,2026-08-01,2026-08-02,Em andamento,Teste,`;
const result = parseFinalizedProductionOrderCsv(csv);
assert.equal(result.totalRows, 2);
assert.equal(result.records.length, 1);
assert.equal(result.issues.length, 1);
assert.equal(result.records[0].familyKey, 'BLCS');
assert.equal(result.records[0].rollQuantity, 30);
assert.equal(result.records[0].fabricKg, 577.9);
assert.equal(result.records[0].yieldPiecesPerKg, 5.44);
});