Throttle Olist sync requests
All checks were successful
Build and Deploy / build-and-deploy (push) Successful in 57s

This commit is contained in:
Cauê Faleiros
2026-08-10 13:13:29 -03:00
parent 1a430d4fb0
commit 838354f784
4 changed files with 16 additions and 8 deletions

View File

@@ -67,6 +67,8 @@ OLIST_SYNC_ENABLED=true
`OLIST_TOKEN_ENCRYPTION_KEY` must be stable between deploys because Graphs uses it to encrypt OAuth tokens stored in PostgreSQL. The initial sync is full; later scheduled runs are incremental and use the product update date. Manual JSON import remains available as a fallback. `OLIST_TOKEN_ENCRYPTION_KEY` must be stable between deploys because Graphs uses it to encrypt OAuth tokens stored in PostgreSQL. The initial sync is full; later scheduled runs are incremental and use the product update date. Manual JSON import remains available as a fallback.
Graphs waits at least 2.5 seconds between every Olist API request (24 requests/minute). This keeps capacity available for other Tiny/Olist integrations. `OLIST_SYNC_REQUEST_DELAY_MS` can increase that delay but cannot lower it below 2500.
### Cutting Workbook Import ### Cutting Workbook Import
Manual cutting/replenishment workbooks can be normalized into JSON before they Manual cutting/replenishment workbooks can be normalized into JSON before they

View File

@@ -44,7 +44,7 @@ module.exports = {
OLIST_TOKEN_ENCRYPTION_KEY: process.env.OLIST_TOKEN_ENCRYPTION_KEY || '', OLIST_TOKEN_ENCRYPTION_KEY: process.env.OLIST_TOKEN_ENCRYPTION_KEY || '',
OLIST_SYNC_ENABLED: process.env.OLIST_SYNC_ENABLED === 'true', OLIST_SYNC_ENABLED: process.env.OLIST_SYNC_ENABLED === 'true',
OLIST_SYNC_INTERVAL_MS: Number(process.env.OLIST_SYNC_INTERVAL_MS) || 24 * 60 * 60 * 1000, OLIST_SYNC_INTERVAL_MS: Number(process.env.OLIST_SYNC_INTERVAL_MS) || 24 * 60 * 60 * 1000,
OLIST_SYNC_REQUESTS_PER_MINUTE: Number(process.env.OLIST_SYNC_REQUESTS_PER_MINUTE) || 25, OLIST_SYNC_REQUEST_DELAY_MS: Math.max(2_500, Number(process.env.OLIST_SYNC_REQUEST_DELAY_MS) || 2_500),
TURNSTILE_SITE_KEY, TURNSTILE_SITE_KEY,
TURNSTILE_SECRET TURNSTILE_SECRET
}; };

View File

@@ -6,7 +6,7 @@ const {
OLIST_FRONTEND_URL, OLIST_FRONTEND_URL,
OLIST_REDIRECT_URI, OLIST_REDIRECT_URI,
OLIST_SYNC_INTERVAL_MS, OLIST_SYNC_INTERVAL_MS,
OLIST_SYNC_REQUESTS_PER_MINUTE, OLIST_SYNC_REQUEST_DELAY_MS,
OLIST_TOKEN_ENCRYPTION_KEY OLIST_TOKEN_ENCRYPTION_KEY
} = require('../config'); } = require('../config');
const { upsertTinyProductComposition } = require('./productionOrderService'); const { upsertTinyProductComposition } = require('./productionOrderService');
@@ -18,6 +18,7 @@ const OAUTH_STATE_TTL_MS = 10 * 60 * 1000;
const RUN_LOCK_KEY = 8_371_204; const RUN_LOCK_KEY = 8_371_204;
let activeSync = null; let activeSync = null;
let scheduledSync = null; let scheduledSync = null;
let nextOlistRequestAt = 0;
const normalizeText = (value) => String(value || '').trim(); const normalizeText = (value) => String(value || '').trim();
const stateHash = (state) => crypto.createHash('sha256').update(state).digest('hex'); const stateHash = (state) => crypto.createHash('sha256').update(state).digest('hex');
@@ -225,7 +226,17 @@ const getAccessToken = async (forceRefresh = false) => {
return token.access_token; return token.access_token;
}; };
const sleep = (ms) => new Promise(resolve => setTimeout(resolve, ms));
const waitForOlistRequestSlot = async () => {
const now = Date.now();
const waitMs = Math.max(0, nextOlistRequestAt - now);
if (waitMs) await sleep(waitMs);
nextOlistRequestAt = Date.now() + OLIST_SYNC_REQUEST_DELAY_MS;
};
const olistRequest = async (path, retry = true) => { const olistRequest = async (path, retry = true) => {
await waitForOlistRequestSlot();
const response = await fetch(`${OLIST_API_URL}${path}`, { const response = await fetch(`${OLIST_API_URL}${path}`, {
headers: { Authorization: `Bearer ${await getAccessToken(!retry)}` } headers: { Authorization: `Bearer ${await getAccessToken(!retry)}` }
}); });
@@ -239,8 +250,6 @@ const olistRequest = async (path, retry = true) => {
return data; return data;
}; };
const sleep = (ms) => new Promise(resolve => setTimeout(resolve, ms));
const listOlistProducts = async () => { const listOlistProducts = async () => {
const products = []; const products = [];
let offset = 0; let offset = 0;
@@ -307,8 +316,6 @@ const runOlistSync = async ({ trigger = 'manual', fullSync = false } = {}) => {
const candidates = changedAfter const candidates = changedAfter
? allProducts.filter(product => !product.dataAlteracao || new Date(product.dataAlteracao).getTime() >= changedAfter.getTime()) ? allProducts.filter(product => !product.dataAlteracao || new Date(product.dataAlteracao).getTime() >= changedAfter.getTime())
: allProducts; : allProducts;
const minimumDelay = Math.ceil(60_000 / Math.max(1, OLIST_SYNC_REQUESTS_PER_MINUTE));
for (const product of candidates) { for (const product of candidates) {
summary.productsScanned += 1; summary.productsScanned += 1;
try { try {
@@ -339,7 +346,6 @@ const runOlistSync = async ({ trigger = 'manual', fullSync = false } = {}) => {
summary.details.push({ productId: product.id, sku: product.sku || '', error: error.message }); summary.details.push({ productId: product.id, sku: product.sku || '', error: error.message });
} }
} }
await sleep(minimumDelay);
} }
return await finishRun(run.id, summary); return await finishRun(run.id, summary);
} catch (error) { } catch (error) {

View File

@@ -34,7 +34,7 @@ services:
- OLIST_TOKEN_ENCRYPTION_KEY=${OLIST_TOKEN_ENCRYPTION_KEY:-} - OLIST_TOKEN_ENCRYPTION_KEY=${OLIST_TOKEN_ENCRYPTION_KEY:-}
- OLIST_SYNC_ENABLED=${OLIST_SYNC_ENABLED:-false} - OLIST_SYNC_ENABLED=${OLIST_SYNC_ENABLED:-false}
- OLIST_SYNC_INTERVAL_MS=${OLIST_SYNC_INTERVAL_MS:-86400000} - OLIST_SYNC_INTERVAL_MS=${OLIST_SYNC_INTERVAL_MS:-86400000}
- OLIST_SYNC_REQUESTS_PER_MINUTE=${OLIST_SYNC_REQUESTS_PER_MINUTE:-25} - OLIST_SYNC_REQUEST_DELAY_MS=${OLIST_SYNC_REQUEST_DELAY_MS:-2500}
- TURNSTILE_SITE_KEY=${TURNSTILE_SITE_KEY:-} - TURNSTILE_SITE_KEY=${TURNSTILE_SITE_KEY:-}
- TURNSTILE_SITEKEY=${TURNSTILE_SITEKEY:-} - TURNSTILE_SITEKEY=${TURNSTILE_SITEKEY:-}
- VITE_TURNSTILE_SITE_KEY=${VITE_TURNSTILE_SITE_KEY:-} - VITE_TURNSTILE_SITE_KEY=${VITE_TURNSTILE_SITE_KEY:-}