Throttle Olist sync requests
All checks were successful
Build and Deploy / build-and-deploy (push) Successful in 57s
All checks were successful
Build and Deploy / build-and-deploy (push) Successful in 57s
This commit is contained in:
@@ -67,6 +67,8 @@ OLIST_SYNC_ENABLED=true
|
|||||||
|
|
||||||
`OLIST_TOKEN_ENCRYPTION_KEY` must be stable between deploys because Graphs uses it to encrypt OAuth tokens stored in PostgreSQL. The initial sync is full; later scheduled runs are incremental and use the product update date. Manual JSON import remains available as a fallback.
|
`OLIST_TOKEN_ENCRYPTION_KEY` must be stable between deploys because Graphs uses it to encrypt OAuth tokens stored in PostgreSQL. The initial sync is full; later scheduled runs are incremental and use the product update date. Manual JSON import remains available as a fallback.
|
||||||
|
|
||||||
|
Graphs waits at least 2.5 seconds between every Olist API request (24 requests/minute). This keeps capacity available for other Tiny/Olist integrations. `OLIST_SYNC_REQUEST_DELAY_MS` can increase that delay but cannot lower it below 2500.
|
||||||
|
|
||||||
### Cutting Workbook Import
|
### Cutting Workbook Import
|
||||||
|
|
||||||
Manual cutting/replenishment workbooks can be normalized into JSON before they
|
Manual cutting/replenishment workbooks can be normalized into JSON before they
|
||||||
|
|||||||
@@ -44,7 +44,7 @@ module.exports = {
|
|||||||
OLIST_TOKEN_ENCRYPTION_KEY: process.env.OLIST_TOKEN_ENCRYPTION_KEY || '',
|
OLIST_TOKEN_ENCRYPTION_KEY: process.env.OLIST_TOKEN_ENCRYPTION_KEY || '',
|
||||||
OLIST_SYNC_ENABLED: process.env.OLIST_SYNC_ENABLED === 'true',
|
OLIST_SYNC_ENABLED: process.env.OLIST_SYNC_ENABLED === 'true',
|
||||||
OLIST_SYNC_INTERVAL_MS: Number(process.env.OLIST_SYNC_INTERVAL_MS) || 24 * 60 * 60 * 1000,
|
OLIST_SYNC_INTERVAL_MS: Number(process.env.OLIST_SYNC_INTERVAL_MS) || 24 * 60 * 60 * 1000,
|
||||||
OLIST_SYNC_REQUESTS_PER_MINUTE: Number(process.env.OLIST_SYNC_REQUESTS_PER_MINUTE) || 25,
|
OLIST_SYNC_REQUEST_DELAY_MS: Math.max(2_500, Number(process.env.OLIST_SYNC_REQUEST_DELAY_MS) || 2_500),
|
||||||
TURNSTILE_SITE_KEY,
|
TURNSTILE_SITE_KEY,
|
||||||
TURNSTILE_SECRET
|
TURNSTILE_SECRET
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -6,7 +6,7 @@ const {
|
|||||||
OLIST_FRONTEND_URL,
|
OLIST_FRONTEND_URL,
|
||||||
OLIST_REDIRECT_URI,
|
OLIST_REDIRECT_URI,
|
||||||
OLIST_SYNC_INTERVAL_MS,
|
OLIST_SYNC_INTERVAL_MS,
|
||||||
OLIST_SYNC_REQUESTS_PER_MINUTE,
|
OLIST_SYNC_REQUEST_DELAY_MS,
|
||||||
OLIST_TOKEN_ENCRYPTION_KEY
|
OLIST_TOKEN_ENCRYPTION_KEY
|
||||||
} = require('../config');
|
} = require('../config');
|
||||||
const { upsertTinyProductComposition } = require('./productionOrderService');
|
const { upsertTinyProductComposition } = require('./productionOrderService');
|
||||||
@@ -18,6 +18,7 @@ const OAUTH_STATE_TTL_MS = 10 * 60 * 1000;
|
|||||||
const RUN_LOCK_KEY = 8_371_204;
|
const RUN_LOCK_KEY = 8_371_204;
|
||||||
let activeSync = null;
|
let activeSync = null;
|
||||||
let scheduledSync = null;
|
let scheduledSync = null;
|
||||||
|
let nextOlistRequestAt = 0;
|
||||||
|
|
||||||
const normalizeText = (value) => String(value || '').trim();
|
const normalizeText = (value) => String(value || '').trim();
|
||||||
const stateHash = (state) => crypto.createHash('sha256').update(state).digest('hex');
|
const stateHash = (state) => crypto.createHash('sha256').update(state).digest('hex');
|
||||||
@@ -225,7 +226,17 @@ const getAccessToken = async (forceRefresh = false) => {
|
|||||||
return token.access_token;
|
return token.access_token;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
const sleep = (ms) => new Promise(resolve => setTimeout(resolve, ms));
|
||||||
|
|
||||||
|
const waitForOlistRequestSlot = async () => {
|
||||||
|
const now = Date.now();
|
||||||
|
const waitMs = Math.max(0, nextOlistRequestAt - now);
|
||||||
|
if (waitMs) await sleep(waitMs);
|
||||||
|
nextOlistRequestAt = Date.now() + OLIST_SYNC_REQUEST_DELAY_MS;
|
||||||
|
};
|
||||||
|
|
||||||
const olistRequest = async (path, retry = true) => {
|
const olistRequest = async (path, retry = true) => {
|
||||||
|
await waitForOlistRequestSlot();
|
||||||
const response = await fetch(`${OLIST_API_URL}${path}`, {
|
const response = await fetch(`${OLIST_API_URL}${path}`, {
|
||||||
headers: { Authorization: `Bearer ${await getAccessToken(!retry)}` }
|
headers: { Authorization: `Bearer ${await getAccessToken(!retry)}` }
|
||||||
});
|
});
|
||||||
@@ -239,8 +250,6 @@ const olistRequest = async (path, retry = true) => {
|
|||||||
return data;
|
return data;
|
||||||
};
|
};
|
||||||
|
|
||||||
const sleep = (ms) => new Promise(resolve => setTimeout(resolve, ms));
|
|
||||||
|
|
||||||
const listOlistProducts = async () => {
|
const listOlistProducts = async () => {
|
||||||
const products = [];
|
const products = [];
|
||||||
let offset = 0;
|
let offset = 0;
|
||||||
@@ -307,8 +316,6 @@ const runOlistSync = async ({ trigger = 'manual', fullSync = false } = {}) => {
|
|||||||
const candidates = changedAfter
|
const candidates = changedAfter
|
||||||
? allProducts.filter(product => !product.dataAlteracao || new Date(product.dataAlteracao).getTime() >= changedAfter.getTime())
|
? allProducts.filter(product => !product.dataAlteracao || new Date(product.dataAlteracao).getTime() >= changedAfter.getTime())
|
||||||
: allProducts;
|
: allProducts;
|
||||||
const minimumDelay = Math.ceil(60_000 / Math.max(1, OLIST_SYNC_REQUESTS_PER_MINUTE));
|
|
||||||
|
|
||||||
for (const product of candidates) {
|
for (const product of candidates) {
|
||||||
summary.productsScanned += 1;
|
summary.productsScanned += 1;
|
||||||
try {
|
try {
|
||||||
@@ -339,7 +346,6 @@ const runOlistSync = async ({ trigger = 'manual', fullSync = false } = {}) => {
|
|||||||
summary.details.push({ productId: product.id, sku: product.sku || '', error: error.message });
|
summary.details.push({ productId: product.id, sku: product.sku || '', error: error.message });
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
await sleep(minimumDelay);
|
|
||||||
}
|
}
|
||||||
return await finishRun(run.id, summary);
|
return await finishRun(run.id, summary);
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
|
|||||||
@@ -34,7 +34,7 @@ services:
|
|||||||
- OLIST_TOKEN_ENCRYPTION_KEY=${OLIST_TOKEN_ENCRYPTION_KEY:-}
|
- OLIST_TOKEN_ENCRYPTION_KEY=${OLIST_TOKEN_ENCRYPTION_KEY:-}
|
||||||
- OLIST_SYNC_ENABLED=${OLIST_SYNC_ENABLED:-false}
|
- OLIST_SYNC_ENABLED=${OLIST_SYNC_ENABLED:-false}
|
||||||
- OLIST_SYNC_INTERVAL_MS=${OLIST_SYNC_INTERVAL_MS:-86400000}
|
- OLIST_SYNC_INTERVAL_MS=${OLIST_SYNC_INTERVAL_MS:-86400000}
|
||||||
- OLIST_SYNC_REQUESTS_PER_MINUTE=${OLIST_SYNC_REQUESTS_PER_MINUTE:-25}
|
- OLIST_SYNC_REQUEST_DELAY_MS=${OLIST_SYNC_REQUEST_DELAY_MS:-2500}
|
||||||
- TURNSTILE_SITE_KEY=${TURNSTILE_SITE_KEY:-}
|
- TURNSTILE_SITE_KEY=${TURNSTILE_SITE_KEY:-}
|
||||||
- TURNSTILE_SITEKEY=${TURNSTILE_SITEKEY:-}
|
- TURNSTILE_SITEKEY=${TURNSTILE_SITEKEY:-}
|
||||||
- VITE_TURNSTILE_SITE_KEY=${VITE_TURNSTILE_SITE_KEY:-}
|
- VITE_TURNSTILE_SITE_KEY=${VITE_TURNSTILE_SITE_KEY:-}
|
||||||
|
|||||||
Reference in New Issue
Block a user