fix: slow and control Graphs backfill

src/index.ts

@@ -3,6 +3,7 @@ import dotenv from 'dotenv';
 import webhookRoutes from './routes/webhook.route';
 import { spawn } from 'child_process';
 import path from 'path';
+import fs from 'fs';
 
 dotenv.config();
 
@@ -50,6 +51,7 @@ app.get('/api/trigger-graphs-backfill', (req: Request, res: Response) => {
 
   const scriptPath = path.join(__dirname, 'scripts', 'backfill-graphs.js');
   const env = { ...process.env };
+  const stopFile = env.GRAPHS_BACKFILL_STOP_FILE || path.join(process.cwd(), 'graphs_backfill_stop.json');
 
   const optionalEnvMap: Record<string, string> = {
     start: 'GRAPHS_BACKFILL_START_DATE',
@@ -57,7 +59,10 @@ app.get('/api/trigger-graphs-backfill', (req: Request, res: Response) => {
     dryRun: 'GRAPHS_BACKFILL_DRY_RUN',
     maxOrders: 'GRAPHS_BACKFILL_MAX_ORDERS',
     maxDays: 'GRAPHS_BACKFILL_MAX_DAYS',
-    contactFallback: 'GRAPHS_BACKFILL_CONTACT_FALLBACK'
+    contactFallback: 'GRAPHS_BACKFILL_CONTACT_FALLBACK',
+    orderDelayMs: 'GRAPHS_BACKFILL_ORDER_DELAY_MS',
+    blockDelayMs: 'GRAPHS_BACKFILL_TINY_BLOCK_DELAY_MS',
+    maxTinyBlockRetries: 'GRAPHS_BACKFILL_MAX_TINY_BLOCK_RETRIES'
   };
 
   for (const [queryKey, envKey] of Object.entries(optionalEnvMap)) {
@@ -67,6 +72,10 @@ app.get('/api/trigger-graphs-backfill', (req: Request, res: Response) => {
     }
   }
 
+  if (fs.existsSync(stopFile)) {
+    fs.rmSync(stopFile, { force: true });
+  }
+
   const child = spawn('node', [scriptPath], {
     detached: true,
     stdio: 'inherit',
@@ -85,11 +94,61 @@ app.get('/api/trigger-graphs-backfill', (req: Request, res: Response) => {
       dryRun: env.GRAPHS_BACKFILL_DRY_RUN || 'false',
       maxOrders: env.GRAPHS_BACKFILL_MAX_ORDERS || null,
       maxDays: env.GRAPHS_BACKFILL_MAX_DAYS || null,
-      contactFallback: env.GRAPHS_BACKFILL_CONTACT_FALLBACK || 'true'
+      contactFallback: env.GRAPHS_BACKFILL_CONTACT_FALLBACK || 'true',
+      orderDelayMs: env.GRAPHS_BACKFILL_ORDER_DELAY_MS || '6000',
+      blockDelayMs: env.GRAPHS_BACKFILL_TINY_BLOCK_DELAY_MS || '600000',
+      maxTinyBlockRetries: env.GRAPHS_BACKFILL_MAX_TINY_BLOCK_RETRIES || null,
+      graphsApiUrlConfigured: Boolean(env.GRAPHS_API_URL || env.NEXSTAR_GRAPHS_API_URL),
+      graphsApiKeyConfigured: Boolean(env.GRAPHS_API_KEY || env.NEXSTAR_GRAPHS_API_KEY)
     }
   });
 });
 
+app.get('/api/graphs-backfill-status', (req: Request, res: Response) => {
+  const expectedToken = process.env.TINY_WEBHOOK_SECRET;
+  if (expectedToken && req.query.token !== expectedToken) {
+    res.status(401).json({ error: 'Unauthorized' });
+    return;
+  }
+
+  const stateFile = process.env.GRAPHS_BACKFILL_STATE_FILE || path.join(process.cwd(), 'graphs_backfill_state.json');
+  const stopFile = process.env.GRAPHS_BACKFILL_STOP_FILE || path.join(process.cwd(), 'graphs_backfill_stop.json');
+  let state = null;
+
+  if (fs.existsSync(stateFile)) {
+    try {
+      state = JSON.parse(fs.readFileSync(stateFile, 'utf-8'));
+    } catch (error: any) {
+      res.status(500).json({ error: `Could not read backfill state: ${error.message}` });
+      return;
+    }
+  }
+
+  res.status(200).json({
+    status: state?.status || 'unknown',
+    stopRequested: fs.existsSync(stopFile),
+    state
+  });
+});
+
+app.get('/api/stop-graphs-backfill', (req: Request, res: Response) => {
+  const expectedToken = process.env.TINY_WEBHOOK_SECRET;
+  if (expectedToken && req.query.token !== expectedToken) {
+    res.status(401).json({ error: 'Unauthorized' });
+    return;
+  }
+
+  const stopFile = process.env.GRAPHS_BACKFILL_STOP_FILE || path.join(process.cwd(), 'graphs_backfill_stop.json');
+  fs.writeFileSync(stopFile, JSON.stringify({
+    requestedAt: new Date().toISOString()
+  }, null, 2));
+
+  res.status(200).json({
+    status: 'STOP_REQUESTED',
+    message: 'Graphs backfill will stop after the current wait/request finishes.'
+  });
+});
+
 // Hidden endpoint to download the stock CSV log directly from the browser
 app.get('/api/stock-logs/download', (req: Request, res: Response) => {
   const expectedToken = process.env.TINY_WEBHOOK_SECRET;