[eric] browser: passively capture redacted shadow-API routes while browsing (tier 2)

electron/cdp-routes.js

@@ -0,0 +1,121 @@
+// Pure helpers for tier-2 shadow-API route capture. No Electron deps so this
+// unit-tests under plain node; main.js wires the CDP Network events to it.
+//
+// The idea: a site's UI is a shell over its own internal HTTP API. While the
+// agent drives the UI we passively record the XHR/fetch endpoints the page
+// fires, so a later task can replay a safe (GET) one directly instead of
+// re-scraping. We NEVER persist secrets: auth/cookie/csrf headers are redacted,
+// and only the body's KEY SHAPE is kept (no values).
+
+// Prefix match (no end-anchor) so header variants like x-csrf-token and
+// x-auth-token are caught too; over-redacting a secret-shaped header is the
+// safe direction.
+const REDACT = /^(authorization|cookie|set-cookie|proxy-authorization|x-csrf|x-xsrf|x-api-key|x-auth)/i;
+const CAPTURE_RESOURCE_TYPES = new Set(['XHR', 'Fetch']);
+const MAX_ROUTES_PER_WC = 200; // bound memory; evict least-recently-seen past this
+
+// Collapse volatile path segments (numeric ids, long hex / uuids) so
+// /orders/4821 and /orders/4822 share one route template.
+function templateUrl(raw) {
+  try {
+    const u = new URL(raw);
+    const path = u.pathname.replace(
+      /\/(\d+|[0-9a-fA-F]{8,}(?:-[0-9a-fA-F]+)*)(?=\/|$)/g,
+      '/{id}',
+    );
+    const keys = [...u.searchParams.keys()].sort().join(',');
+    return u.origin + path + (keys ? '?' + keys : '');
+  } catch {
+    return raw;
+  }
+}
+
+function redactHeaders(headers) {
+  const out = {};
+  for (const k of Object.keys(headers || {})) {
+    out[k] = REDACT.test(k) ? '<redacted>' : headers[k];
+  }
+  return out;
+}
+
+// Keep the JSON body's key skeleton with value TYPES, never values.
+function bodyShape(postData) {
+  if (!postData) return null;
+  try {
+    const skel = (v) =>
+      Array.isArray(v)
+        ? [v.length ? skel(v[0]) : 'empty']
+        : v && typeof v === 'object'
+          ? Object.fromEntries(Object.keys(v).map((k) => [k, skel(v[k])]))
+          : typeof v;
+    return skel(JSON.parse(postData));
+  } catch {
+    return 'raw';
+  }
+}
+
+function isSafeMethod(method) {
+  const m = String(method || '').toUpperCase();
+  return m === 'GET' || m === 'HEAD';
+}
+
+function shouldCapture(resourceType) {
+  return CAPTURE_RESOURCE_TYPES.has(resourceType);
+}
+
+function routeKey(method, template) {
+  return String(method || 'GET').toUpperCase() + ' ' + template;
+}
+
+function makeRouteEntry(request, resourceType) {
+  const method = String(request.method || 'GET').toUpperCase();
+  const template = templateUrl(request.url);
+  return {
+    method,
+    template,
+    resourceType,
+    headers: redactHeaders(request.headers),
+    bodyShape: bodyShape(request.postData),
+    safe: isSafeMethod(method),
+    hits: 1,
+    lastSeen: Date.now(),
+  };
+}
+
+// Merge a captured request into a per-wc Map<routeKey, entry>. Dedupes by
+// (method, templated url): repeats just bump the hit count. Evicts the
+// least-recently-seen entry past the cap.
+function recordRoute(routesMap, request, resourceType, now = Date.now()) {
+  if (!shouldCapture(resourceType)) return;
+  const entry = makeRouteEntry(request, resourceType);
+  entry.lastSeen = now;
+  const key = routeKey(entry.method, entry.template);
+  const existing = routesMap.get(key);
+  if (existing) {
+    existing.hits += 1;
+    existing.lastSeen = now;
+  } else {
+    routesMap.set(key, entry);
+    if (routesMap.size > MAX_ROUTES_PER_WC) {
+      let oldestKey = null;
+      let oldest = Infinity;
+      for (const [k, v] of routesMap) {
+        if (v.lastSeen < oldest) { oldest = v.lastSeen; oldestKey = k; }
+      }
+      if (oldestKey) routesMap.delete(oldestKey);
+    }
+  }
+}
+
+module.exports = {
+  templateUrl,
+  redactHeaders,
+  bodyShape,
+  isSafeMethod,
+  shouldCapture,
+  routeKey,
+  makeRouteEntry,
+  recordRoute,
+  CAPTURE_RESOURCE_TYPES,
+  MAX_ROUTES_PER_WC,
+};

electron/cdp-routes.test.js

@@ -0,0 +1,99 @@
+// Run: node --test electron/cdp-routes.test.js
+const { test } = require('node:test');
+const assert = require('node:assert/strict');
+const R = require('./cdp-routes');
+
+test('templateUrl collapses numeric ids', () => {
+  assert.equal(R.templateUrl('https://x.com/api/orders/4821'), 'https://x.com/api/orders/{id}');
+  assert.equal(R.templateUrl('https://x.com/api/orders/4821/items/9'), 'https://x.com/api/orders/{id}/items/{id}');
+});
+
+test('templateUrl collapses hex/uuid ids', () => {
+  assert.equal(
+    R.templateUrl('https://x.com/u/3f9a8b7c6d5e4f3a/profile'),
+    'https://x.com/u/{id}/profile',
+  );
+  assert.equal(
+    R.templateUrl('https://x.com/r/550e8400-e29b-41d4-a716-446655440000'),
+    'https://x.com/r/{id}',
+  );
+});
+
+test('templateUrl keeps sorted query keys, drops values', () => {
+  assert.equal(R.templateUrl('https://x.com/search?q=shoes&page=2'), 'https://x.com/search?page,q');
+  assert.equal(R.templateUrl('https://x.com/search?q=hats'), 'https://x.com/search?q');
+});
+
+test('templateUrl returns input unchanged on garbage', () => {
+  assert.equal(R.templateUrl('not a url'), 'not a url');
+});
+
+test('redactHeaders scrubs secret headers, keeps the rest', () => {
+  const out = R.redactHeaders({
+    Authorization: 'Bearer abc', Cookie: 'sid=xyz', 'X-CSRF-Token': 't',
+    'Content-Type': 'application/json', Accept: '*/*',
+  });
+  assert.equal(out.Authorization, '<redacted>');
+  assert.equal(out.Cookie, '<redacted>');
+  assert.equal(out['X-CSRF-Token'], '<redacted>');
+  assert.equal(out['Content-Type'], 'application/json');
+  assert.equal(out.Accept, '*/*');
+});
+
+test('bodyShape keeps key skeleton with value TYPES, never values', () => {
+  const shape = R.bodyShape(JSON.stringify({ id: 7, name: 'secret-name', tags: ['a'], nested: { x: true } }));
+  assert.deepEqual(shape, { id: 'number', name: 'string', tags: ['string'], nested: { x: 'boolean' } });
+  // crucially the real value "secret-name" is not present anywhere
+  assert.ok(!JSON.stringify(shape).includes('secret-name'));
+});
+
+test('bodyShape handles non-JSON and empty', () => {
+  assert.equal(R.bodyShape('<xml/>'), 'raw');
+  assert.equal(R.bodyShape(null), null);
+});
+
+test('isSafeMethod only GET/HEAD', () => {
+  assert.equal(R.isSafeMethod('get'), true);
+  assert.equal(R.isSafeMethod('HEAD'), true);
+  assert.equal(R.isSafeMethod('POST'), false);
+  assert.equal(R.isSafeMethod('DELETE'), false);
+});
+
+test('shouldCapture only XHR/Fetch', () => {
+  assert.equal(R.shouldCapture('XHR'), true);
+  assert.equal(R.shouldCapture('Fetch'), true);
+  assert.equal(R.shouldCapture('Document'), false);
+  assert.equal(R.shouldCapture('Image'), false);
+  assert.equal(R.shouldCapture('Script'), false);
+});
+
+test('recordRoute dedupes by (method,template) and counts hits', () => {
+  const m = new Map();
+  R.recordRoute(m, { method: 'GET', url: 'https://x.com/api/orders/1', headers: {} }, 'XHR');
+  R.recordRoute(m, { method: 'GET', url: 'https://x.com/api/orders/2', headers: {} }, 'XHR');
+  assert.equal(m.size, 1);
+  assert.equal([...m.values()][0].hits, 2);
+  assert.equal([...m.values()][0].safe, true);
+});
+
+test('recordRoute marks non-GET unsafe and skips non-XHR resource types', () => {
+  const m = new Map();
+  R.recordRoute(m, { method: 'POST', url: 'https://x.com/api/cart', headers: {}, postData: '{"id":1}' }, 'Fetch');
+  R.recordRoute(m, { method: 'GET', url: 'https://x.com/page.css', headers: {} }, 'Stylesheet');
+  assert.equal(m.size, 1);
+  const e = [...m.values()][0];
+  assert.equal(e.method, 'POST');
+  assert.equal(e.safe, false);
+  assert.deepEqual(e.bodyShape, { id: 'number' });
+});
+
+test('recordRoute evicts least-recently-seen past the cap', () => {
+  const m = new Map();
+  let t = 1000;
+  for (let i = 0; i < R.MAX_ROUTES_PER_WC + 5; i++) {
+    R.recordRoute(m, { method: 'GET', url: `https://x.com/p${i}/a`, headers: {} }, 'XHR', t++);
+  }
+  assert.equal(m.size, R.MAX_ROUTES_PER_WC);
+  // the earliest few paths should have been evicted
+  assert.ok(!m.has('GET https://x.com/p0/a'));
+});

electron/main.js

@@ -53,6 +53,7 @@ const fs = require('fs');
 const getPort = require('get-port');
 const http = require('http');
 const affiliateTracking = require('./affiliateTracking');
+const cdpRoutes = require('./cdp-routes');
 
 // Squirrel makes the APP create its own shortcuts: on --squirrel-install it must
 // call Update.exe --createShortcut and exit, else the user finds only Setup.exe
@@ -1877,6 +1878,7 @@ app.on('web-contents-created', (_event, contents) => {
       cdpQueueByWcId.delete(contents.id);
       cdpChildSessions.delete(contents.id);
       cdpAutoAttachWired.delete(contents.id);
+      cdpRoutesByWcId.delete(contents.id);
     });
 
     contents.on('render-process-gone', () => {
@@ -1884,6 +1886,7 @@ app.on('web-contents-created', (_event, contents) => {
       cdpQueueByWcId.delete(contents.id);
       cdpChildSessions.delete(contents.id);
       cdpAutoAttachWired.delete(contents.id);
+      cdpRoutesByWcId.delete(contents.id);
     });
 
     // WebAuthn/passkey shim. Injected on every dom-ready in the main world
@@ -2242,15 +2245,27 @@ const cdpQueueByWcId = new Map();  // wcId -> Promise (serialization tail)
 // attached child-frame session and where it sits in the frame tree.
 const cdpChildSessions = new Map();   // wcId -> Map<sessionId, {frameId, parentSessionId, url}>
 const cdpAutoAttachWired = new Set(); // wcIds whose 'message' listener is attached
+const cdpRoutesByWcId = new Map();    // wcId -> Map<routeKey, entry> (tier-2 shadow-API capture)
 
 function wireChildSessions(wc) {
   const wcId = wc.id;
   if (cdpAutoAttachWired.has(wcId)) return;
   cdpAutoAttachWired.add(wcId);
   cdpChildSessions.set(wcId, new Map());
+  cdpRoutesByWcId.set(wcId, new Map());
   wc.debugger.on('message', (_e, method, params, sessionId) => {
     const sessions = cdpChildSessions.get(wcId);
     if (!sessions) return;
+    if (method === 'Network.requestWillBeSent') {
+      // Tier-2 passive shadow-API capture: record the XHR/fetch endpoints the
+      // page fires (from root or any child session) so a later task can replay
+      // a safe one. Secrets are redacted inside cdp-routes.
+      const routes = cdpRoutesByWcId.get(wcId);
+      if (routes && params && params.request) {
+        cdpRoutes.recordRoute(routes, params.request, params.type);
+      }
+      return;
+    }
     if (method === 'Target.attachedToTarget') {
       const info = params.targetInfo || {};
       if (info.type !== 'iframe') return;
@@ -2260,10 +2275,11 @@ function wireChildSessions(wc) {
         parentSessionId: sessionId || null,
         url: info.url || '',
       });
-      // Enable perception domains and propagate auto-attach into nested OOPIF.
+      // Enable perception + network domains and propagate auto-attach into nested OOPIF.
       const sid = params.sessionId;
       wc.debugger.sendCommand('Accessibility.enable', {}, sid).catch(() => {});
       wc.debugger.sendCommand('DOM.enable', {}, sid).catch(() => {});
+      wc.debugger.sendCommand('Network.enable', {}, sid).catch(() => {});
       wc.debugger.sendCommand('Target.setAutoAttach',
         { autoAttach: true, waitForDebuggerOnStart: false, flatten: true }, sid).catch(() => {});
     } else if (method === 'Target.detachedFromTarget') {
@@ -2296,6 +2312,10 @@ async function ensureDebuggerAttached(wc) {
     await wc.debugger.sendCommand('Target.setAutoAttach',
       { autoAttach: true, waitForDebuggerOnStart: false, flatten: true });
   } catch (_) {}
+  // Tier-2: record the page's own XHR/fetch endpoints as the agent drives it.
+  try {
+    await wc.debugger.sendCommand('Network.enable', {});
+  } catch (_) {}
 }
 
 async function sendCdpCommandSerialized(wcId, method, params, sessionId) {
@@ -2358,6 +2378,16 @@ ipcMain.handle('cdp-child-sessions-get', (_event, wcId) => {
   return [...m.entries()].map(([sessionId, info]) => ({ sessionId, ...info }));
 });
 
+// Tier-2: captured shadow-API routes for a webContents, newest-busiest first,
+// optionally filtered to an origin. Secrets were already redacted at capture.
+ipcMain.handle('cdp-routes-get', (_event, wcId, originFilter) => {
+  const m = cdpRoutesByWcId.get(wcId);
+  if (!m) return [];
+  let list = [...m.values()];
+  if (originFilter) list = list.filter((r) => r.template.startsWith(originFilter));
+  return list.sort((a, b) => b.hits - a.hits || b.lastSeen - a.lastSeen);
+});
+
 ipcMain.handle('connect-slack', async () => {
   const win = new BrowserWindow({
     width: 900,

electron/preload.js

@@ -55,6 +55,7 @@ contextBridge.exposeInMainWorld('openswarm', {
   cdpCacheGet: (wcId) => ipcRenderer.invoke('cdp-cache-get', wcId),
   cdpCacheClear: (wcId) => ipcRenderer.invoke('cdp-cache-clear', wcId),
   cdpChildSessionsGet: (wcId) => ipcRenderer.invoke('cdp-child-sessions-get', wcId),
+  cdpRoutesGet: (wcId, originFilter) => ipcRenderer.invoke('cdp-routes-get', wcId, originFilter),
   capturePage: (rect) => ipcRenderer.invoke('capture-page', rect),
   getUpdateStatus: () => ipcRenderer.invoke('get-update-status'),
   getCrashRecoveryInfo: () => ipcRenderer.invoke('get-crash-recovery-info'),