From e0359e07522ec1a30ae6472c0e7b29c0e2e261b5 Mon Sep 17 00:00:00 2001 From: ciregenz Date: Tue, 2 Jun 2026 04:08:10 -0700 Subject: [PATCH] [eric] browser: passively capture redacted shadow-API routes while browsing (tier 2) --- electron/cdp-routes.js | 121 ++++++++++++++++++++++++++++++++++++ electron/cdp-routes.test.js | 99 +++++++++++++++++++++++++++++ electron/main.js | 32 +++++++++- electron/preload.js | 1 + 4 files changed, 252 insertions(+), 1 deletion(-) create mode 100644 electron/cdp-routes.js create mode 100644 electron/cdp-routes.test.js diff --git a/electron/cdp-routes.js b/electron/cdp-routes.js new file mode 100644 index 00000000..d6ae602d --- /dev/null +++ b/electron/cdp-routes.js @@ -0,0 +1,121 @@ +// Pure helpers for tier-2 shadow-API route capture. No Electron deps so this +// unit-tests under plain node; main.js wires the CDP Network events to it. +// +// The idea: a site's UI is a shell over its own internal HTTP API. While the +// agent drives the UI we passively record the XHR/fetch endpoints the page +// fires, so a later task can replay a safe (GET) one directly instead of +// re-scraping. We NEVER persist secrets: auth/cookie/csrf headers are redacted, +// and only the body's KEY SHAPE is kept (no values). + +// Prefix match (no end-anchor) so header variants like x-csrf-token and +// x-auth-token are caught too; over-redacting a secret-shaped header is the +// safe direction. +const REDACT = /^(authorization|cookie|set-cookie|proxy-authorization|x-csrf|x-xsrf|x-api-key|x-auth)/i; +const CAPTURE_RESOURCE_TYPES = new Set(['XHR', 'Fetch']); +const MAX_ROUTES_PER_WC = 200; // bound memory; evict least-recently-seen past this + +// Collapse volatile path segments (numeric ids, long hex / uuids) so +// /orders/4821 and /orders/4822 share one route template. +function templateUrl(raw) { + try { + const u = new URL(raw); + const path = u.pathname.replace( + /\/(\d+|[0-9a-fA-F]{8,}(?:-[0-9a-fA-F]+)*)(?=\/|$)/g, + '/{id}', + ); + const keys = [...u.searchParams.keys()].sort().join(','); + return u.origin + path + (keys ? '?' + keys : ''); + } catch { + return raw; + } +} + +function redactHeaders(headers) { + const out = {}; + for (const k of Object.keys(headers || {})) { + out[k] = REDACT.test(k) ? '' : headers[k]; + } + return out; +} + +// Keep the JSON body's key skeleton with value TYPES, never values. +function bodyShape(postData) { + if (!postData) return null; + try { + const skel = (v) => + Array.isArray(v) + ? [v.length ? skel(v[0]) : 'empty'] + : v && typeof v === 'object' + ? Object.fromEntries(Object.keys(v).map((k) => [k, skel(v[k])])) + : typeof v; + return skel(JSON.parse(postData)); + } catch { + return 'raw'; + } +} + +function isSafeMethod(method) { + const m = String(method || '').toUpperCase(); + return m === 'GET' || m === 'HEAD'; +} + +function shouldCapture(resourceType) { + return CAPTURE_RESOURCE_TYPES.has(resourceType); +} + +function routeKey(method, template) { + return String(method || 'GET').toUpperCase() + ' ' + template; +} + +function makeRouteEntry(request, resourceType) { + const method = String(request.method || 'GET').toUpperCase(); + const template = templateUrl(request.url); + return { + method, + template, + resourceType, + headers: redactHeaders(request.headers), + bodyShape: bodyShape(request.postData), + safe: isSafeMethod(method), + hits: 1, + lastSeen: Date.now(), + }; +} + +// Merge a captured request into a per-wc Map. Dedupes by +// (method, templated url): repeats just bump the hit count. Evicts the +// least-recently-seen entry past the cap. +function recordRoute(routesMap, request, resourceType, now = Date.now()) { + if (!shouldCapture(resourceType)) return; + const entry = makeRouteEntry(request, resourceType); + entry.lastSeen = now; + const key = routeKey(entry.method, entry.template); + const existing = routesMap.get(key); + if (existing) { + existing.hits += 1; + existing.lastSeen = now; + } else { + routesMap.set(key, entry); + if (routesMap.size > MAX_ROUTES_PER_WC) { + let oldestKey = null; + let oldest = Infinity; + for (const [k, v] of routesMap) { + if (v.lastSeen < oldest) { oldest = v.lastSeen; oldestKey = k; } + } + if (oldestKey) routesMap.delete(oldestKey); + } + } +} + +module.exports = { + templateUrl, + redactHeaders, + bodyShape, + isSafeMethod, + shouldCapture, + routeKey, + makeRouteEntry, + recordRoute, + CAPTURE_RESOURCE_TYPES, + MAX_ROUTES_PER_WC, +}; diff --git a/electron/cdp-routes.test.js b/electron/cdp-routes.test.js new file mode 100644 index 00000000..8dad08d7 --- /dev/null +++ b/electron/cdp-routes.test.js @@ -0,0 +1,99 @@ +// Run: node --test electron/cdp-routes.test.js +const { test } = require('node:test'); +const assert = require('node:assert/strict'); +const R = require('./cdp-routes'); + +test('templateUrl collapses numeric ids', () => { + assert.equal(R.templateUrl('https://x.com/api/orders/4821'), 'https://x.com/api/orders/{id}'); + assert.equal(R.templateUrl('https://x.com/api/orders/4821/items/9'), 'https://x.com/api/orders/{id}/items/{id}'); +}); + +test('templateUrl collapses hex/uuid ids', () => { + assert.equal( + R.templateUrl('https://x.com/u/3f9a8b7c6d5e4f3a/profile'), + 'https://x.com/u/{id}/profile', + ); + assert.equal( + R.templateUrl('https://x.com/r/550e8400-e29b-41d4-a716-446655440000'), + 'https://x.com/r/{id}', + ); +}); + +test('templateUrl keeps sorted query keys, drops values', () => { + assert.equal(R.templateUrl('https://x.com/search?q=shoes&page=2'), 'https://x.com/search?page,q'); + assert.equal(R.templateUrl('https://x.com/search?q=hats'), 'https://x.com/search?q'); +}); + +test('templateUrl returns input unchanged on garbage', () => { + assert.equal(R.templateUrl('not a url'), 'not a url'); +}); + +test('redactHeaders scrubs secret headers, keeps the rest', () => { + const out = R.redactHeaders({ + Authorization: 'Bearer abc', Cookie: 'sid=xyz', 'X-CSRF-Token': 't', + 'Content-Type': 'application/json', Accept: '*/*', + }); + assert.equal(out.Authorization, ''); + assert.equal(out.Cookie, ''); + assert.equal(out['X-CSRF-Token'], ''); + assert.equal(out['Content-Type'], 'application/json'); + assert.equal(out.Accept, '*/*'); +}); + +test('bodyShape keeps key skeleton with value TYPES, never values', () => { + const shape = R.bodyShape(JSON.stringify({ id: 7, name: 'secret-name', tags: ['a'], nested: { x: true } })); + assert.deepEqual(shape, { id: 'number', name: 'string', tags: ['string'], nested: { x: 'boolean' } }); + // crucially the real value "secret-name" is not present anywhere + assert.ok(!JSON.stringify(shape).includes('secret-name')); +}); + +test('bodyShape handles non-JSON and empty', () => { + assert.equal(R.bodyShape(''), 'raw'); + assert.equal(R.bodyShape(null), null); +}); + +test('isSafeMethod only GET/HEAD', () => { + assert.equal(R.isSafeMethod('get'), true); + assert.equal(R.isSafeMethod('HEAD'), true); + assert.equal(R.isSafeMethod('POST'), false); + assert.equal(R.isSafeMethod('DELETE'), false); +}); + +test('shouldCapture only XHR/Fetch', () => { + assert.equal(R.shouldCapture('XHR'), true); + assert.equal(R.shouldCapture('Fetch'), true); + assert.equal(R.shouldCapture('Document'), false); + assert.equal(R.shouldCapture('Image'), false); + assert.equal(R.shouldCapture('Script'), false); +}); + +test('recordRoute dedupes by (method,template) and counts hits', () => { + const m = new Map(); + R.recordRoute(m, { method: 'GET', url: 'https://x.com/api/orders/1', headers: {} }, 'XHR'); + R.recordRoute(m, { method: 'GET', url: 'https://x.com/api/orders/2', headers: {} }, 'XHR'); + assert.equal(m.size, 1); + assert.equal([...m.values()][0].hits, 2); + assert.equal([...m.values()][0].safe, true); +}); + +test('recordRoute marks non-GET unsafe and skips non-XHR resource types', () => { + const m = new Map(); + R.recordRoute(m, { method: 'POST', url: 'https://x.com/api/cart', headers: {}, postData: '{"id":1}' }, 'Fetch'); + R.recordRoute(m, { method: 'GET', url: 'https://x.com/page.css', headers: {} }, 'Stylesheet'); + assert.equal(m.size, 1); + const e = [...m.values()][0]; + assert.equal(e.method, 'POST'); + assert.equal(e.safe, false); + assert.deepEqual(e.bodyShape, { id: 'number' }); +}); + +test('recordRoute evicts least-recently-seen past the cap', () => { + const m = new Map(); + let t = 1000; + for (let i = 0; i < R.MAX_ROUTES_PER_WC + 5; i++) { + R.recordRoute(m, { method: 'GET', url: `https://x.com/p${i}/a`, headers: {} }, 'XHR', t++); + } + assert.equal(m.size, R.MAX_ROUTES_PER_WC); + // the earliest few paths should have been evicted + assert.ok(!m.has('GET https://x.com/p0/a')); +}); diff --git a/electron/main.js b/electron/main.js index c84794d2..5923a9d3 100644 --- a/electron/main.js +++ b/electron/main.js @@ -53,6 +53,7 @@ const fs = require('fs'); const getPort = require('get-port'); const http = require('http'); const affiliateTracking = require('./affiliateTracking'); +const cdpRoutes = require('./cdp-routes'); // Squirrel makes the APP create its own shortcuts: on --squirrel-install it must // call Update.exe --createShortcut and exit, else the user finds only Setup.exe @@ -1877,6 +1878,7 @@ app.on('web-contents-created', (_event, contents) => { cdpQueueByWcId.delete(contents.id); cdpChildSessions.delete(contents.id); cdpAutoAttachWired.delete(contents.id); + cdpRoutesByWcId.delete(contents.id); }); contents.on('render-process-gone', () => { @@ -1884,6 +1886,7 @@ app.on('web-contents-created', (_event, contents) => { cdpQueueByWcId.delete(contents.id); cdpChildSessions.delete(contents.id); cdpAutoAttachWired.delete(contents.id); + cdpRoutesByWcId.delete(contents.id); }); // WebAuthn/passkey shim. Injected on every dom-ready in the main world @@ -2242,15 +2245,27 @@ const cdpQueueByWcId = new Map(); // wcId -> Promise (serialization tail) // attached child-frame session and where it sits in the frame tree. const cdpChildSessions = new Map(); // wcId -> Map const cdpAutoAttachWired = new Set(); // wcIds whose 'message' listener is attached +const cdpRoutesByWcId = new Map(); // wcId -> Map (tier-2 shadow-API capture) function wireChildSessions(wc) { const wcId = wc.id; if (cdpAutoAttachWired.has(wcId)) return; cdpAutoAttachWired.add(wcId); cdpChildSessions.set(wcId, new Map()); + cdpRoutesByWcId.set(wcId, new Map()); wc.debugger.on('message', (_e, method, params, sessionId) => { const sessions = cdpChildSessions.get(wcId); if (!sessions) return; + if (method === 'Network.requestWillBeSent') { + // Tier-2 passive shadow-API capture: record the XHR/fetch endpoints the + // page fires (from root or any child session) so a later task can replay + // a safe one. Secrets are redacted inside cdp-routes. + const routes = cdpRoutesByWcId.get(wcId); + if (routes && params && params.request) { + cdpRoutes.recordRoute(routes, params.request, params.type); + } + return; + } if (method === 'Target.attachedToTarget') { const info = params.targetInfo || {}; if (info.type !== 'iframe') return; @@ -2260,10 +2275,11 @@ function wireChildSessions(wc) { parentSessionId: sessionId || null, url: info.url || '', }); - // Enable perception domains and propagate auto-attach into nested OOPIF. + // Enable perception + network domains and propagate auto-attach into nested OOPIF. const sid = params.sessionId; wc.debugger.sendCommand('Accessibility.enable', {}, sid).catch(() => {}); wc.debugger.sendCommand('DOM.enable', {}, sid).catch(() => {}); + wc.debugger.sendCommand('Network.enable', {}, sid).catch(() => {}); wc.debugger.sendCommand('Target.setAutoAttach', { autoAttach: true, waitForDebuggerOnStart: false, flatten: true }, sid).catch(() => {}); } else if (method === 'Target.detachedFromTarget') { @@ -2296,6 +2312,10 @@ async function ensureDebuggerAttached(wc) { await wc.debugger.sendCommand('Target.setAutoAttach', { autoAttach: true, waitForDebuggerOnStart: false, flatten: true }); } catch (_) {} + // Tier-2: record the page's own XHR/fetch endpoints as the agent drives it. + try { + await wc.debugger.sendCommand('Network.enable', {}); + } catch (_) {} } async function sendCdpCommandSerialized(wcId, method, params, sessionId) { @@ -2358,6 +2378,16 @@ ipcMain.handle('cdp-child-sessions-get', (_event, wcId) => { return [...m.entries()].map(([sessionId, info]) => ({ sessionId, ...info })); }); +// Tier-2: captured shadow-API routes for a webContents, newest-busiest first, +// optionally filtered to an origin. Secrets were already redacted at capture. +ipcMain.handle('cdp-routes-get', (_event, wcId, originFilter) => { + const m = cdpRoutesByWcId.get(wcId); + if (!m) return []; + let list = [...m.values()]; + if (originFilter) list = list.filter((r) => r.template.startsWith(originFilter)); + return list.sort((a, b) => b.hits - a.hits || b.lastSeen - a.lastSeen); +}); + ipcMain.handle('connect-slack', async () => { const win = new BrowserWindow({ width: 900, diff --git a/electron/preload.js b/electron/preload.js index 9b081498..e8dff1fb 100644 --- a/electron/preload.js +++ b/electron/preload.js @@ -55,6 +55,7 @@ contextBridge.exposeInMainWorld('openswarm', { cdpCacheGet: (wcId) => ipcRenderer.invoke('cdp-cache-get', wcId), cdpCacheClear: (wcId) => ipcRenderer.invoke('cdp-cache-clear', wcId), cdpChildSessionsGet: (wcId) => ipcRenderer.invoke('cdp-child-sessions-get', wcId), + cdpRoutesGet: (wcId, originFilter) => ipcRenderer.invoke('cdp-routes-get', wcId, originFilter), capturePage: (rect) => ipcRenderer.invoke('capture-page', rect), getUpdateStatus: () => ipcRenderer.invoke('get-update-status'), getCrashRecoveryInfo: () => ipcRenderer.invoke('get-crash-recovery-info'),