// Pure helpers for tier-2 shadow-API route capture. No Electron deps so this // unit-tests under plain node; main.js wires the CDP Network events to it. // // The idea: a site's UI is a shell over its own internal HTTP API. While the // agent drives the UI we passively record the XHR/fetch endpoints the page // fires, so a later task can replay a safe (GET) one directly instead of // re-scraping. We NEVER persist secrets: auth/cookie/csrf headers are redacted, // and only the body's KEY SHAPE is kept (no values). // Prefix match (no end-anchor) so header variants like x-csrf-token and // x-auth-token are caught too; over-redacting a secret-shaped header is the // safe direction. const REDACT = /^(authorization|cookie|set-cookie|proxy-authorization|x-csrf|x-xsrf|x-api-key|x-auth)/i; const CAPTURE_RESOURCE_TYPES = new Set(['XHR', 'Fetch']); const MAX_ROUTES_PER_WC = 200; // bound memory; evict least-recently-seen past this // Collapse volatile path segments (numeric ids, long hex / uuids) so // /orders/4821 and /orders/4822 share one route template. function templateUrl(raw) { try { const u = new URL(raw); const path = u.pathname.replace( /\/(\d+|[0-9a-fA-F]{8,}(?:-[0-9a-fA-F]+)*)(?=\/|$)/g, '/{id}', ); const keys = [...u.searchParams.keys()].sort().join(','); return u.origin + path + (keys ? '?' + keys : ''); } catch { return raw; } } function redactHeaders(headers) { const out = {}; for (const k of Object.keys(headers || {})) { out[k] = REDACT.test(k) ? '' : headers[k]; } return out; } // A concrete EXAMPLE url (real values) so the agent can see the param shape and // template its own input in. The template alone (value-stripped) isn't composable // for multi-param endpoints. We redact token-shaped query VALUES so a session // token / api key in the query never lands in the agent's context. The user's own // search terms stay (the agent typed them; not secret). Path stays as-is. const SECRET_PARAM = /token|secret|sig|session|password|pwd|jwt|bearer|auth|access|refresh|csrf|api[-_]?key/i; const TOKEN_PREFIX = /^(sk-|ghp_|gho_|pk_|xox[bap]-|AIza|eyJ|Bearer )/; function looksSecretValue(v) { if (!v) return false; if (TOKEN_PREFIX.test(v)) return true; return v.length >= 20 && /[A-Za-z]/.test(v) && /[0-9]/.test(v) && !/\s/.test(v); } function redactExampleUrl(raw) { try { const u = new URL(raw); for (const [k, v] of [...u.searchParams.entries()]) { if (SECRET_PARAM.test(k) || looksSecretValue(v)) { u.searchParams.set(k, ''); } } return u.origin + u.pathname + (u.search || ''); } catch { return raw; } } // Keep the JSON body's key skeleton with value TYPES, never values. function bodyShape(postData) { if (!postData) return null; try { const skel = (v) => Array.isArray(v) ? [v.length ? skel(v[0]) : 'empty'] : v && typeof v === 'object' ? Object.fromEntries(Object.keys(v).map((k) => [k, skel(v[k])])) : typeof v; return skel(JSON.parse(postData)); } catch { return 'raw'; } } function isSafeMethod(method) { const m = String(method || '').toUpperCase(); return m === 'GET' || m === 'HEAD'; } function shouldCapture(resourceType) { return CAPTURE_RESOURCE_TYPES.has(resourceType); } function routeKey(method, template) { return String(method || 'GET').toUpperCase() + ' ' + template; } // The mutating body WITH values, but secret-shaped string leaves redacted (belt; the backend // recipe learner redacts again). Only JSON, capped: a write recipe needs the real body (the user's // payload sits in one leaf), which bodyShape (types only) can't provide. Non-JSON / oversized = null. const MAX_BODY_CHARS = 65536; function redactBodyValues(postData) { if (!postData || postData.length > MAX_BODY_CHARS) return null; try { const walk = (v) => typeof v === 'string' ? (looksSecretValue(v) ? '' : v) : Array.isArray(v) ? v.map(walk) : v && typeof v === 'object' ? Object.fromEntries(Object.keys(v).map((k) => [k, walk(v[k])])) : v; return JSON.stringify(walk(JSON.parse(postData))); } catch { return null; } } function makeRouteEntry(request, resourceType) { const method = String(request.method || 'GET').toUpperCase(); const template = templateUrl(request.url); const safe = isSafeMethod(method); return { method, template, example: redactExampleUrl(request.url), resourceType, headers: redactHeaders(request.headers), bodyShape: bodyShape(request.postData), // Only mutating routes carry a replayable body; a GET's body (if any) is never a write recipe. lastBody: safe ? null : redactBodyValues(request.postData), safe, hits: 1, lastSeen: Date.now(), }; } // Merge a captured request into a per-wc Map. Dedupes by // (method, templated url): repeats just bump the hit count. Evicts the // least-recently-seen entry past the cap. function recordRoute(routesMap, request, resourceType, now = Date.now()) { if (!shouldCapture(resourceType)) return; const entry = makeRouteEntry(request, resourceType); entry.lastSeen = now; const key = routeKey(entry.method, entry.template); const existing = routesMap.get(key); if (existing) { existing.hits += 1; existing.lastSeen = now; // Refresh the body so a later replay learns from the FRESHEST call (queryId/token rotation // lives in the URL/headers, but a stale body could hold an old nonce); keep the latest. if (entry.lastBody) existing.lastBody = entry.lastBody; } else { routesMap.set(key, entry); if (routesMap.size > MAX_ROUTES_PER_WC) { let oldestKey = null; let oldest = Infinity; for (const [k, v] of routesMap) { if (v.lastSeen < oldest) { oldest = v.lastSeen; oldestKey = k; } } if (oldestKey) routesMap.delete(oldestKey); } } } module.exports = { templateUrl, redactExampleUrl, looksSecretValue, redactHeaders, bodyShape, isSafeMethod, shouldCapture, routeKey, makeRouteEntry, recordRoute, CAPTURE_RESOURCE_TYPES, MAX_ROUTES_PER_WC, };