diff --git a/backend/apps/agents/browser/browser_session_import.py b/backend/apps/agents/browser/browser_session_import.py index 7f4d6c54..8a2fdb10 100644 --- a/backend/apps/agents/browser/browser_session_import.py +++ b/backend/apps/agents/browser/browser_session_import.py @@ -54,7 +54,7 @@ P_CHROMIUM_EPOCH_OFFSET_S = 11644473600 # challenge instead of letting us through, which is WORSE than arriving with no clearance at all. # Everything else in the jar is the actual session, so we carry that and let the edge re-challenge # us honestly. -P_FINGERPRINT_BOUND = {"cf_clearance", "__cf_bm", "_cfuvid", "datadome", "incap_ses", "reese84"} +P_FINGERPRINT_BOUND: set = set() class SessionImportResult(BaseModel): diff --git a/backend/tests/test_browser_session_import.py b/backend/tests/test_browser_session_import.py index 3b078f3e..206650bf 100644 --- a/backend/tests/test_browser_session_import.py +++ b/backend/tests/test_browser_session_import.py @@ -125,19 +125,22 @@ def test_expiry_is_translated_out_of_chromium_time(monkeypatch): assert out[1]["expires"] == 0.0, "a session entry must stay session-scoped, not become 1601" -def test_fingerprint_bound_clearance_is_left_behind(monkeypatch): - """Anti-bot clearance is minted against the UA and IP that earned it, and our webview keeps an - 'openswarm/' token in its UA, so a borrowed clearance can never match. Replaying a mismatched - one reads as token theft and gets us challenged HARDER than arriving with none, while the real - session cookies beside it are perfectly portable.""" +def test_the_whole_jar_travels_including_clearance_tokens(monkeypatch): + """A real browser sends everything it has, so we do too. + + An earlier version held back the anti-bot clearance tokens (cf_clearance and friends) on the + theory that ours could never match the user agent they were minted for. That was measured live + on medium with the UA swap CONFIRMED firing in the Electron log, and it changed nothing in + either direction, so the filter was carrying a story rather than its weight. Holding a cookie + back is a claim about the site's auth that we could not support.""" monkeypatch.setattr(si.browser_cookies, "read_provider_cookie_records", lambda d: [ {"name": "sid", "value": "opaque", "expires_utc": 0}, {"name": "uid", "value": "opaque", "expires_utc": 0}, {"name": "cf_clearance", "value": "opaque", "expires_utc": 0}, {"name": "__cf_bm", "value": "opaque", "expires_utc": 0}, - {"name": "datadome", "value": "opaque", "expires_utc": 0}, ]) - assert sorted(r["name"] for r in si.read_site_records("medium.com")) == ["sid", "uid"] + assert sorted(r["name"] for r in si.read_site_records("medium.com")) == [ + "__cf_bm", "cf_clearance", "sid", "uid"] def test_google_reads_go_through_the_named_sso_scope(monkeypatch): diff --git a/electron/borrowedSessionUa.test.js b/electron/borrowedSessionUa.test.js new file mode 100644 index 00000000..13abf6b9 --- /dev/null +++ b/electron/borrowedSessionUa.test.js @@ -0,0 +1,96 @@ +// The user-agent half of borrowing a sign-in out of the user's real Chrome. +// +// Borrowing the session and presenting as the browser that earned it are ONE decision. Our browser +// cards deliberately advertise an "openswarm/" product token, because Google's sign-in rejects +// a bare Chrome UA as not-genuine-Chrome. But the anti-bot layer in front of a borrowed site checks +// the session against the UA it was minted for, so that same token reads as "this is not the +// browser that logged in" and the session is refused. Measured live: medium (7 entries) and +// instagram (10 entries) both imported cleanly and both still reported signed-out. +// +// main.js needs a real Electron to load, so the two pure functions are lifted out of the source and +// exercised directly. That keeps the test honest about WHICH code it covers: if either function is +// renamed or reshaped, the extraction fails loudly rather than silently testing a stale copy. +const test = require('node:test'); +const assert = require('node:assert'); +const fs = require('node:fs'); +const path = require('node:path'); + +const SRC = fs.readFileSync(path.join(__dirname, 'main.js'), 'utf8'); + +function lift(name) { + const m = SRC.match(new RegExp(`function ${name}\\([\\s\\S]*?\\n\\}`)); + assert.ok(m, `${name} not found in main.js; did the borrowed-session UA path change shape?`); + return m[0]; +} + +// eslint-disable-next-line no-eval +eval(`${lift('bareChromeUserAgent')}\nvar p_borrowedSessionDomains = new Set();\n${lift('hostHasBorrowedSession')}`); + +const APP_UA = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 ' + + '(KHTML, like Gecko) openswarm/1.5.8 Chrome/148.0.7778.218 Electron/42.3.3 Safari/537.36'; +const REAL_CHROME = /^Mozilla\/5\.0 \(Macintosh; Intel Mac OS X 10_15_7\) AppleWebKit\/537\.36 \(KHTML, like Gecko\) Chrome\/[\d.]+ Safari\/537\.36$/; + +test('a borrowed site sees a UA indistinguishable from real Chrome', () => { + const bare = bareChromeUserAgent(APP_UA); + assert.match(bare, REAL_CHROME); + assert.ok(!/openswarm/i.test(bare), 'the product token is the whole tell; it must be gone'); + assert.ok(!/Electron/i.test(bare), 'the Electron token must be gone too'); +}); + +test('the Chrome version is preserved, not invented', () => { + // sec-ch-ua headers carry the real version. Substituting a different one here would make the UA + // and the client hints disagree, which is a louder tell than the token we just removed. + assert.ok(bareChromeUserAgent(APP_UA).includes('Chrome/148.0.7778.218')); +}); + +test('a UA with no product token is left exactly alone', () => { + const already = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 ' + + '(KHTML, like Gecko) Chrome/148.0.7778.218 Safari/537.36'; + assert.strictEqual(bareChromeUserAgent(already), already); +}); + +test('empty and malformed input degrade to a string, never a throw', () => { + for (const bad of [undefined, null, '', 123, {}]) { + assert.strictEqual(typeof bareChromeUserAgent(bad), 'string'); + } +}); + +test('only borrowed sites are rewritten', () => { + p_borrowedSessionDomains.clear(); + assert.strictEqual(hostHasBorrowedSession('https://medium.com/'), false, + 'nothing borrowed yet means nothing is rewritten'); + p_borrowedSessionDomains.add('medium.com'); + assert.strictEqual(hostHasBorrowedSession('https://medium.com/me'), true); + assert.strictEqual(hostHasBorrowedSession('https://cdn.medium.com/x.js'), true, + 'subdomains of a borrowed site carry the same session'); +}); + +test('a lookalike domain is never treated as borrowed', () => { + // Suffix matching done wrong is how "notmedium.com" or "medium.com.evil.net" would inherit the + // borrowed identity. Only the exact host or a real dot-separated subdomain may match. + p_borrowedSessionDomains.clear(); + p_borrowedSessionDomains.add('medium.com'); + assert.strictEqual(hostHasBorrowedSession('https://notmedium.com/'), false); + assert.strictEqual(hostHasBorrowedSession('https://medium.com.evil.net/'), false); + assert.strictEqual(hostHasBorrowedSession('https://google.com/'), false, + 'Google must keep the product token: its own sign-in is what the token exists to satisfy'); +}); + +test('a malformed URL is not a borrowed site', () => { + p_borrowedSessionDomains.clear(); + p_borrowedSessionDomains.add('medium.com'); + assert.strictEqual(hostHasBorrowedSession('not a url'), false); + assert.strictEqual(hostHasBorrowedSession(''), false); +}); + +test('importing a session is what registers the domain', () => { + // The two halves must stay wired together: cookies applied without the matching UA get refused, + // which is exactly the bug this whole path exists to fix. + assert.match(SRC, /if \(set > 0\) p_borrowedSessionDomains\.add\(d\);/, + 'writePartitionCookies must register the domain it just borrowed for'); +}); + +test('the request header is actually rewritten for borrowed sites', () => { + assert.match(SRC, /borrowed && lk === 'user-agent'/, + 'the onBeforeSendHeaders hook must swap the UA on borrowed sites'); +}); diff --git a/electron/main.js b/electron/main.js index 783c52eb..31cb6a56 100644 --- a/electron/main.js +++ b/electron/main.js @@ -3,6 +3,37 @@ const { app, components, BrowserWindow, ipcMain, shell, session, dialog, crashRe // Browser cards live in their own persistent partition so cookies/localStorage/IndexedDB survive reload + quit (Discord etc. stay logged in) and site data stays isolated from the app's defaultSession. The "clear browsing data" wipe nukes only this partition. MUST match BROWSER_PARTITION in frontend BrowserCard.tsx. const BROWSER_PARTITION = 'persist:openswarm-browser'; +// Sites whose sign-in we borrowed out of the user's real Chrome. Populated when a session is +// imported, and it changes exactly one thing: what user agent we present to that site. +// +// Our normal browser-card UA deliberately carries an "openswarm/" product token, because +// Google's sign-in rejects a BARE Chrome UA as not-genuine-Chrome (see BrowserCard.tsx). But a +// borrowed session was minted by real Chrome, and the anti-bot layer in front of these sites checks +// the session against the UA that earned it, so that same token reads as "this is not the browser +// that logged in" and the session is refused. On a borrowed site only, we drop the token and +// present the same Chrome version bare, which is exactly what the onboarding harvest window does +// (hiddenBrowser.js) and how it gets through Cloudflare with borrowed cookies. Google keeps the +// token because we never borrow for it: its own sign-in is the thing the token exists to satisfy. +const p_borrowedSessionDomains = new Set(); +const p_uaSwapLogged = new Set(); + +function bareChromeUserAgent(ua) { + return String(ua || '').replace(/\s*(?:openswarm|Electron)\/\S+/gi, '').replace(/\s{2,}/g, ' ').trim(); +} + +function hostHasBorrowedSession(url) { + if (!p_borrowedSessionDomains.size) return false; + try { + const host = new URL(url).hostname.toLowerCase(); + for (const d of p_borrowedSessionDomains) { + if (host === d || host.endsWith(`.${d}`)) return true; + } + } catch { + // A malformed URL simply isn't a borrowed site. + } + return false; +} + // E2E flag: when OPENSWARM_E2E=1, append a Chromium command-line switch the // renderer reads at startup to set window.__OPENSWARM_E2E__ = true BEFORE any // page script parses, so the production-build store-on-window gate fires @@ -1814,10 +1845,20 @@ app.whenReady().then(async () => { { urls: ['http://*/*', 'https://*/*'] }, (details, callback) => { const headers = { ...(details.requestHeaders || {}) }; + const borrowed = hostHasBorrowedSession(details.url); for (const k of Object.keys(headers)) { const lk = k.toLowerCase(); if (lk === 'sec-ch-ua' || lk === 'sec-ch-ua-full-version-list') { headers[k] = addGoogleChromeBrand(headers[k]); + } else if (borrowed && lk === 'user-agent') { + const swapped = bareChromeUserAgent(headers[k]); + // Once per site: proof the swap actually fired, so "borrowed but still signed out" can be + // read as the site refusing us rather than as this code silently never running. + if (swapped !== headers[k] && !p_uaSwapLogged.has(details.url.split('/')[2])) { + p_uaSwapLogged.add(details.url.split('/')[2]); + console.log(`[borrowed-ua] ${details.url.split('/')[2]} -> ${swapped}`); + } + headers[k] = swapped; } } callback({ requestHeaders: headers }); @@ -2339,6 +2380,25 @@ app.on('web-contents-created', (_event, contents) => { `).catch(() => {}); }); + // On a site whose sign-in we borrowed we send a bare Chrome UA header (see + // p_borrowedSessionDomains), so navigator.userAgent has to say the same thing. A page that + // reads one UA in JS while the request carried another is a louder automation tell than the + // product token we removed, and plenty of anti-bot scripts compare exactly those two. + contents.on('dom-ready', () => { + let borrowed = false; + try { borrowed = hostHasBorrowedSession(contents.getURL()); } catch { borrowed = false; } + if (!borrowed) return; + const bare = bareChromeUserAgent(contents.getUserAgent()); + contents.executeJavaScript(` + (function(){ + try { + if (navigator.userAgent === ${JSON.stringify(bare)}) return; + Object.defineProperty(navigator, 'userAgent', { get: function(){ return ${JSON.stringify(bare)}; }, configurable: true }); + } catch (e) {} + })(); + `).catch(() => {}); + }); + // Real headed Chrome exposes window.chrome.app/csi/loadTimes; an Electron webview's window.chrome is empty ({}), the single most-checked headless/automation tell (PerimeterX/DataDome et al). Stub the same shape real Chrome reports (app = object, csi + loadTimes = functions, NO runtime, matching a non-extension page). Also restore the base 'en' language Electron drops. Page-world (contextIsolation hides the preload), measured to flip every bot.sannysoft row to its Chrome value. contents.on('dom-ready', () => { contents.executeJavaScript(` @@ -2834,6 +2894,9 @@ async function writePartitionCookies(domain, cookies) { // One malformed cookie must not sink the whole sign-in. } } + // Borrowing the session and presenting as the browser that earned it are one decision, not two: + // apply the cookies without the matching UA and the site refuses them. + if (set > 0) p_borrowedSessionDomains.add(d); return { ok: set > 0, set, total: list.length }; } ipcMain.handle('set-partition-cookies', (_e, domain, cookies) => writePartitionCookies(domain, cookies));