WEBDIENSTE/openswarm
1
0
Fork 0
mirror of https://github.com/openswarm-ai/openswarm.git synced 2026-09-11 20:27:44 +02:00
Code Issues Packages Projects Releases Wiki Activity

[eric] onboarding: bound harvest pagination (ChatGPT ~4s/page) so it can't wedge the offscreen read

Browse Source
This commit is contained in:
ciregenz
2026-07-16 10:28:35 -07:00
parent 2d499f8316
commit 7ee9837f7a
2 changed files with 36 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files
electron/hiddenBrowser.js
+9 -4
View File
@@ -36,9 +36,9 @@ function makeWindow(partition) {
}
}
async function withWindow(partition, fn) {
async function withWindow(partition, fn, extraGraceMs = 0) {
const win = makeWindow(partition);
const killer = setTimeout(() => { try { win.destroy(); } catch (_) {} }, LOAD_TIMEOUT_MS + SETTLE_MS + 8000);
const killer = setTimeout(() => { try { win.destroy(); } catch (_) {} }, LOAD_TIMEOUT_MS + SETTLE_MS + 8000 + extraGraceMs);
try {
return await fn(win);
} finally {
@@ -47,6 +47,11 @@ async function withWindow(partition, fn) {
}
}
// The provider-history harvest paginates a genuinely slow endpoint (ChatGPT's
// /backend-api/conversations runs ~4s/page), so its offscreen window must outlive a
// plain fetch/search window or it gets killed mid-pagination and the whole read is lost.
const HARVEST_GRACE_MS = 20000;
async function loadAndSettle(win, url) {
// loadURL rejects on a sub-resource abort even when the main frame is fine, so a rejection is a warning, not a failure; we still try to read the DOM.
const load = win.loadURL(url, { userAgent: SCRAPE_UA }).catch(() => {});
@@ -78,7 +83,7 @@ async function hiddenEval(partition, url, js) {
return withWindow(partition, async (win) => {
await loadAndSettle(win, url);
return win.webContents.executeJavaScript(js, true).catch(() => null);
});
}, HARVEST_GRACE_MS);
}
// Inject the user's own session cookies (read + decrypted by the Python backend) into a
@@ -109,7 +114,7 @@ async function hiddenEvalWithCookies(url, cookieRecords, js) {
return await withWindow(HARVEST_PARTITION, async (win) => {
await loadAndSettle(win, url);
return win.webContents.executeJavaScript(js, true).catch(() => null);
});
}, HARVEST_GRACE_MS);
} finally {
await wipe();
}
electron/usageHarvest.js
+27 -19
View File
@@ -27,30 +27,40 @@ function configure(opts) {
if (opts && typeof opts.readCookies === 'function') p_readCookies = opts.readCookies;
}
// Runs in the page context. Sweeps the full conversation history (all titles,
// paginated + deduped) plus ChatGPT Memory. Hard caps bound the work + PII footprint
// even for a user with thousands of chats; every fetch fails open to empty.
// Runs in the page context. Sweeps recent conversation titles (paginated + deduped)
// plus ChatGPT Memory. Bounded on EVERY dimension so no provider's endpoint speed can
// wedge the read: a wall-clock BUDGET_MS (ChatGPT's /conversations is ~4s/page, so an
// unbounded loop over a many-chat account would run minutes and outlive the offscreen
// window, losing everything), a per-fetch abort, and hard page/title caps. The most
// recent titles are the strongest personalization signal, so a partial is a good result.
const PREAMBLE = `
const BUDGET_MS=14000, PAGE=100, CAP_PAGES=60, CAP_TITLES=1000, GAP_MS=120, FETCH_MS=6000;
const startedAt = Date.now();
const haveTime = () => Date.now() - startedAt < BUDGET_MS;
const jget = async (url, extra) => {
const ac = new AbortController(); const t = setTimeout(() => ac.abort(), FETCH_MS);
try { const r = await fetch(url, Object.assign({credentials:'include', signal:ac.signal}, extra||{})); return r.ok ? await r.json() : null; }
catch (e) { return null; } finally { clearTimeout(t); }
};`;
const SCRIPT = {
codex: `(async () => {
const PAGE=100, CAP_PAGES=60, CAP_TITLES=1000, GAP_MS=90;
codex: `(async () => {${PREAMBLE}
try {
const sess = await fetch('/api/auth/session', {credentials:'include'}).then(r=>r.json());
const sess = await jget('/api/auth/session');
if (!sess || !sess.accessToken) return {ok:false, total:0, titles:[], memories:[]};
const H = {headers:{Authorization:'Bearer '+sess.accessToken, accept:'application/json'}, credentials:'include'};
const H = {headers:{Authorization:'Bearer '+sess.accessToken, accept:'application/json'}};
const seen = new Set(); const titles = [];
let offset = 0, page = 0;
while (page < CAP_PAGES && titles.length < CAP_TITLES) {
const j = await fetch('/backend-api/conversations?offset='+offset+'&limit='+PAGE+'&order=updated', H).then(r=>r.ok?r.json():null).catch(()=>null);
while (page < CAP_PAGES && titles.length < CAP_TITLES && haveTime()) {
const j = await jget('/backend-api/conversations?offset='+offset+'&limit='+PAGE+'&order=updated', H);
const items = (j && j.items) || [];
if (!items.length) break;
let fresh = 0;
for (const c of items) { if (c && c.id && !seen.has(c.id)) { seen.add(c.id); if (c.title) titles.push(c.title); fresh++; } }
if (fresh === 0) break;
if (items.length < PAGE) break;
if (fresh === 0 || items.length < PAGE) break;
offset += PAGE; page++;
await new Promise(r=>setTimeout(r, GAP_MS));
}
const mem = await fetch('/backend-api/memories?include_memory_entries=true', H).then(r=>r.ok?r.json():null).catch(()=>null);
const mem = await jget('/backend-api/memories?include_memory_entries=true', H);
return {
ok: true,
total: seen.size,
@@ -59,22 +69,20 @@ const SCRIPT = {
};
} catch (e) { return {ok:false, total:0, titles:[], memories:[]}; }
})()`,
claude: `(async () => {
const PAGE=100, CAP_PAGES=60, CAP_TITLES=1000, GAP_MS=90;
claude: `(async () => {${PREAMBLE}
try {
const orgs = await fetch('/api/organizations', {credentials:'include', headers:{accept:'application/json'}}).then(r=>r.ok?r.json():null).catch(()=>null);
const orgs = await jget('/api/organizations', {headers:{accept:'application/json'}});
if (!Array.isArray(orgs) || !orgs.length) return {ok:false, total:0, titles:[], memories:[]};
const org = orgs[0].uuid;
const seen = new Set(); const titles = [];
let offset = 0, page = 0;
while (page < CAP_PAGES && titles.length < CAP_TITLES) {
const convs = await fetch('/api/organizations/'+org+'/chat_conversations?limit='+PAGE+'&offset='+offset, {credentials:'include', headers:{accept:'application/json'}}).then(r=>r.ok?r.json():null).catch(()=>null);
while (page < CAP_PAGES && titles.length < CAP_TITLES && haveTime()) {
const convs = await jget('/api/organizations/'+org+'/chat_conversations?limit='+PAGE+'&offset='+offset, {headers:{accept:'application/json'}});
const items = Array.isArray(convs) ? convs : [];
if (!items.length) break;
let fresh = 0;
for (const c of items) { const id = c && c.uuid; if (id && !seen.has(id)) { seen.add(id); if (c.name) titles.push(c.name); fresh++; } }
if (fresh === 0) break;
if (items.length < PAGE) break;
if (fresh === 0 || items.length < PAGE) break;
offset += PAGE; page++;
await new Promise(r=>setTimeout(r, GAP_MS));
}