mirror of
https://github.com/openswarm-ai/openswarm.git
synced 2026-09-09 19:27:45 +02:00
[eric] perf: exclude duplicated python-env + build-staging from the asar (#9 item 4)
- inventory: 607MB asar was ~605MB duplication (python-env 408MB + build-staging 197MB) - both already ship unpacked in resources/; runtime reads resources/, never the asar - add build.files exclusion -> asar ~607MB -> ~2MB; kills the 639MB cold-read on first launch - add inspect_asar.js inventory tool; validate asar size + boot on a packaged exe
This commit is contained in:
@@ -162,7 +162,7 @@ validated on a real packaged EXE (Task #10).
|
||||
1. [DRAFTED, build-gated] Zip the Python stdlib -> python313.zip (medium risk). Draft: scripts/zip-python-stdlib.ps1 (dry-run by default; NOT wired into the release build yet). Measured on the real env: 910 stdlib .py/.pyc files (15.1 MB) collapse into one zip. CPython auto-adds <prefix>/python313.zip to sys.path, so no python._pth is needed; site-packages + DLLs (native .pyd) stay loose; a keep-list keeps data-file stdlib dirs (lib2to3, idlelib, tkinter, ...) loose. Impact: ~7% of total python-env file count, but it collapses the stdlib import-time file-opens (the cold-launch Defender scan storm) into a single scanned file; bigger combined with #3. Validation (Task #10): -Apply on a copy, then import backend.main, importtime parity, boot the packaged backend, measure cold backend-http-ready vs baseline. Wire into build-app-win.ps1 behind an off-by-default -ZipStdlib switch only after it passes.
|
||||
2. [DONE] Ship webapp_template node_modules PRE-EXTRACTED in resources + junction to it (kills the 14.2 s extract -> ~0 s). build-app-win.ps1 step 4b robocopies the tree into resources; runtime _bundled_extracted_modules()/_ensure_warm_cache() prefer it; tests in test_bundled_extracted_modules.py. Mac still ships the .tar.gz (unchanged).
|
||||
3. [DRAFTED, build-gated] Ship site-packages as sourceless .pyc only (drop .py). Draft: scripts/strip-py-to-pyc.ps1 (dry-run default; NOT wired into the build). Measured: 3,352 .py (26.9 MB) + 362 __pycache__ dirs strippable from site-packages (keep-list excludes pip/setuptools). compileall -b writes legacy module.pyc next to source; we delete the .py whose .pyc exists and drop __pycache__. Sourceless import proven with the bundled 3.13 interpreter. Scope: site-packages ONLY (NOT backend app code -- the swarm-debug debugger reads our own source for frame annotation). .pyc magic must match the shipped interpreter, so compile with the bundled python. Validate on a packaged EXE (Task #10); some packages use inspect.getsource and may need the keep-list. Combined with #1 + #2 this takes python-env from ~13,554 files toward ~9,300 (~31% fewer for Defender).
|
||||
4. Inventory + trim app.asar (639 MB): source maps, dev-only deps, duplicate bundles. Single file (not a count issue) but shrinks cold-read I/O.
|
||||
4. [APPLIED, build-gated] Trim app.asar. Inventory (docs/perf/winv2/inspect_asar.js) found the 607 MB asar is almost entirely DUPLICATION: python-env (408 MB, incl. a 242 MB bundled claude.exe) and build-staging (197 MB: node.exe 67 MB, uv.exe 65 MB, mcp-bundles, frontend) are packed into the asar AND already shipped UNPACKED in resources/ via extraResources. The runtime reads from resources/ (confirmed: "Starting backend: ...resources\python-env\python.exe"), never from inside the asar. Source maps were a red herring (0.4 MB). Fix: added a build.files exclusion in electron/package.json ("!python-env/**", "!build-staging/**") so those trees no longer pack into the asar -> ~607 MB -> ~2 MB (just main.js/preload/node_modules). Removes the entire 639 MB cold-read on first launch. Validate on a packaged EXE (Task #10): app still boots (python/node/router resolved from resources), asar size shrunk.
|
||||
5. Opt-in Defender exclusion for install/data dirs, documented, never silent (needs admin/UAC; security-sensitive). Settings toggle only; do not auto-apply.
|
||||
|
||||
Recommended order: #2 (biggest UX win, lowest risk), then #1 (largest cold win, careful import testing), then #3/#4. Validation: re-run profile_startup.sh + a fresh-extract timing on the packaged EXE after each change, diff vs baseline_startup.csv.
|
||||
|
||||
@@ -0,0 +1,57 @@
|
||||
// #9 item 4: inventory an app.asar without extracting it. Parses the asar header
|
||||
// (a Chromium Pickle: [u32 payloadSize][u32 headerSize] then [u32 payloadSize]
|
||||
// [u32 jsonLen][json...]) and reports total size, biggest top-level dirs, biggest
|
||||
// individual files, and trimmable categories (source maps, etc.).
|
||||
// Usage: node inspect_asar.js <path-to-app.asar>
|
||||
const fs = require('fs');
|
||||
|
||||
const asar = process.argv[2];
|
||||
if (!asar) { console.error('usage: node inspect_asar.js <app.asar>'); process.exit(1); }
|
||||
|
||||
const fd = fs.openSync(asar, 'r');
|
||||
const head = Buffer.alloc(8);
|
||||
fs.readSync(fd, head, 0, 8, 0);
|
||||
const headerSize = head.readUInt32LE(4); // size of the header pickle
|
||||
const hp = Buffer.alloc(headerSize);
|
||||
fs.readSync(fd, hp, 0, headerSize, 8);
|
||||
const jsonLen = hp.readUInt32LE(4); // string length inside the pickle
|
||||
const json = hp.slice(8, 8 + jsonLen).toString('utf8');
|
||||
const header = JSON.parse(json);
|
||||
fs.closeSync(fd);
|
||||
|
||||
let total = 0, fileCount = 0;
|
||||
const byExt = {};
|
||||
const files = []; // {path, size}
|
||||
const topDirs = {}; // top-level entry -> size
|
||||
|
||||
function walk(node, parts) {
|
||||
if (node.files) {
|
||||
for (const [name, child] of Object.entries(node.files)) walk(child, parts.concat(name));
|
||||
} else if (typeof node.size === 'number') {
|
||||
const p = parts.join('/');
|
||||
total += node.size; fileCount++;
|
||||
files.push({ p, size: node.size });
|
||||
const ext = (p.match(/\.[^./]+$/) || ['(none)'])[0].toLowerCase();
|
||||
byExt[ext] = (byExt[ext] || 0) + node.size;
|
||||
topDirs[parts[0]] = (topDirs[parts[0]] || 0) + node.size;
|
||||
}
|
||||
}
|
||||
walk(header, []);
|
||||
|
||||
const mb = (b) => (b / 1048576).toFixed(1) + ' MB';
|
||||
const sortObj = (o) => Object.entries(o).sort((a, b) => b[1] - a[1]);
|
||||
|
||||
console.log(`asar total: ${mb(total)} across ${fileCount} files\n`);
|
||||
console.log('=== biggest top-level entries ===');
|
||||
for (const [d, s] of sortObj(topDirs).slice(0, 15)) console.log(` ${mb(s).padStart(10)} ${d}`);
|
||||
console.log('\n=== biggest single files ===');
|
||||
for (const f of files.sort((a, b) => b.size - a.size).slice(0, 20)) console.log(` ${mb(f.size).padStart(10)} ${f.p}`);
|
||||
console.log('\n=== by extension (top 15) ===');
|
||||
for (const [e, s] of sortObj(byExt).slice(0, 15)) console.log(` ${mb(s).padStart(10)} ${e}`);
|
||||
console.log('\n=== trimmable categories ===');
|
||||
const cat = (re) => files.filter(f => re.test(f.p)).reduce((n, f) => n + f.size, 0);
|
||||
console.log(` source maps (*.map): ${mb(cat(/\.map$/))}`);
|
||||
console.log(` .ts/.tsx sources: ${mb(cat(/\.tsx?$/))}`);
|
||||
console.log(` markdown/license/readme: ${mb(cat(/(\.md|license|readme|changelog)/i))}`);
|
||||
console.log(` test/spec/__tests__: ${mb(cat(/(\/test\/|\/tests\/|__tests__|\.spec\.|\.test\.)/i))}`);
|
||||
console.log(` node_modules inside asar: ${mb(cat(/(^|\/)node_modules\//))}`);
|
||||
@@ -41,6 +41,13 @@
|
||||
"directories": {
|
||||
"output": "dist"
|
||||
},
|
||||
"files": [
|
||||
"**/*",
|
||||
"!python-env",
|
||||
"!python-env/**",
|
||||
"!build-staging",
|
||||
"!build-staging/**"
|
||||
],
|
||||
"icon": "build/icon.png",
|
||||
"mac": {
|
||||
"icon": "build/icon.icns",
|
||||
|
||||
Reference in New Issue
Block a user