From dcd4f6f149bdc17b8b30ec55a4f13f6f6e24d311 Mon Sep 17 00:00:00 2001 From: Eric Date: Tue, 16 Jun 2026 22:15:45 -0700 Subject: [PATCH] [eric] perf: draft pyc-only site-packages strip to shrink defender cold-start (#9 item 3) - standalone, dry-run by default, NOT wired into the release build (build-gated) - strips 3352 .py (26.9MB) from site-packages to sourceless .pyc; mechanism proven on bundled 3.13 - scope: site-packages only (keeps backend source for the debugger); validate on a packaged exe --- docs/perf/winv2/README.md | 2 +- scripts/strip-py-to-pyc.ps1 | 80 +++++++++++++++++++++++++++++++++++++ 2 files changed, 81 insertions(+), 1 deletion(-) create mode 100644 scripts/strip-py-to-pyc.ps1 diff --git a/docs/perf/winv2/README.md b/docs/perf/winv2/README.md index d74e3f3f..4d38b39c 100644 --- a/docs/perf/winv2/README.md +++ b/docs/perf/winv2/README.md @@ -161,7 +161,7 @@ validated on a real packaged EXE (Task #10). 1. [DRAFTED, build-gated] Zip the Python stdlib -> python313.zip (medium risk). Draft: scripts/zip-python-stdlib.ps1 (dry-run by default; NOT wired into the release build yet). Measured on the real env: 910 stdlib .py/.pyc files (15.1 MB) collapse into one zip. CPython auto-adds /python313.zip to sys.path, so no python._pth is needed; site-packages + DLLs (native .pyd) stay loose; a keep-list keeps data-file stdlib dirs (lib2to3, idlelib, tkinter, ...) loose. Impact: ~7% of total python-env file count, but it collapses the stdlib import-time file-opens (the cold-launch Defender scan storm) into a single scanned file; bigger combined with #3. Validation (Task #10): -Apply on a copy, then import backend.main, importtime parity, boot the packaged backend, measure cold backend-http-ready vs baseline. Wire into build-app-win.ps1 behind an off-by-default -ZipStdlib switch only after it passes. 2. [DONE] Ship webapp_template node_modules PRE-EXTRACTED in resources + junction to it (kills the 14.2 s extract -> ~0 s). build-app-win.ps1 step 4b robocopies the tree into resources; runtime _bundled_extracted_modules()/_ensure_warm_cache() prefer it; tests in test_bundled_extracted_modules.py. Mac still ships the .tar.gz (unchanged). -3. Precompile + ship only .pyc (drop .py) for app + pure-python deps. Halves remaining loose-file count; low risk; stacks with #1. +3. [DRAFTED, build-gated] Ship site-packages as sourceless .pyc only (drop .py). Draft: scripts/strip-py-to-pyc.ps1 (dry-run default; NOT wired into the build). Measured: 3,352 .py (26.9 MB) + 362 __pycache__ dirs strippable from site-packages (keep-list excludes pip/setuptools). compileall -b writes legacy module.pyc next to source; we delete the .py whose .pyc exists and drop __pycache__. Sourceless import proven with the bundled 3.13 interpreter. Scope: site-packages ONLY (NOT backend app code -- the swarm-debug debugger reads our own source for frame annotation). .pyc magic must match the shipped interpreter, so compile with the bundled python. Validate on a packaged EXE (Task #10); some packages use inspect.getsource and may need the keep-list. Combined with #1 + #2 this takes python-env from ~13,554 files toward ~9,300 (~31% fewer for Defender). 4. Inventory + trim app.asar (639 MB): source maps, dev-only deps, duplicate bundles. Single file (not a count issue) but shrinks cold-read I/O. 5. Opt-in Defender exclusion for install/data dirs, documented, never silent (needs admin/UAC; security-sensitive). Settings toggle only; do not auto-apply. diff --git a/scripts/strip-py-to-pyc.ps1 b/scripts/strip-py-to-pyc.ps1 new file mode 100644 index 00000000..083eef86 --- /dev/null +++ b/scripts/strip-py-to-pyc.ps1 @@ -0,0 +1,80 @@ +<# +.SYNOPSIS + #9 item 3 (DRAFT, build-gated): ship site-packages as sourceless .pyc only, so + Windows Defender has ~half as many loose files to scan on a cold launch after + an update. Compiles each module.py -> legacy module.pyc (next to the source, + NOT in __pycache__), then deletes the .py whose .pyc exists and removes the + redundant __pycache__ dirs. Python imports the sourceless .pyc directly. + +.SCOPE + TARGET SITE-PACKAGES ONLY by default. Do NOT strip the backend app code: the + swarm-debug debugger reads our own .py source for frame annotation, and we want + readable tracebacks for first-party code. Stdlib is handled by #9 item 1 + (zip-python-stdlib.ps1); this is the dependency tree. + +.STATUS + UNVALIDATED. Default is -DryRun (reports only). The .pyc magic must match the + SHIPPED interpreter, so compile with the bundled python (-PythonExe). Some + packages read their own source (inspect.getsource) and break sourceless; keep + a keep-list and validate on a packaged EXE (Task #10) BEFORE wiring into a + release. Intentionally NOT called by build-app-win.ps1 yet. + +.USAGE + pwsh scripts\strip-py-to-pyc.ps1 -TargetDir electron\python-env\Lib\site-packages # dry run + pwsh scripts\strip-py-to-pyc.ps1 -TargetDir \site-packages -PythonExe \python.exe -Apply +#> +param( + [Parameter(Mandatory = $true)][string]$TargetDir, + [string]$PythonExe, + [switch]$Apply +) + +$ErrorActionPreference = 'Stop' +if (-not (Test-Path $TargetDir)) { throw "no target dir: $TargetDir" } + +# Packages that read their own .py at runtime (inspect.getsource / exec of source +# / .py-relative data) -> keep their source. Conservative starting set; expand +# whatever validation flags. Matched against the top-level package dir name. +$KeepSource = @('pip', 'setuptools', 'pkg_resources', '_distutils_hack') + +$allPy = Get-ChildItem -Recurse -File $TargetDir -Filter *.py -ErrorAction SilentlyContinue +$py = $allPy | Where-Object { + $rel = $_.FullName.Substring($TargetDir.Length).TrimStart('\', '/') + $top = ($rel -split '[\\/]')[0] + $KeepSource -notcontains $top +} +$pyCount = ($py | Measure-Object).Count +$pyMB = [math]::Round((($py | Measure-Object -Property Length -Sum).Sum) / 1MB, 1) +$pycacheDirs = (Get-ChildItem -Recurse -Directory $TargetDir -Filter __pycache__ -ErrorAction SilentlyContinue | Measure-Object).Count +Write-Host ("#9 item 3: {0} .py files ({1} MB) eligible under {2}" -f $pyCount, $pyMB, $TargetDir) +Write-Host ("keep-source packages: {0} | __pycache__ dirs present: {1}" -f ($KeepSource -join ', '), $pycacheDirs) + +if (-not $Apply) { + Write-Host "DRY RUN. -Apply compiles to legacy .pyc (compileall -b) next to each source," + Write-Host "deletes each .py whose .pyc now exists, and removes __pycache__. Validate (Task #10):" + Write-Host " 1. python.exe -c 'import backend.main' resolves (deps import sourceless)" + Write-Host " 2. boot the packaged backend; exercise agents/app-builder/skills/MCP" + Write-Host " 3. measure cold backend-http-ready vs baseline_startup.csv" + return +} + +if (-not $PythonExe) { throw "-PythonExe is required for -Apply (must be the SHIPPED interpreter; .pyc magic must match)" } +if (-not (Test-Path $PythonExe)) { throw "no python at $PythonExe" } + +# 1. Compile to legacy sourceless .pyc next to each source (-b). -q quiet; it +# continues past files that fail to compile (py2-only, optional) -> those keep +# their .py since no sibling .pyc is produced. +& $PythonExe -m compileall -b -q $TargetDir +# compileall returns nonzero if ANY file failed; that is expected for odd files, +# so we don't treat it as fatal -- we only delete .py that actually got a .pyc. +$global:LASTEXITCODE = 0 + +# 2. Delete each eligible .py that now has a sibling .pyc. +$deleted = 0 +foreach ($f in $py) { + $pyc = [System.IO.Path]::ChangeExtension($f.FullName, '.pyc') + if (Test-Path $pyc) { Remove-Item -Force $f.FullName; $deleted++ } +} +# 3. Remove redundant __pycache__ (we use the legacy .pyc next to source). +Get-ChildItem -Recurse -Directory $TargetDir -Filter __pycache__ -ErrorAction SilentlyContinue | Remove-Item -Recurse -Force +Write-Host ("Removed {0} .py (kept {1} that did not compile). UNVALIDATED -- verify on the packaged EXE before shipping." -f $deleted, ($pyCount - $deleted))