diff --git a/docs/perf/winv2/TASK10_CHECKLIST.md b/docs/perf/winv2/TASK10_CHECKLIST.md new file mode 100644 index 00000000..dfa38e04 --- /dev/null +++ b/docs/perf/winv2/TASK10_CHECKLIST.md @@ -0,0 +1,55 @@ +# Task #10 — validate the winv2 changes on the REAL signed Windows build + +Do not call winv2 "done" until a **code-signed, downloaded, installed, launched** +build passes this. Unit tests + dry-runs are necessary but not sufficient — the +build-script changes (asar exclusion, node_modules pre-extract) and the cold-start +wins only exist in a packaged EXE. + +## 0. Produce the signed build +- Tag the winv2 HEAD and push: `git tag v1.3.86 && git push origin v1.3.86`. +- This runs `.github/workflows/release-windows.yml` (Azure code-signing) and creates + a **draft** GitHub release (drafts do NOT auto-update existing users). +- Watch it: `gh run watch` / `gh run list --workflow=release-windows.yml`. +- If `build-app-win.ps1` errors on the new step 4b (pre-extract) or the asar + `files` exclusion, fix and re-tag (delete the draft + tag first; never force-push + an existing release tag). + +## 1. Download + verify the signature (must be real signed bits) +- `gh release download v1.3.86 --pattern "*Setup*.exe" --dir .` (or from the draft release page). +- Verify Authenticode: `Get-AuthenticodeSignature .\OpenSwarm-Setup-x64.exe` → Status must be `Valid`, signer = the Azure Trusted Signing cert. NOT "NotSigned"/"UnknownError". + +## 2. Install + first (COLD) launch — the headline metric +- Install the downloaded EXE (Squirrel → `%LOCALAPPDATA%\openswarm`). +- Launch once and let it fully load. This is the COLD launch (Defender scans fresh files). +- Then run the automated checker: `pwsh docs/perf/winv2/validate_packaged.ps1`. +- Acceptance (perf): cold `backend-http-ready` should be **far below the 54-138s baseline** + (target: well under the 10s goal even cold, given the 639MB asar read is gone + + fewer files to scan). Relaunch once for the warm number (target ~2-3s). + +## 3. Automated structural checks (validate_packaged.ps1 must be all PASS) +- app.asar < 50 MB (was ~607 MB) — #9 item 4. +- app.asar does NOT contain python-env / build-staging — #9 item 4. +- `resources/python-env/python.exe` present (still shipped unpacked). +- `resources/node/x64/node.exe` present. +- `resources/backend/apps/skill_registry/skills_snapshot.json` present — Bug #1. +- webapp_template_cache has a pre-extracted `/node_modules/vite/bin/vite.js` + (#9 item 2) — or a `.tar.gz` fallback. + +## 4. Manual GUI checks (can't be automated) +- **Skills (Bug #1):** open the Skills page on a fresh launch → the catalog shows + immediately (NOT empty). Run onboarding step 6/8 "Install a skill" → it finds the + pdf skill (no `waitForSelector "skill-item-pdf" 15000ms` timeout). +- **App Builder (Bug #2):** create an app → the preview goes LIVE. No `[WinError 2]`, + no "backend exited with code 1". First app should be quick (pre-extracted nm + vite). + Bonus: test on a machine WITHOUT Git Bash to confirm the no-bash vite path. +- Sanity: send an agent message (9Router now starts in the background → first message + may wait a moment for it; confirm it still answers). + +## 5. Optional cold-start levers (only after 1-4 pass) +- #9 item 1 (`zip-python-stdlib.ps1 -Apply`) and item 3 (`strip-py-to-pyc.ps1 -Apply`) + on a build copy, then re-run 1-4 + re-measure. Enable in the build only if green. +- #9 item 5 (`add-defender-exclusion.ps1`) is a user opt-in, validate separately. + +## 6. Sign-off +- All of 1-4 green on the signed build → publish the draft release (un-draft) to ship 1.3.86. +- Record the real cold/warm numbers in `boot_breakdown.csv` / README "Results (AFTER)". diff --git a/docs/perf/winv2/validate_packaged.ps1 b/docs/perf/winv2/validate_packaged.ps1 new file mode 100644 index 00000000..b3d4b7a5 --- /dev/null +++ b/docs/perf/winv2/validate_packaged.ps1 @@ -0,0 +1,62 @@ +<# +.SYNOPSIS + Task #10 automated checks: run AFTER installing the signed build. Confirms the + winv2 structural fixes landed in the packaged app and reads the REAL cold/warm + backend-http-ready from the app's own perf log. Manual GUI checks are in + TASK10_CHECKLIST.md (App Builder preview, Skills list, onboarding). +.USAGE + pwsh docs\perf\winv2\validate_packaged.ps1 +#> +param( + [string]$InstallRoot = (Join-Path $env:LOCALAPPDATA 'openswarm'), + [string]$BackendLog = (Join-Path $env:APPDATA 'openswarm\data\backend.log') +) +$ErrorActionPreference = 'Stop' +$pass = 0; $fail = 0 +function ok($m) { Write-Host " PASS $m" -ForegroundColor Green; $script:pass++ } +function bad($m) { Write-Host " FAIL $m" -ForegroundColor Red; $script:fail++ } +function info($m) { Write-Host " .. $m" -ForegroundColor DarkGray } + +$app = Get-ChildItem $InstallRoot -Directory -Filter 'app-*' -EA SilentlyContinue | Sort-Object Name | Select-Object -Last 1 +if (-not $app) { throw "no app-* under $InstallRoot (install the build first)" } +$res = Join-Path $app.FullName 'resources' +Write-Host "Validating packaged build: $res`n" + +# #9 item 4: asar trimmed +$asar = Join-Path $res 'app.asar' +if (Test-Path $asar) { + $asarMB = [math]::Round((Get-Item $asar).Length / 1MB, 1) + if ($asarMB -lt 50) { ok "app.asar = ${asarMB} MB (trimmed; was ~607 MB)" } else { bad "app.asar = ${asarMB} MB (expected < 50)" } + $insp = Join-Path $PSScriptRoot 'inspect_asar.js' + if ((Get-Command node -EA SilentlyContinue) -and (Test-Path $insp)) { + $out = & node $insp $asar 2>&1 | Out-String + if ($out -match 'python-env|build-staging') { bad "asar STILL contains python-env/build-staging" } else { ok "asar excludes python-env + build-staging" } + } +} else { bad "app.asar not found" } + +# still shipped unpacked (runtime reads these) +if (Test-Path (Join-Path $res 'python-env\python.exe')) { ok "python-env shipped unpacked" } else { bad "python-env\python.exe missing" } +if (Test-Path (Join-Path $res 'node\x64\node.exe')) { ok "node bundled" } else { bad "node\x64\node.exe missing" } + +# Bug #1: skills snapshot +if (Test-Path (Join-Path $res 'backend\apps\skill_registry\skills_snapshot.json')) { ok "skills snapshot shipped (catalog never empty)" } else { bad "skills_snapshot.json missing" } + +# #9 item 2 / Bug #2: webapp node_modules pre-extracted or archive +$cache = Join-Path $res 'backend\apps\outputs\webapp_template_cache' +if (Test-Path (Join-Path $cache '*\node_modules\vite\bin\vite.js')) { ok "webapp node_modules PRE-EXTRACTED (zero first-app extract)" } +elseif (Test-Path (Join-Path $cache 'node_modules.*.tar.gz')) { info "webapp node_modules shipped as .tar.gz (extract path, not pre-extracted)" } +else { bad "no webapp node_modules tree/archive in resources" } + +# perf: real cold/warm backend-http-ready from the app's own log +if (Test-Path $BackendLog) { + $m = Select-String -Path $BackendLog -Pattern 'backend-http-ready t=(\d+)' -AllMatches + $vals = @($m.Matches | ForEach-Object { [int]$_.Groups[1].Value }) + if ($vals.Count) { + $recent = ($vals | Select-Object -Last 6 | ForEach-Object { [math]::Round($_ / 1000, 1) }) -join 's, ' + info "backend-http-ready recent: ${recent}s (baseline: warm ~9-10s, cold 54-138s)" + info "latest: $([math]::Round($vals[-1]/1000,1))s -- first launch after install = COLD; relaunch for warm" + } else { info "no backend-http-ready markers yet" } +} else { info "no backend.log yet (launch the app once first)" } + +Write-Host "`n$pass passed, $fail failed. Manual GUI checks: TASK10_CHECKLIST.md (App Builder, Skills, onboarding)." +if ($fail) { exit 1 }