From b7366cce492e6ab24f30b6ba6d9b965038af3cdc Mon Sep 17 00:00:00 2001 From: ciregenz Date: Fri, 7 Aug 2026 09:48:12 -0700 Subject: [PATCH] [eric] scripts: the verifier drives the CDP proofs too, so dictation, INP and idle rAF land in the same scoreboard --- scripts/verify-175.py | 15 +++- scripts/verify175/forced.py | 3 + scripts/verify175/ui.py | 156 ++++++++++++++++++++++++++++++++++++ 3 files changed, 170 insertions(+), 4 deletions(-) create mode 100644 scripts/verify175/ui.py diff --git a/scripts/verify-175.py b/scripts/verify-175.py index 4ea19e13..1147e512 100644 --- a/scripts/verify-175.py +++ b/scripts/verify-175.py @@ -22,7 +22,9 @@ import time import urllib.request from typing import List, Optional, Tuple -from scripts.verify175.forced import check_forced_router_unavailable, check_forced_silent_noop +from scripts.verify175.forced import (check_boot_lifespan, check_forced_router_unavailable, + check_forced_silent_noop) +from scripts.verify175.ui import check_dictation, check_idle_raf, check_inp, check_scroll_both_halves from scripts.verify175.shared import ROOT, ROWS, p_api, row @@ -173,6 +175,11 @@ def main() -> None: check_boot_lifespan() check_live_ttft(token) check_forced_silent_noop(token) + print("\nCDP checks (need headless Chrome on :9223 against the dev frontend):") + check_idle_raf() + check_inp() + check_dictation() + check_scroll_both_halves() if sink: check_forced_router_unavailable(token, sink) else: @@ -185,9 +192,9 @@ def main() -> None: print(f"{len(ROWS) - len(fails) - len(skips)} pass, {len(fails)} fail, {len(skips)} skipped") if fails: print("FAILING: " + ", ".join(f"{n} ({d})" for n, _, d in fails)) - print("\nCovered by hand, recorded on ENG-175 (need a stack, a hidden CLI binary, or a GUI):") - print(" 6 of the 8 forced classes (401 shims, overflow, missing CLI, webview kill, renderer wedge),") - print(" the CDP UI proofs (scroll/fly-to-fit/overlay/dictation), and the packaged-build check.") + print("\nStill hand-run, recorded on ENG-175 (need a hidden CLI binary, Electron, or a signed bundle):") + print(" forced classes: 401 shims, overflow, missing CLI, webview kill, renderer wedge") + print(" plus the fly-to-fit/overlay proofs and the packaged-build check.") sys.exit(1 if fails else 0) diff --git a/scripts/verify175/forced.py b/scripts/verify175/forced.py index 16e55ed6..b8f8b6ab 100644 --- a/scripts/verify175/forced.py +++ b/scripts/verify175/forced.py @@ -4,6 +4,7 @@ Split from verify-175.py only to stay under the file-size cap; it is the same ru import json import os +import statistics import subprocess import sys import time @@ -12,6 +13,8 @@ from typing import List from scripts.verify175.shared import ROOT, p_api, row +PY = os.path.join(ROOT, "backend", ".venv", "bin", "python") + def p_sink_rows(path: str) -> List[dict]: try: diff --git a/scripts/verify175/ui.py b/scripts/verify175/ui.py new file mode 100644 index 00000000..6cb08c1f --- /dev/null +++ b/scripts/verify175/ui.py @@ -0,0 +1,156 @@ +"""The CDP half of the 1.7.5 verification: the checks that need a real renderer. + +Drives headless Chrome against the dev frontend over raw CDP. Every check here follows two rules +that were learned by getting them wrong first: + + 1. Assert the POSITIVE and the NEGATIVE together. "The canvas camera did not move" is not evidence + a surface works: a surface that cannot scroll at all passes that trivially. + 2. Confirm the hit point with elementFromPoint BEFORE dispatching. Locating a surface by DOM scan + hands you elements that something else is painting over, which produced three confident wrong + answers in one afternoon. +""" + +import json +import subprocess +import time +from typing import Optional + +from scripts.verify175.shared import row + +NODE = "node" +WS = "/Users/ericzeng/Downloads/openswarm/frontend/node_modules/ws" + + +def p_cdp(body: str, timeout: int = 180) -> Optional[dict]: + """Run a snippet in a page context and return whatever JSON it printed.""" + js = ( + "const WebSocket=require(%r);const http=require('http');\n" + "function tg(){return new Promise((res,rej)=>{http.get('http://127.0.0.1:9223/json/list',r=>{" + "let d='';r.on('data',c=>d+=c);r.on('end',()=>res(JSON.parse(d)))}).on('error',rej)})}\n" + "const sleep=ms=>new Promise(r=>setTimeout(r,ms));\n" + "(async()=>{const p=(await tg()).find(t=>t.type==='page'&&t.url.includes(':3000'));\n" + "if(!p){console.log(JSON.stringify({error:'no page'}));return;}\n" + "const ws=new WebSocket(p.webSocketDebuggerUrl,{perMessageDeflate:false});\n" + "await new Promise(r=>ws.on('open',r));let id=0;const pend=new Map();\n" + "ws.on('message',m=>{const g=JSON.parse(m);if(g.id&&pend.has(g.id)){pend.get(g.id)(g);pend.delete(g.id)}});\n" + "const send=(m,pa={})=>new Promise(res=>{const i=++id;pend.set(i,res);ws.send(JSON.stringify({id:i,method:m,params:pa}))});\n" + "const ev=async e=>{const r=await send('Runtime.evaluate',{expression:e,returnByValue:true,awaitPromise:true});return r.result?.result?.value};\n" + "%s\n" + "ws.close();})().catch(e=>console.log(JSON.stringify({error:String(e.message)})));\n" + ) % (WS, body) + p = subprocess.run([NODE, "-e", js], capture_output=True, text=True, timeout=timeout) + for line in reversed((p.stdout or "").strip().splitlines()): + try: + return json.loads(line) + except Exception: + continue + return None + + +def check_idle_raf() -> None: + """Counts the APP's rAF scheduling. The old probe used its own rAF loop and therefore always + reported ~120 ticks, which could never distinguish idle from spinning.""" + out = p_cdp( + "await ev(\"(function(){if(window.__RC__)return 1;window.__RC__={n:0};var o=window.requestAnimationFrame;" + "window.requestAnimationFrame=function(cb){window.__RC__.n++;return o.apply(window,arguments)};return 1})()\");\n" + "await ev('window.__RC__.n=0;1');\n" + "await sleep(2000);\n" + "const n=await ev('window.__RC__.n');\n" + "console.log(JSON.stringify({calls:n}));" + ) + if out is None or "calls" not in out: + row("idle renderer (0 rAF in 2s)", "SKIP", f"probe returned {out}") + return + row("idle renderer (0 rAF in 2s)", "PASS" if out["calls"] == 0 else "FAIL", f"{out['calls']} app rAF calls") + + +def check_inp() -> None: + """Reads real `event` PerformanceObserver entries, which is what INP is computed from.""" + out = p_cdp( + "await ev(\"(function(){if(window.__IN__)return 1;window.__IN__=[];try{var o=new PerformanceObserver(function(l){" + "l.getEntries().forEach(function(e){if(e.duration>0)window.__IN__.push(Math.round(e.duration))})});" + "o.observe({type:'event',buffered:true,durationThreshold:0});}catch(e){}return 1})()\");\n" + "const pts=await ev(\"(function(){var t=Array.prototype.slice.call(document.querySelectorAll('.osw-dock-tile')).slice(0,8);" + "return JSON.stringify(t.map(function(x){var r=x.getBoundingClientRect();return{x:Math.round(r.left+r.width/2),y:Math.round(r.top+r.height/2)}}))})()\");\n" + "const list=JSON.parse(pts||'[]');\n" + "for(let r=0;r<3;r++){for(const q of list){\n" + " await send('Input.dispatchMouseEvent',{type:'mousePressed',x:q.x,y:q.y,button:'left',clickCount:1});await sleep(25);\n" + " await send('Input.dispatchMouseEvent',{type:'mouseReleased',x:q.x,y:q.y,button:'left',clickCount:1});await sleep(110);}}\n" + "await sleep(1200);\n" + "const s=await ev(\"(function(){var a=(window.__IN__||[]).slice().sort(function(x,y){return x-y});" + "if(!a.length)return JSON.stringify({n:0});var p=function(q){return a[Math.min(a.length-1,Math.floor(a.length*q))]};" + "return JSON.stringify({n:a.length,p50:p(0.5),p95:p(0.95),max:a[a.length-1]})})()\");\n" + "console.log(s);" + , timeout=300) + if not out or not out.get("n"): + row("INP p95 (<=200ms)", "SKIP", f"no interaction entries ({out})") + return + row("INP p95 (<=200ms)", "PASS" if out["p95"] <= 200 else "FAIL", + f"p95 {out['p95']}ms, p50 {out['p50']}ms, max {out['max']}ms, n={out['n']}") + + +def check_dictation() -> None: + """ENG-176. Four scenarios, including the two that were bugs: a target that dies mid-decode must + drop the words rather than type them into whatever holds focus, and the composer fallback must + survive that refusal.""" + out = p_cdp( + "const r=await ev(`(async()=>{\n" + " const mk=()=>{document.querySelectorAll('.vprobe').forEach(e=>e.remove());\n" + " const A=document.createElement('textarea');A.className='vprobe';document.body.appendChild(A);\n" + " const B=document.createElement('textarea');B.className='vprobe';document.body.appendChild(B);return[A,B]};\n" + " const snap=()=>window.dispatchEvent(new CustomEvent('osw-test:snapshot'));\n" + " const inj=t=>window.dispatchEvent(new CustomEvent('osw-test:inject',{detail:{text:t}}));\n" + " const w=ms=>new Promise(r=>setTimeout(r,ms));\n" + " let A,B;\n" + " [A,B]=mk();A.focus();snap();await w(60);B.focus();await w(60);inj('switch words');await w(160);\n" + " const s1={A:A.value,B:B.value};A.remove();B.remove();\n" + " [A,B]=mk();A.focus();snap();await w(60);A.remove();B.focus();inj('orphan words');await w(160);\n" + " const s2={B:B.value};B.remove();\n" + " [A,B]=mk();A.focus();snap();await w(60);inj('normal words');await w(160);\n" + " const s3={A:A.value};A.remove();B.remove();\n" + " [A,B]=mk();if(document.activeElement&&document.activeElement.blur)document.activeElement.blur();\n" + " document.body.focus();const before=[...document.querySelectorAll('textarea,input')].map(e=>e.value).join('|');\n" + " snap();await w(60);inj('composer words');await w(260);\n" + " const after=[...document.querySelectorAll('textarea,input')].map(e=>e.value).join('|');\n" + " A.remove();B.remove();\n" + " return JSON.stringify({s1,s2,s3,fallbackRouted:before!==after});\n" + "})()`);\n" + "console.log(r);" + , timeout=200) + if not out: + row("ENG-176 dictation (4 scenarios)", "SKIP", "probe returned nothing (test seam is dev-only)") + return + s1, s2, s3 = out.get("s1", {}), out.get("s2", {}), out.get("s3", {}) + ok = (s1.get("A") == "switch words" and s1.get("B") == "" + and s2.get("B") == "" and s3.get("A") == "normal words") + row("ENG-176 dictation (4 scenarios)", "PASS" if ok else "FAIL", + f"switch={s1.get('A')!r}/B={s1.get('B')!r}, lost-target B={s2.get('B')!r}, " + f"normal={s3.get('A')!r}, fallback routed={out.get('fallbackRouted')}") + + +def check_scroll_both_halves() -> None: + """Wheel-storm asserting BOTH halves on every surface it can confirm.""" + out = p_cdp( + "const CAM=\"(function(){var c=document.querySelector('[data-select-type]');if(!c)return 'NC';" + "var el=c.parentElement;while(el){var t=getComputedStyle(el).transform;" + "if(t&&t!=='none'&&!/matrix\\\\(1, 0, 0, 1/.test(t))return t;el=el.parentElement}return 'NM'})()\";\n" + "const FIND=\"(function(){var vw=innerWidth,vh=innerHeight;for(var y=70;yn.clientHeight+8){window.__S__=n;" + "return JSON.stringify({x:x,y:y})}n=n.parentElement}}}return 'none'})()\";\n" + "const f=await ev(FIND);\n" + "if(f==='none'){console.log(JSON.stringify({found:false}));}else{\n" + " const q=JSON.parse(f);const cb=await ev(CAM);const tb=await ev('window.__S__.scrollTop');\n" + " for(let i=0;i<10;i++){await send('Input.dispatchMouseEvent',{type:'mouseWheel',x:q.x,y:q.y,deltaX:0,deltaY:120});await sleep(70);}\n" + " await sleep(800);\n" + " const ta=await ev('window.__S__.scrollTop');const ca=await ev(CAM);\n" + " console.log(JSON.stringify({found:true,scrolled:ta>tb,cameraStill:cb===ca,from:tb,to:ta}));}" + , timeout=200) + if not out or not out.get("found"): + row("wheel-storm (both halves)", "SKIP", "no confirmable scrollable surface on screen") + return + ok = out["scrolled"] and out["cameraStill"] + row("wheel-storm (both halves)", "PASS" if ok else "FAIL", + f"scrollTop {out['from']}->{out['to']} (positive={'ok' if out['scrolled'] else 'FAIL'}), " + f"camera {'byte-identical' if out['cameraStill'] else 'MOVED'}")