diff --git a/e2e/browser-v3/arena/ARENA.md b/e2e/browser-v3/arena/ARENA.md index 37759843..20c110cb 100644 --- a/e2e/browser-v3/arena/ARENA.md +++ b/e2e/browser-v3/arena/ARENA.md @@ -1045,3 +1045,19 @@ MINIWOB last-mile: 94.1 is the honest near-ceiling for the scripted-primitive ap MiniWoB's own reward machinery (search-engine pick_result tried 2 ways, 0/3 -- disabled). Remaining 0.9pt to 95 needs framework-native primitives with diminishing returns; deprioritized BELOW the higher-ROI verified-writes clause. Two seeds already hit 95.2; 94.1 mean is variance-dragged. + +## Verified-writes: independent re-read measurement (2026-08-16, clause 5) +Built vw_verify.py -- independent server-state re-read per write task (program_html url + +required_contents), decoupled from WebArena's composite reward. KEY HONEST FINDING: the composite +reward MISLED toward optimism -- 11/15 write tasks scored 0.5+ (looked like persisted writes) but +independent re-read shows only ~8% (1/13 clean) ACTUALLY persisted. The 0.5 was navigation/partial +credit, NOT write success. Manual spot-check: simple MR-comment writes DO persist (confirmed +'Thanks, working on reviews' on MR 450); but complex/multi-step writes (create repo, edit post, +private project, add file) mostly fail -- a genuine capability wall, NOT an instrument bug. This is +the independent-channel verification pattern catching a FALSE-POSITIVE (opposite of the WebArena +instrument bug which caused false-negatives) -- the discipline works both directions. Clause 5 +(>=95) is FAR OPEN: verified-write ~8-15% on this sample; the agent handles atomic comments but not +multi-step create/edit writes. Scorer caveats: a few url=last tasks need the episode's final page +(not independently checkable), and the write-filter caught a couple read tasks -- number will +tighten with a cleaned write-only partition, but the signal (far below 95) is unambiguous. +Honest verdict: verified-writes is capability-gated for complex writes, not a quick clause to close. diff --git a/e2e/browser-v3/arena/vw_verify.py b/e2e/browser-v3/arena/vw_verify.py new file mode 100644 index 00000000..6a726d43 --- /dev/null +++ b/e2e/browser-v3/arena/vw_verify.py @@ -0,0 +1,67 @@ +"""Independent verified-write scorer: re-read each write task's target from the server and check +the written content persisted -- the honest write-success signal, decoupled from WebArena's +composite reward (which the answer-schema caps). No LLM judge; pure server-state re-read.""" +import json, re, sys +from pathlib import Path +from playwright.sync_api import sync_playwright + +WA = json.load(open("/Users/eric/.cache/arena/study/webarena/config_files/test.raw.json")) +BYID = {str(t["task_id"]): t for t in WA} +SUB = {"__GITLAB__": "http://localhost:8023", "__REDDIT__": "http://localhost:9999"} + +def check(pg, task) -> tuple[bool, str]: + ev = task.get("eval", {}) + for c in ev.get("program_html", []): + url = c.get("url", "") + for k, v in SUB.items(): + url = url.replace(k, v) + if url in ("last", ""): + return None, "url=last (needs episode final page, skipped)" + try: + pg.goto(url, timeout=15000, wait_until="domcontentloaded"); pg.wait_for_timeout(1200) + except Exception as e: + return False, f"fetch-fail {str(e)[:40]}" + loc = c.get("locator", "") + if loc and loc.startswith("document."): + try: + text = pg.evaluate(f"() => {{ const e = {loc}; return e ? (e.textContent||e.outerHTML) : ''; }}") + except Exception: + text = pg.content() + else: + text = pg.content() + req = c.get("required_contents", {}) + if "must_include" in req: + if not all(str(x) in text for x in req["must_include"]): + return False, "must_include absent" + if "exact_match" in req: + if req["exact_match"].strip() not in text: + return False, "exact_match absent" + return True, "write persisted" + +def main(): + sample = json.load(open(sys.argv[1]))["write_task_ids"] + with sync_playwright() as p: + b = p.chromium.launch(headless=True); pg = b.new_page() + out = [] + for tid in sample: + t = BYID.get(tid) + if not t: + continue + try: + ok, why = check(pg, t) + except Exception as e: + ok, why = None, f"err {str(e)[:40]}" + out.append({"id": tid, "persisted": ok, "why": why, "intent": t["intent"][:50]}) + print(out[-1]) + b.close() + scored = [r for r in out if r["persisted"] in (True, False)] + ok = sum(1 for r in scored if r["persisted"]) + print(f"\nVERIFIED-WRITE (independent re-read): {ok}/{len(scored)} persisted", end="") + if scored: + import math + n, ph = len(scored), ok/len(scored); z=1.96 + lo=(ph+z*z/(2*n)-z*math.sqrt((ph*(1-ph)+z*z/(4*n))/n))/(1+z*z/n) + print(f" = {100*ph:.0f}% (Wilson95 lo {100*lo:.0f}%)") + +if __name__ == "__main__": + main()