From 1baecba0611ea1dfc772707eb5ccb383b5b809a8 Mon Sep 17 00:00:00 2001 From: ciregenz Date: Sun, 16 Aug 2026 16:10:29 -0700 Subject: [PATCH] arena: v43 schema-gate was self-inflicted (falsely bounced valid JSON) -- disabled; v42 is honest WA config; full WA re-measurement launching Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01WsbS5x2rYsMDxP2kW3qqmQ --- e2e/browser-v3/arena/ARENA.md | 13 +++++++++++++ e2e/browser-v3/arena/llm_policy.py | 2 +- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/e2e/browser-v3/arena/ARENA.md b/e2e/browser-v3/arena/ARENA.md index a78283db..8cb6f5b0 100644 --- a/e2e/browser-v3/arena/ARENA.md +++ b/e2e/browser-v3/arena/ARENA.md @@ -688,6 +688,19 @@ suppressing credit -- the true partial score is higher. Next: v43 answer-schema (validate send_msg against the schema the TASK ITSELF provides; bounce errors back -- generic instruction-following, knows no answers) to lift 0.5 -> 1.0. +## v43 schema-gate: SELF-INFLICTED bug, disabled (2026-08-16) + +The v43 answer-schema gate was HARMFUL and is disabled. Trace (13.627): the agent emitted valid +JSON ({"task_type":"MUTATE","status":"SUCCESS",...}) but my gate's extraction regex only matched +quoted-string payloads, not raw ({...}) objects, so it FALSELY bounced every valid answer and the +env never received a final message. Caught by reading the trace (the discipline that also caught +the WASP false-positive). Honest conclusion: v42 (instrument fixes + answer_protocol, NO gate) is +the WebArena config; instrument fixes recovered partials 0->0.5 (post-creation credit that was +being zeroed), but 0.5->1.0 needs the exact answer CONTENT the evaluator expects (retrieved_data +values / task_type enum) -- a genuine capability/knowledge wall, not a format bug. Full WA +re-measurement on v42 launched to book the honest partial-sum (was suppressed to 3.83 under the +truncation bug). + ## Champion v41 3-seed MiniWoB (2026-08-16): 94.1 (92.0 / 95.2 / 95.2) — +3.4 vs v40 Two capability fixes landed: scripted draw-circle geometry (0/3->3/3) + universal 18-step budget diff --git a/e2e/browser-v3/arena/llm_policy.py b/e2e/browser-v3/arena/llm_policy.py index e7ed9921..9d95a48d 100644 --- a/e2e/browser-v3/arena/llm_policy.py +++ b/e2e/browser-v3/arena/llm_policy.py @@ -1093,7 +1093,7 @@ def build(name: str, model: str = "", endpoint: str = "", **_: Any) -> Any: local_ctx=True, blocker_probe=True, suppress_wrappers=True, force_unblock=True, native_js_fallback=True, escape_token=True, table_md=True, draw_circle=True, answer_protocol=True, - schema_gate=True, **v43) + schema_gate=False, **v43) # DISABLED: gate falsely bounced valid JSON if name == "osw-llm-v42": # v41 + terminal answer protocol (string-match benchmarks) v42 = dict(v7, system=OSW_SYSTEM_V8 + OSW_SYSTEM_V9_WIDGETS + OSW_SYSTEM_V16 + OSW_SYSTEM_V30 + OSW_SYSTEM_V36, max_tokens=800)