From 3c8751c0b87fcc8a079e9961622f4993cf125bbd Mon Sep 17 00:00:00 2001 From: ciregenz Date: Wed, 12 Aug 2026 20:52:51 -0700 Subject: [PATCH] arena: runway scales with instruction complexity -- the first clean CompWoB pattern Clean-harness CompWoB opened 6/6 on 2-3-clause compositions and 0/3 on 7-8-clause monsters whose traces show correct work hitting the step cap. Episode budget now grows with clause count (feature-triggered from the goal text, capped at 3x) -- a general long-instruction mechanism, not a benchmark tweak. Resume keeps the six clean wins. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01WsbS5x2rYsMDxP2kW3qqmQ --- e2e/browser-v3/arena/run.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/e2e/browser-v3/arena/run.py b/e2e/browser-v3/arena/run.py index 68a1699d..465b230a 100644 --- a/e2e/browser-v3/arena/run.py +++ b/e2e/browser-v3/arena/run.py @@ -107,6 +107,15 @@ def run_episode(arm: str, task: str, seed: int, rec: Recorder, args: argparse.Na obs, _ = with_deadline(setup, args.setup_timeout) env = holder[0] + # Runway scales with instruction complexity -- a 7-clause goal legitimately needs ~3 steps + # per clause. Feature-triggered (comma/then/and counts), never task names; capped at 3x. + goal_now = str(obs.get("goal") or "") + import re as _re + clauses = 1 + len(_re.findall(r",| then | and then |after you|after clicking", goal_now)) + if clauses >= 4: + grown = min(args.max_steps * 3, max(args.max_steps, clauses * 5)) + if grown > args.max_steps: + env._max_episode_steps = grown # gym TimeLimit wrapper attribute except Exception as exc: ep.error_class = classify(exc) ep.error_detail = f"{type(exc).__name__}: {exc}"[:200]