WEBDIENSTE/openswarm
1
0
Fork 0
mirror of https://github.com/openswarm-ai/openswarm.git synced 2026-09-09 19:27:45 +02:00
Code Issues Packages Projects Releases Wiki Activity

arena: bu_real scores non-MiniWoB suites through the task's own validate()

Browse Source 
The MiniWoB page global does not exist on AssistantBench pages, so every competitor
episode there scored a silent 0.0 -- a perfect-zero that looked like a result. Dotted
task ids now score through env.task.validate() exactly as BrowserGym's own step does,
and the retracted numbers can be re-measured honestly.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01WsbS5x2rYsMDxP2kW3qqmQ
This commit is contained in:
ciregenz
2026-08-12 10:53:36 -07:00
co-authored by Claude Fable 5
parent 9456402524
commit d667e6cda9
1 changed files with 17 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files
e2e/browser-v3/arena/bu_real.py
+17 -6
View File
@@ -180,12 +180,23 @@ def drive_in_thread(goal: str, cdp_url: str, model: str, endpoint: str, max_step
return result
def score(env) -> tuple[float, float]:
"""MiniWoB's verdict, read straight off the page globals -- the only grader in this file."""
def score(env, task: str = "") -> tuple[float, float]:
"""The suite's OWN verdict. MiniWoB exposes page globals; every other suite scores through its
task object's validate(). Reading the MiniWoB global on an AssistantBench page silently
returned 0.0 for every episode -- a scoring bug that read as a perfect-zero result."""
page = env.unwrapped.page
reward = float(page.evaluate("typeof WOB_REWARD_GLOBAL !== 'undefined' ? WOB_REWARD_GLOBAL : 0") or 0)
raw = float(page.evaluate("typeof WOB_RAW_REWARD_GLOBAL !== 'undefined' ? WOB_RAW_REWARD_GLOBAL : 0") or 0)
return reward, raw
if "." not in task:
reward = float(page.evaluate("typeof WOB_REWARD_GLOBAL !== 'undefined' ? WOB_REWARD_GLOBAL : 0") or 0)
raw = float(page.evaluate("typeof WOB_RAW_REWARD_GLOBAL !== 'undefined' ? WOB_RAW_REWARD_GLOBAL : 0") or 0)
return reward, raw
# Non-MiniWoB suites: ask the task itself, exactly as BrowserGym's own step() would.
try:
chat = getattr(env.unwrapped, "chat", None)
msgs = list(getattr(chat, "messages", []) or [])
r, _done, _msg, _info = env.unwrapped.task.validate(page, msgs)
return float(r or 0), float(r or 0)
except Exception:
return 0.0, 0.0
def main() -> None:
@@ -239,7 +250,7 @@ def main() -> None:
args.max_steps, args.episode_timeout) if cdp_url else {}
ep.wall_s = time.time() - t0
try:
ep.reward, ep.raw_reward = score(env)
ep.reward, ep.raw_reward = score(env, task)
except Exception as exc:
ep.error_class = ep.error_class or "infra_score_readback"
ep.error_detail = f"{type(exc).__name__}: {exc}"[:200]