WEBDIENSTE/openswarm
1
0
Fork 0
mirror of https://github.com/openswarm-ai/openswarm.git synced 2026-08-17 18:25:42 +02:00
Code Issues Packages Projects Releases Wiki Activity

arena: v42 terminal answer protocol (bare final answer on string-match goals) -- generic output hygiene, offline-verified

Browse Source 
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01WsbS5x2rYsMDxP2kW3qqmQ
This commit is contained in:
ciregenz
2026-08-16 12:57:38 -07:00
co-authored by Claude Fable 5
parent b4c011f8a3
commit 3972f543fc
2 changed files with 24 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files
e2e/browser-v3/arena/llm_policy.py
+12
View File
@@ -192,6 +192,8 @@ class LlmPolicy:
table_md: bool = False
# v41: scripted freehand-circle geometry (run.py-side ring path). Feature-gated on circle goals.
draw_circle: bool = False
# v42: terminal answer protocol (run.py-side). Force a bare final answer on string-match goals.
answer_protocol: bool = False
# v39: mutation-diff action feedback -- run.py appends the page-text delta after each action
# (Agent-E's MutationObserver, approximated text-side). Autocomplete popups, error banners,
# and new rows become explicit feedback instead of something the model must notice unaided.
@@ -1028,6 +1030,16 @@ def build(name: str, model: str = "", endpoint: str = "", **_: Any) -> Any:
local_ctx=True, blocker_probe=True, suppress_wrappers=True,
force_unblock=True, native_js_fallback=True,
table_md=True, mutation_diff=True, **c)
if name == "osw-llm-v42": # v41 + terminal answer protocol (string-match benchmarks)
v42 = dict(v7, system=OSW_SYSTEM_V8 + OSW_SYSTEM_V9_WIDGETS + OSW_SYSTEM_V16 + OSW_SYSTEM_V30
+ OSW_SYSTEM_V36, max_tokens=800)
return OpenSwarmLlmPolicy(name=name, multi=True, vision="progressive", fastpath=True,
scripted_drag=True, auto_complete=True, som=False,
native_pickers=True, verify_terminal=True, post_mouse_vision=True,
multi_cap=6, fill_verify=True, dispatch=True, offscreen=True,
local_ctx=True, blocker_probe=True, suppress_wrappers=True,
force_unblock=True, native_js_fallback=True, escape_token=True,
table_md=True, draw_circle=True, answer_protocol=True, **v42)
if name == "osw-llm-v41": # v40 champion + scripted draw-circle geometry primitive
v41 = dict(v7, system=OSW_SYSTEM_V8 + OSW_SYSTEM_V9_WIDGETS + OSW_SYSTEM_V16 + OSW_SYSTEM_V30
+ OSW_SYSTEM_V36, max_tokens=800)
e2e/browser-v3/arena/run.py
+12 -8
View File
@@ -191,14 +191,18 @@ def run_episode(arm: str, task: str, seed: int, rec: Recorder, args: argparse.Na
try:
for step in range(1, args.max_steps + 1):
t_perc = time.time()
# Q&A goals are scored on the CHAT ANSWER; an episode that pages forever and never
# answers scores 0 no matter what it learned (measured: 1/17 chore episodes sent one).
# On the last budgeted step, tell the policy the budget is up so it answers NOW.
if (step == args.max_steps and hasattr(policy, "history")
and re.search(r"^(calculate|how many|what|which|find|count|tell me|list|give me)|\?\s*$",
ep.goal.strip(), re.I)):
policy.history.append("(FINAL STEP: budget exhausted -- if the goal asks for "
"information, reply ONLY send_msg_to_user(\"<your best answer>\") now)")
# Terminal answer protocol (gated by policy.answer_protocol): string-match benchmarks
# grade ONLY the last chat message and auto-fail on none; exact-match dies on wrapper
# text. On the last budgeted step force a BARE answer -- no markdown, no hedging.
if (getattr(policy, "answer_protocol", False) and step == args.max_steps
and hasattr(policy, "history")
and re.search(r"^(calculate|how many|what|which|find|count|tell me|list|give me"
r"|name|identify|when|who|where)\b|\?\s*$", ep.goal.strip(), re.I)):
policy.history.append(
"(FINAL STEP -- budget is up. Reply with EXACTLY ONE action: "
"send_msg_to_user(\"<answer>\") where <answer> is the BARE value only -- a "
"number, a name, or a comma-separated list. NO markdown, NO 'the answer is', "
"NO units unless the goal asked. If genuinely unknown, answer your best guess.)")
nodes, ax_chars = perception.axtree_stats(obs)
# The LLM call rides inside act(); urlopen's timeout does not cover every hang mode.
decision = with_deadline(lambda: policy.act(obs, ep.goal), args.step_timeout + 90)