mirror of
https://github.com/openswarm-ai/openswarm.git
synced 2026-09-09 11:17:44 +02:00
[eric] update cross-module + test refs to promoted service/version symbols
This commit is contained in:
@@ -1891,10 +1891,10 @@ async def run_browser_agent(
|
||||
{"type": "text", "text": f"\n\n💡 Suggested next step: {guidance}"}
|
||||
]
|
||||
|
||||
_ok = "error" not in result
|
||||
p_ok = "error" not in result
|
||||
browser_metrics.record_tool(
|
||||
session_id, browser_id, turn, tu.name, elapsed_ms,
|
||||
ok=_ok, error=result.get("error", ""),
|
||||
ok=p_ok, error=result.get("error", ""),
|
||||
is_loop=is_loop, stagnation_streak=stagnation_streak,
|
||||
result_len=len(str(result.get("text") or result.get("error") or "")),
|
||||
)
|
||||
|
||||
@@ -46,7 +46,7 @@ from .runtime_proc import (
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Module-level lock so only ONE vite optimizeDeps runs at a time; must be acquired before manager._lock to avoid deadlock with manager.attach.
|
||||
# Module-level lock so only ONE vite optimizeDeps runs at a time; must be acquired before manager.p_lock to avoid deadlock with manager.attach.
|
||||
p_vite_boot_lock = asyncio.Lock()
|
||||
|
||||
|
||||
@@ -105,7 +105,7 @@ class AppRuntime:
|
||||
self._stderr_task: Optional[asyncio.Task] = None
|
||||
self._wait_task: Optional[asyncio.Task] = None
|
||||
self._frontend_ready_task: Optional[asyncio.Task] = None
|
||||
self._lock = asyncio.Lock()
|
||||
self.p_lock = asyncio.Lock()
|
||||
|
||||
def drain_errors(self) -> list[str]:
|
||||
"""Pop and return all accumulated error lines. Used by the
|
||||
@@ -178,7 +178,7 @@ class AppRuntime:
|
||||
burst of "create 3 apps in 5 seconds" doesn't trigger 3 parallel
|
||||
MUI pre-bundle runs each pegging a core.
|
||||
"""
|
||||
async with self._lock:
|
||||
async with self.p_lock:
|
||||
if self.running:
|
||||
return True
|
||||
|
||||
@@ -444,7 +444,7 @@ class AppRuntime:
|
||||
return env
|
||||
|
||||
async def stop(self) -> None:
|
||||
async with self._lock:
|
||||
async with self.p_lock:
|
||||
if not self.process or self.process.returncode is not None:
|
||||
# Still cancel the bind poller in case stop() races a
|
||||
# never-launched runtime; defensive no-op otherwise.
|
||||
@@ -558,7 +558,7 @@ class AppRuntimeManager:
|
||||
# alive. OrderedDict gives O(1) move_to_end + popitem(last=False)
|
||||
# for LRU semantics.
|
||||
self._idle_lru: "OrderedDict[str, AppRuntime]" = OrderedDict()
|
||||
self._lock = asyncio.Lock()
|
||||
self.p_lock = asyncio.Lock()
|
||||
|
||||
async def attach(self, workspace_id: str, workspace_path: str) -> AppRuntime:
|
||||
revived = False
|
||||
@@ -566,7 +566,7 @@ class AppRuntimeManager:
|
||||
# revive-idle branch used to skip the assignment, leaving the
|
||||
# post-lock `if dead is not None:` check throwing UnboundLocalError.
|
||||
dead: Optional[AppRuntime] = None
|
||||
async with self._lock:
|
||||
async with self.p_lock:
|
||||
rt = self.runtimes.get(workspace_id)
|
||||
if rt is None:
|
||||
# Maybe the runtime is sitting idle in the LRU; revive
|
||||
@@ -609,7 +609,7 @@ class AppRuntimeManager:
|
||||
async def detach(self, workspace_id: str) -> None:
|
||||
to_idle: Optional[AppRuntime] = None
|
||||
to_reap: list[AppRuntime] = []
|
||||
async with self._lock:
|
||||
async with self.p_lock:
|
||||
count = self._attached.get(workspace_id, 0) - 1
|
||||
if count > 0:
|
||||
self._attached[workspace_id] = count
|
||||
@@ -714,7 +714,7 @@ class AppRuntimeManager:
|
||||
so they can run their own shutdown. Parallel via gather; with the
|
||||
per-runtime 3s SIGTERM grace, worst case is one ~3s wait rather than
|
||||
N*3s. Idempotent; safe to invoke from multiple shutdown paths."""
|
||||
async with self._lock:
|
||||
async with self.p_lock:
|
||||
victims: list[AppRuntime] = []
|
||||
for rt in list(self.runtimes.values()):
|
||||
victims.append(rt)
|
||||
|
||||
@@ -171,7 +171,7 @@ async def discover_mcp_tools_stdio(command: str, args: list[str] | None = None,
|
||||
|
||||
stderr_task = asyncio.create_task(_drain_stderr())
|
||||
|
||||
async def _send(msg: dict) -> None:
|
||||
async def p_send(msg: dict) -> None:
|
||||
line = json.dumps(msg) + "\n"
|
||||
proc.stdin.write(line.encode())
|
||||
await proc.stdin.drain()
|
||||
@@ -204,7 +204,7 @@ async def discover_mcp_tools_stdio(command: str, args: list[str] | None = None,
|
||||
return data
|
||||
|
||||
try:
|
||||
await _send({
|
||||
await p_send({
|
||||
"jsonrpc": "2.0", "id": 1, "method": "initialize",
|
||||
"params": {
|
||||
"protocolVersion": "2025-03-26",
|
||||
@@ -219,9 +219,9 @@ async def discover_mcp_tools_stdio(command: str, args: list[str] | None = None,
|
||||
# against an already-running server and stay at the default 30 s.
|
||||
await _recv(timeout_s=120.0)
|
||||
|
||||
await _send({"jsonrpc": "2.0", "method": "notifications/initialized"})
|
||||
await p_send({"jsonrpc": "2.0", "method": "notifications/initialized"})
|
||||
|
||||
await _send({"jsonrpc": "2.0", "id": 2, "method": "tools/list", "params": {}})
|
||||
await p_send({"jsonrpc": "2.0", "id": 2, "method": "tools/list", "params": {}})
|
||||
data = await _recv()
|
||||
|
||||
tools_list = data.get("result", {}).get("tools", [])
|
||||
|
||||
@@ -31,7 +31,7 @@ def test_host_of():
|
||||
assert sk.host_of("https://docs.google.com/x") == "docs.google.com"
|
||||
|
||||
|
||||
def _log():
|
||||
def p_log():
|
||||
return [
|
||||
{"tool": "BrowserScreenshot", "input": {}, "ok": False},
|
||||
{"tool": "BrowserNavigate", "input": {"url": "http://h/form"}, "ok": True},
|
||||
@@ -43,7 +43,7 @@ def _log():
|
||||
|
||||
|
||||
def test_distill_builds_robust_steps():
|
||||
steps = sk.distill_steps(_log())
|
||||
steps = sk.distill_steps(p_log())
|
||||
tools = [s["tool"] for s in steps]
|
||||
# reads/screenshots dropped; click_index becomes a robust click-by-name
|
||||
assert tools == ["BrowserNavigate", "BrowserType", "BrowserClickByName"]
|
||||
@@ -101,14 +101,14 @@ def test_distill_bails_on_batched_click_index():
|
||||
|
||||
|
||||
def test_record_and_find_roundtrip():
|
||||
assert sk.record_skill("localhost:8901", "type hello and click Send", _log()) is True
|
||||
assert sk.record_skill("localhost:8901", "type hello and click Send", p_log()) is True
|
||||
found = sk.find_skill("localhost:8901", "Please type hello and click Send")
|
||||
assert found is not None
|
||||
assert [s["tool"] for s in found["steps"]] == ["BrowserNavigate", "BrowserType", "BrowserClickByName"]
|
||||
|
||||
|
||||
def test_find_is_host_scoped():
|
||||
sk.record_skill("a.com", "do thing now", _log())
|
||||
sk.record_skill("a.com", "do thing now", p_log())
|
||||
assert sk.find_skill("b.com", "do thing now") is None
|
||||
|
||||
|
||||
@@ -122,7 +122,7 @@ def test_record_refuses_unrecordable_run():
|
||||
def test_skill_persists_across_restart(_isolated_skills):
|
||||
# record, then simulate a process restart by wiping ONLY the in-memory cache;
|
||||
# find must re-load it from disk.
|
||||
assert sk.record_skill("localhost:8901", "type hello and click Send", _log()) is True
|
||||
assert sk.record_skill("localhost:8901", "type hello and click Send", p_log()) is True
|
||||
sk.clear(wipe_disk=False) # in-memory gone, disk intact (== restart)
|
||||
assert not sk.SKILLS # cache truly empty
|
||||
found = sk.find_skill("localhost:8901", "type hello and click Send")
|
||||
@@ -187,7 +187,7 @@ def test_navigate_url_userinfo_and_fragment_stripped_on_disk(_isolated_skills):
|
||||
|
||||
|
||||
def test_format_version_mismatch_is_ignored(_isolated_skills, monkeypatch):
|
||||
sk.record_skill("v.com", "do a thing now", _log())
|
||||
sk.record_skill("v.com", "do a thing now", p_log())
|
||||
sk.clear(wipe_disk=False)
|
||||
monkeypatch.setattr(sk, "P_SKILL_FORMAT_VERSION", 999) # pretend the format moved on
|
||||
assert sk.find_skill("v.com", "do a thing now") is None
|
||||
@@ -236,7 +236,7 @@ def test_rehydrate_aborts_when_slot_cannot_be_filled(_isolated_skills):
|
||||
|
||||
def test_unquoted_text_stays_literal_backward_compatible(_isolated_skills):
|
||||
# no quotes -> behaves exactly as before (literal text, exact-ish key)
|
||||
assert sk.record_skill("localhost:8901", "type hello and click Send", _log()) is True
|
||||
assert sk.record_skill("localhost:8901", "type hello and click Send", p_log()) is True
|
||||
found = sk.find_skill("localhost:8901", "Please type hello and click Send")
|
||||
assert found is not None
|
||||
concrete = sk.rehydrate(found, "Please type hello and click Send")
|
||||
@@ -246,9 +246,9 @@ def test_unquoted_text_stays_literal_backward_compatible(_isolated_skills):
|
||||
|
||||
# --- skill self-awareness (list / deprecate) ------------------------------
|
||||
def test_list_skills_for_host(_isolated_skills):
|
||||
sk.record_skill("shop.com", "search for shoes now", _log())
|
||||
sk.record_skill("shop.com", "add item to the cart now", _log())
|
||||
sk.record_skill("other.com", "do a thing now", _log())
|
||||
sk.record_skill("shop.com", "search for shoes now", p_log())
|
||||
sk.record_skill("shop.com", "add item to the cart now", p_log())
|
||||
sk.record_skill("other.com", "do a thing now", p_log())
|
||||
listed = sk.list_skills("shop.com")
|
||||
tasks = {x["task"] for x in listed}
|
||||
assert len(listed) == 2 and all("steps" in x and "replays" in x for x in listed)
|
||||
@@ -256,13 +256,13 @@ def test_list_skills_for_host(_isolated_skills):
|
||||
|
||||
|
||||
def test_list_skills_reads_disk_after_restart(_isolated_skills):
|
||||
sk.record_skill("shop.com", "search for shoes now", _log())
|
||||
sk.record_skill("shop.com", "search for shoes now", p_log())
|
||||
sk.clear(wipe_disk=False) # restart: memory gone, disk intact
|
||||
assert len(sk.list_skills("shop.com")) == 1
|
||||
|
||||
|
||||
def test_deprecate_removes_skill_from_memory_and_disk(_isolated_skills):
|
||||
sk.record_skill("shop.com", "search for shoes now", _log())
|
||||
sk.record_skill("shop.com", "search for shoes now", p_log())
|
||||
sig = sk.compute_sig("search for shoes now")
|
||||
assert os.path.exists(sk.skill_path("shop.com", sig))
|
||||
# deprecate using the task_sig as list_skills would surface it
|
||||
@@ -281,20 +281,20 @@ def test_deprecate_unknown_is_false(_isolated_skills):
|
||||
# or run slower-than-baseline; re-deriving different steps is a re-versioned EDIT.
|
||||
|
||||
def test_new_skill_starts_on_probation(_isolated_skills):
|
||||
sk.record_skill("shop.com", "do a thing now", _log())
|
||||
sk.record_skill("shop.com", "do a thing now", p_log())
|
||||
s = sk.find_skill("shop.com", "do a thing now")
|
||||
assert s["state"] == sk.PROBATION and s["rev"] == 1 and s["replays"] == 0
|
||||
|
||||
|
||||
def test_replay_success_promotes_probation_to_trusted(_isolated_skills):
|
||||
sk.record_skill("shop.com", "do a thing now", _log())
|
||||
sk.record_skill("shop.com", "do a thing now", p_log())
|
||||
sk.mark_replay_succeeded("shop.com", "do a thing now")
|
||||
s = sk.find_skill("shop.com", "do a thing now")
|
||||
assert s["state"] == sk.TRUSTED and s["replays"] == 1 and s["fails"] == 0
|
||||
|
||||
|
||||
def test_probation_failure_quarantines_and_blocks_future_replay(_isolated_skills):
|
||||
sk.record_skill("shop.com", "do a thing now", _log()) # probation
|
||||
sk.record_skill("shop.com", "do a thing now", p_log()) # probation
|
||||
verdict = sk.mark_replay_failed("shop.com", "do a thing now")
|
||||
assert verdict == "quarantined"
|
||||
# the ghost guard: a quarantined skill is NEVER handed back for replay...
|
||||
@@ -305,21 +305,21 @@ def test_probation_failure_quarantines_and_blocks_future_replay(_isolated_skills
|
||||
|
||||
|
||||
def test_quarantined_skill_re_recorded_identical_stays_quarantined(_isolated_skills):
|
||||
sk.record_skill("shop.com", "do a thing now", _log())
|
||||
sk.record_skill("shop.com", "do a thing now", p_log())
|
||||
sk.mark_replay_failed("shop.com", "do a thing now") # quarantined
|
||||
# the full LLM agent re-runs and distills the SAME (still-lossy) steps:
|
||||
sk.record_skill("shop.com", "do a thing now", _log())
|
||||
sk.record_skill("shop.com", "do a thing now", p_log())
|
||||
# it must stay quarantined -> pure-LLM baseline, never a wasted replay again
|
||||
assert sk.find_skill("shop.com", "do a thing now") is None
|
||||
assert sk.list_skills("shop.com")[0]["state"] == sk.QUARANTINE
|
||||
|
||||
|
||||
def test_quarantined_skill_unquarantines_on_a_real_edit(_isolated_skills):
|
||||
sk.record_skill("shop.com", "do a thing now", _log())
|
||||
sk.record_skill("shop.com", "do a thing now", p_log())
|
||||
sk.mark_replay_failed("shop.com", "do a thing now") # quarantined
|
||||
# now the page changed and the LLM derives a DIFFERENT click -> a real edit,
|
||||
# which earns the skill another chance (back on probation, re-versioned)
|
||||
edited = _log()[:-1] + [{"tool": "BrowserClickIndex", "input": {}, "ok": True,
|
||||
edited = p_log()[:-1] + [{"tool": "BrowserClickIndex", "input": {}, "ok": True,
|
||||
"clicked_role": "button", "clicked_name": "Submit"}]
|
||||
sk.record_skill("shop.com", "do a thing now", edited)
|
||||
s = sk.find_skill("shop.com", "do a thing now")
|
||||
@@ -327,7 +327,7 @@ def test_quarantined_skill_unquarantines_on_a_real_edit(_isolated_skills):
|
||||
|
||||
|
||||
def test_trusted_skill_tolerates_one_transient_miss_then_demotes(_isolated_skills):
|
||||
sk.record_skill("shop.com", "do a thing now", _log())
|
||||
sk.record_skill("shop.com", "do a thing now", p_log())
|
||||
sk.mark_replay_succeeded("shop.com", "do a thing now") # trusted
|
||||
assert sk.mark_replay_failed("shop.com", "do a thing now") == "kept"
|
||||
s = sk.find_skill("shop.com", "do a thing now")
|
||||
@@ -337,18 +337,18 @@ def test_trusted_skill_tolerates_one_transient_miss_then_demotes(_isolated_skill
|
||||
|
||||
|
||||
def test_re_record_identical_keeps_trust_and_rev(_isolated_skills):
|
||||
sk.record_skill("shop.com", "do a thing now", _log())
|
||||
sk.record_skill("shop.com", "do a thing now", p_log())
|
||||
sk.mark_replay_succeeded("shop.com", "do a thing now")
|
||||
sk.find_skill("shop.com", "do a thing now")["replays"] = 5 # pretend reused a lot
|
||||
sk.record_skill("shop.com", "do a thing now", _log()) # identical re-derive
|
||||
sk.record_skill("shop.com", "do a thing now", p_log()) # identical re-derive
|
||||
s = sk.find_skill("shop.com", "do a thing now")
|
||||
assert s["state"] == sk.TRUSTED and s["rev"] == 1 and s["replays"] == 5
|
||||
|
||||
|
||||
def test_re_record_different_is_an_edit_that_reversions_to_probation(_isolated_skills):
|
||||
sk.record_skill("shop.com", "do a thing now", _log())
|
||||
sk.record_skill("shop.com", "do a thing now", p_log())
|
||||
sk.mark_replay_succeeded("shop.com", "do a thing now") # trusted, rev 1
|
||||
edited = _log()[:-1] + [{"tool": "BrowserClickIndex", "input": {}, "ok": True,
|
||||
edited = p_log()[:-1] + [{"tool": "BrowserClickIndex", "input": {}, "ok": True,
|
||||
"clicked_role": "button", "clicked_name": "Submit"}]
|
||||
sk.record_skill("shop.com", "do a thing now", edited) # different -> EDIT
|
||||
s = sk.find_skill("shop.com", "do a thing now")
|
||||
@@ -358,9 +358,9 @@ def test_re_record_different_is_an_edit_that_reversions_to_probation(_isolated_s
|
||||
|
||||
|
||||
def test_rev_and_state_persist_across_restart(_isolated_skills):
|
||||
sk.record_skill("shop.com", "do a thing now", _log())
|
||||
sk.record_skill("shop.com", "do a thing now", p_log())
|
||||
sk.mark_replay_succeeded("shop.com", "do a thing now")
|
||||
edited = _log()[:-1] + [{"tool": "BrowserClickIndex", "input": {}, "ok": True,
|
||||
edited = p_log()[:-1] + [{"tool": "BrowserClickIndex", "input": {}, "ok": True,
|
||||
"clicked_role": "button", "clicked_name": "Submit"}]
|
||||
sk.record_skill("shop.com", "do a thing now", edited) # rev 2, probation
|
||||
sk.clear(wipe_disk=False) # restart
|
||||
@@ -386,7 +386,7 @@ def test_mark_replay_helpers_on_unknown_are_safe(_isolated_skills):
|
||||
|
||||
|
||||
def test_demoted_skill_can_be_re_proven(_isolated_skills):
|
||||
sk.record_skill("shop.com", "do a thing now", _log())
|
||||
sk.record_skill("shop.com", "do a thing now", p_log())
|
||||
sk.mark_replay_succeeded("shop.com", "do a thing now") # trusted
|
||||
sk.mark_replay_failed("shop.com", "do a thing now")
|
||||
sk.mark_replay_failed("shop.com", "do a thing now") # demoted to probation
|
||||
@@ -398,8 +398,8 @@ def test_demoted_skill_can_be_re_proven(_isolated_skills):
|
||||
# --- composition: build on what's already proven, propagate staleness -------
|
||||
|
||||
def _log_plus():
|
||||
# distills to _log()'s 3 steps PLUS a 4th click -> a strict superset sequence
|
||||
return _log() + [{"tool": "BrowserClickIndex", "input": {}, "ok": True,
|
||||
# distills to p_log()'s 3 steps PLUS a 4th click -> a strict superset sequence
|
||||
return p_log() + [{"tool": "BrowserClickIndex", "input": {}, "ok": True,
|
||||
"clicked_role": "button", "clicked_name": "Checkout"}]
|
||||
|
||||
|
||||
@@ -409,21 +409,21 @@ def _trust(host, task, log):
|
||||
|
||||
|
||||
def test_composition_links_to_trusted_sub_skill(_isolated_skills):
|
||||
_trust("shop.com", "search shoes now", _log()) # trusted foundation
|
||||
_trust("shop.com", "search shoes now", p_log()) # trusted foundation
|
||||
sk.record_skill("shop.com", "search shoes and checkout now", _log_plus())
|
||||
c = sk.find_skill("shop.com", "search shoes and checkout now")
|
||||
assert c["composed_of"] == [sk.compute_sig("search shoes now")]
|
||||
|
||||
|
||||
def test_composition_ignores_untrusted_foundation(_isolated_skills):
|
||||
sk.record_skill("shop.com", "search shoes now", _log()) # probation, NOT trusted
|
||||
sk.record_skill("shop.com", "search shoes now", p_log()) # probation, NOT trusted
|
||||
sk.record_skill("shop.com", "search shoes and checkout now", _log_plus())
|
||||
c = sk.find_skill("shop.com", "search shoes and checkout now")
|
||||
assert c["composed_of"] == [] # only a PROVEN sub-skill is built upon
|
||||
|
||||
|
||||
def test_deprecating_a_foundation_demotes_everything_built_on_it(_isolated_skills):
|
||||
_trust("shop.com", "search shoes now", _log())
|
||||
_trust("shop.com", "search shoes now", p_log())
|
||||
_trust("shop.com", "search shoes and checkout now", _log_plus()) # composed + trusted
|
||||
assert sk.find_skill("shop.com", "search shoes and checkout now")["state"] == sk.TRUSTED
|
||||
sk.deprecate_skill("shop.com", "search shoes now") # foundation pulled
|
||||
@@ -433,7 +433,7 @@ def test_deprecating_a_foundation_demotes_everything_built_on_it(_isolated_skill
|
||||
|
||||
|
||||
def test_demoting_a_foundation_demotes_its_dependents(_isolated_skills):
|
||||
_trust("shop.com", "search shoes now", _log())
|
||||
_trust("shop.com", "search shoes now", p_log())
|
||||
_trust("shop.com", "search shoes and checkout now", _log_plus())
|
||||
sk.mark_replay_failed("shop.com", "search shoes now")
|
||||
sk.mark_replay_failed("shop.com", "search shoes now") # foundation demoted
|
||||
@@ -441,16 +441,16 @@ def test_demoting_a_foundation_demotes_its_dependents(_isolated_skills):
|
||||
|
||||
|
||||
def test_editing_a_foundation_demotes_its_dependents(_isolated_skills):
|
||||
_trust("shop.com", "search shoes now", _log())
|
||||
_trust("shop.com", "search shoes now", p_log())
|
||||
_trust("shop.com", "search shoes and checkout now", _log_plus())
|
||||
edited = _log()[:-1] + [{"tool": "BrowserClickIndex", "input": {}, "ok": True,
|
||||
edited = p_log()[:-1] + [{"tool": "BrowserClickIndex", "input": {}, "ok": True,
|
||||
"clicked_role": "button", "clicked_name": "Find"}]
|
||||
sk.record_skill("shop.com", "search shoes now", edited) # foundation changed
|
||||
assert sk.find_skill("shop.com", "search shoes and checkout now")["state"] == sk.PROBATION
|
||||
|
||||
|
||||
def test_list_skills_surfaces_state_rev_and_builds_on(_isolated_skills):
|
||||
_trust("shop.com", "search shoes now", _log())
|
||||
_trust("shop.com", "search shoes now", p_log())
|
||||
sk.record_skill("shop.com", "search shoes and checkout now", _log_plus())
|
||||
listed = {x["task"]: x for x in sk.list_skills("shop.com")}
|
||||
foundation = listed[sk.compute_sig("search shoes now")]
|
||||
|
||||
@@ -108,23 +108,23 @@ def test_advance_fires_again_at_max():
|
||||
# Catches the worst measured ghost: multi-minute runs, every tool errored, still
|
||||
# reported 'completed'. Must NOT cry wolf on real successes (it overrides status).
|
||||
|
||||
def _ok(tool, summary="done"):
|
||||
def p_ok(tool, summary="done"):
|
||||
return {"tool": tool, "ok": True, "result_summary": summary}
|
||||
|
||||
|
||||
def _err(tool):
|
||||
def p_err(tool):
|
||||
return {"tool": tool, "ok": False, "result_summary": "Element not found: '.x'"}
|
||||
|
||||
|
||||
def test_completion_honest_when_an_action_succeeded():
|
||||
log = [_ok("BrowserListInteractives", "1 button"), _ok("BrowserClickIndex", "Clicked")]
|
||||
log = [p_ok("BrowserListInteractives", "1 button"), p_ok("BrowserClickIndex", "Clicked")]
|
||||
honest, reason = completion_is_honest(log)
|
||||
assert honest and reason == ""
|
||||
|
||||
|
||||
def test_completion_ghost_when_every_action_errored():
|
||||
# the exact LinkedIn ghost: 8 tools, all errored, model said 'completed'
|
||||
log = [_err("BrowserClick") for _ in range(8)]
|
||||
log = [p_err("BrowserClick") for _ in range(8)]
|
||||
honest, reason = completion_is_honest(log)
|
||||
assert not honest and "every state-changing action failed" in reason
|
||||
|
||||
@@ -143,14 +143,14 @@ def test_completion_ghost_when_only_looked_around_with_no_content():
|
||||
|
||||
def test_completion_honest_for_a_read_only_task_that_returned_content():
|
||||
# a legit "tell me what's on the page" task: no action, but a read got content
|
||||
log = [_ok("BrowserGetText", "The page says hello world")]
|
||||
log = [p_ok("BrowserGetText", "The page says hello world")]
|
||||
honest, reason = completion_is_honest(log)
|
||||
assert honest and reason == ""
|
||||
|
||||
|
||||
def test_completion_honest_when_some_errors_but_an_action_landed():
|
||||
# partial failure is fine as long as a real action ultimately succeeded
|
||||
log = [_err("BrowserClick"), _err("BrowserClick"), _ok("BrowserClickIndex", "Clicked Submit")]
|
||||
log = [p_err("BrowserClick"), p_err("BrowserClick"), p_ok("BrowserClickIndex", "Clicked Submit")]
|
||||
honest, reason = completion_is_honest(log)
|
||||
assert honest
|
||||
|
||||
|
||||
@@ -82,13 +82,13 @@ def install_sync_sink():
|
||||
"properties": props,
|
||||
})
|
||||
|
||||
old_sink = svc_client._test_sink
|
||||
old_iid = svc_client._install_id
|
||||
old_sink = svc_client.test_sink
|
||||
old_iid = svc_client.install_id
|
||||
svc_client.set_test_sink(_sink)
|
||||
svc_client._install_id = "test-install-id"
|
||||
svc_client.install_id = "test-install-id"
|
||||
yield
|
||||
svc_client.set_test_sink(old_sink)
|
||||
svc_client._install_id = old_iid
|
||||
svc_client.install_id = old_iid
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
|
||||
@@ -24,7 +24,7 @@ def _load_analyzer():
|
||||
return mod
|
||||
|
||||
|
||||
def _log():
|
||||
def p_log():
|
||||
return [
|
||||
{"tool": "BrowserNavigate", "input": {"url": "http://h/form"}, "ok": True},
|
||||
{"tool": "BrowserType", "input": {"selector": "#q", "text": "shoes"}, "ok": True},
|
||||
@@ -36,13 +36,13 @@ def _task_row(sig, path, dur_s, turns=None, playbook_seeded=False):
|
||||
# started_at in the past makes record_task compute a realistic total_ms.
|
||||
bm.record_task("s-" + sig + path + str(turns) + str(playbook_seeded), "b", sig, "completed",
|
||||
time.time() - dur_s, turns if turns is not None else (0 if path == "replay" else 3),
|
||||
_log(), {"input": 10, "output": 5}, path=path, task_sig=sig,
|
||||
p_log(), {"input": 10, "output": 5}, path=path, task_sig=sig,
|
||||
playbook_seeded=playbook_seeded)
|
||||
|
||||
|
||||
def test_skill_events_are_emitted_for_each_transition(_metrics_dir):
|
||||
sk.clear(wipe_disk=True)
|
||||
sk.record_skill("shop.com", "search now", _log()) # learn
|
||||
sk.record_skill("shop.com", "search now", p_log()) # learn
|
||||
sk.mark_replay_succeeded("shop.com", "search now") # promote
|
||||
sk.mark_replay_failed("shop.com", "search now") # kept (trusted, 1)
|
||||
sk.mark_replay_failed("shop.com", "search now") # demote
|
||||
@@ -56,7 +56,7 @@ def test_skill_events_are_emitted_for_each_transition(_metrics_dir):
|
||||
def test_analyzer_measures_replay_speedup_when_the_layer_helps(_metrics_dir, capsys):
|
||||
sk.clear(wipe_disk=True)
|
||||
# A repeated task: 1 slow LLM run, then 2 fast replays -> measurable speedup.
|
||||
sk.record_skill("shop.com", "search now", _log())
|
||||
sk.record_skill("shop.com", "search now", p_log())
|
||||
_task_row(sk.compute_sig("search now"), "llm", 4.0)
|
||||
sk.mark_replay_succeeded("shop.com", "search now")
|
||||
_task_row(sk.compute_sig("search now"), "replay", 0.04)
|
||||
@@ -75,9 +75,9 @@ def test_analyzer_flags_silent_non_help_thrash(_metrics_dir, capsys):
|
||||
sk.clear(wipe_disk=True)
|
||||
# A task that keeps getting re-learned/edited and quarantined, never promoted,
|
||||
# and whose runs always go via the LLM (never the fast path) = the ghost.
|
||||
sk.record_skill("bad.com", "do thing now", _log()) # learn
|
||||
sk.record_skill("bad.com", "do thing now", p_log()) # learn
|
||||
sk.mark_replay_failed("bad.com", "do thing now") # quarantine
|
||||
edited = _log()[:-1] + [{"tool": "BrowserClickIndex", "input": {}, "ok": True,
|
||||
edited = p_log()[:-1] + [{"tool": "BrowserClickIndex", "input": {}, "ok": True,
|
||||
"clicked_role": "button", "clicked_name": "Other"}]
|
||||
sk.record_skill("bad.com", "do thing now", edited) # edit (un-quarantine)
|
||||
sk.mark_replay_failed("bad.com", "do thing now") # quarantine again
|
||||
@@ -95,9 +95,9 @@ def test_analyzer_flags_silent_non_help_thrash(_metrics_dir, capsys):
|
||||
|
||||
def test_analyzer_reports_composition(_metrics_dir, capsys):
|
||||
sk.clear(wipe_disk=True)
|
||||
sk.record_skill("shop.com", "search now", _log())
|
||||
sk.record_skill("shop.com", "search now", p_log())
|
||||
sk.mark_replay_succeeded("shop.com", "search now") # trusted foundation
|
||||
plus = _log() + [{"tool": "BrowserClickIndex", "input": {}, "ok": True,
|
||||
plus = p_log() + [{"tool": "BrowserClickIndex", "input": {}, "ok": True,
|
||||
"clicked_role": "button", "clicked_name": "Checkout"}]
|
||||
sk.record_skill("shop.com", "search and checkout now", plus) # composes on foundation
|
||||
sk.mark_replay_succeeded("shop.com", "search and checkout now") # dependent earns trust too
|
||||
|
||||
@@ -2678,7 +2678,7 @@ def _make_mock_9router(initial_nodes=None, initial_conns=None, fail_endpoints=No
|
||||
return _resp(200, {"connections": state["connections"]})
|
||||
return _resp(404)
|
||||
|
||||
async def _post(url, json=None, **kw):
|
||||
async def p_post(url, json=None, **kw):
|
||||
state["calls"].append(("POST", url, json))
|
||||
if "/api/provider-nodes" in url and not url.endswith("/provider-nodes/"):
|
||||
if "POST:provider-nodes" in fail:
|
||||
@@ -2739,7 +2739,7 @@ def _make_mock_9router(initial_nodes=None, initial_conns=None, fail_endpoints=No
|
||||
async def __aexit__(self, *a):
|
||||
return False
|
||||
get = AsyncMock(side_effect=_get)
|
||||
post = AsyncMock(side_effect=_post)
|
||||
post = AsyncMock(side_effect=p_post)
|
||||
put = AsyncMock(side_effect=_put)
|
||||
patch = AsyncMock(side_effect=_patch)
|
||||
delete = AsyncMock(side_effect=_delete)
|
||||
|
||||
@@ -8,7 +8,7 @@ from unittest.mock import patch
|
||||
from backend.apps.agents.tools.web import should_register_web_mcp
|
||||
|
||||
|
||||
def _call(**kw):
|
||||
def p_call(**kw):
|
||||
base = dict(model="m", router_model_id="cc/opus", api_type="anthropic",
|
||||
anthropic_api_key=None, connection_mode="own_key")
|
||||
base.update(kw)
|
||||
@@ -18,23 +18,23 @@ def _call(**kw):
|
||||
|
||||
def test_custom_session_always_registers():
|
||||
# ANTHROPIC_BASE_URL points at 9Router with no Claude connection -> native WebSearch 401s.
|
||||
assert _call(api_type="custom") is True
|
||||
assert p_call(api_type="custom") is True
|
||||
|
||||
|
||||
def test_non_claude_primary_registers():
|
||||
# A Gemini/GPT primary has no native Anthropic web path; Pro pool is not counted for it.
|
||||
assert _call(router_model_id="gemini/flash", api_type="google") is True
|
||||
assert p_call(router_model_id="gemini/flash", api_type="google") is True
|
||||
|
||||
|
||||
def test_claude_pro_uses_native_path():
|
||||
# Claude primary on Pro: the managed pool entitles the built-in WebSearch, so don't register.
|
||||
assert _call(router_model_id="cc/opus", api_type="anthropic", connection_mode="openswarm-pro") is False
|
||||
assert p_call(router_model_id="cc/opus", api_type="anthropic", connection_mode="openswarm-pro") is False
|
||||
|
||||
|
||||
def test_subscription_route_claude_non_pro_registers():
|
||||
# opus-4-8 on a non-Pro own-key account: the aux haiku call 401s through 9Router, so a bare
|
||||
# key isn't enough -> fall back to openswarm-web.
|
||||
assert _call(router_model_id="cc/opus", api_type="anthropic",
|
||||
assert p_call(router_model_id="cc/opus", api_type="anthropic",
|
||||
connection_mode="own_key", anthropic_api_key="sk-ant-xxx") is True
|
||||
|
||||
|
||||
|
||||
@@ -140,9 +140,9 @@ import backend.apps.agents.tools.ssrf_guard as _ssrf
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _allow_urls(monkeypatch):
|
||||
async def _ok(url):
|
||||
async def p_ok(url):
|
||||
return None
|
||||
monkeypatch.setattr(_ssrf, "assert_safe_url", _ok)
|
||||
monkeypatch.setattr(_ssrf, "assert_safe_url", p_ok)
|
||||
|
||||
|
||||
def _local_returns(monkeypatch, text):
|
||||
|
||||
Reference in New Issue
Block a user