diff --git a/backend/apps/agents/browser/browser_agent.py b/backend/apps/agents/browser/browser_agent.py index 2d71bb5c..26e854b1 100644 --- a/backend/apps/agents/browser/browser_agent.py +++ b/backend/apps/agents/browser/browser_agent.py @@ -388,11 +388,18 @@ async def run_browser_agent( if m: replay_host = browser_skills.host_of(m.group(0)) skill = browser_skills.find_skill(replay_host, task) if replay_host else None - if skill: - logger.info(f"[browser-skills] REPLAY attempt: {len(skill['steps'])} steps on {replay_host}") + # Fill any parameter slots from THIS task's quoted values (so one learned + # skill serves "do the same thing with a different input"). If a slot can't + # be filled, concrete_steps is None and we run the full agent instead. + concrete_steps = browser_skills.rehydrate(skill, task) if skill else None + if skill and not concrete_steps: + logger.info(f"[browser-skills] skill matched on {replay_host} but slots unfillable from task; running full agent") + skill = None + if skill and concrete_steps: + logger.info(f"[browser-skills] REPLAY attempt: {len(concrete_steps)} steps on {replay_host}") replay_log: list[dict] = [] replay_ok = True - for step in skill["steps"]: + for step in concrete_steps: if cancel_event.is_set(): replay_ok = False break @@ -440,7 +447,7 @@ async def run_browser_agent( }) return { "session_id": session_id, "browser_id": browser_id, - "summary": f"Completed via learned skill replay ({len(skill['steps'])} steps, no LLM).", + "summary": f"Completed via learned skill replay ({len(concrete_steps)} steps, no LLM).", "action_log": replay_log, "final_screenshot": final_screenshot, "replayed": True, } diff --git a/backend/apps/agents/browser/browser_skills.py b/backend/apps/agents/browser/browser_skills.py index 2e85d5f6..914c92e7 100644 --- a/backend/apps/agents/browser/browser_skills.py +++ b/backend/apps/agents/browser/browser_skills.py @@ -111,6 +111,72 @@ def normalize_task(task: str) -> str: return " ".join(toks) +# --- parameterization (reuse one skill for "the same task, different input") --- +# A quoted value in the task is treated as a SLOT: it's abstracted out of the +# skill key (so `search "shoes"` and `search "hats"` share one skill) and the +# value is filled from the LIVE task at replay (so the value is never stored on +# disk, a redaction win, and the skill generalizes). Quoting is the explicit, +# high-precision signal that this token is a parameter; we never guess. +_QUOTE_RE = re.compile(r'["“”‘’\']([^"“”‘’\']{1,200})["“”‘’\']') +_SLOT_TOKEN = " slotvalue " + + +def template_task(task: str) -> tuple[str, list[str]]: + """Replace each quoted span with a fixed token; return (templated, [values]).""" + values: list[str] = [] + + def _repl(m): + values.append(m.group(1)) + return _SLOT_TOKEN + + return _QUOTE_RE.sub(_repl, task or ""), values + + +def _sig(task: str) -> str: + """Skill key signature: template out quoted values, then normalize, so the + same task with different quoted inputs maps to the same key.""" + templated, _ = template_task(task) + return normalize_task(templated) + + +def _parameterize(steps: list[dict], task: str) -> list[dict]: + """Convert any BrowserType whose text is a quoted task value into a slot + step (value_slot index), so the value is sourced live at replay, not stored.""" + _, values = template_task(task) + if not values: + return steps + vlower = [v.strip().lower() for v in values] + out = [] + for s in steps: + if s["tool"] == "BrowserType": + t = (s["params"].get("text") or "").strip().lower() + if t and t in vlower: + out.append({"tool": "BrowserType", "params": {"selector": s["params"].get("selector"), "value_slot": vlower.index(t)}}) + continue + out.append(s) + return out + + +def rehydrate(skill: dict | None, task: str) -> list[dict] | None: + """Fill a skill's value_slot steps from the current task's quoted values. + Returns runnable steps, or None if any slot can't be filled (caller then + falls back to the full LLM agent, never a wrong value).""" + if not skill: + return None + _, values = template_task(task) + out = [] + for s in skill["steps"]: + p = s.get("params", {}) + if s["tool"] == "BrowserType" and "value_slot" in p: + idx = p["value_slot"] + if not isinstance(idx, int) or idx < 0 or idx >= len(values): + return None # slot has no matching live value -> abort replay + out.append({"tool": "BrowserType", "params": {"selector": p.get("selector"), "text": values[idx]}}) + else: + out.append({"tool": s["tool"], "params": dict(p)}) + return out + + def host_of(url: str) -> str: """host:port of a url (so different sites/ports never share a skill).""" try: @@ -312,9 +378,10 @@ def record_skill(host: str, task: str, action_log: list[dict]) -> bool: steps = distill_steps(action_log) if not steps: return False - sig = normalize_task(task) + sig = _sig(task) if not sig: return False + steps = _parameterize(steps, task) # quoted values -> slots (not stored) persistable = steps_are_persistable(steps) skill = { "host": host, "task_sig": sig, "steps": steps, @@ -340,7 +407,7 @@ def find_skill(host: str, task: str) -> dict | None: (no corpus scan). Returns the skill or None. Cheap + flat as the library grows.""" if not host: return None - sig = normalize_task(task) + sig = _sig(task) if not sig: return None k = _key(host, sig) diff --git a/backend/tests/test_browser_skills.py b/backend/tests/test_browser_skills.py index e40521c4..2e6999b8 100644 --- a/backend/tests/test_browser_skills.py +++ b/backend/tests/test_browser_skills.py @@ -191,3 +191,54 @@ def test_format_version_mismatch_is_ignored(_isolated_skills, monkeypatch): sk.clear(wipe_disk=False) monkeypatch.setattr(sk, "_SKILL_FORMAT_VERSION", 999) # pretend the format moved on assert sk.find_skill("v.com", "do a thing now") is None + + +# --- parameterization: "same task, different input" ----------------------- +def test_quoted_value_becomes_a_slot_and_reuses_across_inputs(_isolated_skills): + # learn from a task with a quoted value + log = [ + {"tool": "BrowserNavigate", "input": {"url": "https://shop.com/search"}, "ok": True}, + {"tool": "BrowserType", "input": {"selector": "#q", "text": "running shoes"}, "ok": True}, + {"tool": "BrowserClickIndex", "input": {}, "ok": True, "clicked_role": "button", "clicked_name": "Search"}, + ] + assert sk.record_skill("shop.com", 'search for "running shoes"', log) is True + # a DIFFERENT quoted input matches the SAME skill (templated key) + found = sk.find_skill("shop.com", 'search for "winter boots"') + assert found is not None + concrete = sk.rehydrate(found, 'search for "winter boots"') + type_step = next(s for s in concrete if s["tool"] == "BrowserType") + assert type_step["params"]["text"] == "winter boots" # filled from the NEW task + + +def test_parameterized_value_is_not_persisted(_isolated_skills): + log = [ + {"tool": "BrowserType", "input": {"selector": "#q", "text": "running shoes"}, "ok": True}, + {"tool": "BrowserClickIndex", "input": {}, "ok": True, "clicked_role": "button", "clicked_name": "Search"}, + ] + sk.record_skill("shop.com", 'search for "running shoes"', log) + path = sk._skill_path("shop.com", sk._sig('search for "running shoes"')) + blob = open(path).read() + assert "running shoes" not in blob # the quoted value never hits disk + assert '"value_slot": 0' in blob or '"value_slot":0' in blob + + +def test_rehydrate_aborts_when_slot_cannot_be_filled(_isolated_skills): + log = [ + {"tool": "BrowserType", "input": {"selector": "#q", "text": "shoes"}, "ok": True}, + {"tool": "BrowserClickIndex", "input": {}, "ok": True, "clicked_role": "button", "clicked_name": "Go"}, + ] + sk.record_skill("shop.com", 'search for "shoes"', log) + found = sk.find_skill("shop.com", "search for shoes") # no quotes -> no value to fill + # find still matches if signatures align; rehydrate must refuse (no ghost) + if found is not None: + assert sk.rehydrate(found, "search for shoes") is None + + +def test_unquoted_text_stays_literal_backward_compatible(_isolated_skills): + # no quotes -> behaves exactly as before (literal text, exact-ish key) + assert sk.record_skill("localhost:8901", "type hello and click Send", _log()) is True + found = sk.find_skill("localhost:8901", "Please type hello and click Send") + assert found is not None + concrete = sk.rehydrate(found, "Please type hello and click Send") + type_step = next(s for s in concrete if s["tool"] == "BrowserType") + assert type_step["params"]["text"] == "hello world" # literal, unchanged