diff --git a/backend/apps/agents/browser/browser_agent.py b/backend/apps/agents/browser/browser_agent.py index 97603da1..01990ccc 100644 --- a/backend/apps/agents/browser/browser_agent.py +++ b/backend/apps/agents/browser/browser_agent.py @@ -81,6 +81,7 @@ from backend.apps.agents.browser import browser_metrics from backend.apps.agents.browser import browser_send_script from backend.apps.agents.browser import browser_send_parse from backend.apps.agents.browser import browser_login_handoff +from backend.apps.agents.browser import detect_captcha from backend.apps.agents.browser import browser_session_import from backend.apps.agents.browser import browser_delivery_check from backend.apps.agents.browser import browser_submit_click @@ -1350,6 +1351,7 @@ async def run_browser_agent( card_gone_streak = 0 # consecutive "card is gone" results -> fail fast, don't spin route_hinted_hosts: set[str] = set() # surface the fast network tier once per host p_login_prompted: set[str] = set() # login-once handoff: at most one sign-in pause per domain per run + p_captcha_prompted = False # bot-detection handoff: asked at most once per run, never re-nagged # Stagnation state: busy-but-stuck detection (no URL change + failures across a run of actions), distinct from the exact-repeat loop above. stagnation_streak = 0 @@ -1907,6 +1909,30 @@ async def run_browser_agent( # RequestHumanIntervention stays as the fallback for walls this detector misses. # Soft signed-out (composer withheld behind a "Sign in") only counts once the agent has # actually tried and is still stuck, so a first-turn glance can't raise a false prompt. + # A bot-detection challenge outranks the login wall: a page can show both (tiktok serves + # its audio captcha over a logged-out feed), and signing in is not what unblocks it. + # Never attempted, only handed over. Once per run, because a user who declines should + # not be asked again every turn. + if not p_captcha_prompted: + p_cap_kind = detect_captcha.captcha_kind("\n".join(attached_state_seen)) + if p_cap_kind: + p_captcha_prompted = True + p_cap_host = browser_login_handoff.registrable_domain(last_seen_url) or "" + p_cap_problem, p_cap_instruction = detect_captcha.prompt_copy(p_cap_kind, p_cap_host) + logger.info(f"[browser-agent {session_id}] {p_cap_kind} detected on " + f"{p_cap_host or last_seen_url[:60]}; handing to the user, not solving it") + p_cap_decision = await p_request_browser_approval( + session, "RequestHumanIntervention", + {"problem": p_cap_problem, "instruction": p_cap_instruction}) + if cancel_event.is_set(): + break + if p_cap_decision.get("behavior") == "deny": + done_called = True + done_success = False + done_message = (f"{p_cap_problem} I can't solve these, so I stopped rather " + f"than guess at it.") + break + p_wall_dom = browser_login_handoff.login_wall_domain( last_seen_url, "\n".join(attached_state_seen), allow_soft=(turn >= 2)) if p_wall_dom and p_wall_dom not in p_login_prompted: diff --git a/backend/apps/agents/browser/detect_captcha.py b/backend/apps/agents/browser/detect_captcha.py new file mode 100644 index 00000000..9e27e0c9 --- /dev/null +++ b/backend/apps/agents/browser/detect_captcha.py @@ -0,0 +1,61 @@ +import re +from typing import Optional + +from typeguard import typechecked + +# A bot-detection challenge in the page's own words. We NEVER solve one of these: defeating +# bot-detection is off-limits, and an agent that tries produces exactly the confident nonsense this +# codebase exists to prevent (it cannot hear an audio clip, and "press and hold" against a +# behavioural detector is a coin flip it will report as a success either way). +# +# Named separately from the login wall because the remedy is different. A login wall is fixed by +# signing in once and it stays fixed; a challenge is a one-shot the human has to clear right now, +# in this card, before anything else can proceed. +P_CAPTCHA_RE = re.compile( + r"enter (?:the )?(?:code|characters) you hear|play the audio|" + r"press (?:and|&) hold|click and hold|hold to (?:verify|continue)|" + r"drag (?:the )?(?:slider|puzzle|piece|handle)|slide to (?:verify|complete)|" + r"i'?m not a robot|verify you are (?:a )?human|are you a human|" + r"security check|complete the (?:security )?(?:check|challenge)|" + r"recaptcha|hcaptcha|are you a robot|unusual traffic|" + r"select all (?:images|squares) (?:with|containing)", + re.I, +) +# Vendor iframes and containers, for challenges that render no readable copy at all. +P_CAPTCHA_DOM_RE = re.compile( + r"recaptcha|hcaptcha|turnstile|arkose|funcaptcha|px-captcha|geetest|captcha-?(?:container|frame)", + re.I, +) + + +@typechecked +def captcha_kind(state_text: str) -> Optional[str]: + """Which challenge is on the page, or None. + + The KIND is the whole point of returning a string: "solve the captcha" tells a user nothing, + while "it wants you to play an audio clip and type what you hear" tells them what they are + about to do. Measured on tiktok 2026-08-04, where the challenge was an audio one and the run + burned its whole budget clicking at a page it could never get past. + """ + text = state_text or "" + if re.search(r"enter (?:the )?(?:code|characters) you hear|play the audio", text, re.I): + return "an audio challenge (play the clip and type what you hear)" + if re.search(r"press (?:and|&) hold|click and hold|hold to (?:verify|continue)", text, re.I): + return "a press-and-hold challenge" + if re.search(r"drag (?:the )?(?:slider|puzzle|piece|handle)|slide to (?:verify|complete)", text, re.I): + return "a slider/puzzle challenge" + if re.search(r"select all (?:images|squares) (?:with|containing)", text, re.I): + return "an image-selection challenge" + if P_CAPTCHA_RE.search(text) or P_CAPTCHA_DOM_RE.search(text): + return "a bot-detection challenge" + return None + + +@typechecked +def prompt_copy(kind: str, host: str) -> tuple[str, str]: + """(problem, instruction) for the human handoff. Says plainly that we will not attempt it.""" + return ( + f"{host or 'This site'} is showing {kind}.", + "I can't solve these, and guessing at one would be worse than stopping. Please complete it " + "in the browser card, then tell me to continue.", + ) diff --git a/backend/tests/test_detect_captcha.py b/backend/tests/test_detect_captcha.py new file mode 100644 index 00000000..06793656 --- /dev/null +++ b/backend/tests/test_detect_captcha.py @@ -0,0 +1,56 @@ +"""Bot-detection challenges are handed to the user, never attempted. + +Eric screenshotted the case on 2026-08-04: TikTok showing "Play the audio and enter the code you +hear" over a logged-out feed, while the run underneath burned its budget clicking at a page it could +never get past. These come in several shapes (audio, press-and-hold, slider/puzzle, image grid) and +they share one property: an agent cannot honestly clear any of them. It cannot hear the clip, and a +press-and-hold against a behavioural detector is a coin flip that reports success either way, which +is precisely the false-success class the rest of this suite exists to prevent. + +So the rule is refuse and escalate. What these tests protect is (a) that the common shapes are +detected at all, and (b) that ordinary pages are NOT, because a detector that cries captcha on a +checkout form interrupts the user for nothing. +""" + +from backend.apps.agents.browser import detect_captcha as dc + + +def test_the_tiktok_audio_challenge_from_the_screenshot(): + kind = dc.captcha_kind("Play the audio and enter the code you hear") + assert kind and "audio" in kind, kind + + +def test_the_interaction_shaped_challenges_are_each_named(): + """The kind is load-bearing: "solve the captcha" tells a user nothing, "it wants you to play a + clip and type what you hear" tells them what they are about to do.""" + assert "press-and-hold" in (dc.captcha_kind('[3]