From 1508e03c5ff1c4c16e82ec4e1862c9fffe738d9b Mon Sep 17 00:00:00 2001 From: ciregenz Date: Sun, 2 Aug 2026 15:30:17 -0700 Subject: [PATCH] [eric] browser: half an emoji in page text no longer kills the whole prestage --- backend/apps/agents/browser/browser_agent.py | 6 +----- .../apps/agents/browser/browser_prestage.py | 8 ++++++-- .../agents/browser/strip_lone_surrogates.py | 14 +++++++++++++ backend/tests/test_browser_prestage_opener.py | 20 +++++++++++++++++++ 4 files changed, 41 insertions(+), 7 deletions(-) create mode 100644 backend/apps/agents/browser/strip_lone_surrogates.py diff --git a/backend/apps/agents/browser/browser_agent.py b/backend/apps/agents/browser/browser_agent.py index 6874a0e7..6e5ad5eb 100644 --- a/backend/apps/agents/browser/browser_agent.py +++ b/backend/apps/agents/browser/browser_agent.py @@ -53,6 +53,7 @@ from backend.apps.agents.browser.browser_loop import ( stagnation_exhausted, ) from backend.apps.agents.browser.browser_validator import adjudicate_stuck +from backend.apps.agents.browser.strip_lone_surrogates import strip_lone_surrogates # Single actions the model could have folded into one BrowserBatch turn; reads, waits, and the batch tools themselves don't count toward the streak. P_BATCHABLE_ACTION_TOOLS = { @@ -531,11 +532,6 @@ async def run_api_write(tool_input: dict, current_url: str, browser_id: str = "" return p_api_write_result(res) -def strip_lone_surrogates(s: str) -> str: - # The JS/webview hands us page text as UTF-16, so an emoji can arrive as half of its surrogate pair; Python carries the orphan but .encode('utf-8') later (the SDK serializing the request to the LLM) detonates with "surrogates not allowed" and kills the turn. Swap any orphan for the replacement char. - return re.sub(r"[\ud800-\udfff]", "�", s) if s else s - - def format_tool_result(result: dict, tool_name: str) -> list[dict]: """Convert a browser command result dict into Anthropic API content blocks.""" if "error" in result: diff --git a/backend/apps/agents/browser/browser_prestage.py b/backend/apps/agents/browser/browser_prestage.py index 128ed641..08bbc21a 100644 --- a/backend/apps/agents/browser/browser_prestage.py +++ b/backend/apps/agents/browser/browser_prestage.py @@ -19,6 +19,7 @@ import time from typing import Awaitable, Callable from backend.apps.agents.browser import browser_send_parse, compose_discovery, compose_entry +from backend.apps.agents.browser.strip_lone_surrogates import strip_lone_surrogates logger = logging.getLogger(__name__) @@ -224,8 +225,11 @@ async def run_prestage( li = li if isinstance(li, dict) else {} gt = gt if isinstance(gt, dict) else {} url = str(li.get("url") or gt.get("url") or "") - li_text = str(li.get("text") or "") if "error" not in li else "" - gt_text = str(gt.get("text") or "") if "error" not in gt else "" + # Scrub here, the one door page text comes through: an unpaired surrogate (half an + # emoji, and twitch chat is made of them) survives in Python but detonates the moment + # the aux request is encoded, and the whole stage was dying in a blanket except. + li_text = strip_lone_surrogates(str(li.get("text") or "")) if "error" not in li else "" + gt_text = strip_lone_surrogates(str(gt.get("text") or "")) if "error" not in gt else "" return li_text, gt_text, url current_url = start_url diff --git a/backend/apps/agents/browser/strip_lone_surrogates.py b/backend/apps/agents/browser/strip_lone_surrogates.py new file mode 100644 index 00000000..062994c3 --- /dev/null +++ b/backend/apps/agents/browser/strip_lone_surrogates.py @@ -0,0 +1,14 @@ +import re + +from typeguard import typechecked + + +@typechecked +def strip_lone_surrogates(s: str) -> str: + # The JS/webview hands us page text as UTF-16, so an emoji can arrive as half of its surrogate + # pair; Python carries the orphan but .encode('utf-8') later (anything serializing the text to + # an LLM) detonates with "surrogates not allowed" and kills the turn. Swap any orphan for the + # replacement char. It lives in its own file because the agent loop learned this the hard way + # and prestage then learned it again, live on twitch, where half an emoji killed the whole + # composer-reach stage. + return re.sub(r"[\ud800-\udfff]", "�", s) if s else s diff --git a/backend/tests/test_browser_prestage_opener.py b/backend/tests/test_browser_prestage_opener.py index 7e4e89cb..2e1c8312 100644 --- a/backend/tests/test_browser_prestage_opener.py +++ b/backend/tests/test_browser_prestage_opener.py @@ -83,3 +83,23 @@ def test_send_script_enables_opener_mode(monkeypatch): assert pre.opener_mode() is True monkeypatch.setenv("OSW_SEND_SCRIPT", "0") assert pre.opener_mode() is False + + +def test_half_an_emoji_cannot_kill_the_whole_prestage(): + """Live on twitch: a lone surrogate in the page text raised "'utf-8' codec can't encode + character '\\ud83e'" out of the aux request encode, prestage's blanket except swallowed it as + "[browser-prestage] skipped (...)", and the site lost its entire composer-reach stage. The + agent loop already knew this (strip_lone_surrogates, written for the same detonation); prestage + just never applied it. The scrub now sits on perceive(), the one door page text comes through. + """ + from backend.apps.agents.browser.strip_lone_surrogates import strip_lone_surrogates + + raw = "chat \ud83e is half an emoji" + with pytest.raises(UnicodeEncodeError): + raw.encode("utf-8") + cleaned = strip_lone_surrogates(raw) + assert cleaned.encode("utf-8"), "must survive the encode that killed the stage" + assert "\ud83e" not in cleaned + # A well-formed emoji is left alone; scrubbing real content would be its own bug. + assert strip_lone_surrogates("done \U0001f9e0 ok") == "done \U0001f9e0 ok" + assert strip_lone_surrogates("") == ""