[eric] browser: half an emoji in page text no longer kills the whole prestage

backend/apps/agents/browser/browser_agent.py

@@ -53,6 +53,7 @@ from backend.apps.agents.browser.browser_loop import (
     stagnation_exhausted,
 )
 from backend.apps.agents.browser.browser_validator import adjudicate_stuck
+from backend.apps.agents.browser.strip_lone_surrogates import strip_lone_surrogates
 
 # Single actions the model could have folded into one BrowserBatch turn; reads, waits, and the batch tools themselves don't count toward the streak.
 P_BATCHABLE_ACTION_TOOLS = {
@@ -531,11 +532,6 @@ async def run_api_write(tool_input: dict, current_url: str, browser_id: str = ""
     return p_api_write_result(res)
 
 
-def strip_lone_surrogates(s: str) -> str:
-    # The JS/webview hands us page text as UTF-16, so an emoji can arrive as half of its surrogate pair; Python carries the orphan but .encode('utf-8') later (the SDK serializing the request to the LLM) detonates with "surrogates not allowed" and kills the turn. Swap any orphan for the replacement char.
-    return re.sub(r"[\ud800-\udfff]", "�", s) if s else s
-
-
 def format_tool_result(result: dict, tool_name: str) -> list[dict]:
     """Convert a browser command result dict into Anthropic API content blocks."""
     if "error" in result:

backend/apps/agents/browser/browser_prestage.py

@@ -19,6 +19,7 @@ import time
 from typing import Awaitable, Callable
 
 from backend.apps.agents.browser import browser_send_parse, compose_discovery, compose_entry
+from backend.apps.agents.browser.strip_lone_surrogates import strip_lone_surrogates
 
 logger = logging.getLogger(__name__)
 
@@ -224,8 +225,11 @@ async def run_prestage(
             li = li if isinstance(li, dict) else {}
             gt = gt if isinstance(gt, dict) else {}
             url = str(li.get("url") or gt.get("url") or "")
-            li_text = str(li.get("text") or "") if "error" not in li else ""
-            gt_text = str(gt.get("text") or "") if "error" not in gt else ""
+            # Scrub here, the one door page text comes through: an unpaired surrogate (half an
+            # emoji, and twitch chat is made of them) survives in Python but detonates the moment
+            # the aux request is encoded, and the whole stage was dying in a blanket except.
+            li_text = strip_lone_surrogates(str(li.get("text") or "")) if "error" not in li else ""
+            gt_text = strip_lone_surrogates(str(gt.get("text") or "")) if "error" not in gt else ""
             return li_text, gt_text, url
 
         current_url = start_url

backend/apps/agents/browser/strip_lone_surrogates.py

@@ -0,0 +1,14 @@
+import re
+
+from typeguard import typechecked
+
+
+@typechecked
+def strip_lone_surrogates(s: str) -> str:
+    # The JS/webview hands us page text as UTF-16, so an emoji can arrive as half of its surrogate
+    # pair; Python carries the orphan but .encode('utf-8') later (anything serializing the text to
+    # an LLM) detonates with "surrogates not allowed" and kills the turn. Swap any orphan for the
+    # replacement char. It lives in its own file because the agent loop learned this the hard way
+    # and prestage then learned it again, live on twitch, where half an emoji killed the whole
+    # composer-reach stage.
+    return re.sub(r"[\ud800-\udfff]", "�", s) if s else s

backend/tests/test_browser_prestage_opener.py

@@ -83,3 +83,23 @@ def test_send_script_enables_opener_mode(monkeypatch):
     assert pre.opener_mode() is True
     monkeypatch.setenv("OSW_SEND_SCRIPT", "0")
     assert pre.opener_mode() is False
+
+
+def test_half_an_emoji_cannot_kill_the_whole_prestage():
+    """Live on twitch: a lone surrogate in the page text raised "'utf-8' codec can't encode
+    character '\\ud83e'" out of the aux request encode, prestage's blanket except swallowed it as
+    "[browser-prestage] skipped (...)", and the site lost its entire composer-reach stage. The
+    agent loop already knew this (strip_lone_surrogates, written for the same detonation); prestage
+    just never applied it. The scrub now sits on perceive(), the one door page text comes through.
+    """
+    from backend.apps.agents.browser.strip_lone_surrogates import strip_lone_surrogates
+
+    raw = "chat \ud83e is half an emoji"
+    with pytest.raises(UnicodeEncodeError):
+        raw.encode("utf-8")
+    cleaned = strip_lone_surrogates(raw)
+    assert cleaned.encode("utf-8"), "must survive the encode that killed the stage"
+    assert "\ud83e" not in cleaned
+    # A well-formed emoji is left alone; scrubbing real content would be its own bug.
+    assert strip_lone_surrogates("done \U0001f9e0 ok") == "done \U0001f9e0 ok"
+    assert strip_lone_surrogates("") == ""