[eric] agents: hidden harness messages self-identify, so a misfired nudge can never become 'the user told me to stop' (field report)

This commit is contained in:
ciregenz
2026-08-16 18:49:50 -07:00
parent 2d0fc643f1
commit f92b920e4e
3 changed files with 68 additions and 0 deletions
+6
View File
@@ -114,6 +114,12 @@ class Messaging(AgentManagerProtocol):
"session": session.model_dump(mode="json"),
})
# Hidden messages are harness plumbing (nudges, lost-step retries, auth heals) but ride the
# USER role, so agents stopped mid-task by a misfired nudge truthfully reported "the user
# told me to stop", and others read them as prompt injection (field reports, 2026-08-16).
# One attribution prefix at the one send chokepoint keeps every explanation honest.
if hidden and prompt and not prompt.startswith("[Automated"):
prompt = "[Automated message from OpenSwarm itself, not written by your user] " + prompt
skill_meta = [{"id": s["id"], "name": s["name"]} for s in (attached_skills or [])] or None
image_meta = [{"data": img["data"], "media_type": img.get("media_type", "image/png")} for img in (images or [])] or None
user_msg = Message(
+1
View File
@@ -97,6 +97,7 @@ P_RELEASES: List[ReleaseNote] = [
"An app or browser card whose page process dies now reloads itself instead of sitting as a solid black rectangle. The crash fired no load event at all, so nothing ever repainted it.",
"The mouse-wheel Zoom/Scroll setting works on real mice now. Accelerated wheels (Magic Mouse, Logitech smooth scrolling) report fractional scroll amounts that were being mistaken for a trackpad, so the wheel always panned no matter what the setting said.",
"A browser helper that talks itself into refusing (\"I\u2019m a text-based AI\") no longer poisons its browser for every later task. Refused and fabricated runs are forgotten instead of remembered, and agents can ask for a completely fresh browser when one misbehaves.",
"Agents no longer claim \"the user told me to stop\" when it was OpenSwarm\u2019s own housekeeping talking. Internal wrap-up and retry messages now identify themselves, so an agent\u2019s explanation of why it stopped reflects what actually happened.",
"Heavy sessions no longer vanish without a trace. When memory climbs past the safe line the app now sheds weight itself: preview thumbnails pause and refetchable caches drop, instead of growing until the operating system kills it mid-task.",
],
),
@@ -0,0 +1,61 @@
"""Agents stopped mid-task by a misfired nudge reported "the user told me to stop", and others
read hidden harness messages as prompt injection (field reports, 2026-08-16). Mechanism: every
hidden prompt (silent-quit nudges incl. the FINAL one that literally opens "Stop. Do not call any
more tools.", lost-step retries, auth heals, context-break continuations) rides the USER role, so
the model's misattribution is honest from its chair. Seal: one attribution prefix at the ONE send
chokepoint, so no hidden message can ever read as the user's words.
"""
import inspect
import pytest
from unittest.mock import AsyncMock, patch
from backend.apps.agents.core.models import AgentSession
@pytest.mark.asyncio
async def test_hidden_prompt_gets_the_attribution_prefix():
from backend.apps.agents.agent_manager import agent_manager
from backend.apps.agents.manager import Messaging
session = AgentSession(id="hm-1", name="t", model="sonnet", dashboard_id="d")
agent_manager.sessions["hm-1"] = session
try:
with patch.object(Messaging, "ws_manager") as p_ws, \
patch.object(agent_manager, "run_agent_loop", new=AsyncMock(), create=True):
p_ws.send_to_session = AsyncMock()
try:
await agent_manager.send_message("hm-1", "Stop. Do not call any more tools.", hidden=True)
except Exception:
pass # downstream turn machinery may bail in a unit context; the append happened first
hidden = [m for m in session.messages if m.role == "user" and m.hidden]
assert hidden, "hidden message never appended"
assert hidden[-1].content.startswith("[Automated message from OpenSwarm itself"), \
"a nudge the model attributes to the user is the fabricated-stop bug"
finally:
agent_manager.sessions.pop("hm-1", None)
@pytest.mark.asyncio
async def test_visible_user_prompt_is_untouched():
from backend.apps.agents.agent_manager import agent_manager
from backend.apps.agents.manager import Messaging
session = AgentSession(id="hm-2", name="t", model="sonnet", dashboard_id="d")
agent_manager.sessions["hm-2"] = session
try:
with patch.object(Messaging, "ws_manager") as p_ws, \
patch.object(agent_manager, "run_agent_loop", new=AsyncMock(), create=True):
p_ws.send_to_session = AsyncMock()
try:
await agent_manager.send_message("hm-2", "please stop and summarize", hidden=False)
except Exception:
pass
visible = [m for m in session.messages if m.role == "user" and not m.hidden]
assert visible and visible[-1].content == "please stop and summarize", "real user words must never be rewritten"
finally:
agent_manager.sessions.pop("hm-2", None)
def test_no_double_prefix():
from backend.apps.agents.manager import Messaging
src = inspect.getsource(Messaging)
assert 'not prompt.startswith("[Automated")' in src, "re-sent continuations must not stack prefixes"