mirror of
https://github.com/openswarm-ai/openswarm.git
synced 2026-08-17 18:25:42 +02:00
[eric] agents: hidden harness messages self-identify, so a misfired nudge can never become 'the user told me to stop' (field report)
This commit is contained in:
@@ -114,6 +114,12 @@ class Messaging(AgentManagerProtocol):
|
||||
"session": session.model_dump(mode="json"),
|
||||
})
|
||||
|
||||
# Hidden messages are harness plumbing (nudges, lost-step retries, auth heals) but ride the
|
||||
# USER role, so agents stopped mid-task by a misfired nudge truthfully reported "the user
|
||||
# told me to stop", and others read them as prompt injection (field reports, 2026-08-16).
|
||||
# One attribution prefix at the one send chokepoint keeps every explanation honest.
|
||||
if hidden and prompt and not prompt.startswith("[Automated"):
|
||||
prompt = "[Automated message from OpenSwarm itself, not written by your user] " + prompt
|
||||
skill_meta = [{"id": s["id"], "name": s["name"]} for s in (attached_skills or [])] or None
|
||||
image_meta = [{"data": img["data"], "media_type": img.get("media_type", "image/png")} for img in (images or [])] or None
|
||||
user_msg = Message(
|
||||
|
||||
@@ -97,6 +97,7 @@ P_RELEASES: List[ReleaseNote] = [
|
||||
"An app or browser card whose page process dies now reloads itself instead of sitting as a solid black rectangle. The crash fired no load event at all, so nothing ever repainted it.",
|
||||
"The mouse-wheel Zoom/Scroll setting works on real mice now. Accelerated wheels (Magic Mouse, Logitech smooth scrolling) report fractional scroll amounts that were being mistaken for a trackpad, so the wheel always panned no matter what the setting said.",
|
||||
"A browser helper that talks itself into refusing (\"I\u2019m a text-based AI\") no longer poisons its browser for every later task. Refused and fabricated runs are forgotten instead of remembered, and agents can ask for a completely fresh browser when one misbehaves.",
|
||||
"Agents no longer claim \"the user told me to stop\" when it was OpenSwarm\u2019s own housekeeping talking. Internal wrap-up and retry messages now identify themselves, so an agent\u2019s explanation of why it stopped reflects what actually happened.",
|
||||
"Heavy sessions no longer vanish without a trace. When memory climbs past the safe line the app now sheds weight itself: preview thumbnails pause and refetchable caches drop, instead of growing until the operating system kills it mid-task.",
|
||||
],
|
||||
),
|
||||
|
||||
@@ -0,0 +1,61 @@
|
||||
"""Agents stopped mid-task by a misfired nudge reported "the user told me to stop", and others
|
||||
read hidden harness messages as prompt injection (field reports, 2026-08-16). Mechanism: every
|
||||
hidden prompt (silent-quit nudges incl. the FINAL one that literally opens "Stop. Do not call any
|
||||
more tools.", lost-step retries, auth heals, context-break continuations) rides the USER role, so
|
||||
the model's misattribution is honest from its chair. Seal: one attribution prefix at the ONE send
|
||||
chokepoint, so no hidden message can ever read as the user's words.
|
||||
"""
|
||||
import inspect
|
||||
|
||||
import pytest
|
||||
from unittest.mock import AsyncMock, patch
|
||||
|
||||
from backend.apps.agents.core.models import AgentSession
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_hidden_prompt_gets_the_attribution_prefix():
|
||||
from backend.apps.agents.agent_manager import agent_manager
|
||||
from backend.apps.agents.manager import Messaging
|
||||
session = AgentSession(id="hm-1", name="t", model="sonnet", dashboard_id="d")
|
||||
agent_manager.sessions["hm-1"] = session
|
||||
try:
|
||||
with patch.object(Messaging, "ws_manager") as p_ws, \
|
||||
patch.object(agent_manager, "run_agent_loop", new=AsyncMock(), create=True):
|
||||
p_ws.send_to_session = AsyncMock()
|
||||
try:
|
||||
await agent_manager.send_message("hm-1", "Stop. Do not call any more tools.", hidden=True)
|
||||
except Exception:
|
||||
pass # downstream turn machinery may bail in a unit context; the append happened first
|
||||
hidden = [m for m in session.messages if m.role == "user" and m.hidden]
|
||||
assert hidden, "hidden message never appended"
|
||||
assert hidden[-1].content.startswith("[Automated message from OpenSwarm itself"), \
|
||||
"a nudge the model attributes to the user is the fabricated-stop bug"
|
||||
finally:
|
||||
agent_manager.sessions.pop("hm-1", None)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_visible_user_prompt_is_untouched():
|
||||
from backend.apps.agents.agent_manager import agent_manager
|
||||
from backend.apps.agents.manager import Messaging
|
||||
session = AgentSession(id="hm-2", name="t", model="sonnet", dashboard_id="d")
|
||||
agent_manager.sessions["hm-2"] = session
|
||||
try:
|
||||
with patch.object(Messaging, "ws_manager") as p_ws, \
|
||||
patch.object(agent_manager, "run_agent_loop", new=AsyncMock(), create=True):
|
||||
p_ws.send_to_session = AsyncMock()
|
||||
try:
|
||||
await agent_manager.send_message("hm-2", "please stop and summarize", hidden=False)
|
||||
except Exception:
|
||||
pass
|
||||
visible = [m for m in session.messages if m.role == "user" and not m.hidden]
|
||||
assert visible and visible[-1].content == "please stop and summarize", "real user words must never be rewritten"
|
||||
finally:
|
||||
agent_manager.sessions.pop("hm-2", None)
|
||||
|
||||
|
||||
def test_no_double_prefix():
|
||||
from backend.apps.agents.manager import Messaging
|
||||
src = inspect.getsource(Messaging)
|
||||
assert 'not prompt.startswith("[Automated")' in src, "re-sent continuations must not stack prefixes"
|
||||
Reference in New Issue
Block a user