From 71eecf5cdc2f856a3a909bac04463631a5753ed6 Mon Sep 17 00:00:00 2001 From: ciregenz Date: Thu, 27 Aug 2026 12:02:04 -0700 Subject: [PATCH] [eric] agents: a shaped tool result names the file we already wrote, and where the cut starts --- .../manager/streaming/tool_output_shaper.py | 31 ++++++--- backend/tests/test_overcorrection_audit.py | 7 +- backend/tests/test_shaper_recovery_path.py | 68 +++++++++++++++++++ backend/tests/test_tool_output_shaper.py | 18 +++-- 4 files changed, 104 insertions(+), 20 deletions(-) create mode 100644 backend/tests/test_shaper_recovery_path.py diff --git a/backend/apps/agents/manager/streaming/tool_output_shaper.py b/backend/apps/agents/manager/streaming/tool_output_shaper.py index 7ffa7e7b..40d8124a 100644 --- a/backend/apps/agents/manager/streaming/tool_output_shaper.py +++ b/backend/apps/agents/manager/streaming/tool_output_shaper.py @@ -59,13 +59,22 @@ def shape_text(body: str, recovery: str) -> str: carried = [ln.strip()[:CARRIED_LINE_CHARS] for ln in middle.splitlines() if P_NOTABLE.search(ln)][:MAX_CARRIED_LINES] # Reads as ordinary output truncation, the way head/tail/grep already do, and names NOTHING - # about the harness. Measured 2026-08-25, same task 8 times per arm: the previous wording, - # "[... N chars elided by OpenSwarm. Full output: ...]", blocked 8/8 against 3/8 for no - # shaping at all (Fisher p=0.026). CLAUDE.md already said why: on a lane whose terms restrict - # third-party automated use, naming the harness is a signed confession, and this fires on ~6% of - # every tool result rather than only on nudges. Recovery is the same one hermes relies on: the - # tool call is still in the transcript, so the agent can simply run it again. - note = f"[... {len(middle)} characters omitted ...]" + # about the harness: on a lane whose terms restrict third-party automated use, naming it is a + # signed confession, and this fires on ~6% of every tool result rather than only on nudges. + # + # The PATH is back, and the reason it left is worth recording. It was removed on "the previous + # wording blocked 8/8 against 3/8, Fisher p=0.026" -- a measurement this repo has since + # RETRACTED as a vacuous control (every treatment run was later in the session than every + # control; re-running the control late gave 6/7, interleaved arms are level). What that + # measurement actually tested was the string "elided by OpenSwarm", which is gone either way. + # Dropping the path cost real capability: we write the blob, we REFUSE to shape at all when it + # cannot be written, and then we said nothing about it, so the model's only recovery was to + # re-run the command. hermes hands the model an exact re-read call; this is the same idea, + # phrased as output rather than as instructions from a harness. + p_from_line = body.count("\n", 0, HEAD_CHARS) + 1 + note = (f"[... {len(middle)} characters omitted; full output: {recovery} " + f"(the omitted part starts at line {p_from_line}) ...]") if recovery else ( + f"[... {len(middle)} characters omitted ...]") if carried: note += "\n" + "\n".join(carried) return f"{head}\n{note}\n{tail}" @@ -149,12 +158,12 @@ def shape_for_model(session: object, session_id: str, response: object, msg_id: # guard is present, reachable, and doing nothing. Caught live on `Read`, whose payload nests # the text under `file.content` and so matched none of the flat fields. Say the shape out # loud rather than returning a silent None. - p_size = len(p_payload_text(response)) or _rough_size(response) + p_size = len(p_payload_text(response)) or p_rough_size(response) if p_size > SHAPE_OVER_BYTES: bump_shaping_stat(session, "unrecognised", 1) logger.warning( f"tool-output shaping skipped a {p_size:,}-byte {tool_name} result: unrecognised " - f"payload shape {_shape_of(response)}. It is being sent to the model in full.") + f"payload shape {p_shape_of(response)}. It is being sent to the model in full.") return None p_body = p_payload_text(response) @@ -229,7 +238,7 @@ def shaping_report(session: object) -> Optional[str]: @typechecked -def _rough_size(response: object) -> int: +def p_rough_size(response: object) -> int: try: import json as p_json return len(p_json.dumps(response, default=str)) @@ -238,7 +247,7 @@ def _rough_size(response: object) -> int: @typechecked -def _shape_of(response: object) -> str: +def p_shape_of(response: object) -> str: """A description of an unrecognised payload, enough to add a field without a repro.""" if isinstance(response, dict): return "dict(" + ",".join(sorted(response)[:8]) + ")" diff --git a/backend/tests/test_overcorrection_audit.py b/backend/tests/test_overcorrection_audit.py index 31d385c6..dd02ba9d 100644 --- a/backend/tests/test_overcorrection_audit.py +++ b/backend/tests/test_overcorrection_audit.py @@ -67,8 +67,11 @@ def test_shaping_never_removes_without_a_way_back(): from backend.apps.agents.manager.streaming.tool_output_shaper import shape_text out = shape_text("q" * 9_000, "/blobs/x-model.txt") assert "characters omitted" in out, "a cut is always visible as a cut" - assert "OpenSwarm" not in out and "/blobs/" not in out, \ - "but never by naming the harness or an internal path (p=0.026 block regression)" + assert "/blobs/x-model.txt" in out, \ + "and the docstring above is the reason: recoverability MEANS naming where the full text is" + # The path ban this used to carry cited a retracted p=0.026 control. Harness PROSE stays banned. + note = out[out.index("[..."):out.index("]", out.index("[..."))] + assert "OpenSwarm" not in note.replace("/blobs/x-model.txt", "") def test_shaping_leaves_a_normal_result_completely_alone(): diff --git a/backend/tests/test_shaper_recovery_path.py b/backend/tests/test_shaper_recovery_path.py new file mode 100644 index 00000000..b6d8ddcb --- /dev/null +++ b/backend/tests/test_shaper_recovery_path.py @@ -0,0 +1,68 @@ +"""A shaped tool result tells the model where the rest is. We were paying for the file and hiding it. + +`shape_tool_response` writes the full body to a blob and REFUSES to shape at all when it cannot +("no recovery path means the cut would be unrecoverable, which is the one thing this must never +do"). The path was then threaded through four call sites into `shape_text(body, recovery)` and +dropped on the floor: the model saw only `[... 27431 characters omitted ...]`. + +It was removed on a measurement this repo later retracted as a vacuous control. hermes's equivalent +hands the model an exact re-read call; this restores ours, phrased as output rather than as a note +from a harness. +""" + +from backend.apps.agents.manager.streaming.tool_output_shaper import HEAD_CHARS, shape_text + +BLOB = "/Users/eric/Library/Application Support/OpenSwarm/data/sessions/abc/blobs/m1-model.txt" + + +def p_body() -> str: + return "alpha line\n" * 500 + "NEEDLE\n" + "omega line\n" * 200 + + +def p_note(out: str) -> str: + return next(ln for ln in out.splitlines() if ln.startswith("[...")) + + +def test_the_note_names_the_file_the_shaper_already_wrote(): + assert BLOB in p_note(shape_text(p_body(), BLOB)) + + +def test_it_says_where_in_that_file_the_omitted_part_begins(): + # hermes's trick: without a starting offset the model's first read lands in text it already has. + body = p_body() + expected = body.count("\n", 0, HEAD_CHARS) + 1 + assert f"starts at line {expected}" in p_note(shape_text(body, BLOB)) + + +def test_the_notes_PROSE_names_nothing_about_the_harness(): + """The reason the path left in the first place, stated precisely. + + On a packaged install the blob really does live under `.../Application Support/OpenSwarm/data`, + so the app name is unavoidably inside the path. That is a filesystem fact any `ls` would print, + and it is a different thing from prose announcing the harness ("elided by OpenSwarm"), which is + what the original wording did and what the rule is actually about. So: the PATH may say + anything, the WORDS around it may not.""" + out = shape_text(p_body(), BLOB) + i = out.index("[...") + note = out[i:out.index("]", i)] + prose = note.replace(BLOB, "").lower() + for word in ("openswarm", "harness", "elided by", "automated", "assistant", "we ", "our "): + assert word not in prose, f"the note's prose must not say {word!r}: {prose}" + assert "" in prose, "and the path itself must still be there" + + +def test_no_blob_means_no_promise(): + # A path we could not write must never be advertised; a broken recovery is worse than none. + note = p_note(shape_text(p_body(), "")) + assert "full output" not in note and "line" not in note + + +def test_a_body_too_small_to_cut_is_returned_untouched(): + for small in ("", "just a line", "x" * (HEAD_CHARS - 1)): + assert shape_text(small, BLOB) == small + + +def test_the_answer_still_survives_the_cut(): + # The whole point of carrying notable lines; a recovery path is not a licence to delete answers. + body = "noise\n" * 400 + "FATAL: the database is on fire\n" + "noise\n" * 400 + assert "FATAL: the database is on fire" in shape_text(body, BLOB) diff --git a/backend/tests/test_tool_output_shaper.py b/backend/tests/test_tool_output_shaper.py index ac2d76b0..77f4b425 100644 --- a/backend/tests/test_tool_output_shaper.py +++ b/backend/tests/test_tool_output_shaper.py @@ -22,16 +22,20 @@ def p_big(marker: str = "") -> str: return ("a" * 3_000) + f"\n{marker}\n" + ("b" * 5_000) -def test_an_elision_is_marked_and_never_names_the_harness(): - """Recoverability is still the guarantee, but it is carried by the TOOL CALL surviving in the - transcript (the agent can re-run it), not by pointing at a blob path. +def test_an_elision_is_marked_and_never_names_the_harness_IN_PROSE(): + """CORRECTED 2026-08-27. This used to also ban the blob PATH, citing "8/8 policy blocks against + 3/8, p=0.026". That measurement is retracted: every treatment run was later in the session than + every control, and re-running the control late gave 6/7 while interleaved arms were level. What + it really tested was the phrase "elided by OpenSwarm", which is still banned here. - Naming the harness in the body cost 8/8 policy blocks against 3/8 for no shaping (p=0.026, - 2026-08-25). CLAUDE.md: never announce automation on a lane whose terms restrict it.""" + Re-run the shaper's own guarantee: it writes the full body to a blob and REFUSES to shape when + it cannot, so withholding the path was paying for a recovery nobody could use.""" out = shape_text(p_big(), "/data/blobs/x.txt") assert "characters omitted" in out, "a cut must be visible as a cut" - assert "OpenSwarm" not in out, "the harness may not name itself inside tool output" - assert "/data/blobs/" not in out, "nor leak an internal path into the conversation" + assert "/data/blobs/x.txt" in out, "the file it already wrote must be reachable" + note = out[out.index("[..."):out.index("]", out.index("[..."))] + assert "OpenSwarm" not in note.replace("/data/blobs/x.txt", ""), \ + "the harness may not name ITSELF; a path is data, prose is a confession" def test_the_answer_line_survives_the_middle():