[eric] windows: the ghost reaper scans and kills through PowerShell and taskkill instead of raising on SIGCONT, the publish gate and skill files use / keys, the time pin drops the glibc-only strftime flags, the shutdown fuse arms on Windows; eleven tests state their platform (ENG-490)

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
ciregenz
2026-09-07 10:23:58 -07:00
co-authored by Claude Fable 5.1
parent 46ab862ef4
commit 87ebcc47d7
17 changed files with 145 additions and 36 deletions
@@ -69,8 +69,9 @@ def compose_turn_system_prompt(
tz_abbr = now_local.strftime("%Z") or tz_name
time_ctx = (
"<current_time>\n"
f"Today is {now_local.strftime('%A, %B %-d, %Y')}.\n"
f"Local time: {now_local.strftime('%-I:%M %p')} {tz_abbr} ({tz_name}).\n"
# No %-d / %-I: those are glibc and BSD flags, Windows' strftime raises on them, and the whole pin vanished there.
f"Today is {now_local.strftime('%A, %B')} {now_local.day}, {now_local.year}.\n"
f"Local time: {int(now_local.strftime('%I'))}:{now_local.strftime('%M %p')} {tz_abbr} ({tz_name}).\n"
"Use this as ground truth for any date/time/day-of-week question. The timezone also "
"gives the user's coarse region; when they say 'here' or 'near me' without a place, "
"infer the likely city from it (say you inferred it) instead of claiming you can't know.\n"
+1 -1
View File
@@ -99,7 +99,7 @@ def p_api_callers(root: str) -> List[str]:
continue
with open(full, "r", encoding="utf-8", errors="replace") as fh:
if p_reaches_unserved_backend(fh.read()):
hits.append(os.path.relpath(full, root))
hits.append(os.path.relpath(full, root).replace(os.sep, "/"))
except OSError:
continue
return sorted(hits)
+53 -6
View File
@@ -24,6 +24,38 @@ from backend.config.paths import OUTPUTS_WORKSPACE_DIR as WORKSPACE_DIR
logger = logging.getLogger(__name__)
def p_is_windows() -> bool:
return os.name == "nt"
# Windows has no `ps`, `lsof` or `pgrep`: one CIM query gives pid, ppid and the full command line, and
# taskkill /T walks the tree. Read at call time through p_is_windows() so a test can drive either path.
P_WIN_PROCESS_QUERY = (
"Get-CimInstance Win32_Process | Select-Object ProcessId,ParentProcessId,CommandLine | ConvertTo-Csv -NoTypeInformation"
)
@typechecked
def p_windows_process_table() -> List[tuple]:
"""(pid, ppid, command line) for every process, or [] when PowerShell is unavailable."""
try:
out = subprocess.run(
["powershell", "-NoProfile", "-NonInteractive", "-Command", P_WIN_PROCESS_QUERY],
capture_output=True, text=True, timeout=20,
)
except Exception:
return []
import csv
import io
rows: List[tuple] = []
for rec in csv.DictReader(io.StringIO(out.stdout or "")):
try:
rows.append((int(rec.get("ProcessId") or 0), int(rec.get("ParentProcessId") or 0), rec.get("CommandLine") or ""))
except ValueError:
continue
return [r for r in rows if r[0] > 0]
# Grace between TERM and KILL. Long enough for a run.sh EXIT trap to clean up its ports, short enough
# that boot does not visibly stall on it.
REAP_GRACE_SECONDS = float(os.environ.get("OSW_REAP_GRACE_SECONDS", "1.5"))
@@ -45,6 +77,8 @@ def p_live_backend_pids() -> set:
"""PIDs of every running backend. A workspace process descended from one of these is ALIVE and
owned, not a ghost; a first draft of this reaper matched on the workspace path alone and would
have killed 14 working app runtimes on a machine where the owning backend was up."""
if p_is_windows():
return {pid for pid, _ppid, args in p_windows_process_table() if is_backend_argv(args)}
try:
out = subprocess.run(["ps", "-eo", "pid=,args="], capture_output=True, text=True, timeout=8)
except Exception:
@@ -61,6 +95,8 @@ def p_live_backend_pids() -> set:
@typechecked
def p_ppid_map() -> dict:
if p_is_windows():
return {pid: ppid for pid, ppid, _args in p_windows_process_table()}
try:
out = subprocess.run(["ps", "-eo", "pid=,ppid="], capture_output=True, text=True, timeout=8)
except Exception:
@@ -113,10 +149,15 @@ def find_ghost_runtime_pids() -> List[int]:
# `.../openswarm/...` while our resolved path is `.../OpenSwarm/...`: the same folder, but a
# case-sensitive `in` check misses it and the ghost survives (found live on a packaged smoke).
needle = os.path.abspath(WORKSPACE_DIR).casefold()
try:
out = subprocess.run(["ps", "-eo", "pid=,args="], capture_output=True, text=True, timeout=8)
except Exception:
return []
if p_is_windows():
# No cwd map on Windows: a runtime is matched on its command line alone, the same fact ps gives.
p_lines = [f"{pid} {args}" for pid, _ppid, args in p_windows_process_table()]
else:
try:
out = subprocess.run(["ps", "-eo", "pid=,args="], capture_output=True, text=True, timeout=8)
except Exception:
return []
p_lines = (out.stdout or "").splitlines()
mine = os.getpid()
owners = p_live_backend_pids()
parents = p_ppid_map()
@@ -125,9 +166,9 @@ def find_ghost_runtime_pids() -> List[int]:
# Boot relied on running before anything spawned; the 10-minute sweep gets no such alibi.
if not owners or not parents:
return []
by_cwd = p_cwd_map(needle)
by_cwd = {} if p_is_windows() else p_cwd_map(needle)
candidates = dict.fromkeys(by_cwd)
for line in (out.stdout or "").splitlines():
for line in p_lines:
line = line.strip()
if needle not in line.casefold():
continue
@@ -172,6 +213,12 @@ def reap_ghost_runtimes() -> int:
len(pids), pids[:12],
)
killed = 0
if p_is_windows():
# No SIGSTOP freeze on Windows, so nothing to thaw; taskkill /T /F takes the whole tree at once.
for pid in pids:
kill_descendant_tree(pid, "TERM")
killed += 1
return killed
for pid in pids:
try:
# THAW FIRST. Idle app runtimes are frozen with SIGSTOP, and a stopped process never
+7 -2
View File
@@ -40,6 +40,13 @@ def p_descendant_pids() -> List[int]:
@typechecked
def p_burn() -> None:
if os.name == "nt":
# taskkill /T takes the whole tree, ourselves included; _exit is the belt in case it refuses.
try:
subprocess.run(["taskkill", "/T", "/F", "/PID", str(os.getpid())], capture_output=True, timeout=10)
except Exception:
pass
os._exit(0)
for pid in p_descendant_pids():
try:
os.kill(pid, 9)
@@ -55,8 +62,6 @@ p_armed: Optional[threading.Timer] = None
def arm_shutdown_fuse() -> None:
"""Called at lifespan-shutdown START (already past TERM), so no signal handling: just the timer. Touching signal.signal here would clobber uvicorn's asyncio-installed handlers."""
global p_armed
if os.name == "nt":
return
disarm_shutdown_fuse()
p_armed = threading.Timer(FUSE_S, p_burn)
p_armed.daemon = True
+1 -1
View File
@@ -493,7 +493,7 @@ async def list_skill_files(skill_id: str):
dirs[:] = [d for d in dirs if not d.startswith(".")]
for n in sorted(names):
path = os.path.join(root, n)
rel = os.path.relpath(path, base_abs)
rel = os.path.relpath(path, base_abs).replace(os.sep, "/")
if n.startswith(".") or os.path.getsize(path) > 512_000:
continue
try:
@@ -34,7 +34,7 @@ def p_make_app(ws_root, *, workspace: bool) -> str:
if wsid:
folder = ws_root / wsid
folder.mkdir()
(folder / "app.py").write_text("print('v1')\n")
(folder / "app.py").write_bytes(b"print('v1')\n")
o = Output(
name="Demo", description="", icon="view_quilt",
input_schema={"type": "object", "properties": {}, "required": []},
@@ -50,8 +50,8 @@ def test_workspace_app_export_omits_stale_inline_files(p_ws_root):
o = workspace_io.load_output(oid)
folder = p_ws_root / o.workspace_id
# agent edit -> v2 on disk: modify app.py + add new.py (output.files stays v1)
(folder / "app.py").write_text("print('v2 EDITED')\n")
(folder / "new.py").write_text("print('v2 NEW')\n")
(folder / "app.py").write_bytes(b"print('v2 EDITED')\n")
(folder / "new.py").write_bytes(b"print('v2 NEW')\n")
exp = AppExportable.load(oid)
payload = exp.serialize(P_Ctx())
@@ -64,8 +64,9 @@ def test_truncated_ciphertext_degrades(on_windows):
assert bc.decrypt_cookie_value(b"v10" + NONCE, KEY) is None
def test_the_mac_branch_does_not_try_gcm():
def test_the_mac_branch_does_not_try_gcm(monkeypatch):
"""Same bytes, no Windows flag: the CBC branch must not accidentally accept a GCM blob."""
monkeypatch.setattr(bc, "IS_WIN", False)
assert bc.decrypt_cookie_value(p_win_blob(b"sessionid=abc123"), KEY) is None
+6 -3
View File
@@ -47,9 +47,12 @@ def test_first_seen_preserved_last_login_advances():
assert second["last_login"] >= first["last_login"]
def test_record_login_is_fail_open(monkeypatch):
# an unwritable path must not raise; the run just treats it as a fresh sign-in next time
monkeypatch.setattr(h, "P_STORE_PATH", "/nonexistent-dir-xyz/authenticated_domains.json")
def test_record_login_is_fail_open(monkeypatch, tmp_path):
# an unwritable path must not raise; the run just treats it as a fresh sign-in next time.
# A path beneath a regular FILE is unwritable on every OS; /nonexistent-dir-xyz was creatable on a Windows runner.
blocker = tmp_path / "blocker"
blocker.write_text("x")
monkeypatch.setattr(h, "P_STORE_PATH", str(blocker / "authenticated_domains.json"))
h.record_login("x.com") # no exception
assert h.is_authenticated("x.com") is False
+4 -3
View File
@@ -119,6 +119,7 @@ def test_task_secrets_are_scrubbed_from_tasks_jsonl(tmp_path, monkeypatch):
line = open(p_os.path.join(str(tmp_path), "tasks.jsonl")).read()
assert "hunter2" not in line and "sk-abc" not in line
assert "password [redacted]" in line
# owner-only file perms
mode = p_os.stat(p_os.path.join(str(tmp_path), "tasks.jsonl")).st_mode & 0o777
assert mode == 0o600
# owner-only file perms (POSIX mode bits; on Windows the data root lives under the per-user profile, whose ACL is the boundary)
if p_os.name != "nt":
mode = p_os.stat(p_os.path.join(str(tmp_path), "tasks.jsonl")).st_mode & 0o777
assert mode == 0o600
@@ -42,7 +42,10 @@ def test_warm_cache_is_complete_requires_the_sentinel(tmp_path):
assert vt.warm_cache_is_complete(str(nm)) is False
bindir = nm / ".bin"
bindir.mkdir()
(bindir / "vite").symlink_to("../vite/bin/vite.js")
if os.name == "nt":
(bindir / "vite").write_text("shim")
else:
(bindir / "vite").symlink_to("../vite/bin/vite.js")
assert vt.warm_cache_is_complete(str(nm)) is False
(tmp_path / ".install-complete").write_text("digest")
assert vt.warm_cache_is_complete(str(nm)) is True
+2
View File
@@ -122,6 +122,8 @@ async def test_restoring_a_refresh_token_round_trips(p_router):
@pytest.mark.asyncio
async def test_written_db_is_owner_only(p_router):
if os.name == "nt":
pytest.skip("POSIX mode bits do not exist on Windows; the router's data dir sits under the per-user profile, whose ACL is the boundary")
await store.apply_to_connection("conn-1", changes={}, drop=["refreshToken"])
mode = stat.S_IMODE(os.stat(store.db_path()).st_mode)
assert mode & 0o077 == 0
+4 -1
View File
@@ -99,7 +99,10 @@ def test_atomic_write_fsyncs_directory_after_rename(tmp_path, monkeypatch):
atomic_write_json(str(tmp_path / "x.json"), {"k": "v"})
assert "file" in fsync_targets, "expected fsync on the data file"
assert "dir" in fsync_targets, "expected fsync on the parent directory"
if os.name == "nt":
assert "dir" not in fsync_targets, "Windows cannot open a directory for fsync; the writer skips it there"
else:
assert "dir" in fsync_targets, "expected fsync on the parent directory"
# ---------------- read_json_or_none ----------------
+10 -9
View File
@@ -18,12 +18,13 @@ sys.path.insert(0, ".")
from backend.apps.system import loop_liveness_watchdog as w
w.PROBE_INTERVAL_S = 0.3
w.PROBE_TIMEOUT_S = 0.3
w.DUMP_PATH = "/tmp/loop_watchdog_test_dump.log"
w.DUMP_PATH = %(dump)r
"""
def test_wedged_loop_is_killed_with_forensics(tmp_path):
code = P_PRELUDE + """
dump = str(tmp_path / "loop_watchdog_test_dump.log")
code = P_PRELUDE % {"dump": dump} + """
async def main():
loop = asyncio.get_running_loop()
assert w.start_loop_liveness_watchdog(loop) is not None
@@ -35,13 +36,13 @@ print("SURVIVED")
r = p_run_child(code)
assert r.returncode == w.RESTART_EXIT_CODE, f"expected exit {w.RESTART_EXIT_CODE}, got {r.returncode}: {r.stderr[:300]}"
assert "SURVIVED" not in r.stdout
dump = open("/tmp/loop_watchdog_test_dump.log").read()
dump = open(dump, encoding="utf-8").read()
assert "loop watchdog fired" in dump
assert "Thread" in dump, "faulthandler stack dump missing"
def test_healthy_loop_never_killed():
code = P_PRELUDE + """
def test_healthy_loop_never_killed(tmp_path):
code = P_PRELUDE % {"dump": str(tmp_path / "loop_watchdog_test_dump.log")} + """
async def main():
loop = asyncio.get_running_loop()
stop = w.start_loop_liveness_watchdog(loop)
@@ -55,9 +56,9 @@ print("SURVIVED")
assert r.returncode == 0 and "SURVIVED" in r.stdout
def test_slow_but_alive_loop_survives_single_strikes():
def test_slow_but_alive_loop_survives_single_strikes(tmp_path):
"""Blocks shorter than MAX_STRIKES consecutive misses must never kill (sync httpx on the loop is a known 2s block)."""
code = P_PRELUDE + """
code = P_PRELUDE % {"dump": str(tmp_path / "loop_watchdog_test_dump.log")} + """
async def main():
loop = asyncio.get_running_loop()
stop = w.start_loop_liveness_watchdog(loop)
@@ -73,8 +74,8 @@ print("SURVIVED")
assert r.returncode == 0 and "SURVIVED" in r.stdout
def test_closed_loop_ends_watchdog_quietly():
code = P_PRELUDE + """
def test_closed_loop_ends_watchdog_quietly(tmp_path):
code = P_PRELUDE % {"dump": str(tmp_path / "loop_watchdog_test_dump.log")} + """
async def main():
loop = asyncio.get_running_loop()
w.start_loop_liveness_watchdog(loop)
@@ -113,7 +113,8 @@ def test_the_routes_start_a_job_and_report_it(monkeypatch: Any) -> None:
started = client.post("/api/marketplace/install/start", json={"id": "git-graph"})
assert started.status_code == 200
job_id = started.json()["job_id"]
for _ in range(50):
# The job runs on its own thread; a Windows runner took more than the old 1 s to schedule it.
for _ in range(500):
status = client.get(f"/api/marketplace/install/{job_id}").json()
if status["phase"] == "failed":
break
+5 -2
View File
@@ -41,10 +41,13 @@ def test_an_export_that_yields_nothing_sets_no_variable(tmp_path, monkeypatch):
assert node_trust.node_ca_env(str(tmp_path / "r.pem")) == {}
def test_an_unwritable_destination_sets_no_variable(monkeypatch):
def test_an_unwritable_destination_sets_no_variable(monkeypatch, tmp_path):
monkeypatch.setattr(platform, "system", lambda: "Darwin")
monkeypatch.setattr(node_trust, "p_mac_roots", lambda: ["-----BEGIN CERTIFICATE-----\nx\n"])
assert node_trust.node_ca_env("/proc/nope/cannot/write.pem") == {}
# A path beneath a regular FILE cannot be created on any OS; /proc/nope was creatable on a Windows runner.
blocker = tmp_path / "blocker"
blocker.write_text("x")
assert node_trust.node_ca_env(str(blocker / "cannot" / "write.pem")) == {}
def test_linux_is_left_alone(monkeypatch, tmp_path):
@@ -49,6 +49,8 @@ def test_an_abandoned_flow_ages_out(p_store):
def test_the_verifier_is_not_world_readable(p_store):
if os.name == "nt":
pytest.skip("POSIX mode bits do not exist on Windows; the settings dir sits under the per-user profile, whose ACL is the boundary")
p_store.pending_oauth["state-abc"] = {"provider": "claude", "code_verifier": "secret"}
mode = os.stat(p_store.PENDING_PATH).st_mode & 0o777
assert mode == 0o600, f"pending verifiers must be owner-only, got {oct(mode)}"
+36
View File
@@ -5,6 +5,7 @@ have killed 14 running app runtimes whose backend was up. These pin the discrimi
"""
import os
import pytest
from unittest.mock import patch
from backend.apps.outputs import reap_ghost_runtimes as mod
@@ -163,6 +164,7 @@ def test_a_cwd_orphan_owned_by_a_live_backend_is_spared(monkeypatch):
assert rg.find_ghost_runtime_pids() == [], "a live backend's own app runtime must never be killed"
@pytest.mark.skipif(os.name == "nt", reason="Windows has no SIGSTOP freeze, so there is nothing to thaw; the Windows kill path is pinned below")
def test_a_frozen_ghost_is_thawed_before_being_signalled(monkeypatch):
"""Idle app runtimes are parked with SIGSTOP, and a STOPPED process never handles SIGTERM: it
queues it and lives forever. Found live as a frozen `bash run.sh` that had survived every reap
@@ -213,3 +215,37 @@ def test_the_two_backend_argv_shapes_are_the_ones_the_spawns_use():
assert mod.is_backend_argv("/x/python-env/bin/python3 -m backend.serve --port 8324")
assert not mod.is_backend_argv("node /tmp/ws/frontend/node_modules/.bin/vite")
assert not mod.is_backend_argv("bash run.sh")
def p_win_table(csv_text: str):
class R:
def __init__(self, out):
self.stdout = out
def run(cmd, **kw):
assert cmd[0] == "powershell", "the Windows path must never call ps/lsof/pgrep"
return R(csv_text)
return run
def test_windows_scan_finds_the_orphan_and_spares_the_owned_runtime(monkeypatch):
ws = os.path.abspath(mod.WORKSPACE_DIR)
monkeypatch.setattr(mod, "p_is_windows", lambda: True)
table = (
'"ProcessId","ParentProcessId","CommandLine"\n'
'"100","4","python -m backend.serve --port 20128"\n'
f'"200","100","node {ws}\\app\\vite"\n'
f'"300","1","node {ws}\\other\\vite"\n'
)
with patch.object(mod.subprocess, "run", side_effect=p_win_table(table)):
assert mod.find_ghost_runtime_pids() == [300]
def test_windows_reap_uses_taskkill_and_never_a_posix_signal(monkeypatch):
monkeypatch.setattr(mod, "p_is_windows", lambda: True)
monkeypatch.setattr(mod, "find_ghost_runtime_pids", lambda: [300])
killed = []
monkeypatch.setattr(mod, "kill_descendant_tree", lambda pid, sig="TERM": killed.append((pid, sig)))
monkeypatch.setattr(mod.os, "kill", lambda *a: (_ for _ in ()).throw(AssertionError("os.kill on the Windows path")))
assert mod.reap_ghost_runtimes() == 1
assert killed == [(300, "TERM")]