mirror of
https://github.com/openswarm-ai/openswarm.git
synced 2026-08-20 19:52:23 +02:00
204 lines
10 KiB
Python
204 lines
10 KiB
Python
"""The no-suicide invariant for the agent-editable settings tool, proved by
|
|
exhaustive enumeration rather than a few hand-picked cases.
|
|
|
|
The state space here is small and finite (every shipped model row x every
|
|
connection mode x which keys are present), so we walk ALL of it deterministically
|
|
instead of reaching for randomized property testing. A failure is a concrete,
|
|
reproducible (model, mode, keys) tuple, not a flaky seed.
|
|
|
|
The one invariant under test: the settings-meta write guard must NEVER let an
|
|
agent blank the credential powering its own run, while still allowing it to
|
|
clear any OTHER provider's key. Plus two drift seals: every shipped model lane
|
|
classifies (no "unknown"), and the redactor catches every credential field.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import itertools
|
|
|
|
import pytest
|
|
|
|
from backend.apps.settings.models import AppSettings, CustomProvider
|
|
from backend.apps.agents.providers.registry import BUILTIN_MODELS
|
|
from backend.apps.agents.session_credential import (
|
|
ALL_API_KEY_FIELDS,
|
|
resolve_powering_credential,
|
|
write_would_suicide,
|
|
)
|
|
from backend.apps.settings.redaction import is_secret_field, redact_settings
|
|
|
|
|
|
CONNECTION_MODES = ["own_key", "openswarm-pro", "free-trial"]
|
|
|
|
# Every credential field the settings PUT path already treats as secret. Kept here as the contract the redactor must honor; if PUT's notion of "secret" grows, this list should too, and the drift-seal test fails until the redactor also covers it.
|
|
KNOWN_SECRET_FIELDS = [
|
|
"anthropic_api_key", "openai_api_key", "google_api_key", "openrouter_api_key",
|
|
"claude_subscription_token", "openai_subscription_token", "gemini_subscription_token",
|
|
"openswarm_bearer_token", "free_trial_token", "installation_id",
|
|
]
|
|
|
|
|
|
def p_all_model_values() -> list[str]:
|
|
vals = [m["value"] for rows in BUILTIN_MODELS.values() for m in rows]
|
|
# Plus synthesized lanes the resolver must also place.
|
|
vals += ["or:anthropic/claude-3.5", "custom/lmstudio/llama-3", "totally-made-up-model"]
|
|
return vals
|
|
|
|
|
|
def p_settings_with(mode: str, keys: set[str], custom: bool = False) -> AppSettings:
|
|
s = AppSettings(connection_mode=mode)
|
|
if "anthropic" in keys:
|
|
s.anthropic_api_key = "sk-ant-live-aaaa"
|
|
if "openai" in keys:
|
|
s.openai_api_key = "sk-openai-live-bbbb"
|
|
if "google" in keys:
|
|
s.google_api_key = "goog-live-cccc"
|
|
if "openrouter" in keys:
|
|
s.openrouter_api_key = "or-live-dddd"
|
|
if mode in ("openswarm-pro", "free-trial"):
|
|
s.openswarm_bearer_token = "bearer-live-eeee"
|
|
if mode == "free-trial":
|
|
s.free_trial_token = "ft-live-ffff"
|
|
if custom:
|
|
s.custom_providers = [CustomProvider(name="LMStudio", base_url="http://localhost:1234/v1", api_key="local")]
|
|
return s
|
|
|
|
|
|
# --------------------------------------------------------------------------- The invariant: the live credential can never be blanked; others always can. ---------------------------------------------------------------------------
|
|
|
|
def test_live_api_key_can_never_be_blanked_but_others_can():
|
|
key_subsets = [set(c) for r in range(5)
|
|
for c in itertools.combinations(["anthropic", "openai", "google", "openrouter"], r)]
|
|
checked_api_key_runs = 0
|
|
for model in p_all_model_values():
|
|
for mode in CONNECTION_MODES:
|
|
for keys in key_subsets:
|
|
for custom in (False, True):
|
|
s = p_settings_with(mode, keys, custom=custom)
|
|
p = resolve_powering_credential(model, s)
|
|
|
|
if p.kind == "api_key" and p.protected_field:
|
|
checked_api_key_runs += 1
|
|
# Blanking the live key, in any blank form, is refused.
|
|
for blank in (None, "", " "):
|
|
assert write_would_suicide(p.protected_field, blank, p), (
|
|
f"suicide allowed: model={model} mode={mode} field={p.protected_field}={blank!r}"
|
|
)
|
|
# Replacing it with a real key is a reconnect, allowed.
|
|
assert not write_would_suicide(p.protected_field, "sk-fresh-9999", p)
|
|
# Clearing any OTHER provider's key stays allowed.
|
|
for other in ALL_API_KEY_FIELDS - {p.protected_field}:
|
|
assert not write_would_suicide(other, "", p), (
|
|
f"over-blocked unrelated key {other}: model={model} mode={mode}"
|
|
)
|
|
|
|
elif p.kind == "subscription":
|
|
# The live credential isn't a settings field, so clearing ANY api key is safe (it can't be the powering one).
|
|
for field in ALL_API_KEY_FIELDS:
|
|
assert not write_would_suicide(field, "", p), (
|
|
f"subscription run wrongly protected {field}: model={model} mode={mode}"
|
|
)
|
|
|
|
elif p.kind == "unknown":
|
|
# Fail safe: every credential field is protected.
|
|
for field in ALL_API_KEY_FIELDS:
|
|
assert write_would_suicide(field, "", p)
|
|
|
|
assert checked_api_key_runs > 0, "enumeration never exercised an api-key run; test is vacuous"
|
|
|
|
|
|
def test_custom_provider_run_protects_its_entry():
|
|
s = p_settings_with("own_key", set(), custom=True)
|
|
p = resolve_powering_credential("custom/lmstudio/llama-3", s)
|
|
assert p.kind == "api_key" and p.provider == "custom"
|
|
|
|
# Dropping the powering provider's entry is suicide.
|
|
assert write_would_suicide("custom_providers", [], p)
|
|
# Keeping it (even with a blanked placeholder key, local servers don't need one) is fine.
|
|
keep = [{"name": "LMStudio", "base_url": "http://localhost:1234/v1", "api_key": ""}]
|
|
assert not write_would_suicide("custom_providers", keep, p)
|
|
# Swapping in a different provider but losing the live one is suicide.
|
|
other = [{"name": "Together", "base_url": "https://api.together.xyz/v1", "api_key": "k"}]
|
|
assert write_would_suicide("custom_providers", other, p)
|
|
|
|
|
|
def test_disconnect_all_models_spec_scenario():
|
|
"""The spec's worked example: Claude (api key) + OpenAI (api key) both
|
|
connected, run on an Anthropic model, asked to disconnect everything. It
|
|
must refuse to kill Claude (the live one) and allow killing OpenAI."""
|
|
s = p_settings_with("own_key", {"anthropic", "openai"})
|
|
p = resolve_powering_credential("opus-4-8", s) # default Anthropic row, own_key -> api key
|
|
assert p.kind == "api_key" and p.protected_field == "anthropic_api_key"
|
|
assert write_would_suicide("anthropic_api_key", "", p) # refuse self
|
|
assert not write_would_suicide("openai_api_key", "", p) # allow the other
|
|
|
|
# Same connections, but the run is on the OpenAI key instead: mirror image.
|
|
p2 = resolve_powering_credential("gpt-5.5-api", s)
|
|
assert p2.kind == "api_key" and p2.protected_field == "openai_api_key"
|
|
assert write_would_suicide("openai_api_key", "", p2)
|
|
assert not write_would_suicide("anthropic_api_key", "", p2)
|
|
|
|
|
|
# --------------------------------------------------------------------------- Drift seals. ---------------------------------------------------------------------------
|
|
|
|
def test_every_shipped_model_lane_classifies():
|
|
"""A new model row that the resolver can't place would silently fall to the
|
|
fail-safe 'unknown' lane (over-blocking every key). Force every shipped row
|
|
to resolve to a real api_key/subscription so new lanes get classified."""
|
|
s_pro = p_settings_with("openswarm-pro", {"anthropic", "openai", "google", "openrouter"})
|
|
s_key = p_settings_with("own_key", {"anthropic", "openai", "google", "openrouter"})
|
|
for rows in BUILTIN_MODELS.values():
|
|
for m in rows:
|
|
for s in (s_pro, s_key):
|
|
p = resolve_powering_credential(m["value"], s)
|
|
assert p.kind in ("api_key", "subscription"), (
|
|
f"unclassified model lane {m['value']!r} -> {p.kind}"
|
|
)
|
|
|
|
|
|
def test_redactor_catches_every_known_secret():
|
|
for field in KNOWN_SECRET_FIELDS:
|
|
assert is_secret_field(field), f"redactor would leak {field}"
|
|
# And every AppSettings field that NAMES itself a secret is caught by the rule.
|
|
for name in AppSettings.model_fields:
|
|
if name.endswith(("_key", "_token", "_secret")):
|
|
assert is_secret_field(name)
|
|
|
|
|
|
def test_redaction_fail_safe_catches_misnamed_secret_by_value():
|
|
# The name rule (_key/_token/_secret) would MISS a field named off-convention. The value-shape backstop must still redact it, so a leak needs BOTH a bad name AND a non-credential-shaped value, not just one.
|
|
import json
|
|
raw = {"theme": "dark", "weird_field": "sk-ant-api03-AAAABBBBCCCCDDDDEEEEFFFF"}
|
|
red = redact_settings(raw)
|
|
assert red["theme"] == "dark"
|
|
assert isinstance(red["weird_field"], dict) and red["weird_field"]["configured"] is True
|
|
assert "sk-ant-api03" not in json.dumps(red)
|
|
|
|
|
|
def test_redact_settings_never_emits_a_raw_secret():
|
|
s = p_settings_with("openswarm-pro", {"anthropic", "openai", "google", "openrouter"}, custom=True)
|
|
s.claude_subscription_token = "should-never-appear"
|
|
raw = s.model_dump()
|
|
red = redact_settings(raw)
|
|
|
|
for field in KNOWN_SECRET_FIELDS:
|
|
if field in red:
|
|
assert isinstance(red[field], dict), f"{field} not redacted to a state dict"
|
|
assert "configured" in red[field]
|
|
raw_val = raw.get(field)
|
|
if isinstance(raw_val, str) and raw_val.strip():
|
|
# Configured: state only, never the whole value (last4 at most).
|
|
assert red[field]["configured"] is True
|
|
assert red[field].get("last4") != raw_val
|
|
assert len(red[field].get("last4") or "") <= 4
|
|
# The nested custom-provider key is redacted too.
|
|
assert isinstance(red["custom_providers"][0]["api_key"], dict)
|
|
# Non-secret fields pass through untouched.
|
|
assert red["theme"] == raw["theme"]
|
|
assert red["connection_mode"] == raw["connection_mode"]
|
|
|
|
# The strongest check: the literal secret string appears nowhere in the output.
|
|
import json
|
|
assert "should-never-appear" not in json.dumps(red)
|
|
assert "sk-ant-live-aaaa" not in json.dumps(red)
|