[eric] web: race the free search engines so a dead one costs the hedge delay once, not 8s on every launch

backend/apps/web/cascade.py

@@ -14,12 +14,6 @@ from typing import Awaitable, Callable, Dict, List, Optional
 from pydantic import BaseModel, ConfigDict, Field, InstanceOf
 from typeguard import typechecked
 
-from backend.apps.web.tier_breaker import (
-    record_tier_failure,
-    record_tier_success,
-    tier_cooldown_left,
-)
-
 # A tier handed less than this has no realistic chance, and reporting it as a timeout would be a lie; we say the budget ran out instead.
 MIN_TIER_SECONDS = 3.0
 
@@ -30,8 +24,6 @@ class CascadeTier(BaseModel):
     name: str
     run: InstanceOf[Callable[[], Awaitable[Optional[Dict]]]]
     budget: float
-    # Only for tiers whose failure is a property of the HOST, not of this request; see tier_breaker.
-    breaker: bool = False
 
 
 class CascadeOutcome(BaseModel):
@@ -56,25 +48,14 @@ async def run_cascade(tiers: List[CascadeTier], total_budget: float) -> CascadeO
                 f"{total_budget:.0f}s cascade budget spent; not attempted: {', '.join(skipped)}"
             )
             break
-        cooling = tier_cooldown_left(tier.name) if tier.breaker else 0.0
-        if cooling:
-            errors.append(f"{tier.name}: skipped, still failing (retry in {cooling:.0f}s)")
-            continue
         slice_seconds = min(tier.budget, remaining)
         try:
             result = await asyncio.wait_for(tier.run(), timeout=slice_seconds)
         except asyncio.TimeoutError:
             errors.append(f"{tier.name}: timed out after {slice_seconds:.0f}s")
-            if tier.breaker:
-                record_tier_failure(tier.name)
         except Exception as exc:
             errors.append(f"{tier.name}: {str(exc)[:150]}")
-            if tier.breaker:
-                record_tier_failure(tier.name)
         else:
-            # Answering "no hits" still proves the host is up, so it clears the failure streak.
-            if tier.breaker:
-                record_tier_success(tier.name)
             if result is not None:
                 return CascadeOutcome(result=result, errors=errors)
 

backend/apps/web/keyless_race.py

@@ -0,0 +1,126 @@
+"""Race the free search engines instead of queueing them.
+
+Walking them in order made whether search was FAST depend on whether the first
+engine happened to be alive, which is not a property we control. Measured from
+a fresh backend on a machine where DuckDuckGo had stopped answering TCP: the
+first three searches took 8.8s each, because each one waited out the dead
+engine's whole tier budget before Startpage was allowed to try. A desktop app
+starts its backend on every launch, so that was the first thing a user saw.
+
+So the second engine no longer waits for the first to fail; it waits only for
+the first to be SLOW. If the leader answers inside the hedge delay, which a
+healthy frontend does with room to spare, nothing else is sent and the traffic
+is exactly what it was before. If it doesn't, the next engine starts alongside
+it and the first good answer wins. A dead engine now costs the hedge delay
+once, instead of its full budget forever.
+
+The breaker underneath is what makes it cost nothing on the queries after that,
+but it is now an optimisation rather than the thing standing between the user
+and an answer.
+"""
+
+import asyncio
+from typing import Awaitable, Callable, Dict, List, Optional, Set
+
+from pydantic import BaseModel, ConfigDict, Field, InstanceOf
+from typeguard import typechecked
+
+from backend.apps.web.tier_breaker import (
+    record_tier_failure,
+    record_tier_success,
+    tier_cooldown_left,
+)
+
+# Measured across 44 queries: a healthy DuckDuckGo answered in 0.68-1.14s and a healthy Startpage in 0.46-1.33s, so a frontend still silent at 1.5s is not about to win the race.
+HEDGE_AFTER_SECONDS = 1.5
+
+
+class KeylessEngine(BaseModel):
+    model_config = ConfigDict(validate_assignment=True)
+
+    name: str
+    run: InstanceOf[Callable[[], Awaitable[Optional[Dict]]]]
+
+
+class RaceOutcome(BaseModel):
+    model_config = ConfigDict(validate_assignment=True)
+
+    result: Optional[Dict] = None
+    errors: List[str] = Field(default_factory=list)
+
+
+@typechecked
+async def race_keyless(
+    engines: List[KeylessEngine],
+    budget: float,
+    hedge_after: float = HEDGE_AFTER_SECONDS,
+) -> RaceOutcome:
+    """First good answer wins; a slow engine pulls in the next one rather than blocking it."""
+    loop = asyncio.get_running_loop()
+    deadline = loop.time() + budget
+    errors: List[str] = []
+
+    live: List[KeylessEngine] = []
+    for engine in engines:
+        cooling = tier_cooldown_left(engine.name)
+        if cooling:
+            errors.append(f"{engine.name}: skipped, still failing (retry in {cooling:.0f}s)")
+        else:
+            live.append(engine)
+    if not live:
+        return RaceOutcome(errors=errors)
+
+    running: Dict[asyncio.Task, str] = {}
+    started: Dict[str, float] = {}
+    next_engine = 0
+
+    def start_next() -> None:
+        nonlocal next_engine
+        engine = live[next_engine]
+        next_engine += 1
+        started[engine.name] = loop.time()
+        running[asyncio.ensure_future(engine.run())] = engine.name
+
+    start_next()
+    result: Optional[Dict] = None
+
+    while running and result is None:
+        remaining = deadline - loop.time()
+        if remaining <= 0:
+            break
+        wait_for = remaining
+        if next_engine < len(live):
+            # Hedge off the engine that has been waiting longest, so a stalled leader pulls the next one in on time.
+            oldest = min(started[name] for name in running.values())
+            wait_for = min(wait_for, max(0.0, oldest + hedge_after - loop.time()))
+        done: Set[asyncio.Task] = set()
+        done, _ = await asyncio.wait(set(running), timeout=wait_for,
+                                     return_when=asyncio.FIRST_COMPLETED)
+        for task in done:
+            name = running.pop(task)
+            try:
+                answer = task.result()
+            except asyncio.CancelledError:
+                continue
+            except Exception as exc:
+                errors.append(f"{name}: {str(exc)[:150]}")
+                record_tier_failure(name)
+                continue
+            # Answering "no hits" still proves the host is up, so it clears the failure streak.
+            record_tier_success(name)
+            if answer is not None and result is None:
+                result = answer
+        if result is None and next_engine < len(live) and (not done or not running):
+            start_next()
+
+    for task, name in running.items():
+        task.cancel()
+        silent_for = loop.time() - started[name]
+        # It never answered in the window a healthy engine answers three times over, so stop asking it for a while.
+        if silent_for >= hedge_after:
+            errors.append(f"{name}: no response in {silent_for:.0f}s")
+            record_tier_failure(name, conclusive=True)
+    if running:
+        await asyncio.gather(*running, return_exceptions=True)
+
+    return RaceOutcome(result=result, errors=errors)

backend/apps/web/tier_breaker.py

@@ -53,12 +53,15 @@ def tier_cooldown_left(name: str, now: Optional[float] = None) -> float:
 
 
 @typechecked
-def record_tier_failure(name: str, now: Optional[float] = None) -> None:
-    """A timeout or exception. Three in a row shuts the tier for a doubling cooldown."""
+def record_tier_failure(name: str, now: Optional[float] = None, *, conclusive: bool = False) -> None:
+    """A failure. An error ANSWER is one strike of three, because a 202 or a 403 can be a bad
+    minute. SILENCE is conclusive and shuts the tier at once: a frontend that returns nothing at
+    all in the time a healthy one answers three times over is not having a bad minute, and making
+    the user prove it three times is what put 8.8s on their first three searches after launch."""
     stamp = time.monotonic() if now is None else now
     entry = p_entry(name)
     entry.consecutive_failures += 1
-    if entry.consecutive_failures < FAILURES_TO_OPEN:
+    if not conclusive and entry.consecutive_failures < FAILURES_TO_OPEN:
         return
     # The half-open probe that fails again doubles the wait, so a permanently dead engine stops costing anything.
     entry.cooldown = min(max(entry.cooldown * 2, FIRST_COOLDOWN_SECONDS), MAX_COOLDOWN_SECONDS)

backend/apps/web/web.py

@@ -18,6 +18,7 @@ from pydantic import BaseModel, Field
 from typeguard import typechecked
 
 from backend.apps.web.cascade import CascadeTier, run_cascade
+from backend.apps.web.keyless_race import KeylessEngine, race_keyless
 from backend.apps.web.grounded import (
     format_grounded_as_fetch,
     format_grounded_as_search_results,
@@ -189,9 +190,20 @@ async def search(body: SearchBody) -> Dict:
         return {"query": body.query, "results": format_grounded_as_search_results(grounded, body.query),
                 "backend": "openai_subscription"}
 
+    # Collected out-of-band because the race reports per-engine outcomes and a cascade tier can only report one.
+    keyless_errors: List[str] = []
+
+    async def try_keyless_engines() -> Optional[Dict]:
+        outcome = await race_keyless(
+            [KeylessEngine(name="ddg", run=try_keyless),
+             KeylessEngine(name="startpage", run=try_startpage)],
+            KEYLESS_TIER_SECONDS,
+        )
+        keyless_errors.extend(outcome.errors)
+        return outcome.result
+
     tiers = [
-        CascadeTier(name="ddg", run=try_keyless, budget=KEYLESS_TIER_SECONDS, breaker=True),
-        CascadeTier(name="startpage", run=try_startpage, budget=KEYLESS_TIER_SECONDS, breaker=True),
+        CascadeTier(name="keyless", run=try_keyless_engines, budget=KEYLESS_TIER_SECONDS),
         CascadeTier(name="browser_search", run=try_browser_search, budget=BROWSER_TIER_SECONDS),
     ] + p_grounded_tiers("search", body.primary, {
         "gemini_native": try_gemini,
@@ -201,11 +213,21 @@ async def search(body: SearchBody) -> Dict:
     })
 
     outcome = await run_cascade(tiers, SEARCH_BUDGET_SECONDS)
+    outcome.errors = keyless_errors + [e for e in outcome.errors if not e.startswith("keyless:")]
     if outcome.result is not None:
         if outcome.errors:
             outcome.result["cascade_errors"] = outcome.errors
         return outcome.result
 
+    # Nothing refused us, the engines simply had no matches; saying otherwise sends the model hunting for an outage that isn't there.
+    if not keyless_errors:
+        return {
+            "query": body.query,
+            "results": f"No results for: {body.query}\n\nThe search engines answered normally "
+                       "and had no matches for this query.",
+            "backend": "none",
+        }
+
     # Everything failed. Be honest about why instead of an empty "no results".
     connected = await refresh_9r_connected()
     has_subscription = bool(connected & {"codex", "antigravity", "gemini-cli"})

backend/tests/test_keyless_race.py

@@ -0,0 +1,228 @@
+"""The free engines race; a dead one must not stand between the user and an answer.
+
+The bug these pin is a cold-start bug, so it hides from any average taken over a
+long-lived process. Measured on a fresh backend against a DuckDuckGo that had
+stopped answering TCP: q1 8,812ms, q2 8,825ms, q3 8,934ms, then 585ms. Breaker
+state is per-process and the desktop app starts a backend on every launch, so
+every user's first three searches after opening the app paid full price to
+rediscover what the last process already knew.
+"""
+
+import asyncio
+
+import pytest
+
+from backend.apps.web.keyless_race import KeylessEngine, race_keyless
+from backend.apps.web.tier_breaker import (
+    FIRST_COOLDOWN_SECONDS,
+    reset_tier_health,
+    tier_cooldown_left,
+)
+
+
+@pytest.fixture(autouse=True)
+def p_clean():
+    reset_tier_health()
+    yield
+    reset_tier_health()
+
+
+def p_engine(name, fn):
+    return KeylessEngine(name=name, run=fn)
+
+
+def p_answers(name, after=0.0, calls=None):
+    async def run():
+        if calls is not None:
+            calls.append(name)
+        if after:
+            await asyncio.sleep(after)
+        return {"backend": name}
+    return run
+
+
+def p_silent(name, calls=None):
+    async def run():
+        if calls is not None:
+            calls.append(name)
+        await asyncio.sleep(30)
+    return run
+
+
+def p_raises(name, calls=None):
+    async def run():
+        if calls is not None:
+            calls.append(name)
+        raise RuntimeError(f"{name} served its bot challenge")
+    return run
+
+
+def p_empty(name, calls=None):
+    async def run():
+        if calls is not None:
+            calls.append(name)
+        return None
+    return run
+
+
+@pytest.mark.asyncio
+async def test_healthy_leader_wins_alone_and_sends_no_extra_traffic():
+    """The whole cost case for racing rests on this: a healthy engine must not double our requests."""
+    calls = []
+    out = await race_keyless(
+        [p_engine("ddg", p_answers("ddg", 0.01, calls)),
+         p_engine("startpage", p_answers("startpage", 0.01, calls))],
+        budget=5.0, hedge_after=0.3,
+    )
+    assert out.result == {"backend": "ddg"}
+    assert calls == ["ddg"], "the second engine must not be dispatched when the first is fast"
+
+
+@pytest.mark.asyncio
+async def test_a_silent_leader_costs_the_hedge_not_the_budget():
+    """This is the cold-start fix: a blackholed engine used to cost its full 8s tier budget."""
+    loop = asyncio.get_running_loop()
+    t0 = loop.time()
+    out = await race_keyless(
+        [p_engine("ddg", p_silent("ddg")),
+         p_engine("startpage", p_answers("startpage", 0.01))],
+        budget=5.0, hedge_after=0.3,
+    )
+    elapsed = loop.time() - t0
+    assert out.result == {"backend": "startpage"}
+    assert elapsed < 1.0, f"a dead leader should cost about the hedge delay, took {elapsed:.2f}s"
+
+
+@pytest.mark.asyncio
+async def test_a_fast_failure_starts_the_next_engine_immediately():
+    """An engine that refuses outright should not make us wait out the hedge as well."""
+    loop = asyncio.get_running_loop()
+    t0 = loop.time()
+    out = await race_keyless(
+        [p_engine("ddg", p_raises("ddg")),
+         p_engine("startpage", p_answers("startpage", 0.01))],
+        budget=5.0, hedge_after=2.0,
+    )
+    assert out.result == {"backend": "startpage"}
+    assert loop.time() - t0 < 1.0
+    assert any("bot challenge" in e for e in out.errors)
+
+
+@pytest.mark.asyncio
+async def test_silence_shuts_the_engine_after_one_query():
+    """One conclusive silence, not three, so the tax is one slow search rather than three."""
+    await race_keyless(
+        [p_engine("ddg", p_silent("ddg")),
+         p_engine("startpage", p_answers("startpage", 0.01))],
+        budget=5.0, hedge_after=0.3,
+    )
+    assert tier_cooldown_left("ddg") > 0
+    assert tier_cooldown_left("startpage") == 0.0
+
+
+@pytest.mark.asyncio
+async def test_a_cooling_engine_is_not_dispatched_at_all():
+    calls = []
+    for _ in range(2):
+        await race_keyless(
+            [p_engine("ddg", p_silent("ddg", calls)),
+             p_engine("startpage", p_answers("startpage", 0.01, calls))],
+            budget=5.0, hedge_after=0.3,
+        )
+    assert calls.count("ddg") == 1, "the second query must not re-probe a conclusively dead engine"
+    out = await race_keyless(
+        [p_engine("ddg", p_silent("ddg", calls)),
+         p_engine("startpage", p_answers("startpage", 0.01, calls))],
+        budget=5.0, hedge_after=0.3,
+    )
+    assert any("skipped, still failing" in e for e in out.errors)
+
+
+@pytest.mark.asyncio
+async def test_a_recovered_engine_is_used_again(monkeypatch):
+    """The fix must not quietly blacklist an engine that comes back."""
+    import backend.apps.web.tier_breaker as TB
+    calls = []
+    await race_keyless(
+        [p_engine("ddg", p_silent("ddg", calls)),
+         p_engine("startpage", p_answers("startpage", 0.01, calls))],
+        budget=5.0, hedge_after=0.3,
+    )
+    assert tier_cooldown_left("ddg") > 0
+
+    # Jump past the cooldown rather than sleep through it.
+    real_monotonic = TB.time.monotonic
+    monkeypatch.setattr(TB.time, "monotonic",
+                        lambda: real_monotonic() + FIRST_COOLDOWN_SECONDS + 1)
+    calls.clear()
+    out = await race_keyless(
+        [p_engine("ddg", p_answers("ddg", 0.01, calls)),
+         p_engine("startpage", p_answers("startpage", 0.01, calls))],
+        budget=5.0, hedge_after=0.3,
+    )
+    assert out.result == {"backend": "ddg"}, "a healthy engine must be used again after the cooldown"
+    assert tier_cooldown_left("ddg") == 0.0
+
+
+@pytest.mark.asyncio
+async def test_both_empty_reports_no_hits_not_a_failure():
+    """A nonsense query must not slowly cool down two perfectly healthy engines."""
+    for _ in range(5):
+        out = await race_keyless(
+            [p_engine("ddg", p_empty("ddg")), p_engine("startpage", p_empty("startpage"))],
+            budget=5.0, hedge_after=0.3,
+        )
+        assert out.result is None
+    assert tier_cooldown_left("ddg") == 0.0
+    assert tier_cooldown_left("startpage") == 0.0
+
+
+@pytest.mark.asyncio
+async def test_an_empty_leader_still_lets_the_other_answer():
+    out = await race_keyless(
+        [p_engine("ddg", p_empty("ddg")),
+         p_engine("startpage", p_answers("startpage", 0.01))],
+        budget=5.0, hedge_after=2.0,
+    )
+    assert out.result == {"backend": "startpage"}
+
+
+@pytest.mark.asyncio
+async def test_every_engine_closed_returns_honestly():
+    out = await race_keyless(
+        [p_engine("ddg", p_raises("ddg")), p_engine("startpage", p_raises("startpage"))],
+        budget=5.0, hedge_after=0.3,
+    )
+    assert out.result is None
+    assert len(out.errors) == 2
+
+
+@pytest.mark.asyncio
+async def test_budget_bounds_the_whole_race():
+    loop = asyncio.get_running_loop()
+    t0 = loop.time()
+    out = await race_keyless(
+        [p_engine("ddg", p_silent("ddg")), p_engine("startpage", p_silent("startpage"))],
+        budget=0.6, hedge_after=0.2,
+    )
+    assert out.result is None
+    assert loop.time() - t0 < 2.0
+
+
+@pytest.mark.asyncio
+async def test_losing_engines_do_not_outlive_the_race():
+    """A cancelled request must not keep running and surprise the user's network later."""
+    live = {"n": 0}
+
+    async def clingy():
+        live["n"] += 1
+        try:
+            await asyncio.sleep(30)
+        finally:
+            live["n"] -= 1
+
+    await race_keyless(
+        [p_engine("ddg", clingy), p_engine("startpage", p_answers("startpage", 0.01))],
+        budget=5.0, hedge_after=0.2,
+    )
+    assert live["n"] == 0

backend/tests/test_tier_breaker.py

@@ -1,19 +1,15 @@
-"""A dead search frontend must stop costing us its whole tier budget.
+"""Engine health: how many failures shut a frontend, and for how long.
 
 Measured before this existed: once DuckDuckGo stopped answering TCP, three
-consecutive 44-query rounds each paid the full 8s DuckDuckGo budget on EVERY
-query, so keyless p50 went 1.0s -> 8.5s while Startpage still served 40/40.
-These pin the breaker that makes that unrepresentable, and pin the two ways it
-could go wrong instead: skipping a healthy tier, or leaking a per-URL failure
-into a fixed-host one.
-"""
+consecutive 44-query rounds each paid its full 8s budget on EVERY query, so
+keyless p50 went 1.0s -> 8.5s while Startpage still served 40/40.
 
-import asyncio
+The racing behaviour that keeps a dead engine off the critical path lives in
+test_keyless_race.py; this file pins the state machine underneath it.
+"""
 
 import pytest
 
-import backend.apps.web.cascade as C
-from backend.apps.web.cascade import CascadeTier, run_cascade
 from backend.apps.web.tier_breaker import (
     FAILURES_TO_OPEN,
     FIRST_COOLDOWN_SECONDS,
@@ -32,110 +28,55 @@ def p_clean():
     reset_tier_health()
 
 
-@pytest.fixture
-def p_tiny_floor(monkeypatch):
-    monkeypatch.setattr(C, "MIN_TIER_SECONDS", 0.01)
-
-
-async def p_boom():
-    raise RuntimeError("engine closed")
-
-
-async def p_hit():
-    return {"backend": "second"}
-
-
-def test_streak_opens_then_success_clears():
+def test_an_error_answer_takes_three_strikes():
+    """A 202 or a 403 can be a bad minute, so one is not enough to shut an engine out."""
     for _ in range(FAILURES_TO_OPEN - 1):
         record_tier_failure("ddg")
     assert tier_cooldown_left("ddg") == 0.0
     record_tier_failure("ddg")
     assert 0 < tier_cooldown_left("ddg") <= FIRST_COOLDOWN_SECONDS
+
+
+def test_silence_is_conclusive_on_the_first_failure():
+    """A frontend returning nothing at all is not ambiguous, and making the user prove it
+    three times is what put 8.8s on their first three searches after launch."""
+    record_tier_failure("ddg", conclusive=True)
+    assert 0 < tier_cooldown_left("ddg") <= FIRST_COOLDOWN_SECONDS
+
+
+def test_success_clears_the_streak():
+    for _ in range(FAILURES_TO_OPEN):
+        record_tier_failure("ddg")
+    assert tier_cooldown_left("ddg") > 0
     record_tier_success("ddg")
     assert tier_cooldown_left("ddg") == 0.0
 
 
+def test_a_recovered_engine_is_not_blacklisted_forever():
+    """The cooldown must expire on its own, or one bad afternoon costs us an engine for good."""
+    record_tier_failure("ddg", now=0.0, conclusive=True)
+    assert tier_cooldown_left("ddg", now=0.0) == pytest.approx(FIRST_COOLDOWN_SECONDS)
+    assert tier_cooldown_left("ddg", now=FIRST_COOLDOWN_SECONDS + 1) == 0.0
+    record_tier_success("ddg")
+    for _ in range(FAILURES_TO_OPEN - 1):
+        record_tier_failure("ddg")
+    assert tier_cooldown_left("ddg") == 0.0, "a success must reset the streak, not just the clock"
+
+
 def test_cooldown_doubles_and_is_capped():
     seen = []
     now = 0.0
-    for round_no in range(12):
-        for _ in range(FAILURES_TO_OPEN):
-            record_tier_failure("ddg", now=now)
+    for _ in range(12):
+        record_tier_failure("ddg", now=now, conclusive=True)
         seen.append(tier_cooldown_left("ddg", now=now))
         now += seen[-1] + 1
-        record_tier_failure("ddg", now=now)  # the half-open probe fails again
     assert seen[0] == pytest.approx(FIRST_COOLDOWN_SECONDS)
     assert seen[1] > seen[0]
     assert max(seen) <= MAX_COOLDOWN_SECONDS
 
 
-@pytest.mark.asyncio
-async def test_dead_tier_is_skipped_instantly(p_tiny_floor):
-    calls = []
-
-    async def p_slow_boom():
-        calls.append(1)
-        await asyncio.sleep(0.05)
-        raise RuntimeError("engine closed")
-
-    tiers = [
-        CascadeTier(name="ddg", run=p_slow_boom, budget=5.0, breaker=True),
-        CascadeTier(name="startpage", run=p_hit, budget=5.0, breaker=True),
-    ]
+def test_engines_are_tracked_independently():
     for _ in range(FAILURES_TO_OPEN):
-        out = await run_cascade(tiers, 5.0)
-        assert out.result == {"backend": "second"}
-    assert len(calls) == FAILURES_TO_OPEN
-
-    out = await run_cascade(tiers, 5.0)
-    assert out.result == {"backend": "second"}
-    assert len(calls) == FAILURES_TO_OPEN, "a cooling tier must not be called at all"
-    assert any("skipped, still failing" in e for e in out.errors)
-
-
-@pytest.mark.asyncio
-async def test_timeout_counts_as_failure(p_tiny_floor):
-    async def p_hangs():
-        await asyncio.sleep(30)
-
-    tiers = [
-        CascadeTier(name="ddg", run=p_hangs, budget=0.05, breaker=True),
-        CascadeTier(name="startpage", run=p_hit, budget=5.0, breaker=True),
-    ]
-    for _ in range(FAILURES_TO_OPEN):
-        await run_cascade(tiers, 5.0)
+        record_tier_failure("ddg")
     assert tier_cooldown_left("ddg") > 0
-
-
-@pytest.mark.asyncio
-async def test_no_hits_is_not_a_failure(p_tiny_floor):
-    """An engine that answers 'nothing matched' is alive; only errors count against it."""
-    async def p_empty():
-        return None
-
-    tiers = [
-        CascadeTier(name="ddg", run=p_empty, budget=5.0, breaker=True),
-        CascadeTier(name="startpage", run=p_hit, budget=5.0, breaker=True),
-    ]
-    for _ in range(FAILURES_TO_OPEN + 2):
-        await run_cascade(tiers, 5.0)
-    assert tier_cooldown_left("ddg") == 0.0
-
-
-@pytest.mark.asyncio
-async def test_tiers_without_breaker_always_run(p_tiny_floor):
-    """A fetch tier fails per-URL, so one dead page must never cool the tier for every other URL."""
-    calls = []
-
-    async def p_fail():
-        calls.append(1)
-        raise RuntimeError("404")
-
-    tiers = [
-        CascadeTier(name="local", run=p_fail, budget=5.0),
-        CascadeTier(name="wayback", run=p_hit, budget=5.0),
-    ]
-    for _ in range(FAILURES_TO_OPEN + 3):
-        await run_cascade(tiers, 5.0)
-    assert len(calls) == FAILURES_TO_OPEN + 3
-    assert tier_cooldown_left("local") == 0.0
+    assert tier_cooldown_left("startpage") == 0.0

backend/tests/test_web_search_cascade.py

@@ -213,3 +213,26 @@ async def test_an_honestly_empty_startpage_does_not_count_against_it(monkeypatch
         out = await search(SearchBody(query="zxqvbnmklwertyuiopasdfg", num_results=5))
         assert out["backend"] == "none"
     assert tier_cooldown_left("startpage") == 0.0
+
+
+@pytest.mark.asyncio
+async def test_a_genuinely_empty_search_does_not_claim_an_outage(monkeypatch):
+    """Both engines answering 'no matches' is an answer; calling it a refusal sends the model
+    hunting for an outage that isn't there."""
+    ddg_returns(monkeypatch, "")
+    startpage_returns(monkeypatch, "")
+    out = await search(SearchBody(query="xyzzyplughnothinghere1234567", num_results=5))
+    assert out["backend"] == "none"
+    assert "had no matches" in out["results"]
+    assert "refused" not in out["results"]
+    assert not out.get("cascade_errors")
+
+
+@pytest.mark.asyncio
+async def test_a_real_outage_still_says_so(monkeypatch):
+    ddg_throttled(monkeypatch)
+    startpage_refuses(monkeypatch)
+    out = await search(SearchBody(query="capital of Burkina Faso", num_results=5))
+    assert out["backend"] == "none"
+    assert "refused" in out["results"]
+    assert out["cascade_errors"]