From ce7939b94d8f7135e5bc4ff5a8e3dd05c3d6333d Mon Sep 17 00:00:00 2001 From: ciregenz Date: Thu, 30 Jul 2026 17:20:41 -0700 Subject: [PATCH] [eric] web: race the free search engines so a dead one costs the hedge delay once, not 8s on every launch --- backend/apps/web/cascade.py | 19 -- backend/apps/web/keyless_race.py | 126 +++++++++++++ backend/apps/web/tier_breaker.py | 9 +- backend/apps/web/web.py | 26 ++- backend/tests/test_keyless_race.py | 228 +++++++++++++++++++++++ backend/tests/test_tier_breaker.py | 133 ++++--------- backend/tests/test_web_search_cascade.py | 23 +++ 7 files changed, 444 insertions(+), 120 deletions(-) create mode 100644 backend/apps/web/keyless_race.py create mode 100644 backend/tests/test_keyless_race.py diff --git a/backend/apps/web/cascade.py b/backend/apps/web/cascade.py index af5cf6eb..16229271 100644 --- a/backend/apps/web/cascade.py +++ b/backend/apps/web/cascade.py @@ -14,12 +14,6 @@ from typing import Awaitable, Callable, Dict, List, Optional from pydantic import BaseModel, ConfigDict, Field, InstanceOf from typeguard import typechecked -from backend.apps.web.tier_breaker import ( - record_tier_failure, - record_tier_success, - tier_cooldown_left, -) - # A tier handed less than this has no realistic chance, and reporting it as a timeout would be a lie; we say the budget ran out instead. MIN_TIER_SECONDS = 3.0 @@ -30,8 +24,6 @@ class CascadeTier(BaseModel): name: str run: InstanceOf[Callable[[], Awaitable[Optional[Dict]]]] budget: float - # Only for tiers whose failure is a property of the HOST, not of this request; see tier_breaker. - breaker: bool = False class CascadeOutcome(BaseModel): @@ -56,25 +48,14 @@ async def run_cascade(tiers: List[CascadeTier], total_budget: float) -> CascadeO f"{total_budget:.0f}s cascade budget spent; not attempted: {', '.join(skipped)}" ) break - cooling = tier_cooldown_left(tier.name) if tier.breaker else 0.0 - if cooling: - errors.append(f"{tier.name}: skipped, still failing (retry in {cooling:.0f}s)") - continue slice_seconds = min(tier.budget, remaining) try: result = await asyncio.wait_for(tier.run(), timeout=slice_seconds) except asyncio.TimeoutError: errors.append(f"{tier.name}: timed out after {slice_seconds:.0f}s") - if tier.breaker: - record_tier_failure(tier.name) except Exception as exc: errors.append(f"{tier.name}: {str(exc)[:150]}") - if tier.breaker: - record_tier_failure(tier.name) else: - # Answering "no hits" still proves the host is up, so it clears the failure streak. - if tier.breaker: - record_tier_success(tier.name) if result is not None: return CascadeOutcome(result=result, errors=errors) diff --git a/backend/apps/web/keyless_race.py b/backend/apps/web/keyless_race.py new file mode 100644 index 00000000..db64ab80 --- /dev/null +++ b/backend/apps/web/keyless_race.py @@ -0,0 +1,126 @@ +"""Race the free search engines instead of queueing them. + +Walking them in order made whether search was FAST depend on whether the first +engine happened to be alive, which is not a property we control. Measured from +a fresh backend on a machine where DuckDuckGo had stopped answering TCP: the +first three searches took 8.8s each, because each one waited out the dead +engine's whole tier budget before Startpage was allowed to try. A desktop app +starts its backend on every launch, so that was the first thing a user saw. + +So the second engine no longer waits for the first to fail; it waits only for +the first to be SLOW. If the leader answers inside the hedge delay, which a +healthy frontend does with room to spare, nothing else is sent and the traffic +is exactly what it was before. If it doesn't, the next engine starts alongside +it and the first good answer wins. A dead engine now costs the hedge delay +once, instead of its full budget forever. + +The breaker underneath is what makes it cost nothing on the queries after that, +but it is now an optimisation rather than the thing standing between the user +and an answer. +""" + +import asyncio +from typing import Awaitable, Callable, Dict, List, Optional, Set + +from pydantic import BaseModel, ConfigDict, Field, InstanceOf +from typeguard import typechecked + +from backend.apps.web.tier_breaker import ( + record_tier_failure, + record_tier_success, + tier_cooldown_left, +) + +# Measured across 44 queries: a healthy DuckDuckGo answered in 0.68-1.14s and a healthy Startpage in 0.46-1.33s, so a frontend still silent at 1.5s is not about to win the race. +HEDGE_AFTER_SECONDS = 1.5 + + +class KeylessEngine(BaseModel): + model_config = ConfigDict(validate_assignment=True) + + name: str + run: InstanceOf[Callable[[], Awaitable[Optional[Dict]]]] + + +class RaceOutcome(BaseModel): + model_config = ConfigDict(validate_assignment=True) + + result: Optional[Dict] = None + errors: List[str] = Field(default_factory=list) + + +@typechecked +async def race_keyless( + engines: List[KeylessEngine], + budget: float, + hedge_after: float = HEDGE_AFTER_SECONDS, +) -> RaceOutcome: + """First good answer wins; a slow engine pulls in the next one rather than blocking it.""" + loop = asyncio.get_running_loop() + deadline = loop.time() + budget + errors: List[str] = [] + + live: List[KeylessEngine] = [] + for engine in engines: + cooling = tier_cooldown_left(engine.name) + if cooling: + errors.append(f"{engine.name}: skipped, still failing (retry in {cooling:.0f}s)") + else: + live.append(engine) + if not live: + return RaceOutcome(errors=errors) + + running: Dict[asyncio.Task, str] = {} + started: Dict[str, float] = {} + next_engine = 0 + + def start_next() -> None: + nonlocal next_engine + engine = live[next_engine] + next_engine += 1 + started[engine.name] = loop.time() + running[asyncio.ensure_future(engine.run())] = engine.name + + start_next() + result: Optional[Dict] = None + + while running and result is None: + remaining = deadline - loop.time() + if remaining <= 0: + break + wait_for = remaining + if next_engine < len(live): + # Hedge off the engine that has been waiting longest, so a stalled leader pulls the next one in on time. + oldest = min(started[name] for name in running.values()) + wait_for = min(wait_for, max(0.0, oldest + hedge_after - loop.time())) + done: Set[asyncio.Task] = set() + done, _ = await asyncio.wait(set(running), timeout=wait_for, + return_when=asyncio.FIRST_COMPLETED) + for task in done: + name = running.pop(task) + try: + answer = task.result() + except asyncio.CancelledError: + continue + except Exception as exc: + errors.append(f"{name}: {str(exc)[:150]}") + record_tier_failure(name) + continue + # Answering "no hits" still proves the host is up, so it clears the failure streak. + record_tier_success(name) + if answer is not None and result is None: + result = answer + if result is None and next_engine < len(live) and (not done or not running): + start_next() + + for task, name in running.items(): + task.cancel() + silent_for = loop.time() - started[name] + # It never answered in the window a healthy engine answers three times over, so stop asking it for a while. + if silent_for >= hedge_after: + errors.append(f"{name}: no response in {silent_for:.0f}s") + record_tier_failure(name, conclusive=True) + if running: + await asyncio.gather(*running, return_exceptions=True) + + return RaceOutcome(result=result, errors=errors) diff --git a/backend/apps/web/tier_breaker.py b/backend/apps/web/tier_breaker.py index e574c0f0..8c0dd126 100644 --- a/backend/apps/web/tier_breaker.py +++ b/backend/apps/web/tier_breaker.py @@ -53,12 +53,15 @@ def tier_cooldown_left(name: str, now: Optional[float] = None) -> float: @typechecked -def record_tier_failure(name: str, now: Optional[float] = None) -> None: - """A timeout or exception. Three in a row shuts the tier for a doubling cooldown.""" +def record_tier_failure(name: str, now: Optional[float] = None, *, conclusive: bool = False) -> None: + """A failure. An error ANSWER is one strike of three, because a 202 or a 403 can be a bad + minute. SILENCE is conclusive and shuts the tier at once: a frontend that returns nothing at + all in the time a healthy one answers three times over is not having a bad minute, and making + the user prove it three times is what put 8.8s on their first three searches after launch.""" stamp = time.monotonic() if now is None else now entry = p_entry(name) entry.consecutive_failures += 1 - if entry.consecutive_failures < FAILURES_TO_OPEN: + if not conclusive and entry.consecutive_failures < FAILURES_TO_OPEN: return # The half-open probe that fails again doubles the wait, so a permanently dead engine stops costing anything. entry.cooldown = min(max(entry.cooldown * 2, FIRST_COOLDOWN_SECONDS), MAX_COOLDOWN_SECONDS) diff --git a/backend/apps/web/web.py b/backend/apps/web/web.py index 597b8a5f..adc29893 100644 --- a/backend/apps/web/web.py +++ b/backend/apps/web/web.py @@ -18,6 +18,7 @@ from pydantic import BaseModel, Field from typeguard import typechecked from backend.apps.web.cascade import CascadeTier, run_cascade +from backend.apps.web.keyless_race import KeylessEngine, race_keyless from backend.apps.web.grounded import ( format_grounded_as_fetch, format_grounded_as_search_results, @@ -189,9 +190,20 @@ async def search(body: SearchBody) -> Dict: return {"query": body.query, "results": format_grounded_as_search_results(grounded, body.query), "backend": "openai_subscription"} + # Collected out-of-band because the race reports per-engine outcomes and a cascade tier can only report one. + keyless_errors: List[str] = [] + + async def try_keyless_engines() -> Optional[Dict]: + outcome = await race_keyless( + [KeylessEngine(name="ddg", run=try_keyless), + KeylessEngine(name="startpage", run=try_startpage)], + KEYLESS_TIER_SECONDS, + ) + keyless_errors.extend(outcome.errors) + return outcome.result + tiers = [ - CascadeTier(name="ddg", run=try_keyless, budget=KEYLESS_TIER_SECONDS, breaker=True), - CascadeTier(name="startpage", run=try_startpage, budget=KEYLESS_TIER_SECONDS, breaker=True), + CascadeTier(name="keyless", run=try_keyless_engines, budget=KEYLESS_TIER_SECONDS), CascadeTier(name="browser_search", run=try_browser_search, budget=BROWSER_TIER_SECONDS), ] + p_grounded_tiers("search", body.primary, { "gemini_native": try_gemini, @@ -201,11 +213,21 @@ async def search(body: SearchBody) -> Dict: }) outcome = await run_cascade(tiers, SEARCH_BUDGET_SECONDS) + outcome.errors = keyless_errors + [e for e in outcome.errors if not e.startswith("keyless:")] if outcome.result is not None: if outcome.errors: outcome.result["cascade_errors"] = outcome.errors return outcome.result + # Nothing refused us, the engines simply had no matches; saying otherwise sends the model hunting for an outage that isn't there. + if not keyless_errors: + return { + "query": body.query, + "results": f"No results for: {body.query}\n\nThe search engines answered normally " + "and had no matches for this query.", + "backend": "none", + } + # Everything failed. Be honest about why instead of an empty "no results". connected = await refresh_9r_connected() has_subscription = bool(connected & {"codex", "antigravity", "gemini-cli"}) diff --git a/backend/tests/test_keyless_race.py b/backend/tests/test_keyless_race.py new file mode 100644 index 00000000..9d65ba95 --- /dev/null +++ b/backend/tests/test_keyless_race.py @@ -0,0 +1,228 @@ +"""The free engines race; a dead one must not stand between the user and an answer. + +The bug these pin is a cold-start bug, so it hides from any average taken over a +long-lived process. Measured on a fresh backend against a DuckDuckGo that had +stopped answering TCP: q1 8,812ms, q2 8,825ms, q3 8,934ms, then 585ms. Breaker +state is per-process and the desktop app starts a backend on every launch, so +every user's first three searches after opening the app paid full price to +rediscover what the last process already knew. +""" + +import asyncio + +import pytest + +from backend.apps.web.keyless_race import KeylessEngine, race_keyless +from backend.apps.web.tier_breaker import ( + FIRST_COOLDOWN_SECONDS, + reset_tier_health, + tier_cooldown_left, +) + + +@pytest.fixture(autouse=True) +def p_clean(): + reset_tier_health() + yield + reset_tier_health() + + +def p_engine(name, fn): + return KeylessEngine(name=name, run=fn) + + +def p_answers(name, after=0.0, calls=None): + async def run(): + if calls is not None: + calls.append(name) + if after: + await asyncio.sleep(after) + return {"backend": name} + return run + + +def p_silent(name, calls=None): + async def run(): + if calls is not None: + calls.append(name) + await asyncio.sleep(30) + return run + + +def p_raises(name, calls=None): + async def run(): + if calls is not None: + calls.append(name) + raise RuntimeError(f"{name} served its bot challenge") + return run + + +def p_empty(name, calls=None): + async def run(): + if calls is not None: + calls.append(name) + return None + return run + + +@pytest.mark.asyncio +async def test_healthy_leader_wins_alone_and_sends_no_extra_traffic(): + """The whole cost case for racing rests on this: a healthy engine must not double our requests.""" + calls = [] + out = await race_keyless( + [p_engine("ddg", p_answers("ddg", 0.01, calls)), + p_engine("startpage", p_answers("startpage", 0.01, calls))], + budget=5.0, hedge_after=0.3, + ) + assert out.result == {"backend": "ddg"} + assert calls == ["ddg"], "the second engine must not be dispatched when the first is fast" + + +@pytest.mark.asyncio +async def test_a_silent_leader_costs_the_hedge_not_the_budget(): + """This is the cold-start fix: a blackholed engine used to cost its full 8s tier budget.""" + loop = asyncio.get_running_loop() + t0 = loop.time() + out = await race_keyless( + [p_engine("ddg", p_silent("ddg")), + p_engine("startpage", p_answers("startpage", 0.01))], + budget=5.0, hedge_after=0.3, + ) + elapsed = loop.time() - t0 + assert out.result == {"backend": "startpage"} + assert elapsed < 1.0, f"a dead leader should cost about the hedge delay, took {elapsed:.2f}s" + + +@pytest.mark.asyncio +async def test_a_fast_failure_starts_the_next_engine_immediately(): + """An engine that refuses outright should not make us wait out the hedge as well.""" + loop = asyncio.get_running_loop() + t0 = loop.time() + out = await race_keyless( + [p_engine("ddg", p_raises("ddg")), + p_engine("startpage", p_answers("startpage", 0.01))], + budget=5.0, hedge_after=2.0, + ) + assert out.result == {"backend": "startpage"} + assert loop.time() - t0 < 1.0 + assert any("bot challenge" in e for e in out.errors) + + +@pytest.mark.asyncio +async def test_silence_shuts_the_engine_after_one_query(): + """One conclusive silence, not three, so the tax is one slow search rather than three.""" + await race_keyless( + [p_engine("ddg", p_silent("ddg")), + p_engine("startpage", p_answers("startpage", 0.01))], + budget=5.0, hedge_after=0.3, + ) + assert tier_cooldown_left("ddg") > 0 + assert tier_cooldown_left("startpage") == 0.0 + + +@pytest.mark.asyncio +async def test_a_cooling_engine_is_not_dispatched_at_all(): + calls = [] + for _ in range(2): + await race_keyless( + [p_engine("ddg", p_silent("ddg", calls)), + p_engine("startpage", p_answers("startpage", 0.01, calls))], + budget=5.0, hedge_after=0.3, + ) + assert calls.count("ddg") == 1, "the second query must not re-probe a conclusively dead engine" + out = await race_keyless( + [p_engine("ddg", p_silent("ddg", calls)), + p_engine("startpage", p_answers("startpage", 0.01, calls))], + budget=5.0, hedge_after=0.3, + ) + assert any("skipped, still failing" in e for e in out.errors) + + +@pytest.mark.asyncio +async def test_a_recovered_engine_is_used_again(monkeypatch): + """The fix must not quietly blacklist an engine that comes back.""" + import backend.apps.web.tier_breaker as TB + calls = [] + await race_keyless( + [p_engine("ddg", p_silent("ddg", calls)), + p_engine("startpage", p_answers("startpage", 0.01, calls))], + budget=5.0, hedge_after=0.3, + ) + assert tier_cooldown_left("ddg") > 0 + + # Jump past the cooldown rather than sleep through it. + real_monotonic = TB.time.monotonic + monkeypatch.setattr(TB.time, "monotonic", + lambda: real_monotonic() + FIRST_COOLDOWN_SECONDS + 1) + calls.clear() + out = await race_keyless( + [p_engine("ddg", p_answers("ddg", 0.01, calls)), + p_engine("startpage", p_answers("startpage", 0.01, calls))], + budget=5.0, hedge_after=0.3, + ) + assert out.result == {"backend": "ddg"}, "a healthy engine must be used again after the cooldown" + assert tier_cooldown_left("ddg") == 0.0 + + +@pytest.mark.asyncio +async def test_both_empty_reports_no_hits_not_a_failure(): + """A nonsense query must not slowly cool down two perfectly healthy engines.""" + for _ in range(5): + out = await race_keyless( + [p_engine("ddg", p_empty("ddg")), p_engine("startpage", p_empty("startpage"))], + budget=5.0, hedge_after=0.3, + ) + assert out.result is None + assert tier_cooldown_left("ddg") == 0.0 + assert tier_cooldown_left("startpage") == 0.0 + + +@pytest.mark.asyncio +async def test_an_empty_leader_still_lets_the_other_answer(): + out = await race_keyless( + [p_engine("ddg", p_empty("ddg")), + p_engine("startpage", p_answers("startpage", 0.01))], + budget=5.0, hedge_after=2.0, + ) + assert out.result == {"backend": "startpage"} + + +@pytest.mark.asyncio +async def test_every_engine_closed_returns_honestly(): + out = await race_keyless( + [p_engine("ddg", p_raises("ddg")), p_engine("startpage", p_raises("startpage"))], + budget=5.0, hedge_after=0.3, + ) + assert out.result is None + assert len(out.errors) == 2 + + +@pytest.mark.asyncio +async def test_budget_bounds_the_whole_race(): + loop = asyncio.get_running_loop() + t0 = loop.time() + out = await race_keyless( + [p_engine("ddg", p_silent("ddg")), p_engine("startpage", p_silent("startpage"))], + budget=0.6, hedge_after=0.2, + ) + assert out.result is None + assert loop.time() - t0 < 2.0 + + +@pytest.mark.asyncio +async def test_losing_engines_do_not_outlive_the_race(): + """A cancelled request must not keep running and surprise the user's network later.""" + live = {"n": 0} + + async def clingy(): + live["n"] += 1 + try: + await asyncio.sleep(30) + finally: + live["n"] -= 1 + + await race_keyless( + [p_engine("ddg", clingy), p_engine("startpage", p_answers("startpage", 0.01))], + budget=5.0, hedge_after=0.2, + ) + assert live["n"] == 0 diff --git a/backend/tests/test_tier_breaker.py b/backend/tests/test_tier_breaker.py index b2183518..dfbffcac 100644 --- a/backend/tests/test_tier_breaker.py +++ b/backend/tests/test_tier_breaker.py @@ -1,19 +1,15 @@ -"""A dead search frontend must stop costing us its whole tier budget. +"""Engine health: how many failures shut a frontend, and for how long. Measured before this existed: once DuckDuckGo stopped answering TCP, three -consecutive 44-query rounds each paid the full 8s DuckDuckGo budget on EVERY -query, so keyless p50 went 1.0s -> 8.5s while Startpage still served 40/40. -These pin the breaker that makes that unrepresentable, and pin the two ways it -could go wrong instead: skipping a healthy tier, or leaking a per-URL failure -into a fixed-host one. -""" +consecutive 44-query rounds each paid its full 8s budget on EVERY query, so +keyless p50 went 1.0s -> 8.5s while Startpage still served 40/40. -import asyncio +The racing behaviour that keeps a dead engine off the critical path lives in +test_keyless_race.py; this file pins the state machine underneath it. +""" import pytest -import backend.apps.web.cascade as C -from backend.apps.web.cascade import CascadeTier, run_cascade from backend.apps.web.tier_breaker import ( FAILURES_TO_OPEN, FIRST_COOLDOWN_SECONDS, @@ -32,110 +28,55 @@ def p_clean(): reset_tier_health() -@pytest.fixture -def p_tiny_floor(monkeypatch): - monkeypatch.setattr(C, "MIN_TIER_SECONDS", 0.01) - - -async def p_boom(): - raise RuntimeError("engine closed") - - -async def p_hit(): - return {"backend": "second"} - - -def test_streak_opens_then_success_clears(): +def test_an_error_answer_takes_three_strikes(): + """A 202 or a 403 can be a bad minute, so one is not enough to shut an engine out.""" for _ in range(FAILURES_TO_OPEN - 1): record_tier_failure("ddg") assert tier_cooldown_left("ddg") == 0.0 record_tier_failure("ddg") assert 0 < tier_cooldown_left("ddg") <= FIRST_COOLDOWN_SECONDS + + +def test_silence_is_conclusive_on_the_first_failure(): + """A frontend returning nothing at all is not ambiguous, and making the user prove it + three times is what put 8.8s on their first three searches after launch.""" + record_tier_failure("ddg", conclusive=True) + assert 0 < tier_cooldown_left("ddg") <= FIRST_COOLDOWN_SECONDS + + +def test_success_clears_the_streak(): + for _ in range(FAILURES_TO_OPEN): + record_tier_failure("ddg") + assert tier_cooldown_left("ddg") > 0 record_tier_success("ddg") assert tier_cooldown_left("ddg") == 0.0 +def test_a_recovered_engine_is_not_blacklisted_forever(): + """The cooldown must expire on its own, or one bad afternoon costs us an engine for good.""" + record_tier_failure("ddg", now=0.0, conclusive=True) + assert tier_cooldown_left("ddg", now=0.0) == pytest.approx(FIRST_COOLDOWN_SECONDS) + assert tier_cooldown_left("ddg", now=FIRST_COOLDOWN_SECONDS + 1) == 0.0 + record_tier_success("ddg") + for _ in range(FAILURES_TO_OPEN - 1): + record_tier_failure("ddg") + assert tier_cooldown_left("ddg") == 0.0, "a success must reset the streak, not just the clock" + + def test_cooldown_doubles_and_is_capped(): seen = [] now = 0.0 - for round_no in range(12): - for _ in range(FAILURES_TO_OPEN): - record_tier_failure("ddg", now=now) + for _ in range(12): + record_tier_failure("ddg", now=now, conclusive=True) seen.append(tier_cooldown_left("ddg", now=now)) now += seen[-1] + 1 - record_tier_failure("ddg", now=now) # the half-open probe fails again assert seen[0] == pytest.approx(FIRST_COOLDOWN_SECONDS) assert seen[1] > seen[0] assert max(seen) <= MAX_COOLDOWN_SECONDS -@pytest.mark.asyncio -async def test_dead_tier_is_skipped_instantly(p_tiny_floor): - calls = [] - - async def p_slow_boom(): - calls.append(1) - await asyncio.sleep(0.05) - raise RuntimeError("engine closed") - - tiers = [ - CascadeTier(name="ddg", run=p_slow_boom, budget=5.0, breaker=True), - CascadeTier(name="startpage", run=p_hit, budget=5.0, breaker=True), - ] +def test_engines_are_tracked_independently(): for _ in range(FAILURES_TO_OPEN): - out = await run_cascade(tiers, 5.0) - assert out.result == {"backend": "second"} - assert len(calls) == FAILURES_TO_OPEN - - out = await run_cascade(tiers, 5.0) - assert out.result == {"backend": "second"} - assert len(calls) == FAILURES_TO_OPEN, "a cooling tier must not be called at all" - assert any("skipped, still failing" in e for e in out.errors) - - -@pytest.mark.asyncio -async def test_timeout_counts_as_failure(p_tiny_floor): - async def p_hangs(): - await asyncio.sleep(30) - - tiers = [ - CascadeTier(name="ddg", run=p_hangs, budget=0.05, breaker=True), - CascadeTier(name="startpage", run=p_hit, budget=5.0, breaker=True), - ] - for _ in range(FAILURES_TO_OPEN): - await run_cascade(tiers, 5.0) + record_tier_failure("ddg") assert tier_cooldown_left("ddg") > 0 - - -@pytest.mark.asyncio -async def test_no_hits_is_not_a_failure(p_tiny_floor): - """An engine that answers 'nothing matched' is alive; only errors count against it.""" - async def p_empty(): - return None - - tiers = [ - CascadeTier(name="ddg", run=p_empty, budget=5.0, breaker=True), - CascadeTier(name="startpage", run=p_hit, budget=5.0, breaker=True), - ] - for _ in range(FAILURES_TO_OPEN + 2): - await run_cascade(tiers, 5.0) - assert tier_cooldown_left("ddg") == 0.0 - - -@pytest.mark.asyncio -async def test_tiers_without_breaker_always_run(p_tiny_floor): - """A fetch tier fails per-URL, so one dead page must never cool the tier for every other URL.""" - calls = [] - - async def p_fail(): - calls.append(1) - raise RuntimeError("404") - - tiers = [ - CascadeTier(name="local", run=p_fail, budget=5.0), - CascadeTier(name="wayback", run=p_hit, budget=5.0), - ] - for _ in range(FAILURES_TO_OPEN + 3): - await run_cascade(tiers, 5.0) - assert len(calls) == FAILURES_TO_OPEN + 3 - assert tier_cooldown_left("local") == 0.0 + assert tier_cooldown_left("startpage") == 0.0 diff --git a/backend/tests/test_web_search_cascade.py b/backend/tests/test_web_search_cascade.py index 99a64140..2654cf0a 100644 --- a/backend/tests/test_web_search_cascade.py +++ b/backend/tests/test_web_search_cascade.py @@ -213,3 +213,26 @@ async def test_an_honestly_empty_startpage_does_not_count_against_it(monkeypatch out = await search(SearchBody(query="zxqvbnmklwertyuiopasdfg", num_results=5)) assert out["backend"] == "none" assert tier_cooldown_left("startpage") == 0.0 + + +@pytest.mark.asyncio +async def test_a_genuinely_empty_search_does_not_claim_an_outage(monkeypatch): + """Both engines answering 'no matches' is an answer; calling it a refusal sends the model + hunting for an outage that isn't there.""" + ddg_returns(monkeypatch, "") + startpage_returns(monkeypatch, "") + out = await search(SearchBody(query="xyzzyplughnothinghere1234567", num_results=5)) + assert out["backend"] == "none" + assert "had no matches" in out["results"] + assert "refused" not in out["results"] + assert not out.get("cascade_errors") + + +@pytest.mark.asyncio +async def test_a_real_outage_still_says_so(monkeypatch): + ddg_throttled(monkeypatch) + startpage_refuses(monkeypatch) + out = await search(SearchBody(query="capital of Burkina Faso", num_results=5)) + assert out["backend"] == "none" + assert "refused" in out["results"] + assert out["cascade_errors"]