mirror of
https://github.com/openswarm-ai/openswarm.git
synced 2026-08-24 13:32:22 +02:00
81 lines
3.0 KiB
Python
81 lines
3.0 KiB
Python
"""Short-lived circuit breaker for the cascade's fixed-host tiers.
|
|
|
|
Measured on this machine over three 44-query rounds: DuckDuckGo answered the
|
|
first 7 searches, then served its bot challenge, then 403'd, then stopped
|
|
answering TCP altogether. Once that happened EVERY search paid the full 8s
|
|
DuckDuckGo tier budget before Startpage answered, so the keyless p50 went from
|
|
1.0s to 8.5s while the success rate stayed at 100%. The engine wasn't broken,
|
|
our retrying of a known-dead engine was.
|
|
|
|
A tier only opts in when "closed" is a property of the HOST rather than of the
|
|
request, which is true for a search frontend and false for a page fetch (one
|
|
404 says nothing about the next URL). State is per-process and time-boxed, so
|
|
the worst a wrong guess costs is one cooldown window of a tier we skip.
|
|
"""
|
|
|
|
import time
|
|
from typing import Dict, Optional
|
|
|
|
from pydantic import BaseModel, ConfigDict
|
|
from typeguard import typechecked
|
|
|
|
# Two failures can be one bad minute; three in a row is a closed door.
|
|
FAILURES_TO_OPEN = 3
|
|
FIRST_COOLDOWN_SECONDS = 120.0
|
|
MAX_COOLDOWN_SECONDS = 900.0
|
|
|
|
|
|
class TierHealth(BaseModel):
|
|
model_config = ConfigDict(validate_assignment=True)
|
|
|
|
consecutive_failures: int = 0
|
|
open_until: float = 0.0
|
|
cooldown: float = 0.0
|
|
|
|
|
|
p_health: Dict[str, TierHealth] = {}
|
|
|
|
|
|
@typechecked
|
|
def p_entry(name: str) -> TierHealth:
|
|
if name not in p_health:
|
|
p_health[name] = TierHealth()
|
|
return p_health[name]
|
|
|
|
|
|
@typechecked
|
|
def tier_cooldown_left(name: str, now: Optional[float] = None) -> float:
|
|
"""Seconds until this tier is worth trying again; 0 when it is open for business."""
|
|
entry = p_health.get(name)
|
|
if entry is None:
|
|
return 0.0
|
|
return max(0.0, entry.open_until - (time.monotonic() if now is None else now))
|
|
|
|
|
|
@typechecked
|
|
def record_tier_failure(name: str, now: Optional[float] = None, *, conclusive: bool = False) -> None:
|
|
"""A failure. An error ANSWER is one strike of three, because a 202 or a 403 can be a bad
|
|
minute. SILENCE is conclusive and shuts the tier at once: a frontend that returns nothing at
|
|
all in the time a healthy one answers three times over is not having a bad minute, and making
|
|
the user prove it three times is what put 8.8s on their first three searches after launch."""
|
|
stamp = time.monotonic() if now is None else now
|
|
entry = p_entry(name)
|
|
entry.consecutive_failures += 1
|
|
if not conclusive and entry.consecutive_failures < FAILURES_TO_OPEN:
|
|
return
|
|
# The half-open probe that fails again doubles the wait, so a permanently dead engine stops costing anything.
|
|
entry.cooldown = min(max(entry.cooldown * 2, FIRST_COOLDOWN_SECONDS), MAX_COOLDOWN_SECONDS)
|
|
entry.open_until = stamp + entry.cooldown
|
|
|
|
|
|
@typechecked
|
|
def record_tier_success(name: str) -> None:
|
|
"""The tier answered, even if the answer was 'no hits'. It is alive; forget the history."""
|
|
if name in p_health:
|
|
p_health[name] = TierHealth()
|
|
|
|
|
|
@typechecked
|
|
def reset_tier_health() -> None:
|
|
p_health.clear()
|