From 8fedff40c13d6e58e367d9d745923d7bf7cdb38f Mon Sep 17 00:00:00 2001 From: ciregenz Date: Wed, 29 Jul 2026 01:01:26 -0700 Subject: [PATCH] [eric] efficiency: preflight classifier stamps cooldown whenever it runs (was re-firing Haiku every turn on concrete prompts); remove warm_prompt_cache (billed N no-op requests/dashboard-mount, warmed nothing: wrong prefix, below cache floor, no cache_control) --- backend/apps/agents/agents.py | 15 ++----- backend/apps/agents/manager/RunSupport.py | 44 ------------------- .../hooks/lifecycle/useDashboardLifecycle.ts | 27 ------------ 3 files changed, 4 insertions(+), 82 deletions(-) diff --git a/backend/apps/agents/agents.py b/backend/apps/agents/agents.py index e5e03391..8c40003f 100644 --- a/backend/apps/agents/agents.py +++ b/backend/apps/agents/agents.py @@ -136,8 +136,11 @@ async def send_message(session_id: str, body: dict): async def p_emit_preflight(): try: result = await run_preflight(prompt, task_id=session_id) + # Stamp the cooldown whenever the classifier RAN, not only when it suggested: + # a concrete prompt returns no suggestions, so the old placement never throttled + # and the Haiku classifier re-fired on every single turn for the session's life. + p_mcp_suggest_cooldown[session_id] = time.monotonic() if result.get("suggestions") or result.get("is_vague"): - p_mcp_suggest_cooldown[session_id] = time.monotonic() await p_ws.send_to_session(session_id, "agent:mcp_suggestions", { "session_id": session_id, "suggestions": result.get("suggestions", []), @@ -341,16 +344,6 @@ async def resume_session(session_id: str): return {"session": session.model_dump(mode="json")} -@agents.router.post("/sessions/{session_id}/warm-cache") -async def warm_session_cache(session_id: str): - """Fire a max_tokens=1 dummy request to prime the Anthropic prompt cache; best-effort.""" - try: - await agent_manager.warm_prompt_cache(session_id) - except Exception: - pass - return {"ok": True} - - @agents.router.post("/sessions/{session_id}/compact") async def compact_session(session_id: str): """Run the summarizer over older turns to free up context. diff --git a/backend/apps/agents/manager/RunSupport.py b/backend/apps/agents/manager/RunSupport.py index f4def84d..774e0a46 100644 --- a/backend/apps/agents/manager/RunSupport.py +++ b/backend/apps/agents/manager/RunSupport.py @@ -245,50 +245,6 @@ class RunSupport(AgentManagerProtocol): async def generate_turn_label(self, session_id: str, turn_id: str, user_prompt: str) -> None: return await metadata.generate_turn_label(self.sessions.get(session_id), session_id, turn_id, user_prompt) - @typechecked - async def warm_prompt_cache(self, session_id: str) -> None: - """Pre-warm Anthropic's prompt cache for a session by firing a - max_tokens=1 dummy request through the same agent path. Anthropic - processes the system+tools prefix and writes the cache; the next - real user turn lands a cache hit instead of paying cold-start. - - Skips silently if the session doesn't exist, isn't on Anthropic, - or has no Anthropic credentials. Skips if a real request is - already in flight on this session, Anthropic permits parallel - requests but it just wastes the warm. - """ - session = self.sessions.get(session_id) - if not session: - return - # If a real run is in flight, the cache will be warmed by it; firing again is wasted tokens. - existing = self.tasks.get(session_id) - if existing and not existing.done(): - return - - try: - from backend.apps.agents.providers.registry import find_builtin_model as find_builtin_model - entry = find_builtin_model(session.model) - if not entry or entry.get("api") != "anthropic": - return # other providers handle caching automatically - - from backend.apps.settings.credentials import get_anthropic_client - global_settings = load_settings() - # Free lane rotates pool accounts per call, so a warm ping primes a cache the next call won't hit, and worse it'd burn a metered run at idle (this fires on dashboard mount, not a user query). Skip it on the free trial. - if getattr(global_settings, "connection_mode", "own_key") == "free-trial": - return - client = get_anthropic_client(global_settings) - - # Single ping with the same system + minimal user message. max_tokens=1 keeps it cheap; we don't care about the output. - await client.messages.create( - model=entry.get("model_id", session.model), - max_tokens=1, - system="You are a helpful assistant. Reply with one character.", - messages=[{"role": "user", "content": "ping"}], - ) - logger.debug(f"Cache pre-warm fired for session {session_id}") - except Exception as e: - logger.debug(f"Cache pre-warm failed (non-fatal): {e}") - @typechecked async def generate_group_meta(self, session_id: str, group_id: str, tool_calls: List[dict], results_summary: Optional[List[str]] = None, is_refinement: bool = False) -> Dict: return await metadata.generate_group_meta(self.sessions.get(session_id), session_id, group_id, tool_calls, results_summary, is_refinement) diff --git a/frontend/src/app/pages/Dashboard/hooks/lifecycle/useDashboardLifecycle.ts b/frontend/src/app/pages/Dashboard/hooks/lifecycle/useDashboardLifecycle.ts index 94275855..fa4520b0 100644 --- a/frontend/src/app/pages/Dashboard/hooks/lifecycle/useDashboardLifecycle.ts +++ b/frontend/src/app/pages/Dashboard/hooks/lifecycle/useDashboardLifecycle.ts @@ -156,34 +156,7 @@ export function useDashboardLifecycle({ ? (window as any).requestIdleCallback(loadDeferred, { timeout: 2000 }) : window.setTimeout(loadDeferred, 200); - // Pre-warm Anthropic's prompt cache for sessions on this dashboard ~250ms after mount (debounced; AbortController cancels on dashboard switch). Fires a max_tokens=1 ping per session so the user's first real message hits a warm cache instead of paying cold-start TTFT. Cheap (~$0.0001/session) and non-blocking. Skips for non-Anthropic sessions server-side. - const warmAbort = new AbortController(); - const warmTimer = setTimeout(async () => { - try { - const sessionsState = store.getState().agents.sessions; - const dashSessions = Object.values(sessionsState).filter( - (s) => s.dashboard_id === dashboardId && - s.status !== 'draft' && - s.mode !== 'browser-agent' && - s.mode !== 'sub-agent' && - s.mode !== 'invoked-agent', - ); - for (const s of dashSessions) { - if (warmAbort.signal.aborted) break; - // Fire-and-forget, the endpoint always 200s and the side effect is invisible cache population. - fetch(`${API_BASE}/agents/sessions/${s.id}/warm-cache`, { - method: 'POST', - signal: warmAbort.signal, - }).catch(() => {}); - } - } catch { - /* best-effort */ - } - }, 250); - return () => { - clearTimeout(warmTimer); - warmAbort.abort(); cleanupBrowserHandler(); unsubReconnect(); dashboardWs.disconnect();