WEBDIENSTE/openswarm
1
0
Fork 0
mirror of https://github.com/openswarm-ai/openswarm.git synced 2026-08-17 18:25:42 +02:00
Code Issues Packages Projects Releases Wiki Activity

[eric] efficiency: preflight classifier stamps cooldown whenever it runs (was re-firing Haiku every turn on concrete prompts); remove warm_prompt_cache (billed N no-op requests/dashboard-mount, warmed nothing: wrong prefix, below cache floor, no cache_control)

Browse Source
This commit is contained in:
ciregenz
2026-07-29 01:01:26 -07:00
parent b1f0f5b74e
commit 8fedff40c1
3 changed files with 4 additions and 82 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/apps/agents/agents.py
+4 -11
View File
@@ -136,8 +136,11 @@ async def send_message(session_id: str, body: dict):
async def p_emit_preflight():
try:
result = await run_preflight(prompt, task_id=session_id)
# Stamp the cooldown whenever the classifier RAN, not only when it suggested:
# a concrete prompt returns no suggestions, so the old placement never throttled
# and the Haiku classifier re-fired on every single turn for the session's life.
p_mcp_suggest_cooldown[session_id] = time.monotonic()
if result.get("suggestions") or result.get("is_vague"):
p_mcp_suggest_cooldown[session_id] = time.monotonic()
await p_ws.send_to_session(session_id, "agent:mcp_suggestions", {
"session_id": session_id,
"suggestions": result.get("suggestions", []),
@@ -341,16 +344,6 @@ async def resume_session(session_id: str):
return {"session": session.model_dump(mode="json")}
@agents.router.post("/sessions/{session_id}/warm-cache")
async def warm_session_cache(session_id: str):
"""Fire a max_tokens=1 dummy request to prime the Anthropic prompt cache; best-effort."""
try:
await agent_manager.warm_prompt_cache(session_id)
except Exception:
pass
return {"ok": True}
@agents.router.post("/sessions/{session_id}/compact")
async def compact_session(session_id: str):
"""Run the summarizer over older turns to free up context.
backend/apps/agents/manager/RunSupport.py
-44
View File
@@ -245,50 +245,6 @@ class RunSupport(AgentManagerProtocol):
async def generate_turn_label(self, session_id: str, turn_id: str, user_prompt: str) -> None:
return await metadata.generate_turn_label(self.sessions.get(session_id), session_id, turn_id, user_prompt)
@typechecked
async def warm_prompt_cache(self, session_id: str) -> None:
"""Pre-warm Anthropic's prompt cache for a session by firing a
max_tokens=1 dummy request through the same agent path. Anthropic
processes the system+tools prefix and writes the cache; the next
real user turn lands a cache hit instead of paying cold-start.
Skips silently if the session doesn't exist, isn't on Anthropic,
or has no Anthropic credentials. Skips if a real request is
already in flight on this session, Anthropic permits parallel
requests but it just wastes the warm.
"""
session = self.sessions.get(session_id)
if not session:
return
# If a real run is in flight, the cache will be warmed by it; firing again is wasted tokens.
existing = self.tasks.get(session_id)
if existing and not existing.done():
return
try:
from backend.apps.agents.providers.registry import find_builtin_model as find_builtin_model
entry = find_builtin_model(session.model)
if not entry or entry.get("api") != "anthropic":
return # other providers handle caching automatically
from backend.apps.settings.credentials import get_anthropic_client
global_settings = load_settings()
# Free lane rotates pool accounts per call, so a warm ping primes a cache the next call won't hit, and worse it'd burn a metered run at idle (this fires on dashboard mount, not a user query). Skip it on the free trial.
if getattr(global_settings, "connection_mode", "own_key") == "free-trial":
return
client = get_anthropic_client(global_settings)
# Single ping with the same system + minimal user message. max_tokens=1 keeps it cheap; we don't care about the output.
await client.messages.create(
model=entry.get("model_id", session.model),
max_tokens=1,
system="You are a helpful assistant. Reply with one character.",
messages=[{"role": "user", "content": "ping"}],
)
logger.debug(f"Cache pre-warm fired for session {session_id}")
except Exception as e:
logger.debug(f"Cache pre-warm failed (non-fatal): {e}")
@typechecked
async def generate_group_meta(self, session_id: str, group_id: str, tool_calls: List[dict], results_summary: Optional[List[str]] = None, is_refinement: bool = False) -> Dict:
return await metadata.generate_group_meta(self.sessions.get(session_id), session_id, group_id, tool_calls, results_summary, is_refinement)
frontend/src/app/pages/Dashboard/hooks/lifecycle/useDashboardLifecycle.ts
-27
View File
@@ -156,34 +156,7 @@ export function useDashboardLifecycle({
? (window as any).requestIdleCallback(loadDeferred, { timeout: 2000 })
: window.setTimeout(loadDeferred, 200);
// Pre-warm Anthropic's prompt cache for sessions on this dashboard ~250ms after mount (debounced; AbortController cancels on dashboard switch). Fires a max_tokens=1 ping per session so the user's first real message hits a warm cache instead of paying cold-start TTFT. Cheap (~$0.0001/session) and non-blocking. Skips for non-Anthropic sessions server-side.
const warmAbort = new AbortController();
const warmTimer = setTimeout(async () => {
try {
const sessionsState = store.getState().agents.sessions;
const dashSessions = Object.values(sessionsState).filter(
(s) => s.dashboard_id === dashboardId &&
s.status !== 'draft' &&
s.mode !== 'browser-agent' &&
s.mode !== 'sub-agent' &&
s.mode !== 'invoked-agent',
);
for (const s of dashSessions) {
if (warmAbort.signal.aborted) break;
// Fire-and-forget, the endpoint always 200s and the side effect is invisible cache population.
fetch(`${API_BASE}/agents/sessions/${s.id}/warm-cache`, {
method: 'POST',
signal: warmAbort.signal,
}).catch(() => {});
}
} catch {
/* best-effort */
}
}, 250);
return () => {
clearTimeout(warmTimer);
warmAbort.abort();
cleanupBrowserHandler();
unsubReconnect();
dashboardWs.disconnect();