[eric] browser: prune stale screenshots + capture downscaled JPEG to cut agent context cost

backend/apps/agents/browser/browser_agent.py

@@ -103,7 +103,7 @@ def _format_tool_result(result: dict, tool_name: str) -> list[dict]:
                 "type": "image",
                 "source": {
                     "type": "base64",
-                    "media_type": "image/png",
+                    "media_type": result.get("image_mime", "image/png"),
                     "data": result["image"],
                 },
             },
@@ -587,6 +587,11 @@ async def run_browser_agent(
             if cancel_event.is_set():
                 break
 
+            # Drop stale screenshots before each call: keep first + previous +
+            # current, stub the rest. Images are ~1.3-2k tokens each and get
+            # re-read every turn, so this is the biggest per-turn context win on
+            # any visual task (measured ~2.9x fewer image tokens, ~5x less upload).
+            browser_history.prune_old_screenshots(messages)
             response = await _cancellable(client.messages.create(
                 model=api_model,
                 max_tokens=4096,

backend/apps/agents/browser/browser_history.py

@@ -40,6 +40,60 @@ def clear_browser_history(browser_id: str) -> None:
     _browser_history.pop(browser_id, None)
 
 
+_OMITTED_SCREENSHOT_STUB = "[earlier screenshot omitted to save context]"
+
+
+def _iter_image_block_refs(messages: list[dict]):
+    """Yield (container_list, index) for every image block, in document order.
+
+    Screenshots live either directly in a message's content list or nested inside
+    a tool_result block's content list; handle both so nothing slips through.
+    """
+    for msg in messages:
+        content = msg.get("content")
+        if not isinstance(content, list):
+            continue
+        for i, block in enumerate(content):
+            if not isinstance(block, dict):
+                continue
+            if block.get("type") == "image":
+                yield (content, i)
+            elif block.get("type") == "tool_result" and isinstance(block.get("content"), list):
+                for j, inner in enumerate(block["content"]):
+                    if isinstance(inner, dict) and inner.get("type") == "image":
+                        yield (block["content"], j)
+
+
+def prune_old_screenshots(messages: list[dict], keep_first: bool = True, keep_recent: int = 2) -> int:
+    """Collapse stale screenshot images to a one-line text stub, in place.
+
+    A vision image is ~1.3-2k tokens and the model re-reads EVERY one on EVERY
+    turn, so a task that screenshots a handful of times quietly re-prefills them
+    all each loop (measured ~2.9x the image tokens, ~5x the bytes uploaded per
+    turn). We keep only the orientation anchor (first) plus the `keep_recent` most
+    recent shots (previous + current) and swap the rest for a marker; the URL and
+    the agent's own ReportProgress already carry where/what, so only the pixels
+    are dropped, not the memory. If the agent must re-see, it just re-screenshots.
+    Returns how many images were collapsed.
+    """
+    refs = list(_iter_image_block_refs(messages))
+    keep_count = keep_recent + (1 if keep_first else 0)
+    if len(refs) <= keep_count:
+        return 0
+    keep: set[int] = set()
+    if keep_first:
+        keep.add(0)
+    for k in range(1, keep_recent + 1):
+        keep.add(len(refs) - k)
+    collapsed = 0
+    for idx, (container, i) in enumerate(refs):
+        if idx in keep:
+            continue
+        container[i] = {"type": "text", "text": _OMITTED_SCREENSHOT_STUB}
+        collapsed += 1
+    return collapsed
+
+
 def _validate_message_pairing(messages: list[dict]) -> bool:
     """Verify every tool_result references a tool_use_id from a prior assistant
     message in the same list. Returns False if there's an orphan, which means

backend/apps/agents/browser_agent_mcp_server.py

@@ -156,6 +156,16 @@ def call_backend(tasks: list[dict]) -> dict:
 MAX_IMAGE_B64_BYTES = 400_000
 
 
+def _sniff_image_mime(b64: str) -> str:
+    """PNG vs JPEG from the base64 magic bytes. Capture now sends JPEG, but older
+    callers / cached shots may be PNG, so we label by content, not assumption."""
+    if b64.startswith("/9j/"):
+        return "image/jpeg"
+    if b64.startswith("iVBORw0KGgo"):
+        return "image/png"
+    return "image/png"
+
+
 def compress_screenshot(b64_png: str) -> tuple[str, str] | None:
     """Resize and re-encode as JPEG to stay under the stdio buffer limit."""
     if not HAS_PIL:
@@ -204,7 +214,7 @@ def format_result(result: dict) -> dict:
     screenshot = result.get("final_screenshot")
     if screenshot:
         image_data = screenshot
-        mime_type = "image/png"
+        mime_type = _sniff_image_mime(screenshot)
 
         if len(image_data) > MAX_IMAGE_B64_BYTES:
             compressed = compress_screenshot(image_data)

backend/tests/test_browser_screenshot_prune.py

@@ -0,0 +1,81 @@
+"""Screenshot pruning: keep first + previous + current, collapse the rest.
+
+Vision images are ~1.3-2k tokens each and the model re-reads every one on every
+turn; pruning to 3 anchors (first/previous/current) and stubbing the rest cuts the
+re-prefilled image tokens without losing the agent's memory (URL + ReportProgress
+text stay). These pin the keep-set, the in-place mutation, and tool_result safety.
+"""
+
+from backend.apps.agents.browser.browser_history import (
+    prune_old_screenshots,
+    _OMITTED_SCREENSHOT_STUB,
+)
+
+
+def _img(tag):
+    return {"type": "image", "source": {"type": "base64", "media_type": "image/jpeg", "data": tag}}
+
+
+def _shot_turn(tag, url):
+    # mirrors _format_tool_result for BrowserScreenshot: [image, text(url)]
+    return {"role": "user", "content": [{
+        "type": "tool_result", "tool_use_id": f"t_{tag}",
+        "content": [_img(tag), {"type": "text", "text": f"Screenshot captured. URL: {url}"}],
+    }]}
+
+
+def _count_images(messages):
+    n = 0
+    for m in messages:
+        for b in m.get("content", []):
+            if isinstance(b, dict):
+                if b.get("type") == "image":
+                    n += 1
+                elif b.get("type") == "tool_result":
+                    n += sum(1 for x in b.get("content", []) if isinstance(x, dict) and x.get("type") == "image")
+    return n
+
+
+def test_keeps_first_and_last_two_collapses_middle():
+    msgs = [_shot_turn(str(i), f"https://site/{i}") for i in range(5)]  # images 0..4
+    collapsed = prune_old_screenshots(msgs)
+    assert collapsed == 2  # 5 images - (first + last 2) = 2 stubbed
+    assert _count_images(msgs) == 3
+    # image tags that survive are 0 (first), 3 and 4 (last two)
+    surviving = [b["source"]["data"] for m in msgs for tr in m["content"]
+                 for b in tr["content"] if b.get("type") == "image"]
+    assert surviving == ["0", "3", "4"]
+
+
+def test_three_or_fewer_is_a_noop():
+    msgs = [_shot_turn(str(i), f"u{i}") for i in range(3)]
+    assert prune_old_screenshots(msgs) == 0
+    assert _count_images(msgs) == 3
+
+
+def test_stub_preserves_the_url_text_block():
+    msgs = [_shot_turn(str(i), f"https://site/{i}") for i in range(4)]
+    prune_old_screenshots(msgs)
+    # the collapsed shot (#1) keeps its "URL:" text, only the image became a stub
+    collapsed_tr = msgs[1]["content"][0]["content"]
+    assert any(b.get("text") == _OMITTED_SCREENSHOT_STUB for b in collapsed_tr)
+    assert any("URL: https://site/1" in b.get("text", "") for b in collapsed_tr)
+
+
+def test_handles_direct_image_blocks_too():
+    msgs = [
+        {"role": "user", "content": [_img("a"), {"type": "text", "text": "hi"}]},
+        {"role": "user", "content": [_img("b")]},
+        {"role": "user", "content": [_img("c")]},
+        {"role": "user", "content": [_img("d")]},
+    ]
+    collapsed = prune_old_screenshots(msgs)
+    assert collapsed == 1  # keep a (first), c+d (last two); stub b
+    assert msgs[1]["content"][0] == {"type": "text", "text": _OMITTED_SCREENSHOT_STUB}
+
+
+def test_keep_recent_is_tunable():
+    msgs = [_shot_turn(str(i), f"u{i}") for i in range(6)]
+    prune_old_screenshots(msgs, keep_first=False, keep_recent=1)
+    # only the most recent survives
+    assert _count_images(msgs) == 1

frontend/src/shared/browserCommandHandler.ts

@@ -67,9 +67,23 @@ async function handleScreenshot(wv: BrowserWebview): Promise<Record<string, any>
     try {
       const nativeImage = await wv.capturePage();
       if (!nativeImage.isEmpty()) {
-        const dataUrl = nativeImage.toDataURL();
-        const base64 = dataUrl.replace(/^data:image\/\w+;base64,/, '');
-        return { image: base64, url: wv.getURL(), title: wv.getTitle() };
+        // Send a downscaled JPEG, not a full-res PNG: on real pages JPEG cuts the
+        // wire/upload bytes ~10x (the model reads images by dimensions, so this is
+        // a network + memory win), and capping near 1280 actual px keeps text
+        // legible while trimming tokens a little. Native ops, sub-10ms.
+        // Electron-42 retina gotchas, verified empirically: toJPEG() on a raw
+        // scaleFactor-2 capture returns an EMPTY image, and resize({width}) emits
+        // `width` ACTUAL pixels at scaleFactor 1, EXCEPT resizing to the source's
+        // own logical width is a no-op that leaves it retina (and unencodable). So
+        // we ALWAYS resize to a distinct width to force a clean scaleFactor-1 image.
+        const TARGET_W = 1280;
+        const dpr = (typeof window !== 'undefined' && window.devicePixelRatio) || 1;
+        const { width: dipW } = nativeImage.getSize();
+        const backingW = Math.round(dipW * dpr);
+        let target = Math.min(TARGET_W, backingW);
+        if (target === dipW) target = Math.max(1, target - 1); // dodge the retina no-op
+        const base64 = nativeImage.resize({ width: target, quality: 'good' }).toJPEG(72).toString('base64');
+        return { image: base64, image_mime: 'image/jpeg', url: wv.getURL(), title: wv.getTitle() };
       }
       lastErr = new Error('capturePage returned an empty image (frame not painted yet)');
     } catch (err: any) {

frontend/src/shared/browserRegistry.ts

@@ -1,3 +1,18 @@
+// Subset of Electron's NativeImage we actually call. resize() returns another
+// NativeImage, hence the self-reference.
+export interface ElectronNativeImage {
+  toDataURL: () => string;
+  toPNG: () => Buffer;
+  toJPEG: (quality: number) => Buffer;
+  isEmpty: () => boolean;
+  getSize: () => { width: number; height: number };
+  resize: (options: {
+    width?: number;
+    height?: number;
+    quality?: 'good' | 'better' | 'best';
+  }) => ElectronNativeImage;
+}
+
 export interface BrowserWebview extends HTMLElement {
   src: string;
   loadURL: (url: string) => Promise<void>;
@@ -8,11 +23,7 @@ export interface BrowserWebview extends HTMLElement {
   canGoForward: () => boolean;
   getURL: () => string;
   getTitle: () => string;
-  capturePage: (rect?: { x: number; y: number; width: number; height: number }) => Promise<{
-    toDataURL: () => string;
-    toPNG: () => Buffer;
-    isEmpty: () => boolean;
-  }>;
+  capturePage: (rect?: { x: number; y: number; width: number; height: number }) => Promise<ElectronNativeImage>;
   executeJavaScript: (code: string) => Promise<any>;
   sendInputEvent: (event: any) => void;
   getWebContentsId: () => number;