WEBDIENSTE/openswarm
1
0
Fork 0
mirror of https://github.com/openswarm-ai/openswarm.git synced 2026-09-30 13:34:50 +02:00
Code Issues Packages Projects Releases Wiki Activity

[eric] browser: numbered index overlays on screenshots, vision and clicks share one index language

Browse Source
This commit is contained in:
ciregenz
2026-06-05 11:03:20 -07:00
parent 4c5962d8ac
commit 0992e25bf2
2 changed files with 100 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/apps/agents/browser/browser_schema.py
+13 -2
View File
@@ -71,11 +71,22 @@ BROWSER_TOOLS_SCHEMA = [
"name": "BrowserScreenshot",
"description": (
"Capture a screenshot of the browser page. Returns the screenshot as a "
"base64-encoded PNG image. Use this to see what is currently displayed."
"base64-encoded PNG image. Use this to see what is currently displayed. "
"Elements from your last BrowserListInteractives come back with numbered "
"colored boxes drawn on them, the same numbers you click with, so you can "
"go straight from what you see to BrowserClickIndex."
),
"input_schema": {
"type": "object",
"properties": {},
"properties": {
"annotate": {
"type": "boolean",
"description": (
"Default true. Pass false for a clean, unannotated shot, e.g. "
"when capturing proof of a completed action."
),
},
},
"required": [],
},
},