"""Web tools: WebSearch and WebFetch.""" from __future__ import annotations import html import re from typing import Any import httpx from backend.apps.agents.tools.base import BaseTool, ToolContext _MAX_OUTPUT_BYTES = 100 * 1024 # ~100 KB _HTTP_TIMEOUT = 30 # seconds _USER_AGENT = ( "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) " "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36" ) def _truncate(text: str, limit: int = _MAX_OUTPUT_BYTES) -> str: if len(text) > limit: return text[:limit] + "\n... (output truncated)" return text def _strip_html(raw_html: str) -> str: """Naive but effective HTML → plain-text conversion.""" # Remove script/style blocks text = re.sub(r"<(script|style)[^>]*>.*?", "", raw_html, flags=re.DOTALL | re.IGNORECASE) # Remove HTML tags text = re.sub(r"<[^>]+>", " ", text) # Decode HTML entities text = html.unescape(text) # Collapse whitespace text = re.sub(r"[ \t]+", " ", text) text = re.sub(r"\n{3,}", "\n\n", text) return text.strip() # ─────────────────────────────────────────────────────────────────────────── # WebSearchTool # ─────────────────────────────────────────────────────────────────────────── class WebSearchTool(BaseTool): name = "WebSearch" description = ( "Search the web using DuckDuckGo and return titles, URLs, and " "snippets for the top results." ) def get_schema(self) -> dict: return { "type": "object", "properties": { "query": { "type": "string", "description": "The search query.", }, "num_results": { "type": "integer", "description": "Maximum number of results to return (default 5).", "default": 5, }, }, "required": ["query"], "additionalProperties": False, } async def execute(self, input_data: dict, context: ToolContext) -> list[dict]: query: str = input_data["query"] num_results: int = input_data.get("num_results", 5) try: results = await self._search_ddg(query, num_results) if not results: return [{"type": "text", "text": f"No search results found for: {query}"}] return [{"type": "text", "text": results}] except Exception as exc: return [{"type": "text", "text": f"Web search error: {exc}"}] @staticmethod async def _search_ddg(query: str, num_results: int) -> str: """Query DuckDuckGo HTML endpoint and parse results.""" async with httpx.AsyncClient( timeout=_HTTP_TIMEOUT, follow_redirects=True, headers={"User-Agent": _USER_AGENT}, ) as client: resp = await client.post( "https://html.duckduckgo.com/html/", data={"q": query}, ) resp.raise_for_status() body = resp.text # Parse result blocks – DuckDuckGo wraps each result in #
...
result_blocks = re.findall( r']*class="[^"]*result[^"]*"[^>]*>(.*?)\s*(?=]*class="[^"]*result|$)', body, flags=re.DOTALL, ) entries: list[str] = [] for block in result_blocks: if len(entries) >= num_results: break # Title + URL — handle both class-before-href and href-before-class link_match = re.search( r']*class="[^"]*result__a[^"]*"[^>]*href="([^"]*)"[^>]*>(.*?)', block, flags=re.DOTALL, ) if not link_match: # Try reversed attribute order link_match = re.search( r']*href="([^"]*)"[^>]*class="[^"]*result__a[^"]*"[^>]*>(.*?)', block, flags=re.DOTALL, ) if not link_match: continue raw_url = html.unescape(link_match.group(1)) title = _strip_html(link_match.group(2)).strip() # Snippet snippet_match = re.search( r']*class="[^"]*result__snippet[^"]*"[^>]*>(.*?)', block, flags=re.DOTALL, ) snippet = _strip_html(snippet_match.group(1)).strip() if snippet_match else "" # DuckDuckGo wraps URLs through a redirect; try to extract the real URL real_url_match = re.search(r"uddg=([^&]+)", raw_url) if real_url_match: from urllib.parse import unquote url = unquote(real_url_match.group(1)) else: url = raw_url entry = f"[{len(entries) + 1}] {title}\n {url}" if snippet: entry += f"\n {snippet}" entries.append(entry) return "\n\n".join(entries) # ─────────────────────────────────────────────────────────────────────────── # WebFetchTool # ─────────────────────────────────────────────────────────────────────────── class WebFetchTool(BaseTool): name = "WebFetch" description = ( "Fetch the contents of a URL and return the extracted text. " "HTML is stripped to plain text. Output is truncated to ~100 KB." ) def get_schema(self) -> dict: return { "type": "object", "properties": { "url": { "type": "string", "description": "The URL to fetch.", }, "prompt": { "type": "string", "description": "Optional prompt/context describing what information to look for.", }, }, "required": ["url"], "additionalProperties": False, } async def execute(self, input_data: dict, context: ToolContext) -> list[dict]: url: str = input_data["url"] prompt: str | None = input_data.get("prompt") try: async with httpx.AsyncClient( timeout=_HTTP_TIMEOUT, follow_redirects=True, headers={"User-Agent": _USER_AGENT}, ) as client: resp = await client.get(url) resp.raise_for_status() except httpx.HTTPStatusError as exc: return [{"type": "text", "text": f"HTTP error {exc.response.status_code} fetching {url}"}] except Exception as exc: return [{"type": "text", "text": f"Error fetching {url}: {exc}"}] content_type = resp.headers.get("content-type", "") if "html" in content_type or resp.text.strip().startswith("