Update notebook

examples/web-navigation/web_voyager.ipynb

@@ -122,6 +122,9 @@
     "class BBox(TypedDict):\n",
     "    x: float\n",
     "    y: float\n",
+    "    text: str\n",
+    "    type: str\n",
+    "    ariaLabel: str\n",
     "\n",
     "\n",
     "class Prediction(TypedDict):\n",
@@ -352,6 +355,18 @@
     "    return {**state, **marked_page}\n",
     "\n",
     "\n",
+    "def format_descriptions(state):\n",
+    "    labels = []\n",
+    "    for i, bbox in enumerate(state[\"bboxes\"]):\n",
+    "        text = bbox.get(\"ariaLabel\") or \"\"\n",
+    "        if not text.strip():\n",
+    "            text = bbox[\"text\"]\n",
+    "        el_type = bbox.get(\"type\")\n",
+    "        labels.append(f'{i} (<{el_type}/>): \"{text}\"')\n",
+    "    bbox_descriptions = \"\\nValid Bounding Boxes:\\n\" + \"\\n\".join(labels)\n",
+    "    return {**state, \"bbox_descriptions\": bbox_descriptions}\n",
+    "\n",
+    "\n",
     "def parse(text: str) -> dict:\n",
     "    action_prefix = \"Action: \"\n",
     "    if not text.strip().split(\"\\n\")[-1].startswith(action_prefix):\n",
@@ -386,7 +401,7 @@
    "source": [
     "llm = ChatOpenAI(model=\"gpt-4-vision-preview\", max_tokens=4096)\n",
     "agent = annotate | RunnablePassthrough.assign(\n",
-    "    prediction=prompt | llm | StrOutputParser() | parse\n",
+    "    prediction=format_descriptions | prompt | llm | StrOutputParser() | parse\n",
     ")"
    ]
   },