From 38eb16f131b4bbb314c7aedc0a067706b5efafa5 Mon Sep 17 00:00:00 2001
From: William Fu-Hinthorn <13333726+hinthornw@users.noreply.github.com>
Date: Tue, 6 Feb 2024 06:35:27 -0800
Subject: [PATCH] Update notebook

---
 examples/web-navigation/web_voyager.ipynb | 17 ++++++++++++++++-
 1 file changed, 16 insertions(+), 1 deletion(-)

diff --git a/examples/web-navigation/web_voyager.ipynb b/examples/web-navigation/web_voyager.ipynb
index 87a6fbc82..dd6a17435 100644
--- a/examples/web-navigation/web_voyager.ipynb
+++ b/examples/web-navigation/web_voyager.ipynb
@@ -122,6 +122,9 @@
     "class BBox(TypedDict):\n",
     "    x: float\n",
     "    y: float\n",
+    "    text: str\n",
+    "    type: str\n",
+    "    ariaLabel: str\n",
     "\n",
     "\n",
     "class Prediction(TypedDict):\n",
@@ -352,6 +355,18 @@
     "    return {**state, **marked_page}\n",
     "\n",
     "\n",
+    "def format_descriptions(state):\n",
+    "    labels = []\n",
+    "    for i, bbox in enumerate(state[\"bboxes\"]):\n",
+    "        text = bbox.get(\"ariaLabel\") or \"\"\n",
+    "        if not text.strip():\n",
+    "            text = bbox[\"text\"]\n",
+    "        el_type = bbox.get(\"type\")\n",
+    "        labels.append(f'{i} (<{el_type}/>): \"{text}\"')\n",
+    "    bbox_descriptions = \"\\nValid Bounding Boxes:\\n\" + \"\\n\".join(labels)\n",
+    "    return {**state, \"bbox_descriptions\": bbox_descriptions}\n",
+    "\n",
+    "\n",
     "def parse(text: str) -> dict:\n",
     "    action_prefix = \"Action: \"\n",
     "    if not text.strip().split(\"\\n\")[-1].startswith(action_prefix):\n",
@@ -386,7 +401,7 @@
    "source": [
     "llm = ChatOpenAI(model=\"gpt-4-vision-preview\", max_tokens=4096)\n",
     "agent = annotate | RunnablePassthrough.assign(\n",
-    "    prediction=prompt | llm | StrOutputParser() | parse\n",
+    "    prediction=format_descriptions | prompt | llm | StrOutputParser() | parse\n",
     ")"
    ]
   },