mirror of
https://github.com/langchain-ai/langgraph.git
synced 2026-10-25 01:11:35 +02:00
Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Lance Martin
|
d1b142212d |
@@ -0,0 +1,600 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "99484f34-fde4-4df5-ae5c-f547489d5943",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# Chat LangGraph\n",
|
||||
"\n",
|
||||
"Deep dive on composition of LangGraph docs to inform indexing.\n",
|
||||
"\n",
|
||||
"## Index Design\n",
|
||||
"\n",
|
||||
"### Documentation Loading & Analysis\n",
|
||||
"\n",
|
||||
"Follow what [is done](https://github.com/langchain-ai/chat-langchain/blob/master/backend/ingest.py) in `ChatLangChain`."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 1,
|
||||
"id": "a08800b4-c24f-425c-9df4-c4132fc1adf4",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"USER_AGENT environment variable not set, consider setting it to identify your requests.\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"252"
|
||||
]
|
||||
},
|
||||
"execution_count": 1,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"import re\n",
|
||||
"import asyncio\n",
|
||||
"import nest_asyncio\n",
|
||||
"from typing import Generator\n",
|
||||
"from bs4 import BeautifulSoup, SoupStrainer\n",
|
||||
"from bs4 import BeautifulSoup, Doctype, NavigableString, Tag\n",
|
||||
"from langchain.document_loaders import RecursiveUrlLoader, SitemapLoader\n",
|
||||
"\n",
|
||||
"%matplotlib inline\n",
|
||||
"\n",
|
||||
"def metadata_extractor(html: str, url: str) -> dict:\n",
|
||||
" soup = BeautifulSoup(html, \"lxml\")\n",
|
||||
" title = soup.find(\"title\")\n",
|
||||
" description = soup.find(\"meta\", attrs={\"name\": \"description\"})\n",
|
||||
" html = soup.find(\"html\")\n",
|
||||
" return {\n",
|
||||
" \"source\": url,\n",
|
||||
" \"title\": title.get_text() if title else \"\",\n",
|
||||
" \"description\": description.get(\"content\", \"\") if description else \"\",\n",
|
||||
" \"language\": html.get(\"lang\", \"\") if html else \"\",\n",
|
||||
" }\n",
|
||||
"\n",
|
||||
"def langchain_docs_extractor(html: str) -> str:\n",
|
||||
" soup = BeautifulSoup(html, \"lxml\")\n",
|
||||
" # Remove all the tags that are not meaningful for the extraction.\n",
|
||||
" SCAPE_TAGS = [\"nav\", \"footer\", \"aside\", \"script\", \"style\"]\n",
|
||||
" [tag.decompose() for tag in soup.find_all(SCAPE_TAGS)]\n",
|
||||
"\n",
|
||||
" def get_text(tag: Tag) -> Generator[str, None, None]:\n",
|
||||
" for child in tag.children:\n",
|
||||
" if isinstance(child, Doctype):\n",
|
||||
" continue\n",
|
||||
"\n",
|
||||
" if isinstance(child, NavigableString):\n",
|
||||
" yield child\n",
|
||||
" elif isinstance(child, Tag):\n",
|
||||
" if child.name in [\"h1\", \"h2\", \"h3\", \"h4\", \"h5\", \"h6\"]:\n",
|
||||
" yield f\"{'#' * int(child.name[1:])} {child.get_text()}\\n\\n\"\n",
|
||||
" elif child.name == \"a\":\n",
|
||||
" yield f\"[{child.get_text(strip=False)}]({child.get('href')})\"\n",
|
||||
" elif child.name == \"img\":\n",
|
||||
" yield f\"})\"\n",
|
||||
" elif child.name in [\"strong\", \"b\"]:\n",
|
||||
" yield f\"**{child.get_text(strip=False)}**\"\n",
|
||||
" elif child.name in [\"em\", \"i\"]:\n",
|
||||
" yield f\"_{child.get_text(strip=False)}_\"\n",
|
||||
" elif child.name == \"br\":\n",
|
||||
" yield \"\\n\"\n",
|
||||
" elif child.name == \"code\":\n",
|
||||
" parent = child.find_parent()\n",
|
||||
" if parent is not None and parent.name == \"pre\":\n",
|
||||
" classes = parent.attrs.get(\"class\", \"\")\n",
|
||||
"\n",
|
||||
" language = next(\n",
|
||||
" filter(lambda x: re.match(r\"language-\\w+\", x), classes),\n",
|
||||
" None,\n",
|
||||
" )\n",
|
||||
" if language is None:\n",
|
||||
" language = \"\"\n",
|
||||
" else:\n",
|
||||
" language = language.split(\"-\")[1]\n",
|
||||
"\n",
|
||||
" lines: list[str] = []\n",
|
||||
" for span in child.find_all(\"span\", class_=\"token-line\"):\n",
|
||||
" line_content = \"\".join(\n",
|
||||
" token.get_text() for token in span.find_all(\"span\")\n",
|
||||
" )\n",
|
||||
" lines.append(line_content)\n",
|
||||
"\n",
|
||||
" code_content = \"\\n\".join(lines)\n",
|
||||
" yield f\"```{language}\\n{code_content}\\n```\\n\\n\"\n",
|
||||
" else:\n",
|
||||
" yield f\"`{child.get_text(strip=False)}`\"\n",
|
||||
"\n",
|
||||
" elif child.name == \"p\":\n",
|
||||
" yield from get_text(child)\n",
|
||||
" yield \"\\n\\n\"\n",
|
||||
" elif child.name == \"ul\":\n",
|
||||
" for li in child.find_all(\"li\", recursive=False):\n",
|
||||
" yield \"- \"\n",
|
||||
" yield from get_text(li)\n",
|
||||
" yield \"\\n\\n\"\n",
|
||||
" elif child.name == \"ol\":\n",
|
||||
" for i, li in enumerate(child.find_all(\"li\", recursive=False)):\n",
|
||||
" yield f\"{i + 1}. \"\n",
|
||||
" yield from get_text(li)\n",
|
||||
" yield \"\\n\\n\"\n",
|
||||
" elif child.name == \"div\" and \"tabs-container\" in child.attrs.get(\n",
|
||||
" \"class\", [\"\"]\n",
|
||||
" ):\n",
|
||||
" tabs = child.find_all(\"li\", {\"role\": \"tab\"})\n",
|
||||
" tab_panels = child.find_all(\"div\", {\"role\": \"tabpanel\"})\n",
|
||||
" for tab, tab_panel in zip(tabs, tab_panels):\n",
|
||||
" tab_name = tab.get_text(strip=True)\n",
|
||||
" yield f\"{tab_name}\\n\"\n",
|
||||
" yield from get_text(tab_panel)\n",
|
||||
" elif child.name == \"table\":\n",
|
||||
" thead = child.find(\"thead\")\n",
|
||||
" header_exists = isinstance(thead, Tag)\n",
|
||||
" if header_exists:\n",
|
||||
" headers = thead.find_all(\"th\")\n",
|
||||
" if headers:\n",
|
||||
" yield \"| \"\n",
|
||||
" yield \" | \".join(header.get_text() for header in headers)\n",
|
||||
" yield \" |\\n\"\n",
|
||||
" yield \"| \"\n",
|
||||
" yield \" | \".join(\"----\" for _ in headers)\n",
|
||||
" yield \" |\\n\"\n",
|
||||
"\n",
|
||||
" tbody = child.find(\"tbody\")\n",
|
||||
" tbody_exists = isinstance(tbody, Tag)\n",
|
||||
" if tbody_exists:\n",
|
||||
" for row in tbody.find_all(\"tr\"):\n",
|
||||
" yield \"| \"\n",
|
||||
" yield \" | \".join(\n",
|
||||
" cell.get_text(strip=True) for cell in row.find_all(\"td\")\n",
|
||||
" )\n",
|
||||
" yield \" |\\n\"\n",
|
||||
"\n",
|
||||
" yield \"\\n\\n\"\n",
|
||||
" elif child.name in [\"button\"]:\n",
|
||||
" continue\n",
|
||||
" else:\n",
|
||||
" yield from get_text(child)\n",
|
||||
"\n",
|
||||
" joined = \"\".join(get_text(soup))\n",
|
||||
" return re.sub(r\"\\n\\n+\", \"\\n\\n\", joined).strip()\n",
|
||||
"\n",
|
||||
"nest_asyncio.apply()\n",
|
||||
"\n",
|
||||
"def load_docs():\n",
|
||||
" loader = RecursiveUrlLoader(\n",
|
||||
" \"https://langchain-ai.github.io/langgraph\",\n",
|
||||
" extractor=langchain_docs_extractor,\n",
|
||||
" metadata_extractor=metadata_extractor,\n",
|
||||
" max_depth=10\n",
|
||||
" )\n",
|
||||
" docs = loader.load()\n",
|
||||
" return docs\n",
|
||||
"\n",
|
||||
"docs = load_docs()\n",
|
||||
"docs_texts = [d.page_content for d in docs]\n",
|
||||
"len(docs)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "6a662ed4-624e-4f70-a034-fee7c363518d",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"### Overview of docs\n",
|
||||
" \n",
|
||||
"* Visualize tokens per doc\n",
|
||||
"* Clean outliers"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 2,
|
||||
"id": "e560d06c-b76a-4fd4-9ccc-1bea1004fb47",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import pandas as pd\n",
|
||||
"import tiktoken\n",
|
||||
"\n",
|
||||
"def num_tokens_from_string(string: str, encoding_name: str) -> int:\n",
|
||||
" encoding = tiktoken.get_encoding(encoding_name)\n",
|
||||
" num_tokens = len(encoding.encode(string))\n",
|
||||
" return num_tokens\n",
|
||||
"\n",
|
||||
"docs_data = []\n",
|
||||
"for doc in docs:\n",
|
||||
" doc_text = doc.page_content # Getting text from each document\n",
|
||||
" tokens = num_tokens_from_string(doc_text, \"cl100k_base\") # Calculate tokens\n",
|
||||
" char_count = len(doc_text) # Calculate character count\n",
|
||||
" source = doc.metadata['source'] # Extract source from metadata\n",
|
||||
" docs_data.append({\n",
|
||||
" 'doc_text': doc_text,\n",
|
||||
" 'tokens': tokens,\n",
|
||||
" 'char_count': char_count, # Add character count here\n",
|
||||
" 'source': source\n",
|
||||
" })\n",
|
||||
"\n",
|
||||
"docs_df = pd.DataFrame(docs_data)\n",
|
||||
"exclude = []\n",
|
||||
"docs_df_clean=docs_df[~docs_df.index.isin(exclude)].copy()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 3,
|
||||
"id": "ba36cf3a-0e7a-4f47-b1ac-4c98acbd320a",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"image/png": "iVBORw0KGgoAAAANSUhEUgAAA1EAAAIoCAYAAACI32EXAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy80BEi2AAAACXBIWXMAAA9hAAAPYQGoP6dpAAEAAElEQVR4nOx9eXxU1fn+MzPZM1nJvpCVkA3IQhIBERQVUbBuda3iUrVWRFu1/Vlb3Ldq1aoo2moVtNW6f611QUWwREL2QAhLQgJhyzpLZpJJZjm/P673MpPZ7r2z3YTzfD7zSXLe9z1z7skzz7ln7jnvkRFCCCgoKCgoKCgoKCgoKCh4QR7oBlBQUFBQUFBQUFBQUEwl0EkUBQUFBQUFBQUFBQWFANBJFAUFBQUFBQUFBQUFhQDQSRQFBQUFBQUFBQUFBYUA0EkUBQUFBQUFBQUFBQWFANBJFAUFBQUFBQUFBQUFhQDQSRQFBQUFBQUFBQUFBYUA0EkUBQUFBQUFBQUFBQWFANBJFAUFBQUFBQUFBQUFhQDQSRQFBYUo9PT0QCaT4c033wx0U2zw5ZdfoqysDGFhYZDJZFCr1T59P5lMhjVr1vj0PaYrli5ditLS0kA3g0IC2LRpEwoLCxEcHIzY2NhAN4fCDbKzs7Fy5cpAN4OCIqCgkygKiknYtWsXLrvsMmRlZSEsLAzp6ek455xz8OKLL/rsPf/5z3/i+eeftys/duwYHnzwQbS0tPjsvSfj+++/h0wm417BwcHIzc3Fddddh4MHD3rlPWpra/Hggw96fYIzNDSEyy+/HOHh4Vi/fj02bdqEyMhIOz/r63P1+v77773avlMFWq0WDz30EObNmwelUonw8HCUlpbi97//PY4dOxbo5nkFjz/+OD755BO/v6/BYMBzzz2HmpoaxMTEICwsDAUFBVizZg3279/v9/Y4gtDP9969e3H99dcjLy8Pf/vb3/Daa6/5tH0PPvggZDIZBgcHffo+nkKr1eKxxx7D/PnzERMTg9DQUGRlZeGKK67A559/HujmeQRrnQ0KCkJ8fDwqKytx5513Ys+ePYFuHgUFLwQFugEUFFJCbW0tzjzzTMycORM333wzUlJS0Nvbix07duCvf/0r7rjjDp+87z//+U/s3r0bd911l035sWPH8NBDDyE7OxtlZWU+eW9nWLt2LaqqqmA0GtHU1ITXXnsNn3/+OXbt2oW0tDSP6q6trcVDDz2E66+/3qvfOtfX12NkZASPPPIIzj77bKd+mzZtsvl748aN2Lx5s115UVGR19p2quDgwYM4++yzcfjwYfz85z/HLbfcgpCQELS1teH111/Hxx9/LJmbfU/w+OOP47LLLsNFF13kt/ccHBzEeeedh8bGRqxcuRJXX301lEol9u3bh3fffRevvfYaJiYm/NYeZxD6+f7+++9hsVjw17/+Ffn5+b5v4BRAZ2cnli9fjkOHDuHiiy/GddddB6VSid7eXvz3v//FypUrsXHjRlx77bWBbqponHPOObjuuutACIFGo0FrayveeustvPzyy3jqqafw29/+NtBNpKBwCTqJoqCwwmOPPYaYmBjU19fbDf79/f2BaZQPoNfrHT6hscbixYtx2WWXAQBuuOEGFBQUYO3atXjrrbdw3333+aOZgsH+j9zduP3iF7+w+XvHjh3YvHmzXTmFMJhMJlxyySXo6+vD999/j9NPP93G/thjj+Gpp57ye5ssFgtCQkL8+r5iYDAYEBISArnc8SKR66+/Hs3Nzfjggw9w6aWX2tgeeeQR3H///f5optfB93MrBKOjo4iIiPBaff6EyWTCxRdfjL6+PmzduhWLFi2ysT/wwAP4+uuvYTabXdbDR+cDiYKCAjvNffLJJ7Fq1SrcfffdKCwsxPnnnx+g1lFQ8AChoKDgMHv2bLJ06VLe/ps2bSJVVVUkPDycxMbGksWLF5OvvvqKs3/yySfk/PPPJ6mpqSQkJITk5uaShx9+mJhMJs5nyZIlBIDNKysri2zZssWuHAD5xz/+wcXu2LGDLF++nERHR5Pw8HByxhlnkP/97382bXzggQcIANLe3k6uuuoqEhsbS8rKypxeE/u+77//vk357t27CQBy8803E0II6e7utmsPIYR8++235PTTTycREREkJiaGXHjhhWTPnj127Zn86u7udtnX//73v0lFRQUJCwsjM2bMINdccw05cuSIy35cvXq1yzpZ3H777WSyHOp0OvLb3/6WZGRkkJCQEFJQUECefvppYrFYbPwAkNtvv92m7JFHHiEymYy88MILXNl///tfrl+USiU5//zzye7du23iVq9eTSIjI8mRI0fIz372MxIZGUkSEhLI3XffbcMZQgj517/+RSoqKohSqSRRUVGktLSUPP/8806vcWJigsTFxZHrr7/ezqbRaEhoaCi5++67ubIXXniBFBcXc9yurKwk77zzjtP6CSHk3XffJQDIY4895tKPxZIlS0hJSQlpb28nS5cuJeHh4SQtLY089dRTNn7j4+PkT3/6E6moqCDR0dEkIiKCnH766eS7776z8WM5+fTTT5PnnnuO5ObmErlcTpqbm3nXQQghZrOZPP/886S0tJSEhoaShIQEsnz5clJfX08IIQ75a821I0eOkBtuuIEkJSWRkJAQUlxcTF5//XWb92A/Z//617/I/fffT9LS0ohMJiMqlcphX+3YscPm88cH7j6LhDCcy8rKsotlP6fWYLn+8ccfk5KSEu7avvjiC7s4vp/vrKwsO98HHniAs69fv54UFxeTkJAQkpqaSn7961/b9RHLo4aGBrJ48WISHh5O7rzzTqf9wrZxYGDAqc/Q0BC5++67SWlpKYmMjCRRUVHkvPPOIy0tLTZ+7P/xvffeI48++ihJT08noaGh5KyzziIHDhywq/ell14iOTk5JCwsjFRVVZFt27aRJUuWkCVLlnA+//znPwkA8uSTTzpt32T84x//IADI999/T2677TaSmJhIYmNjCSGE9PT0kNtuu40UFBSQsLAwEh8fTy677DK7/wlbx9atW8ktt9xC4uPjSVRUFLn22mvJ8PCwjW9WVha54IILyA8//ECqqqpIaGgoycnJIW+99Rav9jrSTRaHDh0iQUFBZOHChTblfX195MYbbyRJSUkkNDSUzJ07l7z55pt28e4+v4QQ8vXXX5NFixaRmJgYEhkZSQoKCsh9993Hq+0UFCzokygKCitkZWXhxx9/xO7du91ueH/ooYfw4IMPYuHChXj44YcREhKCuro6fPfddzj33HMBAG+++SaUSiV++9vfQqlU4rvvvsO6deug1Wrx9NNPAwDuv/9+aDQaHDlyBM899xwAQKlUoqioCA8//DDWrVuHW265BYsXLwYALFy4EADw3XffYcWKFaisrMQDDzwAuVyOf/zjHzjrrLPwww8/oLq62qa9P//5zzFr1iw8/vjjIIQI7puuri4AwIwZM5z6fPPNN1ixYgVyc3Px4IMPYmxsDC+++CIWLVqEpqYmZGdn45JLLsH+/fvxr3/9C8899xwSEhIAAImJiU7rffPNN3HDDTegqqoKTzzxBPr6+vDXv/4V27dvR3NzM2JjY3H//fdj9uzZeO211/Dwww8jJycHeXl5gq8TAAghuPDCC7FlyxbcdNNNKCsrw1dffYV7770XR48e5f5PjvDHP/4Rjz/+OF599VXcfPPNAJjlg6tXr8by5cvx1FNPYXR0FK+88gpOP/10NDc3Izs7m4s3m81Yvnw5ampq8Mwzz+Cbb77BX/7yF+Tl5eG2224DAGzevBlXXXUVli1bxj3Z6ejowPbt23HnnXc6bFdwcDAuvvhifPTRR3j11Vdtnsx88sknGB8fx5VXXgkA+Nvf/oa1a9fisssuw5133gmDwYC2tjbU1dXh6quvdnrt//d//wcAgpYYqVQqnHfeebjkkktw+eWX44MPPsDvf/97zJkzBytWrADA7A35+9//jquuugo333wzRkZG8Prrr2P58uXYuXOn3VLXf/zjHzAYDLjlllsQGhqK+Ph4QXXcdNNNePPNN7FixQr88pe/hMlkwg8//IAdO3Zg/vz52LRpE375y1+iuroat9xyCwBwXOvr68Npp53GJRxJTEzEF198gZtuuglardZuye4jjzyCkJAQ3HPPPRgfH3f6xExo3/L5LIrB//73P3z00Uf49a9/jaioKLzwwgu49NJLcfjwYcyYMUPw5/v555/Hxo0b8fHHH+OVV16BUqnE3LlzATB7lx566CGcffbZuO2227Bv3z688sorqK+vx/bt2xEcHMzVMzQ0hBUrVuDKK6/EL37xCyQnJ4u6PhYHDx7EJ598gp///OfIyclBX18fXn31VSxZsgR79uyxW9L85JNPQi6X45577oFGo8Gf//xnXHPNNairq+N8XnnlFaxZswaLFy/Gb37zG/T09OCiiy5CXFwcMjIyOL/PPvsMgP0Tcz749a9/jcTERKxbtw56vR4As8y5trYWV155JTIyMtDT04NXXnkFS5cuxZ49e+ye2K1ZswaxsbF48MEHuT4/dOgQt1+WRWdnJy677DLcdNNNWL16Nd544w1cf/31qKysRElJieC2s5g5cyaWLFmCLVu2QKvVIjo6GmNjY1i6dCk6OzuxZs0a5OTk4P3338f1118PtVpto3vuPr/t7e1YuXIl5s6di4cffhihoaHo7OzE9u3bRbeZ4hRFoGdxFBRSwtdff00UCgVRKBRkwYIF5He/+x356quvyMTEhI3fgQMHiFwuJxdffDExm802NusnFaOjo3bvceutt5KIiAhiMBi4sgsuuMDht8H19fUOn/ZYLBYya9Yssnz5crv3y8nJIeeccw5Xxn7retVVV/HqA/ab1TfeeIMMDAyQY8eOkc8//5xkZ2cTmUzGfZvn6ElUWVkZSUpKIkNDQ1xZa2srkcvl5LrrruPKnn76aV5PnwhhnqAkJSWR0tJSMjY2xpX/5z//IQDLine truncated
|
||||
"text/plain": [
|
||||
"<Figure size 1000x600 with 1 Axes>"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"import matplotlib.pyplot as plt\n",
|
||||
"# Tokens vs chars\n",
|
||||
"plt.figure(figsize=(10, 6))\n",
|
||||
"plt.scatter(docs_df_clean['tokens'], docs_df_clean['char_count'], color=\"blue\", alpha=0.7)\n",
|
||||
"plt.xscale('log')\n",
|
||||
"plt.yscale('log')\n",
|
||||
"plt.title(\"Scatter Plot of Tokens vs Character Count for LangGraph Docs\")\n",
|
||||
"plt.xlabel(\"Tokens (log scale)\")\n",
|
||||
"plt.ylabel(\"Character Count (log scale)\")\n",
|
||||
"plt.grid(True, which=\"both\", ls=\"--\", linewidth=0.5)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 37,
|
||||
"id": "d90279d6",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def get_doc_type(url):\n",
|
||||
" if 'langgraphjs' in url:\n",
|
||||
" parts = url.split('https://langchain-ai.github.io/langgraphjs/')\n",
|
||||
" else:\n",
|
||||
" parts = url.split('https://langchain-ai.github.io/langgraph/')\n",
|
||||
" if len(parts) > 1:\n",
|
||||
" return parts[1].split('/')[0]\n",
|
||||
" return 'Landing Page'\n",
|
||||
"\n",
|
||||
"# Add a new column 'language' to docs_df_clean\n",
|
||||
"docs_df_clean['language'] = docs_df_clean['source'].apply(\n",
|
||||
" lambda x: 'js' if 'langgraphjs' in x.lower() else 'python'\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"# Apply the function to create the 'doc_type' column\n",
|
||||
"docs_df_clean['doc_type'] = docs_df_clean['source'].apply(get_doc_type)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 38,
|
||||
"id": "0c13ad73-15ae-426a-a5a3-87ba4b472b7c",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"image/png": "iVBORw0KGgoAAAANSUhEUgAAA1EAAAJNCAYAAADQ5v/+AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy80BEi2AAAACXBIWXMAAA9hAAAPYQGoP6dpAABewklEQVR4nO3dd3gV1f7+/Xunh1SBUAKB0CEgPXAQkE4ogqCIgmBoYgkKgp3vESKKoIKIRJFDVxEEORZApCq2I6GJSBPpRQg1JEDqev7gyf65SQKZkLAT8n5d175kr1mz5jMzmOybmVnbZowxAgAAAADkiIuzCwAAAACAwoQQBQAAAAAWEKIAAAAAwAJCFAAAAABYQIgCAAAAAAsIUQAAAABgASEKAAAAACwgRAEAAACABYQoAAAAALCAEAUAThIaGqoBAwY4u4zb3ltvvaXKlSvL1dVV9evXz9dtfffdd7LZbFqyZEm+bgcA4FyEKADIA3PnzpXNZtOmTZuyXN66dWvVqVPnprezYsUKjR079qbHKSpWrVql559/Xs2bN9ecOXM0fvz4TH0ygk9OXoXRlStX9M4776hp06YKCAiQl5eXqlevrmHDhmnv3r3OLk+S9PPPP2vs2LE6f/68s0sBgBxxc3YBAFBU7dmzRy4u1v4ta8WKFYqJiSFI5dC6devk4uKiWbNmycPDI8s+tWrV0kcffeTQ9tJLL8nX11ejR4++FWXmm9OnT6tTp07avHmz7rnnHvXt21e+vr7as2ePFi5cqBkzZig5OdnZZernn39WdHS0BgwYoMDAQGeXAwA3RIgCACfx9PR0dgmWJSYmysfHx9ll5NipU6fk7e2dbYCSpNKlS6tfv34ObRMmTFDJkiUztRc2AwYM0NatW7VkyRLdf//9DsvGjRtX6EMiADgLt/MBgJNc+0xUSkqKoqOjVa1aNXl5ealEiRJq0aKFVq9eLenqB+KYmBhJyvIWs8TERI0aNUohISHy9PRUjRo19Pbbb8sY47Ddy5cv6+mnn1bJkiXl5+en7t2769ixY7LZbA5XuMaOHSubzaadO3eqb9++uuOOO9SiRQtJ0vbt2zVgwABVrlxZXl5eKlOmjAYNGqQzZ844bCtjjL1796pfv34KCAhQUFCQ/v3vf8sYoyNHjujee++Vv7+/ypQpo0mTJuXo2KWmpmrcuHGqUqWKPD09FRoaqpdffllJSUn2PjabTXPmzFFiYqL9WM2dOzdH42dl//79euCBB1S8eHEVK1ZM//rXv7R8+fIbrpeUlKR77rlHAQEB+vnnnyVJ6enpmjJlimrXri0vLy+VLl1ajz32mM6dO+ewbmhoqO655x79+OOPatKkiby8vFS5cmXNnz//htv99ddftXz5cg0ePDhTgJKuhvi3337boW3dunVq2bKlfHx8FBgYqHvvvVe7du1y6DNgwACFhoZmGi/jXP+TzWbTsGHD9MUXX6hOnTry9PRU7dq1tXLlSof1nnvuOUlSpUqV7Ofq4MGDkqTVq1erRYsWCgwMlK+vr2rUqKGXX375hvsPAPmJK1EAkIcuXLig06dPZ2pPSUm54bpjx47VG2+8oSFDhqhJkyaKj4/Xpk2btGXLFnXo0EGPPfaYjh8/rtWrV2e6/cwYo+7du2v9+vUaPHiw6tevr2+//VbPPfecjh07pnfeecfed8CAAfrss8/Uv39//etf/9L333+vrl27ZlvXAw88oGrVqmn8+PH2QLZ69Wrt379fAwcOVJkyZfTHH39oxowZ+uOPP/S///0v04fpBx98ULVq1dKECRO0fPlyvfbaaypevLg+/PBDtW3bVhMnTtQnn3yiZ599VuHh4br77ruve6yGDBmiefPmqVevXho1apR+/fVXvfHGG9q1a5f++9//SpI++ugjzZgxQxs3btTMmTMlSXfdddcNz0NWTp48qbvuukuXLl3S008/rRIlSmjevHnq3r27lixZop49e2a53uXLl3Xvvfdq06ZNWrNmjcLDwyVJjz32mObOnauBAwfq6aef1oEDBzRt2jRt3bpVP/30k9zd3e1j7Nu3T7169dLgwYMVGRmp2bNna8CAAWrUqJFq166dbc1fffWVJKl///452sc1a9aoc+fOqly5ssaOHavLly/rvffeU/PmzbVly5Ysg1NO/Pjjj1q6dKmefPJJ+fn5aerUqbr//vt1+PBhlShRQvfdd5/27t2rTz/9VO+8845KliwpSQoKCtIff/yhe+65R3Xr1tWrr74qT09P7du3Tz/99FOuagGAPGMAADdtzpw5RtJ1X7Vr13ZYp2LFiiYyMtL+vl69eqZr167X3U5UVJTJ6kf3F198YSSZ1157zaG9V69exmazmX379hljjNm8ebORZEaMGOHQb8CAAUaSGTNmjL1tzJgxRpLp06dPpu1dunQpU9unn35qJJkNGzZkGmPo0KH2ttTUVFO+fHljs9nMhAkT7O3nzp0z3t7eDsckK9u2bTOSzJAhQxzan332WSPJrFu3zt4WGRlpfHx8rjteVmrXrm1atWplfz9ixAgjyfzwww/2tosXL5pKlSqZ0NBQk5aWZowxZv369UaSWbx4sbl48aJp1aqVKVmypNm6dat9vR9++MFIMp988onDNleuXJmpvWLFipmO6alTp4ynp6cZNWrUdfehZ8+eRpI5d+5cjva5fv36plSpUubMmTP2tt9++824uLiYRx55xN4WGRlpKlasmGn9jHP9T5KMh4eH/e9fxpiSzHvvvWdve+utt4wkc+DAAYf133nnHSPJxMXF5WgfAOBW4XY+AMhDMTExWr16daZX3bp1b7huYGCg/vjjD/3555+Wt7tixQq5urrq6aefdmgfNWqUjDH65ptvJMl+G9WTTz7p0O+pp57KduzHH388U5u3t7f9z1euXNHp06f1r3/9S5K0ZcuWTP2HDBli/7Orq6saN24sY4wGDx5sbw8MDFSNGjW0f//+bGuRru6rJI0cOdKhfdSoUZKUo1vsrFqxYoWaNGliv51Rknx9fTV06FAdPHhQO3fudOh/4cIFdezYUbt379Z3333nMLX64sWLFRAQoA4dOuj06dP2V6NGjeTr66v169c7jBUWFqaWLVva3wcFBeXoOMXHx0uS/Pz8brh/J06c0LZt2zRgwAAVL17c3l63bl116NDBfsxzo3379qpSpYrDmP7+/jesX5J9kokvv/xS6enpua4BAPIaIQoA8lCTJk3Uvn37TK877rjjhuu++uqrOn/+vKpXr64777xTzz33nLZv356j7R46dEjBwcGZPjDXqlXLvjzjvy4uLqpUqZJDv6pVq2Y79rV9Jens2bMaPny4SpcuLW9vbwUFBdn7XbhwIVP/ChUqOLzPmGo749atf7Zf+1zQtTL24dqay5Qpo8DAQPu+5qVDhw6pRo0amdqvPb4ZRowYodjYWK1ZsybTLXd//vmnLly4oFKlSikoKMjhlZCQoFOnTjn0v/bYSdIdd9xxw+Pk7+8vSbp48WKO9k9Stvt4+vRpJSYm3nCcrOS2funqbaDNmzfXkCFDVLp0aT300EP67LPPCFQAnI5nogCggLj77rv1119/6csvv9SqVas0c+ZMvfPOO5o+fbrDlZxb7Z9XnTL07t1bP//8s5577jnVr19fvr6+Sk9PV6dOnbL8gOvq6pqjNkmZJsLITkH+3qZ7771XCxcu1IQJEzR//nyHqezT09NVqlQpffLJJ1muGxQU5PA+t8epZs2akqTff//d4UrWzcruuKelpWXZfjPn2dvbWxs2bND69eu1fPlyrVy5UosWLVLbtm21atWqbMcGgPzGlSgAKECKFy+ugQMH6tNPP9WRI0dUt25dhxnzsvsAW7FiRR0/fjzTVYfdu3fbl2f8Nz09XQcOHHDot2/fvhzXeO7cOa1du1YvvviioqOj1bNnT3Xo0EGVK1fO8Rg3I2Mfrr3t8eTJkzp//rx9X/N6m3v27MnUfu3xzdCjRw/Nnj1bCxYsUFRUlMOyKlWq6MyZM2revHmWVy3r1auXJzV369ZNkvTxxx/fsG9G/dntY8mSJe1T299xxx1ZfinuzVwBvF4gdnFxUbt27TR58mTt3LlTr7/+utatW5fptkcAuJUIUQBQQFw7Pbivr6+qVq3qMG13xgfZaz/EdunSRWlpaZo2bZpD+zvvvCObzabOnTtLkiIiIiRJ77//vkO/9957L8d1Zvzr/7VXEqZMmZLjMW5Gly5dstze5MmTJem6Mw3ezDY3btyoX375xd6WmJioGTNmKDQ0VGFhYZnWeeSRRzR16lRNnz5dL7zwgr29d+/eSktL07hx4zKtk5qammVAyY1mzZqpU6dOmjlzpr744otMy5OTk/Xss89KksqWLav69etr3rx5DtvfsWOHVq1aZT/m0tUQeOHCBYdbTU+cOGGfFTE3svt7ffbs2Ux9M54v++f/FwBwq3E7HwAUEGFhYWrdurUaNWqk4sWLa9OmTVqyZImGDRtm79OoUSNJ0tNPP62IiAi5urrqoYceUrdu3dSmTRuNHj1aBw8eVL169bRq1Sp9+eWXGjFihP3B/kaNGun+++/XlClTdObMGfsU53v37pWUs1vk/P39dffdd+vNN99USkqKypUrp1WrVmW6upVf6tWrp8jISM2YMUPnz59Xq1attHHjRs2bN089evRQmzZt8nybL774oj799FN17txZTz/9tIoXL6558+bpwIED+vzzzx1u1/unYcOGKT4+XqNHj1ZAQIBefvlltWrVSo899pjeeOMNbdu2TR07dpS7u7v+/PNPLV68WO+++6569eqVJ3XPnz9fHTt21H333adu3bqpXbt28vHx0Z9//qmFCxfqxIkT9u+Keuutt9S5c2c1a9ZMgwcPtk9xHhAQ4HA19KGHHtILL7ygnj176umnn9alS5f0wQcfqHr16llOKpITGX+vR48erYceekju7u7q1q2bXn31VW3YsEFdu3ZVxYoVderUKb3Line truncated
|
||||
"text/plain": [
|
||||
"<Figure size 1000x600 with 1 Axes>"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"# Calculate the number of tokens for each document\n",
|
||||
"plt.figure(figsize=(10, 6))\n",
|
||||
"plt.hist(docs_df_clean['tokens'], bins=30, color=\"blue\", edgecolor=\"black\", alpha=0.7)\n",
|
||||
"total_tok = docs_df_clean.tokens.sum()\n",
|
||||
"total_docs = docs_df_clean.shape[0]\n",
|
||||
"plt.title(f\"Histogram of Token Counts \\n Total tokens: {total_tok} \\n Total docs: {total_docs}\")\n",
|
||||
"plt.xlabel(\"Token Count\")\n",
|
||||
"plt.ylabel(\"Frequency\")\n",
|
||||
"plt.grid(axis=\"y\", alpha=0.75)\n",
|
||||
"plt.yscale('log')"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 39,
|
||||
"id": "124fe6a4-9226-4788-a3e1-557ab8524ee0",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"/var/folders/l9/bpjxdmfx7lvd1fbdjn38y5dh0000gn/T/ipykernel_82193/2326669627.py:38: UserWarning: FigureCanvasAgg is non-interactive, and thus cannot be shown\n",
|
||||
" fig.show()\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"image/png": "iVBORw0KGgoAAAANSUhEUgAAA90AAAXRCAYAAACD3P7HAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy80BEi2AAAACXBIWXMAAA9hAAAPYQGoP6dpAAEAAElEQVR4nOzdeVxU9eLG8WcGAUEFlMUV3FM0d9TUcs0t06tpZVmaWxuZqVlZWdpi3collZvdFpf29Je23dw1yzTN3Eo0d9wVXFARgZnv7w9jcgQUlMOM+nm/XryU75yZec45w8AzZ7MZY4wAAAAAAEC+s3s6AAAAAAAA1ypKNwAAAAAAFqF0AwAAAABgEUo3AAAAAAAWoXQDAAAAAGARSjcAAAAAABahdAMAAAAAYBFKNwAAAAAAFqF0AwAAAABgEUo3AOTS0qVLZbPZtHTpUk9HQR5MmzZNNptNv/32m6ejeJ1Vq1bJz89Pu3fv9nQUr7Fr1y7ZbDZNmzYtz/edO3euihYtqiNHjuR/MADAVYvSDcCr2Wy2XH3lpgiPGTNGc+bMsTxzZsnL/CpcuLDKlCmj9u3ba+LEiTp58qTlGa5m//nPfy6r8HhK5ocxufnyNs8995zuuecelS9f3jXWsmVL3XjjjR5M9Y+0tDS9/fbbqlevnoKCghQSEqKaNWvqwQcf1ObNmz0dL4sOHTqoSpUqeu211zwdBQDgRQp5OgAAXMxHH33k9v2MGTO0YMGCLOPR0dGXfKwxY8aoR48e6tq1a35GzNFLL72kihUrKj09XQcPHtTSpUv1xBNPaNy4cfrmm29Uu3btAslxtfnPf/6jsLAwPfDAA56OkivR0dFZXo8jRoxQ0aJF9dxzz3ko1aWtW7dOCxcu1C+//OLpKDnq3r27fvjhB91zzz0aOHCg0tPTtXnzZn333Xdq2rSpqlev7umIWTz00EN68sknNXr0aBUrVszTcQAAXoDSDcCr3XfffW7fr1y5UgsWLMgy7o06duyomJgY1/cjRozQ4sWLdfvtt6tLly6Kj49XQECABxMiP5QsWTLL6/H1119XWFiYV79Op06dqqioKN10002ejpKt1atX67vvvtOrr76qZ5991u22yZMn6/jx454Jdgndu3fXoEGDNHPmTPXr18/TcQAAXoDdywFc9U6fPq1hw4YpMjJS/v7+qlatmt566y0ZY1zT2Gw2nT59WtOnT3ft6pu5JXX37t169NFHVa1aNQUEBCg0NFR33nmndu3ale9ZW7durZEjR2r37t36+OOP3W5bvHixbrnlFhUpUkQhISH617/+pfj4+CyPsW/fPvXv319lypSRv7+/KlasqEceeURpaWmSpFGjRmW7K3Pmbu/nz1eFChV0++23a+nSpYqJiVFAQIBq1arl2l3/q6++Uq1atVS4cGE1aNBAa9euzfK4mzdvVo8ePVSiRAkVLlxYMTEx+uabb7J97uXLl2vo0KEKDw9XkSJF1K1bN7fjXytUqKA///xTP/74o2s9tWzZUpKUnp6u0aNHq2rVqipcuLBCQ0N18803a8GCBbla9ikpKXrooYcUGhqqoKAg9e7dW8eOHXPd3qdPH4WFhSk9PT3Lfdu1a6dq1arl6nkuZIxRhQoV9K9//SvLbampqQoODtZDDz0k6Z9d1b/44gs9++yzKlWqlIoUKaIuXbpoz549We7/66+/qkOHDgoODlZgYKBatGih5cuX5yrXnDlz1Lp168ve7f0///mPatasKX9/f5UpU0axsbHZFuG4uDhVqlRJAQEBatSokX766Se1bNnStV5zsn37dklSs2bNstzm4+Oj0NBQt7FL/VwcPXpUTz75pGrVqqWiRYsqKChIHTt21Pr163M1v7l5nUtSRESEateura+//jpXjwsAuPZRugFc1Ywx6tKli8aPH68OHTpo3LhxqlatmoYPH66hQ4e6pvvoo4/k7++vW265RR999JE++ugjV9FZvXq1fvnlF/Xs2VMTJ07Uww8/rEWLFqlly5ZKSUnJ98z333+/JGn+/PmusYULF6p9+/Y6fPiwRo0apaFDh+qXX35Rs2bN3Ery/v371ahRI33++ee6++67NXHiRN1///368ccfLzvrtm3bdO+996pz58567bXXdOzYMXXu3FmffPKJhgwZovvuu0+jR4/W9u3bddddd8npdLru++eff+qmm25SfHy8nnnmGY0dO1ZFihRR165dNXv27CzPNWjQIK1fv14vvviiHnnkEX377bd67LHHXLdPmDBB5cqVU/Xq1V3rKXMX7VGjRmn06NFq1aqVJk+erOeee05RUVH6/fffczWfjz32mOLj4zVq1Cj17t1bn3zyibp27er6cOb+++9XUlKS5s2b53a/gwcPavHixZe91dpms+m+++7TDz/8oKNHj7rd9u233yo5OTnLY7/66qv6/vvv9fTTT+vxxx/XggULdOutt+rMmTOuaRYvXqzmzZsrOTlZL774osaMGaPjx4+rdevWWrVq1UUz7du3TwkJCapfv/5lzdOoUaMUGxurMmXKaOzYserevbveffddtWvXzu1Di3feeUePPfaYypUrpzfeeEO33HKLunbtqr17917yOTKPM//kk0+UkZFx0Wlz83OxY8cOzZkzR7fffrvGjRun4cOHa+PGjWrRooX2799/0cfP6+u8QYMGXr3bPgCggBkAuIrExsaa89+65syZYySZV155xW26Hj16GJvNZrZt2+YaK1KkiOnTp0+Wx0xJSckytmLFCiPJzJgxwzW2ZMkSI8ksWbLkohmnTp1qJJnVq1fnOE1wcLCpV6+e6/u6deuaiIgIk5SU5Bpbv369sdvtpnfv3q6x3r17G7vdnu1jO51OY4wxL774osnu7T0z186dO11j5cuXN5LML7/84hqbN2+ekWQCAgLM7t27XePvvvtulvlv06aNqVWrlklNTXXL0bRpU1O1atUsz33rrbe6chpjzJAhQ4yPj485fvy4a6xmzZqmRYsWWfLXqVPHdOrUKcv4pWQ+d4MGDUxaWppr/I033jCSzNdff22MMcbhcJhy5cqZu+++2+3+48aNMzabzezYsSPXz3nhPGzZssVIMu+8847bdF26dDEVKlRwLZPM11jZsmVNcnKya7ovv/zSSDJvv/22MebcMq5atapp37692/JMSUkxFStWNG3btr1ovoULFxpJ5ttvv81yW4sWLUzNmjVzvO/hw4eNn5+fadeunXE4HK7xyZMnG0nmww8/NMYYc/bsWRMaGmoaNmxo0tPTXdNNmzbNSMp2HZ/P6XSaFi1aGEmmZMmS5p577jFxcXFur8lMufm5SE1NdctrjDE7d+40/v7+5qWXXnIbk2SmTp3qGsvt6zzTmDFjjCRz6NChi84jAOD6wJZuAFe1//3vf/Lx8dHjjz/uNj5s2DAZY/TDDz9c8jHOP646PT1dSUlJqlKlikJCQnK9FTWvihYt6jqL+YEDB7Ru3To98MADKlGihGua2rVrq23btvrf//4nSXI6nZozZ446d+7sdqx4psvdTbhGjRpq0qSJ6/vGjRtLOrcrfFRUVJbxHTt2SDq3u+7ixYt111136eTJk0pMTFRiYqKSkpLUvn17bd26Vfv27XN7rgcffNAt5y233CKHw5GrS1aFhITozz//1NatWy9rPh988EH5+vq6vn/kkUdUqFAh1/K12+3q1auXvvnmG7czzH/yySdq2rSpKlaseFnPK0k33HCDGjdurE8++cQ1dvToUf3www/q1atXlnXXu3dvt5Nw9ejRQ6VLl3ZlXbdunbZu3ap7771XSUlJrmV/+vRptWnTRsuWLXPbI+FCSUlJkqTixYvneV4WLlyotLQ0PfHEE7Lb//kzYuDAgQoKCtL3338vSfrtt9+UlJSkgQMHqlChf04h06tXr1w9r81m07x58/TKK6+oePHi+uyzzxQbG6vy5cvr7rvvdu3KntufC39/f1deh8OhpKQkFS1aVNWqVbvoz/nlvM4z5y8xMfGS8wkAuPZRugFc1Xbv3q0yZcpkOUtw5tnMc1Pmzpw5oxdeeMF1THhYWJjCw8N1/PhxnThxwpLcp06dcmXOzJjdMcPR0dGuMnXkyBElJyfn++Wczi/WkhQcHCxJioyMzHY88zjobdu2yRijkSNHKjw83O3rxRdflCQdPnz4os+VWU7OP7Y6Jy+99JKOHz+uG264QbVq1dLw4cO1YcOG3M6mqlat6vZ90aJFVbp0abfd93v37q0zZ864dhnesmWL1qxZ4zok4Er07t1by5cvd63vmTNnKj09PdvHvjCrzWZTlSpVXFkzP3jo06dPlmX//vvv6+zZs7l67ZrzznuQWzm9Xv38/FSpUiXX7Zn/VqlSxW26QoUKqUKFCrl6Ln9/fz333HOKj4/X/v379dlnn+mmm27Sl19+6TosIbc/F06nU+PHj1fVqlXdfs43bNhw0WV1Oa/zzOXqjZeJAwAUPM5eDuC6N2jQIE2dOlVPPPGEmjRpouDgYNlsNvXs2fOiWwsv1969e3XixIksZSS/5PSHvsPhyHbcx8cnT+OZhSJz2Tz55JNq3759ttNeOI+XesyLad68ubZv366vv/5a8+fP1/vvv6/x48drypQpGjBgwCXvnxs1atRQgwYN9PHHH6t37976+OOP5efnp7vuuuuKH7tnz54aMmSIPvnkEz377LP6+OOPFRMTc1knaMtc9m+++abq1q2b7TRFixbN8f6ZJyHLzYcd3qJ06dLq2bOnunfvrpo1a+rLL7/M0/Xcx4wZo5EjR6pfv356+eWXVaJECdntdj3xxBMX/Tm/nNd55nINCwvLdT4AwLWL0g3gqla+fHktXLhQJ0+edNvavXnzZtftmXIqo7NmzVKfPn00duxY11hqaqpllyTKvKZz5h/wmRm3bNmSZdrNmzcrLCxMRYoUUUBAgIKCgvTHH39c9PEztx4fP35cISEhrvHcbPXPi0qVKkmSfH19deutt+bLine truncated
|
||||
"text/plain": [
|
||||
"<Figure size 1000x1500 with 2 Axes>"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"def plt_doc_num_and_tok(df):\n",
|
||||
" \n",
|
||||
" fig, (ax1, ax2) = plt.subplots(2, 1, figsize=(10, 15))\n",
|
||||
"\n",
|
||||
" total_tok = df.tokens.sum()\n",
|
||||
" total_docs = df.shape[0]\n",
|
||||
"\n",
|
||||
" # First subplot for Total Documents by Type\n",
|
||||
" doc_counts = df.groupby(['doc_type', 'language']).size().unstack(fill_value=0)\n",
|
||||
" doc_counts['total'] = doc_counts.sum(axis=1)\n",
|
||||
" doc_counts_sorted = doc_counts.sort_values('total', ascending=False).drop('total', axis=1)\n",
|
||||
" doc_counts_sorted.plot(kind='bar', ax=ax1, edgecolor='black')\n",
|
||||
" ax1.set_title(f'Total Documents by Type (Log Scale)\\nTotal docs: {total_docs}')\n",
|
||||
" ax1.set_xlabel('Document Type')\n",
|
||||
" ax1.set_ylabel('Total Documents')\n",
|
||||
" ax1.set_xticklabels(ax1.get_xticklabels(), rotation=90)\n",
|
||||
" ax1.grid(axis='y', linestyle='--', alpha=0.7)\n",
|
||||
" ax1.set_yscale('log')\n",
|
||||
" ax1.legend(title='Language', bbox_to_anchor=(1.05, 1), loc='upper left')\n",
|
||||
" \n",
|
||||
" # Second subplot for Total Tokens by Document Type\n",
|
||||
" token_counts = df.groupby(['doc_type', 'language'])['tokens'].sum().unstack(fill_value=0)\n",
|
||||
" token_counts['total'] = token_counts.sum(axis=1)\n",
|
||||
" token_counts_sorted = token_counts.sort_values('total', ascending=False).drop('total', axis=1)\n",
|
||||
" token_counts_sorted.plot(kind='bar', ax=ax2, edgecolor='black')\n",
|
||||
" ax2.set_title(f'Total Tokens by Document Type (Log Scale)\\nTotal tokens: {total_tok}')\n",
|
||||
" ax2.set_xlabel('Document Type')\n",
|
||||
" ax2.set_ylabel('Total Tokens')\n",
|
||||
" ax2.set_xticklabels(ax2.get_xticklabels(), rotation=90)\n",
|
||||
" ax2.grid(axis='y', linestyle='--', alpha=0.7)\n",
|
||||
" ax2.set_yscale('log')\n",
|
||||
" ax2.legend(title='Language', bbox_to_anchor=(1.05, 1), loc='upper left')\n",
|
||||
" \n",
|
||||
" plt.tight_layout()\n",
|
||||
" return fig\n",
|
||||
"\n",
|
||||
"fig = plt_doc_num_and_tok(docs_df_clean)\n",
|
||||
"fig.show()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "e3ac84ef-699a-4f20-afc9-25967adca366",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"Let's tier our docs for index design"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 33,
|
||||
"id": "ba467d0b",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/html": [
|
||||
"<div>\n",
|
||||
"<style scoped>\n",
|
||||
" .dataframe tbody tr th:only-of-type {\n",
|
||||
" vertical-align: middle;\n",
|
||||
" }\n",
|
||||
"\n",
|
||||
" .dataframe tbody tr th {\n",
|
||||
" vertical-align: top;\n",
|
||||
" }\n",
|
||||
"\n",
|
||||
" .dataframe thead th {\n",
|
||||
" text-align: right;\n",
|
||||
" }\n",
|
||||
"</style>\n",
|
||||
"<table border=\"1\" class=\"dataframe\">\n",
|
||||
" <thead>\n",
|
||||
" <tr style=\"text-align: right;\">\n",
|
||||
" <th></th>\n",
|
||||
" <th>doc_text</th>\n",
|
||||
" <th>tokens</th>\n",
|
||||
" <th>char_count</th>\n",
|
||||
" <th>source</th>\n",
|
||||
" <th>doc_type</th>\n",
|
||||
" <th>language</th>\n",
|
||||
" </tr>\n",
|
||||
" </thead>\n",
|
||||
" <tbody>\n",
|
||||
" <tr>\n",
|
||||
" <th>0</th>\n",
|
||||
" <td>ü¶úüï∏Ô∏èLangGraph\\n\\n[\\n Skip to c...</td>\n",
|
||||
" <td>1695</td>\n",
|
||||
" <td>7252</td>\n",
|
||||
" <td>https://langchain-ai.github.io/langgraph</td>\n",
|
||||
" <td>Unknown</td>\n",
|
||||
" <td>python</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>1</th>\n",
|
||||
" <td>Tutorials\\n\\n[\\n Skip to content\\n ...</td>\n",
|
||||
" <td>870</td>\n",
|
||||
" <td>3669</td>\n",
|
||||
" <td>https://langchain-ai.github.io/langgraph/tutor...</td>\n",
|
||||
" <td>tutorials</td>\n",
|
||||
" <td>python</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>2</th>\n",
|
||||
" <td>SQL Agent\\n\\n[\\n Skip to content\\n ...</td>\n",
|
||||
" <td>379788</td>\n",
|
||||
" <td>580711</td>\n",
|
||||
" <td>https://langchain-ai.github.io/langgraph/tutor...</td>\n",
|
||||
" <td>tutorials</td>\n",
|
||||
" <td>python</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>3</th>\n",
|
||||
" <td>Adaptive RAG\\n\\n[\\n Skip to content\\n...</td>\n",
|
||||
" <td>137960</td>\n",
|
||||
" <td>220299</td>\n",
|
||||
" <td>https://langchain-ai.github.io/langgraph/tutor...</td>\n",
|
||||
" <td>tutorials</td>\n",
|
||||
" <td>python</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>4</th>\n",
|
||||
" <td>Graphs\\n\\n[\\n Skip to content\\n ...</td>\n",
|
||||
" <td>6536</td>\n",
|
||||
" <td>28729</td>\n",
|
||||
" <td>https://langchain-ai.github.io/langgraph/refer...</td>\n",
|
||||
" <td>reference</td>\n",
|
||||
" <td>python</td>\n",
|
||||
" </tr>\n",
|
||||
" </tbody>\n",
|
||||
"</table>\n",
|
||||
"</div>"
|
||||
],
|
||||
"text/plain": [
|
||||
" doc_text tokens char_count \\\n",
|
||||
"0 ü¶úüï∏Ô∏èLangGraph\\n\\n[\\n Skip to c... 1695 7252 \n",
|
||||
"1 Tutorials\\n\\n[\\n Skip to content\\n ... 870 3669 \n",
|
||||
"2 SQL Agent\\n\\n[\\n Skip to content\\n ... 379788 580711 \n",
|
||||
"3 Adaptive RAG\\n\\n[\\n Skip to content\\n... 137960 220299 \n",
|
||||
"4 Graphs\\n\\n[\\n Skip to content\\n ... 6536 28729 \n",
|
||||
"\n",
|
||||
" source doc_type language \n",
|
||||
"0 https://langchain-ai.github.io/langgraph Unknown python \n",
|
||||
"1 https://langchain-ai.github.io/langgraph/tutor... tutorials python \n",
|
||||
"2 https://langchain-ai.github.io/langgraph/tutor... tutorials python \n",
|
||||
"3 https://langchain-ai.github.io/langgraph/tutor... tutorials python \n",
|
||||
"4 https://langchain-ai.github.io/langgraph/refer... reference python "
|
||||
]
|
||||
},
|
||||
"execution_count": 33,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"docs_df_clean.head()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "c841210d-9563-4bbd-8143-21be3bc41de9",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"Ideas:\n",
|
||||
"\n",
|
||||
"* Split js vs python \n",
|
||||
"* Remove notebook cells outputs \n",
|
||||
"* Narrow down to `how-to`, `concepts`, `tutorials`\n",
|
||||
"* Can likely context stuff them (no index) or simple doc-wise retrieval w/ page summary (parent-document) \n",
|
||||
"\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "de592dba-30ba-4065-83ef-75d2cc0d2d48",
|
||||
"metadata": {},
|
||||
"source": []
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "3124fffc-f48f-4cbf-b1d4-95217e0f9a65",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "89dc982a-ce8c-42b0-8de7-1768049c25b4",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "6d558881-5b2a-4a9f-a1cf-dd974a419ef0",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "0cddd4e6-1fd6-43e7-8a09-6630ecdbd6a5",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "93624d25-e186-4568-92e7-7519ae70429b",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "443bb43a-8100-4335-8e37-abdd45f69c73",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "2beee471-8e7e-4430-8981-c501dd5409b0",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3 (ipykernel)",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.11.8"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 5
|
||||
}
|
||||
Reference in new issue
Block a user