mirror of
https://github.com/langchain-ai/langgraph.git
synced 2026-08-28 18:59:42 +02:00
Include classification steps
This commit is contained in:
William Fu-Hinthorn
+327
-17
@@ -80,20 +80,25 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 1,
|
||||
"execution_count": 69,
|
||||
"id": "580d82b5-b60c-47a4-9c8b-e28be22ca0e3",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import logging\n",
|
||||
"import operator\n",
|
||||
"from typing import Annotated, List, Optional, TypedDict\n",
|
||||
"\n",
|
||||
"logging.basicConfig(level=logging.WARNING)\n",
|
||||
"logger = logging.getLogger(\"tnt-llm\")\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"class Doc(TypedDict):\n",
|
||||
" id: str\n",
|
||||
" content: str\n",
|
||||
" summary: Optional[str]\n",
|
||||
" explanation: Optional[str]\n",
|
||||
" category: Optional[str]\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"class TaxonomyGenerationState(TypedDict):\n",
|
||||
@@ -346,6 +351,7 @@
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# We will share an LLM for each step of the generate -> update -> review cycle\n",
|
||||
"# You may want to consider using Opus or another more powerful model for this\n",
|
||||
"taxonomy_generation_llm = ChatAnthropic(\n",
|
||||
" model=\"claude-3-haiku-20240307\", max_tokens_to_sample=2000\n",
|
||||
")\n",
|
||||
@@ -531,12 +537,14 @@
|
||||
"\n",
|
||||
"The docs can contain __any__ content, but we've found it works really well on chat bot logs, such as those captured by [LangSmith](https://smith.langchain.com).\n",
|
||||
"\n",
|
||||
"We will use that as an example below. Update the `project_name` to your own LangSmith project."
|
||||
"We will use that as an example below. Update the `project_name` to your own LangSmith project.\n",
|
||||
"\n",
|
||||
"You will likely have to customize the `run_to_doc` function below, since your expected keys may differ from those of this notebook's author."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 18,
|
||||
"execution_count": 193,
|
||||
"id": "bcc65649-157f-4848-9ef0-8a9932a98d85",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
@@ -552,7 +560,7 @@
|
||||
"past_week = datetime.now() - timedelta(days=7)\n",
|
||||
"runs = list(\n",
|
||||
" client.list_runs(\n",
|
||||
" project_name=\"chat-langchain\",\n",
|
||||
" project_name=project_name,\n",
|
||||
" filter=\"eq(is_root, true)\",\n",
|
||||
" start_time=past_week,\n",
|
||||
" # We only need to return the inputs + outputs\n",
|
||||
@@ -565,7 +573,7 @@
|
||||
"def run_to_doc(run) -> Doc:\n",
|
||||
" turns = []\n",
|
||||
" idx = 0\n",
|
||||
" for turn in run.inputs[\"chat_history\"] or []:\n",
|
||||
" for turn in run.inputs.get(\"chat_history\") or []:\n",
|
||||
" key, value = next(iter(turn.items()))\n",
|
||||
" turns.append(f\"<{key} idx={idx}>\\n{value}\\n</{key}>\")\n",
|
||||
" idx += 1\n",
|
||||
@@ -667,7 +675,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 42,
|
||||
"execution_count": 202,
|
||||
"id": "38085515-6cda-452d-9f44-de440058cb3f",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
@@ -678,22 +686,37 @@
|
||||
"\n",
|
||||
"| ID | Name | Description |\n",
|
||||
"|----|------|-------------|\n",
|
||||
"| 1 | Manage Account | Tasks related to managing user accounts, such as password reset, profile update, and account settings. |\n",
|
||||
"| 2 | Seek Customer Support | Requests for assistance, such as reporting an issue, requesting a refund, or asking about a product or service. |\n",
|
||||
"| 3 | Inquire about Products and Services | General inquiries about product information, pricing, availability, and other details. |\n",
|
||||
"| 4 | Extract and Process Data | Tasks related to extracting, processing, and parsing data from various sources like invoices, PDFs, and web pages. |\n",
|
||||
"| 5 | Analyze Demographic Data | Tasks related to analyzing tabular data on age, drinking/smoking age, marital status, and other demographic information. |\n",
|
||||
"| 6 | Configure Conversational AI | Tasks related to setting up memory, responses, and prompts for conversational AI systems. |\n",
|
||||
"| 7 | Enhance Conversational Retrieval | Tasks related to improving search and retrieval for conversational agents using techniques like query decomposition. |\n",
|
||||
"| 8 | Build Chatbots and Agents | Tasks related to developing conversational AI agents with memory, tools, and language models. |\n",
|
||||
"| 9 | Optimize Web and Search | Tasks related to improving website performance and search engine visibility through techniques like URL loading and parsing. |\n",
|
||||
"| 10 | Deploy and Scale AI | Tasks related to setting up and optimizing infrastructure for deploying and scaling AI applications. |\n"
|
||||
"| 1 | Troubleshooting Network Connectivity Issues | Resolving problems with DNS, network connections, and GitHub extension activation. |\n",
|
||||
"| 2 | Extracting and Analyzing Data | Retrieving and processing data from various sources like text files, databases, and APIs. |\n",
|
||||
"| 3 | Providing Healthcare Insights | Generating medical diagnosis, symptom checking, drug information, and skin condition analysis. |\n",
|
||||
"| 4 | Configuring and Optimizing Models | Adjusting model parameters and hyperparameters to improve performance for a given task. |\n",
|
||||
"| 5 | Generating Creative Poetry | Creating poems using language models and AI-powered tools. |\n",
|
||||
"| 6 | Interacting with Databases | Querying databases, extracting data, and managing errors during data processing. |\n",
|
||||
"| 7 | Querying Vector Databases | Interacting with vector databases like Milvus to store and retrieve high-dimensional data. |\n",
|
||||
"| 8 | Generating Synthetic Data | Creating synthetic data using language models and machine learning techniques. |\n",
|
||||
"| 9 | Integrating Tools and Workflows | Incorporating various tools and libraries into a cohesive workflow for different tasks. |\n",
|
||||
"| 10 | Improving Information Retrieval | Storing and querying multiple vectors per document for better semantic understanding. |\n",
|
||||
"| 11 | Processing Documents and Extracting Text | Parsing and extracting text from various document formats like PDF, DOCX, and HTML. |\n",
|
||||
"| 12 | Building Local Knowledge Bases | Creating knowledge bases from text files, handling text splitting, embeddings, and storage. |\n",
|
||||
"| 13 | Optimizing Conversational Retrieval | Troubleshooting and improving the performance of the ConversationalRetrievalChain in LangChain. |\n",
|
||||
"| 14 | Connecting Databases and Using Agents | Connecting to databases, using agents, and understanding the differences between agent types. |\n",
|
||||
"| 15 | Introspecting LangChain Tools | Accessing and retrieving details about the functions and source code of LangChain tools. |\n",
|
||||
"| 16 | Generating Styled Answers with Retrieval Augmentation | Creating a QA system that generates well-cited answers in a specific style. |\n",
|
||||
"| 17 | Using ZERO_SHOT_REACT_DESCRIPTION Agents | Applying the ZERO_SHOT_REACT_DESCRIPTION agent type in LangChain for chat models. |\n",
|
||||
"| 18 | Automating Microlearning Course Creation | Generating microlearning courses based on input parameters like topic, volume, and learning style. |\n",
|
||||
"| 19 | Integrating with Chroma Vector Store | Storing and retrieving data in the Chroma vector database, including handling document embeddings. |\n",
|
||||
"| 20 | Managing LangChain Callback Tokens | Understanding and utilizing the callback token feature in the LCEL chain. |\n",
|
||||
"| 21 | Troubleshooting FastAPI Deployments | Resolving issues with deploying a React app with a FastAPI backend. |\n",
|
||||
"| 22 | Analyzing Data with LangChain Agents | Using LangChain agents to interact with Pandas and Spark DataFrames for data exploration. |\n",
|
||||
"| 23 | Implementing the OpenAI Chat API | Implementing the OpenAI chat completion API and understanding the required inputs and outputs. |\n",
|
||||
"| 24 | Comparing LangChain and LLMIndex | Evaluating the differences between LangChain and LLMIndex, including their UI support for Markdown. |\n",
|
||||
"| 25 | Suppressing Tools in AgentExecutor | Temporarily disabling tools in an AgentExecutor for a fixed number of invocations. |\n"
|
||||
],
|
||||
"text/plain": [
|
||||
"<IPython.core.display.Markdown object>"
|
||||
]
|
||||
},
|
||||
"execution_count": 42,
|
||||
"execution_count": 202,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
@@ -723,6 +746,293 @@
|
||||
"Markdown(format_taxonomy_md(step[\"__end__\"][\"clusters\"][-1]))"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "3cf106db-6cff-4847-bff9-e25f00b9c015",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Phase 2: Labeling\n",
|
||||
"\n",
|
||||
"Now that we have our taxonomy, it's time to label a subset of our data to train a classifier.\n",
|
||||
"\n",
|
||||
"Input classification can be useful for anything from in-line prompt optimization (tailor the prompt for each classified intent), to system improvements (identifying categories for which the system doesn't produce good responses) to product analytics (understand which intent categories could be improved to drive profits).\n",
|
||||
"\n",
|
||||
"The problem is that LLM-based tagging can be expensive.\n",
|
||||
"\n",
|
||||
"Embeddings can be ~100x cheaper to compute, and a simple logistic regression classifier on top of that would add negligible cost. \n",
|
||||
"\n",
|
||||
"Let's tag and train a classifier!\n",
|
||||
"\n",
|
||||
"#### Label Training Data\n",
|
||||
"\n",
|
||||
"Use an LLM to label the data in a fully-automated fashion. For beter accuracy, you can sample a portion of the results to label by hand as well to verify the quality."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 89,
|
||||
"id": "8aa8a6f5-f53a-41e5-b09d-c6e8476e5471",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"labeling_prompt = hub.pull(\"wfh/tnt-llm-classify\")\n",
|
||||
"\n",
|
||||
"labeling_llm = ChatAnthropic(model=\"claude-3-haiku-20240307\", max_tokens_to_sample=2000)\n",
|
||||
"labeling_llm_chain = (labeling_prompt | labeling_llm | StrOutputParser()).with_config(\n",
|
||||
" run_name=\"ClassifyDocs\"\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def parse_labels(output_text: str) -> Dict:\n",
|
||||
" \"\"\"Parse the generated labels from the predictions.\"\"\"\n",
|
||||
" category_matches = re.findall(\n",
|
||||
" r\"\\s*<category>(.*?)</category>.*\",\n",
|
||||
" output_text,\n",
|
||||
" re.DOTALL,\n",
|
||||
" )\n",
|
||||
" categories = [{\"category\": category.strip()} for category in category_matches]\n",
|
||||
" if len(categories) > 1:\n",
|
||||
" logger.warning(f\"Multiple selected categories: {categories}\")\n",
|
||||
" label = categories[0]\n",
|
||||
" stripped = re.sub(r\"^\\d+\\.\\s*\", \"\", label[\"category\"]).strip()\n",
|
||||
" return {\"category\": stripped}\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"labeling_chain = labeling_llm_chain | parse_labels"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 148,
|
||||
"id": "59c06eea-ecbf-43af-a292-71816ccd92b8",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"final_taxonomy = step[\"__end__\"][\"clusters\"][-1]\n",
|
||||
"xml_taxonomy = format_taxonomy(final_taxonomy)\n",
|
||||
"results = labeling_chain.batch(\n",
|
||||
" [\n",
|
||||
" {\n",
|
||||
" \"content\": doc[\"content\"],\n",
|
||||
" \"taxonomy\": xml_taxonomy,\n",
|
||||
" }\n",
|
||||
" for doc in docs\n",
|
||||
" ],\n",
|
||||
" {\"max_concurrency\": 5},\n",
|
||||
" return_exceptions=True,\n",
|
||||
")\n",
|
||||
"# Update the docs to include the categories\n",
|
||||
"updated_docs = [{**doc, **category} for doc, category in zip(docs, results)]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 149,
|
||||
"id": "c21f787e-2dcb-49c2-9cc1-5284a1732fbc",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from langchain_openai import OpenAIEmbeddings\n",
|
||||
"\n",
|
||||
"encoder = OpenAIEmbeddings(model=\"text-embedding-3-large\")\n",
|
||||
"vectors = encoder.embed_documents([doc[\"content\"] for doc in docs])\n",
|
||||
"embedded_docs = [{**doc, \"embedding\": v} for doc, v in zip(updated_docs, vectors)]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "72284867-727d-467a-891a-7c7881967675",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"#### Train Classifier\n",
|
||||
"\n",
|
||||
"Now that we've extracted the features from the text, we can generate the classifier on them."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "d8b8d386-d1eb-4894-abe7-9346ca363f50",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"%pip install sklearn"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 196,
|
||||
"id": "eb19d42b-97c9-466b-b7b3-c17cc4db5180",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Train Accuracy: 0.515\n",
|
||||
"Test Accuracy: 0.330\n",
|
||||
"Train F1 Score: 0.493\n",
|
||||
"Test F1 Score: 0.335\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"import numpy as np\n",
|
||||
"from sklearn.linear_model import LogisticRegression\n",
|
||||
"from sklearn.metrics import accuracy_score, f1_score, roc_auc_score\n",
|
||||
"from sklearn.model_selection import train_test_split\n",
|
||||
"from sklearn.preprocessing import label_binarize\n",
|
||||
"from sklearn.utils import class_weight\n",
|
||||
"\n",
|
||||
"# Create a dictionary mapping category names to their indices in the taxonomy\n",
|
||||
"category_to_index = {d[\"name\"]: i for i, d in enumerate(final_taxonomy)}\n",
|
||||
"category_to_index[\"Other\"] = len(category_to_index)\n",
|
||||
"# Convert category strings to numeric labels\n",
|
||||
"labels = [\n",
|
||||
" category_to_index.get(d[\"category\"], category_to_index[\"Other\"])\n",
|
||||
" for d in embedded_docs\n",
|
||||
"]\n",
|
||||
"\n",
|
||||
"label_vectors = [d[\"embedding\"] for d in embedded_docs]\n",
|
||||
"\n",
|
||||
"X_train, X_test, y_train, y_test = train_test_split(\n",
|
||||
" label_vectors, labels, test_size=0.2, random_state=42\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"# Calculate class weights\n",
|
||||
"class_weights = class_weight.compute_class_weight(\n",
|
||||
" class_weight=\"balanced\", classes=np.unique(y_train), y=y_train\n",
|
||||
")\n",
|
||||
"class_weight_dict = dict(enumerate(class_weights))\n",
|
||||
"\n",
|
||||
"# Weight the classes to partially handle imbalanced data\n",
|
||||
"model = LogisticRegression(class_weight=class_weight_dict)\n",
|
||||
"model.fit(X_train, y_train)\n",
|
||||
"\n",
|
||||
"train_preds = model.predict(X_train)\n",
|
||||
"test_preds = model.predict(X_test)\n",
|
||||
"\n",
|
||||
"train_acc = accuracy_score(y_train, train_preds)\n",
|
||||
"test_acc = accuracy_score(y_test, test_preds)\n",
|
||||
"train_f1 = f1_score(y_train, train_preds, average=\"weighted\")\n",
|
||||
"test_f1 = f1_score(y_test, test_preds, average=\"weighted\")\n",
|
||||
"\n",
|
||||
"print(f\"Train Accuracy: {train_acc:.3f}\")\n",
|
||||
"print(f\"Test Accuracy: {test_acc:.3f}\")\n",
|
||||
"print(f\"Train F1 Score: {train_f1:.3f}\")\n",
|
||||
"print(f\"Test F1 Score: {test_f1:.3f}\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "32e7e994-a3c3-47f3-a860-32b847dfefee",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Deploy\n",
|
||||
"\n",
|
||||
"Now that you have your classifier, you can easily deploy it and apply to future runs! All you need is to embed the input and apply your LogisticRegression classifier. Let's try it.\n",
|
||||
"\n",
|
||||
"Assuming you've had some more data come in, you can fetch it and apply it below"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 197,
|
||||
"id": "c27cbb6b-4d0f-476a-bef3-31ed307ce45f",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from joblib import dump as jl_dump\n",
|
||||
"\n",
|
||||
"categories = list(category_to_index)\n",
|
||||
"\n",
|
||||
"# Save the model and categories to a file\n",
|
||||
"with open(\"model.joblib\", \"wb\") as file:\n",
|
||||
" jl_dump((model, categories), file)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "443f5f92-7f3c-4ce8-9104-b2b32ace0c42",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"#### To deploy\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 198,
|
||||
"id": "28f0b88a-b308-4208-b482-6c157357dfc6",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from joblib import load as jl_load\n",
|
||||
"from langchain_openai import OpenAIEmbeddings\n",
|
||||
"\n",
|
||||
"loaded_model, loaded_categories = jl_load(\"model.joblib\")\n",
|
||||
"encoder = OpenAIEmbeddings(model=\"text-embedding-3-large\")\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def get_category_name(predictions):\n",
|
||||
" return [loaded_categories[pred] for pred in predictions]\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"classifier = (\n",
|
||||
" RunnableLambda(encoder.embed_documents, encoder.aembed_documents)\n",
|
||||
" | loaded_model.predict\n",
|
||||
" | get_category_name\n",
|
||||
")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 194,
|
||||
"id": "6cdb9d8a-2aa1-4f48-8b23-f311fdf36416",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"client = Client()\n",
|
||||
"\n",
|
||||
"past_5_min = datetime.now() - timedelta(minutes=5)\n",
|
||||
"runs = list(\n",
|
||||
" client.list_runs(\n",
|
||||
" project_name=project_name,\n",
|
||||
" filter=\"eq(is_root, true)\",\n",
|
||||
" start_time=past_5_min,\n",
|
||||
" # We only need to return the inputs + outputs\n",
|
||||
" select=[\"inputs\", \"outputs\"],\n",
|
||||
" limit=100,\n",
|
||||
" )\n",
|
||||
")\n",
|
||||
"docs = [run_to_doc(r) for r in runs]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 199,
|
||||
"id": "b154439b-902e-42f3-afbc-89edb0cfea81",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"INFO:httpx:HTTP Request: POST https://api.openai.com/v1/embeddings \"HTTP/1.1 200 OK\"\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"['Interacting with Databases', 'Optimizing Conversational Retrieval']\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"classes = classifier.invoke([doc[\"content\"] for doc in docs])\n",
|
||||
"print(classes[:2])"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "fe415d54-fa94-4c62-b68b-d4437c446d42",
|
||||
|
||||
Reference in New Issue
Block a user