adding tutorials to notebook checks (#1747)

* wip

* actually catch errors

* fixing tool-calling-errors and persistence-redis

* poetry changes

* poetry update

* run tutorials

* only tutorials (testing)

* print errors

* rewoo fixes

* remove customer support because of user input

* skip notebooks programatically

* add back how-tos

* remove redundant if

* add cassetes for tutorials

* remove no execution since it is generated by CI,

* remove multi-agent/usaco

* try to run in parallel

* skip notebooks fix

* remove magic/non-magic cells

* msgpack instead of yaml

* prepare_notebooks change

* ignore msgpack for spelling

* use compression for cassettes

* update spell check

* reset poetry changes

* upgrade packages for latest

* no update

* poetry changes

docs/_scripts/execute_notebooks.sh

@@ -1,5 +1,32 @@
-for file in $(find docs/docs/how-tos -name "*.ipynb" | grep -v ".ipynb_checkpoints")
-do
-  echo "Executing $file"
-  poetry run jupyter execute "$file"
-done
+#!/bin/bash
+
+# Read the list of notebooks to skip from the JSON file
+SKIP_NOTEBOOKS=$(python -c "import json; print('\n'.join(json.load(open('docs/notebooks_no_execution.json'))))")
+
+# Function to execute a single notebook
+execute_notebook() {
+    file="$1"
+    echo "Starting execution of $file"
+    start_time=$(date +%s)
+    if ! output=$(time poetry run jupyter execute --allow-errors "$file" 2>&1); then
+        end_time=$(date +%s)
+        execution_time=$((end_time - start_time))
+        echo "Error in $file. Execution time: $execution_time seconds"
+        echo "Error details: $output"
+        exit 1
+    fi
+    end_time=$(date +%s)
+    execution_time=$((end_time - start_time))
+    echo "Finished $file. Execution time: $execution_time seconds"
+}
+
+export -f execute_notebook
+
+# Find all notebooks and filter out those in the skip list
+notebooks=$(find docs/docs/tutorials docs/docs/how-tos -name "*.ipynb" | grep -v ".ipynb_checkpoints" | grep -vFf <(echo "$SKIP_NOTEBOOKS"))
+
+# Run notebooks in parallel
+if ! parallel execute_notebook ::: $notebooks; then
+    echo "Errors occurred during notebook execution"
+    exit 1
+fi

docs/_scripts/prepare_notebooks_for_ci.py

@@ -2,12 +2,12 @@
 
 import logging
 import os
-
+import json
 import click
 import nbformat
 
 logger = logging.getLogger(__name__)
-NOTEBOOK_DIRS = ("docs/docs/how-tos",)
+NOTEBOOK_DIRS = ("docs/docs/how-tos","docs/docs/tutorials")
 DOCS_PATH = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 CASSETTES_PATH = os.path.join(DOCS_PATH, "cassettes")
 
@@ -16,6 +16,11 @@ NOTEBOOKS_NO_CASSETTES = (
     "docs/docs/how-tos/many-tools.ipynb"
 )
 
+NOTEBOOKS_NO_EXECUTION = [
+    "docs/docs/tutorials/customer-support/customer-support.ipynb",
+    "docs/docs/tutorials/usaco/usaco.ipynb",
+]
+
 
 def comment_install_cells(notebook: nbformat.NotebookNode) -> nbformat.NotebookNode:
     for cell in notebook.cells:
@@ -71,17 +76,71 @@ def add_vcr_to_notebook(
             continue
 
         cell_id = cell.get("id", idx)
-        cassette_name = f"{cassette_prefix}_{cell_id}.yaml"
-        cell.source = f"with vcr.use_cassette('{cassette_name}', filter_headers=['x-api-key', 'authorization'], record_mode='once'):\n" + "\n".join(
+        cassette_name = f"{cassette_prefix}_{cell_id}.msgpack.zlib"
+        cell.source = f"with custom_vcr.use_cassette('{cassette_name}', filter_headers=['x-api-key', 'authorization'], record_mode='once', serializer='advanced_compressed'):\n" + "\n".join(
             f"    {line}" for line in lines
         )
 
     # Add import statement
     vcr_import_lines = [
         "import vcr",
-        # this is needed for ChatAnthropic
+        "import msgpack",
         "import nest_asyncio",
+        "import base64",
+        "import zlib",
+        "import re",
+        "",
+        "custom_vcr = vcr.VCR()",
+        "",
+        "def compress_data(data, compression_level=9):",
+        "    packed = msgpack.packb(data, use_bin_type=True)",
+        "    compressed = zlib.compress(packed, level=compression_level)",
+        "    return base64.b64encode(compressed).decode('utf-8')",
+        "",
+        "def decompress_data(compressed_string):",
+        "    try:",
+        "        decoded = base64.b64decode(compressed_string)",
+        "        decompressed = zlib.decompress(decoded)",
+        "        return msgpack.unpackb(decompressed, raw=False)",
+        "    except (ValueError, zlib.error, msgpack.exceptions.ExtraData, msgpack.exceptions.UnpackValueError):",
+        "        return {\"requests\": [], \"responses\": []}",
+        "",
+        "def filter_cassette_data(cassette_dict):",
+        "    for interaction in cassette_dict['interactions']:",
+        "        if len(interaction['response']['body']['string']) > 1000:",
+        "            interaction['response']['body']['string'] = interaction['response']['body']['string'][:1000] + '... (truncated)'",
+        "        for req_or_res in [interaction['request'], interaction['response']]:",
+        "            headers_to_remove = ['date', 'server', 'content-length']",
+        "            for header in headers_to_remove:",
+        "                req_or_res['headers'].pop(header, None)",
+        "    return cassette_dict",
+        "",
+        "class AdvancedCompressedSerializer:",
+        "    def serialize(self, cassette_dict):",
+        "        filtered_dict = filter_cassette_data(cassette_dict)",
+        "        return compress_data(filtered_dict)",
+        "",
+        "    def deserialize(self, cassette_string):",
+        "        return decompress_data(cassette_string)",
+        "",
+        "custom_vcr.register_serializer('advanced_compressed', AdvancedCompressedSerializer())",
+        "",
+        "def custom_matcher(r1, r2):",
+        "    return (r1.method == r2.method and",
+        "            r1.url == r2.url and",
+        "            normalize_body(r1.body) == normalize_body(r2.body))",
+        "",
+        "def normalize_body(body):",
+        "    return re.sub(r'\\s+', '', body.lower()) if body else ''",
+        "",
         "nest_asyncio.apply()",
+        "",
+        "custom_vcr.serializer = 'advanced_compressed'",
+        "custom_vcr.record_mode = 'new_episodes'",
+        "custom_vcr.match_on = ['custom']",
+        "custom_vcr.register_matcher('custom', custom_matcher)",
+        "custom_vcr.filter_headers = ['authorization', 'user-agent', 'date', 'server']",
+        "custom_vcr.filter_post_data_parameters = ['password', 'token']",
     ]
     import_cell = nbformat.v4.new_code_cell(source="\n".join(vcr_import_lines))
     import_cell.pop("id", None)
@@ -110,10 +169,24 @@ def process_notebooks(should_comment_install_cells: bool) -> None:
                             notebook, cassette_prefix=cassette_prefix
                         )
 
+                    if notebook_path in NOTEBOOKS_NO_EXECUTION:
+                        # Add a cell at the beginning to indicate that this notebook should not be executed
+                        warning_cell = nbformat.v4.new_markdown_cell(
+                            source="**Warning:** This notebook is not meant to be executed automatically."
+                        )
+                        notebook.cells.insert(0, warning_cell)
+
+                        # Add a special tag to the first code cell
+                        if notebook.cells and notebook.cells[1].cell_type == "code":
+                            notebook.cells[1].metadata["tags"] = notebook.cells[1].metadata.get("tags", []) + ["no_execution"]
+
                     nbformat.write(notebook, notebook_path)
                     logger.info(f"Processed: {notebook_path}")
                 except Exception as e:
                     logger.error(f"Error processing {notebook_path}: {e}")
+    
+    with open(os.path.join(DOCS_PATH, "notebooks_no_execution.json"), "w") as f:
+        json.dump(NOTEBOOKS_NO_EXECUTION, f)
 
 
 @click.command()