perf: add benchmark profiling mode and fix quadratic repr bottleneck

Add --profile flag to bench/__main__.py that bypasses pyperf and runs
each benchmark under cProfile, printing per-benchmark hotspot summaries
and writing .prof files for later analysis. Add benchmark-profile and
benchmark-profile-spy Makefile targets.

Fix O(n^2) performance regression in _get_model_input_state where
f-strings eagerly evaluated repr(state) on every call, triggering
pydantic __repr__ across all accumulated messages. Move error message
construction into the error path so repr is only called when needed.
This yields a 3-5x speedup on react_agent_100x benchmarks.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
John Kennedy
2026-03-12 00:01:43 +00:00
co-authored by Claude Opus 4.6
parent f4a18e0409
commit d08be136b2
3 changed files with 66 additions and 8 deletions
+9 -1
View File
@@ -1,4 +1,4 @@
.PHONY: all format lint type test test_watch integration_tests spell_check spell_fix benchmark profile start-dev-server integration_tests
.PHONY: all format lint type test test_watch integration_tests spell_check spell_fix benchmark benchmark-profile benchmark-profile-spy profile start-dev-server integration_tests
# Default target executed when no arguments are given to make.
all: help
@@ -24,6 +24,14 @@ benchmark-fast:
rm -f $(OUTPUT)
uv run python -m bench -o $(OUTPUT) --fast
benchmark-profile:
mkdir -p out
uv run python -m bench --profile
benchmark-profile-spy:
mkdir -p out
sudo uv run py-spy record -g -o out/benchmark-flamegraph.svg -- python -m bench --fast
GRAPH ?= bench/fanout_to_subgraph.py
profile:
+49 -2
View File
@@ -1,10 +1,9 @@
import random
import sys
from uuid import uuid4
from langchain_core.messages import HumanMessage
from langgraph.checkpoint.memory import InMemorySaver
from pyperf._runner import Runner
from uvloop import new_event_loop
from bench.fanout_to_subgraph import fanout_to_subgraph, fanout_to_subgraph_sync
from bench.pydantic_state import pydantic_state
@@ -464,6 +463,54 @@ benchmarks = (
)
if "--profile" in sys.argv:
import asyncio
import cProfile
import pstats
from pathlib import Path
out_dir = Path("out")
out_dir.mkdir(exist_ok=True)
for name, agraph, graph, input_data in benchmarks:
# Profile async variant
prof = cProfile.Profile()
prof.enable()
asyncio.run(arun(agraph, input_data))
prof.disable()
prof_path = out_dir / f"{name}.prof"
prof.dump_stats(str(prof_path))
print(f"\n{'=' * 60}")
print(f"PROFILE: {name}")
print(f"{'=' * 60}")
stats = pstats.Stats(prof)
stats.sort_stats("cumulative")
stats.print_stats(20)
# Profile sync variant
if graph is not None:
prof = cProfile.Profile()
prof.enable()
run(graph, input_data)
prof.disable()
prof_path = out_dir / f"{name}_sync.prof"
prof.dump_stats(str(prof_path))
print(f"\n{'=' * 60}")
print(f"PROFILE: {name}_sync")
print(f"{'=' * 60}")
stats = pstats.Stats(prof)
stats.sort_stats("cumulative")
stats.print_stats(20)
sys.exit(0)
from pyperf._runner import Runner
from uvloop import new_event_loop
r = Runner()
# Full graph run time
@@ -638,15 +638,18 @@ def create_react_agent(
messages = (
_get_state_value(state, "llm_input_messages")
) or _get_state_value(state, "messages")
error_msg = f"Expected input to call_model to have 'llm_input_messages' or 'messages' key, but got {state}"
else:
messages = _get_state_value(state, "messages")
error_msg = (
f"Expected input to call_model to have 'messages' key, but got {state}"
)
if messages is None:
raise ValueError(error_msg)
if pre_model_hook is not None:
raise ValueError(
f"Expected input to call_model to have 'llm_input_messages' or 'messages' key, but got {state}"
)
else:
raise ValueError(
f"Expected input to call_model to have 'messages' key, but got {state}"
)
_validate_chat_history(messages)
# we're passing messages under `messages` key, as this is expected by the prompt