From b440b14fa7d70bfd10e1b697f8557151185399bc Mon Sep 17 00:00:00 2001 From: Eugene Yurtsev Date: Tue, 18 Mar 2025 21:53:56 -0400 Subject: [PATCH 1/5] x --- libs/langgraph/bench/__main__.py | 46 ++++++++++++++++++++++++++++++ libs/langgraph/bench/sequential.py | 2 +- 2 files changed, 47 insertions(+), 1 deletion(-) diff --git a/libs/langgraph/bench/__main__.py b/libs/langgraph/bench/__main__.py index f45ffd5e0..0841282bf 100644 --- a/libs/langgraph/bench/__main__.py +++ b/libs/langgraph/bench/__main__.py @@ -29,6 +29,21 @@ async def arun(graph: Pregel, input: dict): ) +async def arun_first_event_latency(graph: Pregel, input: dict) -> None: + """Latency for the first event. + + Run the graph until the first event is processed and then stop. + """ + async for _ in graph.astream( + input, + { + "configurable": {"thread_id": str(uuid4())}, + "recursion_limit": 1000000000, + }, + ): + break + + def run(graph: Pregel, input: dict): len( [ @@ -44,6 +59,21 @@ def run(graph: Pregel, input: dict): ) +def run_first_event_latency(graph: Pregel, input: dict) -> None: + """Latency for the first event. + + Run the graph until the first event is processed and then stop. + """ + for _ in graph.stream( + input, + { + "configurable": {"thread_id": str(uuid4())}, + "recursion_limit": 1000000000, + }, + ): + break + + benchmarks = ( ( "fanout_to_subgraph_10x", @@ -330,7 +360,23 @@ benchmarks = ( r = Runner() +# Full graph run time for name, agraph, graph, input in benchmarks: r.bench_async_func(name, arun, agraph, input, loop_factory=new_event_loop) if graph is not None: r.bench_func(name + "_sync", run, graph, input) + + +# First event latency +for name, agraph, graph, input in benchmarks: + r.bench_async_func( + name + "_first_event_latency", + arun_first_event_latency, + agraph, + input, + loop_factory=new_event_loop, + ) + if graph is not None: + r.bench_func( + name + "_first_event_latency_sync", run_first_event_latency, graph, input + ) diff --git a/libs/langgraph/bench/sequential.py b/libs/langgraph/bench/sequential.py index 3ab92912f..ef3ce6329 100644 --- a/libs/langgraph/bench/sequential.py +++ b/libs/langgraph/bench/sequential.py @@ -4,7 +4,7 @@ from langgraph.graph import MessagesState, StateGraph from langgraph.utils.runnable import RunnableCallable -def create_sequential(number_nodes) -> StateGraph: +def create_sequential(number_nodes: int) -> StateGraph: """Create a sequential no-op graph consisting of a few hundred nodes.""" builder = StateGraph(MessagesState) From 4b9cdb4107282eefbff44af289b9ff99065e8ce8 Mon Sep 17 00:00:00 2001 From: Eugene Yurtsev Date: Tue, 18 Mar 2025 22:16:10 -0400 Subject: [PATCH 2/5] add compilation benchmark --- libs/langgraph/bench/__main__.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/libs/langgraph/bench/__main__.py b/libs/langgraph/bench/__main__.py index 0841282bf..d6d57543a 100644 --- a/libs/langgraph/bench/__main__.py +++ b/libs/langgraph/bench/__main__.py @@ -11,6 +11,7 @@ from bench.react_agent import react_agent from bench.sequential import create_sequential from bench.wide_state import wide_state from langgraph.checkpoint.memory import MemorySaver +from langgraph.graph import StateGraph from langgraph.pregel import Pregel @@ -74,6 +75,11 @@ def run_first_event_latency(graph: Pregel, input: dict) -> None: break +def compile_graph(graph: StateGraph) -> None: + """Compile the graph.""" + graph.compile() + + benchmarks = ( ( "fanout_to_subgraph_10x", @@ -380,3 +386,18 @@ for name, agraph, graph, input in benchmarks: r.bench_func( name + "_first_event_latency_sync", run_first_event_latency, graph, input ) + +# Graph compilation times +compilation_benchmarks = ( + ( + "sequential_1000_compilation", + create_sequential(1_000), + ), + ( + "sequential_10000_compilation", + create_sequential(10_000), + ), +) + +for name, graph, input in compilation_benchmarks: + r.bench_func(name, "_compilation", compile_graph, graph, input) From cb95821eb0f0f40918e2ae1999b19ae82dda6e15 Mon Sep 17 00:00:00 2001 From: Eugene Yurtsev Date: Tue, 18 Mar 2025 22:20:18 -0400 Subject: [PATCH 3/5] x --- libs/langgraph/bench/__main__.py | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/libs/langgraph/bench/__main__.py b/libs/langgraph/bench/__main__.py index d6d57543a..45986e7f0 100644 --- a/libs/langgraph/bench/__main__.py +++ b/libs/langgraph/bench/__main__.py @@ -390,13 +390,33 @@ for name, agraph, graph, input in benchmarks: # Graph compilation times compilation_benchmarks = ( ( - "sequential_1000_compilation", + "sequential_1000", create_sequential(1_000), ), ( - "sequential_10000_compilation", + "sequential_10000", create_sequential(10_000), ), + ( + "pydantic_state_25x300", + pydantic_state(300), + ), + ( + "pydantic_state_15x600", + pydantic_state(600), + ), + ( + "pydantic_state_9x1200", + pydantic_state(1200), + ), + ( + "wide_state_15x600", + wide_state(600), + ), + ( + "wide_state_9x1200", + wide_state(1200), + ), ) for name, graph, input in compilation_benchmarks: From 347ab0165e41159c176bcba3bd956690f0156218 Mon Sep 17 00:00:00 2001 From: Eugene Yurtsev Date: Tue, 18 Mar 2025 22:23:39 -0400 Subject: [PATCH 4/5] x --- libs/langgraph/bench/__main__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libs/langgraph/bench/__main__.py b/libs/langgraph/bench/__main__.py index 45986e7f0..61c8581f7 100644 --- a/libs/langgraph/bench/__main__.py +++ b/libs/langgraph/bench/__main__.py @@ -419,5 +419,5 @@ compilation_benchmarks = ( ), ) -for name, graph, input in compilation_benchmarks: - r.bench_func(name, "_compilation", compile_graph, graph, input) +for name, graph in compilation_benchmarks: + r.bench_func(name, "_compilation", compile_graph, graph) From 8be0fb675a48c6bcbbbaca2343960281bbc7cd46 Mon Sep 17 00:00:00 2001 From: Eugene Yurtsev Date: Fri, 21 Mar 2025 15:25:28 -0400 Subject: [PATCH 5/5] x --- libs/langgraph/bench/__main__.py | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/libs/langgraph/bench/__main__.py b/libs/langgraph/bench/__main__.py index ad6c7e65d..06ec43a28 100644 --- a/libs/langgraph/bench/__main__.py +++ b/libs/langgraph/bench/__main__.py @@ -35,14 +35,19 @@ async def arun_first_event_latency(graph: Pregel, input: dict) -> None: Run the graph until the first event is processed and then stop. """ - async for _ in graph.astream( + stream = graph.astream( input, { "configurable": {"thread_id": str(uuid4())}, "recursion_limit": 1000000000, }, - ): - break + ) + + try: + async for _ in stream: + break + finally: + await stream.aclose() def run(graph: Pregel, input: dict): @@ -65,14 +70,19 @@ def run_first_event_latency(graph: Pregel, input: dict) -> None: Run the graph until the first event is processed and then stop. """ - for _ in graph.stream( + stream = graph.stream( input, { "configurable": {"thread_id": str(uuid4())}, "recursion_limit": 1000000000, }, - ): - break + ) + + try: + for _ in stream: + break + finally: + stream.close() def compile_graph(graph: StateGraph) -> None: