Add an optional "main" chain which has implicit input and output channels

This commit is contained in:
Nuno Campos
2023-10-22 21:12:30 +01:00
parent 82d927a421
commit 9dfffe5fc0
9 changed files with 243 additions and 170 deletions
+7 -5
View File
@@ -66,7 +66,7 @@ def recursive_web_loader(
extractor = extractor or (lambda x: x)
metadata_extractor = metadata_extractor or _metadata_extractor
# the main chain that gets executed recursively
chain = (
visitor = (
# while there are urls in next_urls
# run the chain below for each url in next_urls
# adding the current values of visited set, base_url and httpx client
@@ -96,10 +96,12 @@ def recursive_web_loader(
)
)
return Pregel(
# use the base_url as the first url to visit
Pregel.subscribe_to("base_url") | Pregel.send_to("next_urls"),
# add the main chain
chain,
chains={
# use the base_url as the first url to visit
"input": Pregel.subscribe_to("base_url") | Pregel.send_to("next_urls"),
# add the main chain
"visitor": visitor,
},
# define the channels
channels={
"base_url": channels.LastValue(str),