mirror of
https://github.com/langchain-ai/langgraph.git
synced 2026-09-08 10:47:52 +02:00
Caspar BroekhuizenandGitHub
77d98b426b
This PR improves standard integration tests in prebuilt to ensure logical equivalence between the Python and JavaScript implementations of `create_agent`. * Cleans up `test_responses_int` test harness * Adds new test utils to dynamically load JSON test specs * Adds `test_return_direct_int` test harness to validate model behavior when `return_direct` tool property is set * Adds support for when the user instantiates `ToolOutput` with multiple JSON schemas unified by the `oneOf` keyword Failing tests: * `test_inference_to_native_output`: there is some odd behavior where the model makes a second call to `get_weather` despite having just received the tool message, so there are 6 messages instead of the 4 expected. * `test_responses_integration_matrix[asking for information that does not fit into the response format]`: `XFAIL`, currently failing due to undefined behavior when the model cannot conform to any of the structured response formats. TODO in future PRs: * Add exception handling to pass `test_responses_integration_matrix`.
87 lines
2.6 KiB
JSON
87 lines
2.6 KiB
JSON
[
|
|
{
|
|
"name": "updated structured response",
|
|
"responseFormat": [
|
|
{
|
|
"title": "role_schema_structured_output",
|
|
"type": "object",
|
|
"properties": {
|
|
"name": { "type": "string" },
|
|
"role": { "type": "string" }
|
|
},
|
|
"required": ["name", "role"]
|
|
},
|
|
{
|
|
"title": "department_schema_structured_output",
|
|
"type": "object",
|
|
"properties": {
|
|
"name": { "type": "string" },
|
|
"department": { "type": "string" }
|
|
},
|
|
"required": ["name", "department"]
|
|
}
|
|
],
|
|
"assertionsByInvocation": [
|
|
{
|
|
"prompt": "What is the role of Sabine?",
|
|
"toolsWithExpectedCalls": {
|
|
"getEmployeeRole": 1,
|
|
"getEmployeeDepartment": 0
|
|
},
|
|
"expectedLastMessage": "Returning structured response: {'name': 'Sabine', 'role': 'Developer'}",
|
|
"expectedStructuredResponse": { "name": "Sabine", "role": "Developer" },
|
|
"llmRequestCount": 2
|
|
},
|
|
{
|
|
"prompt": "In which department does Henrik work?",
|
|
"toolsWithExpectedCalls": {
|
|
"getEmployeeRole": 1,
|
|
"getEmployeeDepartment": 1
|
|
},
|
|
"expectedLastMessage": "Returning structured response: {'name': 'Henrik', 'department': 'IT'}",
|
|
"expectedStructuredResponse": { "name": "Henrik", "department": "IT" },
|
|
"llmRequestCount": 4
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"name": "asking for information that does not fit into the response format",
|
|
"responseFormat": [
|
|
{
|
|
"schema": {
|
|
"type": "object",
|
|
"properties": {
|
|
"name": { "type": "string" },
|
|
"role": { "type": "string" }
|
|
},
|
|
"required": ["name", "role"]
|
|
}
|
|
},
|
|
{
|
|
"schema": {
|
|
"type": "object",
|
|
"properties": {
|
|
"name": { "type": "string" },
|
|
"department": { "type": "string" }
|
|
},
|
|
"required": ["name", "department"]
|
|
}
|
|
}
|
|
],
|
|
"assertionsByInvocation": [
|
|
{
|
|
"prompt": "How much does Saskia earn?",
|
|
"toolsWithExpectedCalls": {
|
|
"getEmployeeRole": 1,
|
|
"getEmployeeDepartment": 0
|
|
},
|
|
"expectedLastMessage": "Returning structured response: {'name': 'Saskia', 'role': 'Software Engineer'}",
|
|
"expectedStructuredResponse": {
|
|
"name": "Saskia",
|
|
"role": "Software Engineer"
|
|
},
|
|
"llmRequestCount": 2
|
|
}
|
|
]
|
|
}
|
|
] |