Skip to content

Commit 1d9aaae

Browse files
authored
feat: record the outputs_to_string tool result on the Agent tool span (#13720)
1 parent 9ed539b commit 1d9aaae

3 files changed

Lines changed: 162 additions & 27 deletions

File tree

‎haystack/components/agents/tool_calling.py‎

Lines changed: 59 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -188,7 +188,8 @@ def _create_tool_span(tool: Tool, tool_call: ToolCall, parent_span: tracing.Span
188188
"execute tool" span per invocation
189189
(https://github.com/open-telemetry/semantic-conventions-genai/blob/main/docs/gen-ai/gen-ai-agent-spans.md#execute-tool-span),
190190
and tracing backends such as Langfuse follow the same model. The span is tagged with the tool's identity; the caller
191-
adds the call arguments and result as content tags, and sets `haystack.tool.error` to `True` when the call fails.
191+
adds the call arguments, the raw tool output, and the result string the LLM sees as content tags, and sets
192+
`haystack.tool.error` to `True` when the call fails.
192193
193194
The parent is passed in rather than read from the tracer here: the calls of a step run concurrently, so a tracer
194195
that tracks the active span globally would report a sibling tool span as current and the spans would end up chained
@@ -202,18 +203,19 @@ def _create_tool_span(tool: Tool, tool_call: ToolCall, parent_span: tracing.Span
202203

203204

204205
def _make_context_bound_invoke(
205-
tool: Tool, args: dict[str, Any], tool_call: ToolCall, parent_span: tracing.Span | None
206-
) -> Callable[[], Any]:
206+
tool: Tool, args: dict[str, Any], tool_call: ToolCall, parent_span: tracing.Span | None, *, raise_on_failure: bool
207+
) -> Callable[[], ToolInvocationError | tuple[Any, ChatMessage]]:
207208
"""
208209
Return a zero-arg callable that runs `tool.invoke(**args)` under the current contextvars snapshot.
209210
210211
This preserves tracing spans and other context-local state across thread-pool boundaries, so spans opened by the
211-
tool itself stay attached to the calling trace. The callable returns a ToolInvocationError instead of raising so
212-
that parallel executions can collect failures without aborting the whole batch.
212+
tool itself stay attached to the calling trace. The callable returns the raw result paired with its tool-result
213+
message, or the ToolInvocationError instead of raising so that parallel executions can collect failures without
214+
aborting the whole batch.
213215
"""
214216
ctx = contextvars.copy_context()
215217

216-
def _invoke() -> Any:
218+
def _invoke() -> ToolInvocationError | tuple[Any, ChatMessage]:
217219
with _create_tool_span(tool, tool_call, parent_span) as span:
218220
span.set_content_tag("haystack.agent.step.tool.input", tool_call.arguments)
219221
try:
@@ -223,7 +225,11 @@ def _invoke() -> Any:
223225
span.set_content_tag("haystack.agent.step.tool.output", {"error": str(e)})
224226
return e
225227
span.set_content_tag("haystack.agent.step.tool.output", result)
226-
return result
228+
message = _build_tool_result_message(
229+
result=result, tool_call=tool_call, tool=tool, raise_on_failure=raise_on_failure
230+
)
231+
span.set_content_tag("haystack.agent.step.tool.result", message.tool_call_results[0].result)
232+
return result, message
227233

228234
def _runner() -> Any:
229235
return ctx.run(_invoke)
@@ -237,6 +243,8 @@ def _make_bounded_invoke_async(
237243
semaphore: asyncio.Semaphore,
238244
tool_call: ToolCall,
239245
parent_span: tracing.Span | None,
246+
*,
247+
raise_on_failure: bool,
240248
) -> Callable[[], Any]:
241249
"""
242250
Return a zero-arg async callable that awaits `tool.invoke_async(**args)` while holding `semaphore`.
@@ -246,11 +254,11 @@ def _make_bounded_invoke_async(
246254
the native-async branch, and `asyncio.to_thread` propagates them for the fallback branch, so spans opened by
247255
the tool itself stay attached to the calling trace.
248256
249-
Returns a `ToolInvocationError` instead of raising so that gathered executions can collect failures
250-
without aborting the whole batch.
257+
The callable returns the raw result paired with its tool-result message, or the `ToolInvocationError` instead of
258+
raising so that gathered executions can collect failures without aborting the whole batch.
251259
"""
252260

253-
async def _runner() -> Any:
261+
async def _runner() -> ToolInvocationError | tuple[Any, ChatMessage]:
254262
async with semaphore:
255263
with _create_tool_span(tool, tool_call, parent_span) as span:
256264
span.set_content_tag("haystack.agent.step.tool.input", tool_call.arguments)
@@ -261,7 +269,11 @@ async def _runner() -> Any:
261269
span.set_content_tag("haystack.agent.step.tool.output", {"error": str(e)})
262270
return e
263271
span.set_content_tag("haystack.agent.step.tool.output", result)
264-
return result
272+
message = _build_tool_result_message(
273+
result=result, tool_call=tool_call, tool=tool, raise_on_failure=raise_on_failure
274+
)
275+
span.set_content_tag("haystack.agent.step.tool.result", message.tool_call_results[0].result)
276+
return result, message
265277

266278
return _runner
267279

@@ -496,22 +508,29 @@ def _schedule_tool_calls(tool_calls: list[ToolCall], tools: list[Tool]) -> list[
496508

497509

498510
def _finalize_tool_result(
499-
result: Any, tool_call: ToolCall, tool: Tool, state: State, *, raise_on_failure: bool
511+
outcome: ToolInvocationError | tuple[Any, ChatMessage],
512+
tool_call: ToolCall,
513+
tool: Tool,
514+
state: State,
515+
*,
516+
raise_on_failure: bool,
500517
) -> ChatMessage:
501518
"""
502-
Turn a single tool invocation result into a tool-result ChatMessage, merging outputs into State.
519+
Merge a single tool invocation outcome into State and return its tool-result ChatMessage.
503520
504521
On a `ToolInvocationError`, either re-raise (when `raise_on_failure`) or return an error message. Otherwise
505-
merge the tool's outputs into State and build the result message.
522+
`outcome` is the raw result and the message built inside the tool span: merge the result into State and return
523+
the message.
506524
"""
507-
if isinstance(result, ToolInvocationError):
525+
if isinstance(outcome, ToolInvocationError):
508526
if raise_on_failure:
509-
raise result
510-
logger.error("{error_exception}", error_exception=result)
511-
return ChatMessage.from_tool(tool_result=str(result), origin=tool_call, error=True)
527+
raise outcome
528+
logger.error("{error_exception}", error_exception=outcome)
529+
return ChatMessage.from_tool(tool_result=str(outcome), origin=tool_call, error=True)
512530

531+
result, message = outcome
513532
_merge_tool_outputs_into_state(tool, result, state)
514-
return _build_tool_result_message(result, tool_call, tool, raise_on_failure=raise_on_failure)
533+
return message
515534

516535

517536
def _run_tool(
@@ -574,17 +593,21 @@ def _run_tool(
574593
)
575594
futures[idx] = executor.submit(
576595
_make_context_bound_invoke(
577-
tool=resolved_tools[idx], args=args, tool_call=tool_calls[idx], parent_span=parent_span
596+
tool=resolved_tools[idx],
597+
args=args,
598+
tool_call=tool_calls[idx],
599+
parent_span=parent_span,
600+
raise_on_failure=raise_on_failure,
578601
)
579602
)
580603

581604
# Merge results in call order within the batch so write-write merges stay deterministic.
582605
for idx in batch:
583606
message = _finalize_tool_result(
584-
futures[idx].result(),
585-
tool_calls[idx],
586-
resolved_tools[idx],
587-
state,
607+
outcome=futures[idx].result(),
608+
tool_call=tool_calls[idx],
609+
tool=resolved_tools[idx],
610+
state=state,
588611
raise_on_failure=raise_on_failure,
589612
)
590613
results[idx] = message
@@ -669,13 +692,22 @@ async def _run_tool_async(
669692
semaphore=semaphore,
670693
tool_call=tool_calls[idx],
671694
parent_span=parent_span,
695+
raise_on_failure=raise_on_failure,
672696
)()
673-
batch_results = await asyncio.gather(*tasks.values())
697+
# Wait for every call of the batch even if one raises, so no tool keeps running after the step fails
698+
batch_results = await asyncio.gather(*tasks.values(), return_exceptions=True)
674699

675700
# Merge results in call order within the batch so write-write merges stay deterministic.
676-
for idx, result in zip(tasks.keys(), batch_results, strict=True):
701+
for idx, outcome in zip(tasks.keys(), batch_results, strict=True):
702+
# A raised exception, such as a failing `outputs_to_string` handler, is re-raised in call order
703+
if isinstance(outcome, BaseException) and not isinstance(outcome, ToolInvocationError):
704+
raise outcome
677705
message = _finalize_tool_result(
678-
result, tool_calls[idx], resolved_tools[idx], state, raise_on_failure=raise_on_failure
706+
outcome=outcome,
707+
tool_call=tool_calls[idx],
708+
tool=resolved_tools[idx],
709+
state=state,
710+
raise_on_failure=raise_on_failure,
679711
)
680712
results[idx] = message
681713
if streaming_callback is not None:
Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
---
2+
enhancements:
3+
- |
4+
The ``haystack.agent.step.tool`` tracing span of the ``Agent`` now carries a ``haystack.agent.step.tool.result``
5+
content tag with the tool result string the LLM receives, after any ``outputs_to_string`` source or handler is
6+
applied. ``haystack.agent.step.tool.output`` still records the raw value returned by the tool.

‎test/components/agents/test_agent.py‎

Lines changed: 97 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
import os
77
import re
88
import threading
9+
import time
910
from collections.abc import Callable, Iterator
1011
from datetime import datetime
1112
from typing import Any
@@ -91,6 +92,37 @@ def weather_tool():
9192
)
9293

9394

95+
@pytest.fixture
96+
def uppercase_weather_tool():
97+
return Tool(
98+
name="weather_tool",
99+
description="Provides weather information for a given location.",
100+
parameters={"type": "object", "properties": {"location": {"type": "string"}}, "required": ["location"]},
101+
function=weather_function,
102+
outputs_to_string={"source": "weather", "handler": str.upper},
103+
)
104+
105+
106+
@pytest.fixture
107+
def failing_handler_weather_tool():
108+
def slow_paris_weather_function(location: str) -> dict[str, Any]:
109+
# Paris finishes after Berlin's handler has raised
110+
if location == "Paris":
111+
time.sleep(0.1)
112+
return weather_function(location)
113+
114+
def failing_handler(value: str) -> str:
115+
raise ValueError("cannot format weather")
116+
117+
return Tool(
118+
name="weather_tool",
119+
description="Provides weather information for a given location.",
120+
parameters={"type": "object", "properties": {"location": {"type": "string"}}, "required": ["location"]},
121+
function=slow_paris_weather_function,
122+
outputs_to_string={"source": "weather", "handler": failing_handler},
123+
)
124+
125+
94126
@pytest.fixture
95127
def failing_tool():
96128
def failing_function(location: str) -> dict[str, str]:
@@ -1510,10 +1542,75 @@ def test_tracing_span_run_with_tool_call(self, spying_tracer, weather_tool):
15101542
"haystack.tool.description": "Provides weather information for a given location.",
15111543
"haystack.agent.step.tool.input": {"location": "Berlin"},
15121544
"haystack.agent.step.tool.output": {"weather": "mostly sunny", "temperature": 7, "unit": "celsius"},
1545+
"haystack.agent.step.tool.result": '{"weather": "mostly sunny", "temperature": 7, "unit": "celsius"}',
15131546
}
15141547

15151548
assert spying_tracer.spans[0].tags["haystack.agent.steps_taken"] == 2
15161549

1550+
def test_tracing_span_tool_result_uses_outputs_to_string(self, spying_tracer, uppercase_weather_tool):
1551+
"""The tool span records the result the LLM sees, after the `outputs_to_string` handler ran."""
1552+
agent = Agent(chat_generator=_parallel_tool_calling_generator(), tools=[uppercase_weather_tool])
1553+
1554+
agent.run([ChatMessage.from_user("What's the weather in Berlin and Paris?")])
1555+
1556+
tool_spans = [s for s in spying_tracer.spans if s.operation_name == "haystack.agent.step.tool"]
1557+
assert {
1558+
(span.tags["haystack.agent.step.tool.output"]["weather"], span.tags["haystack.agent.step.tool.result"])
1559+
for span in tool_spans
1560+
} == {("mostly sunny", "MOSTLY SUNNY"), ("mostly cloudy", "MOSTLY CLOUDY")}
1561+
1562+
@pytest.mark.asyncio
1563+
async def test_tracing_span_run_async_tool_result_uses_outputs_to_string(
1564+
self, spying_tracer, uppercase_weather_tool
1565+
):
1566+
"""The async path also records the `outputs_to_string` result on each tool span."""
1567+
agent = Agent(chat_generator=_parallel_tool_calling_generator(), tools=[uppercase_weather_tool])
1568+
1569+
await agent.run_async([ChatMessage.from_user("What's the weather in Berlin and Paris?")])
1570+
1571+
tool_spans = [s for s in spying_tracer.spans if s.operation_name == "haystack.agent.step.tool"]
1572+
assert {
1573+
(span.tags["haystack.agent.step.tool.output"]["weather"], span.tags["haystack.agent.step.tool.result"])
1574+
for span in tool_spans
1575+
} == {("mostly sunny", "MOSTLY SUNNY"), ("mostly cloudy", "MOSTLY CLOUDY")}
1576+
1577+
def test_tracing_span_run_with_failing_outputs_to_string_handler(self, spying_tracer, failing_handler_weather_tool):
1578+
agent = Agent(
1579+
chat_generator=_parallel_tool_calling_generator(),
1580+
tools=[failing_handler_weather_tool],
1581+
raise_on_tool_invocation_failure=True,
1582+
)
1583+
1584+
with pytest.raises(ValueError, match="cannot format weather"):
1585+
agent.run([ChatMessage.from_user("What's the weather in Berlin and Paris?")])
1586+
1587+
tool_spans = [s for s in spying_tracer.spans if s.operation_name == "haystack.agent.step.tool"]
1588+
assert len(tool_spans) == 2
1589+
for span in tool_spans:
1590+
assert "haystack.agent.step.tool.output" in span.tags
1591+
assert "haystack.agent.step.tool.result" not in span.tags
1592+
assert spying_tracer.current_span() is None
1593+
1594+
@pytest.mark.asyncio
1595+
async def test_tracing_span_run_async_with_failing_outputs_to_string_handler(
1596+
self, spying_tracer, failing_handler_weather_tool
1597+
):
1598+
agent = Agent(
1599+
chat_generator=_parallel_tool_calling_generator(),
1600+
tools=[failing_handler_weather_tool],
1601+
raise_on_tool_invocation_failure=True,
1602+
)
1603+
1604+
with pytest.raises(ValueError, match="cannot format weather"):
1605+
await agent.run_async([ChatMessage.from_user("What's the weather in Berlin and Paris?")])
1606+
1607+
tool_spans = [s for s in spying_tracer.spans if s.operation_name == "haystack.agent.step.tool"]
1608+
assert len(tool_spans) == 2
1609+
for span in tool_spans:
1610+
assert "haystack.agent.step.tool.output" in span.tags
1611+
assert "haystack.agent.step.tool.result" not in span.tags
1612+
assert spying_tracer.current_span() is None
1613+
15171614
def test_tracing_span_run_with_failing_tool_call(self, spying_tracer, failing_tool):
15181615
agent = Agent(chat_generator=_parallel_tool_calling_generator(), tools=[failing_tool])
15191616

0 commit comments

Comments
 (0)