diff --git a/haystack/components/agents/tool_calling.py b/haystack/components/agents/tool_calling.py index d2fac33ad93..fc38627cf7d 100644 --- a/haystack/components/agents/tool_calling.py +++ b/haystack/components/agents/tool_calling.py @@ -188,7 +188,8 @@ def _create_tool_span(tool: Tool, tool_call: ToolCall, parent_span: tracing.Span "execute tool" span per invocation (https://github.com/open-telemetry/semantic-conventions-genai/blob/main/docs/gen-ai/gen-ai-agent-spans.md#execute-tool-span), and tracing backends such as Langfuse follow the same model. The span is tagged with the tool's identity; the caller - adds the call arguments and result as content tags, and sets `haystack.tool.error` to `True` when the call fails. + adds the call arguments, the raw tool output, and the result string the LLM sees as content tags, and sets + `haystack.tool.error` to `True` when the call fails. The parent is passed in rather than read from the tracer here: the calls of a step run concurrently, so a tracer that tracks the active span globally would report a sibling tool span as current and the spans would end up chained @@ -202,18 +203,19 @@ def _create_tool_span(tool: Tool, tool_call: ToolCall, parent_span: tracing.Span def _make_context_bound_invoke( - tool: Tool, args: dict[str, Any], tool_call: ToolCall, parent_span: tracing.Span | None -) -> Callable[[], Any]: + tool: Tool, args: dict[str, Any], tool_call: ToolCall, parent_span: tracing.Span | None, *, raise_on_failure: bool +) -> Callable[[], ToolInvocationError | tuple[Any, ChatMessage]]: """ Return a zero-arg callable that runs `tool.invoke(**args)` under the current contextvars snapshot. This preserves tracing spans and other context-local state across thread-pool boundaries, so spans opened by the - tool itself stay attached to the calling trace. The callable returns a ToolInvocationError instead of raising so - that parallel executions can collect failures without aborting the whole batch. + tool itself stay attached to the calling trace. The callable returns the raw result paired with its tool-result + message, or the ToolInvocationError instead of raising so that parallel executions can collect failures without + aborting the whole batch. """ ctx = contextvars.copy_context() - def _invoke() -> Any: + def _invoke() -> ToolInvocationError | tuple[Any, ChatMessage]: with _create_tool_span(tool, tool_call, parent_span) as span: span.set_content_tag("haystack.agent.step.tool.input", tool_call.arguments) try: @@ -223,7 +225,11 @@ def _invoke() -> Any: span.set_content_tag("haystack.agent.step.tool.output", {"error": str(e)}) return e span.set_content_tag("haystack.agent.step.tool.output", result) - return result + message = _build_tool_result_message( + result=result, tool_call=tool_call, tool=tool, raise_on_failure=raise_on_failure + ) + span.set_content_tag("haystack.agent.step.tool.result", message.tool_call_results[0].result) + return result, message def _runner() -> Any: return ctx.run(_invoke) @@ -237,6 +243,8 @@ def _make_bounded_invoke_async( semaphore: asyncio.Semaphore, tool_call: ToolCall, parent_span: tracing.Span | None, + *, + raise_on_failure: bool, ) -> Callable[[], Any]: """ Return a zero-arg async callable that awaits `tool.invoke_async(**args)` while holding `semaphore`. @@ -246,11 +254,11 @@ def _make_bounded_invoke_async( the native-async branch, and `asyncio.to_thread` propagates them for the fallback branch, so spans opened by the tool itself stay attached to the calling trace. - Returns a `ToolInvocationError` instead of raising so that gathered executions can collect failures - without aborting the whole batch. + The callable returns the raw result paired with its tool-result message, or the `ToolInvocationError` instead of + raising so that gathered executions can collect failures without aborting the whole batch. """ - async def _runner() -> Any: + async def _runner() -> ToolInvocationError | tuple[Any, ChatMessage]: async with semaphore: with _create_tool_span(tool, tool_call, parent_span) as span: span.set_content_tag("haystack.agent.step.tool.input", tool_call.arguments) @@ -261,7 +269,11 @@ async def _runner() -> Any: span.set_content_tag("haystack.agent.step.tool.output", {"error": str(e)}) return e span.set_content_tag("haystack.agent.step.tool.output", result) - return result + message = _build_tool_result_message( + result=result, tool_call=tool_call, tool=tool, raise_on_failure=raise_on_failure + ) + span.set_content_tag("haystack.agent.step.tool.result", message.tool_call_results[0].result) + return result, message return _runner @@ -496,22 +508,29 @@ def _schedule_tool_calls(tool_calls: list[ToolCall], tools: list[Tool]) -> list[ def _finalize_tool_result( - result: Any, tool_call: ToolCall, tool: Tool, state: State, *, raise_on_failure: bool + outcome: ToolInvocationError | tuple[Any, ChatMessage], + tool_call: ToolCall, + tool: Tool, + state: State, + *, + raise_on_failure: bool, ) -> ChatMessage: """ - Turn a single tool invocation result into a tool-result ChatMessage, merging outputs into State. + Merge a single tool invocation outcome into State and return its tool-result ChatMessage. On a `ToolInvocationError`, either re-raise (when `raise_on_failure`) or return an error message. Otherwise - merge the tool's outputs into State and build the result message. + `outcome` is the raw result and the message built inside the tool span: merge the result into State and return + the message. """ - if isinstance(result, ToolInvocationError): + if isinstance(outcome, ToolInvocationError): if raise_on_failure: - raise result - logger.error("{error_exception}", error_exception=result) - return ChatMessage.from_tool(tool_result=str(result), origin=tool_call, error=True) + raise outcome + logger.error("{error_exception}", error_exception=outcome) + return ChatMessage.from_tool(tool_result=str(outcome), origin=tool_call, error=True) + result, message = outcome _merge_tool_outputs_into_state(tool, result, state) - return _build_tool_result_message(result, tool_call, tool, raise_on_failure=raise_on_failure) + return message def _run_tool( @@ -574,17 +593,21 @@ def _run_tool( ) futures[idx] = executor.submit( _make_context_bound_invoke( - tool=resolved_tools[idx], args=args, tool_call=tool_calls[idx], parent_span=parent_span + tool=resolved_tools[idx], + args=args, + tool_call=tool_calls[idx], + parent_span=parent_span, + raise_on_failure=raise_on_failure, ) ) # Merge results in call order within the batch so write-write merges stay deterministic. for idx in batch: message = _finalize_tool_result( - futures[idx].result(), - tool_calls[idx], - resolved_tools[idx], - state, + outcome=futures[idx].result(), + tool_call=tool_calls[idx], + tool=resolved_tools[idx], + state=state, raise_on_failure=raise_on_failure, ) results[idx] = message @@ -669,13 +692,22 @@ async def _run_tool_async( semaphore=semaphore, tool_call=tool_calls[idx], parent_span=parent_span, + raise_on_failure=raise_on_failure, )() - batch_results = await asyncio.gather(*tasks.values()) + # Wait for every call of the batch even if one raises, so no tool keeps running after the step fails + batch_results = await asyncio.gather(*tasks.values(), return_exceptions=True) # Merge results in call order within the batch so write-write merges stay deterministic. - for idx, result in zip(tasks.keys(), batch_results, strict=True): + for idx, outcome in zip(tasks.keys(), batch_results, strict=True): + # A raised exception, such as a failing `outputs_to_string` handler, is re-raised in call order + if isinstance(outcome, BaseException) and not isinstance(outcome, ToolInvocationError): + raise outcome message = _finalize_tool_result( - result, tool_calls[idx], resolved_tools[idx], state, raise_on_failure=raise_on_failure + outcome=outcome, + tool_call=tool_calls[idx], + tool=resolved_tools[idx], + state=state, + raise_on_failure=raise_on_failure, ) results[idx] = message if streaming_callback is not None: diff --git a/releasenotes/notes/tool-span-result-tag-57e1ebf51829eafa.yaml b/releasenotes/notes/tool-span-result-tag-57e1ebf51829eafa.yaml new file mode 100644 index 00000000000..876dc3a7027 --- /dev/null +++ b/releasenotes/notes/tool-span-result-tag-57e1ebf51829eafa.yaml @@ -0,0 +1,6 @@ +--- +enhancements: + - | + The ``haystack.agent.step.tool`` tracing span of the ``Agent`` now carries a ``haystack.agent.step.tool.result`` + content tag with the tool result string the LLM receives, after any ``outputs_to_string`` source or handler is + applied. ``haystack.agent.step.tool.output`` still records the raw value returned by the tool. diff --git a/test/components/agents/test_agent.py b/test/components/agents/test_agent.py index a3b084c32ab..a47e4f1ad75 100644 --- a/test/components/agents/test_agent.py +++ b/test/components/agents/test_agent.py @@ -6,6 +6,7 @@ import os import re import threading +import time from collections.abc import Callable, Iterator from datetime import datetime from typing import Any @@ -91,6 +92,37 @@ def weather_tool(): ) +@pytest.fixture +def uppercase_weather_tool(): + return Tool( + name="weather_tool", + description="Provides weather information for a given location.", + parameters={"type": "object", "properties": {"location": {"type": "string"}}, "required": ["location"]}, + function=weather_function, + outputs_to_string={"source": "weather", "handler": str.upper}, + ) + + +@pytest.fixture +def failing_handler_weather_tool(): + def slow_paris_weather_function(location: str) -> dict[str, Any]: + # Paris finishes after Berlin's handler has raised + if location == "Paris": + time.sleep(0.1) + return weather_function(location) + + def failing_handler(value: str) -> str: + raise ValueError("cannot format weather") + + return Tool( + name="weather_tool", + description="Provides weather information for a given location.", + parameters={"type": "object", "properties": {"location": {"type": "string"}}, "required": ["location"]}, + function=slow_paris_weather_function, + outputs_to_string={"source": "weather", "handler": failing_handler}, + ) + + @pytest.fixture def failing_tool(): def failing_function(location: str) -> dict[str, str]: @@ -1510,10 +1542,75 @@ def test_tracing_span_run_with_tool_call(self, spying_tracer, weather_tool): "haystack.tool.description": "Provides weather information for a given location.", "haystack.agent.step.tool.input": {"location": "Berlin"}, "haystack.agent.step.tool.output": {"weather": "mostly sunny", "temperature": 7, "unit": "celsius"}, + "haystack.agent.step.tool.result": '{"weather": "mostly sunny", "temperature": 7, "unit": "celsius"}', } assert spying_tracer.spans[0].tags["haystack.agent.steps_taken"] == 2 + def test_tracing_span_tool_result_uses_outputs_to_string(self, spying_tracer, uppercase_weather_tool): + """The tool span records the result the LLM sees, after the `outputs_to_string` handler ran.""" + agent = Agent(chat_generator=_parallel_tool_calling_generator(), tools=[uppercase_weather_tool]) + + agent.run([ChatMessage.from_user("What's the weather in Berlin and Paris?")]) + + tool_spans = [s for s in spying_tracer.spans if s.operation_name == "haystack.agent.step.tool"] + assert { + (span.tags["haystack.agent.step.tool.output"]["weather"], span.tags["haystack.agent.step.tool.result"]) + for span in tool_spans + } == {("mostly sunny", "MOSTLY SUNNY"), ("mostly cloudy", "MOSTLY CLOUDY")} + + @pytest.mark.asyncio + async def test_tracing_span_run_async_tool_result_uses_outputs_to_string( + self, spying_tracer, uppercase_weather_tool + ): + """The async path also records the `outputs_to_string` result on each tool span.""" + agent = Agent(chat_generator=_parallel_tool_calling_generator(), tools=[uppercase_weather_tool]) + + await agent.run_async([ChatMessage.from_user("What's the weather in Berlin and Paris?")]) + + tool_spans = [s for s in spying_tracer.spans if s.operation_name == "haystack.agent.step.tool"] + assert { + (span.tags["haystack.agent.step.tool.output"]["weather"], span.tags["haystack.agent.step.tool.result"]) + for span in tool_spans + } == {("mostly sunny", "MOSTLY SUNNY"), ("mostly cloudy", "MOSTLY CLOUDY")} + + def test_tracing_span_run_with_failing_outputs_to_string_handler(self, spying_tracer, failing_handler_weather_tool): + agent = Agent( + chat_generator=_parallel_tool_calling_generator(), + tools=[failing_handler_weather_tool], + raise_on_tool_invocation_failure=True, + ) + + with pytest.raises(ValueError, match="cannot format weather"): + agent.run([ChatMessage.from_user("What's the weather in Berlin and Paris?")]) + + tool_spans = [s for s in spying_tracer.spans if s.operation_name == "haystack.agent.step.tool"] + assert len(tool_spans) == 2 + for span in tool_spans: + assert "haystack.agent.step.tool.output" in span.tags + assert "haystack.agent.step.tool.result" not in span.tags + assert spying_tracer.current_span() is None + + @pytest.mark.asyncio + async def test_tracing_span_run_async_with_failing_outputs_to_string_handler( + self, spying_tracer, failing_handler_weather_tool + ): + agent = Agent( + chat_generator=_parallel_tool_calling_generator(), + tools=[failing_handler_weather_tool], + raise_on_tool_invocation_failure=True, + ) + + with pytest.raises(ValueError, match="cannot format weather"): + await agent.run_async([ChatMessage.from_user("What's the weather in Berlin and Paris?")]) + + tool_spans = [s for s in spying_tracer.spans if s.operation_name == "haystack.agent.step.tool"] + assert len(tool_spans) == 2 + for span in tool_spans: + assert "haystack.agent.step.tool.output" in span.tags + assert "haystack.agent.step.tool.result" not in span.tags + assert spying_tracer.current_span() is None + def test_tracing_span_run_with_failing_tool_call(self, spying_tracer, failing_tool): agent = Agent(chat_generator=_parallel_tool_calling_generator(), tools=[failing_tool])