From fcb865d638c06f30f4e2438d26107f1b180a9920 Mon Sep 17 00:00:00 2001
From: melmennaoui <mohamed.elmennaoui@docker.com>
Date: Thu, 29 Oct 2026 23:02:31 +0100
Subject: [PATCH] fix(evaluation): preserve tool evidence in judge transcripts

Remove the implicit 500-byte tool-result cutoff so relevance judges can assess the same evidence recorded for the agent. Cover boundary lengths, tail and middle evidence, UTF-8, and large outputs.

Closes #4770

Assisted-By: docker-agent
Signed-off-by: melmennaoui <mohamed.elmennaoui@docker.com>
---
 pkg/evaluation/eval.go      |  4 +---
 pkg/evaluation/eval_test.go | 38 +++++++++++++++++++++++++++++++++++--
 2 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/pkg/evaluation/eval.go b/pkg/evaluation/eval.go
index acb5eb661c..8160abdb7e 100644
--- a/pkg/evaluation/eval.go
+++ b/pkg/evaluation/eval.go
@@ -650,9 +650,7 @@ func buildTranscript(events []map[string]any) string {
 				name, _ = td["name"].(string)
 			}
 			response, _ := event["response"].(string)
-			if len(response) > 500 {
-				response = response[:500] + "...(truncated)"
-			}
+			// Keep the evidence the agent saw; truncation can turn correct answers into grading failures.
 			fmt.Fprintf(&transcript, "[Tool %q returns: %s]\n\n", name, response)
 			if output, ok := outputs[index]; ok {
 				agentName, _ := event["agent_name"].(string)
diff --git a/pkg/evaluation/eval_test.go b/pkg/evaluation/eval_test.go
index b9d59403cb..2b913f1085 100644
--- a/pkg/evaluation/eval_test.go
+++ b/pkg/evaluation/eval_test.go
@@ -977,7 +977,7 @@ func TestBuildTranscript(t *testing.T) {
 			},
 		},
 		{
-			name: "long tool response truncated",
+			name: "long tool response preserved",
 			events: []map[string]any{
 				{
 					"type":         "tool_call_response",
@@ -989,7 +989,7 @@ func TestBuildTranscript(t *testing.T) {
 				},
 			},
 			wantContains: []string{
-				"...(truncated)",
+				`[Tool "shell" returns: ` + strings.Repeat("x", 600) + "]",
 			},
 		},
 		{
@@ -1039,6 +1039,40 @@ func TestBuildTranscript(t *testing.T) {
 	}
 }
 
+func TestBuildTranscriptPreservesToolEvidence(t *testing.T) {
+	t.Parallel()
+
+	for _, tt := range []struct {
+		name     string
+		response string
+	}{
+		{name: "empty"},
+		{name: "below former limit", response: strings.Repeat("x", 499)},
+		{name: "at former limit", response: strings.Repeat("x", 500)},
+		{name: "above former limit", response: strings.Repeat("x", 501)},
+		{name: "verification at end", response: strings.Repeat("build output\n", 100) + "verification: PASS\n"},
+		{name: "evidence in middle", response: strings.Repeat("x", 600) + "\nERROR: tests failed\n" + strings.Repeat("y", 600)},
+		{name: "UTF-8 across former boundary", response: strings.Repeat("x", 499) + "世界\n"},
+		{name: "large result", response: strings.Repeat("log output\n", 10000) + "verification: PASS\n"},
+	} {
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+
+			events := []map[string]any{
+				{
+					"type":            "tool_call_response",
+					"response":        tt.response,
+					"tool_definition": map[string]any{"name": "shell"},
+				},
+				{"type": "agent_choice", "agent_name": "root", "content": "Reported the command result."},
+			}
+			transcript := buildTranscript(events)
+			assert.Equal(t, "[Tool \"shell\" returns: "+tt.response+"]\n\n[Agent root says]:\nReported the command result.\n\n", transcript)
+			assert.True(t, utf8.ValidString(transcript))
+		})
+	}
+}
+
 func TestMatchesAnyPattern(t *testing.T) {
 	t.Parallel()