feat(github-reporter): Show eval usage in summaries

.github/workflows/github-reporter-smoke.yml

@@ -343,7 +343,11 @@ jobs:
             2> tmp/github-reporter-smoke/stderr.txt
 
           grep -F "# vitest-evals" "$GITHUB_STEP_SUMMARY"
-          grep -F "| Metric | Value |" "$GITHUB_STEP_SUMMARY"
+          grep -F "## Summary" "$GITHUB_STEP_SUMMARY"
+          grep -F "**Failed** — 8 passed, 2 failed, 10 total (80.0% pass rate)." "$GITHUB_STEP_SUMMARY"
+          grep -F -- "- **Score:** avg 0.59, min 0.10" "$GITHUB_STEP_SUMMARY"
+          grep -F "## Usage" "$GITHUB_STEP_SUMMARY"
+          grep -F -- "- **Duration:**" "$GITHUB_STEP_SUMMARY"
           grep -F "## Scores" "$GITHUB_STEP_SUMMARY"
           grep -F "0-19%   | #######              1" "$GITHUB_STEP_SUMMARY"
           grep -F "20-39%  | #############        2" "$GITHUB_STEP_SUMMARY"
@@ -366,17 +370,6 @@ jobs:
           grep -F "Final Output" "$GITHUB_STEP_SUMMARY"
           grep -F "Tool           Status" "$GITHUB_STEP_SUMMARY"
           grep -F "lookupInvoice  ok" "$GITHUB_STEP_SUMMARY"
-          grep -F "| Status | failed |" "$GITHUB_STEP_SUMMARY"
-          grep -F "| Evals | 8 passed, 2 failed, 10 total |" "$GITHUB_STEP_SUMMARY"
-          grep -F "| Score | avg 0.59, min 0.10 |" "$GITHUB_STEP_SUMMARY"
-          if grep -F "| Tests |" "$GITHUB_STEP_SUMMARY"; then
-            echo "test totals row should not be shown"
-            exit 1
-          fi
-          if grep -F "| Usage |" "$GITHUB_STEP_SUMMARY"; then
-            echo "top-level usage row should not be shown"
-            exit 1
-          fi
           grep -F "::error file=apps/demo/evals/smoke.eval.ts,line=12,col=5,title=vitest-evals::refund eval > rejects fraud - score 0.10 - SmokeJudge - expected approved but got denied" tmp/github-reporter-smoke/stdout.txt
           grep -F "::error file=apps/demo/evals/smoke.eval.ts,line=28,col=5,title=vitest-evals::refund eval > returns wrong refund amount - score 0.35 - AmountJudge - expected amount=5000, got amount=499" tmp/github-reporter-smoke/stdout.txt
           grep -F "::warning::GitHub Check Run skipped: missing GITHUB_TOKEN" tmp/github-reporter-smoke/stdout.txt

docs/github-actions.md

@@ -41,6 +41,10 @@ jobs:
 The report step still runs after failed evals because of `if: always()`. The
 job remains failed when the eval step fails.
 
+The summary includes the eval count, pass rate, scores, and duration. When the
+harness provides usage data, it also includes tokens, estimated USD cost, tool
+calls, retries, and models. Store estimated cost as `usage.metadata.costUsd`.
+
 ## Check Run
 
 Add `checks: write` and enable `publish-check` when you want a dedicated PR

packages/github-reporter/src/collect.ts

@@ -4,6 +4,7 @@ import {
   type ToolCall,
   type TranscriptToolCallEvent,
   type TranscriptToolResultEvent,
+  UsageSummarySchema as HarnessUsageSummarySchema,
   collectReportWorkspace,
 } from "@vitest-evals/core";
 import type {
@@ -232,30 +233,64 @@ function stringifyReason(value: unknown) {
 }
 
 function sumUsage(cases: EvalCase[]) {
-  const usage: Required<UsageSummary> = {
+  const usage: UsageSummary = {
     inputTokens: 0,
     outputTokens: 0,
     reasoningTokens: 0,
     totalTokens: 0,
     toolCalls: 0,
+    retries: 0,
+    models: [],
   };
+  const models = new Set<string>();
+  let costUsd: number | undefined;
 
   for (const testCase of cases) {
-    const caseUsage = testCase.harness?.usage;
-    usage.inputTokens += caseUsage?.inputTokens ?? 0;
-    usage.outputTokens += caseUsage?.outputTokens ?? 0;
-    usage.reasoningTokens += caseUsage?.reasoningTokens ?? 0;
-    usage.totalTokens +=
-      caseUsage?.totalTokens ??
-      (caseUsage?.inputTokens ?? 0) +
-        (caseUsage?.outputTokens ?? 0) +
-        (caseUsage?.reasoningTokens ?? 0);
+    const usageEntries = [
+      testCase.harness?.usage,
+      ...(testCase.eval?.scores ?? []).map((score) =>
+        usageFromMetadata(score.metadata),
+      ),
+    ].filter((entry): entry is NonNullable<typeof entry> => Boolean(entry));
+
+    for (const entry of usageEntries) {
+      usage.inputTokens += entry.inputTokens ?? 0;
+      usage.outputTokens += entry.outputTokens ?? 0;
+      usage.reasoningTokens += entry.reasoningTokens ?? 0;
+      usage.totalTokens +=
+        entry.totalTokens ??
+        (entry.inputTokens ?? 0) +
+          (entry.outputTokens ?? 0) +
+          (entry.reasoningTokens ?? 0);
+      usage.retries += entry.retries ?? 0;
+      if (entry.model) models.add(entry.model);
+
+      const entryCost = costFromMetadata(entry.metadata);
+      if (entryCost !== undefined) costUsd = (costUsd ?? 0) + entryCost;
+    }
+
     usage.toolCalls += toolCallCount(testCase);
   }
 
+  usage.models = [...models].sort();
+  if (costUsd !== undefined) usage.costUsd = costUsd;
   return usage;
 }
 
+function usageFromMetadata(metadata: EvalScore["metadata"]) {
+  const result = HarnessUsageSummarySchema.safeParse(metadata?.usage);
+  return result.success ? result.data : undefined;
+}
+
+function costFromMetadata(metadata: unknown) {
+  if (!metadata || typeof metadata !== "object" || Array.isArray(metadata)) {
+    return undefined;
+  }
+
+  const value = (metadata as Record<string, unknown>).costUsd;
+  return isFiniteNumber(value) && value >= 0 ? value : undefined;
+}
+
 function toolCallCount(testCase: EvalCase) {
   const usageToolCalls = testCase.harness?.usage?.toolCalls;
   if (usageToolCalls !== undefined) {

packages/github-reporter/src/merge.ts

@@ -49,13 +49,21 @@ export function mergeEvalReports(reports: EvalReport[]): EvalReport {
   };
 }
 
-function mergeUsage(usages: Array<Required<UsageSummary>>) {
+function mergeUsage(usages: UsageSummary[]) {
+  const costs = usages
+    .map((usage) => usage.costUsd)
+    .filter((cost): cost is number => cost !== undefined);
   return {
     inputTokens: sum(usages, (usage) => usage.inputTokens),
     outputTokens: sum(usages, (usage) => usage.outputTokens),
     reasoningTokens: sum(usages, (usage) => usage.reasoningTokens),
     totalTokens: sum(usages, (usage) => usage.totalTokens),
     toolCalls: sum(usages, (usage) => usage.toolCalls),
+    retries: sum(usages, (usage) => usage.retries),
+    ...(costs.length > 0
+      ? { costUsd: costs.reduce((total, cost) => total + cost, 0) }
+      : {}),
+    models: [...new Set(usages.flatMap((usage) => usage.models))].sort(),
   };
 }
 

packages/github-reporter/src/report.test.ts

@@ -214,21 +214,42 @@ describe("collectEvalReport", () => {
     });
   });
 
-  test("does not normalize provider-specific cost as usage", () => {
+  test("summarizes harness and judge usage when metadata is available", () => {
     const json = structuredClone(sampleJson);
-    const usage = (json.testResults[0]?.assertionResults[0]?.meta as any)
-      .harness.run.usage;
-    usage.estimatedCost = 20;
-    usage.metadata = {
-      costUSD: 20,
+    const assertion = json.testResults[0]?.assertionResults[0];
+    const usage = (assertion?.meta as any).harness.run.usage;
+    usage.provider = "vercel-ai-gateway";
+    usage.model = "openai/gpt-5.4";
+    usage.retries = 2;
+    usage.metadata = { costUsd: 0.04 };
+    (assertion?.meta as any).eval.scores[0].metadata.usage = {
+      provider: "vercel-ai-gateway",
+      model: "openai/gpt-5-mini",
+      inputTokens: 10,
+      totalTokens: 10,
+      retries: 1,
+      metadata: { costUsd: 0.01 },
     };
 
     const report = collectEvalReport(json, {
       workspace: "/repo",
     });
+    const summary = renderJobSummary(report);
 
-    expect("estimatedCost" in report.usage).toBe(false);
-    expect(renderJobSummary(report)).not.toContain("$20");
+    expect(report.usage).toMatchObject({
+      inputTokens: 10,
+      totalTokens: 1230,
+      retries: 3,
+      costUsd: 0.05,
+      models: ["openai/gpt-5-mini", "openai/gpt-5.4"],
+    });
+    expect(summary).toContain("- **Tokens:** 1,230 total (10 input)");
+    expect(summary).toContain("- **Cost:** $0.05");
+    expect(summary).toContain("- **Tool calls:** 2");
+    expect(summary).toContain("- **Retries:** 3");
+    expect(summary).toContain(
+      "- **Models:** openai/gpt-5-mini, openai/gpt-5.4",
+    );
   });
 
   test("ignores non-finite eval scores", () => {
@@ -357,8 +378,12 @@ describe("mergeEvalReports", () => {
             run: {
               output: { status: "approved" },
               usage: {
+                provider: "vercel-ai-gateway",
+                model: "openai/gpt-5-mini",
                 totalTokens: 300,
                 toolCalls: 1,
+                retries: 2,
+                metadata: { costUsd: 0.03 },
               },
               timings: {
                 totalMs: 2000,
@@ -391,8 +416,13 @@ describe("mergeEvalReports", () => {
       average: 0.5,
       minimum: 0.2,
     });
-    expect(report.usage.totalTokens).toBe(1520);
-    expect(report.usage.toolCalls).toBe(3);
+    expect(report.usage).toMatchObject({
+      totalTokens: 1520,
+      toolCalls: 3,
+      retries: 2,
+      costUsd: 0.03,
+      models: ["openai/gpt-5-mini"],
+    });
     expect(report.cases).toHaveLength(2);
     expect(report.failures).toHaveLength(1);
     expect(report.durationMs).toBe(7000);
@@ -880,29 +910,27 @@ describe("formatDuration", () => {
 });
 
 describe("renderJobSummary", () => {
-  test("renders the summary table before result details", () => {
+  test("renders a scan-first overview before result details", () => {
     const report = collectEvalReport(sampleJson, {
       workspace: "/repo",
     });
     const summary = renderJobSummary(report);
 
     expect(summary).toContain("# vitest-evals");
     expect(summary).toContain("## Results");
-    expect(summary.indexOf("| Metric | Value |")).toBeLessThan(
+    expect(summary.indexOf("## Summary")).toBeLessThan(
       summary.indexOf("## Results"),
     );
     expect(summary).toContain("### Failures");
     expect(summary).toContain("<details>");
     expect(summary).toContain(
       "<summary>1. refund agent &gt; rejects fraud - StructuredOutputJudge - 0.20</summary>",
     );
-    expect(summary).toContain("| Metric | Value |");
-    expect(summary).toContain("| Status | failed |");
-    expect(summary).toContain("| Evals | 0 passed, 1 failed, 1 total |");
-    expect(summary).toContain("| Pass Rate | 0.0% |");
-    expect(summary).not.toContain("| Tests |");
-    expect(summary).toContain("| Score | avg 0.20, min 0.20 |");
-    expect(summary).not.toContain("| Usage |");
+    expect(summary).toContain(
+      "**Failed** — 0 passed, 1 failed, 1 total (0.0% pass rate).",
+    );
+    expect(summary).toContain("- **Score:** avg 0.20, min 0.20");
+    expect(summary).toContain("## Usage");
     expect(summary).toContain("## Scores");
     expect(summary.indexOf("## Scores")).toBeLessThan(
       summary.indexOf("## Results"),
@@ -973,24 +1001,11 @@ describe("renderJobSummary", () => {
       }),
     );
 
-    expect(summary).toContain("| Status | failed |");
-    expect(summary).toContain("| Evals | 0 passed, 0 failed, 0 total |");
-    expect(summary).not.toContain("| Tests |");
-    expect(summary).toContain("| Other Failures | 1 non-eval test failure |");
+    expect(summary).toContain("**Failed** — 0 passed, 0 failed, 0 total.");
+    expect(summary).toContain("- **Other failures:** 1 non-eval test failure");
     expect(summary).toContain("## Results");
     expect(summary).toContain("No eval metadata was found");
   });
-
-  test("escapes table cell control characters", () => {
-    const report = collectEvalReport(sampleJson, {
-      workspace: "/repo",
-    });
-    report.status = "failed \\ | escaped" as typeof report.status;
-
-    expect(renderJobSummary(report)).toContain(
-      String.raw`| Status | failed \\ \| escaped |`,
-    );
-  });
 });
 
 describe("renderWorkflowCommands", () => {
@@ -1293,9 +1308,10 @@ describe("publishCheckRun", () => {
     const body = JSON.parse(request.body);
     expect(body.conclusion).toBe("success");
     expect(body.output.title).toContain("90.0%");
-    expect(body.output.summary).toContain("| Status | passed |");
-    expect(body.output.summary).toContain("| Pass Rate | 90.0% |");
-    expect(body.output.summary).toContain("| Gate |");
+    expect(body.output.summary).toContain(
+      "**Passed gate** — 9 passed, 1 failed, 10 total (90.0% pass rate).",
+    );
+    expect(body.output.summary).toContain("- **Gate:**");
     expect(body.output.summary).toContain("### Quality Misses");
     expect(body.output.annotations[0]?.annotation_level).toBe("warning");
   });