feat(usage): report application and judge costs separately

.github/workflows/github-reporter-smoke.yml

@@ -373,10 +373,9 @@ jobs:
             echo "test totals row should not be shown"
             exit 1
           fi
-          if grep -F "| Usage |" "$GITHUB_STEP_SUMMARY"; then
-            echo "top-level usage row should not be shown"
-            exit 1
-          fi
+          grep -F "| Application Usage | 1,302 tokens, 9 tools |" "$GITHUB_STEP_SUMMARY"
+          grep -F "| Judge Usage | none |" "$GITHUB_STEP_SUMMARY"
+          grep -F "| Total Usage | 1,302 tokens, 9 tools |" "$GITHUB_STEP_SUMMARY"
           grep -F "::error file=apps/demo/evals/smoke.eval.ts,line=12,col=5,title=vitest-evals::refund eval > rejects fraud - score 0.10 - SmokeJudge - expected approved but got denied" tmp/github-reporter-smoke/stdout.txt
           grep -F "::error file=apps/demo/evals/smoke.eval.ts,line=28,col=5,title=vitest-evals::refund eval > returns wrong refund amount - score 0.35 - AmountJudge - expected amount=5000, got amount=499" tmp/github-reporter-smoke/stdout.txt
           grep -F "::warning::GitHub Check Run skipped: missing GITHUB_TOKEN" tmp/github-reporter-smoke/stdout.txt

AGENTS.md

@@ -29,7 +29,7 @@ apps/                     # demo-ai-sdk, demo-openai-agents, demo-pi (realistic
 - Root API work is harness-first and judge-first.
 - Legacy scorer-first changes stay under `packages/vitest-evals/src/legacy/...`.
 - Keep normalized session and run data JSON-serializable.
-- Keep `UsageSummary` to stable usage units such as tokens, tools, retries, provider, and model. Do not add first-class cost fields; provider-specific cost estimates belong in `usage.metadata`.
+- Keep `UsageSummary` to stable usage units such as tokens, USD cost, tools, retries, provider, and model. Set `costUsd` only when cost is known; provider-specific pricing details belong in `usage.metadata`.
 - GitHub reporting is action-first: document workflows with `uses: getsentry/vitest-evals@v0`, not CLI commands.
 - Sharded eval reporting uses distinct JSON artifacts per matrix job and one reducer job that publishes the combined action report.
 - Root GitHub Action releases keep source and bundled tags separate: Craft publishes the source release, `vX.Y.Z-src` preserves the source baseline, and `vX`/`vX.Y.Z` point at the bundled action commit. Keep the Craft `github` target filtered away from package artifacts.

README.md

@@ -128,8 +128,8 @@ soft-fail defaults, and score-gate options.
 
 Use `vitest-evals serve` to inspect one or more JSON reports locally. It
 accepts files, simple globs, and directories of JSON artifacts, then serves a
-React SPA with run summaries, eval cases, harness output, sessions, tools, and
-trace details.
+React SPA with run summaries, eval cases, separate application and judge usage,
+harness output, sessions, tools, and trace details.
 
 ```sh
 pnpm exec vitest-evals serve vitest-results.json
@@ -232,9 +232,9 @@ Harness-backed suites stay close to plain Vitest:
   `ctx.runJudge(...)` from a configured `judgeHarness`
 - pass scenario-specific judge criteria explicitly to matcher options, or bind
   suite-wide criteria on the judge instance
-- reporter output, replay, usage, tool calls, and spans come from the
-  normalized run. First-party harnesses attach native spans when
-  provider/runtime data is available; `createHarness(...)` attaches a fallback
+- reporter output separates application, judge, and combined token and
+  `costUsd` usage from normalized runs. First-party harnesses attach native
+  spans when provider/runtime data is available; `createHarness(...)` attaches a fallback
   run span when a custom harness does not return traces itself. Span attributes
   include typed OpenTelemetry GenAI semantic keys while keeping
   provider-specific keys JSON-safe.

docs/architecture.md

@@ -94,10 +94,10 @@ not return traces themselves. Span attributes include typed OpenTelemetry GenAI
 semantic keys for common model, agent, tool, and token fields while still
 allowing provider-specific attributes.
 
-`UsageSummary` is intentionally limited to stable usage units such as tokens,
-tool counts, retries, provider, and model. Provider-specific cost estimates are
-not normalized because pricing semantics vary by runtime and can be stale; if a
-harness needs to retain them, store them under `usage.metadata`.
+`UsageSummary` standardizes tokens, USD cost, tool counts, retries, provider,
+and model. Harnesses report provider-supplied or suite-estimated cost through
+`costUsd`; omit it when cost is unknown, and use zero only for known-free runs.
+Provider-specific pricing details remain under `usage.metadata`.
 
 ### `packages/vitest-evals/src/index.ts`
 
@@ -141,7 +141,10 @@ adapters. Custom judges should use `createJudge("Name", assess)` for stable
 reporter labels, or `createJudge({ name, judgeHarness, assess })` when the
 judge should carry a reusable judge-side harness default.
 `createJudgeHarness(...)` is the shared abstraction for judge-side provider
-shims.
+shims. Each `ctx.runJudge(...)` call returns the normalized judge output while
+core records the complete judge harness run on the resulting score. Multiple
+calls from one judge remain separate runs. The GitHub reporter aggregates tokens
+and `costUsd` separately for application and judge usage, plus a derived total.
 
 ### `packages/vitest-evals/src/legacy/*`
 
@@ -171,7 +174,11 @@ Provides the custom Vitest reporter that reads normalized run metadata from
 `packages/core` owns dependency-light primitives shared by the Vitest
 integration, GitHub reporter, and report UI. Its main entry stays browser-safe,
 while `@vitest-evals/core/node` exposes filesystem helpers for local and CI
-report consumers. It exports stable schemas, TypeScript types, and helpers for:
+report consumers. The collected workspace is versioned. Judge runs and standardized cost change
+its persisted shape in schema version 2; readers reject unsupported versions
+instead of silently misreading them. Raw Vitest `task.meta` is strict but not
+independently versioned, so producers and artifact readers must use compatible
+package releases. Core exports schemas, TypeScript types, and helpers for:
 
 - JSON-safe values
 - normalized harness runs, sessions, transcript-derived tool calls, usage,
@@ -238,8 +245,10 @@ For each eval test in a harness-backed suite:
    `usage`, `timings`, `artifacts`, and `errors`.
 6. Core stores that run on `task.meta.harness` for the reporter.
 7. Automatic suite-level judges run against the normalized run/session pair.
-8. The eval test asserts on the same returned result and session.
-9. The reporter renders the recorded metadata without re-executing the harness.
+8. Each judge-harness invocation is stored under its judge score as `judgeRuns`.
+9. The eval test asserts on the same returned result and session.
+10. The reporter renders the recorded metadata and includes judge usage in totals
+   without re-executing either harness.
 
 Explicit `expect(result).toSatisfyJudge(...)` calls use the run's typed output
 and reuse registered input, metadata, and harness context

docs/github-actions.md

@@ -8,7 +8,13 @@ Public docs mirror this guide at
 [vitest-evals.sentry.dev/docs/github](https://vitest-evals.sentry.dev/docs/github).
 
 Use JSON as the eval artifact because it preserves `task.meta.eval` and
-`task.meta.harness`; JUnit XML does not carry the full eval metadata.
+`task.meta.harness`; JUnit XML does not carry the full eval metadata. Usage
+reports show separate application, judge, and combined token and `costUsd`
+totals. A combined cost is shown only when every usage category reports cost;
+omitted cost means unknown, while zero means known free. Judge runs remain
+attached to their scores for detailed usage data. Keep the package producing
+the JSON artifact and the `getsentry/vitest-evals` action on compatible
+releases; raw Vitest task metadata is strict and not independently versioned.
 
 ## Minimal Workflow
 

docs/provider-transformations.md

@@ -110,8 +110,9 @@ Harness adapters should:
   transcript events remain unambiguous
 - preserve the application-facing result separately in `run.output`
 - attach provider/model and stable usage data when available
-- keep provider-specific cost estimates in `usage.metadata`, not as normalized
-  usage fields
+- map provider-reported or suite-estimated USD cost to `usage.costUsd`; omit it
+  when unknown, use zero only for known-free runs, and keep provider-specific
+  pricing details in `usage.metadata`
 - attach replay/cache metadata in the tool record metadata rather than in
   provider-specific side channels
 - expose common assertion data through helpers such as `toolCalls(result)`,

packages/core/src/harness/index.ts

@@ -39,6 +39,7 @@ export const UsageSummarySchema = z
     outputTokens: FiniteNumberSchema.optional(),
     reasoningTokens: FiniteNumberSchema.optional(),
     totalTokens: FiniteNumberSchema.optional(),
+    costUsd: FiniteNumberSchema.nonnegative().optional(),
     toolCalls: FiniteNumberSchema.optional(),
     retries: FiniteNumberSchema.optional(),
     metadata: JsonObjectSchema.optional(),
@@ -59,11 +60,13 @@ export type UsageSummary = {
   reasoningTokens?: number;
   /** Total token count reported by the provider or adapter. */
   totalTokens?: number;
+  /** USD-denominated cost attributed to this run. Omit when unknown; zero means known free. */
+  costUsd?: number;
   /** Count of tool calls observed during the run. */
   toolCalls?: number;
   /** Retry count observed during the run. */
   retries?: number;
-  /** Provider-specific JSON-safe usage details. Cost estimates belong here. */
+  /** Provider-specific JSON-safe usage details. */
   metadata?: Record<string, JsonValue>;
 };
 

packages/core/src/index.test.ts

@@ -1,12 +1,16 @@
 import { describe, expect, test } from "vitest";
 import {
+  HarnessRunSchema,
+  TranscriptToolResultEventSchema,
+  UsageSummarySchema,
+  type VitestJsonReport,
   assistantMessages,
   collectReportWorkspace,
   failedSpans,
-  HarnessRunSchema,
   latestAssistantMessageContent,
-  messagesToTranscriptEvents,
   messagesByRole,
+  messagesToTranscriptEvents,
+  parseReportWorkspace,
   parseVitestJsonReport,
   readEvalTaskMeta,
   spans,
@@ -15,10 +19,7 @@ import {
   toolCalls,
   toolMessages,
   traceSpans,
-  TranscriptToolResultEventSchema,
-  UsageSummarySchema,
   userMessages,
-  type VitestJsonReport,
 } from "./index";
 
 const sampleJson: VitestJsonReport = {
@@ -884,22 +885,24 @@ describe("messagesToTranscriptEvents", () => {
 });
 
 describe("UsageSummarySchema", () => {
-  test("keeps provider-specific usage data under metadata", () => {
+  test("accepts standardized USD cost and keeps provider details in metadata", () => {
     expect(
       UsageSummarySchema.safeParse({
         totalTokens: 120,
-        estimatedCostUsd: 0.02,
+        costUsd: 0.02,
+        metadata: { providerRequestId: "req_123" },
       }).success,
-    ).toBe(false);
+    ).toBe(true);
 
     expect(
       UsageSummarySchema.safeParse({
         totalTokens: 120,
-        metadata: {
-          estimatedCostUsd: 0.02,
-        },
+        estimatedCostUsd: 0.02,
       }).success,
-    ).toBe(true);
+    ).toBe(false);
+    expect(UsageSummarySchema.safeParse({ costUsd: -0.01 }).success).toBe(
+      false,
+    );
   });
 });
 
@@ -933,7 +936,7 @@ describe("collectReportWorkspace", () => {
         },
       },
     ]);
-    expect(workspace.schemaVersion).toBe(1);
+    expect(workspace.schemaVersion).toBe(2);
     expect(workspace.cases).toHaveLength(1);
     expect(workspace.cases[0]).toMatchObject({
       displayFile: "apps/demo/evals/refund.eval.ts",
@@ -965,6 +968,17 @@ describe("collectReportWorkspace", () => {
     });
   });
 
+  test("migrates schema version 1 workspaces in memory", () => {
+    const workspace = collectReportWorkspace(sampleJson);
+
+    expect(
+      parseReportWorkspace({
+        ...workspace,
+        schemaVersion: 1,
+      }),
+    ).toEqual(workspace);
+  });
+
   test("defaults harness-only case scores from Vitest status", () => {
     const json = structuredClone(sampleJson);
     const passed = json.testResults[0]!.assertionResults[0]!;

packages/core/src/report/metadata.ts

@@ -19,6 +19,7 @@ export const EvalScoreSchema = z
   .object({
     name: z.string().optional(),
     score: NullableFiniteNumberSchema,
+    judgeRuns: z.array(HarnessRunSchema).optional(),
     metadata: JsonObjectSchema.optional(),
   })
   .strict();

packages/core/src/report/workspace.ts

@@ -1,20 +1,20 @@
 import { z } from "zod";
+import { FiniteNumberSchema, isRecord, parseWithSchema } from "../schema-utils";
 import {
   EvalMetaSchema,
   HarnessMetaSchema,
   readEvalTaskMeta,
 } from "./metadata";
-import { FiniteNumberSchema, parseWithSchema, isRecord } from "../schema-utils";
 import {
-  parseVitestJsonReport,
-  VitestJsonLocationSchema,
   type VitestJsonAssertion,
+  VitestJsonLocationSchema,
   type VitestJsonReport,
+  parseVitestJsonReport,
 } from "./vitest-json";
 import { VitestJsonStatusSchema } from "./vitest-json";
 
 /** Current schema version for collected report workspaces. */
-export const REPORT_WORKSPACE_SCHEMA_VERSION = 1;
+export const REPORT_WORKSPACE_SCHEMA_VERSION = 2;
 
 /** One collected Vitest JSON report source in a multi-run workspace. */
 export const ReportRunSchema = z
@@ -64,12 +64,23 @@ export const ReportCaseSchema = z
 /** One eval or harness-backed test case collected from Vitest JSON. */
 export type ReportCase = z.infer<typeof ReportCaseSchema>;
 
+const reportWorkspaceShape = {
+  runs: z.array(ReportRunSchema),
+  cases: z.array(ReportCaseSchema),
+};
+
+const LegacyReportWorkspaceSchema = z
+  .object({
+    schemaVersion: z.literal(1),
+    ...reportWorkspaceShape,
+  })
+  .strict();
+
 /** Full multi-run report workspace consumed by rich report UIs. */
 export const ReportWorkspaceSchema = z
   .object({
     schemaVersion: z.literal(REPORT_WORKSPACE_SCHEMA_VERSION),
-    runs: z.array(ReportRunSchema),
-    cases: z.array(ReportCaseSchema),
+    ...reportWorkspaceShape,
   })
   .strict();
 
@@ -92,6 +103,18 @@ export type CollectReportWorkspaceOptions = {
 
 /** Parses and validates an unknown value as a collected report workspace. */
 export function parseReportWorkspace(input: unknown): ReportWorkspace {
+  if (isRecord(input) && input.schemaVersion === 1) {
+    const legacy = parseWithSchema(
+      LegacyReportWorkspaceSchema,
+      input,
+      "report workspace",
+    );
+    return {
+      ...legacy,
+      schemaVersion: REPORT_WORKSPACE_SCHEMA_VERSION,
+    };
+  }
+
   return parseWithSchema(ReportWorkspaceSchema, input, "report workspace");
 }