From fceba7d4d768fec99b40364eb2ea5a528a631210 Mon Sep 17 00:00:00 2001 From: Parker Bibus Date: Wed, 14 Oct 2026 09:37:44 -0700 Subject: [PATCH 1/6] Update Azure Cobalt profiles to four cores (#2217) * Update Azure Cobalt profiles to four cores Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> * Remove unnecessary README update Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --------- Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- build/azure.profile.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/build/azure.profile.yml b/build/azure.profile.yml index 23282690f..be38c8d1c 100644 --- a/build/azure.profile.yml +++ b/build/azure.profile.yml @@ -305,7 +305,7 @@ profiles: variables: serverPort: 5000 serverAddress: 10.0.4.17 - cores: 16 + cores: 4 jobs: db: endpoints: @@ -330,7 +330,7 @@ profiles: variables: serverPort: 5000 serverAddress: 10.0.4.17 - cores: 16 + cores: 4 jobs: db: endpoints: @@ -355,7 +355,7 @@ profiles: variables: serverPort: 5000 serverAddress: 10.0.4.20 - cores: 16 + cores: 4 jobs: db: endpoints: @@ -380,7 +380,7 @@ profiles: variables: serverPort: 5000 serverAddress: 10.0.4.20 - cores: 16 + cores: 4 jobs: db: endpoints: @@ -766,7 +766,7 @@ profiles: variables: serverPort: 5000 serverAddress: 10.0.4.18 - cores: 16 + cores: 4 agents: application: endpoints: @@ -802,7 +802,7 @@ profiles: variables: serverPort: 5000 serverAddress: 10.0.4.19 - cores: 16 + cores: 4 agents: application: endpoints: @@ -838,7 +838,7 @@ profiles: variables: serverPort: 5000 serverAddress: 10.0.4.17 - cores: 16 + cores: 4 agents: application: endpoints: @@ -874,7 +874,7 @@ profiles: variables: serverPort: 5000 serverAddress: 10.0.4.20 - cores: 16 + cores: 4 agents: application: endpoints: From d8e961112fe925b6be4317a38997658dd9408f83 Mon Sep 17 00:00:00 2001 From: Parker Bibus Date: Wed, 14 Oct 2026 10:01:06 -0700 Subject: [PATCH 2/6] Remove workload port overrides from Azure profiles (#2216) Let workload defaults select the listening port instead of overriding container scenarios to port 5000. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- build/azure.profile.yml | 30 ------------------------------ 1 file changed, 30 deletions(-) diff --git a/build/azure.profile.yml b/build/azure.profile.yml index be38c8d1c..f2da86db8 100644 --- a/build/azure.profile.yml +++ b/build/azure.profile.yml @@ -3,7 +3,6 @@ profiles: aspnet-azure-lin: variables: - serverPort: 5000 serverAddress: 10.0.4.4 cores: 4 jobs: @@ -28,7 +27,6 @@ profiles: aspnet-azure-lin-relay: variables: - serverPort: 5000 serverAddress: 10.0.4.4 cores: 4 jobs: @@ -53,7 +51,6 @@ profiles: idna-intel-lin: variables: - serverPort: 5000 serverAddress: 10.0.4.12 cores: 4 jobs: @@ -78,7 +75,6 @@ profiles: idna-intel-lin-relay: variables: - serverPort: 5000 serverAddress: 10.0.4.12 cores: 4 jobs: @@ -103,7 +99,6 @@ profiles: idna-amd-lin: variables: - serverPort: 5000 serverAddress: 10.0.4.13 cores: 4 jobs: @@ -128,7 +123,6 @@ profiles: idna-amd-lin-relay: variables: - serverPort: 5000 serverAddress: 10.0.4.13 cores: 4 jobs: @@ -153,7 +147,6 @@ profiles: idna-intel-win: variables: - serverPort: 5000 serverAddress: 10.0.4.14 cores: 4 jobs: @@ -178,7 +171,6 @@ profiles: idna-intel-win-relay: variables: - serverPort: 5000 serverAddress: 10.0.4.14 cores: 4 jobs: @@ -203,7 +195,6 @@ profiles: idna-amd-win: variables: - serverPort: 5000 serverAddress: 10.0.4.15 cores: 4 jobs: @@ -228,7 +219,6 @@ profiles: idna-amd-win-relay: variables: - serverPort: 5000 serverAddress: 10.0.4.15 cores: 4 jobs: @@ -253,7 +243,6 @@ profiles: aspnet-azurearm64-lin: variables: - serverPort: 5000 serverAddress: 10.0.4.7 cores: 4 jobs: @@ -278,7 +267,6 @@ profiles: aspnet-azurearm64-lin-relay: variables: - serverPort: 5000 serverAddress: 10.0.4.7 cores: 4 jobs: @@ -303,7 +291,6 @@ profiles: cobalt-cloud-lin: variables: - serverPort: 5000 serverAddress: 10.0.4.17 cores: 4 jobs: @@ -328,7 +315,6 @@ profiles: cobalt-cloud-lin-relay: variables: - serverPort: 5000 serverAddress: 10.0.4.17 cores: 4 jobs: @@ -353,7 +339,6 @@ profiles: cobalt-cloud-lin-al3: variables: - serverPort: 5000 serverAddress: 10.0.4.20 cores: 4 jobs: @@ -378,7 +363,6 @@ profiles: cobalt-cloud-lin-al3-relay: variables: - serverPort: 5000 serverAddress: 10.0.4.20 cores: 4 jobs: @@ -404,7 +388,6 @@ profiles: # Add individual machine profiles for azure-db similar to ci.profile.yml azure-db-app: # azurearm64db variables: - serverPort: 5000 serverAddress: 10.0.4.9 cores: 4 agents: @@ -440,7 +423,6 @@ profiles: # Individual machine profiles for azure-client azure-client-app: # azurearm64client variables: - serverPort: 5000 serverAddress: 10.0.4.8 cores: 4 agents: @@ -476,7 +458,6 @@ profiles: # Individual machine profiles for azure-server-arm64 azure-server-arm64-app: # azurearm64server variables: - serverPort: 5000 serverAddress: 10.0.4.7 cores: 4 agents: @@ -512,7 +493,6 @@ profiles: # Individual machine profiles for azure2-client azure2-client-app: variables: - serverPort: 5000 serverAddress: 10.0.4.5 cores: 2 agents: @@ -548,7 +528,6 @@ profiles: # Individual machine profiles for azure2-db azure2-db-app: variables: - serverPort: 5000 serverAddress: 10.0.4.6 cores: 2 agents: @@ -584,7 +563,6 @@ profiles: # Individual machine profiles for azure2-server-amd64 azure2-server-amd64-app: variables: - serverPort: 5000 serverAddress: 10.0.4.4 cores: 4 agents: @@ -620,7 +598,6 @@ profiles: # Individual machine profiles for idna-amd-lin idna-amd-lin-app: variables: - serverPort: 5000 serverAddress: 10.0.4.13 cores: 4 agents: @@ -656,7 +633,6 @@ profiles: # Individual machine profiles for idna-amd-win idna-amd-win-app: variables: - serverPort: 5000 serverAddress: 10.0.4.15 cores: 4 agents: @@ -692,7 +668,6 @@ profiles: # Individual machine profiles for idna-intel-lin idna-intel-lin-app: variables: - serverPort: 5000 serverAddress: 10.0.4.12 cores: 4 agents: @@ -728,7 +703,6 @@ profiles: # Individual machine profiles for idna-intel-win idna-intel-win-app: variables: - serverPort: 5000 serverAddress: 10.0.4.14 cores: 4 agents: @@ -764,7 +738,6 @@ profiles: # Individual machine profiles for cobalt-cloud-lin-client cobalt-cloud-lin-client-app: variables: - serverPort: 5000 serverAddress: 10.0.4.18 cores: 4 agents: @@ -800,7 +773,6 @@ profiles: # Individual machine profiles for cobalt-cloud-lin-db cobalt-cloud-lin-db-app: variables: - serverPort: 5000 serverAddress: 10.0.4.19 cores: 4 agents: @@ -836,7 +808,6 @@ profiles: # Individual machine profiles for cobalt-cloud-lin-server cobalt-cloud-lin-server-app: variables: - serverPort: 5000 serverAddress: 10.0.4.17 cores: 4 agents: @@ -872,7 +843,6 @@ profiles: # Individual machine profiles for cobalt-cloud-lin-server-azure-linux3 cobalt-cloud-lin-server-azure-linux3-app: variables: - serverPort: 5000 serverAddress: 10.0.4.20 cores: 4 agents: From 16ab502fd7b9fbee187dc8d70cfb8b10bce25049 Mon Sep 17 00:00:00 2001 From: Parker Bibus Date: Wed, 14 Oct 2026 10:05:17 -0700 Subject: [PATCH 3/6] Add generated Minimal Todo container benchmarks (#2215) * Add generated Minimal Todo container benchmarks Adds a frozen, reviewed Minimal Todo benchmark cohort (C#, Go, Rust) with a versioned HTTP contract and one native Crank scenario file. Source generation, admission review, pre-promotion comparison, and quota calibration tooling for this cohort -- including the HTTP contract fixture and the one-time state/body verification that uses it -- are owned by the separate dotnet-performance-tools repository; this change is the shipped, day-to-day Crank runtime surface only. - One scenario/minimal-todo/minimal-todo.benchmarks.yml defines all three language scenarios with two roles each (application, load; no separate contract-check/before/after roles and no checks/ fixture fetch). Regular runs validate: application readiness (the startup gate), then the measured Bombardier result strictly (exact URL/method/client/rate/duration/ connections/timeout/request headers, response counters all 2xx with zero other status classes, zero transport errors, delivered rate/duration within 5% of offered), plus CPU/memory tail coverage and sample-gap checks. No requiredArchitecture/architecture gating: Docker images are multi-arch, so the same scenario runs unmodified on x64 or ARM64 agents. Manual connected runs use the shared build/ci.profile.yml / build/azure.profile.yml profiles like every other scenario; there is no scenario-specific profile file. - Each app's own Dockerfile builds directly from floating, multi-arch base images: .NET 11 nightly SDK/ASP.NET (no stable .NET 12 tag is published yet), golang:1.27-bookworm, and rust:1.96-bookworm builders, all finishing on a shared ubuntu:26.04 runtime image (verified against the live .NET 11 nightly manifest). No pinned digests, no Dockerfile.daily/global.json SDK pin, no build-context receipt tooling. - Scheduler template (build/minimal-todo-scenarios.yml) is enabled with the previously accepted matched-contract rate (25200 req/s, 1 CPU, 512 MiB, 128 connections, 30s warmup, 120s duration, 2s request timeout, port 8080), dispatched to the gold-lin, azure-arm64, and cobalt-cloud-lin pods. Validation: offline Crank config resolution (--profile offline) succeeds for all three scenarios (now two roles each); the full scripts/pod-scheduler test suite (43 tests, including the committed-YAML snapshot check) passes; no dead references to any removed file remain. Docker and the floating .NET 11/ Go 1.27/Rust 1.96 toolchains were not available in this environment, so the Dockerfiles could not be build-verified here. This round changed the Docker base images and the C# project's target framework; the previously accepted 25200 req/s rate was calibrated against the prior pinned images and has not been re-run against these new images. The first post-merge scheduled run is the first operational confirmation under the new images, not a re-validation. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 2d9d74a4-18fa-4e63-a55d-792c3aabe5f8 * Remove redundant job options and duplicate metadata from Minimal Todo scenario Follow-up nits on the generated Minimal Todo scenario, no history rewrite: - Remove the custom offline variable/profile entirely (no other scenario in this repository uses one). The application/contract-check jobs no longer clear endpoints conditionally; the bombardier job's onConfigure simply validates that application/load each resolve to exactly one agent endpoint, matching the connected-run assumption every other scenario makes. - Remove job-level options that only restated Crank's own defaults (reuseBuild/reuseSource/collectCounters/downloadOutput/ downloadOutputOutput/displayBuild/displayOutput/noClean: false) and that have no prior art anywhere else in scenarios/*.benchmarks.yml; options blocks now contain only requiredOperatingSystem: linux, matching the repository-wide convention. - Remove the per-app job variables language/framework/sdkImage/runtimeImage and the onResultsCreated block that copied them into todo/* properties: they duplicated the todo/language and todo/framework --property values the scheduler template already passes, and were not read by the build itself. The calibration summary (todo/calibration/*, todo/cpu/*, todo/memory/*) is unchanged. - README: removed the offline-resolution section describing the now-removed profile; no other content changed. Validation: static config assertions confirm the offline variable/profile, the removed options, and the removed job variables are all absent. The full scripts/pod-scheduler test suite (43 tests) passes; regenerating both CI pipelines produces no diff. Structural resolution was re-verified with crank --debug against a test-only sanitized temp copy of the scenario file (outside this repository, never committed) with a non-listening loopback placeholder endpoint per role -- --debug only resolves and serializes the job graph, so no real agent was contacted. Docker and the floating .NET 11/ Go 1.27/Rust 1.96 toolchains remain unavailable in this environment. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 2d9d74a4-18fa-4e63-a55d-792c3aabe5f8 * Use the official dotnet/crank Bombardier job instead of redefining it Follow-up nit, no history rewrite: - Import the official, unmodified Bombardier job via https://raw.githubusercontent.com/dotnet/crank/main/src/Microsoft.Crank.Jobs.Bombardier/bombardier.yml instead of redefining its source/framework/SDK/runtime/arguments/ onConfigure/options locally. Deleted the entire local jobs.bombardier definition, including its workload-variable range checks and the application/load endpoint-count/URL-shape validation (no custom endpoint validation needed: the shared ci/azure profile always supplies a real endpoint, matching every other scenario in this repository). - Renamed the top-level requestTimeout variable to timeout (default unchanged at 2) to match the imported job's own variable name, and updated build/minimal-todo-scenarios.yml's --variable requestTimeout=2 to --variable timeout=2. - Each scenario's load role now references the imported bombardier job with explicit variables path: /todos, transport: http1, presetHeaders: none, and customHeaders for Accept: application/json, Connection: keep-alive, and Accept-Encoding: identity. rate/connections/warmup/duration/timeout/ serverAddress/serverPort are intentionally not set per-role: they are supplied the same way as before, via the scheduler template's explicit --variable flags (or an operator's own CLI variables for a manual run), not a scenario-file override. - README: describes the imported Bombardier job and its explicit variables instead of the old pinned-commit wrapper description. Validation: resolved a test-only sanitized temp copy of the scenario file (outside this repository, never committed) with crank --debug, using a non-listening loopback placeholder endpoint -- confirmed the import resolves and merges correctly for all three languages, both with no CLI variables (imported job's own defaults apply, as expected) and with the same --variable overrides the scheduler template passes (connections, warmup, duration, rate, timeout, serverPort all bind correctly into the imported job, and the resulting Bombardier arguments match exactly what was previously hardcoded). The only network access during this validation was crank's own normal fetch of the imported YAML's raw GitHub URL (the same mechanism every scenario that imports wrk.yml/bombardier.yml already relies on) plus a refused local loopback connection; no real agent was contacted. The full scripts/pod-scheduler test suite (43 tests) passes; regenerating both CI pipelines produces no diff beyond the already-staged timeout rename in build/minimal-todo-scenarios.yml. Docker and the floating .NET 11/Go 1.27/Rust 1.96 toolchains remain unavailable in this environment. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 2d9d74a4-18fa-4e63-a55d-792c3aabe5f8 * Add optional direct-Crank-build C# comparison scenario (not scheduled) Follow-up, no history rewrite. Adds a side-by-side comparison point alongside the existing Docker csharp scenario, which remains unchanged (verified byte-identical in the diff). - New job todo-csharp-project builds src/TodoApi/TodoApi.csproj directly with Crank (project: + framework: net11.0 + channel: latest, confirmed real Crank Job model fields via the actual dotnet/crank source), no Docker fields at all. Reuses the exact same sources/sourceRevision main-or-40hex validation and CPU/memory/port onConfigure logic as todo-csharp. - New scenario csharp-project pairs todo-csharp-project with the same imported Bombardier load job and variables (path: /todos, transport: http1, presetHeaders: none, customHeaders for Accept/Connection/Accept-Encoding) as the csharp scenario. - Not added to build/minimal-todo-scenarios.yml or any pod/CI configuration: the scheduled csharp scenario stays the Docker baseline; csharp-project is only ever run manually via an explicit --scenario csharp-project. - README: documents the optional scenario, why channel: latest is used (coherent latest SDK/runtime/ASP.NET Core, not a floating Docker digest), and that Crank's own CPU/memory limits (cgroups) apply identically to both jobs so a comparison isolates packaging/runtime/host-OS effects rather than a resource-quota difference. Validation: resolved a test-only sanitized temp copy of the scenario file (outside this repository, never committed) with crank --debug for all four scenarios (csharp, go, rust, csharp-project) using a non-listening loopback placeholder endpoint -- all resolve cleanly; confirmed the csharp-project application job resolves Project/Framework/Channel/Port correctly and has no DockerFile. The full scripts/pod-scheduler test suite (43 tests) passes; regenerating both CI pipelines produces no diff (this scenario is not referenced by any pod/CI configuration). No real agent was contacted. Docker and the floating .NET 11/Go 1.27/Rust 1.96 toolchains remain unavailable in this environment. Live/lab execution (the requested one-time paired Docker-vs-direct comparison on real agents) is out of scope for this commit and this session: this checkout has no credentials or network path to real lab agents, consistent with this effort's established division of responsibility where live dispatch is performed by the parent/native executor, never by this candidate-integration checkout. Reported this limitation back to the requester; the comparison itself has not been run. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 2d9d74a4-18fa-4e63-a55d-792c3aabe5f8 * Fix C# Dockerfile locked-mode restore order; correct stale README claims Addresses five PR review comments on #2215 (normal follow-up commit, no history rewrite, no scenario/scheduler/workload change): 1. (csharp/Dockerfile, review 4213810222) The pre-restore COPY only included TodoApi.csproj, so `dotnet restore --locked-mode` ran with no packages.lock.json on disk; the real lock file only arrived in the later `COPY src/TodoApi/ src/TodoApi/` step, after restore had already run. Added packages.lock.json to the pre-restore COPY alongside the csproj (not excluded by .dockerignore). Locked-mode and the lock file itself are unchanged -- this only fixes when the lock file becomes visible to restore. 2. (csharp/README.md, review 4213810258) The "Verification in the supplied environment" section (Windows SDK 10.0.401, runtimes 10.0.12) is now explicitly labeled as historical/original .NET 10 evidence, predating this round's floating .NET 11 images; states plainly that the current .NET 11 configuration was not build-verified in this checkout. Also corrected two stale present-tense claims: "strict runtime selection" (the RuntimeFrameworkVersion/roll-forward pin it described was removed from Directory.Build.props this round) and "the pinned runtimes" for framework-dependent RID publishes (now "an installed, compatible runtime"), plus the package-lock description (the app does have its own packages.lock.json, just with an empty recorded dependency set, not "no package lock"). 3/4. (go/README.md, reviews 4213810293 and 4213810316) Corrected the top-of-file claim that "the Docker builder pins that version and its image digest" -- the builder floats on `golang:1.27-bookworm` with no digest pin, and `go.mod`'s `go 1.27` is a minimum-version directive, not an exact pin. Corrected the Container section and Files list, which claimed "native builder to cross-compile for the selected target" and "pinned multi-stage Linux container build": the Dockerfile has no BUILDPLATFORM/TARGETARCH, so `docker build --platform ...` selects a single target platform for the whole build (native for that platform, via emulation when needed) rather than cross-compiling from one builder architecture to another; that's a different mechanism from the standalone GOOS/GOARCH cross-compilation commands already documented separately (left unchanged). 5. (rust/README.md, review 4213810360) Corrected "a minimal Debian runtime" in the current "Linux containers" section to `ubuntu:26.04` (the actual current runtime base). The Validation section's historical "Runtime image" row (Debian glibc 2.36-9+deb12u14) is preserved verbatim but its heading/intro now explicitly label it as the original Debian-based build from before this round's switch to Ubuntu, so it can't be misread as evidence the current image was tested. Validation: for item 1, Docker is unavailable in this environment, so I built the smallest possible local reproduction instead -- a byte-for-byte copy of the real build context, staged exactly like each Dockerfile COPY step, restored/published with the local SDK (TargetFramework temporarily overridden net11.0->net10.0 in that throwaway copy only, since net11.0 isn't installed locally; no change to the real project). Both the old and the fixed COPY order restore and publish successfully (exit 0) for this specific project today: TodoApi.csproj has zero package references, and neither the csproj nor Directory.Build.props sets RestorePackagesWithLockFile, so no lock file is auto-generated and --locked-mode has nothing to validate against either way right now -- reproducing NU1004 was not possible today, and I am not claiming it was observed. The fix is still correct and makes --locked-mode meaningful going forward (e.g. once a real dependency is added, restore will then actually see and validate against the committed lock file before building, instead of silently not reading it). All four touched files: git diff --check is clean (one pre-existing CRLF/cr-at-eol cosmetic flag per file, consistent with each file's existing line-ending convention, not a real defect, same pattern as every prior round in this PR). No scenario, scheduler, pod, or workload file was touched. Per-comment coverage: all five identified review comments (4213810222, 4213810258, 4213810293, 4213810316, 4213810360) are addressed by this commit. Threads are intentionally left unresolved pending confirmation this fix is accepted. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 2d9d74a4-18fa-4e63-a55d-792c3aabe5f8 * Schedule Minimal Todo C# daily via direct build; Go/Rust weekly via Docker Normal follow-up commit, no history rewrite/scheduler-architecture change/ main-branch change. Implements a new scheduling policy while keeping the Docker generation/comparison path fully intact: - build/minimal-todo-scenarios.yml: the C# scenario entry now submits `--scenario csharp-project` (the existing direct, non-Docker build) instead of `--scenario csharp`, unconditionally on every regular scheduled run (condition: 'true', same cadence as before). todo/language=csharp and todo/framework=net11.0 properties are unchanged. The Go and Rust entries keep their existing Docker `go`/`rust` scenarios but add `condition: Math.round(Date.now() / 43200000) % 14 == 0` (true once every 14 half-day scheduled slots, i.e. once per ~7 days for each pipeline's own 12-hour cron), sharing the same slot value so both languages fall on the same calendar day within a given pipeline. The published Service Bus message's condition now combines both: `(parameters.condition) && (scenario.condition)`, matching the existing idiom in build/containers-scenarios.yml, rather than bypassing the pod-level condition. - The Docker todo-csharp/csharp, todo-go/go, and todo-rust/rust jobs and scenarios in scenarios/minimal-todo/minimal-todo.benchmarks.yml are completely unchanged (not renamed, not removed) and remain the canonical generation/update/admission/pre-promotion comparison path. Rate (25200), 1 CPU, 512 MiB, 128 connections, 30s/120s warmup/duration, 2s timeout, port 8080, and the shared steadystate config are all unchanged. - scenarios/minimal-todo/README.md: replaced the old "optional, not scheduled" description of csharp-project with a new "Execution paths" section mapping which scenario is scheduled where and why, including an explicit, hedged disclosure of the preliminary Docker-vs-direct comparison (~3% lower CPU quota for direct, similar latency; process-exit verification had an unresolved local-reader-bug caveat) -- stated plainly as not a fully accepted paired test or recalibration; the retained 25200 rate is the prior operational policy value, not re-derived from this comparison. - build/README.md: updated the Minimal Todo section to describe the new per-language cadence and point at the scenario README for the full mapping. Validation: regenerating both CI pipelines produces no diff (the cadence condition lives entirely inside the scenario-dispatch template, which is expanded at actual pipeline-run time, not at pod-scheduler generation time). The full scripts/pod-scheduler test suite (43 tests) passes. Confirmed no other file overrides minimal-todo-scenarios.yml's `scenarios` parameter default (the only two pipeline call sites pass only connection/serviceBusQueueName/serviceBusNamespace/arguments), so adding a `condition` key to every default scenario item cannot cause an Azure template "object does not contain property" expansion error elsewhere. Verified the weekly-condition formula numerically against both pipelines' actual cron schedules (ci01/ci02 "0 3/12 * * *", Azure "0 2/12 * * *") across 28 consecutive triggers (14 days) from several arbitrary start dates: exactly 2 of 28 (i.e. exactly 1 per 7 days) evaluate true for each cadence, every time -- a direct consequence of the triggers being an exact-12-hour arithmetic sequence, mod 14. Re-verified all four scenarios (csharp, go, rust, csharp-project) still resolve cleanly via `crank --debug` against a test-only sanitized temp copy (outside this repository, never committed, non-listening loopback placeholder endpoint; no real agent contacted). git diff --check is clean (pre-existing per-file CRLF/cr-at-eol cosmetic flag, same pattern as every prior round, not a real defect). No app code, build mode, or toolchain was changed. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 2d9d74a4-18fa-4e63-a55d-792c3aabe5f8 * Simplify app job runtime config to normal Crank job/application.* options Normal follow-up commit, no history rewrite/scheduler-architecture change/ live jobs. Replaces the bespoke per-job onConfigure hooks (resource-variable -> typed-property mapping, main-or-SHA format validation, forced LocalFolder=null) with plain Crank job properties, matching the established pattern in build/containers-scenarios.yml (--application.cpuLimitRatio / --application.memoryLimitInBytes). - Removed all four app onConfigure blocks (todo-csharp, todo-csharp-project, todo-go, todo-rust): no replacement runtime JS, no forced LocalFolder clear, no main-or-40hex script guard. Normal Crank job/profile/CLI semantics now apply, including local-source overrides (previously impossible: the removed hook unconditionally reset LocalFolder to null on every run regardless of any --application.sources.app.localFolder override). - Each app job now sets typed defaults directly: cpuLimitRatio: 1, memoryLimitInBytes: 536870912, port: 8080 -- preserving the existing 1 CPU / 512 MiB manual default (deleting the hook with no replacement would have silently made these 0/unlimited). All three are overridable via --application.cpuLimitRatio / --application.memoryLimitInBytes / --application.port, same as any other scenario. - The "app" source keeps its name/destination/remote-repository/cache policy; branchOrCommit now defaults to the literal main (a normal branch clone) instead of a templated "#{{sourceRevision}}" marker. The scenario's sourceRevision variable is removed; pinning an exact commit is now a normal CLI source override, --application.sources.app.branchOrCommit "#" (the nested sources dictionary form, not the obsolete --application.source.* alias -- "app" is a named source, not the default unnamed one). The obsolete cpu/memoryBytes/sourceRevision scenario variables are removed now that every consumer has been rewired; the load/workload variables (serverAddress/serverPort/rate/warmup/duration/ connections/timeout) and the imported Bombardier headers are unchanged. - Each app's PORT environment variable now derives from the job's own resolved port ("{{job.Port}}") instead of the removed serverPort-reading hook, so --application.port overrides correctly flow into the container's listening port. The imported Bombardier load job's own default port (5000) no longer matches the app's default (8080); each scenario's load role now sets variables.serverPort: 8080 (a plain per-role variable default, no JS, the same mechanism already used for path/transport/ presetHeaders/customHeaders) so the two match by default, while a CLI --variable serverPort= still overrides both app and load together when passed explicitly. - build/minimal-todo-scenarios.yml: replaced --variable cpu=1 --variable memoryBytes=536870912 --variable sourceRevision=$(Build.SourceVersion) with --application.cpuLimitRatio 1 --application.memoryLimitInBytes 536870912 --application.port 8080 --application.sources.app.branchOrCommit \"#$(Build.SourceVersion)\" (JSON- escaped double quotes around the commit value, matching this same messageBody's existing quoting convention for other values, so the leading "#" cannot be read as a shell comment start by whatever splits this string downstream). --variable serverPort=8080 and the other workload variables are unchanged; result properties (cpu/memoryBytes/ todo/*) are unchanged and remain truthful. - Docs: updated scenarios/minimal-todo/README.md's "Scenario" section to describe the new typed-property/CLI-override shape instead of sourceRevision/onConfigure, and fixed the two stale runtime YAML comment blocks (one job comment, one scenario comment) that still described csharp-project as "optional"/"not part of any scheduled lane" -- it has been the scheduled day-to-day C# lane since commit 3a802b00. Also corrected a "ci01/ci02" imprecision to "ci01" (ci02's cron and contents are unrelated to Minimal Todo). Validation: all four scenarios (csharp, go, rust, csharp-project) resolve cleanly via crank --debug against a test-only sanitized temp copy (outside this repository, never committed, non-listening loopback placeholder endpoint; no real agent contacted), confirmed for every combination the exact requirement calls for: - No CLI overrides: CpuLimitRatio=1.0, MemoryLimitInBytes=536870912, Port=8080, BranchOrCommit="main", EnvironmentVariables.PORT="8080", and the load job's resolved URL is http://.../8080/todos (serverPort default now matches) -- for all four scenarios. - With --application.cpuLimitRatio 0.5 --application.memoryLimitInBytes 268435456 --application.port 8091 --application.sources.app.branchOrCommit "#<40-hex-sha>" --variable serverPort=8091: CpuLimitRatio=0.5, MemoryLimitInBytes=268435456, Port=8091, BranchOrCommit="#" (not truncated at the "#"), EnvironmentVariables.PORT="8091" (matches Job.Port, not left stale), and the load job's resolved URL is http://.../8091/todos. - --application.sources.app.localFolder override is honored (previously impossible due to the removed forced-clear hook). - The imported official Bombardier job is still used and its arguments (headers, --http1, path) are unchanged in both cases. Separately simulated the exact build/minimal-todo-scenarios.yml messageBody JSON with placeholder values substituted for every ${{ }}/$() token: the result parses as valid JSON and the resolved args string contains --application.sources.app.branchOrCommit "#" with the quotes intact around the "#", matching this exact quoting convention already used successfully elsewhere in the same messageBody for other special-character values. The full scripts/pod-scheduler test suite (43 tests) passes; regenerating both CI pipelines produces no diff beyond the minimal-todo-scenarios.yml edit itself. git diff --check is clean (the one remaining flag is the same pre-existing per-file CRLF/cr-at-eol cosmetic pattern as every prior round, confirmed not a real defect). No app code, Dockerfile, job/scenario rename, SelfContained/framework/channel, or generation/comparison Docker path was changed; the Go/Rust weekly and C# regular-cadence scheduling from the prior commit is unchanged. No live agents or Docker commands were used. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 2d9d74a4-18fa-4e63-a55d-792c3aabe5f8 * Add memory/rate annotations; remove test/fixture gunk from runtime repo Normal follow-up commit, no history rewrite/scheduler-architecture change/ merge/live jobs. The PR was already marked ready for review (draft: false); that state is preserved. RUNTIME CONFIG (no workload/value change): - Added a `# 512 MiB` human comment to each of the four app jobs' memoryLimitInBytes: 536870912 (todo-csharp, todo-csharp-project, todo-go, todo-rust). - Added an explicit `rate: 100 # Override with --variable rate=...` to each of the four scenarios' load role (csharp, go, rust, csharp-project). The imported Bombardier job's own default rate is 0 (uncapped); the scenario's former root-level `rate: 100` variable never actually reached the load job (the same non-propagation behavior already documented for serverPort), so a manual run with no --variable override was silently uncapped rather than the intended low smoke-test default. Removed the now-redundant root `rate` variable (nothing in this file templates `{{rate}}` directly; only the per-role default has any effect). A CLI `--variable rate=...` override (e.g. the scheduled 25200) still wins over this default, confirmed empirically. TEST/FIXTURE/EVIDENCE REMOVAL (runtime repo only -- tools-side canonical archive is untouched, already independently preserved per 248's prior confirmation): - Deleted csharp/tests/TodoApi.Tests/ entirely (ApiFixture.cs, AppProcess.cs, ContractTests.cs, LifecycleTests.cs, TodoApi.Tests.csproj, minimal-todo-v1.expected.json, packages.lock.json). - Deleted go/lifecycle_test.go, go/server_test.go, go/testdata/ (the golden fixture). - Deleted rust/tests/ entirely (contract.rs, lifecycle.rs, fixtures/minimal-todo-v1.expected.json). - Zero production .cs/.go/.rs files were touched (confirmed via `git diff --cached --stat` filtered to those paths: no output). The fixed five-Todo dataset, Program.cs/Todo.cs, main.go/server.go, main.rs/lib.rs (including its embedded `#[cfg(test)]` unit tests, which are part of the production file and were left untouched), Cargo.toml/Cargo.lock/go.mod, and all Dockerfiles needed for regular runs are all unchanged. WIRED BUILD SURFACES: - rust/Dockerfile: removed `COPY tests ./tests` and the `FROM build AS test` / `RUN cargo test --release --locked` stage; kept the production `cargo build --release --locked`, the ubuntu:26.04 runtime stage, nonroot user, and entrypoint unchanged. - csharp/TodoApi.slnx: removed the `/tests/` folder and TodoApi.Tests.csproj reference, now a single-project solution (src/TodoApi/TodoApi.csproj). - csharp/src/TodoApi/TodoApi.csproj: removed the dead `` (csproj metadata only, no .cs edit). - Dead dockerignore/gitignore patterns removed: csharp/.dockerignore's `tests` line, csharp/.gitignore's now-unreachable `TestResults/` line, go/.dockerignore's `*_test.go`/`testdata/` lines, rust/.dockerignore's `!tests/`/`!tests/**` allow-list lines (its "ignore everything, allow-list specific paths" pattern no longer needs to admit a path that doesn't exist). README CLEANUP (operator-facing only; full generated test suite, fixture, and validation/campaign history remain owned by the separate dotnet-performance-tools canonical archive): - csharp/README.md, go/README.md, rust/README.md: rewritten concisely to app/framework description, build, run, contract, PORT, readiness, shutdown, and container build/run instructions only. Removed the historical Windows-SDK-10/original-Debian-image validation sections, test run commands/counts, fixture SHA-256 hashes, local artifact/bin/log/ evidence directory listings, and the Files sections' now-dead test-file entries. - scenarios/minimal-todo/README.md: documents the new rate/memory/serverPort per-role defaults; states plainly that production code only ships here (no test projects/fixtures); adds an explicit operational note that the 25200 req/s rate is a uniform policy value applied to all three languages and both the Docker and direct C# builds, not a per-language or per-resource-point capacity guarantee; reinforces that the full generated test suite/fixture/checker/review/campaign history is owned by dotnet-performance-tools. The C#-direct-regular/Go-Rust-weekly cadence and Docker-comparison-availability design from the prior two commits is unchanged and explicitly preserved. Validation: `git diff --cached --stat` filtered to *.cs/*.go/*.rs shows zero production file changes (only test-file deletions). Grepped the full repository for references to every deleted file/directory name: zero remaining matches (no dead links in any README or build/*.yml). All four scenarios (csharp, go, rust, csharp-project) resolve cleanly via `crank --debug` against a test-only sanitized temp copy (outside this repository, never committed, non-listening loopback placeholder endpoint; no real agent or Docker command used), confirming: default load rate is now 100 (previously an uncapped 0), memoryLimitInBytes is unchanged at 536870912, and a `--variable rate=25200` CLI override still wins. The full scripts/pod-scheduler test suite (43 tests) passes; regenerating both CI pipelines produces zero diff. git diff --check is clean (the only flagged lines are the same pre-existing per-file CRLF/cr-at-eol cosmetic pattern seen in every prior round for a fully-rewritten file -- confirmed zero genuine trailing whitespace). Docker and the floating toolchains remain unavailable in this environment, so the Dockerfiles could not be build-verified here; this is an honest, disclosed limitation, not a passed build. No shared agent, Docker daemon, install, or remote probe was started. No live/native/lab/SQL/workflow dispatch was performed. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 2d9d74a4-18fa-4e63-a55d-792c3aabe5f8 --------- Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 2d9d74a4-18fa-4e63-a55d-792c3aabe5f8 --- build/README.md | 17 + build/benchmarks-ci-01.yml | 40 +- build/benchmarks-ci-02.yml | 18 +- build/benchmarks-ci-azure.yml | 46 +- build/benchmarks_ci_azure_pods.json | 8 + build/benchmarks_ci_pods.json | 8 + build/job-variables.yml | 2 + build/minimal-todo-scenarios.yml | 59 ++ scenarios/README.md | 6 + scenarios/minimal-todo/.gitattributes | 1 + scenarios/minimal-todo/README.md | 161 +++++ .../minimal-todo/minimal-todo.benchmarks.yml | 169 +++++ src/BenchmarksApps/MinimalTodo/.gitattributes | 4 + .../MinimalTodo/Directory.Build.targets | 1 + .../MinimalTodo/csharp/.dockerignore | 6 + .../MinimalTodo/csharp/.gitignore | 3 + .../MinimalTodo/csharp/Directory.Build.props | 8 + .../MinimalTodo/csharp/Dockerfile | 15 + .../MinimalTodo/csharp/NuGet.Config | 10 + .../MinimalTodo/csharp/README.md | 81 +++ .../MinimalTodo/csharp/TodoApi.slnx | 5 + .../MinimalTodo/csharp/src/TodoApi/Program.cs | 92 +++ .../MinimalTodo/csharp/src/TodoApi/Todo.cs | 9 + .../csharp/src/TodoApi/TodoApi.csproj | 2 + .../csharp/src/TodoApi/packages.lock.json | 6 + .../MinimalTodo/go/.dockerignore | 3 + src/BenchmarksApps/MinimalTodo/go/Dockerfile | 16 + src/BenchmarksApps/MinimalTodo/go/README.md | 116 ++++ src/BenchmarksApps/MinimalTodo/go/go.mod | 3 + src/BenchmarksApps/MinimalTodo/go/main.go | 107 ++++ src/BenchmarksApps/MinimalTodo/go/server.go | 89 +++ .../MinimalTodo/rust/.dockerignore | 7 + .../MinimalTodo/rust/.gitignore | 1 + .../MinimalTodo/rust/Cargo.lock | 603 ++++++++++++++++++ .../MinimalTodo/rust/Cargo.toml | 24 + .../MinimalTodo/rust/Dockerfile | 15 + src/BenchmarksApps/MinimalTodo/rust/README.md | 93 +++ .../MinimalTodo/rust/rust-toolchain.toml | 3 + .../MinimalTodo/rust/src/lib.rs | 144 +++++ .../MinimalTodo/rust/src/main.rs | 100 +++ 40 files changed, 2072 insertions(+), 29 deletions(-) create mode 100644 build/minimal-todo-scenarios.yml create mode 100644 scenarios/minimal-todo/.gitattributes create mode 100644 scenarios/minimal-todo/README.md create mode 100644 scenarios/minimal-todo/minimal-todo.benchmarks.yml create mode 100644 src/BenchmarksApps/MinimalTodo/.gitattributes create mode 100644 src/BenchmarksApps/MinimalTodo/Directory.Build.targets create mode 100644 src/BenchmarksApps/MinimalTodo/csharp/.dockerignore create mode 100644 src/BenchmarksApps/MinimalTodo/csharp/.gitignore create mode 100644 src/BenchmarksApps/MinimalTodo/csharp/Directory.Build.props create mode 100644 src/BenchmarksApps/MinimalTodo/csharp/Dockerfile create mode 100644 src/BenchmarksApps/MinimalTodo/csharp/NuGet.Config create mode 100644 src/BenchmarksApps/MinimalTodo/csharp/README.md create mode 100644 src/BenchmarksApps/MinimalTodo/csharp/TodoApi.slnx create mode 100644 src/BenchmarksApps/MinimalTodo/csharp/src/TodoApi/Program.cs create mode 100644 src/BenchmarksApps/MinimalTodo/csharp/src/TodoApi/Todo.cs create mode 100644 src/BenchmarksApps/MinimalTodo/csharp/src/TodoApi/TodoApi.csproj create mode 100644 src/BenchmarksApps/MinimalTodo/csharp/src/TodoApi/packages.lock.json create mode 100644 src/BenchmarksApps/MinimalTodo/go/.dockerignore create mode 100644 src/BenchmarksApps/MinimalTodo/go/Dockerfile create mode 100644 src/BenchmarksApps/MinimalTodo/go/README.md create mode 100644 src/BenchmarksApps/MinimalTodo/go/go.mod create mode 100644 src/BenchmarksApps/MinimalTodo/go/main.go create mode 100644 src/BenchmarksApps/MinimalTodo/go/server.go create mode 100644 src/BenchmarksApps/MinimalTodo/rust/.dockerignore create mode 100644 src/BenchmarksApps/MinimalTodo/rust/.gitignore create mode 100644 src/BenchmarksApps/MinimalTodo/rust/Cargo.lock create mode 100644 src/BenchmarksApps/MinimalTodo/rust/Cargo.toml create mode 100644 src/BenchmarksApps/MinimalTodo/rust/Dockerfile create mode 100644 src/BenchmarksApps/MinimalTodo/rust/README.md create mode 100644 src/BenchmarksApps/MinimalTodo/rust/rust-toolchain.toml create mode 100644 src/BenchmarksApps/MinimalTodo/rust/src/lib.rs create mode 100644 src/BenchmarksApps/MinimalTodo/rust/src/main.rs diff --git a/build/README.md b/build/README.md index 205883ac0..3b6c7477d 100644 --- a/build/README.md +++ b/build/README.md @@ -69,3 +69,20 @@ After any edit, regenerate the YAML and commit both the JSON and the YAML together. The snapshot tests in `scripts/pod-scheduler/tests/test_snapshots.py` will fail in CI if the generated YAML drifts from the configs. + +## Minimal Todo + +`minimal-todo-scenarios.yml` dispatches `scenarios/minimal-todo/minimal-todo.benchmarks.yml` +(1 CPU, 536,870,912 bytes / 512 MiB, 25200 req/s, enabled) for the `gold-lin` +and Azure `azure-arm64`/`cobalt-cloud-lin` pods. C# runs on every regular +scheduled slot using the direct (non-Docker) `csharp-project` build; Go and +Rust run the Docker `go`/`rust` builds but only on a shared weekly slot +(`condition: Math.round(Date.now() / 43200000) % 14 == 0`, combined with the +pod's own condition, matching the `containers-scenarios.yml` idiom). The +Docker `csharp`/`go`/`rust` builds (floating, multi-arch base images, no +pinned digests, no `requiredArchitecture` gating) remain the canonical +generation/update/admission/pre-promotion comparison path and are unaffected +-- see `scenarios/minimal-todo/README.md` for the full execution-path +mapping. Generation, admission, and calibration tooling for this cohort live +in the separate `dotnet-performance-tools` +repository; this directory only carries the scheduled dispatch template. diff --git a/build/benchmarks-ci-01.yml b/build/benchmarks-ci-01.yml index 42688238f..577e7ee55 100644 --- a/build/benchmarks-ci-01.yml +++ b/build/benchmarks-ci-01.yml @@ -240,27 +240,43 @@ jobs: # GROUP 8 -- job: Trends_Database_gold_lin - displayName: 8- Trends Database gold-lin +- job: Minimal_Todo_gold_lin + displayName: 8- Minimal Todo gold-lin pool: server - timeoutInMinutes: 120 + timeoutInMinutes: 40 dependsOn: [Frameworks_gold_lin, Crossgen_gold_win] condition: succeededOrFailed() steps: - - template: trend-database-scenarios.yml + - template: minimal-todo-scenarios.yml parameters: connection: ASPNET Benchmarks Service Bus serviceBusQueueName: citrine1 serviceBusNamespace: aspnetbenchmarks - arguments: "$(ciProfile) --profile gold-lin-app --profile gold-load-load --profile gold-db-db " + arguments: "$(ciProfile) --profile gold-lin-app --profile gold-load-load " # GROUP 9 +- job: Single_File_gold_lin + displayName: 9- Single File gold-lin + pool: server + timeoutInMinutes: 120 + dependsOn: [Minimal_Todo_gold_lin] + condition: succeededOrFailed() + steps: + - template: singlefile-scenarios.yml + parameters: + connection: ASPNET Benchmarks Service Bus + serviceBusQueueName: citrine1 + serviceBusNamespace: aspnetbenchmarks + arguments: "$(ciProfile) --profile gold-lin-app --profile gold-load-load " + +# GROUP 10 + - job: WebSockets_gold_lin - displayName: 9- WebSockets gold-lin + displayName: 10- WebSockets gold-lin pool: server timeoutInMinutes: 120 - dependsOn: [Trends_Database_gold_lin] + dependsOn: [Single_File_gold_lin] condition: succeededOrFailed() steps: - template: websocket-scenarios.yml @@ -270,19 +286,19 @@ jobs: serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile gold-lin-app --profile gold-load-load " -# GROUP 10 +# GROUP 11 -- job: Custom_Proxies_gold_lin - displayName: 10- Custom Proxies gold-lin +- job: Build_gold_lin + displayName: 11- Build gold-lin pool: server timeoutInMinutes: 120 dependsOn: [WebSockets_gold_lin] condition: succeededOrFailed() steps: - - template: proxies-custom-scenarios.yml + - template: build-perf-scenarios.yml parameters: connection: ASPNET Benchmarks Service Bus serviceBusQueueName: citrine1 serviceBusNamespace: aspnetbenchmarks - arguments: "$(ciProfile) --profile gold-lin-app --profile gold-load-load --profile gold-db-db " + arguments: "$(ciProfile) --profile gold-lin-app " diff --git a/build/benchmarks-ci-02.yml b/build/benchmarks-ci-02.yml index 1eab26370..ebf4c37b5 100644 --- a/build/benchmarks-ci-02.yml +++ b/build/benchmarks-ci-02.yml @@ -300,19 +300,19 @@ jobs: # GROUP 10 -- job: Single_File_gold_lin - displayName: 10- Single File gold-lin +- job: Trends_Database_gold_lin + displayName: 10- Trends Database gold-lin pool: server timeoutInMinutes: 120 dependsOn: [GC_gold_lin, Build_gold_win] condition: succeededOrFailed() steps: - - template: singlefile-scenarios.yml + - template: trend-database-scenarios.yml parameters: connection: ASPNET Benchmarks Service Bus serviceBusQueueName: citrine1 serviceBusNamespace: aspnetbenchmarks - arguments: "$(ciProfile) --profile gold-lin-app --profile gold-load-load " + arguments: "$(ciProfile) --profile gold-lin-app --profile gold-load-load --profile gold-db-db " # GROUP 11 @@ -320,7 +320,7 @@ jobs: displayName: 11- Crossgen gold-lin pool: server timeoutInMinutes: 120 - dependsOn: [Single_File_gold_lin] + dependsOn: [Trends_Database_gold_lin] condition: succeededOrFailed() steps: - template: crossgen2-scenarios.yml @@ -332,17 +332,17 @@ jobs: # GROUP 12 -- job: Build_gold_lin - displayName: 12- Build gold-lin +- job: Custom_Proxies_gold_lin + displayName: 12- Custom Proxies gold-lin pool: server timeoutInMinutes: 120 dependsOn: [Crossgen_gold_lin] condition: succeededOrFailed() steps: - - template: build-perf-scenarios.yml + - template: proxies-custom-scenarios.yml parameters: connection: ASPNET Benchmarks Service Bus serviceBusQueueName: citrine1 serviceBusNamespace: aspnetbenchmarks - arguments: "$(ciProfile) --profile gold-lin-app " + arguments: "$(ciProfile) --profile gold-lin-app --profile gold-load-load --profile gold-db-db " diff --git a/build/benchmarks-ci-azure.yml b/build/benchmarks-ci-azure.yml index 2406d43af..efe88d70a 100644 --- a/build/benchmarks-ci-azure.yml +++ b/build/benchmarks-ci-azure.yml @@ -360,11 +360,41 @@ jobs: # GROUP 12 +- job: Minimal_Todo_azure_arm64 + displayName: 12- Minimal Todo azure-arm64 + pool: server + timeoutInMinutes: 40 + dependsOn: [Trends_idna_intel_lin, Trends_idna_intel_win] + condition: succeededOrFailed() + steps: + - template: minimal-todo-scenarios.yml + parameters: + connection: ASPNET Benchmarks Service Bus + serviceBusQueueName: azure + serviceBusNamespace: aspnetbenchmarks + arguments: "$(ciProfile) --profile azure-server-arm64-app --profile azure-client-load " + +- job: Minimal_Todo_cobalt_cloud_lin + displayName: 12- Minimal Todo cobalt-cloud-lin + pool: server + timeoutInMinutes: 40 + dependsOn: [Trends_idna_intel_lin, Trends_idna_intel_win] + condition: succeededOrFailed() + steps: + - template: minimal-todo-scenarios.yml + parameters: + connection: ASPNET Benchmarks Service Bus + serviceBusQueueName: azurearm64 + serviceBusNamespace: aspnetbenchmarks + arguments: "$(ciProfile) --profile cobalt-cloud-lin-server-app --profile cobalt-cloud-lin-client-load " + +# GROUP 13 + - job: Trends_Database_azure_arm64 - displayName: 12- Trends Database azure-arm64 + displayName: 13- Trends Database azure-arm64 pool: server timeoutInMinutes: 120 - dependsOn: [Trends_idna_intel_lin, Trends_idna_intel_win] + dependsOn: [Minimal_Todo_azure_arm64, Minimal_Todo_cobalt_cloud_lin] condition: succeededOrFailed() steps: - template: trend-database-scenarios.yml @@ -375,10 +405,10 @@ jobs: arguments: "$(ciProfile) --profile azure-server-arm64-app --profile azure-client-load --profile azure-db-db " - job: Trends_Database_azure2_amd64 - displayName: 12- Trends Database azure2-amd64 + displayName: 13- Trends Database azure2-amd64 pool: server timeoutInMinutes: 120 - dependsOn: [Trends_idna_intel_lin, Trends_idna_intel_win] + dependsOn: [Minimal_Todo_azure_arm64, Minimal_Todo_cobalt_cloud_lin] condition: succeededOrFailed() steps: - template: trend-database-scenarios.yml @@ -388,10 +418,10 @@ jobs: serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile azure2-server-amd64-app --profile azure2-client-load --profile azure2-db-db " -# GROUP 13 +# GROUP 14 - job: Trends_Database_cobalt_cloud_lin - displayName: 13- Trends Database cobalt-cloud-lin + displayName: 14- Trends Database cobalt-cloud-lin pool: server timeoutInMinutes: 120 dependsOn: [Trends_Database_azure_arm64, Trends_Database_azure2_amd64] @@ -404,10 +434,10 @@ jobs: serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile cobalt-cloud-lin-server-app --profile cobalt-cloud-lin-client-load --profile cobalt-cloud-lin-db-db " -# GROUP 14 +# GROUP 15 - job: Trends_Database_cobalt_cloud_lin_azl3 - displayName: 14- Trends Database cobalt-cloud-lin-azl3 + displayName: 15- Trends Database cobalt-cloud-lin-azl3 pool: server timeoutInMinutes: 120 dependsOn: [Trends_Database_cobalt_cloud_lin] diff --git a/build/benchmarks_ci_azure_pods.json b/build/benchmarks_ci_azure_pods.json index fead30052..12cfefca9 100644 --- a/build/benchmarks_ci_azure_pods.json +++ b/build/benchmarks_ci_azure_pods.json @@ -61,6 +61,14 @@ } ], "scenarios": [ + { + "name": "Minimal Todo", + "template": "minimal-todo-scenarios.yml", + "type": 2, + "pods": ["azure-arm64", "cobalt-cloud-lin"], + "estimated_runtime": 15.0, + "timeout": 40 + }, { "name": "Containers", "template": "containers-scenarios.yml", diff --git a/build/benchmarks_ci_pods.json b/build/benchmarks_ci_pods.json index 3486ca0b6..23b7411d2 100644 --- a/build/benchmarks_ci_pods.json +++ b/build/benchmarks_ci_pods.json @@ -119,6 +119,14 @@ "pods": ["gold-lin", "gold-win"], "estimated_runtime": 45.0 }, + { + "name": "Minimal Todo", + "template": "minimal-todo-scenarios.yml", + "type": 2, + "pods": ["gold-lin"], + "estimated_runtime": 15.0, + "timeout": 40 + }, { "name": "MVC", "template": "mvc-scenarios.yml", diff --git a/build/job-variables.yml b/build/job-variables.yml index 59259217f..69216f750 100644 --- a/build/job-variables.yml +++ b/build/job-variables.yml @@ -87,6 +87,8 @@ variables: value: --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/orchard.benchmarks.yml - name: containerMatrixJobs value: --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/containers.benchmarks.yml +- name: minimalTodoJobs + value: --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/minimal-todo/minimal-todo.benchmarks.yml - name: minimalJobs value: --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/src/BenchmarksApps/TechEmpower/Minimal/minimal.benchmarks.yml - name: blazorSsrJobs diff --git a/build/minimal-todo-scenarios.yml b/build/minimal-todo-scenarios.yml new file mode 100644 index 000000000..5cacc8076 --- /dev/null +++ b/build/minimal-todo-scenarios.yml @@ -0,0 +1,59 @@ +# Daily candidate, enabled with the accepted calibrated rate shared by all languages. +# C# uses the direct (non-Docker) csharp-project build day-to-day, every +# regular scheduled run. Go and Rust use the Docker csharp/go/rust build and +# run on a shared weekly slot (once per 14 half-day scheduled slots, i.e. once +# per ~7 days for each pipeline's own 12-hour cron, not a single global +# schedule across pods). The Docker todo-csharp/csharp build, and the +# todo-go/todo-rust/go/rust builds, remain the canonical generation/update/ +# admission/pre-promotion comparison path and are not removed -- see +# scenarios/minimal-todo/README.md for the full execution-path mapping. +parameters: +- name: arguments + type: string + default: '' +- name: connection + type: string + default: '' +- name: serviceBusQueueName + type: string + default: '' +- name: serviceBusNamespace + type: string + default: '' +- name: condition + type: string + default: 'true' +- name: rate + type: string + default: '25200' +- name: scenarios + type: object + default: + - displayName: C# nightly (direct) + arguments: --scenario csharp-project --property todo/language=csharp --property todo/framework=net11.0 + condition: 'true' + - displayName: Go + arguments: --scenario go --property todo/language=go --property todo/framework=net-http + condition: Math.round(Date.now() / 43200000) % 14 == 0 # once every 14 half-days (weekly) + - displayName: Rust + arguments: --scenario rust --property todo/language=rust --property todo/framework=axum-tokio + condition: Math.round(Date.now() / 43200000) % 14 == 0 # once every 14 half-days (weekly) + +steps: +- ${{ each scenario in parameters.scenarios }}: + - task: PublishToAzureServiceBus@2 + condition: succeededOrFailed() + displayName: Minimal Todo ${{ scenario.displayName }} - 1 CPU - 512 MiB + timeoutInMinutes: 10 + inputs: + connectedServiceName: ${{ parameters.connection }} + serviceBusQueueName: ${{ parameters.serviceBusQueueName }} + serviceBusNamespace: ${{ parameters.serviceBusNamespace }} + waitForCompletion: true + useDataContractSerializer: "false" + messageBody: | + { + "name": "crank", + "condition": "(${{ parameters.condition }}) && (${{ scenario.condition }})", + "args": [ "--session $(session) $(minimalTodoJobs) ${{ scenario.arguments }} --application.cpuLimitRatio 1 --application.memoryLimitInBytes 536870912 --application.port 8080 --application.sources.app.branchOrCommit \"#$(Build.SourceVersion)\" --variable rate=${{ parameters.rate }} --variable connections=128 --variable warmup=30 --variable duration=120 --variable timeout=2 --variable serverPort=8080 --property todo/scenario=minimal-todo-v1 --property todo/cohort=minimal-todo-v1-20260929-01 --property todo/ratePolicy=csharp-quota-75-fixed-v1 --property cpu=1 --property memoryBytes=536870912 --property rate=${{ parameters.rate }} --property todo/cpu=1 --property todo/memoryBytes=536870912 --property todo/rate=${{ parameters.rate }} ${{ parameters.arguments }} $(azureProfile) --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/steadystate.profile.yml --no-metadata --no-measurements --command-line-property --table ContainerBenchmarks --sql SQL_CONNECTION_STRING --cert-tenant-id SQL_SERVER_TENANTID --cert-client-id SQL_SERVER_CLIENTID --cert-path SQL_SERVER_CERT_PATH --cert-sni --chart" ] + } diff --git a/scenarios/README.md b/scenarios/README.md index 50c475800..6f3d4bd7b 100644 --- a/scenarios/README.md +++ b/scenarios/README.md @@ -47,6 +47,12 @@ dotnet tool install Microsoft.Crank.Agent --version "0.2.0-*" --global crank-agent ``` +## Minimal Todo + +The [Minimal Todo package](minimal-todo/README.md) contains frozen standalone +C#, Go and Rust applications, a versioned HTTP contract, and native Crank +Dockerfile scenarios with separate operator topology profiles. + ## Plaintext benchmarks The source code for these benchmarks is located [here](https://github.com/aspnet/benchmarks/tree/main/src/Benchmarks). diff --git a/scenarios/minimal-todo/.gitattributes b/scenarios/minimal-todo/.gitattributes new file mode 100644 index 000000000..ccd2cd7bd --- /dev/null +++ b/scenarios/minimal-todo/.gitattributes @@ -0,0 +1 @@ +minimal-todo.benchmarks.yml text eol=crlf diff --git a/scenarios/minimal-todo/README.md b/scenarios/minimal-todo/README.md new file mode 100644 index 000000000..3775f04f4 --- /dev/null +++ b/scenarios/minimal-todo/README.md @@ -0,0 +1,161 @@ +# Minimal Todo v1 + +A small HTTP CRUD workload (`/todos`, `/healthz`) implemented three times — +C#, Go, and Rust — to compare minimal API frameworks under an identical +fixed-rate load. Source generation, admission review, quota-calibration, and +the full generated test suite/fixture/validation history for this cohort are +owned by the separate `dotnet-performance-tools` repository, which holds the +canonical candidate archive. This package is the shipped, day-to-day Crank +runtime surface only: the applications (production code only, no test +projects/fixtures) and one scenario file. + +## Applications + +Applications live in `src/BenchmarksApps/MinimalTodo/{csharp,go,rust}`, each +with its own `Dockerfile` and README covering its local build/run lifecycle. +Production code only; no test projects, fixtures, or validation evidence are +shipped here (see "Results" below for where that history lives). + +Base images are floating (no pinned digests), multi-arch, and share one final +runtime OS across all three languages: + +| Language | Build image | Final image | +| --- | --- | --- | +| C# (.NET 11, ASP.NET Core Minimal APIs) | `mcr.microsoft.com/dotnet/nightly/sdk:11.0` | `mcr.microsoft.com/dotnet/nightly/aspnet:11.0` (Ubuntu 26.04) | +| Go (`net/http`) | `golang:1.27-bookworm` | `ubuntu:26.04` | +| Rust (Axum/Tokio) | `rust:1.96-bookworm` | `ubuntu:26.04` | + +.NET 12 images are not yet published; C# intentionally floats on the .NET 11 +nightly channel until a stable .NET 12 tag exists. Floating tags mean the +exact compiler/runtime/OS patch level can change between runs; this is a +deliberate simplification over pinning digests, at the cost of exact +reproducibility between invocations. + +## Scenario + +[minimal-todo.benchmarks.yml](minimal-todo.benchmarks.yml) defines three +Docker-building scenarios (`csharp`, `go`, `rust`), each with two roles: +`application` and `load`; plus one direct, non-Docker scenario, +`csharp-project` (see "Execution paths" below for which one is scheduled +day-to-day). In each Docker scenario, the application job +clones `aspnet/Benchmarks` (the `app` source, `branchOrCommit` defaulting to +`main`), builds that language's +`Dockerfile`, and waits for `Application started.` +(the only readiness gate a regular run performs — the full HTTP state/body +contract is verified once, separately, by the `dotnet-performance-tools` +generation/admission flow, not on every run). `load` imports and runs the +official, unmodified +[`dotnet/crank` Bombardier job](https://raw.githubusercontent.com/dotnet/crank/main/src/Microsoft.Crank.Jobs.Bombardier/bombardier.yml) +(floating on its own `main`, matching this round's floating-image intent) +against `/todos` at a fixed offered rate, with `transport: http1`, +`presetHeaders: none`, and explicit `customHeaders` for +`Accept: application/json`, `Connection: keep-alive`, and +`Accept-Encoding: identity`. No per-job architecture gating is applied: +images are multi-arch, so the scenario runs unmodified on x64 or ARM64 +agents. + +Each app job uses normal, typed Crank job properties -- `cpuLimitRatio: 1`, +`memoryLimitInBytes: 536870912` (512 MiB), `port: 8080` -- with no custom +`onConfigure` hook. These are overridable the same way `build/containers-scenarios.yml` +overrides its own jobs: `--application.cpuLimitRatio`, +`--application.memoryLimitInBytes`, `--application.port`. Each scenario's +`load` role sets its own `rate: 100` (a safe, low manual-smoke default; the +imported Bombardier job's own default is `0`, meaning uncapped) and +`serverPort: 8080` (matching the application's default port, since the +imported job's own default is `5000`); both are overridable via +`--variable rate=...`/`--variable serverPort=...`, exactly like the +scheduled pipeline does for its accepted rate. The scheduled +pipeline pins the exact triggering build's commit via +`--application.sources.app.branchOrCommit "#"` (see +`build/minimal-todo-scenarios.yml`); a manual run defaults to the `app` +source's `main` branch, or can pass any normal Crank source override +(`--application.sources.app.branchOrCommit`, `.localFolder`, etc.) the same +way as any other scenario. A SHA must use the full-clone `"#" + sha` form +Crank requires for an exact commit — a bare SHA would instead attempt a +shallow branch clone and fail; this is enforced by normal Crank source +semantics, not a scenario-specific guard. Exact-commit generation provenance +is enforced by the separate `dotnet-performance-tools` admission flow, not by +this YAML. Manual connected runs use the shared `build/ci.profile.yml` / +`build/azure.profile.yml` profiles (or explicit +`--variable serverAddress=...`/endpoint overrides) the same way every other +scenario in this repository does; there is no scenario-specific profile +file. + +### Execution paths: Docker (generation/comparison) vs. direct (day-to-day C#) + +The Docker scenarios (`csharp`, `go`, `rust`, backed by the `todo-csharp`/ +`todo-go`/`todo-rust` jobs) are unchanged and remain the canonical +generation/update/admission/pre-promotion comparison path: fully resolvable +for manual runs and for the separate `dotnet-performance-tools` generation +flow, exactly as before. + +`build/minimal-todo-scenarios.yml`'s scheduled dispatch now differs by +language: + +- **C#** submits `--scenario csharp-project` (the direct, non-Docker build + described below) on every regular scheduled run, at the same cadence as + before (each pipeline's own ~12-hour cron, unconditionally). +- **Go and Rust** still submit the Docker `go`/`rust` scenarios, but now run + **weekly** instead of twice a day: each scenario entry adds + `condition: Math.round(Date.now() / 43200000) % 14 == 0` (true once every + 14 half-day scheduled slots, i.e. once per ~7 days), combined with the + existing pod-level condition (`(pod condition) && (scenario condition)`, + matching the idiom already used in `build/containers-scenarios.yml`). This + is evaluated independently per pipeline/pod (the `gold-lin` pod via the + `ci01` pipeline's cron, `azure-arm64`/`cobalt-cloud-lin` via the + Azure pipeline's own cron) -- it is **not** one single global weekly + schedule shared across every pod, and an ad hoc or duplicate pipeline run + is not deduplicated by this condition alone. + +The `csharp-project` scenario builds the exact same `src/TodoApi/TodoApi.csproj` +directly with Crank (`project:` + `framework: net11.0` + `channel: latest`), +with no Docker image at all, using the same source/CPU/memory/port +defaults and the same imported Bombardier load job as the Docker `csharp` +scenario. `channel: latest` selects a coherent latest SDK/runtime/ASP.NET +Core build (the goal is comparing against current released/floating +versions, not an incompatible bleeding-edge combination). This only changes +how the app is **packaged and started** — a native process started by Crank +on the agent's host OS instead of a container image pulled from a registry. +Crank's own CPU/memory limits (`CpuLimitRatio`/`MemoryLimitInBytes`, via +Linux cgroups) are applied identically to both the Docker and direct jobs, +so a Docker-vs-direct comparison isolates packaging/runtime/host-OS effects, +not a CPU/memory quota difference. The distinct scenario name +(`csharp-project` vs. `csharp`) is itself recorded via the existing +`--command-line-property` flag already used by the scheduled dispatch, so +direct and Docker runs are already distinguishable in persisted results +without any new custom property. + +**Comparison status (disclosed, not a formal acceptance):** a preliminary, +exploratory comparison between the Docker and direct C# builds showed +roughly 3% lower CPU quota usage for the direct build with similar latency; +process-exit verification had an unresolved local-reader-bug caveat and was +not conclusively proven either way. This is **not** a fully accepted paired +test or a recalibration — the existing `rate` (25200 req/s) is retained as +the prior operational policy value, unchanged and not re-derived from this +comparison. The direct build was still adopted for day-to-day C# scheduling +based on this preliminary closeness, as an explicit operational decision. + +### Results + +Append the **unchanged** `../steadystate.profile.yml` once as the final +`--config` (not an import). Reporting is otherwise stock Crank: the official +Bombardier wrapper's own request/bad-response/latency/RPS/throughput +measurements, the agent's own CPU/memory measurements, and the shared +steadystate profile's CPU/working-set P90 helpers. `bombardier/raw` (the raw +per-request JSON payload) is excluded from persistence to avoid storing that +large aggregate; no other custom result definitions or hooks are shipped — +calibration-quality analysis belongs to `dotnet-performance-tools`. + +The shipped `rate` (25200 req/s, see `build/minimal-todo-scenarios.yml`) was +calibrated and accepted against the **prior** pinned-digest C#/Go/Rust images +and project settings. This round floated the Docker base images and moved the +C# project to `net11.0`; that calibration has not been re-run against the new +images, so a post-merge scheduled run is the first operational confirmation +under the new images, not a re-validation. 25200 req/s is the current +**operational policy value** applied uniformly to all three languages and +both the Docker and direct C# builds — it is not a per-language or +per-resource-point capacity guarantee, and resource headroom is not +independently verified for every combination. Full generated test +suites, fixture/contract checkers, review, and campaign/validation history +for this cohort are owned by the separate `dotnet-performance-tools` +repository's canonical candidate archive. diff --git a/scenarios/minimal-todo/minimal-todo.benchmarks.yml b/scenarios/minimal-todo/minimal-todo.benchmarks.yml new file mode 100644 index 000000000..6b01b39d8 --- /dev/null +++ b/scenarios/minimal-todo/minimal-todo.benchmarks.yml @@ -0,0 +1,169 @@ +# Remote Crank scenario. App jobs use normal Crank job/source/profile/CLI +# semantics: typed cpuLimitRatio/memoryLimitInBytes/port defaults below are +# overridable via --application.cpuLimitRatio/--application.memoryLimitInBytes/ +# --application.port, and the "app" source's branchOrCommit (default "main") +# via --application.sources.app.branchOrCommit, exactly like +# build/containers-scenarios.yml does. Append the unchanged +# ../steadystate.profile.yml once as the FINAL --config. +imports: + - https://raw.githubusercontent.com/dotnet/crank/main/src/Microsoft.Crank.Jobs.Bombardier/bombardier.yml + +variables: + serverAddress: "" + serverPort: 8080 + warmup: 5 + duration: 15 + connections: 8 + timeout: 2 + +jobs: + todo-csharp: + sources: + app: + localFolder: null + repository: https://github.com/aspnet/Benchmarks.git + branchOrCommit: main + destinationFolder: app + cacheOnAgent: false + dockerFile: app/src/BenchmarksApps/MinimalTodo/csharp/Dockerfile + dockerContextDirectory: app/src/BenchmarksApps/MinimalTodo/csharp + dockerImageName: "minimal-todo-csharp-{{job.RunId}}" + readyStateText: Application started. + cpuLimitRatio: 1 + memoryLimitInBytes: 536870912 # 512 MiB + port: 8080 + environmentVariables: + PORT: "{{job.Port}}" + options: + requiredOperatingSystem: linux + + # Day-to-day scheduled C# comparison job: builds src/TodoApi/TodoApi.csproj + # directly with Crank (no Docker) -- see README "Execution paths". Same + # source/CPU/memory/port defaults as todo-csharp. + todo-csharp-project: + sources: + app: + localFolder: null + repository: https://github.com/aspnet/Benchmarks.git + branchOrCommit: main + destinationFolder: app + cacheOnAgent: false + project: app/src/BenchmarksApps/MinimalTodo/csharp/src/TodoApi/TodoApi.csproj + framework: net11.0 + channel: latest + readyStateText: Application started. + cpuLimitRatio: 1 + memoryLimitInBytes: 536870912 # 512 MiB + port: 8080 + environmentVariables: + PORT: "{{job.Port}}" + options: + requiredOperatingSystem: linux + + todo-go: + sources: + app: + localFolder: null + repository: https://github.com/aspnet/Benchmarks.git + branchOrCommit: main + destinationFolder: app + cacheOnAgent: false + dockerFile: app/src/BenchmarksApps/MinimalTodo/go/Dockerfile + dockerContextDirectory: app/src/BenchmarksApps/MinimalTodo/go + dockerImageName: "minimal-todo-go-{{job.RunId}}" + readyStateText: Application started. + cpuLimitRatio: 1 + memoryLimitInBytes: 536870912 # 512 MiB + port: 8080 + environmentVariables: + PORT: "{{job.Port}}" + options: + requiredOperatingSystem: linux + + todo-rust: + sources: + app: + localFolder: null + repository: https://github.com/aspnet/Benchmarks.git + branchOrCommit: main + destinationFolder: app + cacheOnAgent: false + dockerFile: app/src/BenchmarksApps/MinimalTodo/rust/Dockerfile + dockerContextDirectory: app/src/BenchmarksApps/MinimalTodo/rust + dockerImageName: "minimal-todo-rust-{{job.RunId}}" + readyStateText: Application started. + cpuLimitRatio: 1 + memoryLimitInBytes: 536870912 # 512 MiB + port: 8080 + environmentVariables: + PORT: "{{job.Port}}" + options: + requiredOperatingSystem: linux + +scenarios: + csharp: + application: {job: todo-csharp} + load: + job: bombardier + variables: + serverPort: 8080 + rate: 100 # Override with --variable rate=... + path: /todos + transport: http1 + presetHeaders: none + customHeaders: + - "Accept: application/json" + - "Connection: keep-alive" + - "Accept-Encoding: identity" + go: + application: {job: todo-go} + load: + job: bombardier + variables: + serverPort: 8080 + rate: 100 # Override with --variable rate=... + path: /todos + transport: http1 + presetHeaders: none + customHeaders: + - "Accept: application/json" + - "Connection: keep-alive" + - "Accept-Encoding: identity" + rust: + application: {job: todo-rust} + load: + job: bombardier + variables: + serverPort: 8080 + rate: 100 # Override with --variable rate=... + path: /todos + transport: http1 + presetHeaders: none + customHeaders: + - "Accept: application/json" + - "Connection: keep-alive" + - "Accept-Encoding: identity" + # Day-to-day scheduled C# comparison scenario (see README "Execution + # paths"): direct Crank build of the C# app, no Docker. + csharp-project: + application: {job: todo-csharp-project} + load: + job: bombardier + variables: + serverPort: 8080 + rate: 100 # Override with --variable rate=... + path: /todos + transport: http1 + presetHeaders: none + customHeaders: + - "Accept: application/json" + - "Connection: keep-alive" + - "Accept-Encoding: identity" + +results: + # bombardier/raw is the raw per-request Bombardier JSON payload; excluded + # here only to avoid persisting that large aggregate -- standard Bombardier + # scalars (requests, bad responses, latencies, RPS, throughput) and the + # shared steadystate.profile.yml CPU/working-set P90 helpers already cover + # day-to-day reporting. + - {name: bombardier/raw, excluded: true} diff --git a/src/BenchmarksApps/MinimalTodo/.gitattributes b/src/BenchmarksApps/MinimalTodo/.gitattributes new file mode 100644 index 000000000..bf4f82169 --- /dev/null +++ b/src/BenchmarksApps/MinimalTodo/.gitattributes @@ -0,0 +1,4 @@ +# Preserve the frozen source bytes across checkouts. +csharp/** -text +go/** -text +rust/** -text diff --git a/src/BenchmarksApps/MinimalTodo/Directory.Build.targets b/src/BenchmarksApps/MinimalTodo/Directory.Build.targets new file mode 100644 index 000000000..676190f5e --- /dev/null +++ b/src/BenchmarksApps/MinimalTodo/Directory.Build.targets @@ -0,0 +1 @@ + diff --git a/src/BenchmarksApps/MinimalTodo/csharp/.dockerignore b/src/BenchmarksApps/MinimalTodo/csharp/.dockerignore new file mode 100644 index 000000000..69f5366e0 --- /dev/null +++ b/src/BenchmarksApps/MinimalTodo/csharp/.dockerignore @@ -0,0 +1,6 @@ +**/bin +**/obj +artifacts +TestResults +.git +README.md diff --git a/src/BenchmarksApps/MinimalTodo/csharp/.gitignore b/src/BenchmarksApps/MinimalTodo/csharp/.gitignore new file mode 100644 index 000000000..5e45def14 --- /dev/null +++ b/src/BenchmarksApps/MinimalTodo/csharp/.gitignore @@ -0,0 +1,3 @@ +**/bin/ +**/obj/ +artifacts/ diff --git a/src/BenchmarksApps/MinimalTodo/csharp/Directory.Build.props b/src/BenchmarksApps/MinimalTodo/csharp/Directory.Build.props new file mode 100644 index 000000000..ebf6da6f8 --- /dev/null +++ b/src/BenchmarksApps/MinimalTodo/csharp/Directory.Build.props @@ -0,0 +1,8 @@ + + + net11.0 + enable + enable + true + + diff --git a/src/BenchmarksApps/MinimalTodo/csharp/Dockerfile b/src/BenchmarksApps/MinimalTodo/csharp/Dockerfile new file mode 100644 index 000000000..94056099e --- /dev/null +++ b/src/BenchmarksApps/MinimalTodo/csharp/Dockerfile @@ -0,0 +1,15 @@ +FROM mcr.microsoft.com/dotnet/nightly/sdk:11.0 AS build +WORKDIR /source +COPY Directory.Build.props NuGet.Config ./ +COPY src/TodoApi/TodoApi.csproj src/TodoApi/packages.lock.json src/TodoApi/ +RUN dotnet restore src/TodoApi/TodoApi.csproj --locked-mode +COPY src/TodoApi/ src/TodoApi/ +RUN dotnet publish src/TodoApi/TodoApi.csproj --configuration Release --no-restore --self-contained false --output /app + +FROM mcr.microsoft.com/dotnet/nightly/aspnet:11.0 AS final +WORKDIR /app +ENV PORT=8080 +EXPOSE 8080 +COPY --from=build /app ./ +USER $APP_UID +ENTRYPOINT ["dotnet", "TodoApi.dll"] diff --git a/src/BenchmarksApps/MinimalTodo/csharp/NuGet.Config b/src/BenchmarksApps/MinimalTodo/csharp/NuGet.Config new file mode 100644 index 000000000..e6ba01fc5 --- /dev/null +++ b/src/BenchmarksApps/MinimalTodo/csharp/NuGet.Config @@ -0,0 +1,10 @@ + + + + + + + + + + diff --git a/src/BenchmarksApps/MinimalTodo/csharp/README.md b/src/BenchmarksApps/MinimalTodo/csharp/README.md new file mode 100644 index 000000000..c5bc7a325 --- /dev/null +++ b/src/BenchmarksApps/MinimalTodo/csharp/README.md @@ -0,0 +1,81 @@ +# Minimal Todo API + +A standalone C# / ASP.NET Core Minimal API with five immutable in-memory +records. There are no package dependencies, external services, file reads, +database, or credentials. + +## Build + +Install .NET SDK **11.0** (development/nightly channel), floating to whatever +patch build is currently published; no `global.json` SDK pin and no runtime +version pin in `Directory.Build.props`. The application has no package +dependencies, but still has its own `src/TodoApi/packages.lock.json` (used +for the Dockerfile's `--locked-mode` restore); with zero dependencies, its +recorded `net11.0` dependency set is empty. + +From this directory (PowerShell; on Linux use `/` instead of `\`): + +```powershell +dotnet restore TodoApi.slnx --locked-mode +dotnet build TodoApi.slnx --configuration Release --no-restore +``` + +## Run + +```powershell +$env:PORT = '8080' +dotnet src\TodoApi\bin\Release\net11.0\TodoApi.dll +``` + +`PORT` defaults to `8080`; when set it must contain only ASCII decimal digits +and represent 1 through 65535. Invalid values and bind failures log +diagnostics and exit nonzero. The service listens on `0.0.0.0` over +HTTP/1.1, including persistent connections, and prints the exact line +`Application started.` once the listener and dataset are ready. + +## Contract + +- `GET /todos`: the five fixed Todos as a compact JSON array (property order + `id`, `title`, `dueBy`, `isComplete`). +- `GET /todos/1` through `/todos/5`: one Todo. +- `GET /healthz`: plain UTF-8 `ready`. +- Invalid/missing IDs and unknown paths: empty `404`. +- Mutating methods (`POST`/`PUT`/`PATCH`/`DELETE`) on the list or a known + item: empty `405`. + +There is no compression, output caching, conditional response processing, +HTTPS redirection, or representation negotiation. HEAD/OPTIONS/trailing-slash +aliases are left to the framework and are not part of the contract. + +## Shutdown and logging + +Startup/readiness, shutdown, and unexpected failures are logged; routine +ASP.NET Core access logs are disabled. Unexpected request errors return 500. +Use Ctrl+C locally or SIGTERM on Linux: accepted requests are drained where +practical and new work stops, with an **8-second graceful shutdown timeout** +(inside the standard 10-second exit target). Do not use forced termination +for a normal stop. + +The all-interface listener is not a public deployment: use a trusted local +machine/firewall. No deployment or cloud resources are configured. + +## Container + +```powershell +docker build --pull --tag minimal-todo:v1 . +docker run --detach --name minimal-todo-v1 --publish 127.0.0.1:8080:8080 --stop-timeout 10 minimal-todo:v1 +curl.exe --fail --http1.1 http://127.0.0.1:8080/healthz +curl.exe --fail --http1.1 http://127.0.0.1:8080/todos +docker stop --time 10 minimal-todo-v1 +docker rm minimal-todo-v1 +``` + +The Dockerfile uses the floating multi-architecture Microsoft nightly +SDK/ASP.NET Core images (`mcr.microsoft.com/dotnet/nightly/{sdk,aspnet}:11.0`), +a multi-stage Release publish, the built-in non-root `$APP_UID` user, and an +exec-form entrypoint for signal delivery. One application process runs per +container; the SDK is needed at build time only. + +Generated candidate source, admission review, and full test/validation +history for this cohort are owned by the separate +`dotnet-performance-tools` repository. diff --git a/src/BenchmarksApps/MinimalTodo/csharp/TodoApi.slnx b/src/BenchmarksApps/MinimalTodo/csharp/TodoApi.slnx new file mode 100644 index 000000000..56ec9c1ed --- /dev/null +++ b/src/BenchmarksApps/MinimalTodo/csharp/TodoApi.slnx @@ -0,0 +1,5 @@ + + + + + diff --git a/src/BenchmarksApps/MinimalTodo/csharp/src/TodoApi/Program.cs b/src/BenchmarksApps/MinimalTodo/csharp/src/TodoApi/Program.cs new file mode 100644 index 000000000..43ac81908 --- /dev/null +++ b/src/BenchmarksApps/MinimalTodo/csharp/src/TodoApi/Program.cs @@ -0,0 +1,92 @@ +using System.Collections.Immutable; +using System.Globalization; +using System.Net; +using System.Text; +using System.Text.Json; +using System.Text.Json.Serialization; +using Microsoft.AspNetCore.Server.Kestrel.Core; + +namespace TodoApi; + +public static class Program +{ + public static async Task Main(string[] args) + { + try + { + await using var app = CreateApplication(args, Environment.GetEnvironmentVariable("PORT")); + await app.RunAsync(); + return 0; + } + catch (Exception exception) + { + Console.Error.WriteLine($"Application failed: {exception}"); + return 1; + } + } + + internal static int ParsePort(string? value) + { + if (value is null) + { + return 8080; + } + + if (value.Length == 0 || + value.Any(character => character is < '0' or > '9') || + !int.TryParse(value, NumberStyles.None, CultureInfo.InvariantCulture, out var port) || + port is < 1 or > 65535) + { + throw new ArgumentException("Invalid PORT: expected decimal digits in the range 1 through 65535."); + } + + return port; + } + + internal static WebApplication CreateApplication(string[] args, string? portValue) + { + var port = ParsePort(portValue); + ImmutableArray todos = + [ + new(1, "Walk the dog", null, false), + new(2, "Do the dishes", "2026-01-01", false), + new(3, "Do the laundry", "2026-01-02", false), + new(4, "Clean the bathroom", null, false), + new(5, "Clean the car", "2026-01-03", false) + ]; + + var builder = WebApplication.CreateBuilder(args); + builder.WebHost.ConfigureKestrel(options => + options.Listen(IPAddress.Any, port, listener => listener.Protocols = HttpProtocols.Http1)); + builder.Logging.ClearProviders(); + builder.Logging.AddSimpleConsole(options => options.SingleLine = true); + builder.Logging.SetMinimumLevel(LogLevel.Information); + builder.Logging.AddFilter("Microsoft", LogLevel.Warning); + builder.Services.Configure(options => options.ShutdownTimeout = TimeSpan.FromSeconds(8)); + builder.Services.ConfigureHttpJsonOptions(options => + { + options.SerializerOptions.PropertyNamingPolicy = JsonNamingPolicy.CamelCase; + options.SerializerOptions.DefaultIgnoreCondition = JsonIgnoreCondition.Never; + options.SerializerOptions.WriteIndented = false; + }); + + var app = builder.Build(); + app.UseExceptionHandler(handler => handler.Run(context => + { + context.Response.StatusCode = StatusCodes.Status500InternalServerError; + return Task.CompletedTask; + })); + + app.MapGet("/todos", () => Results.Json(todos)); + app.MapGet("/todos/{id}", (string id) => + id.Length == 1 && id[0] is >= '1' and <= '5' + ? Results.Json(todos[id[0] - '1']) + : Results.NotFound()); + app.MapGet("/healthz", () => Results.Text("ready", "text/plain", Encoding.UTF8)); + + app.Lifetime.ApplicationStarted.Register(() => Console.WriteLine("Application started.")); + app.Lifetime.ApplicationStopping.Register(() => app.Logger.LogInformation("Application stopping.")); + app.Lifetime.ApplicationStopped.Register(() => app.Logger.LogInformation("Application stopped.")); + return app; + } +} diff --git a/src/BenchmarksApps/MinimalTodo/csharp/src/TodoApi/Todo.cs b/src/BenchmarksApps/MinimalTodo/csharp/src/TodoApi/Todo.cs new file mode 100644 index 000000000..899cf2a82 --- /dev/null +++ b/src/BenchmarksApps/MinimalTodo/csharp/src/TodoApi/Todo.cs @@ -0,0 +1,9 @@ +using System.Text.Json.Serialization; + +namespace TodoApi; + +public sealed record Todo( + [property: JsonPropertyOrder(0)] int Id, + [property: JsonPropertyOrder(1)] string Title, + [property: JsonPropertyOrder(2)] string? DueBy, + [property: JsonPropertyOrder(3)] bool IsComplete); diff --git a/src/BenchmarksApps/MinimalTodo/csharp/src/TodoApi/TodoApi.csproj b/src/BenchmarksApps/MinimalTodo/csharp/src/TodoApi/TodoApi.csproj new file mode 100644 index 000000000..c9cfed081 --- /dev/null +++ b/src/BenchmarksApps/MinimalTodo/csharp/src/TodoApi/TodoApi.csproj @@ -0,0 +1,2 @@ + + diff --git a/src/BenchmarksApps/MinimalTodo/csharp/src/TodoApi/packages.lock.json b/src/BenchmarksApps/MinimalTodo/csharp/src/TodoApi/packages.lock.json new file mode 100644 index 000000000..0ecce8c56 --- /dev/null +++ b/src/BenchmarksApps/MinimalTodo/csharp/src/TodoApi/packages.lock.json @@ -0,0 +1,6 @@ +{ + "version": 1, + "dependencies": { + "net11.0": {} + } +} \ No newline at end of file diff --git a/src/BenchmarksApps/MinimalTodo/go/.dockerignore b/src/BenchmarksApps/MinimalTodo/go/.dockerignore new file mode 100644 index 000000000..5321c3a07 --- /dev/null +++ b/src/BenchmarksApps/MinimalTodo/go/.dockerignore @@ -0,0 +1,3 @@ +bin/ +validation/ +README.md diff --git a/src/BenchmarksApps/MinimalTodo/go/Dockerfile b/src/BenchmarksApps/MinimalTodo/go/Dockerfile new file mode 100644 index 000000000..333addb02 --- /dev/null +++ b/src/BenchmarksApps/MinimalTodo/go/Dockerfile @@ -0,0 +1,16 @@ +FROM golang:1.27-bookworm AS build +ENV GOTOOLCHAIN=local \ + GOPROXY=off \ + GOSUMDB=off +WORKDIR /src +COPY go.mod main.go server.go ./ +RUN CGO_ENABLED=0 go build -trimpath -o /out/todo . + +FROM ubuntu:26.04 +RUN groupadd --gid 65532 nonroot && useradd --uid 65532 --gid 65532 --no-create-home --shell /usr/sbin/nologin nonroot +COPY --from=build --chown=65532:65532 /out/todo /todo +USER 65532:65532 +ENV PORT=8080 +EXPOSE 8080 +STOPSIGNAL SIGTERM +ENTRYPOINT ["/todo"] diff --git a/src/BenchmarksApps/MinimalTodo/go/README.md b/src/BenchmarksApps/MinimalTodo/go/README.md new file mode 100644 index 000000000..ab17fb05f --- /dev/null +++ b/src/BenchmarksApps/MinimalTodo/go/README.md @@ -0,0 +1,116 @@ +# Minimal Todo API + +A standalone, read-only service using **Go 1.27** and the standard-library +`net/http` router and `encoding/json` serializer. There are no third-party +dependencies, so no `go.sum` is needed. `go.mod` declares `go 1.27` as a +minimum toolchain version (not an exact pin); the Docker builder floats on +the `golang:1.27-bookworm` tag, with no image digest pin. + +## Contract + +- `GET /todos`: the five fixed Todos, in ID order. +- `GET /todos/1` through `/todos/5`: one Todo. +- `GET /healthz`: the exact UTF-8 body `ready`. +- Invalid/missing IDs and unknown paths: empty `404`. +- Mutating methods on the list or a known item: empty `405`, with `Allow: GET`. + +Todo responses are compact JSON with property order `id`, `title`, `dueBy`, +`isComplete`, explicit null dates, and no newline. The records are initialized +once per server in a private, read-only captured array; no code mutates or +exposes that array. Every successful Todo request calls `json.Marshal` on the +typed records. No response caching, compression, conditional responses, +alternate representations, per-request access logs, database, or outbound +service calls are enabled. HTTP/1.1 keep-alive is supported; TLS, HTTP/2, and +a proxy are not required. HEAD, OPTIONS, and trailing-slash aliases are +outside the v1 contract. Noncanonical paths containing duplicate slashes or +dot segments return an empty 404 rather than a router-generated redirect. + +## Build and run + +Prerequisites: Go **1.27**; a Linux x64 (`amd64`) or ARM64 (`arm64`) host for +native execution. Docker with BuildKit is optional for container builds. The +commands below use a POSIX shell from this project directory. + +```sh +export GOTOOLCHAIN=local +mkdir -p bin +CGO_ENABLED=0 go build -trimpath -o bin/todo . +PORT=8080 ./bin/todo +``` + +`CGO_ENABLED=0` produces a self-contained release executable suitable for the +`ubuntu:26.04` container. Use `Ctrl+C` to stop a foreground process, or send +`kill -TERM ` to its specific PID. Run the binary, not `go run`, for +reliable signal delivery. + +```sh +curl --http1.1 --fail http://127.0.0.1:8080/healthz +curl --http1.1 --fail http://127.0.0.1:8080/todos +curl --http1.1 --fail http://127.0.0.1:8080/todos/1 +``` + +`PORT` defaults to `8080` only when unset. A supplied value must consist of +ASCII decimal digits representing `1` through `65535`; an empty value, signs, +spaces, overflow, and out-of-range values fail with an error and exit code 1. +The service binds IPv4 `0.0.0.0` as required. Use a trusted local environment: +direct execution listens on all IPv4 interfaces and provides no +authentication. Container examples below publish only to loopback; nothing is +deployed remotely. + +After binding and initializing its handlers, the service prints the exact +line `Application started.` to stdout once. Startup, shutdown, and unexpected +errors are logged to stderr. SIGINT/SIGTERM stop new work and allow active +requests to finish, with an **8-second graceful-shutdown deadline**. If that +expires, the service logs forced connection closure and exits unsuccessfully. +Non-default server limits are a 5-second request header timeout and a +60-second idle keep-alive timeout. + +## Linux release targets + +```sh +CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -o bin/todo-linux-amd64 . +CGO_ENABLED=0 GOOS=linux GOARCH=arm64 go build -trimpath -o bin/todo-linux-arm64 . +``` + +Cross-compilation is not native execution: run the resulting binary on its +matching native Linux architecture. + +## Container + +The multi-stage Dockerfile builds without network dependency access (no +third-party Go modules to resolve). `docker build --platform ...` selects a +single target platform for the whole build: BuildKit pulls that platform's +`golang:1.27-bookworm` manifest and runs the build stage natively for it (via +emulation when the host architecture differs) rather than cross-compiling +from one native builder architecture to another -- this is different from +the standalone `GOARCH=...` cross-compilation commands above, which do +cross-compile from a single host. The final `ubuntu:26.04` image contains +only the static release executable, runs as non-root UID/GID 65532, and uses +an exec-form entrypoint for signal delivery. It needs neither a writable +filesystem nor runtime dependencies. + +```sh +docker build --platform linux/amd64 -t minimal-todo:amd64 . +docker run --detach --name minimal-todo-local --platform linux/amd64 \ + --user 65532:65532 --read-only --cap-drop ALL \ + --security-opt no-new-privileges \ + --publish 127.0.0.1:8080:8080 minimal-todo:amd64 +curl --http1.1 --fail http://127.0.0.1:8080/healthz +curl --http1.1 --fail http://127.0.0.1:8080/todos +docker stop --timeout 10 minimal-todo-local +docker rm minimal-todo-local +``` + +On a native ARM64 host, use `--platform linux/arm64` instead. Stop/remove +only your own named container. Image acquisition can require internet access +at build time; the running application needs none. + +## Files + +- `go.mod`, `main.go`, `server.go`: module (minimum Go version, no + third-party dependencies) and application. +- `Dockerfile`, `.dockerignore`: floating multi-stage Linux container build. + +Generated candidate source, admission review, and full test/validation +history for this cohort are owned by the separate +`dotnet-performance-tools` repository. diff --git a/src/BenchmarksApps/MinimalTodo/go/go.mod b/src/BenchmarksApps/MinimalTodo/go/go.mod new file mode 100644 index 000000000..2c41d8b70 --- /dev/null +++ b/src/BenchmarksApps/MinimalTodo/go/go.mod @@ -0,0 +1,3 @@ +module minimal-todo + +go 1.27 diff --git a/src/BenchmarksApps/MinimalTodo/go/main.go b/src/BenchmarksApps/MinimalTodo/go/main.go new file mode 100644 index 000000000..0015fd989 --- /dev/null +++ b/src/BenchmarksApps/MinimalTodo/go/main.go @@ -0,0 +1,107 @@ +package main + +import ( + "context" + "errors" + "fmt" + "log" + "net" + "net/http" + "os" + "os/signal" + "strconv" + "syscall" + "time" +) + +const shutdownTimeout = 8 * time.Second + +func main() { + os.Exit(run()) +} + +func run() int { + logger := log.New(os.Stderr, "", log.LstdFlags) + port, err := configuredPort() + if err != nil { + logger.Printf("Invalid configuration: %v", err) + return 1 + } + + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer stop() + + server := &http.Server{ + Handler: newHandler(logger), + ReadHeaderTimeout: 5 * time.Second, + IdleTimeout: 60 * time.Second, + ErrorLog: logger, + } + listener, err := net.Listen("tcp4", net.JoinHostPort("0.0.0.0", strconv.Itoa(port))) + if err != nil { + logger.Printf("Could not listen on port %d: %v", port, err) + return 1 + } + logger.Printf("Listening on %s", listener.Addr()) + serveResult := make(chan error, 1) + go func() { + serveResult <- server.Serve(listener) + }() + + if _, err := fmt.Fprintln(os.Stdout, "Application started."); err != nil { + logger.Printf("Could not announce readiness: %v", err) + if err := server.Close(); err != nil { + logger.Printf("Could not close server: %v", err) + } + return 1 + } + + select { + case err := <-serveResult: + logger.Printf("HTTP server stopped unexpectedly: %v", err) + return 1 + case <-ctx.Done(): + stop() + } + + logger.Print("Shutdown requested.") + shutdownCtx, cancel := context.WithTimeout(context.Background(), shutdownTimeout) + defer cancel() + if err := server.Shutdown(shutdownCtx); err != nil { + logger.Printf("Graceful shutdown failed; forcing connections closed: %v", err) + if err := server.Close(); err != nil { + logger.Printf("Could not close server: %v", err) + } + return 1 + } + if err := <-serveResult; err != nil && !errors.Is(err, http.ErrServerClosed) { + logger.Printf("HTTP server failed during shutdown: %v", err) + return 1 + } + logger.Print("Application stopped.") + return 0 +} + +func configuredPort() (int, error) { + value, present := os.LookupEnv("PORT") + if !present { + return 8080, nil + } + return parsePort(value) +} + +func parsePort(value string) (int, error) { + if value == "" { + return 0, fmt.Errorf("PORT must contain decimal digits in the range 1 through 65535; got %q", value) + } + for _, digit := range value { + if digit < '0' || digit > '9' { + return 0, fmt.Errorf("PORT must contain decimal digits in the range 1 through 65535; got %q", value) + } + } + port, err := strconv.Atoi(value) + if err != nil || port < 1 || port > 65535 { + return 0, fmt.Errorf("PORT must be in the range 1 through 65535; got %q", value) + } + return port, nil +} diff --git a/src/BenchmarksApps/MinimalTodo/go/server.go b/src/BenchmarksApps/MinimalTodo/go/server.go new file mode 100644 index 000000000..d0295f3f7 --- /dev/null +++ b/src/BenchmarksApps/MinimalTodo/go/server.go @@ -0,0 +1,89 @@ +package main + +import ( + "encoding/json" + "log" + "net/http" + "path" + "strconv" +) + +type todo struct { + ID int `json:"id"` + Title string `json:"title"` + DueBy *string `json:"dueBy"` + IsComplete bool `json:"isComplete"` +} + +func newHandler(logger *log.Logger) http.Handler { + first, second, third := "2026-01-01", "2026-01-02", "2026-01-03" + // Go has no const collections; this private array is captured read-only. + todos := [...]todo{ + {ID: 1, Title: "Walk the dog"}, + {ID: 2, Title: "Do the dishes", DueBy: &first}, + {ID: 3, Title: "Do the laundry", DueBy: &second}, + {ID: 4, Title: "Clean the bathroom"}, + {ID: 5, Title: "Clean the car", DueBy: &third}, + } + + mux := http.NewServeMux() + mux.HandleFunc("/todos", func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet { + methodNotAllowed(w) + return + } + writeJSON(w, todos, logger) + }) + mux.HandleFunc("/todos/{id}", func(w http.ResponseWriter, r *http.Request) { + rawID := r.PathValue("id") + id, err := strconv.Atoi(rawID) + if err != nil || id < 1 || id > len(todos) || rawID != strconv.Itoa(id) { + w.WriteHeader(http.StatusNotFound) + return + } + if r.Method != http.MethodGet { + methodNotAllowed(w) + return + } + writeJSON(w, todos[id-1], logger) + }) + mux.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet { + methodNotAllowed(w) + return + } + w.Header().Set("Content-Type", "text/plain; charset=utf-8") + if _, err := w.Write([]byte("ready")); err != nil { + logger.Printf("Could not write readiness response: %v", err) + } + }) + mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusNotFound) + }) + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // Reject noncanonical paths before ServeMux can redirect them. + if r.URL.Path != path.Clean(r.URL.Path) { + w.WriteHeader(http.StatusNotFound) + return + } + mux.ServeHTTP(w, r) + }) +} + +func methodNotAllowed(w http.ResponseWriter) { + w.Header().Set("Allow", http.MethodGet) + w.WriteHeader(http.StatusMethodNotAllowed) +} + +func writeJSON(w http.ResponseWriter, value any, logger *log.Logger) { + body, err := json.Marshal(value) + if err != nil { + logger.Printf("Could not serialize Todo response: %v", err) + w.WriteHeader(http.StatusInternalServerError) + return + } + w.Header().Set("Content-Type", "application/json") + if _, err := w.Write(body); err != nil { + logger.Printf("Could not write Todo response: %v", err) + } +} diff --git a/src/BenchmarksApps/MinimalTodo/rust/.dockerignore b/src/BenchmarksApps/MinimalTodo/rust/.dockerignore new file mode 100644 index 000000000..928f919af --- /dev/null +++ b/src/BenchmarksApps/MinimalTodo/rust/.dockerignore @@ -0,0 +1,7 @@ +** +!Cargo.toml +!Cargo.lock +!rust-toolchain.toml +!src/ +!src/** +!Dockerfile diff --git a/src/BenchmarksApps/MinimalTodo/rust/.gitignore b/src/BenchmarksApps/MinimalTodo/rust/.gitignore new file mode 100644 index 000000000..24d64373c --- /dev/null +++ b/src/BenchmarksApps/MinimalTodo/rust/.gitignore @@ -0,0 +1 @@ +/target/ diff --git a/src/BenchmarksApps/MinimalTodo/rust/Cargo.lock b/src/BenchmarksApps/MinimalTodo/rust/Cargo.lock new file mode 100644 index 000000000..535cf0db0 --- /dev/null +++ b/src/BenchmarksApps/MinimalTodo/rust/Cargo.lock @@ -0,0 +1,603 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + +[[package]] +name = "axum" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31b698c5f9a010f6573133b09e0de5408834d0c82f8d7475a89fc1867a71cd90" +dependencies = [ + "axum-core", + "bytes", + "futures-util", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-util", + "itoa", + "matchit", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "serde_core", + "serde_json", + "serde_path_to_error", + "sync_wrapper", + "tokio", + "tower", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "axum-core" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08c78f31d7b1291f7ee735c1c6780ccde7785daae9a9206026862dab7d8792d1" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "http-body-util", + "mime", + "pin-project-lite", + "sync_wrapper", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "bitflags" +version = "2.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" + +[[package]] +name = "bytes" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" + +[[package]] +name = "cfg-if" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600" + +[[package]] +name = "cfg_aliases" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys", +] + +[[package]] +name = "futures-channel" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f9e3d69d39e4862ffed03ed071a76f9a13ba1d9109d355b0f0aa6b15e393c4" +dependencies = [ + "futures-core", +] + +[[package]] +name = "futures-core" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" + +[[package]] +name = "futures-task" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd" + +[[package]] +name = "futures-util" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" +dependencies = [ + "futures-core", + "futures-task", + "pin-project-lite", + "slab", +] + +[[package]] +name = "http" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "918d3568bebf352712bc2ef3d46a8bcf1a75b373be6539de198e9105cbbf9ce0" +dependencies = [ + "bytes", + "itoa", +] + +[[package]] +name = "http-body" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca2a8f2913ee65f60facd6a5905613afaa448497a0230cc41ce022d93290bc2c" +dependencies = [ + "bytes", + "http", +] + +[[package]] +name = "http-body-util" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + +[[package]] +name = "hyper" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb3aa54a13a0dfe7fbe3a59e0c76093041720fdc77b110cc0fc260fafb4dc51e" +dependencies = [ + "atomic-waker", + "bytes", + "futures-channel", + "futures-core", + "http", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "pin-utils", + "smallvec", + "tokio", + "want", +] + +[[package]] +name = "hyper-util" +version = "0.1.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c6995591a8f1380fcb4ba966a252a4b29188d51d2b89e3a252f5305be65aea8" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "hyper", + "pin-project-lite", + "tokio", + "tower-service", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + +[[package]] +name = "libc" +version = "0.2.189" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" + +[[package]] +name = "matchit" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + +[[package]] +name = "minimal-todo" +version = "1.0.0" +dependencies = [ + "axum", + "http-body-util", + "hyper", + "hyper-util", + "nix", + "serde", + "serde_json", + "tokio", + "tower", + "tracing", + "tracing-subscriber", +] + +[[package]] +name = "mio" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b18443e9c262bfe8fa82f51666e2642c53393f7e5c27b3e1aeab922cff5b9d8" +dependencies = [ + "libc", + "wasi", + "windows-sys", +] + +[[package]] +name = "nix" +version = "0.30.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74523f3a35e05aba87a1d978330aef40f67b0304ac79c1c00b294c9830543db6" +dependencies = [ + "bitflags", + "cfg-if", + "cfg_aliases", + "libc", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "ryu" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" + +[[package]] +name = "serde" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "serde_json" +version = "1.0.145" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "402a6f66d8c709116cf22f558eab210f5a50187f702eb4d7e5ef38d9a7f1c79c" +dependencies = [ + "itoa", + "memchr", + "ryu", + "serde", + "serde_core", +] + +[[package]] +name = "serde_path_to_error" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10a9ff822e371bb5403e391ecd83e182e0e77ba7f6fe0160b795797109d1b457" +dependencies = [ + "itoa", + "serde", + "serde_core", +] + +[[package]] +name = "sharded-slab" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "signal-hook-registry" +version = "1.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" +dependencies = [ + "errno", + "libc", +] + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "smallvec" +version = "1.16.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9395f0f0eee849a9b707b2f06bb92a6a422090e2123bb2ef8e87a0e61892a8e" + +[[package]] +name = "socket2" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4" +dependencies = [ + "libc", + "windows-sys", +] + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8593e8e72159ed2257d083c7a454a85cbf854f37a0966d8d483aff8c8a3ebcee" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "sync_wrapper" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" + +[[package]] +name = "thread_local" +version = "1.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ad99c4c6d32803332c548b1af0540b357b3f5fc0be8f6c6bfe8b2e6ae784070" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "tokio" +version = "1.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "202caea871b69668250d242070849eb495be178ed697a3e98aebce5bc81a0bed" +dependencies = [ + "bytes", + "libc", + "mio", + "pin-project-lite", + "signal-hook-registry", + "socket2", + "tokio-macros", + "windows-sys", +] + +[[package]] +name = "tokio-macros" +version = "2.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78773a2a397f451582ce068015985c33193cf6dea8b74d2a639fe457b2f07b0e" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.6", +] + +[[package]] +name = "tower" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d039ad9159c98b70ecfd540b2573b97f7f52c3e8d9f8ad57a24b916a536975f9" +dependencies = [ + "futures-core", + "futures-util", + "pin-project-lite", + "sync_wrapper", + "tokio", + "tower-layer", + "tower-service", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + +[[package]] +name = "tower-service" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" + +[[package]] +name = "tracing" +version = "0.1.41" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "784e0ac535deb450455cbfa28a6f0df145ea1bb7ae51b821cf5e7927fdcfbdd0" +dependencies = [ + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2054a14f5307d601f88daf0553e1cbf472acc4f2c51afab632431cdcd72124d5" +dependencies = [ + "sharded-slab", + "thread_local", + "tracing-core", +] + +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + +[[package]] +name = "unicode-ident" +version = "1.0.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d245f478577f809a851594d02313b640fb437e0bb33866753cff937863096954" + +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] diff --git a/src/BenchmarksApps/MinimalTodo/rust/Cargo.toml b/src/BenchmarksApps/MinimalTodo/rust/Cargo.toml new file mode 100644 index 000000000..a7d56ca72 --- /dev/null +++ b/src/BenchmarksApps/MinimalTodo/rust/Cargo.toml @@ -0,0 +1,24 @@ +[package] +name = "minimal-todo" +version = "1.0.0" +edition = "2024" +rust-version = "1.96" +publish = false + +[dependencies] +axum = { version = "=0.8.9", default-features = false, features = ["http1", "json", "tokio", "tracing"] } +serde = { version = "=1.0.228", features = ["derive", "rc"] } +tokio = { version = "=1.53.1", features = ["macros", "net", "rt-multi-thread", "signal", "sync", "time"] } +tracing = "=0.1.41" +tracing-subscriber = { version = "=0.3.20", default-features = false, features = ["fmt"] } + +[dev-dependencies] +http-body-util = "=0.1.3" +hyper = { version = "=1.7.0", default-features = false, features = ["client", "http1"] } +hyper-util = { version = "=0.1.17", features = ["tokio"] } +serde_json = "=1.0.145" +tokio = { version = "=1.53.1", features = ["io-util", "process"] } +tower = { version = "=0.5.2", features = ["util"] } + +[target.'cfg(unix)'.dev-dependencies] +nix = { version = "=0.30.1", default-features = false, features = ["signal", "process"] } diff --git a/src/BenchmarksApps/MinimalTodo/rust/Dockerfile b/src/BenchmarksApps/MinimalTodo/rust/Dockerfile new file mode 100644 index 000000000..613197022 --- /dev/null +++ b/src/BenchmarksApps/MinimalTodo/rust/Dockerfile @@ -0,0 +1,15 @@ +FROM rust:1.96-bookworm AS build +WORKDIR /app +COPY Cargo.toml Cargo.lock rust-toolchain.toml ./ +COPY src ./src +RUN cargo build --release --locked + +FROM ubuntu:26.04 AS runtime +RUN groupadd --gid 10001 app && useradd --uid 10001 --gid app --no-create-home --shell /usr/sbin/nologin app +WORKDIR /app +COPY --from=build /app/target/release/minimal-todo /app/minimal-todo +USER 10001:10001 +ENV PORT=8080 +EXPOSE 8080 +STOPSIGNAL SIGTERM +ENTRYPOINT ["/app/minimal-todo"] diff --git a/src/BenchmarksApps/MinimalTodo/rust/README.md b/src/BenchmarksApps/MinimalTodo/rust/README.md new file mode 100644 index 000000000..78d0a8f74 --- /dev/null +++ b/src/BenchmarksApps/MinimalTodo/rust/README.md @@ -0,0 +1,93 @@ +# Minimal Todo API + +A standalone, read-only service built with **Rust 1.96, Axum 0.8.9, and +Tokio 1.53.1**. Five immutable structured records are initialized once at +startup; Axum's `Json` serializer serializes those records on every +successful request. There is no database, request-path filesystem access, +external service, UI, or mutable business state. + +## Build and run + +Prerequisites: Rust/Cargo 1.96 and the platform's native C linker. Rustup's +`rust-toolchain.toml` selects the requested compiler (a floating minor +version, not an exact patch pin). Direct dependencies are exactly pinned in +`Cargo.toml`; `Cargo.lock` pins the complete dependency graph. The intended +deployment platforms are native Linux x64 and ARM64 with glibc. Windows x64 +is also supported for local development. + +```sh +cargo build --release --locked +PORT=8080 ./target/release/minimal-todo +``` + +PowerShell: + +```powershell +$env:PORT = '8080' +.\target\release\minimal-todo.exe +``` + +`PORT` defaults to `8080`. It must contain only decimal digits and represent a +value from 1 through 65535. Whitespace, signs, empty values, zero, and overflow +are rejected with a diagnostic and a nonzero exit code. The service binds to +`0.0.0.0`; use your local firewall to restrict native-process access. Examples +use loopback only; no automatic deployment or public exposure is configured. + +After successful binding and signal-handler initialization, stdout contains +exactly one readiness line: + +```text +Application started. +``` + +```sh +curl --http1.1 -i http://127.0.0.1:8080/healthz +curl --http1.1 -i http://127.0.0.1:8080/todos +curl --http1.1 -i http://127.0.0.1:8080/todos/1 +``` + +`GET /todos` returns all five records in ascending ID order, with exact compact +UTF-8 JSON bytes and property order `id`, `title`, `dueBy`, `isComplete`. +`GET /todos/1` through `/todos/5` return individual records. Missing or malformed +IDs and unrelated paths return empty `404` responses. POST, PUT, PATCH, and +DELETE on `/todos` or known item paths return empty `405` responses. +`GET /healthz` returns plain text `ready`. HTTP/1.1 persistent connections are +supported. Compression, response caching, conditional responses, content +negotiation, HTTPS redirects, and per-request access logging are not enabled. +No runtime network access is needed except serving client requests. + +## Shutdown and logging + +Use Ctrl+C locally, SIGTERM/SIGINT on Linux, or `docker stop --time 10`. +Windows also handles Ctrl+Break and console-close notifications. The service +stops accepting new connections and gives accepted work up to **8 seconds** to +drain. A timeout is logged as an error and exits unsuccessfully; this leaves +headroom inside the 10-second target. Startup failures, shutdown, and unexpected +errors go to stderr. Normal Tokio worker-thread/runtime defaults are retained; +the explicit drain deadline is the only lifecycle customization. + +## Linux containers + +The multi-stage build produces a release executable and a minimal +`ubuntu:26.04` runtime with UID/GID 10001. Exec-form entrypoint preserves +signal delivery; one application process runs per container. The image has +no application credentials and does not need a writable filesystem. + +On a **native Linux x64** Docker engine: + +```sh +docker build --platform linux/amd64 -t minimal-todo:1 . +docker run --rm --name minimal-todo-local --read-only --cap-drop ALL --security-opt no-new-privileges -p 127.0.0.1:8080:8080 minimal-todo:1 +# In a second terminal: +curl --http1.1 -i http://127.0.0.1:8080/todos +docker stop --time 10 minimal-todo-local +``` + +On a **native Linux ARM64** Docker engine, use `--platform linux/arm64` +instead. Both base images are floating multi-architecture tags supporting +x64 and ARM64; the locked Cargo graph still pins the complete dependency +graph exactly. + +Generated candidate source, admission review, and full test/validation +history for this cohort are owned by the separate +`dotnet-performance-tools` repository. diff --git a/src/BenchmarksApps/MinimalTodo/rust/rust-toolchain.toml b/src/BenchmarksApps/MinimalTodo/rust/rust-toolchain.toml new file mode 100644 index 000000000..1ee3ba8ed --- /dev/null +++ b/src/BenchmarksApps/MinimalTodo/rust/rust-toolchain.toml @@ -0,0 +1,3 @@ +[toolchain] +channel = "1.96" +profile = "minimal" diff --git a/src/BenchmarksApps/MinimalTodo/rust/src/lib.rs b/src/BenchmarksApps/MinimalTodo/rust/src/lib.rs new file mode 100644 index 000000000..4bbf638a9 --- /dev/null +++ b/src/BenchmarksApps/MinimalTodo/rust/src/lib.rs @@ -0,0 +1,144 @@ +use std::{fmt, sync::Arc}; + +use axum::{ + Json, Router, + extract::{Path, State, rejection::PathRejection}, + http::StatusCode, + routing::get, +}; +use serde::Serialize; + +#[derive(Clone, Copy, Serialize)] +#[serde(rename_all = "camelCase")] +struct Todo { + id: u8, + title: &'static str, + due_by: Option<&'static str>, + is_complete: bool, +} + +type Todos = Arc<[Todo; 5]>; + +pub fn router() -> Router { + let todos = Arc::new([ + Todo { + id: 1, + title: "Walk the dog", + due_by: None, + is_complete: false, + }, + Todo { + id: 2, + title: "Do the dishes", + due_by: Some("2026-01-01"), + is_complete: false, + }, + Todo { + id: 3, + title: "Do the laundry", + due_by: Some("2026-01-02"), + is_complete: false, + }, + Todo { + id: 4, + title: "Clean the bathroom", + due_by: None, + is_complete: false, + }, + Todo { + id: 5, + title: "Clean the car", + due_by: Some("2026-01-03"), + is_complete: false, + }, + ]); + + Router::new() + .route("/todos", get(list_todos)) + .route("/todos/{id}", get(get_todo)) + .route("/healthz", get(|| async { "ready" })) + .fallback(|| async { StatusCode::NOT_FOUND }) + .with_state(todos) +} + +async fn list_todos(State(todos): State) -> Json { + Json(todos) +} + +async fn get_todo( + State(todos): State, + path: Result, PathRejection>, +) -> Result, StatusCode> { + let Path(id) = path.map_err(|_| StatusCode::NOT_FOUND)?; + if id.starts_with('0') || !id.bytes().all(|byte| byte.is_ascii_digit()) { + return Err(StatusCode::NOT_FOUND); + } + let index = id + .parse::() + .ok() + .and_then(|id| id.checked_sub(1)) + .ok_or(StatusCode::NOT_FOUND)?; + todos + .get(index) + .copied() + .map(Json) + .ok_or(StatusCode::NOT_FOUND) +} + +#[derive(Debug, PartialEq, Eq)] +pub struct InvalidPort; + +impl fmt::Display for InvalidPort { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("PORT must contain only decimal digits and be in the range 1..=65535") + } +} + +impl std::error::Error for InvalidPort {} + +pub fn parse_port(value: Option<&str>) -> Result { + let Some(value) = value else { + return Ok(8080); + }; + if value.is_empty() || !value.bytes().all(|byte| byte.is_ascii_digit()) { + return Err(InvalidPort); + } + value + .parse::() + .ok() + .filter(|port| *port != 0) + .ok_or(InvalidPort) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn port_defaults_and_boundaries() { + assert_eq!(parse_port(None), Ok(8080)); + for (value, expected) in [("1", 1), ("8080", 8080), ("65535", 65535), ("00080", 80)] { + assert_eq!(parse_port(Some(value)), Ok(expected)); + } + } + + #[test] + fn invalid_ports_are_rejected() { + for value in [ + "", + "0", + "000", + "65536", + "-1", + "+80", + " 80", + "80 ", + "8.0", + "abc", + "\u{ff18}\u{ff10}", + "999999999999999999999999999999", + ] { + assert_eq!(parse_port(Some(value)), Err(InvalidPort), "{value:?}"); + } + } +} diff --git a/src/BenchmarksApps/MinimalTodo/rust/src/main.rs b/src/BenchmarksApps/MinimalTodo/rust/src/main.rs new file mode 100644 index 000000000..6545e9bcf --- /dev/null +++ b/src/BenchmarksApps/MinimalTodo/rust/src/main.rs @@ -0,0 +1,100 @@ +use std::{ + env, + error::Error, + future::IntoFuture, + io::{self, Write}, + process::ExitCode, + sync::Arc, + time::Duration, +}; + +use tokio::{net::TcpListener, sync::Notify}; +use tracing::{error, info}; + +const SHUTDOWN_TIMEOUT: Duration = Duration::from_secs(8); + +#[tokio::main] +async fn main() -> ExitCode { + tracing_subscriber::fmt() + .with_max_level(tracing::Level::INFO) + .with_writer(io::stderr) + .init(); + + match run().await { + Ok(()) => ExitCode::SUCCESS, + Err(error) => { + error!(%error, "Application failed."); + ExitCode::FAILURE + } + } +} + +async fn run() -> Result<(), Box> { + let port_value = match env::var("PORT") { + Ok(value) => Some(value), + Err(env::VarError::NotPresent) => None, + Err(error) => return Err(format!("Cannot read PORT: {error}").into()), + }; + let port = minimal_todo::parse_port(port_value.as_deref())?; + let signal = shutdown_signal()?; + let app = minimal_todo::router(); + let listener = TcpListener::bind(("0.0.0.0", port)) + .await + .map_err(|error| format!("Cannot bind 0.0.0.0:{port}: {error}"))?; + let shutdown = Arc::new(Notify::new()); + let server = axum::serve(listener, app) + .with_graceful_shutdown(shutdown.clone().notified_owned()) + .into_future(); + tokio::pin!(server); + + println!("Application started."); + io::stdout().flush()?; + + tokio::select! { + result = &mut server => { + result?; + return Err("HTTP server stopped unexpectedly".into()); + } + result = signal => result?, + } + + info!("Shutdown requested; allowing up to 8 seconds to drain."); + shutdown.notify_one(); + tokio::time::timeout(SHUTDOWN_TIMEOUT, &mut server) + .await + .map_err(|_| "Graceful shutdown timed out after 8 seconds; terminating remaining work")??; + info!("Application stopped."); + Ok(()) +} + +#[cfg(unix)] +fn shutdown_signal() -> io::Result>> { + use tokio::signal::unix::{SignalKind, signal}; + + let mut terminate = signal(SignalKind::terminate())?; + let mut interrupt = signal(SignalKind::interrupt())?; + Ok(async move { + let received = tokio::select! { + received = terminate.recv() => received, + received = interrupt.recv() => received, + }; + received.ok_or_else(|| io::Error::other("Termination signal stream closed unexpectedly")) + }) +} + +#[cfg(windows)] +fn shutdown_signal() -> io::Result>> { + use tokio::signal::windows; + + let mut interrupt = windows::ctrl_c()?; + let mut terminate = windows::ctrl_break()?; + let mut close = windows::ctrl_close()?; + Ok(async move { + let received = tokio::select! { + received = interrupt.recv() => received, + received = terminate.recv() => received, + received = close.recv() => received, + }; + received.ok_or_else(|| io::Error::other("Termination signal stream closed unexpectedly")) + }) +} From 4db6f23d53b6b45fd4cbe0db4952655a9891d677 Mon Sep 17 00:00:00 2001 From: Parker Bibus Date: Mon, 19 Oct 2026 14:02:39 -0700 Subject: [PATCH 4/6] Move PR benchmarks to Gold Linux and Windows machines (#2219) * Move PR benchmarks to Gold x64 relay machines Replace disconnected Perf/Citrine selections with existing Gold Linux and Windows profiles. Retire unavailable runtime ARM64 dispatch and configs, document support, and validate scenario imports and pipeline architecture selections. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: efd56148-f715-4c87-b1ec-7f70ad6e66c2 * Remove migration-specific PR benchmark tests Keep the Gold machine migration while removing its bespoke validation harness and README command. Completed offline validation remains documented in the PR. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: efd56148-f715-4c87-b1ec-7f70ad6e66c2 --------- Copilot-Session: efd56148-f715-4c87-b1ec-7f70ad6e66c2 --- build/README.md | 34 ++++++++++++- build/prbenchmarks.aspnetcore.config.yml | 24 +++------ build/prbenchmarks.efcore.config.yml | 10 ++-- ...rbenchmarks.runtime.linux_arm64.config.yml | 50 ------------------ .../prbenchmarks.runtime.linux_x64.config.yml | 10 ++-- ...enchmarks.runtime.windows_arm64.config.yml | 50 ------------------ ...rbenchmarks.runtime.windows_x64.config.yml | 10 ++-- build/prbenchmarks.yml | 51 ++----------------- 8 files changed, 51 insertions(+), 188 deletions(-) delete mode 100644 build/prbenchmarks.runtime.linux_arm64.config.yml delete mode 100644 build/prbenchmarks.runtime.windows_arm64.config.yml diff --git a/build/README.md b/build/README.md index 3b6c7477d..2d4941624 100644 --- a/build/README.md +++ b/build/README.md @@ -11,7 +11,7 @@ JSON config that lists the available pods (machine groups) and scenarios. | Generated YAML | Source config | Notes | |---|---|---| -| `benchmarks-ci-01.yml`, `benchmarks-ci-02.yml` | `benchmarks_ci_pods.json` | On-prem fleet (citrine + mono); split across two YAMLs to spread load | +| `benchmarks-ci-01.yml`, `benchmarks-ci-02.yml` | `benchmarks_ci_pods.json` | On-prem fleet (Gold); split across two YAMLs to spread load | | `benchmarks-ci-azure.yml` | `benchmarks_ci_azure_pods.json` | Azure cloud pods (includes the merged eastus2 cobalt cloud pods) | | `benchmarks-ci-cobalt.yml` | `benchmarks_ci_cobalt_pods.json` | Cobalt-hosted pods | @@ -36,6 +36,9 @@ python ./scripts/pod-scheduler/main.py \ The header of each generated YAML embeds the exact regen command for that file. +The `citrine1`, `citrine2`, and `citrine3` names in the on-prem config are +controller queues, not retired benchmark machines or relay targets. + ## Hand-maintained scenario templates Everything else in this directory is hand-maintained. The most important @@ -45,13 +48,40 @@ groups: (e.g. `proxies-scenarios.yml`, `grpc-scenarios.yml`). One per benchmark family. The pod-scheduler picks which template to run via the `template` field on each scenario in the JSON config. -- `prbenchmarks.*.config.yml` — PR-trigger pipelines for individual PR runs. +- `prbenchmarks.yml` — scheduled PR-benchmark bot pipeline. +- `prbenchmarks.*.config.yml` — components, scenarios, and machine choices for + individual PR-benchmark requests. - `job-variables.yml` — shared variable definitions consumed by the scenario templates. When adding a new benchmark family, drop a `*-scenarios.yml` template in this directory and reference it from the relevant `benchmarks_ci_*_pods.json`. +## PR benchmarks + +`prbenchmarks.yml` polls benchmark requests on `dotnet/aspnetcore`, +`dotnet/efcore`, and `dotnet/runtime` every 30 minutes. The controller image +determines where PR components are built, not which machine runs the benchmark. +The first profile in each bot config is the default machine selection. + +| Config | Controller | Supported benchmark machines (default first) | +|---|---|---| +| `prbenchmarks.aspnetcore.config.yml` | Windows | `aspnet-gold-lin`, `aspnet-gold-win` (x64, 56 cores) | +| `prbenchmarks.efcore.config.yml` | Windows | `aspnet-gold-win` (x64, 56 cores) | +| `prbenchmarks.runtime.windows_x64.config.yml` | Windows | `aspnet-gold-win` (x64, 56 cores) | +| `prbenchmarks.runtime.linux_x64.config.yml` | Linux | `aspnet-gold-lin` (x64, 56 cores) | + +These selections use the existing `aspnet-gold-*-relay` profiles imported by +the benchmark scenarios. Gold Linux uses `goldlin`, `goldload`, and `golddb`; +Gold Windows uses `goldwin`, `goldload2`, and `golddb`. The disconnected Perf +and Citrine choices are no longer offered. Runtime ARM64 jobs/configs have +been retired because their relay machines are unavailable; Gold is x64 and +does not replace ARM64 coverage. + +The ASP.NET Core, EF Core, and runtime scenario/component definitions are +unchanged. This migration does not change the polling schedule, controller +images, or Crank SDK/build configuration. + ## Editing the schedule Common edits, all done in JSON: diff --git a/build/prbenchmarks.aspnetcore.config.yml b/build/prbenchmarks.aspnetcore.config.yml index 6e411c414..d69638173 100644 --- a/build/prbenchmarks.aspnetcore.config.yml +++ b/build/prbenchmarks.aspnetcore.config.yml @@ -25,25 +25,13 @@ defaults: --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/buil # the first value is the default if none is specified profiles: - aspnet-perf-lin: - description: INTEL/Linux 12 Cores - arguments: --profile aspnet-perf-lin-relay + aspnet-gold-lin: + description: Gold/Linux x64 56 Cores + arguments: --profile aspnet-gold-lin-relay - aspnet-perf-win: - description: INTEL/Windows 12 Cores - arguments: --profile aspnet-perf-win-relay - - aspnet-citrine-lin: - description: INTEL/Linux 28 Cores - arguments: --profile aspnet-citrine-lin-relay - - aspnet-citrine-win: - description: INTEL/Windows 28 Cores - arguments: --profile aspnet-citrine-win-relay - - aspnet-citrine-ampere: - description: Ampere/Linux 80 Cores - arguments: --profile aspnet-citrine-arm-lin-relay + aspnet-gold-win: + description: Gold/Windows x64 56 Cores + arguments: --profile aspnet-gold-win-relay benchmarks: diff --git a/build/prbenchmarks.efcore.config.yml b/build/prbenchmarks.efcore.config.yml index f6c64b9a5..588dc8a77 100644 --- a/build/prbenchmarks.efcore.config.yml +++ b/build/prbenchmarks.efcore.config.yml @@ -13,13 +13,9 @@ defaults: --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/buil # the first value is the default if none is specified profiles: - aspnet-perf-win: - description: INTEL/Windows 12 Cores - arguments: --profile aspnet-perf-win-relay - - aspnet-citrine-win: - description: INTEL/Windows 28 Cores - arguments: --profile aspnet-citrine-win-relay + aspnet-gold-win: + description: Gold/Windows x64 56 Cores + arguments: --profile aspnet-gold-win-relay benchmarks: AddDataVariations: diff --git a/build/prbenchmarks.runtime.linux_arm64.config.yml b/build/prbenchmarks.runtime.linux_arm64.config.yml deleted file mode 100644 index 3d7073db5..000000000 --- a/build/prbenchmarks.runtime.linux_arm64.config.yml +++ /dev/null @@ -1,50 +0,0 @@ -# multi-line script used to build the project -components: - runtime: - script: | - ./build.sh clr.runtime+clr.alljits+clr.iltools+clr.tools+clr.corelib+clr.nativecorelib -c Release -runtimeconfiguration Release -arch arm64 - - arguments: '--{{job}}.options.outputFiles ./artifacts/bin/coreclr/Linux.arm64.Release/clrjit.dll --{{job}}.options.outputFiles ./artifacts/bin/coreclr/Linux.arm64.Release/coreclr.dll --{{job}}.options.outputFiles ./artifacts/bin/coreclr/Linux.arm64.Release/System.Private.CoreLib.dll' - - libs: - script: | - ./build.sh libs -c Release -runtimeconfiguration Release -arch arm64 - ./src/tests/build.sh Release arm64 generatelayoutonly /p:LibrariesConfiguration=Release - rm -rf ./artifacts/tests/coreclr/Linux.arm64.Release/tests/Core_Root/*.pdb - rm -rf ./artifacts/tests/coreclr/Linux.arm64.Release/tests/Core_Root/CommandLine.dll - - arguments: - --{{job}}.options.outputFiles ./artifacts/tests/coreclr/Linux.arm64.Release/tests/Core_Root/ - -# default arguments that are always used on crank commands -defaults: --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/build/azure.profile.yml --{{job}}.framework net11.0 --relay - -variables: - job: application - -# the first value is the default if none is specified -profiles: - aspnet-citrine-ampere: - description: Ampere/Linux 80 Cores - arguments: --profile aspnet-citrine-arm-lin-relay - -benchmarks: - micro: - description: '.NET Performance micro benchmarks (set filter by adding `--variable filter=...`; by default `filter=*LinqBenchmarks*`)' - arguments: --config https://raw.githubusercontent.com/aspnet/benchmarks/main/scenarios/dotnet.benchmarks.yml --scenario custom - - plaintext: - description: TechEmpower Plaintext Scenario - ASP.NET Platform implementation - arguments: --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/platform.benchmarks.yml --scenario plaintext - - json: - description: TechEmpower JSON Scenario - ASP.NET Platform implementation - arguments: --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/platform.benchmarks.yml --scenario json - - fortunes: - description: TechEmpower Fortunes Scenario - ASP.NET Platform implementation - arguments: --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/platform.benchmarks.yml --scenario fortunes - - fortunes_ef: - description: TechEmpower Fortunes Scenario with EF Core - ASP.NET Platform implementation - arguments: --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/platform.benchmarks.yml --scenario fortunes_ef diff --git a/build/prbenchmarks.runtime.linux_x64.config.yml b/build/prbenchmarks.runtime.linux_x64.config.yml index fc7d30b3b..a1da6399d 100644 --- a/build/prbenchmarks.runtime.linux_x64.config.yml +++ b/build/prbenchmarks.runtime.linux_x64.config.yml @@ -25,13 +25,9 @@ variables: # the first value is the default if none is specified profiles: - aspnet-citrine-lin: - description: Intel/Linux 28 Cores - arguments: --profile aspnet-citrine-lin-relay - - aspnet-perf-lin: - description: Intel/Linux 12 Cores - arguments: --profile aspnet-perf-lin-relay + aspnet-gold-lin: + description: Gold/Linux x64 56 Cores + arguments: --profile aspnet-gold-lin-relay benchmarks: diff --git a/build/prbenchmarks.runtime.windows_arm64.config.yml b/build/prbenchmarks.runtime.windows_arm64.config.yml deleted file mode 100644 index 8359d409e..000000000 --- a/build/prbenchmarks.runtime.windows_arm64.config.yml +++ /dev/null @@ -1,50 +0,0 @@ -# multi-line script used to build the project -components: - runtime: - script: | - call .\build.cmd clr.runtime+clr.alljits+clr.iltools+clr.tools+clr.corelib+clr.nativecorelib -c Release -runtimeconfiguration Release -arch arm64 - - arguments: '--{{job}}.options.outputFiles .\artifacts\bin\coreclr\windows.arm64.Release\clrjit.dll --{{job}}.options.outputFiles .\artifacts\bin\coreclr\windows.arm64.Release\coreclr.dll --{{job}}.options.outputFiles .\artifacts\bin\coreclr\windows.arm64.Release\System.Private.CoreLib.dll' - - libs: - script: | - call .\build.cmd libs -c Release -runtimeconfiguration Release -arch arm64 - call .\src\tests\build.cmd Release arm64 generatelayoutonly /p:LibrariesConfiguration=Release - del /F /S /Q .\artifacts\tests\coreclr\windows.arm64.Release\tests\Core_Root\*.pdb - del /F /S /Q .\artifacts\tests\coreclr\windows.arm64.Release\tests\Core_Root\CommandLine.dll - - arguments: - --{{job}}.options.outputFiles .\artifacts\tests\coreclr\windows.arm64.Release\tests\Core_Root\ - -# default arguments that are always used on crank commands -defaults: --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/build/azure.profile.yml --{{job}}.framework net11.0 --relay - -variables: - job: application - -# the first value is the default if none is specified -profiles: - aspnet-citrine-win-ampere: - description: Ampere/Windows 80 Cores - arguments: --profile aspnet-citrine-arm-win-relay - -benchmarks: - micro: - description: '.NET Performance micro benchmarks (set filter by adding `--variable filter=...`; by default `filter=*LinqBenchmarks*`)' - arguments: --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/dotnet.benchmarks.yml --scenario custom - - plaintext: - description: TechEmpower Plaintext Scenario - ASP.NET Platform implementation - arguments: --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/platform.benchmarks.yml --scenario plaintext - - json: - description: TechEmpower JSON Scenario - ASP.NET Platform implementation - arguments: --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/platform.benchmarks.yml --scenario json - - fortunes: - description: TechEmpower Fortunes Scenario - ASP.NET Platform implementation - arguments: --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/platform.benchmarks.yml --scenario fortunes - - fortunes_ef: - description: TechEmpower Fortunes Scenario with EF Core - ASP.NET Platform implementation - arguments: --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/platform.benchmarks.yml --scenario fortunes_ef diff --git a/build/prbenchmarks.runtime.windows_x64.config.yml b/build/prbenchmarks.runtime.windows_x64.config.yml index cfe348ddf..a194c0097 100644 --- a/build/prbenchmarks.runtime.windows_x64.config.yml +++ b/build/prbenchmarks.runtime.windows_x64.config.yml @@ -24,13 +24,9 @@ variables: # the first value is the default if none is specified profiles: - aspnet-perf-win: - description: Intel/Windows 12 Cores - arguments: --profile aspnet-perf-win-relay - - aspnet-citrine-win: - description: Intel/Windows 28 Cores - arguments: --profile aspnet-citrine-win-relay + aspnet-gold-win: + description: Gold/Windows x64 56 Cores + arguments: --profile aspnet-gold-win-relay benchmarks: micro: diff --git a/build/prbenchmarks.yml b/build/prbenchmarks.yml index 0922a594b..f38cc4e1c 100644 --- a/build/prbenchmarks.yml +++ b/build/prbenchmarks.yml @@ -98,7 +98,7 @@ extends: --app-key "env:APP_KEY" ` --publish-results true ` --external-log-uri $buildUrl - displayName: ASP.NET Core + displayName: ASP.NET Core Gold X64 continueOnError: true env: APP_KEY: $(github.privatekey) @@ -118,7 +118,7 @@ extends: --app-key "env:APP_KEY" ` --publish-results true ` --external-log-uri $buildUrl - displayName: EF Core + displayName: EF Core Gold Windows X64 continueOnError: true env: APP_KEY: $(github.privatekey) @@ -138,32 +138,10 @@ extends: --app-key "env:APP_KEY" ` --publish-results true ` --external-log-uri $buildUrl - displayName: Runtime Windows X64 + displayName: Runtime Gold Windows X64 continueOnError: true env: APP_KEY: $(github.privatekey) - - - - task: AzureCLI@2 - inputs: - azureSubscription: ASPNET Benchmarks Service Bus - scriptType: ps - scriptLocation: inlineScript - inlineScript: | - $buildUrl="$(System.TeamFoundationCollectionUri)$(System.TeamProject)/_build/results?buildId=$(Build.BuildId)" - dotnet artifacts/bin/Microsoft.Crank.PullRequestBot/release/net8.0/crank-pr.dll ` - --repository https://github.com/dotnet/runtime ` - --config $(Pipeline.Workspace)/benchmarks/build/prbenchmarks.runtime.windows_arm64.config.yml ` - --app-id $(github.appid) ` - --install-id $(github.installid) ` - --app-key "env:APP_KEY" ` - --publish-results true ` - --external-log-uri $buildUrl - displayName: Runtime Windows X64 - continueOnError: true - env: - APP_KEY: $(github.privatekey) - - job: BenchmarkLinux displayName: Benchmarks Linux Image pool: @@ -199,28 +177,7 @@ extends: --app-key "env:APP_KEY" ` --publish-results true ` --external-log-uri $buildUrl - displayName: Runtime Linux X64 + displayName: Runtime Gold Linux X64 continueOnError: true env: APP_KEY: $(github.privatekey) - - - task: AzureCLI@2 - inputs: - azureSubscription: ASPNET Benchmarks Service Bus - scriptType: pscore - scriptLocation: inlineScript - inlineScript: | - $buildUrl="$(System.TeamFoundationCollectionUri)$(System.TeamProject)/_build/results?buildId=$(Build.BuildId)" - dotnet artifacts/bin/Microsoft.Crank.PullRequestBot/release/net8.0/crank-pr.dll ` - --repository https://github.com/dotnet/runtime ` - --config $(Pipeline.Workspace)/benchmarks/build/prbenchmarks.runtime.linux_arm64.config.yml ` - --app-id $(github.appid) ` - --install-id $(github.installid) ` - --app-key "env:APP_KEY" ` - --publish-results true ` - --external-log-uri $buildUrl - displayName: Runtime Linux ARM64 - continueOnError: true - env: - APP_KEY: $(github.privatekey) - From 9dd895b564e1b3c68714a8cb45deeff2837faaa1 Mon Sep 17 00:00:00 2001 From: Parker Bibus Date: Mon, 19 Oct 2026 14:02:55 -0700 Subject: [PATCH 5/6] Remove disconnected Citrine benchmark profiles (#2218) Remove retired Citrine and dependent ARM/Siryn profile definitions, update examples to active Gold profiles, and reuse shared profiles in legacy scenarios. Correct the Gold Windows relay database address while preserving controller queues and active machine configurations. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 947a7758-5660-4f23-b447-e7006ace68fd --- build/ci.profile.yml | 193 ----------------- scenarios/README.md | 10 +- scenarios/aspnet.profiles.standard.yml | 168 --------------- scenarios/aspnet.profiles.yml | 202 +----------------- scenarios/mono.benchmarks.yml | 2 +- scenarios/platform.benchmarks.yml | 2 +- src/Benchmarks/json.benchmarks.yml | 25 +-- .../BuildPerformance/buildperformance.yml | 22 +- .../Samples/Template/sample.yml | 2 +- 9 files changed, 14 insertions(+), 612 deletions(-) diff --git a/build/ci.profile.yml b/build/ci.profile.yml index e1570132a..d6ef3fa78 100644 --- a/build/ci.profile.yml +++ b/build/ci.profile.yml @@ -18,177 +18,6 @@ profiles: - secondary - warmup - intel-db-app: - variables: - serverAddress: 10.0.0.103 - cores: 28 - agents: - application: - endpoints: - - http://asp-citrine-db:5001 - aliases: - - main - - intel-db-db: - variables: - databaseServer: 10.0.0.103 - downstreamAddress: 10.0.0.103 - agents: - db: - endpoints: - - http://asp-citrine-db:5001 - aliases: - - downstream - - extra - - intel-db-load: - variables: - secondaryAddress: 10.0.0.103 - agents: - load: - endpoints: - - http://asp-citrine-load:5001 - aliases: - - warmup - - secondary - - intel-load-app: - variables: - serverAddress: 10.0.0.105 - cores: 28 - agents: - application: - endpoints: - - http://asp-citrine-load:5001 - aliases: - - main - - intel-load-db: - variables: - databaseServer: 10.0.0.105 - downstreamAddress: 10.0.0.105 - agents: - db: - endpoints: - - http://asp-citrine-load:5001 - aliases: - - downstream - - extra - - intel-load-load: - variables: - secondaryAddress: 10.0.0.105 - agents: - load: - endpoints: - - http://asp-citrine-load:5001 - aliases: - - warmup - - secondary - - intel-load2-app: - variables: - serverAddress: 10.0.0.101 - cores: 28 - agents: - application: - endpoints: - - http://asp-citrine-load2:5001 - aliases: - - main - - intel-load2-db: - variables: - databaseServer: 10.0.0.101 - downstreamAddress: 10.0.0.101 - agents: - db: - endpoints: - - http://asp-citrine-load2:5001 - aliases: - - downstream - - extra - - intel-load2-load: - variables: - secondaryAddress: 10.0.0.101 - agents: - load: - endpoints: - - http://asp-citrine-load2:5001 - aliases: - - warmup - - secondary - - - arm-lin-app: - variables: - serverAddress: 10.0.0.108 - cores: 80 - agents: - application: - endpoints: - - http://asp-citrine-ampere:5001 - aliases: - - main - - arm-lin-load: - variables: - secondaryAddress: 10.0.0.108 - agents: - load: - endpoints: - - http://asp-citrine-ampere:5001 - aliases: - - warmup - - secondary - - arm-lin-28-app: - variables: - serverAddress: 10.0.0.108 - cores: 28 - agents: - application: - cpuSet: 0-27 - endpoints: - - http://asp-citrine-ampere:5001 - aliases: - - main - - arm-win-app: - variables: - serverAddress: 10.0.0.110 - cores: 80 - agents: - application: - endpoints: - - http://10.240.177.200:5001 - aliases: - - main - - arm-win-28-app: - variables: - serverAddress: 10.0.0.110 - cores: 28 - agents: - application: - cpuSet: 0-27 - endpoints: - - http://10.240.177.200:5001 - aliases: - - main - - arm-win-load: - variables: - secondaryAddress: 10.0.0.110 - agents: - load: - endpoints: - - http://10.240.177.200:5001 - aliases: - - warmup - - secondary - # INTEL Xeon Gold machines gold-lin-app: @@ -282,28 +111,6 @@ profiles: - warmup - secondary - siryn-lin-app: - variables: - serverAddress: 10.0.0.111 - cores: 128 - agents: - application: - endpoints: - - http://asp-siryn-lin:5001 - aliases: - - main - - siryn-lin-load: - variables: - secondaryAddress: 10.0.0.111 - agents: - load: - endpoints: - - http://asp-siryn-lin:5001 - aliases: - - warmup - - secondary - ############################################# # # # COBALT NETWORKED MACHINES # diff --git a/scenarios/README.md b/scenarios/README.md index 6f3d4bd7b..8f173280e 100644 --- a/scenarios/README.md +++ b/scenarios/README.md @@ -32,13 +32,11 @@ Each profile defines a set of machines, private IPs and ports that are used to r | `local` | (local machine) | (local machine) | (local machine) | | `aspnet-perf-lin` | INTEL, 12 logical cores, 1 socket, 32GB | Ubuntu 22.04, Kernel 5.15.0 | Intel(R) Xeon(R) E-2336 CPU @ 2.90GHz | | `aspnet-perf-win` | INTEL, 12 logical cores, 1 socket, 32GB | Windows Server 2022 | Intel(R) Xeon(R) E-2336 CPU @ 2.90GHz | -| `aspnet-citrine-lin` | INTEL, logical 28 cores, 1 socket, 32GB | Ubuntu 20.04, Kernel 5.4.0 | Intel(R) Xeon(R) Gold 5120 CPU @ 2.20GHz | -| `aspnet-citrine-win` | INTEL, logical 28 cores, 1 socket, 32GB | Windows Server 2022 | Intel(R) Xeon(R) Gold 5120 CPU @ 2.20GHz | -| `aspnet-citrine-amd2` | AMD, logical 48 cores, 1 socket, 64GB, 4 NUMA | Ubuntu 20.04, Kernel 5.4.0 | AMD EPYC 7402P 24-Core Processor | -| `aspnet-citrine-arm-lin` | ARM64, 80 logical cores, 1 socket, 1 NUMA, 128 GB | Ubuntu 20.04, Kernel 5.4.0 | Neoverse-N1 | | `aspnet-gold-lin` | INTEL, 56 logical cores, 1 socket, 1 NUMA, 64 GB | Ubuntu 22.04, Kernel 6.8.0-45-generic | Intel(R) Xeon(R) Gold 6330 CPU @ 2.00GHz | | `aspnet-gold-win` | INTEL, 56 logical cores, 1 socket, 1 NUMA, 64 GB | Windows Server 2022 | Intel(R) Xeon(R) Gold 6330 CPU @ 2.00GHz | +The retired Citrine and Siryn machine profiles are no longer available. + For testing purpose only, the __local__ profile requires a local agent to run: ``` @@ -328,7 +326,7 @@ The filename and size can be changed by adapting these variables: --variable filename=file.txt ``` -This scenario can easily reach the max network bandwidth. To verify that use `--load.options.displayOutput true` which will display the __wrk__ output, including the transfer rate. Example of saturated network for a 40Gb/s NIC (Citrine). +This scenario can easily reach the max network bandwidth. To verify that use `--load.options.displayOutput true` which will display the __wrk__ output, including the transfer rate. Historical example of saturated network for a 40Gb/s NIC: ``` [load] Transfer/sec: 4.37GB @@ -546,7 +544,7 @@ These scenarios provide benchmarks to help improve the performance of the .NET G ### Sample ``` -crank --config https://raw.githubusercontent.com/dotnet/performance/main/src/benchmarks/gc/scenarios/CrankConfiguration.yaml --scenario 2gb-pinning --profile aspnet-citrine-win --application.framework net10.0 +crank --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/aspnet.profiles.yml --config https://raw.githubusercontent.com/dotnet/performance/main/src/benchmarks/gc/scenarios/CrankConfiguration.yaml --scenario 2gb-pinning --profile aspnet-gold-win --application.framework net10.0 ``` ### Available scenarios diff --git a/scenarios/aspnet.profiles.standard.yml b/scenarios/aspnet.profiles.standard.yml index 5d64a8b2f..89ef38396 100644 --- a/scenarios/aspnet.profiles.standard.yml +++ b/scenarios/aspnet.profiles.standard.yml @@ -48,118 +48,6 @@ profiles: - db - downstream - aspnet-citrine-arm-lin: - variables: - mainAddress: 10.0.0.108 - secondaryAddress: 10.0.0.105 - extraAddress: 10.0.0.103 - serverAddress: 10.0.0.108 - databaseServer: 10.0.0.103 - downstreamAddress: 10.0.0.103 - cores: 80 - agents: - main: - endpoints: - - http://asp-citrine-ampere:5001 - aliases: - - application - secondary: - endpoints: - - http://asp-citrine-load:5001 - aliases: - - load - - warmup - extra: - endpoints: - - http://asp-citrine-db:5001 - aliases: - - db - - downstream - - aspnet-citrine-arm-lin-relay: - variables: - mainAddress: 10.0.0.108 - secondaryAddress: 10.0.0.105 - extraAddress: 10.0.0.103 - serverAddress: 10.0.0.108 - databaseServer: 10.0.0.103 - downstreamAddress: 10.0.0.103 - cores: 80 - agents: - main: - endpoints: - - https://aspnetperf.servicebus.windows.net/citrinearmlin - aliases: - - application - secondary: - endpoints: - - https://aspnetperf.servicebus.windows.net/citrineload - aliases: - - load - - warmup - extra: - endpoints: - - https://aspnetperf.servicebus.windows.net/citrinedb - aliases: - - db - - downstream - - aspnet-citrine-arm-win: - variables: - mainAddress: 10.0.0.110 - secondaryAddress: 10.0.0.105 - extraAddress: 10.0.0.103 - serverAddress: 10.0.0.110 - databaseServer: 10.0.0.103 - downstreamAddress: 10.0.0.103 - cores: 80 - agents: - main: - endpoints: - - http://10.240.177.200:5001 - aliases: - - application - secondary: - endpoints: - - http://asp-citrine-load:5001 - aliases: - - load - - warmup - extra: - endpoints: - - http://asp-citrine-db:5001 - aliases: - - db - - downstream - - aspnet-citrine-arm-win-relay: - variables: - mainAddress: 10.0.0.110 - secondaryAddress: 10.0.0.105 - extraAddress: 10.0.0.103 - serverAddress: 10.0.0.110 - databaseServer: 10.0.0.103 - downstreamAddress: 10.0.0.103 - cores: 80 - agents: - main: - endpoints: - - https://aspnetperf.servicebus.windows.net/citrinearmwin - aliases: - - application - secondary: - endpoints: - - https://aspnetperf.servicebus.windows.net/citrineload - aliases: - - load - - warmup - extra: - endpoints: - - https://aspnetperf.servicebus.windows.net/citrinedb - aliases: - - db - - downstream - aspnet-gold-lin: variables: mainAddress: 10.0.0.121 @@ -272,62 +160,6 @@ profiles: - db - downstream - aspnet-siryn-arm-lin: - variables: - mainAddress: 10.0.0.111 - secondaryAddress: 10.0.0.105 - extraAddress: 10.0.0.103 - serverAddress: 10.0.0.111 - databaseServer: 10.0.0.103 - downstreamAddress: 10.0.0.103 - cores: 128 - agents: - main: - endpoints: - - http://asp-siryn-lin:5001 - aliases: - - application - secondary: - endpoints: - - http://asp-citrine-load:5001 - aliases: - - load - - warmup - extra: - endpoints: - - http://asp-citrine-db:5001 - aliases: - - db - - downstream - - aspnet-siryn-arm-lin-relay: - variables: - mainAddress: 10.0.0.111 - secondaryAddress: 10.0.0.105 - extraAddress: 10.0.0.103 - serverAddress: 10.0.0.111 - databaseServer: 10.0.0.103 - downstreamAddress: 10.0.0.103 - cores: 128 - agents: - main: - endpoints: - - https://aspnetperf.servicebus.windows.net/sirynlin - aliases: - - application - secondary: - endpoints: - - https://aspnetperf.servicebus.windows.net/citrineload - aliases: - - load - - warmup - extra: - endpoints: - - https://aspnetperf.servicebus.windows.net/citrinedb - aliases: - - db - - downstream - aspnet-cobalt-hosted-lin: variables: mainAddress: 10.57.32.73 diff --git a/scenarios/aspnet.profiles.yml b/scenarios/aspnet.profiles.yml index 03ef9ceb2..1d74dff5c 100644 --- a/scenarios/aspnet.profiles.yml +++ b/scenarios/aspnet.profiles.yml @@ -47,156 +47,6 @@ profiles: - warmup - secondary - aspnet-citrine-arm-relay: - variables: - serverAddress: 10.0.0.107 - cores: 32 - jobs: - db: - endpoints: - - https://aspnetperf.servicebus.windows.net/citrinedb - aliases: - - downstream - - extra - application: - endpoints: - - https://aspnetperf.servicebus.windows.net/citrinearm - variables: - databaseServer: 10.0.0.103 - aliases: - - main - load: - endpoints: - - https://aspnetperf.servicebus.windows.net/citrineload - aliases: - - warmup - - secondary - - aspnet-citrine-ampere: - variables: - serverAddress: 10.0.0.108 - cores: 80 - jobs: - db: - endpoints: - - http://asp-citrine-db:5001 - aliases: - - downstream - - extra - application: - endpoints: - - http://asp-citrine-ampere:5001 - variables: - databaseServer: 10.0.0.103 - aliases: - - main - load: - endpoints: - - http://asp-citrine-load:5001 - aliases: - - warmup - - secondary - - aspnet-citrine-arm-lin: - variables: - serverAddress: 10.0.0.108 - cores: 80 - jobs: - db: - endpoints: - - http://asp-citrine-db:5001 - aliases: - - downstream - - extra - application: - endpoints: - - http://asp-citrine-ampere:5001 - variables: - databaseServer: 10.0.0.103 - aliases: - - main - load: - endpoints: - - http://asp-citrine-load:5001 - aliases: - - warmup - - secondary - - aspnet-citrine-arm-lin-relay: - variables: - serverAddress: 10.0.0.108 - cores: 80 - jobs: - db: - endpoints: - - https://aspnetperf.servicebus.windows.net/citrinedb - aliases: - - downstream - - extra - application: - endpoints: - - https://aspnetperf.servicebus.windows.net/citrinearmlin - variables: - databaseServer: 10.0.0.103 - aliases: - - main - load: - endpoints: - - https://aspnetperf.servicebus.windows.net/citrineload - aliases: - - warmup - - secondary - - aspnet-citrine-arm-win: - variables: - serverAddress: 10.0.0.110 - cores: 80 - jobs: - db: - endpoints: - - http://asp-citrine-db:5001 - aliases: - - downstream - - extra - application: - endpoints: - - http://10.240.177.200:5001 - variables: - databaseServer: 10.0.0.103 - aliases: - - main - load: - endpoints: - - http://asp-citrine-load:5001 - aliases: - - warmup - - secondary - - aspnet-citrine-arm-win-relay: - variables: - serverAddress: 10.0.0.110 - cores: 80 - jobs: - db: - endpoints: - - https://aspnetperf.servicebus.windows.net/citrinedb - aliases: - - downstream - - extra - application: - endpoints: - - https://aspnetperf.servicebus.windows.net/citrinearmwin - variables: - databaseServer: 10.0.0.103 - aliases: - - main - load: - endpoints: - - https://aspnetperf.servicebus.windows.net/citrineload - aliases: - - warmup - - secondary - aspnet-gold-lin: variables: serverAddress: 10.0.0.121 @@ -287,7 +137,7 @@ profiles: endpoints: - https://aspnetperf.servicebus.windows.net/goldwin variables: - databaseServer: 10.0.0.204 + databaseServer: 10.0.0.125 aliases: - main load: @@ -297,56 +147,6 @@ profiles: - warmup - secondary - aspnet-siryn-arm-lin: - variables: - serverAddress: 10.0.0.111 - cores: 128 - jobs: - db: - endpoints: - - http://asp-citrine-db:5001 - aliases: - - downstream - - extra - application: - endpoints: - - http://asp-siryn-lin:5001 - variables: - databaseServer: 10.0.0.103 - aliases: - - main - load: - endpoints: - - http://asp-citrine-load:5001 - aliases: - - warmup - - secondary - - aspnet-siryn-arm-lin-relay: - variables: - serverAddress: 10.0.0.111 - cores: 128 - jobs: - db: - endpoints: - - https://aspnetperf.servicebus.windows.net/citrinedb - aliases: - - downstream - - extra - application: - endpoints: - - https://aspnetperf.servicebus.windows.net/sirynlin - variables: - databaseServer: 10.0.0.103 - aliases: - - main - load: - endpoints: - - https://aspnetperf.servicebus.windows.net/citrineload - aliases: - - warmup - - secondary - aspnet-cobalt-hosted-lin: variables: serverAddress: 10.57.32.73 diff --git a/scenarios/mono.benchmarks.yml b/scenarios/mono.benchmarks.yml index 59d7d0ed9..6837290c4 100644 --- a/scenarios/mono.benchmarks.yml +++ b/scenarios/mono.benchmarks.yml @@ -1,5 +1,5 @@ # Examples: -# --config mono.benchmarks.yml --scenario plaintext-mono-jit --profile aspnet-citrine-lin +# --config mono.benchmarks.yml --scenario plaintext-mono-jit --profile aspnet-gold-lin imports: - https://raw.githubusercontent.com/dotnet/crank/main/src/Microsoft.Crank.Jobs.Wrk/wrk.yml diff --git a/scenarios/platform.benchmarks.yml b/scenarios/platform.benchmarks.yml index 2b1009e54..7d9363a86 100644 --- a/scenarios/platform.benchmarks.yml +++ b/scenarios/platform.benchmarks.yml @@ -1,5 +1,5 @@ # Examples: -# --config platform.benchmarks.yml --scenario json --profile aspnet-citrine-lin +# --config platform.benchmarks.yml --scenario json --profile aspnet-gold-lin imports: - https://raw.githubusercontent.com/dotnet/crank/main/src/Microsoft.Crank.Jobs.Wrk/wrk.yml diff --git a/src/Benchmarks/json.benchmarks.yml b/src/Benchmarks/json.benchmarks.yml index 046e9418f..bc78f4d8f 100644 --- a/src/Benchmarks/json.benchmarks.yml +++ b/src/Benchmarks/json.benchmarks.yml @@ -1,9 +1,9 @@ # Examples: -# --config plaintext.benchmarks.yml --scenario plaintext --profile aspnet-physical -# --config benchmarks.compose.yml --scenario fortunes --profile aspnet-physical +# --config json.benchmarks.yml --scenario json --profile aspnet-gold-lin imports: - - https://raw.githubusercontent.com/aspnet/Benchmarks/main/src/WrkClient/wrk.yml + - https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/aspnet.profiles.yml + - https://raw.githubusercontent.com/dotnet/crank/main/src/Microsoft.Crank.Jobs.Wrk/wrk.yml jobs: aspnetbenchmarks: @@ -57,22 +57,3 @@ scenarios: job: wrk variables: path: /mvc/json - -profiles: - aspnet-citrine-arm: - variables: - serverPort: 5000 - serverAddress: 10.0.0.107 - cores: 32 - jobs: - db: - endpoints: - - http://asp-citrine-db:5001 - application: - endpoints: - - http://asp-citrine-arm:5001 - variables: - databaseServer: 10.0.0.103 - load: - endpoints: - - http://asp-citrine-load:5001 diff --git a/src/BenchmarksApps/BuildPerformance/buildperformance.yml b/src/BenchmarksApps/BuildPerformance/buildperformance.yml index a6fc91ec8..e9d7a412d 100644 --- a/src/BenchmarksApps/BuildPerformance/buildperformance.yml +++ b/src/BenchmarksApps/BuildPerformance/buildperformance.yml @@ -1,3 +1,6 @@ +imports: + - https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/aspnet.profiles.yml + scenarios: blazorwasm: application: @@ -38,22 +41,3 @@ jobs: verboseLogs: false performanceSummary: false arguments: '-s {{build_scenario}} -v {{verboseLogs}} --performanceSummary {{performanceSummary}}' - -profiles: - aspnet-citrine-arm: - variables: - serverPort: 5000 - serverAddress: 10.0.0.107 - cores: 32 - jobs: - db: - endpoints: - - http://asp-citrine-db:5001 - application: - endpoints: - - http://asp-citrine-arm:5001 - variables: - databaseServer: 10.0.0.103 - load: - endpoints: - - http://asp-citrine-load:5001 diff --git a/src/BenchmarksApps/Samples/Template/sample.yml b/src/BenchmarksApps/Samples/Template/sample.yml index 6ca472c0f..119c5521d 100644 --- a/src/BenchmarksApps/Samples/Template/sample.yml +++ b/src/BenchmarksApps/Samples/Template/sample.yml @@ -1,5 +1,5 @@ # Examples: -# --config platform.benchmarks.yml --scenario json --profile aspnet-citrine-lin +# --config platform.benchmarks.yml --scenario json --profile aspnet-gold-lin imports: - https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/aspnet.profiles.yml From cc79cfa2edca5d230762e5f47cb9e34eb0b01abc Mon Sep 17 00:00:00 2001 From: Parker Bibus Date: Wed, 21 Oct 2026 15:38:27 -0700 Subject: [PATCH 6/6] Reuse benchmark builds within each CI pipeline run Cache application and supporting builds for container, gRPC, HttpClient, SignalR, and WebSocket matrices using run-scoped source and build keys. Isolate Docker and compression variants, document the cache lifecycle, and add configuration regression coverage. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: d203afa3-f8a9-47b3-9353-e38e91a647e9 --- build/README.md | 51 +++++ build/build-reuse.profile.yml | 97 +++++++++ build/containers-scenarios.yml | 2 +- build/grpc-scenarios.yml | 20 +- build/httpclient-scenarios.yml | 4 +- build/job-variables.yml | 2 + build/signalr-scenarios.yml | 3 +- build/tests/test_build_reuse.py | 301 ++++++++++++++++++++++++++++ build/websocket-scenarios.yml | 2 +- scenarios/containers.benchmarks.yml | 6 + 10 files changed, 472 insertions(+), 16 deletions(-) create mode 100644 build/build-reuse.profile.yml create mode 100644 build/tests/test_build_reuse.py diff --git a/build/README.md b/build/README.md index 2d4941624..c06bd3e1d 100644 --- a/build/README.md +++ b/build/README.md @@ -100,6 +100,57 @@ together. The snapshot tests in `scripts/pod-scheduler/tests/test_snapshots.py` will fail in CI if the generated YAML drifts from the configs. +## Build reuse within a pipeline run + +`build-reuse.profile.yml` supplies opt-in Crank profiles for the high-repeat +container, gRPC, HttpClient, SignalR, and WebSocket matrices. Their scenario +templates load `$(buildReuseProfile)` and pass +`--variable buildCacheId=$(Build.BuildId)`. The first use of each build on an +agent checks out sources and builds; later compatible entries reuse it. A new +pipeline build ID changes both the source key and the build key, so floating +branches, .NET versions, and Docker base images are refreshed on the next run. +`--session` alone does not invalidate Crank caches. Source and build keys have +different prefixes because their metadata lives in the same cache directory. + +| Suite | Builds reused across entries | +|---|---| +| Containers | Application and database images across 9 sizes and 3 request rates | +| gRPC | Separate ASP.NET Core/Go servers and .NET/Go/h2load clients across calls, protocols, and connection/stream counts | +| HttpClient | Client and Kestrel server across GET/POST, HTTP versions, concurrency, and payload sizes | +| SignalR | Server and client across scenarios, transports, and protocols; warmup and load share the client build | +| WebSockets | Client across payload sizes; separate compressed/uncompressed server builds | + +These profiles reuse builds, not running processes or database contents. +Containers and applications are still started and stopped for each benchmark; +warmup and measurement durations are unchanged. Caches are local to each agent, +not shared between machines or architectures, and remain subject to the agent's +normal cache cleanup/lifetime. These profiles do not add a cache TTL; retained +builds need the same agent cleanup/restart policy as other Crank caches. + +Container cache keys include the image name. Different Dockerfiles must have +different image names; the ASP.NET default, Alpine, Composite, and current +variants therefore have separate tags. gRPC scenario entries specify +`serverBuildKey` and `clientBuildKey` to separate implementations, and WebSocket +server keys include the compile-time compression mode. When adding a new +compile-time variant to these matrices, give it a distinct build identity +rather than reusing an existing key. + +Other potential follow-ups are the proxy payload/protocol matrix and the +baseline/trend suites. They are not opted in here: proxies vary configuration +files copied into Docker images, while baseline/trend suites vary projects, +frameworks, and publishing settings that need their own cache identities. +The SslStream/QUIC matrix is currently disabled. Build-performance benchmarks +must continue building because build time is what they measure. + +To check cache isolation without contacting agents or running benchmarks: + +```bash +python -m unittest discover -s build/tests +``` + +The profile tests use the existing `crank --debug` configuration-only mode and +skip when the Crank CLI is not installed. Template wiring checks run without it. + ## Minimal Todo `minimal-todo-scenarios.yml` dispatches `scenarios/minimal-todo/minimal-todo.benchmarks.yml` diff --git a/build/build-reuse.profile.yml b/build/build-reuse.profile.yml new file mode 100644 index 000000000..9e33bc871 --- /dev/null +++ b/build/build-reuse.profile.yml @@ -0,0 +1,97 @@ +# Opt-in CI caches: rebuild on each pipeline run, reuse only compatible variants. +# Source and build keys need separate namespaces because they share cache metadata. +profiles: + containers-build-reuse: + agents: + application: + buildKey: 'containers-build-{{ job.Source.DockerImageName }}-{{ buildCacheId }}' + source: + sourceKey: 'containers-source-{{ job.Source.DockerImageName }}-{{ buildCacheId }}' + options: + reuseBuild: true + onConfigure: + - 'if (!/^[0-9]+$/.test(String(job.Variables.buildCacheId))) { throw "buildCacheId must be set to the pipeline build ID."; }' + db: + buildKey: 'containers-build-{{ job.Source.DockerImageName }}-{{ buildCacheId }}' + source: + sourceKey: 'containers-source-{{ job.Source.DockerImageName }}-{{ buildCacheId }}' + options: + reuseBuild: true + load: {} + + grpc-build-reuse: + agents: + application: + buildKey: 'grpc-server-build-{{ serverBuildKey }}-{{ job.Framework }}-{{ buildCacheId }}' + source: + sourceKey: 'grpc-server-source-{{ serverBuildKey }}-{{ buildCacheId }}' + options: + reuseBuild: true + onConfigure: + - 'if (!/^[0-9]+$/.test(String(job.Variables.buildCacheId))) { throw "buildCacheId must be set to the pipeline build ID."; }' + - 'if (!job.Variables.serverBuildKey || !job.Variables.clientBuildKey || !/^[a-z0-9-]+$/.test(String(job.Variables.serverBuildKey)) || !/^[a-z0-9-]+$/.test(String(job.Variables.clientBuildKey))) { throw "serverBuildKey and clientBuildKey must identify the gRPC variants using lowercase letters, digits, or hyphens."; }' + load: + buildKey: 'grpc-client-build-{{ clientBuildKey }}-{{ job.Framework }}-{{ buildCacheId }}' + source: + sourceKey: 'grpc-client-source-{{ clientBuildKey }}-{{ buildCacheId }}' + options: + reuseBuild: true + + httpclient-build-reuse: + agents: + load: + aliases: + - secondary + buildKey: 'httpclient-server-build-{{ job.Framework }}-{{ buildCacheId }}' + source: + sourceKey: 'httpclient-source-{{ buildCacheId }}' + options: + reuseBuild: true + onConfigure: + - 'if (!/^[0-9]+$/.test(String(job.Variables.buildCacheId))) { throw "buildCacheId must be set to the pipeline build ID."; }' + application: + aliases: + - main + buildKey: 'httpclient-client-build-{{ job.Framework }}-{{ buildCacheId }}' + source: + sourceKey: 'httpclient-source-{{ buildCacheId }}' + options: + reuseBuild: true + + signalr-build-reuse: + agents: + application: + buildKey: 'signalr-server-build-{{ job.Framework }}-{{ buildCacheId }}' + source: + sourceKey: 'signalr-source-{{ buildCacheId }}' + options: + reuseBuild: true + onConfigure: + - 'if (!/^[0-9]+$/.test(String(job.Variables.buildCacheId))) { throw "buildCacheId must be set to the pipeline build ID."; }' + load: + aliases: + - warmup + - secondary + buildKey: 'signalr-client-build-{{ job.Framework }}-{{ buildCacheId }}' + source: + sourceKey: 'signalr-source-{{ buildCacheId }}' + options: + reuseBuild: true + + websocket-build-reuse: + agents: + application: + buildKey: 'websocket-server-build-{{ job.Framework }}-{{ compress }}-{{ buildCacheId }}' + source: + sourceKey: 'websocket-source-{{ buildCacheId }}' + options: + reuseBuild: true + onConfigure: + - 'if (!/^[0-9]+$/.test(String(job.Variables.buildCacheId))) { throw "buildCacheId must be set to the pipeline build ID."; }' + - 'if (!/^(true|false)$/.test(String(job.Variables.compress))) { throw "compress must be true or false for WebSocket build reuse."; }' + load: + buildKey: 'websocket-client-build-{{ job.Framework }}-{{ buildCacheId }}' + source: + sourceKey: 'websocket-source-{{ buildCacheId }}' + options: + reuseBuild: true diff --git a/build/containers-scenarios.yml b/build/containers-scenarios.yml index cedd393d4..6f1c565f2 100644 --- a/build/containers-scenarios.yml +++ b/build/containers-scenarios.yml @@ -145,6 +145,6 @@ steps: { "name": "crank", "condition": "(${{ parameters.condition }}) && (${{ scenario.condition }})", - "args": [ "--session $(session) $(containerMatrixJobs) ${{ scenario.arguments }} ${{ size.arguments }} ${{ rate.arguments }} ${{ parameters.arguments }} $(azureProfile) --no-metadata --no-measurements --load.options.reuseBuild true --command-line-property --table ContainerBenchmarks --sql SQL_CONNECTION_STRING --cert-tenant-id SQL_SERVER_TENANTID --cert-client-id SQL_SERVER_CLIENTID --cert-path SQL_SERVER_CERT_PATH --cert-sni --chart" ] + "args": [ "--session $(session) $(containerMatrixJobs) ${{ scenario.arguments }} ${{ size.arguments }} ${{ rate.arguments }} ${{ parameters.arguments }} $(azureProfile) $(buildReuseProfile) --profile containers-build-reuse --variable buildCacheId=$(Build.BuildId) --no-metadata --no-measurements --load.options.reuseBuild true --command-line-property --table ContainerBenchmarks --sql SQL_CONNECTION_STRING --cert-tenant-id SQL_SERVER_TENANTID --cert-client-id SQL_SERVER_CLIENTID --cert-path SQL_SERVER_CERT_PATH --cert-sni --chart" ] } diff --git a/build/grpc-scenarios.yml b/build/grpc-scenarios.yml index b2bcc648c..ed4f0fee9 100644 --- a/build/grpc-scenarios.yml +++ b/build/grpc-scenarios.yml @@ -26,14 +26,14 @@ parameters: default: - displayName: "ASP.NET Core - Client: Grpc.Net.Client" - arguments: --scenario grpcaspnetcoreserver-grpcnetclient $(grpcJobs) --property server=grpcaspnetcoreserver --property client=grpcnetclient --application.options.collectCounters true + arguments: --scenario grpcaspnetcoreserver-grpcnetclient $(grpcJobs) --property server=grpcaspnetcoreserver --property client=grpcnetclient --application.options.collectCounters true --variable serverBuildKey=aspnetcore --variable clientBuildKey=dotnet - displayName: "ASP.NET Core - Client: Go" - arguments: --scenario grpcaspnetcoreserver-grpcgoclient $(grpcJobs) --property server=grpcaspnetcoreserver --property client=grpcgoclient --application.options.collectCounters true + arguments: --scenario grpcaspnetcoreserver-grpcgoclient $(grpcJobs) --property server=grpcaspnetcoreserver --property client=grpcgoclient --application.options.collectCounters true --variable serverBuildKey=aspnetcore --variable clientBuildKey=go - displayName: "Go - Client: Grpc.Net.Client" - arguments: --scenario grpcgoserver-grpcnetclient $(grpcJobs) --property server=grpcgoserver --property client=grpcnetclient + arguments: --scenario grpcgoserver-grpcnetclient $(grpcJobs) --property server=grpcgoserver --property client=grpcnetclient --variable serverBuildKey=go --variable clientBuildKey=dotnet - displayName: "Go - Client: Go" - arguments: --scenario grpcgoserver-grpcgoclient $(grpcJobs) --property server=grpcgoserver --property client=grpcgoclient + arguments: --scenario grpcgoserver-grpcgoclient $(grpcJobs) --property server=grpcgoserver --property client=grpcgoclient --variable serverBuildKey=go --variable clientBuildKey=go # H2Load scenarios - name: h2loadscenarios @@ -41,9 +41,9 @@ parameters: default: - displayName: "ASP.NET Core - Client: h2load" - arguments: --scenario grpcaspnetcoreserver-h2loadclient $(grpcJobs) --property server=grpcaspnetcoreserver --property client=h2loadclient --application.options.collectCounters true + arguments: --scenario grpcaspnetcoreserver-h2loadclient $(grpcJobs) --property server=grpcaspnetcoreserver --property client=h2loadclient --application.options.collectCounters true --variable serverBuildKey=aspnetcore --variable clientBuildKey=h2load - displayName: "Go - Client: h2load" - arguments: --scenario grpcgoserver-h2loadclient $(grpcJobs) --property server=grpcgoserver --property client=h2loadclient + arguments: --scenario grpcgoserver-h2loadclient $(grpcJobs) --property server=grpcgoserver --property client=h2loadclient --variable serverBuildKey=go --variable clientBuildKey=h2load # grpc-net only scenarios - name: grpcnetscenarios @@ -51,7 +51,7 @@ parameters: default: - displayName: "ASP.NET Core - Client: Grpc.Net.Client" - arguments: --scenario grpcaspnetcoreserver-grpcnetclient $(grpcJobs) --property server=grpcaspnetcoreserver --property client=grpcnetclient --application.options.collectCounters true + arguments: --scenario grpcaspnetcoreserver-grpcnetclient $(grpcJobs) --property server=grpcaspnetcoreserver --property client=grpcnetclient --application.options.collectCounters true --variable serverBuildKey=aspnetcore --variable clientBuildKey=dotnet - name: callTypes type: object @@ -107,7 +107,7 @@ steps: { "condition": "(${{ parameters.condition }}) && ((new Date().getUTCHours() - 7 + 24) % 24 >= 13)", "name": "crank", - "args": [ "--application.framework net11.0 --command-line-property --table GrpcBenchmarks --sql SQL_CONNECTION_STRING --cert-tenant-id SQL_SERVER_TENANTID --cert-client-id SQL_SERVER_CLIENTID --cert-path SQL_SERVER_CERT_PATH --cert-sni --chart --session $(session) --no-metadata --no-measurements ${{ parameters.arguments }} ${{ s.arguments }} ${{ stream.arguments }} --variable protocol=h2c --property protocol=h2c --property scenario=unary --property payload=0 --variable body=AAAAAAcKBVdvcmxk --variable path=/grpc.testing.BenchmarkService/UnaryCall" ] + "args": [ "--application.framework net11.0 --command-line-property --table GrpcBenchmarks --sql SQL_CONNECTION_STRING --cert-tenant-id SQL_SERVER_TENANTID --cert-client-id SQL_SERVER_CLIENTID --cert-path SQL_SERVER_CERT_PATH --cert-sni --chart --session $(session) --no-metadata --no-measurements ${{ parameters.arguments }} ${{ s.arguments }} ${{ stream.arguments }} $(buildReuseProfile) --profile grpc-build-reuse --variable buildCacheId=$(Build.BuildId) --variable protocol=h2c --property protocol=h2c --property scenario=unary --property payload=0 --variable body=AAAAAAcKBVdvcmxk --variable path=/grpc.testing.BenchmarkService/UnaryCall" ] } # h2/h3 HttpClient to ASP.NET Core @@ -132,7 +132,7 @@ steps: { "condition": "(${{ parameters.condition }}) && ((new Date().getUTCHours() - 7 + 24) % 24 >= 13)", "name": "crank", - "args": [ "--application.framework net11.0 --load.framework net11.0 --command-line-property --table GrpcBenchmarks --sql SQL_CONNECTION_STRING --cert-tenant-id SQL_SERVER_TENANTID --cert-client-id SQL_SERVER_CLIENTID --cert-path SQL_SERVER_CERT_PATH --cert-sni --chart --session $(session) --no-metadata --no-measurements ${{ parameters.arguments }} ${{ s.arguments }} ${{ callType.arguments }} ${{ payload.arguments }} ${{ stream.arguments }} ${{ protocol.arguments }}" ] + "args": [ "--application.framework net11.0 --load.framework net11.0 --command-line-property --table GrpcBenchmarks --sql SQL_CONNECTION_STRING --cert-tenant-id SQL_SERVER_TENANTID --cert-client-id SQL_SERVER_CLIENTID --cert-path SQL_SERVER_CERT_PATH --cert-sni --chart --session $(session) --no-metadata --no-measurements ${{ parameters.arguments }} ${{ s.arguments }} ${{ callType.arguments }} ${{ payload.arguments }} ${{ stream.arguments }} ${{ protocol.arguments }} $(buildReuseProfile) --profile grpc-build-reuse --variable buildCacheId=$(Build.BuildId)" ] } # h2c between various clients and servers @@ -154,5 +154,5 @@ steps: { "condition": "(${{ parameters.condition }}) && ((new Date().getUTCHours() - 7 + 24) % 24 >= 13)", "name": "crank", - "args": [ "--application.framework net11.0 --load.framework net11.0 --command-line-property --table GrpcBenchmarks --sql SQL_CONNECTION_STRING --cert-tenant-id SQL_SERVER_TENANTID --cert-client-id SQL_SERVER_CLIENTID --cert-path SQL_SERVER_CERT_PATH --cert-sni --chart --session $(session) ${{ parameters.arguments }} --no-metadata --no-measurements ${{ s.arguments }} ${{ callType.arguments }} ${{ payload.arguments }} ${{ stream.arguments }} --variable protocol=h2c --property protocol=h2c" ] + "args": [ "--application.framework net11.0 --load.framework net11.0 --command-line-property --table GrpcBenchmarks --sql SQL_CONNECTION_STRING --cert-tenant-id SQL_SERVER_TENANTID --cert-client-id SQL_SERVER_CLIENTID --cert-path SQL_SERVER_CERT_PATH --cert-sni --chart --session $(session) ${{ parameters.arguments }} --no-metadata --no-measurements ${{ s.arguments }} ${{ callType.arguments }} ${{ payload.arguments }} ${{ stream.arguments }} $(buildReuseProfile) --profile grpc-build-reuse --variable buildCacheId=$(Build.BuildId) --variable protocol=h2c --property protocol=h2c" ] } diff --git a/build/httpclient-scenarios.yml b/build/httpclient-scenarios.yml index 2931468b9..c5ce3c8b9 100644 --- a/build/httpclient-scenarios.yml +++ b/build/httpclient-scenarios.yml @@ -94,7 +94,7 @@ steps: { "condition": "(${{ parameters.condition }}) && (Math.round(Date.now() / 43200000) % 2 == 0)", "name": "crank", - "args": [ "--description \"${{ version.displayName }} ${{ s.displayName }} ${{ response.displayName }} ${{ ct.displayName }} $(System.JobDisplayName)\" ${{ parameters.tfm }} --command-line-property --table HttpClientBenchmarks --sql SQL_CONNECTION_STRING --cert-tenant-id SQL_SERVER_TENANTID --cert-client-id SQL_SERVER_CLIENTID --cert-path SQL_SERVER_CERT_PATH --cert-sni --chart --session $(session) ${{ parameters.arguments }} ${{ parameters.httpClientJobs }} --no-metadata --no-measurements ${{ s.arguments }} ${{ version.arguments }} ${{ ct.arguments }} ${{ response.arguments }}" ] + "args": [ "--description \"${{ version.displayName }} ${{ s.displayName }} ${{ response.displayName }} ${{ ct.displayName }} $(System.JobDisplayName)\" ${{ parameters.tfm }} --command-line-property --table HttpClientBenchmarks --sql SQL_CONNECTION_STRING --cert-tenant-id SQL_SERVER_TENANTID --cert-client-id SQL_SERVER_CLIENTID --cert-path SQL_SERVER_CERT_PATH --cert-sni --chart --session $(session) ${{ parameters.arguments }} ${{ parameters.httpClientJobs }} --no-metadata --no-measurements ${{ s.arguments }} ${{ version.arguments }} ${{ ct.arguments }} ${{ response.arguments }} $(buildReuseProfile) --profile httpclient-build-reuse --variable buildCacheId=$(Build.BuildId)" ] } # POST scenarios @@ -116,5 +116,5 @@ steps: { "condition": "(${{ parameters.condition }}) && (Math.round(Date.now() / 43200000) % 2 == 0)", "name": "crank", - "args": [ "--description \"${{ version.displayName }} ${{ s.displayName }} ${{ content.displayName }} ${{ ct.displayName }} $(System.JobDisplayName)\" ${{ parameters.tfm }} --command-line-property --table HttpClientBenchmarks --sql SQL_CONNECTION_STRING --cert-tenant-id SQL_SERVER_TENANTID --cert-client-id SQL_SERVER_CLIENTID --cert-path SQL_SERVER_CERT_PATH --cert-sni --chart --session $(session) ${{ parameters.arguments }} ${{ parameters.httpClientJobs }} --no-metadata --no-measurements ${{ s.arguments }} ${{ version.arguments }} ${{ ct.arguments }} ${{ content.arguments }}" ] + "args": [ "--description \"${{ version.displayName }} ${{ s.displayName }} ${{ content.displayName }} ${{ ct.displayName }} $(System.JobDisplayName)\" ${{ parameters.tfm }} --command-line-property --table HttpClientBenchmarks --sql SQL_CONNECTION_STRING --cert-tenant-id SQL_SERVER_TENANTID --cert-client-id SQL_SERVER_CLIENTID --cert-path SQL_SERVER_CERT_PATH --cert-sni --chart --session $(session) ${{ parameters.arguments }} ${{ parameters.httpClientJobs }} --no-metadata --no-measurements ${{ s.arguments }} ${{ version.arguments }} ${{ ct.arguments }} ${{ content.arguments }} $(buildReuseProfile) --profile httpclient-build-reuse --variable buildCacheId=$(Build.BuildId)" ] } diff --git a/build/job-variables.yml b/build/job-variables.yml index 69216f750..798dfda33 100644 --- a/build/job-variables.yml +++ b/build/job-variables.yml @@ -77,6 +77,8 @@ variables: value: --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/build/azure.profile.yml - name: ciProfile value: --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/build/ci.profile.yml +- name: buildReuseProfile + value: --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/build/build-reuse.profile.yml - name: grpcJobs value: --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/grpc.benchmarks.yml - name: efcoreJobs diff --git a/build/signalr-scenarios.yml b/build/signalr-scenarios.yml index 9961df0ce..72111840e 100644 --- a/build/signalr-scenarios.yml +++ b/build/signalr-scenarios.yml @@ -74,6 +74,5 @@ steps: "condition": "${{ parameters.condition }}", "retries": 1, "name": "crank", - "args": [ "${{ s.arguments }} ${{ transport.arguments }} ${{ protocol.arguments }} --application.options.collectCounters true --load.options.collectCounters true ${{ parameters.arguments }} --load.framework net11.0 --warmup.framework net11.0 --application.framework net11.0 --session $(session) --command-line-property --table SignalRBenchmarks --sql SQL_CONNECTION_STRING --cert-tenant-id SQL_SERVER_TENANTID --cert-client-id SQL_SERVER_CLIENTID --cert-path SQL_SERVER_CERT_PATH --cert-sni --chart --no-measurements --load.options.reuseBuild true --no-metadata --warmup.options.discardResults true" ] + "args": [ "${{ s.arguments }} ${{ transport.arguments }} ${{ protocol.arguments }} --application.options.collectCounters true --load.options.collectCounters true ${{ parameters.arguments }} --load.framework net11.0 --warmup.framework net11.0 --application.framework net11.0 $(buildReuseProfile) --profile signalr-build-reuse --variable buildCacheId=$(Build.BuildId) --session $(session) --command-line-property --table SignalRBenchmarks --sql SQL_CONNECTION_STRING --cert-tenant-id SQL_SERVER_TENANTID --cert-client-id SQL_SERVER_CLIENTID --cert-path SQL_SERVER_CERT_PATH --cert-sni --chart --no-measurements --load.options.reuseBuild true --no-metadata --warmup.options.discardResults true" ] } - diff --git a/build/tests/test_build_reuse.py b/build/tests/test_build_reuse.py new file mode 100644 index 000000000..acd0d4877 --- /dev/null +++ b/build/tests/test_build_reuse.py @@ -0,0 +1,301 @@ +"""Validate CI cache profiles with Crank's configuration-only debug mode. + +Run from the repo root: python -m unittest discover -s build/tests +Requires the existing crank CLI; no agents, builds, or network access are used. +""" + +import json +from pathlib import Path +import re +import shutil +import subprocess +import tempfile +import unittest + + +ROOT = Path(__file__).resolve().parents[2] +CRANK = shutil.which("crank") +PROFILES = { + "containers": ("application", "db"), + "grpc": ("application", "load"), + "httpclient": ("server", "client"), + "signalr": ("application", "warmup", "load"), + "websocket": ("application", "load"), +} + + +class TestTemplateWiring(unittest.TestCase): + def test_all_messages_enable_run_scoped_reuse(self): + for suite in PROFILES: + with self.subTest(suite=suite): + text = (ROOT / "build" / f"{suite}-scenarios.yml").read_text( + encoding="utf-8-sig" + ) + messages = [ + line for line in text.splitlines() if '"args": [' in line + ] + self.assertTrue(messages) + for message in messages: + self.assertIn("$(buildReuseProfile)", message) + self.assertIn(f"--profile {suite}-build-reuse", message) + self.assertIn( + "--variable buildCacheId=$(Build.BuildId)", message + ) + + def test_grpc_scenarios_identify_both_build_variants(self): + text = (ROOT / "build" / "grpc-scenarios.yml").read_text() + scenarios = re.findall( + r"arguments: --scenario (grpc\S+)([^\n]+)", text + ) + self.assertEqual(len(scenarios), 7) + for scenario, arguments in scenarios: + with self.subTest(scenario=scenario): + server, client = scenario.split("-") + server_key = "aspnetcore" if "aspnetcore" in server else "go" + client_key = { + "grpcnetclient": "dotnet", + "grpcgoclient": "go", + "h2loadclient": "h2load", + }[client] + self.assertIn(f"serverBuildKey={server_key}", arguments) + self.assertIn(f"clientBuildKey={client_key}", arguments) + + def test_container_dockerfile_variants_have_distinct_tags(self): + text = (ROOT / "scenarios" / "containers.benchmarks.yml").read_text() + variants = re.findall( + r"dockerFile: (docker/container-matrix/\S+)\n" + r"\s+dockerImageName: (\S+)", text + ) + self.assertEqual(len(variants), 7) + tags = {} + for dockerfile, tag in variants: + self.assertEqual(tags.setdefault(tag, dockerfile), dockerfile) + self.assertEqual(len(tags), 4) + + +@unittest.skipUnless(CRANK, "Requires the crank CLI") +class TestBuildReuseProfiles(unittest.TestCase): + def configure(self, suite, run_id="123", variables=None, patches=None, error=None): + roles = PROFILES[suite] + if suite == "containers": + roles = (*roles, "load") + jobs = { + role: { + "source": { + "repository": "https://github.com/aspnet/Benchmarks.git", + "branchOrCommit": "main", + "project": f"src/{role}/App.csproj", + }, + "framework": "net11.0", + } + for role in roles + } + if suite == "containers": + for role in PROFILES[suite]: + jobs[role]["source"] = { + "repository": "https://github.com/TechEmpower/FrameworkBenchmarks.git", + "branchOrCommit": "master", + "dockerFile": f"{role}/Dockerfile", + "dockerImageName": role, + } + if suite == "signalr": + jobs["warmup"]["source"]["project"] = "src/load/App.csproj" + for role, patch in (patches or {}).items(): + jobs[role].update(patch) + + scenario = {role: {"job": role} for role in roles} + if suite == "httpclient": + scenario["client"]["agent"] = "main" + scenario["server"]["agent"] = "secondary" + config = { + "jobs": jobs, + "profiles": { + "machines": { + "agents": { + "application": {"endpoints": [], "aliases": ["main"]}, + "load": { + "endpoints": [], + "aliases": ["warmup", "secondary"], + }, + "db": {"endpoints": []}, + } + } + }, + "scenarios": {"test": scenario}, + } + values = { + "compress": "false", + "serverBuildKey": "aspnetcore", + "clientBuildKey": "dotnet", + } + if run_id is not None: + values["buildCacheId"] = run_id + values.update(variables or {}) + with tempfile.TemporaryDirectory() as temp: + source = Path(temp) / "scenario.json" + output = Path(temp) / "debug.json" + source.write_text(json.dumps(config), encoding="utf-8") + command = [ + CRANK, + "--config", str(source), + "--config", str(ROOT / "build" / "build-reuse.profile.yml"), + "--scenario", "test", + "--profile", "machines", + "--profile", f"{suite}-build-reuse", + "--debug", str(output), + ] + for key, value in values.items(): + command.extend(("--variable", f"{key}={value}")) + result = subprocess.run( + command, capture_output=True, text=True, timeout=60 + ) + if run_id is None or not str(run_id).isdigit(): + error = "buildCacheId must be set" + if error is not None: + self.assertNotEqual(result.returncode, 0, result.stdout) + self.assertIn(error, result.stdout + result.stderr) + return None + self.assertEqual( + result.returncode, 0, result.stdout + result.stderr + ) + return json.loads(output.read_text(encoding="utf-8"))["Jobs"] + + def assert_cached(self, jobs, suite): + for role in PROFILES[suite]: + with self.subTest(suite=suite, role=role): + job = jobs[role] + self.assertTrue(job["Options"]["ReuseBuild"]) + self.assertTrue(job["NoBuild"]) + self.assertTrue(job["BuildKey"].endswith("-123")) + source_key = job["Source"]["SourceKey"] + self.assertTrue(source_key.endswith("-123")) + self.assertNotEqual(job["BuildKey"], source_key) + + def test_each_profile_invalidates_sources_and_builds_on_new_run(self): + for suite in PROFILES: + with self.subTest(suite=suite): + first = self.configure(suite) + self.assert_cached(first, suite) + second = self.configure(suite, "124") + for role in PROFILES[suite]: + self.assertNotEqual( + first[role]["BuildKey"], second[role]["BuildKey"] + ) + self.assertNotEqual( + first[role]["Source"]["SourceKey"], + second[role]["Source"]["SourceKey"], + ) + + def test_missing_or_invalid_run_id_is_rejected(self): + for suite in PROFILES: + for run_id in (None, "not-a-build"): + with self.subTest(suite=suite, run_id=run_id): + self.configure(suite, run_id) + + def test_missing_or_invalid_build_variants_are_rejected(self): + for key, value in ( + ("serverBuildKey", ""), + ("clientBuildKey", "../invalid"), + ): + with self.subTest(key=key): + self.configure( + "grpc", + variables={key: value}, + error="serverBuildKey and clientBuildKey must identify", + ) + self.configure( + "websocket", + variables={"compress": ""}, + error="compress must be true or false", + ) + + def test_container_runtime_limits_and_rate_do_not_change_cache(self): + first = self.configure("containers", variables={"rate": "1000"}) + second = self.configure( + "containers", + variables={"rate": "10000"}, + patches={ + "application": { + "cpuLimitRatio": 0.25, + "memoryLimitInBytes": 256000000, + "cpuSet": "0", + } + }, + ) + for role in PROFILES["containers"]: + self.assertEqual(first[role]["BuildKey"], second[role]["BuildKey"]) + self.assertEqual( + first[role]["Source"]["SourceKey"], + second[role]["Source"]["SourceKey"], + ) + + def test_container_image_variants_have_separate_caches(self): + first = self.configure("containers") + second = self.configure( + "containers", + patches={ + "application": { + "source": { + "repository": "https://github.com/aspnet/Benchmarks.git", + "branchOrCommit": "main", + "dockerFile": "docker/container-matrix/Middleware-alpine.dockerfile", + "dockerImageName": "app-alpine", + } + } + }, + ) + self.assertNotEqual( + first["application"]["BuildKey"], second["application"]["BuildKey"] + ) + self.assertNotEqual( + first["application"]["Source"]["SourceKey"], + second["application"]["Source"]["SourceKey"], + ) + self.assertEqual(first["db"]["BuildKey"], second["db"]["BuildKey"]) + + def test_websocket_compression_builds_are_separate(self): + plain = self.configure("websocket") + compressed = self.configure( + "websocket", + variables={"compress": "true"}, + patches={"application": {"buildArguments": ["/p:UseCompression=true"]}}, + ) + self.assertNotEqual( + plain["application"]["BuildKey"], + compressed["application"]["BuildKey"], + ) + self.assertEqual(plain["load"]["BuildKey"], compressed["load"]["BuildKey"]) + + def test_signalr_warmup_and_load_share_client_build(self): + jobs = self.configure("signalr") + self.assertEqual(jobs["warmup"]["BuildKey"], jobs["load"]["BuildKey"]) + self.assertNotEqual(jobs["application"]["BuildKey"], jobs["load"]["BuildKey"]) + + def test_grpc_server_and_client_variants_are_separate(self): + dotnet = self.configure("grpc") + go = self.configure( + "grpc", variables={"serverBuildKey": "go", "clientBuildKey": "go"} + ) + h2load = self.configure("grpc", variables={"clientBuildKey": "h2load"}) + self.assertNotEqual( + dotnet["application"]["BuildKey"], go["application"]["BuildKey"] + ) + self.assertEqual( + dotnet["application"]["BuildKey"], h2load["application"]["BuildKey"] + ) + self.assertEqual( + len({jobs["load"]["BuildKey"] for jobs in (dotnet, go, h2load)}), 3 + ) + + def test_framework_versions_are_separate(self): + first = self.configure("httpclient") + second = self.configure( + "httpclient", + patches={"client": {"framework": "net10.0"}}, + ) + self.assertNotEqual(first["client"]["BuildKey"], second["client"]["BuildKey"]) + self.assertEqual(first["server"]["BuildKey"], second["server"]["BuildKey"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/build/websocket-scenarios.yml b/build/websocket-scenarios.yml index 73babf846..b48d64408 100644 --- a/build/websocket-scenarios.yml +++ b/build/websocket-scenarios.yml @@ -65,5 +65,5 @@ steps: "condition": "(${{ parameters.condition }})", "retries": 1, "name": "crank", - "args": [ "${{ s.arguments }} ${{ size.arguments }} ${{ compress.arguments }} --application.options.collectCounters true --load.options.collectCounters true ${{ parameters.arguments }} --load.framework net11.0 --application.framework net11.0 --session $(session) --command-line-property --table WebsocketBenchmarks --sql SQL_CONNECTION_STRING --cert-tenant-id SQL_SERVER_TENANTID --cert-client-id SQL_SERVER_CLIENTID --cert-path SQL_SERVER_CERT_PATH --cert-sni --chart --no-measurements --load.options.reuseBuild true --no-metadata " ] + "args": [ "${{ s.arguments }} ${{ size.arguments }} ${{ compress.arguments }} --application.options.collectCounters true --load.options.collectCounters true ${{ parameters.arguments }} --load.framework net11.0 --application.framework net11.0 $(buildReuseProfile) --profile websocket-build-reuse --variable buildCacheId=$(Build.BuildId) --session $(session) --command-line-property --table WebsocketBenchmarks --sql SQL_CONNECTION_STRING --cert-tenant-id SQL_SERVER_TENANTID --cert-client-id SQL_SERVER_CLIENTID --cert-path SQL_SERVER_CERT_PATH --cert-sni --chart --no-measurements --load.options.reuseBuild true --no-metadata " ] } diff --git a/scenarios/containers.benchmarks.yml b/scenarios/containers.benchmarks.yml index 21161cba4..0bef6f654 100644 --- a/scenarios/containers.benchmarks.yml +++ b/scenarios/containers.benchmarks.yml @@ -39,6 +39,7 @@ scenarios: job: aspnet source: dockerFile: docker/container-matrix/Middleware-alpine.dockerfile + dockerImageName: app-alpine variables: scenario: json load: @@ -53,6 +54,7 @@ scenarios: job: aspnet source: dockerFile: docker/container-matrix/Middleware-composite.dockerfile + dockerImageName: app-composite variables: scenario: json load: @@ -83,6 +85,7 @@ scenarios: job: aspnet source: dockerFile: docker/container-matrix/Middleware-alpine.dockerfile + dockerImageName: app-alpine variables: scenario: dbfortunesraw load: @@ -99,6 +102,7 @@ scenarios: job: aspnet source: dockerFile: docker/container-matrix/Middleware-composite.dockerfile + dockerImageName: app-composite variables: scenario: dbfortunesraw load: @@ -113,6 +117,7 @@ scenarios: job: aspnet source: dockerFile: docker/container-matrix/Middleware-current.dockerfile + dockerImageName: app-current variables: scenario: json load: @@ -129,6 +134,7 @@ scenarios: job: aspnet source: dockerFile: docker/container-matrix/Middleware-current.dockerfile + dockerImageName: app-current variables: scenario: dbfortunesraw load: