diff --git a/docs/sdk-scenarios.md b/docs/sdk-scenarios.md index c09f165f198..c72027e1323 100644 --- a/docs/sdk-scenarios.md +++ b/docs/sdk-scenarios.md @@ -47,6 +47,28 @@ does not download arbitrary Helix uploads into pipeline artifacts. Asynchronous skip the submitter's `artifacts/helix-results` downloads, which were not published by this pipeline. Local `--send-to-helix` runs still wait and download performance reports. +## Blazor workload preparation + +SDK Blazor jobs install `wasm-tools` once on the build agent through +`PreparePayloadWorkItems`, using the SDK in the correlation payload, its bundled +manifests (`--skip-manifest-update`), and the payload's `NuGet.config`. An installation +failure stops payload preparation before the job is sent to Helix. + +Helix work items use `pre.py publish --has-workload --readonly-dotnet` and +`post.py --readonly-dotnet`. These flags prevent the scenario helpers from changing +the shared workload installation; they do not set filesystem permissions. Application +restore, publish, size measurements, and cleanup of work-item-local outputs still run +on Helix. Standalone scenario scripts retain their existing workload-management +defaults. + +Workload availability does not select the benchmark's compilation mode. The Minimum +Template explicitly enables native relinking, while the non-AOT New Template, Pizza, +and Localized scenarios disable it with `WasmNativeWorkload=false`. AOT scenarios +enable `RunAOTCompilation=true`. The Localized AOT command passes this property directly +instead of referencing item metadata as an MSBuild property, which previously dropped +the AOT argument. Its reported sizes may therefore change when it first runs with AOT +actually enabled. + ## SDK Build Throughput Scenario **SDK Build Throughput** is a scenario test that measures the throughput of SDK build process. To be more specific, our test *implicitly calls* diff --git a/eng/performance/blazor_scenarios.proj b/eng/performance/blazor_scenarios.proj index 4cba378c7fa..03c2b5a06ee 100644 --- a/eng/performance/blazor_scenarios.proj +++ b/eng/performance/blazor_scenarios.proj @@ -7,9 +7,13 @@ %PERFLAB_TARGET_FRAMEWORKS% %24{PERFLAB_TARGET_FRAMEWORKS} <_MSBuildArgs>/p:_TrimmerDumpDependencies=true;/warnaserror:NU1602,NU1604 - --msbuild "$(_MSBuildArgs)" --msbuild-static AdditionalMonoLinkerOptions=%27"%24(AdditionalMonoLinkerOptions) --dump-dependencies"%27 --binlog $(LogDirectory)blazor_publish.binlog + --has-workload --readonly-dotnet --msbuild "$(_MSBuildArgs)" --msbuild-static AdditionalMonoLinkerOptions=%27"%24(AdditionalMonoLinkerOptions) --dump-dependencies"%27 --binlog $(LogDirectory)blazor_publish.binlog $(Python) pre.py publish $(PublishArgs) + <_BlazorDotNet>$([MSBuild]::NormalizePath('$(CorrelationPayloadDirectory)', 'dotnet', 'dotnet')) + <_BlazorDotNet Condition="'$(TargetsWindows)' == 'true'">$(_BlazorDotNet).exe + <_BlazorNuGetConfig>$([MSBuild]::NormalizePath('$(CorrelationPayloadDirectory)', 'performance', 'NuGet.config')) + pub\wwwroot pub/wwwroot @@ -27,6 +31,11 @@ + + "$(_BlazorDotNet)" workload install wasm-tools --skip-manifest-update --configfile "$(_BlazorNuGetConfig)" + $(CorrelationPayloadDirectory)performance + + %(Identity) @@ -35,7 +44,7 @@ - $(Python) post.py + $(Python) post.py --readonly-dotnet 4:00 @@ -78,17 +87,19 @@ $(PublishCommand) -f $(PerfLabTargetFrameworksEnvVar) && $(Python) test.py sod --scenario-name "%(Identity)" --dirs $(PizzaAppPubLocation) $(ScenarioArgs) + $(ScenariosDir)blazorlocalized - $(PublishCommand) -f $(PerfLabTargetFrameworksEnvVar) && $(Python) test.py sod --scenario-name "%(Identity)" --dirs $(PizzaAppPubLocation) $(ScenarioArgs) + $(PublishCommand) --msbuild "$(_MSBuildArgs);/p:WasmNativeWorkload=false" -f $(PerfLabTargetFrameworksEnvVar) && $(Python) test.py sod --scenario-name "%(Identity)" --dirs $(PizzaAppPubLocation) $(ScenarioArgs) - <_PublishArgsWithAOT>--msbuild "$(_MSBuildArgs);/p:RunAOTCompilation=true" $(ScenariosDir)blazorlocalized - $(PublishCommand) $(_PublishArgsWithAOT) -f $(PerfLabTargetFrameworksEnvVar) && $(Python) test.py sod --scenario-name "%(Identity)" --dirs $(PizzaAppPubLocation) $(ScenarioArgs) + $(PublishCommand) --msbuild "$(_MSBuildArgs);/p:RunAOTCompilation=true" -f $(PerfLabTargetFrameworksEnvVar) && $(Python) test.py sod --scenario-name "%(Identity)" --dirs $(PizzaAppPubLocation) $(ScenarioArgs) diff --git a/scripts/tests/test_blazor_workload.py b/scripts/tests/test_blazor_workload.py new file mode 100644 index 00000000000..5fab9879d76 --- /dev/null +++ b/scripts/tests/test_blazor_workload.py @@ -0,0 +1,181 @@ +import importlib +import json +import os +from pathlib import Path +import shlex +import shutil +import subprocess +import sys +from unittest.mock import Mock +import xml.etree.ElementTree as ET + +import pytest + + +repo_root = Path(__file__).resolve().parents[2] + + +@pytest.fixture +def scenario_commands(monkeypatch): + monkeypatch.syspath_prepend(str(repo_root / "scripts")) + monkeypatch.syspath_prepend(str(repo_root / "src" / "scenarios")) + return ( + importlib.import_module("shared.precommands").PreCommands, + importlib.import_module("shared.postcommands").PostCommands, + ) + + +@pytest.fixture(params=["true", "false"], ids=["windows", "unix"]) +def blazor_items(request, tmp_path): + dotnet = shutil.which("dotnet") + if dotnet is None: + pytest.skip("MSBuild evaluation requires a .NET SDK") + + project = ET.parse(repo_root / "eng" / "performance" / "blazor_scenarios.proj") + # Evaluate the real work items without downloading the Helix SDK. + project.getroot().attrib.pop("Sdk") + project.find("Import").set( + "Project", str(repo_root / "eng" / "performance" / "PreparePayloadWorkItems.targets") + ) + project_path = tmp_path / "blazor.proj" + project.write(project_path, encoding="utf-8") + payload = tmp_path / "correlation payload" + result = subprocess.run( + [ + dotnet, "msbuild", str(project_path), "-nologo", + "-getItem:PreparePayloadWorkItem,HelixWorkItem", + f"-p:CorrelationPayloadDirectory={payload}{os.sep}", + f"-p:TargetsWindows={request.param}", + "-p:Python=python", + ], + cwd=tmp_path, + check=True, + capture_output=True, + text=True, + ) + return json.loads(result.stdout)["Items"], payload, request.param == "true" + + +def test_workload_is_prepared_once_in_payload(blazor_items): + items, payload, windows = blazor_items + preparation, = items["PreparePayloadWorkItem"] + executable = payload / "dotnet" / ("dotnet.exe" if windows else "dotnet") + config = payload / "performance" / "NuGet.config" + + assert preparation["Command"] == ( + f'"{executable}" workload install wasm-tools --skip-manifest-update ' + f'--configfile "{config}"' + ) + assert Path(preparation["WorkingDirectory"]) == payload / "performance" + + +@pytest.mark.parametrize("exit_code", [0, 23]) +def test_preparation_propagates_workload_exit_code(blazor_items, tmp_path, exit_code): + _, payload, windows = blazor_items + working_directory = payload / "performance" + working_directory.mkdir(parents=True) + # Substitute only the executable; run the real preparation target and arguments. + (working_directory / "workload").write_text( + "import sys\n" + "assert sys.argv[1:4] == ['install', 'wasm-tools', '--skip-manifest-update']\n" + "print('Workload preparation executed')\n" + f"sys.exit({exit_code})\n", + encoding="utf-8", + ) + result = subprocess.run( + [ + shutil.which("dotnet"), "msbuild", str(tmp_path / "blazor.proj"), "-nologo", + "-t:PreparePayloadWorkItems", + f"-p:CorrelationPayloadDirectory={payload}{os.sep}", + f"-p:TargetsWindows={str(windows).lower()}", + f"-p:_BlazorDotNet={sys.executable}", + ], + cwd=tmp_path, + capture_output=True, + text=True, + ) + + assert "Workload preparation executed" in result.stdout + if exit_code: + assert result.returncode != 0 + assert "MSB3073" in result.stdout + else: + assert result.returncode == 0, result.stdout + result.stderr + + +def test_all_work_items_leave_shared_workload_untouched( + blazor_items, scenario_commands, monkeypatch +): + items, _, _ = blazor_items + PreCommands, PostCommands = scenario_commands + run = Mock(side_effect=AssertionError("A work item must not modify the workload")) + monkeypatch.setattr(subprocess, "run", run) + + assert len(items["HelixWorkItem"]) == 8 + for item in items["HelixWorkItem"]: + command = item["Command"].split(" && ", 1)[0] + monkeypatch.setattr(sys, "argv", shlex.split(command)[1:]) + pre = PreCommands() + assert pre.has_workload + assert pre.readonly_dotnet + pre.install_workload("wasm-tools") + pre.uninstall_workload("wasm-tools") + + monkeypatch.setattr(sys, "argv", shlex.split(item["PostCommands"])[1:]) + post = PostCommands() + assert post.readonly_dotnet + post.uninstall_workload("wasm-tools") + + run.assert_not_called() + + +@pytest.mark.parametrize( + ("name", "property_argument"), + [ + ("SOD - Localized App - Publish", "/p:WasmNativeWorkload=false"), + ("SOD - Localized App - Publish - AOT", "/p:RunAOTCompilation=true"), + ], +) +def test_localized_publish_mode_is_forwarded( + blazor_items, scenario_commands, monkeypatch, name, property_argument +): + items, _, _ = blazor_items + PreCommands, _ = scenario_commands + item = next(item for item in items["HelixWorkItem"] if item["Identity"] == name) + command = item["Command"].split(" && ", 1)[0] + monkeypatch.setattr(sys, "argv", shlex.split(command)[1:]) + pre = PreCommands() + pre.project = Mock() + pre.execute() + + assert pre.project.publish.call_args.args[6] == [ + "/p:_TrimmerDumpDependencies=true", + "/warnaserror:NU1602,NU1604", + property_argument, + ] + + +def test_standalone_workload_install_is_unchanged(scenario_commands, monkeypatch): + PreCommands, _ = scenario_commands + monkeypatch.setattr(sys, "argv", ["pre.py", "publish"]) + run = Mock() + monkeypatch.setattr(subprocess, "run", run) + + PreCommands().install_workload("wasm-tools") + + run.assert_called_once_with( + ["dotnet", "workload", "install", "wasm-tools", "--skip-manifest-update"], + check=True, + ) + + +def test_readonly_sdk_requires_preinstalled_workload(scenario_commands, monkeypatch): + PreCommands, _ = scenario_commands + monkeypatch.setattr(sys, "argv", ["pre.py", "publish", "--readonly-dotnet"]) + run = Mock() + monkeypatch.setattr(subprocess, "run", run) + + with pytest.raises(Exception, match="has_workload=false"): + PreCommands().install_workload("wasm-tools") + + run.assert_not_called()