Add staged MAUI desktop benchmarks

eng/performance/maui_desktop_benchmarks.proj

@@ -0,0 +1,36 @@
+<Project Sdk="Microsoft.DotNet.Helix.Sdk" DefaultTargets="Test">
+
+  <Import Project="Scenarios.Common.props" />
+
+  <PropertyGroup>
+    <MauiDesktopPayloadDirectory>$(CorrelationPayloadDirectory)scenarios_out\mauidesktopbenchmarks</MauiDesktopPayloadDirectory>
+  </PropertyGroup>
+
+  <ItemGroup Condition="'$(TargetsWindows)' == 'true'">
+    <PreparePayloadWorkItem Include="MAUI Desktop Benchmark Payload">
+      <Command>$(Python) pre.py --framework $(PERFLAB_Framework) --output &quot;$(MauiDesktopPayloadDirectory)&quot;</Command>
+      <WorkingDirectory>$(ScenariosDir)mauidesktopbenchmarks</WorkingDirectory>
+    </PreparePayloadWorkItem>
+  </ItemGroup>
+
+  <ItemGroup Condition="'$(TargetsWindows)' == 'true'">
+    <MauiDesktopSuite Include="MAUI Desktop BDN - Core">
+      <Suite>core</Suite>
+    </MauiDesktopSuite>
+    <MauiDesktopSuite Include="MAUI Desktop BDN - XAML">
+      <Suite>xaml</Suite>
+    </MauiDesktopSuite>
+    <MauiDesktopSuite Include="MAUI Desktop BDN - Graphics">
+      <Suite>graphics</Suite>
+    </MauiDesktopSuite>
+  </ItemGroup>
+
+  <ItemGroup>
+    <HelixWorkItem Include="@(MauiDesktopSuite)">
+      <PayloadDirectory>$(ScenariosDir)mauidesktopbenchmarks</PayloadDirectory>
+      <Command>$(Python) test.py --framework $(PERFLAB_Framework) --suite %(HelixWorkItem.Suite) --bdn-arguments &quot;$(BenchmarkDotNetArguments)&quot; $(ScenarioArgs)</Command>
+    </HelixWorkItem>
+  </ItemGroup>
+
+  <Import Project="PreparePayloadWorkItems.targets" />
+</Project>

eng/pipelines/sdk-perf-jobs.yml

@@ -93,6 +93,21 @@ jobs:
           ${{ each parameter in parameters.jobParameters }}:
             ${{ parameter.key }}: ${{ parameter.value }}
 
+  # MAUI Desktop BenchmarkDotNet benchmarks
+  - template: /eng/pipelines/templates/build-machine-matrix.yml
+    parameters:
+      jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml
+      buildMachines:
+        - win-x64
+      isPublic: true
+      jobParameters:
+        runKind: maui_desktop_benchmarks
+        projectFileName: maui_desktop_benchmarks.proj
+        channels:
+          - main
+        ${{ each parameter in parameters.jobParameters }}:
+          ${{ parameter.key }}: ${{ parameter.value }}
+
   # Blazor scenario benchmarks
   - template: /eng/pipelines/templates/build-machine-matrix.yml
     parameters:
@@ -688,6 +703,21 @@ jobs:
           ${{ each parameter in parameters.jobParameters }}:
             ${{ parameter.key }}: ${{ parameter.value }}
 
+  # MAUI Desktop BenchmarkDotNet benchmarks
+  - template: /eng/pipelines/templates/build-machine-matrix.yml
+    parameters:
+      jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml
+      buildMachines:
+        - win-x64-viper
+      isPublic: false
+      jobParameters:
+        runKind: maui_desktop_benchmarks
+        projectFileName: maui_desktop_benchmarks.proj
+        channels:
+          - main
+        ${{ each parameter in parameters.jobParameters }}:
+          ${{ parameter.key }}: ${{ parameter.value }}
+
   # NativeAOT scenario benchmarks
   - template: /eng/pipelines/templates/build-machine-matrix.yml
     parameters:

pyrightconfig.json

@@ -65,6 +65,7 @@
         { "root": "./src/scenarios/mauiblazordesktop" },
         { "root": "./src/scenarios/mauiblazorios" },
         { "root": "./src/scenarios/mauidesktop" },
+        { "root": "./src/scenarios/mauidesktopbenchmarks" },
         { "root": "./src/scenarios/mauiios" },
         { "root": "./src/scenarios/mauimaccatalyst" },
         { "root": "./src/scenarios/mauisamplecontentandroid" },

scripts/run_performance_job.py

@@ -1312,7 +1312,12 @@ def publish_dotnet_app_to_payload(payload_dir_name: str, csproj_path: str, self_
                 verbose=True).run()
 
             # Search for additional binlogs generated by the maui scenarios prepare payload work items to copy to the artifacts log dir
-            if args.run_kind in ["maui_scenarios_android", "maui_scenarios_ios", "maui_scenarios_android_innerloop"]:
+            if args.run_kind in [
+                "maui_scenarios_android",
+                "maui_scenarios_ios",
+                "maui_scenarios_android_innerloop",
+                "maui_desktop_benchmarks",
+            ]:
                 for binlog_path in glob(os.path.join(payload_dir, "scenarios_out", "**", "*.binlog"), recursive=True):
                     shutil.copy(binlog_path, ci_artifacts_log_dir)
 

scripts/tests/test_maui_desktop_benchmarks.py

@@ -0,0 +1,43 @@
+import sys
+from pathlib import Path
+
+
+REPOSITORY_ROOT = Path(__file__).parents[2]
+sys.path.insert(0, str(REPOSITORY_ROOT))
+sys.path.insert(0, str(REPOSITORY_ROOT / "scripts"))
+
+from src.scenarios.mauidesktopbenchmarks.pre import get_maui_branch
+from src.scenarios.mauidesktopbenchmarks.test import get_bdn_arguments, parse_bdn_arguments
+
+
+def test_framework_maps_to_maui_branch():
+    assert get_maui_branch("net11.0-windows10.0.19041.0") == "net11.0"
+
+
+def test_public_bdn_arguments_drop_all_category_values():
+    configured = parse_bdn_arguments(
+        "--anyCategories Libraries Runtime ThirdParty "
+        "--iterationCount 1 --warmupCount 0"
+    )
+
+    assert get_bdn_arguments(
+        "graphics",
+        configured,
+        ["--filter", "*ParseBlack*"],
+    ) == [
+        "--iterationCount",
+        "1",
+        "--warmupCount",
+        "0",
+        "--filter",
+        "*ParseBlack*",
+    ]
+
+
+def test_core_defaults_include_filter_and_log_exclusion():
+    assert get_bdn_arguments("core", [], []) == [
+        "--filter",
+        "*",
+        "--exclusion-filter",
+        "*MauiLoggerWithLoggerMinLevelErrorBenchmarker*",
+    ]

src/harness/BenchmarkDotNet.Extensions/RecommendedConfig.cs

@@ -6,6 +6,7 @@
 using BenchmarkDotNet.Jobs;
 using BenchmarkDotNet.Loggers;
 using BenchmarkDotNet.Reports;
+using BenchmarkDotNet.Toolchains;
 using Perfolizer.Horology;
 using Reporting;
 using System;
@@ -28,7 +29,8 @@ public static IConfig Create(
             List<string>? exclusionFilterValue = null,
             List<string>? categoryExclusionFilterValue = null,
             Job? job = null,
-            bool getDiffableDisasm = false)
+            bool getDiffableDisasm = false,
+            IToolchain? toolchain = null)
         {
             if (job is null)
             {
@@ -43,6 +45,11 @@ public static IConfig Create(
                 #pragma warning restore CS0618
             }
 
+            if (toolchain is not null)
+            {
+                job = job.WithToolchain(toolchain);
+            }
+
             var config = ManualConfig.CreateEmpty()
                 .WithBuildTimeout(TimeSpan.FromMinutes(15)) // for slow machines
                 .AddLogger(ConsoleLogger.Default) // log output to console
@@ -58,11 +65,15 @@ public static IConfig Create(
                 .AddFilter(new CategoryExclusionFilter(categoryExclusionFilterValue))
                 .AddExporter(JsonExporter.Full) // make sure we export to Json
                 .AddColumn(StatisticColumn.Median, StatisticColumn.Min, StatisticColumn.Max)
-                .AddValidator(new MandatoryCategoryValidator(mandatoryCategories))
                 .AddValidator(TooManyTestCasesValidator.FailOnError)
                 .AddValidator(new UniqueArgumentsValidator()) // don't allow for duplicated arguments #404
                 .WithSummaryStyle(SummaryStyle.Default.WithMaxParameterColumnWidth(36)); // the default is 20 and trims too aggressively some benchmark results
 
+            if (mandatoryCategories.Count > 0)
+            {
+                config = config.AddValidator(new MandatoryCategoryValidator(mandatoryCategories));
+            }
+
             if (Environment.IsLabEnvironment())
             {
                 config = config.AddExporter(new PerfLabExporter());

src/scenarios/mauidesktopbenchmarks/overlay/MauiPerfLab.props

@@ -0,0 +1,10 @@
+<Project>
+  <PropertyGroup>
+    <!-- Keep Windows enabled so MAUI product projects retain a desktop target. -->
+    <IncludeAndroidTargetFrameworks>false</IncludeAndroidTargetFrameworks>
+    <IncludeIosTargetFrameworks>false</IncludeIosTargetFrameworks>
+    <IncludeMacCatalystTargetFrameworks>false</IncludeMacCatalystTargetFrameworks>
+    <IncludeMacOSTargetFrameworks>false</IncludeMacOSTargetFrameworks>
+    <IncludeTizenTargetFrameworks>false</IncludeTizenTargetFrameworks>
+  </PropertyGroup>
+</Project>

src/scenarios/mauidesktopbenchmarks/overlay/MauiPerfLab.targets

@@ -0,0 +1,30 @@
+<Project>
+  <PropertyGroup>
+    <StartupObject>MauiPerfLab.EntryPoint</StartupObject>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Remove="BenchmarkDotNet" />
+    <PackageReference Include="BenchmarkDotNet" Version="@BENCHMARK_DOTNET_VERSION@" />
+    <PackageReference Include="BenchmarkDotNet.Diagnostics.Windows" Version="@BENCHMARK_DOTNET_VERSION@" />
+
+    <Compile Include="$(MSBuildThisFileDirectory)PerfLabEntry.cs" Link="PerfLabEntry.cs" />
+
+    <Reference Include="BenchmarkDotNet.Extensions">
+      <HintPath>$(MSBuildThisFileDirectory)harness\BenchmarkDotNet.Extensions.dll</HintPath>
+      <Private>true</Private>
+    </Reference>
+    <Reference Include="Reporting">
+      <HintPath>$(MSBuildThisFileDirectory)harness\Reporting.dll</HintPath>
+      <Private>true</Private>
+    </Reference>
+    <Reference Include="Newtonsoft.Json">
+      <HintPath>$(MSBuildThisFileDirectory)harness\Newtonsoft.Json.dll</HintPath>
+      <Private>true</Private>
+    </Reference>
+    <Reference Include="Microsoft.DotNet.PlatformAbstractions">
+      <HintPath>$(MSBuildThisFileDirectory)harness\Microsoft.DotNet.PlatformAbstractions.dll</HintPath>
+      <Private>true</Private>
+    </Reference>
+  </ItemGroup>
+</Project>

src/scenarios/mauidesktopbenchmarks/overlay/PerfLabEntry.cs

@@ -0,0 +1,50 @@
+#nullable enable
+
+using BenchmarkDotNet.Extensions;
+using BenchmarkDotNet.Running;
+using BenchmarkDotNet.Toolchains;
+using BenchmarkDotNet.Toolchains.InProcess.Emit;
+using System;
+using System.Collections.Generic;
+using System.Collections.Immutable;
+using System.IO;
+using System.Linq;
+using System.Threading.Tasks;
+
+namespace MauiPerfLab;
+
+internal static class EntryPoint
+{
+    public static async Task<int> Main(string[] args)
+    {
+        var argsList = args.ToList();
+        argsList = CommandLineOptions.ParseAndRemoveStringsParameter(
+            argsList,
+            "--exclusion-filter",
+            out List<string> exclusionFilter);
+
+        string? artifactsPath = Environment.GetEnvironmentVariable("MAUI_PERFLAB_ARTIFACTS_PATH");
+        if (string.IsNullOrWhiteSpace(artifactsPath))
+        {
+            throw new InvalidOperationException("MAUI_PERFLAB_ARTIFACTS_PATH must be set.");
+        }
+
+        IToolchain? toolchain =
+            Environment.GetEnvironmentVariable("MAUI_PERFLAB_IN_PROCESS") == "1"
+                ? InProcessEmitToolchain.Default
+                : null;
+
+        var config = RecommendedConfig.Create(
+            new DirectoryInfo(artifactsPath),
+            ImmutableHashSet<string>.Empty,
+            exclusionFilterValue: exclusionFilter,
+            toolchain: toolchain);
+
+        var summaries = await BenchmarkSwitcher
+            .FromAssembly(typeof(EntryPoint).Assembly)
+            .RunAsync(argsList.ToArray(), config)
+            .ConfigureAwait(false);
+
+        return summaries.ToExitCode();
+    }
+}