Official build: validate + promote with the newly built Arcade (3/3)

azure-pipelines.yml

@@ -20,6 +20,19 @@ trigger:
     - THIRD-PARTY-NOTICES.TXT
 pr: none
 
+parameters:
+# Bootstrap-breakage escape hatch. The ValidateSdk stage self-bootstraps arcade with the newly
+# produced SDK and runs the validation RepoTests; publishing depends on it. Override to false at
+# queue time to ship a new Arcade when the current one cannot self-validate.
+- name: runValidateSdk
+  displayName: Run the Validate Arcade SDK stage (self-bootstrap + RepoTests)
+  type: boolean
+  default: true
+- name: usePrebuiltArcadeForPublishing
+  displayName: Validate promotion with the newly built Arcade (main only)
+  type: boolean
+  default: true
+
 variables:
 - template: /eng/common-variables.yml@self
   parameters:
@@ -59,3 +72,5 @@ extends:
         ${{ if eq(variables['Build.SourceBranch'], 'refs/heads/main') }}:
           oneLocEnabled: true
         microbuildUseESRP: true
+        runValidateSdk: ${{ parameters.runValidateSdk }}
+        usePrebuiltArcadeForPublishing: ${{ parameters.usePrebuiltArcadeForPublishing }}

eng/build.yml

@@ -5,6 +5,18 @@ parameters:
 - name: microbuildUseESRP
   default: false
   type: boolean
+# When true (default), run the in-build Arcade SDK validation stage (self-bootstrap with the newly
+# produced SDK + RepoTests). Set to false as a bootstrap-breakage escape hatch to ship a new Arcade
+# when the current one cannot self-validate; publishing then no longer depends on ValidateSdk.
+- name: runValidateSdk
+  default: true
+  type: boolean
+# When true (default), the main-branch promotion validates that promotion works when driven by the
+# newly built Arcade (promote to a test channel from a branch bumped to the new Arcade) before adding
+# the build to '.NET Eng - Latest'. Set to false to promote with the current (global.json) Arcade.
+- name: usePrebuiltArcadeForPublishing
+  default: true
+  type: boolean
 
 stages:
 - stage: build
@@ -49,17 +61,20 @@ stages:
             /p:Test=false
           displayName: Windows Build / Publish
 
-- stage: ValidateSdk
-  displayName: Validate Arcade SDK
-  dependsOn: build
-  jobs:
-  - template: /eng/validate-sdk.yml@self
-    parameters:
-      microbuildUseESRP: ${{ parameters.microbuildUseESRP }}
-      buildArgs: -configuration $(_BuildConfig)
-        -prepareMachine
-        $(_InternalBuildArgs)
-        /p:Test=false
+- ${{ if eq(parameters.runValidateSdk, true) }}:
+  - stage: ValidateSdk
+    displayName: Validate Arcade SDK
+    dependsOn: build
+    jobs:
+    - template: /eng/validate-sdk.yml@self
+      parameters:
+        microbuildUseESRP: ${{ parameters.microbuildUseESRP }}
+        # Self-bootstrap with the newly produced SDK, then run the non-Helix validation RepoTests
+        # against it (RunNonHelixTestsOnly skips the Helix-routed unit tests' inline execution).
+        buildArgs: -configuration $(_BuildConfig)
+          -prepareMachine
+          $(_InternalBuildArgs)
+          /p:RunNonHelixTestsOnly=true
 
 - template: /eng/common/templates-official/post-build/post-build.yml@self
   parameters:
@@ -68,4 +83,68 @@ stages:
     enableSigningValidation: false
     publishDependsOn:
     - Validate
-    - ValidateSdk
+    - ${{ if eq(parameters.runValidateSdk, true) }}:
+      - ValidateSdk
+
+# Promotion to '.NET Eng - Latest', validating that promotion works with the newly built Arcade.
+# Main only: release branches keep using dotnet/arcade-validation for this gate (see #17046). The
+# build's assets are already published to '.NET Eng - Validation' by the post-build publish stage.
+- ${{ if eq(variables['Build.SourceBranch'], 'refs/heads/main') }}:
+  - stage: PromoteWithNewArcade
+    displayName: Promote using the newly built Arcade
+    dependsOn:
+    - publish_using_darc
+    - ${{ if eq(parameters.runValidateSdk, true) }}:
+      - ValidateSdk
+    jobs:
+    - template: /eng/common/templates-official/job/job.yml@self
+      parameters:
+        name: PromoteWithNewArcade
+        displayName: Promote using the newly built Arcade
+        enableMicrobuild: false
+        timeoutInMinutes: 120
+        artifacts:
+          download:
+            name: ReleaseConfigs
+            path: release_configs
+        variables:
+        - group: Publish-Build-Assets
+        - template: /eng/common/templates-official/variables/pool-providers.yml@self
+        pool:
+          name: $(DncEngInternalBuildPool)
+          demands: ImageOverride -equals windows.vs2026.amd64
+        preSteps:
+        - checkout: self
+          clean: true
+        steps:
+        # The publish-build-assets job writes the BAR build id as the first line of ReleaseConfigs.txt.
+        - powershell: |
+            $releaseConfigs = Get-Content "$(System.DefaultWorkingDirectory)/release_configs/ReleaseConfigs.txt"
+            $barId = ($releaseConfigs | Select-Object -First 1).Trim()
+            if ([string]::IsNullOrEmpty($barId)) { Write-Error "Could not read BAR build id from ReleaseConfigs.txt"; exit 1 }
+            Write-Host "BAR build id: $barId"
+            Write-Host "##vso[task.setvariable variable=BARBuildId]$barId"
+          displayName: Read BAR build id
+        # Token with code read/write on the internal AzDO mirror, to push the update-dependencies branch.
+        - task: AzureCLI@2
+          displayName: Get AzDO token (WIF)
+          name: GetAzDoToken
+          inputs:
+            azureSubscription: "dnceng-build-rw-code-rw-wif"
+            scriptType: ps
+            scriptLocation: inlineScript
+            inlineScript: |
+              $azdoToken = az account get-access-token --resource "499b84ac-1321-427f-aa17-267ca6975798" --query accessToken -o tsv
+              if ($LASTEXITCODE -ne 0) { Write-Error "Failed to get AzDO token"; exit 1 }
+              Write-Host "##vso[task.setvariable variable=WifAzDoToken;issecret=true]$azdoToken"
+        - task: AzureCLI@2
+          displayName: Promote using the newly built Arcade
+          inputs:
+            azureSubscription: "Darc: Maestro Production"
+            scriptType: ps
+            scriptLocation: scriptPath
+            scriptPath: $(System.DefaultWorkingDirectory)/eng/validation/promote-with-new-arcade.ps1
+            arguments: >
+              -BuildId $(BARBuildId)
+              -AzdoToken $(WifAzDoToken)
+              -UsePrebuiltArcade ${{ parameters.usePrebuiltArcadeForPublishing }}

eng/validate-sdk.yml

@@ -12,6 +12,8 @@ jobs:
     displayName: Validate Arcade SDK
     enableMicrobuild: false
     microbuildUseESRP: ${{ parameters.microbuildUseESRP }}
+    # Upload the non-Helix (inline) validation RepoTests results from artifacts/TestResults.
+    enablePublishTestResults: true
     artifacts:
       download:
         path: build_stage_artifacts

eng/validation/promote-with-new-arcade.ps1

@@ -0,0 +1,142 @@
+# Promotes an Arcade build to the final channel, validating that promotion works when driven by the
+# *newly built* Arcade rather than the bootstrap Arcade pinned in global.json.
+#
+# This replaces the promotion validation that dotnet/arcade-validation performed (see that repo's
+# eng/validation/test-publishing.ps1 and update-channel.ps1). The flow mirrors it:
+#
+#   1. The official build has already published this build's assets to the source channel
+#      (".NET Eng - Validation") using the current/bootstrap Arcade (normal post-build publishing).
+#   2. Create a dev branch off the built commit, run `darc update-dependencies --id <BARBuildId>` to
+#      bump Arcade to the version this build produced, and push it to the internal AzDO mirror.
+#   3. Promote the build to a test channel ("General Testing") with --source-branch <devBranch>. Darc
+#      runs the promotion (publishing) pipeline from that branch, so PublishArtifactsInManifest /
+#      Microsoft.DotNet.Build.Tasks.Feed come from the NEWLY BUILT Arcade -> this validates promotion
+#      with the new Arcade.
+#   4. On success, add the build to the final channel (".NET Eng - Latest") with
+#      --skip-assets-publishing (assets are already published; this is the promotion gate).
+#   5. Delete the dev branch.
+#
+# When -UsePrebuiltArcade:$false, steps 2-4 are collapsed into a plain promotion to the final channel
+# using the current Arcade (the bootstrap-breakage escape hatch).
+#
+# NOTE: This only runs in the internal official build and cannot be validated locally or in PR. It
+# requires review by @dotnet/dnceng and a real official-build run to confirm.
+
+Param(
+  [Parameter(Mandatory=$true)][int]    $BuildId,           # BAR build id of the build being promoted.
+  [Parameter(Mandatory=$true)][string] $AzdoToken,         # AzDO token with code read/write on the mirror (System.AccessToken).
+  [string] $AzdoOrg          = 'dnceng',
+  [string] $AzdoProject      = 'internal',
+  [string] $AzdoRepoName     = 'dotnet-arcade',
+  [string] $SourceChannel    = '.NET Eng - Validation',
+  [string] $TestChannel      = 'General Testing',
+  [string] $FinalChannel     = '.NET Eng - Latest',
+  # 'true' (default): validate promotion with the newly built Arcade before the final promotion.
+  # Anything else: escape hatch - promote to the final channel with the current (global.json) Arcade.
+  [string] $UsePrebuiltArcade = 'true'
+)
+
+set-strictmode -version 2.0
+$ErrorActionPreference = 'Stop'
+
+. $PSScriptRoot\..\common\tools.ps1
+
+# Install a private copy of darc so we don't disturb the repo's tool manifest.
+$darcPath = Join-Path $PSScriptRoot "darc\$(New-Guid)"
+& $PSScriptRoot\..\common\darc-init.ps1 -toolpath $darcPath | Out-Host
+$darc = Join-Path $darcPath 'darc.exe'
+
+function Invoke-Darc {
+  Write-Host "darc $args"
+  & $darc @args --azdev-pat $AzdoToken --ci
+  if ($LASTEXITCODE -ne 0) {
+    throw "darc exited with code $LASTEXITCODE"
+  }
+}
+
+function Add-BuildToChannel([string] $channel, [string[]] $extraArgs) {
+  $darcArgs = @('add-build-to-channel', '--id', $BuildId, '--channel', $channel) + $extraArgs
+  Invoke-Darc @darcArgs
+}
+
+try {
+  if ($UsePrebuiltArcade -ne 'true') {
+    # Escape hatch: promote to the final channel with the current (global.json) Arcade.
+    Write-Host "UsePrebuiltArcade is '$UsePrebuiltArcade': promoting build $BuildId to '$FinalChannel' with the current Arcade."
+    Add-BuildToChannel $FinalChannel @('--skip-assets-publishing')
+    Write-Host 'done.'
+    ExitWithExitCode 0
+  }
+
+  # Discover the commit this build was produced from so we can branch off it.
+  $buildJson = & $darc get-build --id $BuildId --output-format json --ci | ConvertFrom-Json
+  if ($LASTEXITCODE -ne 0 -or $null -eq $buildJson) {
+    throw "Could not retrieve BAR build $BuildId."
+  }
+  $sha = $buildJson.commit
+
+  $targetBranch = "validation/promote-arcade-$BuildId"
+  $repoUri = "https://dev.azure.com/$AzdoOrg/$AzdoProject/_git/$AzdoRepoName"
+
+  # Work in an agent-local scratch directory.
+  $scratchBase = if ($env:AGENT_TEMPDIRECTORY) { $env:AGENT_TEMPDIRECTORY } else { [System.IO.Path]::GetTempPath() }
+  $repoRoot = Join-Path $scratchBase "promote-$BuildId"
+  if (Test-Path $repoRoot) { Remove-Item -Recurse -Force $repoRoot }
+
+  # Push using the AzDO token via an http extraheader so we don't persist credentials on disk.
+  $authHeader = "AUTHORIZATION: bearer $AzdoToken"
+
+  Write-Host "Cloning '$AzdoRepoName' and creating branch '$targetBranch' at $sha."
+  & git -c "http.extraheader=$authHeader" clone -c core.longpaths=true $repoUri $repoRoot
+  if ($LASTEXITCODE -ne 0) { throw "git clone failed." }
+
+  Push-Location $repoRoot
+  try {
+    & git config user.email 'dotnet-maestro[bot]@users.noreply.github.com'
+    & git config user.name 'dotnet-maestro[bot]'
+    & git checkout -b $targetBranch $sha
+    if ($LASTEXITCODE -ne 0) { throw "git checkout of $sha failed." }
+
+    # Bump Arcade (and coherent dependencies) to the versions this build produced.
+    & $darc update-dependencies --id $BuildId --azdev-pat $AzdoToken --ci
+    if ($LASTEXITCODE -ne 0) { throw "darc update-dependencies failed." }
+
+    & git commit -am "Update Arcade to the newly built version for promotion validation (BAR $BuildId)"
+    if ($LASTEXITCODE -ne 0) { throw "git commit failed." }
+
+    & git -c "http.extraheader=$authHeader" push origin HEAD
+    if ($LASTEXITCODE -ne 0) { throw "git push of '$targetBranch' failed." }
+  }
+  finally {
+    Pop-Location
+  }
+
+  try {
+    # Test promotion with the new Arcade: darc runs the promotion pipeline from $targetBranch, whose
+    # global.json now points at the newly built Arcade, so the new publishing infra does the work.
+    Write-Host "Validating promotion with the newly built Arcade by promoting build $BuildId to '$TestChannel' from '$targetBranch'."
+    Add-BuildToChannel $TestChannel @('--source-branch', $targetBranch, '--publishing-infra-version', '3')
+
+    # The new Arcade published assets successfully; register the build on the final channel.
+    Write-Host "Promotion validated. Adding build $BuildId to '$FinalChannel'."
+    Add-BuildToChannel $FinalChannel @('--skip-assets-publishing')
+  }
+  finally {
+    Write-Host "Cleaning up branch '$targetBranch'."
+    try {
+      & git -c "http.extraheader=$authHeader" -C $repoRoot push origin --delete $targetBranch
+    }
+    catch {
+      Write-Warning "Unable to delete branch '$targetBranch': $_"
+    }
+  }
+
+  Write-Host 'done.'
+}
+catch {
+  Write-Host $_.ScriptStackTrace
+  Write-PipelineTelemetryError -Category 'PromoteWithNewArcade' -Message $_
+  ExitWithExitCode 1
+}
+
+ExitWithExitCode 0