diff --git a/azure-pipelines.yml b/azure-pipelines.yml index c8514906f39..530b3f0f180 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -20,6 +20,15 @@ trigger: - THIRD-PARTY-NOTICES.TXT pr: none +parameters: +# Bootstrap-breakage escape hatch. The ValidateSdk stage self-bootstraps arcade with the newly +# produced SDK and validates signing + promotion using that SDK; publishing depends on it. +# Override to false at queue time to ship a new Arcade when the current one cannot self-validate. +- name: runValidateSdk + displayName: Run the Validate Arcade SDK stage (signing + self-build + promotion validation) + type: boolean + default: true + variables: - template: /eng/common-variables.yml@self parameters: @@ -59,3 +68,4 @@ extends: ${{ if eq(variables['Build.SourceBranch'], 'refs/heads/main') }}: oneLocEnabled: true microbuildUseESRP: true + runValidateSdk: ${{ parameters.runValidateSdk }} diff --git a/eng/build.yml b/eng/build.yml index 895ada05323..15e7765b1e5 100644 --- a/eng/build.yml +++ b/eng/build.yml @@ -5,6 +5,14 @@ parameters: - name: microbuildUseESRP default: false type: boolean +# When true (default), run the Validate Arcade SDK stage: three parallel validations of the newly +# produced SDK - signing validation, self-build (self-bootstrap), and promotion to a test channel +# with the new Arcade. Set to false as a bootstrap-breakage escape hatch to ship a new Arcade when +# the current one cannot self-validate; the promotion to '.NET Eng - Latest' still runs on build +# success. +- name: runValidateSdk + default: true + type: boolean stages: - stage: build @@ -49,17 +57,25 @@ stages: /p:Test=false displayName: Windows Build / Publish -- stage: ValidateSdk - displayName: Validate Arcade SDK - dependsOn: build - jobs: - - template: /eng/validate-sdk.yml@self - parameters: - microbuildUseESRP: ${{ parameters.microbuildUseESRP }} - buildArgs: -configuration $(_BuildConfig) - -prepareMachine - $(_InternalBuildArgs) - /p:Test=false +- ${{ if eq(parameters.runValidateSdk, true) }}: + - stage: ValidateSdk + displayName: Validate Arcade SDK + dependsOn: build + jobs: + # Self-bootstrap: rebuild arcade with the newly produced SDK (build + sign + pack + publish). + - template: /eng/validate-sdk.yml@self + parameters: + microbuildUseESRP: ${{ parameters.microbuildUseESRP }} + buildArgs: -configuration $(_BuildConfig) + -prepareMachine + $(_InternalBuildArgs) + /p:Test=false + # Sign a curated set of artifact types with the newly built SignTool. + - template: /eng/validate-signing.yml@self + parameters: + microbuildUseESRP: ${{ parameters.microbuildUseESRP }} + # Validate that build promotion works when driven by the newly built Arcade. + - template: /eng/validate-promotion.yml@self - template: /eng/common/templates-official/post-build/post-build.yml@self parameters: @@ -68,4 +84,72 @@ stages: enableSigningValidation: false publishDependsOn: - Validate - - ValidateSdk + - ${{ if eq(parameters.runValidateSdk, true) }}: + - ValidateSdk + +# Promote the build to '.NET Eng - Latest' using the current/default Arcade. Runs after a successful +# build, and - when validation is enabled - after ValidateSdk passes, so a build can still be promoted +# when validation is disabled. Main only: release branches keep using dotnet/arcade-validation for +# this gate (see #17046). The assets were already published to '.NET Eng - Validation' by the +# post-build publish stage, so this uses --skip-assets-publishing. +- ${{ if eq(variables['Build.SourceBranch'], 'refs/heads/main') }}: + - stage: PromoteToLatest + displayName: Promote to '.NET Eng - Latest' + dependsOn: + - publish_using_darc + - ${{ if eq(parameters.runValidateSdk, true) }}: + - ValidateSdk + jobs: + - template: /eng/common/templates-official/job/job.yml@self + parameters: + name: PromoteToLatest + displayName: Promote to '.NET Eng - Latest' + enableMicrobuild: false + timeoutInMinutes: 60 + # The publish-build-assets job writes the BAR build id as the first line of ReleaseConfigs.txt. + artifacts: + download: + name: ReleaseConfigs + path: release_configs + variables: + - group: Publish-Build-Assets + - template: /eng/common/templates-official/variables/pool-providers.yml@self + pool: + name: $(DncEngInternalBuildPool) + demands: ImageOverride -equals windows.vs2026.amd64 + preSteps: + - checkout: self + clean: true + steps: + - powershell: | + $releaseConfigs = Get-Content "$(System.DefaultWorkingDirectory)/release_configs/ReleaseConfigs.txt" + $barId = ($releaseConfigs | Select-Object -First 1).Trim() + if ([string]::IsNullOrEmpty($barId)) { Write-Error "Could not read BAR build id from ReleaseConfigs.txt"; exit 1 } + Write-Host "BAR build id: $barId" + Write-Host "##vso[task.setvariable variable=BARBuildId]$barId" + displayName: Read BAR build id + - task: AzureCLI@2 + displayName: Get AzDO token (WIF) + name: GetAzDoToken + inputs: + azureSubscription: "dnceng-build-rw-code-rw-wif" + scriptType: ps + scriptLocation: inlineScript + inlineScript: | + $azdoToken = az account get-access-token --resource "499b84ac-1321-427f-aa17-267ca6975798" --query accessToken -o tsv + if ($LASTEXITCODE -ne 0) { Write-Error "Failed to get AzDO token"; exit 1 } + Write-Host "##vso[task.setvariable variable=WifAzDoToken;issecret=true]$azdoToken" + - task: AzureCLI@2 + displayName: Promote to '.NET Eng - Latest' + inputs: + azureSubscription: "Darc: Maestro Production" + scriptType: ps + scriptLocation: inlineScript + inlineScript: | + $ci = $true + $disableConfigureToolsetImport = $true + . $(Build.SourcesDirectory)/eng/common/tools.ps1 + $darc = Get-Darc + # Assets were already published to '.NET Eng - Validation'; this is the promotion gate. + & $darc add-build-to-channel --id $(BARBuildId) --channel '.NET Eng - Latest' --azdev-pat $(WifAzDoToken) --ci --skip-assets-publishing + if ($LASTEXITCODE -ne 0) { Write-Error "Failed to promote build $(BARBuildId) to '.NET Eng - Latest'."; exit 1 } diff --git a/eng/update-packagesource.ps1 b/eng/update-packagesource.ps1 deleted file mode 100644 index fd92963ded2..00000000000 --- a/eng/update-packagesource.ps1 +++ /dev/null @@ -1,90 +0,0 @@ -Param( - [string] $gitHubPat, - [string] $packagesSource -) - -$ErrorActionPreference = "Stop" -. $PSScriptRoot\common\tools.ps1 - -# Batch and executable files exit and define $LASTEXITCODE. Powershell commands exit and define $? -function CheckExitCode ([string]$stage, [bool]$commandExitCode = $True) -{ - $exitCode = 0 - if($commandExitCode -eq -$False) { - $exitCode = 1 - } - else { - if ( Test-Path "LASTEXITCODE" -ErrorAction SilentlyContinue) - { - $exitCode = $LASTEXITCODE - } - } - - if ($exitCode -ne 0) { - Write-PipelineTelemetryError -Category "UpdatePackageSource" -Message "Something failed in stage: '$stage'. Check for errors above. Exiting now with exit code $exitCode..." - ExitWithExitCode $exitCode - } -} - -function StopDotnetIfRunning -{ - $dotnet = Get-Process "dotnet" -ErrorAction SilentlyContinue - if ($dotnet) { - stop-process $dotnet - } -} - -function AddSourceToNugetConfig([string]$nugetConfigPath, [string]$source) -{ - Write-Host "Adding '$source' to '$nugetConfigPath'..." - $nugetConfig = New-Object XML - $nugetConfig.PreserveWhitespace = $true - $nugetConfig.Load($nugetConfigPath) - $packageSources = $nugetConfig.SelectSingleNode("//packageSources") - $keyAttribute = $nugetConfig.CreateAttribute("key") - $keyAttribute.Value = "arcade-local" - $valueAttribute = $nugetConfig.CreateAttribute("value") - $valueAttribute.Value = $source - $newSource = $nugetConfig.CreateElement("add") - $newSource.Attributes.Append($keyAttribute) | Out-Null - $newSource.Attributes.Append($valueAttribute) | Out-Null - $packageSources.AppendChild($newSource) | Out-Null - $nugetConfig.Save($nugetConfigPath) -} - -try { - Push-Location $PSScriptRoot - $nugetConfigPath = Join-Path $RepoRoot "NuGet.config" - - Write-Host "Adding local source to NuGet.config" - AddSourceToNugetConfig $nugetConfigPath $packagesSource - CheckExitCode "Adding source to NuGet.config" $? - - Write-Host "Updating dependencies using Darc..." - $dotnetRoot = InitializeDotNetCli -install:$true - $DarcExe = "$dotnetRoot\tools" - Create-Directory $DarcExe - $DarcExe = Resolve-Path $DarcExe - . .\common\darc-init.ps1 -toolpath $DarcExe - CheckExitCode "Running darc-init" - - $Env:dotnet_root = $dotnetRoot - & $DarcExe\darc.exe update-dependencies ` - --packages-folder "$packagesSource" ` - --github-pat "$gitHubPat" ` - --channel ".NET Tools - Latest" ` - --ci - - CheckExitCode "Updating dependencies" -} -catch { - Write-Host $_.ScriptStackTrace - Write-PipelineTelemetryError -Category "UpdatePackageSource" -Message $_ - ExitWithExitCode 1 -} -finally { - Write-Host "Cleaning up workspace..." - StopDotnetIfRunning - Pop-Location -} -ExitWithExitCode 0 \ No newline at end of file diff --git a/eng/validate-promotion.yml b/eng/validate-promotion.yml new file mode 100644 index 00000000000..1df3a024054 --- /dev/null +++ b/eng/validate-promotion.yml @@ -0,0 +1,59 @@ +# Validates that Arcade build promotion works when driven by the newly built Arcade. One of the +# parallel jobs of the official build's ValidateSdk stage. See eng/validation/validate-promotion.ps1. +# +# NOTE: runs only in the internal official build; not testable locally or in PR. Needs +# @dotnet/dnceng review and a real official-build run to confirm. + +jobs: +- template: /eng/common/templates-official/job/job.yml@self + parameters: + name: ValidatePromotion + displayName: Validate promotion (newly built Arcade) + enableMicrobuild: false + timeoutInMinutes: 120 + # The publish-build-assets job writes the BAR build id as the first line of ReleaseConfigs.txt. + artifacts: + download: + name: ReleaseConfigs + path: release_configs + variables: + - group: Publish-Build-Assets + - template: /eng/common/templates-official/variables/pool-providers.yml@self + pool: + name: $(DncEngInternalBuildPool) + demands: ImageOverride -equals windows.vs2026.amd64 + preSteps: + - checkout: self + clean: true + steps: + - powershell: | + $releaseConfigs = Get-Content "$(System.DefaultWorkingDirectory)/release_configs/ReleaseConfigs.txt" + $barId = ($releaseConfigs | Select-Object -First 1).Trim() + if ([string]::IsNullOrEmpty($barId)) { Write-Error "Could not read BAR build id from ReleaseConfigs.txt"; exit 1 } + Write-Host "BAR build id: $barId" + Write-Host "##vso[task.setvariable variable=BARBuildId]$barId" + displayName: Read BAR build id + # Token with code read/write on the internal AzDO mirror, to push the update-dependencies branch. + - task: AzureCLI@2 + displayName: Get AzDO token (WIF) + name: GetAzDoToken + inputs: + azureSubscription: "dnceng-build-rw-code-rw-wif" + scriptType: ps + scriptLocation: inlineScript + inlineScript: | + $azdoToken = az account get-access-token --resource "499b84ac-1321-427f-aa17-267ca6975798" --query accessToken -o tsv + if ($LASTEXITCODE -ne 0) { Write-Error "Failed to get AzDO token"; exit 1 } + Write-Host "##vso[task.setvariable variable=WifAzDoToken;issecret=true]$azdoToken" + - task: AzureCLI@2 + displayName: Validate promotion with the newly built Arcade + inputs: + azureSubscription: "Darc: Maestro Production" + scriptType: ps + scriptLocation: scriptPath + scriptPath: $(System.DefaultWorkingDirectory)/eng/validation/validate-promotion.ps1 + arguments: > + -BuildId $(BARBuildId) + -Commit $(Build.SourceVersion) + -AzdoToken $(WifAzDoToken) + -GitHubPat $(BotAccount-dotnet-maestro-bot-PAT) diff --git a/eng/validate-sdk.yml b/eng/validate-sdk.yml index 8e8f57ad13f..3780e514f1a 100644 --- a/eng/validate-sdk.yml +++ b/eng/validate-sdk.yml @@ -34,16 +34,14 @@ jobs: - checkout: self clean: true steps: - - task: AzureCLI@2 - displayName: Update package source - inputs: - azureSubscription: "Darc: Maestro Production" - scriptType: ps - scriptLocation: inlineScript - inlineScript: > - .\eng\update-packagesource.ps1 - -gitHubPat $(BotAccount-dotnet-maestro-bot-PAT) - -packagesSource $(System.DefaultWorkingDirectory)/build_stage_artifacts + # Point the repo at the newly produced Arcade/Helix SDK (bump global.json + add the local feed) + # so the build below actually validates the SDK this build produced, not the bootstrap SDK. This + # is a pure file edit (no darc / no auth); darc update-dependencies is not used because arcade + # does not track its own SDK in Version.Details.xml, so it would have nothing to update. + - powershell: > + .\eng\validation\use-built-sdk.ps1 + -PackagesSource $(System.DefaultWorkingDirectory)/build_stage_artifacts + displayName: Use newly built SDK - script: eng\common\cibuild.cmd $(_BuildArgs) displayName: Build / Validate diff --git a/eng/validate-signing.yml b/eng/validate-signing.yml new file mode 100644 index 00000000000..85a0cdd5ffb --- /dev/null +++ b/eng/validate-signing.yml @@ -0,0 +1,239 @@ +# Validates that the *newly built* Arcade SDK's SignTool can sign a curated set of artifact types +# (packages, VSIXes, .deb/.rpm/.pkg, loose assemblies, nested containers, ...). Ported from +# dotnet/arcade-validation's Validate_Signing_{Windows,Linux,MacOS} jobs, with an added step that +# bootstraps the newly produced SDK (use-built-sdk.ps1) before signing so the new SignTool is used. +# +# The signing inputs are the existing SignTool unit-test resources +# (src/Microsoft.DotNet.SignTool.Tests/Resources) - a superset of arcade-validation's - so no binary +# assets are duplicated here. The signingTestPackages / signingTestVSIXes lists below are the curated +# set of clean, signable inputs (that folder also holds deliberately-malformed/non-signable inputs +# used by the unit tests), defined once and shared by all three jobs. +# +# One of the parallel jobs of the official build's ValidateSdk stage. +# +# NOTE: real signing uses MicroBuild/ESRP and only runs in the internal official build; not testable +# locally or in PR. Needs @dotnet/dnceng review and a real official-build run to confirm (especially +# the cross-platform use-built-sdk and Mac/Linux real-signing wiring). + +parameters: + microbuildUseESRP: false + # Curated, clean signable inputs from src/Microsoft.DotNet.SignTool.Tests/Resources, shared by all + # jobs (CopyFiles 'contents' minimatch, one pattern per line). + signingTestPackages: | + ContainerOne.1.0.0.nupkg + CustomTargetFrameworkAttribute.dll + EmptyPKT.dll + NestedContainer.1.0.0.nupkg + NestedZip.zip + PackageWithZip.nupkg + SignedLibrary.dll + test.deb + test.mpack + test.pkg + test.rpm + test.tgz + test.zip + signingTestVSIXes: | + PackageWithRelationships.vsix + test.vsix + +jobs: +- template: /eng/common/templates-official/job/job.yml@self + parameters: + name: Validate_Signing_Windows + displayName: Validate Signing (Windows) + enableMicrobuild: true + microbuildUseESRP: ${{ parameters.microbuildUseESRP }} + artifacts: + download: + name: Artifacts_Windows_NT_Release + path: build_stage_artifacts + publish: + logs: + name: Logs_ValidateSigning_Windows_$(_BuildConfig) + timeoutInMinutes: 90 + variables: + - group: Publish-Build-Assets + - name: TeamName + value: DotNetCore + - template: /eng/common/templates-official/variables/pool-providers.yml@self + pool: + name: $(DncEngInternalBuildPool) + demands: ImageOverride -equals windows.vs2026.amd64 + strategy: + matrix: + Test_Signing: + _BuildConfig: Debug + _SignType: test + Real_Signing: + _BuildConfig: Release + _SignType: real + preSteps: + - checkout: self + clean: true + steps: + # Bootstrap the newly produced SDK so the sign below uses the new SignTool. + - task: AzureCLI@2 + displayName: Update package source (use newly built SDK) + inputs: + azureSubscription: "Darc: Maestro Production" + scriptType: ps + scriptLocation: inlineScript + inlineScript: > + .\eng\validation\use-built-sdk.ps1 + -PackagesSource $(System.DefaultWorkingDirectory)/build_stage_artifacts + - task: CopyFiles@2 + displayName: Stage signing test packages + inputs: + sourceFolder: $(Build.SourcesDirectory)\src\Microsoft.DotNet.SignTool.Tests\Resources + targetFolder: $(Build.SourcesDirectory)\artifacts\packages\$(_BuildConfig)\NonShipping + contents: ${{ parameters.signingTestPackages }} + - task: CopyFiles@2 + displayName: Stage signing test VSIXes + inputs: + sourceFolder: $(Build.SourcesDirectory)\src\Microsoft.DotNet.SignTool.Tests\Resources + targetFolder: $(Build.SourcesDirectory)\artifacts\VSSetup\$(_BuildConfig) + contents: ${{ parameters.signingTestVSIXes }} + - powershell: eng\common\build.ps1 + -configuration $(_BuildConfig) + -restore + -prepareMachine + -sign + -ci + /p:DotNetSignType=$(_SignType) + /p:TeamName=$(TeamName) + /p:OfficialBuildId=$(Build.BuildNumber) + displayName: Sign with the newly built SignTool + env: + SYSTEM_ACCESSTOKEN: $(System.AccessToken) + +- template: /eng/common/templates-official/job/job.yml@self + parameters: + name: Validate_Signing_Linux + displayName: Validate Signing (Linux) + enableMicrobuild: true + enableMicrobuildForMacAndLinux: true + microbuildUseESRP: ${{ parameters.microbuildUseESRP }} + artifacts: + download: + name: Artifacts_Windows_NT_Release + path: build_stage_artifacts + publish: + logs: + name: Logs_ValidateSigning_Linux_$(_BuildConfig) + timeoutInMinutes: 90 + variables: + - group: Publish-Build-Assets + - name: TeamName + value: DotNetCore + - name: _BuildConfig + value: Release + - name: _SignType + value: real + - template: /eng/common/templates-official/variables/pool-providers.yml@self + pool: + name: $(DncEngInternalBuildPool) + image: build.azurelinux.3.amd64 + os: linux + preSteps: + - checkout: self + clean: true + steps: + - task: AzureCLI@2 + displayName: Update package source (use newly built SDK) + inputs: + azureSubscription: "Darc: Maestro Production" + scriptType: pscore + scriptLocation: inlineScript + inlineScript: > + ./eng/validation/use-built-sdk.ps1 + -PackagesSource $(System.DefaultWorkingDirectory)/build_stage_artifacts + - task: CopyFiles@2 + displayName: Stage signing test packages + inputs: + sourceFolder: $(Build.SourcesDirectory)/src/Microsoft.DotNet.SignTool.Tests/Resources + targetFolder: $(Build.SourcesDirectory)/artifacts/packages/$(_BuildConfig)/NonShipping + contents: ${{ parameters.signingTestPackages }} + - task: CopyFiles@2 + displayName: Stage signing test VSIXes + inputs: + sourceFolder: $(Build.SourcesDirectory)/src/Microsoft.DotNet.SignTool.Tests/Resources + targetFolder: $(Build.SourcesDirectory)/artifacts/VSSetup/$(_BuildConfig) + contents: ${{ parameters.signingTestVSIXes }} + - script: eng/common/build.sh + --configuration $(_BuildConfig) + --restore + --prepareMachine + --sign + --ci + /p:DotNetSignType=$(_SignType) + /p:TeamName=$(TeamName) + /p:OfficialBuildId=$(Build.BuildNumber) + displayName: Sign with the newly built SignTool + env: + SYSTEM_ACCESSTOKEN: $(System.AccessToken) + +- template: /eng/common/templates-official/job/job.yml@self + parameters: + name: Validate_Signing_MacOS + displayName: Validate Signing (macOS) + enableMicrobuild: true + enableMicrobuildForMacAndLinux: true + microbuildUseESRP: ${{ parameters.microbuildUseESRP }} + artifacts: + download: + name: Artifacts_Windows_NT_Release + path: build_stage_artifacts + publish: + logs: + name: Logs_ValidateSigning_MacOS_$(_BuildConfig) + timeoutInMinutes: 90 + variables: + - group: Publish-Build-Assets + - name: TeamName + value: DotNetCore + - name: _BuildConfig + value: Release + - name: _SignType + value: real + pool: + name: Azure Pipelines + image: macos-latest + os: macOS + preSteps: + - checkout: self + clean: true + steps: + - task: AzureCLI@2 + displayName: Update package source (use newly built SDK) + inputs: + azureSubscription: "Darc: Maestro Production" + scriptType: pscore + scriptLocation: inlineScript + inlineScript: > + ./eng/validation/use-built-sdk.ps1 + -PackagesSource $(System.DefaultWorkingDirectory)/build_stage_artifacts + - task: CopyFiles@2 + displayName: Stage signing test packages + inputs: + sourceFolder: $(Build.SourcesDirectory)/src/Microsoft.DotNet.SignTool.Tests/Resources + targetFolder: $(Build.SourcesDirectory)/artifacts/packages/$(_BuildConfig)/NonShipping + contents: ${{ parameters.signingTestPackages }} + - task: CopyFiles@2 + displayName: Stage signing test VSIXes + inputs: + sourceFolder: $(Build.SourcesDirectory)/src/Microsoft.DotNet.SignTool.Tests/Resources + targetFolder: $(Build.SourcesDirectory)/artifacts/VSSetup/$(_BuildConfig) + contents: ${{ parameters.signingTestVSIXes }} + - script: eng/common/build.sh + --configuration $(_BuildConfig) + --restore + --prepareMachine + --sign + --ci + /p:DotNetSignType=$(_SignType) + /p:TeamName=$(TeamName) + /p:OfficialBuildId=$(Build.BuildNumber) + displayName: Sign with the newly built SignTool + env: + SYSTEM_ACCESSTOKEN: $(System.AccessToken) diff --git a/eng/validation/use-built-sdk.ps1 b/eng/validation/use-built-sdk.ps1 new file mode 100644 index 00000000000..4fbb57e7099 --- /dev/null +++ b/eng/validation/use-built-sdk.ps1 @@ -0,0 +1,94 @@ +# Points the repo at a locally built Arcade/Helix SDK (bumps the msbuild-sdks versions in global.json +# and adds the local package feed) so a subsequent `build` uses the newly built SDK - both the +# signing validation (`build -sign`, exercising the new SignTool) and the self-build validation. +# +# Cross-platform and dependency-free (pure file edits): it does NOT run dotnet or darc, so it works +# on the Windows/Linux/macOS agents. darc update-dependencies is deliberately not used: arcade does +# not track its own Arcade/Helix SDK in Version.Details.xml, so darc would find nothing to update and +# would leave global.json (and therefore the SDK actually used) unchanged. + +Param( + [Parameter(Mandatory=$true)][string] $PackagesSource # Folder containing the freshly built *.nupkg (e.g. the downloaded build artifacts). +) + +$ErrorActionPreference = 'Stop' + +$repoRoot = (Resolve-Path "$PSScriptRoot/../..").Path + +# Discover the produced Arcade SDK version from the built package. +$arcadePkg = Get-ChildItem -Path $PackagesSource -Recurse -Filter 'Microsoft.DotNet.Arcade.Sdk.*.nupkg' -ErrorAction SilentlyContinue | Select-Object -First 1 +if (-not $arcadePkg) { + throw "Could not find Microsoft.DotNet.Arcade.Sdk.*.nupkg under '$PackagesSource'." +} +if ($arcadePkg.Name -notmatch '^Microsoft\.DotNet\.Arcade\.Sdk\.(.+)\.nupkg$') { + throw "Unexpected Arcade SDK package name '$($arcadePkg.Name)'." +} +$version = $Matches[1] +Write-Host "Using locally built Arcade/Helix SDK version '$version'." + +# Bump the msbuild-sdks versions in global.json (targeted replace to preserve formatting). +$globalJsonPath = Join-Path $repoRoot 'global.json' +$globalJson = Get-Content -Path $globalJsonPath -Raw +$globalJson = $globalJson -replace '("Microsoft\.DotNet\.Arcade\.Sdk"\s*:\s*")[^"]*(")', "`${1}$version`${2}" +$globalJson = $globalJson -replace '("Microsoft\.DotNet\.Helix\.Sdk"\s*:\s*")[^"]*(")', "`${1}$version`${2}" +Set-Content -Path $globalJsonPath -Value $globalJson -NoNewline +Write-Host "Updated Arcade/Helix SDK versions in '$globalJsonPath'." + +# Add the local package feed(s) to NuGet.config so the new SDK resolves. Add each distinct directory +# that actually contains *.nupkg (e.g. packages//Shipping and .../NonShipping) as a flat +# feed - a NuGet folder source pointing at the artifacts root does not reliably resolve the nested +# packages (and the MSBuild SDK resolver needs the SDK package findable at a source root). +$feedDirs = @(Get-ChildItem -Path $PackagesSource -Recurse -Filter '*.nupkg' -ErrorAction SilentlyContinue | + Select-Object -ExpandProperty DirectoryName -Unique) +if ($feedDirs.Count -eq 0) { + throw "No *.nupkg found under '$PackagesSource'." +} + +$nugetConfigPath = Join-Path $repoRoot 'NuGet.config' +$nugetConfig = New-Object System.Xml.XmlDocument +$nugetConfig.PreserveWhitespace = $true +$nugetConfig.Load($nugetConfigPath) +$packageSources = $nugetConfig.SelectSingleNode("//packageSources") + +# If the config uses packageSourceMapping, a bare source is never consulted for a package +# whose ID is claimed by a more/equally specific pattern on another source. The newly built SDK is +# 'Microsoft.DotNet.Arcade.Sdk', which matches the remote feeds' 'microsoft.*' mapping, so without a +# matching mapping entry NuGet would never look at our local feed. Mirror the broad patterns +# ('microsoft.*' and '*') on each local feed so it *ties* with the remotes: the local feed becomes +# eligible for the new SDK (which only it has) but, because it only ties (never exceeds) the remote +# patterns, it never blocks resolution of packages it does not contain. +$packageSourceMapping = $nugetConfig.SelectSingleNode("//packageSourceMapping") + +$index = 0 +foreach ($dir in $feedDirs) { + $key = "arcade-local-$index" + Write-Host "Adding local feed '$dir' (key '$key') to '$nugetConfigPath'." + $newSource = $nugetConfig.CreateElement("add") + $keyAttribute = $nugetConfig.CreateAttribute("key") + $keyAttribute.Value = $key + $valueAttribute = $nugetConfig.CreateAttribute("value") + $valueAttribute.Value = $dir + $newSource.Attributes.Append($keyAttribute) | Out-Null + $newSource.Attributes.Append($valueAttribute) | Out-Null + $packageSources.AppendChild($newSource) | Out-Null + + if ($null -ne $packageSourceMapping) { + $mappingSource = $nugetConfig.CreateElement("packageSource") + $mappingKey = $nugetConfig.CreateAttribute("key") + $mappingKey.Value = $key + $mappingSource.Attributes.Append($mappingKey) | Out-Null + foreach ($pattern in @('microsoft.*', '*')) { + $pkg = $nugetConfig.CreateElement("package") + $patternAttribute = $nugetConfig.CreateAttribute("pattern") + $patternAttribute.Value = $pattern + $pkg.Attributes.Append($patternAttribute) | Out-Null + $mappingSource.AppendChild($pkg) | Out-Null + } + $packageSourceMapping.AppendChild($mappingSource) | Out-Null + } + + $index++ +} +$nugetConfig.Save($nugetConfigPath) + +Write-Host "done." diff --git a/eng/validation/validate-promotion.ps1 b/eng/validation/validate-promotion.ps1 new file mode 100644 index 00000000000..39269030070 --- /dev/null +++ b/eng/validation/validate-promotion.ps1 @@ -0,0 +1,140 @@ +# Validates that Arcade build promotion works when driven by the *newly built* Arcade (rather than +# the bootstrap Arcade pinned in global.json). This is one of the parallel validations that make up +# the official build's ValidateSdk stage; it does NOT perform the final promotion to the release +# channel (that is done separately, with the default Arcade, only after all validation succeeds). +# +# Mirrors dotnet/arcade-validation's eng/validation/test-publishing.ps1: +# 1. Create a dev branch off the commit that produced this build and run +# `darc update-dependencies --id ` to bump Arcade to the version this build produced; +# push it to the internal AzDO mirror. +# 2. Promote the build to a test channel ("General Testing") with --source-branch . Darc +# runs the promotion (publishing) pipeline from that branch, so PublishArtifactsInManifest / +# Microsoft.DotNet.Build.Tasks.Feed come from the NEWLY BUILT Arcade -> promotion is validated +# against the new Arcade. +# 3. Delete the dev branch. +# +# NOTE: runs only in the internal official build; not testable locally or in PR. Needs +# @dotnet/dnceng review and a real official-build run to confirm. + +Param( + [Parameter(Mandatory=$true)][int] $BuildId, # BAR build id of the build being validated. + [Parameter(Mandatory=$true)][string] $Commit, # The commit that produced this build (Build.SourceVersion). + [Parameter(Mandatory=$true)][string] $AzdoToken, # AzDO OAuth/AAD access token (WIF), not a PAT; needs code read/write on the mirror. + [Parameter(Mandatory=$true)][string] $GitHubPat, # GitHub PAT; darc update-dependencies --id needs it for coherency updates. + [string] $AzdoOrg = 'dnceng', + [string] $AzdoProject = 'internal', + [string] $AzdoRepoName = 'dotnet-arcade', + [string] $TestChannel = 'General Testing' +) + +set-strictmode -version 2.0 +$ErrorActionPreference = 'Stop' + +. $PSScriptRoot\..\common\tools.ps1 + +# Install a private copy of darc so we don't disturb the repo's tool manifest. +$darcPath = Join-Path $PSScriptRoot "darc\$([guid]::NewGuid())" +& $PSScriptRoot\..\common\darc-init.ps1 -toolpath $darcPath | Out-Host +$darc = Join-Path $darcPath 'darc.exe' + +# The promotion validation re-publishes this build via darc from a branch bumped to the newly built +# Arcade; that publishing pipeline restores the new Arcade toolset from the build's channel feed. If +# the build is not assigned to any channel (e.g. dev/PR branches have no default channel), its assets +# are registered in BAR but not available on a restorable feed, so promotion can't be validated - +# skip it in that case. +$buildJson = & $darc get-build --id $BuildId --output-format json --ci | ConvertFrom-Json +if ($LASTEXITCODE -ne 0 -or $null -eq $buildJson) { + throw "Could not retrieve BAR build $BuildId." +} +# darc get-build may return an array; take the first element. Access 'channels' via PSObject so we +# don't throw under StrictMode when the property is omitted (darc omits it when the build is on no +# channel - which is precisely the dev/PR-branch skip case we want to detect). +$build = if ($buildJson -is [System.Array]) { $buildJson | Select-Object -First 1 } else { $buildJson } +# Note: an empty @() emitted from inside an if/else block yields $null (nothing on the output +# stream), which then throws under StrictMode on the .Count access below. Initialize to an empty +# array up front and only reassign when channels are actually present, and read Count via @(...). +$channels = @() +$channelsProperty = $build.PSObject.Properties['channels'] +if ($channelsProperty -and $channelsProperty.Value) { $channels = @($channelsProperty.Value) } +if (@($channels).Count -eq 0) { + Write-Host "Build $BuildId is not assigned to any channel; skipping promotion validation (no restorable feed for the build's assets)." + ExitWithExitCode 0 +} +Write-Host "Build $BuildId is assigned to channel(s): $($channels -join ', '). Proceeding with promotion validation." + +$targetBranch = "validation/promote-arcade-$BuildId" +$repoUri = "https://dev.azure.com/$AzdoOrg/$AzdoProject/_git/$AzdoRepoName" + +# Push using the AzDO token via an http extraheader so we don't persist credentials on disk. +$authHeader = "Authorization: Bearer $AzdoToken" + +# Work in an agent-local scratch directory. +$scratchBase = if ($env:AGENT_TEMPDIRECTORY) { $env:AGENT_TEMPDIRECTORY } else { [System.IO.Path]::GetTempPath() } +$repoRoot = Join-Path $scratchBase "promote-$BuildId" +if (Test-Path $repoRoot) { Remove-Item -Recurse -Force $repoRoot } + +try { + Write-Host "Cloning '$AzdoRepoName' and creating branch '$targetBranch' at $Commit." + & git -c "http.extraheader=$authHeader" clone -c core.longpaths=true $repoUri $repoRoot + if ($LASTEXITCODE -ne 0) { throw "git clone failed." } + + Push-Location $repoRoot + try { + & git config user.email 'dotnet-maestro[bot]@users.noreply.github.com' + & git config user.name 'dotnet-maestro[bot]' + & git checkout -b $targetBranch $Commit + if ($LASTEXITCODE -ne 0) { throw "git checkout of $Commit failed." } + + # Bump Arcade (and coherent dependencies) to the versions this build produced. The --id form + # performs coherency updates, which require a GitHub PAT. + & $darc update-dependencies --id $BuildId --azdev-pat $AzdoToken --github-pat $GitHubPat --ci + if ($LASTEXITCODE -ne 0) { throw "darc update-dependencies failed." } + + # A no-op update means darc produced the same versions already pinned, which is unexpected for a + # new build - treat it as an error rather than silently promoting an unchanged branch. + & git add -A + if ([string]::IsNullOrWhiteSpace((& git status --porcelain))) { + throw "darc update-dependencies produced no changes; the build produced the same versions already pinned." + } + & git commit -m "Update Arcade to the newly built version for promotion validation (BAR $BuildId)" + if ($LASTEXITCODE -ne 0) { throw "git commit failed." } + + & git -c "http.extraheader=$authHeader" push origin "HEAD:refs/heads/$targetBranch" + if ($LASTEXITCODE -ne 0) { throw "git push of '$targetBranch' failed." } + } + finally { + Pop-Location + } + + try { + # Validate promotion with the new Arcade: darc runs the promotion pipeline from $targetBranch, + # whose global.json now points at the newly built Arcade, so the new publishing infra does the + # work. Let darc pick the publishing infra version (do not pin it here). + Write-Host "Validating promotion with the newly built Arcade by promoting build $BuildId to '$TestChannel' from '$targetBranch'." + & $darc add-build-to-channel ` + --id $BuildId ` + --channel "$TestChannel" ` + --source-branch $targetBranch ` + --azdev-pat $AzdoToken ` + --ci + if ($LASTEXITCODE -ne 0) { throw "darc add-build-to-channel to '$TestChannel' failed." } + } + finally { + Write-Host "Cleaning up branch '$targetBranch'." + try { + & git -c "http.extraheader=$authHeader" -C $repoRoot push origin --delete $targetBranch + } + catch { + Write-Warning "Unable to delete branch '$targetBranch': $_" + } + } + + Write-Host 'done.' +} +catch { + Write-Host $_.ScriptStackTrace + Write-PipelineTelemetryError -Category 'ValidatePromotion' -Message $_ + ExitWithExitCode 1 +} + +ExitWithExitCode 0