diff --git a/NuGet.config b/NuGet.config index 59a6a1e57..35a81656f 100644 --- a/NuGet.config +++ b/NuGet.config @@ -14,7 +14,6 @@ - diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index c50d8bdac..48b216832 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -39,9 +39,9 @@ - - https://github.com/dotnet/dotnet - dc6e7082d768b79e69b16e8fd146a509dfa6130d + + https://github.com/dotnet/arcade + aa5dcbaa4fbec9c9f1ea9b3e9e4b7fc067f34d0b diff --git a/eng/common/AGENTS.md b/eng/common/AGENTS.md new file mode 100644 index 000000000..a5ed8f729 --- /dev/null +++ b/eng/common/AGENTS.md @@ -0,0 +1,5 @@ +# `eng/common` + +Files under `eng/common` come from [Arcade](https://github.com/dotnet/arcade). +Edits in `eng/common` will be overwritten by automation unless the changes are made directly in the Arcade repository. +For more information, see the [Arcade documentation](https://github.com/dotnet/arcade/tree/main/Documentation). diff --git a/eng/common/SetupNugetSources.ps1 b/eng/common/SetupNugetSources.ps1 index 65ed3a8ad..fc8d61801 100644 --- a/eng/common/SetupNugetSources.ps1 +++ b/eng/common/SetupNugetSources.ps1 @@ -1,7 +1,6 @@ # This script adds internal feeds required to build commits that depend on internal package sources. For instance, -# dotnet6-internal would be added automatically if dotnet6 was found in the nuget.config file. Similarly, -# dotnet-eng-internal and dotnet-tools-internal are added if dotnet-eng and dotnet-tools are present. -# In addition, this script also enables disabled internal Maestro (darc-int*) feeds. +# dotnet6-internal would be added automatically if dotnet6 was found in the nuget.config file. In addition also enables +# disabled internal Maestro (darc-int*) feeds. # # Optionally, this script also adds a credential entry for each of the internal feeds if supplied. # @@ -174,16 +173,4 @@ foreach ($dotnetVersion in $dotnetVersions) { } } -# Check for dotnet-eng and add dotnet-eng-internal if present -$dotnetEngSource = $sources.SelectSingleNode("add[@key='dotnet-eng']") -if ($dotnetEngSource -ne $null) { - AddOrEnablePackageSource -Sources $sources -DisabledPackageSources $disabledSources -SourceName "dotnet-eng-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet-eng-internal/nuget/$feedSuffix" -Creds $creds -Username $userName -pwd $Password -} - -# Check for dotnet-tools and add dotnet-tools-internal if present -$dotnetToolsSource = $sources.SelectSingleNode("add[@key='dotnet-tools']") -if ($dotnetToolsSource -ne $null) { - AddOrEnablePackageSource -Sources $sources -DisabledPackageSources $disabledSources -SourceName "dotnet-tools-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet-tools-internal/nuget/$feedSuffix" -Creds $creds -Username $userName -pwd $Password -} - $doc.Save($filename) diff --git a/eng/common/SetupNugetSources.sh b/eng/common/SetupNugetSources.sh index b2163abbe..b97cc5363 100755 --- a/eng/common/SetupNugetSources.sh +++ b/eng/common/SetupNugetSources.sh @@ -1,9 +1,8 @@ #!/usr/bin/env bash # This script adds internal feeds required to build commits that depend on internal package sources. For instance, -# dotnet6-internal would be added automatically if dotnet6 was found in the nuget.config file. Similarly, -# dotnet-eng-internal and dotnet-tools-internal are added if dotnet-eng and dotnet-tools are present. -# In addition, this script also enables disabled internal Maestro (darc-int*) feeds. +# dotnet6-internal would be added automatically if dotnet6 was found in the nuget.config file. In addition also enables +# disabled internal Maestro (darc-int*) feeds. # # Optionally, this script also adds a credential entry for each of the internal feeds if supplied. # @@ -174,18 +173,6 @@ for DotNetVersion in ${DotNetVersions[@]} ; do fi done -# Check for dotnet-eng and add dotnet-eng-internal if present -grep -i " /dev/null -if [ "$?" == "0" ]; then - AddOrEnablePackageSource "dotnet-eng-internal" "https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet-eng-internal/nuget/$FeedSuffix" -fi - -# Check for dotnet-tools and add dotnet-tools-internal if present -grep -i " /dev/null -if [ "$?" == "0" ]; then - AddOrEnablePackageSource "dotnet-tools-internal" "https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet-tools-internal/nuget/$FeedSuffix" -fi - # I want things split line by line PrevIFS=$IFS IFS=$'\n' diff --git a/eng/common/build.ps1 b/eng/common/build.ps1 index 8cfee107e..4b4f6b092 100644 --- a/eng/common/build.ps1 +++ b/eng/common/build.ps1 @@ -6,6 +6,7 @@ Param( [string][Alias('v')]$verbosity = "minimal", [string] $msbuildEngine = $null, [bool] $warnAsError = $true, + [string] $warnNotAsError = '', [bool] $nodeReuse = $true, [switch] $buildCheck = $false, [switch][Alias('r')]$restore, @@ -23,6 +24,7 @@ Param( [switch][Alias('pb')]$productBuild, [switch]$fromVMR, [switch][Alias('bl')]$binaryLog, + [string][Alias('bln')]$binaryLogName = '', [switch][Alias('nobl')]$excludeCIBinarylog, [switch] $ci, [switch] $prepareMachine, @@ -45,6 +47,7 @@ function Print-Usage() { Write-Host " -platform Platform configuration: 'x86', 'x64' or any valid Platform value to pass to msbuild" Write-Host " -verbosity Msbuild verbosity: q[uiet], m[inimal], n[ormal], d[etailed], and diag[nostic] (short: -v)" Write-Host " -binaryLog Output binary log (short: -bl)" + Write-Host " -binaryLogName Binary log file name or path; implies -binaryLog (short: -bln)" Write-Host " -help Print help and exit" Write-Host "" @@ -70,6 +73,7 @@ function Print-Usage() { Write-Host " -excludeCIBinarylog Don't output binary log (short: -nobl)" Write-Host " -prepareMachine Prepare machine for CI run, clean up processes after build" Write-Host " -warnAsError Sets warnaserror msbuild parameter ('true' or 'false')" + Write-Host " -warnNotAsError Sets a semi-colon delimited list of warning codes that should not be treated as errors" Write-Host " -msbuildEngine Msbuild engine to use to run build ('dotnet', 'vs', or unspecified)." Write-Host " -excludePrereleaseVS Set to exclude build engines in prerelease versions of Visual Studio" Write-Host " -nativeToolsOnMachine Sets the native tools on machine environment variable (indicating that the script should use native tools on machine)" @@ -100,7 +104,19 @@ function Build { $toolsetBuildProj = InitializeToolset InitializeCustomToolset - $bl = if ($binaryLog) { '/bl:' + (Join-Path $LogDir 'Build.binlog') } else { '' } + $bl = '' + if ($binaryLog) { + $binaryLogPath = if ([string]::IsNullOrEmpty($binaryLogName)) { + Join-Path $LogDir 'Build.binlog' + } elseif ([System.IO.Path]::IsPathRooted($binaryLogName)) { + $binaryLogName + } else { + Join-Path $LogDir $binaryLogName + } + + Create-Directory (Split-Path -Parent $binaryLogPath) + $bl = '/bl:' + $binaryLogPath + } $platformArg = if ($platform) { "/p:Platform=$platform" } else { '' } $check = if ($buildCheck) { '/check' } else { '' } @@ -160,6 +176,10 @@ try { $nodeReuse = $false } + if (-not [string]::IsNullOrEmpty($binaryLogName)) { + $binaryLog = $true + } + if ($nativeToolsOnMachine) { $env:NativeToolsOnMachine = $true } diff --git a/eng/common/build.sh b/eng/common/build.sh index 9767bb411..719ee4b58 100755 --- a/eng/common/build.sh +++ b/eng/common/build.sh @@ -13,6 +13,7 @@ usage() echo " --configuration Build configuration: 'Debug' or 'Release' (short: -c)" echo " --verbosity Msbuild verbosity: q[uiet], m[inimal], n[ormal], d[etailed], and diag[nostic] (short: -v)" echo " --binaryLog Create MSBuild binary log (short: -bl)" + echo " --binaryLogName Binary log file name or path; implies --binaryLog (short: -bln)" echo " --help Print help and exit (short: -h)" echo "" @@ -42,6 +43,7 @@ usage() echo " --prepareMachine Prepare machine for CI run, clean up processes after build" echo " --nodeReuse Sets nodereuse msbuild parameter ('true' or 'false')" echo " --warnAsError Sets warnaserror msbuild parameter ('true' or 'false')" + echo " --warnNotAsError Sets a semi-colon delimited list of warning codes that should not be treated as errors" echo " --buildCheck Sets /check msbuild parameter" echo " --fromVMR Set when building from within the VMR" echo "" @@ -78,11 +80,12 @@ ci=false clean=false warn_as_error=true +warn_not_as_error='' node_reuse=true build_check=false binary_log=false +binary_log_name='' exclude_ci_binary_log=false -pipelines_log=false projects='' configuration='' @@ -92,7 +95,7 @@ runtime_source_feed='' runtime_source_feed_key='' properties=() -while [[ $# > 0 ]]; do +while [[ $# -gt 0 ]]; do opt="$(echo "${1/#--/-}" | tr "[:upper:]" "[:lower:]")" case "$opt" in -help|-h) @@ -113,12 +116,14 @@ while [[ $# > 0 ]]; do -binarylog|-bl) binary_log=true ;; + -binarylogname|-bln) + binary_log=true + binary_log_name=$2 + shift + ;; -excludecibinarylog|-nobl) exclude_ci_binary_log=true ;; - -pipelineslog|-pl) - pipelines_log=true - ;; -restore|-r) restore=true ;; @@ -176,6 +181,10 @@ while [[ $# > 0 ]]; do warn_as_error=$2 shift ;; + -warnnotaserror) + warn_not_as_error=$2 + shift + ;; -nodereuse) node_reuse=$2 shift @@ -204,7 +213,6 @@ if [[ -z "$configuration" ]]; then fi if [[ "$ci" == true ]]; then - pipelines_log=true node_reuse=false if [[ "$exclude_ci_binary_log" == false ]]; then binary_log=true @@ -231,7 +239,17 @@ function Build { local bl="" if [[ "$binary_log" == true ]]; then - bl="/bl:\"$log_dir/Build.binlog\"" + local binary_log_path="" + if [[ -z "$binary_log_name" ]]; then + binary_log_path="$log_dir/Build.binlog" + elif [[ "$binary_log_name" = /* ]]; then + binary_log_path="$binary_log_name" + else + binary_log_path="$log_dir/$binary_log_name" + fi + + mkdir -p "$(dirname "$binary_log_path")" + bl="/bl:\"$binary_log_path\"" fi local check="" diff --git a/eng/common/core-templates/job/helix-job-monitor.yml b/eng/common/core-templates/job/helix-job-monitor.yml new file mode 100644 index 000000000..767450da2 --- /dev/null +++ b/eng/common/core-templates/job/helix-job-monitor.yml @@ -0,0 +1,217 @@ +parameters: +# Maximum run time of the monitor job in minutes. Also used for --max-wait-minutes. +- name: timeoutInMinutes + type: number + default: 360 + +# Owner segment of the source repository (e.g. 'dotnet' for 'dotnet/runtime') passed via --organization. +# Defaults to the owner segment of BUILD_REPOSITORY_NAME when empty. +- name: organization + type: string + default: '' + +# Name of the source repository (e.g. 'runtime' for 'dotnet/runtime') passed via --repository. +# Defaults to the repo segment of BUILD_REPOSITORY_NAME when empty. +- name: repository + type: string + default: '' + +# Optional dependency list for the generated job. +- name: dependsOn + type: object + default: [] + +# Optional condition for the generated job. +- name: condition + type: string + default: '' + +# NuGet package id of the Helix job monitor tool. +- name: toolPackageId + type: string + default: Microsoft.DotNet.Helix.JobMonitor + +# Console command exposed by the installed tool package. +- name: toolCommand + type: string + default: dotnet-helix-job-monitor + +# Optional explicit tool version. Only honored when 'toolNupkgArtifactName' is set; in the +# default code path the version is taken from the consuming repo's .config/dotnet-tools.json. +- name: toolVersion + type: string + default: '' + +# Base URI for the Helix service (--helix-base-uri). +- name: helixBaseUri + type: string + default: https://helix.dot.net/ + +# Helix API access token forwarded to the tool via the HELIX_ACCESSTOKEN environment variable. +- name: helixAccessToken + type: string + default: '' + +# Polling interval in seconds (--polling-interval-seconds). +- name: pollingIntervalSeconds + type: number + default: 30 + +# Advanced: optional pipeline artifact (produced earlier in this run) that contains the tool +# nupkg. When set, the artifact is downloaded and the tool is installed from the nupkg into +# a local tool-path; this bypasses the repo's .config/dotnet-tools.json manifest and is +# primarily intended for the Arcade repository itself, where the Helix job monitor tool is +# built in the same pipeline that runs this template. +# +# When this parameter is empty (the default), the consuming repository must declare the tool +# in its .config/dotnet-tools.json manifest (alongside other local .NET tools); the template +# will check out the repo and run 'dotnet tool restore' to install the version pinned there. +- name: toolNupkgArtifactName + type: string + default: '' + +# Advanced: sub-path within the downloaded artifact where the tool nupkg is located. Defaults +# to the standard Arcade non-shipping packages location for a Release build (relative to the +# pipeline artifact root, which is itself the build's 'artifacts' directory). +- name: toolNupkgArtifactSubPath + type: string + default: 'packages/Release/NonShipping' + +jobs: +- job: HelixJobMonitor + displayName: Monitor Helix Jobs + timeoutInMinutes: ${{ parameters.timeoutInMinutes }} + ${{ if ne(length(parameters.dependsOn), 0) }}: + dependsOn: ${{ parameters.dependsOn }} + ${{ if ne(parameters.condition, '') }}: + condition: ${{ parameters.condition }} + pool: + ${{ if eq(variables['System.TeamProject'], 'public') }}: + name: $(DncEngPublicBuildPool) + demands: ImageOverride -equals build.azurelinux.3.amd64.open + ${{ else }}: + name: $(DncEngInternalBuildPool) + demands: ImageOverride -equals build.azurelinux.3.amd64 + steps: + - checkout: self + fetchDepth: 1 + + - ${{ if ne(parameters.toolNupkgArtifactName, '') }}: + - task: DownloadPipelineArtifact@2 + displayName: Download Helix Job Monitor artifact + inputs: + buildType: current + artifactName: ${{ parameters.toolNupkgArtifactName }} + itemPattern: '${{ parameters.toolNupkgArtifactSubPath }}/${{ parameters.toolPackageId }}.*.nupkg' + targetPath: $(Agent.TempDirectory)/helix-job-monitor-nupkg + + - bash: | + set -euo pipefail + + toolPath="$AGENT_TEMPDIRECTORY/helix-job-monitor-tool" + mkdir -p "$toolPath" + + packageId='${{ parameters.toolPackageId }}' + toolVersion='${{ parameters.toolVersion }}' + nupkgArtifactSubPath='${{ parameters.toolNupkgArtifactSubPath }}' + nupkgDir="$AGENT_TEMPDIRECTORY/helix-job-monitor-nupkg/$nupkgArtifactSubPath" + + if [ ! -d "$nupkgDir" ]; then + echo "Expected nupkg directory '$nupkgDir' was not produced by the artifact download." >&2 + exit 1 + fi + + nupkg=$(find "$nupkgDir" -maxdepth 1 -type f -name "$packageId.*.nupkg" | head -n 1) + if [ -z "$nupkg" ]; then + echo "No '$packageId.*.nupkg' found in '$nupkgDir'." >&2 + exit 1 + fi + + # Derive the version from the nupkg filename so the local package is selected + # deterministically instead of resolving against any other configured feed. + nupkgBase=$(basename "$nupkg" .nupkg) + derivedVersion="${nupkgBase#${packageId}.}" + if [ -z "$toolVersion" ]; then + toolVersion="$derivedVersion" + fi + + echo "Using locally built '$packageId' version '$toolVersion' from '$nupkgDir'." + + # Create a minimal NuGet.config that only references the local nupkg directory. + # This avoids conflicts with the repo's package source mapping which blocks --add-source. + toolNugetConfig="$AGENT_TEMPDIRECTORY/helix-job-monitor-nuget.config" + printf '\n\n \n \n \n \n\n' "$nupkgDir" > "$toolNugetConfig" + + pushd "$(Build.SourcesDirectory)" > /dev/null + ./eng/common/dotnet.sh tool install \ + --tool-path "$toolPath" "$packageId" \ + --version "$toolVersion" \ + --configfile "$toolNugetConfig" + + # Locate the tool DLL so the run step can invoke it via ./eng/common/dotnet.sh exec. + toolDll=$(find "$toolPath/.store" -path '*/tools/*/any/*.deps.json' -type f | head -n 1) + toolDll="${toolDll%.deps.json}.dll" + if [ ! -f "$toolDll" ]; then + echo "Could not find tool DLL in '$toolPath/.store'." >&2 + exit 1 + fi + + echo "Tool DLL: $toolDll" + echo "##vso[task.setvariable variable=HelixJobMonitorDll]$toolDll" + displayName: Install Helix Job Monitor + + - ${{ else }}: + - bash: ./eng/common/dotnet.sh tool restore + displayName: Restore Helix Job Monitor + + - bash: | + set -euo pipefail + + toolArgs=( + --helix-base-uri '${{ parameters.helixBaseUri }}' + --polling-interval-seconds '${{ parameters.pollingIntervalSeconds }}' + --max-wait-minutes "$((${{ parameters.timeoutInMinutes }} - 2))" # Set the tool's timeout slightly lower than the Azure DevOps job timeout to allow it to exit gracefully. + --stage-name '$(System.StageName)' + ) + + organization='${{ parameters.organization }}' + repository='${{ parameters.repository }}' + + # Fall back to Azure DevOps-provided environment variables when the caller did not + # supply organization / repository explicitly. BUILD_REPOSITORY_NAME is typically + # 'owner/repo' for GitHub-backed builds. + if [ -z "$organization" ] || [ -z "$repository" ]; then + buildRepoName="${BUILD_REPOSITORY_NAME:-}" + if [ -n "$buildRepoName" ] && [[ "$buildRepoName" == */* ]]; then + repoOwner="${buildRepoName%%/*}" + repoName="${buildRepoName#*/}" + if [ -z "$organization" ]; then organization="$repoOwner"; fi + if [ -z "$repository" ]; then repository="$repoName"; fi + fi + fi + + if [ -n "$organization" ]; then toolArgs+=( --organization "$organization" ); fi + if [ -n "$repository" ]; then toolArgs+=( --repository "$repository" ); fi + + # Build.Reason and Build.SourceBranch are required to derive the Helix source filter + # the same way the Helix SDK submitter does (PR -> 'pr', internal -> 'official', + # otherwise -> 'ci'). Without these, manually-queued / scheduled / CI builds would + # be looked up under the wrong source prefix and find zero jobs. + toolArgs+=( --build-reason "$(Build.Reason)" ) + toolArgs+=( --source-branch "$(Build.SourceBranch)" ) + + if [ -n '${{ parameters.toolNupkgArtifactName }}' ]; then + # Tool was installed from a local nupkg; run the DLL via the repo-local dotnet. + export DOTNET_ROOT="$(Build.SourcesDirectory)/.dotnet" + ./eng/common/dotnet.sh exec "$(HelixJobMonitorDll)" "${toolArgs[@]}" + else + # Tool was restored from the local .config/dotnet-tools.json manifest; invoke it + # through the manifest from the repo root. + pushd "$BUILD_SOURCESDIRECTORY" > /dev/null + trap 'popd > /dev/null' EXIT + ./eng/common/dotnet.sh tool run '${{ parameters.toolCommand }}' -- "${toolArgs[@]}" + fi + displayName: Monitor Helix Jobs + env: + SYSTEM_ACCESSTOKEN: $(System.AccessToken) + HELIX_ACCESSTOKEN: ${{ parameters.helixAccessToken }} diff --git a/eng/common/core-templates/job/job.yml b/eng/common/core-templates/job/job.yml index eaed6d87e..66c7988f2 100644 --- a/eng/common/core-templates/job/job.yml +++ b/eng/common/core-templates/job/job.yml @@ -19,6 +19,8 @@ parameters: # publishing defaults artifacts: '' enableMicrobuild: false + enablePreviewMicrobuild: false + microbuildPluginVersion: 'latest' enableMicrobuildForMacAndLinux: false microbuildUseESRP: true enablePublishBuildArtifacts: false @@ -71,6 +73,8 @@ jobs: templateContext: ${{ parameters.templateContext }} variables: + - name: AllowPtrToDetectTestRunRetryFiles + value: true - ${{ if ne(parameters.enableTelemetry, 'false') }}: - name: DOTNET_CLI_TELEMETRY_PROFILE value: '$(Build.Repository.Uri)' @@ -128,6 +132,8 @@ jobs: - template: /eng/common/core-templates/steps/install-microbuild.yml parameters: enableMicrobuild: ${{ parameters.enableMicrobuild }} + enablePreviewMicrobuild: ${{ parameters.enablePreviewMicrobuild }} + microbuildPluginVersion: ${{ parameters.microbuildPluginVersion }} enableMicrobuildForMacAndLinux: ${{ parameters.enableMicrobuildForMacAndLinux }} microbuildUseESRP: ${{ parameters.microbuildUseESRP }} continueOnError: ${{ parameters.continueOnError }} @@ -150,6 +156,8 @@ jobs: - template: /eng/common/core-templates/steps/cleanup-microbuild.yml parameters: enableMicrobuild: ${{ parameters.enableMicrobuild }} + enablePreviewMicrobuild: ${{ parameters.enablePreviewMicrobuild }} + microbuildPluginVersion: ${{ parameters.microbuildPluginVersion }} enableMicrobuildForMacAndLinux: ${{ parameters.enableMicrobuildForMacAndLinux }} continueOnError: ${{ parameters.continueOnError }} diff --git a/eng/common/core-templates/job/onelocbuild.yml b/eng/common/core-templates/job/onelocbuild.yml index eefed3b66..86ea9f635 100644 --- a/eng/common/core-templates/job/onelocbuild.yml +++ b/eng/common/core-templates/job/onelocbuild.yml @@ -22,6 +22,7 @@ parameters: GitHubOrg: dotnet MirrorRepo: '' MirrorBranch: main + xLocCustomPowerShellScript: '' condition: '' JobNameSuffix: '' is1ESPipeline: '' @@ -97,6 +98,8 @@ jobs: gitHubOrganization: ${{ parameters.GitHubOrg }} mirrorRepo: ${{ parameters.MirrorRepo }} mirrorBranch: ${{ parameters.MirrorBranch }} + ${{ if ne(parameters.xLocCustomPowerShellScript, '') }}: + xLocCustomPowerShellScript: ${{ parameters.xLocCustomPowerShellScript }} condition: ${{ parameters.condition }} # Copy the locProject.json to the root of the Loc directory, then publish a pipeline artifact diff --git a/eng/common/core-templates/job/publish-build-assets.yml b/eng/common/core-templates/job/publish-build-assets.yml index 06f2eed03..700f77114 100644 --- a/eng/common/core-templates/job/publish-build-assets.yml +++ b/eng/common/core-templates/job/publish-build-assets.yml @@ -91,8 +91,8 @@ jobs: fetchDepth: 3 clean: true - - ${{ if eq(parameters.isAssetlessBuild, 'false') }}: - - ${{ if eq(parameters.publishingVersion, 3) }}: + - ${{ if eq(parameters.isAssetlessBuild, 'false') }}: + - ${{ if eq(parameters.publishingVersion, 3) }}: - task: DownloadPipelineArtifact@2 displayName: Download Asset Manifests inputs: @@ -117,7 +117,7 @@ jobs: flattenFolders: true condition: ${{ parameters.condition }} continueOnError: ${{ parameters.continueOnError }} - + - task: NuGetAuthenticate@1 # Populate internal runtime variables. @@ -125,7 +125,7 @@ jobs: ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: parameters: legacyCredential: $(dn-bot-dnceng-artifact-feeds-rw) - + - template: /eng/common/templates/steps/enable-internal-runtimes.yml - task: AzureCLI@2 @@ -145,7 +145,7 @@ jobs: condition: ${{ parameters.condition }} continueOnError: ${{ parameters.continueOnError }} - + - task: powershell@2 displayName: Create ReleaseConfigs Artifact inputs: @@ -191,7 +191,7 @@ jobs: BARBuildId: ${{ parameters.BARBuildId }} PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} is1ESPipeline: ${{ parameters.is1ESPipeline }} - + # Darc is targeting 8.0, so make sure it's installed - task: UseDotNet@2 inputs: diff --git a/eng/common/core-templates/job/renovate.yml b/eng/common/core-templates/job/renovate.yml new file mode 100644 index 000000000..ff86c80b4 --- /dev/null +++ b/eng/common/core-templates/job/renovate.yml @@ -0,0 +1,196 @@ +# -------------------------------------------------------------------------------------- +# Renovate Bot Job Template +# -------------------------------------------------------------------------------------- +# This Azure DevOps pipeline job template runs Renovate (https://docs.renovatebot.com/) +# to automatically update dependencies in a GitHub repository. +# +# Renovate scans the repository for dependency files and creates pull requests to update +# outdated dependencies based on the configuration specified in the renovateConfigPath +# parameter. +# +# Usage: +# For each product repo wanting to make use of Renovate, this template is called from +# an internal Azure DevOps pipeline, typically with a schedule trigger, to check for +# and propose dependency updates. +# +# For more info, see https://github.com/dotnet/arcade/blob/main/Documentation/Renovate.md +# -------------------------------------------------------------------------------------- + +parameters: + +# Path to the Renovate configuration file within the repository. +- name: renovateConfigPath + type: string + default: 'eng/renovate.json' + +# GitHub repository to run Renovate against, in the format 'owner/repo'. +# This could technically be any repo but convention is to target the same +# repo that contains the calling pipeline. The Renovate config file would +# be co-located with the pipeline's repo and, in most cases, the config +# file is specific to the repo being targeted. +- name: gitHubRepo + type: string + +# List of base branches to target for Renovate PRs. +# NOTE: The Renovate configuration file is always read from the branch where the +# pipeline is run, NOT from the target branches specified here. If you need different +# configurations for different branches, run the pipeline from each branch separately. +- name: baseBranches + type: object + default: + - main + +# When true, Renovate will run in dry run mode, which previews changes without creating PRs. +# See the 'Run Renovate' step log output for details of what would have been changed. +- name: dryRun + type: boolean + default: false + +# By default, Renovate will not recreate a PR for a given dependency/version pair that was +# previously closed. This allows opting in to always recreating PRs even if they were +# previously closed. +- name: forceRecreatePR + type: boolean + default: false + +# Name of the arcade repository resource in the pipeline. +# This allows repos which haven't been onboarded to Arcade to still use this +# template by checking out the repo as a resource with a custom name and pointing +# this parameter to it. +- name: arcadeRepoResource + type: string + default: self + +# Directory name for the self repo under $(Build.SourcesDirectory) in multi-checkout. +# In multi-checkout (when arcadeRepoResource != 'self'), Azure DevOps checks out the +# self repo to $(Build.SourcesDirectory)/. Set this to match the auto-generated +# directory name. Using the auto-generated name is necessary rather than explicitly +# defining a checkout path because container jobs expect repos to live under the agent's +# workspace ($(Pipeline.Workspace)). On some self-hosted setups the host path +# (e.g., /mnt/vss/_work) differs from the container path (e.g., /__w), and a custom checkout +# path can fail validation. Using the default checkout location keeps the paths consistent +# and avoids this issue. +- name: selfRepoName + type: string + default: '' +- name: arcadeRepoName + type: string + default: '' + +# Pool configuration for the job. +- name: pool + type: object + default: + name: NetCore1ESPool-Internal + image: build.azurelinux.3.amd64 + os: linux + +jobs: +- job: Renovate + displayName: Run Renovate + container: RenovateContainer + variables: + - group: dotnet-renovate-bot + # The Renovate version is automatically updated by https://github.com/dotnet/arcade/blob/main/azure-pipelines-renovate.yml. + # Changing the variable name here would require updating the name in https://github.com/dotnet/arcade/blob/main/eng/renovate.json as well. + - name: renovateVersion + value: '42' + readonly: true + - name: renovateLogFilePath + value: '$(Build.ArtifactStagingDirectory)/renovate.json' + readonly: true + - name: dryRunArg + readonly: true + ${{ if eq(parameters.dryRun, true) }}: + value: 'full' + ${{ else }}: + value: '' + - name: recreateWhenArg + readonly: true + ${{ if eq(parameters.forceRecreatePR, true) }}: + value: 'always' + ${{ else }}: + value: '' + # In multi-checkout (without custom paths), Azure DevOps places each repo under + # $(Build.SourcesDirectory)/. selfRepoName must be provided in that case. + - name: selfRepoPath + readonly: true + ${{ if eq(parameters.arcadeRepoResource, 'self') }}: + value: '$(Build.SourcesDirectory)' + ${{ else }}: + value: '$(Build.SourcesDirectory)/${{ parameters.selfRepoName }}' + - name: arcadeRepoPath + readonly: true + ${{ if eq(parameters.arcadeRepoResource, 'self') }}: + value: '$(Build.SourcesDirectory)' + ${{ else }}: + value: '$(Build.SourcesDirectory)/${{ parameters.arcadeRepoName }}' + pool: ${{ parameters.pool }} + + templateContext: + outputParentDirectory: $(Build.ArtifactStagingDirectory) + outputs: + - output: pipelineArtifact + displayName: Publish Renovate Log + condition: succeededOrFailed() + targetPath: $(Build.ArtifactStagingDirectory) + artifactName: $(Agent.JobName)_Logs_Attempt$(System.JobAttempt) + isProduction: false # logs are non-production artifacts + + steps: + - checkout: self + fetchDepth: 1 + + - ${{ if ne(parameters.arcadeRepoResource, 'self') }}: + - checkout: ${{ parameters.arcadeRepoResource }} + fetchDepth: 1 + + - script: | + renovate-config-validator $(selfRepoPath)/${{parameters.renovateConfigPath}} 2>&1 | tee /tmp/renovate-config-validator.out + validatorExit=${PIPESTATUS[0]} + if grep -q '^ WARN:' /tmp/renovate-config-validator.out; then + echo "##vso[task.logissue type=warning]Renovate config validator produced warnings." + echo "##vso[task.complete result=SucceededWithIssues]" + fi + exit $validatorExit + displayName: Validate Renovate config + env: + LOG_LEVEL: info + LOG_FILE_LEVEL: debug + LOG_FILE: $(Build.ArtifactStagingDirectory)/renovate-config-validator.json + + - script: | + . $(arcadeRepoPath)/eng/common/renovate.env + renovate 2>&1 | tee /tmp/renovate.out + renovateExit=${PIPESTATUS[0]} + if grep -q '^ WARN:' /tmp/renovate.out; then + echo "##vso[task.logissue type=warning]Renovate produced warnings." + echo "##vso[task.complete result=SucceededWithIssues]" + fi + exit $renovateExit + displayName: Run Renovate + env: + RENOVATE_FORK_TOKEN: $(BotAccount-dotnet-renovate-bot-PAT) + RENOVATE_TOKEN: $(BotAccount-dotnet-renovate-bot-PAT) + RENOVATE_REPOSITORIES: ${{parameters.gitHubRepo}} + RENOVATE_BASE_BRANCHES: ${{ convertToJson(parameters.baseBranches) }} + RENOVATE_DRY_RUN: $(dryRunArg) + RENOVATE_RECREATE_WHEN: $(recreateWhenArg) + LOG_LEVEL: info + LOG_FILE_LEVEL: debug + LOG_FILE: $(renovateLogFilePath) + RENOVATE_CONFIG_FILE: $(selfRepoPath)/${{parameters.renovateConfigPath}} + + - script: | + echo "PRs created by Renovate:" + if [ -s "$(renovateLogFilePath)" ]; then + if ! jq -r 'select(.msg == "PR created" and .pr != null) | "https://github.com/\(.repository)/pull/\(.pr)"' "$(renovateLogFilePath)" | sort -u; then + echo "##vso[task.logissue type=warning]Failed to parse Renovate log file with jq." + echo "##vso[task.complete result=SucceededWithIssues]" + fi + else + echo "##vso[task.logissue type=warning]No Renovate log file found or file is empty." + echo "##vso[task.complete result=SucceededWithIssues]" + fi + displayName: List created PRs + condition: and(succeededOrFailed(), eq('${{ parameters.dryRun }}', false)) diff --git a/eng/common/core-templates/job/source-index-stage1.yml b/eng/common/core-templates/job/source-index-stage1.yml index 76baf5c27..bac6ac5fa 100644 --- a/eng/common/core-templates/job/source-index-stage1.yml +++ b/eng/common/core-templates/job/source-index-stage1.yml @@ -15,6 +15,8 @@ jobs: variables: - name: BinlogPath value: ${{ parameters.binlogPath }} + - name: skipComponentGovernanceDetection + value: true - template: /eng/common/core-templates/variables/pool-providers.yml parameters: is1ESPipeline: ${{ parameters.is1ESPipeline }} @@ -25,10 +27,10 @@ jobs: pool: ${{ if eq(variables['System.TeamProject'], 'public') }}: name: $(DncEngPublicBuildPool) - image: windows.vs2026preview.scout.amd64.open + image: windows.vs2026.amd64.open ${{ if eq(variables['System.TeamProject'], 'internal') }}: name: $(DncEngInternalBuildPool) - image: windows.vs2026preview.scout.amd64 + image: windows.vs2026.amd64 steps: - ${{ if eq(parameters.is1ESPipeline, '') }}: diff --git a/eng/common/core-templates/jobs/codeql-build.yml b/eng/common/core-templates/jobs/codeql-build.yml deleted file mode 100644 index dbc14ac58..000000000 --- a/eng/common/core-templates/jobs/codeql-build.yml +++ /dev/null @@ -1,32 +0,0 @@ -parameters: - # See schema documentation in /Documentation/AzureDevOps/TemplateSchema.md - continueOnError: false - # Required: A collection of jobs to run - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#job - jobs: [] - # Optional: if specified, restore and use this version of Guardian instead of the default. - overrideGuardianVersion: '' - is1ESPipeline: '' - -jobs: -- template: /eng/common/core-templates/jobs/jobs.yml - parameters: - is1ESPipeline: ${{ parameters.is1ESPipeline }} - enableMicrobuild: false - enablePublishBuildArtifacts: false - enablePublishTestResults: false - enablePublishBuildAssets: false - enableTelemetry: true - - variables: - - group: Publish-Build-Assets - # The Guardian version specified in 'eng/common/sdl/packages.config'. This value must be kept in - # sync with the packages.config file. - - name: DefaultGuardianVersion - value: 0.109.0 - - name: GuardianPackagesConfigFile - value: $(System.DefaultWorkingDirectory)\eng\common\sdl\packages.config - - name: GuardianVersion - value: ${{ coalesce(parameters.overrideGuardianVersion, '$(DefaultGuardianVersion)') }} - - jobs: ${{ parameters.jobs }} - diff --git a/eng/common/core-templates/post-build/common-variables.yml b/eng/common/core-templates/post-build/common-variables.yml index d5627a994..db298ae16 100644 --- a/eng/common/core-templates/post-build/common-variables.yml +++ b/eng/common/core-templates/post-build/common-variables.yml @@ -11,8 +11,6 @@ variables: - name: MaestroApiVersion value: "2020-02-20" - - name: SourceLinkCLIVersion - value: 3.0.0 - name: SymbolToolVersion value: 1.0.1 - name: BinlogToolVersion diff --git a/eng/common/core-templates/post-build/post-build.yml b/eng/common/core-templates/post-build/post-build.yml index 905a6315e..8aa86e304 100644 --- a/eng/common/core-templates/post-build/post-build.yml +++ b/eng/common/core-templates/post-build/post-build.yml @@ -1,118 +1,108 @@ parameters: - # Which publishing infra should be used. THIS SHOULD MATCH THE VERSION ON THE BUILD MANIFEST. - # Publishing V1 is no longer supported - # Publishing V2 is no longer supported - # Publishing V3 is the default - - name: publishingInfraVersion - displayName: Which version of publishing should be used to promote the build definition? - type: number - default: 3 - values: - - 3 - - 4 - - - name: BARBuildId - displayName: BAR Build Id - type: number - default: 0 - - - name: PromoteToChannelIds - displayName: Channel to promote BARBuildId to - type: string - default: '' - - - name: enableSourceLinkValidation - displayName: Enable SourceLink validation - type: boolean - default: false - - - name: enableSigningValidation - displayName: Enable signing validation - type: boolean - default: true - - - name: enableSymbolValidation - displayName: Enable symbol validation - type: boolean - default: false - - - name: enableNugetValidation - displayName: Enable NuGet validation - type: boolean - default: true - - - name: publishInstallersAndChecksums - displayName: Publish installers and checksums - type: boolean - default: true - - - name: requireDefaultChannels - displayName: Fail the build if there are no default channel(s) registrations for the current build - type: boolean - default: false - - - name: SDLValidationParameters - type: object - default: - enable: false - publishGdn: false - continueOnError: false - params: '' - artifactNames: '' - downloadArtifacts: true - - - name: isAssetlessBuild - type: boolean - displayName: Is Assetless Build - default: false - - # These parameters let the user customize the call to sdk-task.ps1 for publishing - # symbols & general artifacts as well as for signing validation - - name: symbolPublishingAdditionalParameters - displayName: Symbol publishing additional parameters - type: string - default: '' - - - name: artifactsPublishingAdditionalParameters - displayName: Artifact publishing additional parameters - type: string - default: '' - - - name: signingValidationAdditionalParameters - displayName: Signing validation additional parameters - type: string - default: '' - - # Which stages should finish execution before post-build stages start - - name: validateDependsOn - type: object - default: - - build - - - name: publishDependsOn - type: object - default: - - Validate - - # Optional: Call asset publishing rather than running in a separate stage - - name: publishAssetsImmediately - type: boolean - default: false - - - name: is1ESPipeline - type: boolean - default: false +# Which publishing infra should be used. THIS SHOULD MATCH THE VERSION ON THE BUILD MANIFEST. +# Publishing V1 is no longer supported +# Publishing V2 is no longer supported +# Publishing V3 is the default +- name: publishingInfraVersion + displayName: Which version of publishing should be used to promote the build definition? + type: number + default: 3 + values: + - 3 + - 4 + +- name: BARBuildId + displayName: BAR Build Id + type: number + default: 0 + +- name: PromoteToChannelIds + displayName: Channel to promote BARBuildId to + type: string + default: '' + +- name: enableSourceLinkValidation + displayName: Enable SourceLink validation + type: boolean + default: false + +- name: enableSigningValidation + displayName: Enable signing validation + type: boolean + default: true + +- name: enableSymbolValidation + displayName: Enable symbol validation + type: boolean + default: false + +- name: enableNugetValidation + displayName: Enable NuGet validation + type: boolean + default: true + +- name: publishInstallersAndChecksums + displayName: Publish installers and checksums + type: boolean + default: true + +- name: requireDefaultChannels + displayName: Fail the build if there are no default channel(s) registrations for the current build + type: boolean + default: false + +- name: isAssetlessBuild + type: boolean + displayName: Is Assetless Build + default: false + +# These parameters let the user customize the call to sdk-task.ps1 for publishing +# symbols & general artifacts as well as for signing validation +- name: symbolPublishingAdditionalParameters + displayName: Symbol publishing additional parameters + type: string + default: '' + +- name: artifactsPublishingAdditionalParameters + displayName: Artifact publishing additional parameters + type: string + default: '' + +- name: signingValidationAdditionalParameters + displayName: Signing validation additional parameters + type: string + default: '' + +# Which stages should finish execution before post-build stages start +- name: validateDependsOn + type: object + default: + - build + +- name: publishDependsOn + type: object + default: + - Validate + +# Optional: Call asset publishing rather than running in a separate stage +- name: publishAssetsImmediately + type: boolean + default: false + +- name: is1ESPipeline + type: boolean + default: false stages: -- ${{ if or(eq( parameters.enableNugetValidation, 'true'), eq(parameters.enableSigningValidation, 'true'), eq(parameters.enableSourceLinkValidation, 'true'), eq(parameters.SDLValidationParameters.enable, 'true')) }}: +- ${{ if or(eq( parameters.enableNugetValidation, 'true'), eq(parameters.enableSigningValidation, 'true'), eq(parameters.enableSourceLinkValidation, 'true')) }}: - stage: Validate dependsOn: ${{ parameters.validateDependsOn }} displayName: Validate Build Assets variables: - - template: /eng/common/core-templates/post-build/common-variables.yml - - template: /eng/common/core-templates/variables/pool-providers.yml - parameters: - is1ESPipeline: ${{ parameters.is1ESPipeline }} + - template: /eng/common/core-templates/post-build/common-variables.yml + - template: /eng/common/core-templates/variables/pool-providers.yml + parameters: + is1ESPipeline: ${{ parameters.is1ESPipeline }} jobs: - job: displayName: NuGet Validation @@ -128,49 +118,49 @@ stages: ${{ else }}: ${{ if eq(parameters.is1ESPipeline, true) }}: name: $(DncEngInternalBuildPool) - image: windows.vs2026preview.scout.amd64 + image: windows.vs2026.amd64 os: windows ${{ else }}: name: $(DncEngInternalBuildPool) - demands: ImageOverride -equals windows.vs2026preview.scout.amd64 + demands: ImageOverride -equals windows.vs2026.amd64 steps: - - template: /eng/common/core-templates/post-build/setup-maestro-vars.yml - parameters: - BARBuildId: ${{ parameters.BARBuildId }} - PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} - is1ESPipeline: ${{ parameters.is1ESPipeline }} - - - ${{ if ne(parameters.publishingInfraVersion, 4) }}: - - task: DownloadBuildArtifacts@0 - displayName: Download Package Artifacts - inputs: - buildType: specific - buildVersionToDownload: specific - project: $(AzDOProjectName) - pipeline: $(AzDOPipelineId) - buildId: $(AzDOBuildId) - artifactName: PackageArtifacts - checkDownloadedFiles: true - - ${{ if eq(parameters.publishingInfraVersion, 4) }}: - - task: DownloadPipelineArtifact@2 - displayName: Download Pipeline Artifacts (V4) - inputs: - itemPattern: '*/packages/**/*.nupkg' - targetPath: '$(Build.ArtifactStagingDirectory)/PipelineArtifactsDownload' - - task: CopyFiles@2 - displayName: Flatten packages to PackageArtifacts - inputs: - SourceFolder: '$(Build.ArtifactStagingDirectory)/PipelineArtifactsDownload' - Contents: '**/*.nupkg' - TargetFolder: '$(Build.ArtifactStagingDirectory)/PackageArtifacts' - flattenFolders: true - - - task: PowerShell@2 - displayName: Validate + - template: /eng/common/core-templates/post-build/setup-maestro-vars.yml + parameters: + BARBuildId: ${{ parameters.BARBuildId }} + PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} + is1ESPipeline: ${{ parameters.is1ESPipeline }} + + - ${{ if ne(parameters.publishingInfraVersion, 4) }}: + - task: DownloadBuildArtifacts@0 + displayName: Download Package Artifacts + inputs: + buildType: specific + buildVersionToDownload: specific + project: $(AzDOProjectName) + pipeline: $(AzDOPipelineId) + buildId: $(AzDOBuildId) + artifactName: PackageArtifacts + checkDownloadedFiles: true + - ${{ if eq(parameters.publishingInfraVersion, 4) }}: + - task: DownloadPipelineArtifact@2 + displayName: Download Pipeline Artifacts (V4) + inputs: + itemPattern: '*/packages/**/*.nupkg' + targetPath: '$(Build.ArtifactStagingDirectory)/PipelineArtifactsDownload' + - task: CopyFiles@2 + displayName: Flatten packages to PackageArtifacts inputs: - filePath: $(System.DefaultWorkingDirectory)/eng/common/post-build/nuget-validation.ps1 - arguments: -PackagesPath $(Build.ArtifactStagingDirectory)/PackageArtifacts/ + SourceFolder: '$(Build.ArtifactStagingDirectory)/PipelineArtifactsDownload' + Contents: '**/*.nupkg' + TargetFolder: '$(Build.ArtifactStagingDirectory)/PackageArtifacts' + flattenFolders: true + + - task: PowerShell@2 + displayName: Validate + inputs: + filePath: $(System.DefaultWorkingDirectory)/eng/common/post-build/nuget-validation.ps1 + arguments: -PackagesPath $(Build.ArtifactStagingDirectory)/PackageArtifacts/ - job: displayName: Signing Validation @@ -184,143 +174,96 @@ stages: os: windows # If it's not devdiv, it's dnceng ${{ else }}: - ${{ if eq(parameters.is1ESPipeline, true) }}: + ${{ if eq(parameters.is1ESPipeline, true) }}: name: $(DncEngInternalBuildPool) image: windows.vs2026.amd64 os: windows ${{ else }}: name: $(DncEngInternalBuildPool) - demands: ImageOverride -equals windows.vs2026preview.scout.amd64 + demands: ImageOverride -equals windows.vs2026.amd64 steps: - - template: /eng/common/core-templates/post-build/setup-maestro-vars.yml - parameters: - BARBuildId: ${{ parameters.BARBuildId }} - PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} - is1ESPipeline: ${{ parameters.is1ESPipeline }} - - - ${{ if ne(parameters.publishingInfraVersion, 4) }}: - - task: DownloadBuildArtifacts@0 - displayName: Download Package Artifacts - inputs: - buildType: specific - buildVersionToDownload: specific - project: $(AzDOProjectName) - pipeline: $(AzDOPipelineId) - buildId: $(AzDOBuildId) - artifactName: PackageArtifacts - checkDownloadedFiles: true - - ${{ if eq(parameters.publishingInfraVersion, 4) }}: - - task: DownloadPipelineArtifact@2 - displayName: Download Pipeline Artifacts (V4) - inputs: - itemPattern: '*/packages/**/*.nupkg' - targetPath: '$(Build.ArtifactStagingDirectory)/PipelineArtifactsDownload' - - task: CopyFiles@2 - displayName: Flatten packages to PackageArtifacts - inputs: - SourceFolder: '$(Build.ArtifactStagingDirectory)/PipelineArtifactsDownload' - Contents: '**/*.nupkg' - TargetFolder: '$(Build.ArtifactStagingDirectory)/PackageArtifacts' - flattenFolders: true - - # This is necessary whenever we want to publish/restore to an AzDO private feed - # Since sdk-task.ps1 tries to restore packages we need to do this authentication here - # otherwise it'll complain about accessing a private feed. - - task: NuGetAuthenticate@1 - displayName: 'Authenticate to AzDO Feeds' - - # Signing validation will optionally work with the buildmanifest file which is downloaded from - # Azure DevOps above. - - task: PowerShell@2 - displayName: Validate - inputs: - filePath: eng\common\sdk-task.ps1 - arguments: -task SigningValidation -restore -msbuildEngine vs - /p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts' - /p:SignCheckExclusionsFile='$(System.DefaultWorkingDirectory)/eng/SignCheckExclusionsFile.txt' - ${{ parameters.signingValidationAdditionalParameters }} - - - template: /eng/common/core-templates/steps/publish-logs.yml - parameters: - is1ESPipeline: ${{ parameters.is1ESPipeline }} - StageLabel: 'Validation' - JobLabel: 'Signing' - BinlogToolVersion: $(BinlogToolVersion) + - template: /eng/common/core-templates/post-build/setup-maestro-vars.yml + parameters: + BARBuildId: ${{ parameters.BARBuildId }} + PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} + is1ESPipeline: ${{ parameters.is1ESPipeline }} - - job: - displayName: SourceLink Validation - condition: eq( ${{ parameters.enableSourceLinkValidation }}, 'true') - pool: - # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) - ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: - name: AzurePipelines-EO - image: 1ESPT-Windows2025 - demands: Cmd - os: windows - # If it's not devdiv, it's dnceng - ${{ else }}: - ${{ if eq(parameters.is1ESPipeline, true) }}: - name: $(DncEngInternalBuildPool) - image: windows.vs2026.amd64 - os: windows - ${{ else }}: - name: $(DncEngInternalBuildPool) - demands: ImageOverride -equals windows.vs2026preview.scout.amd64 - steps: - - template: /eng/common/core-templates/post-build/setup-maestro-vars.yml - parameters: - BARBuildId: ${{ parameters.BARBuildId }} - PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} - is1ESPipeline: ${{ parameters.is1ESPipeline }} - - - ${{ if ne(parameters.publishingInfraVersion, 4) }}: - - task: DownloadBuildArtifacts@0 - displayName: Download Blob Artifacts - inputs: - buildType: specific - buildVersionToDownload: specific - project: $(AzDOProjectName) - pipeline: $(AzDOPipelineId) - buildId: $(AzDOBuildId) - artifactName: BlobArtifacts - checkDownloadedFiles: true - - ${{ if eq(parameters.publishingInfraVersion, 4) }}: - - task: DownloadPipelineArtifact@2 - displayName: Download Pipeline Artifacts (V4) - inputs: - itemPattern: '*/assets/**' - targetPath: '$(Build.ArtifactStagingDirectory)/PipelineArtifactsDownload' - - task: CopyFiles@2 - displayName: Flatten assets to BlobArtifacts - inputs: - SourceFolder: '$(Build.ArtifactStagingDirectory)/PipelineArtifactsDownload' - Contents: '**/*' - TargetFolder: '$(Build.ArtifactStagingDirectory)/BlobArtifacts' - flattenFolders: true - - - task: PowerShell@2 - displayName: Validate + - ${{ if ne(parameters.publishingInfraVersion, 4) }}: + - task: DownloadBuildArtifacts@0 + displayName: Download Package Artifacts + inputs: + buildType: specific + buildVersionToDownload: specific + project: $(AzDOProjectName) + pipeline: $(AzDOPipelineId) + buildId: $(AzDOBuildId) + artifactName: PackageArtifacts + checkDownloadedFiles: true + - ${{ if eq(parameters.publishingInfraVersion, 4) }}: + - task: DownloadPipelineArtifact@2 + displayName: Download Pipeline Artifacts (V4) + inputs: + itemPattern: '*/packages/**/*.nupkg' + targetPath: '$(Build.ArtifactStagingDirectory)/PipelineArtifactsDownload' + - task: CopyFiles@2 + displayName: Flatten packages to PackageArtifacts + inputs: + SourceFolder: '$(Build.ArtifactStagingDirectory)/PipelineArtifactsDownload' + Contents: '**/*.nupkg' + TargetFolder: '$(Build.ArtifactStagingDirectory)/PackageArtifacts' + flattenFolders: true + + # This is necessary whenever we want to publish/restore to an AzDO private feed + # Since sdk-task.ps1 tries to restore packages we need to do this authentication here + # otherwise it'll complain about accessing a private feed. + - task: NuGetAuthenticate@1 + displayName: 'Authenticate to AzDO Feeds' + + # Signing validation will optionally work with the buildmanifest file which is downloaded from + # Azure DevOps above. + - task: PowerShell@2 + displayName: Validate + inputs: + filePath: eng\common\sdk-task.ps1 + arguments: -task SigningValidation -restore -msbuildEngine dotnet + /p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts' + /p:SignCheckExclusionsFile='$(System.DefaultWorkingDirectory)/eng/SignCheckExclusionsFile.txt' + ${{ parameters.signingValidationAdditionalParameters }} + + - template: /eng/common/core-templates/steps/publish-logs.yml + parameters: + is1ESPipeline: ${{ parameters.is1ESPipeline }} + StageLabel: 'Validation' + JobLabel: 'Signing' + BinlogToolVersion: $(BinlogToolVersion) + + # SourceLink validation has been removed — the underlying CLI tool + # (targeting netcoreapp2.1) has not functioned for years. + # The enableSourceLinkValidation parameter is kept but ignored so + # existing pipelines that pass it are not broken. + # See https://github.com/dotnet/arcade/issues/16647 + - ${{ if eq(parameters.enableSourceLinkValidation, 'true') }}: + - job: + displayName: 'SourceLink Validation Removed - please remove enableSourceLinkValidation from your pipeline' + pool: server + steps: + - task: Delay@1 + displayName: 'Warning: SourceLink validation removed (see https://github.com/dotnet/arcade/issues/16647)' inputs: - filePath: $(System.DefaultWorkingDirectory)/eng/common/post-build/sourcelink-validation.ps1 - arguments: -InputPath $(Build.ArtifactStagingDirectory)/BlobArtifacts/ - -ExtractPath $(Agent.BuildDirectory)/Extract/ - -GHRepoName $(Build.Repository.Name) - -GHCommit $(Build.SourceVersion) - -SourcelinkCliVersion $(SourceLinkCLIVersion) - continueOnError: true + delayForMinutes: '0' - ${{ if ne(parameters.publishAssetsImmediately, 'true') }}: - stage: publish_using_darc - ${{ if or(eq(parameters.enableNugetValidation, 'true'), eq(parameters.enableSigningValidation, 'true'), eq(parameters.enableSourceLinkValidation, 'true'), eq(parameters.SDLValidationParameters.enable, 'true')) }}: + ${{ if or(eq(parameters.enableNugetValidation, 'true'), eq(parameters.enableSigningValidation, 'true'), eq(parameters.enableSourceLinkValidation, 'true')) }}: dependsOn: ${{ parameters.publishDependsOn }} ${{ else }}: dependsOn: ${{ parameters.validateDependsOn }} displayName: Publish using Darc variables: - - template: /eng/common/core-templates/post-build/common-variables.yml - - template: /eng/common/core-templates/variables/pool-providers.yml - parameters: - is1ESPipeline: ${{ parameters.is1ESPipeline }} + - template: /eng/common/core-templates/post-build/common-variables.yml + - template: /eng/common/core-templates/variables/pool-providers.yml + parameters: + is1ESPipeline: ${{ parameters.is1ESPipeline }} jobs: - job: displayName: Publish Using Darc @@ -334,7 +277,7 @@ stages: os: windows # If it's not devdiv, it's dnceng ${{ else }}: - ${{ if eq(parameters.is1ESPipeline, true) }}: + ${{ if eq(parameters.is1ESPipeline, true) }}: name: NetCore1ESPool-Publishing-Internal image: windows.vs2026.amd64 os: windows @@ -342,34 +285,33 @@ stages: name: NetCore1ESPool-Publishing-Internal demands: ImageOverride -equals windows.vs2026.amd64 steps: - - template: /eng/common/core-templates/post-build/setup-maestro-vars.yml - parameters: - BARBuildId: ${{ parameters.BARBuildId }} - PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} - is1ESPipeline: ${{ parameters.is1ESPipeline }} - - - task: NuGetAuthenticate@1 - - # Populate internal runtime variables. - - template: /eng/common/templates/steps/enable-internal-sources.yml - parameters: - legacyCredential: $(dn-bot-dnceng-artifact-feeds-rw) - - - template: /eng/common/templates/steps/enable-internal-runtimes.yml + - template: /eng/common/core-templates/post-build/setup-maestro-vars.yml + parameters: + BARBuildId: ${{ parameters.BARBuildId }} + PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} + is1ESPipeline: ${{ parameters.is1ESPipeline }} - # Darc is targeting 8.0, so make sure it's installed - - task: UseDotNet@2 - inputs: - version: 8.0.x + - task: NuGetAuthenticate@1 - - task: AzureCLI@2 - displayName: Publish Using Darc - inputs: - azureSubscription: "Darc: Maestro Production" - scriptType: ps - scriptLocation: scriptPath - scriptPath: $(System.DefaultWorkingDirectory)/eng/common/post-build/publish-using-darc.ps1 - arguments: > + # Populate internal runtime variables. + - template: /eng/common/templates/steps/enable-internal-sources.yml + parameters: + legacyCredential: $(dn-bot-dnceng-artifact-feeds-rw) + + - template: /eng/common/templates/steps/enable-internal-runtimes.yml + + - task: UseDotNet@2 + inputs: + version: 8.0.x + + - task: AzureCLI@2 + displayName: Publish Using Darc + inputs: + azureSubscription: "Darc: Maestro Production" + scriptType: ps + scriptLocation: scriptPath + scriptPath: $(System.DefaultWorkingDirectory)/eng/common/post-build/publish-using-darc.ps1 + arguments: > -BuildId $(BARBuildId) -PublishingInfraVersion 3 -AzdoToken '$(System.AccessToken)' diff --git a/eng/common/core-templates/stages/renovate.yml b/eng/common/core-templates/stages/renovate.yml new file mode 100644 index 000000000..edab28182 --- /dev/null +++ b/eng/common/core-templates/stages/renovate.yml @@ -0,0 +1,111 @@ +# -------------------------------------------------------------------------------------- +# Renovate Pipeline Template +# -------------------------------------------------------------------------------------- +# This template provides a complete reusable pipeline definition for running Renovate +# in a 1ES Official pipeline. Pipelines can extend from this template and only need +# to pass the Renovate job parameters. +# +# For more info, see https://github.com/dotnet/arcade/blob/main/Documentation/Renovate.md +# -------------------------------------------------------------------------------------- + +parameters: + +# Path to the Renovate configuration file within the repository. +- name: renovateConfigPath + type: string + default: 'eng/renovate.json' + +# GitHub repository to run Renovate against, in the format 'owner/repo'. +- name: gitHubRepo + type: string + +# List of base branches to target for Renovate PRs. +- name: baseBranches + type: object + default: + - main + +# When true, Renovate will run in dry run mode. +- name: dryRun + type: boolean + default: false + +# When true, Renovate will recreate PRs even if they were previously closed. +- name: forceRecreatePR + type: boolean + default: false + +# Name of the arcade repository resource in the pipeline. +# This allows repos which haven't been onboarded to Arcade to still use this +# template by checking out the repo as a resource with a custom name and pointing +# this parameter to it. +- name: arcadeRepoResource + type: string + default: 'self' + +- name: selfRepoName + type: string + default: '' +- name: arcadeRepoName + type: string + default: '' + +# Pool configuration for the pipeline. +- name: pool + type: object + default: + name: NetCore1ESPool-Internal + image: build.azurelinux.3.amd64 + os: linux + +# Renovate version used in the container image tag. +- name: renovateVersion + default: 43 + type: number + +# Pool configuration for SDL analysis. +- name: sdlPool + type: object + default: + name: NetCore1ESPool-Internal + image: windows.vs2026.amd64 + os: windows + +resources: + repositories: + - repository: 1ESPipelineTemplates + type: git + name: 1ESPipelineTemplates/1ESPipelineTemplates + ref: refs/tags/release + +extends: + template: v1/1ES.Official.PipelineTemplate.yml@1ESPipelineTemplates + parameters: + pool: ${{ parameters.pool }} + sdl: + sourceAnalysisPool: ${{ parameters.sdlPool }} + # When repos that aren't onboarded to Arcade use this template, they set the + # arcadeRepoResource parameter to point to their Arcade repo resource. In that case, + # Aracde will be excluded from SDL analysis. + ${{ if ne(parameters.arcadeRepoResource, 'self') }}: + sourceRepositoriesToScan: + exclude: + - repository: ${{ parameters.arcadeRepoResource }} + containers: + RenovateContainer: + image: mcr.microsoft.com/dotnet-buildtools/prereqs:azurelinux-3.0-renovate-${{ parameters.renovateVersion }}-amd64 + stages: + - stage: Renovate + displayName: Run Renovate + jobs: + - template: /eng/common/core-templates/job/renovate.yml@${{ parameters.arcadeRepoResource }} + parameters: + renovateConfigPath: ${{ parameters.renovateConfigPath }} + gitHubRepo: ${{ parameters.gitHubRepo }} + baseBranches: ${{ parameters.baseBranches }} + dryRun: ${{ parameters.dryRun }} + forceRecreatePR: ${{ parameters.forceRecreatePR }} + pool: ${{ parameters.pool }} + arcadeRepoResource: ${{ parameters.arcadeRepoResource }} + selfRepoName: ${{ parameters.selfRepoName }} + arcadeRepoName: ${{ parameters.arcadeRepoName }} diff --git a/eng/common/core-templates/steps/install-microbuild-impl.yml b/eng/common/core-templates/steps/install-microbuild-impl.yml new file mode 100644 index 000000000..da22beb3f --- /dev/null +++ b/eng/common/core-templates/steps/install-microbuild-impl.yml @@ -0,0 +1,34 @@ +parameters: + - name: microbuildTaskInputs + type: object + default: {} + + - name: microbuildEnv + type: object + default: {} + + - name: enablePreviewMicrobuild + type: boolean + default: false + + - name: condition + type: string + + - name: continueOnError + type: boolean + +steps: +- ${{ if eq(parameters.enablePreviewMicrobuild, true) }}: + - task: MicroBuildSigningPluginPreview@4 + displayName: Install Preview MicroBuild plugin + inputs: ${{ parameters.microbuildTaskInputs }} + env: ${{ parameters.microbuildEnv }} + continueOnError: ${{ parameters.continueOnError }} + condition: ${{ parameters.condition }} +- ${{ else }}: + - task: MicroBuildSigningPlugin@4 + displayName: Install MicroBuild plugin + inputs: ${{ parameters.microbuildTaskInputs }} + env: ${{ parameters.microbuildEnv }} + continueOnError: ${{ parameters.continueOnError }} + condition: ${{ parameters.condition }} diff --git a/eng/common/core-templates/steps/install-microbuild.yml b/eng/common/core-templates/steps/install-microbuild.yml index 553fce66b..76a54e157 100644 --- a/eng/common/core-templates/steps/install-microbuild.yml +++ b/eng/common/core-templates/steps/install-microbuild.yml @@ -4,6 +4,8 @@ parameters: # Enable install tasks for MicroBuild on Mac and Linux # Will be ignored if 'enableMicrobuild' is false or 'Agent.Os' is 'Windows_NT' enableMicrobuildForMacAndLinux: false + # Enable preview version of MB signing plugin + enablePreviewMicrobuild: false # Determines whether the ESRP service connection information should be passed to the signing plugin. # This overlaps with _SignType to some degree. We only need the service connection for real signing. # It's important that the service connection not be passed to the MicroBuildSigningPlugin task in this place. @@ -13,6 +15,8 @@ parameters: microbuildUseESRP: true # Microbuild installation directory microBuildOutputFolder: $(Agent.TempDirectory)/MicroBuild + # Microbuild version + microbuildPluginVersion: 'latest' continueOnError: false @@ -69,42 +73,46 @@ steps: # YAML expansion, and Windows vs. Linux/Mac uses different service connections. However, # we can avoid including the MB install step if not enabled at all. This avoids a bunch of # extra pipeline authorizations, since most pipelines do not sign on non-Windows. - - task: MicroBuildSigningPlugin@4 - displayName: Install MicroBuild plugin (Windows) - inputs: - signType: $(_SignType) - zipSources: false - feedSource: https://dnceng.pkgs.visualstudio.com/_packaging/MicroBuildToolset/nuget/v3/index.json - ${{ if eq(parameters.microbuildUseESRP, true) }}: - ConnectedServiceName: 'MicroBuild Signing Task (DevDiv)' - ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: - ConnectedPMEServiceName: 6cc74545-d7b9-4050-9dfa-ebefcc8961ea - ${{ else }}: - ConnectedPMEServiceName: 248d384a-b39b-46e3-8ad5-c2c210d5e7ca - env: - TeamName: $(_TeamName) - MicroBuildOutputFolderOverride: ${{ parameters.microBuildOutputFolder }} - SYSTEM_ACCESSTOKEN: $(System.AccessToken) - continueOnError: ${{ parameters.continueOnError }} - condition: and(succeeded(), eq(variables['Agent.Os'], 'Windows_NT'), in(variables['_SignType'], 'real', 'test')) - - - ${{ if eq(parameters.enableMicrobuildForMacAndLinux, true) }}: - - task: MicroBuildSigningPlugin@4 - displayName: Install MicroBuild plugin (non-Windows) - inputs: + - template: /eng/common/core-templates/steps/install-microbuild-impl.yml + parameters: + enablePreviewMicrobuild: ${{ parameters.enablePreviewMicrobuild }} + microbuildTaskInputs: signType: $(_SignType) zipSources: false feedSource: https://dnceng.pkgs.visualstudio.com/_packaging/MicroBuildToolset/nuget/v3/index.json - workingDirectory: ${{ parameters.microBuildOutputFolder }} + version: ${{ parameters.microbuildPluginVersion }} ${{ if eq(parameters.microbuildUseESRP, true) }}: ConnectedServiceName: 'MicroBuild Signing Task (DevDiv)' ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: - ConnectedPMEServiceName: beb8cb23-b303-4c95-ab26-9e44bc958d39 + ConnectedPMEServiceName: 6cc74545-d7b9-4050-9dfa-ebefcc8961ea ${{ else }}: - ConnectedPMEServiceName: c24de2a5-cc7a-493d-95e4-8e5ff5cad2bc - env: + ConnectedPMEServiceName: 248d384a-b39b-46e3-8ad5-c2c210d5e7ca + microbuildEnv: TeamName: $(_TeamName) MicroBuildOutputFolderOverride: ${{ parameters.microBuildOutputFolder }} SYSTEM_ACCESSTOKEN: $(System.AccessToken) continueOnError: ${{ parameters.continueOnError }} - condition: and(succeeded(), ne(variables['Agent.Os'], 'Windows_NT'), eq(variables['_SignType'], 'real')) + condition: and(succeeded(), eq(variables['Agent.Os'], 'Windows_NT'), in(variables['_SignType'], 'real', 'test')) + + - ${{ if eq(parameters.enableMicrobuildForMacAndLinux, true) }}: + - template: /eng/common/core-templates/steps/install-microbuild-impl.yml + parameters: + enablePreviewMicrobuild: ${{ parameters.enablePreviewMicrobuild }} + microbuildTaskInputs: + signType: $(_SignType) + zipSources: false + feedSource: https://dnceng.pkgs.visualstudio.com/_packaging/MicroBuildToolset/nuget/v3/index.json + version: ${{ parameters.microbuildPluginVersion }} + workingDirectory: ${{ parameters.microBuildOutputFolder }} + ${{ if eq(parameters.microbuildUseESRP, true) }}: + ConnectedServiceName: 'MicroBuild Signing Task (DevDiv)' + ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: + ConnectedPMEServiceName: beb8cb23-b303-4c95-ab26-9e44bc958d39 + ${{ else }}: + ConnectedPMEServiceName: c24de2a5-cc7a-493d-95e4-8e5ff5cad2bc + microbuildEnv: + TeamName: $(_TeamName) + MicroBuildOutputFolderOverride: ${{ parameters.microBuildOutputFolder }} + SYSTEM_ACCESSTOKEN: $(System.AccessToken) + continueOnError: ${{ parameters.continueOnError }} + condition: and(succeeded(), ne(variables['Agent.Os'], 'Windows_NT'), eq(variables['_SignType'], 'real')) diff --git a/eng/common/core-templates/steps/publish-logs.yml b/eng/common/core-templates/steps/publish-logs.yml index 4eed0312b..2731e48cc 100644 --- a/eng/common/core-templates/steps/publish-logs.yml +++ b/eng/common/core-templates/steps/publish-logs.yml @@ -33,10 +33,6 @@ steps: '$(publishing-dnceng-devdiv-code-r-build-re)' '$(dn-bot-all-orgs-artifact-feeds-rw)' '$(akams-client-id)' - '$(microsoft-symbol-server-pat)' - '$(symweb-symbol-server-pat)' - '$(dnceng-symbol-server-pat)' - '$(dn-bot-all-orgs-build-rw-code-rw)' '$(System.AccessToken)' ${{parameters.CustomSensitiveDataList}} continueOnError: true @@ -61,3 +57,4 @@ steps: condition: always() retryCountOnTaskFailure: 10 # for any files being locked isProduction: false # logs are non-production artifacts + diff --git a/eng/common/core-templates/steps/source-build.yml b/eng/common/core-templates/steps/source-build.yml index 09ae5cd73..b75f59c42 100644 --- a/eng/common/core-templates/steps/source-build.yml +++ b/eng/common/core-templates/steps/source-build.yml @@ -24,7 +24,7 @@ steps: # in the default public locations. internalRuntimeDownloadArgs= if [ '$(dotnetbuilds-internal-container-read-token-base64)' != '$''(dotnetbuilds-internal-container-read-token-base64)' ]; then - internalRuntimeDownloadArgs='/p:DotNetRuntimeSourceFeed=https://ci.dot.net/internal /p:DotNetRuntimeSourceFeedKey=$(dotnetbuilds-internal-container-read-token-base64) --runtimesourcefeed https://ci.dot.net/internal --runtimesourcefeedkey '$(dotnetbuilds-internal-container-read-token-base64)'' + internalRuntimeDownloadArgs='/p:DotNetRuntimeSourceFeed=https://ci.dot.net/internal /p:DotNetRuntimeSourceFeedKey=$(dotnetbuilds-internal-container-read-token-base64) --runtimesourcefeed https://ci.dot.net/internal --runtimesourcefeedkey $(dotnetbuilds-internal-container-read-token-base64)' fi buildConfig=Release diff --git a/eng/common/core-templates/steps/source-index-stage1-publish.yml b/eng/common/core-templates/steps/source-index-stage1-publish.yml index e9a694afa..fdca62235 100644 --- a/eng/common/core-templates/steps/source-index-stage1-publish.yml +++ b/eng/common/core-templates/steps/source-index-stage1-publish.yml @@ -1,21 +1,21 @@ parameters: - sourceIndexUploadPackageVersion: 2.0.0-20250818.1 - sourceIndexProcessBinlogPackageVersion: 1.0.1-20250818.1 + sourceIndexUploadPackageVersion: 2.0.0-20260521.2 + sourceIndexProcessBinlogPackageVersion: 1.0.1-20260521.2 sourceIndexPackageSource: https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json binlogPath: artifacts/log/Debug/Build.binlog steps: - task: UseDotNet@2 - displayName: "Source Index: Use .NET 9 SDK" + displayName: "Source Index: Use .NET 10 SDK" inputs: packageType: sdk - version: 9.0.x + version: 10.0.x installationPath: $(Agent.TempDirectory)/dotnet workingDirectory: $(Agent.TempDirectory) - script: | - $(Agent.TempDirectory)/dotnet/dotnet tool install BinLogToSln --version ${{parameters.sourceIndexProcessBinlogPackageVersion}} --add-source ${{parameters.SourceIndexPackageSource}} --tool-path $(Agent.TempDirectory)/.source-index/tools - $(Agent.TempDirectory)/dotnet/dotnet tool install UploadIndexStage1 --version ${{parameters.sourceIndexUploadPackageVersion}} --add-source ${{parameters.SourceIndexPackageSource}} --tool-path $(Agent.TempDirectory)/.source-index/tools + $(Agent.TempDirectory)/dotnet/dotnet tool install BinLogToSln --version ${{parameters.sourceIndexProcessBinlogPackageVersion}} --source ${{parameters.sourceIndexPackageSource}} --tool-path $(Agent.TempDirectory)/.source-index/tools + $(Agent.TempDirectory)/dotnet/dotnet tool install UploadIndexStage1 --version ${{parameters.sourceIndexUploadPackageVersion}} --source ${{parameters.sourceIndexPackageSource}} --tool-path $(Agent.TempDirectory)/.source-index/tools displayName: "Source Index: Download netsourceindex Tools" # Set working directory to temp directory so 'dotnet' doesn't try to use global.json and use the repo's sdk. workingDirectory: $(Agent.TempDirectory) diff --git a/eng/common/cross/build-rootfs.sh b/eng/common/cross/build-rootfs.sh index 8abfb71f7..f06854ccc 100755 --- a/eng/common/cross/build-rootfs.sh +++ b/eng/common/cross/build-rootfs.sh @@ -5,10 +5,11 @@ set -e usage() { echo "Usage: $0 [BuildArch] [CodeName] [lldbx.y] [llvmx[.y]] [--skipunmount] --rootfsdir ]" - echo "BuildArch can be: arm(default), arm64, armel, armv6, loongarch64, ppc64le, riscv64, s390x, x64, x86" + echo "BuildArch can be: arm(default), arm64, loongarch64, ppc64le, riscv64, s390x, x64, x86" echo "CodeName - optional, Code name for Linux, can be: xenial(default), zesty, bionic, alpine" echo " for alpine can be specified with version: alpineX.YY or alpineedge" echo " for FreeBSD can be: freebsd13, freebsd14" + echo " for OpenBSD can be: openbsd" echo " for illumos can be: illumos" echo " for Haiku can be: haiku." echo "lldbx.y - optional, LLDB version, can be: lldb3.9(default), lldb4.0, lldb5.0, lldb6.0 no-lldb. Ignored for alpine and FreeBSD" @@ -27,6 +28,8 @@ __BuildArch=arm __AlpineArch=armv7 __FreeBSDArch=arm __FreeBSDMachineArch=armv7 +__OpenBSDArch=arm +__OpenBSDMachineArch=armv7 __IllumosArch=arm7 __HaikuArch=arm __QEMUArch=arm @@ -72,8 +75,8 @@ __AlpinePackages+=" krb5-dev" __AlpinePackages+=" openssl-dev" __AlpinePackages+=" zlib-dev" -__FreeBSDBase="13.4-RELEASE" -__FreeBSDPkg="1.21.3" +__FreeBSDBase="13.5-RELEASE" +__FreeBSDPkg="2.7.5" __FreeBSDABI="13" __FreeBSDPackages="libunwind" __FreeBSDPackages+=" icu" @@ -82,6 +85,12 @@ __FreeBSDPackages+=" openssl" __FreeBSDPackages+=" krb5" __FreeBSDPackages+=" terminfo-db" +__OpenBSDVersion="7.8" +__OpenBSDPackages="heimdal-libs" +__OpenBSDPackages+=" icu4c" +__OpenBSDPackages+=" inotify-tools" +__OpenBSDPackages+=" openssl" + __IllumosPackages="icu" __IllumosPackages+=" mit-krb5" __IllumosPackages+=" openssl" @@ -130,7 +139,6 @@ __AlpineKeys=' 616db30d:MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAnpUpyWDWjlUk3smlWeA0\nlIMW+oJ38t92CRLHH3IqRhyECBRW0d0aRGtq7TY8PmxjjvBZrxTNDpJT6KUk4LRm\na6A6IuAI7QnNK8SJqM0DLzlpygd7GJf8ZL9SoHSH+gFsYF67Cpooz/YDqWrlN7Vw\ntO00s0B+eXy+PCXYU7VSfuWFGK8TGEv6HfGMALLjhqMManyvfp8hz3ubN1rK3c8C\nUS/ilRh1qckdbtPvoDPhSbTDmfU1g/EfRSIEXBrIMLg9ka/XB9PvWRrekrppnQzP\nhP9YE3x/wbFc5QqQWiRCYyQl/rgIMOXvIxhkfe8H5n1Et4VAorkpEAXdsfN8KSVv\nLSMazVlLp9GYq5SUpqYX3KnxdWBgN7BJoZ4sltsTpHQ/34SXWfu3UmyUveWj7wp0\nx9hwsPirVI00EEea9AbP7NM2rAyu6ukcm4m6ATd2DZJIViq2es6m60AE6SMCmrQF\nwmk4H/kdQgeAELVfGOm2VyJ3z69fQuywz7xu27S6zTKi05Qlnohxol4wVb6OB7qG\nLPRtK9ObgzRo/OPumyXqlzAi/Yvyd1ZQk8labZps3e16bQp8+pVPiumWioMFJDWV\nGZjCmyMSU8V6MB6njbgLHoyg2LCukCAeSjbPGGGYhnKLm1AKSoJh3IpZuqcKCk5C\n8CM1S15HxV78s9dFntEqIokCAwEAAQ== 66ba20fe:MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAtfB12w4ZgqsXWZDfUAV/\n6Y4aHUKIu3q4SXrNZ7CXF9nXoAVYrS7NAxJdAodsY3vPCN0g5O8DFXR+390LdOuQ\n+HsGKCc1k5tX5ZXld37EZNTNSbR0k+NKhd9h6X3u6wqPOx7SIKxwAQR8qeeFq4pP\nrt9GAGlxtuYgzIIcKJPwE0dZlcBCg+GnptCUZXp/38BP1eYC+xTXSL6Muq1etYfg\nodXdb7Yl+2h1IHuOwo5rjgY5kpY7GcAs8AjGk3lDD/av60OTYccknH0NCVSmPoXK\nvrxDBOn0LQRNBLcAfnTKgHrzy0Q5h4TNkkyTgxkoQw5ObDk9nnabTxql732yy9BY\ns+hM9+dSFO1HKeVXreYSA2n1ndF18YAvAumzgyqzB7I4pMHXq1kC/8bONMJxwSkS\nYm6CoXKyavp7RqGMyeVpRC7tV+blkrrUml0BwNkxE+XnwDRB3xDV6hqgWe0XrifD\nYTfvd9ScZQP83ip0r4IKlq4GMv/R5shcCRJSkSZ6QSGshH40JYSoiwJf5FHbj9ND\n7do0UAqebWo4yNx63j/wb2ULorW3AClv0BCFSdPsIrCStiGdpgJDBR2P2NZOCob3\nG9uMj+wJD6JJg2nWqNJxkANXX37Qf8plgzssrhrgOvB0fjjS7GYhfkfmZTJ0wPOw\nA8+KzFseBh4UFGgue78KwgkCAwEAAQ== ' -__Keyring= __KeyringFile="/usr/share/keyrings/ubuntu-archive-keyring.gpg" __SkipSigCheck=0 __SkipEmulation=0 @@ -153,6 +161,10 @@ while :; do __AlpineArch=armv7 __QEMUArch=arm ;; + armel) + # this is only used for tizen-build-rootfs.sh + __BuildArch=armel + ;; arm64) __BuildArch=arm64 __UbuntuArch=arm64 @@ -160,31 +172,8 @@ while :; do __QEMUArch=aarch64 __FreeBSDArch=arm64 __FreeBSDMachineArch=aarch64 - ;; - armel) - __BuildArch=armel - __UbuntuArch=armel - __UbuntuRepo="http://archive.debian.org/debian/" - __CodeName=buster - __KeyringFile="/usr/share/keyrings/debian-archive-keyring.gpg" - __LLDB_Package="liblldb-6.0-dev" - __UbuntuPackages="${__UbuntuPackages// libomp-dev/}" - __UbuntuPackages="${__UbuntuPackages// libomp5/}" - __UbuntuSuites= - ;; - armv6) - __BuildArch=armv6 - __UbuntuArch=armhf - __QEMUArch=arm - __UbuntuRepo="http://raspbian.raspberrypi.org/raspbian/" - __CodeName=buster - __KeyringFile="/usr/share/keyrings/raspbian-archive-keyring.gpg" - __LLDB_Package="liblldb-6.0-dev" - __UbuntuSuites= - - if [[ -e "$__KeyringFile" ]]; then - __Keyring="--keyring $__KeyringFile" - fi + __OpenBSDArch=arm64 + __OpenBSDMachineArch=aarch64 ;; loongarch64) __BuildArch=loongarch64 @@ -193,10 +182,6 @@ while :; do __UbuntuArch=loong64 __UbuntuSuites=unreleased __LLDB_Package="liblldb-19-dev" - - if [[ "$__CodeName" == "sid" ]]; then - __UbuntuRepo="http://ftp.ports.debian.org/debian-ports/" - fi ;; riscv64) __BuildArch=riscv64 @@ -212,7 +197,7 @@ while :; do __AlpineArch=ppc64le __QEMUArch=ppc64le __UbuntuArch=ppc64el - __UbuntuRepo="http://ports.ubuntu.com/ubuntu-ports/" + __UbuntuRepo="https://ports.ubuntu.com/ubuntu-ports/" __UbuntuPackages="${__UbuntuPackages// libunwind8-dev/}" __UbuntuPackages="${__UbuntuPackages// libomp-dev/}" __UbuntuPackages="${__UbuntuPackages// libomp5/}" @@ -223,7 +208,7 @@ while :; do __AlpineArch=s390x __QEMUArch=s390x __UbuntuArch=s390x - __UbuntuRepo="http://ports.ubuntu.com/ubuntu-ports/" + __UbuntuRepo="https://ports.ubuntu.com/ubuntu-ports/" __UbuntuPackages="${__UbuntuPackages// libunwind8-dev/}" __UbuntuPackages="${__UbuntuPackages// libomp-dev/}" __UbuntuPackages="${__UbuntuPackages// libomp5/}" @@ -235,15 +220,17 @@ while :; do __UbuntuArch=amd64 __FreeBSDArch=amd64 __FreeBSDMachineArch=amd64 + __OpenBSDArch=amd64 + __OpenBSDMachineArch=amd64 __illumosArch=x86_64 __HaikuArch=x86_64 - __UbuntuRepo="http://archive.ubuntu.com/ubuntu/" + __UbuntuRepo="https://archive.ubuntu.com/ubuntu/" ;; x86) __BuildArch=x86 __UbuntuArch=i386 __AlpineArch=x86 - __UbuntuRepo="http://archive.ubuntu.com/ubuntu/" + __UbuntuRepo="https://archive.ubuntu.com/ubuntu/" ;; lldb*) version="$(echo "$lowerI" | tr -d '[:alpha:]-=')" @@ -295,9 +282,7 @@ while :; do ;; noble) # Ubuntu 24.04 __CodeName=noble - if [[ -z "$__LLDB_Package" ]]; then - __LLDB_Package="liblldb-19-dev" - fi + __LLDB_Package="liblldb-19-dev" ;; stretch) # Debian 9 __CodeName=stretch @@ -305,7 +290,7 @@ while :; do __KeyringFile="/usr/share/keyrings/debian-archive-keyring.gpg" if [[ -z "$__UbuntuRepo" ]]; then - __UbuntuRepo="http://ftp.debian.org/debian/" + __UbuntuRepo="https://archive.debian.org/debian/" fi ;; buster) # Debian 10 @@ -314,7 +299,7 @@ while :; do __KeyringFile="/usr/share/keyrings/debian-archive-keyring.gpg" if [[ -z "$__UbuntuRepo" ]]; then - __UbuntuRepo="http://archive.debian.org/debian/" + __UbuntuRepo="https://archive.debian.org/debian/" fi ;; bullseye) # Debian 11 @@ -322,7 +307,7 @@ while :; do __KeyringFile="/usr/share/keyrings/debian-archive-keyring.gpg" if [[ -z "$__UbuntuRepo" ]]; then - __UbuntuRepo="http://ftp.debian.org/debian/" + __UbuntuRepo="https://ftp.debian.org/debian/" fi ;; bookworm) # Debian 12 @@ -330,7 +315,7 @@ while :; do __KeyringFile="/usr/share/keyrings/debian-archive-keyring.gpg" if [[ -z "$__UbuntuRepo" ]]; then - __UbuntuRepo="http://ftp.debian.org/debian/" + __UbuntuRepo="https://ftp.debian.org/debian/" fi ;; sid) # Debian sid @@ -339,25 +324,21 @@ while :; do # Debian-Ports architectures need different values case "$__UbuntuArch" in - amd64|arm64|armel|armhf|i386|mips64el|ppc64el|riscv64|s390x) + amd64|arm64|armhf|i386|mips64el|ppc64el|riscv64|s390x) __KeyringFile="/usr/share/keyrings/debian-archive-keyring.gpg" if [[ -z "$__UbuntuRepo" ]]; then - __UbuntuRepo="http://ftp.debian.org/debian/" + __UbuntuRepo="https://ftp.debian.org/debian/" fi ;; *) __KeyringFile="/usr/share/keyrings/debian-ports-archive-keyring.gpg" if [[ -z "$__UbuntuRepo" ]]; then - __UbuntuRepo="http://ftp.ports.debian.org/debian-ports/" + __UbuntuRepo="https://ftp.debian.org/debian-ports/" fi ;; esac - - if [[ -e "$__KeyringFile" ]]; then - __Keyring="--keyring $__KeyringFile" - fi ;; tizen) __CodeName= @@ -383,10 +364,14 @@ while :; do ;; freebsd14) __CodeName=freebsd - __FreeBSDBase="14.2-RELEASE" + __FreeBSDBase="14.3-RELEASE" __FreeBSDABI="14" __SkipUnmount=1 ;; + openbsd) + __CodeName=openbsd + __SkipUnmount=1 + ;; illumos) __CodeName=illumos __SkipUnmount=1 @@ -457,7 +442,7 @@ fi __UbuntuPackages+=" ${__LLDB_Package:-}" if [[ -z "$__UbuntuRepo" ]]; then - __UbuntuRepo="http://ports.ubuntu.com/" + __UbuntuRepo="https://ports.ubuntu.com/" fi if [[ -n "$__LLVM_MajorVersion" ]]; then @@ -544,15 +529,15 @@ if [[ "$__CodeName" == "alpine" ]]; then # initialize DB # shellcheck disable=SC2086 "$__ApkToolsDir/apk.static" \ - -X "http://dl-cdn.alpinelinux.org/alpine/$version/main" \ - -X "http://dl-cdn.alpinelinux.org/alpine/$version/community" \ + -X "https://dl-cdn.alpinelinux.org/alpine/$version/main" \ + -X "https://dl-cdn.alpinelinux.org/alpine/$version/community" \ -U $__ApkSignatureArg --root "$__RootfsDir" --arch "$__AlpineArch" --initdb add if [[ "$__AlpineLlvmLibsLookup" == 1 ]]; then # shellcheck disable=SC2086 __AlpinePackages+=" $("$__ApkToolsDir/apk.static" \ - -X "http://dl-cdn.alpinelinux.org/alpine/$version/main" \ - -X "http://dl-cdn.alpinelinux.org/alpine/$version/community" \ + -X "https://dl-cdn.alpinelinux.org/alpine/$version/main" \ + -X "https://dl-cdn.alpinelinux.org/alpine/$version/community" \ -U $__ApkSignatureArg --root "$__RootfsDir" --arch "$__AlpineArch" \ search 'llvm*-libs' | grep -E '^llvm' | sort | tail -1 | sed 's/-[^-]*//2g')" fi @@ -560,8 +545,8 @@ if [[ "$__CodeName" == "alpine" ]]; then # install all packages in one go # shellcheck disable=SC2086 "$__ApkToolsDir/apk.static" \ - -X "http://dl-cdn.alpinelinux.org/alpine/$version/main" \ - -X "http://dl-cdn.alpinelinux.org/alpine/$version/community" \ + -X "https://dl-cdn.alpinelinux.org/alpine/$version/main" \ + -X "https://dl-cdn.alpinelinux.org/alpine/$version/community" \ -U $__ApkSignatureArg --root "$__RootfsDir" --arch "$__AlpineArch" $__NoEmulationArg \ add $__AlpinePackages @@ -578,7 +563,7 @@ elif [[ "$__CodeName" == "freebsd" ]]; then curl -SL "https://download.freebsd.org/ftp/releases/${__FreeBSDArch}/${__FreeBSDMachineArch}/${__FreeBSDBase}/base.txz" | tar -C "$__RootfsDir" -Jxf - ./lib ./usr/lib ./usr/libdata ./usr/include ./usr/share/keys ./etc ./bin/freebsd-version fi echo "ABI = \"FreeBSD:${__FreeBSDABI}:${__FreeBSDMachineArch}\"; FINGERPRINTS = \"${__RootfsDir}/usr/share/keys\"; REPOS_DIR = [\"${__RootfsDir}/etc/pkg\"]; REPO_AUTOUPDATE = NO; RUN_SCRIPTS = NO;" > "${__RootfsDir}"/usr/local/etc/pkg.conf - echo "FreeBSD: { url: \"pkg+http://pkg.FreeBSD.org/\${ABI}/quarterly\", mirror_type: \"srv\", signature_type: \"fingerprints\", fingerprints: \"/usr/share/keys/pkg\", enabled: yes }" > "${__RootfsDir}"/etc/pkg/FreeBSD.conf + echo "FreeBSD: { url: \"pkg+https://pkg.FreeBSD.org/\${ABI}/quarterly\", mirror_type: \"srv\", signature_type: \"fingerprints\", fingerprints: \"/usr/share/keys/pkg\", enabled: yes }" > "${__RootfsDir}"/etc/pkg/FreeBSD.conf mkdir -p "$__RootfsDir"/tmp # get and build package manager if [[ "$__hasWget" == 1 ]]; then @@ -592,9 +577,65 @@ elif [[ "$__CodeName" == "freebsd" ]]; then ./autogen.sh && ./configure --prefix="$__RootfsDir"/host && make -j "$JOBS" && make install rm -rf "$__RootfsDir/tmp/pkg-${__FreeBSDPkg}" # install packages we need. - INSTALL_AS_USER=$(whoami) "$__RootfsDir"/host/sbin/pkg -r "$__RootfsDir" -C "$__RootfsDir"/usr/local/etc/pkg.conf update + INSTALL_AS_USER=$(whoami) IGNORE_OSVERSION=yes "$__RootfsDir"/host/sbin/pkg -r "$__RootfsDir" -C "$__RootfsDir"/usr/local/etc/pkg.conf update # shellcheck disable=SC2086 INSTALL_AS_USER=$(whoami) "$__RootfsDir"/host/sbin/pkg -r "$__RootfsDir" -C "$__RootfsDir"/usr/local/etc/pkg.conf install --yes $__FreeBSDPackages +elif [[ "$__CodeName" == "openbsd" ]]; then + # determine mirrors + OPENBSD_MIRROR="https://cdn.openbsd.org/pub/OpenBSD/$__OpenBSDVersion/$__OpenBSDMachineArch" + + # download base system sets + ensureDownloadTool + + BASE_SETS=(base comp) + for set in "${BASE_SETS[@]}"; do + FILE="${set}${__OpenBSDVersion//./}.tgz" + echo "Downloading $FILE..." + if [[ "$__hasWget" == 1 ]]; then + wget -O- "$OPENBSD_MIRROR/$FILE" | tar -C "$__RootfsDir" -xzpf - + else + curl -SL "$OPENBSD_MIRROR/$FILE" | tar -C "$__RootfsDir" -xzpf - + fi + done + + PKG_MIRROR="https://cdn.openbsd.org/pub/OpenBSD/${__OpenBSDVersion}/packages/${__OpenBSDMachineArch}" + + echo "Installing packages into sysroot..." + + # Fetch package index once + if [[ "$__hasWget" == 1 ]]; then + PKG_INDEX=$(wget -qO- "$PKG_MIRROR/") + else + PKG_INDEX=$(curl -s "$PKG_MIRROR/") + fi + + for pkg in $__OpenBSDPackages; do + PKG_FILE=$(echo "$PKG_INDEX" | grep -Po ">\K${pkg}-[0-9][^\" ]*\.tgz" \ + | sort -V | tail -n1) + + echo "Resolved package filename for $pkg: $PKG_FILE" + + [[ -z "$PKG_FILE" ]] && { echo "ERROR: Package $pkg not found"; exit 1; } + + if [[ "$__hasWget" == 1 ]]; then + wget -O- "$PKG_MIRROR/$PKG_FILE" | tar -C "$__RootfsDir" -xzpf - + else + curl -SL "$PKG_MIRROR/$PKG_FILE" | tar -C "$__RootfsDir" -xzpf - + fi + done + + echo "Creating versionless symlinks for shared libraries..." + # Find all versioned .so files and create the base .so symlink + for lib in "$__RootfsDir/usr/lib/libc++.so."* "$__RootfsDir/usr/lib/libc++abi.so."* "$__RootfsDir/usr/lib/libpthread.so."*; do + if [ -f "$lib" ]; then + # Extract the filename (e.g., libc++.so.12.0) + VERSIONED_NAME=$(basename "$lib") + # Remove the trailing version numbers (e.g., libc++.so) + BASE_NAME=${VERSIONED_NAME%.so.*}.so + # Create the symlink in the same directory + ln -sf "$VERSIONED_NAME" "$__RootfsDir/usr/lib/$BASE_NAME" + fi + done elif [[ "$__CodeName" == "illumos" ]]; then mkdir "$__RootfsDir/tmp" pushd "$__RootfsDir/tmp" @@ -759,6 +800,14 @@ elif [[ "$__CodeName" == "haiku" ]]; then elif [[ -n "$__CodeName" ]]; then __Suites="$__CodeName $(for suite in $__UbuntuSuites; do echo -n "$__CodeName-$suite "; done)" + __SigCheckArgs= + if [[ "$__SkipSigCheck" == "0" ]]; then + if [[ -e "$__KeyringFile" ]]; then + __SigCheckArgs="--keyring $__KeyringFile" + fi + __SigCheckArgs="$__SigCheckArgs --force-check-gpg" + fi + if [[ "$__SkipEmulation" == "1" ]]; then if [[ -z "$AR" ]]; then if command -v ar &>/dev/null; then @@ -774,31 +823,23 @@ elif [[ -n "$__CodeName" ]]; then PYTHON=${PYTHON_EXECUTABLE:-python3} # shellcheck disable=SC2086,SC2046 - echo running "$PYTHON" "$__CrossDir/install-debs.py" --arch "$__UbuntuArch" --mirror "$__UbuntuRepo" --rootfsdir "$__RootfsDir" --artool "$AR" \ + echo running "$PYTHON" "$__CrossDir/install-debs.py" $__SigCheckArgs --arch "$__UbuntuArch" --mirror "$__UbuntuRepo" --rootfsdir "$__RootfsDir" --artool "$AR" \ $(for suite in $__Suites; do echo -n "--suite $suite "; done) \ $__UbuntuPackages # shellcheck disable=SC2086,SC2046 - "$PYTHON" "$__CrossDir/install-debs.py" --arch "$__UbuntuArch" --mirror "$__UbuntuRepo" --rootfsdir "$__RootfsDir" --artool "$AR" \ + "$PYTHON" "$__CrossDir/install-debs.py" $__SigCheckArgs --arch "$__UbuntuArch" --mirror "$__UbuntuRepo" --rootfsdir "$__RootfsDir" --artool "$AR" \ $(for suite in $__Suites; do echo -n "--suite $suite "; done) \ $__UbuntuPackages exit 0 fi - __UpdateOptions= - if [[ "$__SkipSigCheck" == "0" ]]; then - __Keyring="$__Keyring --force-check-gpg" - else - __Keyring= - __UpdateOptions="--allow-unauthenticated --allow-insecure-repositories" - fi - # shellcheck disable=SC2086 - echo running debootstrap "--variant=minbase" $__Keyring --arch "$__UbuntuArch" "$__CodeName" "$__RootfsDir" "$__UbuntuRepo" + echo running debootstrap "--variant=minbase" $__SigCheckArgs --arch "$__UbuntuArch" "$__CodeName" "$__RootfsDir" "$__UbuntuRepo" # shellcheck disable=SC2086 - if ! debootstrap "--variant=minbase" $__Keyring --arch "$__UbuntuArch" "$__CodeName" "$__RootfsDir" "$__UbuntuRepo"; then + if ! debootstrap "--variant=minbase" $__SigCheckArgs --arch "$__UbuntuArch" "$__CodeName" "$__RootfsDir" "$__UbuntuRepo"; then echo "debootstrap failed! dumping debootstrap.log" cat "$__RootfsDir/debootstrap/debootstrap.log" exit 1 @@ -816,6 +857,11 @@ Components: main universe Signed-By: $__KeyringFile EOF + __UpdateOptions= + if [[ "$__SkipSigCheck" == "1" ]]; then + __UpdateOptions="--allow-unauthenticated --allow-insecure-repositories" + fi + # shellcheck disable=SC2086 chroot "$__RootfsDir" apt-get update $__UpdateOptions chroot "$__RootfsDir" apt-get -f -y install diff --git a/eng/common/cross/install-debs.py b/eng/common/cross/install-debs.py index c81eb37e5..20ca770a1 100755 --- a/eng/common/cross/install-debs.py +++ b/eng/common/cross/install-debs.py @@ -4,6 +4,7 @@ import asyncio import aiohttp import gzip +import hashlib import os import re import shutil @@ -16,7 +17,7 @@ from collections import deque from functools import cmp_to_key -async def download_file(session, url, dest_path, max_retries=3, retry_delay=2, timeout=60): +async def download_file(session, url, dest_path, max_retries=3, retry_delay=2, timeout=60, checksum=None): """Asynchronous file download with retries.""" attempt = 0 while attempt < max_retries: @@ -25,19 +26,25 @@ async def download_file(session, url, dest_path, max_retries=3, retry_delay=2, t if response.status == 200: with open(dest_path, "wb") as f: content = await response.read() + + # verify checksum if provided + if checksum: + sha256 = hashlib.sha256(content).hexdigest() + if sha256 != checksum: + raise Exception(f"SHA256 mismatch for {url}: expected {checksum}, got {sha256}") + f.write(content) print(f"Downloaded {url} at {dest_path}") return else: - print(f"Failed to download {url}, Status Code: {response.status}") - break + raise Exception(f"Failed to download {url}, Status Code: {response.status}") except (asyncio.CancelledError, asyncio.TimeoutError, aiohttp.ClientError) as e: print(f"Error downloading {url}: {type(e).__name__} - {e}. Retrying...") attempt += 1 await asyncio.sleep(retry_delay) - print(f"Failed to download {url} after {max_retries} attempts.") + raise Exception(f"Failed to download {url} after {max_retries} attempts.") async def download_deb_files_parallel(mirror, packages, tmp_dir): """Download .deb files in parallel.""" @@ -51,11 +58,11 @@ async def download_deb_files_parallel(mirror, packages, tmp_dir): if filename: url = f"{mirror}/{filename}" dest_path = os.path.join(tmp_dir, os.path.basename(filename)) - tasks.append(asyncio.create_task(download_file(session, url, dest_path))) + tasks.append(asyncio.create_task(download_file(session, url, dest_path, checksum=info.get("SHA256")))) await asyncio.gather(*tasks) -async def download_package_index_parallel(mirror, arch, suites): +async def download_package_index_parallel(mirror, arch, suites, check_sig, keyring): """Download package index files for specified suites and components entirely in memory.""" tasks = [] timeout = aiohttp.ClientTimeout(total=60) @@ -63,10 +70,9 @@ async def download_package_index_parallel(mirror, arch, suites): async with aiohttp.ClientSession(timeout=timeout) as session: for suite in suites: for component in ["main", "universe"]: - url = f"{mirror}/dists/{suite}/{component}/binary-{arch}/Packages.gz" - tasks.append(fetch_and_decompress(session, url)) + tasks.append(fetch_and_decompress(session, mirror, arch, suite, component, check_sig, keyring)) - results = await asyncio.gather(*tasks, return_exceptions=True) + results = await asyncio.gather(*tasks) merged_content = "" for result in results: @@ -77,20 +83,71 @@ async def download_package_index_parallel(mirror, arch, suites): return merged_content -async def fetch_and_decompress(session, url): +async def fetch_and_decompress(session, mirror, arch, suite, component, check_sig, keyring): """Fetch and decompress the Packages.gz file.""" - try: - async with session.get(url) as response: - if response.status == 200: - compressed_data = await response.read() - decompressed_data = gzip.decompress(compressed_data).decode('utf-8') - print(f"Downloaded index: {url}") - return decompressed_data - else: - print(f"Skipped index: {url} (doesn't exist)") - return None - except Exception as e: - print(f"Error fetching {url}: {e}") + + path = f"{component}/binary-{arch}/Packages.gz" + url = f"{mirror}/dists/{suite}/{path}" + + async with session.get(url) as response: + if response.status == 200: + compressed_data = await response.read() + decompressed_data = gzip.decompress(compressed_data).decode('utf-8') + print(f"Downloaded index: {url}") + + if check_sig: + # Verify the package index against the sha256 recorded in the Release file + release_file_content = await fetch_release_file(session, mirror, suite, keyring) + packages_sha = parse_release_file(release_file_content, path) + + sha256 = hashlib.sha256(compressed_data).hexdigest() + if sha256 != packages_sha: + raise Exception(f"SHA256 mismatch for {path}: expected {packages_sha}, got {sha256}") + print(f"Checksum verified for {path}") + + return decompressed_data + else: + print(f"Skipped index: {url} (doesn't exist)") + return None + +async def fetch_release_file(session, mirror, suite, keyring): + """Fetch Release and Release.gpg files and verify the signature.""" + + release_url = f"{mirror}/dists/{suite}/Release" + release_gpg_url = f"{mirror}/dists/{suite}/Release.gpg" + + with tempfile.NamedTemporaryFile() as release_file, tempfile.NamedTemporaryFile() as release_gpg_file: + await download_file(session, release_url, release_file.name) + await download_file(session, release_gpg_url, release_gpg_file.name) + + print("Verifying signature of Release with Release.gpg.") + verify_command = ["gpg"] + if keyring: + verify_command += ["--keyring", keyring] + verify_command += ["--verify", release_gpg_file.name, release_file.name] + result = subprocess.run(verify_command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + + if result.returncode != 0: + raise Exception(f"Signature verification failed: {result.stderr.decode('utf-8')}") + + print("Signature verified successfully.") + + with open(release_file.name) as f: + return f.read() + +def parse_release_file(content, path): + """Parses the Release file and returns sha256 checksum of the specified path.""" + + # data looks like this: + # + matches = re.findall(r'^ (\S*) +(\S*) +(\S*)$', content, re.MULTILINE) + + for entry in matches: + # the file has both md5 and sha256 checksums, we want sha256 which has a length of 64 + if entry[2] == path and len(entry[0]) == 64: + return entry[0] + + raise Exception(f"Could not find checksum for {path} in Release file.") def parse_debian_version(version): """Parse a Debian package version into epoch, upstream version, and revision.""" @@ -171,13 +228,15 @@ def parse_package_index(content): filename = fields.get("Filename") depends = fields.get("Depends") provides = fields.get("Provides", None) + sha256 = fields.get("SHA256") # Only update if package_name is not in packages or if the new version is higher if package_name not in packages or compare_debian_versions(version, packages[package_name]["Version"]) > 0: packages[package_name] = { "Version": version, "Filename": filename, - "Depends": depends + "Depends": depends, + "SHA256": sha256 } # Update aliases if package provides any alternatives @@ -233,7 +292,7 @@ def extract_deb_file(deb_file, tmp_dir, extract_dir, ar_tool): os.makedirs(extract_dir, exist_ok=True) with tempfile.TemporaryDirectory(dir=tmp_dir) as tmp_subdir: - result = subprocess.run(f"{ar_tool} t {os.path.abspath(deb_file)}", cwd=tmp_subdir, check=True, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + result = subprocess.run([ar_tool, "t", os.path.abspath(deb_file)], cwd=tmp_subdir, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) tar_filename = None for line in result.stdout.decode().splitlines(): @@ -247,7 +306,8 @@ def extract_deb_file(deb_file, tmp_dir, extract_dir, ar_tool): tar_file_path = os.path.join(tmp_subdir, tar_filename) print(f"Extracting {tar_filename} from {deb_file}..") - subprocess.run(f"{ar_tool} p {os.path.abspath(deb_file)} {tar_filename} > {tar_file_path}", check=True, shell=True) + with open(tar_file_path, "wb") as outfile: + subprocess.run([ar_tool, "p", os.path.abspath(deb_file), tar_filename], check=True, stdout=outfile, stderr=subprocess.PIPE) file_extension = os.path.splitext(tar_file_path)[1].lower() @@ -268,7 +328,18 @@ def extract_deb_file(deb_file, tmp_dir, extract_dir, ar_tool): raise ValueError(f"Unsupported compression format: {file_extension}") with tarfile.open(tar_file_path, mode) as tar: - tar.extractall(path=extract_dir, filter='fully_trusted') + tar.extractall(path=extract_dir, filter=_rootfs_extraction_filter) + +def _rootfs_extraction_filter(member, dest_path): + """Tarfile extraction filter based on the 'data' filter that additionally + rewrites absolute-target symlinks/hardlinks into rootfs-relative paths. + """ + if (member.issym() or member.islnk()) and os.path.isabs(member.linkname): + link_dir = os.path.dirname(member.name) + new_linkname = os.path.relpath(member.linkname.lstrip('/'), + start=link_dir or '.') + member = member.replace(linkname=new_linkname, deep=False) + return tarfile.data_filter(member, dest_path) def finalize_setup(rootfsdir): lib_dir = os.path.join(rootfsdir, 'lib') @@ -295,24 +366,17 @@ def finalize_setup(rootfsdir): if __name__ == "__main__": parser = argparse.ArgumentParser(description="Generate rootfs for .NET runtime on Debian-like OS") - parser.add_argument("--distro", required=False, help="Distro name (e.g., debian, ubuntu, etc.)") parser.add_argument("--arch", required=True, help="Architecture (e.g., amd64, loong64, etc.)") parser.add_argument("--rootfsdir", required=True, help="Destination directory.") parser.add_argument('--suite', required=True, action='append', help='Specify one or more repository suites to collect index data.') - parser.add_argument("--mirror", required=False, help="Mirror (e.g., http://ftp.debian.org/debian-ports etc.)") + parser.add_argument("--mirror", required=True, help="Mirror (e.g., http://ftp.debian.org/debian-ports etc.)") parser.add_argument("--artool", required=False, default="ar", help="ar tool to extract debs (e.g., ar, llvm-ar etc.)") + parser.add_argument("--force-check-gpg", required=False, action='store_true', help="Verify the packages against signatures in Release file.") + parser.add_argument("--keyring", required=False, default='', help="Keyring file to check signature of Release file.") parser.add_argument("packages", nargs="+", help="List of package names to be installed.") args = parser.parse_args() - if args.mirror is None: - if args.distro == "ubuntu": - args.mirror = "http://archive.ubuntu.com/ubuntu" if args.arch in ["amd64", "i386"] else "http://ports.ubuntu.com/ubuntu-ports" - elif args.distro == "debian": - args.mirror = "http://ftp.debian.org/debian-ports" - else: - raise Exception("Unsupported distro") - DESIRED_PACKAGES = args.packages + [ # base packages "dpkg", "busybox", @@ -322,9 +386,16 @@ def finalize_setup(rootfsdir): "debianutils" ] - print(f"Creating rootfs. rootfsdir: {args.rootfsdir}, distro: {args.distro}, arch: {args.arch}, suites: {args.suite}, mirror: {args.mirror}") + print(f"Creating rootfs. rootfsdir: {args.rootfsdir}, arch: {args.arch}, suites: {args.suite}, mirror: {args.mirror}") + + check_sig = args.force_check_gpg + if check_sig and not args.keyring: + print("ERROR: --force-check-gpg requires --keyring to specify a keyring file for signature verification.") + print("Install the appropriate keyring package (e.g., debian-ports-archive-keyring, ubuntu-archive-keyring)") + print("or pass --skipsigcheck to build-rootfs.sh to disable signature checking.") + sys.exit(1) - package_index_content = asyncio.run(download_package_index_parallel(args.mirror, args.arch, args.suite)) + package_index_content = asyncio.run(download_package_index_parallel(args.mirror, args.arch, args.suite, check_sig, args.keyring)) packages_info, aliases = parse_package_index(package_index_content) diff --git a/eng/common/cross/toolchain.cmake b/eng/common/cross/toolchain.cmake index 0ff85cf03..99d6dfe82 100644 --- a/eng/common/cross/toolchain.cmake +++ b/eng/common/cross/toolchain.cmake @@ -3,15 +3,22 @@ set(CROSS_ROOTFS $ENV{ROOTFS_DIR}) # reset platform variables (e.g. cmake 3.25 sets LINUX=1) unset(LINUX) unset(FREEBSD) +unset(OPENBSD) unset(ILLUMOS) unset(ANDROID) unset(TIZEN) unset(HAIKU) set(TARGET_ARCH_NAME $ENV{TARGET_BUILD_ARCH}) + +file(GLOB OPENBSD_PROBE "${CROSS_ROOTFS}/etc/signify/openbsd-*.pub") + if(EXISTS ${CROSS_ROOTFS}/bin/freebsd-version) set(CMAKE_SYSTEM_NAME FreeBSD) set(FREEBSD 1) +elseif(OPENBSD_PROBE) + set(CMAKE_SYSTEM_NAME OpenBSD) + set(OPENBSD 1) elseif(EXISTS ${CROSS_ROOTFS}/usr/platform/i86pc) set(CMAKE_SYSTEM_NAME SunOS) set(ILLUMOS 1) @@ -53,6 +60,8 @@ elseif(TARGET_ARCH_NAME STREQUAL "arm64") endif() elseif(FREEBSD) set(triple "aarch64-unknown-freebsd12") + elseif(OPENBSD) + set(triple "aarch64-unknown-openbsd") endif() elseif(TARGET_ARCH_NAME STREQUAL "armel") set(CMAKE_SYSTEM_PROCESSOR armv7l) @@ -109,6 +118,8 @@ elseif(TARGET_ARCH_NAME STREQUAL "x64") endif() elseif(FREEBSD) set(triple "x86_64-unknown-freebsd12") + elseif(OPENBSD) + set(triple "x86_64-unknown-openbsd") elseif(ILLUMOS) set(TOOLCHAIN "x86_64-illumos") elseif(HAIKU) @@ -193,7 +204,7 @@ if(ANDROID) # include official NDK toolchain script include(${CROSS_ROOTFS}/../build/cmake/android.toolchain.cmake) -elseif(FREEBSD) +elseif(FREEBSD OR OPENBSD) # we cross-compile by instructing clang set(CMAKE_C_COMPILER_TARGET ${triple}) set(CMAKE_CXX_COMPILER_TARGET ${triple}) @@ -214,13 +225,19 @@ elseif(ILLUMOS) locate_toolchain_exec(g++ CMAKE_CXX_COMPILER) elseif(HAIKU) set(CMAKE_SYSROOT "${CROSS_ROOTFS}") - set(CMAKE_PROGRAM_PATH "${CMAKE_PROGRAM_PATH};${CROSS_ROOTFS}/cross-tools-x86_64/bin") set(CMAKE_SYSTEM_PREFIX_PATH "${CROSS_ROOTFS}") set(CMAKE_C_STANDARD_LIBRARIES "${CMAKE_C_STANDARD_LIBRARIES} -lssp") set(CMAKE_CXX_STANDARD_LIBRARIES "${CMAKE_CXX_STANDARD_LIBRARIES} -lssp") - locate_toolchain_exec(gcc CMAKE_C_COMPILER) - locate_toolchain_exec(g++ CMAKE_CXX_COMPILER) + if ($ENV{CCC_CC} MATCHES ".*gcc.*") + set(CMAKE_PROGRAM_PATH "${CMAKE_PROGRAM_PATH};${CROSS_ROOTFS}/cross-tools-x86_64/bin") + locate_toolchain_exec(gcc CMAKE_C_COMPILER) + locate_toolchain_exec(g++ CMAKE_CXX_COMPILER) + else() + set(CMAKE_C_COMPILER_EXTERNAL_TOOLCHAIN "${CROSS_ROOTFS}/cross-tools-x86_64") + set(CMAKE_CXX_COMPILER_EXTERNAL_TOOLCHAIN "${CROSS_ROOTFS}/cross-tools-x86_64") + set(CMAKE_ASM_COMPILER_EXTERNAL_TOOLCHAIN "${CROSS_ROOTFS}/cross-tools-x86_64") + endif() # let CMake set up the correct search paths include(Platform/Haiku) @@ -291,7 +308,7 @@ endif() # Specify compile options -if((TARGET_ARCH_NAME MATCHES "^(arm|arm64|armel|armv6|loongarch64|ppc64le|riscv64|s390x|x64|x86)$" AND NOT ANDROID AND NOT FREEBSD) OR ILLUMOS OR HAIKU) +if((TARGET_ARCH_NAME MATCHES "^(arm|arm64|armel|armv6|loongarch64|ppc64le|riscv64|s390x|x64|x86)$" AND NOT ANDROID AND NOT FREEBSD AND NOT OPENBSD) OR ILLUMOS OR HAIKU) set(CMAKE_C_COMPILER_TARGET ${TOOLCHAIN}) set(CMAKE_CXX_COMPILER_TARGET ${TOOLCHAIN}) set(CMAKE_ASM_COMPILER_TARGET ${TOOLCHAIN}) diff --git a/eng/common/darc-init.ps1 b/eng/common/darc-init.ps1 index e33743105..a5be41db6 100644 --- a/eng/common/darc-init.ps1 +++ b/eng/common/darc-init.ps1 @@ -29,11 +29,11 @@ function InstallDarcCli ($darcVersion, $toolpath) { Write-Host "Installing Darc CLI version $darcVersion..." Write-Host 'You may need to restart your command window if this is the first dotnet tool you have installed.' if (-not $toolpath) { - Write-Host "'$dotnet' tool install $darcCliPackageName --version $darcVersion --add-source '$arcadeServicesSource' -v $verbosity -g" - & "$dotnet" tool install $darcCliPackageName --version $darcVersion --add-source "$arcadeServicesSource" -v $verbosity -g + Write-Host "'$dotnet' tool install $darcCliPackageName --version $darcVersion --source '$arcadeServicesSource' -v $verbosity -g" + & "$dotnet" tool install $darcCliPackageName --version $darcVersion --source "$arcadeServicesSource" -v $verbosity -g }else { - Write-Host "'$dotnet' tool install $darcCliPackageName --version $darcVersion --add-source '$arcadeServicesSource' -v $verbosity --tool-path '$toolpath'" - & "$dotnet" tool install $darcCliPackageName --version $darcVersion --add-source "$arcadeServicesSource" -v $verbosity --tool-path "$toolpath" + Write-Host "'$dotnet' tool install $darcCliPackageName --version $darcVersion --source '$arcadeServicesSource' -v $verbosity --tool-path '$toolpath'" + & "$dotnet" tool install $darcCliPackageName --version $darcVersion --source "$arcadeServicesSource" -v $verbosity --tool-path "$toolpath" } } diff --git a/eng/common/darc-init.sh b/eng/common/darc-init.sh index e889f439b..b56d40e57 100755 --- a/eng/common/darc-init.sh +++ b/eng/common/darc-init.sh @@ -5,7 +5,7 @@ darcVersion='' versionEndpoint='https://maestro.dot.net/api/assets/darc-version?api-version=2020-02-20' verbosity='minimal' -while [[ $# > 0 ]]; do +while [[ $# -gt 0 ]]; do opt="$(echo "$1" | tr "[:upper:]" "[:lower:]")" case "$opt" in --darcversion) @@ -73,9 +73,9 @@ function InstallDarcCli { echo "Installing Darc CLI version $darcVersion..." echo "You may need to restart your command shell if this is the first dotnet tool you have installed." if [ -z "$toolpath" ]; then - echo $($dotnet_root/dotnet tool install $darc_cli_package_name --version $darcVersion --add-source "$arcadeServicesSource" -v $verbosity -g) + echo $($dotnet_root/dotnet tool install $darc_cli_package_name --version $darcVersion --source "$arcadeServicesSource" -v $verbosity -g) else - echo $($dotnet_root/dotnet tool install $darc_cli_package_name --version $darcVersion --add-source "$arcadeServicesSource" -v $verbosity --tool-path "$toolpath") + echo $($dotnet_root/dotnet tool install $darc_cli_package_name --version $darcVersion --source "$arcadeServicesSource" -v $verbosity --tool-path "$toolpath") fi } diff --git a/eng/common/dotnet-install.ps1 b/eng/common/dotnet-install.ps1 index 811f0f717..50ae62737 100644 --- a/eng/common/dotnet-install.ps1 +++ b/eng/common/dotnet-install.ps1 @@ -10,7 +10,11 @@ Param( . $PSScriptRoot\tools.ps1 -$dotnetRoot = Join-Path $RepoRoot '.dotnet' +if (-not [string]::IsNullOrEmpty($env:DOTNET_GLOBAL_INSTALL_DIR)) { + $dotnetRoot = $env:DOTNET_GLOBAL_INSTALL_DIR +} else { + $dotnetRoot = Join-Path $RepoRoot '.dotnet' +} $installdir = $dotnetRoot try { diff --git a/eng/common/dotnet-install.sh b/eng/common/dotnet-install.sh index 7b9d97e3b..1cb3f5aba 100755 --- a/eng/common/dotnet-install.sh +++ b/eng/common/dotnet-install.sh @@ -18,7 +18,7 @@ architecture='' runtime='dotnet' runtimeSourceFeed='' runtimeSourceFeedKey='' -while [[ $# > 0 ]]; do +while [[ $# -gt 0 ]]; do opt="$(echo "$1" | tr "[:upper:]" "[:lower:]")" case "$opt" in -version|-v) @@ -80,7 +80,11 @@ case $cpuname in ;; esac -dotnetRoot="${repo_root}.dotnet" +if [[ -n "${DOTNET_GLOBAL_INSTALL_DIR:-}" ]]; then + dotnetRoot="$DOTNET_GLOBAL_INSTALL_DIR" +else + dotnetRoot="${repo_root}.dotnet" +fi if [[ $architecture != "" ]] && [[ $architecture != $buildarch ]]; then dotnetRoot="$dotnetRoot/$architecture" fi diff --git a/eng/common/dotnet.sh b/eng/common/dotnet.sh index 2ef682356..f6d24871c 100755 --- a/eng/common/dotnet.sh +++ b/eng/common/dotnet.sh @@ -19,7 +19,7 @@ source $scriptroot/tools.sh InitializeDotNetCli true # install # Invoke acquired SDK with args if they are provided -if [[ $# > 0 ]]; then +if [[ $# -gt 0 ]]; then __dotnetDir=${_InitializeDotNetCli} dotnetPath=${__dotnetDir}/dotnet ${dotnetPath} "$@" diff --git a/eng/common/internal-feed-operations.sh b/eng/common/internal-feed-operations.sh index 9378223ba..6299e7eff 100755 --- a/eng/common/internal-feed-operations.sh +++ b/eng/common/internal-feed-operations.sh @@ -100,7 +100,7 @@ operation='' authToken='' repoName='' -while [[ $# > 0 ]]; do +while [[ $# -gt 0 ]]; do opt="$(echo "$1" | tr "[:upper:]" "[:lower:]")" case "$opt" in --operation) diff --git a/eng/common/native/init-distro-rid.sh b/eng/common/native/init-distro-rid.sh index 83ea7aab0..8fc6d2fec 100755 --- a/eng/common/native/init-distro-rid.sh +++ b/eng/common/native/init-distro-rid.sh @@ -39,6 +39,8 @@ getNonPortableDistroRid() # $rootfsDir can be empty. freebsd-version is a shell script and should always work. __freebsd_major_version=$("$rootfsDir"/bin/freebsd-version | cut -d'.' -f1) nonPortableRid="freebsd.$__freebsd_major_version-${targetArch}" + elif [ "$targetOs" = "openbsd" ]; then + nonPortableRid="openbsd.$(uname -r)-${targetArch}" elif command -v getprop >/dev/null && getprop ro.product.system.model | grep -qi android; then __android_sdk_version=$(getprop ro.build.version.sdk) nonPortableRid="android.$__android_sdk_version-${targetArch}" diff --git a/eng/common/native/install-dependencies.sh b/eng/common/native/install-dependencies.sh index 477a44f33..aff839fa0 100755 --- a/eng/common/native/install-dependencies.sh +++ b/eng/common/native/install-dependencies.sh @@ -24,14 +24,16 @@ case "$os" in apt update apt install -y build-essential gettext locales cmake llvm clang lld lldb liblldb-dev libunwind8-dev libicu-dev liblttng-ust-dev \ - libssl-dev libkrb5-dev pigz cpio + libssl-dev libkrb5-dev pigz cpio ninja-build file localedef -i en_US -c -f UTF-8 -A /usr/share/locale/locale.alias en_US.UTF-8 - elif [ "$ID" = "fedora" ] || [ "$ID" = "rhel" ] || [ "$ID" = "azurelinux" ]; then + elif [ "$ID" = "fedora" ] || [ "$ID" = "rhel" ] || [ "$ID" = "azurelinux" ] || [ "$ID" = "centos" ]; then pkg_mgr="$(command -v tdnf 2>/dev/null || command -v dnf)" - $pkg_mgr install -y cmake llvm lld lldb clang python curl libicu-devel openssl-devel krb5-devel lttng-ust-devel pigz cpio + $pkg_mgr install -y cmake llvm lld lldb clang python curl libicu-devel openssl-devel krb5-devel lttng-ust-devel pigz cpio ninja-build file + elif [ "$ID" = "amzn" ]; then + dnf install -y cmake llvm lld lldb clang python libicu-devel openssl-devel krb5-devel lttng-ust-devel pigz cpio ninja-build file elif [ "$ID" = "alpine" ]; then - apk add build-base cmake bash curl clang llvm-dev lld lldb krb5-dev lttng-ust-dev icu-dev openssl-dev pigz cpio + apk add build-base cmake bash curl clang llvm llvm-dev lld lldb-dev krb5-dev lttng-ust-dev icu-dev openssl-dev pigz cpio ninja file else echo "Unsupported distro. distro: $ID" exit 1 @@ -52,6 +54,7 @@ brew "openssl@3" brew "pkgconf" brew "python3" brew "pigz" +brew "ninja" EOF ;; diff --git a/eng/common/pipeline-logging-functions.ps1 b/eng/common/pipeline-logging-functions.ps1 index 8e422c561..9f85c2917 100644 --- a/eng/common/pipeline-logging-functions.ps1 +++ b/eng/common/pipeline-logging-functions.ps1 @@ -32,7 +32,7 @@ function Write-PipelineTelemetryError { $PSBoundParameters.Remove('Category') | Out-Null if ($Force -Or ((Test-Path variable:ci) -And $ci)) { - $Message = "(NETCORE_ENGINEERING_TELEMETRY=$Category) $Message" + $Message = "($Category) $Message" } $PSBoundParameters.Remove('Message') | Out-Null $PSBoundParameters.Add('Message', $Message) diff --git a/eng/common/post-build/redact-logs.ps1 b/eng/common/post-build/redact-logs.ps1 index 472d5bb56..672f4e265 100644 --- a/eng/common/post-build/redact-logs.ps1 +++ b/eng/common/post-build/redact-logs.ps1 @@ -9,7 +9,8 @@ param( [Parameter(Mandatory=$false)][string] $TokensFilePath, [Parameter(ValueFromRemainingArguments=$true)][String[]]$TokensToRedact, [Parameter(Mandatory=$false)][string] $runtimeSourceFeed, - [Parameter(Mandatory=$false)][string] $runtimeSourceFeedKey) + [Parameter(Mandatory=$false)][string] $runtimeSourceFeedKey +) try { $ErrorActionPreference = 'Stop' @@ -48,8 +49,8 @@ try { Write-Host "Installing Binlog redactor CLI..." Write-Host "'$dotnet' new tool-manifest" & "$dotnet" new tool-manifest - Write-Host "'$dotnet' tool install $packageName --local --add-source '$PackageFeed' -v $verbosity --version $BinlogToolVersion" - & "$dotnet" tool install $packageName --local --add-source "$PackageFeed" -v $verbosity --version $BinlogToolVersion + Write-Host "'$dotnet' tool install $packageName --local --source '$PackageFeed' -v $verbosity --version $BinlogToolVersion" + & "$dotnet" tool install $packageName --local --source "$PackageFeed" -v $verbosity --version $BinlogToolVersion if (Test-Path $TokensFilePath) { Write-Host "Adding additional sensitive data for redaction from file: " $TokensFilePath diff --git a/eng/common/post-build/sourcelink-validation.ps1 b/eng/common/post-build/sourcelink-validation.ps1 deleted file mode 100644 index 1976ef70f..000000000 --- a/eng/common/post-build/sourcelink-validation.ps1 +++ /dev/null @@ -1,327 +0,0 @@ -param( - [Parameter(Mandatory=$true)][string] $InputPath, # Full path to directory where Symbols.NuGet packages to be checked are stored - [Parameter(Mandatory=$true)][string] $ExtractPath, # Full path to directory where the packages will be extracted during validation - [Parameter(Mandatory=$false)][string] $GHRepoName, # GitHub name of the repo including the Org. E.g., dotnet/arcade - [Parameter(Mandatory=$false)][string] $GHCommit, # GitHub commit SHA used to build the packages - [Parameter(Mandatory=$true)][string] $SourcelinkCliVersion # Version of SourceLink CLI to use -) - -$ErrorActionPreference = 'Stop' -Set-StrictMode -Version 2.0 - -# `tools.ps1` checks $ci to perform some actions. Since the post-build -# scripts don't necessarily execute in the same agent that run the -# build.ps1/sh script this variable isn't automatically set. -$ci = $true -$disableConfigureToolsetImport = $true -. $PSScriptRoot\..\tools.ps1 - -# Cache/HashMap (File -> Exist flag) used to consult whether a file exist -# in the repository at a specific commit point. This is populated by inserting -# all files present in the repo at a specific commit point. -$global:RepoFiles = @{} - -# Maximum number of jobs to run in parallel -$MaxParallelJobs = 16 - -$MaxRetries = 5 -$RetryWaitTimeInSeconds = 30 - -# Wait time between check for system load -$SecondsBetweenLoadChecks = 10 - -if (!$InputPath -or !(Test-Path $InputPath)){ - Write-Host "No files to validate." - ExitWithExitCode 0 -} - -$ValidatePackage = { - param( - [string] $PackagePath # Full path to a Symbols.NuGet package - ) - - . $using:PSScriptRoot\..\tools.ps1 - - # Ensure input file exist - if (!(Test-Path $PackagePath)) { - Write-Host "Input file does not exist: $PackagePath" - return [pscustomobject]@{ - result = 1 - packagePath = $PackagePath - } - } - - # Extensions for which we'll look for SourceLink information - # For now we'll only care about Portable & Embedded PDBs - $RelevantExtensions = @('.dll', '.exe', '.pdb') - - Write-Host -NoNewLine 'Validating ' ([System.IO.Path]::GetFileName($PackagePath)) '...' - - $PackageId = [System.IO.Path]::GetFileNameWithoutExtension($PackagePath) - $ExtractPath = Join-Path -Path $using:ExtractPath -ChildPath $PackageId - $FailedFiles = 0 - - Add-Type -AssemblyName System.IO.Compression.FileSystem - - [System.IO.Directory]::CreateDirectory($ExtractPath) | Out-Null - - try { - $zip = [System.IO.Compression.ZipFile]::OpenRead($PackagePath) - - $zip.Entries | - Where-Object {$RelevantExtensions -contains [System.IO.Path]::GetExtension($_.Name)} | - ForEach-Object { - $FileName = $_.FullName - $Extension = [System.IO.Path]::GetExtension($_.Name) - $FakeName = -Join((New-Guid), $Extension) - $TargetFile = Join-Path -Path $ExtractPath -ChildPath $FakeName - - # We ignore resource DLLs - if ($FileName.EndsWith('.resources.dll')) { - return [pscustomobject]@{ - result = 0 - packagePath = $PackagePath - } - } - - [System.IO.Compression.ZipFileExtensions]::ExtractToFile($_, $TargetFile, $true) - - $ValidateFile = { - param( - [string] $FullPath, # Full path to the module that has to be checked - [string] $RealPath, - [ref] $FailedFiles - ) - - $sourcelinkExe = "$env:USERPROFILE\.dotnet\tools" - $sourcelinkExe = Resolve-Path "$sourcelinkExe\sourcelink.exe" - $SourceLinkInfos = & $sourcelinkExe print-urls $FullPath | Out-String - - if ($LASTEXITCODE -eq 0 -and -not ([string]::IsNullOrEmpty($SourceLinkInfos))) { - $NumFailedLinks = 0 - - # We only care about Http addresses - $Matches = (Select-String '(http[s]?)(:\/\/)([^\s,]+)' -Input $SourceLinkInfos -AllMatches).Matches - - if ($Matches.Count -ne 0) { - $Matches.Value | - ForEach-Object { - $Link = $_ - $CommitUrl = "https://raw.githubusercontent.com/${using:GHRepoName}/${using:GHCommit}/" - - $FilePath = $Link.Replace($CommitUrl, "") - $Status = 200 - $Cache = $using:RepoFiles - - $attempts = 0 - - while ($attempts -lt $using:MaxRetries) { - if ( !($Cache.ContainsKey($FilePath)) ) { - try { - $Uri = $Link -as [System.URI] - - if ($Link -match "submodules") { - # Skip submodule links until sourcelink properly handles submodules - $Status = 200 - } - elseif ($Uri.AbsoluteURI -ne $null -and ($Uri.Host -match 'github' -or $Uri.Host -match 'githubusercontent')) { - # Only GitHub links are valid - $Status = (Invoke-WebRequest -Uri $Link -UseBasicParsing -Method HEAD -TimeoutSec 5).StatusCode - } - else { - # If it's not a github link, we want to break out of the loop and not retry. - $Status = 0 - $attempts = $using:MaxRetries - } - } - catch { - Write-Host $_ - $Status = 0 - } - } - - if ($Status -ne 200) { - $attempts++ - - if ($attempts -lt $using:MaxRetries) - { - $attemptsLeft = $using:MaxRetries - $attempts - Write-Warning "Download failed, $attemptsLeft attempts remaining, will retry in $using:RetryWaitTimeInSeconds seconds" - Start-Sleep -Seconds $using:RetryWaitTimeInSeconds - } - else { - if ($NumFailedLinks -eq 0) { - if ($FailedFiles.Value -eq 0) { - Write-Host - } - - Write-Host "`tFile $RealPath has broken links:" - } - - Write-Host "`t`tFailed to retrieve $Link" - - $NumFailedLinks++ - } - } - else { - break - } - } - } - } - - if ($NumFailedLinks -ne 0) { - $FailedFiles.value++ - $global:LASTEXITCODE = 1 - } - } - } - - &$ValidateFile $TargetFile $FileName ([ref]$FailedFiles) - } - } - catch { - Write-Host $_ - } - finally { - $zip.Dispose() - } - - if ($FailedFiles -eq 0) { - Write-Host 'Passed.' - return [pscustomobject]@{ - result = 0 - packagePath = $PackagePath - } - } - else { - Write-PipelineTelemetryError -Category 'SourceLink' -Message "$PackagePath has broken SourceLink links." - return [pscustomobject]@{ - result = 1 - packagePath = $PackagePath - } - } -} - -function CheckJobResult( - $result, - $packagePath, - [ref]$ValidationFailures, - [switch]$logErrors) { - if ($result -ne '0') { - if ($logErrors) { - Write-PipelineTelemetryError -Category 'SourceLink' -Message "$packagePath has broken SourceLink links." - } - $ValidationFailures.Value++ - } -} - -function ValidateSourceLinkLinks { - if ($GHRepoName -ne '' -and !($GHRepoName -Match '^[^\s\/]+/[^\s\/]+$')) { - if (!($GHRepoName -Match '^[^\s-]+-[^\s]+$')) { - Write-PipelineTelemetryError -Category 'SourceLink' -Message "GHRepoName should be in the format / or -. '$GHRepoName'" - ExitWithExitCode 1 - } - else { - $GHRepoName = $GHRepoName -replace '^([^\s-]+)-([^\s]+)$', '$1/$2'; - } - } - - if ($GHCommit -ne '' -and !($GHCommit -Match '^[0-9a-fA-F]{40}$')) { - Write-PipelineTelemetryError -Category 'SourceLink' -Message "GHCommit should be a 40 chars hexadecimal string. '$GHCommit'" - ExitWithExitCode 1 - } - - if ($GHRepoName -ne '' -and $GHCommit -ne '') { - $RepoTreeURL = -Join('http://api.github.com/repos/', $GHRepoName, '/git/trees/', $GHCommit, '?recursive=1') - $CodeExtensions = @('.cs', '.vb', '.fs', '.fsi', '.fsx', '.fsscript') - - try { - # Retrieve the list of files in the repo at that particular commit point and store them in the RepoFiles hash - $Data = Invoke-WebRequest $RepoTreeURL -UseBasicParsing | ConvertFrom-Json | Select-Object -ExpandProperty tree - - foreach ($file in $Data) { - $Extension = [System.IO.Path]::GetExtension($file.path) - - if ($CodeExtensions.Contains($Extension)) { - $RepoFiles[$file.path] = 1 - } - } - } - catch { - Write-Host "Problems downloading the list of files from the repo. Url used: $RepoTreeURL . Execution will proceed without caching." - } - } - elseif ($GHRepoName -ne '' -or $GHCommit -ne '') { - Write-Host 'For using the http caching mechanism both GHRepoName and GHCommit should be informed.' - } - - if (Test-Path $ExtractPath) { - Remove-Item $ExtractPath -Force -Recurse -ErrorAction SilentlyContinue - } - - $ValidationFailures = 0 - - # Process each NuGet package in parallel - Get-ChildItem "$InputPath\*.symbols.nupkg" | - ForEach-Object { - Write-Host "Starting $($_.FullName)" - Start-Job -ScriptBlock $ValidatePackage -ArgumentList $_.FullName | Out-Null - $NumJobs = @(Get-Job -State 'Running').Count - - while ($NumJobs -ge $MaxParallelJobs) { - Write-Host "There are $NumJobs validation jobs running right now. Waiting $SecondsBetweenLoadChecks seconds to check again." - sleep $SecondsBetweenLoadChecks - $NumJobs = @(Get-Job -State 'Running').Count - } - - foreach ($Job in @(Get-Job -State 'Completed')) { - $jobResult = Wait-Job -Id $Job.Id | Receive-Job - CheckJobResult $jobResult.result $jobResult.packagePath ([ref]$ValidationFailures) -LogErrors - Remove-Job -Id $Job.Id - } - } - - foreach ($Job in @(Get-Job)) { - $jobResult = Wait-Job -Id $Job.Id | Receive-Job - CheckJobResult $jobResult.result $jobResult.packagePath ([ref]$ValidationFailures) - Remove-Job -Id $Job.Id - } - if ($ValidationFailures -gt 0) { - Write-PipelineTelemetryError -Category 'SourceLink' -Message "$ValidationFailures package(s) failed validation." - ExitWithExitCode 1 - } -} - -function InstallSourcelinkCli { - $sourcelinkCliPackageName = 'sourcelink' - - $dotnetRoot = InitializeDotNetCli -install:$true - $dotnet = "$dotnetRoot\dotnet.exe" - $toolList = & "$dotnet" tool list --global - - if (($toolList -like "*$sourcelinkCliPackageName*") -and ($toolList -like "*$sourcelinkCliVersion*")) { - Write-Host "SourceLink CLI version $sourcelinkCliVersion is already installed." - } - else { - Write-Host "Installing SourceLink CLI version $sourcelinkCliVersion..." - Write-Host 'You may need to restart your command window if this is the first dotnet tool you have installed.' - & "$dotnet" tool install $sourcelinkCliPackageName --version $sourcelinkCliVersion --verbosity "minimal" --global - } -} - -try { - InstallSourcelinkCli - - foreach ($Job in @(Get-Job)) { - Remove-Job -Id $Job.Id - } - - ValidateSourceLinkLinks -} -catch { - Write-Host $_.Exception - Write-Host $_.ScriptStackTrace - Write-PipelineTelemetryError -Category 'SourceLink' -Message $_ - ExitWithExitCode 1 -} diff --git a/eng/common/sdk-task.ps1 b/eng/common/sdk-task.ps1 index b64b66a62..68119de60 100644 --- a/eng/common/sdk-task.ps1 +++ b/eng/common/sdk-task.ps1 @@ -22,7 +22,7 @@ $warnAsError = if ($noWarnAsError) { $false } else { $true } function Print-Usage() { Write-Host "Common settings:" - Write-Host " -task Name of Arcade task (name of a project in SdkTasks directory of the Arcade SDK package)" + Write-Host " -task Name of Arcade task (name of a project in toolset directory of the Arcade SDK package)" Write-Host " -restore Restore dependencies" Write-Host " -verbosity Msbuild verbosity: q[uiet], m[inimal], n[ormal], d[etailed], and diag[nostic]" Write-Host " -help Print help and exit" @@ -66,20 +66,7 @@ try { if( $msbuildEngine -eq "vs") { # Ensure desktop MSBuild is available for sdk tasks. - if( -not ($GlobalJson.tools.PSObject.Properties.Name -contains "vs" )) { - $GlobalJson.tools | Add-Member -Name "vs" -Value (ConvertFrom-Json "{ `"version`": `"16.5`" }") -MemberType NoteProperty - } - if( -not ($GlobalJson.tools.PSObject.Properties.Name -match "xcopy-msbuild" )) { - $GlobalJson.tools | Add-Member -Name "xcopy-msbuild" -Value "18.0.0" -MemberType NoteProperty - } - if ($GlobalJson.tools."xcopy-msbuild".Trim() -ine "none") { - $xcopyMSBuildToolsFolder = InitializeXCopyMSBuild $GlobalJson.tools."xcopy-msbuild" -install $true - } - if ($xcopyMSBuildToolsFolder -eq $null) { - throw 'Unable to get xcopy downloadable version of msbuild' - } - - $global:_MSBuildExe = "$($xcopyMSBuildToolsFolder)\MSBuild\Current\Bin\MSBuild.exe" + $global:_MSBuildExe = InitializeVisualStudioMSBuild } $taskProject = GetSdkTaskProject $task diff --git a/eng/common/sdk-task.sh b/eng/common/sdk-task.sh index 3270f83fa..1cf71bb2a 100755 --- a/eng/common/sdk-task.sh +++ b/eng/common/sdk-task.sh @@ -2,7 +2,7 @@ show_usage() { echo "Common settings:" - echo " --task Name of Arcade task (name of a project in SdkTasks directory of the Arcade SDK package)" + echo " --task Name of Arcade task (name of a project in toolset directory of the Arcade SDK package)" echo " --restore Restore dependencies" echo " --verbosity Msbuild verbosity: q[uiet], m[inimal], n[ormal], d[etailed], and diag[nostic]" echo " --help Print help and exit" diff --git a/eng/common/sdl/NuGet.config b/eng/common/sdl/NuGet.config deleted file mode 100644 index 3849bdb3c..000000000 --- a/eng/common/sdl/NuGet.config +++ /dev/null @@ -1,18 +0,0 @@ - - - - - - - - - - - - - - - - - - diff --git a/eng/common/sdl/configure-sdl-tool.ps1 b/eng/common/sdl/configure-sdl-tool.ps1 deleted file mode 100644 index 27f5a4115..000000000 --- a/eng/common/sdl/configure-sdl-tool.ps1 +++ /dev/null @@ -1,130 +0,0 @@ -Param( - [string] $GuardianCliLocation, - [string] $WorkingDirectory, - [string] $TargetDirectory, - [string] $GdnFolder, - # The list of Guardian tools to configure. For each object in the array: - # - If the item is a [hashtable], it must contain these entries: - # - Name = The tool name as Guardian knows it. - # - Scenario = (Optional) Scenario-specific name for this configuration entry. It must be unique - # among all tool entries with the same Name. - # - Args = (Optional) Array of Guardian tool configuration args, like '@("Target > C:\temp")' - # - If the item is a [string] $v, it is treated as '@{ Name="$v" }' - [object[]] $ToolsList, - [string] $GuardianLoggerLevel='Standard', - # Optional: Additional params to add to any tool using CredScan. - [string[]] $CrScanAdditionalRunConfigParams, - # Optional: Additional params to add to any tool using PoliCheck. - [string[]] $PoliCheckAdditionalRunConfigParams, - # Optional: Additional params to add to any tool using CodeQL/Semmle. - [string[]] $CodeQLAdditionalRunConfigParams, - # Optional: Additional params to add to any tool using Binskim. - [string[]] $BinskimAdditionalRunConfigParams -) - -$ErrorActionPreference = 'Stop' -Set-StrictMode -Version 2.0 -$disableConfigureToolsetImport = $true -$global:LASTEXITCODE = 0 - -try { - # `tools.ps1` checks $ci to perform some actions. Since the SDL - # scripts don't necessarily execute in the same agent that run the - # build.ps1/sh script this variable isn't automatically set. - $ci = $true - . $PSScriptRoot\..\tools.ps1 - - # Normalize tools list: all in [hashtable] form with defined values for each key. - $ToolsList = $ToolsList | - ForEach-Object { - if ($_ -is [string]) { - $_ = @{ Name = $_ } - } - - if (-not ($_['Scenario'])) { $_.Scenario = "" } - if (-not ($_['Args'])) { $_.Args = @() } - $_ - } - - Write-Host "List of tools to configure:" - $ToolsList | ForEach-Object { $_ | Out-String | Write-Host } - - # We store config files in the r directory of .gdn - $gdnConfigPath = Join-Path $GdnFolder 'r' - $ValidPath = Test-Path $GuardianCliLocation - - if ($ValidPath -eq $False) - { - Write-PipelineTelemetryError -Force -Category 'Sdl' -Message "Invalid Guardian CLI Location." - ExitWithExitCode 1 - } - - foreach ($tool in $ToolsList) { - # Put together the name and scenario to make a unique key. - $toolConfigName = $tool.Name - if ($tool.Scenario) { - $toolConfigName += "_" + $tool.Scenario - } - - Write-Host "=== Configuring $toolConfigName..." - - $gdnConfigFile = Join-Path $gdnConfigPath "$toolConfigName-configure.gdnconfig" - - # For some tools, add default and automatic args. - switch -Exact ($tool.Name) { - 'credscan' { - if ($targetDirectory) { - $tool.Args += "`"TargetDirectory < $TargetDirectory`"" - } - $tool.Args += "`"OutputType < pre`"" - $tool.Args += $CrScanAdditionalRunConfigParams - } - 'policheck' { - if ($targetDirectory) { - $tool.Args += "`"Target < $TargetDirectory`"" - } - $tool.Args += $PoliCheckAdditionalRunConfigParams - } - {$_ -in 'semmle', 'codeql'} { - if ($targetDirectory) { - $tool.Args += "`"SourceCodeDirectory < $TargetDirectory`"" - } - $tool.Args += $CodeQLAdditionalRunConfigParams - } - 'binskim' { - if ($targetDirectory) { - # Binskim crashes due to specific PDBs. GitHub issue: https://github.com/microsoft/binskim/issues/924. - # We are excluding all `_.pdb` files from the scan. - $tool.Args += "`"Target < $TargetDirectory\**;-:file|$TargetDirectory\**\_.pdb`"" - } - $tool.Args += $BinskimAdditionalRunConfigParams - } - } - - # Create variable pointing to the args array directly so we can use splat syntax later. - $toolArgs = $tool.Args - - # Configure the tool. If args array is provided or the current tool has some default arguments - # defined, add "--args" and splat each element on the end. Arg format is "{Arg id} < {Value}", - # one per parameter. Doc page for "guardian configure": - # https://dev.azure.com/securitytools/SecurityIntegration/_wiki/wikis/Guardian/1395/configure - Exec-BlockVerbosely { - & $GuardianCliLocation configure ` - --working-directory $WorkingDirectory ` - --tool $tool.Name ` - --output-path $gdnConfigFile ` - --logger-level $GuardianLoggerLevel ` - --noninteractive ` - --force ` - $(if ($toolArgs) { "--args" }) @toolArgs - Exit-IfNZEC "Sdl" - } - - Write-Host "Created '$toolConfigName' configuration file: $gdnConfigFile" - } -} -catch { - Write-Host $_.ScriptStackTrace - Write-PipelineTelemetryError -Force -Category 'Sdl' -Message $_ - ExitWithExitCode 1 -} diff --git a/eng/common/sdl/execute-all-sdl-tools.ps1 b/eng/common/sdl/execute-all-sdl-tools.ps1 deleted file mode 100644 index 4715d75e9..000000000 --- a/eng/common/sdl/execute-all-sdl-tools.ps1 +++ /dev/null @@ -1,167 +0,0 @@ -Param( - [string] $GuardianPackageName, # Required: the name of guardian CLI package (not needed if GuardianCliLocation is specified) - [string] $NugetPackageDirectory, # Required: directory where NuGet packages are installed (not needed if GuardianCliLocation is specified) - [string] $GuardianCliLocation, # Optional: Direct location of Guardian CLI executable if GuardianPackageName & NugetPackageDirectory are not specified - [string] $Repository=$env:BUILD_REPOSITORY_NAME, # Required: the name of the repository (e.g. dotnet/arcade) - [string] $BranchName=$env:BUILD_SOURCEBRANCH, # Optional: name of branch or version of gdn settings; defaults to master - [string] $SourceDirectory=$env:BUILD_SOURCESDIRECTORY, # Required: the directory where source files are located - [string] $ArtifactsDirectory = (Join-Path $env:BUILD_ARTIFACTSTAGINGDIRECTORY ('artifacts')), # Required: the directory where build artifacts are located - [string] $AzureDevOpsAccessToken, # Required: access token for dnceng; should be provided via KeyVault - - # Optional: list of SDL tools to run on source code. See 'configure-sdl-tool.ps1' for tools list - # format. - [object[]] $SourceToolsList, - # Optional: list of SDL tools to run on built artifacts. See 'configure-sdl-tool.ps1' for tools - # list format. - [object[]] $ArtifactToolsList, - # Optional: list of SDL tools to run without automatically specifying a target directory. See - # 'configure-sdl-tool.ps1' for tools list format. - [object[]] $CustomToolsList, - - [bool] $TsaPublish=$False, # Optional: true will publish results to TSA; only set to true after onboarding to TSA; TSA is the automated framework used to upload test results as bugs. - [string] $TsaBranchName=$env:BUILD_SOURCEBRANCH, # Optional: required for TSA publish; defaults to $(Build.SourceBranchName); TSA is the automated framework used to upload test results as bugs. - [string] $TsaRepositoryName=$env:BUILD_REPOSITORY_NAME, # Optional: TSA repository name; will be generated automatically if not submitted; TSA is the automated framework used to upload test results as bugs. - [string] $BuildNumber=$env:BUILD_BUILDNUMBER, # Optional: required for TSA publish; defaults to $(Build.BuildNumber) - [bool] $UpdateBaseline=$False, # Optional: if true, will update the baseline in the repository; should only be run after fixing any issues which need to be fixed - [bool] $TsaOnboard=$False, # Optional: if true, will onboard the repository to TSA; should only be run once; TSA is the automated framework used to upload test results as bugs. - [string] $TsaInstanceUrl, # Optional: only needed if TsaOnboard or TsaPublish is true; the instance-url registered with TSA; TSA is the automated framework used to upload test results as bugs. - [string] $TsaCodebaseName, # Optional: only needed if TsaOnboard or TsaPublish is true; the name of the codebase registered with TSA; TSA is the automated framework used to upload test results as bugs. - [string] $TsaProjectName, # Optional: only needed if TsaOnboard or TsaPublish is true; the name of the project registered with TSA; TSA is the automated framework used to upload test results as bugs. - [string] $TsaNotificationEmail, # Optional: only needed if TsaOnboard is true; the email(s) which will receive notifications of TSA bug filings (e.g. alias@microsoft.com); TSA is the automated framework used to upload test results as bugs. - [string] $TsaCodebaseAdmin, # Optional: only needed if TsaOnboard is true; the aliases which are admins of the TSA codebase (e.g. DOMAIN\alias); TSA is the automated framework used to upload test results as bugs. - [string] $TsaBugAreaPath, # Optional: only needed if TsaOnboard is true; the area path where TSA will file bugs in AzDO; TSA is the automated framework used to upload test results as bugs. - [string] $TsaIterationPath, # Optional: only needed if TsaOnboard is true; the iteration path where TSA will file bugs in AzDO; TSA is the automated framework used to upload test results as bugs. - [string] $GuardianLoggerLevel='Standard', # Optional: the logger level for the Guardian CLI; options are Trace, Verbose, Standard, Warning, and Error - [string[]] $CrScanAdditionalRunConfigParams, # Optional: Additional Params to custom build a CredScan run config in the format @("xyz:abc","sdf:1") - [string[]] $PoliCheckAdditionalRunConfigParams, # Optional: Additional Params to custom build a Policheck run config in the format @("xyz:abc","sdf:1") - [string[]] $CodeQLAdditionalRunConfigParams, # Optional: Additional Params to custom build a Semmle/CodeQL run config in the format @("xyz < abc","sdf < 1") - [string[]] $BinskimAdditionalRunConfigParams, # Optional: Additional Params to custom build a Binskim run config in the format @("xyz < abc","sdf < 1") - [bool] $BreakOnFailure=$False # Optional: Fail the build if there were errors during the run -) - -try { - $ErrorActionPreference = 'Stop' - Set-StrictMode -Version 2.0 - $disableConfigureToolsetImport = $true - $global:LASTEXITCODE = 0 - - # `tools.ps1` checks $ci to perform some actions. Since the SDL - # scripts don't necessarily execute in the same agent that run the - # build.ps1/sh script this variable isn't automatically set. - $ci = $true - . $PSScriptRoot\..\tools.ps1 - - #Replace repo names to the format of org/repo - if (!($Repository.contains('/'))) { - $RepoName = $Repository -replace '(.*?)-(.*)', '$1/$2'; - } - else{ - $RepoName = $Repository; - } - - if ($GuardianPackageName) { - $guardianCliLocation = Join-Path $NugetPackageDirectory (Join-Path $GuardianPackageName (Join-Path 'tools' 'guardian.cmd')) - } else { - $guardianCliLocation = $GuardianCliLocation - } - - $workingDirectory = (Split-Path $SourceDirectory -Parent) - $ValidPath = Test-Path $guardianCliLocation - - if ($ValidPath -eq $False) - { - Write-PipelineTelemetryError -Force -Category 'Sdl' -Message 'Invalid Guardian CLI Location.' - ExitWithExitCode 1 - } - - Exec-BlockVerbosely { - & $(Join-Path $PSScriptRoot 'init-sdl.ps1') -GuardianCliLocation $guardianCliLocation -Repository $RepoName -BranchName $BranchName -WorkingDirectory $workingDirectory -AzureDevOpsAccessToken $AzureDevOpsAccessToken -GuardianLoggerLevel $GuardianLoggerLevel - } - $gdnFolder = Join-Path $workingDirectory '.gdn' - - if ($TsaOnboard) { - if ($TsaCodebaseName -and $TsaNotificationEmail -and $TsaCodebaseAdmin -and $TsaBugAreaPath) { - Exec-BlockVerbosely { - & $guardianCliLocation tsa-onboard --codebase-name "$TsaCodebaseName" --notification-alias "$TsaNotificationEmail" --codebase-admin "$TsaCodebaseAdmin" --instance-url "$TsaInstanceUrl" --project-name "$TsaProjectName" --area-path "$TsaBugAreaPath" --iteration-path "$TsaIterationPath" --working-directory $workingDirectory --logger-level $GuardianLoggerLevel - } - if ($LASTEXITCODE -ne 0) { - Write-PipelineTelemetryError -Force -Category 'Sdl' -Message "Guardian tsa-onboard failed with exit code $LASTEXITCODE." - ExitWithExitCode $LASTEXITCODE - } - } else { - Write-PipelineTelemetryError -Force -Category 'Sdl' -Message 'Could not onboard to TSA -- not all required values ($TsaCodebaseName, $TsaNotificationEmail, $TsaCodebaseAdmin, $TsaBugAreaPath) were specified.' - ExitWithExitCode 1 - } - } - - # Configure a list of tools with a default target directory. Populates the ".gdn/r" directory. - function Configure-ToolsList([object[]] $tools, [string] $targetDirectory) { - if ($tools -and $tools.Count -gt 0) { - Exec-BlockVerbosely { - & $(Join-Path $PSScriptRoot 'configure-sdl-tool.ps1') ` - -GuardianCliLocation $guardianCliLocation ` - -WorkingDirectory $workingDirectory ` - -TargetDirectory $targetDirectory ` - -GdnFolder $gdnFolder ` - -ToolsList $tools ` - -AzureDevOpsAccessToken $AzureDevOpsAccessToken ` - -GuardianLoggerLevel $GuardianLoggerLevel ` - -CrScanAdditionalRunConfigParams $CrScanAdditionalRunConfigParams ` - -PoliCheckAdditionalRunConfigParams $PoliCheckAdditionalRunConfigParams ` - -CodeQLAdditionalRunConfigParams $CodeQLAdditionalRunConfigParams ` - -BinskimAdditionalRunConfigParams $BinskimAdditionalRunConfigParams - if ($BreakOnFailure) { - Exit-IfNZEC "Sdl" - } - } - } - } - - # Configure Artifact and Source tools with default Target directories. - Configure-ToolsList $ArtifactToolsList $ArtifactsDirectory - Configure-ToolsList $SourceToolsList $SourceDirectory - # Configure custom tools with no default Target directory. - Configure-ToolsList $CustomToolsList $null - - # At this point, all tools are configured in the ".gdn" directory. Run them all in a single call. - # (If we used "run" multiple times, each run would overwrite data from earlier runs.) - Exec-BlockVerbosely { - & $(Join-Path $PSScriptRoot 'run-sdl.ps1') ` - -GuardianCliLocation $guardianCliLocation ` - -WorkingDirectory $SourceDirectory ` - -UpdateBaseline $UpdateBaseline ` - -GdnFolder $gdnFolder - } - - if ($TsaPublish) { - if ($TsaBranchName -and $BuildNumber) { - if (-not $TsaRepositoryName) { - $TsaRepositoryName = "$($Repository)-$($BranchName)" - } - Exec-BlockVerbosely { - & $guardianCliLocation tsa-publish --all-tools --repository-name "$TsaRepositoryName" --branch-name "$TsaBranchName" --build-number "$BuildNumber" --onboard $True --codebase-name "$TsaCodebaseName" --notification-alias "$TsaNotificationEmail" --codebase-admin "$TsaCodebaseAdmin" --instance-url "$TsaInstanceUrl" --project-name "$TsaProjectName" --area-path "$TsaBugAreaPath" --iteration-path "$TsaIterationPath" --working-directory $workingDirectory --logger-level $GuardianLoggerLevel - } - if ($LASTEXITCODE -ne 0) { - Write-PipelineTelemetryError -Force -Category 'Sdl' -Message "Guardian tsa-publish failed with exit code $LASTEXITCODE." - ExitWithExitCode $LASTEXITCODE - } - } else { - Write-PipelineTelemetryError -Force -Category 'Sdl' -Message 'Could not publish to TSA -- not all required values ($TsaBranchName, $BuildNumber) were specified.' - ExitWithExitCode 1 - } - } - - if ($BreakOnFailure) { - Write-Host "Failing the build in case of breaking results..." - Exec-BlockVerbosely { - & $guardianCliLocation break --working-directory $workingDirectory --logger-level $GuardianLoggerLevel - } - } else { - Write-Host "Letting the build pass even if there were breaking results..." - } -} -catch { - Write-Host $_.ScriptStackTrace - Write-PipelineTelemetryError -Force -Category 'Sdl' -Message $_ - exit 1 -} diff --git a/eng/common/sdl/extract-artifact-archives.ps1 b/eng/common/sdl/extract-artifact-archives.ps1 deleted file mode 100644 index 68da4fbf2..000000000 --- a/eng/common/sdl/extract-artifact-archives.ps1 +++ /dev/null @@ -1,63 +0,0 @@ -# This script looks for each archive file in a directory and extracts it into the target directory. -# For example, the file "$InputPath/bin.tar.gz" extracts to "$ExtractPath/bin.tar.gz.extracted/**". -# Uses the "tar" utility added to Windows 10 / Windows 2019 that supports tar.gz and zip. -param( - # Full path to directory where archives are stored. - [Parameter(Mandatory=$true)][string] $InputPath, - # Full path to directory to extract archives into. May be the same as $InputPath. - [Parameter(Mandatory=$true)][string] $ExtractPath -) - -$ErrorActionPreference = 'Stop' -Set-StrictMode -Version 2.0 - -$disableConfigureToolsetImport = $true - -try { - # `tools.ps1` checks $ci to perform some actions. Since the SDL - # scripts don't necessarily execute in the same agent that run the - # build.ps1/sh script this variable isn't automatically set. - $ci = $true - . $PSScriptRoot\..\tools.ps1 - - Measure-Command { - $jobs = @() - - # Find archive files for non-Windows and Windows builds. - $archiveFiles = @( - Get-ChildItem (Join-Path $InputPath "*.tar.gz") - Get-ChildItem (Join-Path $InputPath "*.zip") - ) - - foreach ($targzFile in $archiveFiles) { - $jobs += Start-Job -ScriptBlock { - $file = $using:targzFile - $fileName = [System.IO.Path]::GetFileName($file) - $extractDir = Join-Path $using:ExtractPath "$fileName.extracted" - - New-Item $extractDir -ItemType Directory -Force | Out-Null - - Write-Host "Extracting '$file' to '$extractDir'..." - - # Pipe errors to stdout to prevent PowerShell detecting them and quitting the job early. - # This type of quit skips the catch, so we wouldn't be able to tell which file triggered the - # error. Save output so it can be stored in the exception string along with context. - $output = tar -xf $file -C $extractDir 2>&1 - # Handle NZEC manually rather than using Exit-IfNZEC: we are in a background job, so we - # don't have access to the outer scope. - if ($LASTEXITCODE -ne 0) { - throw "Error extracting '$file': non-zero exit code ($LASTEXITCODE). Output: '$output'" - } - - Write-Host "Extracted to $extractDir" - } - } - - Receive-Job $jobs -Wait - } -} -catch { - Write-Host $_ - Write-PipelineTelemetryError -Force -Category 'Sdl' -Message $_ - ExitWithExitCode 1 -} diff --git a/eng/common/sdl/extract-artifact-packages.ps1 b/eng/common/sdl/extract-artifact-packages.ps1 deleted file mode 100644 index f031ed5b2..000000000 --- a/eng/common/sdl/extract-artifact-packages.ps1 +++ /dev/null @@ -1,82 +0,0 @@ -param( - [Parameter(Mandatory=$true)][string] $InputPath, # Full path to directory where artifact packages are stored - [Parameter(Mandatory=$true)][string] $ExtractPath # Full path to directory where the packages will be extracted -) - -$ErrorActionPreference = 'Stop' -Set-StrictMode -Version 2.0 - -$disableConfigureToolsetImport = $true - -function ExtractArtifacts { - if (!(Test-Path $InputPath)) { - Write-Host "Input Path does not exist: $InputPath" - ExitWithExitCode 0 - } - $Jobs = @() - Get-ChildItem "$InputPath\*.nupkg" | - ForEach-Object { - $Jobs += Start-Job -ScriptBlock $ExtractPackage -ArgumentList $_.FullName - } - - foreach ($Job in $Jobs) { - Wait-Job -Id $Job.Id | Receive-Job - } -} - -try { - # `tools.ps1` checks $ci to perform some actions. Since the SDL - # scripts don't necessarily execute in the same agent that run the - # build.ps1/sh script this variable isn't automatically set. - $ci = $true - . $PSScriptRoot\..\tools.ps1 - - $ExtractPackage = { - param( - [string] $PackagePath # Full path to a NuGet package - ) - - if (!(Test-Path $PackagePath)) { - Write-PipelineTelemetryError -Category 'Build' -Message "Input file does not exist: $PackagePath" - ExitWithExitCode 1 - } - - $RelevantExtensions = @('.dll', '.exe', '.pdb') - Write-Host -NoNewLine 'Extracting ' ([System.IO.Path]::GetFileName($PackagePath)) '...' - - $PackageId = [System.IO.Path]::GetFileNameWithoutExtension($PackagePath) - $ExtractPath = Join-Path -Path $using:ExtractPath -ChildPath $PackageId - - Add-Type -AssemblyName System.IO.Compression.FileSystem - - [System.IO.Directory]::CreateDirectory($ExtractPath); - - try { - $zip = [System.IO.Compression.ZipFile]::OpenRead($PackagePath) - - $zip.Entries | - Where-Object {$RelevantExtensions -contains [System.IO.Path]::GetExtension($_.Name)} | - ForEach-Object { - $TargetPath = Join-Path -Path $ExtractPath -ChildPath (Split-Path -Path $_.FullName) - [System.IO.Directory]::CreateDirectory($TargetPath); - - $TargetFile = Join-Path -Path $ExtractPath -ChildPath $_.FullName - [System.IO.Compression.ZipFileExtensions]::ExtractToFile($_, $TargetFile) - } - } - catch { - Write-Host $_ - Write-PipelineTelemetryError -Force -Category 'Sdl' -Message $_ - ExitWithExitCode 1 - } - finally { - $zip.Dispose() - } - } - Measure-Command { ExtractArtifacts } -} -catch { - Write-Host $_ - Write-PipelineTelemetryError -Force -Category 'Sdl' -Message $_ - ExitWithExitCode 1 -} diff --git a/eng/common/sdl/init-sdl.ps1 b/eng/common/sdl/init-sdl.ps1 deleted file mode 100644 index 3ac1d92b3..000000000 --- a/eng/common/sdl/init-sdl.ps1 +++ /dev/null @@ -1,55 +0,0 @@ -Param( - [string] $GuardianCliLocation, - [string] $Repository, - [string] $BranchName='master', - [string] $WorkingDirectory, - [string] $AzureDevOpsAccessToken, - [string] $GuardianLoggerLevel='Standard' -) - -$ErrorActionPreference = 'Stop' -Set-StrictMode -Version 2.0 -$disableConfigureToolsetImport = $true -$global:LASTEXITCODE = 0 - -# `tools.ps1` checks $ci to perform some actions. Since the SDL -# scripts don't necessarily execute in the same agent that run the -# build.ps1/sh script this variable isn't automatically set. -$ci = $true -. $PSScriptRoot\..\tools.ps1 - -# Don't display the console progress UI - it's a huge perf hit -$ProgressPreference = 'SilentlyContinue' - -# Construct basic auth from AzDO access token; construct URI to the repository's gdn folder stored in that repository; construct location of zip file -$encodedPat = [Convert]::ToBase64String([System.Text.Encoding]::ASCII.GetBytes(":$AzureDevOpsAccessToken")) -$escapedRepository = [Uri]::EscapeDataString("/$Repository/$BranchName/.gdn") -$uri = "https://dev.azure.com/dnceng/internal/_apis/git/repositories/sdl-tool-cfg/Items?path=$escapedRepository&versionDescriptor[versionOptions]=0&`$format=zip&api-version=5.0" -$zipFile = "$WorkingDirectory/gdn.zip" - -Add-Type -AssemblyName System.IO.Compression.FileSystem -$gdnFolder = (Join-Path $WorkingDirectory '.gdn') - -try { - # if the folder does not exist, we'll do a guardian init and push it to the remote repository - Write-Host 'Initializing Guardian...' - Write-Host "$GuardianCliLocation init --working-directory $WorkingDirectory --logger-level $GuardianLoggerLevel" - & $GuardianCliLocation init --working-directory $WorkingDirectory --logger-level $GuardianLoggerLevel - if ($LASTEXITCODE -ne 0) { - Write-PipelineTelemetryError -Force -Category 'Build' -Message "Guardian init failed with exit code $LASTEXITCODE." - ExitWithExitCode $LASTEXITCODE - } - # We create the mainbaseline so it can be edited later - Write-Host "$GuardianCliLocation baseline --working-directory $WorkingDirectory --name mainbaseline" - & $GuardianCliLocation baseline --working-directory $WorkingDirectory --name mainbaseline - if ($LASTEXITCODE -ne 0) { - Write-PipelineTelemetryError -Force -Category 'Build' -Message "Guardian baseline failed with exit code $LASTEXITCODE." - ExitWithExitCode $LASTEXITCODE - } - ExitWithExitCode 0 -} -catch { - Write-Host $_.ScriptStackTrace - Write-PipelineTelemetryError -Force -Category 'Sdl' -Message $_ - ExitWithExitCode 1 -} diff --git a/eng/common/sdl/packages.config b/eng/common/sdl/packages.config deleted file mode 100644 index e5f543ea6..000000000 --- a/eng/common/sdl/packages.config +++ /dev/null @@ -1,4 +0,0 @@ - - - - diff --git a/eng/common/sdl/run-sdl.ps1 b/eng/common/sdl/run-sdl.ps1 deleted file mode 100644 index 2eac8c78f..000000000 --- a/eng/common/sdl/run-sdl.ps1 +++ /dev/null @@ -1,49 +0,0 @@ -Param( - [string] $GuardianCliLocation, - [string] $WorkingDirectory, - [string] $GdnFolder, - [string] $UpdateBaseline, - [string] $GuardianLoggerLevel='Standard' -) - -$ErrorActionPreference = 'Stop' -Set-StrictMode -Version 2.0 -$disableConfigureToolsetImport = $true -$global:LASTEXITCODE = 0 - -try { - # `tools.ps1` checks $ci to perform some actions. Since the SDL - # scripts don't necessarily execute in the same agent that run the - # build.ps1/sh script this variable isn't automatically set. - $ci = $true - . $PSScriptRoot\..\tools.ps1 - - # We store config files in the r directory of .gdn - $gdnConfigPath = Join-Path $GdnFolder 'r' - $ValidPath = Test-Path $GuardianCliLocation - - if ($ValidPath -eq $False) - { - Write-PipelineTelemetryError -Force -Category 'Sdl' -Message "Invalid Guardian CLI Location." - ExitWithExitCode 1 - } - - $gdnConfigFiles = Get-ChildItem $gdnConfigPath -Recurse -Include '*.gdnconfig' - Write-Host "Discovered Guardian config files:" - $gdnConfigFiles | Out-String | Write-Host - - Exec-BlockVerbosely { - & $GuardianCliLocation run ` - --working-directory $WorkingDirectory ` - --baseline mainbaseline ` - --update-baseline $UpdateBaseline ` - --logger-level $GuardianLoggerLevel ` - --config @gdnConfigFiles - Exit-IfNZEC "Sdl" - } -} -catch { - Write-Host $_.ScriptStackTrace - Write-PipelineTelemetryError -Force -Category 'Sdl' -Message $_ - ExitWithExitCode 1 -} diff --git a/eng/common/sdl/sdl.ps1 b/eng/common/sdl/sdl.ps1 deleted file mode 100644 index 648c5068d..000000000 --- a/eng/common/sdl/sdl.ps1 +++ /dev/null @@ -1,38 +0,0 @@ - -function Install-Gdn { - param( - [Parameter(Mandatory=$true)] - [string]$Path, - - # If omitted, install the latest version of Guardian, otherwise install that specific version. - [string]$Version - ) - - $ErrorActionPreference = 'Stop' - Set-StrictMode -Version 2.0 - $disableConfigureToolsetImport = $true - $global:LASTEXITCODE = 0 - - # `tools.ps1` checks $ci to perform some actions. Since the SDL - # scripts don't necessarily execute in the same agent that run the - # build.ps1/sh script this variable isn't automatically set. - $ci = $true - . $PSScriptRoot\..\tools.ps1 - - $argumentList = @("install", "Microsoft.Guardian.Cli", "-Source https://securitytools.pkgs.visualstudio.com/_packaging/Guardian/nuget/v3/index.json", "-OutputDirectory $Path", "-NonInteractive", "-NoCache") - - if ($Version) { - $argumentList += "-Version $Version" - } - - Start-Process nuget -Verbose -ArgumentList $argumentList -NoNewWindow -Wait - - $gdnCliPath = Get-ChildItem -Filter guardian.cmd -Recurse -Path $Path - - if (!$gdnCliPath) - { - Write-PipelineTelemetryError -Category 'Sdl' -Message 'Failure installing Guardian' - } - - return $gdnCliPath.FullName -} \ No newline at end of file diff --git a/eng/common/sdl/trim-assets-version.ps1 b/eng/common/sdl/trim-assets-version.ps1 deleted file mode 100644 index 0daa2a9e9..000000000 --- a/eng/common/sdl/trim-assets-version.ps1 +++ /dev/null @@ -1,75 +0,0 @@ -<# -.SYNOPSIS -Install and run the 'Microsoft.DotNet.VersionTools.Cli' tool with the 'trim-artifacts-version' command to trim the version from the NuGet assets file name. - -.PARAMETER InputPath -Full path to directory where artifact packages are stored - -.PARAMETER Recursive -Search for NuGet packages recursively - -#> - -Param( - [string] $InputPath, - [bool] $Recursive = $true -) - -$CliToolName = "Microsoft.DotNet.VersionTools.Cli" - -function Install-VersionTools-Cli { - param( - [Parameter(Mandatory=$true)][string]$Version - ) - - Write-Host "Installing the package '$CliToolName' with a version of '$version' ..." - $feed = "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json" - - $argumentList = @("tool", "install", "--local", "$CliToolName", "--add-source $feed", "--no-cache", "--version $Version", "--create-manifest-if-needed") - Start-Process "$dotnet" -Verbose -ArgumentList $argumentList -NoNewWindow -Wait -} - -# ------------------------------------------------------------------- - -if (!(Test-Path $InputPath)) { - Write-Host "Input Path '$InputPath' does not exist" - ExitWithExitCode 1 -} - -$ErrorActionPreference = 'Stop' -Set-StrictMode -Version 2.0 - -$disableConfigureToolsetImport = $true -$global:LASTEXITCODE = 0 - -# `tools.ps1` checks $ci to perform some actions. Since the SDL -# scripts don't necessarily execute in the same agent that run the -# build.ps1/sh script this variable isn't automatically set. -$ci = $true -. $PSScriptRoot\..\tools.ps1 - -try { - $dotnetRoot = InitializeDotNetCli -install:$true - $dotnet = "$dotnetRoot\dotnet.exe" - - $toolsetVersion = Read-ArcadeSdkVersion - Install-VersionTools-Cli -Version $toolsetVersion - - $cliToolFound = (& "$dotnet" tool list --local | Where-Object {$_.Split(' ')[0] -eq $CliToolName}) - if ($null -eq $cliToolFound) { - Write-PipelineTelemetryError -Force -Category 'Sdl' -Message "The '$CliToolName' tool is not installed." - ExitWithExitCode 1 - } - - Exec-BlockVerbosely { - & "$dotnet" $CliToolName trim-assets-version ` - --assets-path $InputPath ` - --recursive $Recursive - Exit-IfNZEC "Sdl" - } -} -catch { - Write-Host $_ - Write-PipelineTelemetryError -Force -Category 'Sdl' -Message $_ - ExitWithExitCode 1 -} diff --git a/eng/common/template-guidance.md b/eng/common/template-guidance.md index e2b07a865..f772aa3d7 100644 --- a/eng/common/template-guidance.md +++ b/eng/common/template-guidance.md @@ -71,7 +71,6 @@ eng\common\ source-build.yml (shim) source-index-stage1.yml (shim) jobs\ - codeql-build.yml (shim) jobs.yml (shim) source-build.yml (shim) post-build\ @@ -88,7 +87,6 @@ eng\common\ source-build.yml (shim) variables\ pool-providers.yml (logic + redirect) # templates/variables/pool-providers.yml will redirect to templates-official/variables/pool-providers.yml if you are running in the internal project - sdl-variables.yml (logic) core-templates\ job\ job.yml (logic) @@ -97,7 +95,6 @@ eng\common\ source-build.yml (logic) source-index-stage1.yml (logic) jobs\ - codeql-build.yml (logic) jobs.yml (logic) source-build.yml (logic) post-build\ diff --git a/eng/common/templates-official/jobs/codeql-build.yml b/eng/common/templates-official/jobs/codeql-build.yml deleted file mode 100644 index a726322ec..000000000 --- a/eng/common/templates-official/jobs/codeql-build.yml +++ /dev/null @@ -1,7 +0,0 @@ -jobs: -- template: /eng/common/core-templates/jobs/codeql-build.yml - parameters: - is1ESPipeline: true - - ${{ each parameter in parameters }}: - ${{ parameter.key }}: ${{ parameter.value }} diff --git a/eng/common/templates-official/variables/sdl-variables.yml b/eng/common/templates-official/variables/sdl-variables.yml deleted file mode 100644 index f1311bbb1..000000000 --- a/eng/common/templates-official/variables/sdl-variables.yml +++ /dev/null @@ -1,7 +0,0 @@ -variables: -# The Guardian version specified in 'eng/common/sdl/packages.config'. This value must be kept in -# sync with the packages.config file. -- name: DefaultGuardianVersion - value: 0.109.0 -- name: GuardianPackagesConfigFile - value: $(System.DefaultWorkingDirectory)\eng\common\sdl\packages.config \ No newline at end of file diff --git a/eng/common/templates/jobs/codeql-build.yml b/eng/common/templates/jobs/codeql-build.yml deleted file mode 100644 index 517f24d6a..000000000 --- a/eng/common/templates/jobs/codeql-build.yml +++ /dev/null @@ -1,7 +0,0 @@ -jobs: -- template: /eng/common/core-templates/jobs/codeql-build.yml - parameters: - is1ESPipeline: false - - ${{ each parameter in parameters }}: - ${{ parameter.key }}: ${{ parameter.value }} diff --git a/eng/common/templates/steps/vmr-sync.yml b/eng/common/templates/steps/vmr-sync.yml index eb619c502..cdc6a28ff 100644 --- a/eng/common/templates/steps/vmr-sync.yml +++ b/eng/common/templates/steps/vmr-sync.yml @@ -45,11 +45,11 @@ steps: workingDirectory: ${{ parameters.vmrPath }} - script: | - ./eng/common/vmr-sync.sh \ - --vmr ${{ parameters.vmrPath }} \ - --tmp $(Agent.TempDirectory) \ - --azdev-pat '$(dn-bot-all-orgs-code-r)' \ - --ci \ + ./eng/common/vmr-sync.sh \ + --vmr ${{ parameters.vmrPath }} \ + --tmp $(Agent.TempDirectory) \ + --azdev-pat '$(AzdoToken)' \ + --ci \ --debug if [ "$?" -ne 0 ]; then @@ -67,11 +67,11 @@ steps: condition: eq(variables['Agent.OS'], 'Windows_NT') - powershell: | - ./eng/common/vmr-sync.ps1 ` - -vmr ${{ parameters.vmrPath }} ` - -tmp $(Agent.TempDirectory) ` - -azdevPat '$(dn-bot-all-orgs-code-r)' ` - -ci ` + ./eng/common/vmr-sync.ps1 ` + -vmr ${{ parameters.vmrPath }} ` + -tmp $(Agent.TempDirectory) ` + -azdevPat '$(AzdoToken)' ` + -ci ` -debugOutput if ($LASTEXITCODE -ne 0) { diff --git a/eng/common/tools.ps1 b/eng/common/tools.ps1 index 977a2d4b1..77dfd5eaa 100644 --- a/eng/common/tools.ps1 +++ b/eng/common/tools.ps1 @@ -13,12 +13,6 @@ # Set to true to output binary log from msbuild. Note that emitting binary log slows down the build. [bool]$binaryLog = if (Test-Path variable:binaryLog) { $binaryLog } else { $ci -and !$excludeCIBinarylog } -# Set to true to use the pipelines logger which will enable Azure logging output. -# https://github.com/Microsoft/azure-pipelines-tasks/blob/master/docs/authoring/commands.md -# This flag is meant as a temporary opt-opt for the feature while validate it across -# our consumers. It will be deleted in the future. -[bool]$pipelinesLog = if (Test-Path variable:pipelinesLog) { $pipelinesLog } else { $ci } - # Turns on machine preparation/clean up code that changes the machine state (e.g. kills build processes). [bool]$prepareMachine = if (Test-Path variable:prepareMachine) { $prepareMachine } else { $false } @@ -34,6 +28,9 @@ # Configures warning treatment in msbuild. [bool]$warnAsError = if (Test-Path variable:warnAsError) { $warnAsError } else { $true } +# Specifies semi-colon delimited list of warning codes that should not be treated as errors. +[string]$warnNotAsError = if (Test-Path variable:warnNotAsError) { $warnNotAsError } else { '' } + # Specifies which msbuild engine to use for build: 'vs', 'dotnet' or unspecified (determined based on presence of tools.vs in global.json). [string]$msbuildEngine = if (Test-Path variable:msbuildEngine) { $msbuildEngine } else { $null } @@ -157,9 +154,6 @@ function InitializeDotNetCli([bool]$install, [bool]$createSdkLocationFile) { return $global:_DotNetInstallDir } - # Don't resolve runtime, shared framework, or SDK from other locations to ensure build determinism - $env:DOTNET_MULTILEVEL_LOOKUP=0 - # Disable first run since we do not need all ASP.NET packages restored. $env:DOTNET_NOLOGO=1 @@ -185,7 +179,11 @@ function InitializeDotNetCli([bool]$install, [bool]$createSdkLocationFile) { if ((-not $globalJsonHasRuntimes) -and (-not [string]::IsNullOrEmpty($env:DOTNET_INSTALL_DIR)) -and (Test-Path(Join-Path $env:DOTNET_INSTALL_DIR "sdk\$dotnetSdkVersion"))) { $dotnetRoot = $env:DOTNET_INSTALL_DIR } else { - $dotnetRoot = Join-Path $RepoRoot '.dotnet' + if (-not [string]::IsNullOrEmpty($env:DOTNET_GLOBAL_INSTALL_DIR)) { + $dotnetRoot = $env:DOTNET_GLOBAL_INSTALL_DIR + } else { + $dotnetRoot = Join-Path $RepoRoot '.dotnet' + } if (-not (Test-Path(Join-Path $dotnetRoot "sdk\$dotnetSdkVersion"))) { if ($install) { @@ -225,7 +223,6 @@ function InitializeDotNetCli([bool]$install, [bool]$createSdkLocationFile) { # Make Sure that our bootstrapped dotnet cli is available in future steps of the Azure Pipelines build Write-PipelinePrependPath -Path $dotnetRoot - Write-PipelineSetVariable -Name 'DOTNET_MULTILEVEL_LOOKUP' -Value '0' Write-PipelineSetVariable -Name 'DOTNET_NOLOGO' -Value '1' return $global:_DotNetInstallDir = $dotnetRoot @@ -299,6 +296,8 @@ function InstallDotNet([string] $dotnetRoot, $dotnetVersionLabel = "'sdk v$version'" + # For performance this check is duplicated in src/Microsoft.DotNet.Arcade.Sdk/src/InstallDotNetCore.cs + # if you are making changes here, consider if you need to make changes there as well. if ($runtime -ne '' -and $runtime -ne 'sdk') { $runtimePath = $dotnetRoot $runtimePath = $runtimePath + "\shared" @@ -374,12 +373,11 @@ function InstallDotNet([string] $dotnetRoot, # # 1. MSBuild from an active VS command prompt # 2. MSBuild from a compatible VS installation -# 3. MSBuild from the xcopy tool package # # Returns full path to msbuild.exe. # Throws on failure. # -function InitializeVisualStudioMSBuild([bool]$install, [object]$vsRequirements = $null) { +function InitializeVisualStudioMSBuild([object]$vsRequirements = $null) { if (-not (IsWindowsPlatform)) { throw "Cannot initialize Visual Studio on non-Windows" } @@ -389,13 +387,7 @@ function InitializeVisualStudioMSBuild([bool]$install, [object]$vsRequirements = } # Minimum VS version to require. - $vsMinVersionReqdStr = '17.7' - $vsMinVersionReqd = [Version]::new($vsMinVersionReqdStr) - - # If the version of msbuild is going to be xcopied, - # use this version. Version matches a package here: - # https://dev.azure.com/dnceng/public/_artifacts/feed/dotnet-eng/NuGet/Microsoft.DotNet.Arcade.MSBuild.Xcopy/versions/18.0.0 - $defaultXCopyMSBuildVersion = '18.0.0' + $vsMinVersionReqdStr = '18.0' if (!$vsRequirements) { if (Get-Member -InputObject $GlobalJson.tools -Name 'vs') { @@ -425,46 +417,16 @@ function InitializeVisualStudioMSBuild([bool]$install, [object]$vsRequirements = } } - # Locate Visual Studio installation or download x-copy msbuild. + # Locate Visual Studio installation. $vsInfo = LocateVisualStudio $vsRequirements - if ($vsInfo -ne $null -and $env:ForceUseXCopyMSBuild -eq $null) { + if ($vsInfo -ne $null) { # Ensure vsInstallDir has a trailing slash $vsInstallDir = Join-Path $vsInfo.installationPath "\" $vsMajorVersion = $vsInfo.installationVersion.Split('.')[0] InitializeVisualStudioEnvironmentVariables $vsInstallDir $vsMajorVersion } else { - if (Get-Member -InputObject $GlobalJson.tools -Name 'xcopy-msbuild') { - $xcopyMSBuildVersion = $GlobalJson.tools.'xcopy-msbuild' - $vsMajorVersion = $xcopyMSBuildVersion.Split('.')[0] - } else { - #if vs version provided in global.json is incompatible (too low) then use the default version for xcopy msbuild download - if($vsMinVersion -lt $vsMinVersionReqd){ - Write-Host "Using xcopy-msbuild version of $defaultXCopyMSBuildVersion since VS version $vsMinVersionStr provided in global.json is not compatible" - $xcopyMSBuildVersion = $defaultXCopyMSBuildVersion - $vsMajorVersion = $xcopyMSBuildVersion.Split('.')[0] - } - else{ - # If the VS version IS compatible, look for an xcopy msbuild package - # with a version matching VS. - # Note: If this version does not exist, then an explicit version of xcopy msbuild - # can be specified in global.json. This will be required for pre-release versions of msbuild. - $vsMajorVersion = $vsMinVersion.Major - $vsMinorVersion = $vsMinVersion.Minor - $xcopyMSBuildVersion = "$vsMajorVersion.$vsMinorVersion.0" - } - } - - $vsInstallDir = $null - if ($xcopyMSBuildVersion.Trim() -ine "none") { - $vsInstallDir = InitializeXCopyMSBuild $xcopyMSBuildVersion $install - if ($vsInstallDir -eq $null) { - throw "Could not xcopy msbuild. Please check that package 'Microsoft.DotNet.Arcade.MSBuild.Xcopy @ $xcopyMSBuildVersion' exists on feed 'dotnet-eng'." - } - } - if ($vsInstallDir -eq $null) { - throw 'Unable to find Visual Studio that has required version and components installed' - } + throw 'Unable to find Visual Studio that has required version and components installed' } $msbuildVersionDir = if ([int]$vsMajorVersion -lt 16) { "$vsMajorVersion.0" } else { "Current" } @@ -491,38 +453,6 @@ function InitializeVisualStudioEnvironmentVariables([string] $vsInstallDir, [str } } -function InstallXCopyMSBuild([string]$packageVersion) { - return InitializeXCopyMSBuild $packageVersion -install $true -} - -function InitializeXCopyMSBuild([string]$packageVersion, [bool]$install) { - $packageName = 'Microsoft.DotNet.Arcade.MSBuild.Xcopy' - $packageDir = Join-Path $ToolsDir "msbuild\$packageVersion" - $packagePath = Join-Path $packageDir "$packageName.$packageVersion.nupkg" - - if (!(Test-Path $packageDir)) { - if (!$install) { - return $null - } - - Create-Directory $packageDir - - Write-Host "Downloading $packageName $packageVersion" - $ProgressPreference = 'SilentlyContinue' # Don't display the console progress UI - it's a huge perf hit - Retry({ - Invoke-WebRequest "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/flat2/$packageName/$packageVersion/$packageName.$packageVersion.nupkg" -UseBasicParsing -OutFile $packagePath - }) - - if (!(Test-Path $packagePath)) { - Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "See https://dev.azure.com/dnceng/internal/_wiki/wikis/DNCEng%20Services%20Wiki/1074/Updating-Microsoft.DotNet.Arcade.MSBuild.Xcopy-WAS-RoslynTools.MSBuild-(xcopy-msbuild)-generation?anchor=troubleshooting for help troubleshooting issues with XCopy MSBuild" - throw - } - Unzip $packagePath $packageDir - } - - return Join-Path $packageDir 'tools' -} - # # Locates Visual Studio instance that meets the minimal requirements specified by tools.vs object in global.json. # @@ -544,7 +474,6 @@ function LocateVisualStudio([object]$vsRequirements = $null){ if (Get-Member -InputObject $GlobalJson.tools -Name 'vswhere') { $vswhereVersion = $GlobalJson.tools.vswhere } else { - # keep this in sync with the VSWhereVersion in DefaultVersions.props $vswhereVersion = '3.1.7' } @@ -592,6 +521,11 @@ function LocateVisualStudio([object]$vsRequirements = $null){ return $null } + if ($null -eq $vsInfo -or $vsInfo.Count -eq 0) { + throw "No instance of Visual Studio meeting the requirements specified was found. Requirements: $($args -join ' ')" + return $null + } + # use first matching instance return $vsInfo[0] } @@ -627,7 +561,7 @@ function InitializeBuildTool() { $buildTool = @{ Path = $dotnetPath; Command = 'msbuild'; Tool = 'dotnet'; Framework = 'net' } } elseif ($msbuildEngine -eq "vs") { try { - $msbuildPath = InitializeVisualStudioMSBuild -install:$restore + $msbuildPath = InitializeVisualStudioMSBuild } catch { Write-PipelineTelemetryError -Category 'InitializeToolset' -Message $_ ExitWithExitCode 1 @@ -656,10 +590,9 @@ function GetDefaultMSBuildEngine() { ExitWithExitCode 1 } -function GetNuGetPackageCachePath() { +function InitializeNuGetPackageCachePath() { if ($env:NUGET_PACKAGES -eq $null) { # Use local cache on CI to ensure deterministic build. - # Avoid using the http cache as workaround for https://github.com/NuGet/Home/issues/3116 # use global cache in dev builds to avoid cost of downloading packages. # For directory normalization, see also: https://github.com/NuGet/Home/issues/7968 if ($useGlobalNuGetCache) { @@ -674,7 +607,13 @@ function GetNuGetPackageCachePath() { # Returns a full path to an Arcade SDK task project file. function GetSdkTaskProject([string]$taskName) { - return Join-Path (Split-Path (InitializeToolset) -Parent) "SdkTasks\$taskName.proj" + $toolsetDir = Split-Path (InitializeToolset) -Parent + $proj = Join-Path $toolsetDir "$taskName.proj" + if (Test-Path $proj) { + return $proj + } + + throw "Unable to find $taskName.proj in toolset at: $toolsetDir" } function InitializeNativeTools() { @@ -708,16 +647,19 @@ function InitializeToolset() { return $global:_InitializeToolset } - $nugetCache = GetNuGetPackageCachePath - $toolsetVersion = Read-ArcadeSdkVersion - $toolsetLocationFile = Join-Path $ToolsetDir "$toolsetVersion.txt" + $toolsetToolsDir = Join-Path $ToolsetDir $toolsetVersion - if (Test-Path $toolsetLocationFile) { - $path = Get-Content $toolsetLocationFile -TotalCount 1 - if (Test-Path $path) { - return $global:_InitializeToolset = $path - } + # Check if the toolset has already been extracted + $toolsetBuildProj = $null + $buildProjPath = Join-Path $toolsetToolsDir 'Build.proj' + + if (Test-Path $buildProjPath) { + $toolsetBuildProj = $buildProjPath + } + + if ($toolsetBuildProj -ne $null) { + return $global:_InitializeToolset = $toolsetBuildProj } if (-not $restore) { @@ -725,21 +667,41 @@ function InitializeToolset() { ExitWithExitCode 1 } - $buildTool = InitializeBuildTool + $downloadArgs = @("package", "download", "Microsoft.DotNet.Arcade.Sdk@$toolsetVersion", "--verbosity", "minimal", "--prerelease", "--output", "$nugetPackageCachePath") + $nugetConfig = $env:NUGET_CONFIG + if (-not $nugetConfig) { + # Search for any variation of nuget.config in the RepoRoot + $configFile = Get-ChildItem -Path $RepoRoot -File | Where-Object { $_.Name -ieq "nuget.config" } | Select-Object -First 1 + + if ($configFile) { + $nugetConfig = $configFile.FullName + } + } + + if ($nugetConfig) { + $downloadArgs += "--configfile" + $downloadArgs += $nugetConfig + } + DotNet @downloadArgs - $proj = Join-Path $ToolsetDir 'restore.proj' - $bl = if ($binaryLog) { '/bl:' + (Join-Path $LogDir 'ToolsetRestore.binlog') } else { '' } + $packageDir = Join-Path $nugetPackageCachePath (Join-Path 'microsoft.dotnet.arcade.sdk' $toolsetVersion) + $packageToolsetDir = Join-Path $packageDir 'toolset' - '' | Set-Content $proj + if (!(Test-Path $packageToolsetDir)) { + Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "Arcade SDK package does not contain a toolset or tools folder: $packageDir" + ExitWithExitCode 3 + } - MSBuild-Core $proj $bl /t:__WriteToolsetLocation /clp:ErrorsOnly`;NoSummary /p:__ToolsetLocationOutputFile=$toolsetLocationFile + New-Item -ItemType Directory -Path $toolsetToolsDir -Force | Out-Null + Copy-Item -Path "$packageToolsetDir\*" -Destination $toolsetToolsDir -Recurse -Force - $path = Get-Content $toolsetLocationFile -Encoding UTF8 -TotalCount 1 - if (!(Test-Path $path)) { - throw "Invalid toolset path: $path" + if (Test-Path $buildProjPath) { + $toolsetBuildProj = $buildProjPath + } else { + throw "Unable to find Build.proj in toolset at: $toolsetToolsDir" } - return $global:_InitializeToolset = $path + return $global:_InitializeToolset = $toolsetBuildProj } function ExitWithExitCode([int] $exitCode) { @@ -773,39 +735,6 @@ function Stop-Processes() { # Terminates the script if the build fails. # function MSBuild() { - if ($pipelinesLog) { - $buildTool = InitializeBuildTool - - if ($ci -and $buildTool.Tool -eq 'dotnet') { - $env:NUGET_PLUGIN_HANDSHAKE_TIMEOUT_IN_SECONDS = 20 - $env:NUGET_PLUGIN_REQUEST_TIMEOUT_IN_SECONDS = 20 - Write-PipelineSetVariable -Name 'NUGET_PLUGIN_HANDSHAKE_TIMEOUT_IN_SECONDS' -Value '20' - Write-PipelineSetVariable -Name 'NUGET_PLUGIN_REQUEST_TIMEOUT_IN_SECONDS' -Value '20' - } - - Enable-Nuget-EnhancedRetry - - $toolsetBuildProject = InitializeToolset - $basePath = Split-Path -parent $toolsetBuildProject - $selectedPath = Join-Path $basePath (Join-Path $buildTool.Framework 'Microsoft.DotNet.ArcadeLogging.dll') - - if (-not $selectedPath) { - Write-PipelineTelemetryError -Category 'Build' -Message "Unable to find arcade sdk logger assembly: $selectedPath" - ExitWithExitCode 1 - } - - $args += "/logger:$selectedPath" - } - - MSBuild-Core @args -} - -# -# Executes msbuild (or 'dotnet msbuild') with arguments passed to the function. -# The arguments are automatically quoted. -# Terminates the script if the build fails. -# -function MSBuild-Core() { if ($ci) { if (!$binaryLog -and !$excludeCIBinarylog) { Write-PipelineTelemetryError -Category 'Build' -Message 'Binary log must be enabled in CI build, or explicitly opted-out from with the -excludeCIBinarylog switch.' @@ -818,8 +747,6 @@ function MSBuild-Core() { } } - Enable-Nuget-EnhancedRetry - $buildTool = InitializeBuildTool $cmdArgs = "$($buildTool.Command) /m /nologo /clp:Summary /v:$verbosity /nr:$nodeReuse /p:ContinuousIntegrationBuild=$ci" @@ -836,6 +763,10 @@ function MSBuild-Core() { $cmdArgs += ' /p:TreatWarningsAsErrors=false' } + if ($warnAsError -and $warnNotAsError) { + $cmdArgs += " /warnnotaserror:$warnNotAsError /p:AdditionalWarningsNotAsErrors=$warnNotAsError" + } + foreach ($arg in $args) { if ($null -ne $arg -and $arg.Trim() -ne "") { if ($arg.EndsWith('\')) { @@ -873,6 +804,40 @@ function MSBuild-Core() { } } +# +# Executes a dotnet command with arguments passed to the function. +# Terminates the script if the command fails. +# +function DotNet() { + $dotnetRoot = InitializeDotNetCli -install:$restore + $dotnetPath = Join-Path $dotnetRoot (GetExecutableFileName 'dotnet') + + $cmdArgs = "" + foreach ($arg in $args) { + if ($null -ne $arg -and $arg.Trim() -ne "") { + if ($arg.EndsWith('\')) { + $arg = $arg + "\" + } + $cmdArgs += " `"$arg`"" + } + } + + $env:ARCADE_BUILD_TOOL_COMMAND = "`"$dotnetPath`" $cmdArgs" + + $exitCode = Exec-Process $dotnetPath $cmdArgs + + if ($exitCode -ne 0) { + Write-Host "dotnet command failed with exit code $exitCode. Check errors above." -ForegroundColor Red + + if ($ci -and $env:SYSTEM_TEAMPROJECT -ne $null -and !$fromVMR) { + Write-PipelineSetResult -Result "Failed" -Message "dotnet command execution failed." + ExitWithExitCode 0 + } else { + ExitWithExitCode $exitCode + } + } +} + function GetMSBuildBinaryLogCommandLineArgument($arguments) { foreach ($argument in $arguments) { if ($argument -ne $null) { @@ -930,6 +895,12 @@ Create-Directory $ToolsetDir Create-Directory $TempDir Create-Directory $LogDir +# Direct MSBuild crash diagnostics (MSB4166 failure.txt files) to a known location +# under artifacts/log so they are captured as build artifacts in CI. +if (-not $env:MSBUILDDEBUGPATH) { + $env:MSBUILDDEBUGPATH = Join-Path $LogDir 'MsbuildDebugLogs' +} + Write-PipelineSetVariable -Name 'Artifacts' -Value $ArtifactsDir Write-PipelineSetVariable -Name 'Artifacts.Toolset' -Value $ToolsetDir Write-PipelineSetVariable -Name 'Artifacts.Log' -Value $LogDir @@ -951,19 +922,5 @@ if (!$disableConfigureToolsetImport) { } } -# -# If $ci flag is set, turn on (and log that we did) special environment variables for improved Nuget client retry logic. -# -function Enable-Nuget-EnhancedRetry() { - if ($ci) { - Write-Host "Setting NUGET enhanced retry environment variables" - $env:NUGET_ENABLE_ENHANCED_HTTP_RETRY = 'true' - $env:NUGET_ENHANCED_MAX_NETWORK_TRY_COUNT = 6 - $env:NUGET_ENHANCED_NETWORK_RETRY_DELAY_MILLISECONDS = 1000 - $env:NUGET_RETRY_HTTP_429 = 'true' - Write-PipelineSetVariable -Name 'NUGET_ENABLE_ENHANCED_HTTP_RETRY' -Value 'true' - Write-PipelineSetVariable -Name 'NUGET_ENHANCED_MAX_NETWORK_TRY_COUNT' -Value '6' - Write-PipelineSetVariable -Name 'NUGET_ENHANCED_NETWORK_RETRY_DELAY_MILLISECONDS' -Value '1000' - Write-PipelineSetVariable -Name 'NUGET_RETRY_HTTP_429' -Value 'true' - } -} +# Initialize the nuget package cache vars +$nugetPackageCachePath = InitializeNuGetPackageCachePath diff --git a/eng/common/tools.sh b/eng/common/tools.sh index 1b296f646..48cab70eb 100755 --- a/eng/common/tools.sh +++ b/eng/common/tools.sh @@ -8,16 +8,6 @@ ci=${ci:-false} # Build mode source_build=${source_build:-false} -# Set to true to use the pipelines logger which will enable Azure logging output. -# https://github.com/Microsoft/azure-pipelines-tasks/blob/master/docs/authoring/commands.md -# This flag is meant as a temporary opt-opt for the feature while validate it across -# our consumers. It will be deleted in the future. -if [[ "$ci" == true ]]; then - pipelines_log=${pipelines_log:-true} -else - pipelines_log=${pipelines_log:-false} -fi - # Build configuration. Common values include 'Debug' and 'Release', but the repository may use other names. configuration=${configuration:-'Debug'} @@ -52,6 +42,9 @@ fi # Configures warning treatment in msbuild. warn_as_error=${warn_as_error:-true} +# Specifies semi-colon delimited list of warning codes that should not be treated as errors. +warn_not_as_error=${warn_not_as_error:-''} + # True to attempt using .NET Core already that meets requirements specified in global.json # installed on the machine instead of downloading one. use_installed_dotnet_cli=${use_installed_dotnet_cli:-true} @@ -115,9 +108,6 @@ function InitializeDotNetCli { local install=$1 - # Don't resolve runtime, shared framework, or SDK from other locations to ensure build determinism - export DOTNET_MULTILEVEL_LOOKUP=0 - # Disable first run since we want to control all package sources export DOTNET_NOLOGO=1 @@ -148,7 +138,11 @@ function InitializeDotNetCli { if [[ $global_json_has_runtimes == false && -n "${DOTNET_INSTALL_DIR:-}" && -d "$DOTNET_INSTALL_DIR/sdk/$dotnet_sdk_version" ]]; then dotnet_root="$DOTNET_INSTALL_DIR" else - dotnet_root="${repo_root}.dotnet" + if [[ -n "${DOTNET_GLOBAL_INSTALL_DIR:-}" ]]; then + dotnet_root="$DOTNET_GLOBAL_INSTALL_DIR" + else + dotnet_root="${repo_root}.dotnet" + fi export DOTNET_INSTALL_DIR="$dotnet_root" @@ -166,7 +160,6 @@ function InitializeDotNetCli { # build steps from using anything other than what we've downloaded. Write-PipelinePrependPath -path "$dotnet_root" - Write-PipelineSetVariable -name "DOTNET_MULTILEVEL_LOOKUP" -value "0" Write-PipelineSetVariable -name "DOTNET_NOLOGO" -value "1" # return value @@ -188,6 +181,8 @@ function InstallDotNet { local version=$2 local runtime=$4 + # For performance this check is duplicated in src/Microsoft.DotNet.Arcade.Sdk/src/InstallDotNetCore.cs + # if you are making changes here, consider if you need to make changes there as well. local dotnetVersionLabel="'$runtime v$version'" if [[ -n "${4:-}" ]] && [ "$4" != 'sdk' ]; then runtimePath="$root" @@ -369,7 +364,7 @@ function InitializeBuildTool { _InitializeBuildToolCommand="msbuild" } -function GetNuGetPackageCachePath { +function InitializeNuGetPackageCachePath { if [[ -z ${NUGET_PACKAGES:-} ]]; then if [[ "$use_global_nuget_cache" == true ]]; then export NUGET_PACKAGES="$HOME/.nuget/packages/" @@ -379,7 +374,7 @@ function GetNuGetPackageCachePath { fi # return value - _GetNuGetPackageCachePath=$NUGET_PACKAGES + _InitializeNuGetPackageCachePath=$NUGET_PACKAGES } function InitializeNativeTools() { @@ -401,20 +396,21 @@ function InitializeToolset { return fi - GetNuGetPackageCachePath - ReadGlobalVersion "Microsoft.DotNet.Arcade.Sdk" local toolset_version=$_ReadGlobalVersion - local toolset_location_file="$toolset_dir/$toolset_version.txt" + local toolset_tools_dir="$toolset_dir/$toolset_version" - if [[ -a "$toolset_location_file" ]]; then - local path=`cat "$toolset_location_file"` - if [[ -a "$path" ]]; then - # return value - _InitializeToolset="$path" - return - fi + # Check if the toolset has already been extracted + local toolset_build_proj="" + if [[ -a "$toolset_tools_dir/Build.proj" ]]; then + toolset_build_proj="$toolset_tools_dir/Build.proj" + fi + + if [[ -n "$toolset_build_proj" ]]; then + # return value + _InitializeToolset="$toolset_build_proj" + return fi if [[ "$restore" != true ]]; then @@ -422,20 +418,37 @@ function InitializeToolset { ExitWithExitCode 2 fi - local proj="$toolset_dir/restore.proj" + local download_args=("package" "download" "Microsoft.DotNet.Arcade.Sdk@$toolset_version" "--verbosity" "minimal" "--prerelease" "--output" "$_InitializeNuGetPackageCachePath") + local nuget_config="${NUGET_CONFIG:-}" + if [[ -z "$nuget_config" ]]; then + # Search for any variation of nuget.config in the RepoRoot + local found_config + found_config=$(find "$repo_root" -maxdepth 1 -type f -iname "nuget.config" -print -quit) + + if [[ -n "$found_config" ]]; then + nuget_config="$found_config" + fi + fi - local bl="" - if [[ "$binary_log" == true ]]; then - bl="/bl:$log_dir/ToolsetRestore.binlog" + if [[ -n "$nuget_config" ]]; then + download_args+=("--configfile" "$nuget_config") fi + DotNet "${download_args[@]}" - echo '' > "$proj" - MSBuild-Core "$proj" $bl /t:__WriteToolsetLocation /clp:ErrorsOnly\;NoSummary /p:__ToolsetLocationOutputFile="$toolset_location_file" + local package_dir="$_InitializeNuGetPackageCachePath/microsoft.dotnet.arcade.sdk/$toolset_version" - local toolset_build_proj=`cat "$toolset_location_file"` + if [[ ! -d "$package_dir/toolset" ]]; then + Write-PipelineTelemetryError -category 'InitializeToolset' "Arcade SDK package does not contain a toolset folder: $package_dir" + ExitWithExitCode 3 + fi - if [[ ! -a "$toolset_build_proj" ]]; then - Write-PipelineTelemetryError -category 'Build' "Invalid toolset path: $toolset_build_proj" + mkdir -p "$toolset_tools_dir" + cp -r "$package_dir/toolset/." "$toolset_tools_dir" + + if [[ -a "$toolset_tools_dir/Build.proj" ]]; then + toolset_build_proj="$toolset_tools_dir/Build.proj" + else + Write-PipelineTelemetryError -category 'Build' "Unable to find Build.proj in toolset at: $toolset_tools_dir" ExitWithExitCode 3 fi @@ -457,34 +470,27 @@ function StopProcesses { return 0 } -function MSBuild { - local args=( "$@" ) - if [[ "$pipelines_log" == true ]]; then - InitializeBuildTool - InitializeToolset - - if [[ "$ci" == true ]]; then - export NUGET_PLUGIN_HANDSHAKE_TIMEOUT_IN_SECONDS=20 - export NUGET_PLUGIN_REQUEST_TIMEOUT_IN_SECONDS=20 - Write-PipelineSetVariable -name "NUGET_PLUGIN_HANDSHAKE_TIMEOUT_IN_SECONDS" -value "20" - Write-PipelineSetVariable -name "NUGET_PLUGIN_REQUEST_TIMEOUT_IN_SECONDS" -value "20" - fi +function DotNet { + InitializeDotNetCli $restore - local toolset_dir="${_InitializeToolset%/*}" - local selectedPath="$toolset_dir/net/Microsoft.DotNet.ArcadeLogging.dll" + local dotnet_path="$_InitializeDotNetCli/dotnet" - if [[ -z "$selectedPath" ]]; then - Write-PipelineTelemetryError -category 'Build' "Unable to find arcade sdk logger assembly: $selectedPath" - ExitWithExitCode 1 - fi + export ARCADE_BUILD_TOOL_COMMAND="$dotnet_path $@" - args+=( "-logger:$selectedPath" ) - fi + "$dotnet_path" "$@" || { + local exit_code=$? + echo "dotnet command failed with exit code $exit_code. Check errors above." - MSBuild-Core "${args[@]}" + if [[ "$ci" == true && -n ${SYSTEM_TEAMPROJECT:-} && "$from_vmr" != true ]]; then + Write-PipelineSetResult -result "Failed" -message "dotnet command execution failed." + ExitWithExitCode 0 + else + ExitWithExitCode $exit_code + fi + } } -function MSBuild-Core { +function MSBuild { if [[ "$ci" == true ]]; then if [[ "$binary_log" != true && "$exclude_ci_binary_log" != true ]]; then Write-PipelineTelemetryError -category 'Build' "Binary log must be enabled in CI build, or explicitly opted-out from with the -noBinaryLog switch." @@ -532,7 +538,12 @@ function MSBuild-Core { mt_switch="-mt" fi - RunBuildTool "$_InitializeBuildToolCommand" /m /nologo /clp:Summary /v:$verbosity /nr:$node_reuse $warnaserror_switch $mt_switch /p:TreatWarningsAsErrors=$warn_as_error /p:ContinuousIntegrationBuild=$ci "$@" + local warnnotaserror_switch="" + if [[ -n "$warn_not_as_error" && "$warn_as_error" == true ]]; then + warnnotaserror_switch="/warnnotaserror:$warn_not_as_error /p:AdditionalWarningsNotAsErrors=$warn_not_as_error" + fi + + RunBuildTool "$_InitializeBuildToolCommand" /m /nologo /clp:Summary /v:$verbosity /nr:$node_reuse $warnaserror_switch $mt_switch $warnnotaserror_switch /p:TreatWarningsAsErrors=$warn_as_error /p:ContinuousIntegrationBuild=$ci "$@" } function GetDarc { @@ -549,8 +560,17 @@ function GetDarc { # Returns a full path to an Arcade SDK task project file. function GetSdkTaskProject { - taskName=$1 - echo "$(dirname $_InitializeToolset)/SdkTasks/$taskName.proj" + local taskName=$1 + local toolsetDir + toolsetDir="$(dirname "$_InitializeToolset")" + local proj="$toolsetDir/$taskName.proj" + if [[ -a "$proj" ]]; then + echo "$proj" + return + fi + + Write-PipelineTelemetryError -category 'Build' "Unable to find $taskName.proj in toolset at: $toolsetDir" + ExitWithExitCode 3 } ResolvePath "${BASH_SOURCE[0]}" @@ -588,6 +608,12 @@ mkdir -p "$toolset_dir" mkdir -p "$temp_dir" mkdir -p "$log_dir" +# Direct MSBuild crash diagnostics (MSB4166 failure.txt files) to a known location +# under artifacts/log so they are captured as build artifacts in CI. +if [[ -z "${MSBUILDDEBUGPATH:-}" ]]; then + export MSBUILDDEBUGPATH="$log_dir/MsbuildDebugLogs" +fi + Write-PipelineSetVariable -name "Artifacts" -value "$artifacts_dir" Write-PipelineSetVariable -name "Artifacts.Toolset" -value "$toolset_dir" Write-PipelineSetVariable -name "Artifacts.Log" -value "$log_dir" @@ -608,3 +634,6 @@ fi if [[ -n "${useInstalledDotNetCli:-}" ]]; then use_installed_dotnet_cli="$useInstalledDotNetCli" fi + +# Initialize the nuget package cache vars +InitializeNuGetPackageCachePath diff --git a/eng/pipelines/devflow-official.yml b/eng/pipelines/devflow-official.yml index f49a5e9f8..e3144937f 100644 --- a/eng/pipelines/devflow-official.yml +++ b/eng/pipelines/devflow-official.yml @@ -87,6 +87,7 @@ extends: - template: /eng/common/templates-official/jobs/jobs.yml@self parameters: enableMicrobuild: true + enablePreviewMicrobuild: true enablePublishBuildAssets: true enablePublishBuildArtifacts: true enablePublishTestResults: true diff --git a/global.json b/global.json index 4ddff5f13..9a3fef8b7 100644 --- a/global.json +++ b/global.json @@ -1,17 +1,17 @@ { "sdk": { - "version": "10.0.106", + "version": "11.0.100-preview.5.26227.104", "paths": [ ".dotnet", "$host$" ], "rollForward": "latestMinor", - "allowPrerelease": false + "allowPrerelease": true }, "tools": { - "dotnet": "10.0.106" + "dotnet": "11.0.100-preview.5.26227.104" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "10.0.0-beta.26257.101" + "Microsoft.DotNet.Arcade.Sdk": "11.0.0-beta.26302.1" } }