diff --git a/.buildkite/build-linux.sh b/.buildkite/build-linux.sh index 0ad594166..63d864706 100755 --- a/.buildkite/build-linux.sh +++ b/.buildkite/build-linux.sh @@ -24,6 +24,9 @@ if ! command -v go >/dev/null 2>&1; then fi go version +echo "--- :package: fetch go-search-replace" +.buildkite/fetch-search-replace.sh linux/amd64 linux/arm64 + VERSION="$(go run -mod=mod ./cmd/stamp-version)" COMMIT="$(git rev-parse --short HEAD 2>/dev/null || echo unknown)" @@ -35,14 +38,14 @@ build() { go build -buildvcs=false -trimpath \ -ldflags="-s -w -X github.com/Automattic/vip/internal/version.Version=${VERSION} -X github.com/Automattic/vip/internal/version.Commit=${COMMIT}" \ -o "${out}" ./cmd/vip-next - shasum -a 256 "${out}" > "${out}.sha256" } +checksum() { shasum -a 256 "$1" > "$1.sha256"; } + mkdir -p dist build amd64 build arm64 -# Smoke-test only the arch matching this agent (a cross-built slice won't run here). case "$(uname -m)" in x86_64|amd64) native=amd64 ;; aarch64|arm64) native=arm64 ;; @@ -54,4 +57,12 @@ if [ -n "${native}" ]; then "dist/${BIN_BASE}-linux-${native}" whoami --help fi +for arch in amd64 arm64; do + bin="dist/${BIN_BASE}-linux-${arch}" + helper="third_party/go-search-replace/linux-${arch}/go-search-replace" + echo "--- :package: tarball linux/${arch}" + .buildkite/pack-release.sh linux "${arch}" "${bin}" "${helper}" + checksum "dist/${BIN_BASE}-linux-${arch}.tar.gz" + rm -f "${bin}" +done diff --git a/.buildkite/build-macos.sh b/.buildkite/build-macos.sh index 94687183e..4b2948a2b 100755 --- a/.buildkite/build-macos.sh +++ b/.buildkite/build-macos.sh @@ -1,9 +1,8 @@ #!/usr/bin/env bash set -euo pipefail # Build, sign, and notarize the macOS vip-next artifacts on a Buildkite macOS -# agent (queue: mac). Two bare per-arch binaries: codesigned + notarized -# (online-verified; a bare Mach-O can't be stapled). Checksums are written -# AFTER signing (signing changes the bytes). +# agent (queue: mac). Each arch is a tarball of vip-next + go-search-replace, +# both codesigned and notarized. Checksums are written AFTER signing. [ -f .buildkite/shared-pipeline-vars ] && . .buildkite/shared-pipeline-vars : "${BIN_BASE:=vip-next}" @@ -12,7 +11,6 @@ echo "--- :ruby: install gems" if command -v install_gems >/dev/null 2>&1; then install_gems; else bundle install; fi echo "--- :go: toolchain" -# Any Go will do: go.mod's `toolchain` directive makes it fetch go1.27.0 itself. if ! command -v go >/dev/null 2>&1; then echo "--- :package: install go" brew install go @@ -22,6 +20,9 @@ go version # Bundler installs to `vendor/bundle`, which makes Go take `-mod=vendor`. export GOFLAGS="${GOFLAGS:--mod=mod}" +echo "--- :package: fetch go-search-replace" +.buildkite/fetch-search-replace.sh darwin/arm64 darwin/amd64 + VERSION="$(go run -mod=mod ./cmd/stamp-version)" COMMIT="$(git rev-parse --short HEAD 2>/dev/null || echo unknown)" @@ -41,7 +42,6 @@ mkdir -p dist build arm64 build amd64 -# Smoke-test the native arch (a cross-built slice may not run without Rosetta). case "$(uname -m)" in arm64) native=arm64 ;; x86_64) native=amd64 ;; @@ -58,7 +58,11 @@ bundle exec fastlane configure_code_signing for arch in arm64 amd64; do bin="dist/${BIN_BASE}-darwin-${arch}" - echo "--- :closed_lock_with_key: sign and notarize ${arch}" + helper="third_party/go-search-replace/darwin-${arch}/go-search-replace" + echo "--- :closed_lock_with_key: sign and notarize darwin/${arch}" bundle exec fastlane sign_and_notarize binary:"${bin}" - checksum "${bin}" + bundle exec fastlane sign_and_notarize binary:"${helper}" + .buildkite/pack-release.sh darwin "${arch}" "${bin}" "${helper}" + checksum "dist/${BIN_BASE}-darwin-${arch}.tar.gz" + rm -f "${bin}" done diff --git a/.buildkite/build-windows.ps1 b/.buildkite/build-windows.ps1 index de51043ef..699f22df5 100644 --- a/.buildkite/build-windows.ps1 +++ b/.buildkite/build-windows.ps1 @@ -38,23 +38,38 @@ Write-Host "--- :test_tube: smoke" & $out --version & $out whoami --help +Write-Host "--- :package: fetch go-search-replace" +bash .buildkite/fetch-search-replace.sh windows/amd64 +if ($LASTEXITCODE -ne 0) { throw 'fetch-search-replace.sh failed' } +$helper = 'third_party/go-search-replace/windows-amd64/go-search-replace.exe' + Write-Host "--- :closed_lock_with_key: Azure Trusted Signing" $setupScript = (Get-Command setup_azure_trusted_signing.ps1 -ErrorAction Stop).Source & $setupScript if ($LASTEXITCODE -ne 0) { throw 'setup_azure_trusted_signing.ps1 failed' } -Write-Host "--- :closed_lock_with_key: Authenticode sign" -& $env:SIGNTOOL_PATH sign /v ` - /fd $env:AZURE_FILE_DIGEST ` - /tr $env:AZURE_TIMESTAMP_SERVER ` - /td $env:AZURE_TIMESTAMP_DIGEST ` - /dlib $env:AZURE_CODE_SIGNING_DLIB ` - /dmdf $env:AZURE_METADATA_JSON ` - $out -if ($LASTEXITCODE -ne 0) { throw 'signtool sign failed' } -& $env:SIGNTOOL_PATH verify /pa /v $out -if ($LASTEXITCODE -ne 0) { throw 'signtool verify failed' } - -Write-Host "--- checksum" -$hash = (Get-FileHash -Algorithm SHA256 $out).Hash.ToLower() -"$hash *$(Split-Path $out -Leaf)" | Set-Content "$out.sha256" -NoNewline +function Sign-File([string]$path) { + Write-Host "--- :closed_lock_with_key: Authenticode sign $path" + & $env:SIGNTOOL_PATH sign /v ` + /fd $env:AZURE_FILE_DIGEST ` + /tr $env:AZURE_TIMESTAMP_SERVER ` + /td $env:AZURE_TIMESTAMP_DIGEST ` + /dlib $env:AZURE_CODE_SIGNING_DLIB ` + /dmdf $env:AZURE_METADATA_JSON ` + $path + if ($LASTEXITCODE -ne 0) { throw "signtool sign failed for $path" } + & $env:SIGNTOOL_PATH verify /pa /v $path + if ($LASTEXITCODE -ne 0) { throw "signtool verify failed for $path" } +} + +Sign-File $out +Sign-File $helper + +Write-Host "--- :package: tarball" +$env:BIN_BASE = $binBase +bash .buildkite/pack-release.sh windows amd64 $out $helper +if ($LASTEXITCODE -ne 0) { throw 'pack-release.sh failed' } +$tar = "dist/$binBase-windows-amd64.tar.gz" +$hash = (Get-FileHash -Algorithm SHA256 $tar).Hash.ToLower() +"$hash *$(Split-Path $tar -Leaf)" | Set-Content "$tar.sha256" -NoNewline +Remove-Item $out -Force diff --git a/.buildkite/fetch-search-replace.sh b/.buildkite/fetch-search-replace.sh new file mode 100755 index 000000000..65663dd0d --- /dev/null +++ b/.buildkite/fetch-search-replace.sh @@ -0,0 +1,65 @@ +#!/usr/bin/env bash +set -euo pipefail +# Fetch go-search-replace for each goos/goarch argument, verify against +# third_party/go-search-replace/MANIFEST, install under that tree. +# Usage: fetch-search-replace.sh darwin/arm64 darwin/amd64 +# No args: this host's GOOS/GOARCH. + +GSR_DIR=third_party/go-search-replace +GSR_REPO=Automattic/go-search-replace + +sha256() { + if command -v shasum >/dev/null 2>&1; then + shasum -a 256 "$1" | awk '{ print $1 }' + else + sha256sum "$1" | awk '{ print $1 }' + fi +} + +tag="$(awk '$1=="TAG"{print $2}' "${GSR_DIR}/MANIFEST")" +[ -n "${tag}" ] || { echo "no TAG in ${GSR_DIR}/MANIFEST" >&2; exit 1; } + +if [ "$#" -eq 0 ]; then + set -- "$(go env GOOS)/$(go env GOARCH)" +fi + +tmp="$(mktemp -d)" +trap 'rm -rf "${tmp}"' EXIT + +for t in "$@"; do + os="${t%%/*}" + arch="${t##*/}" + want="$(awk -v k="${t}" '$1==k{print $2}' "${GSR_DIR}/MANIFEST")" + if [ -z "${want}" ]; then + echo "ERROR: ${t} is not pinned in ${GSR_DIR}/MANIFEST" >&2 + exit 1 + fi + name="go-search-replace_${os}_${arch}" + if [ "${os}" = windows ]; then + name="${name}.exe" + fi + url="https://github.com/${GSR_REPO}/releases/download/${tag}/${name}.gz" + echo " fetching ${name} (${tag})" + if ! curl -fsSL "${url}" -o "${tmp}/${name}.gz"; then + echo "ERROR: could not download ${url}" >&2 + exit 1 + fi + gunzip -f "${tmp}/${name}.gz" + got="$(sha256 "${tmp}/${name}")" + if [ "${got}" != "${want}" ]; then + echo "ERROR: checksum mismatch for ${t}" >&2 + echo " expected (from upstream SLSA provenance): ${want}" >&2 + echo " got: ${got}" >&2 + echo " Refusing to install. Do not bypass this." >&2 + exit 1 + fi + out="${GSR_DIR}/${os}-${arch}" + mkdir -p "${out}" + d="${out}/go-search-replace" + if [ "${os}" = windows ]; then + d="${d}.exe" + fi + mv "${tmp}/${name}" "${d}" + chmod +x "${d}" + echo " verified + installed ${d}" +done diff --git a/.buildkite/pack-release.sh b/.buildkite/pack-release.sh new file mode 100755 index 000000000..61387cf93 --- /dev/null +++ b/.buildkite/pack-release.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +set -euo pipefail +# Pack vip-next + go-search-replace into dist/${BIN_BASE}--.tar.gz +# Usage: pack-release.sh + +: "${BIN_BASE:=vip-next}" + +os="${1:?}" +arch="${2:?}" +bin="${3:?}" +helper="${4:?}" + +stage="$(mktemp -d)" +trap 'rm -rf "${stage}"' EXIT + +if [ "${os}" = windows ]; then + cp "${bin}" "${stage}/vip-next.exe" + cp "${helper}" "${stage}/go-search-replace.exe" + chmod +x "${stage}/vip-next.exe" "${stage}/go-search-replace.exe" + tar -czf "dist/${BIN_BASE}-windows-${arch}.tar.gz" -C "${stage}" vip-next.exe go-search-replace.exe +else + cp "${bin}" "${stage}/vip-next" + cp "${helper}" "${stage}/go-search-replace" + chmod +x "${stage}/vip-next" "${stage}/go-search-replace" + tar -czf "dist/${BIN_BASE}-${os}-${arch}.tar.gz" -C "${stage}" vip-next go-search-replace +fi diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index a3f5c9308..1de6397ad 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -24,7 +24,7 @@ steps: - github_commit_status: context: 'Build & sign (macOS)' artifact_paths: - - 'dist/vip-next-darwin-*' + - 'dist/vip-next-darwin-*.tar.gz*' - label: ':windows: Build & sign (Windows)' command: powershell -NoProfile -ExecutionPolicy Bypass -File .buildkite/build-windows.ps1 @@ -36,7 +36,7 @@ steps: - github_commit_status: context: 'Build & sign (Windows)' artifact_paths: - - 'dist/vip-next-windows-amd64.exe*' + - 'dist/vip-next-windows-amd64.tar.gz*' - label: ':linux: Build (Linux)' command: .buildkite/build-linux.sh @@ -46,4 +46,4 @@ steps: - github_commit_status: context: 'Build (Linux)' artifact_paths: - - 'dist/vip-next-linux-*' + - 'dist/vip-next-linux-*.tar.gz*' diff --git a/Makefile b/Makefile index 4945fc2d5..0eab7a4eb 100644 --- a/Makefile +++ b/Makefile @@ -56,42 +56,20 @@ GSR_REPO := Automattic/go-search-replace # assets GZIPPED but attests the UNCOMPRESSED binaries, so we gunzip first and # then hash — verified against release 0.0.11. # -# Binaries are gitignored; only MANIFEST is tracked. Upgrade with -# `make vendor-search-replace TAG=` and commit the MANIFEST diff. +# Binaries are gitignored; only MANIFEST is tracked. Upgrade by editing +# MANIFEST (tag + digests) then `make vendor-search-replace`. # # ALL is the release build's entry point: bundling every platform is what makes # the shipped tarball self-contained. vendor-search-replace: - @tag="$${TAG:-$$(awk '$$1=="TAG"{print $$2}' $(GSR_DIR)/MANIFEST)}"; \ - if [ -z "$$tag" ]; then echo "no TAG in $(GSR_DIR)/MANIFEST" >&2; exit 1; fi; \ - if [ -n "$$ALL" ]; then \ - targets=$$(awk '/^(darwin|linux|windows)\//{print $$1}' $(GSR_DIR)/MANIFEST); \ + @if [ -n "$$ALL" ]; then \ + set -- $$(awk '/^(darwin|linux|windows)\//{print $$1}' $(GSR_DIR)/MANIFEST); \ + elif [ -n "$$TARGETS" ]; then \ + set -- $$TARGETS; \ else \ - targets="$$($(GO) env GOOS)/$$($(GO) env GOARCH)"; \ + set -- "$$($(GO) env GOOS)/$$($(GO) env GOARCH)"; \ fi; \ - tmp=$$(mktemp -d); trap 'rm -rf "$$tmp"' EXIT; \ - for t in $$targets; do \ - os=$${t%%/*}; arch=$${t##*/}; \ - want=$$(awk -v k="$$t" '$$1==k{print $$2}' $(GSR_DIR)/MANIFEST); \ - if [ -z "$$want" ]; then echo "ERROR: $$t is not pinned in $(GSR_DIR)/MANIFEST" >&2; exit 1; fi; \ - name=go-search-replace_$${os}_$${arch}; \ - if [ "$$os" = "windows" ]; then name=$$name.exe; fi; \ - echo " fetching $$name ($$tag)"; \ - if ! gh release download "$$tag" --repo $(GSR_REPO) --pattern "$$name.gz" --dir "$$tmp" --clobber >/dev/null 2>&1; then \ - echo "ERROR: could not download $$name.gz from $(GSR_REPO)@$$tag" >&2; \ - echo " needs the gh CLI, authenticated. See docs/BUILD-SIGNING.md." >&2; exit 1; fi; \ - gunzip -f "$$tmp/$$name.gz"; \ - got=$$(shasum -a 256 "$$tmp/$$name" | cut -d' ' -f1); \ - if [ "$$got" != "$$want" ]; then \ - echo "ERROR: checksum mismatch for $$t" >&2; \ - echo " expected (from upstream SLSA provenance): $$want" >&2; \ - echo " got: $$got" >&2; \ - echo " Refusing to install. Do not bypass this." >&2; exit 1; fi; \ - out=$(GSR_DIR)/$${os}-$${arch}; mkdir -p "$$out"; \ - d=$$out/go-search-replace; if [ "$$os" = "windows" ]; then d=$$d.exe; fi; \ - mv "$$tmp/$$name" "$$d"; chmod +x "$$d"; \ - echo " verified + installed $$d"; \ - done + .buildkite/fetch-search-replace.sh "$$@" search-replace-bin: @os=$$($(GO) env GOOS); arch=$$($(GO) env GOARCH); \ diff --git a/docs/BUILD-SIGNING.md b/docs/BUILD-SIGNING.md index 354d77047..a88b7c428 100644 --- a/docs/BUILD-SIGNING.md +++ b/docs/BUILD-SIGNING.md @@ -243,11 +243,10 @@ Actions. See `.buildkite/pipeline.yml` and the per-platform scripts. (`a8c-fastlane-match`). Signing and notarization are the `sign_and_notarize` lane: Developer ID Application, identifier `com.automattic.vip-cli`, no staple on a bare Mach-O. -- **macOS artifacts:** two signed + notarized bare binaries (arm64, amd64; - online-verified). -- **Windows / Linux artifacts:** signed `.exe` (Azure Trusted Signing via - `setup_azure_trusted_signing.ps1`) and the two Linux binaries with - `.sha256` checksums. +- **macOS artifacts:** per-arch `.tar.gz` of `vip-next` + `go-search-replace`, + both signed and notarized (online-verified). +- **Windows / Linux artifacts:** the same tarball layout. Windows Authenticode- + signs both PEs via Azure Trusted Signing; Linux checksums only. ### Verifying a real run @@ -318,42 +317,13 @@ Upstream 0.0.11 publishes `darwin_{amd64,arm64}`, `linux_{386,amd64,arm64}`, `windows_{386,amd64,arm64}` — so `linux/arm64` (Graviton, ARM CI, Docker on Apple Silicon), previously unsupported, is covered with no self-building. -### Still needed in the signing pipeline +CI fetches the helper with `.buildkite/fetch-search-replace.sh` (curl of the +public GitHub release, then the MANIFEST sha256). Each arch is packed as +`vip-next` + `go-search-replace` in a `.tar.gz` after signing. -1. **Build agents need the `gh` CLI, authenticated**, for - `make vendor-search-replace`. Alternatively pre-populate - `third_party/go-search-replace/` from an internal mirror — but whatever - supplies it, the MANIFEST check must still run. - -2. **Release builds must run `ALL=1 make vendor-search-replace` before - packaging**, so the tarball is self-contained. Add it ahead of the build step - in `.buildkite/build-*.{sh,ps1}`. - -3. **macOS — this is the one that will bite.** `go-search-replace` is a nested - Mach-O executable inside our distributable. Under the hardened runtime, - notarization **fails** unless every nested executable is signed. So: - - - sign `go-search-replace` with the same Developer ID Application identity as - `vip-next`, with `--options runtime --timestamp`, - - sign it **before** the enclosing `.pkg`/archive is built and submitted, - - staple only the outer artifact. - - Re-signing a third-party binary with our identity is expected for bundled - helpers, but it is a deliberate supply-chain decision: we are attesting a - binary we did not build. The MANIFEST checksum + upstream SLSA provenance is - what makes that defensible — do not weaken either. - -4. **Windows.** Authenticode-signing the bundled helper is optional; nothing - blocks execution if it is unsigned, but SmartScreen reputation is per-binary. - Recommendation: sign it, same cert as `vip-next`. - -5. **Linux.** Nothing extra — the existing checksum/detached-signature step - should cover the bundled helper as well as the main binary. - -6. **Verify after a real run:** the artifact contains - `go-search-replace[.exe]` next to `vip-next`, `codesign -vvv --deep` passes - on macOS, and `vip-next search-replace` works on a machine that never had the - binary on `PATH`. +Re-signing the upstream helper with our Developer ID is expected. The MANIFEST +checksum plus upstream SLSA provenance is what makes that defensible — do not +weaken either. ### Open diff --git a/third_party/go-search-replace/MANIFEST b/third_party/go-search-replace/MANIFEST index 71ba9ec1b..3f1bfd776 100644 --- a/third_party/go-search-replace/MANIFEST +++ b/third_party/go-search-replace/MANIFEST @@ -17,8 +17,7 @@ # sha256. `make vendor-search-replace` does exactly that and refuses to install # anything that does not match. # -# To upgrade: `make vendor-search-replace TAG=` rewrites this file, so -# an upgrade is one reviewable commit whose diff is the tag and the digests. +# To upgrade: edit this file (tag + digests) then `make vendor-search-replace`. # # Format: /