From 730f43f0de9284d3e34c5c270bceb554598076e4 Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Thu, 13 Aug 2026 12:33:09 -0500 Subject: [PATCH] fix(bench): repair the skopeo install and pin the competitor tools skopeo renamed its module path to go.podman.io/skopeo at v1.23.0. The github.com path still serves tags, but their go.mod declares the new path, so the install fails on a module path mismatch. set -euo pipefail aborts the bootstrap there, before dregsy, regsync, the ocync build and the bench-proxy CA, and Terraform reports success because nothing waits on user-data. Instance bootstrap has been broken since that release. The three tools ocync is measured against were also installed with @latest, so every rebuild silently changed what the benchmark compared against. bench/results/{registry}.json is tracked for longitudinal comparison, which that undermines: a metric shift could come from a competitor release rather than from a change here. dregsy tags releases without the v prefix, so the module proxy serves only pseudo-versions for it. Its pin is the pseudo-version for release 0.5.2. dregsy and the credential helper report no usable version at runtime, one having no version flag and the other needing ldflags go install does not set, so the bootstrap log echoes their pins rather than probing them. --- bench/CLAUDE.md | 2 ++ bench/terraform/aws/user-data.sh | 33 +++++++++++++++++++++++++------- 2 files changed, 28 insertions(+), 7 deletions(-) diff --git a/bench/CLAUDE.md b/bench/CLAUDE.md index 0e94da45..8d7b1f5e 100644 --- a/bench/CLAUDE.md +++ b/bench/CLAUDE.md @@ -127,6 +127,8 @@ cargo xtask bench --tools ocync,dregsy,regsync sync Managed by Terraform in `bench/terraform/aws/`. `user_data_replace_on_change = true` -- bootstrap changes recreate the instance. +Competitor tool versions are pinned at the top of `user-data.sh`. Bumping one means editing that block, which recreates the instance, so treat the first run afterward as a new baseline rather than comparing it against older records. Two of them are awkward: skopeo's module path is `go.podman.io/skopeo` since v1.23.0, and dregsy tags releases without a `v` prefix so it can only be pinned by pseudo-version. + ```bash cd bench/terraform/aws && terraform init && terraform apply # create cd bench/terraform/aws && terraform destroy # destroy diff --git a/bench/terraform/aws/user-data.sh b/bench/terraform/aws/user-data.sh index d85a311f..c0f86cf3 100644 --- a/bench/terraform/aws/user-data.sh +++ b/bench/terraform/aws/user-data.sh @@ -92,10 +92,25 @@ echo "Go: $(go version)" export HOME=/root GOPATH=/root/go GOCACHE=/root/.cache/go-build export PATH="/usr/local/go/bin:$GOPATH/bin:$PATH" +# Competitor tool versions. These are pinned rather than floating because +# bench/results/{registry}.json is compared across runs: a tool that changes +# version between instance rebuilds makes a metric shift unattributable. +# Bump deliberately, and treat the first run afterward as a new baseline. +ECR_CREDENTIAL_HELPER_VERSION="v0.12.0" +REGCLIENT_VERSION="v0.11.5" +# skopeo moved its module path to go.podman.io/skopeo at v1.23.0. The github.com +# path still serves tags, but their go.mod declares the new path, so installing +# from it fails on a module path mismatch rather than a missing version. +SKOPEO_VERSION="v1.24.0" +# dregsy tags releases without the v prefix, so the module proxy cannot resolve +# them and only serves pseudo-versions. This one is release 0.5.2, whose tag +# points at commit e92e79a. Re-derive it from the tag when bumping. +DREGSY_VERSION="v0.0.0-20250317074629-e92e79a50145" + # ── ECR credential helper ──────────────────────────────────────────────────── echo "--- Installing ECR credential helper" -go install github.com/awslabs/amazon-ecr-credential-helper/ecr-login/cli/docker-credential-ecr-login@latest +go install "github.com/awslabs/amazon-ecr-credential-helper/ecr-login/cli/docker-credential-ecr-login@$${ECR_CREDENTIAL_HELPER_VERSION}" cp /root/go/bin/docker-credential-ecr-login /usr/local/bin/ mkdir -p /home/ec2-user/.docker @@ -104,14 +119,16 @@ cat > /home/ec2-user/.docker/config.json <<'DCEOF' DCEOF chown -R ec2-user:ec2-user /home/ec2-user/.docker -echo "ecr-credential-helper: $(docker-credential-ecr-login version 2>&1 || true)" +# go install sets none of the ldflags the project's Makefile uses, so the binary +# reports "development" whatever is pinned. Echo the pin instead. +echo "ecr-credential-helper: $${ECR_CREDENTIAL_HELPER_VERSION}" # ── skopeo (dregsy transfer backend) ───────────────────────────────────────── echo "--- Installing skopeo" CGO_ENABLED=1 go install \ - -tags "exclude_graphdriver_btrfs exclude_graphdriver_devicemapper containers_image_openpgp" \ - github.com/containers/skopeo/cmd/skopeo@latest + -tags "exclude_graphdriver_btrfs containers_image_openpgp" \ + "go.podman.io/skopeo/cmd/skopeo@$${SKOPEO_VERSION}" cp /root/go/bin/skopeo /usr/local/bin/skopeo echo "skopeo: $(skopeo --version 2>&1)" @@ -119,15 +136,17 @@ echo "skopeo: $(skopeo --version 2>&1)" # ── dregsy ──────────────────────────────────────────────────────────────────── echo "--- Installing dregsy" -go install github.com/xelalexv/dregsy/cmd/dregsy@latest +go install "github.com/xelalexv/dregsy/cmd/dregsy@$${DREGSY_VERSION}" cp /root/go/bin/dregsy /usr/local/bin/dregsy -echo "dregsy: $(dregsy --version 2>&1 || true)" +# dregsy has no version flag and injects its version via release-build ldflags, +# so probing the binary prints a usage dump. Echo the pin instead. +echo "dregsy: $${DREGSY_VERSION}" # ── regsync ─────────────────────────────────────────────────────────────────── echo "--- Installing regsync" -go install github.com/regclient/regclient/cmd/regsync@latest +go install "github.com/regclient/regclient/cmd/regsync@$${REGCLIENT_VERSION}" cp /root/go/bin/regsync /usr/local/bin/regsync echo "regsync: $(regsync version 2>&1 || true)"