diff --git a/bench/CLAUDE.md b/bench/CLAUDE.md index 0e94da4..8d7b1f5 100644 --- a/bench/CLAUDE.md +++ b/bench/CLAUDE.md @@ -127,6 +127,8 @@ cargo xtask bench --tools ocync,dregsy,regsync sync Managed by Terraform in `bench/terraform/aws/`. `user_data_replace_on_change = true` -- bootstrap changes recreate the instance. +Competitor tool versions are pinned at the top of `user-data.sh`. Bumping one means editing that block, which recreates the instance, so treat the first run afterward as a new baseline rather than comparing it against older records. Two of them are awkward: skopeo's module path is `go.podman.io/skopeo` since v1.23.0, and dregsy tags releases without a `v` prefix so it can only be pinned by pseudo-version. + ```bash cd bench/terraform/aws && terraform init && terraform apply # create cd bench/terraform/aws && terraform destroy # destroy diff --git a/bench/terraform/aws/user-data.sh b/bench/terraform/aws/user-data.sh index d85a311..c0f86cf 100644 --- a/bench/terraform/aws/user-data.sh +++ b/bench/terraform/aws/user-data.sh @@ -92,10 +92,25 @@ echo "Go: $(go version)" export HOME=/root GOPATH=/root/go GOCACHE=/root/.cache/go-build export PATH="/usr/local/go/bin:$GOPATH/bin:$PATH" +# Competitor tool versions. These are pinned rather than floating because +# bench/results/{registry}.json is compared across runs: a tool that changes +# version between instance rebuilds makes a metric shift unattributable. +# Bump deliberately, and treat the first run afterward as a new baseline. +ECR_CREDENTIAL_HELPER_VERSION="v0.12.0" +REGCLIENT_VERSION="v0.11.5" +# skopeo moved its module path to go.podman.io/skopeo at v1.23.0. The github.com +# path still serves tags, but their go.mod declares the new path, so installing +# from it fails on a module path mismatch rather than a missing version. +SKOPEO_VERSION="v1.24.0" +# dregsy tags releases without the v prefix, so the module proxy cannot resolve +# them and only serves pseudo-versions. This one is release 0.5.2, whose tag +# points at commit e92e79a. Re-derive it from the tag when bumping. +DREGSY_VERSION="v0.0.0-20250317074629-e92e79a50145" + # ── ECR credential helper ──────────────────────────────────────────────────── echo "--- Installing ECR credential helper" -go install github.com/awslabs/amazon-ecr-credential-helper/ecr-login/cli/docker-credential-ecr-login@latest +go install "github.com/awslabs/amazon-ecr-credential-helper/ecr-login/cli/docker-credential-ecr-login@$${ECR_CREDENTIAL_HELPER_VERSION}" cp /root/go/bin/docker-credential-ecr-login /usr/local/bin/ mkdir -p /home/ec2-user/.docker @@ -104,14 +119,16 @@ cat > /home/ec2-user/.docker/config.json <<'DCEOF' DCEOF chown -R ec2-user:ec2-user /home/ec2-user/.docker -echo "ecr-credential-helper: $(docker-credential-ecr-login version 2>&1 || true)" +# go install sets none of the ldflags the project's Makefile uses, so the binary +# reports "development" whatever is pinned. Echo the pin instead. +echo "ecr-credential-helper: $${ECR_CREDENTIAL_HELPER_VERSION}" # ── skopeo (dregsy transfer backend) ───────────────────────────────────────── echo "--- Installing skopeo" CGO_ENABLED=1 go install \ - -tags "exclude_graphdriver_btrfs exclude_graphdriver_devicemapper containers_image_openpgp" \ - github.com/containers/skopeo/cmd/skopeo@latest + -tags "exclude_graphdriver_btrfs containers_image_openpgp" \ + "go.podman.io/skopeo/cmd/skopeo@$${SKOPEO_VERSION}" cp /root/go/bin/skopeo /usr/local/bin/skopeo echo "skopeo: $(skopeo --version 2>&1)" @@ -119,15 +136,17 @@ echo "skopeo: $(skopeo --version 2>&1)" # ── dregsy ──────────────────────────────────────────────────────────────────── echo "--- Installing dregsy" -go install github.com/xelalexv/dregsy/cmd/dregsy@latest +go install "github.com/xelalexv/dregsy/cmd/dregsy@$${DREGSY_VERSION}" cp /root/go/bin/dregsy /usr/local/bin/dregsy -echo "dregsy: $(dregsy --version 2>&1 || true)" +# dregsy has no version flag and injects its version via release-build ldflags, +# so probing the binary prints a usage dump. Echo the pin instead. +echo "dregsy: $${DREGSY_VERSION}" # ── regsync ─────────────────────────────────────────────────────────────────── echo "--- Installing regsync" -go install github.com/regclient/regclient/cmd/regsync@latest +go install "github.com/regclient/regclient/cmd/regsync@$${REGCLIENT_VERSION}" cp /root/go/bin/regsync /usr/local/bin/regsync echo "regsync: $(regsync version 2>&1 || true)"