fix(container-run.sh): pin the dev/build container image by digest

.claude/CLAUDE.md

@@ -2,13 +2,20 @@
 
 All commands should be run from the repository root (`/ic`).
 
-Never manually edit `ci/container/TAG`. It is bumped only by the
+Never manually edit `ci/container/TAG`, `ci/container/ic-dev.digest` or
+`ci/container/ic-build.digest`. They are written only by the
 `container-autobuild.yml` GitHub Actions workflow, which builds the new
-dev-container image and pushes it to the registry *before* the tag change
-takes effect. Hand-editing it points the tag at an image that was never built
-or published, breaking CI for everyone. If the dev container image genuinely
+dev-container images and pushes them to the registry *before* committing the
+new tag and digests. `ci/container/container-run.sh` pulls the image by the
+committed digest: a malformed pin is refused outright, and a digest the
+registry does not serve cannot be pulled, so a hand-edited pin breaks the dev
+container for everyone. If the dev container image genuinely
 needs to change (e.g. `ci/container/Dockerfile` was edited), let that workflow
-bump `TAG` — don't do it in your commit.
+bump them — don't do it in your commit. Until its bot commit lands (this
+includes the sanctioned trick of putting a random string in `TAG` to force a
+rebuild), `TAG` does not match the working tree and `container-run.sh` refuses
+to run; `CONTAINER_RUN_ALLOW_UNPINNED=1` makes it build the image locally from
+the checkout instead.
 
 # Rust
 

.claude/skills/run-in-dev-container/SKILL.md

@@ -47,7 +47,13 @@ runtime's daemon isn't reachable it prints which command it tried.
 - The repo is mounted at `/ic` and that's the working directory, so invoke
   repo-local scripts with a relative path, e.g.
   `CONTAINER_RUNTIME=docker ./ci/container/container-run.sh ./path/to/script.sh`.
-- The image is pulled from `ghcr.io` on first use (large, one-time).
+- The image is pulled from `ghcr.io` on first use (large, one-time) by the
+  digest committed in `ci/container/ic-dev.digest` and verified against it
+  before it runs; registry tags are never pulled. If `ci/container/Dockerfile`,
+  `init.sh` or `files/*` differ from what `ci/container/TAG` was built from
+  (you edited them, or the autobuild's commit hasn't landed on your branch yet)
+  the script refuses to run; `CONTAINER_RUN_ALLOW_UNPINNED=1` makes it build the
+  image locally instead, which is slow.
 - Anything the command writes under `/ic` (or `~/.cache`) persists on the host,
   since those are bind-mounted.
 - Don't nest: the script refuses to run inside an existing container.

.github/workflows/ci-main.yml

@@ -61,6 +61,8 @@ jobs:
             fuzzers:
               - ".github/workflows/ci-main.yml"
               - "ci/container/TAG"
+              - "ci/container/ic-build.digest"
+              - "ci/container/ic-dev.digest"
               - "bin/fuzzing/*.sh"
               - "**/BUILD.bazel"
               - "**/*.bzl"
@@ -822,6 +824,8 @@ jobs:
               - "**/*.lock"
               - "ci/scripts/rust-lint.sh"
               - "ci/container/TAG"
+              - "ci/container/ic-build.digest"
+              - "ci/container/ic-dev.digest"
               - ".github/**"
       - name: Run Cargo ${{ matrix.name }} Linux
         if: |
@@ -873,15 +877,91 @@ jobs:
     runs-on: ubuntu-latest
     timeout-minutes: 30
     strategy:
+      fail-fast: false
       matrix:
         include:
-          - name: TestCase 1
+          # Happy paths: the pinned image is pulled by digest, verified and run.
+          - name: ic-dev default
+            setup: ""
             command: ""
-          - name: TestCase 2
+            expect: success
+            runtime: podman
+          - name: ic-build
+            setup: ""
             command: "--image ic-build bash"
+            expect: success
+            runtime: podman
+          # A local image squatting on the tag must be ignored: the pinned digest
+          # is pulled and run instead, and the tag is re-pointed to it (checked by
+          # the "Verify the poisoned tag ..." step below). The squatter is an
+          # empty image that cannot run anything, so a successful `true` already
+          # proves the pinned image ran.
+          - name: poisoned local tag
+            setup: |
+              tar -cf empty.tar -T /dev/null
+              repo=ghcr.io/dfinity
+              sudo podman import empty.tar "$repo/ic-dev:$(< ci/container/TAG)"
+            command: "true"
+            expect: success
+            runtime: podman
+          # A tampered pin must be refused before anything is pulled or built.
+          - name: malformed pin refused
+            setup: echo not-a-digest > ci/container/ic-dev.digest
+            command: "true"
+            expect: "refusing to pull an unpinned image"
+            runtime: podman
+          # A well-formed pin the registry does not serve: the pull fails and, by
+          # default, neither a local build nor a cached image is used.
+          - name: unknown digest refused
+            setup: printf 'sha256:%064d\n' 0 > ci/container/ic-dev.digest
+            command: "true"
+            expect: "refusing to build an unpinned image locally"
+            runtime: podman
+          # The same failed pull while a local image squats on the tag: the cached
+          # mutable-tag image must not be run either (only
+          # CONTAINER_RUN_ALLOW_UNPINNED=1 permits that), so this still refuses.
+          - name: unknown digest with poisoned local tag refused
+            setup: |
+              tar -cf empty.tar -T /dev/null
+              repo=ghcr.io/dfinity
+              sudo podman import empty.tar "$repo/ic-dev:$(< ci/container/TAG)"
+              printf 'sha256:%064d\n' 0 > ci/container/ic-dev.digest
+            command: "true"
+            expect: "refusing to build an unpinned image locally"
+            runtime: podman
+          # Edited container inputs (ci/container/{Dockerfile,init.sh,files/*}): no
+          # reviewed digest exists for the computed tag, so by default the script
+          # refuses without pulling or building anything. The setup step edits the
+          # input after the TAG-sync gate below has passed on the pristine checkout.
+          - name: edited inputs refused
+            setup: echo '# edited after the sync check' >> ci/container/Dockerfile
+            command: "true"
+            expect: "refusing to run an unpinned image"
+            runtime: podman
+          # With the explicit opt-out, an image cached under the computed tag is
+          # reused instead of being rebuilt, and the script must say that it is
+          # unverified. The cache is seeded by re-tagging the pinned image.
+          - name: edited inputs with opt-out reuses local image
+            setup: |
+              echo '# edited after the sync check' >> ci/container/Dockerfile
+              repo=ghcr.io/dfinity
+              pinned="$repo/ic-dev@$(< ci/container/ic-dev.digest)"
+              sudo podman pull "$pinned"
+              sudo podman tag "$pinned" "$repo/ic-dev:$(ci/container/get-image-tag.sh)"
+            command: "true"
+            allow_unpinned: "1"
+            expect: success
+            expect_warning: "NOT verified against a reviewed digest pin"
+            runtime: podman
+          - name: docker runtime
+            setup: ""
+            command: "true"
+            expect: success
+            runtime: docker
     steps:
       - *checkout
       - name: Workaround for actions/runner-images#14473 (crun pinning)
+        if: matrix.runtime == 'podman'
         run: |
           # Workaround for actions/runner-images#14473:
           # Podman 5.x on ubuntu-24.04/22.04 picks /usr/bin/crun (1.14.1) over
@@ -904,14 +984,153 @@ jobs:
           ref: ${{ inputs.commit-sha }}
           filters: |
             container-run:
+              - '.github/workflows/ci-main.yml'
               - '.github/workflows/ci-pr-only.yml'
               - '.github/workflows/container-autobuild.yml'
               - 'ci/container/**'
-      - name: Test Container Run
+            pins:
+              - '.devcontainer/**'
+              - '.github/workflows/**'
+      - name: Decide whether to run
+        id: decide
         if: |
           steps.filter.outputs.container-run == 'true' ||
           env.BRANCH_NAME == 'master' ||
           startsWith(env.BRANCH_NAME, 'rc--') ||
           startsWith(env.BRANCH_NAME, 'hotfix-')
+        run: echo "run=true" >> "$GITHUB_OUTPUT"
+      - name: Check the digest pins are consistent
+        if: steps.decide.outputs.run == 'true' || steps.filter.outputs.pins == 'true'
+        run: |
+          # ci/container/*.digest, the ic-dev pin in .devcontainer/ and the ic-build
+          # pins in the workflows are all written by container-autobuild.yml and
+          # must agree. Never compare against what the registry serves under
+          # ci/container/TAG: that would make the mutable tag the oracle again.
+          #
+          # container-autobuild.yml rewrites every "<prefix>ic-build[:@]..." string
+          # under .github/workflows/ with sed, so the image names below are kept
+          # in a variable to keep this script from being rewritten too.
+          set -euo pipefail
+          prefix="ghcr.io/dfinity/"
+          dev_digest="$(< ci/container/ic-dev.digest)"
+          build_digest="$(< ci/container/ic-build.digest)"
+          for digest in "$dev_digest" "$build_digest"; do
+            if ! [[ $digest =~ ^sha256:[0-9a-f]{64}$ ]]; then
+              echo "::error::malformed digest '$digest' in ci/container/*.digest (they are written by container-autobuild.yml)"
+              exit 1
+            fi
+          done
+          # Every reference to either image (":tag" or "@digest", well-formed or
+          # not) in both consumer locations must be exactly the pinned digest
+          # reference, so a reference changed to a mutable tag or a malformed
+          # digest is rejected too, not only a mismatching well-formed one. At
+          # least one reference per image must exist (it does today), which also
+          # guards against the grep pattern going stale.
+          check_refs() {
+            local image=$1 digest=$2 refs
+            # grep exits 1 when nothing matches; treat that as an empty set so the
+            # comparison below reports it instead of pipefail aborting the step.
+            refs="$({ grep -rhoE "${prefix}${image}[:@][^\"'[:space:]]+" .devcontainer .github/workflows || true; } | sort -u)"
+            if [ "$refs" != "${prefix}${image}@${digest}" ]; then
+              echo "::error file=ci/container/${image}.digest::every ${image} reference in .devcontainer/ and .github/workflows/ must be ${prefix}${image}@${digest} (ci/container/${image}.digest; all are written by container-autobuild.yml), found:"
+              echo "${refs:-<none>}"
+              exit 1
+            fi
+          }
+          check_refs ic-dev "$dev_digest"
+          check_refs ic-build "$build_digest"
+      - name: Check ci/container/TAG matches the container inputs
+        id: sync
+        if: steps.decide.outputs.run == 'true'
+        run: |
+          # When TAG is out of sync there is no reviewed digest for the working
+          # tree and container-run.sh refuses to run (it only builds locally with
+          # CONTAINER_RUN_ALLOW_UNPINNED=1), so every leg below would fail. On a
+          # pull request that state is transient: the container-autobuild.yml bot
+          # commit brings TAG and *.digest in sync and re-triggers CI. (PRs from
+          # forks cannot change the container inputs: touching ci/* closes them
+          # automatically, see .github/repo_policies/EXTERNAL_CONTRIB_BLACKLIST.)
+          # Anywhere else (push to master, the merge queue, release-testing on
+          # rc--/hotfix-, a manual dispatch) it means that commit never landed,
+          # so fail.
+          set -euo pipefail
+          computed="$(ci/container/get-image-tag.sh)"
+          pinned="$(< ci/container/TAG)"
+          if [ "$computed" = "$pinned" ]; then
+            echo "in_sync=true" >> "$GITHUB_OUTPUT"
+          elif [ "$CI_EVENT_NAME" = pull_request ]; then
+            echo "::notice file=ci/container/TAG::ci/container/TAG ($pinned) does not match the container inputs (computed $computed); skipping the container-run tests: the Container IC Build Image workflow will commit TAG and *.digest to this branch and re-trigger CI"
+            echo "in_sync=false" >> "$GITHUB_OUTPUT"
+          else
+            echo "::error file=ci/container/TAG::ci/container/TAG ($pinned) does not match the container inputs (computed $computed): the container-autobuild.yml commit did not land on $BRANCH_NAME ($CI_EVENT_NAME)"
+            exit 1
+          fi
+      - name: Set up "${{ matrix.name }}"
+        if: steps.sync.outputs.in_sync == 'true' && matrix.setup != ''
+        run: ${{ matrix.setup }}
+      - name: Test Container Run
+        if: steps.sync.outputs.in_sync == 'true'
+        env:
+          CONTAINER_RUNTIME: ${{ matrix.runtime }}
+          # Empty for every leg that does not set allow_unpinned, which the script
+          # treats as unset.
+          CONTAINER_RUN_ALLOW_UNPINNED: ${{ matrix.allow_unpinned }}
         run: |
-          ./ci/container/container-run.sh ${{ matrix.command }}
+          set -euo pipefail
+          if [ "${{ matrix.expect }}" = success ]; then
+            set +e
+            out="$(./ci/container/container-run.sh ${{ matrix.command }} 2>&1)"
+            rc=$?
+            set -e
+            printf '%s\n' "$out"
+            if [ "$rc" -ne 0 ]; then
+              echo "::error::container-run.sh exited with $rc"
+              exit 1
+            fi
+          else
+            # A tampered pin or edited inputs must fail fast: not succeed, not hit
+            # the timeout, and not fall through to a local build that merely times
+            # the job out. Snapshot the store first: a refusal must leave it
+            # untouched (nothing pulled or built) and must not have run anything,
+            # in particular not an image the setup step cached under a tag.
+            images_before="$(sudo podman images --no-trunc -q | sort -u)"
+            containers_before="$(sudo podman ps -aq)"
+            set +e
+            out="$(timeout 300 ./ci/container/container-run.sh ${{ matrix.command }} 2>&1)"
+            rc=$?
+            set -e
+            printf '%s\n' "$out"
+            if [ "$rc" -eq 0 ] || [ "$rc" -eq 124 ]; then
+              echo "::error::expected container-run.sh to refuse quickly, got exit code $rc"
+              exit 1
+            fi
+            if ! grep -qF -- "${{ matrix.expect }}" <<<"$out"; then
+              echo "::error::expected the output to contain '${{ matrix.expect }}'"
+              exit 1
+            fi
+            if [ "$(sudo podman images --no-trunc -q | sort -u)" != "$images_before" ]; then
+              echo "::error::an image was pulled or built despite the refusal"
+              exit 1
+            fi
+            if [ "$(sudo podman ps -aq)" != "$containers_before" ]; then
+              echo "::error::a container was run despite the refusal"
+              exit 1
+            fi
+          fi
+          if [ -n "${{ matrix.expect_warning }}" ] && ! grep -qF -- "${{ matrix.expect_warning }}" <<<"$out"; then
+            echo "::error::expected the output to contain '${{ matrix.expect_warning }}'"
+            exit 1
+          fi
+      - name: Verify the poisoned tag was re-pointed to the verified image
+        if: steps.sync.outputs.in_sync == 'true' && matrix.name == 'poisoned local tag'
+        run: |
+          set -euo pipefail
+          repo=ghcr.io/dfinity
+          tag="$repo/ic-dev:$(< ci/container/TAG)"
+          want="$repo/ic-dev@$(< ci/container/ic-dev.digest)"
+          digests="$(sudo podman image inspect --format '{{range .RepoDigests}}{{println .}}{{end}}' "$tag")"
+          printf '%s\n' "$digests"
+          if ! grep -qxF "$want" <<<"$digests"; then
+            echo "::error::$tag does not carry the pinned digest $want after the run"
+            exit 1
+          fi

.github/workflows/container-autobuild.yml

@@ -45,7 +45,9 @@ jobs:
           if grep $image_tag ci/container/TAG; then
             echo "build_image=false" >> "$GITHUB_OUTPUT"
             echo "Update with this image already in this PR!"
-            echo "Modify ci/container/TAG with random string to trigger new build."
+            echo "Modify ci/container/TAG with a random string to trigger a new build. Until this"
+            echo "workflow's commit lands, container-run.sh on that commit refuses to run because no"
+            echo "reviewed digest exists for it; set CONTAINER_RUN_ALLOW_UNPINNED=1 to build the image locally instead."
           else
             echo "build_image=true" >> "$GITHUB_OUTPUT"
             echo "image_tag=$image_tag" >> "$GITHUB_OUTPUT"
@@ -160,10 +162,25 @@ jobs:
           set -xeuo pipefail
 
           IMAGE_PREFIX="ghcr.io/dfinity/"
-          IMG_NAME_IC_BUILD="${IMAGE_PREFIX}ic-build@${{ needs.ic-build-image.outputs.ic-build-imagedigest }}"
-          IMG_NAME_IC_DEV="${IMAGE_PREFIX}ic-dev@${{ needs.ic-build-image.outputs.ic-dev-imagedigest }}"
-
+          IC_BUILD_DIGEST='${{ needs.ic-build-image.outputs.ic-build-imagedigest }}'
+          IC_DEV_DIGEST='${{ needs.ic-build-image.outputs.ic-dev-imagedigest }}'
+          # container-run.sh pulls by these digests and refuses a malformed one, so
+          # fail here rather than committing a pin that breaks every container run.
+          for digest in "$IC_BUILD_DIGEST" "$IC_DEV_DIGEST"; do
+            if ! [[ $digest =~ ^sha256:[0-9a-f]{64}$ ]]; then
+              echo "::error::malformed image digest '$digest'"
+              exit 1
+            fi
+          done
+          IMG_NAME_IC_BUILD="${IMAGE_PREFIX}ic-build@${IC_BUILD_DIGEST}"
+          IMG_NAME_IC_DEV="${IMAGE_PREFIX}ic-dev@${IC_DEV_DIGEST}"
+
+          # TAG and the *.digest files are read by ci/container/container-run.sh,
+          # which pulls the images by digest; they live directly under
+          # ci/container/ (never in ci/container/files/, an input of get-image-tag.sh).
           echo '${{ needs.build-image-prep.outputs.image_tag }}' > ci/container/TAG
+          echo "$IC_BUILD_DIGEST" > ci/container/ic-build.digest
+          echo "$IC_DEV_DIGEST" > ci/container/ic-dev.digest
           pushd .devcontainer
           sed -i -E "s|${IMAGE_PREFIX}ic-dev(:\|@)[^\"]{5,}|$IMG_NAME_IC_DEV|g" -- *
           popd