refactor: vendor legacy contract artifacts, fold compat e2e into the norm…

.github/ci3_labels_to_env.sh

@@ -57,11 +57,6 @@ function main {
     echo "NO_FAIL_FAST=1" >> $GITHUB_ENV
   fi
 
-  # Handle skip-compat-e2e label (escape hatch for backwards compat test failures on release PRs)
-  if has_label "ci-skip-compat-e2e"; then
-    echo "SKIP_COMPAT_E2E=1" >> $GITHUB_ENV
-  fi
-
   local chonk_input_update=0
   local chonk_input_update_requested=0
   if [ "${GITHUB_EVENT_NAME:-}" = "pull_request" ] && { has_label "ci-refresh-chonk" || head_commit_has_marker "--ci-refresh-chonk"; }; then
@@ -163,7 +158,6 @@ function main {
   if [ "$ci_mode" = "release" ] &&
      [ "$(printf '%s' "${GITHUB_REPOSITORY:-}" | tr 'A-Z' 'a-z')" = "aztecprotocol/aztec-packages-private" ]; then
     echo "PRIVATE_RELEASE=1" >> $GITHUB_ENV
-    echo "SKIP_COMPAT_E2E=1" >> $GITHUB_ENV
   fi
 
   # Determine if benchmarks should be uploaded (merge-queue, full, or full-no-test-cache modes)

.github/workflows/ci3.yml

@@ -127,7 +127,7 @@ jobs:
           PR_NUMBER: ${{ github.event.pull_request.number }}
           GITHUB_REF_NAME: ${{ github.ref_name }}
           GITHUB_ACTOR: ${{ github.actor }}
-          # Forwarded to EC2 so release_compat_e2e's nightly-failure Slack alert can build a real run URL.
+          # Groups this workflow's CI dashboard entries under the GitHub run id (stable across re-runs).
           RUN_ID: ${{ github.run_id }}
           CI3_INSTANCE_PROFILE_NAME: ${{ secrets.CI3_INSTANCE_PROFILE_NAME }}
           CI3_SECURITY_GROUP_ID: ${{ secrets.CI3_SECURITY_GROUP_ID }}

bootstrap.sh

@@ -642,94 +642,48 @@ function private_release {
   done
 }
 
-function release_compat_e2e {
-  # Runs e2e tests with contract artifacts from every prior stable release since 4.2.0 (the version
-  # where we committed to backwards compatibility). Validates that old contract artifacts work on the
-  # current release. Blocking for stable/RC releases; observational (non-blocking) for nightlies.
-  # Set SKIP_COMPAT_E2E=1 to bypass (escape hatch via the ci-skip-compat-e2e label).
-  if [ "${SKIP_COMPAT_E2E:-0}" = "1" ]; then
-    echo "SKIP_COMPAT_E2E=1, skipping backwards compatibility e2e tests."
-    return 0
-  fi
-
-  # Compat e2e only runs on amd64 — the arm64 release job just builds and publishes release-image.
-  if [ "$(arch)" == arm64 ]; then
-    echo "Skipping backwards compatibility e2e tests on arm64 (amd64 only)."
-    return 0
-  fi
-
-  # TODO: bump when v5 commits to backwards-compatible contract artifacts.
-  #   compat_major:       major version that has compat guarantees today.
-  #   compat_min_version: earliest stable tag of that major to test against
-  #                       (artifacts before this are incompatible due to oracle interface changes).
-  local compat_major="4"
-  local compat_min_version="4.2.0"
+function check_compat_artifacts_tracked {
+  # The backwards-compat e2e sweep runs as part of the normal e2e suite, driven by the artifact
+  # tarballs committed under yarn-project/end-to-end/legacy-contracts/ (see its README). This check
+  # guards the history: cutting a release while a prior stable's artifacts are missing there would
+  # silently shrink compat coverage for every future run.
+  #   compat_major:       major version line with a backwards-compatibility guarantee.
+  #   compat_min_version: earliest stable release covered by that guarantee.
+  local compat_major="5"
+  local compat_min_version="5.0.1"
 
   local current_version major
   current_version=$(jq -r '."."' .release-please-manifest.json)
   major=$(semver major "$current_version")
   if [ "$major" != "$compat_major" ]; then
-    echo "Compat e2e tests only apply to v${compat_major}. Current major: v${major}. Skipping."
+    echo "Compat artifact tracking only applies to v${compat_major}. Current major: v${major}. Skipping."
     return 0
   fi
 
   # Fetch tags (EC2 clone may not have them). Fail loud: a silent fetch failure plus an empty
-  # tag list would publish a real release with zero compat coverage.
+  # tag list would vacuously pass the check.
   if ! git fetch origin 'refs/tags/v*:refs/tags/v*'; then
     echo "ERROR: failed to fetch release tags." >&2
     return 1
   fi
 
-  # Discover stable tags for this major version (no prerelease suffixes).
-  local versions=()
+  local missing=()
   local tag ver
   while IFS= read -r tag; do
     ver=${tag#v}
     # Include only versions >= compat_min_version (sort -V puts smaller first).
-    if [ "$(printf '%s\n%s' "$compat_min_version" "$ver" | sort -V | head -1)" = "$compat_min_version" ]; then
-      versions+=("$ver")
-    fi
-  done < <(git tag -l "v${major}.*" | grep -E "^v[0-9]+\.[0-9]+\.[0-9]+$" | sort -V)
-
-  # Exclude the current tag when running on a release tag push.
-  if [[ "${REF_NAME:-}" =~ ^v?([0-9]+\.[0-9]+\.[0-9]+)$ ]]; then
-    local current_tag="${BASH_REMATCH[1]}"
-    local filtered=()
-    local v
-    for v in "${versions[@]}"; do
-      [ "$v" != "$current_tag" ] && filtered+=("$v")
-    done
-    versions=("${filtered[@]}")
-  fi
-
-  if [ ${#versions[@]} -eq 0 ]; then
-    echo "No prior stable versions found for v${major}.x (>= $compat_min_version). Skipping compat tests."
-    return 0
+    [ "$(printf '%s\n%s' "$compat_min_version" "$ver" | sort -V | head -1)" = "$compat_min_version" ] || continue
+    # The tag being released right now can't have artifacts yet — they're published by this release.
+    [ "v$ver" = "${REF_NAME:-}" ] && continue
+    [ -f "yarn-project/end-to-end/legacy-contracts/$ver.tar.gz" ] || missing+=("$ver")
+  done < <(git tag -l "v${compat_major}.*" | grep -E "^v[0-9]+\.[0-9]+\.[0-9]+$" | sort -V)
+
+  if [ ${#missing[@]} -gt 0 ]; then
+    echo "ERROR: stable release(s) missing committed compat artifacts: ${missing[*]}" >&2
+    echo "Add yarn-project/end-to-end/legacy-contracts/<version>.tar.gz for each (see that README), then re-tag." >&2
+    return 1
   fi
-
-  echo_header "Backwards compatibility e2e tests"
-  echo "Testing against ${#versions[@]} prior stable version(s): ${versions[*]}"
-
-  # Pre-populate the legacy contract cache on the host. Test containers run with --net=none, so the
-  # jest resolver's on-demand npm install would fail with EAI_AGAIN. Install here where we have network.
-  for ver in "${versions[@]}"; do
-    node yarn-project/end-to-end/src/install_legacy_contracts.cjs "$ver"
-  done
-
-  # Build and run the compat test commands in an isolated subshell so the bespoke test settings
-  # (no test cache, no fast-fail short-circuit) don't leak into the release build/publish that follows.
-  # set -e re-enables errexit inside this subshell: the caller invokes release_compat_e2e with errexit
-  # disabled (to capture its exit code), so without this a failed build/install would be masked.
-  (
-    set -e
-    export USE_TEST_CACHE=0
-    export CI_FULL=0
-    export NO_FAIL_FAST=1
-    build
-    for ver in "${versions[@]}"; do
-      yarn-project/end-to-end/bootstrap.sh compat_test_cmds "$ver"
-    done | filter_test_cmds | parallelize
-  )
+  echo "Compat artifacts tracked for all prior stable v${compat_major} releases."
 }
 
 ### SELF TESTING #######################################################################################################
@@ -1061,34 +1015,15 @@ case "$cmd" in
   # RELEASES #
   ############
   "ci-release")
-    # Single command that tests and publishes a release. Runs the backwards-compatibility e2e
-    # checks (blocking for stable/RC, observational for nightlies), then builds and publishes.
+    # Single command that tests and publishes a release.
     # DRY_RUN=1 exercises the whole flow without publishing — this is how releases are tested in CI.
     export CI=1
     export USE_TEST_CACHE=1
     if ! semver check $REF_NAME; then
       exit 1
     fi
 
-    # Backwards-compatibility e2e checks. A failure blocks stable/RC releases, but only warns on
-    # nightlies (where compat coverage is observational) so the nightly publish still proceeds.
-    # Toggle errexit explicitly rather than `release_compat_e2e || compat_rc=$?`: calling under `||`
-    # suspends errexit for the whole function (and its subshell), masking build/setup failures there.
-    compat_rc=0
-    set +e
-    release_compat_e2e
-    compat_rc=$?
-    set -e
-    if [ "$compat_rc" -ne 0 ]; then
-      if [[ "${REF_NAME:-}" == *-nightly.* ]]; then
-        run_url="https://github.com/${GITHUB_REPOSITORY:-AztecProtocol/aztec-packages}/actions/runs/${RUN_ID:-unknown}"
-        "$ci3/slack_notify" "Backwards compatibility e2e tests FAILED on nightly tag <${run_url}|${REF_NAME}>" "#team-fairies" || true
-        echo "Compat e2e failed on nightly tag — continuing (non-blocking)."
-      else
-        echo "ERROR: backwards compatibility e2e tests failed — blocking release." >&2
-        exit 1
-      fi
-    fi
+    check_compat_artifacts_tracked
 
     if [[ "$(semver prerelease $REF_NAME)" == private* ]]; then
       echo_header "Private fork release: $REF_NAME"

ci.sh

@@ -131,7 +131,10 @@ case "$cmd" in
   full|full-no-test-cache)
     export CI_DASHBOARD="prs"
     export JOB_ID="x-$cmd"
-    export AWS_SHUTDOWN_TIME=75
+    # Sized for the arm's heaviest workload: full runs include the backwards-compat e2e sweep
+    # (yarn-project/end-to-end test_cmds under CI_FULL). The timer is a runaway backstop, so the
+    # headroom is free for runs that finish sooner.
+    export AWS_SHUTDOWN_TIME=90
     bootstrap_ec2 "./bootstrap.sh ci-$cmd"
     ;;
   chonk-input-update)

ci3/bootstrap_ec2

@@ -402,7 +402,6 @@ start_build() {
     -e DRY_RUN=${DRY_RUN:-} \
     -e PRIVATE_RELEASE=${PRIVATE_RELEASE:-} \
     -e INTERNAL_DOCKER_REGISTRY=${INTERNAL_DOCKER_REGISTRY:-} \
-    -e SKIP_COMPAT_E2E=${SKIP_COMPAT_E2E:-} \
     --pids-limit=65536 \
     --shm-size=2g \
     --ulimit nofile=1048576:1048576 \

yarn-project/end-to-end/README.md

@@ -142,26 +142,28 @@ CI splits each `it` in a `.parallel.test.ts` file into its own docker job, runni
 
 ### CI test discovery — `bootstrap.sh`
 
-`end-to-end/bootstrap.sh` enumerates tests in two arrays, and a test must resolve through the relevant one
-or it **won't run in CI**:
+`end-to-end/bootstrap.sh` enumerates tests in two arrays inside `test_cmds`, and a test must resolve
+through the relevant one or it **won't run in CI**:
 
-- `test_cmds` — the standard run. Covers each category with a recursive glob (e.g.
+- The standard run. Covers each category with a recursive glob (e.g.
   `src/automine/!(simulation)/**/*.test.ts`, `src/multi-node/**/*.test.ts`), so a new file or sub-folder
   inside an existing category is picked up automatically; only a new top-level category needs its own glob
   line. Tests with bespoke handling sit outside the globs: the `single-node/prover/` lanes at the top of
   the function (real proofs and custom resources under `CI_FULL`, `FAKE_PROOFS=1` otherwise) and
   `avm_simulator` (below).
-- `compat_test_cmds` — the forward/legacy-compat run (a subset). This one enumerates **single-level leaf
-  globs** (e.g. `src/automine/token/*.test.ts`), so a new sub-folder whose tests should run against legacy
-  contract artifacts needs its own line here.
+- The backwards-compat sweep (`compat_tests`, a subset, emitted only under `CI_FULL`): reruns the
+  artifact-consuming tests once per prior stable release committed under `legacy-contracts/` (see that
+  directory's README), with `CONTRACT_ARTIFACTS_VERSION` set so the jest resolver swaps in the historical
+  artifact JSON. This one enumerates **single-level leaf globs** (e.g. `src/automine/token/*.test.ts`), so
+  a new sub-folder whose tests should run against legacy contract artifacts needs its own line here.
 
 Bespoke handling to be aware of:
 
-- **`avm_simulator`** (`automine/simulation/avm_simulator.test.ts`) has a dedicated line in `test_cmds`
-  that sets `DUMP_AVM_INPUTS_TO_DIR` (feeds the downstream `avm_check_circuit` job) and is therefore
-  excluded from the generic `simulation/` glob there (`!(avm_simulator)`). In `compat_test_cmds` it runs
+- **`avm_simulator`** (`automine/simulation/avm_simulator.test.ts`) has a dedicated line in the standard
+  run that sets `DUMP_AVM_INPUTS_TO_DIR` (feeds the downstream `avm_check_circuit` job) and is therefore
+  excluded from the generic `simulation/` glob there (`!(avm_simulator)`). In the compat sweep it runs
   as a regular test (no dump line), so it is **not** excluded there.
-- **`kernelless_simulation`** is excluded from `compat_test_cmds` only.
+- **`kernelless_simulation`** is excluded from the compat sweep only.
 
 After editing the arrays, confirm every `*.test.ts` resolves through exactly one line (no duplicate, no
 omission — anything excluded via `!(...)` must be matched by its dedicated line). Per-test bash `TIMEOUT`

yarn-project/end-to-end/bootstrap.sh

@@ -164,6 +164,68 @@ function test_cmds {
     # Run at LOG_LEVEL=verbose so the captured local-network logs are detailed enough for diagnostics.
     echo "$hash:ONLY_TERM_PARENT=1 LOG_LEVEL=verbose $run_test_script compose $flow"
   done
+
+  # Backwards-compat sweep: rerun the artifact-consuming tests against each prior stable release's
+  # contract artifacts, committed as legacy-contracts/<version>.tar.gz (see legacy-contracts/README.md).
+  # The jest resolver swaps artifact JSON imports when CONTRACT_ARTIFACTS_VERSION is set. No tarballs
+  # (a line with no stable releases yet) means no sweep. Full CI only: the sweep multiplies e2e cost
+  # per version, and the merge queue's full run is the enforcement point.
+  # Only simple (jest-based) tests, since compose/docker tests don't use the legacy jest resolver.
+  # Excludes kernelless_simulation, which asserts on the exact number of nullifiers emitted and breaks
+  # whenever contracts add/remove nullifier emissions across versions (unrelated to the compat contract
+  # surface).
+  if [ "${CI_FULL:-0}" -eq 1 ]; then
+    local compat_tests=(
+      src/automine/*.test.ts
+      src/automine/contracts/*.test.ts
+      src/automine/contracts/deploy/*.test.ts
+      src/automine/contracts/nested/*.test.ts
+      src/automine/token/*.test.ts
+      src/automine/accounts/*.test.ts
+      src/automine/effects/*.test.ts
+      src/automine/simulation/!(kernelless_simulation).test.ts
+      src/single-node/fees/*.test.ts
+      src/single-node/cross-chain/*.test.ts
+      src/single-node/bot/*.test.ts
+      src/infra/*.test.ts
+      src/p2p/*.test.ts
+      src/p2p/reqresp/*.test.ts
+    )
+    local version tarball
+    for tarball in legacy-contracts/*.tar.gz; do
+      [ -e "$tarball" ] || continue
+      version=$(basename "$tarball" .tar.gz)
+      # Unpack on the host now: the isolated test containers share this checkout, so extracting once
+      # here saves every container doing it lazily. Status goes to stderr — stdout is the command
+      # stream.
+      node src/install_legacy_contracts.cjs "$version" 1>&2
+      for test in "${compat_tests[@]}"; do
+        local name
+        if [[ "$test" == src/p2p/* || "$test" == src/automine/* ]]; then
+          # The p2p/ and automine/ folders have no `e2e_` prefix to strip; flatten their path into an
+          # e2e_<path> name (matching the historical e2e_p2p/<file> names) by dropping "src/",
+          # ".test.ts", and slashes.
+          name=${test#src/}
+          name=e2e_${name%.test.ts}
+          name=${name//\//_}
+        else
+          name=${test#*e2e_}
+          name=e2e_${name%.test.ts}
+        fi
+
+        if [[ "$test" == *.parallel.test.ts ]]; then
+          while IFS= read -r test_name; do
+            # See the matching note near the top of test_cmds: collapse docker-illegal characters so
+            # NAME is a valid container name for docker_isolate.
+            local safe_test_name=$(echo "$test_name" | sed 's/[^a-zA-Z0-9_.-]/_/g')
+            echo "$prefix:NAME=compat_${version}_${name}_${safe_test_name} CONTRACT_ARTIFACTS_VERSION=$version $run_test_script simple $test \"$test_name\""
+          done < <(extract_test_names "$test")
+        else
+          echo "$prefix:NAME=compat_${version}_${name} CONTRACT_ARTIFACTS_VERSION=$version $run_test_script simple $test"
+        fi
+      done
+    done
+  fi
 }
 
 function test {
@@ -303,61 +365,6 @@ function avm_check_circuit {
   avm_check_circuit_cmds | parallelize
 }
 
-# Generates e2e test commands using contract artifacts from a prior release version.
-# Only includes simple (jest-based) tests since compose/docker tests don't use the legacy jest resolver.
-# Excludes kernelless_simulation, which asserts on the exact number of nullifiers emitted and breaks
-# whenever contracts add/remove nullifier emissions across versions (unrelated to the compat contract
-# surface).
-function compat_test_cmds {
-  local version=${1:?version is required}
-  local run_test_script="yarn-project/end-to-end/scripts/run_test.sh"
-  local prefix="$hash:ISOLATE=1"
-  local compat_env="CONTRACT_ARTIFACTS_VERSION=$version"
-
-  local tests=(
-    src/automine/*.test.ts
-    src/automine/contracts/*.test.ts
-    src/automine/contracts/deploy/*.test.ts
-    src/automine/contracts/nested/*.test.ts
-    src/automine/token/*.test.ts
-    src/automine/accounts/*.test.ts
-    src/automine/effects/*.test.ts
-    src/automine/simulation/!(kernelless_simulation).test.ts
-    src/single-node/fees/*.test.ts
-    src/single-node/cross-chain/*.test.ts
-    src/single-node/bot/*.test.ts
-    src/infra/*.test.ts
-    src/p2p/*.test.ts
-    src/p2p/reqresp/*.test.ts
-  )
-  for test in "${tests[@]}"; do
-    local name
-    if [[ "$test" == src/p2p/* || "$test" == src/automine/* ]]; then
-      # The p2p/ and automine/ folders have no `e2e_` prefix to strip; flatten their path into an
-      # e2e_<path> name (matching the historical e2e_p2p/<file> names) by dropping "src/", ".test.ts",
-      # and slashes.
-      name=${test#src/}
-      name=e2e_${name%.test.ts}
-      name=${name//\//_}
-    else
-      name=${test#*e2e_}
-      name=e2e_${name%.test.ts}
-    fi
-
-    if [[ "$test" == *.parallel.test.ts ]]; then
-      while IFS= read -r test_name; do
-        # See the matching note in test_cmds: collapse docker-illegal characters so NAME is a valid
-        # container name for docker_isolate.
-        local safe_test_name=$(echo "$test_name" | sed 's/[^a-zA-Z0-9_.-]/_/g')
-        local full_name="compat_${version}_${name}_${safe_test_name}"
-        echo "$prefix:NAME=$full_name $compat_env $run_test_script simple $test \"$test_name\""
-      done < <(extract_test_names "$test")
-    else
-      echo "$prefix:NAME=compat_${version}_${name} $compat_env $run_test_script simple $test"
-    fi
-  done
-}
-
 case "$cmd" in
   "")
     build