Skip to content

use full commit verification in blocksync (CON-246) #21730

use full commit verification in blocksync (CON-246)

use full commit verification in blocksync (CON-246) #21730

# This workflow will build a golang project
# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-go
name: Docker Integration Test
on:
push:
branches:
- main
- release/**
pull_request:
branches:
- main
- release/**
merge_group:
branches:
- main
- release/**
concurrency:
group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event_name == 'push' && github.sha || github.ref }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
defaults:
run:
shell: bash
env:
# Non-cone sparse-checkout pattern shared by every job that needs the source
# tree: everything except the darwin/aarch64 wasmvm archives, which CI
# (linux/amd64 only) never links and which make up ~190 MB of the tree.
SPARSE_CHECKOUT: |
/*
!/sei-wasmvm/internal/api/*darwin*
!/sei-wasmvm/internal/api/*aarch64*
!/sei-wasmvm/internal/api/*.dylib
jobs:
# Generate the integration-test matrix from integration-test-matrix.json.
# Runs the full matrix on every event (pull_request, push, merge_group) — all
# Autobahn coverage runs on every PR. The matrix is built here, in a job
# output, because the `matrix` context is NOT available to a job-level `if:`.
#
# Packing policy for integration-test-matrix.json. Every row is its own job
# and all rows start at once, so wall-clock = prepare-cluster + the slowest
# row, while each extra row costs its ~2 min of fixed setup in runner-minutes.
# A row is a list of suites that share one cluster boot (same `env`) and run
# back-to-back against the same chain state. When adding a suite:
# - Keep every row under ~8 min of test time. Join an existing row with a
# matching `env` that has room; otherwise add a new row.
# - Suites that assert absolute chain state (total supply, wasm code
# counts, ...) run first in their row, and the row's `_comment` names the
# assertion so a later suite is not appended ahead of it.
# - Destructive suites stay alone: anything that kills, wipes, or
# state-syncs a validator, disables a module, or runs a chain upgrade
# leaves the cluster in a state no other suite should inherit.
# - A suite that boots its own cluster (Autobahn Basic) is a separate job,
# not a matrix row.
# - A row with "disabled": true stays in the file but does not run. Its
# `_comment` says why and carries the TODO to re-enable it.
set-matrix:
name: Resolve integration test matrix
runs-on: ubuntu-latest
timeout-minutes: 5
outputs:
tests: ${{ steps.build.outputs.tests }}
steps:
# See: https://github.com/actions/checkout/releases/tag/v7.0.0
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0
with:
sparse-checkout: .github/workflows
- name: Build matrix
id: build
run: |
set -euo pipefail
matrix_file=.github/workflows/integration-test-matrix.json
tests=$(jq -c '[.[] | select(.disabled != true)]' "$matrix_file")
echo "tests=$tests" >> "$GITHUB_OUTPUT"
echo "Selected $(echo "$tests" | jq 'length') matrix entries."
jq -r '.[] | select(.disabled == true) | "Skipped disabled entry: \(.name)"' "$matrix_file"
# Build localnode/rpcnode images and seid once; matrix jobs pull the images
# from GHCR (tagged with the run id) and download seid as a small artifact.
prepare-cluster:
name: Prepare integration test cluster
runs-on: ubuntu-large
timeout-minutes: 45
# packages:write required to push images and buildx cache to GHCR. Fork PRs
# cannot publish org packages, so integration tests cannot run from forks; a
# maintainer must push external contributions to a branch on this repo.
# cache-to is restricted to push events to avoid cache thrashing on PRs.
permissions:
packages: write
contents: read
env:
GHCR_LOCALNODE: ghcr.io/sei-protocol/sei-chain-integration-test-localnode
GHCR_RPCNODE: ghcr.io/sei-protocol/sei-chain-integration-test-rpcnode
steps:
# See: https://github.com/actions/checkout/releases/tag/v7.0.0
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0
with:
# CI only builds linux/amd64; leave the darwin/aarch64 wasmvm
# archives (~500 MB of the tree) out of the fetch.
sparse-checkout: ${{ env.SPARSE_CHECKOUT }}
sparse-checkout-cone-mode: false
# Reclaim disk and push Docker + Go caches onto /mnt before the
# image builds so the runner's small root filesystem does not fill up and
# crash the runner mid-job.
- name: Reclaim runner disk
env:
# Image build + seid build need more headroom than test jobs.
CI_FREE_DISK_MIN_GIB: 60
run: bash .github/scripts/ci-free-disk.sh
- name: Relocate Docker data-root to /mnt
run: bash .github/scripts/ci-relocate-docker-data-root.sh
- name: Relocate Go caches to /mnt
run: bash .github/scripts/ci-relocate-go-cache.sh
- name: Login to Docker Hub
# See: https://github.com/docker/login-action/releases/tag/v4.2.0
uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee
if: env.DOCKERHUB_USERNAME != ''
env:
DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
- name: Login to GitHub Container Registry
# See: https://github.com/docker/login-action/releases/tag/v4.2.0
uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee
with:
registry: ghcr.io
username: ${{ github.repository_owner }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Set up Docker Buildx
# See: https://github.com/docker/setup-buildx-action/releases/tag/v4.1.0
uses: docker/setup-buildx-action@d7f5e7f509e45cec5c76c4d5afdd7de93d0b3df5
- name: Build localnode image (cached)
uses: docker/build-push-action@v6
with:
context: docker/localnode
file: docker/localnode/Dockerfile
platforms: linux/amd64
push: false
load: true
tags: sei-chain/localnode
# Run-id label makes each run's image digest unique (config-only
# change, layer cache unaffected) so old run tags can be pruned without
# touching digests other runs still reference.
labels: sei-chain.ci-run-id=${{ github.run_id }}
cache-from: type=registry,ref=${{ env.GHCR_LOCALNODE }}:cache
cache-to: ${{ github.event_name == 'push' && format('type=registry,ref={0}:cache,mode=max', env.GHCR_LOCALNODE) || '' }}
- uses: actions/setup-go@v7
with:
go-version: '1.27.4'
- name: Build seid and frozen RPC router binaries in localnode container
env:
DOCKER_PLATFORM: linux/amd64
run: make build-seid-in-localnode-ci
- name: Push localnode image to GHCR for test jobs
run: |
source .github/scripts/docker-registry-retry.sh
docker tag sei-chain/localnode "${GHCR_LOCALNODE}:${{ github.run_id }}"
push_with_retry "${GHCR_LOCALNODE}:${{ github.run_id }}"
- name: Package CI artifacts
run: tar -czf integration-build.tar.gz build/seid build/frozen-rpc-router
- name: Upload integration CI artifacts
uses: actions/upload-artifact@v4
with:
name: integration-ci-artifacts
path: integration-build.tar.gz
retention-days: 1
# rpcnode does not depend on seid, so it is built and pushed in parallel with
# prepare-cluster instead of on its critical path.
prepare-rpcnode:
name: Prepare integration test rpcnode image
runs-on: ubuntu-latest
timeout-minutes: 20
permissions:
packages: write
contents: read
env:
GHCR_RPCNODE: ghcr.io/sei-protocol/sei-chain-integration-test-rpcnode
steps:
# See: https://github.com/actions/checkout/releases/tag/v7.0.0
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0
with:
sparse-checkout: |
.github
docker/rpcnode
- name: Login to Docker Hub
# See: https://github.com/docker/login-action/releases/tag/v4.2.0
uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee
if: env.DOCKERHUB_USERNAME != ''
env:
DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
- name: Login to GitHub Container Registry
# See: https://github.com/docker/login-action/releases/tag/v4.2.0
uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee
with:
registry: ghcr.io
username: ${{ github.repository_owner }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Set up Docker Buildx
# See: https://github.com/docker/setup-buildx-action/releases/tag/v4.1.0
uses: docker/setup-buildx-action@d7f5e7f509e45cec5c76c4d5afdd7de93d0b3df5
- name: Build rpcnode image (cached)
uses: docker/build-push-action@v6
with:
context: docker/rpcnode
file: docker/rpcnode/Dockerfile
platforms: linux/amd64
push: false
load: true
tags: sei-chain/rpcnode
labels: sei-chain.ci-run-id=${{ github.run_id }}
cache-from: type=registry,ref=${{ env.GHCR_RPCNODE }}:cache
cache-to: ${{ github.event_name == 'push' && format('type=registry,ref={0}:cache,mode=max', env.GHCR_RPCNODE) || '' }}
- name: Push rpcnode image to GHCR for test jobs
run: |
source .github/scripts/docker-registry-retry.sh
docker tag sei-chain/rpcnode "${GHCR_RPCNODE}:${{ github.run_id }}"
push_with_retry "${GHCR_RPCNODE}:${{ github.run_id }}"
integration-tests:
name: Integration Test (${{ matrix.test.name }})
needs: [ set-matrix, prepare-cluster, prepare-rpcnode ]
runs-on: ubuntu-large
timeout-minutes: 30
permissions:
packages: read
contents: read
env:
# Wall-clock budget for a whole row's scripts. Each `go test` invocation
# gets `-timeout $JOB_TIMEOUT` set to whatever is left of this budget, so
# a hang anywhere in the row hits Go's timeout (goroutine dump) before
# timeout-minutes cancels the job.
ROW_BUDGET_SECONDS: 1560
GHCR_LOCALNODE: ghcr.io/sei-protocol/sei-chain-integration-test-localnode
GHCR_RPCNODE: ghcr.io/sei-protocol/sei-chain-integration-test-rpcnode
DAPP_TESTS_MNEMONIC: ${{ secrets.DAPP_TESTS_MNEMONIC }}
strategy:
# other jobs should run even if one integration test fails
fail-fast: false
# Matrix is resolved by the set-matrix job from
# integration-test-matrix.json.
matrix:
test: ${{ fromJSON(needs.set-matrix.outputs.tests) }}
steps:
# See: https://github.com/actions/checkout/releases/tag/v7.0.0
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0
with:
sparse-checkout: ${{ env.SPARSE_CHECKOUT }}
sparse-checkout-cone-mode: false
- name: Reclaim runner disk
run: bash .github/scripts/ci-free-disk.sh
- name: Relocate Docker data-root to /mnt
run: bash .github/scripts/ci-relocate-docker-data-root.sh
- name: Relocate Go caches to /mnt
run: bash .github/scripts/ci-relocate-go-cache.sh
- uses: actions/setup-node@v4
with:
node-version: '22'
cache: npm
cache-dependency-path: |
contracts/package-lock.json
integration_test/dapp_tests/package-lock.json
integration_test/rpc_tests/package-lock.json
integration_test/precompile_tests/package-lock.json
- name: Install jq
run: command -v jq >/dev/null || sudo apt-get install -y jq
- name: Set up Go
uses: actions/setup-go@v7
with:
go-version: '1.27.4'
- name: Login to GitHub Container Registry
# See: https://github.com/docker/login-action/releases/tag/v4.2.0
uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee
with:
registry: ghcr.io
username: ${{ github.repository_owner }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Download integration CI artifacts
uses: actions/download-artifact@v4
with:
name: integration-ci-artifacts
- name: Load prebuilt seid and pull Docker images
run: |
tar -xzf integration-build.tar.gz
# The binaries are now extracted under build/; drop the
# tarball so its copy does not sit on disk for the rest of the job.
rm -f integration-build.tar.gz
source .github/scripts/docker-registry-retry.sh
pull_with_retry "${GHCR_LOCALNODE}:${{ github.run_id }}" & localnode_pull=$!
pull_with_retry "${GHCR_RPCNODE}:${{ github.run_id }}" & rpcnode_pull=$!
wait "${localnode_pull}"
wait "${rpcnode_pull}"
docker tag "${GHCR_LOCALNODE}:${{ github.run_id }}" sei-chain/localnode
docker tag "${GHCR_RPCNODE}:${{ github.run_id }}" sei-chain/rpcnode
# Drop the redundant run-id-tagged references now that the images are
# retagged as sei-chain/*; keeps a single tag per image.
docker image rm "${GHCR_LOCALNODE}:${{ github.run_id }}" "${GHCR_RPCNODE}:${{ github.run_id }}" || true
- name: Start 4 node docker cluster
if: ${{ matrix.test.cluster != 'frozen-rpc-router' }}
run: DOCKER_DETACH=true INVARIANT_CHECK_INTERVAL=10 ${{matrix.test.env}} make docker-cluster-start-ci
- name: Start frozen RPC router docker cluster
if: ${{ matrix.test.cluster == 'frozen-rpc-router' }}
run: DOCKER_DETACH=true INVARIANT_CHECK_INTERVAL=10 make docker-frozen-rpc-cluster-start-ci
- name: Wait for docker cluster to start
run: |
until [ $(cat build/generated/launch.complete 2>/dev/null |wc -l) = 4 ]
do
sleep 2
done
sleep 5
- name: Start rpc node
if: ${{ matrix.test.cluster != 'frozen-rpc-router' }}
run: ${{matrix.test.env}} make run-rpc-node-integration-ci &
# Autobahn clusters serve the EVM JSON-RPC only, so they get the startup
# gate that reads liveness from there instead of from Tendermint RPC.
- name: Verify Sei Chain is running
if: ${{ matrix.test.cluster != 'frozen-rpc-router' }}
run: |
case "${{ matrix.test.env }}" in
*AUTOBAHN=true*) startup=TestAutobahnStartup ;;
*) startup=TestStartup ;;
esac
go test -tags yaml_integration -v -timeout 5m ./integration_test/runner/... -run "$startup"
- name: ${{ matrix.test.name }}
run: |
scripts=$(echo '${{ toJson(matrix.test.scripts) }}' | jq -r '.[]')
row_start=$SECONDS
IFS=$'\n' # change the internal field separator to newline
for script in $scripts
do
remaining=$(( ROW_BUDGET_SECONDS - (SECONDS - row_start) ))
if [ "${remaining}" -lt 60 ]; then remaining=60; fi
export JOB_TIMEOUT="${remaining}s"
echo "::group::${script} (JOB_TIMEOUT=${JOB_TIMEOUT})"
if ! bash -c "${script}"; then
echo "::endgroup::"
echo "::error::suite failed: ${script}"
exit 1
fi
echo "::endgroup::"
done
unset IFS # revert the internal field separator back to default
- name: Print node logs on failure
if: ${{ failure() }}
run: |
set -euo pipefail
for c in sei-node-0 sei-node-1 sei-node-2 sei-node-3; do
echo "==================== ${c} (docker logs tail) ===================="
docker logs --tail 200 "${c}" || true
echo "==================== ${c} (seid log file tail) ===================="
# Logs are accessible on host since build/generated is mounted in containers
NODE_ID=${c#sei-node-}
if [ -f "build/generated/logs/seid-${NODE_ID}.log" ]; then
tail -200 "build/generated/logs/seid-${NODE_ID}.log" || true
fi
done
echo "==================== sei-frozen-rpc-router ===================="
docker logs --tail 200 sei-frozen-rpc-router || true
- name: Prepare log artifact name
if: ${{ always() }}
id: log_artifact_meta
run: |
raw="${{ matrix.test.name }}"
safe=$(echo "$raw" | tr '[:upper:]' '[:lower:]' | sed -E 's/[^a-z0-9]+/-/g' | sed -E 's/^-+|-+$//g')
if [ -z "$safe" ]; then
safe="logs"
fi
echo "artifact_name=$safe" >> $GITHUB_OUTPUT
- name: Collect logs directory
if: ${{ always() }}
run: |
LOG_ROOT="artifacts/sei-${{ steps.log_artifact_meta.outputs.artifact_name }}"
mkdir -p "$LOG_ROOT"
if [ -d build/generated/logs ]; then
cp -r build/generated/logs "$LOG_ROOT/"
else
echo "No logs directory found"
fi
- name: Upload logs directory
if: ${{ always() }}
uses: actions/upload-artifact@v4
with:
name: integration-logs-${{ steps.log_artifact_meta.outputs.artifact_name }}
path: artifacts/sei-${{ steps.log_artifact_meta.outputs.artifact_name }}
if-no-files-found: warn
- name: Upload RPC parity test report
if: ${{ always() }}
uses: actions/upload-artifact@v4
with:
name: rpc-parity-report-${{ steps.log_artifact_meta.outputs.artifact_name }}
path: integration_test/rpc_tests/reports/merged
if-no-files-found: ignore
- name: Upload precompile test report
if: ${{ always() }}
uses: actions/upload-artifact@v4
with:
name: precompile-report-${{ steps.log_artifact_meta.outputs.artifact_name }}
path: integration_test/precompile_tests/reports/merged
if-no-files-found: ignore
# Autobahn integration suite (Autobahn Basic) — boots its own cluster via
# TestMain (docker-cluster-start / -stop), so it runs in a separate job
# rather than as a matrix entry that would share the integration-tests
# cluster.
autobahn-integration-tests:
name: Integration Test (Autobahn Basic)
runs-on: ubuntu-large
timeout-minutes: 45
needs: [ prepare-cluster, prepare-rpcnode ]
permissions:
packages: read
contents: read
env:
GHCR_LOCALNODE: ghcr.io/sei-protocol/sei-chain-integration-test-localnode
GHCR_RPCNODE: ghcr.io/sei-protocol/sei-chain-integration-test-rpcnode
steps:
# See: https://github.com/actions/checkout/releases/tag/v7.0.0
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0
with:
sparse-checkout: ${{ env.SPARSE_CHECKOUT }}
sparse-checkout-cone-mode: false
- name: Reclaim runner disk
run: bash .github/scripts/ci-free-disk.sh
- name: Relocate Docker data-root to /mnt
run: bash .github/scripts/ci-relocate-docker-data-root.sh
- name: Relocate Go caches to /mnt
run: bash .github/scripts/ci-relocate-go-cache.sh
- uses: actions/setup-go@v7
with:
go-version: '1.27.4'
- name: Install jq
run: command -v jq >/dev/null || sudo apt-get install -y jq
- name: Login to Docker Hub
uses: docker/login-action@v3
if: env.DOCKERHUB_USERNAME != ''
env:
DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
- name: Login to GitHub Container Registry
uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ github.repository_owner }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Download integration CI artifacts
uses: actions/download-artifact@v4
with:
name: integration-ci-artifacts
- name: Load prebuilt seid and pull Docker images
run: |
tar -xzf integration-build.tar.gz
# The binaries are now extracted under build/; drop the
# tarball so its copy does not sit on disk for the rest of the job.
rm -f integration-build.tar.gz
source .github/scripts/docker-registry-retry.sh
pull_with_retry "${GHCR_LOCALNODE}:${{ github.run_id }}" & localnode_pull=$!
pull_with_retry "${GHCR_RPCNODE}:${{ github.run_id }}" & rpcnode_pull=$!
wait "${localnode_pull}"
wait "${rpcnode_pull}"
docker tag "${GHCR_LOCALNODE}:${{ github.run_id }}" sei-chain/localnode
docker tag "${GHCR_RPCNODE}:${{ github.run_id }}" sei-chain/rpcnode
# Drop the redundant run-id-tagged references now that the images are
# retagged as sei-chain/*; keeps a single tag per image.
docker image rm "${GHCR_LOCALNODE}:${{ github.run_id }}" "${GHCR_RPCNODE}:${{ github.run_id }}" || true
- name: Run autobahn integration tests
env:
AUTOBAHN_PREBUILT_IMAGES: "true"
run: make autobahn-integration-test
- name: Print node logs on failure
if: ${{ failure() }}
run: |
set -euo pipefail
for c in sei-node-0 sei-node-1 sei-node-2 sei-node-3 sei-rpc-node; do
echo "==================== ${c} (docker logs tail) ===================="
docker logs --tail 200 "${c}" || true
done
- name: Collect logs directory
if: ${{ always() }}
run: |
mkdir -p artifacts/sei-autobahn-integration
if [ -d build/generated/logs ]; then
cp -r build/generated/logs artifacts/sei-autobahn-integration/
fi
- name: Upload logs directory
if: ${{ always() }}
uses: actions/upload-artifact@v4
with:
name: integration-logs-autobahn-integration
path: artifacts/sei-autobahn-integration
if-no-files-found: warn
integration-test-check:
name: Integration Test Check
runs-on: ubuntu-latest
needs: [ prepare-cluster, prepare-rpcnode, integration-tests, autobahn-integration-tests ]
if: always()
steps:
- name: Verify prepare and test jobs succeeded
run: |
if [[ "${{ needs.prepare-cluster.result }}" != "success" ]]; then
echo "prepare-cluster job did not succeed (${{ needs.prepare-cluster.result }})"
exit 1
fi
if [[ "${{ needs.prepare-rpcnode.result }}" != "success" ]]; then
echo "prepare-rpcnode job did not succeed (${{ needs.prepare-rpcnode.result }})"
exit 1
fi
if [[ "${{ needs.integration-tests.result }}" != "success" ]]; then
echo "integration-tests job did not succeed (${{ needs.integration-tests.result }})"
exit 1
fi
if [[ "${{ needs.autobahn-integration-tests.result }}" != "success" ]]; then
echo "autobahn-integration-tests job did not succeed (${{ needs.autobahn-integration-tests.result }})"
exit 1
fi
echo "All integration test jobs passed."