Skip to content

Commit a1b0ae9

Browse files
authored
Give each release upgrade test binary its own shared library tree (#4742)
The minor release upgrade test builds each `seid` with its worktree mounted at `/sei-protocol/sei-chain`, so every cgo rpath points at the same path, and the validators run both binaries from a mount of the main checkout. #4207 staged release libraries only where main lacked a file of the same name, so a library both trees ship under one name, such as `sei-wasmvm/internal/api/libwasmvm.x86_64.so`, always resolved to main's copy and the release nodes ran main's libwasmvm. The same applied to the main binary when `target_ref` differs from the checked-out commit. Today the release/v6.6, v6.7 and v6.8 copies are byte-identical to main's, so no run has been affected yet, but the first libwasmvm change on main would be masked silently. `build_binary` now mounts each worktree at `build/upgrade-test/<label>` under the repo mount, and `stage_shared_libraries` copies every `.so` of that worktree to the same path, so the release and main binaries each resolve only their own tree. `verify_shared_libraries` runs `ldd` in every validator against the running release binary and the staged main binary, and fails the run if a library is missing or resolves from outside that binary's tree. Cleanup removes the whole `build/upgrade-test` directory. This only touches the CI script. A local run of `release-upgrade-test.sh` for release/v6.7 to main passed, with the release `RUNPATH` under `build/upgrade-test/release` and main's under `build/upgrade-test/main`.
1 parent 84e6980 commit a1b0ae9

1 file changed

Lines changed: 46 additions & 23 deletions

File tree

‎.github/scripts/release-upgrade-test.sh‎

Lines changed: 46 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,10 @@ readonly BUILD_ROOT="$RUN_ROOT/bin"
1010
readonly ARTIFACT_ROOT="$RUN_ROOT/artifacts"
1111
readonly NODE_COUNT=4
1212
readonly CROSS_VERSION_ARTIFACT="$ARTIFACT_ROOT/cross-version.json"
13+
readonly CONTAINER_REPO_ROOT=/sei-protocol/sei-chain
14+
# Each binary is built and its shared libraries staged under its own directory
15+
# of the repo mount, so its cgo rpath never resolves to the other tree's files.
16+
readonly BINARY_TREES=build/upgrade-test
1317

1418
RELEASE_BRANCH="${RELEASE_BRANCH:-}"
1519
MAIN_REF="${MAIN_REF:-${GITHUB_SHA:-HEAD}}"
@@ -21,7 +25,6 @@ BOUNDARY_FROM=
2125
UPGRADE_NAME=
2226
UPGRADE_TAG=
2327
CROSS_VERSION_TESTS=
24-
STAGED_RELEASE_LIBS=()
2528

2629
log() {
2730
printf '\n[%s] %s\n' "$(date -u +'%Y-%m-%dT%H:%M:%SZ')" "$*"
@@ -208,14 +211,16 @@ build_binary() {
208211
go_build_cache="$(go env GOCACHE)"
209212
mkdir -p "$go_mod_cache" "$go_build_cache"
210213

214+
local container_source_dir="$CONTAINER_REPO_ROOT/$BINARY_TREES/$label"
215+
211216
log "Building $label seid"
212217
docker run --rm \
213218
--user="$(id -u):$(id -g)" \
214219
--platform linux/amd64 \
215-
-v "$source_dir:/sei-protocol/sei-chain:Z" \
220+
-v "$source_dir:$container_source_dir:Z" \
216221
-v "$go_mod_cache:/root/go/pkg/mod:Z" \
217222
-v "$go_build_cache:/root/.cache/go-build:Z" \
218-
-w /sei-protocol/sei-chain \
223+
-w "$container_source_dir" \
219224
-e LEDGER_ENABLED=false \
220225
-e GOFLAGS=-buildvcs=false \
221226
-e "GOTOOLCHAIN=$source_go_toolchain" \
@@ -228,20 +233,41 @@ build_binary() {
228233
sha256sum "$output_path" | tee "$ARTIFACT_ROOT/$label.sha256"
229234
}
230235

231-
# stage_release_shared_libraries copies release libraries missing from the main worktree.
232-
stage_release_shared_libraries() {
233-
log "Staging release shared libraries missing from $REPO_ROOT"
236+
# stage_shared_libraries copies every shared library of a worktree to the
237+
# path its binary was built under.
238+
stage_shared_libraries() {
239+
local source_dir="$1"
240+
local label="$2"
234241
local lib rel
242+
log "Staging $label shared libraries under $REPO_ROOT/$BINARY_TREES/$label"
235243
while IFS= read -r -d '' lib; do
236-
rel="${lib#"$RELEASE_WORKTREE"/}"
237-
if [[ -e "$REPO_ROOT/$rel" ]]; then
238-
printf 'skipped_release_lib=%s\n' "$rel" | tee -a "$ARTIFACT_ROOT/revisions.txt"
239-
continue
240-
fi
241-
install -D -m 0644 "$lib" "$REPO_ROOT/$rel"
242-
STAGED_RELEASE_LIBS+=("$REPO_ROOT/$rel")
243-
printf 'staged_release_lib=%s\n' "$rel" | tee -a "$ARTIFACT_ROOT/revisions.txt"
244-
done < <(find "$RELEASE_WORKTREE" -name '.git' -prune -o -type f -name '*.so' -print0)
244+
rel="${lib#"$source_dir"/}"
245+
install -D -m 0644 "$lib" "$REPO_ROOT/$BINARY_TREES/$label/$rel"
246+
printf 'staged_%s_lib=%s\n' "$label" "$rel" | tee -a "$ARTIFACT_ROOT/revisions.txt"
247+
done < <(find "$source_dir" -name '.git' -prune -o -type f -name '*.so' -print0)
248+
}
249+
250+
# verify_shared_libraries fails unless every library a binary in each validator
251+
# loads from the repo mount comes from that binary's own tree.
252+
verify_shared_libraries() {
253+
local binary="$1"
254+
local label="$2"
255+
local own_tree="$CONTAINER_REPO_ROOT/$BINARY_TREES/$label/"
256+
local node
257+
local resolved
258+
local path
259+
260+
for ((i = 0; i < NODE_COUNT; i++)); do
261+
node="sei-node-$i"
262+
resolved="$(docker exec "$node" ldd "$binary")" ||
263+
die "unable to resolve the $label shared libraries in $node"
264+
! grep -q 'not found' <<<"$resolved" ||
265+
die "$node cannot resolve every $label shared library: $resolved"
266+
while IFS= read -r path; do
267+
[[ "$path" == "$own_tree"* ]] ||
268+
die "$node $label binary loads $path from outside $own_tree"
269+
done < <(awk -v root="$CONTAINER_REPO_ROOT/" '$2 == "=>" && index($3, root) == 1 { print $3 }' <<<"$resolved")
270+
done
245271
}
246272

247273
height() {
@@ -330,6 +356,7 @@ start_release_cluster() {
330356
local release_hash
331357
release_hash="$(sha256sum "$BUILD_ROOT/release-seid" | awk '{print $1}')"
332358
verify_running_binary_hash "$release_hash" release
359+
verify_shared_libraries /root/go/bin/seid release
333360
assert_all_nodes_progress 5 "pre-upgrade"
334361
}
335362

@@ -392,6 +419,7 @@ stage_main_binary() {
392419
[[ "$actual_hash" == "$MAIN_BINARY_HASH" ]] ||
393420
die "$node staged binary checksum mismatch"
394421
done
422+
verify_shared_libraries /tmp/seid.next main
395423
}
396424

397425
node_has_upgrade_halt() {
@@ -606,13 +634,7 @@ cleanup() {
606634
DOCKER_PLATFORM=linux/amd64 make docker-cluster-stop
607635
)
608636
fi
609-
if ((${#STAGED_RELEASE_LIBS[@]})); then
610-
rm -f "${STAGED_RELEASE_LIBS[@]}"
611-
local lib
612-
for lib in "${STAGED_RELEASE_LIBS[@]}"; do
613-
rmdir -p --ignore-fail-on-non-empty "$(dirname "$lib")" 2>/dev/null || true
614-
done
615-
fi
637+
rm -rf "${REPO_ROOT:?}/$BINARY_TREES"
616638
git -C "$REPO_ROOT" worktree remove --force "$MAIN_WORKTREE" 2>/dev/null
617639
git -C "$REPO_ROOT" worktree remove --force "$RELEASE_WORKTREE" 2>/dev/null
618640
exit "$exit_code"
@@ -630,8 +652,9 @@ main() {
630652
discover_cross_version_tests
631653
build_localnode_image
632654
build_binary "$RELEASE_WORKTREE" "$BUILD_ROOT/release-seid" release
633-
stage_release_shared_libraries
634655
build_binary "$MAIN_WORKTREE" "$BUILD_ROOT/main-seid" main
656+
stage_shared_libraries "$RELEASE_WORKTREE" release
657+
stage_shared_libraries "$MAIN_WORKTREE" main
635658
start_release_cluster
636659
run_cross_version_phase before
637660
stage_main_binary

0 commit comments

Comments
 (0)