From e2b1776ab6adf53badd4b530cb2f75c0c8671f97 Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Wed, 9 Sep 2026 23:39:56 -0700 Subject: [PATCH 1/2] fix: derive the RDMA cgo link line instead of maintaining it by hand scripts/rdma-cgo-libs.txt listed libminio's transitive archives by hand, so it went stale silently. minio-cpp 1.0.0 swapped curlpp for cpp-httplib (adding brotli) and stopped taking vcpkg's zlib; nothing failed until a clean runner could not find -lcurlpp, and a release host still holding the 0.6.0 archives reported green while CI was red. minio-cpp now declares its private dependencies in miniocpp.pc, so scripts/rdma-link-libs.sh derives the list with pkg-config and the file is gone. Only the -l names and -pthread are taken: pkg-config's -L points into the vcpkg tree the prefix was built from, which on a cross build is the wrong architecture, and every caller already passes -L/lib. Taking names only is what makes one derivation serve both architectures. goreleaser cannot run a command and its checkout is wiped per run, so provisioning writes the derived line into each prefix as lib/warp-rdma-cgo-libs.txt and qreleaser.yaml reads it from there. A prefix that was never provisioned now fails loudly at template time, and verify_prefix asserts the file is present. The list still feeds static_libs(), so a dependency minio-cpp newly pulls in appears there and verify_prefix demands an archive for it -- the check that was missing when the 1.0.0 shift went unnoticed. minio-cpp is pinned to a commit because the pkg-config fix landed after v1.0.0; move it to v1.0.1 once that is cut. --- .github/workflows/go-rdma.yml | 13 ++-- .goreleaser/qreleaser.yaml | 12 ++- RDMA.md | 8 +- scripts/build-rdma.sh | 9 +-- scripts/rdma-cgo-libs.txt | 1 - scripts/rdma-cross/triplets/arm64-linux.cmake | 2 +- scripts/rdma-link-libs.sh | 77 +++++++++++++++++++ scripts/setup-rdma-release-host.sh | 52 ++++++++++--- 8 files changed, 146 insertions(+), 28 deletions(-) delete mode 100644 scripts/rdma-cgo-libs.txt create mode 100755 scripts/rdma-link-libs.sh diff --git a/.github/workflows/go-rdma.yml b/.github/workflows/go-rdma.yml index 35a370f3..54d89487 100644 --- a/.github/workflows/go-rdma.yml +++ b/.github/workflows/go-rdma.yml @@ -36,12 +36,14 @@ jobs: # Pinned in lockstep with scripts/build-rdma.sh and # scripts/setup-rdma-release-host.sh: this job copies vendor/s3rdma out of # the checkout, so a floating main makes the build race whatever landed - # there. v1.0.0 installs the library as libminio with a stable soname. + # there. Pinned to a commit rather than a tag: the pkg-config fix that + # lets the link line be derived landed after v1.0.0 and is not yet + # tagged. Move this to v1.0.1 once it is cut. - name: Checkout minio-cpp uses: actions/checkout@v5 with: repository: minio/minio-cpp - ref: v1.0.0 + ref: 92d8b2c3ec6ac2c82012590de572c55f3591f338 path: "minio-cpp" persist-credentials: false @@ -102,9 +104,10 @@ jobs: CGO_ENABLED: "1" run: | # The library is static, so minio-go's own -lminio no longer drags in - # the C++ runtime or the vcpkg archives; name them from the one list the - # build script and goreleaser config also use. - export CGO_LDFLAGS="-L/usr/local/lib $(cat scripts/rdma-cgo-libs.txt)" + # the C++ runtime or the vcpkg archives; derive them from the + # miniocpp.pc that was just installed, the same way the build script + # and the release host do. + export CGO_LDFLAGS="-L/usr/local/lib $(scripts/rdma-link-libs.sh /usr/local ../minio-cpp)" go build -tags=rdma -o /tmp/warp . go vet -tags=rdma ./... go test -race -tags=rdma ./... diff --git a/.goreleaser/qreleaser.yaml b/.goreleaser/qreleaser.yaml index f8cfef75..81266c93 100644 --- a/.goreleaser/qreleaser.yaml +++ b/.goreleaser/qreleaser.yaml @@ -80,10 +80,14 @@ builds: # archive have to be named explicitly. Only libs3rdma stays dynamic -- # minio-cpp vendors it as a shared object with no static form -- and the # /usr/lib/warp rpath is where the RDMA package installs it. - # The library list lives in scripts/rdma-cgo-libs.txt so this config, the - # build script and the CI workflows cannot drift apart. + # The library list is derived from the installed miniocpp.pc by + # scripts/rdma-link-libs.sh and left in the prefix at provisioning time. + # goreleaser cannot run a command, and the checkout it builds from is + # wiped per run, so reading it out of the prefix is what keeps this + # config, the build script and the CI workflows from drifting apart. A + # prefix that was never provisioned fails loudly here. - CGO_CFLAGS={{ envOrDefault "CGO_CFLAGS" "" }} -I{{ envOrDefault "MINIOCPP_PREFIX" "/usr/local" }}/include - - CGO_LDFLAGS={{ envOrDefault "CGO_LDFLAGS" "" }} -L{{ envOrDefault "MINIOCPP_PREFIX" "/usr/local" }}/lib -Wl,-rpath-link,{{ envOrDefault "MINIOCPP_PREFIX" "/usr/local" }}/lib -Wl,-rpath,/usr/lib/warp -Wl,--enable-new-dtags {{ mustReadFile "scripts/rdma-cgo-libs.txt" }} + - CGO_LDFLAGS={{ envOrDefault "CGO_LDFLAGS" "" }} -L{{ envOrDefault "MINIOCPP_PREFIX" "/usr/local" }}/lib -Wl,-rpath-link,{{ envOrDefault "MINIOCPP_PREFIX" "/usr/local" }}/lib -Wl,-rpath,/usr/lib/warp -Wl,--enable-new-dtags {{ mustReadFile (printf "%s/lib/warp-rdma-cgo-libs.txt" (envOrDefault "MINIOCPP_PREFIX" "/usr/local")) }} goos: - linux goarch: @@ -101,7 +105,7 @@ builds: - CC=aarch64-linux-gnu-gcc - CXX=aarch64-linux-gnu-g++ - CGO_CFLAGS={{ envOrDefault "CGO_CFLAGS" "" }} -I{{ envOrDefault "MINIOCPP_PREFIX" "/usr/local" }}/aarch64-linux-gnu/include - - CGO_LDFLAGS={{ envOrDefault "CGO_LDFLAGS" "" }} -L{{ envOrDefault "MINIOCPP_PREFIX" "/usr/local" }}/aarch64-linux-gnu/lib -Wl,-rpath-link,{{ envOrDefault "MINIOCPP_PREFIX" "/usr/local" }}/aarch64-linux-gnu/lib -Wl,-rpath,/usr/lib/warp -Wl,--enable-new-dtags {{ mustReadFile "scripts/rdma-cgo-libs.txt" }} + - CGO_LDFLAGS={{ envOrDefault "CGO_LDFLAGS" "" }} -L{{ envOrDefault "MINIOCPP_PREFIX" "/usr/local" }}/aarch64-linux-gnu/lib -Wl,-rpath-link,{{ envOrDefault "MINIOCPP_PREFIX" "/usr/local" }}/aarch64-linux-gnu/lib -Wl,-rpath,/usr/lib/warp -Wl,--enable-new-dtags {{ mustReadFile (printf "%s/aarch64-linux-gnu/lib/warp-rdma-cgo-libs.txt" (envOrDefault "MINIOCPP_PREFIX" "/usr/local")) }} flags: - -trimpath - --tags=kqueue,rdma diff --git a/RDMA.md b/RDMA.md index 106818a0..bc2eb808 100644 --- a/RDMA.md +++ b/RDMA.md @@ -211,13 +211,17 @@ the same library list the release uses: ```bash λ export CGO_ENABLED=1 λ export CGO_CFLAGS="-I/usr/local/include" -λ export CGO_LDFLAGS="-L/usr/local/lib $(cat scripts/rdma-cgo-libs.txt)" +λ export CGO_LDFLAGS="-L/usr/local/lib $(scripts/rdma-link-libs.sh /usr/local /path/to/minio-cpp)" λ go build -tags=kqueue,rdma ``` libminiocpp is linked statically, so cgo, which links with `gcc` rather than `g++`, has to be told about the C++ runtime and every transitive archive by -name. That list lives in `scripts/rdma-cgo-libs.txt`. +name. `scripts/rdma-link-libs.sh` derives that list from the installed +`miniocpp.pc`, so it tracks whatever minio-cpp actually links against. It needs +the minio-cpp source tree only to find the `.pc` files of the private +dependencies under `vcpkg_installed/`; if those are already on +`PKG_CONFIG_PATH`, the prefix alone is enough. The binary this produces still loads libs3rdma at run time, and nothing tells it where to find it. Choose one: diff --git a/scripts/build-rdma.sh b/scripts/build-rdma.sh index c37a201f..07b54fd7 100755 --- a/scripts/build-rdma.sh +++ b/scripts/build-rdma.sh @@ -125,7 +125,7 @@ else "${MINIO_CPP_REPO:-https://github.com/minio/minio-cpp}" fi git -C "${MINIO_CPP_DIR}" fetch --depth 1 origin \ - "${MINIO_CPP_REF:-v1.0.0}" + "${MINIO_CPP_REF:-92d8b2c3ec6ac2c82012590de572c55f3591f338}" git -C "${MINIO_CPP_DIR}" checkout --detach -f FETCH_HEAD echo ">>> building libminiocpp with RDMA" @@ -151,10 +151,9 @@ echo ">>> building libminiocpp with RDMA" TAGS="kqueue,rdma" NAME="warp-rdma" -# Static libminiocpp plus its transitive vcpkg archives, then libs3rdma. Kept in -# one file because .goreleaser/qreleaser.yaml and the CI workflows have to link -# exactly the same way. -RDMA_LINK_LIBS="$(cat "${REPO_DIR}/scripts/rdma-cgo-libs.txt")" +# Static libminio plus its transitive vcpkg archives, then libs3rdma, derived +# from the miniocpp.pc just installed so this cannot drift from what was built. +RDMA_LINK_LIBS="$("${REPO_DIR}/scripts/rdma-link-libs.sh" "${PREFIX}" "${MINIO_CPP_DIR}")" STAGE="${WORK}/stage/${NAME}" rm -rf "${STAGE}" diff --git a/scripts/rdma-cgo-libs.txt b/scripts/rdma-cgo-libs.txt deleted file mode 100644 index 16a10676..00000000 --- a/scripts/rdma-cgo-libs.txt +++ /dev/null @@ -1 +0,0 @@ --lminio -ls3rdma -lssl -lcrypto -lINIReader -linih -lpugixml -lbrotlienc -lbrotlidec -lbrotlicommon -lz -lstdc++ -lm -ldl -lpthread \ No newline at end of file diff --git a/scripts/rdma-cross/triplets/arm64-linux.cmake b/scripts/rdma-cross/triplets/arm64-linux.cmake index e3aa6847..48bbc3cc 100644 --- a/scripts/rdma-cross/triplets/arm64-linux.cmake +++ b/scripts/rdma-cross/triplets/arm64-linux.cmake @@ -9,7 +9,7 @@ set(VCPKG_TARGET_ARCHITECTURE arm64) set(VCPKG_CRT_LINKAGE dynamic) # libminiocpp is linked statically into warp, so its dependencies must be static -# too; see scripts/rdma-cgo-libs.txt for the resulting link line. +# too; scripts/rdma-link-libs.sh derives the resulting link line. set(VCPKG_LIBRARY_LINKAGE static) set(VCPKG_CMAKE_SYSTEM_NAME Linux) diff --git a/scripts/rdma-link-libs.sh b/scripts/rdma-link-libs.sh new file mode 100755 index 00000000..7e6f2769 --- /dev/null +++ b/scripts/rdma-link-libs.sh @@ -0,0 +1,77 @@ +#!/usr/bin/env bash +# +# Print the cgo link line for the S3-over-RDMA build, derived from minio-cpp's +# own pkg-config metadata. +# +# libminio is linked statically, so cgo -- which links with gcc, not g++ -- has +# to name the C++ runtime and every transitive archive explicitly. That list was +# maintained by hand in scripts/rdma-cgo-libs.txt until minio-cpp learned to +# declare it: 1.0.0 swapped curlpp for cpp-httplib (adding brotli) and stopped +# taking vcpkg's zlib, and the hand-written list went stale without a word. +# +# Only the -l names and -pthread are taken. The -L that pkg-config would emit +# points into the vcpkg tree the prefix was built from, which on a cross build +# is the wrong architecture; every caller already passes -L/lib, where +# the archives are collected. Taking names only is what makes one derivation +# correct for both architectures. +# +# Usage: +# scripts/rdma-link-libs.sh PREFIX [MINIO_CPP_SRC] +# +# PREFIX the libminio install prefix (holds lib/pkgconfig/miniocpp.pc) +# MINIO_CPP_SRC the minio-cpp source tree, whose vcpkg_installed/ holds the +# .pc files for the private dependencies. Omit it only when +# those are already on PKG_CONFIG_PATH. + +set -euo pipefail + +if [ $# -lt 1 ]; then + echo "usage: $0 PREFIX [MINIO_CPP_SRC]" >&2 + exit 1 +fi + +prefix="$1" +src="${2:-}" + +command -v pkg-config >/dev/null 2>&1 || { + echo "rdma-link-libs: pkg-config not found" >&2 + exit 1 +} + +pc_path="${prefix}/lib/pkgconfig" +if [ -n "${src}" ]; then + for dir in "${src}"/vcpkg_installed/*/lib/pkgconfig; do + [ -d "${dir}" ] && pc_path="${pc_path}:${dir}" + done +fi +export PKG_CONFIG_PATH="${pc_path}${PKG_CONFIG_PATH:+:${PKG_CONFIG_PATH}}" + +if ! pkg-config --exists miniocpp; then + echo "rdma-link-libs: no miniocpp.pc on ${PKG_CONFIG_PATH}" >&2 + exit 1 +fi + +libs="$(pkg-config --static --libs-only-l miniocpp)" +other="$(pkg-config --static --libs-only-other miniocpp)" + +# A .pc that resolves but names no library means the prefix is not the one this +# build needs. Fail here rather than emitting a link line that drops libminio. +case " ${libs} " in +*" -lminio "*) ;; +*) + echo "rdma-link-libs: miniocpp.pc does not name -lminio; wrong or stale prefix?" >&2 + exit 1 + ;; +esac + +# -pthread arrives through Libs.private and is not a -l flag, so --libs-only-l +# drops it. Take it from the other flags rather than the whole set, which can +# also carry -Wl options that have no place in a name-only list. +threads="" +case " ${other} " in +*" -pthread "*) threads=" -pthread" ;; +esac + +# pkg-config cannot know about the C++ runtime: it describes a C interface, and +# nothing in the .pc records that the archive behind it is C++. +printf '%s%s -lstdc++\n' "${libs}" "${threads}" diff --git a/scripts/setup-rdma-release-host.sh b/scripts/setup-rdma-release-host.sh index bc14b918..a45d762d 100755 --- a/scripts/setup-rdma-release-host.sh +++ b/scripts/setup-rdma-release-host.sh @@ -15,8 +15,10 @@ # arm64 ${PREFIX}/aarch64-linux-gnu (cross-built) # # Each holds libminio.a built with RDMA enabled plus its headers, the vcpkg -# static archives it links against (scripts/rdma-cgo-libs.txt), and the vendored -# libs3rdma shared object the release packaging copies out. The arm64 prefix +# static archives it links against, the derived cgo link line +# (lib/warp-rdma-cgo-libs.txt, which qreleaser.yaml reads because goreleaser +# cannot run a command), and the vendored libs3rdma shared object the release +# packaging copies out. The arm64 prefix # path is fixed rather than host-dependent because qreleaser.yaml has to name it # in a static override. # @@ -104,10 +106,17 @@ esac # Pinned in lockstep with scripts/build-rdma.sh and .github/workflows/go-rdma.yml: # a floating minio-cpp is what leaves a host with headers too old for the # minio-go revision in go.mod, and vcpkg's port scripts track the newest CMake. -# v1.0.0 installs the library as libminio with a stable soname. -MINIO_CPP_REF="${MINIO_CPP_REF:-v1.0.0}" +# minio-cpp is pinned to a commit rather than a tag: the pkg-config fix +# that lets the link line be derived landed after v1.0.0 and is not yet +# tagged. Move this to v1.0.1 once it is cut. +MINIO_CPP_REF="${MINIO_CPP_REF:-92d8b2c3ec6ac2c82012590de572c55f3591f338}" MINIO_CPP_REPO="${MINIO_CPP_REPO:-https://github.com/minio/minio-cpp}" VCPKG_REF="${VCPKG_REF:-2026.07.29}" + +# Written into each prefix at provisioning time and read back by qreleaser.yaml, +# smoke_build and verify_prefix. The qreleaser checkout is wiped per run, so a +# file generated into the repo would not survive to the build. +LINK_LIBS_NAME="warp-rdma-cgo-libs.txt" CMAKE_MIN="3.31" CMAKE_VERSION="${CMAKE_VERSION:-3.31.6}" @@ -164,14 +173,24 @@ elf_machine() { readelf -h "$1" 2>/dev/null | sed -n 's/^ *Machine: *//p' | head -1 } -# scripts/rdma-cgo-libs.txt names three kinds of library: the toolchain's own, +# The derived link line names three kinds of library: the toolchain's own, # libs3rdma, which minio-cpp vendors as a shared object with no static form, and # everything else, which must come from an archive. A published binary that # picks one of the last group up dynamically has a runtime dependency the # package neither declares nor bundles, and fails to start on a customer host. +# +# Reading the list from the prefix rather than from a file in this repo is what +# makes a dependency minio-cpp newly pulls in fail loudly: it appears here, and +# verify_prefix then demands an archive for it. static_libs() { + local prefix="$1" local lib - for lib in $(tr ' ' '\n' <"${REPO_DIR}/scripts/rdma-cgo-libs.txt" | sed -n 's/^-l//p'); do + # Absent on a first provisioning: build_target sweeps shadowing objects + # before installing, and a prefix with no list has nothing to shadow. On a + # re-provision this is the previous install's list, which is the right one + # to sweep against. + [ -f "${prefix}/lib/${LINK_LIBS_NAME}" ] || return 0 + for lib in $(tr ' ' '\n' <"${prefix}/lib/${LINK_LIBS_NAME}" | sed -n 's/^-l//p'); do case "${lib}" in stdc++ | m | dl | pthread) ;; # zlib is a system library here: minio-cpp 1.0.0 takes vcpkg's only on @@ -229,7 +248,14 @@ verify_prefix() { missing=1 fi - for lib in $(static_libs); do + # qreleaser.yaml reads this out of the prefix; without it a release links + # with an empty library list instead of failing here. + if [ ! -f "${prefix}/lib/${LINK_LIBS_NAME}" ]; then + echo "MISSING: ${prefix}/lib/${LINK_LIBS_NAME} (${arch}, needed by qreleaser.yaml)" >&2 + missing=1 + fi + + for lib in $(static_libs "${prefix}"); do name="lib${lib}" if ! compgen -G "${prefix}/lib/${name}.a" >/dev/null; then echo "MISSING: ${prefix}/lib/${name}.a (${arch})" >&2 @@ -453,7 +479,7 @@ build_target() { # Shared objects that shadow the archives we install. ld picks these over the # .a in the same -L, which is how a release ends up depending on a library it # neither bundles nor declares. - for lib in $(static_libs); do + for lib in $(static_libs "${prefix}"); do for f in "${prefix}"/lib/"lib${lib}".so*; do if [ -e "${f}" ]; then stale+=("${f}") @@ -481,6 +507,12 @@ build_target() { as_root cp -P vcpkg_installed/"${triplet}"/lib/*.a "${prefix}/lib/" command -v ldconfig >/dev/null 2>&1 && as_root ldconfig || true + # Derive the link line from the miniocpp.pc just installed and leave it in + # the prefix: goreleaser cannot run a command, and the checkout it builds + # from is wiped per run, so the prefix is the only place this can live. + "${REPO_DIR}/scripts/rdma-link-libs.sh" "${prefix}" "${src}" | + as_root tee "${prefix}/lib/${LINK_LIBS_NAME}" >/dev/null + verify_prefix "${arch}" } @@ -499,7 +531,7 @@ smoke_build() { CC="$(arch_cc "${arch}")" \ CXX="$(arch_cxx "${arch}")" \ CGO_CFLAGS="-I${prefix}/include" \ - CGO_LDFLAGS="-L${prefix}/lib -Wl,-rpath-link,${prefix}/lib -Wl,-rpath,/usr/lib/warp -Wl,--enable-new-dtags $(cat "${REPO_DIR}/scripts/rdma-cgo-libs.txt")" \ + CGO_LDFLAGS="-L${prefix}/lib -Wl,-rpath-link,${prefix}/lib -Wl,-rpath,/usr/lib/warp -Wl,--enable-new-dtags $(cat "${prefix}/lib/${LINK_LIBS_NAME}")" \ go build -trimpath -tags=kqueue,rdma -o "${out}" . local machine @@ -518,7 +550,7 @@ smoke_build() { # declare, so the binary would fail to start on a customer host. local needed lib needed="$(readelf -d "${out}" | sed -n 's/.*Shared library: \[\(.*\)\]/\1/p')" - for lib in $(static_libs); do + for lib in $(static_libs "${prefix}"); do if printf '%s\n' "${needed}" | grep -q "^lib${lib}\.so"; then echo "smoke build links lib${lib} dynamically; something in $(arch_prefix "${arch}")/lib shadows lib${lib}.a" >&2 return 1 From 0c0ee99ef5c6612a0248236a4bb7c01cc9502e2d Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Thu, 10 Sep 2026 07:51:34 -0700 Subject: [PATCH 2/2] fix: do not truncate the derived link list when derivation fails Piping rdma-link-libs.sh straight into tee let tee create the file before the derivation's exit status was known, so a failure left an empty list where a good one had been. Both guards tested -f, which an empty file satisfies, so static_libs then yielded nothing and every derived archive check in verify_prefix became vacuous while still reporting the prefix as satisfying the link line. Derive into a variable first, so a failure never opens the file, and test -s rather than -f so an empty list is treated as no list at all. --- scripts/setup-rdma-release-host.sh | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/scripts/setup-rdma-release-host.sh b/scripts/setup-rdma-release-host.sh index a45d762d..07f87c6e 100755 --- a/scripts/setup-rdma-release-host.sh +++ b/scripts/setup-rdma-release-host.sh @@ -189,7 +189,7 @@ static_libs() { # before installing, and a prefix with no list has nothing to shadow. On a # re-provision this is the previous install's list, which is the right one # to sweep against. - [ -f "${prefix}/lib/${LINK_LIBS_NAME}" ] || return 0 + [ -s "${prefix}/lib/${LINK_LIBS_NAME}" ] || return 0 for lib in $(tr ' ' '\n' <"${prefix}/lib/${LINK_LIBS_NAME}" | sed -n 's/^-l//p'); do case "${lib}" in stdc++ | m | dl | pthread) ;; @@ -250,7 +250,7 @@ verify_prefix() { # qreleaser.yaml reads this out of the prefix; without it a release links # with an empty library list instead of failing here. - if [ ! -f "${prefix}/lib/${LINK_LIBS_NAME}" ]; then + if [ ! -s "${prefix}/lib/${LINK_LIBS_NAME}" ]; then echo "MISSING: ${prefix}/lib/${LINK_LIBS_NAME} (${arch}, needed by qreleaser.yaml)" >&2 missing=1 fi @@ -510,7 +510,15 @@ build_target() { # Derive the link line from the miniocpp.pc just installed and leave it in # the prefix: goreleaser cannot run a command, and the checkout it builds # from is wiped per run, so the prefix is the only place this can live. - "${REPO_DIR}/scripts/rdma-link-libs.sh" "${prefix}" "${src}" | + # + # Derived into a variable first. Piping straight into tee would truncate a + # previously good list before the derivation's exit status is known, and an + # empty list makes every check that reads it vacuous. Declared separately + # from the assignment on purpose: `local x="$(...)"` returns local's status, + # which would swallow the failure this guards against. + local link_libs + link_libs="$("${REPO_DIR}/scripts/rdma-link-libs.sh" "${prefix}" "${src}")" + printf '%s\n' "${link_libs}" | as_root tee "${prefix}/lib/${LINK_LIBS_NAME}" >/dev/null verify_prefix "${arch}"