From bf2f640923c4342ecdac0ef0a20a447d1c4780a1 Mon Sep 17 00:00:00 2001 From: Giulio Eulisse <10544+ktf@users.noreply.github.com> Date: Sun, 28 Jun 2026 21:10:33 +0200 Subject: [PATCH 1/2] Create reproducible tarballs First step to REAPI support and reproducible builds in general. --- alibuild_helpers/build.py | 5 ++++ alibuild_helpers/build_template.sh | 45 ++++++++++++++++++++++++++++-- tests/test_build.py | 20 +++++++++++++ 3 files changed, 67 insertions(+), 3 deletions(-) diff --git a/alibuild_helpers/build.py b/alibuild_helpers/build.py index 66ca6dc4..614e492a 100644 --- a/alibuild_helpers/build.py +++ b/alibuild_helpers/build.py @@ -1072,6 +1072,11 @@ def performPreferCheckWithTempDir(pkg, cmd): ("GIT_COMMITTER_EMAIL", "unknown"), ("INCREMENTAL_BUILD_HASH", spec.get("incremental_hash", "0")), ("JOBS", str(args.jobs)), + # Produce reproducible, content-stable tarballs for packages that may be + # uploaded to the remote store. Devel packages are never uploaded, so we + # leave their install trees untouched to avoid perturbing mtimes that + # incremental rebuilds might care about. + ("NORMALIZE_TARBALL", "" if spec["is_devel_pkg"] else "1"), ("PKGHASH", spec["hash"]), ("PKGNAME", spec["package"]), ("PKGREVISION", spec["revision"]), diff --git a/alibuild_helpers/build_template.sh b/alibuild_helpers/build_template.sh index 773fc10d..7430178a 100644 --- a/alibuild_helpers/build_template.sh +++ b/alibuild_helpers/build_template.sh @@ -41,6 +41,7 @@ export PATH=$WORK_DIR/wrapper-scripts:$PATH # - DEVEL_PREFIX # - INCREMENTAL_BUILD_HASH # - JOBS +# - NORMALIZE_TARBALL # - PKGHASH # - PKGNAME # - PKGREVISION @@ -291,6 +292,26 @@ mkdir -p "${WORK_DIR}/TARS/$HASH_PATH" \ "${WORK_DIR}/TARS/$ARCHITECTURE/$PKGNAME" PACKAGE_WITH_REV=$PKGNAME-$PKGVERSION-$PKGREVISION.$ARCHITECTURE.tar.gz + +# Decide whether to produce a normalized (reproducible) tarball. Only packages +# that may end up in the remote store are normalized; devel packages have +# NORMALIZE_TARBALL unset so their install trees (and mtimes) are left untouched, +# which matters for incremental rebuilds. +do_normalize= +if [ "$NORMALIZE_TARBALL" = 1 ] && [ "$CAN_DELETE" != 1 ] && [ -z "$CACHED_TARBALL" ]; then + do_normalize=1 + # Normalize mtimes in place to a fixed, reproducible epoch. This must happen + # before the parallel rsync below, so the installed copy and the tarball agree + # and neither reads the tree while we mutate it. We avoid tar's --mtime, as + # old GNU tar (e.g. slc7's 1.26) doesn't support it. date(1) differs between + # GNU (-d @epoch) and BSD (-r epoch), so try both. + norm_epoch="${SOURCE_DATE_EPOCH:-0}" + if norm_stamp=$(date -d "@$norm_epoch" +%%Y%%m%%d%%H%%M.%%S 2>/dev/null) || + norm_stamp=$(date -r "$norm_epoch" +%%Y%%m%%d%%H%%M.%%S 2>/dev/null); then + find "$WORK_DIR/INSTALLROOT/$PKGHASH" -exec touch -h -t "$norm_stamp" {} + + fi +fi + # Copy and tar/compress (if applicable) in parallel. # Use -H to match tar's behaviour of preserving hardlinks. rsync -DgloprH "$WORK_DIR/INSTALLROOT/$PKGHASH/" "$WORK_DIR" & rsync_pid=$! @@ -302,9 +323,27 @@ elif [ -z "$CACHED_TARBALL" ]; then # Use pigz to compress, if we can, because it's multicore. gzip=$(command -v pigz) || gzip=$(command -v gzip) # We don't have an existing tarball, and we want to keep the one we create now. - tar -cC "$WORK_DIR/INSTALLROOT/$PKGHASH" . | - # Avoid having broken left overs if the tar fails. - $gzip -c > "$WORK_DIR/TARS/$HASH_PATH/$PACKAGE_WITH_REV.processing" + if [ -n "$do_normalize" ]; then + # Reproducible archive: deterministic entry order (via a sorted file list, + # since old GNU tar and bsdtar lack --sort), zeroed ownership, and the + # normalized mtimes set above. Ownership flags are spelled differently by + # GNU tar (--owner/--group) and bsdtar (--uid/--gid). gzip -n drops the + # gzip header's name and timestamp. + if tar --version 2>/dev/null | grep -qi 'GNU tar'; then + tar_owner="--owner=0 --group=0 --numeric-owner" + else + tar_owner="--uid 0 --gid 0 --numeric-owner" + fi + ( cd "$WORK_DIR/INSTALLROOT/$PKGHASH" && + find . -mindepth 1 | LC_ALL=C sort | + tar -cf - --no-recursion $tar_owner -T - ) | + # Avoid having broken left overs if the tar fails. + $gzip -n -c > "$WORK_DIR/TARS/$HASH_PATH/$PACKAGE_WITH_REV.processing" + else + tar -cC "$WORK_DIR/INSTALLROOT/$PKGHASH" . | + # Avoid having broken left overs if the tar fails. + $gzip -c > "$WORK_DIR/TARS/$HASH_PATH/$PACKAGE_WITH_REV.processing" + fi mv "$WORK_DIR/TARS/$HASH_PATH/$PACKAGE_WITH_REV.processing" \ "$WORK_DIR/TARS/$HASH_PATH/$PACKAGE_WITH_REV" ln -nfs "../../$HASH_PATH/$PACKAGE_WITH_REV" \ diff --git a/tests/test_build.py b/tests/test_build.py index f260f34c..bc16bbed 100644 --- a/tests/test_build.py +++ b/tests/test_build.py @@ -437,6 +437,26 @@ def test_initdotsh(self) -> None: self.assertIn("export APPEND_ROOT_1=", complete_initdotsh) self.assertIn("export PREPEND_ROOT_1=", complete_initdotsh) + def test_build_template_percent_format(self) -> None: + """build_template.sh is interpolated via printf-style % formatting in + doBuild(), so every literal '%' in it must be doubled. A stray '%' + raises at build time but is not covered by the unit tests that mock the + build out, so check the real template formats cleanly here.""" + template_path = os.path.join( + os.path.dirname(os.path.dirname(os.path.abspath(__file__))), + "alibuild_helpers", "build_template.sh") + with open(template_path) as templatef: + template = templatef.read() + # These are exactly the keys doBuild() substitutes (see build.py). + keys = ("provenance", "initdotsh_deps", "initdotsh_full", "develPrefix", + "workDir", "configDir", "incremental_recipe", "requires", + "build_requires", "runtime_requires") + try: + template % {key: "" for key in keys} + except (TypeError, ValueError, KeyError) as exc: + self.fail("build_template.sh does not %%-format cleanly (likely an " + "unescaped '%%' that should be '%%%%'): %s" % exc) + if __name__ == '__main__': unittest.main() From 30cd972b1f18c94635fc3ae9f79cbd7a30e02637 Mon Sep 17 00:00:00 2001 From: Giulio Eulisse <10544+ktf@users.noreply.github.com> Date: Sun, 28 Jun 2026 22:26:59 +0200 Subject: [PATCH 2/2] Helpers for ac / cas path resolution --- alibuild_helpers/build.py | 63 +++++++++++++++++++++++++++++++++++ alibuild_helpers/utilities.py | 38 +++++++++++++++++++++ tests/test_build.py | 51 +++++++++++++++++++++++++++- tests/test_utilities.py | 27 +++++++++++++++ 4 files changed, 178 insertions(+), 1 deletion(-) diff --git a/alibuild_helpers/build.py b/alibuild_helpers/build.py index 614e492a..9a51435f 100644 --- a/alibuild_helpers/build.py +++ b/alibuild_helpers/build.py @@ -25,6 +25,7 @@ import tempfile import concurrent.futures +import hashlib import importlib import json import socket @@ -110,6 +111,65 @@ def update_repo(package, git_prompt): # Creates a directory in the store which contains symlinks to the package # and its direct / indirect dependencies +def build_ac_entry(spec, specs, architecture): + """Assemble the Action Cache (AC) entry for a freshly built package. + + The entry records what produced the tarball -- recipe, git commit, dependency + action hashes and build environment -- so that the content-addressed store + (CAS) can later be reconstructed from the (small) action cache, and so that a + package can be installed from its runtime closure without a build. See + REMOTE_STORE_CAS_AC.md. + + The action hash is spec["remote_revision_hash"]; this entry is keyed by it. + Dependencies are referenced by their *action* hash (specs[dep]["hash"]), which + is what the action hash itself folds in (see storeHashes()), so the recorded + DAG matches the hash and is independent of whether builds are byte-reproducible. + + result.outputDigest / result.size are filled in by the remote backend at + upload time, once the tarball bytes exist and have been hashed. + """ + # Resolve the real git commit and the tag aliases pointing at it, mirroring + # storeHashes() so the recorded provenance matches the computed action hash. + scm_refs = spec.get("scm_refs", {}) + real_commit_hash = scm_refs.get("refs/tags/" + spec["commit_hash"], + spec["commit_hash"]) + alt_refs = {ref[len("refs/tags/"):]: git_hash + for ref, git_hash in scm_refs.items() + if ref.startswith("refs/tags/") and git_hash == real_commit_hash} + + def dep_refs(dep_names): + return [{"package": dep, "actionHash": specs[dep]["hash"]} + for dep in sorted(dep_names)] + + recipe_text = spec.get("recipe", "") or "" + recipe_digest = hashlib.sha256(recipe_text.encode("utf-8", "ignore")).hexdigest() + + return { + "schemaVersion": 1, + "action": { + "package": spec["package"], + "version": spec["version"], + "revision": spec["revision"], + "architecture": architecture, + "actionHash": spec["remote_revision_hash"], + "commit": { + "ref": spec["commit_hash"], + "commitHash": real_commit_hash, + "altRefs": alt_refs, + }, + "recipeDigest": "sha256:" + recipe_digest, + "env": dict(spec.get("env") or {}), + "append_path": dict(spec.get("append_path") or {}), + "prepend_path": dict(spec.get("prepend_path") or {}), + "track_env": dict(spec.get("track_env") or {}), + "relocatePaths": sorted(spec.get("relocate_paths", [])), + "deps": dep_refs(spec.get("full_requires", [])), + "runtimeDeps": dep_refs(spec.get("full_runtime_requires", [])), + "depsHash": spec.get("deps_hash", ""), + }, + } + + def createDistLinks(spec, specs, args, syncHelper, repoType, requiresType): # At the point we call this function, spec has a single, definitive hash. target_dir = "{work_dir}/TARS/{arch}/{repo}/{package}/{package}-{version}-{revision}" \ @@ -1237,6 +1297,9 @@ def performPreferCheckWithTempDir(pkg, cmd): # Make sure not to upload local-only packages! These might have been # produced in a previous run with a read-only remote store. if not spec["revision"].startswith("local"): + # Assemble the Action Cache entry so AC/CAS-aware backends can record what + # produced this tarball. Backends that don't understand it ignore it. + spec["ac_entry"] = build_ac_entry(spec, specs, args.architecture) syncHelper.upload_symlinks_and_tarball(spec) if not args.onlyDeps: diff --git a/alibuild_helpers/utilities.py b/alibuild_helpers/utilities.py index 99bf03c0..76426985 100644 --- a/alibuild_helpers/utilities.py +++ b/alibuild_helpers/utilities.py @@ -106,6 +106,44 @@ def resolve_links_path(architecture, package): return "/".join(("TARS", architecture, package)) +def resolve_cas_path(content_hash, algo="sha256"): + """Return the store path for a content-addressed blob (REAPI-style CAS). + + Unlike resolve_store_path(), which is keyed by the *action* hash (a hash of + the recipe and its inputs), this is keyed by a hash of the blob's actual + bytes. The path is sharded on the first two hex characters of the hash, like + the action store, to keep directory fan-out manageable on local mirrors. + + The returned path is relative to the working directory or the root of the + remote store. + """ + return "/".join(("cas", algo, content_hash[:2], content_hash)) + + +def resolve_ac_path(architecture, action_hash): + """Return the store path for an Action Cache entry (REAPI-style AC). + + The entry is keyed by the action hash (spec["remote_revision_hash"]) and + records what produced the artifact, so the CAS can be reconstructed from it. + + The returned path is relative to the working directory or the root of the + remote store. + """ + return "/".join(("ac", architecture, action_hash[:2], action_hash + ".json")) + + +def file_digest(path, algo="sha256", _chunk_size=1 << 20): + """Return the hex digest of the bytes of the file at the given path. + + Used to content-address tarballs and recipe blobs for the CAS. + """ + hasher = hashlib.new(algo) + with open(path, "rb") as fileobj: + for chunk in iter(lambda: fileobj.read(_chunk_size), b""): + hasher.update(chunk) + return hasher.hexdigest() + + def short_commit_hash(spec): """Shorten the spec's commit hash to make it more human-readable. diff --git a/tests/test_build.py b/tests/test_build.py index bc16bbed..09a28c99 100644 --- a/tests/test_build.py +++ b/tests/test_build.py @@ -11,7 +11,7 @@ from collections import OrderedDict from alibuild_helpers.utilities import parseRecipe, resolve_tag -from alibuild_helpers.build import doBuild, storeHashes, generate_initdotsh +from alibuild_helpers.build import doBuild, storeHashes, generate_initdotsh, build_ac_entry # Determine architecture based on platform def get_test_architecture(): @@ -400,6 +400,55 @@ def test_hashing(self) -> None: self.assertEqual(len(extra["remote_hashes"]), 3) self.assertEqual(extra["local_hashes"][0], TEST_EXTRA_BUILD_HASH) + def test_build_ac_entry(self) -> None: + """build_ac_entry records the provenance needed to rebuild/install.""" + import hashlib + default = self.setup_spec(TEST_DEFAULT_RELEASE) + zlib = self.setup_spec(TEST_ZLIB_RECIPE) + default["commit_hash"] = "0" + zlib.setdefault("requires", []).append(default["package"]) + zlib["scm_refs"] = {ref: githash for githash, _, ref in ( + line.partition("\t") for line in TEST_ZLIB_GIT_REFS.splitlines())} + try: + zlib["commit_hash"] = zlib["scm_refs"]["refs/tags/" + zlib["tag"]] + except KeyError: + zlib["commit_hash"] = zlib["scm_refs"]["refs/heads/" + zlib["tag"]] + specs = {pkg["package"]: pkg for pkg in (default, zlib)} + for spec in specs.values(): + spec["is_devel_pkg"] = False + + storeHashes("defaults-release", specs, considerRelocation=False) + default["hash"] = default["remote_revision_hash"] + storeHashes("zlib", specs, considerRelocation=False) + zlib["hash"] = zlib["remote_revision_hash"] + zlib["revision"] = "1" + # These closures are normally computed in doBuild() before upload. + zlib["full_requires"] = {"defaults-release"} + zlib["full_runtime_requires"] = set() + zlib["relocate_paths"] = ["lib", "bin"] + + entry = build_ac_entry(zlib, specs, "slc7_x86-64") + self.assertEqual(entry["schemaVersion"], 1) + action = entry["action"] + self.assertEqual(action["package"], "zlib") + self.assertEqual(action["version"], zlib["version"]) + self.assertEqual(action["revision"], "1") + self.assertEqual(action["architecture"], "slc7_x86-64") + # The entry is keyed by the action hash, and deps are referenced by + # *their* action hash, so the recorded DAG matches storeHashes(). + self.assertEqual(action["actionHash"], zlib["remote_revision_hash"]) + self.assertEqual(action["deps"], + [{"package": "defaults-release", "actionHash": default["hash"]}]) + self.assertEqual(action["runtimeDeps"], []) + self.assertEqual(action["depsHash"], zlib["deps_hash"]) + # The recipe digest is the sha256 of the recipe body (a CAS blob). + self.assertEqual(action["recipeDigest"], "sha256:" + + hashlib.sha256(zlib["recipe"].encode("utf-8")).hexdigest()) + self.assertEqual(action["relocatePaths"], ["bin", "lib"]) # sorted + self.assertEqual(action["commit"]["ref"], zlib["commit_hash"]) + # The output digest/size are filled in by the backend at upload time. + self.assertNotIn("result", entry) + def test_initdotsh(self) -> None: """Sanity-check the generated init.sh for a few variables.""" specs = { diff --git a/tests/test_utilities.py b/tests/test_utilities.py index 26619fb0..9318238e 100644 --- a/tests/test_utilities.py +++ b/tests/test_utilities.py @@ -10,9 +10,12 @@ from alibuild_helpers.utilities import resolve_version from alibuild_helpers.utilities import topological_sort from alibuild_helpers.utilities import resolveFilename, resolveDefaultsFilename +from alibuild_helpers.utilities import resolve_store_path, resolve_cas_path, resolve_ac_path, file_digest import alibuild_helpers +import hashlib import os import string +import tempfile UBUNTU_1510_OS_RELEASE = """ NAME="Ubuntu" @@ -280,6 +283,30 @@ def test_UTF8_Hasher(self) -> None: self.assertEqual(h3.hexdigest(), "0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33") self.assertNotEqual(h1.hexdigest(), h2.hexdigest()) + def test_cas_ac_paths(self) -> None: + h = "abcdef0123456789abcdef0123456789abcdef01" + chash = "f" * 64 + # The action store and the CAS/AC namespaces all shard on the first two + # hex characters of the hash, matching resolve_store_path's convention. + self.assertEqual(resolve_store_path("slc7_x86-64", h), + "TARS/slc7_x86-64/store/ab/" + h) + self.assertEqual(resolve_cas_path(chash), + "cas/sha256/ff/" + chash) + self.assertEqual(resolve_cas_path(chash, algo="sha1"), + "cas/sha1/ff/" + chash) + self.assertEqual(resolve_ac_path("slc7_x86-64", h), + "ac/slc7_x86-64/ab/" + h + ".json") + + def test_file_digest(self) -> None: + payload = b"the quick brown fox\n" * 100000 # exceed the 1 MiB read chunk + expected = hashlib.sha256(payload).hexdigest() + with tempfile.NamedTemporaryFile() as tmp: + tmp.write(payload) + tmp.flush() + self.assertEqual(file_digest(tmp.name), expected) + self.assertEqual(file_digest(tmp.name, algo="sha1"), + hashlib.sha1(payload).hexdigest()) + def test_asList(self) -> None: self.assertEqual(asList("a"), ["a"]) self.assertEqual(asList(["a"]), ["a"])