Helpers for ac / cas path resolution

alibuild_helpers/build.py

@@ -25,6 +25,7 @@
 import tempfile
 
 import concurrent.futures
+import hashlib
 import importlib
 import json
 import socket
@@ -110,6 +111,65 @@ def update_repo(package, git_prompt):
 
 # Creates a directory in the store which contains symlinks to the package
 # and its direct / indirect dependencies
+def build_ac_entry(spec, specs, architecture):
+  """Assemble the Action Cache (AC) entry for a freshly built package.
+
+  The entry records what produced the tarball -- recipe, git commit, dependency
+  action hashes and build environment -- so that the content-addressed store
+  (CAS) can later be reconstructed from the (small) action cache, and so that a
+  package can be installed from its runtime closure without a build. See
+  REMOTE_STORE_CAS_AC.md.
+
+  The action hash is spec["remote_revision_hash"]; this entry is keyed by it.
+  Dependencies are referenced by their *action* hash (specs[dep]["hash"]), which
+  is what the action hash itself folds in (see storeHashes()), so the recorded
+  DAG matches the hash and is independent of whether builds are byte-reproducible.
+
+  result.outputDigest / result.size are filled in by the remote backend at
+  upload time, once the tarball bytes exist and have been hashed.
+  """
+  # Resolve the real git commit and the tag aliases pointing at it, mirroring
+  # storeHashes() so the recorded provenance matches the computed action hash.
+  scm_refs = spec.get("scm_refs", {})
+  real_commit_hash = scm_refs.get("refs/tags/" + spec["commit_hash"],
+                                  spec["commit_hash"])
+  alt_refs = {ref[len("refs/tags/"):]: git_hash
+              for ref, git_hash in scm_refs.items()
+              if ref.startswith("refs/tags/") and git_hash == real_commit_hash}
+
+  def dep_refs(dep_names):
+    return [{"package": dep, "actionHash": specs[dep]["hash"]}
+            for dep in sorted(dep_names)]
+
+  recipe_text = spec.get("recipe", "") or ""
+  recipe_digest = hashlib.sha256(recipe_text.encode("utf-8", "ignore")).hexdigest()
+
+  return {
+    "schemaVersion": 1,
+    "action": {
+      "package": spec["package"],
+      "version": spec["version"],
+      "revision": spec["revision"],
+      "architecture": architecture,
+      "actionHash": spec["remote_revision_hash"],
+      "commit": {
+        "ref": spec["commit_hash"],
+        "commitHash": real_commit_hash,
+        "altRefs": alt_refs,
+      },
+      "recipeDigest": "sha256:" + recipe_digest,
+      "env": dict(spec.get("env") or {}),
+      "append_path": dict(spec.get("append_path") or {}),
+      "prepend_path": dict(spec.get("prepend_path") or {}),
+      "track_env": dict(spec.get("track_env") or {}),
+      "relocatePaths": sorted(spec.get("relocate_paths", [])),
+      "deps": dep_refs(spec.get("full_requires", [])),
+      "runtimeDeps": dep_refs(spec.get("full_runtime_requires", [])),
+      "depsHash": spec.get("deps_hash", ""),
+    },
+  }
+
+
 def createDistLinks(spec, specs, args, syncHelper, repoType, requiresType):
   # At the point we call this function, spec has a single, definitive hash.
   target_dir = "{work_dir}/TARS/{arch}/{repo}/{package}/{package}-{version}-{revision}" \
@@ -1072,6 +1132,11 @@ def performPreferCheckWithTempDir(pkg, cmd):
       ("GIT_COMMITTER_EMAIL", "unknown"),
       ("INCREMENTAL_BUILD_HASH", spec.get("incremental_hash", "0")),
       ("JOBS", str(args.jobs)),
+      # Produce reproducible, content-stable tarballs for packages that may be
+      # uploaded to the remote store. Devel packages are never uploaded, so we
+      # leave their install trees untouched to avoid perturbing mtimes that
+      # incremental rebuilds might care about.
+      ("NORMALIZE_TARBALL", "" if spec["is_devel_pkg"] else "1"),
       ("PKGHASH", spec["hash"]),
       ("PKGNAME", spec["package"]),
       ("PKGREVISION", spec["revision"]),
@@ -1232,6 +1297,9 @@ def performPreferCheckWithTempDir(pkg, cmd):
     # Make sure not to upload local-only packages! These might have been
     # produced in a previous run with a read-only remote store.
     if not spec["revision"].startswith("local"):
+      # Assemble the Action Cache entry so AC/CAS-aware backends can record what
+      # produced this tarball. Backends that don't understand it ignore it.
+      spec["ac_entry"] = build_ac_entry(spec, specs, args.architecture)
       syncHelper.upload_symlinks_and_tarball(spec)
 
   if not args.onlyDeps:

alibuild_helpers/build_template.sh

@@ -41,6 +41,7 @@ export PATH=$WORK_DIR/wrapper-scripts:$PATH
 # - DEVEL_PREFIX
 # - INCREMENTAL_BUILD_HASH
 # - JOBS
+# - NORMALIZE_TARBALL
 # - PKGHASH
 # - PKGNAME
 # - PKGREVISION
@@ -291,6 +292,26 @@ mkdir -p "${WORK_DIR}/TARS/$HASH_PATH" \
          "${WORK_DIR}/TARS/$ARCHITECTURE/$PKGNAME"
 
 PACKAGE_WITH_REV=$PKGNAME-$PKGVERSION-$PKGREVISION.$ARCHITECTURE.tar.gz
+
+# Decide whether to produce a normalized (reproducible) tarball. Only packages
+# that may end up in the remote store are normalized; devel packages have
+# NORMALIZE_TARBALL unset so their install trees (and mtimes) are left untouched,
+# which matters for incremental rebuilds.
+do_normalize=
+if [ "$NORMALIZE_TARBALL" = 1 ] && [ "$CAN_DELETE" != 1 ] && [ -z "$CACHED_TARBALL" ]; then
+  do_normalize=1
+  # Normalize mtimes in place to a fixed, reproducible epoch. This must happen
+  # before the parallel rsync below, so the installed copy and the tarball agree
+  # and neither reads the tree while we mutate it. We avoid tar's --mtime, as
+  # old GNU tar (e.g. slc7's 1.26) doesn't support it. date(1) differs between
+  # GNU (-d @epoch) and BSD (-r epoch), so try both.
+  norm_epoch="${SOURCE_DATE_EPOCH:-0}"
+  if norm_stamp=$(date -d "@$norm_epoch" +%%Y%%m%%d%%H%%M.%%S 2>/dev/null) ||
+     norm_stamp=$(date -r "$norm_epoch" +%%Y%%m%%d%%H%%M.%%S 2>/dev/null); then
+    find "$WORK_DIR/INSTALLROOT/$PKGHASH" -exec touch -h -t "$norm_stamp" {} +
+  fi
+fi
+
 # Copy and tar/compress (if applicable) in parallel.
 # Use -H to match tar's behaviour of preserving hardlinks.
 rsync -DgloprH "$WORK_DIR/INSTALLROOT/$PKGHASH/" "$WORK_DIR" & rsync_pid=$!
@@ -302,9 +323,27 @@ elif [ -z "$CACHED_TARBALL" ]; then
   # Use pigz to compress, if we can, because it's multicore.
   gzip=$(command -v pigz) || gzip=$(command -v gzip)
   # We don't have an existing tarball, and we want to keep the one we create now.
-  tar -cC "$WORK_DIR/INSTALLROOT/$PKGHASH" . |
-    # Avoid having broken left overs if the tar fails.
-    $gzip -c > "$WORK_DIR/TARS/$HASH_PATH/$PACKAGE_WITH_REV.processing"
+  if [ -n "$do_normalize" ]; then
+    # Reproducible archive: deterministic entry order (via a sorted file list,
+    # since old GNU tar and bsdtar lack --sort), zeroed ownership, and the
+    # normalized mtimes set above. Ownership flags are spelled differently by
+    # GNU tar (--owner/--group) and bsdtar (--uid/--gid). gzip -n drops the
+    # gzip header's name and timestamp.
+    if tar --version 2>/dev/null | grep -qi 'GNU tar'; then
+      tar_owner="--owner=0 --group=0 --numeric-owner"
+    else
+      tar_owner="--uid 0 --gid 0 --numeric-owner"
+    fi
+    ( cd "$WORK_DIR/INSTALLROOT/$PKGHASH" &&
+      find . -mindepth 1 | LC_ALL=C sort |
+        tar -cf - --no-recursion $tar_owner -T - ) |
+      # Avoid having broken left overs if the tar fails.
+      $gzip -n -c > "$WORK_DIR/TARS/$HASH_PATH/$PACKAGE_WITH_REV.processing"
+  else
+    tar -cC "$WORK_DIR/INSTALLROOT/$PKGHASH" . |
+      # Avoid having broken left overs if the tar fails.
+      $gzip -c > "$WORK_DIR/TARS/$HASH_PATH/$PACKAGE_WITH_REV.processing"
+  fi
   mv "$WORK_DIR/TARS/$HASH_PATH/$PACKAGE_WITH_REV.processing" \
      "$WORK_DIR/TARS/$HASH_PATH/$PACKAGE_WITH_REV"
   ln -nfs "../../$HASH_PATH/$PACKAGE_WITH_REV" \

alibuild_helpers/utilities.py

@@ -106,6 +106,44 @@ def resolve_links_path(architecture, package):
   return "/".join(("TARS", architecture, package))
 
 
+def resolve_cas_path(content_hash, algo="sha256"):
+  """Return the store path for a content-addressed blob (REAPI-style CAS).
+
+  Unlike resolve_store_path(), which is keyed by the *action* hash (a hash of
+  the recipe and its inputs), this is keyed by a hash of the blob's actual
+  bytes. The path is sharded on the first two hex characters of the hash, like
+  the action store, to keep directory fan-out manageable on local mirrors.
+
+  The returned path is relative to the working directory or the root of the
+  remote store.
+  """
+  return "/".join(("cas", algo, content_hash[:2], content_hash))
+
+
+def resolve_ac_path(architecture, action_hash):
+  """Return the store path for an Action Cache entry (REAPI-style AC).
+
+  The entry is keyed by the action hash (spec["remote_revision_hash"]) and
+  records what produced the artifact, so the CAS can be reconstructed from it.
+
+  The returned path is relative to the working directory or the root of the
+  remote store.
+  """
+  return "/".join(("ac", architecture, action_hash[:2], action_hash + ".json"))
+
+
+def file_digest(path, algo="sha256", _chunk_size=1 << 20):
+  """Return the hex digest of the bytes of the file at the given path.
+
+  Used to content-address tarballs and recipe blobs for the CAS.
+  """
+  hasher = hashlib.new(algo)
+  with open(path, "rb") as fileobj:
+    for chunk in iter(lambda: fileobj.read(_chunk_size), b""):
+      hasher.update(chunk)
+  return hasher.hexdigest()
+
+
 def short_commit_hash(spec):
   """Shorten the spec's commit hash to make it more human-readable.
 

tests/test_build.py

@@ -11,7 +11,7 @@
 from collections import OrderedDict
 
 from alibuild_helpers.utilities import parseRecipe, resolve_tag
-from alibuild_helpers.build import doBuild, storeHashes, generate_initdotsh
+from alibuild_helpers.build import doBuild, storeHashes, generate_initdotsh, build_ac_entry
 
 # Determine architecture based on platform
 def get_test_architecture():
@@ -400,6 +400,55 @@ def test_hashing(self) -> None:
         self.assertEqual(len(extra["remote_hashes"]), 3)
         self.assertEqual(extra["local_hashes"][0], TEST_EXTRA_BUILD_HASH)
 
+    def test_build_ac_entry(self) -> None:
+        """build_ac_entry records the provenance needed to rebuild/install."""
+        import hashlib
+        default = self.setup_spec(TEST_DEFAULT_RELEASE)
+        zlib = self.setup_spec(TEST_ZLIB_RECIPE)
+        default["commit_hash"] = "0"
+        zlib.setdefault("requires", []).append(default["package"])
+        zlib["scm_refs"] = {ref: githash for githash, _, ref in (
+            line.partition("\t") for line in TEST_ZLIB_GIT_REFS.splitlines())}
+        try:
+            zlib["commit_hash"] = zlib["scm_refs"]["refs/tags/" + zlib["tag"]]
+        except KeyError:
+            zlib["commit_hash"] = zlib["scm_refs"]["refs/heads/" + zlib["tag"]]
+        specs = {pkg["package"]: pkg for pkg in (default, zlib)}
+        for spec in specs.values():
+            spec["is_devel_pkg"] = False
+
+        storeHashes("defaults-release", specs, considerRelocation=False)
+        default["hash"] = default["remote_revision_hash"]
+        storeHashes("zlib", specs, considerRelocation=False)
+        zlib["hash"] = zlib["remote_revision_hash"]
+        zlib["revision"] = "1"
+        # These closures are normally computed in doBuild() before upload.
+        zlib["full_requires"] = {"defaults-release"}
+        zlib["full_runtime_requires"] = set()
+        zlib["relocate_paths"] = ["lib", "bin"]
+
+        entry = build_ac_entry(zlib, specs, "slc7_x86-64")
+        self.assertEqual(entry["schemaVersion"], 1)
+        action = entry["action"]
+        self.assertEqual(action["package"], "zlib")
+        self.assertEqual(action["version"], zlib["version"])
+        self.assertEqual(action["revision"], "1")
+        self.assertEqual(action["architecture"], "slc7_x86-64")
+        # The entry is keyed by the action hash, and deps are referenced by
+        # *their* action hash, so the recorded DAG matches storeHashes().
+        self.assertEqual(action["actionHash"], zlib["remote_revision_hash"])
+        self.assertEqual(action["deps"],
+                         [{"package": "defaults-release", "actionHash": default["hash"]}])
+        self.assertEqual(action["runtimeDeps"], [])
+        self.assertEqual(action["depsHash"], zlib["deps_hash"])
+        # The recipe digest is the sha256 of the recipe body (a CAS blob).
+        self.assertEqual(action["recipeDigest"], "sha256:" +
+                         hashlib.sha256(zlib["recipe"].encode("utf-8")).hexdigest())
+        self.assertEqual(action["relocatePaths"], ["bin", "lib"])  # sorted
+        self.assertEqual(action["commit"]["ref"], zlib["commit_hash"])
+        # The output digest/size are filled in by the backend at upload time.
+        self.assertNotIn("result", entry)
+
     def test_initdotsh(self) -> None:
         """Sanity-check the generated init.sh for a few variables."""
         specs = {
@@ -437,6 +486,26 @@ def test_initdotsh(self) -> None:
         self.assertIn("export APPEND_ROOT_1=", complete_initdotsh)
         self.assertIn("export PREPEND_ROOT_1=", complete_initdotsh)
 
+    def test_build_template_percent_format(self) -> None:
+        """build_template.sh is interpolated via printf-style % formatting in
+        doBuild(), so every literal '%' in it must be doubled. A stray '%'
+        raises at build time but is not covered by the unit tests that mock the
+        build out, so check the real template formats cleanly here."""
+        template_path = os.path.join(
+            os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
+            "alibuild_helpers", "build_template.sh")
+        with open(template_path) as templatef:
+            template = templatef.read()
+        # These are exactly the keys doBuild() substitutes (see build.py).
+        keys = ("provenance", "initdotsh_deps", "initdotsh_full", "develPrefix",
+                "workDir", "configDir", "incremental_recipe", "requires",
+                "build_requires", "runtime_requires")
+        try:
+            template % {key: "" for key in keys}
+        except (TypeError, ValueError, KeyError) as exc:
+            self.fail("build_template.sh does not %%-format cleanly (likely an "
+                      "unescaped '%%' that should be '%%%%'): %s" % exc)
+
 
 if __name__ == '__main__':
     unittest.main()

tests/test_utilities.py

@@ -10,9 +10,12 @@
 from alibuild_helpers.utilities import resolve_version
 from alibuild_helpers.utilities import topological_sort
 from alibuild_helpers.utilities import resolveFilename, resolveDefaultsFilename
+from alibuild_helpers.utilities import resolve_store_path, resolve_cas_path, resolve_ac_path, file_digest
 import alibuild_helpers
+import hashlib
 import os
 import string
+import tempfile
 
 UBUNTU_1510_OS_RELEASE = """
 NAME="Ubuntu"
@@ -280,6 +283,30 @@ def test_UTF8_Hasher(self) -> None:
     self.assertEqual(h3.hexdigest(), "0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33")
     self.assertNotEqual(h1.hexdigest(), h2.hexdigest())
 
+  def test_cas_ac_paths(self) -> None:
+    h = "abcdef0123456789abcdef0123456789abcdef01"
+    chash = "f" * 64
+    # The action store and the CAS/AC namespaces all shard on the first two
+    # hex characters of the hash, matching resolve_store_path's convention.
+    self.assertEqual(resolve_store_path("slc7_x86-64", h),
+                     "TARS/slc7_x86-64/store/ab/" + h)
+    self.assertEqual(resolve_cas_path(chash),
+                     "cas/sha256/ff/" + chash)
+    self.assertEqual(resolve_cas_path(chash, algo="sha1"),
+                     "cas/sha1/ff/" + chash)
+    self.assertEqual(resolve_ac_path("slc7_x86-64", h),
+                     "ac/slc7_x86-64/ab/" + h + ".json")
+
+  def test_file_digest(self) -> None:
+    payload = b"the quick brown fox\n" * 100000  # exceed the 1 MiB read chunk
+    expected = hashlib.sha256(payload).hexdigest()
+    with tempfile.NamedTemporaryFile() as tmp:
+      tmp.write(payload)
+      tmp.flush()
+      self.assertEqual(file_digest(tmp.name), expected)
+      self.assertEqual(file_digest(tmp.name, algo="sha1"),
+                       hashlib.sha1(payload).hexdigest())
+
   def test_asList(self) -> None:
     self.assertEqual(asList("a"), ["a"])
     self.assertEqual(asList(["a"]), ["a"])