From 6d709e2d50fa6b2a68ebd034ab7536efd14b31b9 Mon Sep 17 00:00:00 2001 From: Giulio Eulisse <10544+ktf@users.noreply.github.com> Date: Wed, 2 Sep 2026 10:19:08 +0200 Subject: [PATCH 1/5] Do not fetch references when the need commit is there already --- alibuild_helpers/git.py | 6 ++++ alibuild_helpers/scm.py | 2 ++ alibuild_helpers/workarea.py | 34 ++++++++++++++++++++ tests/test_workarea.py | 62 ++++++++++++++++++++++++++++++++++++ 4 files changed, 104 insertions(+) diff --git a/alibuild_helpers/git.py b/alibuild_helpers/git.py index 9596a7ea..dbd4f7f2 100644 --- a/alibuild_helpers/git.py +++ b/alibuild_helpers/git.py @@ -76,6 +76,12 @@ def checkoutCmd(self, tag): def fetchCmd(self, remote, *refs): return ["fetch", "-f", "--prune"] + clone_speedup_options() + [remote, *refs] + def hasCommitCmd(self, commit): + # -e is silent and just sets the exit code. The ^{commit} peel matters: a + # bare cat-file -e succeeds for any object, so a hash that happens to name + # a blob or a tree would look like a usable commit. + return ["cat-file", "-e", commit + "^{commit}"] + def setWriteUrlCmd(self, url): return ["remote", "set-url", "--push", "origin", url] diff --git a/alibuild_helpers/scm.py b/alibuild_helpers/scm.py index 0ada8502..71559b45 100644 --- a/alibuild_helpers/scm.py +++ b/alibuild_helpers/scm.py @@ -19,6 +19,8 @@ def checkoutCmd(self, tag): raise NotImplementedError def fetchCmd(self, remote, *refs): raise NotImplementedError + def hasCommitCmd(self, commit): + raise NotImplementedError def cloneReferenceCmd(self, spec, referenceRepo, usePartialClone): raise NotImplementedError def cloneSourceCmd(self, source, destination, referenceRepo, usePartialClone): diff --git a/alibuild_helpers/workarea.py b/alibuild_helpers/workarea.py index 31475cea..d727d4fb 100644 --- a/alibuild_helpers/workarea.py +++ b/alibuild_helpers/workarea.py @@ -2,15 +2,46 @@ import errno import os import os.path +import re import shutil import tempfile from collections import OrderedDict from alibuild_helpers.log import dieOnError, debug, error +from alibuild_helpers.scm import SCMError from alibuild_helpers.utilities import call_ignoring_oserrors, symlink, short_commit_hash, asList FETCH_LOG_NAME = "fetch-log.txt" +#: A tag that is a full commit id cannot move: it names its own content. 40 hex +#: digits in a SHA-1 repository, 64 in a SHA-256 one. +IMMUTABLE_TAG_RE = re.compile(r"[0-9a-fA-F]{40}(?:[0-9a-fA-F]{24})?\Z") + + +def has_pinned_commit(scm, spec, referenceRepo): + """True if the spec pins a commit the reference repo already has. + + Fetching then cannot discover anything. The reason to refresh a reference + repo at all is that a tag may have been moved, and a commit id cannot be + moved -- so if the object is already here, the fetch has nothing to find. + + Worth skipping rather than merely tolerating, because that fetch is not free: + it asks the upstream for every ref in the repository, on every round, on + every builder. For Eigen3 -- pinned to a commit, and mirrored from gitlab.com + rather than from one of the alisw GitHub mirrors -- that is the request most + likely to be throttled, and a 403 there is fatal to the whole build. + """ + tag = str(spec.get("tag", "")) + if not IMMUTABLE_TAG_RE.match(tag): + return False + try: + err, _ = scm.exec(scm.hasCommitCmd(tag), directory=referenceRepo, + check=False, prompt=False) + except (NotImplementedError, SCMError, OSError): + # An SCM that cannot answer the question gets the old behaviour. + return False + return err == 0 + def cleanup_git_log(referenceSources): """Remove a stale fetch-log.txt. @@ -112,6 +143,9 @@ def updateReferenceRepo(referenceSources, p, spec, if not os.path.exists(referenceRepo): cmd = scm.cloneReferenceCmd(spec["source"], referenceRepo, usePartialClone) logged_scm(scm, p, referenceSources, cmd, ".", allowGitPrompt) + elif fetch and has_pinned_commit(scm, spec, referenceRepo): + debug("%s is pinned to commit %s, which %s already has: not fetching", + p, spec["tag"], referenceRepo) elif fetch: ref_match_rule = asList(spec.get("ref_match_rule", ["+refs/tags/*:refs/tags/*", "+refs/heads/*:refs/heads/*"])) cmd = scm.fetchCmd(spec["source"], *ref_match_rule) diff --git a/tests/test_workarea.py b/tests/test_workarea.py index f6d38272..bd077537 100644 --- a/tests/test_workarea.py +++ b/tests/test_workarea.py @@ -119,6 +119,68 @@ def test_reference_sources_created(self, mock_git, mock_makedirs, mock_exists): ], directory=".", check=False, prompt=True) self.assertEqual(spec.get("reference"), "%s/sw/MIRROR/aliroot" % getcwd()) + @patch("os.path.exists") + @patch("os.makedirs") + @patch("codecs.open") + @patch("alibuild_helpers.git.git") + @patch("alibuild_helpers.workarea.is_writeable", new=MagicMock(return_value=True)) + def test_pinned_commit_present_is_not_fetched(self, mock_git, mock_open, + mock_makedirs, mock_exists): + """A tag that is a commit id cannot move, so a present object needs no fetch. + + The point of refreshing a mirror is to notice a moved tag. A commit id + names its own content, so there is nothing to notice, and the fetch is + pure cost -- it asks upstream for every ref, on every round, on every + builder, and upstreams throttle that. + """ + mock_exists.return_value = True + mock_git.return_value = 0, "" # cat-file -e succeeds: the commit is here + spec = MOCK_SPEC.copy() + spec["tag"] = "e7248b26a1ed53fa030c5c459f7ea095dfd276ac" + updateReferenceRepoSpec(referenceSources="sw/MIRROR", p="AliRoot", + spec=spec, fetch=True) + # called ONCE: the existence check, and no fetch after it + mock_git.assert_called_once_with( + ["cat-file", "-e", spec["tag"] + "^{commit}"], + directory="%s/sw/MIRROR/aliroot" % getcwd(), check=False, prompt=False) + self.assertEqual(spec.get("reference"), "%s/sw/MIRROR/aliroot" % getcwd()) + + @patch("os.path.exists") + @patch("os.makedirs") + @patch("codecs.open") + @patch("alibuild_helpers.git.git") + @patch("alibuild_helpers.workarea.is_writeable", new=MagicMock(return_value=True)) + def test_pinned_commit_missing_is_fetched(self, mock_git, mock_open, + mock_makedirs, mock_exists): + """A pinned commit we do NOT have still has to be fetched.""" + mock_exists.return_value = True + mock_git.side_effect = [(1, "not found"), (0, "sentinel output")] + spec = MOCK_SPEC.copy() + spec["tag"] = "0" * 40 + updateReferenceRepoSpec(referenceSources="sw/MIRROR", p="AliRoot", + spec=spec, fetch=True) + self.assertEqual(mock_git.call_count, 2) + self.assertEqual(mock_git.call_args_list[-1][0][0][0], "fetch") + + @patch("os.path.exists") + @patch("os.makedirs") + @patch("codecs.open") + @patch("alibuild_helpers.git.git") + @patch("alibuild_helpers.workarea.is_writeable", new=MagicMock(return_value=True)) + def test_movable_tag_is_always_fetched(self, mock_git, mock_open, + mock_makedirs, mock_exists): + """A named tag can be moved, so it is fetched without even asking.""" + mock_exists.return_value = True + mock_git.return_value = 0, "sentinel output" + spec = MOCK_SPEC.copy() + spec["tag"] = "v3.4.0" + updateReferenceRepoSpec(referenceSources="sw/MIRROR", p="AliRoot", + spec=spec, fetch=True) + mock_git.assert_called_once_with([ + "fetch", "-f", "--prune", "--filter=blob:none", spec["source"], + "+refs/tags/*:refs/tags/*", "+refs/heads/*:refs/heads/*", + ], directory="%s/sw/MIRROR/aliroot" % getcwd(), check=False, prompt=True) + if __name__ == '__main__': unittest.main() From c43997224045e782da2cc0acbd310fea8bf89d29 Mon Sep 17 00:00:00 2001 From: Giulio Eulisse <10544+ktf@users.noreply.github.com> Date: Wed, 2 Sep 2026 10:19:08 +0200 Subject: [PATCH 2/5] Use absolute path for sleep --- alibuild_helpers/cmd.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/alibuild_helpers/cmd.py b/alibuild_helpers/cmd.py index 4b2499fc..68882334 100644 --- a/alibuild_helpers/cmd.py +++ b/alibuild_helpers/cmd.py @@ -166,12 +166,12 @@ def __init__(self, docker_image, docker_run_args=(), extra_env={}, extra_volumes def __enter__(self): if self._docker_image: stream_pull("docker", self._docker_image) - # "sleep inf" pauses forever, until we kill it. + # "/bin/sleep inf" pauses forever, until we kill it. envOpts = [opt for k, v in self._extra_env.items() for opt in ("-e", f"{k}={v}")] volumes = [opt for v in self._extra_volumes for opt in ("-v", v)] - cmd = ["docker", "run", "--detach"] + envOpts + volumes + ["--rm", "--entrypoint="] + cmd = ["docker", "run", "--detach"] + envOpts + volumes + ["--rm", "--entrypoint=/bin/sleep"] cmd += self._docker_run_args - cmd += [self._docker_image, "sleep", "inf"] + cmd += [self._docker_image, "inf"] debug("Starting Docker container (this pulls %s first if it is not present " "locally, which can take a while with no output): %s", self._docker_image, " ".join(cmd)) From 28708a0a4fa8dde6276174681b2c588f1c5c5d84 Mon Sep 17 00:00:00 2001 From: Giulio Eulisse <10544+ktf@users.noreply.github.com> Date: Wed, 2 Sep 2026 10:19:08 +0200 Subject: [PATCH 3/5] Claim packages atomically, and resume a partial publish Publishing writes the package symlink, the dist symlinks and the tarball, with no transaction between them. A build killed part-way left the remote in a state every later build refused to touch ("already exists on S3 but ... does not", or "Conflicts detected" among the dist symlinks) until someone cleaned it up by hand, and clients following the symlink got 404s meanwhile. Write the package symlink with If-None-Match: * and let it arbitrate who owns the revision. If it points at the store path we are about to write, any leftovers are our own and we complete the publish; anything else still aborts. Losing the claim to a build of the same hash is fine, but that build may itself have died, so check for its tarball instead of assuming it will finish. The store must honour If-None-Match (CERN's Ceph RGW verifiably does); ignoring it would silently restore the old racy behaviour, so there is no fallback. botocore support is checked up front, and read-only use is unaffected. --- alibuild_helpers/sync.py | 115 +++++++++++++++++----- tests/test_sync.py | 199 ++++++++++++++++++++++++++++++++++++++- 2 files changed, 290 insertions(+), 24 deletions(-) diff --git a/alibuild_helpers/sync.py b/alibuild_helpers/sync.py index d9612df3..48b6085c 100644 --- a/alibuild_helpers/sync.py +++ b/alibuild_helpers/sync.py @@ -571,10 +571,64 @@ def _s3_init(self) -> None: "variables to aliBuild in order to use the S3 remote store") sys.exit(1) + if self.writeStore: + self._check_conditional_write_support() + + def _check_conditional_write_support(self): + """Fail early if we cannot claim packages before uploading them. + + Only the client side is checkable: a store without conditional writes + ignores If-None-Match rather than rejecting it. + """ + try: + supported = "IfNoneMatch" in self.s3.meta.service_model \ + .operation_model("PutObject").input_shape.members + except Exception: # pylint: disable=broad-except + supported = False + dieOnError(not supported, + "your boto3/botocore is too old to publish to an S3 store: it " + "cannot send If-None-Match, which aliBuild needs to claim a " + "package before uploading it. Upgrade it (S3 conditional writes " + "were added in August 2024) with: pip install -U boto3") + + def _link_is_ours(self, link_path, link_body): + """Does the symlink on S3 point at the tarball we are about to write? + + False if it turned out not to exist. Dies if it belongs to a different + build, which we have no way to arbitrate with. + """ + from botocore.exceptions import ClientError + try: + remote_target = self.s3.get_object( + Bucket=self.writeStore, Key=link_path)["Body"].read().decode("utf-8").strip() + except ClientError as exc: + # Gone since we looked. Anything else and we cannot tell whose it is. + dieOnError(exc.response.get("Error", {}).get("Code") + not in ("404", "NoSuchKey"), + "%s exists on S3 but could not be read, so we cannot tell " + "whether this build owns it: %s" % (link_path, exc)) + return False + dieOnError(remote_target != link_body, + "%s already exists on S3 and points at %s, not %s: another build " + "owns this package. Aborting rather than repointing it." % + (link_path, remote_target, link_body)) + return True + def _put_link(self, link_path, link_body): - """Write the symlink object, whose body is the store path it stands for.""" - self.s3.put_object(Bucket=self.writeStore, Key=link_path, - Body=link_body.encode("utf-8")) + """Claim the symlink object, whose body is the store path it stands for. + + Conditional on the key not existing yet; False if somebody claimed it first. + """ + from botocore.exceptions import ClientError + try: + self.s3.put_object(Bucket=self.writeStore, Key=link_path, IfNoneMatch="*", + Body=link_body.encode("utf-8")) + return True + except ClientError as exc: + if exc.response.get("Error", {}).get("Code") == "PreconditionFailed" or \ + exc.response.get("ResponseMetadata", {}).get("HTTPStatusCode") == 412: + return False + raise def _s3_listdir(self, dirname): """List keys of items under dirname in the read bucket.""" @@ -715,11 +769,11 @@ def upload_symlinks_and_tarball(self, spec) -> None: debug("All %s symlinks already exist on S3, skipping upload", link_dir) continue - # Excluding our own symlinks (above), if there is anything in our link_dir - # on the remote, something else is uploading symlinks (or already has)! - dieOnError(symlinks_existing, - "Conflicts detected in %s on S3; aborting: %s" % - (link_dir, ", ".join(sorted(symlinks_existing)))) + # Leftovers of a publish that died among these. Who owns the revision + # is decided by the package symlink claim below: foreign builds die + # there before writing anything, so rewriting is safe. + if symlinks_existing: + warning("%s is incomplete on S3; rewriting it.", link_dir) dist_symlinks[link_dir] = symlinks @@ -740,10 +794,9 @@ def upload_symlinks_and_tarball(self, spec) -> None: else: # The tarball is in the local store but its link was never made: it was # fetched from a store where the link had not been published, so - # fetch_symlinks had nothing to copy. The body is the store path we are - # about to publish, so write the link rather than failing on a state we - # can repair -- and before the check below, which would otherwise report - # a missing local file as a conflict on the remote. + # fetch_symlinks had nothing to copy. Write the link rather than failing + # on a state we can repair -- and before the ownership check below, which + # would otherwise report a missing local file as a remote conflict. # A link body is the store path relative to TARS/, i.e. # "/store/xx//" -- that is what os.readlink(...) # .lstrip("./") yields for a link the build created, and fetch_symlinks @@ -751,20 +804,37 @@ def upload_symlinks_and_tarball(self, spec) -> None: link_body = tar_path[len("TARS/"):] if tar_path.startswith("TARS/") else tar_path os.makedirs(os.path.dirname(local_link), exist_ok=True) symlink("../../" + link_body, local_link) - dieOnError(tar_exists or link_exists, - "%s already exists on S3 but %s does not, aborting!" % - (tar_path if tar_exists else link_path, - link_path if tar_exists else tar_path)) + if tar_exists or link_exists: + # Half a publish: a previous run died between the two writes, or + # something else is uploading this package right now. + if link_exists: + link_exists = self._link_is_ours(link_path, link_body) + warning("%s was published only partially (%s is missing); completing it.", + tarball, tar_path if link_exists else link_path) debug("Uploading tarball and symlinks for %s %s-%s (%s) to S3", spec["package"], spec["version"], spec["revision"], spec["hash"]) - # Upload the smaller file first, so that any parallel uploads are more - # likely to find it and fail. - self._put_link(link_path, link_body) + # Claim the package before uploading it, so a parallel upload fails here + # rather than overwriting us. + if link_exists: + debug("%s already points at our tarball", link_path) + elif not self._put_link(link_path, link_body): + # Somebody claimed it first. Dies unless they build the same hash. + dieOnError(not self._link_is_ours(link_path, link_body), + "%s was created and then deleted while we were claiming it; " + "refusing to publish a tarball with no symlink." % link_path) + # They may have died mid-publish, so check rather than assume they will + # finish; duplicating a live uploader's work is harmless. + if self._s3_key_exists(tar_path): + debug("%s was published by a concurrent build of the same hash", tarball) + return + warning("%s was claimed by a concurrent build of the same hash, which has " + "not uploaded the tarball; uploading ours.", link_path) - # Second, upload dist symlinks. These should be in place before the main - # tarball, to avoid races in the publisher. + # Second, upload dist symlinks. The publisher takes the tarball appearing + # under store/ as its signal to publish and reads runtime dependencies from + # dist-runtime/, where an incomplete listing means deps silently missing. start_time = time.time() total_symlinks = 0 @@ -805,7 +875,8 @@ def _upload_single_symlink(link_key, hash_path): debug("Uploaded %d dist symlinks in %.2f seconds", total_symlinks, end_time - start_time) - self._upload_tarball(spec, tar_path) + if not tar_exists: + self._upload_tarball(spec, tar_path) def _upload_tarball(self, spec, tar_path) -> None: """Upload the tarball bytes to the remote store under tar_path. diff --git a/tests/test_sync.py b/tests/test_sync.py index 2eb07f00..c91677ec 100644 --- a/tests/test_sync.py +++ b/tests/test_sync.py @@ -15,6 +15,7 @@ GOOD_HASH = "deadbeefdeadbeefdeadbeefdeadbeefdeadbeef" BAD_HASH = "baadf00dbaadf00dbaadf00dbaadf00dbaadf00d" NONEXISTENT_HASH = "TRIGGERS_A_404" +RESUME_HASH = "f00dcafef00dcafef00dcafef00dcafef00dcafe" GOOD_SPEC = { # fully present on the remote store "package": PACKAGE, "version": "v1.3.1", "revision": "1", "hash": GOOD_HASH, @@ -33,6 +34,12 @@ "remote_revision_hash": NONEXISTENT_HASH, "remote_hashes": [NONEXISTENT_HASH], } +RESUME_SPEC = { # symlink published, but the store object never made it + "package": PACKAGE, "version": "v1.3.1", "revision": "4", + "hash": RESUME_HASH, + "remote_revision_hash": RESUME_HASH, + "remote_hashes": [RESUME_HASH], +} def tarball_name(spec): @@ -285,7 +292,8 @@ def mock_s3(self): def paginate_listdir(Bucket, Delimiter, Prefix): dir = Prefix.rstrip(Delimiter) if dir in (resolve_store_path(ARCHITECTURE, NONEXISTENT_HASH), - resolve_store_path(ARCHITECTURE, BAD_HASH)): + resolve_store_path(ARCHITECTURE, BAD_HASH), + resolve_store_path(ARCHITECTURE, RESUME_HASH)): return [{}] elif dir in (resolve_store_path(ARCHITECTURE, GOOD_HASH), resolve_links_path(ARCHITECTURE, PACKAGE)): @@ -311,11 +319,17 @@ def paginate_listdir(Bucket, Delimiter, Prefix): elif dir.endswith("-" + MISSING_SPEC["revision"]): # No pre-existing symlinks under dist*. return [{"Contents": []}] + elif dir.endswith("-" + RESUME_SPEC["revision"]): + # The interrupted publish also died among the dist symlinks. + return [{"Contents": [ + {"Key": dir + Delimiter + "somedep-v1-1.%s.tar.gz" % ARCHITECTURE}, + ]}] else: raise NotImplementedError("unknown dist prefix " + Prefix) def head_object(Bucket, Key): if NONEXISTENT_HASH in Key or BAD_HASH in Key or \ + RESUME_HASH in Key or \ os.path.basename(Key) == tarball_name(MISSING_SPEC): raise ClientError({"Error": {"Code": "404"}}, "head_object") return {} @@ -434,13 +448,194 @@ def test_tarball_upload(self) -> None: b3sync.s3.put_object.assert_not_called() b3sync.s3.upload_file.assert_not_called() - # Make sure conflict detection is working for tarball sync. + # Conflict detection: the remote symlink points at somebody else's + # store path, so they own this package and we must not touch it. b3sync.s3.put_object.reset_mock() b3sync.s3.upload_file.reset_mock() self.assertRaises(SystemExit, b3sync.upload_symlinks_and_tarball, BAD_SPEC) b3sync.s3.put_object.assert_not_called() b3sync.s3.upload_file.assert_not_called() + @patch("os.listdir", new=lambda path: ( + [tarball_name(RESUME_SPEC)] if path.endswith("-" + RESUME_SPEC["revision"]) else + NotImplemented + )) + @patch("os.path.islink", new=MagicMock(return_value=True)) + def test_tarball_upload_resume(self) -> None: + """A publish interrupted between the symlink and the tarball is resumable.""" + link_target = os.path.join( + resolve_store_path(ARCHITECTURE, RESUME_HASH), tarball_name(RESUME_SPEC)) + b3sync = sync.Boto3RemoteSync( + remoteStore="b3://localhost", writeStore="b3://localhost", + architecture=ARCHITECTURE, workdir="/sw") + b3sync.s3 = self.mock_s3() + # The remote symlink points where we are about to write: ours. + b3sync.s3.get_object = MagicMock(return_value={ + "Body": MagicMock(read=lambda: link_target.encode("utf-8")), + }) + + with patch("os.readlink", new=MagicMock(return_value="../../" + link_target)): + b3sync.upload_symlinks_and_tarball(RESUME_SPEC) + + b3sync.s3.upload_file.assert_called() + link_key = os.path.join(resolve_links_path(ARCHITECTURE, PACKAGE), + tarball_name(RESUME_SPEC)) + for call in b3sync.s3.put_object.mock_calls: + self.assertNotEqual(call.kwargs.get("Key"), link_key) + + @patch("os.listdir", new=lambda path: ( + [tarball_name(RESUME_SPEC)] if path.endswith("-" + RESUME_SPEC["revision"]) else + NotImplemented + )) + @patch("os.path.islink", new=MagicMock(return_value=True)) + def test_tarball_present_link_missing(self) -> None: + """Only the missing symlink is written; the tarball is not re-uploaded.""" + link_target = os.path.join( + resolve_store_path(ARCHITECTURE, RESUME_HASH), tarball_name(RESUME_SPEC)) + b3sync = sync.Boto3RemoteSync( + remoteStore="b3://localhost", writeStore="b3://localhost", + architecture=ARCHITECTURE, workdir="/sw") + b3sync.s3 = self.mock_s3() + b3sync._s3_key_exists = lambda path: path == link_target + + with patch("os.readlink", new=MagicMock(return_value="../../" + link_target)): + b3sync.upload_symlinks_and_tarball(RESUME_SPEC) + + b3sync.s3.upload_file.assert_not_called() + b3sync.s3.put_object.assert_any_call( + Bucket="localhost", IfNoneMatch="*", + Key=os.path.join(resolve_links_path(ARCHITECTURE, PACKAGE), + tarball_name(RESUME_SPEC)), + Body=link_target.encode("utf-8")) + + @patch("os.listdir", new=lambda path: ( + [tarball_name(RESUME_SPEC)] if path.endswith("-" + RESUME_SPEC["revision"]) else + NotImplemented + )) + @patch("os.readlink", new=MagicMock(return_value="dummy path")) + @patch("os.path.islink", new=MagicMock(return_value=True)) + def test_tarball_upload_unreadable_link(self) -> None: + """If we cannot tell who owns the existing symlink, we must not touch it.""" + from botocore.exceptions import ClientError + b3sync = sync.Boto3RemoteSync( + remoteStore="b3://localhost", writeStore="b3://localhost", + architecture=ARCHITECTURE, workdir="/sw") + b3sync.s3 = self.mock_s3() + b3sync.s3.get_object = MagicMock(side_effect=ClientError( + {"Error": {"Code": "AccessDenied"}}, "get_object")) + + self.assertRaises(SystemExit, b3sync.upload_symlinks_and_tarball, RESUME_SPEC) + b3sync.s3.put_object.assert_not_called() + b3sync.s3.upload_file.assert_not_called() + + def fresh_upload_sync(self): + """A sync object publishing MISSING_SPEC, which is absent from the remote.""" + b3sync = sync.Boto3RemoteSync( + remoteStore="b3://localhost", writeStore="b3://localhost", + architecture=ARCHITECTURE, workdir="/sw") + b3sync.s3 = self.mock_s3() + return b3sync + + def test_conditional_write_required(self) -> None: + """Publishing without If-None-Match support must fail before any work.""" + import boto3 + b3sync = self.fresh_upload_sync() + # A mock has no service model, standing in for an old botocore. + self.assertRaises(SystemExit, b3sync._check_conditional_write_support) + b3sync.s3 = boto3.client("s3", region_name="us-east-1", + aws_access_key_id="x", aws_secret_access_key="y") + b3sync._check_conditional_write_support() + + @patch("os.listdir", new=lambda path: ( + [] if path.endswith("-" + MISSING_SPEC["revision"]) else NotImplemented)) + @patch("os.readlink", new=MagicMock(return_value="dummy path")) + @patch("os.path.islink", new=MagicMock(return_value=True)) + def test_symlink_claimed_conditionally(self) -> None: + """The symlink is claimed with If-None-Match, where the store supports it.""" + b3sync = self.fresh_upload_sync() + b3sync.upload_symlinks_and_tarball(MISSING_SPEC) + b3sync.s3.put_object.assert_any_call( + IfNoneMatch="*", Bucket="localhost", + Key=os.path.join(resolve_links_path(ARCHITECTURE, PACKAGE), + tarball_name(MISSING_SPEC)), + Body=b"dummy path") + + @patch("os.listdir", new=lambda path: ( + [] if path.endswith("-" + MISSING_SPEC["revision"]) else NotImplemented)) + @patch("os.readlink", new=MagicMock(return_value="dummy path")) + @patch("os.path.islink", new=MagicMock(return_value=True)) + def test_symlink_claim_lost_to_other_build(self) -> None: + """Losing the claim to a build of a different hash must abort the upload.""" + from botocore.exceptions import ClientError + b3sync = self.fresh_upload_sync() + b3sync.s3.put_object = MagicMock(side_effect=ClientError( + {"Error": {"Code": "PreconditionFailed"}}, "put_object")) + # mock_s3's get_object reports a target that is not ours. + self.assertRaises(SystemExit, b3sync.upload_symlinks_and_tarball, MISSING_SPEC) + b3sync.s3.upload_file.assert_not_called() + + @patch("os.listdir", new=lambda path: ( + [] if path.endswith("-" + MISSING_SPEC["revision"]) else NotImplemented)) + @patch("os.readlink", new=MagicMock(return_value="dummy path")) + @patch("os.path.islink", new=MagicMock(return_value=True)) + def test_symlink_claim_lost_to_same_hash(self) -> None: + """We upload anyway: the winner of the claim may have died mid-publish.""" + b3sync = self.claim_losing_sync() + b3sync.upload_symlinks_and_tarball(MISSING_SPEC) + b3sync.s3.upload_file.assert_called() + + @patch("os.listdir", new=lambda path: ( + [] if path.endswith("-" + MISSING_SPEC["revision"]) else NotImplemented)) + @patch("os.readlink", new=MagicMock(return_value="dummy path")) + @patch("os.path.islink", new=MagicMock(return_value=True)) + def test_symlink_claim_lost_to_finished_build(self) -> None: + """...but not if the winner already finished: nothing left to do.""" + b3sync = self.claim_losing_sync() + tar_path = os.path.join(resolve_store_path(ARCHITECTURE, NONEXISTENT_HASH), + tarball_name(MISSING_SPEC)) + # It shows up only after the two existence checks at the top. + checks = [] + + def key_exists(path): + checks.append(path) + return path == tar_path and len(checks) > 2 + + b3sync._s3_key_exists = key_exists + b3sync.upload_symlinks_and_tarball(MISSING_SPEC) + b3sync.s3.upload_file.assert_not_called() + + @patch("os.listdir", new=lambda path: ( + [] if path.endswith("-" + MISSING_SPEC["revision"]) else NotImplemented)) + @patch("os.readlink", new=MagicMock(return_value="dummy path")) + @patch("os.path.islink", new=MagicMock(return_value=True)) + def test_symlink_deleted_under_us(self) -> None: + """A symlink deleted while we claim it must not leave an unreferenced tarball.""" + from botocore.exceptions import ClientError + b3sync = self.claim_losing_sync() + b3sync.s3.get_object = MagicMock(side_effect=ClientError( + {"Error": {"Code": "NoSuchKey"}}, "get_object")) + + self.assertRaises(SystemExit, b3sync.upload_symlinks_and_tarball, MISSING_SPEC) + b3sync.s3.upload_file.assert_not_called() + + def claim_losing_sync(self): + """A sync object that always loses the race to claim the symlink. + + The winner reports the same target as ours, so it is building the same + hash rather than conflicting with us. + """ + from botocore.exceptions import ClientError + + def put_object(**kwargs): + if "IfNoneMatch" in kwargs: + raise ClientError({"Error": {"Code": "PreconditionFailed"}}, "put_object") + + b3sync = self.fresh_upload_sync() + b3sync.s3.put_object = MagicMock(side_effect=put_object) + b3sync.s3.get_object = MagicMock(return_value={ + "Body": MagicMock(read=lambda: b"dummy path")}) + return b3sync + if __name__ == '__main__': unittest.main() From 9cca6773c00d6a9398477f6ddcde6f225689a1fe Mon Sep 17 00:00:00 2001 From: Giulio Eulisse <10544+ktf@users.noreply.github.com> Date: Wed, 2 Sep 2026 10:19:08 +0200 Subject: [PATCH 4/5] Skill to add back support for old botocore / Ceph backends --- .claude/skills/s3-conditional-writes/SKILL.md | 100 ++++++++++++++++++ 1 file changed, 100 insertions(+) create mode 100644 .claude/skills/s3-conditional-writes/SKILL.md diff --git a/.claude/skills/s3-conditional-writes/SKILL.md b/.claude/skills/s3-conditional-writes/SKILL.md new file mode 100644 index 00000000..82bab067 --- /dev/null +++ b/.claude/skills/s3-conditional-writes/SKILL.md @@ -0,0 +1,100 @@ +--- +name: s3-conditional-writes +description: Use this skill when the b3:// S3 backend fails with "your boto3/botocore is too old to publish", when asked to support an S3 store that does not implement conditional writes (If-None-Match on PUT), or when changing how packages are claimed and published in Boto3RemoteSync. +version: 0.1.0 +--- + +# Conditional writes in the b3:// backend + +Publishing a package writes two objects with no transaction spanning them: + +1. `TARS///--..tar.gz` — a symlink object whose + body is the store path of the tarball. +2. `TARS//store///` — the tarball itself. + +The symlink is written first, with `IfNoneMatch="*"`, which makes it an atomic +claim: a second build publishing the same package gets a 412 instead of +silently overwriting. `Boto3RemoteSync._put_link()` does this, and it is the +only write to that key -- completing a publish already established as ours +skips it, because `_link_is_ours()` has just verified the body is what we would +write. There is deliberately no unconditional overwrite anywhere, so no +`IfMatch` is needed (its RGW support is unmeasured in any case). + +**This is a hard requirement, checked at startup.** +`_check_conditional_write_support()` runs from `_s3_init()` whenever a write +store is configured, and exits with an actionable message if botocore cannot +send the parameter. Read-only use is unaffected. + +## Why supporting stores without it was rejected + +A store that does not implement conditional writes **ignores** `If-None-Match` +rather than rejecting it. It returns 200 for a claim that should have failed, +so the fallback is undetectable from the response: the code would believe it +held an exclusive claim while actually running the old check-then-act logic. +Silently degrading a mutual-exclusion guarantee is worse than refusing to run. + +CERN's Ceph RGW was measured to enforce it (a second +`put_object(..., IfNoneMatch="*")` on an existing key returns +`PreconditionFailed`). AWS S3 has supported it since August 2024. + +## What re-adding compatibility would take + +Only do this if a store that genuinely cannot honour the header has to be +supported. In rough order: + +1. **Detect the client side.** Restore the probe as a predicate rather than a + fatal check: + ```python + "IfNoneMatch" in self.s3.meta.service_model \ + .operation_model("PutObject").input_shape.members + ``` + Cache it on the instance; it cannot change at runtime. This is cheap, + offline and reliable. + +2. **Decide about the store side, which is not detectable.** There is no + read-only probe. The only conclusive test is writing the same key twice with + `IfNoneMatch="*"` and seeing whether the second attempt returns 412, which + means a write to the real store at startup. Options, none free: + - trust a config flag / URL parameter set by whoever runs the build; + - probe once against a throwaway key and cache the answer; + - accept the degradation silently, which is what this design refuses. + +3. **Gate the claim.** Give `_put_link` a flag and fall back to a plain + `put_object` when unsupported. + +4. **Keep the recovery path intact.** `_link_is_ours()` and the + partial-publish completion in `upload_symlinks_and_tarball` do not depend on + conditional writes and must keep working either way — they are what makes a + claim stranded by a killed build recoverable at all. + +5. **Test both paths.** The removed test asserted no `IfNoneMatch` appears in + any `put_object` call when support is absent; the surviving + `test_symlink_claimed_conditionally` asserts the opposite when it is + present. Reinstate the pair, parameterising the fixture helper + `fresh_upload_sync()` on support. + +## Known residual race + +A symlink deleted between our claim failing and our reading it aborts the +publish rather than retrying. Retrying does not help: the same race can hit the +retry, and a deleter can remove the link after a successful write anyway. +Nothing in aliBuild deletes links, so this needs a cleanup job running +concurrently with a build. + +## Invariants not to break + +- **Partial dist directories are rewritten, not treated as conflicts.** Safe + only because the package symlink claim arbitrates ownership of the revision, + and foreign builds die before writing anything; do not loosen one without + the other. +- **The tarball is written last.** `aliPublishS3` treats a tarball appearing + under `store/` as its signal to publish and reads the package's runtime + dependencies from `dist-runtime/`. An incomplete listing there is not an + error, it is a package published with dependencies missing — so every + symlink must be in place before the tarball becomes visible. +- **A claim must stay recoverable.** With conditional writes and no recovery + logic, a build killed between the claim and the tarball would strand a + symlink no later build could ever retake. +- **Losing a claim to the same hash is not fatal**, but the winner must not be + assumed to have finished: check for the tarball, and upload if it is absent. + The winner may have died mid-publish, and liveness is not observable. From 92b3acf9b5489e8a5cdbff263b6b4b43890708e0 Mon Sep 17 00:00:00 2001 From: Giulio Eulisse <10544+ktf@users.noreply.github.com> Date: Wed, 2 Sep 2026 23:34:21 +0200 Subject: [PATCH 5/5] Feature commit for alibuild 2.0 * Add new provenance tracking remote store back-end. Modeled after Google's REAPI. * Add install sub command to install packages without alidist and the need to build them. * Add reconstruct sub command to rebuild a tarball using the provenance information stored in the remote store ledger. * Add migrate sub command to populate the new reapi store with packages from the old store * Code-signing support for official tarballs --- REMOTE_STORE_CAS_AC.md | 576 ++++++++++++++ REMOTE_STORE_CAS_AC_LOGBOOK.md | 275 +++++++ aliBuild | 12 + alibuild_helpers/args.py | 87 ++- alibuild_helpers/build.py | 439 ++++++++++- alibuild_helpers/build_template.sh | 43 ++ alibuild_helpers/completions/bash.sh | 69 +- alibuild_helpers/completions/zsh.sh | 61 ++ alibuild_helpers/install.py | 158 ++++ alibuild_helpers/keyring.json | 30 + alibuild_helpers/log.py | 14 +- alibuild_helpers/migrate.py | 1031 +++++++++++++++++++++++++ alibuild_helpers/reconstruct.py | 791 +++++++++++++++++++ alibuild_helpers/signing.py | 317 ++++++++ alibuild_helpers/source.py | 251 ++++++ alibuild_helpers/sync.py | 89 ++- alibuild_helpers/sync_reapi.py | 1054 ++++++++++++++++++++++++++ alibuild_helpers/utilities.py | 57 ++ alienv | 14 +- pyproject.toml | 8 +- setup.py | 7 +- tests/test_build.py | 262 ++++++- tests/test_cmd.py | 6 +- tests/test_install.py | 345 +++++++++ tests/test_migrate.py | 732 ++++++++++++++++++ tests/test_reconstruct.py | 556 ++++++++++++++ tests/test_signing.py | 311 ++++++++ tests/test_source.py | 289 +++++++ tests/test_sync.py | 135 +++- tests/test_sync_reapi.py | 875 +++++++++++++++++++++ tests/test_utilities.py | 28 + tox.ini | 5 + 32 files changed, 8859 insertions(+), 68 deletions(-) create mode 100644 REMOTE_STORE_CAS_AC.md create mode 100644 REMOTE_STORE_CAS_AC_LOGBOOK.md create mode 100644 alibuild_helpers/install.py create mode 100644 alibuild_helpers/keyring.json create mode 100644 alibuild_helpers/migrate.py create mode 100644 alibuild_helpers/reconstruct.py create mode 100644 alibuild_helpers/signing.py create mode 100644 alibuild_helpers/source.py create mode 100644 alibuild_helpers/sync_reapi.py create mode 100644 tests/test_install.py create mode 100644 tests/test_migrate.py create mode 100644 tests/test_reconstruct.py create mode 100644 tests/test_signing.py create mode 100644 tests/test_source.py create mode 100644 tests/test_sync_reapi.py diff --git a/REMOTE_STORE_CAS_AC.md b/REMOTE_STORE_CAS_AC.md new file mode 100644 index 00000000..5acf67b0 --- /dev/null +++ b/REMOTE_STORE_CAS_AC.md @@ -0,0 +1,576 @@ +# Design: S3 remote store with Action Cache + CAS + +Status: **largely implemented.** This document describes aliBuild's remote store, +redesigned along the lines of Google's Remote Execution API (REAPI), splitting it +into a small, authoritative **Action Cache (AC)** and a large, regenerable +**Content Addressable Storage (CAS)**, served from S3. Phase-by-phase +implementation status and the settled decisions live in +[`REMOTE_STORE_CAS_AC_LOGBOOK.md`](REMOTE_STORE_CAS_AC_LOGBOOK.md). + +## Motivation + +Two goals: + +1. **Regenerable cache.** The set of tarballs aliBuild ships is large and + expensive to store. We want the heavy artifact store to be *derived*: if it + is deleted, it can be reconstructed from a small ledger plus the recipes and + sources. The ledger must therefore record enough about *how* each artifact + was produced to re-run the build. + +2. **Install without build.** It should be possible to materialise a working + installation of a package and its runtime closure directly from the remote + store, with no alidist, no git checkout, and no toolchain — a thin client + that only downloads and relocates. + +## Background: today's store is action-addressed, not content-addressed + +It is tempting to call the current `TARS//store///` layout +a CAS, but it is not. The key is `spec["remote_revision_hash"]` +(`build.py:245`), which is a hash of the *action* — recipe text, version, +package name, git commit, `env`/`append_path`/`prepend_path`/`track_env`, and +every dependency's `hash` (`build.py:146-242`). It is keyed by **what produces +the artifact**, never by the bytes of the artifact. The tarball just happens to +be parked under that action key, and the per-package symlink + `.manifest` layer +is a secondary name→action-key index. + +So, in REAPI terms, aliBuild already has a primitive **Action Cache** and *no* +CAS: + +| REAPI concept | keyed by | aliBuild today | +|-----------------|-------------------------|-------------------------------------------------| +| Action Cache | action / recipe digest | `store//` — exists | +| CAS | hash of the content bytes | does not exist — tarball stored under action key | + +Crucial property: the action hash is a pure function of inputs, so it is stable +across rebuilds. A *content* hash of a tarball is only stable if the build is +bit-for-bit reproducible. Importantly, **the dependency DAG is held together by +action hashes, not content hashes**: a package's action hash folds in each +dependency's `hash` (`build.py:224,228`), i.e. the dependency's *action* hash, +never its bytes. This means reconstruction is robust even when builds are not +reproducible (see below). + +## Three actions + +The redesign makes explicit three operations that share one AC: + +- **build** — recipe + dependency closure → tarball. Keyed by the action hash. + This is what `doBuild` does today. The expensive *producer*. +- **install** — action hash + target prefix → materialised installation. Needs + only the CAS blobs of the runtime closure plus each tarball's self-contained + `relocate-me.sh`. No recipes, no toolchain. The cheap, recipe-free *consumer*. +- **reconstruct** — walk the AC DAG and re-run `build` actions for any missing + CAS blobs, bottom-up. Regenerates the CAS from the AC + recipes + sources. + +`install` and `reconstruct` are duals: one materialises from the CAS, the other +repopulates it. + +## Data model + +### CAS + +Content-addressed by a hash of the bytes (e.g. `sha256`): + +``` +cas/sha256// # tarball bytes and recipe blobs +``` + +Because aliBuild already mints several *equivalent* action hashes per build +(tag aliases; `spec["remote_hashes"]`, `build.py:250`), a content-addressed CAS +stores the bytes once and lets all equivalent action-cache entries point at the +same blob. This dedup only pays off if the tarball bytes are stable, which is +why tarball normalisation (below) is a prerequisite. + +### Action Cache + +One small JSON object per action, keyed by the action hash: + +``` +ac///.json +``` + +```jsonc +{ + "schemaVersion": 2, + "action": { + "package": "ROOT", + "version": "v6-28-04", + "revision": "1", + "architecture": "slc7_x86-64", + "actionHash": "", + "commit": { "ref": "v6-28-04", "commitHash": "abc123…", "altRefs": { } }, + "source": "https://github.com/root-project/root", + "tag": "v6-28-04", + "recipeDigest": "sha256:…", // FULL recipe (header + body) as a CAS blob + "container": { // build environment, null for native builds + "runtime": "docker", + "image": "registry.cern.ch/alisw/slc8-builder:latest", + "digest": "registry.cern.ch/alisw/slc8-builder@sha256:…" + }, + "env": { }, "append_path": { }, "prepend_path": { }, "track_env": { }, + "relocatePaths": [ "…" ], + "deps": [ { "package": "GCC-Toolchain", "actionHash": "…" } ], + "runtimeDeps": [ { "package": "GCC-Toolchain", "actionHash": "…" } ], + "depsHash": "" + }, + "result": { + "tarball": "ROOT-v6-28-04-1.slc7_x86-64.tar.gz", + "outputDigest": "sha256:…", // CAS digest of the tarball bytes + "size": 123456789 + } +} +``` + +Notes: + +- `recipeDigest` (the **full** recipe — header + body, so no alidist checkout is + needed), `commit`, `source`, `deps` and `env`/paths are what make the entry + **reconstructing**: they are the full action definition. `container` records + the build environment (image reference + immutable digest) so reconstruction + can pin it. None of this was persisted before; the action hash was computed on + the fly and discarded. +- `runtimeDeps` is the runtime closure (aliBuild's `full_runtime_requires`, + `build.py:1235`), recorded as action hashes so `install` needs only the AC. + It mirrors the existing `dist-runtime` link tree but keyed by action hash. +- `outputDigest` is **not load-bearing for reconstruction** — deps reference + action hashes, so a rebuilt blob simply gets a new digest and that entry's + `outputDigest` is rewritten. If builds are reproducible it doubles as an + integrity check; if not, it is just "what we shipped last time". + +### Prefix independence (portability) + +Both the AC key (the action hash) and the AC entry's contents are independent of +the build/install prefix, which is what lets the same build in different +directories share one cache entry: + +- The action hash already excludes the prefix — it is foundational to the + shared remote cache, which works across users with different `sw/` locations. + `append_path`/`prepend_path` hold package-relative tokens (`lib`, `bin`); the + absolute prefix is only spliced in as `${PKG_ROOT}/...` at init.sh-generation + time (`build.py:465-468`). `relocate_paths` are package-relative. And + `pruneWorkdirFromPaths`/`pruneVersionEnvVars` (`build.py:523,597`) strip the + workdir and `*_VERSION` from the environment before the build. +- `build_ac_entry` copies only those prefix-independent fields and records deps + by action hash; it never stores `workDir`/`INSTALLROOT`/build-prefix. +- CAS content is relocatable (`relocate-me.sh` + `.unrelocated`), so the prefix + is bound only at install time, not baked into the bytes. + +Caveat: this rests on the recipe convention of not hardcoding an absolute build +path into an `env:` value; doing so would already make today's action hash +prefix-dependent, and the AC merely mirrors whatever the hash sees. + +## Two stores by lifetime: ledger vs artifact + +The content above splits into two lifetimes, and `REAPIRemoteSync` can put them +in two separate stores (buckets): + +- **Ledger store** (small, **keep forever**, back it up): Action Cache entries + (`ac/`) **plus the reconstruction-input blobs** — recipe, source bundles and + refs. This is the precious, reproduce-forever set. +- **Artifact store** (large, **deletable / regenerable**, lifecycle-expirable): + the output tarball blobs (`cas/`) and the legacy `TARS/store` redirects/links. + +Crucially the inputs live with the AC, not with the tarballs: they are needed to +*reconstruct*, so they must outlive the tarballs. (Before the split they shared +one `cas/`, so "delete the CAS" would have taken the recipes/sources too and +broken reconstruction — the split fixes that.) Because a tarball blob and its +legacy redirect are both in the artifact store, the redirect stays same-bucket; +no cross-bucket redirect is needed. + +Config: the artifact store is `--remote-store` (`--write-store`/`::rw` to +upload); the ledger store is the optional `--ac-store` (same `::rw` semantics on +`build`), defaulting to the artifact store so single-bucket setups are unchanged. +Both must share the S3 endpoint (one client; only the bucket differs). +`REAPIRemoteSync` routes by role: `read_ac_entry`/`read_blob`/`download_blob`/ +`put_file_as_blob`/`put_bytes_as_blob`/`read_object_json`/`write_object_json` → +ledger; `put_artifact_blob`/`download_artifact`/`artifact_blob_exists` and the +`TARS` redirect/link → artifact. + +### Artifact retention: ephemeral vs permanent + +The artifact store has a bucket **lifecycle rule** (see +`ali-marathon/s3/alibuild-cas-lifecycle.xml`): objects tagged +`retention=ephemeral` are deleted **90 days after their last-modified time**; +untagged objects and anything tagged `retention=permanent` match no expiry rule +and are kept forever. This is fail-safe — a missing tag never causes deletion. +The **ledger** store is never tagged (always keep-forever); only the large +artifact **tarball blobs** carry a retention tag. + +`--storage {ephemeral,permanent}` (default **ephemeral**) drives it, so the CAS +behaves as an LRU cache by default and production pins what it needs: + +- **ephemeral** (CI/dev): uploaded blobs are tagged `retention=ephemeral`. +- **permanent** (production): blobs are tagged `retention=permanent`, and when a + build *reuses* an existing blob (dedup hit) that is still `ephemeral`, it is + **promoted** to permanent. So a blob first produced by a CI build survives + once a production build depends on it. + +Because a tag-based lifecycle counts from *last-modified*, not last-access, the +reader turns this into true LRU: `download_artifact` **touches** a blob (a +server-side copy-to-self with `TaggingDirective=COPY`, preserving the tag) when +it is `ephemeral` and within `REFRESH_WITHIN_DAYS` (30) of the 90-day expiry. +So a blob that keeps being used never expires, while one unused for ~3 months is +reclaimed. The touch is best-effort and only attempted when the client can write +the same bucket it reads. + +## Reconstruction + +``` +reconstruct(top action hash): + for each action in post-order over the deps DAG: + if CAS has result.outputDigest: continue + fetch recipeDigest blob, check out commit, assemble dependency inputs + re-run the build action → new tarball + put tarball in CAS, update result.outputDigest/size in the AC entry +``` + +Correct regardless of build reproducibility, because the DAG edges are action +hashes. Assumes recipes (in CAS) and source repositories (external, referenced +by commit) are still available. + +## Install + +`install` is a deterministic *client materialisation*, not a cached action — it +produces a prefix-specific result (relocation depends on the target path) that +is cheap to redo, so there is nothing worth caching. It is essentially the +existing cached-tarball branch of `build_template.sh:159-172` (unpack + +`relocate-me.sh` + drop `*.unrelocated`) promoted to a first-class, recipe-free +operation: + +``` +install(label, prefix): + top = resolve label → action hash via the store (latest / version-revision) + for node in {top} ∪ runtime closure (from AC runtimeDeps): + blob = CAS[ AC[node].result.outputDigest ] + unpack blob into prefix + run prefix/.../relocate-me.sh against prefix + generate init.sh / modulefiles +``` + +This makes the remote store *self-describing*: today, to know what to fetch, +aliBuild must recompute action hashes from alidist + git. An AC-driven install +reads the closure and digests straight from the store. + +## Signing and trust (implemented) + +Content addressing gives **integrity** (bytes match their hash) but not +**authenticity**: anyone who can write to the bucket can push a malicious tarball, +compute its digest, and write an AC entry pointing at it — a consumer resolving +`action → outputDigest → blob` would then install it. Signing binds each artifact +to a **trusted builder identity** so `install`/`reconstruct`/fetch can refuse +anything not produced by a trusted key. + +**Threat model.** A writer to the store who is not a trusted builder (leaked +credential, insider, or a push path that bypasses CI). *Not* defended by signing +alone: a trusted builder that is itself compromised — that is what key revocation +and a transparency log are for. + +**What is signed — the AC entry, not the blob.** The AC entry already binds the +whole action (`recipeDigest`, `commit`, dependency action hashes, `container`, +`ALIBUILD_ALIDIST_HASH`, …) to `result.outputDigest`. Sign that and the chain is: + +``` +signature → outputDigest → (consumer re-hashes the downloaded blob) → bytes +``` + +The tarball is never signed directly — content addressing already ties the digest +to the bytes. Sign the *claim*, verify the *bytes hash to the claim*. Use a +**DSSE** envelope (Dead Simple Signing Envelope, the in-toto/sigstore standard): +sign the exact payload bytes + a `payloadType` via PAE, so there is no JSON +canonicalisation to get wrong. Record signatures in the AC JSON, in the +**ledger** store (keep-forever — the right home): + +```jsonc +"signatures": [ { "keyid": "…", "sig": "…", "signer": "alice-ci" } ] +``` + +The signed payload must bind at least `actionHash` + `outputDigest` + +`architecture` + `package`, so a valid signature cannot be replayed onto a +different action. + +**Key custody — via the security-proxy.** Consistent with aliBuild's "real secrets +never touch the build shell" model, the builder's private signing key lives in the +security-proxy: a new `sign` route (Ed25519) that takes a payload and returns a +DSSE signature. The key never appears in the build process, CI logs, or an +operator's shell — the same trust boundary that already re-signs S3 requests. +Alternatives: a KMS/HSM, or **keyless** sigstore via CERN OIDC → Fulcio +short-lived certs + a Rekor transparency log (the SLSA-gold path, a larger lift); +the proxy-held key is the MVP. + +**Trust root.** A keyring of trusted public keys, each with an identity and a +validity window, that consumers verify against. MVP: a keyring file (shipped in +alibuild or alidist), itself signed by a root key so it cannot be tampered with, +bootstrapped from a signed alibuild release. Later: a TUF-managed root for +rotation and a transparency log for auditable, revocable signatures. **Revocation +and bootstrap must be designed up front**, not bolted on: on key compromise you +must be able to distrust a key and re-evaluate everything it signed. + +**Verification.** `install`, `reconstruct` and build-with-fetch verify before +trusting a fetched artifact: fetch AC entry → verify signature(s) against the +trust store per policy → download the CAS blob → check it hashes to the signed +`outputDigest` → proceed, else refuse. Policy modes: `--require-signature` (fail +closed), warn-only, off (default during rollout). Trust is verified over the +**whole runtime/build closure, recursively** — every AC entry in the closure +signed by a trusted key, not just the top package (the easy thing to under-scope). +Local/devel builds (`revision local…`) are exempt or signed by a dev key. + +**Rollout.** Optional → warn → enforced, with unsigned legacy entries still +installable throughout the transition (a mixed store). The default is currently +**`warn`**: every consuming command verifies and reports, but nothing is refused, +so a mixed store keeps working while producers start signing. + +### MVP: phased plan — S0–S3 done, S4 partly done + +Ed25519 keys, DSSE envelopes, key held by the security-proxy, a shipped keyring — +no Fulcio/Rekor/TUF/OPA. Each phase is independently testable and lands behind a +flag, so nothing changed for existing (unsigned) stores until enforcement is +switched on. + +Validated end to end on `osx_arm64` against the production store: a signed build +uploaded (`schemaVersion: 3`, signature verifying against the shipped keyring), +then reinstalled with `--require-signature require` on a machine with no alidist +and no `--trusted-keys`. + +- **Phase S0 (done) — Verify primitives + keyring (pure, no infra).** New + `alibuild_helpers/signing.py`: `dsse_pae(payloadType, payload)` (PAE encoding), + `signed_payload(ac_entry)` (canonical bytes binding `actionHash` + + `outputDigest` + `architecture` + `package`), `load_keyring(path)` (keyid → + {ed25519 pubkey, signer, notBefore/notAfter, revoked}), and + `verify(ac_entry, keyring, policy)`. Ed25519 via `PyNaCl`/`cryptography`. + Pure functions, unit-tested against fixed vectors — signing and verification + fixtures, tamper/expiry/revocation cases. No network. +- **Phase S1 (done) — security-proxy `sign` route.** Add an Ed25519 `sign` route in + `~/src/ali-bot/security-proxy/` (key provisioned into a slot by the human, like + the S3 creds; the build never sees it). Thin client `sign_via_proxy(payload) → + {keyid, sig}` in `signing.py`, plus a way to export the public key so the + keyring can be built from it. Testable against a local/mock proxy. +- **Phase S2 (done) — Sign on upload.** In `REAPIRemoteSync._upload_tarball` (once + `result.outputDigest` is known), build the DSSE payload, sign it via the proxy, + and add `signatures: []` to the AC JSON before writing it to the **ledger**. + Gated on a configured signer (`--sign`); unsigned uploads keep working; skip + `revision local…`. `schemaVersion` → 3 (additive; old readers ignore the field). + validate-system entries sign over `actionHash` + `recipeDigest` + `package` + (no tarball); optional in the MVP. +- **Phase S3 (done) — Verify on consume.** Hook verification into `install.py` + (`collect_runtime_closure`/`install_entry`), `reconstruct`, and the + build-with-fetch path (`fetch_tarball`): verify each AC entry against the + keyring **recursively over the closure**, then confirm the downloaded blob + hashes to the signed `outputDigest`. New args on `add_reapi_store_args`: + `--require-signature` (fail closed) / warn-only (default) / off, and + `--trusted-keys `. Policy MVP: "≥1 trusted, in-window, non-revoked + key." Tests: signed→pass; tampered blob→fail; unsigned+require→fail; untrusted + /expired/revoked key→fail; one unsigned dep in the closure→fail under require. +- **Phase S4 (partly done) — Keyring distribution + rollout.** The keyring now + **ships inside the alibuild package** (`alibuild_helpers/keyring.json`) and is + the trust *anchor*: it arrives with the code the user already executes, i.e. + over a different channel than the store being verified. That is what makes the + recipe-free `install` verify anything at all — it has no alidist to read a + keyring from. `alidist/keyring.json` is merged on top when present, so keys can + be added without cutting an alibuild release, and `--trusted-keys` replaces the + set entirely (testing / air-gapped). + + Merging can only ever **narrow** trust: key ids are self-certifying + (`sha256` of the public key) so the union of keys is conflict-free, validity + windows **intersect**, and revocation lists **union**. Neither source can + un-revoke a key or widen a window the other narrowed — which is what lets + revocation ride on alibuild releases, since builds track the latest alibuild. + + Still to do: signing the keyring itself with a root key (TUF-lite, + verify-keyring-before-use) and, if the keyring is ever served *from* the store, + a version counter + expiry so a stale copy cannot be replayed to resurrect a + revoked key. Then flip the default `warn` → `require` once producers sign; + unsigned legacy entries stay installable in warn mode. + +Deferred to "full" (not MVP): keyless OIDC/Fulcio, Rekor transparency log, +TUF-managed root rotation, OPA/Rego policy — see the infrastructure they require +before adopting. + +## S3 backend + +A new sync backend (sibling of `Boto3RemoteSync` in `sync.py`), selected by the +`reapi://` URL scheme — named after the design (REAPI AC/CAS) rather than the +transport, to distinguish it from the byte-dumb `s3://`/`b3://` backends. It: + +- parameterises the endpoint (`endpoint_url`, region) from the URL / env, so it + works with AWS, MinIO, Ceph RGW and CERN — unlike the current `b3://`, which + hardcodes `s3.cern.ch` (`sync.py:521`); +- implements the existing duck-typed interface (`fetch_symlinks`, + `fetch_tarball`, `upload_symlinks_and_tarball`) for compatibility, and adds AC + read/write plus CAS get/put keyed by content digest; +- on upload: put the tarball into CAS by content digest, the recipe into CAS, + and write the AC entry; keep the `store/`+symlink+`.manifest` views as needed + for the existing publisher / `HttpRemoteSync` during migration. + +## Prerequisite: normalized tarballs + +Content addressing is only useful if identical file trees produce identical +bytes. `build_template.sh` produces normalized, reproducible tarballs: mtimes pinned to +`SOURCE_DATE_EPOCH`, deterministic entry order via a sorted file list, zeroed +ownership, and `gzip -n`. This normalises the *wrapper + metadata* only; +embedded dates / RPATHs / codegen are the separate long tail of true +reproducibility. + +Normalisation is gated by the `NORMALIZE_TARBALL` build-env flag, which +`build.py` sets for every package **except devel packages**: byte-stable output +is useful regardless of the remote store (build-to-build determinism, CAS +dedup, rsync/mirror efficiency, easier debugging), so it is on by default. Devel +packages are excluded because they are never uploaded and normalising would +perturb install-tree mtimes that incremental rebuilds rely on. The extra packing +cost (a full tree-walk to `touch` mtimes + a sorted `find`) is small relative to +compile + `gzip`. A future optimisation could drop the `touch` on modern GNU tar +(>= 1.28) by using `--mtime`/`--sort` directly. + +## Fetch action: content-addressed sources + +Today a build depends on a live `git` checkout of `source@commit`; the source is +not preserved. So `reconstruct` is only as durable as the upstream repo — which +can be force-pushed, retagged, made private or deleted. To make a build fully +**hermetic**, source acquisition becomes its own first-class, content-addressed +action (the REAPI "input root in the CAS" idea): + +``` +fetch(source@commit) ─┐ +recipe blob ──────────┼─► build(action hash) ─► tarball (CAS) +dep tarballs ─────────┤ +container digest ─────┘ +``` + +The build action references the fetch action by hash instead of folding in a +live checkout, so every input (recipe, source, dependencies, container) is +content-addressed and preserved. Payoffs: (1) the same source reused across +recipe revisions, architectures and nearby commits is stored once; (2) rebuilds +no longer depend on upstream git being alive or immutable. + +### Source artifact: base + delta + +A fetch artifact is stored as a **base** plus a **delta**, so near-identical +source trees don't duplicate: + +- **Git sources (preferred):** the source CAS is effectively a shared git + object store — store objects content-addressed, write each commit as a thin + pack against what is already present, and `git repack -ad` periodically. Git's + own delta heuristic chooses delta bases better than any hand-rolled rule, and + dedup across commits/arches/packages is automatic. Reconstruct = fetch base + + thin pack, then checkout the commit. +- **Non-git / tarball sources:** the fetched tarball *is* the artifact, + content-addressed directly; base/delta only applies if we choose to snapshot + evolving trees as base tarball + binary patch (`xdelta`/`bsdiff`). + +### SCM support + +aliBuild abstracts the SCM behind `spec["scm"]` (`SCM` base in `scm.py`, with +`Git` and `Sapling` implementations). The fetch-action implementation is +currently **git-only** (`GitSourceStore` uses git bundles; `apply_refs` uses +`git update-ref`). Capture is gated on `isinstance(spec["scm"], Git)`, so +Sapling packages skip cleanly and fall back to upstream at reconstruct time — +nothing breaks, they're just not yet hermetic. + +What is already SCM-agnostic: the whole CAS layer, and the refs *mapping* itself +(`scm_refs` is produced via `scm.parseRefs`, so `store_refs`/`load_refs` are +generic). Git-specific: bundle create/restore and `apply_refs`. + +Generalisation (follow-up): push the snapshot/restore primitives into the SCM +abstraction — e.g. `scm.snapshotSource`/`scm.restoreSource`/`scm.applyRefs` on +`Git` and `Sapling` — so `source.py` becomes a thin generic driver over +`spec["scm"]` with the CAS layer unchanged. Sapling would use its own +bundle/clone mechanism in place of git bundles. + +### Choosing the base + +The base-selection rule is a **pure storage optimisation, not a correctness +input**: each fetch artifact records the exact `{baseDigest, deltaDigest}` it +used, so reconstruction follows that pointer and never re-derives the rule. The +rule can therefore be heuristic and can change over time without invalidating +anything already stored. + +- Git sources: there is no rule to write — git packing chooses the bases. +- Tarball sources: a simple online greedy rule suffices — **nearest + release-tag ancestor as the base, re-anchor past a size threshold** (e.g. when + a delta exceeds ~50% of the base). This keeps chains depth-1 (base→target, no + long chains), bounding both storage waste and reconstruct cost. Anchoring on + the source's own history (nearest release tag) rather than local fetch order + makes independent builders converge on the same base, maximising dedup; even + if they don't, both artifacts reconstruct correctly — only dedup suffers. + +The single knob (re-anchor threshold) is a space/▵ tradeoff — more bases means +more storage but smaller, faster deltas — tunable from telemetry later. + +## Migration: legacy store → reapi + +Existing releases in the old action-addressed store can be migrated into the +reapi layout *without rebuilding*, and made reconstruct-complete — so the old +tarball can be deleted and still regenerated. The key enabler is that every +tarball already embeds its own provenance in `.meta.json` +(`create_provenance_info`, build.py:510): + +- `alidist.commit` — the exact alidist commit that produced the build; +- `defaults` — the `--defaults` name used; +- `package`: `{tag, source, version, revision, hash}`; +- `dependencies.recursive.{build,runtime}` — the full dependency DAG, each entry + carrying its hash (i.e. exactly the AC `deps`/`runtimeDeps` action hashes). + +So migration is metadata extraction, not archaeology. + +### Per-release migration steps + +An offline `migrate-store` batch, for each old tarball: + +1. **Hash the tarball → CAS blob** (sha256); the bytes are preserved. +2. **Extract `.meta.json`** for the provenance above. +3. **Recover the full recipe** via the recorded `alidist.commit` + (`git show :.sh` against an alidist mirror) → recipe blob in CAS + + `recipeDigest`. alidist is a single, well-preserved repo, far more durable + than the scattered upstream sources. +4. **Synthesize the AC entry** (schema v2): `actionHash` = the old store hash, + `commit`/`source`/`tag`/`defaults` from `.meta.json`, `recipeDigest` from + step 3, `deps`/`runtimeDeps` from the recursive dependency hashes, + `result.outputDigest` = the sha256 from step 1. +5. **(Phase 6) snapshot the source** at `commit` into the source CAS, so even + the upstream repo disappearing doesn't block reconstruction. + +### Self-verification + +Because `storeHashes` is deterministic, migration **recomputes the action hash** +from the recovered recipe + commit + dependency hashes and checks it matches the +old store key. A match proves the action definition was recovered faithfully; a +mismatch (e.g. the recorded alidist commit no longer reproduces that hash) is +flagged rather than written. Migration is thus checked and auditable. + +### Container provenance for legacy builds + +Old builds did not record their container. The migrator supplies one: + +- `--container ` to set it explicitly, else +- the architecture's current default builder image — reusing alibuild's own + derivation (`registry.cern.ch/alisw/[-arm]-builder`, build.py:448-451; + worth factoring into a shared helper so build and migration agree). + +This is a *best guess*, not the original environment, so it is marked +`"provenance": "migration-default"` (vs `"recorded"` for fresh builds) — keeping +the distinction between captured and assumed provenance explicit. This matches +the frozen-release contract: reconstruct produces a valid, equivalent build, not +a bit-identical one, and ALICE's per-architecture builder images are stable +enough that the current default almost always reproduces a functionally +equivalent artifact. + +### Caveats + +- Tarballs without `.meta.json` (pre-provenance) → install-only entries (closure + from `dist-runtime` links, no recipe), flagged non-reconstructible but still + installable / frozen. +- alidist history must reach the recorded commit (normally true; rewritten + history would lose some recipes, flagged at step 3). +- `env`/`relocatePaths` aren't in `.meta.json`, but aren't needed for + reconstruction: a rebuild re-derives them from the recipe + defaults. +- Migrated CAS blobs hash the legacy (non-normalized) bytes, so they won't dedup + against future normalized rebuilds — fine for frozen releases. + +## Status, decisions, and implementation log + +The phase-by-phase implementation status and the (now-settled) design decisions +have moved to [`REMOTE_STORE_CAS_AC_LOGBOOK.md`](REMOTE_STORE_CAS_AC_LOGBOOK.md). +In short: the AC/CAS backend, `install`, `reconstruct` (with `--verify`, +`--rebuild`, `--rebaseline`, `--persist`), content-addressed source + refs +snapshots, validate-system actions, legacy-store `migrate`, and signing +(S0–S3 plus keyring distribution) are implemented. **The main outstanding items +are enforcement rollout (`warn` → `require`, which needs producers — notably CI — +to sign), a root-signed keyring, and Sapling source snapshots.** diff --git a/REMOTE_STORE_CAS_AC_LOGBOOK.md b/REMOTE_STORE_CAS_AC_LOGBOOK.md new file mode 100644 index 00000000..c232b009 --- /dev/null +++ b/REMOTE_STORE_CAS_AC_LOGBOOK.md @@ -0,0 +1,275 @@ +# Logbook: S3 remote store (Action Cache + CAS) + +Running record of *what was built and decided when* for the store redesign, and +the (now-settled) decisions. The timeless design lives in +[`REMOTE_STORE_CAS_AC.md`](REMOTE_STORE_CAS_AC.md); this file is kept separate so +the design doc stays readable. + +## Open decisions + +- **Compatibility scope:** keep back-compat with the `TARS`/publisher layout and + add `ac/`+`cas/` alongside, or go greenfield. Reconstruction is cleaner + greenfield but is not readable by today's HTTP frontend without a shim. +- **CAS hash algorithm:** `sha256` (REAPI default) vs reuse of the existing + SHA-1 machinery. +- **Install surface:** standalone `aliBuild install @` with zero + alidist dependency (preferred), factoring the unpack+relocate step out of + `build_template.sh` so build and install share one implementation. +- **Label entry points:** which names are installable (latest, version-revision, + named nightly tags) and how they are recorded in the store. + +## Implementation plan + +Settled decisions: **additive** layout (keep `TARS/store`+symlink+manifest; +add `cas/`+`ac/`; the legacy `store/` object becomes an S3 redirect to the CAS +blob, so bytes are stored once); **CAS digest = sha256**, AC key = the existing +sha1 action hash; **`reapi://`** URL scheme; install is a new alidist-free +subcommand; labels reuse the existing `version-revision`/`latest` resolution. + +- **Phase 0 — Normalized tarballs.** Done (`build.py`, `build_template.sh`, + regression test). +- **Phase 1 — Path helpers + AC entry assembly (pure, no S3).** + `utilities.py`: `resolve_cas_path(algo, h)` → `cas///`, + `resolve_ac_path(arch, h)` → `ac///.json`, and a sha256 + `file_digest` helper. `build.py`: assemble `spec["ac_entry"]` from existing + spec fields before upload, keeping the backend dumb. Tests in + `test_utilities.py` and `test_build.py`. +- **Phase 2 — `reapi://` backend, write path.** New backend in `sync.py` + (generic endpoint), registered in `remote_from_url`. On upload: sha256 the + tarball, put to CAS (skip if present → dedup), put the recipe blob, put the AC + JSON, write the legacy `store/` object as a redirect. Tests extend + `test_sync.py`. +- **Phase 3 — Read path.** `fetch_tarball` resolves action hash → AC → + `outputDigest` → CAS blob into the local store path, with legacy-`store/` + fallback. `fetch_symlinks` unchanged. +- **Phase 4 — `aliBuild install`.** Done. `alibuild_helpers/install.py` adds an + alidist-free subcommand that resolves a label to an action hash (via the + per-package symlink objects), reads the AC runtime closure, fetches each CAS + blob, extracts it into the prefix and runs the in-tarball `relocate-me.sh` + (the relocation logic is not reimplemented -- it ships in the tarball). The + package's own `init.sh` (already inside the tarball) wires the environment. + REAPIRemoteSync gained `read_ac_entry`, `download_blob` and + `resolve_action_hash` read helpers. +- **Phase 5 — `aliBuild reconstruct`.** Done. `alibuild_helpers/reconstruct.py` + walks the AC build closure post-order, finds tarballs missing from the CAS, + and materialises the archived full recipes into a self-contained alidist + directory (surfacing the recorded build container so the env can be pinned). + The actual rebuild reuses the normal build: running the emitted + `aliBuild build … --remote-store reapi://…::rw` against the materialised + config recomputes the same action hashes, rebuilds the missing packages and + re-uploads them (writing fresh CAS blobs + updated `outputDigest`s). + Prerequisite, also done: the AC now archives the **full** recipe (parseRecipe + retains `fullRecipe`) plus `source`/`tag`/`container`, so reconstruction needs + no alidist checkout. + + `reconstruct --verify` (done): a read-only pre-flight that prints the + reconstruction *plan* for a package's closure — which tarballs would be + **reused** from the CAS (blob present) vs **rebuilt** (blob missing) — and + checks the ledger can actually rebuild the missing ones: recipe blob present + + integrity-verified (sha256 == recipeDigest), dependency DAG intact, and source + archived/upstream/none. It certifies the key property that reconstruction is + incremental — present dependencies (toolchains included) are reused, never + recompiled — and flags any missing tarball that is *not* regenerable. Rebuilds + nothing. + + `reconstruct --verify --rebuild` (done): the content-hash capstone. It + materialises the recipes, restores the target's source, then rebuilds **only the + target** via the normal build against a *read-only* store (`--force-rebuild + PACKAGE`, no `::rw`) so every dependency is fetched and reused from the CAS and + nothing is uploaded; it then hashes the produced tarball and compares to the + recorded `outputDigest`. A **match** proves the blob is byte-for-byte + regenerable from the ledger; a **differ** is reported soft (the rebuild is + valid but not bit-identical — expected for pre-normalisation legacy tarballs) + and only fails under `--strict`. The `--force-rebuild` changes the target's + action hash but not the content compared, and only the target is forced, so the + "don't rebuild the toolchain" guarantee holds. Runs in an isolated workdir; the + real store is never written. + + `reconstruct --rebaseline` (done): adopt a legacy tarball's reproducible hash as + the new recorded one, so future verifies are byte-identical instead of a + perpetual soft `differ`. It implies `--verify --rebuild`; when the rebuild + *differs* from the recorded (pre-normalisation) hash, it rewrites the target's AC + entry `outputDigest` — plus the store redirect and per-package link — to point at + the rebuilt hash, keyed by the **unchanged action hash** (an in-place pointer + swap: `REAPIRemoteSync.rebaseline_ac_entry`, reusing `migrate_put`). The new CAS + blob is written **before** the AC is repointed, so a failure never leaves the + entry dangling (unlike a manual delete-then-rebuild, which has a window where the + ledger points at nothing); the retention of the replaced blob is preserved + (untagged == permanent). It is a **dry run** that only prints the plan unless + `--apply` is given, and leaves the now-orphaned old blob in place unless + `--delete-old` is also passed. This is the supported way to normalise legacy + entries in bulk; because the rebuild is deterministic (two runs give the same + hash), the re-baseline target is a fixed point. A `match` rebuild is a no-op. + Note it discards the historical `outputDigest` provenance by design — the ledger + then records the normalised rebuild, not what was originally distributed. + + `reconstruct --persist` (done): the correct way to put a **deleted** artifact blob + back. It implies `--verify --rebuild`, and when the isolated rebuild reproduces the + recorded output digest exactly, it uploads **only that CAS blob** at its + content-addressed key (`put_artifact_blob`, `--storage` retention, default + `permanent`). It does **not** go through `aliBuild build ...::rw`: that path + *re-publishes* — it assigns a fresh revision (a rebuild in a polluted workdir came + out as `-1`, whose tarball hashes differently than the recorded `-6`, so it would + restore the *wrong* blob) and writes a new `dist`/`dist-direct`/`dist-runtime` + publisher graph. A content-addressed restore needs none of that: the AC entry, + legacy store redirect and per-package links still point at the recorded hash (only + the blob was gone), so putting the bytes back at `cas///` is the + whole operation. It **refuses a `differ` rebuild** (that blob is unreferenced -- + re-baseline instead), is a no-op when the blob is already present, and is a **dry + run** unless `--apply`. The rebuild runs in an isolated workdir against a read-only + store, so revision resolution is stable (reproduces the recorded revision) and the + build itself uploads nothing. Validated end to end: delete a CAS blob, `--verify` + flips it to `REBUILD`, `--persist --apply` restores the byte-identical blob, and a + final `--verify` shows `0 to rebuild`. + + **validate-system actions (done).** A satisfied `system_requirement` (make, + yacc-like, ...) or `prefer_system`-from-host package produces no tarball but *is* + an action: its check must run to validate the build host. `getPackageList` + collects every such package into `systemPackageSpecs` -- for **both** categories + and regardless of `--no-system` (a required system tool must be validated even + when everything else is built from source), which `--remote-store` sets -- and + stamps each package's `system_requires` (the satisfied system deps it dropped from + its build requires). A build then: + - writes a **`validate-system`** AC entry per system action + (`build_validate_system_entry` + `put_ac_entry`): the archived recipe (with its + check), **no `result` tarball**, keyed by the **recipe digest** + (`system_recipe_digest`) -- content-addressed, so the same system tool required + by many packages deduplicates to one entry; + - references those system deps in each dependent's AC entry `deps`, **by the same + recipe digest** (`build_ac_entry` + `system_specs`), so the dependent resolves to + the validate-system node. + + reconstruct then **walks to** the node via `deps`, `materialize_recipes` writes its + recipe, and the rebuild re-runs the check on the host -- self-contained, no + `--alidist` needed. `find_missing_blobs` skips them (no artifact); `--verify` + reports them as `system`. Deliberate scope: the system dep is recorded for + walking/materialising/revalidating but **not folded into the dependent's action + hash** (the check is host-dependent by nature, and the built deps that *are* + hashed are unchanged, so nothing rebuilds). + + **Migrate-side population (done, automatic).** Migration gives its entries the same + validate-system nodes (`populate_system_deps`): for each migrated build entry it + reads the archived recipe, finds requires that are `system_requirement` packages + (recovered from `--alidist`), writes a validate-system entry per such dep and + references it in the entry's `deps` -- exactly what a fresh build does. This runs as + a **standard pass on every migration** (not opt-in); `migrate --populate-system` is + the bulk-retroactive form that walks the *whole* ledger to backfill entries migrated + before the feature existed. Idempotent, no rebuild. Net: the whole store (fresh + + migrated) is self-reconstructable for system deps -- the `--alidist` bridge is no + longer needed at reconstruct time. + + Known limitations (next hardening): the build invocation is currently emitted + for the user to run rather than auto-executed; faithful rebuilds assume the + recorded `tag` still resolves to `commit.commitHash` (true for release tags, + not for moving branches — explicit commit pinning is a follow-up); and the + original `--defaults` name is not recorded (the materialised config uses + `--defaults release`, since `defaults-release` is in the closure). + +- **Phase 6 — Fetch action / content-addressed sources.** Core done. + `alibuild_helpers/source.py` provides `GitSourceStore` over the reapi backend: + `snapshot(repo, source, commit)` stores the source as an **incremental chain of + thin git bundles** — a one-off full base bundle (first snapshot of a repo, or + after a re-baseline at `MAX_CHAIN`), then a tiny delta per commit, each thin + against the repo's previous snapshot via a rolling per-repo head. So a stream + of close commits (e.g. **daily builds of O2Physics**) shares one base and stores + only its per-commit delta instead of duplicating the whole source each day; the + artifact records the ordered `segments` to restore. `restore(entry, dest)` + fetches and applies that chain to rematerialise the exact checkout offline, and + **falls back to cloning upstream on any chain failure** — under normal + conditions upstream is available, so the snapshot is a backup + fetch speedup, + not the sole source of truth. Bundle chains can only be built/thinned with the + full commit graph, so shallow clones are unusable (a shallow `git bundle create` + produces an invalid bundle); the legacy `snapshot_legacy_source` therefore uses + a **partial** (`--filter=tree:0`/`blob:none`) mirror — the same filter aliBuild + already uses for reference/source clones — so it does not ingest a package's + whole history to snapshot one commit. Backed by `REAPIRemoteSync` helpers + (`put_file_as_blob`, `read_object_json`, `write_object_json`). Verified by + real-git round-trip tests: incremental chain (big base + tiny deltas, one shared + base), idempotency, multi-segment restore with the upstream wiped, and the + upstream fallback. + + Wiring done: `doBuild` calls `snapshot_source` at upload time (gated on a + reapi write store, non-devel, git source; best-effort — it never breaks a + build, snapshotting from the full reference mirror `spec["reference"]`), and + records the artifact in the AC as `action.sourceArtifact`. `reconstruct` + restores archived sources from the CAS into a reference-sources layout and + adds `--reference-sources` to the emitted build command. + + Refs artifact (done): a rebuild has two upstream touchpoints — resolving + tags->commits (`git ls-remote` at build.py:69) and fetching source objects. + The second is the source artifact above; the first is now a **refs artifact**: + `store_refs`/`load_refs` archive the `scm_refs` ref->commit mapping as a small + CAS blob (`build.py` `snapshot_refs` captures it into `action.refsArtifact`), + and `reconstruct` calls `apply_refs` to recreate the original tag refs in the + restored repo so `ls-remote` against it resolves tags offline. Verified with + real-git tests. + + Source-aware checkout (done, for fresh git builds): `reconstruct` now + pre-populates the build's `SOURCES///` from the source + artifact (`GitSourceStore.restore_to_source_dir`, replicating + `short_commit_hash`) and applies the cached tags, then emits `-w `. + At rebuild, `checkout_sources` takes its `isdir` branch and checks out the tag + **locally**, never cloning the upstream URL; combined with the restored + reference repo (for the `ls-remote` at build.py:69) the rebuild contacts + upstream for nothing. Crucially this does **not** touch `spec["source"]`, + which would change the action hash (build.py:216). Proven by an integration + test that runs the real `checkout_sources` against an unreachable upstream URL + and still checks out. No edit to the build hot path was needed. + + Migrated legacy releases (done): `aliBuild migrate --snapshot-sources` clones + each release's source (once per package, cached under `--source-mirror`), + resolves the tag to a commit, and archives source + refs into the CAS, setting + `commit.ref` to the resolved SHA so the source-aware checkout path matches at + rebuild. Best-effort: if upstream is already gone the release still migrates, + just without offline source. So both fresh builds and migrated legacy releases + are now offline-reconstructible (delete the tarball *and* lose the upstream). + + Enriching an already-migrated release (done): re-running with + `--snapshot-sources` does *not* redo the migration. A fully-migrated package is + enriched in place from the Action Cache: the AC entry already records the + upstream git URL and tag, so `enrich_source_snapshot` clones upstream, + snapshots source + refs into the ledger, and rewrites just the AC entry — no + tarball re-download and no CAS write. Idempotent (a second run is a no-op once + the snapshot exists) and cheap (only the unavoidable per-package upstream + clone), so sources can be back-filled long after the initial migration. The + build container is intentionally *not* pinned by digest: it is provenance only + (not part of the action hash), so patching/deleting builder images never + invalidates the AC/CAS, and reconstruct falls back from digest to tag. + + Remaining: Sapling sources are still git-only (skip cleanly). + +- **Phase 7 — `migrate` (migrate-store).** Core done. + `alibuild_helpers/migrate.py` turns legacy tarballs into reconstruct-complete + reapi entries: `read_meta_json` extracts the embedded provenance, + `recover_recipe` recovers the full recipe from the recorded alidist commit + (`git show`), `ac_entry_from_meta` synthesises the schema-v2 AC entry (deps + from the recorded recursive dependency hashes), and `REAPIRemoteSync.migrate_put` + writes the CAS blob + recipe blob + AC entry + legacy redirect + link (so the + migrated release is installable and publisher-compatible). The build container + is supplied via `--container` or the architecture's default builder (shared + `default_builder_image` helper), marked `"provenance": "migration-default"`. + `aliBuild migrate TARBALL... --alidist DIR --remote-store reapi://...`. + + Inputs: tarballs can be local paths, or `PACKAGE/VERSION-REVISION` specs + fetched from a **read-only** old HTTP store via `--read-store` (the old store + is only ever read, never written). `--dry-run` prints the planned migration + actions (and the URLs it would fetch) without downloading or writing anything. + + Self-verification: a **structural** self-check is wired (`verify_recovered_recipe`, + on by default, `--no-verify` to skip) — it confirms the recovered recipe + parses, its package field matches the metadata, and every recorded dependency + carries a hash, skipping (not writing) entries that fail. This catches the + realistic failure modes (wrong/renamed recipe, corrupt metadata, missing dep + hash) without the false-mismatch risk of a half-done hash recompute. + + Follow-ups: full self-verification by **recomputing the action hash** and + matching the store key (needs replaying defaults + scm_refs — alibuild's + planning phase; best done as a verify mode of `reconstruct`, which already + materialises the closure); tarballs without `.meta.json` are skipped + (install-only fallback TBD); enumerating an old S3 store (vs. taking tarball + paths) is a driver convenience. + +Backbone is 1→2→3 (all testable with mocked S3); 4 and 5 are separable and land +last. Phase 5 is riskiest. Phases 6 (source durability) and 7 (migration) are +separable upgrades that together make legacy releases reconstructible without +their original tarballs or upstreams. Docs: update `docs/docs/user.md`. diff --git a/aliBuild b/aliBuild index 9b173ad7..6780c490 100755 --- a/aliBuild +++ b/aliBuild @@ -17,6 +17,9 @@ from alibuild_helpers.deps import doDeps from alibuild_helpers.log import info, debug, logger, error from alibuild_helpers.utilities import detectArch from alibuild_helpers.build import doBuild +from alibuild_helpers.install import doInstall +from alibuild_helpers.reconstruct import doReconstruct +from alibuild_helpers.migrate import doMigrate from alibuild_helpers.completion import doCompletion @@ -83,6 +86,15 @@ def doMain(args, parser): doBuild(args, parser) sys.exit(0) + if args.action == "install": + sys.exit(0 if doInstall(args, parser) else 1) + + if args.action == "reconstruct": + sys.exit(0 if doReconstruct(args, parser) else 1) + + if args.action == "migrate": + sys.exit(0 if doMigrate(args, parser) else 1) + if __name__ == "__main__": args, parser = doParseArgs() diff --git a/alibuild_helpers/args.py b/alibuild_helpers/args.py index e77be942..c6d5d306 100644 --- a/alibuild_helpers/args.py +++ b/alibuild_helpers/args.py @@ -55,6 +55,13 @@ def doParseArgs(): description="Verify the status of your system.") init_parser = subparsers.add_parser("init", help="initialise local packages", description="Initialise development packages.") + # The reapi:// subcommands register their own parsers from their implementation + # modules (args live with the command), so this shared parser stays free of their + # options. Order is preserved for --help listing. + from alibuild_helpers import install, reconstruct, migrate + install.add_parser(subparsers, detectedArch, DEFAULT_WORK_DIR) + reconstruct.add_parser(subparsers, detectedArch, DEFAULT_WORK_DIR) + migrate.add_parser(subparsers, detectedArch, DEFAULT_WORK_DIR) version_parser = subparsers.add_parser("version", help="display %(prog)s version", description="Display %(prog)s and architecture.") completion_parser = subparsers.add_parser("completion", help="output shell completion code", @@ -142,6 +149,14 @@ def doParseArgs(): 'https://s3.cern.ch/swift/v1/alibuild-repo'. It requires no credentials and provides tarballs for the most common supported architectures. """) + build_parser.add_argument("--plan", dest="plan", action="store_true", + help=("Resolve the build and print what it WOULD do -- for each " + "package the version-revision it lands on, its hash, and " + "whether it comes from the remote store or gets built -- then " + "stop. Unlike -n/--dry-run, which returns before any of that " + "is known, this runs the real resolution and so reads the " + "remote store. The 'reuse' lines are PACKAGE/VERSION-REVISION " + "specs, which is what `aliBuild migrate` consumes.")) build_remote.add_argument("--no-remote-store", action="store_true", help="Disable the use of the remote store, even if it is enabled by default.") build_remote.add_argument("--remote-store", dest="remoteStore", metavar="STORE", default="", @@ -156,6 +171,70 @@ def doParseArgs(): "except ::rw is not recognised. Implies --no-system.")) build_remote.add_argument("--insecure", dest="insecure", action="store_true", help="Don't validate TLS certificates when connecting to an https:// remote store.") + build_remote.add_argument("--ac-store", dest="acStore", metavar="STORE", default="", + help=("For reapi:// stores, a separate ledger store for the " + "Action Cache and reconstruction inputs (recipe/source/refs), " + "which are kept while the artifact tarballs are deletable. " + "Same ::rw syntax as --remote-store. Defaults to --remote-store.")) + build_remote.add_argument("--legacy-links-store", dest="legacyStore", metavar="STORE", + default="", + help=("For reapi:// stores, where to publish the legacy " + "TARS// link and store objects, if not in the " + "artifact store. Use this to keep an existing repo " + "browsable by consumers that only know that layout, " + "while the bytes stay content-addressed. The store " + "objects then redirect to an absolute CAS URL, which " + "needs an aliBuild new enough to follow one.")) + build_remote.add_argument("--cas-public-url", dest="casPublicUrl", metavar="URL", + default="", + help=("Public base URL of the CAS bucket, e.g. " + "https://s3.cern.ch/swift/v1/alibuild-cas. Required " + "with --legacy-links-store: the legacy store objects " + "redirect there by absolute URL, and the endpoint a " + "build uploads through (a proxy, say) is not " + "necessarily reachable by consumers.")) + build_remote.add_argument("--storage", dest="storage", choices=("ephemeral", "permanent"), + default="ephemeral", + help=("Retention for uploaded reapi:// tarball blobs: 'ephemeral' " + "(default; LRU-expired by the bucket lifecycle, refreshed on " + "use) or 'permanent' (pinned; also promotes any ephemeral blob " + "it reuses). Use 'permanent' for production builds.")) + build_remote.add_argument("--no-snapshot-sources", dest="no_snapshot_sources", + action="store_true", + help=("Don't archive built packages' git sources into a reapi:// " + "ledger. Source archival is best-effort and skipped for partial " + "(blobless/treeless) mirrors anyway; use this to skip it entirely " + "(reconstruct then relies on upstream git for sources).")) + build_remote.add_argument("--sign-url", dest="signUrl", metavar="URL", default="", + help=("Endpoint of the security-proxy sign route used to sign " + "Action Cache entries uploaded to a reapi:// store (e.g. " + "https:///sign/alibuild-ac). Uploading to a reapi:// " + "store signs by default and requires this unless --no-sign.")) + build_remote.add_argument("--sign-token", dest="signToken", metavar="TOKEN", default="", + help="Gate token presented to --sign-url. Prefer --sign-token-file " + "for short-lived credentials.") + build_remote.add_argument("--sign-token-file", dest="signTokenFile", metavar="FILE", default="", + help=("Read the credential for --sign-url from FILE, freshly for " + "every signing request. Use this for short-lived tokens that " + "are refreshed in place -- e.g. a Nomad workload-identity JWT, " + "whose TTL is minutes while a build signs entries for hours.")) + build_remote.add_argument("--signer", dest="signer", metavar="NAME", default="alibuild", + help=("Human-readable signer label recorded in the signature " + "(the keyid is authoritative). Default '%(default)s'.")) + build_remote.add_argument("--no-sign", dest="noSign", action="store_true", + help=("Upload to a reapi:// store without signing Action Cache " + "entries. By default uploads are signed and refused if " + "--sign-url/--sign-token are not given.")) + build_remote.add_argument("--require-signature", dest="requireSignature", + choices=("off", "warn", "require"), default="warn", + help=("Verify signatures on prebuilt reapi:// tarballs reused " + "during a build, against --trusted-keys: 'warn' (default; " + "log unverified), 'off' (skip), or 'require' (fail closed).")) + build_remote.add_argument("--trusted-keys", dest="trustedKeys", default="", metavar="KEYRING", + help=("Path to the JSON keyring of trusted signing keys. This " + "REPLACES the defaults, which are the keyring shipped with " + "alibuild merged with keyring.json from the alidist " + "checkout (if any).")) build_dirs = build_parser.add_argument_group(title="Customise aliBuild directories") build_dirs.add_argument("-C", "--chdir", metavar="DIR", dest="chdir", default=DEFAULT_CHDIR, @@ -357,7 +436,7 @@ def doParseArgs(): def optionOrder(x): if x in ["--debug", "-d", "-n", "--dry-run"]: return 0 - if x in ["build", "init", "clean", "analytics", "doctor", "deps", "completion"]: + if x in ["build", "init", "clean", "analytics", "doctor", "deps", "completion", "install", "reconstruct", "migrate"]: return 1 return 2 rest.sort(key=optionOrder) @@ -491,6 +570,12 @@ def finaliseArgs(args, parser): args.remoteStore = args.remoteStore[0:-4] args.writeStore = args.remoteStore + # The optional ledger store mirrors --remote-store's ::rw semantics. + args.acWriteStore = "" + if getattr(args, "acStore", "").endswith("::rw"): + args.acStore = args.acStore[0:-4] + args.acWriteStore = args.acStore + if args.action in ["build", "init"]: if "develPrefix" in args and args.develPrefix is None: if "chdir" in args: diff --git a/alibuild_helpers/build.py b/alibuild_helpers/build.py index b9938161..6882a151 100644 --- a/alibuild_helpers/build.py +++ b/alibuild_helpers/build.py @@ -17,6 +17,8 @@ from alibuild_helpers.sl import Sapling from alibuild_helpers.scm import SCMError from alibuild_helpers.sync import remote_from_url +from alibuild_helpers.sync_reapi import REAPIRemoteSync, signature_checker +from alibuild_helpers.source import GitSourceStore, store_refs from alibuild_helpers.workarea import logged_scm, updateReferenceRepoSpec, checkout_sources from alibuild_helpers.log import ProgressPrint, log_current_package from glob import glob @@ -25,6 +27,7 @@ import tempfile import concurrent.futures +import hashlib import importlib import json import socket @@ -110,6 +113,241 @@ def update_repo(package, git_prompt): # Creates a directory in the store which contains symlinks to the package # and its direct / indirect dependencies +def build_ac_entry(spec, specs, architecture, container=None, source_artifact=None, + refs_artifact=None, system_specs=None): + """Assemble the Action Cache (AC) entry for a freshly built package. + + The entry records what produced the tarball -- the full recipe, git commit, + source, dependency action hashes, build environment and (if any) the build + container -- so that the content-addressed store (CAS) can later be + reconstructed from the (small) action cache, and so that a package can be + installed from its runtime closure without a build. See REMOTE_STORE_CAS_AC.md. + + `container` is an optional dict describing the build container (image + reference + immutable digest); None for native builds. + + The action hash is spec["remote_revision_hash"]; this entry is keyed by it. + Dependencies are referenced by their *action* hash (specs[dep]["hash"]), which + is what the action hash itself folds in (see storeHashes()), so the recorded + DAG matches the hash and is independent of whether builds are byte-reproducible. + + result.outputDigest / result.size are filled in by the remote backend at + upload time, once the tarball bytes exist and have been hashed. + """ + # Resolve the real git commit and the tag aliases pointing at it, mirroring + # storeHashes() so the recorded provenance matches the computed action hash. + scm_refs = spec.get("scm_refs", {}) + real_commit_hash = scm_refs.get("refs/tags/" + spec["commit_hash"], + spec["commit_hash"]) + alt_refs = {ref[len("refs/tags/"):]: git_hash + for ref, git_hash in scm_refs.items() + if ref.startswith("refs/tags/") and git_hash == real_commit_hash} + + def dep_refs(dep_names): + return [{"package": dep, "actionHash": specs[dep]["hash"]} + for dep in sorted(dep_names)] + + # System dependencies (make, yacc-like, a prefer_system package taken from the + # host, ...) produce no tarball, so they are not in specs and not in build_requires; + # but they ARE validate-system actions. Reference them here -- by their recipe + # digest, matching their validate-system entry's action hash -- so reconstruct + # walks to them, materialises the recipe and re-runs the check on the host. + system_specs = system_specs or {} + def system_refs(dep_names): + return [{"package": dep, "actionHash": system_recipe_digest(system_specs[dep])} + for dep in sorted(dep_names) if dep in system_specs] + + # Archive the full recipe (header + body), so the build can be reconstructed + # without an alidist checkout. spec["recipe"] is only the build body. + recipe_text = spec.get("fullRecipe") or spec.get("recipe") or "" + recipe_digest = hashlib.sha256(recipe_text.encode("utf-8", "ignore")).hexdigest() + + return { + "schemaVersion": 2, + # Outside "action": provenance, not identity. Here rather than in the tarball + # because both vary independently of the action hash, which would make + # identical builds produce different bytes. Advisory: not covered by the + # signature. + "provenance": { + "alibuildVersion": __version__ or "unknown", + "alidistCommit": os.environ.get("ALIBUILD_ALIDIST_HASH", ""), + }, + "action": { + "package": spec["package"], + "version": spec["version"], + "revision": spec["revision"], + "architecture": architecture, + "actionHash": spec["remote_revision_hash"], + "commit": { + "ref": spec["commit_hash"], + "commitHash": real_commit_hash, + "altRefs": alt_refs, + }, + "source": spec.get("source"), + "tag": spec.get("tag"), + "recipeDigest": "sha256:" + recipe_digest, + "container": container, + "sourceArtifact": source_artifact, + "refsArtifact": refs_artifact, + "env": dict(spec.get("env") or {}), + "append_path": dict(spec.get("append_path") or {}), + "prepend_path": dict(spec.get("prepend_path") or {}), + "track_env": dict(spec.get("track_env") or {}), + "relocatePaths": sorted(spec.get("relocate_paths", [])), + "deps": dep_refs(spec.get("full_requires", [])) + + system_refs(spec.get("system_requires", [])), + "runtimeDeps": dep_refs(spec.get("full_runtime_requires", [])), + "depsHash": spec.get("deps_hash", ""), + }, + } + + +def system_recipe_digest(spec): + """The sha256 of a system package's full recipe. It is both the recipeDigest and + the *action hash* of the package's validate-system node: content-addressed, so a + dependent and the node itself compute the same value and the dependent's deps + reference resolves, and equal recipes (the same system tool required by many + packages) deduplicate to a single entry. + + For a replaced package (prefer_system_replacement_specs) fullRecipe is the selected + replacement rendered as a recipe, so the digest covers what was adopted.""" + recipe_text = spec.get("fullRecipe") or spec.get("recipe") or "" + return hashlib.sha256(recipe_text.encode("utf-8", "ignore")).hexdigest() + + +def publish_validate_system_entries(syncHelper, systemPackageSpecs, specs, + architecture): + """Record validate-system actions for the system / prefer_system packages. + + The Action Cache has to hold the full recipe closure: reconstruct materialises + these recipes (dependency resolution needs them) and re-validates the system + requirement on the target host, with no alidist checkout. + + Written before the first package is uploaded, not after the build: packages are + published as they are built, and assert_deps_in_ledger refuses to publish one + whose dependencies are absent -- which every dependent of a system package is, + until these exist. Done once per run; the entries are keyed by recipe digest and + are the same for every dependent. + """ + # A read-only store -- reconstruct's isolated rebuild, say -- writes nothing. + if not getattr(syncHelper, "writeStore", "") or \ + getattr(syncHelper, "_validate_system_published", False): + return + for _, sysspec in sorted(systemPackageSpecs.items()): + recipe_text = sysspec.get("fullRecipe") or sysspec.get("recipe") or "" + syncHelper.put_ac_entry(build_validate_system_entry(sysspec, specs, architecture), + recipe_text, sign=True) + syncHelper._validate_system_published = True + + +def build_validate_system_entry(spec, specs, architecture): + """Assemble a 'validate-system' Action Cache entry for a satisfied system / + prefer_system package -- a closure node that produces no tarball. It archives the + recipe (with its system_requirement/prefer_system check) so reconstruct can + materialise it and re-validate on the target host. Keyed by the recipe digest (see + system_recipe_digest), which is what dependents reference in their deps.""" + recipe_digest = system_recipe_digest(spec) + return { + "schemaVersion": 2, + "action": { + "kind": "validate-system", + "package": spec["package"], + "version": spec.get("version"), + "revision": spec.get("revision") or "1", + "architecture": architecture, + "actionHash": recipe_digest, + "recipeDigest": "sha256:" + recipe_digest, + # What this node validates, readable without fetching the recipe. Immutable: + # part of the recipe the entry is keyed by. + **{key: spec[key] for key in ("system_requirement_check", "prefer_system_check") + if spec.get(key)}, + "deps": [], + }, + } + + +def snapshot_source(spec, syncHelper): + """Best-effort: archive the package's git source into the CAS so the build is + reconstructible without upstream git. Returns a source-artifact dict, or None + when not applicable (non-reapi store, devel/source-less package) or on any + failure -- source archival must never break a build. Snapshots from the reference + mirror (spec["reference"]); the bundler backfills a partial mirror first.""" + if not isinstance(syncHelper, REAPIRemoteSync) or not getattr(syncHelper, "writeStore", ""): + return None + if spec.get("is_devel_pkg") or "source" not in spec or "reference" not in spec: + return None + # The source store is git-only for now; other SCMs (Sapling) skip cleanly and + # fall back to upstream at reconstruct time. See REMOTE_STORE_CAS_AC.md. + if not isinstance(spec.get("scm"), Git): + return None + try: + # Resolve the tag/branch label to a commit SHA. `git rev-parse ^{commit}` + # needs the tag *ref* to exist in the local clone -- which it does for a normal + # build off a full mirror, but NOT for a reconstruct rebuild whose source is a + # restored working checkout (no tag refs). Prefer the archived ref map, then the + # commit actually checked out for this build, and only then rev-parse. snapshot() + # is idempotent per commit, so an already-archived source re-uploads nothing and + # its reference is preserved in the regenerated AC entry. + commit_hash = spec["commit_hash"] + scm_refs = spec.get("scm_refs") or {} + commit = (scm_refs.get("refs/tags/" + commit_hash) or + scm_refs.get("refs/heads/" + commit_hash)) + if not commit: + try: + commit = spec["scm"].checkedOutCommitName(directory=spec["source"]) + except SCMError: + commit = git(("rev-parse", commit_hash + "^{commit}"), + directory=spec["reference"]).strip() + return GitSourceStore(syncHelper).snapshot(spec["reference"], spec["source"], commit) + except Exception as exc: # pylint: disable=broad-except + warning("Could not snapshot source for %s, it will not be reconstructible " + "without upstream git: %s", spec["package"], exc) + return None + + +def snapshot_refs(spec, syncHelper): + """Best-effort: archive the package's ref->commit mapping (scm_refs) into the + CAS, so tag resolution at reconstruct time needs no upstream `git ls-remote`. + Returns a refs-artifact dict or None. Gated like snapshot_source().""" + if not isinstance(syncHelper, REAPIRemoteSync) or not getattr(syncHelper, "writeStore", ""): + return None + if spec.get("is_devel_pkg") or not spec.get("scm_refs"): + return None + # apply_refs (reconstruct side) is git-only, so only archive refs for git. + if not isinstance(spec.get("scm"), Git): + return None + try: + return store_refs(syncHelper, spec.get("source"), spec["scm_refs"]) + except Exception as exc: # pylint: disable=broad-except + warning("Could not snapshot refs for %s: %s", spec["package"], exc) + return None + + +def resolve_container_provenance(args): + """Best-effort capture of the build container for reproducibility: its + reference ("location") and immutable digest ("hash"). Returns None for native + (non-container) builds, and leaves digest None if it cannot be resolved.""" + if not getattr(args, "docker", False) or not getattr(args, "dockerImage", None): + return None + image = args.dockerImage + provenance = {"runtime": None, "image": image, "digest": None} + # Prefer docker; fall back to Apple's "container" runtime. + for runtime in ("docker", "container"): + if getstatusoutput("command -v " + runtime)[0]: + continue + provenance["runtime"] = runtime + # RepoDigests carries the registry digest (repo@sha256:...); fall back to + # the local image Id (sha256:...) if the image was never pulled/pushed. + for fmt in ("{{index .RepoDigests 0}}", "{{.Id}}"): + err, out = getstatusoutput("%s image inspect --format %s %s" % + (runtime, quote(fmt), quote(image))) + if not err and "sha256:" in out: + provenance["digest"] = out.strip() + break + break + return provenance + + def createDistLinks(spec, specs, args, syncHelper, repoType, requiresType): # At the point we call this function, spec has a single, definitive hash. target_dir = "{work_dir}/TARS/{arch}/{repo}/{package}/{package}-{version}-{revision}" \ @@ -122,6 +360,90 @@ def createDistLinks(spec, specs, args, syncHelper, repoType, requiresType): symlink(dep_tarball, target_dir) +def report_plan(plan): + """Print what a build would do, one line per package, then a count. + + The per-package lines go to STDOUT with print(), not through the logger, which + writes to stderr: the point of --plan is to be piped, and the reuse lines are + PACKAGE/VERSION-REVISION, exactly the spec `aliBuild migrate` takes. So + + aliBuild build O2Suite --plan ... | awk '$1 == "reuse" { print $2 }' + + is the list of things to migrate before a build can publish against a store + that does not have them yet. The summary stays on stderr so it cannot land in + the middle of that data. + """ + for package, version_revision, pkg_hash, action in plan: + print("%s %s/%s %s" % (action, package, version_revision, pkg_hash)) + reused = sum(1 for entry in plan if entry[3] == "reuse") + banner("plan: %d package(s): %d reused from the remote store, %d to build.", + len(plan), reused, len(plan) - reused) + + +def bound_unpublished_rebuild(package, rebuilt, ac_store): + """Allow ONE rebuild of a package that is built locally but not published. + + The publish check and the "is it published" check can disagree permanently. + upload_symlinks_and_tarball returns early when the tarball and its link are + already in the legacy store, while is_published() asks the Action Cache -- so a + package that exists only as a pre-2.0 legacy tarball is "already uploaded" and + "not published" at the same time, and rebuilding to fix it changes neither. The + loop that results is unbounded: observed as 1743 rebuilds of defaults-release + in 26 minutes, each one succeeding. + + One rebuild is worth attempting, because the usual cause -- an interrupted run + that left the package here but unuploaded -- really is repaired by it. A second + visit means rebuilding is not the answer, so say what is: the tarball predates + the CAS/AC layout and has to be migrated into it. + """ + dieOnError(package in rebuilt, + "%s is built locally but cannot be published to %s: rebuilding it " + "did not produce an Action Cache entry. Its tarball is almost " + "certainly a pre-2.0 legacy one, which upload_symlinks_and_tarball " + "adopts as already-uploaded without ever writing an AC entry, so no " + "number of rebuilds will publish it. Migrate it into the CAS/AC " + "layout first: aliBuild migrate %s/- --closure " + "--read-store ." % (package, ac_store, package)) + rebuilt.add(package) + + +def select_cached_tarball(tarballs, wanted, uploading): + """Pick which local tarball to unpack instead of rebuilding, or "" to rebuild. + + The store is keyed by hash, but a tarball's name -- and the paths inside it -- + carry the revision, so the one built as -1 is not the one we can publish as + -2. Reusing it anyway skips packaging entirely (build_template.sh only tars + when CACHED_TARBALL is empty), and the upload then dies on a missing file + having ALREADY published the symlink claim and the dist links, leaving the + store advertising a tarball that does not exist. + + Re-tarring the unpacked tree is not the way out: unpacking runs + relocate-me.sh and deletes the .unrelocated copies, so by then the tree is + specific to this machine and would poison every other consumer. + + So when we are going to upload, only the tarball named for our revision will + do; anything else is a rebuild. When we are not uploading, any revision is + fine -- the unpack path relocates whatever it finds (see the $PKGVERSION-* + glob in build_template.sh) and nothing downstream needs the name to match. + + A revision is normally stable for a given hash, so this only bites when one + gets reassigned: another builder claimed it first, or -- as when the legacy + link tree was being read from the wrong bucket -- the revision was picked + while blind to what was already published. + """ + for tarball in sorted(tarballs): + if os.path.basename(tarball) == wanted: + return tarball + if not tarballs: + return "" + if uploading: + debug("Ignoring cached tarball(s) %s: built for another revision, and we " + "must produce %s to upload it", ", ".join( + sorted(os.path.basename(t) for t in tarballs)), wanted) + return "" + return sorted(tarballs)[0] + + def storeHashes(package, specs, considerRelocation): """Calculate various hashes for package, and store them in specs[package]. @@ -430,11 +752,20 @@ def spec_info(spec): def dependency_list(key): return [spec_info(specs[dep]) for dep in specs[package].get(key, ())] + # Deterministic fields only: this lives inside the tarball, so anything varying + # independently of the action hash breaks reproducibility. alibuild_version and the + # alidist commit moved to the AC entry for that reason; the recipe replaces them + # and makes the tarball self-describing without the ledger. + recipe_text = specs[package].get("fullRecipe") or specs[package].get("recipe") or "" + return json.dumps({ "comment": args.annotate.get(package), - "alibuild_version": __version__, - "alidist": { - "commit": os.environ["ALIBUILD_ALIDIST_HASH"], + "recipe": { + "digest": "sha256:" + hashlib.sha256(recipe_text.encode("utf-8", "ignore")).hexdigest(), + "text": recipe_text, + "env": dict(specs[package].get("env") or {}), + "append_path": dict(specs[package].get("append_path") or {}), + "prepend_path": dict(specs[package].get("prepend_path") or {}), }, "architecture": args.architecture, "defaults": args.defaults, @@ -453,8 +784,37 @@ def dependency_list(key): def doBuild(args, parser): + # Always sign reapi:// uploads: refuse to upload unsigned unless --no-sign, so + # nothing lands in a reapi ledger without a signature by accident. + if args.writeStore.startswith("reapi://") and not getattr(args, "noSign", False): + dieOnError(not (getattr(args, "signUrl", "") and + (getattr(args, "signToken", "") or getattr(args, "signTokenFile", ""))), + "uploading to a reapi:// store signs Action Cache entries by default: " + "pass --sign-url and --sign-token (or --sign-token-file), or --no-sign " + "to upload unsigned.") + syncHelper = remote_from_url(args.remoteStore, args.writeStore, args.architecture, - args.workDir, getattr(args, "insecure", False)) + args.workDir, getattr(args, "insecure", False), + ac_url=getattr(args, "acStore", "") or "", + ac_write_url=getattr(args, "acWriteStore", "") or "", + storage=getattr(args, "storage", "ephemeral"), + sign_url=getattr(args, "signUrl", "") if not getattr(args, "noSign", False) else "", + sign_token=getattr(args, "signToken", ""), + sign_token_file=getattr(args, "signTokenFile", ""), + signer=getattr(args, "signer", "alibuild"), + legacy_url=getattr(args, "legacyStore", "") or "", + cas_public_url=getattr(args, "casPublicUrl", "") or "") + + # Verify signatures on prebuilt reapi:// tarballs reused during the build. The + # keyring defaults to /keyring.json (args.configDir); under the default + # 'warn' policy this is a no-op when no keyring is present, so builds without a + # keyring are unaffected. + if isinstance(syncHelper, REAPIRemoteSync): + syncHelper.verify_checker = signature_checker(args) + + # Capture the build container (if any) once: it is constant for the run and + # recorded in every Action Cache entry for reproducibility. + container_provenance = resolve_container_provenance(args) packages = args.pkgname specs = {} @@ -754,6 +1114,12 @@ def performPreferCheckWithTempDir(pkg, cmd): mainPackage = buildOrder.pop() warning("Not rebuilding %s because --only-deps option provided.", mainPackage) + # Packages already rebuilt once because they were unpublished; see + # bound_unpublished_rebuild. Per run, not per package: the loop revisits the + # same entry, so the state has to outlive one iteration. + rebuilt_unpublished = set() + # (package, version-revision, hash, "reuse"|"build") for --plan. + plan = [] while buildOrder: p = buildOrder[0] spec = specs[p] @@ -922,6 +1288,16 @@ def performPreferCheckWithTempDir(pkg, cmd): else: spec["hash"] = spec["remote_revision_hash"] + # Everything a DEPENDENT needs -- version, revision, hash -- is settled by + # here, and nothing below has run yet, so this is where a plan can be taken + # without perturbing the packages that follow. Popping and continuing is the + # same shape the "already built" path uses further down. + if getattr(args, "plan", False): + plan.append((spec["package"], "%s-%s" % (spec["version"], spec["revision"]), + spec["hash"], "build" if candidate is None else "reuse")) + buildOrder.pop(0) + continue + # We do not use the override for devel packages, because we # want to avoid having to rebuild things when the /tmp gets cleaned. if spec["is_devel_pkg"]: @@ -975,9 +1351,24 @@ def performPreferCheckWithTempDir(pkg, cmd): fileHash = readHashFile(hashFile) # Development packages have their own rebuild-detection logic above. # spec["hash"] is only useful here for regular packages. - if fileHash == spec["hash"] and not spec["is_devel_pkg"]: - # If we get here, we know we are in sync with whatever remote store. We - # can therefore create a directory which contains all the packages which + # Being built locally says nothing about the store: an interrupted run, or one + # against a different store, leaves the package here but unpublished, and + # skipping it then means it is never uploaded while its dependents are -- a + # dangling closure. Check rather than assume; rebuilding is the cost of being + # wrong, and bound_unpublished_rebuild is what actually bounds it -- this + # comment used to claim the bound without anything implementing it. + published = True + if fileHash == spec["hash"] and not spec["is_devel_pkg"] and \ + isinstance(syncHelper, REAPIRemoteSync) and getattr(syncHelper, "writeStore", ""): + published = syncHelper.is_published(spec["remote_revision_hash"]) + if not published: + bound_unpublished_rebuild(p, rebuilt_unpublished, syncHelper.acWriteStore) + info("%s is built locally but not published to %s; rebuilding it so it can " + "be uploaded", spec["package"], syncHelper.acWriteStore) + + if fileHash == spec["hash"] and not spec["is_devel_pkg"] and published: + # We are in sync with whatever remote store (verified above when there is one). + # We can therefore create a directory which contains all the packages which # were used to compile this one. debug("Package %s was correctly compiled. Moving to next one.", spec["package"]) # If using incremental builds, next time we execute the script we need to remove @@ -1026,7 +1417,11 @@ def performPreferCheckWithTempDir(pkg, cmd): if not spec["is_devel_pkg"]: syncHelper.fetch_tarball(spec) tarballs = glob(os.path.join(tar_hash_dir, "*gz")) - spec["cachedTarball"] = tarballs[0] if len(tarballs) else "" + spec["cachedTarball"] = select_cached_tarball( + tarballs, + "{package}-{version}-{revision}.{arch}.tar.gz".format( + arch=args.architecture, **spec), + uploading=bool(getattr(syncHelper, "writeStore", ""))) debug("Found tarball in %s" % spec["cachedTarball"] if spec["cachedTarball"] else "No cache tarballs found") @@ -1079,10 +1474,12 @@ def performPreferCheckWithTempDir(pkg, cmd): ("GIT_COMMITTER_EMAIL", "unknown"), ("INCREMENTAL_BUILD_HASH", spec.get("incremental_hash", "0")), ("JOBS", str(args.jobs)), - # Produce reproducible, content-stable tarballs for packages that may be - # uploaded to the remote store. Devel packages are never uploaded, so we - # leave their install trees untouched to avoid perturbing mtimes that - # incremental rebuilds might care about. + # Produce reproducible, content-stable tarballs for all packages, since + # byte-stable output is useful regardless of the remote store (build-to- + # build determinism, CAS dedup, rsync/mirror efficiency). Devel packages + # are excluded (NORMALIZE_TARBALL unset): they are never uploaded, and + # normalising would perturb install-tree mtimes that incremental rebuilds + # rely on. See REMOTE_STORE_CAS_AC.md. ("NORMALIZE_TARBALL", "" if spec["is_devel_pkg"] else "1"), ("PKGHASH", spec["hash"]), ("PKGNAME", spec["package"]), @@ -1244,8 +1641,26 @@ def performPreferCheckWithTempDir(pkg, cmd): # Make sure not to upload local-only packages! These might have been # produced in a previous run with a read-only remote store. if not spec["revision"].startswith("local"): + # Only the reapi backend records an Action Cache entry / source snapshot; for + # every other backend (b3://, s3://, rsync, http, ...) this whole block is a + # no-op, so the build+upload path stays byte-identical to before the reapi + # work. The gate is explicit so it can never silently leak into normal builds. + if isinstance(syncHelper, REAPIRemoteSync): + source_artifact = None if getattr(args, "no_snapshot_sources", False) \ + else snapshot_source(spec, syncHelper) + spec["ac_entry"] = build_ac_entry( + spec, specs, args.architecture, container=container_provenance, + source_artifact=source_artifact, + refs_artifact=snapshot_refs(spec, syncHelper), + system_specs=systemPackageSpecs) + publish_validate_system_entries(syncHelper, systemPackageSpecs, specs, + args.architecture) syncHelper.upload_symlinks_and_tarball(spec) + if getattr(args, "plan", False): + report_plan(plan) + return + if not args.onlyDeps: banner("Build of %s successfully completed on `%s'.\n" "Your software installation is at:" diff --git a/alibuild_helpers/build_template.sh b/alibuild_helpers/build_template.sh index 06e83e71..390a05bc 100644 --- a/alibuild_helpers/build_template.sh +++ b/alibuild_helpers/build_template.sh @@ -374,3 +374,46 @@ fi # Mark the build as successful with a placeholder. Allows running incremental # recipe in case the package is in development mode. echo "${DEVEL_HASH}${DEPS_HASH}" > "$BUILDDIR/.build_succeeded" + +# Peak memory for this package, read before the container holding the cgroup +# goes away. O2Physics sizes its `analysis` ninja pool from the cgroup LIMIT at +# an assumed 8 GiB per job -- a worst case measured off the heaviest producers. +# This is what the build actually used, so the assumption can be checked against +# it rather than argued about. cgroup v1 keeps a high-water mark in +# memory.max_usage_in_bytes; v2 only grew memory.peak in Linux 5.19, and macOS +# has neither, so staying quiet is the normal outcome on some builders. +for _peak_file in /sys/fs/cgroup/memory/memory.max_usage_in_bytes \ + /sys/fs/cgroup/memory.peak; do + [ -r "$_peak_file" ] || continue + # No `|| continue` on the read: it reports failure at EOF when the file has no + # trailing newline, having already assigned the value. Let the numeric test + # below be the only gate, and clear the variable so a failed read cannot carry + # the previous iteration's number forward. + _peak_bytes= + read -r _peak_bytes < "$_peak_file" 2>/dev/null || true + case $_peak_bytes in ''|*[!0-9]*) continue ;; esac + # The limit is worth printing beside it: the pool is derived from that number, + # and it is NOT the Nomad reservation -- build-loop.sh keeps some back. + _limit_mib= + for _limit_file in /sys/fs/cgroup/memory/memory.limit_in_bytes \ + /sys/fs/cgroup/memory.max; do + [ -r "$_limit_file" ] || continue + _limit_bytes= + read -r _limit_bytes < "$_limit_file" 2>/dev/null || true + # cgroup v2 spells "no limit" as the literal string "max". + case $_limit_bytes in ''|*[!0-9]*) continue ;; esac + _limit_mib=$((_limit_bytes / 1048576)) + break + done + echo "aliBuild: peak memory $PKGNAME-$PKGVERSION-$PKGREVISION $((_peak_bytes / 1048576)) MiB${_limit_mib:+ of $_limit_mib MiB}" + # Also machine-readable, appended rather than logged, so a caller can turn it + # into metrics without teeing and re-parsing the whole build output. Packages + # build one container at a time, so plain >> needs no locking. Whoever reads + # this owns truncating it: aliBuild has no idea where one CI round ends. + if [ -w "$WORK_DIR" ]; then + printf '%%s\t%%s\t%%s\n' "$PKGNAME-$PKGVERSION-$PKGREVISION" \ + "$((_peak_bytes / 1048576))" "${_limit_mib:-0}" \ + >> "$WORK_DIR/peak-memory.tsv" || true + fi + break +done diff --git a/alibuild_helpers/completions/bash.sh b/alibuild_helpers/completions/bash.sh index a34db4bb..cd9a69d1 100644 --- a/alibuild_helpers/completions/bash.sh +++ b/alibuild_helpers/completions/bash.sh @@ -53,7 +53,7 @@ _aliBuild_complete() { local subcmd="" for (( i=1; i < cword; i++ )); do case "${words[i]}" in - build|clean|deps|doctor|init|analytics|architecture|version|completion) + build|clean|deps|doctor|init|install|reconstruct|migrate|analytics|architecture|version|completion) subcmd="${words[i]}" break ;; @@ -64,7 +64,7 @@ _aliBuild_complete() { if [[ -z "$subcmd" ]]; then COMPREPLY=( $(compgen -W " -d --debug -n --dry-run - build clean deps doctor init analytics architecture version completion + build clean deps doctor init install reconstruct migrate analytics architecture version completion " -- "$cur") ) return fi @@ -73,8 +73,14 @@ _aliBuild_complete() { case "$subcmd" in build) case "$prev" in - -a|--architecture|-z|--devel-prefix|-e|-j|--jobs|--plugin|--docker-image|--docker-extra-args|-v|--remote-store|--write-store) + -a|--architecture|-z|--devel-prefix|-e|-j|--jobs|--plugin|--docker-image|--docker-extra-args|-v|--remote-store|--write-store|--ac-store|--legacy-links-store|--cas-public-url|--sign-url|--sign-token|--signer) return ;; + --storage) + COMPREPLY=( $(compgen -W "ephemeral permanent" -- "$cur") ); return ;; + --require-signature) + COMPREPLY=( $(compgen -W "off warn require" -- "$cur") ); return ;; + --sign-token-file|--trusted-keys) + _filedir; return ;; --defaults) _alibuild_defaults; return ;; --no-local|--disable|--force-rebuild) @@ -89,9 +95,12 @@ _aliBuild_complete() { -a --architecture --defaults --force-unknown-architecture -z --devel-prefix -e -j --jobs -u --fetch-repos --no-local --force-tracked --plugin --disable --force-rebuild - --annotate --only-deps + --annotate --only-deps --plan --docker --docker-image --docker-extra-args -v - --no-remote-store --remote-store --write-store --insecure + --no-remote-store --remote-store --write-store --ac-store --storage --insecure + --legacy-links-store --cas-public-url + --no-snapshot-sources --sign-url --sign-token --sign-token-file --signer --no-sign + --require-signature --trusted-keys -C --chdir -w --work-dir -c --config-dir --reference-sources --aggressive-cleanup --no-auto-cleanup --always-prefer-system --no-system @@ -157,6 +166,56 @@ _aliBuild_complete() { _alibuild_packages fi ;; + install) + case "$prev" in + --version|--revision|-a|--architecture|--remote-store|--ac-store) + return ;; + -w|--work-dir|--prefix) + _filedir -d; return ;; + esac + if [[ "$cur" == -* ]]; then + COMPREPLY=( $(compgen -W " + --version --revision -a --architecture --remote-store --ac-store --insecure + -w --work-dir --prefix + " -- "$cur") ) + else + _alibuild_packages + fi + ;; + reconstruct) + case "$prev" in + --version|--revision|-a|--architecture|--remote-store|--ac-store) + return ;; + -w|--work-dir|--output-config) + _filedir -d; return ;; + esac + if [[ "$cur" == -* ]]; then + COMPREPLY=( $(compgen -W " + --version --revision -a --architecture --remote-store --ac-store --insecure + -w --work-dir --output-config + " -- "$cur") ) + else + _alibuild_packages + fi + ;; + migrate) + case "$prev" in + -a|--architecture|--remote-store|--read-store|--ac-store|--container|-j|--jobs) + return ;; + --storage) + COMPREPLY=( $(compgen -W "ephemeral permanent" -- "$cur") ); return ;; + --alidist|-w|--work-dir|--source-mirror) + _filedir -d; return ;; + esac + if [[ "$cur" == -* ]]; then + COMPREPLY=( $(compgen -W " + --alidist -a --architecture --remote-store --read-store --ac-store --storage --insecure + -w --work-dir --container --no-verify --closure -j --jobs --snapshot-sources --source-mirror + " -- "$cur") ) + else + _filedir + fi + ;; init) case "$prev" in -a|--architecture|--dist|-z|--devel-prefix) diff --git a/alibuild_helpers/completions/zsh.sh b/alibuild_helpers/completions/zsh.sh index 1e51b655..ddd9809b 100644 --- a/alibuild_helpers/completions/zsh.sh +++ b/alibuild_helpers/completions/zsh.sh @@ -66,9 +66,22 @@ _aliBuild_cmd_build() { '--docker-image[Docker image to build inside of]:image: ' \ '--docker-extra-args[Arguments to pass to docker run]:args: ' \ '*-v[Additional volume to mount inside Docker container]:volume: ' \ + '--plan[Resolve the build and print what it would do, then stop]' \ '--no-remote-store[Disable the use of the remote store]' \ '--remote-store[Where to find prebuilt tarballs to reuse]:store: ' \ '--write-store[Where to upload newly built packages]:store: ' \ + '--ac-store[Separate reapi:// ledger store (AC + reconstruction inputs)]:store: ' \ + '--legacy-links-store[Where to publish the legacy TARS/ links and store objects]:store: ' \ + '--cas-public-url[Public base URL of the CAS bucket, for legacy store redirects]:url: ' \ + '--storage[Retention for uploaded tarball blobs]:storage:(ephemeral permanent)' \ + '--no-snapshot-sources[Do not archive built package sources into the reapi ledger]' \ + '--sign-url[Security-proxy sign route for reapi Action Cache uploads]:url: ' \ + '--sign-token[Gate token presented to --sign-url]:token: ' \ + '--sign-token-file[File holding the credential for --sign-url, re-read per request]:file:_files' \ + '--signer[Human-readable signer label recorded in the signature]:name: ' \ + '--no-sign[Upload to a reapi:// store without signing Action Cache entries]' \ + '--require-signature[Verify signatures on reused reapi tarballs]:policy:(off warn require)' \ + '--trusted-keys[JSON keyring of trusted signing keys]:keyring:_files' \ '--insecure[Do not validate TLS certificates for remote store]' \ '(-C --chdir)'{-C,--chdir}'[Change to directory before building]:directory:_directories' \ '(-w --work-dir)'{-w,--work-dir}'[Toplevel directory for builds]:directory:_directories' \ @@ -141,6 +154,51 @@ _aliBuild_cmd_init() { '::package:_alibuild_packages' } +_aliBuild_cmd_install() { + _arguments -s -S \ + '--version[Version of the package to install]:version: ' \ + '--revision[Revision to install]:revision: ' \ + '(-a --architecture)'{-a,--architecture}'[Architecture to install for]:architecture: ' \ + '--remote-store[reapi:// store to install from]:store: ' \ + '--ac-store[Separate reapi:// ledger store]:store: ' \ + '--insecure[Use http instead of https for the reapi:// endpoint]' \ + '(-w --work-dir)'{-w,--work-dir}'[Default install prefix]:directory:_directories' \ + '--prefix[Directory to install into]:directory:_directories' \ + '::package:_alibuild_packages' +} + +_aliBuild_cmd_reconstruct() { + _arguments -s -S \ + '--version[Version of the package to reconstruct]:version: ' \ + '--revision[Revision to reconstruct]:revision: ' \ + '(-a --architecture)'{-a,--architecture}'[Architecture to reconstruct for]:architecture: ' \ + '--remote-store[reapi:// store to reconstruct from / into]:store: ' \ + '--ac-store[Separate reapi:// ledger store]:store: ' \ + '--insecure[Use http instead of https for the reapi:// endpoint]' \ + '(-w --work-dir)'{-w,--work-dir}'[Work directory]:directory:_directories' \ + '--output-config[Where to materialise the recipes]:directory:_directories' \ + '::package:_alibuild_packages' +} + +_aliBuild_cmd_migrate() { + _arguments -s -S \ + '--alidist[alidist checkout/mirror to recover recipes from]:directory:_directories' \ + '(-a --architecture)'{-a,--architecture}'[Architecture being migrated]:architecture: ' \ + '--remote-store[reapi:// store to migrate into]:store: ' \ + '--ac-store[Separate reapi:// ledger store]:store: ' \ + '--read-store[Read-only http(s) old store to fetch tarballs from]:url: ' \ + '--insecure[Use http instead of https for the reapi:// endpoint]' \ + '(-w --work-dir)'{-w,--work-dir}'[Work directory]:directory:_directories' \ + '--container[Container image to record for migrated builds]:image: ' \ + '--storage[Retention for migrated tarball blobs]:storage:(ephemeral permanent)' \ + '--no-verify[Skip the structural self-check of recovered recipes]' \ + '--closure[Migrate the whole build closure of each top package]' \ + '(-j --jobs)'{-j,--jobs}'[Migrate this many packages in parallel]:jobs: ' \ + '--snapshot-sources[Also archive each release git source into the CAS]' \ + '--source-mirror[Where to cache source clones]:directory:_directories' \ + '*:tarball:_files' +} + _aliBuild_cmd_analytics() { _arguments -s -S \ ':state:(on off)' @@ -181,6 +239,9 @@ _aliBuild() { 'deps:Show dependency tree for a package' 'doctor:Check system requirements for a package' 'init:Initialise a local development area' + 'install:Install a prebuilt package from a reapi:// store' + 'reconstruct:Reconstruct missing CAS tarballs from the Action Cache' + 'migrate:Migrate legacy tarballs into a reapi:// store' 'analytics:Turn analysis data reporting on or off' 'architecture:Display detected architecture' 'version:Display aliBuild version' diff --git a/alibuild_helpers/install.py b/alibuild_helpers/install.py new file mode 100644 index 00000000..3624d042 --- /dev/null +++ b/alibuild_helpers/install.py @@ -0,0 +1,158 @@ +"""Recipe-free installation of prebuilt packages from a reapi:// store. + +`aliBuild install` materialises a package and its runtime closure straight from +the Action Cache + CAS, without alidist, a git checkout or a toolchain. It is +the consumer dual of a build: where a build produces a tarball, install fetches +the tarball bytes from the CAS and relocates them into a prefix. See +REMOTE_STORE_CAS_AC.md. +""" + +import os +import os.path +import tarfile +from shlex import quote + +from alibuild_helpers.cmd import execute +from alibuild_helpers.log import info, debug, dieOnError +from alibuild_helpers.sync import remote_from_url +from alibuild_helpers.sync_reapi import REAPIRemoteSync, add_reapi_store_args, signature_checker +from alibuild_helpers.utilities import symlink + + +def add_parser(subparsers, detected_arch, work_dir_default): + """Register the `install` subcommand's parser -- args live with the command, so + the shared top-level argument parser carries no install-specific reapi options.""" + p = subparsers.add_parser( + "install", help="install a prebuilt package from a reapi:// store", + description="Install a prebuilt package and its runtime closure straight " + "from a reapi:// Action Cache + CAS, without recipes or a build.") + p.add_argument("package", metavar="PACKAGE", help="Package to install.") + p.add_argument("--version", required=True, metavar="VERSION", + help="Version of the package to install.") + p.add_argument("--revision", default=None, metavar="REVISION", + help="Revision to install. Defaults to the highest available for the version.") + add_reapi_store_args( + p, remote_help="reapi:// store to install from.", + arch_help="Architecture to install for. Default '%(default)s'.", + detected_arch=detected_arch, work_dir_default=work_dir_default, + work_dir_help="Default install prefix if --prefix is not given. Default '%(default)s'.") + p.add_argument("--prefix", dest="prefix", default=None, metavar="DIR", + help="Directory to install into. Defaults to the work dir.") + return p + + +def collect_runtime_closure(sync, top_hash): + """Return the list of Action Cache entries to install: the requested package + first, followed by its runtime dependency closure. + + The AC entry's runtimeDeps is the already-flattened runtime closure (it is + built from full_runtime_requires), so a single pass over it suffices; we still + deduplicate by action hash defensively. + """ + top = sync.read_ac_entry(top_hash) + dieOnError(top is None, "No Action Cache entry found for action %s" % top_hash) + + entries = [top] + seen = {top_hash} + for dep in top["action"].get("runtimeDeps", []): + dep_hash = dep["actionHash"] + if dep_hash in seen: + continue + seen.add(dep_hash) + entry = sync.read_ac_entry(dep_hash) + dieOnError(entry is None, "Missing Action Cache entry for runtime dependency " + "%s (%s)" % (dep.get("package", "?"), dep_hash)) + entries.append(entry) + return entries + + +def install_entry(sync, entry, prefix, architecture, checker=None): + """Materialise a single Action Cache entry into prefix: fetch its CAS blob, + unpack it (the tarball already contains //-/...) and run + the in-tarball relocate-me.sh against the target prefix. + + When a SignatureChecker is passed, the downloaded blob is bound to the signed + output digest (bytes must hash to it) before it is unpacked into the prefix.""" + action = entry["action"] + result = entry.get("result") or {} + digest = result.get("outputDigest", "") + dieOnError(":" not in digest, + "Action Cache entry for %s has no output digest; cannot install " + "(was it ever uploaded?)" % action["package"]) + algo, _, content_hash = digest.partition(":") + + pkg, version, revision = action["package"], action["version"], action["revision"] + pkgpath = os.path.join(architecture, pkg, "%s-%s" % (version, revision)) + dest_dir = os.path.join(prefix, pkgpath) + + if os.path.isdir(dest_dir): + debug("%s %s-%s already present at %s, skipping", + pkg, version, revision, dest_dir) + else: + info("Installing %s %s-%s", pkg, version, revision) + os.makedirs(prefix, exist_ok=True) + tmp_tarball = os.path.join(prefix, ".%s-%s-%s.tar.gz.part" % (pkg, version, revision)) + try: + sync.download_artifact(content_hash, tmp_tarball, algo) + if checker: + checker.check_blob(tmp_tarball, algo, content_hash, entry) + with tarfile.open(tmp_tarball) as tar: + # The tarball is laid out as //-/..., so extracting + # at the prefix lands it in the right place. + try: + tar.extractall(prefix, filter="data") # python >= 3.12 + except TypeError: + tar.extractall(prefix) + finally: + if os.path.exists(tmp_tarball): + os.unlink(tmp_tarball) + + # Relocate to the final prefix. relocate-me.sh ships inside the tarball and + # rewrites the build-time placeholder paths to $WORK_DIR/$PKGPATH, so it + # must run from the prefix with WORK_DIR pointing at it. + relocate = os.path.join(dest_dir, "relocate-me.sh") + if os.path.exists(relocate): + err = execute("cd %s && WORK_DIR=%s bash -e %s" % ( + quote(prefix), quote(prefix), quote(os.path.join(pkgpath, "relocate-me.sh")))) + dieOnError(err, "Relocation failed for %s %s-%s" % (pkg, version, revision)) + for root, _, files in os.walk(dest_dir): + for fname in files: + if fname.endswith(".unrelocated"): + os.unlink(os.path.join(root, fname)) + + # Point latest at the freshly installed revision, like a build would. + pkg_dir = os.path.join(prefix, architecture, pkg) + os.makedirs(pkg_dir, exist_ok=True) + symlink("%s-%s" % (version, revision), os.path.join(pkg_dir, "latest")) + + +def doInstall(args, parser): + sync = remote_from_url(args.remoteStore, "", args.architecture, args.workDir, + getattr(args, "insecure", False), + ac_url=getattr(args, "acStore", "") or "") + dieOnError(not isinstance(sync, REAPIRemoteSync), + "'aliBuild install' requires a reapi:// remote store, but got %r" % + (args.remoteStore or "(none)")) + + prefix = os.path.abspath(args.prefix or args.workDir) + top_hash = sync.resolve_action_hash(args.package, args.version, args.revision) + dieOnError(not top_hash, "Could not find %s %s%s in %s" % ( + args.package, args.version, + "-" + args.revision if args.revision else "", args.remoteStore)) + + closure = collect_runtime_closure(sync, top_hash) + checker = signature_checker(args) + if checker: + checker.check_closure(closure) + info("Installing %s and %d runtime dependenc%s into %s", args.package, + len(closure) - 1, "y" if len(closure) == 2 else "ies", prefix) + for entry in closure: + install_entry(sync, entry, prefix, args.architecture, checker) + + top = closure[0]["action"] + init_sh = os.path.join(prefix, args.architecture, top["package"], + "%s-%s" % (top["version"], top["revision"]), + "etc", "profile.d", "init.sh") + info("Done. To use %s, run:\n WORK_DIR=%s source %s", + top["package"], quote(prefix), quote(init_sh)) + return True diff --git a/alibuild_helpers/keyring.json b/alibuild_helpers/keyring.json new file mode 100644 index 00000000..68d880df --- /dev/null +++ b/alibuild_helpers/keyring.json @@ -0,0 +1,30 @@ +{ + "_comment": [ + "Trust anchor for reapi:// Action Cache signatures, shipped inside the", + "alibuild package. It is consulted by every consuming command (install,", + "reconstruct, build-with-fetch); `install` in particular has no alidist to", + "read a keyring from, so without this it would verify nothing at all.", + "", + "It is an anchor precisely because it arrives with the code the user already", + "executes -- a different channel from the store being verified. A keyring", + "served from the store itself could be replaced by anyone able to write to", + "the store, which is the attacker signing exists to stop.", + "", + "alidist/keyring.json is merged on top of this when present, so keys can be", + "added without cutting an alibuild release. Merging never widens trust:", + "validity windows intersect and revocations union, so a key revoked here", + "stays revoked no matter what any other keyring says. Revoking by shipping a", + "new alibuild works because builds track the latest alibuild.", + "", + "Key ids are sha256 of the raw 32-byte Ed25519 public key and are checked on", + "load, so an id cannot be pointed at a different key." + ], + "keys": { + "263745eacc18c1794d22fc4527a9d9e5a6e4b700cab6a046edcc8018f8572a98": { + "notAfter": "2028-01-01T00:00:00Z", + "publicKey": "bppFX0aJ4woEoZPBkCo2hV6gGAua/7ul2Ta5dteXsno=", + "signer": "giulio-laptop" + } + }, + "revoked": [] +} diff --git a/alibuild_helpers/log.py b/alibuild_helpers/log.py index 284af8b1..921b6e13 100644 --- a/alibuild_helpers/log.py +++ b/alibuild_helpers/log.py @@ -19,19 +19,25 @@ def __init__(self, fmtstr) -> None: logging.CRITICAL: "\033[1;37;41m", logging.SUCCESS: "\033[1;32m" } if sys.stdout.isatty() else {} def format(self, record): - record.msg = record.msg % record.args + # getMessage() rather than "record.msg % record.args": a record is shared + # by every handler attached to the logger, so substituting into record.msg + # and leaving record.args in place makes the NEXT formatter re-apply the + # args to an already-expanded string -- "not all arguments converted". + # Harmless with only our handler attached, which is why it went unnoticed; + # under pytest, whose logging plugin adds its own handler, it aborts. + msg = record.getMessage() if record.levelno == logging.BANNER and sys.stdout.isatty(): - lines = record.msg.split("\n") + lines = msg.split("\n") return "\n\033[1;34m==>\033[m \033[1m%s\033[m" % lines[0] + \ "".join("\n \033[1m%s\033[m" % x for x in lines[1:]) elif record.levelno == logging.INFO or record.levelno == logging.BANNER: - return record.msg + return msg return "\n".join(self.fmtstr % { "asctime": datetime.datetime.now().strftime("%Y-%m-%d@%H:%M:%S"), "levelname": (self.LEVEL_COLORS.get(record.levelno, self.COLOR_RESET) + record.levelname + self.COLOR_RESET), "message": x, - } for x in record.msg.split("\n")) + } for x in msg.split("\n")) def log_current_package(package, main_package, specs, devel_prefix) -> None: diff --git a/alibuild_helpers/migrate.py b/alibuild_helpers/migrate.py new file mode 100644 index 00000000..2fe1e6cb --- /dev/null +++ b/alibuild_helpers/migrate.py @@ -0,0 +1,1031 @@ +"""Migrate legacy (action-addressed) releases into the reapi CAS + AC layout. + +Every aliBuild tarball embeds its own provenance in a `.meta.json` written by +create_provenance_info(): the alidist commit that produced it, the defaults +name, the package's tag/source, and the full dependency DAG with hashes. So +migration is mostly metadata extraction, not archaeology: read `.meta.json`, +recover the full recipe from the recorded alidist commit, and synthesise a +reconstruct-complete Action Cache entry, with the tarball preserved in the CAS. +See REMOTE_STORE_CAS_AC.md (Migration). + +Legacy builds did not record their container, so migration supplies one (an +explicit override or the architecture's current default builder), marked +`"provenance": "migration-default"` so assumed environment is never confused +with captured environment. +""" + +import hashlib +import json +import os +import os.path +import re +import shutil +import tarfile +import tempfile +import time +from concurrent.futures import ThreadPoolExecutor + +import requests +from requests.exceptions import (ChunkedEncodingError, + ConnectionError as RequestsConnectionError, Timeout) + +from alibuild_helpers.git import git, Git +from alibuild_helpers.log import info, debug, warning, dieOnError, byte_progress +from alibuild_helpers.sync import remote_from_url +from alibuild_helpers.sync_reapi import REAPIRemoteSync, add_reapi_store_args +from alibuild_helpers.source import GitSourceStore, store_refs +from alibuild_helpers.utilities import default_builder_image, parseRecipe + + +def add_parser(subparsers, detected_arch, work_dir_default): + """Register the `migrate` subcommand's parser -- its options live with the command, + keeping the migration surface out of the shared top-level argument parser.""" + p = subparsers.add_parser( + "migrate", help="migrate legacy tarballs into a reapi:// store", + description="Migrate legacy (action-addressed) release tarballs into a " + "reapi:// CAS + Action Cache, using each tarball's embedded " + ".meta.json provenance and the recorded alidist commit.") + p.add_argument("tarballs", metavar="TARBALL", nargs="*", + help="Legacy tarball(s) to migrate: local paths, or PACKAGE/VERSION-REVISION " + "specs when --read-store is given. Not needed with --enrich-sources.") + p.add_argument("--read-store", dest="read_store", default=None, metavar="URL", + help="Read-only http(s) old store to fetch tarballs from (e.g. " + "https://s3.cern.ch/swift/v1/alibuild-repo). The old store is never " + "written to.") + p.add_argument("--alidist", default=None, metavar="DIR", + help="Path to an alidist git checkout/mirror from which to recover recipes at " + "the recorded commits. Required to migrate tarballs; not needed for " + "ledger-only source recovery (--snapshot-sources with no TARBALL).") + add_reapi_store_args( + p, remote_help="reapi:// store to migrate into.", + arch_help="Architecture being migrated. Default '%(default)s'.", + detected_arch=detected_arch, work_dir_default=work_dir_default) + p.add_argument("--container", dest="container", default=None, metavar="IMAGE", + help="Container image to record for the migrated builds (marked as assumed). " + "Defaults to the architecture's default builder.") + p.add_argument("--storage", dest="storage", choices=("ephemeral", "permanent"), + default="ephemeral", + help="Retention for migrated tarball blobs: 'ephemeral' (default) or 'permanent' " + "(pinned; use for real production releases).") + p.add_argument("--no-verify", dest="no_verify", action="store_true", + help="Skip the structural self-check of recovered recipes.") + p.add_argument("--populate-system", dest="populate_system", action="store_true", + help="Bulk-retroactive: walk the WHOLE reapi ledger and add validate-system " + "nodes (make, yacc-like, ... recovered from --alidist) to every build entry. " + "Normal migrations already do this automatically for what they migrate; use " + "this to backfill entries migrated before the feature existed. Idempotent; " + "honours -n/--dry-run.") + p.add_argument("--allow-no-provenance", dest="allow_no_provenance", action="store_true", + help="For pre-provenance tarballs (no .meta.json, so no recipe/AC entry can be " + "recovered), still store the tarball and its store redirect + per-package " + "link, reserving the version-revision so a later fresh build doesn't shadow " + "it. Such packages are preserved and installable but NOT reconstructable " + "(no ledger entry).") + p.add_argument("--closure", dest="closure", action="store_true", + help="Treat each TARBALL as a top package (PACKAGE/VERSION-REVISION) and migrate " + "its whole build closure, read from the old store's dist tree. Requires " + "--read-store.") + p.add_argument("--match", dest="match", default=None, metavar="REGEX", + help="Instead of (or in addition to) explicit TARBALLs, migrate every " + "PACKAGE/VERSION-REVISION published for the architecture in the old store " + "whose spec matches REGEX (Python re.search). '.*' migrates the whole arch. " + "Requires --read-store; composes with --closure (each match is expanded to " + "its closure) and -n/--dry-run (preview the selection without writing).") + p.add_argument("-j", "--jobs", dest="jobs", type=int, default=1, + help="Migrate this many packages in parallel (overlaps the downloads/uploads). " + "Peak disk scales with the number of jobs. Default %(default)d.") + p.add_argument("--snapshot-sources", dest="snapshot_sources", action="store_true", + help="Archive each release's git source into the ledger (clones upstream once " + "per package) so releases become offline-reconstructible. Idempotent: " + "already-migrated releases are enriched in place from the Action Cache (no " + "tarball re-download, no CAS rewrite). With no TARBALL given, walks the whole " + "ledger to recover sources -- works even after the old store has pruned it.") + p.add_argument("--source-mirror", dest="source_mirror", default=None, metavar="DIR", + help="Where to cache source clones for --snapshot-sources. Defaults to " + "WORKDIR/MIRROR-migrate.") + return p + + +class _TextReader: + """Minimal recipe reader over an in-memory string, for parseRecipe.""" + url = "" + + def __init__(self, text): + self.text = text + + def __call__(self): + return self.text + + +def verify_recovered_recipe(meta, recipe_text): + """Structural self-check that the recipe recovered from the recorded alidist + commit matches the tarball's metadata: it parses, its package field matches, + and every recorded dependency carries a hash. Returns (ok, reason). + + This is a structural check, not a full action-hash recompute (which would + require replaying defaults + scm_refs, i.e. alibuild's planning phase). It + catches the realistic failure modes -- wrong/renamed recipe, corrupt metadata, + a missing dependency hash -- without risking false mismatches.""" + try: + err, spec, _ = parseRecipe(_TextReader(recipe_text)) + except Exception as exc: # pylint: disable=broad-except + return False, "recovered recipe does not parse: %s" % exc + if err or not spec: + return False, "recovered recipe does not parse: %s" % err + if spec.get("package", "").lower() != meta["package"]["name"].lower(): + return False, "recovered recipe is for %r, expected %r" % ( + spec.get("package"), meta["package"]["name"]) + recursive = meta.get("dependencies", {}).get("recursive", {}) + for kind in ("build", "runtime"): + for dep in recursive.get(kind, []): + if not dep.get("hash"): + return False, "dependency %r has no recorded hash" % dep.get("name") + return True, "" + + +def read_meta_json(tarball_path): + """Extract and parse the package's .meta.json from a legacy tarball, or None + if the tarball predates embedded provenance. Iterates members lazily and stops + at .meta.json, rather than getmembers() which decompresses the whole archive.""" + with tarfile.open(tarball_path) as tar: + for member in tar: + if os.path.basename(member.name) == ".meta.json": + return json.loads(tar.extractfile(member).read()) + return None + + +def download_from_old_store(read_url, architecture, spec, dest_dir): + """Download a tarball from a read-only HTTP old store. `spec` is + PACKAGE/VERSION-REVISION (e.g. 'ROOT/v6-28-04-1'). + + The per-package object is a *symlink pointer* (its body is a store-relative + path like '/store///'), not the tarball -- the swift + REST endpoint serves the body, not a redirect. So we GET the pointer, resolve + it to the content-addressed store object, and download that. The old store is + only ever read here, never written.""" + pkg, _, verrev = spec.partition("/") + dieOnError(not verrev, "expected PACKAGE/VERSION-REVISION, got %r" % spec) + tarball = "%s-%s.%s.tar.gz" % (pkg, verrev, architecture) + base = read_url.rstrip("/") + link_url = "%s/TARS/%s/%s/%s" % (base, architecture, pkg, tarball) + + debug("HTTP GET %s (resolve symlink)", link_url) + link = requests.get(link_url, timeout=(30, 60)) + link.raise_for_status() + target = re.sub(r"^(\.\./)+", "", link.text.strip()) # tolerate ../.. prefixes + dieOnError("store/" not in target, + "could not resolve %s via its symlink at %s (got %r)" % + (spec, link_url, target[:120])) + if not target.startswith("TARS/"): + target = "TARS/" + target + + dest = os.path.join(dest_dir, tarball) + store_url = "%s/%s" % (base, target) + debug("HTTP GET %s (download tarball)", store_url) + _download_with_resume(store_url, dest) + return dest + + +def _download_with_resume(url, dest, retries=5): + """Stream url to dest, resuming from the bytes already on disk (HTTP Range) if + the connection drops or stalls mid-download -- so a blip on a multi-GB file + doesn't throw away the whole transfer. (connect, read) timeout of (30, 120): + a stalled read errors after 120s and we resume rather than restart.""" + total, progress, offset = None, None, 0 + for attempt in range(retries + 1): + headers = {"Range": "bytes=%d-" % offset} if offset else {} + try: + with requests.get(url, stream=True, headers=headers, timeout=(30, 120)) as resp: + if offset and resp.status_code == 206: + mode = "ab" # server honoured the range: append + else: + resp.raise_for_status() # 200 (fresh, or range ignored): restart + mode, offset = "wb", 0 + if total is None: + total = int(resp.headers.get("content-length", 0)) or None + if progress is None: + progress = byte_progress("download " + os.path.basename(dest), total) + with open(dest, mode) as out: + for chunk in resp.iter_content(1 << 20): + out.write(chunk) + offset += len(chunk) + progress(len(chunk)) + dieOnError(total is not None and offset != total, + "incomplete download of %s (%d/%d bytes)" % (dest, offset, total)) + return + except (ChunkedEncodingError, RequestsConnectionError, Timeout) as exc: + offset = os.path.getsize(dest) if os.path.exists(dest) else 0 + if attempt >= retries: + raise + warning("Download of %s interrupted at %d bytes; resuming (%d/%d): %s", + os.path.basename(dest), offset, attempt + 1, retries, exc) + time.sleep(2) + + +def _list_old_store(read_url, prefix): + """List keys under prefix in a read-only swift/HTTP old store.""" + url = "%s/?prefix=%s&delimiter=/" % (read_url.rstrip("/"), prefix) + debug("HTTP GET %s", url) + resp = requests.get(url, timeout=120) + resp.raise_for_status() + return resp.text.split() + + +def _arch_package_names(read_url, architecture): + """The package-directory names under TARS// (longest first, so + _match_package prefers the more specific name), minus the publisher/store + subtrees. Used to map closure/dep tarball filenames back to package names.""" + pkg_prefix = "TARS/%s/" % architecture + special = {"dist", "dist-direct", "dist-runtime", "store"} + return sorted({k[len(pkg_prefix):].rstrip("/") + for k in _list_old_store(read_url, pkg_prefix) + if k.startswith(pkg_prefix) and k.endswith("/")} - special, + key=len, reverse=True) + + +def _dist_folder_specs(read_url, architecture, subtree, top_spec, names): + """List the PACKAGE/VERSION-REVISION specs recorded in a dist subtree folder for + top_spec. `subtree` is 'dist' (full closure), 'dist-direct' (direct deps) or + 'dist-runtime' (runtime closure). Returns [] if the folder does not exist.""" + pkg, _, verrev = top_spec.partition("/") + suffix = ".%s.tar.gz" % architecture + prefix = "TARS/%s/%s/%s/%s-%s/" % (architecture, subtree, pkg, pkg, verrev) + filenames = sorted({os.path.basename(k) for k in _list_old_store(read_url, prefix) + if k.endswith(suffix)}) + specs = [] + for fname in filenames: + spec = _match_package(read_url, architecture, names, fname[:-len(suffix)], suffix) + if spec: + specs.append(spec) + return specs + + +def enumerate_closure(read_url, architecture, top_spec, strict=True): + """Return the PACKAGE/VERSION-REVISION specs for the full build closure of + top_spec, read cheaply from the old store's dist tree (no tarball downloads). + + Only packages that were a *build target* have a dist/ tree. A dependency-only or + prefer_system package (e.g. ninja) has none: with strict=True (an explicit + `--closure PKG`) that is an error (likely a wrong spec); with strict=False (driven + by `--match`, where every spec is real and enumerated) it just means the closure is + the package itself, so we return [top_spec] instead of failing.""" + pkg, _, verrev = top_spec.partition("/") + dieOnError(not verrev, "expected PACKAGE/VERSION-REVISION, got %r" % top_spec) + names = _arch_package_names(read_url, architecture) + specs = _dist_folder_specs(read_url, architecture, "dist", top_spec, names) + if not specs: + dieOnError(strict, "no dist closure at TARS/%s/dist/%s/%s-%s/ -- is %s right?" % + (architecture, pkg, pkg, verrev, top_spec)) + debug("No dist tree for %s (dependency-only/prefer_system); migrating it alone", + top_spec) + return [top_spec] + # The dist tree includes the top package itself, but guard just in case. + if top_spec not in specs: + specs.append(top_spec) + return specs + + +def enumerate_arch(read_url, architecture, pattern=None): + """Return every PACKAGE/VERSION-REVISION published for `architecture` in the old + store, optionally filtered by a regex (`re.search` against the 'PACKAGE/VERSION- + REVISION' spec). Reads only the per-package link listing -- no tarball downloads. + Used by `migrate --match` to bulk-migrate a whole arch (or a regex subset) so the + new store's revisions are the authoritative old-store ones before any fresh build + claims them.""" + suffix = ".%s.tar.gz" % architecture + pkg_prefix = "TARS/%s/" % architecture + names = _arch_package_names(read_url, architecture) + regex = re.compile(pattern) if pattern else None + specs = set() + for name in names: + for key in _list_old_store(read_url, "%s%s/" % (pkg_prefix, name)): + base = os.path.basename(key.rstrip("/")) + if not base.endswith(suffix): + continue # skips 'latest*' symlinks, manifests, etc. + spec = _match_package(read_url, architecture, names, base[:-len(suffix)], suffix) + if spec and (regex is None or regex.search(spec)): + specs.add(spec) + return sorted(specs) + + +def recover_legacy_deps(read_url, architecture, items, sync): + """Second pass of a legacy migration: for every migrated item that is a legacy + (pre-provenance) AC entry, recover its dependency graph from the old store's + dist-direct/dist-runtime folders and write it into the entry, hash-linked. Each + dep is resolved to the hash it lives under in the *new* store (a legacy dep's + content hash, or a full build's action hash) via the per-package link, so the + graph walks with the existing machinery. Self-contained and idempotent: reads the + new store to find legacy entries, so it also enriches already-present ones on a + re-run. Deps that can't be resolved are dropped (partial graph). Returns the number + of entries enriched.""" + names = _arch_package_names(read_url, architecture) + hash_cache = {} + + def action_hash(spec): + if spec not in hash_cache: + pkg, _, verrev = spec.partition("/") + version, _, revision = verrev.rpartition("-") + try: + hash_cache[spec] = sync.resolve_action_hash(pkg, version, revision) + except Exception: # pylint: disable=broad-except + hash_cache[spec] = None + return hash_cache[spec] + + def refs(specs): + return [{"package": dep.partition("/")[0], "actionHash": action_hash(dep)} + for dep in specs if action_hash(dep)] + + enriched = 0 + for spec in items: + content_hash = action_hash(spec) + if not content_hash: + continue + entry = sync.read_ac_entry(content_hash) + if entry is None or entry["action"].get("kind") != "legacy": + continue + direct = [d for d in _dist_folder_specs(read_url, architecture, "dist-direct", spec, names) + if d != spec] + runtime = [d for d in _dist_folder_specs(read_url, architecture, "dist-runtime", spec, names) + if d != spec] + entry["action"]["deps"] = refs(direct) + entry["action"]["runtimeDeps"] = refs(runtime) + sync.update_ac_entry(entry) + enriched += 1 + return enriched + + +def _match_package(read_url, architecture, names, base, suffix): + """Map a closure tarball basename '--' to PACKAGE/VERSION-REVISION. + + A package name can contain dashes, and one name can be a prefix of another + (e.g. 'ninja' vs 'ninja-fortran'): the tarball 'ninja-fortran-v1.11.1.g9-25' + is package 'ninja' with version 'fortran-v1.11.1.g9', not 'ninja-fortran'. + Longest-prefix alone is wrong, so when several package names match we pick the + one whose per-package symlink actually exists on the store.""" + candidates = [n for n in names if base.startswith(n + "-")] # names are longest-first + if not candidates: + return None + if len(candidates) == 1: + return "%s/%s" % (candidates[0], base[len(candidates[0]) + 1:]) + for name in candidates: + url = "%s/TARS/%s/%s/%s%s" % (read_url.rstrip("/"), architecture, name, base, suffix) + try: + if requests.head(url, timeout=(10, 30)).status_code == 200: + return "%s/%s" % (name, base[len(name) + 1:]) + except Exception: # pylint: disable=broad-except + pass + warning("Ambiguous package for %s (candidates %s); guessing %s", + base, candidates, candidates[0]) + return "%s/%s" % (candidates[0], base[len(candidates[0]) + 1:]) + + +def _alidist_remote(alidist_dir): + """Return the alidist remote URL (origin of the local clone, else canonical).""" + if alidist_dir: + err, out = git(("config", "--get", "remote.origin.url"), + directory=alidist_dir, check=False) + if not err and out.strip(): + return out.strip() + return "https://github.com/alisw/alidist" + + +def _github_raw_base(remote_url): + """Map a github.com remote URL to its raw.githubusercontent.com base.""" + match = re.search(r"github\.com[:/](.+?)(?:\.git)?$", remote_url or "") + return "https://raw.githubusercontent.com/" + (match.group(1) if match else "alisw/alidist") + + +def recover_recipe(alidist_dir, alidist_commit, package): + """Recover the full recipe of `package` at the recorded alidist commit. + + Tries the local alidist checkout first (`git show`); if the commit isn't + present there -- dailies are often built from a CI/branch commit that isn't + reachable from a plain `master` clone -- falls back to fetching the raw recipe + from the alidist remote on GitHub by commit.""" + fname = package.lower() + ".sh" + if alidist_dir: + err, out = git(("show", "%s:%s" % (alidist_commit, fname)), + directory=alidist_dir, check=False) + if not err: + return out + debug("%s not in local alidist; fetching from GitHub", alidist_commit) + url = "%s/%s/%s" % (_github_raw_base(_alidist_remote(alidist_dir)), alidist_commit, fname) + debug("HTTP GET %s (recover recipe)", url) + resp = requests.get(url, timeout=60) + resp.raise_for_status() + return resp.text + + +def container_for_migration(architecture, override=None): + """Return a container record for a migrated build: an explicit override or the + architecture's default builder, marked as assumed (not captured) provenance.""" + image = override or default_builder_image(architecture) + return {"runtime": "docker", "image": image, "digest": None, + "provenance": "migration-default"} + + +def ac_entry_from_meta(meta, recipe_text, container, source_artifact=None, + refs_artifact=None, commit_hash=None): + """Synthesise a (reconstruct-complete) Action Cache entry from a tarball's + embedded .meta.json provenance and the recovered recipe. + + If a source was snapshotted (commit_hash resolved), commit.ref is set to that + SHA -- which is what spec["commit_hash"] becomes at rebuild -- so the + source-aware checkout's SOURCES path matches. Otherwise we fall back to the + tag, all the provenance .meta.json gives us.""" + pkg = meta["package"] + recipe_digest = hashlib.sha256((recipe_text or "").encode("utf-8", "ignore")).hexdigest() + commit_ref = commit_hash or pkg.get("tag") + + def dep_refs(deps): + return [{"package": d["name"], "actionHash": d["hash"]} for d in deps] + + recursive = meta.get("dependencies", {}).get("recursive", {}) + return { + "schemaVersion": 2, + "action": { + "package": pkg["name"], + "version": pkg["version"], + "revision": pkg["revision"], + "architecture": meta["architecture"], + "actionHash": pkg["hash"], + "commit": {"ref": commit_ref, "commitHash": commit_ref, "altRefs": {}}, + "source": pkg.get("source"), + "tag": pkg.get("tag"), + "defaults": meta.get("defaults"), + "recipeDigest": "sha256:" + recipe_digest, + "container": container, + "sourceArtifact": source_artifact, + "refsArtifact": refs_artifact, + "deps": dep_refs(recursive.get("build", [])), + "runtimeDeps": dep_refs(recursive.get("runtime", [])), + "depsHash": "", + }, + } + + +def _ref_candidates(tag): + """Candidate refs to resolve a build's recorded tag, tolerating a deleted rc/ + branch. Dailies are built on an ``rc/`` branch that upstream later deletes, + while the real ```` tag survives -- so try the tag as-is, then with the + ``rc/`` prefix stripped, then the bare basename.""" + candidates = [tag] + if tag.startswith("rc/"): + candidates.append(tag[len("rc/"):]) + base = tag.rsplit("/", 1)[-1] + if base not in candidates: + candidates.append(base) + return candidates + + +def _resolve_source_commit(repo, source, tag): + """Resolve `tag` to a commit in the mirror, fetching candidate refs from + upstream as needed, and falling back to the surviving tag when the recorded + ref (e.g. an rc/ branch built from, then deleted) is gone. Returns the commit + SHA. Raises if nothing resolves.""" + def resolve(ref): + err, out = git(("rev-parse", "--verify", "-q", ref + "^{commit}"), + directory=repo, check=False) + return out.strip() if not err and out.strip() else None + + for cand in _ref_candidates(tag): + commit = resolve(cand) + if commit: + return commit + # Not present locally: try to fetch the candidate as a tag or a branch. + for src_ref in ("refs/tags/%s" % cand, "refs/heads/%s" % cand): + git(("fetch", "--quiet", source, "+%s:refs/_snap/%s" % (src_ref, cand)), + directory=repo, check=False) + commit = resolve("refs/_snap/%s" % cand) + if commit: + return commit + raise RuntimeError( + "could not resolve %r for %s (tried %s): the rc/ branch may be deleted and " + "no surviving tag was found" % (tag, source, _ref_candidates(tag))) + + +def snapshot_legacy_source(sync, meta, mirror_dir): + """Capture a legacy release's git source into the CAS at migrate time (while + upstream is presumably still alive), so the release becomes offline- + reconstructible. Clones a full bare mirror, reused per package across releases: + a snapshot bundle (especially the first, full-history base) must pack objects + locally, so a *partial* clone would force a slow object-by-object network + backfill during `git bundle create` -- a full clone moves the same history once, + in bulk, up front. Resolves the tag to a commit -- tolerating a deleted rc/ + branch by falling back to the surviving tag -- and snapshots source + refs. + Returns (source_artifact, refs_artifact, commit_hash), all None on any failure + (source archival is best-effort and must not abort a migration).""" + source = meta["package"].get("source") + if not source: + return None, None, None + try: + repo = os.path.join(mirror_dir, meta["package"]["name"].lower()) + tag = meta["package"]["tag"] + if not os.path.isdir(repo): + os.makedirs(mirror_dir, exist_ok=True) + git(("clone", "--quiet", "--bare", source, repo), directory=mirror_dir) + else: + # Refresh the reused mirror's tags + branches so a newer daily's refs are + # present (candidates are also fetched on demand in _resolve_source_commit). + git(("fetch", "--quiet", "--tags", source, "+refs/heads/*:refs/heads/*"), + directory=repo, check=False) + commit = _resolve_source_commit(repo, source, tag) + source_artifact = GitSourceStore(sync).snapshot(repo, source, commit) + scm_refs = Git().parseRefs(git(Git().listRefsCmd(repo), directory=repo)) + # Pin the recipe's own tag to the resolved commit so it still resolves offline + # at reconstruct time even if it was an rc/ branch upstream has since deleted: + # apply_refs then recreates it and `git checkout ` works with no upstream. + scm_refs.setdefault("refs/tags/" + tag, commit) + refs_artifact = store_refs(sync, source, scm_refs) + return source_artifact, refs_artifact, commit + except Exception as exc: # pylint: disable=broad-except + warning("Could not snapshot source for %s from %s (it will not be offline-" + "reconstructible): %s", meta["package"]["name"], source, exc) + return None, None, None + + +def _enrich_entry(sync, entry, mirror_dir): + """Add a source snapshot to an already-migrated AC entry, in place, from the + entry itself -- no tarball download, no CAS write. The entry records the + upstream git URL and tag, so we clone upstream, snapshot the (chained) source + + refs bundles into the ledger, and rewrite the entry with sourceArtifact/ + refsArtifact and the resolved commit. Idempotent: a second run is a no-op. + + Returns 'migrated' (newly enriched), 'present' (already has a snapshot, or the + package has no upstream source), or 'skipped' (could not enrich -- upstream + gone; snapshot_legacy_source has already warned).""" + action = entry["action"] + package = action["package"] + if action.get("sourceArtifact"): + return "present" # already offline-reconstructible + source = action.get("source") + if not source: + return "present" # no upstream source (e.g. defaults-release) + + meta = {"package": {"name": package, "source": source, "tag": action.get("tag")}} + source_artifact, refs_artifact, commit = snapshot_legacy_source(sync, meta, mirror_dir) + if not source_artifact: + return "skipped" # snapshot_legacy_source already warned + + action["sourceArtifact"] = source_artifact + action["refsArtifact"] = refs_artifact + if commit: + action["commit"] = {"ref": commit, "commitHash": commit, + "altRefs": action.get("commit", {}).get("altRefs", {})} + sync.update_ac_entry(entry) + info("Enriched %s %s-%s with source snapshot (commit %s)", + package, action.get("version"), action.get("revision"), (commit or "?")[:12]) + return "migrated" + + +def enrich_source_snapshot(sync, package, version, revision, mirror_dir): + """Enrich a single already-migrated release, resolved by (package, version, + revision) via the per-package link. Used from the tarball/old-store migration + path; the ledger-wide recovery uses `doEnrichSources`.""" + action_hash = sync.resolve_action_hash(package, version, revision) + if not action_hash: + warning("Could not resolve action hash for %s %s-%s; cannot enrich source", + package, version, revision) + return "skipped" + entry = sync.read_ac_entry(action_hash) + if entry is None: + warning("No Action Cache entry for %s (%s); cannot enrich source", + package, action_hash) + return "skipped" + return _enrich_entry(sync, entry, mirror_dir) + + +def doEnrichSources(args): + """Recover source snapshots for an already-migrated set by walking the Action + Cache ledger directly -- no old store, no closure, no tarballs. For every AC + entry of the architecture that has an upstream source but no snapshot, clone + upstream and archive the (chained) source into the ledger. This is how a daily + is made offline-reconstructible after the fact, even once the old store has + pruned it. Reached via `migrate --snapshot-sources` with no TARBALL.""" + dieOnError(not args.remoteStore or not args.remoteStore.startswith("reapi://"), + "recovering sources (migrate --snapshot-sources with no TARBALL) requires " + "a reapi:// remote store, got %r" % (args.remoteStore or "(none)")) + ac_store = strip_rw(getattr(args, "acStore", "")) + sync = remote_from_url(args.remoteStore, args.remoteStore, args.architecture, + args.workDir, getattr(args, "insecure", False), + ac_url=ac_store, ac_write_url=ac_store, + storage=getattr(args, "storage", "permanent")) + dieOnError(not isinstance(sync, REAPIRemoteSync), + "recovering sources requires a reapi:// remote store") + + mirror_dir = args.source_mirror or os.path.join(args.workDir, "MIRROR-migrate") + dry_run = getattr(args, "dryRun", False) + jobs = max(1, getattr(args, "jobs", 1) or 1) + hashes = list(sync.iter_ac_entry_hashes(args.architecture)) + total = len(hashes) + info("Enriching sources for %d Action Cache entr%s in %s", total, + "y" if total == 1 else "ies", args.architecture) + + def process(idx, action_hash): + entry = sync.read_ac_entry(action_hash) + if entry is None: + return "skipped" + action = entry["action"] + if dry_run: + if action.get("sourceArtifact"): + state = "already snapshotted" + elif not action.get("source"): + state = "no upstream source" + else: + state = "would snapshot from %s" % action["source"] + info("[%d/%d] %s %s-%s: %s", idx, total, action["package"], + action.get("version"), action.get("revision"), state) + return "migrated" if state.startswith("would") else "present" + result = _enrich_entry(sync, entry, mirror_dir) + info("[%d/%d] %s: %s", idx, total, action["package"], result) + return result + + if jobs > 1 and not dry_run: + info("Enriching with %d parallel jobs", jobs) + with ThreadPoolExecutor(max_workers=jobs) as pool: + results = list(pool.map(lambda pair: process(*pair), enumerate(hashes, 1))) + else: + results = [process(idx, h) for idx, h in enumerate(hashes, 1)] + + info("Source enrichment %sdone: %d enriched, %d already present or source-less, " + "%d skipped", "(dry-run) " if dry_run else "", results.count("migrated"), + results.count("present"), results.count("skipped")) + return results.count("skipped") == 0 + + +def _recipe_requires(recipe_text, architecture): + """The arch-filtered require names (build + runtime) of a recipe text.""" + err, spec, _ = parseRecipe(_TextReader(recipe_text)) + if err or spec is None: + return [] + names = [] + for entry in (list(spec.get("requires", []) or []) + + list(spec.get("build_requires", []) or [])): + pkg, _, regex = str(entry).partition(":") + if regex and not re.match(regex, architecture): + continue # arch-excluded dependency (e.g. "GCC-Toolchain:(?!osx)") + names.append(pkg) + return names + + +def _system_requirement_spec(recipe_text): + """Parsed spec if the recipe declares a system_requirement (make, yacc-like, ...), + else None. These are exactly the deps missing from a migrated entry: they produce + no tarball, so they were never recorded as a (built) dependency.""" + err, spec, _ = parseRecipe(_TextReader(recipe_text)) + if err or spec is None or "system_requirement" not in spec: + return None + return spec + + +def _recipe_recoverer(alidist_dir, alidist_commit): + """A cached `recover_recipe` closure (system recipes like make.sh recur across + packages, so recover each at most once). Returns None on failure.""" + cache = {} + def recipe_of(pkg): + if pkg not in cache: + try: + cache[pkg] = recover_recipe(alidist_dir, alidist_commit, pkg) + except Exception as exc: # pylint: disable=broad-except + debug("Could not recover recipe for %s@%s: %s", pkg, alidist_commit, exc) + cache[pkg] = None + return cache[pkg] + return recipe_of + + +def _populate_entry_system_deps(sync, architecture, entry, recipe_of, dry_run): + """Give one build AC entry its validate-system nodes: read its archived recipe, + find requires that are system_requirement packages (via recipe_of), write a + validate-system entry per such dep and reference it in the entry's deps. Idempotent + (skips deps already present). Returns the number of system deps added.""" + from alibuild_helpers.build import build_validate_system_entry + action = entry["action"] + if action.get("kind") in ("validate-system", "legacy"): + return 0 + recipe_digest = action.get("recipeDigest", "").split(":", 1)[-1] + if not recipe_digest: + return 0 + try: + pkg_recipe = sync.read_blob(recipe_digest).decode("utf-8", "ignore") + except Exception: # pylint: disable=broad-except + return 0 + have = {d["package"] for d in action.get("deps", [])} + new_deps = [] + for req in _recipe_requires(pkg_recipe, architecture): + if req in have: + continue + req_recipe = recipe_of(req) + if req_recipe is None or _system_requirement_spec(req_recipe) is None: + continue + sysspec = dict(_system_requirement_spec(req_recipe), package=req, fullRecipe=req_recipe) + sys_entry = build_validate_system_entry(sysspec, {}, architecture) + new_deps.append({"package": req, "actionHash": sys_entry["action"]["actionHash"]}) + have.add(req) + if not dry_run: + sync.put_ac_entry(sys_entry, req_recipe) + if new_deps: + action["deps"] = action.get("deps", []) + new_deps + if not dry_run: + sync.update_ac_entry(entry) + debug("%s-%s: +%d system dep(s): %s", action["package"], action.get("revision"), + len(new_deps), ", ".join(d["package"] for d in new_deps)) + return len(new_deps) + + +def populate_system_deps(sync, architecture, alidist_dir, alidist_commit, + items=None, dry_run=False): + """Give migrated packages the validate-system nodes a fresh build now writes, so + reconstruct materialises their system recipes and re-runs the checks on the host + with no --alidist bridge. With `items` (a list of PACKAGE/VERSION-REVISION specs) + only those entries are processed -- the automatic pass run for every migration; + with items=None the whole AC ledger is walked (bulk retroactive, `--populate-system`). + Idempotent. Returns (entriesEnriched, depsAdded).""" + recipe_of = _recipe_recoverer(alidist_dir, alidist_commit) + + def entries(): + if items is None: + for action_hash in sync.iter_ac_entry_hashes(architecture): + entry = sync.read_ac_entry(action_hash) + if entry is not None: + yield entry + else: + for spec in items: + pkg, _, verrev = spec.partition("/") + version, _, revision = verrev.rpartition("-") + try: + action_hash = sync.resolve_action_hash(pkg, version, revision) + except Exception: # pylint: disable=broad-except + action_hash = None + entry = sync.read_ac_entry(action_hash) if action_hash else None + if entry is not None: + yield entry + + enriched = added = 0 + for entry in entries(): + n = _populate_entry_system_deps(sync, architecture, entry, recipe_of, dry_run) + if n: + enriched += 1 + added += n + return enriched, added + + +def doPopulateSystem(args): + """migrate --populate-system: the bulk retroactive form -- walk the *whole* AC + ledger and add validate-system nodes to every build entry (normal migrations do + this automatically for what they migrate). Needs the reapi ledger + --alidist.""" + dieOnError(not args.remoteStore or not args.remoteStore.startswith("reapi://"), + "migrate --populate-system requires a reapi:// remote store, got %r" % + (args.remoteStore or "(none)")) + dieOnError(not args.alidist, "--alidist is required to recover system recipes") + ac_store = strip_rw(getattr(args, "acStore", "")) + sync = remote_from_url(args.remoteStore, args.remoteStore, args.architecture, + args.workDir, getattr(args, "insecure", False), + ac_url=ac_store, ac_write_url=ac_store, + storage=getattr(args, "storage", "permanent")) + dieOnError(not isinstance(sync, REAPIRemoteSync), + "migrate --populate-system requires a reapi:// remote store") + commit = getattr(args, "alidist_commit", None) or "HEAD" + dry_run = getattr(args, "dryRun", False) + enriched, added = populate_system_deps(sync, args.architecture, args.alidist, + commit, dry_run=dry_run) + info("System population %sdone: %d entr%s enriched, %d validate-system dep(s) added", + "(dry-run) " if dry_run else "", enriched, "y" if enriched == 1 else "ies", added) + return True + + +def migrate_tarball(sync, tarball_path, alidist_dir, container_override=None, + verify=True, snapshot_sources=False, mirror_dir=None, + dry_run=False): + """Migrate a single legacy tarball into the reapi store. Returns the migrated + package's action hash, or None if it could not be migrated (no provenance, + recipe could not be recovered, or the self-check failed).""" + meta = read_meta_json(tarball_path) + if meta is None: + warning("%s has no .meta.json (pre-provenance); skipping (not migratable)", + tarball_path) + return None + container = container_for_migration(meta["architecture"], container_override) + # Only legacy tarballs carry an alidist commit; current builds record it in the AC + # entry. Checked before the try below, whose handler would otherwise raise a + # second KeyError while reporting the first. + alidist_commit = (meta.get("alidist") or {}).get("commit") + if not alidist_commit: + warning("%s records no alidist commit; skipping (not a legacy tarball -- " + "current builds archive the recipe and are already in the ledger)", + tarball_path) + return None + try: + recipe = recover_recipe(alidist_dir, alidist_commit, meta["package"]["name"]) + except Exception as exc: # pylint: disable=broad-except + warning("Could not recover recipe for %s from alidist@%s: %s", + meta["package"]["name"], alidist_commit, exc) + return None + if verify: + ok, reason = verify_recovered_recipe(meta, recipe) + if not ok: + warning("Self-check failed for %s, skipping: %s", + meta["package"]["name"], reason) + return None + pkg = meta["package"] + if dry_run: + recursive = meta.get("dependencies", {}).get("recursive", {}) + info("[dry-run] would migrate %s %s-%s (action %s): %d build deps, %d runtime " + "deps%s", pkg["name"], pkg["version"], pkg["revision"], pkg["hash"], + len(recursive.get("build", [])), len(recursive.get("runtime", [])), + "; would snapshot source" if snapshot_sources else "") + return pkg["hash"] + + source_artifact = refs_artifact = commit_hash = None + if snapshot_sources: + source_artifact, refs_artifact, commit_hash = \ + snapshot_legacy_source(sync, meta, mirror_dir) + entry = ac_entry_from_meta(meta, recipe, container, source_artifact, + refs_artifact, commit_hash) + sync.migrate_put(entry, tarball_path, recipe) + return entry["action"]["actionHash"] + + +def strip_rw(url): + """Drop the ::rw suffix a store URL may carry. + + finaliseArgs() normalises ::rw only for `build` and `doctor`, so every other + subcommand sees the suffix verbatim -- and boto3 then rejects + "alibuild-cas::rw" as a bucket name before a single request goes out. Since a + working `--remote-store` is exactly what people copy from a build invocation + into a migrate one, accept the suffix rather than fail on it. + + Dropping it loses nothing here: doMigrate passes its remote store as BOTH the + read and the write URL, so migrate writes whether or not the marker is present. + """ + url = (url or "").rstrip() + return url[:-len("::rw")] if url.endswith("::rw") else url + + +def doMigrate(args, parser): + # Both stores, once, before any branch below builds a sync from them. + args.remoteStore = strip_rw(getattr(args, "remoteStore", "")) + if hasattr(args, "acStore"): + args.acStore = strip_rw(args.acStore) + + # `migrate --snapshot-sources` with nothing to migrate = recover sources by + # walking the Action Cache ledger: every migrated entry with an upstream source + # but no snapshot is cloned from upstream and archived. Idempotent, and needs no + # old store/closure, so it works even after the old store pruned the release. + if getattr(args, "populate_system", False): + return doPopulateSystem(args) + match = getattr(args, "match", None) + if getattr(args, "snapshot_sources", False) and not args.tarballs and not match: + return doEnrichSources(args) + dry_run = getattr(args, "dryRun", False) + read_url = getattr(args, "read_store", None) + if read_url: + dieOnError(not read_url.startswith("http"), + "--read-store must be a read-only http(s) URL, got %r" % read_url) + + # --match enumerates the old store's arch and adds every spec matching REGEX to + # the work list (so a whole arch, or a regex subset, can be migrated without + # naming each package). Composes with any explicit TARBALLs and with --closure. + if match: + dieOnError(not read_url, "--match needs --read-store to enumerate the old store") + matched = enumerate_arch(read_url, args.architecture, match) + info("--match %r selects %d package(s) for %s", match, len(matched), args.architecture) + args.tarballs = sorted(set(list(args.tarballs) + matched)) + + dieOnError(not args.tarballs, "no tarballs given to migrate") + dieOnError(not args.alidist, "--alidist DIR is required to migrate tarballs") + + # A dry-run only reads the old store and prints; it needs neither credentials + # nor an S3 client, so don't construct (or require) the reapi write store. + if dry_run: + dieOnError(not args.remoteStore.startswith("reapi://"), + "'aliBuild migrate' requires a reapi:// remote store, but got %r" % + (args.remoteStore or "(none)")) + sync = None + else: + ac_store = strip_rw(getattr(args, "acStore", "")) + sync = remote_from_url(args.remoteStore, args.remoteStore, args.architecture, + args.workDir, getattr(args, "insecure", False), + ac_url=ac_store, ac_write_url=ac_store, + storage=getattr(args, "storage", "ephemeral")) + dieOnError(not isinstance(sync, REAPIRemoteSync), + "'aliBuild migrate' requires a reapi:// remote store, but got %r" % + (args.remoteStore or "(none)")) + + verify = not getattr(args, "no_verify", False) + snapshot_sources = getattr(args, "snapshot_sources", False) + allow_no_provenance = getattr(args, "allow_no_provenance", False) + mirror_dir = args.source_mirror or os.path.join(args.workDir, "MIRROR-migrate") + + items = args.tarballs + if getattr(args, "closure", False): + dieOnError(not read_url, "--closure needs --read-store to enumerate the closure") + # When --match drove the list, every spec is a real enumerated tarball, so a + # missing dist tree (dependency-only/prefer_system package) is not an error -- + # migrate it alone. An explicit --closure PKG stays strict (empty dist = typo). + strict_closure = not bool(match) + seen, items = set(), [] + for top in args.tarballs: + for spec in enumerate_closure(read_url, args.architecture, top, strict=strict_closure): + if spec not in seen: + seen.add(spec) + items.append(spec) + info("Closure of %s expands to %d package(s)", ", ".join(args.tarballs), len(items)) + + download_dir = tempfile.mkdtemp(prefix="alibuild-migrate-") \ + if read_url and not dry_run else None + total = len(items) + jobs = max(1, getattr(args, "jobs", 1) or 1) + + def process(idx, item): + """Migrate one package; returns 'migrated' | 'skipped' | 'present'. Runs + concurrently under a thread pool (I/O-bound; the boto3 client is thread-safe).""" + if read_url and dry_run: + pkg, _, verrev = item.partition("/") + info("[dry-run] would fetch %s/TARS/%s/%s/%s-%s.%s.tar.gz and migrate", + read_url.rstrip("/"), args.architecture, pkg, pkg, verrev, args.architecture) + return "migrated" + # Skip fully-migrated packages without downloading the (large) tarball: the + # per-package link is the last object migrate_put writes, so its presence in + # the artifact store means the entry is complete (idempotent/resumable even + # if a previous run was interrupted mid-write). + if read_url and sync is not None: + pkg, _, verrev = item.partition("/") + tarball_name = "%s-%s.%s.tar.gz" % (pkg, verrev, args.architecture) + if sync.is_fully_migrated(args.architecture, pkg, tarball_name, verrev): + # Already migrated. If sources are being archived, enrich the existing + # AC entry in place (clone upstream -> snapshot -> rewrite entry) without + # re-downloading the tarball; otherwise there is nothing left to do. + if snapshot_sources and not dry_run: + version, _, revision = verrev.rpartition("-") + result = enrich_source_snapshot(sync, pkg, version, revision, mirror_dir) + info("[%d/%d] %s already migrated; source snapshot: %s", + idx, total, item, result) + return result + info("[%d/%d] %s already present, skipping", idx, total, item) + return "present" + info("[%d/%d] Migrating %s", idx, total, item) + tarball = item + try: + if read_url: + tarball = download_from_old_store(read_url, args.architecture, item, download_dir) + # Pre-provenance tarball (no .meta.json)? With --allow-no-provenance, reserve + # its version-revision via a legacy store write (no recipe/AC) instead of + # skipping, so a later fresh build won't shadow it. + if allow_no_provenance and read_meta_json(tarball) is None: + pkg, _, verrev = item.partition("/") + version, _, revision = verrev.rpartition("-") + if not dry_run: + sync.put_legacy_artifact(pkg, version, revision, tarball) + info("[%d/%d] Reserved %s (legacy, no provenance -- not reconstructable)", + idx, total, item) + return "migrated" + ok = migrate_tarball(sync, tarball, args.alidist, args.container, verify=verify, + snapshot_sources=snapshot_sources, mirror_dir=mirror_dir, + dry_run=dry_run) + return "migrated" if ok else "skipped" + except Exception as exc: # pylint: disable=broad-except + warning("Could not migrate %s: %s", item, exc) + return "skipped" + finally: + # Delete each downloaded tarball as we go, so the closure doesn't pile up. + if read_url and not dry_run and tarball != item and os.path.exists(tarball): + os.unlink(tarball) + + try: + if jobs > 1 and not dry_run: + info("Migrating with %d parallel jobs", jobs) + with ThreadPoolExecutor(max_workers=jobs) as pool: + results = list(pool.map(lambda pair: process(*pair), enumerate(items, 1))) + else: + results = [process(idx, item) for idx, item in enumerate(items, 1)] + finally: + if download_dir: + shutil.rmtree(download_dir, ignore_errors=True) + + # Second pass: recover the dependency graph of legacy (pre-provenance) entries + # from the old store's dist tree and hash-link it into their AC entries, so they + # form a connected, walkable graph instead of isolated nodes. Needs the old store + # (dist tree) and the new store (to resolve dep hashes); skipped on dry runs. + if allow_no_provenance and read_url and not dry_run: + enriched = recover_legacy_deps(read_url, args.architecture, items, sync) + if enriched: + info("Recovered the dependency graph for %d legacy package(s) from the dist tree", + enriched) + + # Standard pass: give the just-migrated entries their validate-system nodes (make, + # yacc-like, ...) recovered from alidist, so reconstruct is self-contained for + # system deps -- exactly what a fresh build does. Not opt-in; skipped on dry runs + # (no ledger to read) and when there is no alidist to recover system recipes from. + if sync is not None and not dry_run and args.alidist: + commit = getattr(args, "alidist_commit", None) or "HEAD" + _, added = populate_system_deps(sync, args.architecture, args.alidist, commit, + items=items) + if added: + info("Added %d validate-system dependency node(s) recovered from alidist", added) + + migrated = results.count("migrated") + skipped = results.count("skipped") + present = results.count("present") + info("Migration %sdone: %d migrated, %d skipped, %d already present", + "(dry-run) " if dry_run else "", migrated, skipped, present) + return skipped == 0 diff --git a/alibuild_helpers/reconstruct.py b/alibuild_helpers/reconstruct.py new file mode 100644 index 00000000..e5747191 --- /dev/null +++ b/alibuild_helpers/reconstruct.py @@ -0,0 +1,791 @@ +"""Reconstruct missing CAS blobs from the Action Cache. + +`aliBuild reconstruct` walks the build closure of a package in the Action Cache, +finds which content-addressed tarballs are missing from the CAS, and +materialises a self-contained alidist directory (the archived recipes, plus the +recorded source commits and build container) so the packages can be rebuilt and +the CAS repopulated -- even if every tarball was deleted. See +REMOTE_STORE_CAS_AC.md. + +The actual rebuild reuses the normal build: once the recipes are materialised, +`aliBuild build ... --remote-store reapi://...::rw` recomputes the same action +hashes, fetches whatever blobs still exist, rebuilds the missing ones and +uploads them (writing fresh CAS blobs and updated AC entries). The DAG is held +together by action hashes, so rebuilt blobs that differ byte-for-byte (when a +build is not bit-reproducible) are fine: their AC outputDigest is simply +rewritten. +""" + +import glob +import hashlib +import os +import re +import os.path +import shutil +import subprocess +import sys + +from alibuild_helpers.log import (info, debug, warning, error, dieOnError, + banner, success) +from alibuild_helpers.sync import remote_from_url +from alibuild_helpers.sync_reapi import REAPIRemoteSync, add_reapi_store_args, signature_checker +from alibuild_helpers.source import GitSourceStore, load_refs, apply_refs +from alibuild_helpers.utilities import file_digest + + +def add_parser(subparsers, detected_arch, work_dir_default): + """Register the `reconstruct` subcommand's parser -- its (many) options live with + the command, keeping them out of the shared top-level argument parser.""" + p = subparsers.add_parser( + "reconstruct", help="reconstruct missing CAS tarballs from the Action Cache", + description="Walk the build closure of a package in a reapi:// Action Cache, " + "find tarballs missing from the CAS, and materialise the archived " + "recipes so they can be rebuilt and the CAS repopulated.") + p.add_argument("package", metavar="PACKAGE", help="Package to reconstruct.") + p.add_argument("--version", required=True, metavar="VERSION", + help="Version of the package to reconstruct.") + p.add_argument("--revision", default=None, metavar="REVISION", + help="Revision to reconstruct. Defaults to the highest available for the version.") + add_reapi_store_args( + p, remote_help="reapi:// store to reconstruct from / into.", + arch_help="Architecture to reconstruct for. Default '%(default)s'.", + detected_arch=detected_arch, work_dir_default=work_dir_default) + p.add_argument("--output-config", dest="outputConfig", default=None, metavar="DIR", + help="Where to materialise the recipes. Defaults to WORKDIR/reconstruct-PACKAGE.") + p.add_argument("--verify", dest="verify", action="store_true", + help="Read-only: report the reconstruction plan for the package's closure " + "(which tarballs would be reused from the CAS vs rebuilt) and check the " + "ledger is complete (recipe integrity, dependency DAG, archived sources). " + "Rebuilds nothing.") + p.add_argument("--rebuild", dest="rebuild", action="store_true", + help="With --verify: actually rebuild just the target package (reusing all " + "dependencies from the CAS, no upload) and compare the produced tarball's " + "content hash to the recorded outputDigest. A match proves the blob is " + "byte-for-byte regenerable.") + p.add_argument("--strict", dest="strict", action="store_true", + help="With --verify --rebuild: treat a non-identical rebuilt tarball as a failure " + "(default: soft, since legacy pre-normalisation tarballs are not " + "bit-reproducible).") + p.add_argument("--alidist", dest="alidist", default=None, metavar="DIR", + help="alidist checkout used to supply the defaults config file " + "(defaults-.sh) when it is not archived in the ledger (it is a config, " + "not a package in the closure). Needed to rebuild releases built with a " + "non-'release' defaults, e.g. 'o2'.") + p.add_argument("--rebaseline", dest="rebaseline", action="store_true", + help="Rebuild the target (implies --verify --rebuild) and, if its reproducible " + "hash differs from the recorded legacy one, rewrite the AC entry's " + "outputDigest (and store redirect) to the rebuilt hash so future verifies " + "are byte-identical. A DRY RUN unless --apply is given.") + p.add_argument("--apply", dest="apply", action="store_true", + help="With --rebaseline: actually perform the ledger + CAS writes (default is a " + "dry run that only prints the plan).") + p.add_argument("--delete-old", dest="delete_old", action="store_true", + help="With --rebaseline --apply: also delete the CAS blob orphaned by the " + "re-baseline (default: leave it in place).") + p.add_argument("--persist", dest="persist", action="store_true", + help="Rebuild the target (implies --verify --rebuild) and, if it reproduces the " + "recorded output digest, upload just that CAS blob back -- a content-" + "addressed restore. Unlike an 'aliBuild build ::rw', it writes ONLY the " + "blob: no revision assignment, no dist/publisher symlinks, no AC rewrite " + "(the AC entry, redirect and links already point at it). A DRY RUN unless " + "--apply is given.") + p.add_argument("--storage", dest="storage", default="permanent", + choices=("ephemeral", "permanent"), + help="Retention tag for a blob restored by --persist. Default '%(default)s' (a " + "deliberately reconstructed blob should not be LRU-expired).") + return p + + +def _digest_parts(entry): + """Return (algo, content_hash) from an AC entry's output digest, or None.""" + digest = (entry.get("result") or {}).get("outputDigest", "") + if ":" not in digest: + return None + algo, _, content_hash = digest.partition(":") + return algo, content_hash + + +def walk_build_closure(sync, top_hash): + """Return the AC entries for top_hash and its full build-dependency closure, + in post-order (dependencies before the packages that need them).""" + ordered = [] + visited = set() + + def visit(action_hash): + if action_hash in visited: + return + visited.add(action_hash) + entry = sync.read_ac_entry(action_hash) + dieOnError(entry is None, "Missing Action Cache entry for %s" % action_hash) + for dep in entry["action"].get("deps", []): + visit(dep["actionHash"]) + ordered.append(entry) + + visit(top_hash) + return ordered + + +def find_missing_blobs(sync, entries): + """Return the subset of entries whose output tarball is missing from the CAS. + validate-system entries produce no tarball, so they are never 'missing' -- the + rebuild re-validates them on the host instead.""" + missing = [] + for entry in entries: + # validate-system produces no tarball; legacy artifacts have no recipe, so a + # lost one cannot be rebuilt -- neither is a rebuild candidate. + if entry["action"].get("kind") in ("validate-system", "legacy"): + continue + parts = _digest_parts(entry) + dieOnError(parts is None, "Action Cache entry for %s has no output digest" % + entry["action"]["package"]) + algo, content_hash = parts + if not sync.artifact_blob_exists(content_hash, algo): + missing.append(entry) + return missing + + +def recipe_package_name(recipe): + """The package name a recipe declares in its own header, or "" if none. + + Usually this is the AC entry's package, but not for defaults: alibuild injects + every defaults flavour under the single node name ``defaults-release`` while the + recipe itself still declares the flavour actually used (``defaults-o2``, ...). + That is by design -- it is how defaults are selected without a code change -- + so the ledger is consistent; only a reconstruct that writes the recipe out under + the *node* name gets it wrong, since alibuild then rejects the file for + disagreeing with its own package field. + """ + for line in recipe.decode("utf-8", "replace").splitlines(): + if line.startswith("---"): # end of the YAML header + break + match = re.match(r"package:\s*(\S+)", line) + if match: + return match.group(1) + return "" + + +def defaults_from_closure(sync, entries): + """Recover the ``--defaults`` flavour a build actually used, or "". + + ``action.defaults`` is absent from older (migrated) entries, and the node name + is always ``defaults-release`` whatever the flavour, so the only honest source + is the recipe archived for that node: ``defaults-o2`` means ``--defaults o2``. + Guessing "release" instead makes the rebuild read a defaults file that declares + a different package and fail. + """ + for entry in entries: + action = entry["action"] + if not action.get("package", "").startswith("defaults-"): + continue + digest = action.get("recipeDigest", "") + if ":" not in digest: + continue + algo, _, recipe_hash = digest.partition(":") + declared = recipe_package_name(sync.read_blob(recipe_hash, algo)) + if declared.startswith("defaults-"): + return declared[len("defaults-"):] + return "" + + +def materialize_recipes(sync, entries, config_dir): + """Write the archived recipe of every entry into config_dir as .sh, so + the closure can be rebuilt as a self-contained alidist. Returns the written + paths. + + The file is named after the package the recipe *declares*, not the AC entry's + package name; see :func:`recipe_package_name` for why those differ for defaults. + """ + os.makedirs(config_dir, exist_ok=True) + written = [] + for entry in entries: + action = entry["action"] + digest = action.get("recipeDigest", "") + dieOnError(":" not in digest, "Action Cache entry for %s has no recipe digest " + "(was it written before recipes were archived?)" % action["package"]) + algo, _, recipe_hash = digest.partition(":") + recipe = sync.read_blob(recipe_hash, algo) + declared = recipe_package_name(recipe) or action["package"] + path = os.path.join(config_dir, declared.lower() + ".sh") + with open(path, "wb") as recipef: + recipef.write(recipe) + written.append(path) + return written + + +def restore_sources(sync, entries, reference_dir): + """Restore the archived git source of each entry that has one into the + reference-sources layout (/), so a rebuild can + reuse it via --reference-sources. Returns (restored, from_upstream) package + name lists.""" + store = GitSourceStore(sync) + restored, from_upstream = [], [] + for entry in entries: + action = entry["action"] + artifact = action.get("sourceArtifact") + if not artifact: + from_upstream.append(action["package"]) + continue + dest = os.path.join(reference_dir, action["package"].lower()) + try: + store.restore(artifact, dest) + # Recreate the original tag refs from the cached mapping so that a rebuild + # can resolve tags against this local repo without contacting upstream. + refs_artifact = action.get("refsArtifact") + if refs_artifact: + apply_refs(dest, load_refs(sync, refs_artifact)) + restored.append(action["package"]) + except Exception as exc: # pylint: disable=broad-except + warning("Could not restore source for %s from the CAS: %s", + action["package"], exc) + from_upstream.append(action["package"]) + return restored, from_upstream + + +def _recipe_intact(sync, action): + """Whether an action's recipe blob is present in the ledger and matches its + recorded digest (sha256 == recipeDigest).""" + algo, _, recipe_hash = action.get("recipeDigest", "").partition(":") + if not recipe_hash: + return False + try: + return hashlib.new(algo, sync.read_blob(recipe_hash, algo)).hexdigest() == recipe_hash + except Exception: # pylint: disable=broad-except + return False + + +def verify_closure(sync, closure): + """Read-only reconstruction check for a build closure. For every package it + determines whether its tarball would be *reused* from the CAS (blob present) or + *rebuilt* (blob missing), and whether the ledger can actually rebuild it: recipe + blob present and integrity-verified (sha256 == recipeDigest), dependency DAG + intact, and source either archived (offline) or upstream-only. Returns + (rows, ok) where ok means every would-rebuild package is regenerable.""" + by_hash = {e["action"]["actionHash"] for e in closure} + rows, ok = [], True + for entry in closure: + action = entry["action"] + # System / prefer_system packages produce no tarball: they are re-validated + # on the host at rebuild, not reused or rebuilt. Report and move on. + if action.get("kind") == "validate-system": + recipe_ok = _recipe_intact(sync, action) + if not recipe_ok: + ok = False + rows.append({ + "package": action["package"], "version": action.get("version"), + "revision": action.get("revision"), "action": "system", + "recipe_ok": recipe_ok, "deps_ok": True, "source": "n/a", + "regenerable": recipe_ok, "rebuildable": recipe_ok, + }) + continue + # Legacy (pre-provenance) artifact: preserved and installable, but has no + # recipe, so it can never be rebuilt -- only reused while its blob survives. + if action.get("kind") == "legacy": + parts = _digest_parts(entry) + present = bool(parts) and sync.artifact_blob_exists(parts[1], parts[0]) + if not present: + ok = False # a lost legacy blob is gone for good (nothing to rebuild from) + rows.append({ + "package": action["package"], "version": action.get("version"), + "revision": action.get("revision"), "action": "legacy", + "recipe_ok": False, "deps_ok": True, "source": "n/a", + # `regenerable` means "satisfiable right now" (the blob is there); + # `rebuildable` means "could be regenerated if the blob were lost", which + # for a legacy entry is never, since it has no recipe. + "regenerable": present, "rebuildable": False, + }) + continue + # Would this tarball be reused (present) or rebuilt (missing)? + parts = _digest_parts(entry) + present = bool(parts) and sync.artifact_blob_exists(parts[1], parts[0]) + # Recipe blob present and matching its recorded digest? + recipe_ok = _recipe_intact(sync, action) + deps_ok = all(dep["actionHash"] in by_hash for dep in action.get("deps", [])) + has_snapshot = bool(action.get("sourceArtifact")) + has_upstream = bool(action.get("source")) + # A package we would have to rebuild must be regenerable: its recipe must be + # intact, its deps consistent, and its source obtainable (archived, upstream, + # or none needed). Reused (present) packages don't need to be rebuildable now. + regenerable = recipe_ok and deps_ok and (has_snapshot or has_upstream or + not action.get("source")) + if not present and not regenerable: + ok = False + rows.append({ + "package": action["package"], + "version": action.get("version"), "revision": action.get("revision"), + "action": "reuse" if present else "rebuild", + "recipe_ok": recipe_ok, "deps_ok": deps_ok, + "source": "archived" if has_snapshot else ("upstream" if has_upstream else "none"), + "regenerable": regenerable, "rebuildable": regenerable, + }) + return rows, ok + + +def ensure_defaults_recipe(config_dir, defaults_name, alidist_dir=None): + """Make sure the defaults *config* file the build needs (defaults-.sh) is + present in the materialised alidist. `defaults-release` is usually already there + as a package recipe, but other defaults (e.g. `o2`) are a config file, not a + package in the AC closure, so they are not materialised -- copy them from a + provided --alidist checkout. Returns True if the file is present/available. + + (A future step archives the defaults recipe as a reconstruction input so this + needs no checkout; until then, and for already-migrated releases, --alidist is + the bridge.)""" + if not defaults_name: + return True + target = os.path.join(config_dir, "defaults-%s.sh" % defaults_name) + if os.path.exists(target): + return True + if alidist_dir: + src = os.path.join(alidist_dir, "defaults-%s.sh" % defaults_name) + if os.path.exists(src): + shutil.copyfile(src, target) + debug("Materialised defaults-%s.sh from %s", defaults_name, alidist_dir) + return True + warning("defaults-%s.sh not found under --alidist %s", defaults_name, alidist_dir) + else: + warning("defaults-%s.sh is a config file (not a package in the closure) and is " + "not archived; pass --alidist so reconstruct can supply it.", + defaults_name) + return False + + +def supply_recipes_from_alidist(config_dir, alidist_dir): + """Copy every recipe (and defaults) from an alidist checkout into the + materialised config that isn't already there, so the rebuild's dependency + resolution finds the *full* recipe closure -- including system/prefer_system + packages, which have recipes (with their system-requirement checks) but no + tarballs, so they are absent from the Action Cache. + + Archived recipes already materialised for built packages are kept (never + overwritten), so their content -- and therefore their action hashes -- stays + faithful; only the missing ones come from the checkout. Returns the count + copied. + + (Bridge until the full recipe closure is archived as a reconstruction input; + see the reconstruct notes in REMOTE_STORE_CAS_AC.md.)""" + copied = 0 + for src in glob.glob(os.path.join(alidist_dir, "*.sh")): + dest = os.path.join(config_dir, os.path.basename(src)) + if not os.path.exists(dest): + shutil.copyfile(src, dest) + copied += 1 + info("Supplied %d additional recipe(s) from %s to complete the closure " + "(system/prefer_system packages, defaults, transitive requires)", + copied, alidist_dir) + return copied + + +def _prepare_config_recipes(config_dir, defaults_name, alidist_dir): + """Complete the materialised config so the rebuild's dependency resolution finds + every recipe -- including system/prefer_system packages and the defaults config, + which are not in the Action Cache. With --alidist, supply the whole closure from + it; otherwise fall back to the defaults-only check (and warn).""" + if alidist_dir: + supply_recipes_from_alidist(config_dir, alidist_dir) + else: + ensure_defaults_recipe(config_dir, defaults_name, None) + + +def _rebuilt_tarball(work_dir, architecture, package): + """Return the tarball a rebuild produced for `package` under work_dir's TARS + tree, or None. A force-rebuilt package lands under a fresh action-hash + directory, so glob across hashes and match by package name.""" + pattern = os.path.join(work_dir, "TARS", architecture, "store", "*", "*", + "%s-*.%s.tar.gz" % (package, architecture)) + matches = sorted(glob.glob(pattern)) + return matches[-1] if matches else None + + +def _rebuild_verdict(recorded_hash, rebuilt_hash, strict): + """Interpret a rebuild's content hash vs the recorded one. Returns (ok, kind). + A differing hash means the rebuild works but isn't byte-identical -- expected + for pre-normalisation legacy tarballs, so it is soft unless --strict.""" + if rebuilt_hash == recorded_hash: + return True, "match" + return (not strict), "differ" + + +class RebuildResult: + """Outcome of verify_rebuild. Truthy iff the rebuild passed its verdict, so it + drops into `and`/assertTrue like the bool it replaced, while also carrying the + produced tarball and hashes so a re-baseline can consume them.""" + def __init__(self, ok, kind=None, algo=None, recorded=None, rebuilt=None, + tarball=None): + self.ok, self.kind, self.algo = ok, kind, algo + self.recorded, self.rebuilt, self.tarball = recorded, rebuilt, tarball + + def __bool__(self): + return self.ok + + +def verify_rebuild(args, sync, closure, build_runner=None): + """Rebuild just the target package -- reusing every dependency from the CAS -- + and compare the produced tarball's content hash to the recorded outputDigest. + Proves the ledger regenerates the actual bytes, not merely that the inputs are + present. Builds into an isolated workdir and never uploads to the real store.""" + entry = closure[-1] + parts = _digest_parts(entry) + dieOnError(not parts, "the target has no recorded output digest to compare against") + algo, recorded_hash = parts + + # Materialise recipes + restore the target's source, exactly as a reconstruct. + config_dir = os.path.abspath(getattr(args, "outputConfig", None) or + os.path.join(args.workDir, "reconstruct-" + args.package)) + materialize_recipes(sync, closure, config_dir) + _prepare_config_recipes(config_dir, entry["action"].get("defaults"), + getattr(args, "alidist", None)) + reference_dir = os.path.join(config_dir, "sources") + restore_sources(sync, [entry], reference_dir) + + build_dir = os.path.join(args.workDir, "verify-rebuild-" + args.package) + try: + # Pre-populate SOURCES so the rebuild checks out locally without upstream. + GitSourceStore(sync).restore_to_source_dir(entry, build_dir) + except Exception as exc: # pylint: disable=broad-except + debug("Could not pre-populate SOURCES for %s: %s", args.package, exc) + + ac_store = (getattr(args, "acStore", "") or "").rstrip() + if ac_store.endswith("::rw"): + ac_store = ac_store[:-4] + container = entry["action"].get("container") or {} + image = container.get("digest") or container.get("image") + # Rebuild with the defaults this build actually used (they feed the action + # hash), not a hardcoded guess: prefer the recorded field, else recover the + # flavour from the archived defaults recipe (migrated entries lack the field). + defaults = (entry["action"].get("defaults") + or defaults_from_closure(sync, closure) or "release") + + # Read-only remote store (no ::rw) => dependencies are fetched and reused, + # nothing is uploaded. --force-rebuild only the target, so only it is + # recompiled; its action hash changes but that does not affect the *content* + # we compare against the recorded output digest. Propagate -d so the rebuild + # streams a debug build log when reconstruct is run with --debug (-d is a + # global flag, so it must precede the "build" subcommand). + # Re-invoke through the *same interpreter*, not just the same script: executing + # sys.argv[0] directly would fall back to the shebang's `env python3`, i.e. to + # whatever is on PATH. Running alibuild from a virtualenv that is not on PATH + # (the documented dev setup) would then rebuild with a different, possibly + # dependency-less Python and fail with a bare ImportError. + cmd = [sys.executable, sys.argv[0]] + if getattr(args, "debug", False): + cmd.append("-d") + cmd += ["build", args.package, "-c", config_dir, + "-a", args.architecture, "-w", build_dir, + "--remote-store", args.remoteStore, + # A reconstruct rebuild must be hermetic: a local checkout of any + # dependency (e.g. alibuild-recipe-tools under the cwd) would otherwise + # be picked up as a development package -- rebuilt from local sources and, + # worse, disabling the remote write store for the whole build. + "--force-tracked", + "--force-rebuild", args.package, "--defaults", defaults] + if ac_store: + cmd += ["--ac-store", ac_store] + if getattr(args, "insecure", False): + cmd += ["--insecure"] + if os.path.isdir(reference_dir): + cmd += ["--reference-sources", reference_dir] + if image: + cmd += ["--docker", "--docker-image", image] + + # The materialised recipes are a plain directory, not an SCM checkout; pass the + # alidist provenance explicitly so the build doesn't require a git repo (the + # recipes are already pinned by content). Use the recorded recipe digest. + alidist_hash = entry["action"].get("recipeDigest", "").split(":")[-1] or "reconstruct" + build_env = dict(os.environ, ALIBUILD_ALIDIST_HASH=alidist_hash) + + info("Rebuilding %s in isolation (deps reused from the CAS, no upload):\n %s", + args.package, " ".join(cmd)) + runner = build_runner or (lambda command: subprocess.call(command, env=build_env)) + returncode = runner(cmd) + if returncode != 0: + error("Rebuild of %s failed (exit %d): reconstruction is NOT verified.", + args.package, returncode) + return RebuildResult(False) + + tarball = _rebuilt_tarball(build_dir, args.architecture, args.package) + dieOnError(not tarball, "the rebuild produced no tarball for %s under %s" % + (args.package, build_dir)) + rebuilt_hash = file_digest(tarball, algo) + ok, kind = _rebuild_verdict(recorded_hash, rebuilt_hash, + getattr(args, "strict", False)) + if kind == "match": + success("REPRODUCED: %s rebuilt to a byte-identical tarball (%s:%s). Its CAS " + "blob is fully regenerable from the ledger.", args.package, algo, rebuilt_hash) + else: + warning("Rebuilt %s, but the tarball is NOT byte-identical to the recorded one:\n" + " recorded: %s:%s\n rebuilt : %s:%s\n" + " Expected when the build is not bit-reproducible -- a pre-normalisation " + "legacy tarball, or paths/timestamps baked into the artifact. The rebuild " + "is valid; pass --strict to treat a mismatch as failure.", + args.package, algo, recorded_hash, algo, rebuilt_hash) + return RebuildResult(ok, kind=kind, algo=algo, recorded=recorded_hash, + rebuilt=rebuilt_hash, tarball=tarball) + + +def do_rebaseline(args, sync, closure, result): + """Re-baseline the target's AC entry onto the just-rebuilt tarball: rewrite its + outputDigest (and store redirect + link) to the reproducible hash so future + verifies are byte-identical. Only meaningful when the rebuild *differed* from + the recorded (legacy) hash. Prints the plan; performs it only with --apply. The + new blob is written before the AC is repointed, so it is never left dangling.""" + if result is None or result.tarball is None: + error("Cannot re-baseline %s: its rebuild did not produce a tarball.", + args.package) + return False + if result.kind == "match": + info("Nothing to re-baseline for %s: the rebuild already matches the recorded " + "digest (%s:%s).", args.package, result.algo, result.recorded) + return True + + entry = closure[-1] + action_hash = entry["action"]["actionHash"] + old_cas = "cas/%s/%s/%s" % (result.algo, result.recorded[:2], result.recorded) + new_cas = "cas/%s/%s/%s" % (result.algo, result.rebuilt[:2], result.rebuilt) + info("Re-baseline plan for %s %s (action %s):", args.package, args.version, + action_hash) + info(" store new CAS blob : %s", new_cas) + info(" rewrite AC outputDigest: %s:%s -> %s:%s", result.algo, result.recorded, + result.algo, result.rebuilt) + info(" orphaned old CAS blob: %s%s", old_cas, + " (will delete)" if getattr(args, "delete_old", False) else " (left in place)") + + if not getattr(args, "apply", False): + warning("DRY RUN: nothing written. Re-run with --apply to perform the " + "re-baseline above (a write to the real ledger + CAS).") + return True + + # Reuse the recipe already in the ledger (dedup skips re-upload); pass it + # through so a missing blob would still be restored. + recipe_text = "" + recipe_digest = entry["action"].get("recipeDigest", "").split(":", 1)[-1] + if recipe_digest: + try: + recipe_text = sync.read_blob(recipe_digest).decode("utf-8", "ignore") + except Exception as exc: # pylint: disable=broad-except + debug("Could not pre-read recipe blob %s: %s", recipe_digest, exc) + old_hash, new_hash, old_cas_path = sync.rebaseline_ac_entry( + entry, result.tarball, recipe_text) + success("Re-baselined %s: AC entry %s now points at %s:%s.", args.package, + action_hash, result.algo, new_hash) + if getattr(args, "delete_old", False) and old_hash and old_hash != new_hash: + sync.delete_artifact_blob(old_hash, result.algo) + info("Deleted orphaned CAS blob %s", old_cas_path) + return True + + +def do_persist(args, sync, closure, result): + """Content-addressed restore of a regenerated blob. After an isolated rebuild + that reproduces the recorded output digest, upload ONLY that CAS blob back -- + no revision assignment, no dist/publisher symlinks, no AC rewrite. The AC entry, + legacy store redirect and per-package links still point at this exact hash (only + the blob was deleted), so putting the bytes back at their content-addressed key + is the whole restore. This is the correct reconstruct semantics; an 'aliBuild + build ::rw' is not (it re-publishes: assigns a fresh revision and writes a new + dist graph). A DRY RUN unless --apply.""" + if result is None or result.tarball is None: + error("Cannot persist %s: its rebuild did not produce a tarball.", args.package) + return False + if result.kind != "match": + error("Refusing to persist %s: the rebuild's hash (%s:%s) does not match the " + "recorded digest (%s:%s) -- a content-addressed restore must reproduce the " + "recorded blob exactly. Use --rebaseline to adopt the new hash instead.", + args.package, result.algo, result.rebuilt, result.algo, result.recorded) + return False + + cas_path = "cas/%s/%s/%s" % (result.algo, result.recorded[:2], result.recorded) + if sync.artifact_blob_exists(result.recorded, result.algo): + info("CAS blob %s for %s is already present; nothing to restore.", + cas_path, args.package) + return True + + info("Restore plan for %s: upload the regenerated blob -> %s (retention=%s); " + "no revision, dist symlinks or AC entry are written.", args.package, cas_path, + getattr(args, "storage", "permanent")) + if not getattr(args, "apply", False): + warning("DRY RUN: nothing written. Re-run with --apply to upload the blob.") + return True + + stored = sync.put_artifact_blob(result.tarball, result.algo) + dieOnError(stored != result.recorded, + "restored blob hashed to %s:%s but the recorded digest is %s:%s" % + (result.algo, stored, result.algo, result.recorded)) + success("Restored CAS blob %s for %s from the ledger (content-addressed; no " + "publisher metadata touched).", cas_path, args.package) + return True + + +def doVerify(args, sync, closure): + """Print the reconstruction plan + ledger-completeness report for a closure and + return True if every would-rebuild package is regenerable. With --rebuild, also + regenerate the target and compare its content hash to the recorded digest.""" + rows, ok = verify_closure(sync, closure) + reuse = sum(1 for r in rows if r["action"] == "reuse") + rebuild = [r for r in rows if r["action"] == "rebuild"] + system = sum(1 for r in rows if r["action"] == "system") + legacy = sum(1 for r in rows if r["action"] == "legacy") + info("Reconstruction plan for %s %s (%d packages): %d reused from CAS, %d to " + "rebuild, %d system (revalidated on host), %d legacy (preserved, not " + "reconstructable)", + args.package, args.version, len(rows), reuse, len(rebuild), system, legacy) + markers = {"reuse": "reuse ", "rebuild": "REBUILD", "system": "system ", + "legacy": "legacy "} + for r in rows: + flags = "recipe:%s deps:%s source:%s" % ( + "ok" if r["recipe_ok"] else "MISSING", + "ok" if r["deps_ok"] else "BROKEN", r["source"]) + note = "" if r["regenerable"] else ( + " <- LOST (no recipe)" if r["action"] == "legacy" else + (" <- NOT regenerable" if r["action"] == "rebuild" else "")) + info(" %s %-24s %s-%s [%s]%s", markers.get(r["action"], r["action"]), + r["package"], r["version"], r["revision"], flags, note) + # Two separate claims, and conflating them was actively misleading: "nothing + # needs rebuilding" is about the blobs that happen to be present today, while + # "this closure is reconstructible" is about whether the ledger could regenerate + # them if they were lost. A closure of purely legacy entries satisfies the first + # and fails the second, and used to report SUCCESS -- exactly backwards, since + # the premise of the ledger is that the artifact store is disposable. + fragile = [r for r in rows if not r["rebuildable"]] + fragile_note = "%d of %d package(s) could NOT be regenerated if their blob were " \ + "lost (%s%s)" % ( + len(fragile), len(rows), + ", ".join(r["package"] for r in fragile[:3]), + ", ..." if len(fragile) > 3 else "") + # validate-system nodes produce no tarball, so they must not be counted as one. + artifacts = len(rows) - system + if not rebuild: + if fragile: + warning("All %d tarball(s) are present in the CAS, so nothing would be rebuilt " + "now -- but %s. The artifact store is NOT disposable for this closure.", + artifacts, fragile_note) + else: + success("All %d tarball(s) present in the CAS and the whole closure is " + "regenerable from the ledger (deps, incl. toolchains, reused as-is).", + artifacts) + elif ok: + if fragile: + warning("The %d missing tarball(s) would rebuild, but %s.", len(rebuild), + fragile_note) + else: + success("Reconstruction is possible: %d package(s) would rebuild, all regenerable " + "from the ledger; the other %d are reused from the CAS.", len(rebuild), reuse) + else: + warning("Reconstruction INCOMPLETE: some missing tarballs are not regenerable " + "(see 'NOT regenerable' above).") + + if getattr(args, "rebuild", False): + result = verify_rebuild(args, sync, closure) + if getattr(args, "rebaseline", False): + ok = do_rebaseline(args, sync, closure, result) and ok + if getattr(args, "persist", False): + ok = do_persist(args, sync, closure, result) and ok + ok = bool(result) and ok + return ok + + +def doReconstruct(args, parser): + ac_store = (getattr(args, "acStore", "") or "").rstrip() + if ac_store.endswith("::rw"): + ac_store = ac_store[:-4] + sync = remote_from_url(args.remoteStore, args.remoteStore, args.architecture, + args.workDir, getattr(args, "insecure", False), + ac_url=ac_store, ac_write_url=ac_store, + storage=getattr(args, "storage", "permanent")) + dieOnError(not isinstance(sync, REAPIRemoteSync), + "'aliBuild reconstruct' requires a reapi:// remote store, but got %r" % + (args.remoteStore or "(none)")) + + top_hash = sync.resolve_action_hash(args.package, args.version, args.revision) + dieOnError(not top_hash, "Could not find %s %s%s in %s" % ( + args.package, args.version, + "-" + args.revision if args.revision else "", args.remoteStore)) + + closure = walk_build_closure(sync, top_hash) + + # Before acting on the closure (reusing or rebuilding from it), verify the + # Action Cache entries are trusted, so reconstruct never propagates an + # untrusted recipe/provenance into a fresh CAS blob. + checker = signature_checker(args) + if checker: + checker.check_closure(closure) + + # Re-baselining and persisting both need a rebuild to compare/restore, so they + # imply --verify --rebuild. + if getattr(args, "rebaseline", False) or getattr(args, "persist", False): + args.verify = args.rebuild = True + + if getattr(args, "verify", False): + return doVerify(args, sync, closure) + missing = find_missing_blobs(sync, closure) + info("Build closure of %s %s: %d package(s), %d tarball(s) missing from the CAS", + args.package, args.version, len(closure), len(missing)) + if not missing: + info("Nothing to reconstruct: all CAS blobs are present.") + return True + + for entry in missing: + action = entry["action"] + info(" missing: %s %s-%s (%s)", action["package"], action["version"], + action["revision"], action["actionHash"]) + + config_dir = os.path.abspath(args.outputConfig or + os.path.join(args.workDir, "reconstruct-" + args.package)) + written = materialize_recipes(sync, closure, config_dir) + info("Materialised %d recipe(s) into %s", len(written), config_dir) + _prepare_config_recipes(config_dir, closure[-1]["action"].get("defaults"), + getattr(args, "alidist", None)) + + # Restore archived sources from the CAS so the rebuild doesn't depend on + # upstream git for the bytes (closing the "tarball lost" gap). + reference_dir = os.path.join(config_dir, "sources") + restored, from_upstream = restore_sources(sync, missing, reference_dir) + info("Restored %d source(s) from the CAS into %s; %d will be fetched from " + "upstream (no archived source)", len(restored), reference_dir, + len(from_upstream)) + reference_hint = (" --reference-sources %s" % reference_dir) if restored else "" + + # Pre-populate the build's SOURCES so checkout_sources checks out locally (its + # isdir branch) without cloning the upstream URL -- the "lost upstream" fix. + store = GitSourceStore(sync) + prepopulated = 0 + for entry in missing: + try: + if store.restore_to_source_dir(entry, args.workDir): + prepopulated += 1 + except Exception as exc: # pylint: disable=broad-except + warning("Could not pre-populate build source for %s: %s", + entry["action"]["package"], exc) + if prepopulated: + info("Pre-populated %d build source(s) under %s/SOURCES for an offline " + "rebuild", prepopulated, args.workDir) + + # Surface the recorded build container so the user can pin the environment. + container = closure[-1]["action"].get("container") + docker_hint = "" + if container and container.get("image"): + image = container.get("digest") or container["image"] + docker_hint = " --docker --docker-image %s" % image + info("Recorded build container: %s (digest %s)", container["image"], + container.get("digest") or "unknown") + + # Use the defaults recorded for this build (they feed the action hash), and pass + # the alidist provenance via the environment so the plain materialised recipe + # directory is accepted without being a git checkout. + defaults = (closure[-1]["action"].get("defaults") + or defaults_from_closure(sync, closure) or "release") + alidist_hash = closure[-1]["action"].get("recipeDigest", "").split(":")[-1] or "reconstruct" + # Propagate the connection flags this reconstruct was invoked with, so the + # suggested build actually talks to the same store the same way: --insecure + # (http, e.g. a local proxy) and a separate writable ledger (--ac-store ::rw), + # or the regenerated AC entry would be written into the CAS bucket. Point at the + # binary that was actually invoked, not a bare "aliBuild" that may be a different + # install lacking the reconstruct-side fixes. + program = sys.argv[0] or "aliBuild" + insecure_hint = " --insecure" if getattr(args, "insecure", False) else "" + ac_hint = " --ac-store %s::rw" % ac_store if ac_store else "" + banner("To rebuild the missing tarballs and repopulate the CAS, run:\n" + " ALIBUILD_ALIDIST_HASH=%s %s build %s --defaults %s -c %s -a %s " + "-w %s --force-tracked%s%s%s --remote-store %s::rw%s", + alidist_hash, program, args.package, defaults, config_dir, + args.architecture, args.workDir, docker_hint, reference_hint, + insecure_hint, args.remoteStore, ac_hint) + return True diff --git a/alibuild_helpers/signing.py b/alibuild_helpers/signing.py new file mode 100644 index 00000000..610f0a2c --- /dev/null +++ b/alibuild_helpers/signing.py @@ -0,0 +1,317 @@ +"""Detached signing and verification of reapi Action Cache entries. + +MVP scheme (see REMOTE_STORE_CAS_AC.md, "Signing and trust"): Ed25519 keys, a +DSSE (Dead Simple Signing Envelope) pre-authentication encoding over a canonical +payload derived from the AC entry, and a keyring of trusted public keys. + +The private key is never held here. In production signing goes through the +security-proxy (`sign_via_proxy`, phase S1): the proxy holds the Ed25519 key and +alibuild only sends it the DSSE bytes to sign. This module holds the shared +DSSE/payload construction so signer and verifier agree byte-for-byte, plus +keyring loading and verification. Everything except `sign_via_proxy` is a pure +function with no network. `cryptography` is an optional dependency, imported +lazily, so importing alibuild never requires it -- only sign()/verify()/ +load_keyring()/public_key() do (and `sign_via_proxy` needs neither, since the +proxy does the signing). +""" + +import base64 +import hashlib +import json +import os.path +from datetime import datetime, timezone + +# Payload type for the DSSE PAE. Bump the version suffix if signed_payload changes, +# so an old signature can never be mistaken for one over the new binding. +PAYLOAD_TYPE = "application/vnd.alibuild.ac-signature.v1+json" + + +def _ed25519(): + """Import cryptography's Ed25519 lazily, with a clear error if it is absent.""" + try: + from cryptography.hazmat.primitives.asymmetric import ed25519 + from cryptography.hazmat.primitives.serialization import Encoding, PublicFormat + return ed25519, Encoding, PublicFormat + except ImportError as exc: + raise RuntimeError("signing requires the 'cryptography' package " + "(pip install cryptography)") from exc + + +def dsse_pae(payload_type, payload): + """DSSE v1 Pre-Authentication Encoding -- the exact bytes that get signed. + + PAE = "DSSEv1" SP LEN(type) SP type SP LEN(payload) SP payload + + (LEN is ASCII decimal; everything concatenated as bytes.) Signing the PAE + rather than raw JSON is what frees us from JSON-canonicalisation worries. + """ + typ = payload_type.encode("utf-8") + body = payload if isinstance(payload, bytes) else payload.encode("utf-8") + return b" ".join((b"DSSEv1", + str(len(typ)).encode("ascii"), typ, + str(len(body)).encode("ascii"), body)) + + +def signed_payload(ac_entry): + """Canonical bytes bound by a signature for an AC entry. + + Binds identity + inputs + output so a signature cannot be replayed onto a + different action: actionHash, package, architecture, and the digest -- the + tarball's outputDigest for build entries, or recipeDigest for tarball-less + validate-system entries. Deterministic (sorted keys, no whitespace) so signer + and verifier produce identical bytes. + """ + action = ac_entry.get("action") or {} + result = ac_entry.get("result") or {} + digest = result.get("outputDigest") or action.get("recipeDigest") or "" + body = { + "actionHash": action.get("actionHash", ""), + "package": action.get("package", ""), + "architecture": action.get("architecture", ""), + "digest": digest, + } + return json.dumps(body, sort_keys=True, separators=(",", ":")).encode("utf-8") + + +def keyid_for(public_key_bytes): + """Self-certifying key id: sha256 of the raw 32-byte Ed25519 public key.""" + return hashlib.sha256(public_key_bytes).hexdigest() + + +def public_key(private_key_seed): + """Return (keyid, base64 raw public key) for an Ed25519 seed -- used to build + keyrings and for the proxy to publish its public key.""" + ed25519, encoding, public_format = _ed25519() + sk = ed25519.Ed25519PrivateKey.from_private_bytes(private_key_seed) + pub = sk.public_key().public_bytes(encoding.Raw, public_format.Raw) + return keyid_for(pub), base64.b64encode(pub).decode("ascii") + + +def sign(ac_entry, private_key_seed, signer): + """Return a signatures[] element for the AC entry. + + private_key_seed is the raw 32-byte Ed25519 seed. Used by tests and, with the + identical DSSE/payload construction, by the security-proxy signing route (S1). + """ + ed25519, encoding, public_format = _ed25519() + sk = ed25519.Ed25519PrivateKey.from_private_bytes(private_key_seed) + pub = sk.public_key().public_bytes(encoding.Raw, public_format.Raw) + sig = sk.sign(dsse_pae(PAYLOAD_TYPE, signed_payload(ac_entry))) + return {"keyid": keyid_for(pub), "signer": signer, + "sig": base64.b64encode(sig).decode("ascii")} + + +def sign_via_proxy(ac_entry, endpoint, token, signer, timeout=30): + """Sign an AC entry through the security-proxy and return a signatures[] element. + + The private key lives in the proxy; alibuild never holds it (and this path + needs no `cryptography`). The client builds the full DSSE PAE bytes here -- + ``dsse_pae(PAYLOAD_TYPE, signed_payload(ac_entry))`` -- and the proxy is a + dumb signer: it Ed25519-signs exactly those bytes and returns + ``{"keyid", "sig"}``. Because the PAE is constructed here, the bytes the proxy + signs are identical to what ``sign()`` produces locally, so a proxy signature + verifies against a keyring built from the same key. + + endpoint is the proxy's sign URL and token its gate token, both resolved by + the caller at use-time (the port and token rotate). Returns + ``{"keyid", "signer", "sig"}``. + """ + import requests + message = dsse_pae(PAYLOAD_TYPE, signed_payload(ac_entry)) + response = requests.post( + endpoint, data=message, timeout=timeout, + headers={"Authorization": "Bearer %s" % token, + "Content-Type": "application/octet-stream"}) + response.raise_for_status() + reply = response.json() + return {"keyid": reply["keyid"], "signer": signer, "sig": reply["sig"]} + + +def _parse_time(value): + return datetime.fromisoformat(value.replace("Z", "+00:00")) if value else None + + +class Keyring: + """Trusted Ed25519 public keys (by self-certifying keyid) + a revocation set.""" + + def __init__(self, keys, revoked): + self.keys = keys # keyid -> {"pub", "signer", "notBefore", "notAfter"} + self.revoked = set(revoked) + + +def bundled_keyring_path(): + """Path to the keyring shipped inside the alibuild package. + + This is the trust *anchor*: it arrives with the code the user already runs, + i.e. over a different channel than the store being verified. A keyring served + from the store itself would be forgeable by anyone who can write to the store + -- exactly the attacker signatures exist to stop -- so the store can at best + *distribute* a keyring, never anchor one. + """ + return os.path.join(os.path.dirname(os.path.abspath(__file__)), "keyring.json") + + +def _later(one, two): + return one if two is None else two if one is None else max(one, two) + + +def _earlier(one, two): + return one if two is None else two if one is None else min(one, two) + + +def merge_keyrings(keyrings): + """Merge several keyrings into one, never widening trust. + + Key ids are self-certifying (sha256 of the public key), so the same id can + never denote two different keys and the union of keys is conflict-free. Where + a key appears in more than one source with different validity windows the + *intersection* wins, and revocation lists are unioned. So a later keyring can + add keys, but cannot un-revoke a key or extend a window another source + narrowed -- which is what lets a revocation shipped in alibuild stand even if + a stale alidist keyring still lists the key as good. + """ + keys, revoked = {}, set() + for keyring in keyrings: + revoked |= set(keyring.revoked) + for keyid, entry in keyring.keys.items(): + current = keys.get(keyid) + if current is None: + keys[keyid] = dict(entry) + else: + current["notBefore"] = _later(current["notBefore"], entry["notBefore"]) + current["notAfter"] = _earlier(current["notAfter"], entry["notAfter"]) + return Keyring(keys, revoked) + + +def load_keyrings(sources): + """Load and merge several keyrings (paths or dicts). See merge_keyrings.""" + return merge_keyrings([load_keyring(source) for source in sources]) + + +def load_keyring(source): + """Load a keyring from a JSON file path or an already-parsed dict. + + Format:: + + { "keys": { "": { "publicKey": "", + "signer": "alice-ci", + "notBefore": "2026-01-01T00:00:00Z", # optional + "notAfter": "2027-01-01T00:00:00Z" } }, # optional + "revoked": [ "", ... ] } # optional + + Each keyid must equal sha256(publicKey), so an id cannot be spoofed onto a + different key. + """ + ed25519, _, _ = _ed25519() + if isinstance(source, dict): + data = source + else: + with open(source) as handle: + data = json.load(handle) + keys = {} + for keyid, entry in (data.get("keys") or {}).items(): + pub_raw = base64.b64decode(entry["publicKey"]) + if keyid != keyid_for(pub_raw): + raise ValueError("keyring keyid %s does not match its public key" % keyid) + keys[keyid] = { + "pub": ed25519.Ed25519PublicKey.from_public_bytes(pub_raw), + "signer": entry.get("signer", ""), + "notBefore": _parse_time(entry.get("notBefore")), + "notAfter": _parse_time(entry.get("notAfter")), + } + return Keyring(keys, data.get("revoked") or []) + + +# Signature-enforcement policy for the consume side (install / reconstruct / +# build-with-fetch). "off" ignores signatures entirely; "warn" verifies but only +# logs on failure (the rollout default while producers start signing); "require" +# fails closed on any entry that is not covered by a trusted, in-window, +# non-revoked signature. +POLICY_OFF = "off" +POLICY_WARN = "warn" +POLICY_REQUIRE = "require" +POLICIES = (POLICY_OFF, POLICY_WARN, POLICY_REQUIRE) + + +def evaluate(ac_entry, keyring, policy, min_signatures=1, now=None): + """Apply a signature policy to a single AC entry. + + Returns ``(allowed, level, reason)``: + + * ``off`` -> ``(True, None, "")`` -- signatures ignored. + * ``warn`` -> allowed either way; ``level`` is ``"warn"`` with a reason + when verification fails, else ``None``. + * ``require`` -> allowed only when verification passes; otherwise + ``(False, "error", reason)``. + + Keeping enforcement here (rather than at each call site) means install, + reconstruct and fetch share one policy interpretation. + """ + if policy == POLICY_OFF: + return True, None, "" + ok, reason = verify(ac_entry, keyring, min_signatures=min_signatures, now=now) + if ok: + return True, None, reason + if policy == POLICY_WARN: + return True, "warn", reason + return False, "error", reason + + +def evaluate_closure(entries, keyring, policy, min_signatures=1, now=None): + """Apply a signature policy across a build's dependency closure. + + ``entries`` is an iterable of ``(package, ac_entry)`` pairs -- the AC entry + for the package being consumed plus one for every dependency, since a + tarball is only trustworthy if everything it was built from is too. Returns + ``(allowed, problems)`` where ``problems`` is a list of + ``(package, level, reason)`` for every entry that did not verify cleanly + (both ``warn`` and ``error`` levels). ``allowed`` is False as soon as any + entry fails under ``require``. + """ + allowed, problems = True, [] + for package, ac_entry in entries: + ok, level, reason = evaluate(ac_entry, keyring, policy, + min_signatures=min_signatures, now=now) + if level: + problems.append((package, level, reason)) + if not ok: + allowed = False + return allowed, problems + + +def verify(ac_entry, keyring, min_signatures=1, now=None): + """Verify an AC entry's signatures against a keyring. + + Returns ``(ok, reason)``. ok is True when at least ``min_signatures`` + *distinct* trusted keys -- each within its validity window and not revoked -- + carry a valid signature over the entry's signed payload. + """ + now = now or datetime.now(timezone.utc) + signatures = ac_entry.get("signatures") or [] + if not signatures: + return False, "no signatures" + message = dsse_pae(PAYLOAD_TYPE, signed_payload(ac_entry)) + accepted, reasons = set(), [] + for sig in signatures: + keyid = sig.get("keyid", "") + short = (keyid[:12] or "?") + if keyid in keyring.revoked: + reasons.append("%s revoked" % short) + elif keyid not in keyring.keys: + reasons.append("%s untrusted" % short) + else: + key = keyring.keys[keyid] + if key["notBefore"] and now < key["notBefore"]: + reasons.append("%s not yet valid" % short) + elif key["notAfter"] and now > key["notAfter"]: + reasons.append("%s expired" % short) + else: + try: + key["pub"].verify(base64.b64decode(sig.get("sig", "")), message) + accepted.add(keyid) + except Exception: # bad signature or malformed base64 + reasons.append("%s bad signature" % short) + if len(accepted) >= min_signatures: + return True, "%d trusted signature(s)" % len(accepted) + return False, "insufficient trusted signatures (%d/%d)%s" % ( + len(accepted), min_signatures, (": " + "; ".join(reasons)) if reasons else "") diff --git a/alibuild_helpers/source.py b/alibuild_helpers/source.py new file mode 100644 index 00000000..82e15836 --- /dev/null +++ b/alibuild_helpers/source.py @@ -0,0 +1,251 @@ +"""Content-addressed git sources for hermetic builds and reconstruction. + +A source is stored in the CAS as an **incremental chain of thin git bundles**: a +one-off **base bundle** (the full history the first time a repo is snapshotted or +after a re-baseline) followed by tiny **delta bundles**, each thin against the +repo's previous snapshot. So a stream of close commits -- e.g. daily builds of +the same package -- shares one base and stores only its per-commit delta, instead +of duplicating the whole source every day. + +Each source artifact records the ordered list of segment digests needed to +restore its commit (`segments`), so a restore just fetches and applies them in +order -- no re-derivation of the chain. Restore prefers this CAS path (fast, +offline) and falls back to cloning upstream on any failure: under normal +conditions upstream is available, so the snapshot is a backup + fetch speedup +rather than the sole source of truth. See REMOTE_STORE_CAS_AC.md (Phase 6). +""" + +import hashlib +import json +import os +import os.path +import shutil +import tempfile + +from alibuild_helpers.git import git, clone_speedup_options +from alibuild_helpers.log import warning, debug + + +def _repo_id(source_url): + return hashlib.sha256(source_url.encode("utf-8")).hexdigest() + + +def store_refs(sync, source_url, scm_refs): + """Store the ref->commit mapping (scm_refs, as produced by `git ls-remote`) + as a content-addressed CAS blob, so tag resolution can happen offline at + reconstruct time without contacting upstream. Returns a refs-artifact dict or + None when there are no refs.""" + if not scm_refs: + return None + blob = json.dumps(scm_refs, sort_keys=True).encode("utf-8") + return {"type": "git-refs", "source": source_url, + "digest": sync.put_bytes_as_blob(blob)} + + +def load_refs(sync, artifact): + """Return the ref->commit mapping stored in a refs artifact.""" + return json.loads(sync.read_blob(artifact["digest"])) + + +def apply_refs(repo_dir, scm_refs): + """Recreate tag refs in a restored repo from the cached mapping, so + `git ls-remote` against it resolves tags offline. Best-effort: refs whose + objects are not present in the restored repo are skipped.""" + for ref, sha in scm_refs.items(): + if ref.startswith("refs/tags/"): + git(("update-ref", ref, sha), directory=repo_dir, check=False) + + +def _is_ancestor(repo_dir, maybe_ancestor, commit): + """Whether maybe_ancestor is an ancestor of commit (both present in repo_dir). + Used to decide whether a new snapshot can be a thin delta against the current + chain head; a False (branch switch, force-push, missing commit) re-baselines.""" + if not maybe_ancestor: + return False + err, _ = git(("merge-base", "--is-ancestor", maybe_ancestor, commit), + directory=repo_dir, check=False) + return err == 0 + + +class GitSourceStore: + """Store and restore git sources as base + thin-delta bundles in the CAS.""" + + def __init__(self, sync): + self.sync = sync + + def _backfill_objects(self, repo_dir, commit, base_commit): + """Materialise the objects a bundle of `commit` (minus base_commit) needs. + + aliBuild mirrors are treeless/blobless partial clones, so a bundle's trees and + blobs are absent locally; `git bundle create` would otherwise lazily fetch them + from the promisor remote one object at a time -- thousands of serial round-trips + that look like a hang. Instead, fetch the whole needed closure in a single + packfile. `--refetch` bypasses negotiation (so already-"present" commits don't + suppress the transfer), and the permissive filter override lifts the mirror's + tree:0/blob:none filter for this one fetch. Guarded on there being missing + objects, so it is a no-op on a full mirror (e.g. the migrate path's clones).""" + rev = ["rev-list", "--objects", "--missing=print", commit] + if base_commit: + rev += ["--not", base_commit] + err, out = git(tuple(rev), directory=repo_dir, check=False) + if err or not any(line.startswith("?") for line in out.splitlines()): + return # full mirror / already backfilled -> nothing to fetch + debug("Backfilling partial-clone objects for %s before bundling", commit) + # Bound the backfill explicitly: a first full-history backfill for a monster + # repo can be very large, and we would rather fall back to upstream (a caught, + # non-fatal skip of the snapshot) than stall the build. Tunable for such repos + # via ALIBUILD_GIT_BACKFILL_TIMEOUT; a materialised incremental delta is small + # and completes well within it. + git(("-c", "remote.origin.partialclonefilter=blob:limit=1t", + "fetch", "--refetch", "--no-tags", "origin", commit), directory=repo_dir, + timeout=int(os.environ.get("ALIBUILD_GIT_BACKFILL_TIMEOUT", "600"))) + + def _bundle(self, repo_dir, commit, base_commit, out_path): + """Write a git bundle of `commit` (thin against base_commit if given) to + out_path. Uses throwaway tags, since `git bundle` advertises refs, not bare + SHAs, and cleans them up afterwards.""" + snap = "_alibuild_snap_" + commit + git(("tag", "-f", snap, commit), directory=repo_dir) + try: + # Ensure the objects the bundle must contain are present locally, so bundle + # create doesn't hang lazily fetching them from a partial mirror one at a time. + self._backfill_objects(repo_dir, commit, base_commit) + if base_commit: + base_tag = "_alibuild_base_" + base_commit + git(("tag", "-f", base_tag, base_commit), directory=repo_dir) + try: + git(("bundle", "create", out_path, snap, "--not", base_tag), + directory=repo_dir) + finally: + git(("tag", "-d", base_tag), directory=repo_dir, check=False) + else: + git(("bundle", "create", out_path, snap), directory=repo_dir) + finally: + git(("tag", "-d", snap), directory=repo_dir, check=False) + + # Re-baseline (store a fresh full base bundle) once the incremental chain gets + # this long: bounds the number of bundles a restore must fetch. Each fetch is a + # small local CAS blob, so this can be generous; a fresh base costs storage + # once. Daily builds hit this roughly twice a year at the default. + MAX_CHAIN = 250 + + def snapshot(self, repo_dir, source_url, commit): + """Capture source_url@commit (present in repo_dir) into the CAS as an + incremental segment, thin against the repo's previous snapshot when possible. + Returns a source-artifact dict recording the ordered `segments` to restore. + + Idempotent per commit; deduplicating by content hash means re-snapshotting an + unchanged commit, or one whose delta bytes already exist, re-uploads nothing. + Advances a per-repo rolling head so the next snapshot deltas against this one.""" + repo_id = _repo_id(source_url) + + # Already snapshotted this exact commit -> reuse its recorded chain. + seg_key = "sources/git/%s/segment/%s.json" % (repo_id, commit) + existing = self.sync.read_object_json(seg_key) + if existing: + return {"type": "git", "source": source_url, "commit": commit, + "baseCommit": existing.get("baseCommit"), + "segments": existing["segments"]} + + # Thin against the chain head when it is an ancestor of `commit` (the common + # "next daily" case) and the chain is not too long; otherwise re-baseline. + head_key = "sources/git/%s/head.json" % repo_id + head = self.sync.read_object_json(head_key) or {} + prior = head.get("segments", []) + if (head.get("commit") and len(prior) < self.MAX_CHAIN and + _is_ancestor(repo_dir, head["commit"], commit)): + base_commit = head["commit"] + base_of_chain = head.get("baseCommit") or head["commit"] + else: + base_commit, prior, base_of_chain = None, [], commit # start a new chain + + with tempfile.TemporaryDirectory() as tmp: + bundle = os.path.join(tmp, "segment.bundle") + self._bundle(repo_dir, commit, base_commit, bundle) + digest = self.sync.put_file_as_blob(bundle) + + segments = prior + [digest] + record = {"digest": digest, "parent": base_commit, + "baseCommit": base_of_chain, "segments": segments} + self.sync.write_object_json(seg_key, record) + self.sync.write_object_json(head_key, {"commit": commit, + "baseCommit": base_of_chain, + "segments": segments}) + return {"type": "git", "source": source_url, "commit": commit, + "baseCommit": base_of_chain, "segments": segments} + + def restore_to_source_dir(self, entry, work_dir): + """Restore an entry's archived git source into the SOURCES layout that + checkout_sources expects (work_dir/SOURCES///), with the + original tags applied, so a rebuild checks out it offline (the isdir branch + of checkout_sources) instead of cloning the upstream URL. Returns the source + dir, or None if the entry has no source artifact. + + The directory name replicates short_commit_hash(): the recorded + commit.ref is exactly spec["commit_hash"] from the original build, so this + matches what checkout_sources will compute at rebuild time.""" + action = entry["action"] + artifact = action.get("sourceArtifact") + ref = action.get("commit", {}).get("ref") + if not artifact or not ref: + return None + tag = action.get("tag") + short = ref if tag == ref else ref[:10] + source_dir = os.path.join(work_dir, "SOURCES", action["package"], + action["version"], short) + self.restore(artifact, source_dir) + refs_artifact = action.get("refsArtifact") + if refs_artifact: + apply_refs(source_dir, load_refs(self.sync, refs_artifact)) + return source_dir + + def restore(self, entry, dest_dir): + """Materialise the source described by `entry` into dest_dir as a checkout. + + Prefers the archived CAS chain (fast, offline). On any failure -- a missing + blob, a broken chain, an unexpected bundle error -- falls back to cloning + upstream, since under normal conditions upstream is available and the + snapshot is a backup/speedup, not the sole source of truth.""" + os.makedirs(dest_dir, exist_ok=True) + try: + self._restore_from_cas(entry, dest_dir) + except Exception as exc: # pylint: disable=broad-except + source = entry.get("source") + if not source: + raise + warning("Restoring %s@%s from the CAS failed (%s); cloning upstream instead", + source, entry.get("commit", "?"), exc) + self._restore_from_upstream(entry, dest_dir) + + def _restore_from_cas(self, entry, dest_dir): + """Rebuild the checkout from the archived bundle chain in the CAS.""" + git(("init", "-q"), directory=dest_dir) + with tempfile.TemporaryDirectory() as tmp: + segments = entry.get("segments") + if segments: + # Fetch the chain in order: each thin segment's prerequisites are + # supplied by the segments before it. + for idx, digest in enumerate(segments): + bundle = os.path.join(tmp, "seg%d.bundle" % idx) + self.sync.download_blob(digest, bundle) + git(("fetch", "-q", bundle, "refs/*:refs/_recon/s%d/*" % idx), + directory=dest_dir) + else: + # Backward compatibility: pre-chain base/delta artifacts. + if entry.get("baseDigest"): + base = os.path.join(tmp, "base.bundle") + self.sync.download_blob(entry["baseDigest"], base) + git(("fetch", "-q", base, "refs/*:refs/_recon/base/*"), directory=dest_dir) + delta = os.path.join(tmp, "delta.bundle") + self.sync.download_blob(entry["deltaDigest"], delta) + git(("fetch", "-q", delta, "refs/*:refs/_recon/delta/*"), directory=dest_dir) + git(("-c", "advice.detachedHead=false", "checkout", "-q", entry["commit"]), + directory=dest_dir) + + def _restore_from_upstream(self, entry, dest_dir): + """Fallback: clone the source from upstream and check out the commit.""" + shutil.rmtree(dest_dir, ignore_errors=True) + os.makedirs(os.path.dirname(dest_dir) or ".", exist_ok=True) + git(("clone", "-q", *clone_speedup_options(), entry["source"], dest_dir)) + git(("-c", "advice.detachedHead=false", "checkout", "-q", entry["commit"]), + directory=dest_dir) diff --git a/alibuild_helpers/sync.py b/alibuild_helpers/sync.py index 48b6085c..b6f8621b 100644 --- a/alibuild_helpers/sync.py +++ b/alibuild_helpers/sync.py @@ -16,12 +16,24 @@ from alibuild_helpers.utilities import resolve_store_path, resolve_links_path, symlink -def remote_from_url(read_url, write_url, architecture, work_dir, insecure=False): +def remote_from_url(read_url, write_url, architecture, work_dir, insecure=False, + ac_url="", ac_write_url="", storage="ephemeral", + sign_url="", sign_token="", sign_token_file="", signer="alibuild", + legacy_url="", cas_public_url=""): """Parse remote store URLs and return the correct RemoteSync instance for them.""" if read_url.startswith("http"): return HttpRemoteSync(read_url, architecture, work_dir, insecure) if read_url.startswith("s3://"): return S3RemoteSync(read_url, write_url, architecture, work_dir) + if read_url.startswith("reapi://"): + # Lazy import: sync_reapi imports Boto3RemoteSync from here, so importing + # sync.py must not pull it in at module load (and non-reapi runs never do). + from alibuild_helpers.sync_reapi import REAPIRemoteSync + return REAPIRemoteSync(read_url, write_url, architecture, work_dir, insecure, + ac_url, ac_write_url, storage, + sign_url=sign_url, sign_token=sign_token, + sign_token_file=sign_token_file, signer=signer, + legacyStore=legacy_url, casPublicUrl=cas_public_url) if read_url.startswith("b3://"): return Boto3RemoteSync(read_url, write_url, architecture, work_dir) if read_url.startswith("cvmfs://"): @@ -530,6 +542,21 @@ class Boto3RemoteSync: def __init__(self, remoteStore, writeStore, architecture, workdir) -> None: self.remoteStore = re.sub("^b3://", "", remoteStore) self.writeStore = re.sub("^b3://", "", writeStore) + # Where the legacy TARS// tree goes. The same bucket as the artifacts + # unless a subclass says otherwise: REAPIRemoteSync points it elsewhere so the + # bytes can stay content-addressed while consumers that only know the old + # layout keep resolving. Defined here because the publish path below writes + # BOTH halves of that tree -- the store object and the symlinks -- and they + # have to land together, or a client finds a symlink naming a store object + # that is not in the bucket it is reading. + self.legacyWriteStore = self.writeStore + # Where the legacy tree is READ from. Must follow legacyWriteStore, not the + # artifact store: a build that lists symlinks in one bucket and claims them + # in another cannot see its own previous publications. That is not abstract + # -- it made revision assignment blind, so a rebuild whose dependencies had + # changed reassigned a revision that was already taken, and then died in + # _link_is_ours claiming a link it had itself written the day before. + self.legacyReadStore = self.remoteStore self.architecture = architecture self.workdir = workdir self.endpoint_url = "https://s3.cern.ch" @@ -600,7 +627,7 @@ def _link_is_ours(self, link_path, link_body): from botocore.exceptions import ClientError try: remote_target = self.s3.get_object( - Bucket=self.writeStore, Key=link_path)["Body"].read().decode("utf-8").strip() + Bucket=self.legacyWriteStore, Key=link_path)["Body"].read().decode("utf-8").strip() except ClientError as exc: # Gone since we looked. Anything else and we cannot tell whose it is. dieOnError(exc.response.get("Error", {}).get("Code") @@ -621,7 +648,7 @@ def _put_link(self, link_path, link_body): """ from botocore.exceptions import ClientError try: - self.s3.put_object(Bucket=self.writeStore, Key=link_path, IfNoneMatch="*", + self.s3.put_object(Bucket=self.legacyWriteStore, Key=link_path, IfNoneMatch="*", Body=link_body.encode("utf-8")) return True except ClientError as exc: @@ -633,22 +660,32 @@ def _put_link(self, link_path, link_body): def _s3_listdir(self, dirname): """List keys of items under dirname in the read bucket.""" pages = self.s3.get_paginator("list_objects_v2") \ - .paginate(Bucket=self.remoteStore, Delimiter="/", + .paginate(Bucket=self.legacyReadStore, Delimiter="/", Prefix=dirname.rstrip("/") + "/") return (item["Key"] for pg in pages for item in pg.get("Contents", ())) def _s3_key_exists(self, key): - """Return whether the given key exists in the write bucket already.""" + """Return whether the given legacy-tree key exists already. + + Both callers pass a TARS// key -- the store object or the symlink -- so + this looks in the legacy bucket, which is the artifact bucket unless a + subclass moved it. + """ from botocore.exceptions import ClientError try: - self.s3.head_object(Bucket=self.writeStore, Key=key) + self.s3.head_object(Bucket=self.legacyWriteStore, Key=key) except ClientError as err: if err.response["Error"]["Code"] == "404": return False raise return True - def fetch_tarball(self, spec) -> None: + def fetch_tarball(self, spec): + """Download a prebuilt tarball for spec from the remote store, if one exists. + + Returns ``(pkg_hash, local_path)`` for a tarball *freshly downloaded* in this + call, or ``None`` if one was already present locally or none was found. The + reapi backend uses the return value to verify only freshly fetched bytes.""" debug("Updating remote store for package %s with hashes %s", spec["package"], ", ".join(spec["remote_hashes"])) @@ -657,7 +694,7 @@ def fetch_tarball(self, spec) -> None: store_path = resolve_store_path(self.architecture, pkg_hash) if glob.glob(os.path.join(self.workdir, store_path, "%s-*.tar.gz" % spec["package"])): debug("Reusing existing tarball for %s@%s", spec["package"], pkg_hash) - return + return None for pkg_hash in spec["remote_hashes"]: store_path = resolve_store_path(self.architecture, pkg_hash) @@ -670,28 +707,45 @@ def fetch_tarball(self, spec) -> None: debug("Fetching tarball %s", tarball) # Create containing directory locally. (exist_ok= is python3-specific.) os.makedirs(os.path.join(self.workdir, store_path), exist_ok=True) - meta = self.s3.head_object(Bucket=self.remoteStore, Key=tarball) + meta = self.s3.head_object(Bucket=self.legacyReadStore, Key=tarball) + # A reapi:// store keeps the legacy store object as a website-redirect + # stub pointing at the content-addressed CAS blob rather than the bytes + # themselves (see REAPIRemoteSync._upload_tarball). Object GetObject does + # not follow website redirects, so a plain download here would fetch the + # ~50-byte stub. head_object surfaces the redirect target, so follow it to + # the CAS blob (same bucket). We still save it under the legacy store path + # and name, so the rest of the build finds the tarball where it expects. + # + # Which bucket holds the bytes follows the redirect, not the listing. A + # stub points into the CAS bucket; WITHOUT one the legacy object IS the + # tarball and still lives beside its link, in the legacy bucket. Reading + # it from the CAS bucket 404s the moment the two are different -- as + # they are for a ubuntu2204 tree whose tarballs were published by a + # plain s3:// store, which writes bytes where reapi:// writes stubs. fetch_key = tarball + fetch_bucket = self.legacyReadStore redirect = meta.get("WebsiteRedirectLocation") if redirect: fetch_key = redirect.lstrip("/") + fetch_bucket = self.remoteStore debug("Store object %s redirects to CAS blob %s; fetching that", tarball, fetch_key) - meta = self.s3.head_object(Bucket=self.remoteStore, Key=fetch_key) + meta = self.s3.head_object(Bucket=fetch_bucket, Key=fetch_key) total_size = int(meta.get("ContentLength", 0)) debug("Downloading tarball for %s@%s: %s (%d MB)", spec["package"], spec["version"], tarball, total_size >> 20) + dest = os.path.join(self.workdir, store_path, os.path.basename(tarball)) # boto3 invokes Callback with the per-chunk *delta*, not the cumulative # total; byte_progress accumulates it (a raw delta looks stuck at 256 KB). self.s3.download_file( - Bucket=self.remoteStore, Key=fetch_key, - Filename=os.path.join(self.workdir, store_path, os.path.basename(tarball)), + Bucket=fetch_bucket, Key=fetch_key, Filename=dest, Callback=byte_progress("download %s@%s" % (spec["package"], spec["version"]), total_size)) - return + return pkg_hash, dest debug("Remote has no tarballs for %s with hashes %s", spec["package"], ", ".join(spec["remote_hashes"])) + return None def fetch_symlinks(self, spec) -> None: from botocore.exceptions import ClientError @@ -709,7 +763,7 @@ def fetch_symlinks(self, spec) -> None: debug("Fetching symlink manifest") n_symlinks = 0 try: - manifest = self.s3.get_object(Bucket=self.remoteStore, Key=links_path + ".manifest") + manifest = self.s3.get_object(Bucket=self.legacyReadStore, Key=links_path + ".manifest") except ClientError as exc: debug("Could not fetch manifest: %s", exc) else: @@ -733,7 +787,7 @@ def fetch_symlinks(self, spec) -> None: if os.path.islink(link_path): continue debug("Fetching leftover symlink %s", link_key) - resp = self.s3.get_object(Bucket=self.remoteStore, Key=link_key) + resp = self.s3.get_object(Bucket=self.legacyReadStore, Key=link_key) target = os.fsdecode(resp["Body"].read()).rstrip("\n") if not target.startswith("../../"): target = "../../" + target @@ -842,7 +896,7 @@ def upload_symlinks_and_tarball(self, spec) -> None: max_workers = min(32, (len(dist_symlinks) * 10) or 1) def _upload_single_symlink(link_key, hash_path): - self.s3.put_object(Bucket=self.writeStore, + self.s3.put_object(Bucket=self.legacyWriteStore, Key=link_key, Body=os.fsencode(hash_path), ACL="public-read", @@ -880,6 +934,9 @@ def _upload_single_symlink(link_key, hash_path): def _upload_tarball(self, spec, tar_path) -> None: """Upload the tarball bytes to the remote store under tar_path. + + Factored out so that REAPIRemoteSync can store the bytes content-addressed + in a CAS and write an Action Cache entry instead. """ self.s3.upload_file(Bucket=self.writeStore, Key=tar_path, Filename=os.path.join(self.workdir, tar_path)) diff --git a/alibuild_helpers/sync_reapi.py b/alibuild_helpers/sync_reapi.py new file mode 100644 index 00000000..c8ef49b4 --- /dev/null +++ b/alibuild_helpers/sync_reapi.py @@ -0,0 +1,1054 @@ +"""REAPI (Action Cache + CAS) sync backend for alibuild. + +Split out of sync.py so the reapi:// store -- its content-addressed CAS, the +Action Cache ledger, retention/lifecycle handling and reconstruction helpers -- +lives on its own and keeps the shared sync.py surface small. The base +Boto3RemoteSync (scheme b3://) it extends stays in sync.py; remote_from_url() +there lazy-imports this module so importing sync.py never pulls reapi in. +""" + +import glob +import hashlib +import json +import os +import os.path +import re +import time +from datetime import datetime, timezone + +from alibuild_helpers.log import debug, info, warning, error, dieOnError, byte_progress +from alibuild_helpers.utilities import resolve_store_path, resolve_links_path, symlink +from alibuild_helpers.utilities import resolve_cas_path, resolve_ac_path, file_digest +from alibuild_helpers.sync import Boto3RemoteSync + + +def add_reapi_store_args(parser, remote_help, arch_help, detected_arch, work_dir_default, + work_dir_help="Work directory. Default '%(default)s'."): + """Add the reapi:// store options shared by the install/reconstruct/migrate + subcommands: the (required) --remote-store, --insecure, the separate --ac-store + ledger, -a/--architecture and -w/--work-dir. Per-command help text is passed in. + Keeping this here lets each command register its own parser from its own module, + so the reapi arg surface stays out of the shared top-level argument parser.""" + parser.add_argument("--remote-store", dest="remoteStore", metavar="STORE", + default="", required=True, help=remote_help) + parser.add_argument("--insecure", dest="insecure", action="store_true", + help="Use http instead of https for the reapi:// endpoint.") + parser.add_argument("--ac-store", dest="acStore", default="", metavar="STORE", + help="Separate reapi:// ledger store (Action Cache + reconstruction " + "inputs). Defaults to --remote-store.") + parser.add_argument("-a", "--architecture", dest="architecture", metavar="ARCH", + default=detected_arch, help=arch_help) + parser.add_argument("-w", "--work-dir", dest="workDir", default=work_dir_default, + help=work_dir_help) + # Signature enforcement (consume side). Default "warn": verify against the + # keyring but only log failures, so unsigned legacy stores keep working; "off" + # skips verification, "require" fails closed. Ignored by upload/migrate, which + # produce rather than consume. + parser.add_argument("--require-signature", dest="requireSignature", + choices=("off", "warn", "require"), default="warn", + help="Verify Action Cache signatures against --trusted-keys: " + "'warn' (default; log unverified entries), 'off' (skip), " + "or 'require' (fail closed on any unverified entry in the " + "closure).") + parser.add_argument("--trusted-keys", dest="trustedKeys", default="", metavar="KEYRING", + help="Path to the JSON keyring of trusted signing keys. This " + "REPLACES the defaults, which are the keyring shipped with " + "alibuild merged with keyring.json from the alidist checkout " + "(if any).") + + +class SignatureChecker: + """Consume-side signature enforcement, shared by install and reconstruct. + + Wraps the (policy, keyring) pair and turns the pure results of + ``signing.evaluate_closure`` into alibuild log output and, under ``require``, a + fatal error. ``check_blob`` additionally binds the *bytes* to the signed digest + -- a signature over an ``outputDigest`` only means something once we confirm the + downloaded blob actually hashes to it. Only constructed when policy is not + ``off`` (see :func:`signature_checker`), so call sites can guard on ``if checker``. + """ + + def __init__(self, policy, keyring): + self.policy = policy + self.keyring = keyring + + # Above this many warn-level problems, report one aggregated line per distinct + # reason instead of one per package. A mixed store during rollout means whole + # closures are unsigned, and a hundred identical warnings buries anything else. + # Errors are always listed in full: under 'require' the detail is the point. + WARN_DETAIL_LIMIT = 5 + + def check_closure(self, entries): + """Verify a whole dependency closure. ``entries`` is an iterable of AC + entries; warns per unverified entry and dies if any fail under ``require``.""" + from alibuild_helpers import signing + pairs = [((entry.get("action") or {}).get("package", "?"), entry) + for entry in entries] + allowed, problems = signing.evaluate_closure(pairs, self.keyring, self.policy) + warns = [(package, reason) for package, level, reason in problems if level == "warn"] + for package, level, reason in problems: + if level != "warn": + error("Signature check for %s: %s", package, reason) + if len(warns) > self.WARN_DETAIL_LIMIT: + by_reason = {} + for package, reason in warns: + by_reason.setdefault(reason, []).append(package) + for reason, packages in sorted(by_reason.items()): + warning("Signature check: %d of %d packages in the closure %s (%s%s)", + len(packages), len(pairs), reason, ", ".join(sorted(packages)[:3]), + ", ..." if len(packages) > 3 else "") + else: + for package, reason in warns: + warning("Signature check for %s: %s", package, reason) + dieOnError(not allowed, + "Signature verification failed under --require-signature=require; " + "refusing to install untrusted artifacts.") + + def check_blob(self, path, algo, content_hash, entry=None): + """Confirm downloaded bytes hash to the digest the AC entry claims. + + Fatal when that claim is trustworthy -- policy ``require``, or an entry + carrying a valid signature -- because then a mismatch means the bytes were + swapped after signing, which is the attack this whole mechanism exists to + stop. For an *unsigned* entry under ``warn`` the digest is itself an + unverified claim (and migrated legacy entries can carry stale ones), so a + mismatch is reported without blocking: refusing there would fail closed under + a policy whose entire contract is that it does not. + """ + from alibuild_helpers import signing + actual = file_digest(path, algo) + if actual == content_hash: + return + trustworthy = self.policy == signing.POLICY_REQUIRE or ( + entry is not None and signing.verify(entry, self.keyring)[0]) + detail = ("Downloaded blob hashes to %s:%s but the Action Cache entry " + "expects %s:%s" % (algo, actual, algo, content_hash)) + dieOnError(trustworthy, detail + " -- refusing to use it.") + warning("%s; the entry is not signed by a trusted key, so neither the bytes " + "nor the claim are verified (policy=%s).", detail, self.policy) + + +def keyring_sources(args): + """Keyrings to verify against, in merge order. + + An explicit ``--trusted-keys`` *replaces* the set -- the escape hatch for + testing and air-gapped setups. Otherwise the keyring bundled with alibuild is + always consulted, plus ``keyring.json`` from the alidist checkout when there is + one (``args.configDir`` for build, ``args.alidist`` for reconstruct). + + The bundled keyring is what makes the recipe-free ``install`` verify anything + at all: it has no alidist, so before it shipped, the default ``warn`` policy + silently checked nothing. alidist's copy still matters -- it can add keys + without cutting an alibuild release -- and merging cannot widen trust, so a key + revoked in the shipped keyring stays revoked (see signing.merge_keyrings). + """ + from alibuild_helpers import signing + explicit = getattr(args, "trustedKeys", "") or "" + if explicit: + return [explicit] + sources = [path for path in (signing.bundled_keyring_path(),) + if os.path.exists(path)] + alidist = getattr(args, "configDir", "") or getattr(args, "alidist", "") or "" + if alidist and os.path.exists(os.path.join(alidist, "keyring.json")): + sources.append(os.path.join(alidist, "keyring.json")) + return sources + + +def signature_checker(args): + """Build a :class:`SignatureChecker` from consume-side args, or ``None`` when + verification is not in effect: policy ``off``, or policy ``warn`` (the default) + with no keyring found at all. Policy ``require`` with no keyring fails fast.""" + from alibuild_helpers import signing + policy = getattr(args, "requireSignature", signing.POLICY_WARN) + if policy == signing.POLICY_OFF: + return None + sources = [path for path in keyring_sources(args) if os.path.exists(path)] + if not sources: + dieOnError(policy == signing.POLICY_REQUIRE, + "--require-signature=require needs a keyring, but none was found: " + "pass --trusted-keys (the keyring shipped with alibuild is missing).") + debug("No signing keyring found; skipping signature verification (policy=%s).", + policy) + return None + try: + keyring = signing.load_keyrings(sources) + except RuntimeError as exc: + # 'cryptography' is an optional extra; without it we cannot verify. Under + # 'require' that must fail closed, but the default 'warn' degrades to a + # warning so the optional dependency stays genuinely optional. + dieOnError(policy == signing.POLICY_REQUIRE, + "--require-signature=require needs the 'cryptography' package: " + "pip install alibuild[signing] (%s)" % exc) + warning("Signature verification requested but 'cryptography' is not installed; " + "skipping (pip install alibuild[signing]).") + return None + return SignatureChecker(policy, keyring) + + +class REAPIRemoteSync(Boto3RemoteSync): + """S3 remote store using a REAPI-style Action Cache (AC) + CAS layout. + + Unlike Boto3RemoteSync (scheme ``b3://``), which stores each tarball under its + *action* hash and hardcodes the CERN endpoint, this backend (scheme + ``reapi://``): + + * stores the tarball and recipe bytes content-addressed under + ``cas///``, so equivalent builds (e.g. tag aliases that + share a commit) deduplicate to a single blob; + * writes a small Action Cache entry ``ac///.json`` recording + how the tarball was produced (recipe, commit, dependency action hashes, + build environment), so the CAS can be reconstructed from it; + * keeps the legacy ``TARS/store`` + symlink + ``.manifest`` layout, with the + store object as an S3 redirect to the CAS blob, so the existing publisher + and HttpRemoteSync keep working without duplicating bytes; + * parameterises the endpoint, so it works against AWS, MinIO, Ceph RGW and + CERN. + + URL form: ``reapi:///``. The endpoint scheme defaults + to https; pass ``insecure=True`` (aliBuild ``--insecure``) to use http, e.g. + for a local MinIO. + + See REMOTE_STORE_CAS_AC.md for the full design. + """ + + CAS_ALGO = "sha256" + # What a bare "reapi://" means: the standard three-bucket layout, so + # the common deployment needs no bucket spelled out anywhere. Naming a bucket + # in the URL opts out of all of it -- a single-bucket store stays one bucket, + # as it was before these defaults existed. + DEFAULT_ENDPOINT_HOST = "s3.cern.ch" + DEFAULT_CAS_BUCKET = "alibuild-cas" + DEFAULT_AC_BUCKET = "alibuild-ac" + DEFAULT_LEGACY_BUCKET = "alibuild-repo" + # Where a consumer reaches DEFAULT_CAS_BUCKET. Needed because the default + # layout puts the legacy links in a different bucket from the blobs, which + # makes the redirect absolute; without it a bare reapi:// would die asking + # for --cas-public-url and the default would be unusable. + DEFAULT_CAS_PUBLIC_URL = "https://s3.cern.ch/swift/v1/alibuild-cas" + # Object-tag lifecycle for the artifact store (see the bucket lifecycle rule, + # ali-marathon/s3/alibuild-cas-lifecycle.xml, and REMOTE_STORE_CAS_AC.md): + # objects tagged retention=ephemeral expire 90 days after last-modified; + # untagged / retention=permanent are kept forever. + RETENTION_TAG_KEY = "retention" + EPHEMERAL_TTL_DAYS = 90 + REFRESH_WITHIN_DAYS = 30 # touch (LRU-refresh) ephemeral objects within this of expiry + + def __init__(self, remoteStore, writeStore, architecture, workdir, + insecure=False, acStore="", acWriteStore="", storage="ephemeral", + sign_url="", sign_token="", sign_token_file="", + signer="alibuild", legacyStore="", casPublicUrl="") -> None: + scheme = "http" if insecure else "https" + read_endpoint, self.remoteStore = self._parse_reapi_url(remoteStore, scheme) + write_endpoint, self.writeStore = self._parse_reapi_url(writeStore, scheme) + # "reapi://" with no bucket selects the default layout below. Tracked + # rather than inferred later, because "no bucket given" and "bucket given + # that happens to be alibuild-cas" have to behave differently: only the + # former also moves the AC and legacy stores off the artifact bucket. + default_layout = False + if remoteStore and not self.remoteStore: + self.remoteStore, default_layout = self.DEFAULT_CAS_BUCKET, True + if writeStore and not self.writeStore: + self.writeStore, default_layout = self.DEFAULT_CAS_BUCKET, True + self.architecture = architecture + self.workdir = workdir + # Dep hashes already confirmed present in the ledger (see assert_deps_in_ledger). + self._deps_seen = set() + # Read and write endpoints are normally the same host; prefer the read one. + self.endpoint_url = read_endpoint or write_endpoint + # The artifact store (self.remoteStore/self.writeStore) holds the large, + # deletable/regenerable output tarballs. The *ledger* store holds the small, + # keep-forever set: Action Cache entries plus the reconstruction-input blobs + # (recipe, source, refs). They have different lifetimes, so they can live in + # different buckets with different retention policies -- deleting the + # artifact store is then safe, since reconstruct rebuilds it from the ledger. + # The ledger defaults to the artifact store (single-bucket setups), and must + # share the endpoint (one S3 client; only the bucket differs). + ac_read_ep, self.acRemoteStore = ( + self._parse_reapi_url(acStore, scheme) if acStore else + ("", self.DEFAULT_AC_BUCKET if default_layout else self.remoteStore)) + ac_write_ep, self.acWriteStore = ( + self._parse_reapi_url(acWriteStore, scheme) if acWriteStore else + ("", self.DEFAULT_AC_BUCKET if default_layout else self.writeStore)) + # Where the legacy TARS//... link and store objects go. Defaults to + # the artifact store, which is the single-bucket layout everything used + # before. Point it at an existing legacy repo to keep publishing a classic + # tree there while the bytes live content-addressed here: consumers that + # only know TARS/ keep working, and nothing is stored twice. + # + # The redirect written into that tree then has to be ABSOLUTE (see + # _cas_redirect), because a relative one resolves against whichever bucket + # the client is reading from. + legacy_ep, self.legacyWriteStore = ( + self._parse_reapi_url(legacyStore, scheme) if legacyStore else + ("", self.DEFAULT_LEGACY_BUCKET if default_layout else self.writeStore)) + # Reads of the legacy tree follow its writes. Without this the split is + # write-only: _s3_listdir and fetch_symlinks would still list the artifact + # bucket, whose legacy tree stops being updated the moment the split is + # turned on. Revision assignment reads that listing to learn which revisions + # are taken, so it would see none, reassign one that already exists, and then + # be refused by the ownership check when it tried to claim the link -- a + # build colliding with its own publication from the day before. + self.legacyReadStore = (self.legacyWriteStore + if self.legacyWriteStore != self.writeStore + else self.remoteStore) + # Where a CONSUMER reaches the CAS bucket. Only needed when the legacy tree + # is in a different bucket, because only then is the redirect absolute. + self.casPublicUrl = casPublicUrl or (self.DEFAULT_CAS_PUBLIC_URL + if default_layout else "") + dieOnError(self.legacyWriteStore != self.writeStore and not self.casPublicUrl, + "--legacy-links-store needs --cas-public-url: the store objects " + "written there redirect to the CAS bucket by absolute URL, and " + "the address this build uploads through is not necessarily one " + "a consumer can reach") + for endpoint in (ac_read_ep, ac_write_ep, legacy_ep): + dieOnError(bool(endpoint) and bool(self.endpoint_url) and + endpoint != self.endpoint_url, + "the AC/ledger and legacy stores must share the endpoint with " + "the artifact store (%s); a cross-endpoint split is not " + "supported" % self.endpoint_url) + # Artifact retention: "ephemeral" (default; LRU-expired by the bucket + # lifecycle) or "permanent" (pinned, and promotes any ephemeral blob it + # reuses). The ledger store is never tagged -- it is always keep-forever. + dieOnError(storage not in ("ephemeral", "permanent"), + "storage must be 'ephemeral' or 'permanent', not %r" % storage) + self.storage = storage + # Signing config for Action Cache uploads. When sign_url is set, each AC + # entry is signed via the security-proxy sign route before it is written to + # the ledger (see _sign_ac_entry). The build path enforces "always sign" + # (fail closed unless --no-sign); a bare instantiation stays unsigned. + self.sign_url = sign_url + self.sign_token = sign_token + self.sign_token_file = sign_token_file + self.signer = signer + # Consume-side signature verification. Set by the build path (see build.py) + # to a SignatureChecker; when set, fetch_tarball verifies freshly downloaded + # tarballs against the keyring. None means no verification (the default). + self.verify_checker = None + self._s3_init() + + def _cas_redirect(self, cas_path): + """The x-amz-website-redirect-location to write on a legacy store object. + + Relative while the legacy tree lives in the artifact bucket -- which is the + default, and what every store written so far contains. Absolute once they + differ, because the client resolves a relative target against the bucket it + is READING from, and that is the legacy one: it would look for the blob in + a bucket that does not have it, and get a 404 in the middle of a download. + + The absolute form is NOT derived from our own endpoint. We may be writing + through something the readers cannot reach -- a credential broker on + loopback with a per-allocation port, for instance -- and this URL is baked + into a persistent object that outlives the build. It has to be the address + a CONSUMER uses, which only the caller knows, so it is required rather than + guessed.""" + if self.legacyWriteStore == self.writeStore: + return "/" + cas_path + return "%s/%s" % (self.casPublicUrl.rstrip("/"), cas_path) + + @staticmethod + def _parse_reapi_url(url, scheme): + """Split ``reapi:///`` into ``(endpoint_url, bucket)``. + + Both parts are optional: a bare ``reapi://`` means the default endpoint and + the default bucket layout, so the common deployment spells out neither.""" + if not url: + return "", "" + host, _, bucket = re.sub("^reapi://", "", url).partition("/") + return ("%s://%s" % (scheme, host or REAPIRemoteSync.DEFAULT_ENDPOINT_HOST), + bucket.strip("/")) + + def upload_symlinks_and_tarball(self, spec) -> None: + """Publish a built package, refusing up front if its closure would dangle. + + The check has to happen before the base implementation writes anything: it + claims the per-package link and uploads the legacy symlinks *before* calling + _upload_tarball, so aborting from in there leaves a claimed link with no + artifact behind it for a later build to clean up.""" + ac_entry = spec.get("ac_entry") + if ac_entry: + self.assert_deps_in_ledger(ac_entry["action"]) + super().upload_symlinks_and_tarball(spec) + + def _upload_tarball(self, spec, tar_path) -> None: + """Store the tarball content-addressed in the CAS, write its Action Cache + entry and the recipe blob, and leave the legacy store object as a redirect + to the CAS blob. + + Ledger first, bytes last. Both orders leave a window if the process dies + mid-publish, but they fail very differently. Blob-then-entry leaves an + unreferenced blob and no entry: the package looks unbuilt to the store while + the work area says it is built, so build.py skips it on every later run and it + is never published -- unrecoverable without deleting the local artifacts by + hand. Entry-then-blob leaves an entry whose blob is missing, which is a state + the design already handles: fetch_tarball degrades to a rebuild, and + reconstruct repairs it in bulk. The closure is never dangling either way, + because dependents reference the entry.""" + local_tar = os.path.join(self.workdir, tar_path) + content_hash = file_digest(local_tar, self.CAS_ALGO) + cas_path = resolve_cas_path(content_hash, self.CAS_ALGO) + output_digest = "%s:%s" % (self.CAS_ALGO, content_hash) + + ac_entry = spec.get("ac_entry") + if ac_entry: + # 1. Recipe blob in the *ledger* store (keep-forever reconstruction input), + # before the entry that names it: _recipe_intact checks it is there. + recipe_digest = ac_entry["action"]["recipeDigest"].split(":", 1)[-1] + recipe_cas = resolve_cas_path(recipe_digest, self.CAS_ALGO) + if not self._exists(self.acWriteStore, recipe_cas): + # Store the full recipe (header + body); its digest is recipeDigest. + recipe_text = spec.get("fullRecipe") or spec.get("recipe") or "" + self.s3.put_object(Bucket=self.acWriteStore, Key=recipe_cas, + Body=recipe_text.encode("utf-8", "ignore")) + + # 2. Action Cache entry. The output digest is computed from the local + # tarball above, so it is known before the bytes are uploaded. + ac_entry = dict(ac_entry, result={ + "tarball": os.path.basename(tar_path), + "outputDigest": output_digest, + "size": os.path.getsize(local_tar), + }) + # Sign it (over the just-set output digest) before it is written, so the + # ledger only ever holds signed entries when signing is configured. + ac_entry = self._sign_ac_entry(ac_entry) + ac_path = resolve_ac_path(self.architecture, ac_entry["action"]["actionHash"]) + self.s3.put_object(Bucket=self.acWriteStore, Key=ac_path, + Body=json.dumps(ac_entry, indent=2, sort_keys=True) + .encode("utf-8"), + ContentType="application/json") + + # 3. Content-addressed tarball bytes. Skip the upload if an identical blob + # already exists -- this is where equivalent action hashes deduplicate -- + # but promote it if this is a permanent build reusing an ephemeral blob. + if self._exists(self.writeStore, cas_path): + self._maybe_promote(cas_path) + debug("CAS already has %s, not re-uploading bytes", cas_path) + else: + self.s3.upload_file(Bucket=self.writeStore, Key=cas_path, Filename=local_tar, + ExtraArgs={"Tagging": self._retention_tagging()}, + Callback=byte_progress("upload " + cas_path, + os.path.getsize(local_tar))) + + if not ac_entry: + debug("No Action Cache entry for %s; uploaded CAS blob only", tar_path) + + # 4. Legacy store object: a redirect to the CAS blob, so the existing + # publisher / HttpRemoteSync resolve it without storing the bytes twice. + # Goes to the legacy store, which is the artifact store unless + # --legacy-links-store points it elsewhere. + self.s3.put_object(Bucket=self.legacyWriteStore, Key=tar_path, + Body=os.fsencode(cas_path), + WebsiteRedirectLocation=self._cas_redirect(cas_path)) + + def _sign_ac_entry(self, ac_entry): + """Sign an AC entry (with its result already set) via the security-proxy + sign route, returning a copy with a ``signatures`` list and schemaVersion 3. + + A no-op when signing is not configured (``sign_url`` empty) -- the build path + is what guarantees "always sign" by refusing to upload unsigned unless + --no-sign; a direct instantiation without sign_url uploads unsigned.""" + if not self.sign_url: + return ac_entry + from alibuild_helpers import signing + signature = signing.sign_via_proxy(ac_entry, self.sign_url, + self._current_sign_token(), self.signer) + signed = dict(ac_entry, schemaVersion=3) + signed["signatures"] = [signature] + return signed + + def _current_sign_token(self): + """The credential to present to the sign route, resolved *per request*. + + ``sign_token_file`` is read fresh every time rather than cached, because the + credential may be short-lived and refreshed in place underneath us: a Nomad + workload-identity JWT has a TTL of minutes, while a release build signs Action + Cache entries for as long as it runs. Capturing it once at startup -- which is + what a literal ``--sign-token`` does -- would work for the first upload of a + long build and fail for the rest. + """ + if self.sign_token_file: + try: + with open(self.sign_token_file) as handle: + token = handle.read().strip() + except OSError as exc: + dieOnError(True, "cannot read the signing credential from %s: %s" + % (self.sign_token_file, exc)) + dieOnError(not token, "the signing credential file %s is empty; if it holds a " + "short-lived token, it may not have been renewed yet." + % self.sign_token_file) + return token + dieOnError(not self.sign_token, + "signing a reapi:// upload needs a credential: pass --sign-token or " + "--sign-token-file (or --no-sign to upload unsigned).") + return self.sign_token + + def _verify_download(self, spec, entry, tar_path, algo="", content_hash=""): + """Verify a freshly downloaded tarball before the build reuses it, when a + verifier is configured (build path). The AC entry must carry a trusted + signature (policy-enforced) and the bytes must hash to the signed output + digest. A missing AC entry (legacy/unsigned tarball) is treated as unsigned: + warns, or fails under --require-signature=require.""" + if not self.verify_checker: + return + if entry is None: + self.verify_checker.check_closure([{"action": {"package": spec["package"]}}]) + return + self.verify_checker.check_closure([entry]) + if content_hash: + self.verify_checker.check_blob(tar_path, algo, content_hash, entry) + + def _exists(self, bucket, key): + """Return whether key exists in the given bucket.""" + from botocore.exceptions import ClientError + debug("S3 head_object %s/%s", bucket, key) + try: + self.s3.head_object(Bucket=bucket, Key=key) + except ClientError: + return False + return True + + # --- Ledger store: AC entries + reconstruction-input blobs (recipe/source/ + # refs). Small, keep-forever; read from acRemoteStore, write acWriteStore. + + def read_ac_entry(self, action_hash): + """Return the parsed Action Cache entry for action_hash, or None if absent.""" + from botocore.exceptions import ClientError + ac_path = resolve_ac_path(self.architecture, action_hash) + debug("S3 get_object %s/%s (read AC)", self.acRemoteStore, ac_path) + try: + obj = self.s3.get_object(Bucket=self.acRemoteStore, Key=ac_path) + except ClientError: + return None + return json.loads(obj["Body"].read()) + + def is_published(self, action_hash): + """Whether an action is fully published: its Action Cache entry exists *and* the + CAS blob that entry names is present. + + Both halves matter. A locally built package proves nothing about the store: an + earlier run may have been interrupted between building and publishing, or have + built against a different store, and the entry would be missing. And an entry + whose blob is gone (expired ephemeral artifact, or a publish interrupted between + entry and bytes) is not something a dependent can be published against either. + + Two HEAD-class requests, no download -- the blob is checked by presence, since it + is content-addressed and its key is its own digest.""" + entry = self.read_ac_entry(action_hash) + if entry is None: + return False + algo, _, content_hash = (entry.get("result") or {}).get("outputDigest", "").partition(":") + if not content_hash: + # validate-system nodes produce no tarball: the entry alone is the artifact. + return (entry.get("action") or {}).get("kind") == "validate-system" + return self._exists(self.remoteStore, resolve_cas_path(content_hash, algo)) + + def download_blob(self, content_hash, dest, algo="sha256"): + """Download a ledger (input) blob -- e.g. a source bundle -- to dest.""" + self.s3.download_file(Bucket=self.acRemoteStore, + Key=resolve_cas_path(content_hash, algo), Filename=dest) + + def read_blob(self, content_hash, algo="sha256"): + """Return the bytes of a ledger (input) blob -- e.g. a recipe or refs blob.""" + return self.s3.get_object(Bucket=self.acRemoteStore, + Key=resolve_cas_path(content_hash, algo))["Body"].read() + + def put_file_as_blob(self, path, algo="sha256"): + """Upload a ledger (input) blob from a file -- e.g. a source bundle. Dedups. + Returns the content hash.""" + content_hash = file_digest(path, algo) + cas_path = resolve_cas_path(content_hash, algo) + if not self._exists(self.acWriteStore, cas_path): + self.s3.upload_file(Bucket=self.acWriteStore, Key=cas_path, Filename=path) + return content_hash + + def put_bytes_as_blob(self, data, algo="sha256"): + """Upload an in-memory ledger (input) blob -- e.g. a refs blob. Dedups.""" + content_hash = hashlib.new(algo, data).hexdigest() + cas_path = resolve_cas_path(content_hash, algo) + if not self._exists(self.acWriteStore, cas_path): + self.s3.put_object(Bucket=self.acWriteStore, Key=cas_path, Body=data) + return content_hash + + def read_object_json(self, key): + """Return the JSON object at key in the ledger store, or None.""" + from botocore.exceptions import ClientError + try: + obj = self.s3.get_object(Bucket=self.acRemoteStore, Key=key) + except ClientError: + return None + return json.loads(obj["Body"].read()) + + def write_object_json(self, key, obj): + """Write a small JSON object at key in the ledger store.""" + self.s3.put_object(Bucket=self.acWriteStore, Key=key, + Body=json.dumps(obj, sort_keys=True).encode("utf-8"), + ContentType="application/json") + + # --- Artifact store: large, deletable/regenerable output tarball blobs; + # read from remoteStore, write writeStore. + + def _retention_tagging(self): + """The retention tag to apply to freshly uploaded artifact blobs.""" + return "%s=%s" % (self.RETENTION_TAG_KEY, self.storage) + + def _retention_of(self, bucket, key): + """Return the retention tag value of an object, or None if untagged/absent.""" + from botocore.exceptions import ClientError + try: + tags = self.s3.get_object_tagging(Bucket=bucket, Key=key)["TagSet"] + except ClientError: + return None + return next((t["Value"] for t in tags if t["Key"] == self.RETENTION_TAG_KEY), None) + + def _maybe_promote(self, cas_path): + """When uploading as 'permanent' and the blob already exists tagged + ephemeral, promote it to permanent so it is no longer LRU-expired.""" + if self.storage != "permanent": + return + if self._retention_of(self.writeStore, cas_path) == "ephemeral": + self.s3.put_object_tagging( + Bucket=self.writeStore, Key=cas_path, + Tagging={"TagSet": [{"Key": self.RETENTION_TAG_KEY, "Value": "permanent"}]}) + debug("Promoted %s from ephemeral to permanent", cas_path) + + def put_artifact_blob(self, path, algo="sha256"): + """Upload an output tarball to the artifact CAS keyed by content hash, + tagged with the current retention. Dedups; promotes on a permanent reuse.""" + content_hash = file_digest(path, algo) + cas_path = resolve_cas_path(content_hash, algo) + if self._exists(self.writeStore, cas_path): + self._maybe_promote(cas_path) + else: + size = os.path.getsize(path) + debug("S3 upload_file %s/%s (%d bytes)", self.writeStore, cas_path, size) + self.s3.upload_file(Bucket=self.writeStore, Key=cas_path, Filename=path, + ExtraArgs={"Tagging": self._retention_tagging()}, + Callback=byte_progress("upload " + cas_path, size)) + return content_hash + + def _touch_if_expiring(self, cas_path): + """LRU-refresh: if the blob is ephemeral and within REFRESH_WITHIN_DAYS of + its EPHEMERAL_TTL_DAYS expiry, copy it onto itself (preserving the tag) to + reset last-modified. Best-effort and only when we can write the same bucket.""" + from botocore.exceptions import ClientError + if not self.writeStore or self.writeStore != self.remoteStore: + return + try: + if self._retention_of(self.remoteStore, cas_path) != "ephemeral": + return + head = self.s3.head_object(Bucket=self.remoteStore, Key=cas_path) + age_days = (datetime.now(timezone.utc) - head["LastModified"]).days + if age_days < self.EPHEMERAL_TTL_DAYS - self.REFRESH_WITHIN_DAYS: + return + debug("Refreshing ephemeral %s (%d days old)", cas_path, age_days) + self.s3.copy_object(Bucket=self.writeStore, Key=cas_path, + CopySource={"Bucket": self.remoteStore, "Key": cas_path}, + MetadataDirective="COPY", TaggingDirective="COPY") + except ClientError as exc: + debug("Could not refresh %s: %s", cas_path, exc) + + def download_artifact(self, content_hash, dest, algo="sha256"): + """Download an output tarball blob from the artifact store to dest, LRU- + refreshing it if it is ephemeral and close to expiry.""" + cas_path = resolve_cas_path(content_hash, algo) + self._touch_if_expiring(cas_path) + debug("S3 download_file %s/%s", self.remoteStore, cas_path) + self.s3.download_file(Bucket=self.remoteStore, Key=cas_path, Filename=dest) + + def artifact_blob_exists(self, content_hash, algo="sha256"): + """Return whether an output tarball blob exists in the artifact store.""" + return self._exists(self.remoteStore, resolve_cas_path(content_hash, algo)) + + def is_fully_migrated(self, architecture, package, tarball, verrev=None): + """Whether `tarball` is already in the reapi store, by either route. + + migrate_put writes the per-package link LAST, so its presence means the whole + entry (tarball + recipe + AC + redirect + link) is complete -- a cheap + completion marker for idempotent re-runs, and one the AC cannot give because + it is written earlier. That is the first check. + + It is not sufficient alone. A package published by a BUILD puts its link + wherever that build's --legacy-links-store points, and migrate has no such + option: it can only look in its own store. So a package the release already + published perfectly -- blob, recipe, AC entry, redirect stub in the shared + legacy bucket -- looked unmigrated here and was retried every round. It then + downloaded the 78-byte stub over --read-store (a plain GET, which does not + follow the redirect) and reported "file could not be opened successfully". + Seventeen packages of the ubuntu2204 O2Suite closure did that, every round. + + Hence the fallback: ask the Action Cache, which is the same question a BUILD + asks before reusing something (is_published: entry present AND blob present). + If that holds, migrating again would rewrite identical bytes and an identical + entry, and could only add a legacy link in a bucket no consumer of this store + reads. verrev is optional so older callers keep the link-only behaviour. + """ + if self._exists(self.legacyWriteStore, + os.path.join(resolve_links_path(architecture, package), tarball)): + return True + if not verrev: + return False + version, _, revision = verrev.rpartition("-") + action_hash = self.resolve_action_hash(package, version, revision or None) + return bool(action_hash) and self.is_published(action_hash) + + def iter_ac_entry_hashes(self, architecture): + """Yield the action hash of every Action Cache entry under ac// in the + ledger. Used to walk a migrated set for source enrichment without the old + store or a closure enumeration.""" + prefix = "ac/%s/" % architecture + pages = self.s3.get_paginator("list_objects_v2") \ + .paginate(Bucket=self.acRemoteStore, Prefix=prefix) + for page in pages: + for item in page.get("Contents", ()): + key = item["Key"] + if key.endswith(".json"): + yield os.path.basename(key)[:-len(".json")] + + def assert_deps_in_ledger(self, action): + """Refuse to publish an entry whose dependencies are not in the ledger. + + Nothing else notices a hole: deps are serialised as hashes without being + resolved, and the consume-side check_closure only evaluates signature policy + over entries a client already resolved -- an absent dep never becomes an entry. + reconstruct then dies on the closure long afterwards, and AC entries are + keep-forever, so the hole is permanent. + + The usual cause is a package that was already built locally and therefore never + uploaded (build.py skips it as "in sync with whatever remote store", inferring + remote presence from local), or an earlier run that failed part-way through a + closure.""" + missing = [dep for dep in action.get("deps", ()) + if dep["actionHash"] not in self._deps_seen and + not self._exists(self.acWriteStore, + resolve_ac_path(self.architecture, dep["actionHash"]))] + dieOnError(bool(missing), + "refusing to publish %s: %d of its dependencies have no Action Cache " + "entry in %s, so the closure would be unresolvable:\n%s\n" + "Rebuild them into the store -- delete their install dir and tarballs " + "(TARS/, including the store/ blob) and build again; the action hash is " + "unchanged, so this entry's references stay valid." % ( + action.get("package", "?"), len(missing), self.acWriteStore, + "\n".join(" %s %s" % (dep["actionHash"], dep["package"]) + for dep in missing))) + # Only reached when all resolved: remember them so a closure of N packages + # sharing dependencies costs one HEAD per distinct dep, not per dependent. + self._deps_seen.update(dep["actionHash"] for dep in action.get("deps", ())) + + def put_ac_entry(self, entry, recipe_text="", sign=False): + """Write an Action Cache entry that has no output tarball (e.g. a + 'validate-system' action for a system/prefer_system package): its recipe blob + (a reconstruction input) plus the entry itself, both in the ledger. No CAS + blob, redirect or link -- there is no artifact. Idempotent (dedups the recipe, + overwrites the entry). No-op on a read-only store (no ledger write target). + + With sign=True (and signing configured) the entry is signed over its + recipeDigest -- signed_payload falls back to it when there is no output digest + -- so a validate-system node in a closure is trusted like any other under + --require-signature. Callers that must never sign (migrate: legacy provenance + reconstructed after the fact) leave sign=False.""" + if not self.acWriteStore: + return + if sign: + entry = self._sign_ac_entry(entry) + action = entry["action"] + recipe_digest = action.get("recipeDigest", "").split(":", 1)[-1] + if recipe_digest: + recipe_cas = resolve_cas_path(recipe_digest, self.CAS_ALGO) + if not self._exists(self.acWriteStore, recipe_cas): + debug("S3 put_object %s/%s (recipe, %s)", self.acWriteStore, recipe_cas, + action.get("kind", "build")) + self.s3.put_object(Bucket=self.acWriteStore, Key=recipe_cas, + Body=(recipe_text or "").encode("utf-8", "ignore")) + ac_path = resolve_ac_path(self.architecture, action["actionHash"]) + debug("S3 put_object %s/%s (%s AC entry)", self.acWriteStore, ac_path, + action.get("kind", "build")) + self.s3.put_object(Bucket=self.acWriteStore, Key=ac_path, + Body=json.dumps(entry, indent=2, sort_keys=True).encode("utf-8"), + ContentType="application/json") + + def update_ac_entry(self, entry): + """Overwrite an existing Action Cache entry in the ledger -- e.g. to add a + source snapshot to an already-migrated release. The entry is keyed by its + own action hash, so this rewrites in place and preserves the result block.""" + ac_path = resolve_ac_path(self.architecture, entry["action"]["actionHash"]) + debug("S3 put_object %s/%s (update AC entry)", self.acWriteStore, ac_path) + self.s3.put_object(Bucket=self.acWriteStore, Key=ac_path, + Body=json.dumps(entry, indent=2, sort_keys=True).encode("utf-8"), + ContentType="application/json") + + @staticmethod + def _highest_revision(candidates): + """Pick the entry with the highest revision: numeric first, else lexicographic.""" + def sort_key(item): + rev = item[0] + return (1, int(rev)) if rev.isdigit() else (0, rev) + return max(candidates, key=sort_key) + + def resolve_action_hash(self, package, version, revision=None): + """Resolve a human label (package, version[, revision]) to an action hash. + + Primary path: the per-package symlink objects written at upload time. Those + only exist for actions that produced a *tarball*, so a validate-system node + (``make``, ``yacc-like``, ...) has none and used to be unaddressable by name + -- the failure even pointed at the CAS, which is the wrong place to look for + something that deliberately has no artifact. Hence the fallback: scan the + ledger, which describes every action whether or not it produced bytes. + + With no revision, the highest available one for the version is chosen. + Returns None if nothing matches. + """ + hit = self._resolve_via_links(package, version, revision) + return hit if hit is not None else self._resolve_via_ac(package, version, revision) + + def _resolve_via_ac(self, package, version, revision=None): + """Resolve a label by scanning the Action Cache. O(entries for the arch), so + this is the fallback rather than the primary path.""" + prefix = os.path.dirname(resolve_ac_path(self.architecture, "0" * 40)).rsplit("/", 1)[0] + pages = self.s3.get_paginator("list_objects_v2") \ + .paginate(Bucket=self.acRemoteStore, Prefix=prefix.rstrip("/") + "/") + candidates = [] + for page in pages: + for item in page.get("Contents", ()): + if not item["Key"].endswith(".json"): + continue + entry = self.read_ac_entry(os.path.basename(item["Key"])[:-len(".json")]) + action = (entry or {}).get("action") or {} + if action.get("package") != package or str(action.get("version")) != str(version): + continue + if revision is not None and str(action.get("revision")) != str(revision): + continue + candidates.append((str(action.get("revision", "")), action.get("actionHash"))) + if not candidates: + return None + return self._highest_revision(candidates)[1] + + def _resolve_via_links(self, package, version, revision=None): + links_path = resolve_links_path(self.architecture, package) + name_prefix = "%s-%s-" % (package, version) + name_suffix = ".%s.tar.gz" % self.architecture + candidates = [] # (revision, link key) + for key in self._s3_listdir(links_path): + name = os.path.basename(key) + if name.startswith(name_prefix) and name.endswith(name_suffix): + candidates.append((name[len(name_prefix):-len(name_suffix)], key)) + if revision is not None: + candidates = [(rev, key) for rev, key in candidates if rev == str(revision)] + if not candidates: + return None + _, link_key = self._highest_revision(candidates) + # link_key came out of _s3_listdir, which reads the legacy tree; its body has + # to be read from the same place. Reading it from the artifact bucket only + # worked while the two were one bucket -- the same mistake as the byte fetch + # in Boto3RemoteSync.fetch_tarball, in the one other spot that pairs a + # legacy key with a bucket. + target = os.fsdecode(self.s3.get_object(Bucket=self.legacyReadStore, + Key=link_key)["Body"].read()) + match = re.search(r"store/[0-9a-f]{2}/([0-9a-f]+)/", target) + return match.group(1) if match else None + + def migrate_put(self, ac_entry, tarball_path, recipe_text): + """Write a migrated legacy release into the reapi store: the tarball as a + CAS blob, the recovered recipe blob, the Action Cache entry, plus the legacy + store redirect and per-package link so it stays installable and + publisher-compatible. Returns the tarball's content hash. + + Migrated entries are intentionally left *unsigned*: their provenance is + reconstructed after the fact, so a signature would falsely assert a trusted + signer built them. They are consumed as legacy (tolerated under + --require-signature=warn, refused under require). Do not sign here.""" + action = ac_entry["action"] + arch, pkg = action["architecture"], action["package"] + action_hash = action["actionHash"] + tarball = "{package}-{version}-{revision}.{arch}.tar.gz".format(arch=arch, **action) + + # Tarball -> artifact store; recipe + AC -> ledger store. + content_hash = self.put_artifact_blob(tarball_path, self.CAS_ALGO) + cas_path = resolve_cas_path(content_hash, self.CAS_ALGO) + + recipe_digest = action["recipeDigest"].split(":", 1)[-1] + recipe_cas = resolve_cas_path(recipe_digest, self.CAS_ALGO) + if not self._exists(self.acWriteStore, recipe_cas): + debug("S3 put_object %s/%s (recipe)", self.acWriteStore, recipe_cas) + self.s3.put_object(Bucket=self.acWriteStore, Key=recipe_cas, + Body=(recipe_text or "").encode("utf-8", "ignore")) + + entry = dict(ac_entry, result={ + "tarball": tarball, + "outputDigest": "%s:%s" % (self.CAS_ALGO, content_hash), + "size": os.path.getsize(tarball_path), + }) + ac_path = resolve_ac_path(arch, action_hash) + debug("S3 put_object %s/%s (AC entry)", self.acWriteStore, ac_path) + self.s3.put_object(Bucket=self.acWriteStore, Key=ac_path, + Body=json.dumps(entry, indent=2, sort_keys=True).encode("utf-8"), + ContentType="application/json") + + store_key = os.path.join(resolve_store_path(arch, action_hash), tarball) + debug("S3 put_object %s/%s (store redirect)", self.writeStore, store_key) + self.s3.put_object(Bucket=self.legacyWriteStore, Key=store_key, + Body=os.fsencode(cas_path), + WebsiteRedirectLocation=self._cas_redirect(cas_path)) + + link_target = "../../%s/store/%s/%s/%s" % (arch, action_hash[:2], action_hash, tarball) + link_key = os.path.join(resolve_links_path(arch, pkg), tarball) + debug("S3 put_object %s/%s (link)", self.writeStore, link_key) + # The link stays RELATIVE: it points at the store object beside it in the + # same tree, and the client reads its body rather than following the header. + self.s3.put_object(Bucket=self.legacyWriteStore, Key=link_key, + Body=link_target.encode("utf-8"), + WebsiteRedirectLocation=link_target) + return content_hash + + def put_legacy_artifact(self, package, version, revision, tarball_path): + """Store a pre-provenance legacy tarball (no .meta.json, so no recipe can be + recovered) so its version-revision is reserved in the store and it stays + installable. Writes: the CAS blob (content-addressed), a *legacy* AC entry + (kind='legacy', keyed by the tarball's content hash in place of an action hash, + with no recipe or deps), and the legacy store redirect + per-package link. The + kind='legacy' entry distinguishes these preserved-but-non-reconstructable + artifacts from full builds and validate-system entries in the ledger. + Idempotent (content-addressed dedup). Returns the content hash.""" + arch = self.architecture + tarball = "{package}-{version}-{revision}.{arch}.tar.gz".format( + package=package, version=version, revision=revision, arch=arch) + content_hash = self.put_artifact_blob(tarball_path, self.CAS_ALGO) + cas_path = resolve_cas_path(content_hash, self.CAS_ALGO) + + # Legacy AC entry: no recipe/deps (no provenance), keyed by the content hash. + entry = { + "schemaVersion": 2, + "action": { + "kind": "legacy", "package": package, "version": version, + "revision": revision, "architecture": arch, "actionHash": content_hash, + }, + "result": { + "tarball": tarball, "outputDigest": "%s:%s" % (self.CAS_ALGO, content_hash), + "size": os.path.getsize(tarball_path), + }, + } + ac_path = resolve_ac_path(arch, content_hash) + debug("S3 put_object %s/%s (legacy AC entry)", self.acWriteStore, ac_path) + self.s3.put_object(Bucket=self.acWriteStore, Key=ac_path, + Body=json.dumps(entry, indent=2, sort_keys=True).encode("utf-8"), + ContentType="application/json") + + store_key = os.path.join(resolve_store_path(arch, content_hash), tarball) + debug("S3 put_object %s/%s (legacy store redirect)", self.writeStore, store_key) + self.s3.put_object(Bucket=self.legacyWriteStore, Key=store_key, + Body=os.fsencode(cas_path), + WebsiteRedirectLocation=self._cas_redirect(cas_path)) + + link_target = "../../%s/store/%s/%s/%s" % (arch, content_hash[:2], content_hash, tarball) + link_key = os.path.join(resolve_links_path(arch, package), tarball) + debug("S3 put_object %s/%s (legacy link)", self.writeStore, link_key) + # The link stays RELATIVE: it points at the store object beside it in the + # same tree, and the client reads its body rather than following the header. + self.s3.put_object(Bucket=self.legacyWriteStore, Key=link_key, + Body=link_target.encode("utf-8"), + WebsiteRedirectLocation=link_target) + return content_hash + + def rebaseline_ac_entry(self, entry, tarball_path, recipe_text=""): + """Re-baseline an existing AC entry onto a freshly rebuilt tarball whose + content hash differs from the recorded one (e.g. a legacy pre-normalisation + build). Writes the new CAS blob, then rewrites the entry's outputDigest and + the store redirect + link -- all keyed by the *unchanged* action hash, so it + is an in-place pointer swap. The new blob is stored *before* the AC entry is + repointed, so a failure never leaves the entry dangling; the old blob is left + orphaned (delete it separately via delete_artifact_blob). The retention of the + blob being replaced is preserved (untagged == keep-forever == permanent). + Returns (old_hash, new_hash, old_cas_path).""" + old_digest = (entry.get("result") or {}).get("outputDigest", "") + old_hash = old_digest.split(":", 1)[-1] + old_cas_path = resolve_cas_path(old_hash, self.CAS_ALGO) if old_hash else None + # Preserve the retention of the blob we are replacing (untagged => permanent). + prev_storage = self.storage + if old_cas_path: + self.storage = self._retention_of(self.writeStore, old_cas_path) or "permanent" + try: + new_hash = self.migrate_put( + {k: v for k, v in entry.items() if k != "result"}, tarball_path, recipe_text) + finally: + self.storage = prev_storage + return old_hash, new_hash, old_cas_path + + def delete_artifact_blob(self, content_hash, algo="sha256"): + """Delete an output tarball blob from the artifact store by content hash. + Used to reclaim the blob orphaned by a re-baseline. Safe to call on the + artifact store only -- ledger blobs are keep-forever and never deleted here.""" + cas_path = resolve_cas_path(content_hash, algo) + debug("S3 delete_object %s/%s (orphaned blob)", self.writeStore, cas_path) + self.s3.delete_object(Bucket=self.writeStore, Key=cas_path) + + def fetch_tarball(self, spec) -> None: + """Resolve the tarball via the Action Cache (action hash -> AC entry -> + output digest -> CAS blob). Falls back to the legacy store layout when no + AC entry exists, so mixed / migrating stores keep working.""" + from botocore.exceptions import ClientError + + # If we already have a tarball with any equivalent hash, don't hit S3. + for pkg_hash in spec["remote_hashes"]: + store_path = resolve_store_path(self.architecture, pkg_hash) + if glob.glob(os.path.join(self.workdir, store_path, "%s-*.tar.gz" % spec["package"])): + debug("Reusing existing tarball for %s@%s", spec["package"], pkg_hash) + return + + saw_ac_entry = False + for pkg_hash in spec["remote_hashes"]: + ac_path = resolve_ac_path(self.architecture, pkg_hash) + try: + obj = self.s3.get_object(Bucket=self.acRemoteStore, Key=ac_path) + except ClientError: + continue + entry = json.loads(obj["Body"].read()) + result = entry.get("result", {}) + digest = result.get("outputDigest", "") + if ":" not in digest: + debug("AC entry %s has no usable output digest", ac_path) + continue + saw_ac_entry = True + algo, _, content_hash = digest.partition(":") + cas_path = resolve_cas_path(content_hash, algo) + store_path = resolve_store_path(self.architecture, pkg_hash) + tarball = result.get("tarball") or \ + "{package}-{version}-{revision}.{arch}.tar.gz".format(arch=self.architecture, **spec) + dest = os.path.join(self.workdir, store_path, tarball) + os.makedirs(os.path.join(self.workdir, store_path), exist_ok=True) + debug("Fetching CAS blob %s for %s@%s", cas_path, spec["package"], pkg_hash) + # The AC entry exists but its CAS blob may not (an ephemeral artifact that + # expired, or one deleted for reconstruction). fetch_tarball's contract is + # to leave the local tarball absent so the caller rebuilds -- a missing blob + # must degrade to a rebuild, never crash the whole build. + try: + meta = self.s3.head_object(Bucket=self.remoteStore, Key=cas_path) + total_size = int(meta.get("ContentLength", 0)) + debug("Downloading tarball for %s@%s: %s (%d MB)", spec["package"], + spec["version"], cas_path, total_size >> 20) + # boto3 invokes Callback with the per-chunk *delta*, not the cumulative + # total; byte_progress accumulates it (a raw delta looks stuck at 256 KB). + self.s3.download_file( + Bucket=self.remoteStore, Key=cas_path, Filename=dest, + Callback=byte_progress("download %s@%s" % (spec["package"], spec["version"]), + total_size)) + # Verify the freshly downloaded bytes against the keyring (build path) + # before the build trusts them; fails closed under --require-signature. + self._verify_download(spec, entry, dest, algo, content_hash) + return + except ClientError: + warning("CAS blob %s for %s@%s is missing though its AC entry exists; " + "it will be rebuilt.", cas_path, spec["package"], pkg_hash) + continue + + # Only fall back to the legacy store layout when there was no usable AC entry + # at all (a mixed / pre-AC store). If an AC entry was found but its blob was + # gone, return empty so the package is rebuilt, rather than resurrecting a + # stale legacy redirect that points at the same missing blob. + if saw_ac_entry: + debug("AC entries for %s reference missing CAS blobs; leaving it to rebuild.", + spec["package"]) + return + debug("No Action Cache entry for %s with hashes %s; trying legacy store", + spec["package"], ", ".join(spec["remote_hashes"])) + fetched = super().fetch_tarball(spec) + # A legacy tarball has no AC entry, hence no signature: let the policy decide. + if self.verify_checker and fetched: + self._verify_download(spec, None, fetched[1]) diff --git a/alibuild_helpers/utilities.py b/alibuild_helpers/utilities.py index bb643a52..2a9ce2ca 100644 --- a/alibuild_helpers/utilities.py +++ b/alibuild_helpers/utilities.py @@ -46,6 +46,16 @@ def symlink(link_target, link_name): asList = lambda x : x if type(x) == list else [x] +def plain_data(value): + """Recursively turn OrderedDicts into plain dicts, so a parsed spec fragment can + be re-serialised with yaml.safe_dump (which refuses to represent OrderedDict).""" + if isinstance(value, dict): + return {key: plain_data(item) for key, item in value.items()} + if isinstance(value, list): + return [plain_data(item) for item in value] + return value + + def topological_sort(specs): """Topologically sort specs so that dependencies come before the packages that depend on them. @@ -106,6 +116,43 @@ def resolve_links_path(architecture, package): return "/".join(("TARS", architecture, package)) +def resolve_cas_path(content_hash, algo="sha256"): + """Return the store path for a content-addressed blob (REAPI-style CAS). + + Unlike resolve_store_path(), which is keyed by the *action* hash (a hash of + the recipe and its inputs), this is keyed by a hash of the blob's actual + bytes. The path is sharded on the first two hex characters of the hash, like + the action store, to keep directory fan-out manageable on local mirrors. + + The returned path is relative to the working directory or the root of the + remote store. + """ + return "/".join(("cas", algo, content_hash[:2], content_hash)) + + +def resolve_ac_path(architecture, action_hash): + """Return the store path for an Action Cache entry (REAPI-style AC). + + The entry is keyed by the action hash (spec["remote_revision_hash"]) and + records what produced the artifact, so the CAS can be reconstructed from it. + + The returned path is relative to the working directory or the root of the + remote store. + """ + return "/".join(("ac", architecture, action_hash[:2], action_hash + ".json")) + + +def file_digest(path, algo="sha256", _chunk_size=1 << 20): + """Return the hex digest of the bytes of the file at the given path. + + Used to content-address tarballs and recipe blobs for the CAS. + """ + hasher = hashlib.new(algo) + with open(path, "rb") as fileobj: + for chunk in iter(lambda: fileobj.read(_chunk_size), b""): + hasher.update(chunk) + return hasher.hexdigest() + def default_builder_image(architecture): """Return the default builder container image for an architecture, e.g. @@ -654,6 +701,16 @@ def getPackageList(packages, specs, configDir, preferSystem, noSystem, # Unlike the recipe's own track_env, the value is not shell code to # be run: the check has already computed it for us. spec.setdefault("track_env", OrderedDict())[name.strip()] = value + # A replacement is a header fragment with no fullRecipe of its own; + # without this every replaced package's validate-system node hashes "" + # and they all collide. Render the *selected* replacement rather than + # the original recipe, so a later python3.14 matcher does not invalidate + # the python3.12 validation, which stays distinct via the key. + # sort_keys=False preserves authored env/prepend_path order. + header = {k: v for k, v in spec.items() if k not in ("recipe", "fullRecipe")} + spec["fullRecipe"] = "%s---\n%s" % ( + yaml.safe_dump(plain_data(header), default_flow_style=False, sort_keys=False), + spec.get("recipe", "")) recipe = replacement.get("recipe", "") # If there's an explicitly-specified recipe, we're still building # the package. If not, aliBuild will still "build" it, but it's diff --git a/alienv b/alienv index c31713d6..2966c484 100755 --- a/alienv +++ b/alienv @@ -136,9 +136,17 @@ EOF PKGVER=${PKG##*/} PKGNAME=${PKG%/*} PKGNAME=${PKGNAME##*/} - [[ ! -e "$PKG/etc/modulefiles/$PKGNAME" ]] && continue - mkdir -p "$WORK_DIR/MODULES/$ARCHITECTURE/$PKGNAME" - cp "$PKG/etc/modulefiles/$PKGNAME" "$WORK_DIR/MODULES/$ARCHITECTURE/$PKGNAME/$PKGVER" + [[ ! -d "$PKG/etc/modulefiles" ]] && continue + # A package may ship more than the modulefile named after itself: e.g. + # IgProf also ships IgProfCPU and IgProfMemory, opt-in variants that arm + # the profiler on the very same installation. Publish every modulefile + # found, under the version of the package providing it. + for MODFILE in "$PKG"/etc/modulefiles/*; do + [[ -f "$MODFILE" ]] || continue + MODNAME=${MODFILE##*/} + mkdir -p "$WORK_DIR/MODULES/$ARCHITECTURE/$MODNAME" + cp "$MODFILE" "$WORK_DIR/MODULES/$ARCHITECTURE/$MODNAME/$PKGVER" + done done < <(find $WORK_DIR/$ARCHITECTURE -maxdepth 2 -mindepth 2 2> /dev/null) else printf "${EY}WARNING: not updating modulefiles${EZ}\n" >&2 diff --git a/pyproject.toml b/pyproject.toml index d6112552..af646ecc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -38,6 +38,12 @@ docs = [ 'mkdocs-material', 'mkdocs-redirects', ] +# Ed25519 signing of reapi:// Action Cache entries. Imported lazily, so the core +# tool works without it -- but keep this in sync with setup.py's extras_require: +# because [project] exists here, this table is what the built metadata advertises. +signing = [ +'cryptography', +] [project.urls] homepage = 'https://alisw.github.io/alibuild' @@ -49,7 +55,7 @@ script-files = ["aliBuild", "alienv", "aliDoctor", "aliDeps", "pb"] write_to = "alibuild_helpers/_version.py" [tool.setuptools.package-data] -alibuild_helpers = ['build_template.sh', 'completions/*.sh'] +alibuild_helpers = ['build_template.sh', 'completions/*.sh', 'keyring.json'] [tool.setuptools.packages.find] where = ["."] diff --git a/setup.py b/setup.py index bd557367..ab837a7c 100644 --- a/setup.py +++ b/setup.py @@ -91,12 +91,17 @@ # https://packaging.python.org/en/latest/requirements.html install_requires=install_requires, + # Optional features. Signing/verifying reapi Action Cache entries needs + # Ed25519 from `cryptography`; it is imported lazily so the core tool works + # without it. `pip install alibuild[signing]`. + extras_require={'signing': ['cryptography']}, + # If there are data files included in your packages that need to be # installed, specify them here. If using Python 2.6 or less, then these # have to be included in MANIFEST.in as well. include_package_data=True, package_data={ - 'alibuild_helpers': ['build_template.sh', 'completions/*.sh'], + 'alibuild_helpers': ['build_template.sh', 'completions/*.sh', 'keyring.json'], }, # To provide executable scripts, use entry points in preference to the diff --git a/tests/test_build.py b/tests/test_build.py index 81f27ae2..b1c88fac 100644 --- a/tests/test_build.py +++ b/tests/test_build.py @@ -11,7 +11,8 @@ from collections import OrderedDict from alibuild_helpers.utilities import parseRecipe, resolve_tag -from alibuild_helpers.build import doBuild, storeHashes, generate_initdotsh +from alibuild_helpers import sync_reapi +from alibuild_helpers.build import doBuild, storeHashes, generate_initdotsh, build_ac_entry, select_cached_tarball, bound_unpublished_rebuild # Determine architecture based on platform def get_test_architecture(): @@ -217,6 +218,8 @@ class BuildTestCase(unittest.TestCase): @patch("alibuild_helpers.analytics", new=MagicMock()) @patch("requests.Session.get", new=MagicMock()) @patch("alibuild_helpers.sync.execute", new=dummy_execute) + @patch("alibuild_helpers.build.snapshot_source") + @patch("alibuild_helpers.build.build_ac_entry") @patch("alibuild_helpers.git.git") @patch("alibuild_helpers.build.exists", new=MagicMock(side_effect=dummy_exists)) @patch("os.path.exists", new=MagicMock(side_effect=dummy_exists)) @@ -254,7 +257,8 @@ class BuildTestCase(unittest.TestCase): @patch("alibuild_helpers.workarea.is_writeable", new=MagicMock(return_value=True)) @patch("alibuild_helpers.build.basename", new=MagicMock(return_value="aliBuild")) @patch("alibuild_helpers.build.install_wrapper_script", new=MagicMock()) - def test_coverDoBuild(self, mock_debug, mock_listdir, mock_warning, mock_git_git) -> None: + def test_coverDoBuild(self, mock_debug, mock_listdir, mock_warning, mock_git_git, + mock_build_ac_entry, mock_snapshot_source) -> None: mock_git_git.side_effect = dummy_git mock_debug.side_effect = lambda *args: None mock_warning.side_effect = lambda *args: None @@ -343,6 +347,12 @@ def mkcall(args): ], any_order=True) self.assertEqual(mock_git_git.call_count, len(common_calls) + 1) + # Isolation guard (Layer 2): a non-reapi build (here no remote store) must + # never touch the reapi Action Cache / source-snapshot machinery -- the + # existing build+upload path stays byte-identical for b3://, s3://, ... . + mock_build_ac_entry.assert_not_called() + mock_snapshot_source.assert_not_called() + def setup_spec(self, script): """Parse the alidist recipe in SCRIPT and return its spec.""" err, spec, recipe = parseRecipe(lambda: script) @@ -400,6 +410,163 @@ def test_hashing(self) -> None: self.assertEqual(len(extra["remote_hashes"]), 3) self.assertEqual(extra["local_hashes"][0], TEST_EXTRA_BUILD_HASH) + def test_build_validate_system_entry(self) -> None: + """build_validate_system_entry records a tarball-less validate-system + action: the recipe (with its system check), keyed by the recipe digest so + dependents that reference it (also by recipe digest) resolve to it.""" + import hashlib + from alibuild_helpers.build import build_validate_system_entry + recipe = "package: yacc-like\nsystem_requirement: yacc\n" + digest = hashlib.sha256(recipe.encode("utf-8")).hexdigest() + yacc = {"package": "yacc-like", "version": "v1", "fullRecipe": recipe} + entry = build_validate_system_entry(yacc, {}, "slc7_x86-64") + action = entry["action"] + self.assertEqual(action["kind"], "validate-system") + self.assertEqual(action["package"], "yacc-like") + # Keyed by the recipe digest (not a build action hash), and defaults + # revision to "1" when the system spec has none. + self.assertEqual(action["actionHash"], digest) + self.assertEqual(action["recipeDigest"], "sha256:" + digest) + self.assertEqual(action["revision"], "1") + self.assertEqual(action["architecture"], "slc7_x86-64") + self.assertNotIn("result", entry) # produces no tarball + self.assertEqual(action["deps"], []) + + def test_build_ac_entry_references_system_deps(self) -> None: + """A built package's AC entry references its satisfied system deps by their + recipe digest, so reconstruct walks to the validate-system entry.""" + import hashlib + from alibuild_helpers.build import build_ac_entry, system_recipe_digest + make_recipe = "package: make\nsystem_requirement: '.*'\n" + make_spec = {"package": "make", "fullRecipe": make_recipe} + spec = {"package": "probe", "version": "1", "revision": "1", + "remote_revision_hash": "p" * 40, "commit_hash": "0", + "scm_refs": {}, "full_requires": [], "system_requires": ["make"]} + entry = build_ac_entry(spec, {"probe": spec}, "slc7_x86-64", + system_specs={"make": make_spec}) + self.assertEqual(entry["action"]["deps"], + [{"package": "make", + "actionHash": system_recipe_digest(make_spec)}]) + + def test_build_ac_entry(self) -> None: + """build_ac_entry records the provenance needed to rebuild/install.""" + import hashlib + default = self.setup_spec(TEST_DEFAULT_RELEASE) + zlib = self.setup_spec(TEST_ZLIB_RECIPE) + default["commit_hash"] = "0" + zlib.setdefault("requires", []).append(default["package"]) + zlib["scm_refs"] = {ref: githash for githash, _, ref in ( + line.partition("\t") for line in TEST_ZLIB_GIT_REFS.splitlines())} + try: + zlib["commit_hash"] = zlib["scm_refs"]["refs/tags/" + zlib["tag"]] + except KeyError: + zlib["commit_hash"] = zlib["scm_refs"]["refs/heads/" + zlib["tag"]] + specs = {pkg["package"]: pkg for pkg in (default, zlib)} + for spec in specs.values(): + spec["is_devel_pkg"] = False + + storeHashes("defaults-release", specs, considerRelocation=False) + default["hash"] = default["remote_revision_hash"] + storeHashes("zlib", specs, considerRelocation=False) + zlib["hash"] = zlib["remote_revision_hash"] + zlib["revision"] = "1" + # These closures are normally computed in doBuild() before upload. + zlib["full_requires"] = {"defaults-release"} + zlib["full_runtime_requires"] = set() + zlib["relocate_paths"] = ["lib", "bin"] + + container = {"runtime": "docker", "image": "alisw/slc7-builder:latest", + "digest": "sha256:abc"} + refs_artifact = {"type": "git-refs", "source": "u", "digest": "r" * 64} + entry = build_ac_entry(zlib, specs, "slc7_x86-64", container=container, + refs_artifact=refs_artifact) + self.assertEqual(entry["schemaVersion"], 2) + action = entry["action"] + # Container + refs provenance are recorded verbatim for a reproducible env. + self.assertEqual(action["container"], container) + self.assertEqual(action["refsArtifact"], refs_artifact) + self.assertEqual(action["package"], "zlib") + self.assertEqual(action["version"], zlib["version"]) + self.assertEqual(action["revision"], "1") + self.assertEqual(action["architecture"], "slc7_x86-64") + # The entry is keyed by the action hash, and deps are referenced by + # *their* action hash, so the recorded DAG matches storeHashes(). + self.assertEqual(action["actionHash"], zlib["remote_revision_hash"]) + self.assertEqual(action["deps"], + [{"package": "defaults-release", "actionHash": default["hash"]}]) + self.assertEqual(action["runtimeDeps"], []) + self.assertEqual(action["depsHash"], zlib["deps_hash"]) + # The recipe digest is the sha256 of the *full* recipe (a CAS blob), + # so the build can be reconstructed without an alidist checkout. + self.assertEqual(action["recipeDigest"], "sha256:" + + hashlib.sha256(zlib["fullRecipe"].encode("utf-8")).hexdigest()) + self.assertEqual(action["source"], zlib.get("source")) + self.assertEqual(action["relocatePaths"], ["bin", "lib"]) # sorted + self.assertEqual(action["commit"]["ref"], zlib["commit_hash"]) + # The output digest/size are filled in by the backend at upload time. + self.assertNotIn("result", entry) + + def test_snapshot_source_gating(self) -> None: + """snapshot_source only archives for git, non-devel packages going to a + reapi store; everything else is a no-op (and never touches git).""" + from alibuild_helpers.build import snapshot_source + from alibuild_helpers import sync + # Not a reapi store -> never snapshots. + self.assertIsNone(snapshot_source({"package": "x"}, sync.NoRemoteSync())) + # reapi store, but devel / source-less packages are skipped before git. + with patch.object(sync_reapi.REAPIRemoteSync, "_s3_init", lambda self: None): + reapi = sync_reapi.REAPIRemoteSync("reapi://h/b", "reapi://h/b", + "slc7_x86-64", "/sw") + self.assertIsNone(snapshot_source( + {"package": "x", "is_devel_pkg": True, "source": "u", "reference": "r"}, reapi)) + self.assertIsNone(snapshot_source( + {"package": "x", "is_devel_pkg": False}, reapi)) # no source + + def test_snapshot_source_resolves_commit_from_refs(self) -> None: + """A reconstruct rebuild's restored source is a working checkout without the + tag ref, so `git rev-parse ^{commit}` fails. snapshot_source must instead + resolve commit_hash via scm_refs (idempotent snapshot then preserves the + archived-source reference in the regenerated AC entry).""" + from alibuild_helpers.build import snapshot_source + from alibuild_helpers import sync + from alibuild_helpers.git import Git + with patch.object(sync_reapi.REAPIRemoteSync, "_s3_init", lambda self: None): + reapi = sync_reapi.REAPIRemoteSync("reapi://h/b", "reapi://h/b", "slc7_x86-64", "/sw") + spec = {"package": "zlib", "is_devel_pkg": False, "source": "https://x/zlib", + "reference": "/ref", "commit_hash": "v1.3.1", "scm": Git(), + "scm_refs": {"refs/tags/v1.3.1": "d" * 40}} + captured = {} + + class FakeStore: + def __init__(self, _sync): pass + def snapshot(self, repo, url, commit): + captured["commit"] = commit + return {"type": "git", "commit": commit} + + def fake_git(args, directory=None, check=True, **kw): + if args[:2] == ("config", "--get"): + return ("", "") # not a partial clone + raise AssertionError("git rev-parse must not run when scm_refs resolves") + + with patch("alibuild_helpers.build.GitSourceStore", FakeStore), \ + patch("alibuild_helpers.build.git", side_effect=fake_git): + art = snapshot_source(spec, reapi) + self.assertEqual(captured["commit"], "d" * 40) # resolved from scm_refs + self.assertEqual(art["commit"], "d" * 40) + + def test_snapshot_refs_gating(self) -> None: + """snapshot_refs only archives scm_refs for non-devel packages going to + a reapi store.""" + from alibuild_helpers.build import snapshot_refs + from alibuild_helpers import sync + self.assertIsNone(snapshot_refs({"package": "x"}, sync.NoRemoteSync())) + with patch.object(sync_reapi.REAPIRemoteSync, "_s3_init", lambda self: None): + reapi = sync_reapi.REAPIRemoteSync("reapi://h/b", "reapi://h/b", + "slc7_x86-64", "/sw") + self.assertIsNone(snapshot_refs( + {"package": "x", "is_devel_pkg": True, "scm_refs": {"a": "b"}}, reapi)) + self.assertIsNone(snapshot_refs({"package": "x"}, reapi)) # no scm_refs + def test_initdotsh(self) -> None: """Sanity-check the generated init.sh for a few variables.""" specs = { @@ -505,5 +672,96 @@ def test_missing_hash_without_checkout_dies(self) -> None: self._run_reaching_scm_block() + + +class SelectCachedTarballTestCase(unittest.TestCase): + """A cached tarball is only reusable at the revision it was packaged under. + + The store is keyed by hash, so a rebuilt package lands in the same directory + as the one built before it; only the file name carries the revision. Picking + the wrong one used to skip packaging (build_template.sh tars only when there + is no cached tarball) and then crash in the upload -- after the symlink claim + and the dist links had already gone out, so the store was left advertising a + tarball that was never written. + """ + + WANTED = "QualityControl-v1.195.4-2.slc10_x86-64.tar.gz" + DIR = "sw/TARS/slc10_x86-64/store/7e/7e875fbe" + + def path(self, name): + return os.path.join(self.DIR, name) + + def test_exact_revision_is_reused(self): + self.assertEqual( + select_cached_tarball([self.path(self.WANTED)], self.WANTED, uploading=True), + self.path(self.WANTED)) + + def test_other_revision_is_rejected_when_uploading(self): + """The regression: -1 on disk, -2 to publish. Rebuild instead.""" + stale = self.path("QualityControl-v1.195.4-1.slc10_x86-64.tar.gz") + self.assertEqual(select_cached_tarball([stale], self.WANTED, uploading=True), "") + + def test_other_revision_is_fine_when_not_uploading(self): + """Without a write store nothing needs the name to match, and the unpack + path relocates whatever revision it finds -- so keep the cheap reuse.""" + stale = self.path("QualityControl-v1.195.4-1.slc10_x86-64.tar.gz") + self.assertEqual(select_cached_tarball([stale], self.WANTED, uploading=False), stale) + + def test_exact_match_wins_over_a_stale_neighbour(self): + """Both present: the old code took glob order, which is arbitrary.""" + stale = self.path("QualityControl-v1.195.4-1.slc10_x86-64.tar.gz") + for order in ([stale, self.path(self.WANTED)], [self.path(self.WANTED), stale]): + for uploading in (True, False): + self.assertEqual( + select_cached_tarball(order, self.WANTED, uploading=uploading), + self.path(self.WANTED)) + + def test_no_tarballs_means_no_cache(self): + for uploading in (True, False): + self.assertEqual(select_cached_tarball([], self.WANTED, uploading=uploading), "") + + + +class BoundUnpublishedRebuildTestCase(unittest.TestCase): + """A package that cannot be published must not be rebuilt forever. + + upload_symlinks_and_tarball adopts an existing legacy tarball as + already-uploaded without writing an Action Cache entry, while is_published() + only asks the AC. For a pre-2.0 tarball both stay true forever, and the + rebuild-to-fix-it path loops -- 1743 rebuilds of defaults-release in 26 + minutes on the first ubuntu2204 release attempt. + """ + + def test_first_visit_allows_the_rebuild(self): + rebuilt = set() + bound_unpublished_rebuild("defaults-release", rebuilt, "alibuild-ac") + self.assertEqual(rebuilt, {"defaults-release"}) + + def test_second_visit_dies_rather_than_looping(self): + rebuilt = {"defaults-release"} + with self.assertRaises(SystemExit): + bound_unpublished_rebuild("defaults-release", rebuilt, "alibuild-ac") + + def test_the_error_says_how_to_fix_it(self): + """A bare 'cannot publish' would leave the reader nowhere: the remedy is + migration, not a retry, so the message has to name it.""" + rebuilt = {"defaults-release"} + # dieOnError logs through alibuild_helpers.log.error; build.py never + # imports `error` itself, so patching it there raises AttributeError. + with self.assertRaises(SystemExit), \ + patch("alibuild_helpers.log.error") as mock_error: + bound_unpublished_rebuild("defaults-release", rebuilt, "alibuild-ac") + msg = " ".join(str(a) for a in mock_error.call_args[0]) + self.assertIn("aliBuild migrate", msg) + self.assertIn("defaults-release", msg) + self.assertIn("alibuild-ac", msg) + + def test_packages_are_bounded_independently(self): + """One package exhausting its rebuild must not block another's first.""" + rebuilt = {"defaults-release"} + bound_unpublished_rebuild("zlib", rebuilt, "alibuild-ac") + self.assertEqual(rebuilt, {"defaults-release", "zlib"}) + if __name__ == '__main__': unittest.main() + diff --git a/tests/test_cmd.py b/tests/test_cmd.py index b95d9637..0a312510 100644 --- a/tests/test_cmd.py +++ b/tests/test_cmd.py @@ -24,8 +24,8 @@ def test_DockerRunner(self, mock_getstatusoutput, mock_getoutput): mock_getoutput.side_effect = lambda cmd: "container-id\n" mock_getstatusoutput.return_value = (0, "") # image already present: stream_pull is a no-op with DockerRunner("image", ["extra arg"]) as getstatusoutput_docker: - mock_getoutput.assert_called_with(["docker", "run", "--detach", "--rm", "--entrypoint=", - "extra arg", "image", "sleep", "inf"]) + mock_getoutput.assert_called_with(["docker", "run", "--detach", "--rm", "--entrypoint=/bin/sleep", + "extra arg", "image", "inf"]) getstatusoutput_docker("echo foo") mock_getstatusoutput.assert_called_with(["docker", "container", "exec", "container-id", "bash", "-c", "echo foo"], cwd=None) mock_getstatusoutput.assert_called_with("docker container kill container-id") @@ -74,7 +74,7 @@ def test_DockerRunner_with_env_vars(self, mock_getstatusoutput, mock_getoutput): mock_getoutput.assert_called_with(["docker", "run", "--detach", "-e", "TEST_VAR=test_value", "-e", "ANOTHER_VAR=another_value", - "--rm", "--entrypoint=", "image", "sleep", "inf"]) + "--rm", "--entrypoint=/bin/sleep", "image", "inf"]) # Test that exec command includes environment variables getstatusoutput_docker("echo test") diff --git a/tests/test_install.py b/tests/test_install.py new file mode 100644 index 00000000..0c2194ef --- /dev/null +++ b/tests/test_install.py @@ -0,0 +1,345 @@ +import hashlib +import io +import json +import os +import os.path +import tarfile +import tempfile +import unittest +from argparse import Namespace +from unittest.mock import patch + +from alibuild_helpers import signing +from alibuild_helpers import sync +from alibuild_helpers import sync_reapi +from alibuild_helpers import install +from alibuild_helpers.install import collect_runtime_closure, install_entry, doInstall + +# `cryptography` is an optional extra (`pip install alibuild[signing]`), so these +# tests must skip rather than fail where it is absent -- e.g. a CI job that does +# not install the extra. tox installs it via `extras = signing`, so they normally +# do run. +try: + import cryptography # noqa: F401 + HAVE_CRYPTOGRAPHY = True +except ImportError: + HAVE_CRYPTOGRAPHY = False + +requires_crypto = unittest.skipUnless( + HAVE_CRYPTOGRAPHY, "needs the optional 'cryptography' extra") + + +ARCH = "slc7_x86-64" +SEED_A = bytes(range(32)) +SEED_B = bytes(range(32, 64)) + + +def make_tarball(pkg, version, revision): + """Build an in-memory tarball laid out like a real aliBuild package: + //-/... with an init.sh, a file needing relocation + (plus its .unrelocated pristine copy) and a relocate-me.sh that mimics the + real one (sed from .unrelocated, drop a marker we can assert on).""" + pkgpath = "%s/%s/%s-%s" % (ARCH, pkg, version, revision) + relocate = ( + "#!/bin/bash -e\n" + ': "${WORK_DIR:?Please define WORK_DIR}"\n' + 'PP=%s\n' + 'sed -e "s|@PLACEHOLDER@|$WORK_DIR/$PP|g" "$PP/lib/foo.txt.unrelocated" > "$PP/lib/foo.txt"\n' + 'touch "$WORK_DIR/$PP/relocated.marker"\n' + ) % pkgpath + files = { + pkgpath + "/etc/profile.d/init.sh": "# init for %s\n" % pkg, + pkgpath + "/lib/foo.txt.unrelocated": "prefix is @PLACEHOLDER@\n", + pkgpath + "/lib/foo.txt": "prefix is @PLACEHOLDER@\n", + pkgpath + "/relocate-me.sh": relocate, + } + buf = io.BytesIO() + with tarfile.open(fileobj=buf, mode="w:gz") as tar: + for name, content in sorted(files.items()): + data = content.encode() + info = tarfile.TarInfo(name) + info.size = len(data) + tar.addfile(info, io.BytesIO(data)) + return buf.getvalue() + + +class FakeSync(sync_reapi.REAPIRemoteSync): + """A REAPIRemoteSync whose S3 reads are served from in-memory fixtures.""" + + def __init__(self, entries, label_to_hash, blobs): + self.architecture = ARCH + self._entries = entries # action hash -> AC entry + self._label_to_hash = label_to_hash # (pkg, ver, rev) -> action hash + self._blobs = blobs # content hash -> tarball bytes + + def read_ac_entry(self, action_hash): + return self._entries.get(action_hash) + + def resolve_action_hash(self, package, version, revision=None): + return self._label_to_hash.get((package, version, revision)) + + def download_artifact(self, content_hash, dest, algo="sha256"): + with open(dest, "wb") as destf: + destf.write(self._blobs[content_hash]) + + +def make_entry(pkg, version, revision, content_hash, runtime=()): + return { + "schemaVersion": 1, + "action": { + "package": pkg, "version": version, "revision": revision, + "architecture": ARCH, "actionHash": "hash-" + pkg, + "runtimeDeps": [{"package": p, "actionHash": h} for p, h in runtime], + }, + "result": { + "tarball": "%s-%s-%s.%s.tar.gz" % (pkg, version, revision, ARCH), + "outputDigest": "sha256:" + content_hash, "size": 4096, + }, + } + + +class InstallTestCase(unittest.TestCase): + def setUp(self): + # zlib (top) with one runtime dependency, GCC. + self.entries = { + "hash-zlib": make_entry("zlib", "v1", "1", "c" * 64, + runtime=[("GCC", "hash-GCC")]), + "hash-GCC": make_entry("GCC", "v9", "2", "d" * 64), + } + self.blobs = { + "c" * 64: make_tarball("zlib", "v1", "1"), + "d" * 64: make_tarball("GCC", "v9", "2"), + } + self.sync = FakeSync(self.entries, + {("zlib", "v1", None): "hash-zlib", + ("zlib", "v1", "1"): "hash-zlib"}, + self.blobs) + + def test_collect_runtime_closure(self): + closure = collect_runtime_closure(self.sync, "hash-zlib") + self.assertEqual([e["action"]["package"] for e in closure], ["zlib", "GCC"]) + + def test_collect_runtime_closure_missing_dep(self): + del self.entries["hash-GCC"] + self.assertRaises(SystemExit, collect_runtime_closure, self.sync, "hash-zlib") + + def test_install_entry_extracts_and_relocates(self): + with tempfile.TemporaryDirectory() as prefix: + install_entry(self.sync, self.entries["hash-zlib"], prefix, ARCH) + base = os.path.join(prefix, ARCH, "zlib", "v1-1") + # Extracted. + self.assertTrue(os.path.exists(os.path.join(base, "etc/profile.d/init.sh"))) + # relocate-me.sh ran (marker dropped, with WORK_DIR == prefix). + self.assertTrue(os.path.exists(os.path.join(base, "relocated.marker"))) + # The placeholder was rewritten to the final prefix path. + with open(os.path.join(base, "lib/foo.txt")) as foo: + self.assertIn(os.path.join(prefix, ARCH, "zlib", "v1-1"), foo.read()) + # The .unrelocated pristine copy was cleaned up. + self.assertFalse(os.path.exists(os.path.join(base, "lib/foo.txt.unrelocated"))) + # latest symlink points at the installed revision. + self.assertEqual(os.readlink(os.path.join(prefix, ARCH, "zlib", "latest")), "v1-1") + + def test_install_entry_skips_if_present(self): + with tempfile.TemporaryDirectory() as prefix: + os.makedirs(os.path.join(prefix, ARCH, "zlib", "v1-1")) + with patch.object(self.sync, "download_blob") as dl: + install_entry(self.sync, self.entries["hash-zlib"], prefix, ARCH) + dl.assert_not_called() + + def test_doInstall_end_to_end(self): + with tempfile.TemporaryDirectory() as prefix: + args = Namespace(package="zlib", version="v1", revision=None, + architecture=ARCH, remoteStore="reapi://localhost/bucket", + insecure=False, workDir=prefix, prefix=prefix) + with patch("alibuild_helpers.install.remote_from_url", return_value=self.sync): + self.assertTrue(doInstall(args, None)) + # Both the package and its runtime dependency are installed. + self.assertTrue(os.path.exists( + os.path.join(prefix, ARCH, "zlib", "v1-1", "etc/profile.d/init.sh"))) + self.assertTrue(os.path.exists( + os.path.join(prefix, ARCH, "GCC", "v9-2", "etc/profile.d/init.sh"))) + + def test_doInstall_requires_reapi_store(self): + args = Namespace(package="zlib", version="v1", revision=None, + architecture=ARCH, remoteStore="https://s3.cern.ch/foo", + insecure=False, workDir="/sw", prefix=None) + # A non-reapi store yields a non-REAPIRemoteSync backend, which must abort. + self.assertRaises(SystemExit, doInstall, args, None) + + +def write_keyring(path, *seed_signers): + """Write a JSON keyring trusting the given (seed, signer) public keys.""" + keys = {} + for seed, signer in seed_signers: + keyid, pub = signing.public_key(seed) + keys[keyid] = {"publicKey": pub, "signer": signer} + with open(path, "w") as handle: + json.dump({"keys": keys}, handle) + + +@requires_crypto +class SignatureWiringTestCase(unittest.TestCase): + """End-to-end: --require-signature / --trusted-keys gating doInstall.""" + + def setUp(self): + # Real tarball bytes so their sha256 is the entry's output digest -- the + # signature binds that digest, and install re-hashes to confirm the bytes. + self.zlib_bytes = make_tarball("zlib", "v1", "1") + self.gcc_bytes = make_tarball("GCC", "v9", "2") + self.zlib_h = hashlib.sha256(self.zlib_bytes).hexdigest() + self.gcc_h = hashlib.sha256(self.gcc_bytes).hexdigest() + self.blobs = {self.zlib_h: self.zlib_bytes, self.gcc_h: self.gcc_bytes} + + def _entry(self, pkg, ver, rev, content_hash, runtime=(), seed=SEED_A): + entry = make_entry(pkg, ver, rev, content_hash, runtime) + if seed is not None: + entry["signatures"] = [signing.sign(entry, seed, "ci")] + return entry + + def _sync(self, gcc_seed=SEED_A, blobs=None): + entries = { + "hash-zlib": self._entry("zlib", "v1", "1", self.zlib_h, + runtime=[("GCC", "hash-GCC")]), + "hash-GCC": self._entry("GCC", "v9", "2", self.gcc_h, seed=gcc_seed), + } + return FakeSync(entries, + {("zlib", "v1", None): "hash-zlib", + ("zlib", "v1", "1"): "hash-zlib"}, + blobs if blobs is not None else self.blobs) + + def _run(self, sync_obj, prefix, policy, trusted_seed=SEED_A): + keyring_path = os.path.join(prefix, "keyring.json") + write_keyring(keyring_path, (trusted_seed, "ci")) + args = Namespace(package="zlib", version="v1", revision=None, + architecture=ARCH, remoteStore="reapi://localhost/bucket", + insecure=False, workDir=prefix, prefix=prefix, + requireSignature=policy, trustedKeys=keyring_path) + with patch("alibuild_helpers.install.remote_from_url", return_value=sync_obj): + return doInstall(args, None) + + def test_require_installs_signed_closure(self): + with tempfile.TemporaryDirectory() as prefix: + self.assertTrue(self._run(self._sync(), prefix, "require")) + self.assertTrue(os.path.exists( + os.path.join(prefix, ARCH, "GCC", "v9-2", "etc/profile.d/init.sh"))) + + def test_require_rejects_unsigned_dependency(self): + with tempfile.TemporaryDirectory() as prefix: + self.assertRaises(SystemExit, self._run, + self._sync(gcc_seed=None), prefix, "require") + + def test_require_rejects_untrusted_key(self): + with tempfile.TemporaryDirectory() as prefix: + # Signed with SEED_A, but the keyring only trusts SEED_B. + self.assertRaises(SystemExit, self._run, + self._sync(), prefix, "require", trusted_seed=SEED_B) + + def test_warn_installs_unsigned(self): + with tempfile.TemporaryDirectory() as prefix: + sync_obj = self._sync(gcc_seed=None) + sync_obj._entries["hash-zlib"].pop("signatures", None) + self.assertTrue(self._run(sync_obj, prefix, "warn")) + self.assertTrue(os.path.exists( + os.path.join(prefix, ARCH, "zlib", "v1-1", "etc/profile.d/init.sh"))) + + def test_require_rejects_blob_not_matching_signed_digest(self): + with tempfile.TemporaryDirectory() as prefix: + # Signatures are valid, but the CAS serves the wrong bytes for zlib: + # the blob-digest binding must catch it. + bad = dict(self.blobs, **{self.zlib_h: self.gcc_bytes}) + self.assertRaises(SystemExit, self._run, + self._sync(blobs=bad), prefix, "require") + + def test_off_skips_verification_entirely(self): + with tempfile.TemporaryDirectory() as prefix: + # Unsigned, no keyring needed: policy off installs as before. + sync_obj = self._sync(gcc_seed=None) + sync_obj._entries["hash-zlib"].pop("signatures", None) + args = Namespace(package="zlib", version="v1", revision=None, + architecture=ARCH, remoteStore="reapi://localhost/bucket", + insecure=False, workDir=prefix, prefix=prefix, + requireSignature="off", trustedKeys="") + with patch("alibuild_helpers.install.remote_from_url", return_value=sync_obj): + self.assertTrue(doInstall(args, None)) + + +@requires_crypto +class CheckerFactoryTestCase(unittest.TestCase): + """signature_checker default resolution: policy default + keyring lookup.""" + + def _args(self, **over): + base = dict(requireSignature="warn", trustedKeys="") + base.update(over) + return Namespace(**base) + + def test_bundled_keyring_is_used_without_alidist(self): + # The recipe-free `install` has no alidist; the keyring shipped inside the + # package is what makes it verify anything at all. + checker = sync_reapi.signature_checker(self._args()) + self.assertIsInstance(checker, sync_reapi.SignatureChecker) + self.assertEqual(sync_reapi.keyring_sources(self._args()), + [signing.bundled_keyring_path()]) + + def test_bundled_keyring_ships_and_loads(self): + path = signing.bundled_keyring_path() + self.assertTrue(os.path.exists(path), "keyring.json must ship with the package") + self.assertTrue(signing.load_keyring(path).keys, "shipped keyring has no keys") + + def test_off_is_none(self): + self.assertIsNone(sync_reapi.signature_checker( + self._args(requireSignature="off"))) + + def test_require_without_any_keyring_aborts(self): + with patch("alibuild_helpers.signing.bundled_keyring_path", + return_value="/nonexistent/keyring.json"): + self.assertRaises(SystemExit, sync_reapi.signature_checker, + self._args(requireSignature="require")) + + def test_alidist_keyring_merges_on_top_of_the_bundled_one(self): + with tempfile.TemporaryDirectory() as alidist: + keyring_path = os.path.join(alidist, "keyring.json") + write_keyring(keyring_path, (SEED_A, "ci")) + # configDir (build) and alidist (reconstruct) both resolve to it, and + # neither replaces the bundled keyring -- both are consulted. + for kwargs in ({"configDir": alidist}, {"alidist": alidist}): + self.assertEqual(sync_reapi.keyring_sources(self._args(**kwargs)), + [signing.bundled_keyring_path(), keyring_path]) + checker = sync_reapi.signature_checker(self._args(configDir=alidist)) + self.assertIsInstance(checker, sync_reapi.SignatureChecker) + self.assertEqual(checker.policy, "warn") + # The alidist key is trusted, and so is everything already shipped. + bundled = signing.load_keyring(signing.bundled_keyring_path()) + self.assertIn(signing.public_key(SEED_A)[0], checker.keyring.keys) + for keyid in bundled.keys: + self.assertIn(keyid, checker.keyring.keys) + + def test_missing_cryptography_degrades_under_warn(self): + # cryptography is optional: warn skips when it can't load the keyring, + # require fails closed. + with tempfile.TemporaryDirectory() as d: + keyring_path = os.path.join(d, "keyring.json") + write_keyring(keyring_path, (SEED_A, "ci")) + with patch("alibuild_helpers.signing.load_keyring", + side_effect=RuntimeError("no cryptography")): + self.assertIsNone(sync_reapi.signature_checker( + self._args(trustedKeys=keyring_path))) + self.assertRaises(SystemExit, sync_reapi.signature_checker, + self._args(requireSignature="require", + trustedKeys=keyring_path)) + + def test_explicit_trusted_keys_replaces_the_defaults(self): + # An explicit --trusted-keys is an escape hatch: it must not silently + # keep trusting the shipped keys on top of what the user asked for. + with tempfile.TemporaryDirectory() as d: + explicit = os.path.join(d, "mykeys.json") + write_keyring(explicit, (SEED_A, "ci")) + self.assertEqual( + sync_reapi.keyring_sources( + self._args(trustedKeys=explicit, configDir="/some/alidist")), + [explicit]) + checker = sync_reapi.signature_checker(self._args(trustedKeys=explicit)) + self.assertEqual(set(checker.keyring.keys), {signing.public_key(SEED_A)[0]}) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_migrate.py b/tests/test_migrate.py new file mode 100644 index 00000000..2f683ab8 --- /dev/null +++ b/tests/test_migrate.py @@ -0,0 +1,732 @@ +import hashlib +import io +import json +import os +import os.path +import shutil +import subprocess +import tarfile +import tempfile +import threading +import unittest +from argparse import Namespace +from unittest.mock import patch, MagicMock + +from alibuild_helpers.migrate import strip_rw + +from alibuild_helpers import sync +from alibuild_helpers import sync_reapi + +from alibuild_helpers.migrate import ( + read_meta_json, recover_recipe, container_for_migration, + ac_entry_from_meta, migrate_tarball, doMigrate, verify_recovered_recipe, + download_from_old_store, enumerate_closure, enumerate_arch, enrich_source_snapshot) + + +class _FakeResp: + """Minimal streaming requests.Response stand-in.""" + def __init__(self, data=b"", text=""): + self._data = data + self.text = text + self.status_code = 200 + self.headers = {"content-length": str(len(data))} + def __enter__(self): + return self + def __exit__(self, *exc): + return False + def raise_for_status(self): + pass + def iter_content(self, chunk_size): + yield self._data + +ARCH = "slc7_x86-64" + +META = { + "alibuild_version": "1.0", + "alidist": {"commit": None}, # filled in per-test + "architecture": ARCH, + "defaults": "o2", + "package": {"name": "zlib", "tag": "v1.3.1", "source": "https://example/zlib", + "version": "v1.3.1", "revision": "1", "hash": "z" * 40}, + "dependencies": { + "direct": {"build": [], "runtime": []}, + "recursive": { + "build": [{"name": "GCC", "tag": "v9", "source": "https://e/gcc", + "version": "v9", "revision": "2", "hash": "g" * 40}], + "runtime": [{"name": "GCC", "tag": "v9", "source": "https://e/gcc", + "version": "v9", "revision": "2", "hash": "g" * 40}], + }, + }, +} + + +def make_tarball_with_meta(meta): + """A legacy tarball laid out as //-/.meta.json + a file.""" + pkgpath = "%s/zlib/v1.3.1-1" % ARCH + buf = io.BytesIO() + with tarfile.open(fileobj=buf, mode="w:gz") as tar: + for name, data in ((pkgpath + "/.meta.json", json.dumps(meta).encode()), + (pkgpath + "/lib/libz.so", b"binary")): + info = tarfile.TarInfo(name) + info.size = len(data) + tar.addfile(info, io.BytesIO(data)) + return buf.getvalue() + + +class FakeReapiSync: + def __init__(self): + self.calls = [] + self.blobs = {} + self.objects = {} + + def migrate_put(self, ac_entry, tarball_path, recipe_text): + self.calls.append((ac_entry, tarball_path, recipe_text)) + return "c" * 64 + + # Enough of the CAS/source-store interface for GitSourceStore + store_refs. + def put_file_as_blob(self, path, algo="sha256"): + with open(path, "rb") as blobf: + data = blobf.read() + h = hashlib.sha256(data).hexdigest() + self.blobs.setdefault(h, data) + return h + + def put_bytes_as_blob(self, data, algo="sha256"): + h = hashlib.sha256(data).hexdigest() + self.blobs.setdefault(h, data) + return h + + def read_blob(self, content_hash, algo="sha256"): + return self.blobs[content_hash] + + def read_object_json(self, key): + return self.objects.get(key) + + def write_object_json(self, key, obj): + self.objects[key] = obj + + +class _EnrichSync: + """Minimal ledger stand-in for enrich_source_snapshot: one AC entry keyed by + a fixed action hash, plus a record of any in-place updates.""" + def __init__(self, entry, action_hash="a" * 40): + self._entry = entry + self._hash = action_hash + self.updated = [] + + def resolve_action_hash(self, package, version, revision=None): + return self._hash + + def read_ac_entry(self, action_hash): + return self._entry if action_hash == self._hash else None + + def update_ac_entry(self, entry): + self.updated.append(entry) + + +def _ac_entry(source="https://example.invalid/zlib.git", tag="v1.3.1", + source_artifact=None): + return {"schemaVersion": 2, "action": { + "package": "zlib", "version": "v1.3.1", "revision": "6", + "architecture": ARCH, "actionHash": "a" * 40, + "source": source, "tag": tag, "sourceArtifact": source_artifact, + "refsArtifact": None, + "commit": {"ref": tag, "commitHash": tag, "altRefs": {}}}} + + +class MigrateTestCase(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.tmp, ignore_errors=True) + + def _write_tarball(self, meta): + path = os.path.join(self.tmp, "zlib-v1.3.1-1.%s.tar.gz" % ARCH) + with open(path, "wb") as tarf: + tarf.write(make_tarball_with_meta(meta)) + return path + + def _make_alidist(self): + """A git alidist with a zlib.sh recipe; returns (dir, commit).""" + alidist = os.path.join(self.tmp, "alidist") + os.makedirs(alidist) + env = dict(os.environ, GIT_AUTHOR_NAME="t", GIT_AUTHOR_EMAIL="a@b.c", + GIT_COMMITTER_NAME="t", GIT_COMMITTER_EMAIL="a@b.c") + run = lambda *a: subprocess.run(["git"] + list(a), cwd=alidist, env=env, + check=True, stdout=subprocess.PIPE).stdout.decode().strip() + run("init", "-q") + with open(os.path.join(alidist, "zlib.sh"), "w") as recipef: + recipef.write("package: zlib\nversion: v1.3.1\n---\nbuild zlib\n") + run("add", ".") + run("commit", "-qm", "recipes") + return alidist, run("rev-parse", "HEAD") + + def test_read_meta_json(self): + path = self._write_tarball(META) + meta = read_meta_json(path) + self.assertEqual(meta["package"]["name"], "zlib") + self.assertEqual(meta["defaults"], "o2") + + def test_read_meta_json_absent(self): + path = os.path.join(self.tmp, "nometa.tar.gz") + with tarfile.open(path, "w:gz") as tar: + data = b"x" + info = tarfile.TarInfo("%s/zlib/v1.3.1-1/lib/libz.so" % ARCH) + info.size = len(data) + tar.addfile(info, io.BytesIO(data)) + self.assertIsNone(read_meta_json(path)) + + def test_recover_recipe(self): + alidist, commit = self._make_alidist() + recipe = recover_recipe(alidist, commit, "zlib") + self.assertIn("build zlib", recipe) + self.assertIn("package: zlib", recipe) + + def test_recover_recipe_github_fallback(self): + alidist, _ = self._make_alidist() + bogus = "0" * 40 # not in the local clone -> git show fails -> GitHub + with patch("alibuild_helpers.migrate.requests.get", + return_value=_FakeResp(text="package: zlib\n---\nfrom github\n")) as get: + recipe = recover_recipe(alidist, bogus, "zlib") + self.assertIn("from github", recipe) + url = get.call_args.args[0] + self.assertIn("raw.githubusercontent.com", url) + self.assertTrue(url.endswith(bogus + "/zlib.sh")) + + def test_container_for_migration(self): + default = container_for_migration(ARCH) + self.assertEqual(default["image"], "registry.cern.ch/alisw/slc7-builder") + self.assertEqual(default["provenance"], "migration-default") + override = container_for_migration(ARCH, "myreg/img:tag") + self.assertEqual(override["image"], "myreg/img:tag") + + def test_ac_entry_from_meta(self): + entry = ac_entry_from_meta(META, "package: zlib\n---\nbuild\n", + container_for_migration(ARCH)) + action = entry["action"] + self.assertEqual(entry["schemaVersion"], 2) + self.assertEqual(action["actionHash"], "z" * 40) + self.assertEqual(action["source"], "https://example/zlib") + self.assertEqual(action["deps"], [{"package": "GCC", "actionHash": "g" * 40}]) + self.assertEqual(action["runtimeDeps"], [{"package": "GCC", "actionHash": "g" * 40}]) + self.assertEqual(action["container"]["provenance"], "migration-default") + self.assertEqual(action["recipeDigest"], "sha256:" + + hashlib.sha256(b"package: zlib\n---\nbuild\n").hexdigest()) + + def test_migrate_tarball(self): + alidist, commit = self._make_alidist() + meta = json.loads(json.dumps(META)) + meta["alidist"]["commit"] = commit + path = self._write_tarball(meta) + sync = FakeReapiSync() + action_hash = migrate_tarball(sync, path, alidist) + self.assertEqual(action_hash, "z" * 40) + self.assertEqual(len(sync.calls), 1) + entry, tarball_path, recipe = sync.calls[0] + self.assertEqual(tarball_path, path) + self.assertIn("build zlib", recipe) # recovered from alidist + self.assertEqual(entry["action"]["package"], "zlib") + + def test_migrate_tarball_skips_without_meta(self): + alidist, _ = self._make_alidist() + path = os.path.join(self.tmp, "nometa.tar.gz") + with tarfile.open(path, "w:gz") as tar: + info = tarfile.TarInfo("%s/zlib/v1.3.1-1/x" % ARCH) + info.size = 1 + tar.addfile(info, io.BytesIO(b"x")) + sync = FakeReapiSync() + self.assertIsNone(migrate_tarball(sync, path, alidist)) + self.assertEqual(sync.calls, []) + + def test_enrich_source_snapshot_adds_source(self): + s = _EnrichSync(_ac_entry()) + art = {"type": "git", "commit": "deadbeef", "baseDigest": None, + "deltaDigest": "d" * 64} + refs = {"type": "git-refs", "digest": "r" * 64} + with patch("alibuild_helpers.migrate.snapshot_legacy_source", + return_value=(art, refs, "deadbeef")) as snap: + result = enrich_source_snapshot(s, "zlib", "v1.3.1", "6", "/tmp/mirror") + self.assertEqual(result, "migrated") + # It clones/snapshots using the source+tag recorded in the AC entry, + # never the tarball. + _, meta_arg, mirror_arg = snap.call_args[0] + self.assertEqual(meta_arg["package"]["source"], + "https://example.invalid/zlib.git") + self.assertEqual(meta_arg["package"]["tag"], "v1.3.1") + self.assertEqual(mirror_arg, "/tmp/mirror") + # The AC entry is rewritten in place with the snapshot + resolved commit. + self.assertEqual(len(s.updated), 1) + action = s.updated[0]["action"] + self.assertEqual(action["sourceArtifact"], art) + self.assertEqual(action["refsArtifact"], refs) + self.assertEqual(action["commit"]["commitHash"], "deadbeef") + + def test_enrich_source_snapshot_idempotent(self): + # An entry that already has a snapshot is left untouched. + s = _EnrichSync(_ac_entry(source_artifact={"type": "git", "commit": "x"})) + with patch("alibuild_helpers.migrate.snapshot_legacy_source") as snap: + result = enrich_source_snapshot(s, "zlib", "v1.3.1", "6", "/tmp/mirror") + self.assertEqual(result, "present") + snap.assert_not_called() + self.assertEqual(s.updated, []) + + def test_enrich_source_snapshot_no_upstream_source(self): + # Packages without an upstream source (e.g. defaults-release) are a no-op. + s = _EnrichSync(_ac_entry(source=None)) + with patch("alibuild_helpers.migrate.snapshot_legacy_source") as snap: + result = enrich_source_snapshot(s, "defaults-release", "v1", "1", + "/tmp/mirror") + self.assertEqual(result, "present") + snap.assert_not_called() + self.assertEqual(s.updated, []) + + def test_enrich_source_snapshot_upstream_gone(self): + # If the upstream clone/snapshot fails, nothing is rewritten. + s = _EnrichSync(_ac_entry()) + with patch("alibuild_helpers.migrate.snapshot_legacy_source", + return_value=(None, None, None)): + result = enrich_source_snapshot(s, "zlib", "v1.3.1", "6", "/tmp/mirror") + self.assertEqual(result, "skipped") + self.assertEqual(s.updated, []) + + def test_enrich_source_snapshot_unresolved_hash(self): + class _NoHash(_EnrichSync): + def resolve_action_hash(self, *a, **k): + return None + s = _NoHash(_ac_entry()) + result = enrich_source_snapshot(s, "zlib", "v1.3.1", "6", "/tmp/mirror") + self.assertEqual(result, "skipped") + self.assertEqual(s.updated, []) + + def test_doEnrichSources_walks_ledger(self): + # `migrate --snapshot-sources` with no TARBALL enriches every AC entry + # from the ledger: sourced ones get snapshotted, source-less/already-done + # ones are left alone -- no old store involved. + from alibuild_helpers.migrate import doEnrichSources + entries = { + "h-zlib": _ac_entry(), # has source, no snapshot + "h-def": _ac_entry(source=None), # source-less + "h-done": _ac_entry(source_artifact={"type": "git"}), # already snapshotted + } + for h, e in entries.items(): + e["action"]["actionHash"] = h + + class _LedgerSync(sync_reapi.REAPIRemoteSync): + def __init__(self): self.updated = [] + def iter_ac_entry_hashes(self, arch): return list(entries) + def read_ac_entry(self, h): return entries[h] + def update_ac_entry(self, e): self.updated.append(e["action"]["actionHash"]) + + ledger = _LedgerSync() + args = Namespace(remoteStore="reapi://localhost/cas", acStore="", architecture=ARCH, + workDir=self.tmp, insecure=False, storage="permanent", + source_mirror=os.path.join(self.tmp, "mir"), dryRun=False, jobs=1) + with patch("alibuild_helpers.migrate.remote_from_url", return_value=ledger), \ + patch("alibuild_helpers.migrate.snapshot_legacy_source", + return_value=({"type": "git", "commit": "c"}, {"digest": "r"}, "c")) as snap: + ok = doEnrichSources(args) + # Only the sourced, un-snapshotted entry is cloned + rewritten. + self.assertEqual(snap.call_count, 1) + self.assertEqual(ledger.updated, ["h-zlib"]) + self.assertTrue(ok) + + @unittest.skipUnless(shutil.which("git"), "git required") + def test_snapshot_falls_back_when_rc_branch_deleted(self): + from alibuild_helpers.migrate import snapshot_legacy_source + from alibuild_helpers.source import load_refs + env = dict(os.environ, GIT_AUTHOR_NAME="t", GIT_AUTHOR_EMAIL="a@b.c", + GIT_COMMITTER_NAME="t", GIT_COMMITTER_EMAIL="a@b.c") + + def g(*a): + return subprocess.run(["git", "-C", up, *a], env=env, check=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT).stdout.decode().strip() + + up = os.path.join(self.tmp, "upstream") + os.makedirs(up) + g("init", "-q") + with open(os.path.join(up, "f.txt"), "w") as ff: + ff.write("src") + g("add", "-A") + g("commit", "-qm", "daily") + commit = g("rev-parse", "HEAD") + # Built on rc/, with the real tag also created; then the rc + # branch is deleted upstream (the failure this fallback handles). + g("branch", "rc/daily-20260630-0000") + g("tag", "daily-20260630-0000") + g("branch", "-D", "rc/daily-20260630-0000") + + sync = FakeReapiSync() + meta = {"architecture": ARCH, + "package": {"name": "o2physics", "source": up, + "tag": "rc/daily-20260630-0000"}} + art, refs, got = snapshot_legacy_source( + sync, meta, os.path.join(self.tmp, "mirror")) + # Resolved to the same commit via the surviving tag, and snapshotted. + self.assertEqual(got, commit) + self.assertIsNotNone(art) + # The recipe's rc/ tag is pinned in the refs so offline checkout resolves + # it at reconstruct time even though upstream deleted the branch. + self.assertEqual(load_refs(sync, refs)["refs/tags/rc/daily-20260630-0000"], + commit) + + def test_ref_candidates_strips_rc_prefix(self): + from alibuild_helpers.migrate import _ref_candidates + self.assertEqual(_ref_candidates("rc/daily-20260630-0000"), + ["rc/daily-20260630-0000", "daily-20260630-0000"]) + self.assertEqual(_ref_candidates("v1.2.3"), ["v1.2.3"]) + + def test_verify_recovered_recipe(self): + recipe = "package: zlib\nversion: v1.3.1\n---\nbuild\n" + ok, _ = verify_recovered_recipe(META, recipe) + self.assertTrue(ok) + # Wrong package field is caught. + ok, reason = verify_recovered_recipe(META, "package: other\n---\nbuild\n") + self.assertFalse(ok) + self.assertIn("expected", reason) + # A dependency without a recorded hash is caught. + bad_meta = json.loads(json.dumps(META)) + bad_meta["dependencies"]["recursive"]["build"][0]["hash"] = "" + ok, reason = verify_recovered_recipe(bad_meta, recipe) + self.assertFalse(ok) + self.assertIn("no recorded hash", reason) + + def test_migrate_tarball_skips_on_failed_verify(self): + alidist, commit = self._make_alidist() # provides recipe with package: zlib + meta = json.loads(json.dumps(META)) + meta["alidist"]["commit"] = commit + meta["package"]["name"] = "notzlib" # mismatch -> recover gets zlib.sh? no + # Point the package name at something whose recipe (zlib.sh) won't match. + meta["package"]["name"] = "zlib" + # Force a mismatch by tampering the recovered recipe's expectation: + meta["dependencies"]["recursive"]["build"][0]["hash"] = "" + path = self._write_tarball(meta) + sync = FakeReapiSync() + self.assertIsNone(migrate_tarball(sync, path, alidist)) # verify fails -> skip + self.assertEqual(sync.calls, []) + # With verification disabled, it is migrated. + self.assertIsNotNone(migrate_tarball(sync, path, alidist, verify=False)) + + def _make_source_repo(self, tag): + """A git 'upstream' repo with a tagged commit; returns (path, sha).""" + repo = os.path.join(self.tmp, "upstream") + os.makedirs(repo) + env = dict(os.environ, GIT_AUTHOR_NAME="t", GIT_AUTHOR_EMAIL="a@b.c", + GIT_COMMITTER_NAME="t", GIT_COMMITTER_EMAIL="a@b.c") + run = lambda *a: subprocess.run(["git"] + list(a), cwd=repo, env=env, + check=True, stdout=subprocess.PIPE).stdout.decode().strip() + run("init", "-q") + with open(os.path.join(repo, "src.c"), "w") as srcf: + srcf.write("int main(){}\n") + run("add", ".") + run("commit", "-qm", "code") + run("tag", tag) + return repo, run("rev-parse", "HEAD") + + def test_migrate_snapshots_source(self): + alidist, commit = self._make_alidist() + upstream, sha = self._make_source_repo("v1.3.1") + meta = json.loads(json.dumps(META)) + meta["alidist"]["commit"] = commit + meta["package"]["source"] = upstream # local repo stands in for upstream + path = self._write_tarball(meta) + sync = FakeReapiSync() + mirror = os.path.join(self.tmp, "mirror") + migrate_tarball(sync, path, alidist, snapshot_sources=True, mirror_dir=mirror) + + entry = sync.calls[0][0]["action"] + # Source + refs were archived, and the commit was resolved to a real SHA + # (so the source-aware checkout's SOURCES path matches at rebuild). + self.assertIsNotNone(entry["sourceArtifact"]) + self.assertIsNotNone(entry["refsArtifact"]) + self.assertEqual(entry["commit"]["commitHash"], sha) + self.assertEqual(entry["commit"]["ref"], sha) + + def test_download_from_old_store(self): + urls = [] + pointer = "%s/store/92/abc/ROOT-v6-28-04-1.%s.tar.gz" % (ARCH, ARCH) + link_url = "https://store/repo/TARS/%s/ROOT/ROOT-v6-28-04-1.%s.tar.gz" % (ARCH, ARCH) + + def fake_get(url, **kwargs): + urls.append(url) + # First GET resolves the symlink pointer; second fetches the bytes. + return _FakeResp(text=pointer) if url == link_url else _FakeResp(b"TARBALL-BYTES") + + with patch("alibuild_helpers.migrate.requests.get", side_effect=fake_get): + dest = download_from_old_store("https://store/repo/", ARCH, + "ROOT/v6-28-04-1", self.tmp) + # Two-step: GET the symlink, then the resolved content-addressed object. + self.assertEqual(urls[0], link_url) + self.assertEqual(urls[1], "https://store/repo/TARS/" + pointer) + with open(dest, "rb") as got: + self.assertEqual(got.read(), b"TARBALL-BYTES") + + def test_enumerate_closure(self): + # dist tree lists the closure; TARS// lists package dirs. + dist_prefix = "TARS/%s/dist/O2/O2-daily-1/" % ARCH + pkg_prefix = "TARS/%s/" % ARCH + + def fake_list(read_url, prefix): + if prefix == dist_prefix: + return [dist_prefix + n for n in ( + "O2-daily-1.%s.tar.gz" % ARCH, + "GCC-Toolchain-v14-1.%s.tar.gz" % ARCH, + "ninja-fortran-v1.11.1.g9-25.%s.tar.gz" % ARCH, + "zlib-v1.3.1-6.%s.tar.gz" % ARCH)] + if prefix == pkg_prefix: + # Both GCC/GCC-Toolchain and ninja/ninja-fortran are packages that + # prefix-match a filename -> disambiguated by which symlink exists. + return [pkg_prefix + n + "/" for n in + ("O2", "GCC", "GCC-Toolchain", "ninja", "ninja-fortran", "zlib")] + return [] + + # The *correct* per-package symlink exists; the wrong-prefix one 404s. + exists = {"GCC-Toolchain/GCC-Toolchain-v14-1", + "ninja/ninja-fortran-v1.11.1.g9-25"} + + def fake_head(url, timeout=None): + parts = url.rstrip("/").split("/") + base = parts[-1][:-len(".%s.tar.gz" % ARCH)] + resp = MagicMock() + resp.status_code = 200 if "%s/%s" % (parts[-2], base) in exists else 404 + return resp + + with patch("alibuild_helpers.migrate._list_old_store", side_effect=fake_list), \ + patch("alibuild_helpers.migrate.requests.head", side_effect=fake_head): + specs = enumerate_closure("https://store", ARCH, "O2/daily-1") + self.assertEqual(set(specs), + {"O2/daily-1", "GCC-Toolchain/v14-1", + "ninja/fortran-v1.11.1.g9-25", "zlib/v1.3.1-6"}) + + def test_populate_system_deps(self): + from alibuild_helpers.migrate import populate_system_deps + probe_recipe = "package: probe\nversion: '1'\nrequires:\n - make\n---\nbuild\n" + probe_digest = hashlib.sha256(probe_recipe.encode()).hexdigest() + make_recipe = ("package: make\nversion: '4'\nsystem_requirement: '.*'\n" + "system_requirement_check: |\n type make\n---\n") + make_digest = hashlib.sha256(make_recipe.encode()).hexdigest() + probe_entry = {"schemaVersion": 2, "action": { + "package": "probe", "version": "1", "revision": "1", "architecture": ARCH, + "actionHash": "hp", "recipeDigest": "sha256:" + probe_digest, "deps": []}} + put_entries, updated = {}, {} + + class FakeSync: + def iter_ac_entry_hashes(self, architecture): return ["hp"] + def read_ac_entry(self, h): return probe_entry if h == "hp" else None + def read_blob(self, digest, algo="sha256"): + return probe_recipe.encode() if digest == probe_digest else b"" + def put_ac_entry(self, entry, recipe_text=""): + put_entries[entry["action"]["actionHash"]] = entry + def update_ac_entry(self, entry): + updated[entry["action"]["package"]] = entry + + with patch("alibuild_helpers.migrate.recover_recipe", + side_effect=lambda d, c, pkg: make_recipe if pkg == "make" else None): + enriched, added = populate_system_deps(FakeSync(), ARCH, "/alidist", "HEAD") + self.assertEqual((enriched, added), (1, 1)) + # make got a validate-system entry keyed by its recipe digest, + self.assertEqual(put_entries[make_digest]["action"]["kind"], "validate-system") + # and probe's deps now reference it by the same digest. + self.assertEqual(updated["probe"]["action"]["deps"], + [{"package": "make", "actionHash": make_digest}]) + + def test_recover_legacy_deps(self): + from alibuild_helpers.migrate import recover_legacy_deps + suffix = ".%s.tar.gz" % ARCH + pkg_prefix = "TARS/%s/" % ARCH + hashes = {"zlib/v1.2.8-1": "hz", "GCC-Toolchain/v14-1": "hg"} + + def fake_list(read_url, prefix): + if prefix == pkg_prefix: + return [pkg_prefix + n + "/" for n in + ["zlib", "GCC-Toolchain", "dist", "dist-direct", "dist-runtime", "store"]] + if prefix == "TARS/%s/dist-direct/zlib/zlib-v1.2.8-1/" % ARCH: + # includes self plus the one direct dep + return ["x/GCC-Toolchain-v14-1" + suffix, "x/zlib-v1.2.8-1" + suffix] + if prefix == "TARS/%s/dist-runtime/zlib/zlib-v1.2.8-1/" % ARCH: + return ["x/GCC-Toolchain-v14-1" + suffix] + return [] # GCC-Toolchain has no dist-direct/runtime folder + + updated = {} + + class FakeSync: + def resolve_action_hash(self, pkg, version, revision=None): + return hashes.get("%s/%s-%s" % (pkg, version, revision)) + + def read_ac_entry(self, h): + pkg = {"hz": "zlib", "hg": "GCC-Toolchain"}.get(h) + return None if pkg is None else { + "schemaVersion": 2, + "action": {"kind": "legacy", "package": pkg, "actionHash": h}} + + def update_ac_entry(self, entry): + updated[entry["action"]["package"]] = entry + + with patch("alibuild_helpers.migrate._list_old_store", side_effect=fake_list), \ + patch("alibuild_helpers.migrate.requests.head", + side_effect=lambda url, timeout=None: MagicMock(status_code=200)): + n = recover_legacy_deps("https://store", ARCH, + ["zlib/v1.2.8-1", "GCC-Toolchain/v14-1"], FakeSync()) + self.assertEqual(n, 2) + # zlib's direct dep GCC is hash-linked; self is excluded. + self.assertEqual(updated["zlib"]["action"]["deps"], + [{"package": "GCC-Toolchain", "actionHash": "hg"}]) + self.assertEqual(updated["zlib"]["action"]["runtimeDeps"], + [{"package": "GCC-Toolchain", "actionHash": "hg"}]) + # GCC has no dist folder -> empty (leaf) deps. + self.assertEqual(updated["GCC-Toolchain"]["action"]["deps"], []) + + def test_enumerate_arch(self): + pkg_prefix = "TARS/%s/" % ARCH + suffix = ".%s.tar.gz" % ARCH + per_pkg = {"zlib": ["zlib-v1.3.1-6", "zlib-v1.2.11-1"], + "RapidJSON": ["RapidJSON-v1.1.0-3"], + "ninja-fortran": ["ninja-fortran-v1.11.1.g9-25"]} + + def fake_list(read_url, prefix): + if prefix == pkg_prefix: + # real package dirs plus publisher/store subtrees that must be excluded + return [pkg_prefix + n + "/" for n in + list(per_pkg) + ["dist", "dist-direct", "dist-runtime", "store"]] + for pkg, builds in per_pkg.items(): + if prefix == "%s%s/" % (pkg_prefix, pkg): + return ["%s%s/%s%s" % (pkg_prefix, pkg, b, suffix) for b in builds] + \ + ["%s%s/latest" % (pkg_prefix, pkg)] # non-tarball -> skipped + return [] + + with patch("alibuild_helpers.migrate._list_old_store", side_effect=fake_list), \ + patch("alibuild_helpers.migrate.requests.head", + side_effect=lambda url, timeout=None: MagicMock(status_code=200)): + allspecs = enumerate_arch("https://store", ARCH) + only_rj = enumerate_arch("https://store", ARCH, r"^RapidJSON/") + self.assertEqual(set(allspecs), + {"zlib/v1.3.1-6", "zlib/v1.2.11-1", + "RapidJSON/v1.1.0-3", "ninja-fortran/v1.11.1.g9-25"}) + self.assertEqual(set(only_rj), {"RapidJSON/v1.1.0-3"}) # regex filters; dist/store excluded + + def test_enumerate_closure_no_dist_tolerant(self): + # A dependency-only package (no dist/ tree): strict raises, non-strict + # (--match-driven) falls back to migrating the package alone. + with patch("alibuild_helpers.migrate._list_old_store", return_value=[]): + self.assertEqual( + enumerate_closure("https://store", ARCH, "ninja/fortran-v1.8.2.g3b-1", + strict=False), + ["ninja/fortran-v1.8.2.g3b-1"]) + with self.assertRaises(SystemExit): + enumerate_closure("https://store", ARCH, "ninja/fortran-v1.8.2.g3b-1") + + def test_migrate_tarball_dry_run(self): + alidist, commit = self._make_alidist() + meta = json.loads(json.dumps(META)) + meta["alidist"]["commit"] = commit + path = self._write_tarball(meta) + sync = FakeReapiSync() + action_hash = migrate_tarball(sync, path, alidist, dry_run=True) + self.assertEqual(action_hash, META["package"]["hash"]) + self.assertEqual(sync.calls, []) # dry-run writes nothing + + def test_doMigrate_parallel_processes_all(self): + args = Namespace( + tarballs=["a.tgz", "b.tgz", "c.tgz", "d.tgz", "e.tgz"], + remoteStore="reapi://localhost/cas", acStore="", architecture=ARCH, + workDir="/sw", insecure=False, alidist="/alidist", container=None, + no_verify=False, snapshot_sources=False, source_mirror=None, + read_store=None, closure=False, storage="ephemeral", jobs=4) + + processed, lock = [], threading.Lock() + + def fake_migrate(sync_, tarball, *a, **kw): + with lock: + processed.append(tarball) + return True + + fake_sync = MagicMock(spec=sync_reapi.REAPIRemoteSync) + with patch("alibuild_helpers.migrate.remote_from_url", return_value=fake_sync), \ + patch("alibuild_helpers.migrate.migrate_tarball", side_effect=fake_migrate): + ok = doMigrate(args, None) + self.assertTrue(ok) + # Every package was processed exactly once across the 4 worker threads. + self.assertEqual(sorted(processed), ["a.tgz", "b.tgz", "c.tgz", "d.tgz", "e.tgz"]) + + def test_download_resumes_after_interruption(self): + from requests.exceptions import ChunkedEncodingError + from alibuild_helpers.migrate import _download_with_resume + full = b"0123456789" * 10 # 100 bytes + ranges = [] + + def fake_get(url, stream=False, headers=None, timeout=None): + ranges.append((headers or {}).get("Range")) + first = len(ranges) == 1 + resp = MagicMock() + resp.__enter__ = lambda s: s + resp.__exit__ = lambda s, *a: False + resp.raise_for_status = lambda: None + if first: + resp.status_code = 200 + resp.headers = {"content-length": str(len(full))} + def it(_n): + yield full[:40] + raise ChunkedEncodingError("connection dropped") + resp.iter_content = it + else: + start = int(ranges[-1].split("=")[1].rstrip("-")) + resp.status_code = 206 + resp.headers = {"content-length": str(len(full) - start)} + resp.iter_content = lambda _n: iter([full[start:]]) + return resp + + dest = os.path.join(self.tmp, "resume.bin") + with patch("alibuild_helpers.migrate.requests.get", side_effect=fake_get), \ + patch("alibuild_helpers.migrate.time.sleep"): + _download_with_resume("http://x/obj", dest, retries=3) + + with open(dest, "rb") as got: + self.assertEqual(got.read(), full) # fully reassembled + self.assertIsNone(ranges[0]) # first attempt: no Range + self.assertEqual(ranges[1], "bytes=40-") # resumed from the 40 bytes on disk + + def test_byte_progress(self): + from alibuild_helpers import log + with patch.object(log, "debug") as dbg: + prog = log.byte_progress("upload x", total=1000 << 20, + every_bytes=200 << 20, every_seconds=10 ** 9) + for _ in range(4): + prog(100 << 20) # 100 MB each -> crosses 200 MB twice (at 200, 400) + self.assertEqual(dbg.call_count, 2) + + def test_doMigrate_requires_reapi_store(self): + args = Namespace(remoteStore="https://s3.cern.ch/foo", architecture=ARCH, + workDir="/sw", insecure=False, tarballs=["x.tar.gz"], + alidist="/alidist", container=None) + self.assertRaises(SystemExit, doMigrate, args, None) + + +class StripRwTestCase(unittest.TestCase): + """`migrate` must accept the same store URL `build` does. + + finaliseArgs() normalises the ::rw suffix only for build and doctor, so every + other subcommand saw it verbatim and handed boto3 a bucket literally named + "alibuild-cas::rw", which fails parameter validation before any request goes + out. Copying a working --remote-store from a build invocation into a migrate + one is the obvious thing to do, so it has to work. + """ + + def test_suffix_is_dropped(self): + self.assertEqual(strip_rw("reapi://h/alibuild-cas::rw"), "reapi://h/alibuild-cas") + + def test_url_without_the_suffix_is_untouched(self): + self.assertEqual(strip_rw("reapi://h/alibuild-cas"), "reapi://h/alibuild-cas") + + def test_trailing_whitespace_does_not_hide_the_suffix(self): + self.assertEqual(strip_rw("reapi://h/alibuild-cas::rw "), "reapi://h/alibuild-cas") + + def test_empty_and_none_become_empty(self): + for value in ("", None): + self.assertEqual(strip_rw(value), "") + + def test_rw_elsewhere_in_the_url_is_kept(self): + """Only a SUFFIX is the marker.""" + self.assertEqual(strip_rw("reapi://h/rw-cas"), "reapi://h/rw-cas") + +if __name__ == "__main__": + unittest.main() + diff --git a/tests/test_reconstruct.py b/tests/test_reconstruct.py new file mode 100644 index 00000000..60019efd --- /dev/null +++ b/tests/test_reconstruct.py @@ -0,0 +1,556 @@ +import hashlib +import os +import os.path +import shutil +import tempfile +import unittest +from argparse import Namespace +from unittest.mock import patch, MagicMock + +from alibuild_helpers import sync +from alibuild_helpers import sync_reapi +from alibuild_helpers.reconstruct import ( + walk_build_closure, find_missing_blobs, materialize_recipes, doReconstruct, + recipe_package_name, defaults_from_closure, + restore_sources, verify_closure, verify_rebuild, _rebuilt_tarball, + _rebuild_verdict, do_rebaseline, do_persist, RebuildResult) + +ARCH = "slc7_x86-64" + + +def make_entry(pkg, version, revision, content_hash, recipe_hash, + deps=(), container=None): + return { + "schemaVersion": 2, + "action": { + "package": pkg, "version": version, "revision": revision, + "architecture": ARCH, "actionHash": "hash-" + pkg, + "recipeDigest": "sha256:" + recipe_hash, + "container": container, + "deps": [{"package": p, "actionHash": "hash-" + p} for p in deps], + "runtimeDeps": [], + }, + "result": {"tarball": "%s-%s-%s.%s.tar.gz" % (pkg, version, revision, ARCH), + "outputDigest": "sha256:" + content_hash, "size": 1}, + } + + +class FakeSync(sync_reapi.REAPIRemoteSync): + """A REAPIRemoteSync whose CAS/AC reads come from in-memory fixtures.""" + + def __init__(self, entries, blobs_present, recipe_blobs, label_to_hash): + self.architecture = ARCH + self._entries = entries # action hash -> AC entry + self._present = set(blobs_present) # content hashes present in the CAS + self._recipes = recipe_blobs # recipe content hash -> bytes + self._labels = label_to_hash + + def read_ac_entry(self, action_hash): + return self._entries.get(action_hash) + + def resolve_action_hash(self, package, version, revision=None): + return self._labels.get((package, version, revision)) + + def artifact_blob_exists(self, content_hash, algo="sha256"): + return content_hash in self._present + + def read_blob(self, content_hash, algo="sha256"): + return self._recipes[content_hash] + + +class ReconstructTestCase(unittest.TestCase): + def setUp(self): + # zlib depends on GCC; GCC depends on defaults-release. + self.entries = { + "hash-zlib": make_entry("zlib", "v1", "1", "czlib", "rzlib", + deps=["GCC"], + container={"runtime": "docker", + "image": "alisw/slc7-builder:latest", + "digest": "alisw/slc7-builder@sha256:abc"}), + "hash-GCC": make_entry("GCC", "v9", "2", "cgcc", "rgcc", + deps=["defaults-release"]), + "hash-defaults-release": make_entry("defaults-release", "v1", "1", + "cdef", "rdef"), + } + self.recipes = {"rzlib": b"package: zlib\n---\nbuild zlib\n", + "rgcc": b"package: GCC\n---\nbuild gcc\n", + "rdef": b"package: defaults-release\n---\n"} + self.labels = {("zlib", "v1", None): "hash-zlib", + ("zlib", "v1", "1"): "hash-zlib"} + self.tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.tmp, ignore_errors=True) + + def make_sync(self, present): + return FakeSync(self.entries, present, self.recipes, self.labels) + + def test_walk_build_closure_postorder(self): + s = self.make_sync(present=()) + closure = walk_build_closure(s, "hash-zlib") + names = [e["action"]["package"] for e in closure] + # Dependencies must come before the packages that need them. + self.assertEqual(names, ["defaults-release", "GCC", "zlib"]) + + def test_find_missing_blobs(self): + # Only GCC's blob is present; the other two are missing. + s = self.make_sync(present={"cgcc"}) + missing = find_missing_blobs(s, walk_build_closure(s, "hash-zlib")) + self.assertEqual({e["action"]["package"] for e in missing}, + {"zlib", "defaults-release"}) + + def test_materialize_recipes(self): + s = self.make_sync(present=()) + with tempfile.TemporaryDirectory() as cfg: + written = materialize_recipes(s, walk_build_closure(s, "hash-zlib"), cfg) + self.assertEqual(len(written), 3) + # Files are named .sh (lowercased) and hold the full recipe. + with open(os.path.join(cfg, "zlib.sh"), "rb") as zf: + self.assertEqual(zf.read(), self.recipes["rzlib"]) + self.assertTrue(os.path.exists(os.path.join(cfg, "gcc.sh"))) + self.assertTrue(os.path.exists(os.path.join(cfg, "defaults-release.sh"))) + + def _use_o2_defaults(self): + """alibuild injects every defaults flavour under the single node name + `defaults-release`, so the archived recipe legitimately declares a + different package. Reproduce that shape.""" + self.recipes["rdef"] = b"package: defaults-o2\n---\n" + + def test_recipe_package_name_reads_the_header_only(self): + self.assertEqual(recipe_package_name(b"package: defaults-o2\n---\n"), + "defaults-o2") + # A `package:` line in the *body* must not be mistaken for the header. + self.assertEqual(recipe_package_name(b"---\npackage: nonsense\n"), "") + self.assertEqual(recipe_package_name(b"version: v1\n---\n"), "") + + def test_materialize_names_defaults_after_the_declared_package(self): + # Writing an o2 defaults recipe out as defaults-release.sh makes the + # rebuild reject the file for disagreeing with its own package field. + self._use_o2_defaults() + s = self.make_sync(present=()) + with tempfile.TemporaryDirectory() as cfg: + materialize_recipes(s, walk_build_closure(s, "hash-zlib"), cfg) + self.assertTrue(os.path.exists(os.path.join(cfg, "defaults-o2.sh"))) + self.assertFalse(os.path.exists(os.path.join(cfg, "defaults-release.sh"))) + + def test_defaults_recovered_from_the_archived_recipe(self): + self._use_o2_defaults() + s = self.make_sync(present=()) + closure = walk_build_closure(s, "hash-zlib") + # The AC entries carry no `defaults` field (migrated ones never do), so + # the flavour has to come from the recipe the ledger archived. + self.assertEqual(defaults_from_closure(s, closure), "o2") + + def test_defaults_recovery_is_release_for_a_plain_build(self): + s = self.make_sync(present=()) + self.assertEqual( + defaults_from_closure(s, walk_build_closure(s, "hash-zlib")), "release") + + def test_doReconstruct_nothing_missing(self): + s = self.make_sync(present={"czlib", "cgcc", "cdef"}) + args = Namespace(package="zlib", version="v1", revision=None, architecture=ARCH, + remoteStore="reapi://localhost/bucket", insecure=False, + workDir="/sw", outputConfig=None) + with patch("alibuild_helpers.reconstruct.remote_from_url", return_value=s): + # All blobs present -> succeeds without materialising anything. + self.assertTrue(doReconstruct(args, None)) + + def test_doReconstruct_materializes_when_missing(self): + s = self.make_sync(present=()) + with tempfile.TemporaryDirectory() as workdir: + args = Namespace(package="zlib", version="v1", revision=None, architecture=ARCH, + remoteStore="reapi://localhost/bucket", insecure=False, + workDir=workdir, outputConfig=None) + with patch("alibuild_helpers.reconstruct.remote_from_url", return_value=s): + self.assertTrue(doReconstruct(args, None)) + cfg = os.path.join(workdir, "reconstruct-zlib") + self.assertTrue(os.path.exists(os.path.join(cfg, "zlib.sh"))) + self.assertTrue(os.path.exists(os.path.join(cfg, "gcc.sh"))) + + def test_restore_sources(self): + s = self.make_sync(present=()) + # zlib has an archived source artifact (and refs); the others don't. + self.entries["hash-zlib"]["action"]["sourceArtifact"] = { + "type": "git", "commit": "deadbeef", "baseDigest": None, + "deltaDigest": "abc"} + self.entries["hash-zlib"]["action"]["refsArtifact"] = { + "type": "git-refs", "digest": "r" * 64} + closure = walk_build_closure(s, "hash-zlib") + with patch("alibuild_helpers.reconstruct.GitSourceStore") as gss, \ + patch("alibuild_helpers.reconstruct.load_refs", return_value={}) as load, \ + patch("alibuild_helpers.reconstruct.apply_refs") as apply_: + restored, from_upstream = restore_sources(s, closure, "/tmp/ref") + self.assertEqual(restored, ["zlib"]) + self.assertEqual(set(from_upstream), {"GCC", "defaults-release"}) + gss.return_value.restore.assert_called_once() + # The cached tag mapping is reapplied so tags resolve offline. + load.assert_called_once() + apply_.assert_called_once() + + def _verify_sync(self, present, sources=None, break_recipe=None): + """A FakeSync whose recipe blobs have *real* sha256 digests (so recipe + integrity checks are meaningful), with a configurable set of present CAS + blobs, per-package sources, and an optionally-corrupted recipe.""" + recipes = {"zlib": b"r-zlib", "GCC": b"r-gcc", "defaults-release": b"r-def"} + dig = {p: hashlib.sha256(b).hexdigest() for p, b in recipes.items()} + + def entry(pkg, ver, rev, chash, deps=()): + e = make_entry(pkg, ver, rev, chash, dig[pkg], deps=deps) + if sources and pkg in sources: + e["action"]["source"] = sources[pkg] + return e + + entries = { + "hash-zlib": entry("zlib", "v1", "1", "czlib", deps=["GCC"]), + "hash-GCC": entry("GCC", "v9", "2", "cgcc", deps=["defaults-release"]), + "hash-defaults-release": entry("defaults-release", "v1", "1", "cdef"), + } + recipe_blobs = {dig[p]: b for p, b in recipes.items()} + if break_recipe: + recipe_blobs[dig[break_recipe]] = b"tampered" # sha256 no longer matches + labels = {("zlib", "v1", None): "hash-zlib", ("zlib", "v1", "1"): "hash-zlib"} + return FakeSync(entries, present, recipe_blobs, labels) + + def test_find_missing_blobs_skips_validate_system(self): + # A validate-system entry has no output digest; it must be skipped, not + # flagged as a missing blob. + s = self.make_sync(present=set()) + sys_entry = {"action": {"kind": "validate-system", "package": "yacc-like", + "recipeDigest": "sha256:ryacc"}} + build_entry = make_entry("zlib", "v1", "1", "czlib", "rzlib") + missing = find_missing_blobs(s, [sys_entry, build_entry]) + self.assertEqual([e["action"]["package"] for e in missing], ["zlib"]) + + def test_verify_closure_reports_system(self): + yacc_recipe = b"package: yacc-like\nsystem_requirement: yacc\n" + zlib_recipe = b"package: zlib\n" + yd = hashlib.sha256(yacc_recipe).hexdigest() + zd = hashlib.sha256(zlib_recipe).hexdigest() + zlib_entry = make_entry("zlib", "v1", "1", "czlib", zd) + sys_entry = {"schemaVersion": 2, "action": { + "kind": "validate-system", "package": "yacc-like", "version": "v1", + "revision": "1", "architecture": ARCH, "actionHash": "hash-yacc", + "recipeDigest": "sha256:" + yd, "deps": []}} + s = FakeSync(entries={}, blobs_present={"czlib"}, + recipe_blobs={yd: yacc_recipe, zd: zlib_recipe}, label_to_hash={}) + rows, ok = verify_closure(s, [sys_entry, zlib_entry]) + self.assertTrue(ok) + by = {r["package"]: r for r in rows} + # The system package is reported as 'system' (revalidated on host), never + # reused/rebuilt, with its recipe verified and no source. + self.assertEqual(by["yacc-like"]["action"], "system") + self.assertTrue(by["yacc-like"]["recipe_ok"]) + self.assertEqual(by["yacc-like"]["source"], "n/a") + self.assertEqual(by["zlib"]["action"], "reuse") + + def test_verify_closure_reports_legacy(self): + # A legacy (pre-provenance) entry: no recipe, keyed by content hash. It is + # reported as 'legacy', skipped by find_missing_blobs, and only "ok" while + # its blob survives (it can never be rebuilt). + legacy_entry = {"schemaVersion": 2, "action": { + "kind": "legacy", "package": "bz2", "version": "1.0.8", "revision": "1", + "architecture": ARCH, "actionHash": "chash-bz2"}, + "result": {"tarball": "bz2-1.0.8-1.%s.tar.gz" % ARCH, + "outputDigest": "sha256:chash-bz2", "size": 1}} + present = FakeSync(entries={}, blobs_present={"chash-bz2"}, + recipe_blobs={}, label_to_hash={}) + rows, ok = verify_closure(present, [legacy_entry]) + self.assertTrue(ok) + self.assertEqual(rows[0]["action"], "legacy") + self.assertEqual(rows[0]["source"], "n/a") + self.assertTrue(rows[0]["regenerable"]) # present -> fine + self.assertEqual(find_missing_blobs(present, [legacy_entry]), []) # never a rebuild target + # If its blob is lost, it is flagged non-regenerable (gone for good). + lost = FakeSync(entries={}, blobs_present=set(), recipe_blobs={}, label_to_hash={}) + rows, ok = verify_closure(lost, [legacy_entry]) + self.assertFalse(ok) + self.assertFalse(rows[0]["regenerable"]) + + def test_legacy_is_never_rebuildable_even_when_present(self): + # `regenerable` means "satisfiable right now" (the blob is there); + # `rebuildable` means "could be regenerated if the blob were lost". A + # legacy entry has no recipe, so it is never the latter -- conflating the + # two made a closure of purely legacy entries report SUCCESS. + legacy_entry = {"schemaVersion": 2, "action": { + "kind": "legacy", "package": "bz2", "version": "1.0.8", "revision": "1", + "architecture": ARCH, "actionHash": "chash-bz2"}, + "result": {"outputDigest": "sha256:chash-bz2", "size": 1}} + present = FakeSync(entries={}, blobs_present={"chash-bz2"}, + recipe_blobs={}, label_to_hash={}) + rows, _ = verify_closure(present, [legacy_entry]) + self.assertTrue(rows[0]["regenerable"]) + self.assertFalse(rows[0]["rebuildable"]) + + def test_healthy_closure_is_rebuildable(self): + s = self._verify_sync(present={"czlib", "cgcc", "cdef"}) + rows, ok = verify_closure(s, walk_build_closure(s, "hash-zlib")) + self.assertTrue(ok) + self.assertTrue(all(r["rebuildable"] for r in rows)) + + def test_verify_all_present_reuses_everything(self): + s = self._verify_sync(present={"czlib", "cgcc", "cdef"}) + rows, ok = verify_closure(s, walk_build_closure(s, "hash-zlib")) + self.assertTrue(ok) + self.assertTrue(all(r["action"] == "reuse" for r in rows)) + self.assertTrue(all(r["recipe_ok"] for r in rows)) + + def test_verify_missing_blob_is_regenerable(self): + # zlib's tarball is gone; GCC + defaults are present -> only zlib rebuilds, + # and it's regenerable (recipe intact, deps consistent). + s = self._verify_sync(present={"cgcc", "cdef"}) + rows, ok = verify_closure(s, walk_build_closure(s, "hash-zlib")) + self.assertTrue(ok) + by_pkg = {r["package"]: r for r in rows} + self.assertEqual(by_pkg["zlib"]["action"], "rebuild") + self.assertTrue(by_pkg["zlib"]["regenerable"]) + self.assertEqual(by_pkg["GCC"]["action"], "reuse") # toolchain reused + self.assertEqual(by_pkg["defaults-release"]["action"], "reuse") + + def test_verify_flags_unregenerable_missing_blob(self): + # zlib missing AND its recipe blob is corrupt -> not regenerable -> not ok. + s = self._verify_sync(present={"cgcc", "cdef"}, break_recipe="zlib") + rows, ok = verify_closure(s, walk_build_closure(s, "hash-zlib")) + self.assertFalse(ok) + zlib = next(r for r in rows if r["package"] == "zlib") + self.assertFalse(zlib["recipe_ok"]) + self.assertFalse(zlib["regenerable"]) + + def test_ensure_defaults_recipe(self): + from alibuild_helpers.reconstruct import ensure_defaults_recipe + cfg = os.path.join(self.tmp, "cfg") + os.makedirs(cfg) + # No defaults name -> nothing to do. + self.assertTrue(ensure_defaults_recipe(cfg, None)) + # 'release' is already present as a materialised package recipe. + open(os.path.join(cfg, "defaults-release.sh"), "w").close() + self.assertTrue(ensure_defaults_recipe(cfg, "release")) + # 'o2' is missing and there's no alidist to supply it. + self.assertFalse(ensure_defaults_recipe(cfg, "o2")) + # ...but an alidist that has defaults-o2.sh gets it copied in. + ad = os.path.join(self.tmp, "alidist") + os.makedirs(ad) + with open(os.path.join(ad, "defaults-o2.sh"), "w") as df: + df.write("O2 DEFAULTS") + self.assertTrue(ensure_defaults_recipe(cfg, "o2", ad)) + with open(os.path.join(cfg, "defaults-o2.sh")) as df: + self.assertEqual(df.read(), "O2 DEFAULTS") + # An alidist missing the file still reports False. + self.assertFalse(ensure_defaults_recipe(cfg, "o3", ad)) + + def test_supply_recipes_from_alidist(self): + from alibuild_helpers.reconstruct import supply_recipes_from_alidist + cfg = os.path.join(self.tmp, "cfg") + alidist = os.path.join(self.tmp, "alidist") + os.makedirs(cfg) + os.makedirs(alidist) + # Archived recipe already materialised for a built package. + with open(os.path.join(cfg, "zlib.sh"), "w") as zf: + zf.write("ARCHIVED") + # alidist has the built package (should NOT overwrite) plus system/defaults. + for name, body in (("zlib.sh", "CURRENT"), ("yacc-like.sh", "SYS"), + ("defaults-o2.sh", "O2")): + with open(os.path.join(alidist, name), "w") as af: + af.write(body) + copied = supply_recipes_from_alidist(cfg, alidist) + self.assertEqual(copied, 2) # yacc-like + defaults-o2, not zlib + with open(os.path.join(cfg, "zlib.sh")) as zf: + self.assertEqual(zf.read(), "ARCHIVED") # archived recipe preserved + self.assertTrue(os.path.exists(os.path.join(cfg, "yacc-like.sh"))) + self.assertTrue(os.path.exists(os.path.join(cfg, "defaults-o2.sh"))) + + def test_rebuilt_tarball_glob(self): + store = os.path.join(self.tmp, "w", "TARS", ARCH, "store", "ab", "abcd") + os.makedirs(store) + tar = os.path.join(store, "zlib-v1-1.%s.tar.gz" % ARCH) + open(tar, "wb").close() + wd = os.path.join(self.tmp, "w") + self.assertEqual(_rebuilt_tarball(wd, ARCH, "zlib"), tar) + self.assertIsNone(_rebuilt_tarball(wd, ARCH, "other")) + + def test_rebuild_verdict(self): + self.assertEqual(_rebuild_verdict("a", "a", False), (True, "match")) + self.assertEqual(_rebuild_verdict("a", "a", True), (True, "match")) + # Differ is a soft pass by default, a failure under --strict. + self.assertEqual(_rebuild_verdict("a", "b", False), (True, "differ")) + self.assertEqual(_rebuild_verdict("a", "b", True), (False, "differ")) + + def _rebuild_args(self, strict=False): + return Namespace(package="zlib", version="v1", revision="1", architecture=ARCH, + remoteStore="reapi://x/cas", acStore="", insecure=False, + workDir=self.tmp, outputConfig=None, strict=strict) + + def _fake_builder(self, content): + """A build_runner that drops `content` where _rebuilt_tarball will find it.""" + def run(cmd): + d = os.path.join(self.tmp, "verify-rebuild-zlib", "TARS", ARCH, + "store", "ab", "abcdef") + os.makedirs(d, exist_ok=True) + with open(os.path.join(d, "zlib-v1-1.%s.tar.gz" % ARCH), "wb") as tf: + tf.write(content) + return 0 + return run + + def test_verify_rebuild_match(self): + content = b"reproduced-bytes" + recorded = hashlib.sha256(content).hexdigest() + closure = [make_entry("zlib", "v1", "1", recorded, "rz")] + with patch("alibuild_helpers.reconstruct.materialize_recipes"), \ + patch("alibuild_helpers.reconstruct.restore_sources", return_value=([], [])), \ + patch("alibuild_helpers.reconstruct.GitSourceStore"): + ok = verify_rebuild(self._rebuild_args(), MagicMock(), closure, + build_runner=self._fake_builder(content)) + self.assertTrue(ok) # byte-identical rebuild + + def test_verify_rebuild_differ_soft_and_strict(self): + recorded = hashlib.sha256(b"original").hexdigest() # differs from rebuilt + closure = [make_entry("zlib", "v1", "1", recorded, "rz")] + builder = self._fake_builder(b"rebuilt-differently") + with patch("alibuild_helpers.reconstruct.materialize_recipes"), \ + patch("alibuild_helpers.reconstruct.restore_sources", return_value=([], [])), \ + patch("alibuild_helpers.reconstruct.GitSourceStore"): + self.assertTrue(verify_rebuild(self._rebuild_args(strict=False), + MagicMock(), closure, build_runner=builder)) + self.assertFalse(verify_rebuild(self._rebuild_args(strict=True), + MagicMock(), closure, build_runner=builder)) + + def test_verify_rebuild_build_failure(self): + closure = [make_entry("zlib", "v1", "1", "c" * 64, "rz")] + with patch("alibuild_helpers.reconstruct.materialize_recipes"), \ + patch("alibuild_helpers.reconstruct.restore_sources", return_value=([], [])), \ + patch("alibuild_helpers.reconstruct.GitSourceStore"): + ok = verify_rebuild(self._rebuild_args(), MagicMock(), closure, + build_runner=lambda cmd: 1) # build fails + self.assertFalse(ok) + + def test_verify_rebuild_result_carries_tarball(self): + content = b"rebuilt-differently" + recorded = hashlib.sha256(b"original").hexdigest() + closure = [make_entry("zlib", "v1", "1", recorded, "rz")] + with patch("alibuild_helpers.reconstruct.materialize_recipes"), \ + patch("alibuild_helpers.reconstruct.restore_sources", return_value=([], [])), \ + patch("alibuild_helpers.reconstruct.GitSourceStore"): + result = verify_rebuild(self._rebuild_args(), MagicMock(), closure, + build_runner=self._fake_builder(content)) + self.assertTrue(result) # soft pass, still truthy + self.assertEqual(result.kind, "differ") + self.assertEqual(result.recorded, recorded) + self.assertEqual(result.rebuilt, hashlib.sha256(content).hexdigest()) + self.assertTrue(os.path.exists(result.tarball)) + + def _rebaseline_entry(self): + return make_entry("zlib", "v1", "1", "a" * 64, "rz") + + def test_do_rebaseline_match_is_noop(self): + result = RebuildResult(True, kind="match", algo="sha256", + recorded="a" * 64, rebuilt="a" * 64, tarball="/x") + sync_mock = MagicMock() + ok = do_rebaseline(Namespace(package="zlib", version="v1", apply=True, + delete_old=False), sync_mock, + [self._rebaseline_entry()], result) + self.assertTrue(ok) + sync_mock.rebaseline_ac_entry.assert_not_called() + + def test_do_rebaseline_dry_run_writes_nothing(self): + result = RebuildResult(True, kind="differ", algo="sha256", + recorded="a" * 64, rebuilt="b" * 64, tarball="/x") + sync_mock = MagicMock() + ok = do_rebaseline(Namespace(package="zlib", version="v1", apply=False, + delete_old=False), sync_mock, + [self._rebaseline_entry()], result) + self.assertTrue(ok) # dry run is not a failure + sync_mock.rebaseline_ac_entry.assert_not_called() + sync_mock.delete_artifact_blob.assert_not_called() + + def test_do_rebaseline_apply_rewrites_and_keeps_old(self): + result = RebuildResult(True, kind="differ", algo="sha256", + recorded="a" * 64, rebuilt="b" * 64, tarball="/x") + sync_mock = MagicMock() + sync_mock.read_blob.return_value = b"recipe" + sync_mock.rebaseline_ac_entry.return_value = ("a" * 64, "b" * 64, "cas/old") + ok = do_rebaseline(Namespace(package="zlib", version="v1", apply=True, + delete_old=False), sync_mock, + [self._rebaseline_entry()], result) + self.assertTrue(ok) + sync_mock.rebaseline_ac_entry.assert_called_once() + sync_mock.delete_artifact_blob.assert_not_called() # old blob kept + + def test_do_rebaseline_apply_delete_old(self): + result = RebuildResult(True, kind="differ", algo="sha256", + recorded="a" * 64, rebuilt="b" * 64, tarball="/x") + sync_mock = MagicMock() + sync_mock.read_blob.return_value = b"recipe" + sync_mock.rebaseline_ac_entry.return_value = ("a" * 64, "b" * 64, "cas/old") + ok = do_rebaseline(Namespace(package="zlib", version="v1", apply=True, + delete_old=True), sync_mock, + [self._rebaseline_entry()], result) + self.assertTrue(ok) + sync_mock.delete_artifact_blob.assert_called_once_with("a" * 64, "sha256") + + def test_do_rebaseline_failed_rebuild(self): + ok = do_rebaseline(Namespace(package="zlib", version="v1", apply=True, + delete_old=False), MagicMock(), + [self._rebaseline_entry()], RebuildResult(False)) + self.assertFalse(ok) # no tarball -> cannot re-baseline + + def _persist_args(self, apply): + return Namespace(package="zlib", version="v1", apply=apply, storage="permanent") + + def test_do_persist_apply_restores_blob(self): + result = RebuildResult(True, kind="match", algo="sha256", + recorded="a" * 64, rebuilt="a" * 64, tarball="/x") + sync_mock = MagicMock() + sync_mock.artifact_blob_exists.return_value = False # blob is missing + sync_mock.put_artifact_blob.return_value = "a" * 64 # restores identical hash + ok = do_persist(self._persist_args(apply=True), sync_mock, [], result) + self.assertTrue(ok) + sync_mock.put_artifact_blob.assert_called_once_with("/x", "sha256") + + def test_do_persist_dry_run_writes_nothing(self): + result = RebuildResult(True, kind="match", algo="sha256", + recorded="a" * 64, rebuilt="a" * 64, tarball="/x") + sync_mock = MagicMock() + sync_mock.artifact_blob_exists.return_value = False + ok = do_persist(self._persist_args(apply=False), sync_mock, [], result) + self.assertTrue(ok) # dry run is not a failure + sync_mock.put_artifact_blob.assert_not_called() + + def test_do_persist_already_present_is_noop(self): + result = RebuildResult(True, kind="match", algo="sha256", + recorded="a" * 64, rebuilt="a" * 64, tarball="/x") + sync_mock = MagicMock() + sync_mock.artifact_blob_exists.return_value = True # already there + ok = do_persist(self._persist_args(apply=True), sync_mock, [], result) + self.assertTrue(ok) + sync_mock.put_artifact_blob.assert_not_called() + + def test_do_persist_refuses_differ(self): + # A rebuild that does not reproduce the recorded hash must not be persisted + # (that would upload a blob under a key that no AC entry references). + result = RebuildResult(True, kind="differ", algo="sha256", + recorded="a" * 64, rebuilt="b" * 64, tarball="/x") + sync_mock = MagicMock() + ok = do_persist(self._persist_args(apply=True), sync_mock, [], result) + self.assertFalse(ok) + sync_mock.put_artifact_blob.assert_not_called() + + def test_do_persist_failed_rebuild(self): + ok = do_persist(self._persist_args(apply=True), MagicMock(), [], + RebuildResult(False)) + self.assertFalse(ok) + + def test_doReconstruct_verify_mode(self): + s = self._verify_sync(present={"czlib", "cgcc", "cdef"}) + args = Namespace(package="zlib", version="v1", revision=None, architecture=ARCH, + remoteStore="reapi://localhost/bucket", insecure=False, + workDir="/sw", outputConfig=None, verify=True) + with patch("alibuild_helpers.reconstruct.remote_from_url", return_value=s): + # Verify mode returns True and materialises nothing. + self.assertTrue(doReconstruct(args, None)) + + def test_doReconstruct_requires_reapi_store(self): + args = Namespace(package="zlib", version="v1", revision=None, architecture=ARCH, + remoteStore="https://s3.cern.ch/foo", insecure=False, + workDir="/sw", outputConfig=None) + self.assertRaises(SystemExit, doReconstruct, args, None) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_signing.py b/tests/test_signing.py new file mode 100644 index 00000000..3b002fe2 --- /dev/null +++ b/tests/test_signing.py @@ -0,0 +1,311 @@ +import base64 +import json +import threading +import unittest +from datetime import datetime, timezone +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + +from alibuild_helpers import signing + +# `cryptography` is an optional extra (`pip install alibuild[signing]`), so these +# tests must skip rather than fail where it is absent -- e.g. a CI job that does +# not install the extra. tox installs it via `extras = signing`, so they normally +# do run. +try: + import cryptography # noqa: F401 + HAVE_CRYPTOGRAPHY = True +except ImportError: + HAVE_CRYPTOGRAPHY = False + +requires_crypto = unittest.skipUnless( + HAVE_CRYPTOGRAPHY, "needs the optional 'cryptography' extra") + + +# Deterministic Ed25519 seeds (any 32 bytes work). +SEED_A = bytes(range(32)) +SEED_B = bytes(range(32, 64)) + + +def ac_entry(**override): + entry = { + "action": {"actionHash": "a" * 40, "package": "zlib", + "architecture": "slc7_x86-64", "recipeDigest": "sha256:r"}, + "result": {"outputDigest": "sha256:o", "size": 10, "tarball": "zlib.tgz"}, + } + entry.update(override) + return entry + + +def keyring(*seed_signers, revoked=(), not_before=None, not_after=None): + keys = {} + for seed, signer in seed_signers: + keyid, pub = signing.public_key(seed) + item = {"publicKey": pub, "signer": signer} + if not_before: + item["notBefore"] = not_before + if not_after: + item["notAfter"] = not_after + keys[keyid] = item + return signing.load_keyring({"keys": keys, "revoked": list(revoked)}) + + +@requires_crypto +class SigningTestCase(unittest.TestCase): + def test_dsse_pae_format(self) -> None: + self.assertEqual(signing.dsse_pae("t", b"hi"), b"DSSEv1 1 t 2 hi") + + def test_signed_payload_deterministic_and_binds_output(self) -> None: + self.assertEqual(signing.signed_payload(ac_entry()), + signing.signed_payload(ac_entry())) + # Swapping the artifact the entry points at changes the signed payload. + self.assertNotEqual( + signing.signed_payload(ac_entry()), + signing.signed_payload(ac_entry(result={"outputDigest": "sha256:X"}))) + + def test_validate_system_entry_binds_recipe_digest(self) -> None: + # No result tarball -> bind recipeDigest instead of outputDigest. + entry = {"action": {"actionHash": "h", "package": "make", + "architecture": "slc7_x86-64", "recipeDigest": "sha256:r"}} + self.assertIn(b"sha256:r", signing.signed_payload(entry)) + + def test_sign_then_verify(self) -> None: + entry = ac_entry() + entry["signatures"] = [signing.sign(entry, SEED_A, "alice-ci")] + ok, reason = signing.verify(entry, keyring((SEED_A, "alice-ci"))) + self.assertTrue(ok, reason) + + def test_tampered_output_digest_fails(self) -> None: + entry = ac_entry() + entry["signatures"] = [signing.sign(entry, SEED_A, "ci")] + entry["result"]["outputDigest"] = "sha256:EVIL" # point at another blob + ok, reason = signing.verify(entry, keyring((SEED_A, "ci"))) + self.assertFalse(ok) + self.assertIn("bad signature", reason) + + def test_untrusted_key_fails(self) -> None: + entry = ac_entry() + entry["signatures"] = [signing.sign(entry, SEED_A, "ci")] + ok, reason = signing.verify(entry, keyring((SEED_B, "other"))) + self.assertFalse(ok) + self.assertIn("untrusted", reason) + + def test_unsigned_fails(self) -> None: + ok, reason = signing.verify(ac_entry(), keyring((SEED_A, "ci"))) + self.assertFalse(ok) + self.assertIn("no signatures", reason) + + def test_revoked_key_fails(self) -> None: + entry = ac_entry() + sig = signing.sign(entry, SEED_A, "ci") + entry["signatures"] = [sig] + ok, reason = signing.verify(entry, keyring((SEED_A, "ci"), revoked=[sig["keyid"]])) + self.assertFalse(ok) + self.assertIn("revoked", reason) + + def test_expired_key_fails(self) -> None: + entry = ac_entry() + entry["signatures"] = [signing.sign(entry, SEED_A, "ci")] + past = datetime(2000, 1, 1, tzinfo=timezone.utc).isoformat() + ok, reason = signing.verify(entry, keyring((SEED_A, "ci"), not_after=past)) + self.assertFalse(ok) + self.assertIn("expired", reason) + + def test_not_yet_valid_key_fails(self) -> None: + entry = ac_entry() + entry["signatures"] = [signing.sign(entry, SEED_A, "ci")] + future = datetime(2999, 1, 1, tzinfo=timezone.utc).isoformat() + ok, reason = signing.verify(entry, keyring((SEED_A, "ci"), not_before=future)) + self.assertFalse(ok) + self.assertIn("not yet valid", reason) + + def test_threshold_requires_distinct_keys(self) -> None: + entry = ac_entry() + entry["signatures"] = [signing.sign(entry, SEED_A, "a")] + ring = keyring((SEED_A, "a"), (SEED_B, "b")) + self.assertFalse(signing.verify(entry, ring, min_signatures=2)[0]) + entry["signatures"].append(signing.sign(entry, SEED_B, "b")) + self.assertTrue(signing.verify(entry, ring, min_signatures=2)[0]) + + def test_keyid_must_match_public_key(self) -> None: + _, pub = signing.public_key(SEED_A) + with self.assertRaises(ValueError): + signing.load_keyring({"keys": {"deadbeef": {"publicKey": pub}}}) + + +@requires_crypto +class PolicyTestCase(unittest.TestCase): + def _signed(self): + entry = ac_entry() + entry["signatures"] = [signing.sign(entry, SEED_A, "ci")] + return entry + + def test_off_ignores_signatures(self) -> None: + # Unsigned entry, empty keyring: "off" allows and flags nothing. + allowed, level, _reason = signing.evaluate( + ac_entry(), keyring((SEED_A, "ci")), signing.POLICY_OFF) + self.assertTrue(allowed) + self.assertIsNone(level) + + def test_warn_allows_but_flags_unsigned(self) -> None: + allowed, level, reason = signing.evaluate( + ac_entry(), keyring((SEED_A, "ci")), signing.POLICY_WARN) + self.assertTrue(allowed) + self.assertEqual(level, "warn") + self.assertIn("no signatures", reason) + + def test_warn_clean_on_valid_signature(self) -> None: + allowed, level, _reason = signing.evaluate( + self._signed(), keyring((SEED_A, "ci")), signing.POLICY_WARN) + self.assertTrue(allowed) + self.assertIsNone(level) + + def test_require_fails_closed_on_unsigned(self) -> None: + allowed, level, _reason = signing.evaluate( + ac_entry(), keyring((SEED_A, "ci")), signing.POLICY_REQUIRE) + self.assertFalse(allowed) + self.assertEqual(level, "error") + + def test_require_passes_on_valid_signature(self) -> None: + allowed, level, _reason = signing.evaluate( + self._signed(), keyring((SEED_A, "ci")), signing.POLICY_REQUIRE) + self.assertTrue(allowed) + self.assertIsNone(level) + + def test_closure_require_fails_on_one_unsigned_dep(self) -> None: + # A signed package with one unsigned dependency must fail under require. + ring = keyring((SEED_A, "ci")) + entries = [("zlib", self._signed()), ("boost", ac_entry())] + allowed, problems = signing.evaluate_closure(entries, ring, signing.POLICY_REQUIRE) + self.assertFalse(allowed) + self.assertEqual([p[0] for p in problems], ["boost"]) + self.assertEqual(problems[0][1], "error") + + def test_closure_warn_collects_all_problems_without_blocking(self) -> None: + ring = keyring((SEED_A, "ci")) + entries = [("zlib", self._signed()), ("boost", ac_entry()), + ("root", ac_entry())] + allowed, problems = signing.evaluate_closure(entries, ring, signing.POLICY_WARN) + self.assertTrue(allowed) + self.assertEqual({p[0] for p in problems}, {"boost", "root"}) + self.assertTrue(all(p[1] == "warn" for p in problems)) + + def test_closure_all_signed_passes_clean(self) -> None: + ring = keyring((SEED_A, "ci")) + entries = [("zlib", self._signed()), ("boost", self._signed())] + allowed, problems = signing.evaluate_closure(entries, ring, signing.POLICY_REQUIRE) + self.assertTrue(allowed) + self.assertEqual(problems, []) + + +@requires_crypto +class MergeKeyringTestCase(unittest.TestCase): + """Merging must only ever *narrow* trust: alidist (or any later keyring) may + add keys, but must not be able to un-revoke one or widen a validity window + the shipped keyring narrowed. That is what makes revoking by shipping a new + alibuild release actually stick.""" + + def test_union_of_keys(self) -> None: + merged = signing.merge_keyrings([keyring((SEED_A, "a")), keyring((SEED_B, "b"))]) + self.assertEqual(set(merged.keys), + {signing.public_key(SEED_A)[0], signing.public_key(SEED_B)[0]}) + + def test_revocation_cannot_be_undone_by_a_later_keyring(self) -> None: + entry = ac_entry() + sig = signing.sign(entry, SEED_A, "ci") + entry["signatures"] = [sig] + revoking = keyring((SEED_A, "ci"), revoked=[sig["keyid"]]) + permissive = keyring((SEED_A, "ci")) + for order in ((revoking, permissive), (permissive, revoking)): + ok, reason = signing.verify(entry, signing.merge_keyrings(order)) + self.assertFalse(ok) + self.assertIn("revoked", reason) + + def test_validity_windows_intersect(self) -> None: + entry = ac_entry() + entry["signatures"] = [signing.sign(entry, SEED_A, "ci")] + past = datetime(2000, 1, 1, tzinfo=timezone.utc).isoformat() + expired = keyring((SEED_A, "ci"), not_after=past) + open_ended = keyring((SEED_A, "ci")) + for order in ((expired, open_ended), (open_ended, expired)): + ok, reason = signing.verify(entry, signing.merge_keyrings(order)) + self.assertFalse(ok, "a later keyring must not extend an expired key") + self.assertIn("expired", reason) + + def test_added_key_is_trusted(self) -> None: + entry = ac_entry() + entry["signatures"] = [signing.sign(entry, SEED_B, "later")] + merged = signing.merge_keyrings([keyring((SEED_A, "shipped")), + keyring((SEED_B, "later"))]) + self.assertTrue(signing.verify(entry, merged)[0]) + + +class _DumbSignerProxy(BaseHTTPRequestHandler): + """A stand-in for the security-proxy sign route: Ed25519-signs the exact bytes + posted, gated by a bearer token, and returns {"keyid", "sig"}. Holds SEED_A -- + the private key never leaves the "proxy" (the real one lives out of process).""" + + seed = SEED_A + expected_token = "gate-token" + + def log_message(self, *_args): + pass # keep the test output quiet + + def _respond(self, code, body=b""): + # Always send an explicit Content-Length so the client reads a bounded + # body rather than until connection close (which races under teardown). + self.send_response(code) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + if body: + self.wfile.write(body) + + def do_POST(self): + body_in = self.rfile.read(int(self.headers.get("Content-Length", 0))) + if self.headers.get("Authorization") != "Bearer %s" % self.expected_token: + self._respond(401) + return + from cryptography.hazmat.primitives.asymmetric import ed25519 + sk = ed25519.Ed25519PrivateKey.from_private_bytes(self.seed) + keyid, _pub = signing.public_key(self.seed) + reply = json.dumps({"keyid": keyid, + "sig": base64.b64encode(sk.sign(body_in)).decode("ascii")}) + self._respond(200, reply.encode("utf-8")) + + +@requires_crypto +class SignViaProxyTestCase(unittest.TestCase): + def setUp(self) -> None: + self.server = ThreadingHTTPServer(("127.0.0.1", 0), _DumbSignerProxy) + self.thread = threading.Thread(target=self.server.serve_forever, daemon=True) + self.thread.start() + self.endpoint = "http://127.0.0.1:%d/sign/alibuild-ac" % self.server.server_port + + def tearDown(self) -> None: + self.server.shutdown() + self.server.server_close() + self.thread.join() + + def test_proxy_signature_verifies(self) -> None: + entry = ac_entry() + entry["signatures"] = [ + signing.sign_via_proxy(entry, self.endpoint, "gate-token", "alice-ci")] + ok, reason = signing.verify(entry, keyring((SEED_A, "alice-ci"))) + self.assertTrue(ok, reason) + + def test_proxy_matches_local_sign_byte_for_byte(self) -> None: + # The proxy path and the local sign() path must produce the identical + # signature bytes -- Ed25519 is deterministic and both sign the same PAE. + entry = ac_entry() + via_proxy = signing.sign_via_proxy(entry, self.endpoint, "gate-token", "ci") + local = signing.sign(entry, SEED_A, "ci") + self.assertEqual(via_proxy, local) + + def test_bad_gate_token_rejected(self) -> None: + import requests + with self.assertRaises(requests.HTTPError): + signing.sign_via_proxy(ac_entry(), self.endpoint, "wrong-token", "ci") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_source.py b/tests/test_source.py new file mode 100644 index 00000000..ebbfcff8 --- /dev/null +++ b/tests/test_source.py @@ -0,0 +1,289 @@ +import hashlib +import os +import os.path +import shutil +import subprocess +import tempfile +import unittest + +from alibuild_helpers.source import ( + GitSourceStore, _repo_id, _is_ancestor, + store_refs, load_refs, apply_refs) + + +class FakeCASSync: + """In-memory stand-in for REAPIRemoteSync's CAS + pointer operations.""" + + def __init__(self): + self.blobs = {} # content hash -> bytes + self.objects = {} # key -> JSON pointer dict + self.uploads = 0 # number of blobs actually uploaded + + def put_file_as_blob(self, path, algo="sha256"): + with open(path, "rb") as blobf: + data = blobf.read() + content_hash = hashlib.sha256(data).hexdigest() + if content_hash not in self.blobs: + self.blobs[content_hash] = data + self.uploads += 1 + return content_hash + + def download_blob(self, content_hash, dest, algo="sha256"): + with open(dest, "wb") as destf: + destf.write(self.blobs[content_hash]) + + def put_bytes_as_blob(self, data, algo="sha256"): + content_hash = hashlib.sha256(data).hexdigest() + if content_hash not in self.blobs: + self.blobs[content_hash] = data + self.uploads += 1 + return content_hash + + def read_blob(self, content_hash, algo="sha256"): + return self.blobs[content_hash] + + def read_object_json(self, key): + return self.objects.get(key) + + def write_object_json(self, key, obj): + self.objects[key] = obj + + +def _git(args, cwd): + env = dict(os.environ, GIT_AUTHOR_NAME="t", GIT_AUTHOR_EMAIL="a@b.c", + GIT_COMMITTER_NAME="t", GIT_COMMITTER_EMAIL="a@b.c") + return subprocess.run(["git"] + args, cwd=cwd, env=env, check=True, + stdout=subprocess.PIPE, stderr=subprocess.STDOUT).stdout.decode().strip() + + +@unittest.skipUnless(shutil.which("git"), "git is required for source tests") +class GitSourceStoreTestCase(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.tmp, ignore_errors=True) + self.repo = os.path.join(self.tmp, "src") + os.makedirs(self.repo) + _git(["init", "-q"], self.repo) + self._commit("a\n", "c1", tag="v1") + self.c1 = _git(["rev-parse", "HEAD"], self.repo) + self._commit("a\nb\n", "c2") + self.c2 = _git(["rev-parse", "HEAD"], self.repo) + self._commit("a\nb\nc\n", "c3") + self.c3 = _git(["rev-parse", "HEAD"], self.repo) + self.url = "https://example.com/repo" + + def _commit(self, content, msg, tag=None): + with open(os.path.join(self.repo, "f.txt"), "w") as srcf: + srcf.write(content) + _git(["add", "."], self.repo) + _git(["commit", "-qm", msg], self.repo) + if tag: + _git(["tag", tag], self.repo) + + def test_is_ancestor(self): + self.assertTrue(_is_ancestor(self.repo, self.c1, self.c3)) + self.assertFalse(_is_ancestor(self.repo, self.c3, self.c1)) + self.assertFalse(_is_ancestor(self.repo, None, self.c3)) + + def test_backfill_objects_fetches_when_missing(self): + from unittest.mock import patch + store = GitSourceStore(FakeCASSync()) + calls = [] + + def fake_git(args, directory=None, check=True, **kw): + calls.append(tuple(args)) + if args[0] == "rev-list": + return ("", "abc f.txt\n?deadbeefdeadbeefdeadbeefdeadbeefdeadbeef missing\n") + return ("", "") + + with patch("alibuild_helpers.source.git", side_effect=fake_git): + store._backfill_objects("/repo", "commitX", None) + # A missing object -> one bulk --refetch backfill (not per-object lazy fetch). + self.assertTrue(any("--refetch" in c and "fetch" in c for c in calls)) + + def test_backfill_objects_noop_on_full_mirror(self): + from unittest.mock import patch + store = GitSourceStore(FakeCASSync()) + calls = [] + + def fake_git(args, directory=None, check=True, **kw): + calls.append(tuple(args)) + return ("", "abc f.txt\ndef g.txt\n") # no '?' lines -> nothing missing + + with patch("alibuild_helpers.source.git", side_effect=fake_git): + store._backfill_objects("/repo", "commitX", "baseY") + self.assertFalse(any("--refetch" in c for c in calls)) # no fetch issued + + def test_snapshot_and_restore_roundtrip(self): + sync = FakeCASSync() + store = GitSourceStore(sync) + entry = store.snapshot(self.repo, self.url, self.c2) + # First snapshot of the repo: one full base segment, no ancestor base. + self.assertEqual(entry["baseCommit"], self.c2) + self.assertEqual(len(entry["segments"]), 1) + + # Wipe the upstream entirely: restore must work from the CAS alone. + shutil.rmtree(self.repo) + dest = os.path.join(self.tmp, "restored") + store.restore(entry, dest) + self.assertEqual(_git(["rev-parse", "HEAD"], dest), self.c2) + with open(os.path.join(dest, "f.txt")) as out: + self.assertEqual(out.read(), "a\nb\n") + + def test_incremental_chain_dedups_shared_history(self): + """Close commits share one base and store only their per-commit delta, + instead of duplicating the whole source each time.""" + # A repo with a large incompressible file (lives in the base) plus a tiny + # file that changes each "daily" commit. + repo = os.path.join(self.tmp, "bigrepo") + os.makedirs(repo) + _git(["init", "-q"], repo) + with open(os.path.join(repo, "core.bin"), "wb") as bf: + bf.write(os.urandom(200000)) # incompressible -> stays big in the base + for rev in ("r0", "r1", "r2"): + with open(os.path.join(repo, "f.txt"), "w") as ff: + ff.write(rev) + _git(["add", "."], repo) + _git(["commit", "-qm", rev], repo) + k0 = _git(["rev-parse", "HEAD~2"], repo) + k1 = _git(["rev-parse", "HEAD~1"], repo) + k2 = _git(["rev-parse", "HEAD"], repo) + url = "https://example.com/bigrepo" + + sync = FakeCASSync() + store = GitSourceStore(sync) + e0 = store.snapshot(repo, url, k0) # base (holds core.bin) + e1 = store.snapshot(repo, url, k1) # thin against k0 + e2 = store.snapshot(repo, url, k2) # thin against k1 + # The chain grows by exactly one segment per commit, all sharing the base. + self.assertEqual(e1["segments"], e0["segments"] + [e1["segments"][-1]]) + self.assertEqual(e2["segments"], e1["segments"] + [e2["segments"][-1]]) + self.assertEqual(e2["baseCommit"], k0) + # base + 2 deltas = 3 distinct blobs, nothing re-uploaded. + self.assertEqual(len(sync.blobs), 3) + self.assertEqual(sync.uploads, 3) + # The big file is stored once (in the base); the per-daily deltas are tiny. + base_blob = sync.blobs[e0["segments"][0]] + self.assertGreater(len(base_blob), 150000) + for entry in (e1, e2): + self.assertLess(len(sync.blobs[entry["segments"][-1]]), 2000) + + # Restoring the tip fetches the whole chain and checks out k2 offline. + shutil.rmtree(repo) + dest = os.path.join(self.tmp, "restored3") + store.restore(e2, dest) + self.assertEqual(_git(["rev-parse", "HEAD"], dest), k2) + with open(os.path.join(dest, "f.txt")) as out: + self.assertEqual(out.read(), "r2") + + def test_snapshot_is_idempotent(self): + sync = FakeCASSync() + store = GitSourceStore(sync) + store.snapshot(self.repo, self.url, self.c1) + store.snapshot(self.repo, self.url, self.c2) + uploads_before = sync.uploads + again = store.snapshot(self.repo, self.url, self.c2) # same commit again + self.assertEqual(sync.uploads, uploads_before) # nothing re-uploaded + self.assertEqual(again["commit"], self.c2) + self.assertEqual(len(again["segments"]), 2) + + def test_restore_falls_back_to_upstream(self): + """A broken CAS chain must not be fatal: restore clones upstream instead + (the snapshot is a backup/speedup, upstream is normally available).""" + sync = FakeCASSync() + store = GitSourceStore(sync) + entry = store.snapshot(self.repo, self.url, self.c2) + # Corrupt the archive: drop its blobs so the CAS path must fail. + sync.blobs.clear() + entry = dict(entry, source=self.repo) # a reachable "upstream" to clone + dest = os.path.join(self.tmp, "fallback") + store.restore(entry, dest) + self.assertEqual(_git(["rev-parse", "HEAD"], dest), self.c2) + + def test_store_and_load_refs(self): + sync = FakeCASSync() + refs = {"refs/tags/v1": "a" * 40, "refs/heads/master": "b" * 40} + artifact = store_refs(sync, self.url, refs) + self.assertEqual(artifact["type"], "git-refs") + self.assertEqual(load_refs(sync, artifact), refs) + # No refs -> no artifact. + self.assertIsNone(store_refs(sync, self.url, {})) + + def test_apply_refs_recreates_tags(self): + # apply_refs recreates the tag refs (only refs/tags/*) in a repo. + apply_refs(self.repo, {"refs/tags/recovered": self.c3, + "refs/heads/ignored": self.c1}) + self.assertEqual(_git(["rev-parse", "refs/tags/recovered"], self.repo), self.c3) + # Branch refs are not recreated. + out = subprocess.run(["git", "rev-parse", "--verify", "-q", + "refs/heads/ignored"], cwd=self.repo, + stdout=subprocess.PIPE).returncode + self.assertNotEqual(out, 0) + + def test_restore_to_source_dir_path(self): + sync = FakeCASSync() + store = GitSourceStore(sync) + art = store.snapshot(self.repo, self.url, self.c2) + refs = store_refs(sync, self.url, {"refs/tags/v1": self.c1}) + entry = {"action": {"package": "zlib", "version": "v1.3.1", + "commit": {"ref": self.c2}, "tag": "abranch", + "sourceArtifact": art, "refsArtifact": refs}} + work = os.path.join(self.tmp, "wd") + src = store.restore_to_source_dir(entry, work) + # tag != commit ref -> short = ref[:10], matching short_commit_hash(). + self.assertEqual(src, os.path.join(work, "SOURCES", "zlib", "v1.3.1", + self.c2[:10])) + self.assertEqual(_git(["rev-parse", "HEAD"], src), self.c2) + + def test_offline_checkout_from_restored_source(self): + """The headline 'lost upstream' guarantee: after restoring into SOURCES, + alibuild's own checkout_sources must check out with an UNREACHABLE + upstream URL -- proving no upstream contact.""" + from alibuild_helpers.workarea import checkout_sources + from alibuild_helpers.git import Git + sync = FakeCASSync() + store = GitSourceStore(sync) + art = store.snapshot(self.repo, self.url, self.c1) # c1 is tagged v1 + refs = store_refs(sync, self.url, {"refs/tags/v1": self.c1}) + entry = {"action": {"package": "zlib", "version": "v1.3.1", + "commit": {"ref": self.c1}, "tag": "v1", + "sourceArtifact": art, "refsArtifact": refs}} + work = os.path.join(self.tmp, "wd2") + store.restore_to_source_dir(entry, work) + + spec = {"scm": Git(), "source": "https://invalid.invalid/zlib.git", + "commit_hash": self.c1, "tag": "v1", "package": "zlib", + "version": "v1.3.1", "is_devel_pkg": False} + reference_sources = os.path.join(self.tmp, "refsrc") + os.makedirs(reference_sources) + # Would fail/hang if it tried to reach the bogus URL. + checkout_sources(spec, work, reference_sources, False) + sdir = os.path.join(work, "SOURCES", "zlib", "v1.3.1", self.c1[:10]) + self.assertEqual(_git(["rev-parse", "HEAD"], sdir), self.c1) + + def test_first_snapshot_is_a_self_contained_base(self): + # The first snapshot of a repo is one full base segment (baseCommit is + # the commit itself), restorable standalone. + repo2 = os.path.join(self.tmp, "src2") + os.makedirs(repo2) + _git(["init", "-q"], repo2) + with open(os.path.join(repo2, "g.txt"), "w") as gf: + gf.write("x\n") + _git(["add", "."], repo2) + _git(["commit", "-qm", "only"], repo2) + commit = _git(["rev-parse", "HEAD"], repo2) + + sync = FakeCASSync() + store = GitSourceStore(sync) + entry = store.snapshot(repo2, "https://example.com/repo2", commit) + self.assertEqual(entry["baseCommit"], commit) + self.assertEqual(len(entry["segments"]), 1) + + shutil.rmtree(repo2) + dest = os.path.join(self.tmp, "restored2") + store.restore(entry, dest) + self.assertEqual(_git(["rev-parse", "HEAD"], dest), commit) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_sync.py b/tests/test_sync.py index c91677ec..8b24cf6f 100644 --- a/tests/test_sync.py +++ b/tests/test_sync.py @@ -391,30 +391,51 @@ def test_tarball_download(self) -> None: @patch("os.listdir", new=lambda path: ( [] if path.endswith("-" + MISSING_SPEC["revision"]) else NotImplemented)) + @patch("glob.glob", new=MagicMock(return_value=[])) + @patch("os.listdir", new=MagicMock(return_value=[])) + @patch("os.makedirs", new=MagicMock()) + @patch("os.path.exists", new=MagicMock(return_value=False)) + @patch("os.path.isfile", new=MagicMock(return_value=False)) @patch("os.path.islink", new=MagicMock(return_value=False)) - def test_missing_local_link_is_recreated(self) -> None: - """A tarball in the local store whose link was never made is publishable.""" + def test_tarball_download_follows_reapi_redirect(self) -> None: + """A reapi:// store leaves the legacy store object as a website-redirect + stub pointing at the CAS blob. A b3:// consumer must follow the redirect + and download the real bytes from the CAS key -- not the stub -- while + still saving them under the legacy store path/name the build expects.""" + from botocore.exceptions import ClientError + store_path = resolve_store_path(ARCHITECTURE, GOOD_HASH) + tarball_key = store_path + "/" + tarball_name(GOOD_SPEC) + cas_key = "cas/sha256/aa/" + "a" * 64 + + def paginate(Bucket, Delimiter, Prefix): + if Prefix.rstrip(Delimiter) == store_path: + return [{"Contents": [{"Key": tarball_key}]}] + return [{}] + + def head_object(Bucket, Key): + if Key == tarball_key: # legacy store object -> redirect stub + return {"WebsiteRedirectLocation": "/" + cas_key} + if Key == cas_key: # the real content-addressed bytes + return {"ContentLength": 4096} + raise ClientError({"Error": {"Code": "404"}}, "head_object") + + downloaded = [] b3sync = sync.Boto3RemoteSync( remoteStore="b3://localhost", writeStore="b3://localhost", architecture=ARCHITECTURE, workdir="/sw") - b3sync.s3 = self.mock_s3() - b3sync.upload_symlinks_and_tarball(MISSING_SPEC) - tar_path = os.path.join(resolve_store_path(ARCHITECTURE, NONEXISTENT_HASH), - tarball_name(MISSING_SPEC)) - # The body is the store path relative to TARS/. build.py parses the - # local link, which fetch_symlinks builds as "../../" + body, to work - # out which revisions are taken -- a body carrying the "TARS/" prefix - # produces a link it cannot parse, so the revision looks free and the - # next build collides with what is already published. - body = tar_path[len("TARS/"):] - b3sync.s3.put_object.assert_any_call( - Bucket="localhost", - Key=os.path.join(resolve_links_path(ARCHITECTURE, PACKAGE), - tarball_name(MISSING_SPEC)), - Body=body.encode("utf-8")) - self.assertNotIn("TARS/", body) - self.assertTrue(("../../" + body).startswith("../../%s/store/" % ARCHITECTURE), - "local link would not parse: %r" % ("../../" + body)) + b3sync.s3 = MagicMock( + get_paginator=lambda method: MagicMock(paginate=paginate), + head_object=head_object, + download_file=MagicMock(side_effect=lambda **kw: downloaded.append(kw))) + + b3sync.fetch_tarball(GOOD_SPEC) + + self.assertEqual(len(downloaded), 1) + # Bytes are fetched from the CAS blob, not the redirect stub... + self.assertEqual(downloaded[0]["Key"], cas_key) + # ...but saved under the legacy store path + tarball name. + self.assertTrue(downloaded[0]["Filename"].endswith( + store_path + "/" + tarball_name(GOOD_SPEC))) @patch("os.listdir", new=lambda path: ( [tarball_name(GOOD_SPEC)] if path.endswith("-" + GOOD_SPEC["revision"]) else @@ -528,6 +549,32 @@ def test_tarball_upload_unreadable_link(self) -> None: b3sync.s3.put_object.assert_not_called() b3sync.s3.upload_file.assert_not_called() + @patch("os.listdir", new=lambda path: ( + [] if path.endswith("-" + MISSING_SPEC["revision"]) else NotImplemented)) + @patch("os.path.islink", new=MagicMock(return_value=False)) + def test_missing_local_link_is_recreated(self) -> None: + """A tarball in the local store whose link was never made is publishable.""" + b3sync = self.fresh_upload_sync() + b3sync.upload_symlinks_and_tarball(MISSING_SPEC) + tar_path = os.path.join(resolve_store_path(ARCHITECTURE, NONEXISTENT_HASH), + tarball_name(MISSING_SPEC)) + # The body is the store path relative to TARS/. build.py parses the + # local link, which fetch_symlinks builds as "../../" + body, to work + # out which revisions are taken -- a body carrying the "TARS/" prefix + # produces a link it cannot parse, so the revision looks free and the + # next build collides with what is already published. + body = tar_path[len("TARS/"):] + b3sync.s3.put_object.assert_any_call( + IfNoneMatch="*", Bucket="localhost", + Key=os.path.join(resolve_links_path(ARCHITECTURE, PACKAGE), + tarball_name(MISSING_SPEC)), + Body=body.encode("utf-8")) + # ... and the link we wrote locally must be in the shape that parser + # expects: "../..//store/...", with no "TARS/" in the body. + self.assertNotIn("TARS/", body) + self.assertTrue(("../../" + body).startswith("../../%s/store/" % ARCHITECTURE), + "local link would not parse: %r" % ("../../" + body)) + def fresh_upload_sync(self): """A sync object publishing MISSING_SPEC, which is absent from the remote.""" b3sync = sync.Boto3RemoteSync( @@ -637,5 +684,53 @@ def put_object(**kwargs): return b3sync + + +@patch("alibuild_helpers.sync.Boto3RemoteSync._s3_init", new=MagicMock()) +class LegacyLinkBucketTestCase(unittest.TestCase): + """The legacy TARS/ tree must be written as a UNIT, into one bucket. + + REAPIRemoteSync stores the bytes content-addressed and can put the legacy + tree elsewhere (--legacy-links-store). It overrides _upload_tarball, which + writes the store object -- but the SYMLINKS are written by the base class, + which used to send them to writeStore. The two halves then split across + buckets, and a client reading the legacy bucket found a store object it + could never reach by name, because the symlink naming it was in the other + bucket. Every helper below must therefore follow legacyWriteStore. + """ + + def make_sync(self): + sync_obj = sync.Boto3RemoteSync( + remoteStore="b3://read", writeStore="b3://artifacts", + architecture=ARCHITECTURE, workdir="/sw") + sync_obj.legacyWriteStore = "legacy" # as REAPIRemoteSync sets it + sync_obj.s3 = MagicMock() + return sync_obj + + def test_default_is_the_artifact_bucket(self): + """Unset, it must not change b3:// behaviour.""" + sync_obj = sync.Boto3RemoteSync( + remoteStore="b3://read", writeStore="b3://artifacts", + architecture=ARCHITECTURE, workdir="/sw") + self.assertEqual(sync_obj.legacyWriteStore, sync_obj.writeStore) + + def test_link_claim_goes_to_the_legacy_bucket(self): + sync_obj = self.make_sync() + sync_obj._put_link("TARS/x/pkg/pkg.tar.gz", "store/aa/hash/pkg.tar.gz") + self.assertEqual(sync_obj.s3.put_object.call_args.kwargs["Bucket"], "legacy") + + def test_existence_check_looks_in_the_legacy_bucket(self): + sync_obj = self.make_sync() + sync_obj._s3_key_exists("TARS/x/store/aa/hash/pkg.tar.gz") + self.assertEqual(sync_obj.s3.head_object.call_args.kwargs["Bucket"], "legacy") + + def test_link_ownership_read_uses_the_legacy_bucket(self): + sync_obj = self.make_sync() + body = MagicMock() + body.read.return_value = b"store/aa/hash/pkg.tar.gz" + sync_obj.s3.get_object.return_value = {"Body": body} + sync_obj._link_is_ours("TARS/x/pkg/pkg.tar.gz", "store/aa/hash/pkg.tar.gz") + self.assertEqual(sync_obj.s3.get_object.call_args.kwargs["Bucket"], "legacy") + if __name__ == '__main__': unittest.main() diff --git a/tests/test_sync_reapi.py b/tests/test_sync_reapi.py new file mode 100644 index 00000000..e8b7c9fe --- /dev/null +++ b/tests/test_sync_reapi.py @@ -0,0 +1,875 @@ +"""Tests for the reapi:// sync backend (REAPIRemoteSync), split out of +test_sync.py to mirror the sync.py / sync_reapi.py source split.""" + +import io +import json +import os +import os.path +import unittest +from unittest.mock import patch, MagicMock + +from alibuild_helpers import sync +from alibuild_helpers import sync_reapi +from alibuild_helpers import signing +from alibuild_helpers.utilities import resolve_links_path, resolve_store_path +from alibuild_helpers.utilities import resolve_cas_path, resolve_ac_path + +# `cryptography` is an optional extra (`pip install alibuild[signing]`), so the +# tests that really sign or build a keyring must skip where it is absent. The +# upload tests below mock sign_via_proxy and so need nothing. +try: + import cryptography # noqa: F401 + HAVE_CRYPTOGRAPHY = True +except ImportError: + HAVE_CRYPTOGRAPHY = False + +requires_crypto = unittest.skipUnless( + HAVE_CRYPTOGRAPHY, "needs the optional 'cryptography' extra") + + +# Shared fixtures (kept local so this module stands alone). +ARCHITECTURE = "slc7_x86-64" +PACKAGE = "zlib" +SEED = bytes(range(32)) + + +def one_key_ring(seed=SEED, signer="ci"): + keyid, pub = signing.public_key(seed) + return signing.load_keyring({"keys": {keyid: {"publicKey": pub, "signer": signer}}}) + + +def tarball_name(spec): + return ("{package}-{version}-{revision}.{arch}.tar.gz" + .format(arch=ARCHITECTURE, **spec)) + + +REAPI_HASH = "deadbeef" * 5 # 40 hex chars, like a real action hash +REAPI_RECIPE_DIGEST = "a" * 64 +REAPI_CONTENT_HASH = "c" * 64 +REAPI_SPEC = { + "package": PACKAGE, "version": "v1.3.1", "revision": "1", + "hash": REAPI_HASH, + "remote_revision_hash": REAPI_HASH, + "remote_hashes": [REAPI_HASH], + "recipe": "build steps here", + "ac_entry": { + "schemaVersion": 1, + "action": { + "actionHash": REAPI_HASH, + "recipeDigest": "sha256:" + REAPI_RECIPE_DIGEST, + }, + }, +} + + +@patch("os.makedirs", new=MagicMock(return_value=None)) +@patch("alibuild_helpers.sync.symlink", new=MagicMock(return_value=None)) +@patch("alibuild_helpers.sync.ProgressPrint", new=MagicMock()) +@patch("alibuild_helpers.log.error", new=MagicMock()) +@patch("alibuild_helpers.sync_reapi.REAPIRemoteSync._s3_init", new=MagicMock()) +class REAPIRemoteSyncTestCase(unittest.TestCase): + """Check the reapi:// (Action Cache + CAS) remote store.""" + + def make_sync(self, client): + reapi = sync_reapi.REAPIRemoteSync( + remoteStore="reapi://localhost/bucket", + writeStore="reapi://localhost/bucket", + architecture=ARCHITECTURE, workdir="/sw") + reapi.s3 = client + return reapi + + def make_client(self, existing=()): + """Mock S3 client: head_object 404s unless the key is in `existing`, + and all directory listings are empty (so uploads see no conflicts).""" + from botocore.exceptions import ClientError + + def head_object(Bucket, Key): + if Key in existing: + return {"ContentLength": 4096} + raise ClientError({"Error": {"Code": "404"}}, "head_object") + + return MagicMock( + head_object=MagicMock(side_effect=head_object), + get_paginator=lambda method: MagicMock( + paginate=lambda **kw: [{"Contents": []}]), + put_object=MagicMock(return_value=None), + upload_file=MagicMock(return_value=None), + download_file=MagicMock(return_value=None), + ) + + def put_keys(self, client): + return [c.kwargs["Key"] for c in client.put_object.call_args_list] + + def test_resolve_falls_back_to_the_ledger_for_tarball_less_actions(self): + """A validate-system action produces no tarball, hence no per-package link, + and used to be unaddressable by name -- with an error that pointed at the + CAS. It must resolve by scanning the Action Cache instead.""" + entry = {"schemaVersion": 2, + "action": {"kind": "validate-system", "package": "make", + "version": "4", "revision": "1", + "architecture": ARCHITECTURE, "actionHash": "ahash-make"}} + ac_key = "ac/%s/ah/ahash-make.json" % ARCHITECTURE + + def get_object(Bucket, Key): + if Key == ac_key: + return {"Body": io.BytesIO(json.dumps(entry).encode())} + raise AssertionError("unexpected get_object %s" % Key) + + client = self.make_client() + # No links anywhere; one AC entry in the ledger listing. + client.get_paginator = lambda method: MagicMock( + paginate=lambda **kw: ([{"Contents": [{"Key": ac_key}]}] + if kw.get("Prefix", "").startswith("ac/") else + [{"Contents": []}])) + client.get_object = MagicMock(side_effect=get_object) + sync = self.make_sync(client) + self.assertEqual(sync.resolve_action_hash("make", "4", "1"), "ahash-make") + # A revision that does not exist must still resolve to nothing. + self.assertIsNone(sync.resolve_action_hash("make", "4", "9")) + + def test_parse_url(self) -> None: + self.assertEqual( + sync_reapi.REAPIRemoteSync._parse_reapi_url("reapi://s3.cern.ch/alibuild-repo", "https"), + ("https://s3.cern.ch", "alibuild-repo")) + self.assertEqual( + sync_reapi.REAPIRemoteSync._parse_reapi_url("reapi://localhost:9000/bkt", "http"), + ("http://localhost:9000", "bkt")) + self.assertEqual(sync_reapi.REAPIRemoteSync._parse_reapi_url("", "https"), ("", "")) + + def test_factory(self) -> None: + obj = sync.remote_from_url("reapi://s3.example/bucket", + "reapi://s3.example/bucket", ARCHITECTURE, "/sw") + self.assertIsInstance(obj, sync_reapi.REAPIRemoteSync) + self.assertEqual(obj.remoteStore, "bucket") + self.assertEqual(obj.endpoint_url, "https://s3.example") + + @patch("alibuild_helpers.sync_reapi.file_digest", + new=MagicMock(return_value=REAPI_CONTENT_HASH)) + @patch("os.path.getsize", new=MagicMock(return_value=4096)) + @patch("os.listdir", + new=lambda path: [tarball_name(REAPI_SPEC)] if path.endswith("-1") else []) + @patch("os.readlink", new=MagicMock(return_value="../../store/de/dead/x.tar.gz")) + @patch("os.path.islink", new=MagicMock(return_value=True)) + def test_upload_writes_cas_ac_redirect(self) -> None: + client = self.make_client() + reapi = self.make_sync(client) + reapi.upload_symlinks_and_tarball(REAPI_SPEC) + + cas_path = resolve_cas_path(REAPI_CONTENT_HASH) + recipe_cas = resolve_cas_path(REAPI_RECIPE_DIGEST) + ac_path = resolve_ac_path(ARCHITECTURE, REAPI_HASH) + store_key = resolve_store_path(ARCHITECTURE, REAPI_HASH) + "/" + tarball_name(REAPI_SPEC) + + # Tarball bytes go to the CAS via upload_file (content-addressed). + client.upload_file.assert_called_once() + self.assertEqual(client.upload_file.call_args.kwargs["Key"], cas_path) + + keys = self.put_keys(client) + self.assertIn(recipe_cas, keys) # recipe blob stored in CAS + self.assertIn(ac_path, keys) # Action Cache entry written + self.assertIn(store_key, keys) # legacy store object written + + # The legacy store object is a redirect to the CAS blob, not the bytes. + redirect = next(c for c in client.put_object.call_args_list + if c.kwargs["Key"] == store_key) + self.assertEqual(redirect.kwargs["WebsiteRedirectLocation"], "/" + cas_path) + + # The AC entry records the output digest pointing at the CAS blob. + ac_call = next(c for c in client.put_object.call_args_list + if c.kwargs["Key"] == ac_path) + entry = json.loads(ac_call.kwargs["Body"]) + self.assertEqual(entry["result"]["outputDigest"], + "sha256:" + REAPI_CONTENT_HASH) + self.assertEqual(entry["result"]["size"], 4096) + + @patch("alibuild_helpers.sync_reapi.file_digest", + new=MagicMock(return_value=REAPI_CONTENT_HASH)) + @patch("os.path.getsize", new=MagicMock(return_value=4096)) + @patch("os.listdir", + new=lambda path: [tarball_name(REAPI_SPEC)] if path.endswith("-1") else []) + @patch("os.readlink", new=MagicMock(return_value="../../store/de/dead/x.tar.gz")) + @patch("os.path.islink", new=MagicMock(return_value=True)) + def test_upload_routes_ledger_and_artifact_to_separate_stores(self) -> None: + client = self.make_client() + reapi = sync_reapi.REAPIRemoteSync( + remoteStore="reapi://localhost/artifacts", + writeStore="reapi://localhost/artifacts", + architecture=ARCHITECTURE, workdir="/sw", + acStore="reapi://localhost/ledger", + acWriteStore="reapi://localhost/ledger") + reapi.s3 = client + reapi.upload_symlinks_and_tarball(REAPI_SPEC) + + cas_path = resolve_cas_path(REAPI_CONTENT_HASH) + recipe_cas = resolve_cas_path(REAPI_RECIPE_DIGEST) + ac_path = resolve_ac_path(ARCHITECTURE, REAPI_HASH) + store_key = resolve_store_path(ARCHITECTURE, REAPI_HASH) + "/" + tarball_name(REAPI_SPEC) + bucket_of = {c.kwargs["Key"]: c.kwargs["Bucket"] + for c in client.put_object.call_args_list} + + # Keep-forever ledger: AC entry + recipe blob. + self.assertEqual(bucket_of[ac_path], "ledger") + self.assertEqual(bucket_of[recipe_cas], "ledger") + # Deletable artifact store: tarball bytes + legacy redirect/link. + self.assertEqual(client.upload_file.call_args.kwargs["Bucket"], "artifacts") + self.assertEqual(client.upload_file.call_args.kwargs["Key"], cas_path) + self.assertEqual(bucket_of[store_key], "artifacts") + + @patch("alibuild_helpers.sync_reapi.file_digest", + new=MagicMock(return_value=REAPI_CONTENT_HASH)) + @patch("os.path.getsize", new=MagicMock(return_value=4096)) + @patch("os.listdir", + new=lambda path: [tarball_name(REAPI_SPEC)] if path.endswith("-1") else []) + @patch("os.readlink", new=MagicMock(return_value="../../store/de/dead/x.tar.gz")) + @patch("os.path.islink", new=MagicMock(return_value=True)) + def test_upload_dedups_existing_cas_blob(self) -> None: + # The CAS already has the tarball bytes (e.g. from an equivalent hash). + client = self.make_client(existing={resolve_cas_path(REAPI_CONTENT_HASH)}) + reapi = self.make_sync(client) + reapi.upload_symlinks_and_tarball(REAPI_SPEC) + # We must not re-upload identical bytes, but we still write the AC entry. + client.upload_file.assert_not_called() + self.assertIn(resolve_ac_path(ARCHITECTURE, REAPI_HASH), self.put_keys(client)) + + @patch("alibuild_helpers.sync_reapi.file_digest", + new=MagicMock(return_value=REAPI_CONTENT_HASH)) + @patch("os.path.getsize", new=MagicMock(return_value=4096)) + @patch("os.listdir", + new=lambda path: [tarball_name(REAPI_SPEC)] if path.endswith("-1") else []) + @patch("os.readlink", new=MagicMock(return_value="../../store/de/dead/x.tar.gz")) + @patch("os.path.islink", new=MagicMock(return_value=True)) + @patch("alibuild_helpers.signing.sign_via_proxy") + def test_upload_signs_ac_entry_when_configured(self, sign_via_proxy) -> None: + sign_via_proxy.return_value = {"keyid": "kid", "signer": "ci", "sig": "s"} + client = self.make_client() + reapi = sync_reapi.REAPIRemoteSync( + remoteStore="reapi://localhost/bucket", writeStore="reapi://localhost/bucket", + architecture=ARCHITECTURE, workdir="/sw", + sign_url="https://proxy/sign/alibuild-ac", sign_token="tok", signer="ci") + reapi.s3 = client + reapi.upload_symlinks_and_tarball(REAPI_SPEC) + + ac_call = next(c for c in client.put_object.call_args_list + if c.kwargs["Key"] == resolve_ac_path(ARCHITECTURE, REAPI_HASH)) + entry = json.loads(ac_call.kwargs["Body"]) + self.assertEqual(entry["signatures"], + [{"keyid": "kid", "signer": "ci", "sig": "s"}]) + self.assertEqual(entry["schemaVersion"], 3) + # It signs the entry *after* the output digest is set, so the signature + # binds the uploaded tarball. + signed_entry = sign_via_proxy.call_args.args[0] + self.assertEqual(signed_entry["result"]["outputDigest"], + "sha256:" + REAPI_CONTENT_HASH) + + @patch("alibuild_helpers.sync_reapi.file_digest", + new=MagicMock(return_value=REAPI_CONTENT_HASH)) + @patch("os.path.getsize", new=MagicMock(return_value=4096)) + @patch("os.listdir", + new=lambda path: [tarball_name(REAPI_SPEC)] if path.endswith("-1") else []) + @patch("os.readlink", new=MagicMock(return_value="../../store/de/dead/x.tar.gz")) + @patch("os.path.islink", new=MagicMock(return_value=True)) + def test_upload_unsigned_by_default(self) -> None: + # No sign_url configured: the AC entry is written without signatures. + client = self.make_client() + reapi = self.make_sync(client) + reapi.upload_symlinks_and_tarball(REAPI_SPEC) + ac_call = next(c for c in client.put_object.call_args_list + if c.kwargs["Key"] == resolve_ac_path(ARCHITECTURE, REAPI_HASH)) + self.assertNotIn("signatures", json.loads(ac_call.kwargs["Body"])) + + def test_sign_without_token_aborts(self) -> None: + # Signing configured (sign_url) but no gate token: fail closed. + reapi = sync_reapi.REAPIRemoteSync( + remoteStore="reapi://localhost/bucket", writeStore="reapi://localhost/bucket", + architecture=ARCHITECTURE, workdir="/sw", + sign_url="https://proxy/sign/alibuild-ac", sign_token="") + self.assertRaises(SystemExit, reapi._sign_ac_entry, + {"action": {"actionHash": "h"}, "result": {}}) + + # --- validate-system entries (no tarball) sign over their recipe digest. --- + + def _validate_system_entry(self): + rd = "b" * 64 + return {"schemaVersion": 2, "action": { + "kind": "validate-system", "package": "make", "version": "v1", + "revision": "1", "architecture": ARCHITECTURE, "actionHash": rd, + "recipeDigest": "sha256:" + rd, "deps": []}} + + def _written_ac(self, client, action_hash): + ac_path = resolve_ac_path(ARCHITECTURE, action_hash) + call = next(c for c in client.put_object.call_args_list + if c.kwargs["Key"] == ac_path) + return json.loads(call.kwargs["Body"]) + + @patch("alibuild_helpers.signing.sign_via_proxy") + def test_put_ac_entry_signs_over_recipe_digest_when_requested(self, sign_via_proxy) -> None: + sign_via_proxy.return_value = {"keyid": "kid", "signer": "ci", "sig": "s"} + client = self.make_client() + reapi = sync_reapi.REAPIRemoteSync( + remoteStore="reapi://localhost/bucket", writeStore="reapi://localhost/bucket", + architecture=ARCHITECTURE, workdir="/sw", + sign_url="https://proxy/sign/alibuild-ac", sign_token="tok", signer="ci") + reapi.s3 = client + entry = self._validate_system_entry() + reapi.put_ac_entry(entry, "recipe text", sign=True) + + written = self._written_ac(client, entry["action"]["actionHash"]) + self.assertEqual(written["signatures"], [{"keyid": "kid", "signer": "ci", "sig": "s"}]) + self.assertEqual(written["schemaVersion"], 3) + # It binds the recipe digest: a validate-system entry has no result block. + self.assertNotIn("result", sign_via_proxy.call_args.args[0]) + + def test_put_ac_entry_unsigned_when_signing_not_configured(self) -> None: + # sign=True requested but no sign_url (e.g. build --no-sign): stays unsigned. + client = self.make_client() + reapi = self.make_sync(client) + entry = self._validate_system_entry() + reapi.put_ac_entry(entry, "recipe text", sign=True) + self.assertNotIn("signatures", self._written_ac(client, entry["action"]["actionHash"])) + + @patch("alibuild_helpers.signing.sign_via_proxy", + new=MagicMock(return_value={"keyid": "k", "signer": "s", "sig": "x"})) + @requires_crypto + def test_put_ac_entry_default_never_signs(self) -> None: + # migrate calls put_ac_entry() with the default sign=False, so it never + # signs even on a sync that has signing configured. + client = self.make_client() + reapi = sync_reapi.REAPIRemoteSync( + remoteStore="reapi://localhost/bucket", writeStore="reapi://localhost/bucket", + architecture=ARCHITECTURE, workdir="/sw", + sign_url="https://proxy/sign/alibuild-ac", sign_token="tok") + reapi.s3 = client + entry = self._validate_system_entry() + reapi.put_ac_entry(entry, "recipe text") # default sign=False + self.assertNotIn("signatures", self._written_ac(client, entry["action"]["actionHash"])) + + # --- Consume side: verifying prebuilt tarballs reused during a build. The + # reapi fetch_tarball resolves via the AC entry (get_object) then the CAS + # blob (head/download), so we mock the S3 client at those seams. --- + + def _signed_entry(self, seed=SEED): + entry = { + "action": {"actionHash": REAPI_HASH, "package": PACKAGE, + "architecture": ARCHITECTURE, "recipeDigest": "sha256:r"}, + "result": {"outputDigest": "sha256:" + REAPI_CONTENT_HASH, + "tarball": tarball_name(REAPI_SPEC), "size": 4096}, + } + if seed is not None: + entry["signatures"] = [signing.sign(entry, seed, "ci")] + return entry + + def _ac_client(self, entry=None, blob_missing=False): + """S3 client whose AC get_object returns `entry` (ClientError if None), + the CAS head_object is present unless blob_missing, and download is a + no-op.""" + from botocore.exceptions import ClientError + + def get_object(Bucket, Key): + if entry is None: + raise ClientError({"Error": {"Code": "404"}}, "get_object") + body = MagicMock(read=MagicMock(return_value=json.dumps(entry).encode())) + return {"Body": body} + + def head_object(Bucket, Key): + if blob_missing: + raise ClientError({"Error": {"Code": "404"}}, "head_object") + return {"ContentLength": 4096} + + return MagicMock(get_object=MagicMock(side_effect=get_object), + head_object=MagicMock(side_effect=head_object), + download_file=MagicMock()) + + def _with_checker(self, client, policy, keyring=None): + reapi = self.make_sync(client) + reapi.verify_checker = sync_reapi.SignatureChecker( + policy, keyring if keyring is not None else one_key_ring()) + return reapi + + @patch("alibuild_helpers.sync_reapi.glob.glob", new=MagicMock(return_value=[])) + @patch("alibuild_helpers.sync_reapi.file_digest", + new=MagicMock(return_value=REAPI_CONTENT_HASH)) + @requires_crypto + def test_fetch_verifies_signed_download(self) -> None: + # Signed entry + bytes matching the signed digest + trusted key: no raise. + reapi = self._with_checker(self._ac_client(self._signed_entry()), "require") + reapi.fetch_tarball(REAPI_SPEC) + reapi.s3.download_file.assert_called_once() + + @patch("alibuild_helpers.sync_reapi.glob.glob", new=MagicMock(return_value=[])) + @patch("alibuild_helpers.sync_reapi.file_digest", + new=MagicMock(return_value=REAPI_CONTENT_HASH)) + @requires_crypto + def test_fetch_rejects_unsigned_under_require(self) -> None: + reapi = self._with_checker( + self._ac_client(self._signed_entry(seed=None)), "require") + self.assertRaises(SystemExit, reapi.fetch_tarball, REAPI_SPEC) + + @patch("alibuild_helpers.sync_reapi.glob.glob", new=MagicMock(return_value=[])) + @patch("alibuild_helpers.sync_reapi.file_digest", + new=MagicMock(return_value="f" * 64)) # bytes != signed digest + @requires_crypto + def test_fetch_rejects_tampered_blob(self) -> None: + reapi = self._with_checker(self._ac_client(self._signed_entry()), "require") + self.assertRaises(SystemExit, reapi.fetch_tarball, REAPI_SPEC) + + @patch("alibuild_helpers.sync_reapi.glob.glob", new=MagicMock(return_value=[])) + @patch("alibuild_helpers.sync_reapi.file_digest", + new=MagicMock(return_value=REAPI_CONTENT_HASH)) + @requires_crypto + def test_fetch_signed_under_warn_ok(self) -> None: + reapi = self._with_checker(self._ac_client(self._signed_entry()), "warn") + reapi.fetch_tarball(REAPI_SPEC) # no raise + + @patch("alibuild_helpers.sync.Boto3RemoteSync.fetch_tarball", + new=MagicMock(return_value=(REAPI_HASH, "/sw/store/x.tar.gz"))) + @patch("alibuild_helpers.sync_reapi.glob.glob", new=MagicMock(return_value=[])) + @requires_crypto + def test_fetch_legacy_unsigned_rejected_under_require(self) -> None: + # No AC entry: falls back to the legacy store; an unsigned legacy tarball + # must fail closed under require, and be tolerated under warn. + self.assertRaises(SystemExit, self._with_checker( + self._ac_client(entry=None), "require").fetch_tarball, REAPI_SPEC) + self._with_checker(self._ac_client(entry=None), "warn").fetch_tarball(REAPI_SPEC) + + @patch("alibuild_helpers.sync.Boto3RemoteSync.fetch_tarball", + new=MagicMock(return_value=None)) # legacy path downloaded nothing + @patch("alibuild_helpers.sync_reapi.glob.glob", new=MagicMock(return_value=[])) + @requires_crypto + def test_fetch_skips_verification_when_nothing_downloaded(self) -> None: + # No AC entry and legacy fetch found nothing: verification never runs, so + # even a require policy does not abort. + self._with_checker(self._ac_client(entry=None), "require").fetch_tarball(REAPI_SPEC) + + @patch("alibuild_helpers.sync_reapi.file_digest", + new=MagicMock(return_value=REAPI_CONTENT_HASH)) + @patch("os.path.getsize", new=MagicMock(return_value=4096)) + @patch("os.listdir", + new=lambda path: [tarball_name(REAPI_SPEC)] if path.endswith("-1") else []) + @patch("os.readlink", new=MagicMock(return_value="../../store/de/dead/x.tar.gz")) + @patch("os.path.islink", new=MagicMock(return_value=True)) + def test_upload_tags_ephemeral_by_default(self) -> None: + client = self.make_client() + reapi = self.make_sync(client) # default storage = ephemeral + reapi.upload_symlinks_and_tarball(REAPI_SPEC) + self.assertEqual(client.upload_file.call_args.kwargs["ExtraArgs"], + {"Tagging": "retention=ephemeral"}) + + @patch("alibuild_helpers.sync_reapi.file_digest", + new=MagicMock(return_value=REAPI_CONTENT_HASH)) + @patch("os.path.getsize", new=MagicMock(return_value=4096)) + @patch("os.listdir", + new=lambda path: [tarball_name(REAPI_SPEC)] if path.endswith("-1") else []) + @patch("os.readlink", new=MagicMock(return_value="../../store/de/dead/x.tar.gz")) + @patch("os.path.islink", new=MagicMock(return_value=True)) + def test_permanent_build_promotes_ephemeral_blob(self) -> None: + # The blob already exists tagged ephemeral; a permanent build promotes it. + client = self.make_client(existing={resolve_cas_path(REAPI_CONTENT_HASH)}) + client.get_object_tagging = MagicMock( + return_value={"TagSet": [{"Key": "retention", "Value": "ephemeral"}]}) + client.put_object_tagging = MagicMock() + reapi = sync_reapi.REAPIRemoteSync("reapi://localhost/bucket", "reapi://localhost/bucket", + architecture=ARCHITECTURE, workdir="/sw", + storage="permanent") + reapi.s3 = client + reapi.upload_symlinks_and_tarball(REAPI_SPEC) + client.upload_file.assert_not_called() # deduped, not re-uploaded + client.put_object_tagging.assert_called_once() + self.assertEqual(client.put_object_tagging.call_args.kwargs["Tagging"], + {"TagSet": [{"Key": "retention", "Value": "permanent"}]}) + + @patch("os.path.getsize", new=MagicMock(return_value=4096)) + def test_rebaseline_rewrites_digest_preserving_retention(self) -> None: + old_hash, new_hash = "a" * 64, "b" * 64 + client = self.make_client() + # The blob being replaced is tagged permanent -> the new one must be too. + client.get_object_tagging = MagicMock( + return_value={"TagSet": [{"Key": "retention", "Value": "permanent"}]}) + reapi = self.make_sync(client) # default storage = ephemeral + entry = { + "schemaVersion": 2, + "action": {"package": "zlib", "version": "v1", "revision": "1", + "architecture": ARCHITECTURE, "actionHash": REAPI_HASH, + "recipeDigest": "sha256:" + REAPI_RECIPE_DIGEST}, + "result": {"tarball": "zlib-v1-1.%s.tar.gz" % ARCHITECTURE, + "outputDigest": "sha256:" + old_hash, "size": 1}, + } + with patch("alibuild_helpers.sync_reapi.file_digest", return_value=new_hash): + oh, nh, old_cas = reapi.rebaseline_ac_entry(entry, "/tmp/zlib.tar.gz", "recipe") + self.assertEqual((oh, nh), (old_hash, new_hash)) + self.assertEqual(old_cas, resolve_cas_path(old_hash)) + # New blob uploaded at the rebuilt content hash, tagged permanent (preserved). + self.assertEqual(client.upload_file.call_args.kwargs["Key"], + resolve_cas_path(new_hash)) + self.assertEqual(client.upload_file.call_args.kwargs["ExtraArgs"], + {"Tagging": "retention=permanent"}) + # AC entry rewritten in place with the new outputDigest. + ac_path = resolve_ac_path(ARCHITECTURE, REAPI_HASH) + ac_put = next(c for c in client.put_object.call_args_list + if c.kwargs["Key"] == ac_path) + body = json.loads(ac_put.kwargs["Body"].decode()) + self.assertEqual(body["result"]["outputDigest"], "sha256:" + new_hash) + # storage restored to its original value afterwards (no leak). + self.assertEqual(reapi.storage, "ephemeral") + + @patch("alibuild_helpers.sync_reapi.file_digest", new=MagicMock(return_value="e" * 64)) + @patch("os.path.getsize", new=MagicMock(return_value=4096)) + def test_put_legacy_artifact(self) -> None: + client = self.make_client() + reapi = self.make_sync(client) + ch = reapi.put_legacy_artifact("zlib", "v1.2.8", "1", "/tmp/zlib.tar.gz") + self.assertEqual(ch, "e" * 64) + # Bytes go to the CAS content-addressed. + self.assertEqual(client.upload_file.call_args.kwargs["Key"], resolve_cas_path("e" * 64)) + keys = self.put_keys(client) + tarball = "zlib-v1.2.8-1.%s.tar.gz" % ARCHITECTURE + self.assertIn(resolve_store_path(ARCHITECTURE, "e" * 64) + "/" + tarball, keys) # redirect + self.assertIn(resolve_links_path(ARCHITECTURE, "zlib") + "/" + tarball, keys) # link + # A kind='legacy' AC entry is written (keyed by the content hash), but no + # recipe blob (there is no provenance to reconstruct from). + ac_path = resolve_ac_path(ARCHITECTURE, "e" * 64) + self.assertIn(ac_path, keys) + ac_put = next(c for c in client.put_object.call_args_list + if c.kwargs["Key"] == ac_path) + entry = json.loads(ac_put.kwargs["Body"].decode()) + self.assertEqual(entry["action"]["kind"], "legacy") + self.assertEqual(entry["result"]["outputDigest"], "sha256:" + "e" * 64) + self.assertNotIn("recipeDigest", entry["action"]) # no provenance + + def test_delete_artifact_blob(self) -> None: + client = self.make_client() + client.delete_object = MagicMock() + reapi = self.make_sync(client) + reapi.delete_artifact_blob("b" * 64) + client.delete_object.assert_called_once_with( + Bucket="bucket", Key=resolve_cas_path("b" * 64)) + + @patch("os.path.exists", new=MagicMock(return_value=False)) + def test_download_refreshes_old_ephemeral(self) -> None: + from datetime import datetime, timezone, timedelta + client = self.make_client() + client.get_object_tagging = MagicMock( + return_value={"TagSet": [{"Key": "retention", "Value": "ephemeral"}]}) + client.head_object = MagicMock( + return_value={"LastModified": datetime.now(timezone.utc) - timedelta(days=75)}) + client.copy_object = MagicMock() + reapi = sync_reapi.REAPIRemoteSync("reapi://localhost/bucket", "reapi://localhost/bucket", + architecture=ARCHITECTURE, workdir="/sw", + storage="permanent") + reapi.s3 = client + reapi.download_artifact(REAPI_CONTENT_HASH, "/tmp/x") + client.copy_object.assert_called_once() # LRU-refreshed (75d >= 60d) + + client.copy_object.reset_mock() + client.head_object = MagicMock( + return_value={"LastModified": datetime.now(timezone.utc) - timedelta(days=10)}) + reapi.download_artifact(REAPI_CONTENT_HASH, "/tmp/x") + client.copy_object.assert_not_called() # fresh (10d < 60d), no refresh + + @patch("glob.glob", new=MagicMock(return_value=[])) + @patch("os.makedirs", new=MagicMock()) + def test_fetch_via_action_cache(self) -> None: + cas_path = resolve_cas_path(REAPI_CONTENT_HASH) + # The CAS blob exists, so head_object (for its size) succeeds. + client = self.make_client(existing={cas_path}) + ac_path = resolve_ac_path(ARCHITECTURE, REAPI_HASH) + entry = {"result": {"tarball": tarball_name(REAPI_SPEC), + "outputDigest": "sha256:" + REAPI_CONTENT_HASH}} + + def get_object(Bucket, Key): + if Key == ac_path: + return {"Body": MagicMock(read=lambda: json.dumps(entry).encode())} + raise NotImplementedError(Key) + client.get_object = MagicMock(side_effect=get_object) + + reapi = self.make_sync(client) + reapi.fetch_tarball(REAPI_SPEC) + + # We downloaded the CAS blob to the local action-store path. + client.download_file.assert_called_once() + self.assertEqual(client.download_file.call_args.kwargs["Key"], cas_path) + self.assertTrue(client.download_file.call_args.kwargs["Filename"].endswith( + resolve_store_path(ARCHITECTURE, REAPI_HASH) + "/" + tarball_name(REAPI_SPEC))) + + +@patch("alibuild_helpers.sync_reapi.REAPIRemoteSync._s3_init", new=MagicMock()) +class REAPIDefaultLayoutTestCase(unittest.TestCase): + """A bare reapi:// URL selects the standard endpoint and bucket layout.""" + + def test_bare_url_selects_default_endpoint_and_buckets(self) -> None: + reapi = sync_reapi.REAPIRemoteSync( + remoteStore="reapi://", writeStore="reapi://", + architecture=ARCHITECTURE, workdir="/sw") + self.assertEqual(reapi.endpoint_url, "https://s3.cern.ch") + self.assertEqual(reapi.remoteStore, "alibuild-cas") + self.assertEqual(reapi.writeStore, "alibuild-cas") + self.assertEqual(reapi.acRemoteStore, "alibuild-ac") + self.assertEqual(reapi.acWriteStore, "alibuild-ac") + self.assertEqual(reapi.legacyWriteStore, "alibuild-repo") + # The legacy links land in a different bucket from the blobs, so the + # redirect is absolute and needs a consumer-reachable CAS URL. Without + # a default the bare form would die on the --cas-public-url check. + self.assertEqual(reapi.casPublicUrl, + "https://s3.cern.ch/swift/v1/alibuild-cas") + + def test_explicit_bucket_opts_out_of_the_split(self) -> None: + """Naming a bucket keeps the single-bucket behaviour, so stores that + predate these defaults are unaffected.""" + reapi = sync_reapi.REAPIRemoteSync( + remoteStore="reapi://localhost/bucket", + writeStore="reapi://localhost/bucket", + architecture=ARCHITECTURE, workdir="/sw") + self.assertEqual(reapi.endpoint_url, "https://localhost") + for store in (reapi.acRemoteStore, reapi.acWriteStore, + reapi.legacyWriteStore): + self.assertEqual(store, "bucket") + self.assertEqual(reapi.casPublicUrl, "") + + def test_explicit_flags_still_win_over_defaults(self) -> None: + reapi = sync_reapi.REAPIRemoteSync( + remoteStore="reapi://", writeStore="reapi://", + architecture=ARCHITECTURE, workdir="/sw", + acStore="reapi:///ledger", acWriteStore="reapi:///ledger", + legacyStore="reapi:///old", casPublicUrl="https://example/cas") + self.assertEqual(reapi.acRemoteStore, "ledger") + self.assertEqual(reapi.legacyWriteStore, "old") + self.assertEqual(reapi.casPublicUrl, "https://example/cas") + + + + +@patch("alibuild_helpers.sync_reapi.REAPIRemoteSync._s3_init", new=MagicMock()) +@patch("alibuild_helpers.sync.Boto3RemoteSync._s3_init", new=MagicMock()) +class LegacyReadStoreTestCase(unittest.TestCase): + """The legacy tree must be READ from wherever it is WRITTEN. + + --legacy-links-store used to redirect only the writes. Every read -- the + symlink listing, the manifest, the per-link fetch -- stayed on the artifact + bucket, whose legacy tree stops being updated the moment the split is turned + on. The consequence was not a missing file but a wrong number: revision + assignment learns which revisions are taken from that listing, so it saw + none, reassigned a revision that already existed, and was then refused by the + ownership check while claiming a link the same builder had written the day + before. One bucket for both directions, or neither. + """ + + def test_split_makes_reads_follow_writes(self): + reapi = sync_reapi.REAPIRemoteSync( + remoteStore="reapi://localhost/cas", writeStore="reapi://localhost/cas", + architecture=ARCHITECTURE, workdir="/sw", + legacyStore="reapi://localhost/legacy", + casPublicUrl="https://example/cas") + self.assertEqual(reapi.legacyWriteStore, "legacy") + self.assertEqual(reapi.legacyReadStore, "legacy", + "reads must follow the split, not stay on the artifact bucket") + + def test_unsplit_reads_the_artifact_bucket(self): + """No split -> unchanged behaviour, so plain reapi:// stores are unaffected.""" + reapi = sync_reapi.REAPIRemoteSync( + remoteStore="reapi://localhost/bucket", writeStore="reapi://localhost/bucket", + architecture=ARCHITECTURE, workdir="/sw") + self.assertEqual(reapi.legacyWriteStore, "bucket") + self.assertEqual(reapi.legacyReadStore, "bucket") + + @patch("os.makedirs", new=MagicMock(return_value=None)) + @patch("os.listdir", new=MagicMock(return_value=[])) + @patch("alibuild_helpers.sync.symlink", new=MagicMock(return_value=None)) + def test_symlink_discovery_never_touches_the_artifact_bucket(self): + """The behavioural half: with the tree split, no read may address the CAS + bucket. This is the test that would have caught the bug -- the constructor + assertions above only pin the wiring, while this pins what it is for.""" + buckets = [] + client = MagicMock( + get_paginator=lambda method: MagicMock( + paginate=lambda **kw: (buckets.append(kw["Bucket"]), + [{"Contents": [{"Key": "TARS/x/zlib/z-1-1.tar.gz"}]}])[1]), + get_object=MagicMock(side_effect=lambda Bucket, Key: ( + buckets.append(Bucket), + {"Body": MagicMock(iter_lines=lambda: iter(()), + read=lambda: b"../../store/de/dead/z.tar.gz")})[1]), + ) + reapi = sync_reapi.REAPIRemoteSync( + remoteStore="reapi://localhost/cas", writeStore="reapi://localhost/cas", + architecture=ARCHITECTURE, workdir="/sw", + legacyStore="reapi://localhost/legacy", + casPublicUrl="https://example/cas") + reapi.s3 = client + reapi.fetch_symlinks({"package": "zlib"}) + + self.assertTrue(buckets, "expected the manifest and the listing to be read") + self.assertNotIn("cas", buckets, + "a read addressed the artifact bucket, whose legacy tree " + "stops being written once the split is on") + self.assertEqual(set(buckets), {"legacy"}) + + def test_boto3_defaults_to_the_artifact_bucket(self): + """b3:// has no split at all; legacyReadStore must simply be remoteStore.""" + b3 = sync.Boto3RemoteSync(remoteStore="b3://read", writeStore="b3://write", + architecture=ARCHITECTURE, workdir="/sw") + self.assertEqual(b3.legacyReadStore, b3.remoteStore) + + + +@patch("os.makedirs", new=MagicMock(return_value=None)) +@patch("alibuild_helpers.sync.ProgressPrint", new=MagicMock()) +@patch("alibuild_helpers.sync_reapi.REAPIRemoteSync._s3_init", new=MagicMock()) +class FetchTarballBucketTestCase(unittest.TestCase): + """Where the BYTES are read from follows the redirect, not the listing. + + With the legacy tree split out, a store object is either a stub pointing into + the CAS bucket or the tarball itself sitting in the legacy bucket. Reading an + un-redirected object from the CAS bucket 404s -- which is exactly what the + first ubuntu2204 release did, because its tarballs were published by a plain + s3:// store that writes bytes where reapi:// writes stubs. + """ + + def make_sync(self, client): + reapi = sync_reapi.REAPIRemoteSync( + remoteStore="reapi://localhost/cas", writeStore="reapi://localhost/cas", + architecture=ARCHITECTURE, workdir="/sw", + legacyStore="reapi://localhost/legacy", + casPublicUrl="https://example/cas") + reapi.s3 = client + return reapi + + def make_client(self, redirect=None): + """Record every (bucket, key) the fetch touches.""" + self.reads = [] + head_meta = {"ContentLength": 4096} + if redirect: + head_meta = dict(head_meta, WebsiteRedirectLocation=redirect) + + def head_object(Bucket, Key): + self.reads.append(("head", Bucket, Key)) + # The CAS blob itself never carries a redirect. + return {"ContentLength": 4096} if Key.startswith("cas/") else head_meta + + def download_file(Bucket, Key, Filename, Callback=None): + self.reads.append(("get", Bucket, Key)) + + # The Action Cache is consulted first; miss it so the legacy path runs, + # which is the path under test. + from botocore.exceptions import ClientError + + def get_object(Bucket, Key): + self.reads.append(("get_object", Bucket, Key)) + raise ClientError({"Error": {"Code": "NoSuchKey"}}, "get_object") + + key = "TARS/%s/store/de/dead/x.tar.gz" % ARCHITECTURE + return MagicMock( + head_object=MagicMock(side_effect=head_object), + download_file=MagicMock(side_effect=download_file), + get_object=MagicMock(side_effect=get_object), + get_paginator=lambda method: MagicMock( + paginate=lambda **kw: [{"Contents": [{"Key": key}]}]), + ) + + def fetch(self, redirect=None): + client = self.make_client(redirect) + reapi = self.make_sync(client) + reapi.fetch_tarball({"package": "zlib", "version": "v1", + "remote_hashes": ["dead"]}) + return self.reads + + def test_plain_tarball_is_read_from_the_legacy_bucket(self): + """The regression: no redirect, so the bytes are the legacy object.""" + reads = self.fetch(redirect=None) + # Only the tarball reads: the Action Cache probe addresses the CAS + # bucket by design, and asserting against it would pin the wrong thing. + tarball_reads = [r for r in reads if r[0] in ("head", "get")] + self.assertTrue([r for r in tarball_reads if r[0] == "get"], + "expected a download") + for kind, bucket, key in tarball_reads: + self.assertEqual(bucket, "legacy", + "%s of %s must come from the legacy bucket" % (kind, key)) + + def test_resolve_via_links_reads_the_link_from_the_legacy_bucket(self): + """Same bug, the other place that pairs a legacy key with a bucket. + + _resolve_via_links lists the legacy tree and then reads the winning + link's body to recover the store hash. Reading that body from the + artifact bucket only worked while the two were one bucket. + """ + buckets = [] + body = b"../../%s/store/de/dead/zlib-v1-1.%s.tar.gz" % ( + ARCHITECTURE.encode(), ARCHITECTURE.encode()) + key = "TARS/%s/zlib/zlib-v1-1.%s.tar.gz" % (ARCHITECTURE, ARCHITECTURE) + client = MagicMock( + get_paginator=lambda method: MagicMock( + paginate=lambda **kw: (buckets.append(kw["Bucket"]), + [{"Contents": [{"Key": key}]}])[1]), + get_object=MagicMock(side_effect=lambda Bucket, Key: ( + buckets.append(Bucket), {"Body": MagicMock(read=lambda: body)})[1]), + ) + reapi = sync_reapi.REAPIRemoteSync( + remoteStore="reapi://localhost/cas", writeStore="reapi://localhost/cas", + architecture=ARCHITECTURE, workdir="/sw", + legacyStore="reapi://localhost/legacy", + casPublicUrl="https://example/cas") + reapi.s3 = client + + self.assertEqual(reapi._resolve_via_links("zlib", "v1"), "dead") + self.assertEqual(set(buckets), {"legacy"}, + "listing and link body must address the same bucket") + + def test_stub_is_followed_into_the_cas_bucket(self): + """And the other half still works: a stub redirects into the CAS.""" + reads = self.fetch(redirect="/cas/sha256/de/dead") + gets = [r for r in reads if r[0] == "get"] + self.assertEqual([(b, k) for _, b, k in gets], + [("cas", "cas/sha256/de/dead")]) + # The first head -- the legacy store object -- still addresses the + # legacy bucket; only the redirected fetch leaves it. + heads = [r for r in reads if r[0] == "head"] + self.assertEqual(heads[0][1], "legacy") + + +@patch("alibuild_helpers.sync_reapi.REAPIRemoteSync._s3_init", new=MagicMock()) +class IsFullyMigratedTestCase(unittest.TestCase): + """A package published by a BUILD must not be migrated again. + + migrate has no --legacy-links-store, so it can only look for the per-package + link in its own artifact store. A release publishes that link into the SHARED + legacy bucket instead, so a perfectly published package looked unmigrated, + was retried every round, and failed on the 78-byte redirect stub that + --read-store's plain GET returns. Seventeen packages of the ubuntu2204 + O2Suite closure did that on every round. + """ + + def make_sync(self, link_exists, ac_hash=None, published=False): + reapi = sync_reapi.REAPIRemoteSync( + remoteStore="reapi://localhost/cas", writeStore="reapi://localhost/cas", + architecture=ARCHITECTURE, workdir="/sw") + reapi._exists = MagicMock(return_value=link_exists) + reapi.resolve_action_hash = MagicMock(return_value=ac_hash) + reapi.is_published = MagicMock(return_value=published) + return reapi + + def test_link_present_is_enough(self): + reapi = self.make_sync(link_exists=True) + self.assertTrue(reapi.is_fully_migrated(ARCHITECTURE, "zlib", "zlib-v1-1.tar.gz", "v1-1")) + reapi.resolve_action_hash.assert_not_called() # cheap path, no AC lookup + + def test_published_elsewhere_counts_as_migrated(self): + """The regression: no link here, but the AC says it is published.""" + reapi = self.make_sync(link_exists=False, ac_hash="abc123", published=True) + self.assertTrue(reapi.is_fully_migrated(ARCHITECTURE, "c-ares", "c-ares-1.34.6-1.tar.gz", "1.34.6-1")) + reapi.resolve_action_hash.assert_called_once_with("c-ares", "1.34.6", "1") + + def test_ac_entry_without_its_blob_is_not_migrated(self): + """is_published requires the blob too; an entry alone must not skip it.""" + reapi = self.make_sync(link_exists=False, ac_hash="abc123", published=False) + self.assertFalse(reapi.is_fully_migrated(ARCHITECTURE, "zlib", "zlib-v1-1.tar.gz", "v1-1")) + + def test_unknown_action_is_not_migrated(self): + reapi = self.make_sync(link_exists=False, ac_hash=None) + self.assertFalse(reapi.is_fully_migrated(ARCHITECTURE, "zlib", "zlib-v1-1.tar.gz", "v1-1")) + + def test_without_verrev_it_falls_back_to_the_link_only(self): + """Older callers pass no verrev; they must keep the previous behaviour.""" + reapi = self.make_sync(link_exists=False, ac_hash="abc123", published=True) + self.assertFalse(reapi.is_fully_migrated(ARCHITECTURE, "zlib", "zlib-v1-1.tar.gz")) + reapi.resolve_action_hash.assert_not_called() + +if __name__ == '__main__': + unittest.main() + diff --git a/tests/test_utilities.py b/tests/test_utilities.py index 216c6f5f..0df10932 100644 --- a/tests/test_utilities.py +++ b/tests/test_utilities.py @@ -10,8 +10,11 @@ from alibuild_helpers.utilities import resolve_version from alibuild_helpers.utilities import topological_sort from alibuild_helpers.utilities import resolveFilename, resolveDefaultsFilename +from alibuild_helpers.utilities import resolve_store_path, resolve_cas_path, resolve_ac_path, file_digest from alibuild_helpers.utilities import docker_platform_for import alibuild_helpers +import hashlib + class DockerPlatformTestCase(unittest.TestCase): def test_docker_platform_for(self): @@ -24,6 +27,7 @@ def test_docker_platform_for(self): self.assertIsNone(docker_platform_for("")) import os import string +import tempfile UBUNTU_1510_OS_RELEASE = """ NAME="Ubuntu" @@ -291,6 +295,30 @@ def test_UTF8_Hasher(self) -> None: self.assertEqual(h3.hexdigest(), "0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33") self.assertNotEqual(h1.hexdigest(), h2.hexdigest()) + def test_cas_ac_paths(self) -> None: + h = "abcdef0123456789abcdef0123456789abcdef01" + chash = "f" * 64 + # The action store and the CAS/AC namespaces all shard on the first two + # hex characters of the hash, matching resolve_store_path's convention. + self.assertEqual(resolve_store_path("slc7_x86-64", h), + "TARS/slc7_x86-64/store/ab/" + h) + self.assertEqual(resolve_cas_path(chash), + "cas/sha256/ff/" + chash) + self.assertEqual(resolve_cas_path(chash, algo="sha1"), + "cas/sha1/ff/" + chash) + self.assertEqual(resolve_ac_path("slc7_x86-64", h), + "ac/slc7_x86-64/ab/" + h + ".json") + + def test_file_digest(self) -> None: + payload = b"the quick brown fox\n" * 100000 # exceed the 1 MiB read chunk + expected = hashlib.sha256(payload).hexdigest() + with tempfile.NamedTemporaryFile() as tmp: + tmp.write(payload) + tmp.flush() + self.assertEqual(file_digest(tmp.name), expected) + self.assertEqual(file_digest(tmp.name, algo="sha1"), + hashlib.sha1(payload).hexdigest()) + def test_asList(self) -> None: self.assertEqual(asList("a"), ["a"]) self.assertEqual(asList(["a"]), ["a"]) diff --git a/tox.ini b/tox.ini index c478d53e..77f45a89 100644 --- a/tox.ini +++ b/tox.ini @@ -47,6 +47,11 @@ deps = coverage distro +; `cryptography` is an optional runtime extra, but the signing tests skip without +; it -- so install it here, or signing would silently lose all CI coverage. +extras = + signing + setenv = # `aliBuild analytics` puts preference files under $HOME. HOME = {envtmpdir}