diff --git a/.github/workflows/grid-cli.yml b/.github/workflows/grid-cli.yml new file mode 100644 index 0000000..3a29aec --- /dev/null +++ b/.github/workflows/grid-cli.yml @@ -0,0 +1,74 @@ +name: grid-cli + +on: + push: + pull_request: + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: grid-cli-${{ github.event.pull_request.head.ref || github.ref_name }} + cancel-in-progress: true + +env: + ARTIFACT_NAME: nano-rust-cli-el9-x86_64 + BUILD_IMAGE: cmssw/el9:x86_64 + CARGO_INCREMENTAL: "0" + CARGO_PROFILE_RELEASE_STRIP: symbols + CARGO_TERM_COLOR: always + RUST_FEATURES: nano-cli/http,nano-ui/http,nano-workflow/http + +jobs: + build: + name: Build Rust CLIs (CMS EL9) + runs-on: ubuntu-latest + container: + image: cmssw/el9:x86_64 + options: --user 0 + steps: + - uses: actions/checkout@v4 + + - uses: dtolnay/rust-toolchain@stable + + - name: Build every Rust CLI + run: >- + cargo build --locked --release --workspace --bins + --features "${RUST_FEATURES}" + + - name: Package grid artifact + run: | + python3 scripts/package_grid_cli.py \ + --target-dir target/release \ + --output-dir "dist/${ARTIFACT_NAME}" \ + --build-image "${BUILD_IMAGE}" \ + --features "${RUST_FEATURES}" \ + --repository "${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}" \ + --commit "${GITHUB_SHA}" + python3 "dist/${ARTIFACT_NAME}/verify.py" "dist/${ARTIFACT_NAME}" + tar -C dist -czf "dist/${ARTIFACT_NAME}.tar.gz" "${ARTIFACT_NAME}" + + - uses: actions/upload-artifact@v4 + with: + name: ${{ env.ARTIFACT_NAME }}-${{ github.sha }} + path: dist/${{ env.ARTIFACT_NAME }}.tar.gz + if-no-files-found: error + retention-days: 14 + + grid-smoke: + name: Smoke test artifact (CMS EL9 grid) + needs: build + runs-on: ubuntu-latest + container: + image: cmssw/el9:x86_64-grid + options: --user 0 + steps: + - uses: actions/download-artifact@v4 + with: + name: ${{ env.ARTIFACT_NAME }}-${{ github.sha }} + + - name: Verify in grid worker image + run: | + tar -xzf "${ARTIFACT_NAME}.tar.gz" + python3 "${ARTIFACT_NAME}/verify.py" "${ARTIFACT_NAME}" diff --git a/README.md b/README.md index 5d48162..4afe174 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,7 @@ # nano.rust [![CI](https://github.com/DickyChant/nano.rust/actions/workflows/ci.yml/badge.svg)](https://github.com/DickyChant/nano.rust/actions/workflows/ci.yml) +[![grid CLI](https://github.com/DickyChant/nano.rust/actions/workflows/grid-cli.yml/badge.svg)](https://github.com/DickyChant/nano.rust/actions/workflows/grid-cli.yml) [![docs](https://github.com/DickyChant/nano.rust/actions/workflows/docs.yml/badge.svg)](https://github.com/DickyChant/nano.rust/actions/workflows/docs.yml) [![links](https://github.com/DickyChant/nano.rust/actions/workflows/links.yml/badge.svg)](https://github.com/DickyChant/nano.rust/actions/workflows/links.yml) diff --git a/docs/grid-cli-artifact.md b/docs/grid-cli-artifact.md new file mode 100644 index 0000000..f272329 --- /dev/null +++ b/docs/grid-cli-artifact.md @@ -0,0 +1,30 @@ +# nano.rust Grid CLI Artifact + +This archive contains every Rust CLI in the workspace, built on CMS EL9 and +smoke-tested in the `cmssw/el9:x86_64-grid` worker image. It has no ROOT or +CMSSW runtime dependency. + +The archive layout is: + +- `bin/`: `nano`, `nano-mcp`, `nano-ui`, and `nano-workflow`. +- `configs/`: reviewed run cards, sample catalogues, and correction data. +- `crates/nano-spec/examples/`: physics-facing TOML and ADL specifications. +- `manifest.json`: source commit, toolchain, build image, features, and SHA-256 + checksums. +- `verify.py`: artifact integrity, dynamic-linking, and launch checks. + +Unpack the archive in the job sandbox and run tools from its root so relative +paths in analysis specs continue to resolve: + +```bash +tar -xzf nano-rust-cli-el9-x86_64.tar.gz +cd nano-rust-cli-el9-x86_64 +python3 verify.py . +./bin/nano validate --catalogue-version v15 \ + crates/nano-spec/examples/wz_vbs.toml +``` + +For HTCondor, transfer the `.tar.gz` file with the job and unpack it in the +wrapper before invoking a CLI. The artifact targets Linux x86-64 and is tested +against the official CMS EL9 grid image; rebuild it for another architecture or +OS baseline. diff --git a/scripts/package_grid_cli.py b/scripts/package_grid_cli.py new file mode 100644 index 0000000..a868307 --- /dev/null +++ b/scripts/package_grid_cli.py @@ -0,0 +1,131 @@ +#!/usr/bin/env python3 +"""Assemble all workspace CLI binaries into a self-describing grid artifact.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +from pathlib import Path +import platform +import shutil +import subprocess + + +def command_output(*args: str) -> str: + return subprocess.check_output(args, text=True).strip() + + +def sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def cli_targets() -> list[dict[str, str]]: + metadata = json.loads( + command_output("cargo", "metadata", "--no-deps", "--format-version", "1") + ) + targets = [] + for package in metadata["packages"]: + if package["id"] not in metadata["workspace_members"]: + continue + for target in package["targets"]: + if "bin" in target["kind"]: + targets.append({"name": target["name"], "package": package["name"]}) + return sorted(targets, key=lambda target: target["name"]) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--target-dir", type=Path, default=Path("target/release")) + parser.add_argument( + "--output-dir", type=Path, default=Path("dist/nano-rust-cli-el9-x86_64") + ) + parser.add_argument("--build-image", required=True) + parser.add_argument("--features", default="") + parser.add_argument("--repository") + parser.add_argument("--commit") + return parser.parse_args() + + +def main() -> None: + args = parse_args() + targets = cli_targets() + if not targets: + raise SystemExit("cargo metadata did not report any CLI binary targets") + + missing = [ + target["name"] + for target in targets + if not (args.target_dir / target["name"]).is_file() + ] + if missing: + raise SystemExit(f"release binaries are missing: {', '.join(missing)}") + + shutil.rmtree(args.output_dir, ignore_errors=True) + bin_dir = args.output_dir / "bin" + bin_dir.mkdir(parents=True) + + binaries = [] + for target in targets: + source = args.target_dir / target["name"] + destination = bin_dir / target["name"] + shutil.copy2(source, destination) + destination.chmod(0o755) + binaries.append( + { + **target, + "path": f"bin/{target['name']}", + "bytes": destination.stat().st_size, + "sha256": sha256(destination), + } + ) + + shutil.copytree("configs", args.output_dir / "configs") + spec_root = args.output_dir / "crates/nano-spec/examples" + spec_root.parent.mkdir(parents=True) + shutil.copytree("crates/nano-spec/examples", spec_root) + shutil.copy2("docs/grid-cli-artifact.md", args.output_dir / "README.md") + shutil.copy2("scripts/verify_grid_cli.py", args.output_dir / "verify.py") + if Path("LICENSE").is_file(): + shutil.copy2("LICENSE", args.output_dir / "LICENSE") + + image_info = Path("/image-build-info.txt") + repository = args.repository or command_output( + "git", "config", "--get", "remote.origin.url" + ) + commit = args.commit or command_output("git", "rev-parse", "HEAD") + manifest = { + "schema_version": 1, + "source": { + "repository": repository, + "commit": commit, + }, + "build": { + "image": args.build_image, + "image_info": image_info.read_text().strip() + if image_info.is_file() + else None, + "architecture": platform.machine(), + "rustc": command_output("rustc", "--version"), + "features": [feature for feature in args.features.split(",") if feature], + }, + "binaries": binaries, + "config_root": "configs", + "spec_root": "crates/nano-spec/examples", + } + (args.output_dir / "manifest.json").write_text( + json.dumps(manifest, indent=2) + os.linesep, encoding="utf-8" + ) + + print(f"packaged {len(binaries)} CLI binaries in {args.output_dir}") + for binary in binaries: + print(f" {binary['name']} ({binary['package']}, {binary['bytes']} bytes)") + + +if __name__ == "__main__": + main() diff --git a/scripts/verify_grid_cli.py b/scripts/verify_grid_cli.py new file mode 100644 index 0000000..64e7fa2 --- /dev/null +++ b/scripts/verify_grid_cli.py @@ -0,0 +1,89 @@ +#!/usr/bin/env python3 +"""Verify artifact integrity and launch every CLI through the host loader.""" + +from __future__ import annotations + +import hashlib +import json +from pathlib import Path +import subprocess +import sys + + +def sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def main() -> None: + root = Path(sys.argv[1] if len(sys.argv) > 1 else ".").resolve() + manifest = json.loads((root / "manifest.json").read_text(encoding="utf-8")) + binaries = manifest.get("binaries", []) + if not binaries: + raise SystemExit("manifest contains no binaries") + + for entry in binaries: + binary = root / entry["path"] + if not binary.is_file(): + raise SystemExit(f"missing binary: {binary}") + if sha256(binary) != entry["sha256"]: + raise SystemExit(f"checksum mismatch: {binary}") + + linked = subprocess.run( + ["ldd", str(binary)], capture_output=True, text=True, check=False + ) + link_output = linked.stdout + linked.stderr + if "not found" in link_output: + raise SystemExit(f"unresolved runtime library for {binary}:\n{link_output}") + + launched = subprocess.run( + [str(binary), "--help"], + stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + timeout=10, + check=False, + ) + if launched.returncode < 0 or launched.returncode in (126, 127): + raise SystemExit( + f"failed to launch {binary} (exit {launched.returncode}):\n{launched.stdout}" + ) + print(f"OK {entry['name']}: linked and launchable (exit {launched.returncode})") + + config_root = root / manifest["config_root"] + if not config_root.is_dir(): + raise SystemExit(f"missing config root: {config_root}") + print(f"OK config root: {config_root}") + + spec_root = root / manifest["spec_root"] + if not spec_root.is_dir(): + raise SystemExit(f"missing spec root: {spec_root}") + print(f"OK spec root: {spec_root}") + + wz_spec = spec_root / "wz_vbs.toml" + validation = subprocess.run( + [ + str(root / "bin/nano"), + "validate", + "--catalogue-version", + "v15", + str(wz_spec.relative_to(root)), + ], + cwd=root, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + timeout=30, + check=False, + ) + if validation.returncode != 0: + raise SystemExit(f"WZ spec validation failed:\n{validation.stdout}") + print("OK WZ VBS spec and correction payload") + + +if __name__ == "__main__": + main()