From bb3fb31e303bf656ad1c0f1b3a74e39622e2d049 Mon Sep 17 00:00:00 2001 From: Josef Hak Date: Wed, 7 Oct 2026 09:28:18 +0000 Subject: [PATCH] Support OCI upstream repos in auto-updates Resolve the newest release from the registry tag list instead of skipping non-https repositories. Helm's own range resolution is not usable here: it filters tags through strict semver, so every 'v'-prefixed tag is dropped, which returns a stale version for mixed-tag repos and fails outright for repos tagged only that way. Also enable auto-updates for envoy-gateway. --- .github/workflows/auto-update.yml | 1 + scripts/chart_ctl.py | 130 ++++++++++++++++++++++++++++-- scripts/tests/test_chart_ctl.py | 52 ++++++++++++ 3 files changed, 176 insertions(+), 7 deletions(-) diff --git a/.github/workflows/auto-update.yml b/.github/workflows/auto-update.yml index 3717a41551..bac5d1c2a0 100644 --- a/.github/workflows/auto-update.yml +++ b/.github/workflows/auto-update.yml @@ -41,6 +41,7 @@ jobs: # On cron: auto-update selected apps: all_detected_files=$(find \ apps/cert-manager \ + apps/envoy-gateway \ apps/external-dns \ apps/external-secrets \ apps/nvidia \ diff --git a/scripts/chart_ctl.py b/scripts/chart_ctl.py index 645d9d0ce6..911112f903 100755 --- a/scripts/chart_ctl.py +++ b/scripts/chart_ctl.py @@ -8,8 +8,15 @@ import json import sys import shutil +import urllib.error +import urllib.parse +import urllib.request +from packaging.version import InvalidVersion, Version import utils +# Docker Hub is addressed by a different host in the registry API than in chart repository URLs. +DOCKER_HUB_HOSTS = {"docker.io", "index.docker.io"} + def _semver_parts(version: str): """Return (major, minor, patch) as ints, or None if not semver.""" @@ -174,6 +181,115 @@ def update_charts_cfg(args: str, updates_list: list, cfg: dict): write_charts_cfg(args.app, output) +def oci_chart_ref(repository: str, chart: str) -> str: + return f"{repository.rstrip('/')}/{chart}" + + +def oci_registry_path(repository: str, chart: str): + """Split an OCI chart reference into registry API host and repository path.""" + host, _, path = oci_chart_ref(repository, chart)[len("oci://"):].partition('/') + if host in DOCKER_HUB_HOSTS: + host = "registry-1.docker.io" + if '/' not in path: + path = f"library/{path}" + return host, path + + +def registry_get(url: str, token: str = None): + request = urllib.request.Request(url, headers={"User-Agent": "k0rdent-catalog"}) + if token: + request.add_header("Authorization", f"Bearer {token}") + return urllib.request.urlopen(request, timeout=60) + + +def registry_auth_token(challenge: str) -> str: + """Resolve a pull token from a Www-Authenticate challenge (Docker registry v2 auth).""" + params = dict(re.findall(r'(\w+)="([^"]*)"', challenge)) + realm = params.pop("realm", None) + if not realm: + return None + url = f"{realm}?{urllib.parse.urlencode(params)}" if params else realm + with registry_get(url) as response: + body = json.load(response) + return body.get("token") or body.get("access_token") + + +def oci_list_tags(repository: str, chart: str) -> list: + host, path = oci_registry_path(repository, chart) + url = f"https://{host}/v2/{path}/tags/list?n=1000" + token = None + tags = [] + while url: + try: + response = registry_get(url, token) + except urllib.error.HTTPError as e: + if e.code == 401 and token is None: + token = registry_auth_token(e.headers.get("Www-Authenticate", "")) + if token: + continue + raise + with response: + tags.extend(json.load(response).get("tags") or []) + link = response.headers.get("Link", "") + next_page = re.search(r'<([^>]+)>\s*;\s*rel="?next"?', link) + url = urllib.parse.urljoin(url, next_page.group(1)) if next_page else None + return tags + + +def latest_stable_tag(tags: list) -> str: + """Highest release tag, ignoring pre-releases and anything not version-like.""" + latest, latest_version = None, None + for tag in tags: + try: + version = Version(tag) + except InvalidVersion: + continue + if version.is_prerelease: + continue + if latest_version is None or version > latest_version: + latest, latest_version = tag, version + return latest + + +def show_chart(reference: str, version: str = None) -> dict: + args = ["helm", "show", "chart", reference] + if version: + args += ["--version", version] + result = subprocess.run(args, check=True, capture_output=True, text=True) + return yaml.safe_load(result.stdout) + + +def get_latest_https_chart(chart: str, repository: str) -> dict: + subprocess.run(["helm", "repo", "add", chart, repository], check=True) + try: + subprocess.run(["helm", "repo", "update"], check=True) + return show_chart(f"{chart}/{chart}") + finally: + subprocess.run(["helm", "repo", "remove", chart], check=True) + + +def get_latest_oci_chart(chart: str, repository: str) -> dict: + """Resolve the newest release from the registry tag list. + + `helm show chart --version '>=0.0.0'` would be shorter, but Helm drops every + 'v'-prefixed tag when resolving a range, which silently returns a stale + version (or no version at all) for charts tagged that way. + """ + tag = latest_stable_tag(oci_list_tags(repository, chart)) + if tag is None: + raise RuntimeError(f"no release tags found in '{oci_chart_ref(repository, chart)}'") + return show_chart(oci_chart_ref(repository, chart), tag) + + +def get_latest_chart(chart: str, repository: str) -> dict: + if repository.startswith("https"): + return get_latest_https_chart(chart, repository) + if repository.startswith("oci://"): + return get_latest_oci_chart(chart, repository) + print(f"Unsupported repo '{repository}' to automatically check updates, skipping.") + return None + + def check_updates(args: str): cfg = read_charts_cfg(args.app, allow_return_none=True) if cfg is None: @@ -183,13 +299,14 @@ def check_updates(args: str): updates_list = [] updates_dict = {} for chart, data in last_deps.items(): - if not data['repository'].startswith("https"): - print(f"Unsupported repo '{data['repository']}' to automatically check updates, skipping.") + try: + up_to_date_chart = get_latest_chart(chart, data['repository']) + except (subprocess.CalledProcessError, urllib.error.URLError, RuntimeError) as e: + # One unreachable repo (private registry, outage) must not fail the whole app. + print(f"::warning::Cannot check updates for '{chart}' in '{data['repository']}': {e}") + continue + if up_to_date_chart is None: continue - subprocess.run(["helm", "repo", "add", chart, data['repository']], check=True) - subprocess.run(["helm", "repo", "update"], check=True) - result = subprocess.run(["helm", "show", "chart", f"{chart}/{chart}"], check=True, capture_output=True, text=True) - up_to_date_chart = yaml.safe_load(result.stdout) print(f"Last version found: {up_to_date_chart['version']}") try_ignore_prefix_v(up_to_date_chart, data['version']) if up_to_date_chart['version'] != data['version']: @@ -198,7 +315,6 @@ def check_updates(args: str): item['version'] = up_to_date_chart['version'] updates_list.append(item) updates_dict[item['name']] = item - subprocess.run(["helm", "repo", "remove", chart], check=True) update_charts_cfg(args, updates_list, cfg) if args.generate_charts: generate(args) diff --git a/scripts/tests/test_chart_ctl.py b/scripts/tests/test_chart_ctl.py index 2937703a7f..31d3087ebb 100644 --- a/scripts/tests/test_chart_ctl.py +++ b/scripts/tests/test_chart_ctl.py @@ -71,3 +71,55 @@ def test_prune_old_patches_keeps_non_semver(): pruned = chart_ctl.prune_old_patches("nonexistent-app", charts) versions = sorted(c["version"] for c in pruned) assert versions == ["1.0.1", "main"] + + +def test_oci_registry_path_plain_host(): + assert chart_ctl.oci_registry_path("oci://ghcr.io/kserve/charts", "kserve") == ( + "ghcr.io", "kserve/charts/kserve") + + +def test_oci_registry_path_strips_trailing_slash(): + assert chart_ctl.oci_registry_path("oci://ghcr.io/k0rdent/catalog/charts/", "valkey") == ( + "ghcr.io", "k0rdent/catalog/charts/valkey") + + +def test_oci_registry_path_maps_docker_hub(): + assert chart_ctl.oci_registry_path("oci://docker.io/envoyproxy", "gateway-helm") == ( + "registry-1.docker.io", "envoyproxy/gateway-helm") + + +def test_oci_registry_path_docker_hub_official_repo(): + # A single path segment on Docker Hub lives under the implicit "library" namespace. + assert chart_ctl.oci_registry_path("oci://docker.io", "n8n") == ( + "registry-1.docker.io", "library/n8n") + + +def test_latest_stable_tag_picks_highest(): + assert chart_ctl.latest_stable_tag(["1.2.0", "1.10.0", "1.9.0"]) == "1.10.0" + + +def test_latest_stable_tag_keeps_v_prefix(): + # Helm's own range resolution drops these; we must not. + assert chart_ctl.latest_stable_tag(["v0.15.0", "v0.16.0"]) == "v0.16.0" + + +def test_latest_stable_tag_skips_prereleases(): + assert chart_ctl.latest_stable_tag(["v0.16.0", "v0.17.0-rc0"]) == "v0.16.0" + + +def test_latest_stable_tag_ignores_non_versions(): + assert chart_ctl.latest_stable_tag(["latest", "main", "sha-abc123", "1.0.0"]) == "1.0.0" + + +def test_latest_stable_tag_compares_across_v_prefix(): + # Mixed tagging must not hide a newer release behind a prefix difference. + assert chart_ctl.latest_stable_tag(["1.6.0", "v2.2.1"]) == "v2.2.1" + + +def test_latest_stable_tag_without_releases(): + assert chart_ctl.latest_stable_tag(["latest", "1.0.0-rc1"]) is None + + +def test_oci_chart_ref(): + assert chart_ctl.oci_chart_ref("oci://quay.io/strimzi-helm/", "strimzi-kafka-operator") == ( + "oci://quay.io/strimzi-helm/strimzi-kafka-operator")