From e309cecb535bd609c5dabe27d21ba716f2629f90 Mon Sep 17 00:00:00 2001 From: Wynne Pirini Date: Fri, 16 Oct 2026 14:35:32 +1300 Subject: [PATCH] ci(ns8): gate releases on what actually reached the registry Two silent-failure guards, both for bugs that already shipped once. Post-publish multi-arch assertion. A release once published an amd64-only image as :latest. It installs on amd64 and hard-fails on arm64, the publish step exited 0, and nothing looked at the registry. packaging/ns8/assert- multiarc.py now runs after push-manifest.sh against the published tags and fails the release unless every required architecture is present, that architecture's config really declares it, it carries an org.nethserver.images label, and the app image that label names is itself multi-arch and contains the same architecture. It checks the registry rather than local storage on purpose: a broken publish must not be able to pass by looking healthy from the runner. It needs no extra secrets, since the module repository is public and the token is anonymous, and it retries briefly so push-to-read propagation latency cannot fail a good release. It runs for branch pushes too, because :edge and :dev are published tags and deserve the same guarantee as a version tag. state-include.conf assertion. That file is handed to restic as --files-from, so restic backs up ONLY the paths listed in it. Dropping a line loses data while the backup job still reports success, which is exactly how #142 shipped the module with the database but not the encryption key. Both paths are now asserted in the validate job, with the reason in a comment, so the next person who trims that file finds out from CI rather than from a failed restore. Also py_compiles the new script in the validate job, since the post-publish assertion cannot run on a PR (there is nothing published to assert against) and would otherwise ship completely unexercised. .gitignore now covers __pycache__, which a local py_compile run would otherwise commit. Verified locally: the assertion script exits 1 on the current amd64-only :latest with an accurate diagnosis, and exits 0 on :edge. The state-include check passes against the real file and detects a simulated dropped line. --- .github/workflows/ns8-module.yml | 36 ++++++ .gitignore | 1 + packaging/ns8/assert-multiarc.py | 201 +++++++++++++++++++++++++++++++ 3 files changed, 238 insertions(+) create mode 100644 packaging/ns8/assert-multiarc.py diff --git a/.github/workflows/ns8-module.yml b/.github/workflows/ns8-module.yml index f2436c9a..cbaca127 100644 --- a/.github/workflows/ns8-module.yml +++ b/.github/workflows/ns8-module.yml @@ -33,6 +33,7 @@ jobs: working-directory: packaging/ns8 run: | python3 -m py_compile imageroot/actions/configure-module/20configure imageroot/actions/get-configuration/20read + python3 -m py_compile assert-multiarc.py python3 -c "import json, glob; [json.load(open(f)) for f in glob.glob('imageroot/actions/*/*.json') + glob.glob('ui/public/*.json') + ['ui/public/i18n/en/translation.json']]" bash -n imageroot/actions/configure-module/80start_services bash -n build-images.sh @@ -62,6 +63,27 @@ jobs: fi echo "$scripts" | awk '$1!="100755" {print "not executable in index:", $4; bad=1} END{exit bad}' + # state-include.conf is handed to restic as --files-from, so restic + # backs up ONLY the paths listed in it. Dropping a line is a silent + # data-loss failure: the backup job still reports success while + # quietly omitting that data. Neither line is discoverable from the + # image, so assert the file instead of trusting it to stay correct. + # volumes/calnode-data the SQLite database, the whole app state + # state/calnode.env holds CALNODE_ENCRYPTION_KEY; without it a + # restored database cannot be decrypted and + # the app refuses to start with + # "keyvault: failed to unwrap DEK" + inc=imageroot/etc/state-include.conf + test -f "$inc" || { echo "::error::$inc is missing"; exit 1; } + for required in volumes/calnode-data state/calnode.env; do + if ! grep -qx "$required" "$inc"; then + echo "::error::$inc no longer lists '$required'. restic receives this file as --files-from, so omitting a path silently drops that data from every backup while still reporting success." + cat "$inc" + exit 1 + fi + done + echo "state-include.conf lists both the volume and the encryption key" + # Run the real push-manifest.sh end to end against a scratch storage and # an oci: destination, so no registry and no credentials are needed. PR # builds push nothing, which means the manifest step is skipped on PRs and @@ -221,3 +243,17 @@ jobs: TARGETARCHES: "amd64 arm64" IMAGETAGS: ${{ steps.tags.outputs.module_tags }} run: bash push-manifest.sh + + # The gate that would have caught the amd64-only :latest. It inspects what + # actually landed in the registry rather than local storage, so a broken + # publish cannot pass by looking healthy from the runner. Needs no extra + # secrets: the module repository is public and the token is anonymous. + # Runs for branch pushes too, because :edge and :dev are published tags + # and deserve exactly the same guarantee as a version tag. + - name: Assert the published tags are multi-arch + if: github.event_name != 'pull_request' + working-directory: packaging/ns8 + env: + MODULE_REPO: ghcr.io/${{ github.repository_owner }}/ns8-calnode + IMAGETAGS: ${{ steps.tags.outputs.module_tags }} + run: python3 assert-multiarc.py "$MODULE_REPO" $IMAGETAGS diff --git a/.gitignore b/.gitignore index e32f2a3a..e378b9ae 100644 --- a/.gitignore +++ b/.gitignore @@ -65,3 +65,4 @@ __screenshots__/ # NS8 Robot Framework test output (output.xml, log.html, report.html, screenshots) packaging/ns8/tests/outputs/ +__pycache__/ diff --git a/packaging/ns8/assert-multiarc.py b/packaging/ns8/assert-multiarc.py new file mode 100644 index 00000000..e0394142 --- /dev/null +++ b/packaging/ns8/assert-multiarc.py @@ -0,0 +1,201 @@ +#!/usr/bin/env python3 +"""Assert that published NS8 module tags are genuinely multi-arch and coherent. + +This is the release gate. It exists because a release once published an +amd64-only image as `:latest`, which installs on amd64 and hard-fails on arm64, +and nothing noticed: the publish step succeeded, and no check looked at what had +actually landed in the registry. + +Run it after `push-manifest.sh`. Usage: + + assert-multiarc.py [...] + +Exit status is 0 only if every tag passes every check for every required +architecture. + +For each required architecture this verifies: + + 1. the tag resolves to an image index / manifest list, not a single manifest. + A single manifest means an arm64 node cannot pull the tag at all. + 2. the index contains that architecture. + 3. that architecture's image config really declares that architecture, so an + entry cannot be mislabelled inside the list. + 4. that config carries an `org.nethserver.images` label. NS8 reads it at + install time to learn which app image to run, so a missing label produces a + module that installs and then does nothing. + 5. the app image the label names is itself multi-arch and contains that same + architecture, so the module cannot point at an app image its own node + cannot run. + +Defaults assume the public repository; override with MODULE_REPO / +REQUIRED_ARCHES. +""" +import json +import os +import sys +import time +import urllib.error +import urllib.request + +REGISTRY = "https://ghcr.io" +INDEX_TYPES = { + "application/vnd.oci.image.index.v1+json", + "application/vnd.docker.distribution.manifest.list.v2+json", +} +ACCEPT = ", ".join(INDEX_TYPES | { + "application/vnd.oci.image.manifest.v1+json", + "application/vnd.docker.distribution.manifest.v2+json", +}) + +# A push and an immediate read can race on the registry side. Retry briefly so a +# release is not failed by propagation latency, but fail rather than wait long +# enough to hide a real problem. +ATTEMPTS = 4 +RETRY_SECONDS = 10 + +_module_repo = os.environ.get("MODULE_REPO", "calnode/ns8-calnode") +_required = os.environ.get("REQUIRED_ARCHES", "amd64 arm64").split() +_tokens = {} + + +class Fail(Exception): + """A check failed in a way worth reporting verbatim.""" + + +def token(repo): + if repo not in _tokens: + url = f"{REGISTRY}/token?scope=repository:{repo}:pull&service=ghcr.io" + with urllib.request.urlopen(url, timeout=30) as f: + _tokens[repo] = json.load(f)["token"] + return _tokens[repo] + + +def get(repo, reference): + """GET a manifest, returning (body, mediaType).""" + req = urllib.request.Request( + f"{REGISTRY}/v2/{repo}/manifests/{reference}", + headers={"Authorization": f"Bearer {token(repo)}", "Accept": ACCEPT}, + ) + with urllib.request.urlopen(req, timeout=30) as f: + return json.load(f), (f.headers.get("Content-Type") or "").split(";")[0].strip() + + +def get_blob(repo, digest): + req = urllib.request.Request( + f"{REGISTRY}/v2/{repo}/blobs/{digest}", + headers={"Authorization": f"Bearer {token(repo)}"}, + ) + with urllib.request.urlopen(req, timeout=30) as f: + return json.load(f) + + +def split_ref(ref): + """ghcr.io/calnode/calnode:0.11.1 -> ('calnode/calnode', '0.11.1')""" + body = ref.split("/", 1)[1] if ref.count("/") > 1 else ref + name, _, tag = body.rpartition(":") + return (name or body), tag + + +def arches(body): + return { + (m.get("platform", {}).get("os"), m.get("platform", {}).get("architecture")) + for m in body.get("manifests", []) + if m.get("platform", {}).get("architecture") not in (None, "unknown") + } + + +def fetch_with_retry(repo, reference, what): + """Tolerate a short propagation delay after a push, then give up loudly.""" + last = None + for attempt in range(1, ATTEMPTS + 1): + try: + return get(repo, reference) + except urllib.error.HTTPError as exc: + last = exc + if attempt < ATTEMPTS: + print(f" retrying {what} (HTTP {exc.code}, attempt {attempt}/{ATTEMPTS})") + time.sleep(RETRY_SECONDS) + raise Fail(f"cannot fetch {what}: HTTP {last.code}") + + +def check_tag(repo, tag, arch_names): + print(f"\n=== {repo}:{tag} (require {arch_names})") + body, ctype = fetch_with_retry(repo, tag, f"{repo}:{tag}") + mtype = body.get("mediaType") or ctype + + if mtype not in INDEX_TYPES: + raise Fail( + f"tag is a single-arch manifest ({mtype}), not a multi-arch index; " + "an arm64 node cannot pull it" + ) + have = arches(body) + print(f" ok image index, arches present: {sorted(a for _, a in have)}") + + for arch in arch_names: + if ("linux", arch) not in have: + raise Fail(f"index is missing linux/{arch}") + + entry = next( + m for m in body["manifests"] + if m.get("platform", {}).get("os") == "linux" + and m.get("platform", {}).get("architecture") == arch + ) + sub, _ = get(repo, entry["digest"]) + cfg = get_blob(repo, sub["config"]["digest"]) + + if cfg.get("architecture") != arch: + raise Fail( + f"linux/{arch} entry carries a {cfg.get('architecture')} config, " + "so the entry is mislabelled" + ) + print(f" ok linux/{arch} config really declares {arch}") + + labels = (cfg.get("config") or {}).get("Labels") or {} + app_ref = labels.get("org.nethserver.images") + if not app_ref: + raise Fail(f"linux/{arch} has no org.nethserver.images label") + print(f" ok linux/{arch} org.nethserver.images = {app_ref}") + + app_repo, app_tag = split_ref(app_ref) + abody, actype = fetch_with_retry(app_repo, app_tag, f"app image {app_ref}") + amtype = abody.get("mediaType") or actype + if amtype not in INDEX_TYPES: + raise Fail(f"app image {app_ref} is single-arch ({amtype})") + if ("linux", arch) not in arches(abody): + raise Fail(f"app image {app_ref} does not contain linux/{arch}") + print(f" ok app image {app_ref} is multi-arch and contains linux/{arch}") + + +def main(argv): + repo = argv[1] if len(argv) > 1 else _module_repo + tags = argv[2:] + if not tags: + print(__doc__) + return 2 + + failures = [] + for tag in tags: + try: + check_tag(repo, tag, _required) + except Fail as exc: + print(f" FAIL {exc}") + failures.append(f"{repo}:{tag}: {exc}") + except Exception as exc: # noqa: BLE001 - report, never mask + print(f" ERROR {type(exc).__name__}: {exc}") + failures.append(f"{repo}:{tag}: {type(exc).__name__}: {exc}") + + print("\n" + "=" * 62) + for f in failures: + print(f" FAIL {f}") + if failures: + print( + f"\n{len(failures)} of {len(tags)} published tag(s) are not usable. " + "Refusing to call this release good." + ) + return 1 + print(f" PASS all {len(tags)} published tag(s) are multi-arch and coherent") + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) \ No newline at end of file