diff --git a/.github/workflows/ns8-module.yml b/.github/workflows/ns8-module.yml index cbaca127..e5c5d4a8 100644 --- a/.github/workflows/ns8-module.yml +++ b/.github/workflows/ns8-module.yml @@ -250,6 +250,13 @@ jobs: # secrets: the module repository is public and the token is anonymous. # Runs for branch pushes too, because :edge and :dev are published tags # and deserve exactly the same guarantee as a version tag. + # + # Deliberately module-only: it does NOT resolve the app image named by + # org.nethserver.images. This workflow and docker-publish.yml are triggered + # by the same tag push and run concurrently, so on a fresh release the app + # image is often not published yet and resolving it here races the app + # publish. That check lives in release-verify.yml, which runs once both + # publish workflows have completed. - name: Assert the published tags are multi-arch if: github.event_name != 'pull_request' working-directory: packaging/ns8 diff --git a/.github/workflows/release-verify.yml b/.github/workflows/release-verify.yml new file mode 100644 index 00000000..2ec6fe33 --- /dev/null +++ b/.github/workflows/release-verify.yml @@ -0,0 +1,129 @@ +name: Verify module release + +# The paired check that a module release cannot live without. +# +# ns8-module.yml and docker-publish.yml are both triggered by the same tag push +# and run CONCURRENTLY. That means the module workflow cannot verify that the app +# image exists: on a fresh release the app image is frequently not published yet, +# and checking there races the app publish. It is not a timing bug that a longer +# retry can fix. An app build can lag by however long it takes, and that grows, +# so a longer retry drifts toward never failing a genuinely absent app image, +# while a short one fails releases whose app image was merely slow. +# +# So the pair check lives here instead. This workflow is triggered by the +# COMPLETION of both publish workflows, and the job runs only when both of them +# concluded successfully for the same commit. There is no sleep, no retry budget +# and no polling loop deciding whether to wait: the two completion events are the +# dependency. Whichever workflow finishes last triggers the run that does the +# work, so a slow app publish simply means the job runs later, not that it fails. +# +# Consequence worth knowing: once both have succeeded, an app tag that is still +# absent is genuinely wrong rather than late, so assert-multiarc.py --pair fails +# on it promptly instead of waiting. + +on: + workflow_run: + workflows: ['NS8 module publish', 'Docker publish'] + types: [completed] + +permissions: + actions: read # read the sibling workflow's conclusion for this commit + contents: read # resolve which tag, if any, points at this commit + +concurrency: + group: verify-release-${{ github.event.workflow_run.head_sha }} + cancel-in-progress: false + +jobs: + verify: + name: Verify module and app tags agree + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@v7 + + - name: Wait for both publish workflows to finish + id: gate + env: + GH_TOKEN: ${{ github.token }} + SHA: ${{ github.event.workflow_run.head_sha }} + TRIGGERING: ${{ github.event.workflow_run.name }} + # Both publish workflows must have concluded for this commit. The one + # that triggered this run has already reported; the other may still be + # running, in which case its own completion will trigger us again. + SIBLING: ${{ github.event.workflow_run.name == 'NS8 module publish' && 'Docker publish' || 'NS8 module publish' }} + run: | + set -euo pipefail + echo "triggering workflow: $TRIGGERING (conclusion ${{ github.event.workflow_run.conclusion }})" + echo "waiting for sibling: $SIBLING" + + if [ "${{ github.event.workflow_run.conclusion }}" != "success" ]; then + echo "::notice::the triggering workflow did not succeed; nothing to verify." + echo "ready=false" >> "$GITHUB_OUTPUT" + echo "outcome=not-verified" >> "$GITHUB_OUTPUT" + exit 0 + fi + + status=$(gh api "repos/${{ github.repository }}/actions/workflows/$SIBLING/runs" \ + --jq "[.workflow_runs[] | select(.head_sha == \"$SHA\")] | first | .status // \"absent\"") + conclusion=$(gh api "repos/${{ github.repository }}/actions/workflows/$SIBLING/runs" \ + --jq "[.workflow_runs[] | select(.head_sha == \"$SHA\")] | first | .conclusion // \"absent\"") + + echo "sibling status=$status conclusion=$conclusion" + if [ "$status" != "completed" ]; then + echo "::notice::$SIBLING has not finished for this commit (status=$status)." \ + "Its completion will trigger this workflow again. Not a failure." + echo "ready=false" >> "$GITHUB_OUTPUT" + echo "outcome=waiting-for-sibling" >> "$GITHUB_OUTPUT" + elif [ "$conclusion" != "success" ]; then + echo "::error::$SIBLING concluded '$conclusion' for this commit, so this is not a clean release." + echo "ready=false" >> "$GITHUB_OUTPUT" + echo "outcome=blocked" >> "$GITHUB_OUTPUT" + else + echo "ready=true" >> "$GITHUB_OUTPUT" + echo "outcome=verify" >> "$GITHUB_OUTPUT" + fi + + - name: Resolve the released tag + if: steps.gate.outputs.ready == 'true' + id: tag + env: + GH_TOKEN: ${{ github.token }} + SHA: ${{ github.event.workflow_run.head_sha }} + run: | + set -euo pipefail + # workflow_run does not carry the ref, so find the tag that points at + # this commit. A plain branch push has no such tag and skips, which is + # how this stays out of the way of ordinary commits. + tag=$(gh api "repos/${{ github.repository }}/tags" --paginate \ + --jq "map(select(.commit.sha == \"$SHA\")) | map(.name) | map(select(test(\"^v[0-9]+\\\\.[0-9]+\\\\.[0-9]+$\"))) | first // empty" \ + | head -1 | tr -d '\r') + if [ -z "$tag" ]; then + echo "::notice::no release tag points at $SHA; not a tag release, skipping." + echo "found=false" >> "$GITHUB_OUTPUT" + else + echo "found=true" >> "$GITHUB_OUTPUT" + echo "name=$tag" >> "$GITHUB_OUTPUT" + echo "version=${tag#v}" >> "$GITHUB_OUTPUT" + fi + + - name: Assert the module and app tags form a matched multi-arch pair + if: steps.gate.outputs.ready == 'true' && steps.tag.outputs.found == 'true' + working-directory: packaging/ns8 + env: + MODULE_REPO: ghcr.io/${{ github.repository_owner }}/ns8-calnode + run: | + # Same version tag plus its major.minor alias, plus :latest. The module + # publishes exactly these three, so all three must resolve to the same + # release and carry the same app reference. + # + # No app repository is passed: the app image is taken from each + # module's org.nethserver.images label, which is the only authoritative + # source for what a module actually runs. + version="${{ steps.tag.outputs.version }}" + major_minor="${version%.*}" + python3 assert-multiarc.py \ + --pair \ + --expect-app-tag "$version" \ + "$MODULE_REPO" \ + "$version" "$major_minor" latest \ No newline at end of file diff --git a/packaging/ns8/assert-multiarc.py b/packaging/ns8/assert-multiarc.py index cfb507fa..533e7f80 100644 --- a/packaging/ns8/assert-multiarc.py +++ b/packaging/ns8/assert-multiarc.py @@ -1,40 +1,52 @@ #!/usr/bin/env python3 """Assert that published NS8 module tags are genuinely multi-arch and coherent. -This is the release gate. It exists because a release once published an -amd64-only image as `:latest`, which installs on amd64 and hard-fails on arm64, -and nothing noticed: the publish step succeeded, and no check looked at what had -actually landed in the registry. - -Run it after `push-manifest.sh`. Usage: - - assert-multiarc.py [...] - -Exit status is 0 only if every tag passes every check for every required -architecture. - -For each required architecture this verifies: - - 1. the tag resolves to an image index / manifest list, not a single manifest. - A single manifest means an arm64 node cannot pull the tag at all. - 2. the index contains that architecture. - 3. that architecture's image config really declares that architecture, so an - entry cannot be mislabelled inside the list. - 4. that config carries an `org.nethserver.images` label. NS8 reads it at - install time to learn which app image to run, so a missing label produces a - module that installs and then does nothing. - 5. the app image the label names is itself multi-arch and contains that same - architecture, so the module cannot point at an app image its own node - cannot run. - -Defaults assume the public repository; override with MODULE_REPO / -REQUIRED_ARCHES. +Two modes, because two things have very different lifetimes: + + module-only (default) + Checks the module image itself: that the tag is an image index, that every + required architecture is present, that each architecture's config really + declares it, and that it carries a well-formed `org.nethserver.images` + label. These are intrinsic properties of what the module workflow just + published, so this is safe to run there. + + --pair + Additionally resolves the app image named by each label and asserts it + exists, is multi-arch, and contains each architecture. This must NOT run + in the module workflow: `ns8-module.yml` and `docker-publish.yml` are + triggered by the same tag push and run CONCURRENTLY, so on a fresh release + the app image does not exist yet and the check races the app publish. It + belongs in release-verify.yml, which runs after both have completed. + +Usage: + assert-multiarc.py [--pair] [--expect-app-tag TAG] [...] + + --pair resolve and check the app image named by each label + --expect-app-tag T assert every label names exactly this app tag, so the + module release and the app release are the same version + +There is deliberately NO option to override the app repository. The app image a +module runs is whatever its org.nethserver.images label names, so the label is +the only authoritative source and it is also what a fork needs: a module built +under someone else's namespace names their app image, and deriving from the +label handles that with no configuration. An override here would be worse than +useless, because it would let a pair check be asserted against an app repository +the module does not actually reference, hiding a mislabelled module. + +Exit status is 0 only if every tag passes every applicable check. + +Why the distinction between a slow app image and a missing one is not a guess: +in --pair mode both publish workflows have already finished, so an app tag that +is still absent is genuinely wrong, not merely late. The retry here only absorbs +registry read-after-write propagation, it is bounded, and it then fails. """ +import argparse import json import os import sys import time import urllib.error +import urllib.parse import urllib.request REGISTRY = "https://ghcr.io" @@ -47,14 +59,12 @@ "application/vnd.docker.distribution.manifest.v2+json", }) -# A push and an immediate read can race on the registry side. Retry briefly so a -# release is not failed by propagation latency, but fail rather than wait long -# enough to hide a real problem. +# Read-after-write propagation only. Deliberately short: in --pair mode the app +# publish has already finished, so anything still missing after this is a real +# fault rather than a race, and must fail rather than wait. ATTEMPTS = 4 RETRY_SECONDS = 10 -_module_repo = os.environ.get("MODULE_REPO", "calnode/ns8-calnode") -_required = os.environ.get("REQUIRED_ARCHES", "amd64 arm64").split() _tokens = {} @@ -110,7 +120,7 @@ def arches(body): def fetch_with_retry(repo, reference, what): - """Tolerate a short propagation delay after a push, then give up loudly.""" + """Absorb registry propagation, then fail loudly and quickly.""" last = None for attempt in range(1, ATTEMPTS + 1): try: @@ -123,7 +133,8 @@ def fetch_with_retry(repo, reference, what): raise Fail(f"cannot fetch {what}: HTTP {last.code}") -def check_tag(repo, tag, arch_names): +def app_refs_for(repo, tag, arch_names): + """Check the module image and return the app refs its labels name.""" print(f"\n=== {repo}:{tag} (require {arch_names})") body, ctype = fetch_with_retry(repo, tag, f"{repo}:{tag}") mtype = body.get("mediaType") or ctype @@ -136,6 +147,7 @@ def check_tag(repo, tag, arch_names): have = arches(body) print(f" ok image index, arches present: {sorted(a for _, a in have)}") + found = {} for arch in arch_names: if ("linux", arch) not in have: raise Fail(f"index is missing linux/{arch}") @@ -159,48 +171,88 @@ def check_tag(repo, tag, arch_names): app_ref = labels.get("org.nethserver.images") if not app_ref: raise Fail(f"linux/{arch} has no org.nethserver.images label") + app_repo, app_tag = split_ref(app_ref) + if not app_tag: + raise Fail(f"linux/{arch} label org.nethserver.images={app_ref} has no tag") print(f" ok linux/{arch} org.nethserver.images = {app_ref}") + found[arch] = (app_repo, app_tag) - app_repo, app_tag = split_ref(app_ref) - abody, actype = fetch_with_retry(app_repo, app_tag, f"app image {app_ref}") - amtype = abody.get("mediaType") or actype - if amtype not in INDEX_TYPES: - raise Fail(f"app image {app_ref} is single-arch ({amtype})") + # Every architecture must agree on the app image, or different nodes would + # get different applications from the same module tag. + distinct = set(found.values()) + if len(distinct) > 1: + raise Fail(f"architectures disagree on the app image: {sorted(distinct)}") + return found + + +def check_app_image(app_repo, app_tag, arch_names, what): + abody, actype = fetch_with_retry(app_repo, app_tag, what) + amtype = abody.get("mediaType") or actype + if amtype not in INDEX_TYPES: + raise Fail(f"{what} is single-arch ({amtype}); an arm64 node cannot run it") + for arch in arch_names: if ("linux", arch) not in arches(abody): - raise Fail(f"app image {app_ref} does not contain linux/{arch}") - print(f" ok app image {app_ref} is multi-arch and contains linux/{arch}") + raise Fail(f"{what} does not contain linux/{arch}") + print(f" ok {what} is multi-arch and contains {sorted(arch_names)}") -def main(argv): - repo = argv[1] if len(argv) > 1 else _module_repo - tags = argv[2:] - if not tags: - print(__doc__) - return 2 +def check_tag(repo, tag, arch_names, pair, expect_app_tag): + found = app_refs_for(repo, tag, arch_names) + if not pair: + print(" -- not checking the app image (module-only mode)") + return + for app_repo, app_tag in sorted(set(found.values())): + if expect_app_tag and app_tag != expect_app_tag: + raise Fail( + f"label points at app tag '{app_tag}' but this release publishes " + f"app tag '{expect_app_tag}'; the module and the app would be " + "different versions" + ) + check_app_image( + app_repo, + app_tag, + arch_names, + f"app image {app_repo}:{app_tag}", + ) + if expect_app_tag: + print(f" ok module {tag} and app {expect_app_tag} are the same release") + + +def main(argv=None): + p = argparse.ArgumentParser(add_help=True, description=__doc__) + p.add_argument("repo", help="module image repository, e.g. calnode/ns8-calnode") + p.add_argument("tags", nargs="+", help="published module tags to check") + p.add_argument("--pair", action="store_true", + help="also resolve the app image named by each label") + p.add_argument("--expect-app-tag", default=None, + help="assert every label names exactly this app tag") + p.add_argument("--required-arches", + default=os.environ.get("REQUIRED_ARCHES", "amd64 arm64")) + args = p.parse_args(argv) + + arch_names = args.required_arches.split() failures = [] - for tag in tags: + for tag in args.tags: try: - check_tag(repo, tag, _required) + check_tag(args.repo, tag, arch_names, args.pair, args.expect_app_tag) except Fail as exc: print(f" FAIL {exc}") - failures.append(f"{repo}:{tag}: {exc}") + failures.append(f"{args.repo}:{tag}: {exc}") except Exception as exc: # noqa: BLE001 - report, never mask print(f" ERROR {type(exc).__name__}: {exc}") - failures.append(f"{repo}:{tag}: {type(exc).__name__}: {exc}") + failures.append(f"{args.repo}:{tag}: {type(exc).__name__}: {exc}") print("\n" + "=" * 62) for f in failures: print(f" FAIL {f}") if failures: - print( - f"\n{len(failures)} of {len(tags)} published tag(s) are not usable. " - "Refusing to call this release good." - ) + print(f"\n{len(failures)} of {len(args.tags)} tag(s) are not usable.") return 1 - print(f" PASS all {len(tags)} published tag(s) are multi-arch and coherent") + mode = "multi-arch and coherent" if args.pair else "multi-arch with coherent labels" + print(f" PASS all {len(args.tags)} tag(s) are {mode}") return 0 if __name__ == "__main__": - sys.exit(main(sys.argv)) \ No newline at end of file + sys.exit(main()) \ No newline at end of file