From 11e5320e60b48e1f8befd8d0b848ec885264cf39 Mon Sep 17 00:00:00 2001 From: Wynne Pirini Date: Fri, 16 Oct 2026 12:35:01 +1300 Subject: [PATCH 1/3] packaging(ns8): read manifest arches from the registry, not local storage This is the failure that stopped the v0.11.0 module release. buildah manifest add was given a plain local reference, so it read each architecture out of local containers-storage. On the release runner buildah had two graph roots merged into a single store, and the local image record existed while the manifest blob it pointed at did not: Error: initializing source containers-storage:[overlay@/home/runner/.local/ share/containers/storage+/run/user/1001/containers]...: reading manifest for instance "sha256:43c2abfb...": ... file does not exist The old pre-check missed it. It asked `buildah images -q`, which inspects the same local store, so it answered "present" for an image whose manifest could not be read, and the script failed later and far more cryptically. That guard is gone: a failed add is now the check, and it is authoritative. The fix is to read each arch from the registry, where build-images.sh has already pushed it. This is the documented form: the example in buildah-manifest-add(1) is `buildah manifest add mylist:v1.11 docker://fedora`. It also means the existing refusal to publish a partial list now inspects the content that will actually ship. SOURCE_PREFIX defaults to docker:// and is only overridden by the hermetic self-test. It uses ${x-default} rather than ${x:-default} on purpose: an empty value is a deliberate request to read locally, and ${x:-} would silently replace it. The CI self-test previously covered only the local-read path, which is exactly why this shipped broken. It now runs twice: the existing hermetic test, and a new registry-backed test that assembles from the real published :build- tags and pushes to a local oci: path, so it needs no credentials, publishes nothing, and asserts the assembled index.json really holds two arches. --- .github/workflows/ns8-module.yml | 44 ++++++++++++++++++++++++--- packaging/ns8/push-manifest.sh | 52 ++++++++++++++++++++++++-------- 2 files changed, 79 insertions(+), 17 deletions(-) diff --git a/.github/workflows/ns8-module.yml b/.github/workflows/ns8-module.yml index 5e83009f..f2451ea4 100644 --- a/.github/workflows/ns8-module.yml +++ b/.github/workflows/ns8-module.yml @@ -70,7 +70,7 @@ jobs: # surfaced on a release push, so exercise it here instead. # push-manifest.sh refuses to publish a list that is missing an arch, so a # clean exit is the assertion. - - name: Selftest the multi-arch manifest assembly + - name: Selftest the multi-arch manifest assembly (hermetic) run: | set -euo pipefail tmp=$(mktemp -d) @@ -78,9 +78,45 @@ jobs: buildah commit ocr-selftest-amd64 ocr-selftest/ns8-calnode:build-amd64 >/dev/null buildah from --name ocr-selftest-arm64 --arch arm64 scratch >/dev/null buildah commit ocr-selftest-arm64 ocr-selftest/ns8-calnode:build-arm64 >/dev/null - REPOBASE=ocr-selftest TARGETARCHES="amd64 arm64" IMAGETAGS=selftest \ - PUSH_DESTINATION="oci:${tmp}/out" bash packaging/ns8/push-manifest.sh - echo "manifest selftest OK" + # SOURCE_PREFIX is empty so this reads from local containers-storage, + # which is the only way to exercise the script with no registry. + REPOBASE=ocr-selftest SOURCE_PREFIX="" TARGETARCHES="amd64 arm64" \ + IMAGETAGS=selftest PUSH_DESTINATION="oci:${tmp}/out" \ + bash packaging/ns8/push-manifest.sh + echo "hermetic manifest selftest OK" + + # The hermetic test above reads local storage, so it cannot catch the + # failure that actually broke the v0.11.0 release: buildah running with + # two graph roots merged into one store, where the local image record + # existed but its manifest blob did not, and + # buildah manifest add + # failed with "reading manifest for instance ...: file does not exist". + # + # A release now reads each architecture from the registry instead, so this + # test does too: it assembles from the real published :build- tags + # and pushes the result to a local oci: path, so it needs no credentials + # and publishes nothing. If these tags are missing, that is itself the + # signal worth failing on. + - name: Selftest the manifest assembly against the registry + run: | + set -euo pipefail + tmp=$(mktemp -d) + REPOBASE=ghcr.io/calnode TARGETARCHES="amd64 arm64" \ + IMAGETAGS=selftest-registry PUSH_DESTINATION="oci:${tmp}/out" \ + bash packaging/ns8/push-manifest.sh + # Prove the assembled list really carries both arches rather than + # trusting the exit code alone. buildah writes a plain OCI layout, so + # read its index.json directly. + count=$(jq '[.manifests[] + | select(.platform.architecture != null + and .platform.architecture != "unknown")] + | length' "${tmp}/out/index.json") + if [ "$count" -ne 2 ]; then + echo "::error::assembled index holds ${count} arches, expected 2" + cat "${tmp}/out/index.json" + exit 1 + fi + echo "registry-backed manifest selftest OK (${count} arches)" build: needs: validate diff --git a/packaging/ns8/push-manifest.sh b/packaging/ns8/push-manifest.sh index b8d3dfd7..7b2e1b14 100755 --- a/packaging/ns8/push-manifest.sh +++ b/packaging/ns8/push-manifest.sh @@ -18,6 +18,11 @@ # REPOBASE module image namespace, default ghcr.io/calnode # TARGETARCHES space-separated arches to assemble, required # IMAGETAGS space-separated tags to publish, default "latest" +# SOURCE_PREFIX transport prefix used when reading each per-arch image, +# default "docker://". Set it to the empty string to read from +# local containers-storage instead, which only the hermetic +# self-test wants. Do not do that for a real release: see the +# note on the manifest add below. # PUSH_DESTINATION push target override, default docker://. Set to an # oci: path to exercise the whole script with no registry, # which is what the CI selftest does. @@ -32,31 +37,52 @@ repobase="${repobase,,}" reponame="ns8-calnode" read -r -a arches <<< "${TARGETARCHES:-}" read -r -a imagetags <<< "${IMAGETAGS:-latest}" +# ${x-default}, not ${x:-default}: an empty SOURCE_PREFIX is a deliberate +# request to read locally, and ${x:-} would silently overwrite it with the +# default and defeat the hermetic self-test. +source_prefix="${SOURCE_PREFIX-docker://}" if [ "${#arches[@]}" -eq 0 ]; then echo "TARGETARCHES is required, e.g. TARGETARCHES=\"amd64 arm64\"" >&2 exit 1 fi -for arch in "${arches[@]}"; do - # `images -q` prints the image ID when the tag is present and prints - # nothing when it is absent. There is no singular "image exists" form. - if [ -z "$(buildah images -q "${repobase}/${reponame}:build-${arch}")" ]; then - echo "Missing local image for ${arch}. Run build-images.sh with" \ - "TARGETARCH=${arch} PUSH=1 first." >&2 - exit 1 - fi -done - for imagetag in "${imagetags[@]}"; do manifest="${repobase}/${reponame}:${imagetag}" # Drop any stale manifest so a retry cannot inherit old entries. buildah manifest rm "${manifest}" 2>/dev/null || true buildah manifest create "${manifest}" for arch in "${arches[@]}"; do - # Exactly two arguments: buildah 5.x takes only the list and the - # image, with no instance-name argument. - buildah manifest add "${manifest}" "${repobase}/${reponame}:build-${arch}" + # Read each architecture from the REGISTRY, not from local + # containers-storage. Reading locally is what broke the v0.11.0 release: + # buildah ran with two graph roots merged into a single store, so the + # local image record existed while the manifest blob it pointed at did + # not, and the add failed with + # + # reading manifest for instance "sha256:...": file does not exist + # + # build-images.sh has already pushed the per-arch tags by the time this + # runs, so docker:// is both the documented form (the example in + # buildah-manifest-add(1) is `buildah manifest add mylist:v1.11 + # docker://fedora`) and the source of truth for what actually ships. + # + # It also removes the need for the old `buildah images -q` pre-check, + # which could not answer the question reliably: it inspects the local + # store, so it happily reported a present image whose manifest was + # missing, and would equally have reported a published image as absent. + # A failed add is now the check, and it is authoritative. + # + # Exactly two arguments: buildah 5.x takes only the list and the image, + # with no instance-name argument. + if ! buildah manifest add "${manifest}" \ + "${source_prefix}${repobase}/${reponame}:build-${arch}"; then + echo "Failed to add ${source_prefix}${repobase}/${reponame}:build-${arch}" \ + "to ${manifest}." >&2 + echo "If that tag does not exist in the registry, run build-images.sh" \ + "with TARGETARCH=${arch} PUSH=1 first." >&2 + buildah manifest rm "${manifest}" 2>/dev/null || true + exit 1 + fi done # Refuse to publish a partial manifest. If any arch silently failed to # join the list, an arm64 node would get an image it cannot run. From 691682f28f105a704934770d686f1aa23d088f69 Mon Sep 17 00:00:00 2001 From: Wynne Pirini Date: Fri, 16 Oct 2026 12:49:21 +1300 Subject: [PATCH 2/3] packaging(ns8): keep a prose word out of the buildah subcommand scan The validate job greps the build scripts for 'buildah [a-z]+' and asserts each extracted word is a real subcommand. My explanatory comment said 'buildah ran with two graph roots...', so the scanner read 'ran' as a subcommand and failed the build. Reworded to 'buildah(1) there had...'. The scanner also matches inside comments generally, so any future sentence phrasing a buildah command in prose can trip it. Not changing that here: it is pre-existing behaviour, and it is exactly the check that caught this. --- packaging/ns8/push-manifest.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packaging/ns8/push-manifest.sh b/packaging/ns8/push-manifest.sh index 7b2e1b14..7c5ae8ea 100755 --- a/packaging/ns8/push-manifest.sh +++ b/packaging/ns8/push-manifest.sh @@ -55,7 +55,7 @@ for imagetag in "${imagetags[@]}"; do for arch in "${arches[@]}"; do # Read each architecture from the REGISTRY, not from local # containers-storage. Reading locally is what broke the v0.11.0 release: - # buildah ran with two graph roots merged into a single store, so the + # buildah(1) there had two graph roots merged into a single store, so the # local image record existed while the manifest blob it pointed at did # not, and the add failed with # From 0ccb85049b13828fbd8f60fa8c1045d4858aa6c3 Mon Sep 17 00:00:00 2001 From: Wynne Pirini Date: Fri, 16 Oct 2026 13:02:30 +1300 Subject: [PATCH 3/3] ci(ns8): count arches inside the referenced index blob The registry-backed selftest failed on its own assertion, not on the assembly: the run logged 'Published ... (amd64 arm64)' and wrote a correct two-arch list. buildah stores that list as a single entry in the OCI layout's index.json, and the per-arch manifests live inside the index that entry references, so the top level holds no platform entries to count. Follow the reference when the top level is empty, and fail loudly if there is neither a top-level platform set nor a readable blob. Validated locally against a real OCI layout built from the published :edge index. --- .github/workflows/ns8-module.yml | 26 +++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ns8-module.yml b/.github/workflows/ns8-module.yml index f2451ea4..f2436c9a 100644 --- a/.github/workflows/ns8-module.yml +++ b/.github/workflows/ns8-module.yml @@ -105,12 +105,28 @@ jobs: IMAGETAGS=selftest-registry PUSH_DESTINATION="oci:${tmp}/out" \ bash packaging/ns8/push-manifest.sh # Prove the assembled list really carries both arches rather than - # trusting the exit code alone. buildah writes a plain OCI layout, so - # read its index.json directly. - count=$(jq '[.manifests[] - | select(.platform.architecture != null - and .platform.architecture != "unknown")] + # trusting the exit code alone. buildah writes a plain OCI layout, + # but it stores the assembled list as a single entry in index.json + # whose own mediaType is an image index, with the per-arch manifests + # inside that referenced blob. So look there when the top level holds + # no platform entries of its own. + count=$(jq '[.manifests[]? + | select(.platform.architecture != null + and .platform.architecture != "unknown")] | length' "${tmp}/out/index.json") + if [ "$count" -eq 0 ]; then + top=$(jq -r '.manifests[0].digest' "${tmp}/out/index.json") + inner="${tmp}/out/blobs/sha256/${top#sha256:}" + if [ ! -f "$inner" ]; then + echo "::error::no index entry with platforms and no referenced blob at $inner" + cat "${tmp}/out/index.json" + exit 1 + fi + count=$(jq '[.manifests[]? + | select(.platform.architecture != null + and .platform.architecture != "unknown")] + | length' "$inner") + fi if [ "$count" -ne 2 ]; then echo "::error::assembled index holds ${count} arches, expected 2" cat "${tmp}/out/index.json"