diff --git a/.github/workflows/ns8-module.yml b/.github/workflows/ns8-module.yml index 5e83009f..f2436c9a 100644 --- a/.github/workflows/ns8-module.yml +++ b/.github/workflows/ns8-module.yml @@ -70,7 +70,7 @@ jobs: # surfaced on a release push, so exercise it here instead. # push-manifest.sh refuses to publish a list that is missing an arch, so a # clean exit is the assertion. - - name: Selftest the multi-arch manifest assembly + - name: Selftest the multi-arch manifest assembly (hermetic) run: | set -euo pipefail tmp=$(mktemp -d) @@ -78,9 +78,61 @@ jobs: buildah commit ocr-selftest-amd64 ocr-selftest/ns8-calnode:build-amd64 >/dev/null buildah from --name ocr-selftest-arm64 --arch arm64 scratch >/dev/null buildah commit ocr-selftest-arm64 ocr-selftest/ns8-calnode:build-arm64 >/dev/null - REPOBASE=ocr-selftest TARGETARCHES="amd64 arm64" IMAGETAGS=selftest \ - PUSH_DESTINATION="oci:${tmp}/out" bash packaging/ns8/push-manifest.sh - echo "manifest selftest OK" + # SOURCE_PREFIX is empty so this reads from local containers-storage, + # which is the only way to exercise the script with no registry. + REPOBASE=ocr-selftest SOURCE_PREFIX="" TARGETARCHES="amd64 arm64" \ + IMAGETAGS=selftest PUSH_DESTINATION="oci:${tmp}/out" \ + bash packaging/ns8/push-manifest.sh + echo "hermetic manifest selftest OK" + + # The hermetic test above reads local storage, so it cannot catch the + # failure that actually broke the v0.11.0 release: buildah running with + # two graph roots merged into one store, where the local image record + # existed but its manifest blob did not, and + # buildah manifest add + # failed with "reading manifest for instance ...: file does not exist". + # + # A release now reads each architecture from the registry instead, so this + # test does too: it assembles from the real published :build- tags + # and pushes the result to a local oci: path, so it needs no credentials + # and publishes nothing. If these tags are missing, that is itself the + # signal worth failing on. + - name: Selftest the manifest assembly against the registry + run: | + set -euo pipefail + tmp=$(mktemp -d) + REPOBASE=ghcr.io/calnode TARGETARCHES="amd64 arm64" \ + IMAGETAGS=selftest-registry PUSH_DESTINATION="oci:${tmp}/out" \ + bash packaging/ns8/push-manifest.sh + # Prove the assembled list really carries both arches rather than + # trusting the exit code alone. buildah writes a plain OCI layout, + # but it stores the assembled list as a single entry in index.json + # whose own mediaType is an image index, with the per-arch manifests + # inside that referenced blob. So look there when the top level holds + # no platform entries of its own. + count=$(jq '[.manifests[]? + | select(.platform.architecture != null + and .platform.architecture != "unknown")] + | length' "${tmp}/out/index.json") + if [ "$count" -eq 0 ]; then + top=$(jq -r '.manifests[0].digest' "${tmp}/out/index.json") + inner="${tmp}/out/blobs/sha256/${top#sha256:}" + if [ ! -f "$inner" ]; then + echo "::error::no index entry with platforms and no referenced blob at $inner" + cat "${tmp}/out/index.json" + exit 1 + fi + count=$(jq '[.manifests[]? + | select(.platform.architecture != null + and .platform.architecture != "unknown")] + | length' "$inner") + fi + if [ "$count" -ne 2 ]; then + echo "::error::assembled index holds ${count} arches, expected 2" + cat "${tmp}/out/index.json" + exit 1 + fi + echo "registry-backed manifest selftest OK (${count} arches)" build: needs: validate diff --git a/packaging/ns8/push-manifest.sh b/packaging/ns8/push-manifest.sh index b8d3dfd7..7c5ae8ea 100755 --- a/packaging/ns8/push-manifest.sh +++ b/packaging/ns8/push-manifest.sh @@ -18,6 +18,11 @@ # REPOBASE module image namespace, default ghcr.io/calnode # TARGETARCHES space-separated arches to assemble, required # IMAGETAGS space-separated tags to publish, default "latest" +# SOURCE_PREFIX transport prefix used when reading each per-arch image, +# default "docker://". Set it to the empty string to read from +# local containers-storage instead, which only the hermetic +# self-test wants. Do not do that for a real release: see the +# note on the manifest add below. # PUSH_DESTINATION push target override, default docker://. Set to an # oci: path to exercise the whole script with no registry, # which is what the CI selftest does. @@ -32,31 +37,52 @@ repobase="${repobase,,}" reponame="ns8-calnode" read -r -a arches <<< "${TARGETARCHES:-}" read -r -a imagetags <<< "${IMAGETAGS:-latest}" +# ${x-default}, not ${x:-default}: an empty SOURCE_PREFIX is a deliberate +# request to read locally, and ${x:-} would silently overwrite it with the +# default and defeat the hermetic self-test. +source_prefix="${SOURCE_PREFIX-docker://}" if [ "${#arches[@]}" -eq 0 ]; then echo "TARGETARCHES is required, e.g. TARGETARCHES=\"amd64 arm64\"" >&2 exit 1 fi -for arch in "${arches[@]}"; do - # `images -q` prints the image ID when the tag is present and prints - # nothing when it is absent. There is no singular "image exists" form. - if [ -z "$(buildah images -q "${repobase}/${reponame}:build-${arch}")" ]; then - echo "Missing local image for ${arch}. Run build-images.sh with" \ - "TARGETARCH=${arch} PUSH=1 first." >&2 - exit 1 - fi -done - for imagetag in "${imagetags[@]}"; do manifest="${repobase}/${reponame}:${imagetag}" # Drop any stale manifest so a retry cannot inherit old entries. buildah manifest rm "${manifest}" 2>/dev/null || true buildah manifest create "${manifest}" for arch in "${arches[@]}"; do - # Exactly two arguments: buildah 5.x takes only the list and the - # image, with no instance-name argument. - buildah manifest add "${manifest}" "${repobase}/${reponame}:build-${arch}" + # Read each architecture from the REGISTRY, not from local + # containers-storage. Reading locally is what broke the v0.11.0 release: + # buildah(1) there had two graph roots merged into a single store, so the + # local image record existed while the manifest blob it pointed at did + # not, and the add failed with + # + # reading manifest for instance "sha256:...": file does not exist + # + # build-images.sh has already pushed the per-arch tags by the time this + # runs, so docker:// is both the documented form (the example in + # buildah-manifest-add(1) is `buildah manifest add mylist:v1.11 + # docker://fedora`) and the source of truth for what actually ships. + # + # It also removes the need for the old `buildah images -q` pre-check, + # which could not answer the question reliably: it inspects the local + # store, so it happily reported a present image whose manifest was + # missing, and would equally have reported a published image as absent. + # A failed add is now the check, and it is authoritative. + # + # Exactly two arguments: buildah 5.x takes only the list and the image, + # with no instance-name argument. + if ! buildah manifest add "${manifest}" \ + "${source_prefix}${repobase}/${reponame}:build-${arch}"; then + echo "Failed to add ${source_prefix}${repobase}/${reponame}:build-${arch}" \ + "to ${manifest}." >&2 + echo "If that tag does not exist in the registry, run build-images.sh" \ + "with TARGETARCH=${arch} PUSH=1 first." >&2 + buildah manifest rm "${manifest}" 2>/dev/null || true + exit 1 + fi done # Refuse to publish a partial manifest. If any arch silently failed to # join the list, an arm64 node would get an image it cannot run.