diff --git a/.gitignore b/.gitignore index 853f259..d7a79f7 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,4 @@ tests/test_helper/bats-assert/ # Test artifacts headers.txt +put-headers.txt diff --git a/AGENTS.md b/AGENTS.md index 6cffdf4..b7e07e1 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -33,7 +33,7 @@ Step flow in `action.yml`: 2. `resolve-build-config.sh` / `verify-architecture.sh` — platforms, cache scope and outputs, incl. multi-arch handling. 3. `login-registries.sh` → `docker/build-push-action`. 4. Multi-arch (`multiarch-mode`): `build` jobs push by digest and upload digest artifacts; one `merge` job runs `merge-manifests.sh` and then does the GitOps/Upwind steps. `build` mode skips all GitOps steps. -5. `retag-image.sh` (release/custom tag path) — polls the primary registry's v2 manifest API for the `main-`/`master-` image, retags it, then replicates to additional registries with `docker buildx imagetools create`. +5. `retag-image.sh` (release/custom tag path) — polls the primary registry's v2 manifest API for the `main-`/`master-` image (or exactly the `docker-retag-digest` manifest when set), retags it, then replicates to additional registries with `docker buildx imagetools create`. 6. `update-gitops.sh` — checks out the GitOps repo and edits YAML with `yq`. Environment by ref: `main`/`master` → `gitops-stage`, `dev` → `gitops-dev`, tags → `gitops-prod`; any other ref simulates the dev update without committing. Each input line is ` `; a path ending in `.tag` or pointing at a map with `tag`/`repository` gets only the tag, otherwise the full image reference. Also stamps `/…` annotations. Commit + push retries with backoff. Shared libs in `scripts/lib/` (sourced, never executed): diff --git a/README.md b/README.md index 99463f1..bffa659 100644 --- a/README.md +++ b/README.md @@ -279,6 +279,7 @@ Notes: | `multiarch-digests-path` | Directory holding the per-architecture digest files | `/tmp/gitops-action-digests` | | `docker-build-provenance` | Generate [provenance](https://docs.docker.com/build/attestations/slsa-provenance/) attestation for the build | `false` | | `docker-build-no-cache` | Re-execute Dockerfile steps and skip external cache import/export. Base images are always checked for updates. | `false` | +| `docker-retag-digest` | Digest (`sha256:…`) to retag on a release or custom tag instead of whatever `main-`/`master-` points to at retag time, so a caller retags exactly the digest it verified | | | `docker-disable-retagging` | Disables retagging of existing images and run a new build instead | `false` | | `deployment-annotations` | Stamp deployment-tracking annotations (`deploy.staffbase.com/*`) onto updated GitOps manifests. See [Deployment tracking annotations](#deployment-tracking-annotations) | `true` | | `deployment-domain` | Key namespace for deployment-tracking metadata. Used verbatim for annotation keys (`/...`) and reversed to reverse-DNS for label keys (`com.staffbase.deploy.*`) | `deploy.staffbase.com` | diff --git a/action.yml b/action.yml index b1716dc..94195a3 100644 --- a/action.yml +++ b/action.yml @@ -74,6 +74,10 @@ inputs: description: 'Directory holding the per-architecture digest files.' required: false default: '/tmp/gitops-action-digests' + docker-retag-digest: + description: 'Digest (sha256:<64 hex>) of the image to retag on a release or custom tag. When set, exactly this manifest is retagged instead of whatever main-/master- points to at retag time, so a caller can retag the digest it verified. Optional; defaults to the tag lookup.' + required: false + default: '' docker-disable-retagging: description: 'Disable retagging of existing images' required: false @@ -281,6 +285,7 @@ runs: INPUT_DOCKER_IMAGE: ${{ inputs.docker-image }} INPUT_TAG: ${{ steps.preparation.outputs.tag }} INPUT_LATEST: ${{ steps.preparation.outputs.latest }} + INPUT_RETAG_DIGEST: ${{ inputs.docker-retag-digest }} run: ${{ github.action_path }}/scripts/retag-image.sh - name: Checkout GitOps Repository diff --git a/scripts/retag-image.sh b/scripts/retag-image.sh index d4a153a..e82cef9 100755 --- a/scripts/retag-image.sh +++ b/scripts/retag-image.sh @@ -11,6 +11,8 @@ # INPUT_DOCKER_REGISTRY_API, INPUT_DOCKER_IMAGE, INPUT_TAG, # INPUT_LATEST # Optional env vars: INPUT_DOCKER_USERNAME, INPUT_DOCKER_PASSWORD, +# INPUT_RETAG_DIGEST (sha256:<64 hex>; retag exactly this +# manifest instead of resolving the master-/main- tag), # RETAG_TIMEOUT_SECONDS (default: 300), # RETAG_POLL_INTERVAL (default: 10) # @@ -56,26 +58,58 @@ TIMEOUT="${RETAG_TIMEOUT_SECONDS:-300}" POLL_INTERVAL="${RETAG_POLL_INTERVAL:-10}" CHECK_EXISTING_TAGS="master-${GITHUB_SHA::8} main-${GITHUB_SHA::8}" +RETAG_DIGEST="${INPUT_RETAG_DIGEST:-}" +if [[ -n "$RETAG_DIGEST" ]]; then + if [[ ! "$RETAG_DIGEST" =~ ^sha256:[0-9a-f]{64}$ ]]; then + log_error "INPUT_RETAG_DIGEST must be sha256:<64 lowercase hex characters>" + exit 1 + fi + command -v sha256sum &>/dev/null || require_tool shasum + # A caller that verified a digest gets exactly that manifest; a tag could be moved after verification. + CHECK_EXISTING_TAGS="$RETAG_DIGEST" +fi ACCEPT_HEADER="application/vnd.docker.distribution.manifest.v2+json, application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.oci.image.manifest.v1+json, application/vnd.oci.image.index.v1+json" echo "CHECK_EXISTING_TAGS: ${CHECK_EXISTING_TAGS}" -echo "Check if an image already exists for ${INPUT_DOCKER_IMAGE}:main|master-${GITHUB_SHA::8}" +echo "Check if an image already exists for ${INPUT_DOCKER_IMAGE} (${CHECK_EXISTING_TAGS// /, })" + +# The manifest is kept in a file and uploaded with --data-binary so its bytes, +# and therefore its digest, stay exactly as fetched. +MANIFEST_FILE="$(mktemp)" +trap 'rm -f "$MANIFEST_FILE"' EXIT + +sha256_file() { + if command -v sha256sum &>/dev/null; then + sha256sum "$1" | cut -d' ' -f1 + else + shasum -a 256 "$1" | cut -d' ' -f1 + fi +} retag_manifest() { local target_tag="$1" - local manifest="$2" - local content_type="$3" + local content_type="$2" + local digest_args=() + [[ -z "$RETAG_DIGEST" ]] || digest_args=(-D put-headers.txt) curl --fail-with-body -X PUT \ -H "Content-Type: ${content_type}" \ "${AUTH_ARGS[@]}" \ - -d "${manifest}" \ + ${digest_args[@]+"${digest_args[@]}"} \ + --data-binary "@${MANIFEST_FILE}" \ "${INPUT_DOCKER_REGISTRY_API}${INPUT_DOCKER_IMAGE}/manifests/${target_tag}" + if [[ -n "$RETAG_DIGEST" ]]; then + local stored + stored=$(grep -i "^Docker-Content-Digest:" put-headers.txt | cut -d' ' -f2 | tr -d '\r' || true) + if [[ "$stored" != "$RETAG_DIGEST" ]]; then + log_error "${INPUT_DOCKER_IMAGE}:${target_tag} was stored as ${stored:-an unknown digest}, not ${RETAG_DIGEST}" + exit 1 + fi + fi } foundImage=false DETECTED_CONTENT_TYPE="" DIGEST="" -MANIFEST="" end=$((SECONDS + TIMEOUT)) attempt=1 @@ -84,12 +118,13 @@ while [ $SECONDS -lt $end ]; do echo "Poll attempt ${attempt} (${remaining}s remaining)..." for tag in $CHECK_EXISTING_TAGS; do - MANIFEST=$(curl -s -D headers.txt \ + : > "$MANIFEST_FILE" + curl -s -D headers.txt -o "$MANIFEST_FILE" \ -H "Accept: ${ACCEPT_HEADER}" \ "${AUTH_ARGS[@]}" \ - "${INPUT_DOCKER_REGISTRY_API}${INPUT_DOCKER_IMAGE}/manifests/${tag}") + "${INPUT_DOCKER_REGISTRY_API}${INPUT_DOCKER_IMAGE}/manifests/${tag}" - if [[ $MANIFEST == *"errors"* ]]; then + if grep -q "errors" "$MANIFEST_FILE"; then echo "No image found for ${INPUT_DOCKER_IMAGE}:${tag}" continue else @@ -97,6 +132,14 @@ while [ $SECONDS -lt $end ]; do foundImage=true DETECTED_CONTENT_TYPE=$(grep -i "^Content-Type:" headers.txt | cut -d' ' -f2 | tr -d '\r') DIGEST=$(grep -i "^Docker-Content-Digest:" headers.txt | cut -d' ' -f2 | tr -d '\r') + if [[ -n "$RETAG_DIGEST" && "$DIGEST" != "$RETAG_DIGEST" ]]; then + log_error "The registry returned ${DIGEST:-no digest} for ${INPUT_DOCKER_IMAGE}@${RETAG_DIGEST}" + exit 1 + fi + if [[ -n "$RETAG_DIGEST" && "sha256:$(sha256_file "$MANIFEST_FILE")" != "$RETAG_DIGEST" ]]; then + log_error "The manifest bytes fetched for ${INPUT_DOCKER_IMAGE}@${RETAG_DIGEST} do not match that digest" + exit 1 + fi break 2 fi done @@ -106,15 +149,15 @@ while [ $SECONDS -lt $end ]; do done if [[ $foundImage == false ]]; then - log_error "No image found for ${INPUT_DOCKER_IMAGE}:main|master-${GITHUB_SHA::8} within ${TIMEOUT} seconds" + log_error "No image found for ${INPUT_DOCKER_IMAGE} (${CHECK_EXISTING_TAGS// /, }) within ${TIMEOUT} seconds" exit 1 fi echo "Retagging image with release version and :latest tags for ${INPUT_DOCKER_IMAGE}" echo "Using Content-Type: ${DETECTED_CONTENT_TYPE}" -retag_manifest "$INPUT_TAG" "$MANIFEST" "$DETECTED_CONTENT_TYPE" -retag_manifest "$INPUT_LATEST" "$MANIFEST" "$DETECTED_CONTENT_TYPE" +retag_manifest "$INPUT_TAG" "$DETECTED_CONTENT_TYPE" +retag_manifest "$INPUT_LATEST" "$DETECTED_CONTENT_TYPE" if [[ ${#REGISTRIES[@]} -gt 1 ]]; then require_tool docker diff --git a/tests/retag-image.bats b/tests/retag-image.bats index 672b1c9..a00f150 100644 --- a/tests/retag-image.bats +++ b/tests/retag-image.bats @@ -33,6 +33,8 @@ create_curl_mock() { # Record call echo "curl \$*" >> "${TEST_TEMP_DIR}/curl_calls.log" +out_file=\$(echo "\$*" | sed -n 's/.*-o \([^ ]*\).*/\1/p') + # Handle different call patterns case "\$*" in *manifests/master-*|*manifests/main-*) @@ -45,9 +47,9 @@ Content-Type: application/vnd.docker.distribution.manifest.v2+json Docker-Content-Digest: sha256:abc123def456 HEADERS fi - echo '{"schemaVersion": 2}' + echo '{"schemaVersion": 2}' > "\$out_file" else - echo '{"errors": [{"code": "MANIFEST_UNKNOWN"}]}' + echo '{"errors": [{"code": "MANIFEST_UNKNOWN"}]}' > "\$out_file" fi ;; *"--fail-with-body"*"-X PUT"*) @@ -159,3 +161,144 @@ MOCK_EOF run test -f "${TEST_TEMP_DIR}/docker_calls.log" assert_failure } + +# Digest of the body create_digest_curl_mock serves for a manifest fetched by digest. +served_manifest_digest() { + echo "sha256:$(echo '{"schemaVersion": 2}' | shasum -a 256 | cut -d' ' -f1)" +} + +create_digest_curl_mock() { + local returned_digest="$1" + local stored_digest="${2:-$1}" + cat > "${TEST_TEMP_DIR}/mocks/curl" << MOCK_EOF +#!/usr/bin/env bash +echo "curl \$*" >> "${TEST_TEMP_DIR}/curl_calls.log" +out_file=\$(echo "\$*" | sed -n 's/.*-o \([^ ]*\).*/\1/p') +case "\$*" in + *"--fail-with-body"*"-X PUT"*) + headers_file=\$(echo "\$*" | sed 's/.*-D \([^ ]*\).*/\1/') + printf 'Docker-Content-Digest: %s\n' "$stored_digest" > "\$headers_file" + echo "PUT OK" + ;; + *manifests/sha256:*) + headers_file=\$(echo "\$*" | sed 's/.*-D \([^ ]*\).*/\1/') + printf 'Content-Type: application/vnd.oci.image.index.v1+json\nDocker-Content-Digest: %s\n' "$returned_digest" > "\$headers_file" + echo '{"schemaVersion": 2}' > "\$out_file" + ;; + *) + echo '{"errors": [{"code": "MANIFEST_UNKNOWN"}]}' > "\$out_file" + ;; +esac +MOCK_EOF + chmod +x "${TEST_TEMP_DIR}/mocks/curl" +} + +@test "retags exactly the given digest without resolving the main/master tag" { + digest="$(served_manifest_digest)" + export INPUT_RETAG_DIGEST="$digest" + create_digest_curl_mock "$digest" + + run "$SCRIPT" + assert_success + assert_output_value "digest" "$digest" + + run cat "${TEST_TEMP_DIR}/curl_calls.log" + assert_output --partial "my-service/manifests/${digest}" + refute_output --partial "manifests/main-" + refute_output --partial "manifests/master-" + assert_output --partial "my-service/manifests/1.0.0" + assert_output --partial "my-service/manifests/latest" +} + +@test "fails without retagging when the registry returns another digest" { + export INPUT_RETAG_DIGEST="sha256:$(printf 'a%.0s' {1..64})" + create_digest_curl_mock "sha256:$(printf 'b%.0s' {1..64})" + + run "$SCRIPT" + assert_failure + assert_output --partial "The registry returned sha256:bbbb" + + run cat "${TEST_TEMP_DIR}/curl_calls.log" + refute_output --partial "-X PUT" +} + +@test "rejects a malformed digest before any registry call" { + export INPUT_RETAG_DIGEST="main-abcdef12" + create_digest_curl_mock "unused" + + run "$SCRIPT" + assert_failure + assert_output --partial "INPUT_RETAG_DIGEST must be sha256" + [ ! -e "${TEST_TEMP_DIR}/curl_calls.log" ] +} + +@test "fails when the registry stores the retagged manifest under another digest" { + export INPUT_RETAG_DIGEST="$(served_manifest_digest)" + create_digest_curl_mock "$INPUT_RETAG_DIGEST" "sha256:$(printf 'c%.0s' {1..64})" + + run "$SCRIPT" + assert_failure + assert_output --partial "my-service:1.0.0 was stored as sha256:cccc" +} + +# A registry that, like a real one, addresses manifests by the sha256 of their +# exact bytes: GET serves a newline-terminated manifest and PUT stores whatever +# bytes it received under their own digest. +create_byte_registry_mock() { + local served_digest="${1:-}" + printf '{\n "schemaVersion": 2,\n "mediaType": "application/vnd.oci.image.index.v1+json",\n "manifests": []\n}\n' > "${TEST_TEMP_DIR}/manifest.json" + MANIFEST_DIGEST="sha256:$(shasum -a 256 "${TEST_TEMP_DIR}/manifest.json" | cut -d' ' -f1)" + cat > "${TEST_TEMP_DIR}/mocks/curl" << MOCK_EOF +#!/usr/bin/env bash +echo "curl \$*" >> "${TEST_TEMP_DIR}/curl_calls.log" +headers_file="" out_file="" data="" put=false +args=("\$@") +for ((i = 0; i < \${#args[@]}; i++)); do + case "\${args[i]}" in + -D) headers_file="\${args[i+1]}" ;; + -o) out_file="\${args[i+1]}" ;; + -d|--data-binary) data="\${args[i+1]}" ;; + PUT) put=true ;; + esac +done +if [[ "\$put" == true ]]; then + if [[ "\$data" == @* ]]; then cat "\${data#@}"; else printf '%s' "\$data"; fi > "${TEST_TEMP_DIR}/uploaded.json" + printf 'Docker-Content-Digest: sha256:%s\n' "\$(shasum -a 256 "${TEST_TEMP_DIR}/uploaded.json" | cut -d' ' -f1)" > "\$headers_file" + exit 0 +fi +printf 'Content-Type: application/vnd.oci.image.index.v1+json\nDocker-Content-Digest: %s\n' "${served_digest:-$MANIFEST_DIGEST}" > "\$headers_file" +if [[ -n "\$out_file" ]]; then cat "${TEST_TEMP_DIR}/manifest.json" > "\$out_file"; else cat "${TEST_TEMP_DIR}/manifest.json"; fi +MOCK_EOF + chmod +x "${TEST_TEMP_DIR}/mocks/curl" +} + +@test "retags a newline-terminated manifest byte for byte under the verified digest" { + create_byte_registry_mock + export INPUT_RETAG_DIGEST="$MANIFEST_DIGEST" + + run "$SCRIPT" + assert_success + assert_output_value "digest" "$MANIFEST_DIGEST" + cmp "${TEST_TEMP_DIR}/manifest.json" "${TEST_TEMP_DIR}/uploaded.json" +} + +@test "fails without retagging when the fetched manifest bytes do not match the digest" { + create_byte_registry_mock "sha256:$(printf 'a%.0s' {1..64})" + export INPUT_RETAG_DIGEST="sha256:$(printf 'a%.0s' {1..64})" + + run "$SCRIPT" + assert_failure + assert_output --partial "do not match" + + run cat "${TEST_TEMP_DIR}/curl_calls.log" + refute_output --partial "-X PUT" +} + +@test "uploads the fetched manifest bytes unchanged without a verified digest" { + create_byte_registry_mock + + run "$SCRIPT" + assert_success + assert_output_value "digest" "$MANIFEST_DIGEST" + cmp "${TEST_TEMP_DIR}/manifest.json" "${TEST_TEMP_DIR}/uploaded.json" +}