From c98afb7a36ac311bc5825c1057ac4af47164b1e0 Mon Sep 17 00:00:00 2001 From: wlsdnd2367 <230291092+wlsdnd2367@users.noreply.github.com> Date: Mon, 2 Nov 2026 10:52:03 +0900 Subject: [PATCH 1/4] =?UTF-8?q?feat:=20=EC=8A=B9=EC=9D=B8=EB=90=9C=20?= =?UTF-8?q?=EC=9A=B4=EC=98=81=20=EC=9D=B4=EB=AF=B8=EC=A7=80=20SSH=20?= =?UTF-8?q?=EC=82=AC=EC=A0=84=20=EB=8B=A4=EC=9A=B4=EB=A1=9C=EB=93=9C=20?= =?UTF-8?q?=ED=8C=8C=EC=9D=B4=ED=94=84=EB=9D=BC=EC=9D=B8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/pipeline-self-test.yml | 2 + .github/workflows/vm-image-prefetch.yml | 163 +++++ docs/vm-image-prefetch.md | 159 +++++ scripts/README.md | 7 + scripts/vm_cache_preflight.py | 75 +++ scripts/vm_image_prefetch.py | 407 ++++++++++++ tests/README.md | 7 + .../prob/for_organizer/web/Dockerfile | 4 + tests/test_vm_image_prefetch.py | 591 ++++++++++++++++++ tests/test_workflow_contracts.py | 11 + 10 files changed, 1426 insertions(+) create mode 100644 .github/workflows/vm-image-prefetch.yml create mode 100644 docs/vm-image-prefetch.md create mode 100644 scripts/vm_cache_preflight.py create mode 100644 scripts/vm_image_prefetch.py create mode 100644 tests/test_vm_image_prefetch.py diff --git a/.github/workflows/pipeline-self-test.yml b/.github/workflows/pipeline-self-test.yml index e4d33ed..d16e3b7 100644 --- a/.github/workflows/pipeline-self-test.yml +++ b/.github/workflows/pipeline-self-test.yml @@ -10,6 +10,7 @@ on: - ".github/workflows/pipeline-self-test.yml" - ".github/workflows/ghcr-cleanup-dry-run.yml" - ".github/workflows/node-image-prefetch-plan.yml" + - ".github/workflows/vm-image-prefetch.yml" - "ci/**" - "scripts/**" - "tests/**" @@ -22,6 +23,7 @@ on: - ".github/workflows/pipeline-self-test.yml" - ".github/workflows/ghcr-cleanup-dry-run.yml" - ".github/workflows/node-image-prefetch-plan.yml" + - ".github/workflows/vm-image-prefetch.yml" - "ci/**" - "scripts/**" - "tests/**" diff --git a/.github/workflows/vm-image-prefetch.yml b/.github/workflows/vm-image-prefetch.yml new file mode 100644 index 0000000..a242864 --- /dev/null +++ b/.github/workflows/vm-image-prefetch.yml @@ -0,0 +1,163 @@ +name: 승인된 운영 이미지 VM 사전 다운로드 + +on: + workflow_dispatch: + inputs: + apply: + description: 검수된 계획으로 실제 다운로드 (기본은 목록 검수만) + type: boolean + default: false + expected_plan_sha256: + description: 앞선 목록 검수에서 승인한 plan SHA-256 (실제 실행 시 필수) + type: string + default: "" + +permissions: + contents: read + +concurrency: + group: vm-image-prefetch-maintenance + cancel-in-progress: false + +jobs: + plan: + name: 운영 선택·원본 발행·VM 범위 검수 (SSH 없음) + runs-on: ubuntu-latest + timeout-minutes: 25 + outputs: + plan_sha256: ${{ steps.plan.outputs.plan_sha256 }} + steps: + - name: 공식 저장소 수동 main 실행 확인 + env: + APPLY: ${{ inputs.apply }} + ENABLED: ${{ vars.VM_IMAGE_PREFETCH_ENABLED }} + EXPECTED_SHA: ${{ inputs.expected_plan_sha256 }} + run: | + set -euo pipefail + test "$GITHUB_REPOSITORY" = MSG-CTF/msgctf-devsecops || exit 1 + test "$GITHUB_EVENT_NAME" = workflow_dispatch || exit 1 + test "$GITHUB_REF" = refs/heads/main || exit 1 + if [ "$APPLY" = true ]; then + test "$ENABLED" = true || { echo "실제 다운로드는 아직 비활성입니다" >&2; exit 1; } + [[ "$EXPECTED_SHA" =~ ^[a-f0-9]{64}$ ]] || exit 1 + fi + - uses: actions/checkout@v4 + with: + persist-credentials: false + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - run: python3 -m pip install -r ci/requirements.txt + - name: 비공개 선택 자료와 원본 발행 대조 + id: plan + env: + VM_SCOPE: ${{ secrets.VM_IMAGE_PREFETCH_SCOPE }} + VM_NODES: ${{ secrets.VM_IMAGE_PREFETCH_NODES }} + GH_TOKEN: ${{ secrets.CHALLENGE_ARTIFACT_READ_TOKEN }} + run: | + set -euo pipefail + umask 077 + trap 'rm -rf "$RUNNER_TEMP/vm-prefetch-input" "$RUNNER_TEMP/vm-prefetch-plan"' EXIT + mkdir "$RUNNER_TEMP/vm-prefetch-input" + python3 -c 'import os,pathlib; r=pathlib.Path(os.environ["RUNNER_TEMP"])/"vm-prefetch-input"; (r/"scope.json").write_text(os.environ["VM_SCOPE"]); (r/"nodes.json").write_text(os.environ["VM_NODES"])' + python3 -B -m scripts.vm_image_prefetch \ + --scope "$RUNNER_TEMP/vm-prefetch-input/scope.json" \ + --nodes "$RUNNER_TEMP/vm-prefetch-input/nodes.json" \ + --output-dir "$RUNNER_TEMP/vm-prefetch-plan" + python3 -c 'import os,json,pathlib; r=pathlib.Path(os.environ["RUNNER_TEMP"])/"vm-prefetch-plan"; s=json.loads((r/"summary.json").read_text()); open(os.environ["GITHUB_OUTPUT"],"a").write("plan_sha256="+s["plan_sha256"]+"\n"); pathlib.Path("dist").mkdir(exist_ok=True); pathlib.Path("dist/vm-prefetch-plan-summary.json").write_bytes((r/"summary.json").read_bytes())' + - uses: actions/upload-artifact@v4 + with: + name: vm-prefetch-plan-summary-${{ github.run_id }}-${{ github.run_attempt }} + path: dist/vm-prefetch-plan-summary.json + retention-days: 14 + if-no-files-found: error + + apply: + name: 승인된 VM의 이미지 캐시 채우기 (실행·활성화 없음) + needs: plan + if: inputs.apply + environment: vm-image-prefetch + runs-on: ubuntu-latest + timeout-minutes: 180 + steps: + - name: 승인한 계획 일치 확인 + env: + EXPECTED_SHA: ${{ inputs.expected_plan_sha256 }} + CURRENT_SHA: ${{ needs.plan.outputs.plan_sha256 }} + run: | + set -euo pipefail + test "$GITHUB_REPOSITORY" = MSG-CTF/msgctf-devsecops || exit 1 + test "$GITHUB_EVENT_NAME" = workflow_dispatch || exit 1 + test "$GITHUB_REF" = refs/heads/main || exit 1 + test -n "$EXPECTED_SHA" || exit 1 + test "$EXPECTED_SHA" = "$CURRENT_SHA" || exit 1 + - uses: actions/checkout@v4 + with: + persist-credentials: false + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - run: python3 -m pip install -r ci/requirements.txt + - name: 원본 재검수·전체 VM 사전 검사·순차 다운로드 + env: + VM_SCOPE: ${{ secrets.VM_IMAGE_PREFETCH_SCOPE }} + VM_NODES: ${{ secrets.VM_IMAGE_PREFETCH_NODES }} + VM_SSH_KEY: ${{ secrets.VM_IMAGE_PREFETCH_SSH_KEY }} + REPORT_CERT: ${{ secrets.VM_IMAGE_PREFETCH_REPORT_CERT }} + REPORT_CERT_SHA: ${{ vars.VM_IMAGE_PREFETCH_REPORT_CERT_SHA256 }} + GH_TOKEN: ${{ secrets.CHALLENGE_ARTIFACT_READ_TOKEN }} + EXPECTED_SHA: ${{ inputs.expected_plan_sha256 }} + run: | + set -euo pipefail + umask 077 + trap 'rm -rf "$RUNNER_TEMP/vm-prefetch-input" "$RUNNER_TEMP/vm-prefetch-apply"' EXIT + mkdir "$RUNNER_TEMP/vm-prefetch-input" + python3 -c 'import os,pathlib; r=pathlib.Path(os.environ["RUNNER_TEMP"])/"vm-prefetch-input"; (r/"scope.json").write_text(os.environ["VM_SCOPE"]); (r/"nodes.json").write_text(os.environ["VM_NODES"]); (r/"ssh-key").write_text(os.environ["VM_SSH_KEY"])' + test -n "$REPORT_CERT" || { echo "상세 결과 수신 인증서를 등록하세요" >&2; exit 1; } + [[ "$REPORT_CERT_SHA" =~ ^[a-f0-9]{64}$ ]] || exit 1 + python3 -c 'import os,pathlib; (pathlib.Path(os.environ["RUNNER_TEMP"])/"vm-prefetch-input/recipient.pem").write_text(os.environ["REPORT_CERT"])' + openssl x509 -in "$RUNNER_TEMP/vm-prefetch-input/recipient.pem" -checkend 10800 -noout + actual_cert_sha="$(openssl x509 -in "$RUNNER_TEMP/vm-prefetch-input/recipient.pem" -outform DER | sha256sum | cut -d ' ' -f 1)" + test "$actual_cert_sha" = "$REPORT_CERT_SHA" || exit 1 + set +e + python3 -B -m scripts.vm_image_prefetch \ + --scope "$RUNNER_TEMP/vm-prefetch-input/scope.json" \ + --nodes "$RUNNER_TEMP/vm-prefetch-input/nodes.json" \ + --output-dir "$RUNNER_TEMP/vm-prefetch-apply" \ + --ssh-key "$RUNNER_TEMP/vm-prefetch-input/ssh-key" \ + --expected-plan-sha256 "$EXPECTED_SHA" --apply + status=$? + set -e + if [ -f "$RUNNER_TEMP/vm-prefetch-apply/summary.json" ]; then + mkdir -p dist + cp "$RUNNER_TEMP/vm-prefetch-apply/summary.json" dist/vm-prefetch-apply-summary.json + fi + if [ -f "$RUNNER_TEMP/vm-prefetch-apply/plan.json" ]; then + mkdir -p dist + private_files=(plan.json) + if [ -f "$RUNNER_TEMP/vm-prefetch-apply/results.json" ]; then + private_files+=(results.json) + fi + tar -C "$RUNNER_TEMP/vm-prefetch-apply" -cf "$RUNNER_TEMP/vm-prefetch-input/evidence.tar" "${private_files[@]}" + openssl cms -encrypt -binary -aes-256-cbc -outform DER \ + -in "$RUNNER_TEMP/vm-prefetch-input/evidence.tar" \ + -out dist/vm-prefetch-private-results.cms "$RUNNER_TEMP/vm-prefetch-input/recipient.pem" + sha256sum dist/vm-prefetch-private-results.cms > dist/vm-prefetch-private-results.cms.sha256 + fi + exit "$status" + - uses: actions/upload-artifact@v4 + if: always() && hashFiles('dist/vm-prefetch-apply-summary.json') != '' + with: + name: vm-prefetch-apply-summary-${{ github.run_id }}-${{ github.run_attempt }} + path: dist/vm-prefetch-apply-summary.json + retention-days: 14 + if-no-files-found: error + - uses: actions/upload-artifact@v4 + if: always() && hashFiles('dist/vm-prefetch-private-results.cms') != '' + with: + name: vm-prefetch-encrypted-results-${{ github.run_id }}-${{ github.run_attempt }} + path: | + dist/vm-prefetch-private-results.cms + dist/vm-prefetch-private-results.cms.sha256 + retention-days: 14 + if-no-files-found: error diff --git a/docs/vm-image-prefetch.md b/docs/vm-image-prefetch.md new file mode 100644 index 0000000..e86d137 --- /dev/null +++ b/docs/vm-image-prefetch.md @@ -0,0 +1,159 @@ +# 운영 VM 이미지 사전 다운로드 + +이 경로는 **일반 SSH로 운영 문제 이미지 캐시를 채우는 수동 정비 작업**입니다. +새 Runtime API나 다운로드 엔진을 만들지 않습니다. 기존 Runtime image-agent의 +JSON 요청·전체 내용 검증을 재사용합니다. Backend 준비 operation 등록, Runtime +readiness 등록·정책 승인, Pod 실행·삭제, active 전환·무중단 교체는 수행하지 않습니다. +Broker 후보 자격·예약·Scheduler fallback도 변경하지 않습니다. + +## 실행 전 필수 검수 + +1. 최신 운영 선택에서 일반 Linux K3s release별 전체 컨테이너 digest를 확정합니다. + 최신 발행본을 운영 버전으로 추측하지 않습니다. 기존 버전·롤백 이미지는 보존합니다. +2. 실제 실행 후보 VM 전체와 canonical target·Node UID·machine/boot ID·architecture를 + 대조합니다. Backend·Broker·Runtime의 ID를 같은 값으로 가정하지 않습니다. +3. **VM을 켠 뒤** host identity를 읽어 검수합니다. 재부팅된 VM은 이전 boot ID 승인으로 + 실행하지 않습니다. 자동으로 새 identity를 신뢰하거나 기존 Node UID를 생성하지 않습니다. +4. 각 VM의 실제 image filesystem, 추가 허용량·임시/압축 해제 공간·보존 여유를 확정합니다. + 최소20% 여유와 명시한 reserve를 유지합니다. 예산은 검사 장치이지 OS 강제 quota가 아닙니다. + 한 이미지의 공간 상한도 운영자가 산정해야 합니다. 동시 정비·다른 대량 pull과 조율합니다. +5. 정적 문제는 제외합니다. The Quiet Second의 예전 접속형 이미지를 새 파일형 문제를 위해 + 받지 않습니다. KOTH·Windows·DOCKER·VM과 미승인 digest를 K3s로 임의 변환하지 않습니다. + AFTERIMAGE 등 별도 정책 검토 중인 버전은 승인 전 보류합니다. + +선택 파일은24시간 이내의 운영자 승인 snapshot입니다. 이 도구가 Backend active나 +전체 운영 VM의 누락 여부, Runtime 정책 승인을 독립 조회한 것은 아닙니다. +보고서 범위는 **입력한 release와 VM만**입니다. 제외 항목도 남깁니다. + +## VM 설치 조건 + +다운로드는 기존 Runtime의 [고정 SSH 명령과 agent](https://github.com/MSG-CTF/secure-provisioner/blob/619423a35876d3e823df17b812cb0b1539001c13/docs/operations/image-preparation.md#L93)를 사용합니다. + +- `/usr/local/libexec/runtime-image-agent`: 검토한 Runtime 바이너리. VM별 SHA-256을 고정합니다. +- `/etc/secure-provisioner/image-agent.json`: 실제 Node UID와 K3s binary 절대경로를 담은 기존 설정. +- `/usr/local/libexec/vm-image-cache-preflight`: 이 저장소의 `scripts/vm_cache_preflight.py`. +- `/etc/secure-provisioner/vm-prefetch.json`: `{"image_fs_path":"실제 K3s imageFS 절대경로"}`. + +관리자가 사전에 승인한 설치를 수행합니다. 이 workflow는 설치 파일·서비스·기존 +K3s 설정을 덮어쓰거나 재시작하지 않습니다. 준비 도구가 없으면 보류합니다. +Linux VM에 `/usr/bin/python3`가 있어야 하며, Runtime agent는 원래 저장소의 검토한 +소스·빌드 결과를 사용합니다. 임의의 대체 다운로드 프로그램을 sudo로 실행하지 않습니다. +파일과 상위 디렉터리는 root 소유, group/other 쓰기 금지이며 symlink를 허용하지 않습니다. +preflight 파일은 다음과 같이 고정 경로에 설치할 수 있습니다. + +```bash +sudo install -o root -g root -m 0755 scripts/vm_cache_preflight.py /usr/local/libexec/vm-image-cache-preflight +``` + +preflight는 machine/boot·architecture·설정의 Node UID·binary hash·디스크만 읽습니다. +Node UID는 operator-pinned 설정값이며 Kubernetes API의 독립 UID 재조회가 아닙니다. +관리자는 실제 노드 대응을 먼저 검수해야 합니다. 일반 `docker pull`은 사용하지 않습니다. +agent는 `/run/k3s/containerd/containerd.sock`, `k8s.io`에서 실제 manifest·config·layer를 확인합니다. + +SSH 계정에는 가능하면 위 **고정 명령2개**의 비대화식 sudo 권한만 부여합니다. +공유 프로젝트 관리자 키를 CI에 영구 등록하지 않습니다. 개인키·GCP 계정 키·token은 +채팅·commit·보고서에 넣지 않습니다. 노드의 기존 private GHCR 읽기 인증을 사용하고 +registry token을 SSH 인자나 agent 입력에 전달하지 않습니다. + +## 비공개 입력 + +`scope.json` 예시입니다. 값은 실행 당일 실제 운영 선택과 원본 발행 기록으로 채웁니다. +실제 inventory는 공개 저장소에 commit하지 않습니다. + +```json +{ + "schema_version": 1, + "selection_mode": "OPERATOR_APPROVED_ACTIVE_RELEASES", + "inventory_complete_as_attested": true, + "observed_at": "2026-11-02T00:00:00Z", + "valid_until": "2026-11-02T01:00:00Z", + "releases": [{ + "challenge_slug": "web-example", + "release_id": "00000000-0000-4000-8000-000000000001", + "registry_revision": 1, + "publish_run_id": 1, + "publish_artifact_id": 1, + "source_sha": "원본 main 커밋 소문자40자리", + "artifact_sha256": "원본 artifact-v2.json SHA-256 소문자64자리", + "architecture": "AMD64", + "policy_approved": true, + "approved_target_ids": ["target-example"], + "containers": [{"name":"web", "image":"ghcr.io/msg-ctf/challenges/web-example/web@sha256:실제64자리"}] + }, {"challenge_slug":"crypto-reused-seal", "exclude_reason":"STATIC_CONVERSION"}] +} +``` + +정확한 release의 전체 컨테이너 세트가 공식 성공 main Actions bundle·검사 증거·checksum과 +일치해야 합니다. 원본 조회에는 문제 저장소 Actions artifact 읽기 인증이 필요합니다. +정책 승인은 운영자 확인값입니다. 다운로드 자체가 실행 승인을 부여하지 않습니다. +`exclude_reason` 허용값은 STATIC_CONVERSION, KOTH, WINDOWS, NOT_APPROVED, +NOT_IN_THIS_MAINTENANCE입니다. 제외를 준비 성공으로 집계하지 않습니다. + +`nodes.json`에는 schema_version1과 nodes 배열을 넣습니다. VM마다 다음 값이 필요합니다. + +| 필드 | 의미 | +|---|---| +| node_id | 보고서용 안전한 별칭 | +| target_id, node_uid, machine_id, boot_id | 검수한 실제 대응과 현재 identity | +| architecture | AMD64 또는 ARM64 | +| ssh_host, ssh_port, ssh_user | 일반 SSH 주소·포트·전용 계정 | +| ssh_host_key | 별도 보안 경로로 확인한 `ssh-ed25519 BASE64` 공개 host key | +| agent_sha256 | 설치된 Runtime agent의 검토된 binary SHA-256 | +| image_fs_path | 실제 K3s containerd 데이터 filesystem 경로 | +| reserve_bytes | 보존·운영용 최소 잔여 공간 | +| max_additional_bytes | 이번 전체 정비의 추가 공간 예산 | +| per_image_workspace_bytes | 이미지 한 건의 임시·압축 해제 포함 작업 여유 | + +동일 architecture의 입력 VM 전체를 각 release의 approved_target_ids가 포함해야 합니다. +VM 주소·Node UID·machine ID 중복이나 누락을 허용하지 않습니다. host key를 현장에서 +무인증 `ssh-keyscan` 결과만으로 신뢰하거나 StrictHostKeyChecking을 끄지 않습니다. + +## 로컬 실행과 결과 + +```bash +python3 -B -m scripts.vm_image_prefetch --scope /private/scope.json --nodes /private/nodes.json --output-dir /private/new-plan +``` + +기본은 GitHub 원본 대조와 계획 생성뿐이며 SSH에 접속하지 않습니다. +`plan.json`의 전체 이미지·노드·제외/보류를 검수한 뒤 표시된 plan_sha256을 승인합니다. +실제 실행은 같은 입력을 다시 검수하므로 source·VM·예산이 바뀌면 새 승인이 필요합니다. + +```bash +python3 -B -m scripts.vm_image_prefetch --scope /private/scope.json --nodes /private/nodes.json --output-dir /private/new-apply --ssh-key /private/ssh-key --expected-plan-sha256 APPROVED_SHA256 --apply +``` + +모든 VM의 identity·도구 hash·공간을 먼저 확인한 후 한 VM·한 이미지씩 처리합니다. +캐시가 완전하면 REUSED_VERIFIED, 없거나 불완전해 pull 후 검증하면 PULLED_OR_REPAIRED입니다. +후자는 cold pull의 증거가 아닙니다. agent_prepare_seconds도 실제 registry 전송 시간만을 +측정한 값이 아닙니다. 실패 시 다음 이미지로 넘어가지 않으며, 캐시 삭제·자동 retry는 없습니다. +다시 승인해 실행하면 이미 준비된 이미지를 검증·재사용할 수 있습니다. + +새 출력 디렉터리는0700, plan/results/summary는0600입니다. 결과가 부분 실패해도 완료한 +쌍과 실패 위치를 남깁니다. Runtime readiness·Backend operation에 결과를 등록하지 않으므로 +기존 플랫폼의 별도 조회와 대조해야 합니다. 실제 Pod imageID·HTTP 접속·무중단은 후속 시험입니다. + +이는 관측 당시의 캐시 보유 결과입니다. kubelet의 이미지 GC, VM 삭제·디스크 교체로 +캐시가 사라질 수 있고 이 도구는 GC를 끄거나 영구 보존을 강제하지 않습니다. PWN의 +이미지 준비도 gVisor·실행 격리 승인을 뜻하지 않습니다. 새 서버 자동 준비와 패치 시 +자동 호출은 기존 플랫폼 연동의 후속이며 이번 수동 정비 workflow가 대신하지 않습니다. + +## GitHub 수동 workflow + +`.github/workflows/vm-image-prefetch.yml`은 공식 저장소 main의 수동 실행만 허용합니다. +동시 실행을 막고 실제 실행은 기본 비활성입니다. 일반 SSH에 접근 가능한 승인된 러너가 +필요합니다. 이를 위해 SSH 방화벽을 인터넷 전체에 열지 않습니다. IAP/VPN 어댑터는 포함하지 않습니다. + +- repo Secrets: VM_IMAGE_PREFETCH_SCOPE, VM_IMAGE_PREFETCH_NODES, + CHALLENGE_ARTIFACT_READ_TOKEN (문제 저장소 Actions 읽기). +- `vm-image-prefetch` Environment Secret: VM_IMAGE_PREFETCH_SSH_KEY, + VM_IMAGE_PREFETCH_REPORT_CERT (결과 수신자의 공개 PEM 인증서, 개인키 아님). +- Environment 변수 VM_IMAGE_PREFETCH_REPORT_CERT_SHA256: 인증서 DER SHA-256. +- repo 변수 VM_IMAGE_PREFETCH_ENABLED는 검수 전 미설정/false로 유지합니다. +- Environment에는 main 제한·필수 승인자를 설정한 뒤 사용합니다. 이 보호는 관리자가 + 실제로 설정해야 하며 workflow 파일만으로 승인자가 자동 등록되는 것은 아닙니다. + +먼저 apply=false로 계획 SHA를 확인합니다. 검수·접속·용량·설치가 확인된 뒤에만 +승인 SHA와 apply=true로 실행합니다. 인증서는 실행 전에 fingerprint와 남은 유효기간을 +확인합니다. 공개 artifact에는 집계와 암호화된 CMS 결과만 올립니다. 상세 plan/results는 +수신 인증서로 AES-256-CBC CMS 암호화하고 checksum을 남기며, 원문·키는 러너에서 삭제합니다. +수신 개인키는 러너나 채팅에 전달하지 않습니다. 실제 실행은 별도 정비 일정에 승인합니다. diff --git a/scripts/README.md b/scripts/README.md index 5fbeb63..8c77a93 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -9,6 +9,13 @@ Provisioner API 요청으로 변환합니다. 생성 Operation을 조회한 뒤 ## 파일 +### `vm_image_prefetch.py` / `vm_cache_preflight.py` + +운영자가 선택한 정확한 release와 VM을 공식 발행 원본에 대조하고, 별도 수동 승인 후 +기존 Runtime agent를 일반 SSH로 호출해 이미지 캐시만 채웁니다. 기본은 SSH 없는 +계획 검수입니다. 실행·활성화·삭제·Runtime 설정은 변경하지 않습니다. +[운영 준비와 입력 형식](../docs/vm-image-prefetch.md)을 참고합니다. + ### `collect_node_image_plans.py` 현재 main과 성공한 실제 bundle을 대조해 일반 Linux K3s 문제의 전체 다운로드 diff --git a/scripts/vm_cache_preflight.py b/scripts/vm_cache_preflight.py new file mode 100644 index 0000000..cad871e --- /dev/null +++ b/scripts/vm_cache_preflight.py @@ -0,0 +1,75 @@ +#!/usr/bin/python3 -B +"""Read-only host checks for the manually approved VM cache operation.""" + +import hashlib +import json +import os +from pathlib import Path +import platform +import stat + +CONFIG = Path('/etc/secure-provisioner/vm-prefetch.json') +AGENT_CONFIG = Path('/etc/secure-provisioner/image-agent.json') +AGENT = Path('/usr/local/libexec/runtime-image-agent') + + +def trusted_file(path): + if path.is_symlink(): + raise ValueError('UNTRUSTED_INSTALLATION') + info = path.stat() + if not stat.S_ISREG(info.st_mode) or info.st_uid != 0 or info.st_mode & 0o022: + raise ValueError('UNTRUSTED_INSTALLATION') + for parent in path.parents: + info = parent.stat() + if parent.is_symlink() or info.st_uid != 0 or info.st_mode & 0o022: + raise ValueError('UNTRUSTED_INSTALLATION') + return path + + +def snapshot(): + if os.geteuid() != 0: + raise ValueError('PREFLIGHT_PERMISSION') + config = json.loads(trusted_file(CONFIG).read_bytes()) + agent_config = json.loads(trusted_file(AGENT_CONFIG).read_bytes()) + if set(config) != {'image_fs_path'} or not isinstance(config['image_fs_path'], str): + raise ValueError('PREFLIGHT_CONFIG') + image_fs = Path(config['image_fs_path']) + if not image_fs.is_absolute() or image_fs.is_symlink() or not image_fs.is_dir(): + raise ValueError('PREFLIGHT_CONFIG') + trusted_file(AGENT) + if (not isinstance(agent_config, dict) or set(agent_config) != {'node_uid', 'k3s_binary'} + or not isinstance(agent_config['k3s_binary'], str) + or not Path(agent_config['k3s_binary']).is_absolute()): + raise ValueError('PREFLIGHT_CONFIG') + trusted_file(Path(agent_config['k3s_binary'])) + disk = os.statvfs(image_fs) + arch = {'x86_64': 'AMD64', 'aarch64': 'ARM64'}.get(platform.machine()) + if platform.system() != 'Linux' or arch is None: + raise ValueError('PLATFORM_MISMATCH') + return { + 'schema_version': 1, + 'node_uid': agent_config['node_uid'], + 'node_uid_source': 'OPERATOR_PINNED_AGENT_CONFIG', + 'machine_id': Path('/etc/machine-id').read_text().strip(), + 'boot_id': Path('/proc/sys/kernel/random/boot_id').read_text().strip(), + 'os': 'linux', 'architecture': arch, + 'image_fs_path': str(image_fs), + 'available_bytes': disk.f_bavail * disk.f_frsize, + 'total_bytes': disk.f_blocks * disk.f_frsize, + 'agent_sha256': hashlib.sha256(AGENT.read_bytes()).hexdigest(), + 'preflight_sha256': hashlib.sha256(Path(__file__).read_bytes()).hexdigest(), + } + + +def main(): + try: + result = snapshot() + except (OSError, ValueError, TypeError, KeyError): + print(json.dumps({'error_code': 'PREFLIGHT_FAILED'})) + return 1 + print(json.dumps(result, separators=(',', ':'))) + return 0 + + +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/scripts/vm_image_prefetch.py b/scripts/vm_image_prefetch.py new file mode 100644 index 0000000..b6ef150 --- /dev/null +++ b/scripts/vm_image_prefetch.py @@ -0,0 +1,407 @@ +#!/usr/bin/env python3 +"""Audit active image selections, then optionally fill approved VM caches over SSH.""" + +import argparse +import base64 +from datetime import datetime, timezone, timedelta +import hashlib +import ipaddress +import json +import os +from pathlib import Path +import re +import selectors +import signal +import stat +import subprocess +import sys +import tempfile +import time +from uuid import UUID +import zipfile + +import yaml + +if __package__: + from . import collect_node_image_plans as sources +else: + import collect_node_image_plans as sources + +NAME = re.compile(r'^[a-z0-9][a-z0-9-]{0,62}$') +SHA = re.compile(r'^[a-f0-9]{64}$') +DIGEST = re.compile(r'^sha256:[a-f0-9]{64}$') +IDENTITY = re.compile(r'^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$') +REPOSITORY = sources.publication.REPOSITORY +AGENT_COMMAND = 'sudo -n /usr/local/libexec/runtime-image-agent' +PREFLIGHT_COMMAND = 'sudo -n /usr/local/libexec/vm-image-cache-preflight' +MAX_BYTES = 1024 * 1024 + + +def require(condition): + if not condition: + raise ValueError('INPUT_OR_EVIDENCE_INVALID') + + +def unique_object(pairs): + result = {} + for key, value in pairs: + require(key not in result) + result[key] = value + return result + + +def load_json(raw): + require(len(raw) <= MAX_BYTES) + return json.loads(raw, object_pairs_hook=unique_object, + parse_constant=lambda _: require(False)) + + +def read_json(path): + path = Path(path) + require(not path.is_symlink() and path.is_file() and path.stat().st_size <= MAX_BYTES) + return load_json(path.read_bytes()) + + +def timestamp(value): + require(isinstance(value, str)) + parsed = datetime.fromisoformat(value.replace('Z', '+00:00')) + require(parsed.tzinfo is not None) + return parsed.astimezone(timezone.utc) + + +def positive(value): + return type(value) is int and 0 < value <= 2**63 - 1 + + +def image_set(containers): + require(isinstance(containers, list) and 1 <= len(containers) <= 32) + names = set() + result = {} + for container in containers: + require(isinstance(container, dict) and set(container) == {'name', 'image'}) + name, image = container['name'], container['image'] + require(isinstance(name, str) and NAME.fullmatch(name) and name not in names) + require(isinstance(image, str) and sources.MSGCTF_GHCR_DIGEST_IMAGE.fullmatch(image)) + names.add(name) + result[name] = image + return result + + +def checked_nodes(inventory): + require(isinstance(inventory, dict) and set(inventory) == {'schema_version', 'nodes'}) + require(inventory['schema_version'] == 1 and isinstance(inventory['nodes'], list) + and 1 <= len(inventory['nodes']) <= 50) + nodes, ids, machines, uids, hosts = [], set(), set(), set(), set() + fields = {'node_id', 'target_id', 'node_uid', 'machine_id', 'boot_id', 'architecture', + 'ssh_host', 'ssh_port', 'ssh_user', 'ssh_host_key', 'agent_sha256', + 'image_fs_path', 'reserve_bytes', 'max_additional_bytes', 'per_image_workspace_bytes'} + for row in inventory['nodes']: + require(isinstance(row, dict) and set(row) == fields) + for key in ('node_id', 'target_id', 'node_uid', 'machine_id', 'boot_id'): + require(isinstance(row[key], str) and IDENTITY.fullmatch(row[key])) + require(NAME.fullmatch(row['node_id']) and row['node_id'] not in ids) + require(row['machine_id'] not in machines and row['architecture'] in ('AMD64', 'ARM64')) + require(row['node_uid'] not in uids) + host = row['ssh_host'] + require(isinstance(host, str) and len(host) <= 253 and not host.startswith('-')) + try: + address = ipaddress.ip_address(host) + except ValueError: + require(re.fullmatch(r'[A-Za-z0-9](?:[A-Za-z0-9.-]*[A-Za-z0-9])?', host) + and '..' not in host and host.lower() != 'localhost') + else: + require(not (address.is_loopback or address.is_link_local or address.is_unspecified + or address.is_multicast)) + require(type(row['ssh_port']) is int and 1 <= row['ssh_port'] <= 65535) + require((host, row['ssh_port']) not in hosts) + require(isinstance(row['ssh_user'], str) and re.fullmatch(r'[a-z_][a-z0-9_-]{0,31}', row['ssh_user'])) + require(isinstance(row['ssh_host_key'], str)) + key = row['ssh_host_key'].split() + require(len(key) == 2 and key[0] == 'ssh-ed25519') + decoded = base64.b64decode(key[1], validate=True) + require(len(decoded) == 51 and decoded[:19] == b'\x00\x00\x00\x0bssh-ed25519\x00\x00\x00\x20') + require(isinstance(row['agent_sha256'], str) and SHA.fullmatch(row['agent_sha256'])) + path = row['image_fs_path'] + require(isinstance(path, str) and path.startswith('/') and path != '/' + and re.fullmatch(r'/[A-Za-z0-9_./-]+', path) and '..' not in Path(path).parts) + for field in ('reserve_bytes', 'max_additional_bytes', 'per_image_workspace_bytes'): + require(positive(row[field])) + require(row['per_image_workspace_bytes'] <= row['max_additional_bytes']) + ids.add(row['node_id']) + machines.add(row['machine_id']) + uids.add(row['node_uid']) + hosts.add((host, row['ssh_port'])) + nodes.append(dict(row)) + return sorted(nodes, key=lambda row: row['node_id']) + + +def current_kind(slug, main_sha, api): + data = api(f'contents/{slug}/info.yaml?ref={main_sha}') + require(data.get('encoding') == 'base64' and positive(data.get('size')) and data['size'] <= MAX_BYTES) + spec = yaml.safe_load(base64.b64decode(data['content'])) + require(isinstance(spec, dict)) + deployment = spec.get('deployment') + if deployment is None: + return 'STATIC' + require(isinstance(deployment, dict)) + return deployment.get('runtime_type') + + +def build_plan(scope, inventory, *, api=sources.publication.github, fetch=None, now=None): + now = now or datetime.now(timezone.utc) + require(isinstance(scope, dict) and scope.get('schema_version') == 1 + and scope.get('selection_mode') == 'OPERATOR_APPROVED_ACTIVE_RELEASES' + and scope.get('inventory_complete_as_attested') is True) + observed, expires = timestamp(scope.get('observed_at')), timestamp(scope.get('valid_until')) + require(observed <= now < expires and timedelta(0) < expires - observed <= timedelta(hours=24)) + releases = scope.get('releases') + require(isinstance(releases, list) and 1 <= len(releases) <= 100) + nodes = checked_nodes(inventory) + main_sha = api('git/ref/heads/main')['object']['sha'] + require(isinstance(main_sha, str) and sources.COMMIT_SHA.fullmatch(main_sha)) + plans, held, excluded, seen = [], [], [], set() + for selected in releases: + require(isinstance(selected, dict)) + slug = selected.get('challenge_slug') + require(isinstance(slug, str) and NAME.fullmatch(slug) and slug not in seen) + seen.add(slug) + reason = selected.get('exclude_reason') + if reason is not None: + require(reason in {'STATIC_CONVERSION', 'KOTH', 'WINDOWS', 'NOT_APPROVED', 'NOT_IN_THIS_MAINTENANCE'}) + excluded.append({'challenge_slug': slug, 'reason': reason}) + continue + try: + require(not slug.startswith('koth-') and slug != 'pwn-msgdriver') + require(str(UUID(selected['release_id'])) == selected['release_id']) + if selected.get('policy_approved') is not True: + held.append({'challenge_slug': slug, 'reason': 'POLICY_APPROVAL_NOT_ATTESTED'}) + continue + require(positive(selected['registry_revision']) and positive(selected['publish_run_id']) + and positive(selected['publish_artifact_id'])) + require(isinstance(selected['artifact_sha256'], str) and SHA.fullmatch(selected['artifact_sha256'])) + kind = current_kind(slug, main_sha, api) + if kind != 'KUBERNETES': + held.append({'challenge_slug': slug, 'reason': 'CURRENT_PROBLEM_NOT_KUBERNETES'}) + continue + meta = api(f"actions/artifacts/{selected['publish_artifact_id']}") + origin = {'id': selected['publish_artifact_id'], 'run_id': selected['publish_run_id'], + 'registry_revision': selected['registry_revision'], 'source_sha': selected['source_sha'], + 'name': meta['name']} + found = fetch(slug, origin) if fetch else sources.fetch_plan(slug, origin, api=api) + require(found['artifact_sha256'] == selected['artifact_sha256'] + and found['challenge_slug'] == slug + and found['registry_revision'] == selected['registry_revision'] + and found['source_sha'] == selected['source_sha'] + and found['publish_run_id'] == selected['publish_run_id'] + and found['publish_artifact_id'] == selected['publish_artifact_id'] + and found['runtime_type'] == 'KUBERNETES' and found['source_ref'] == 'refs/heads/main' + and found['architecture'] == selected['architecture'] + and image_set(found['containers']) == image_set(selected['containers'])) + matching = [row for row in nodes if row['architecture'] == found['architecture']] + targets = selected['approved_target_ids'] + require(isinstance(targets, list) and len(targets) == len(set(targets)) + and set(targets) == {row['target_id'] for row in matching} and matching) + plans.append({**found, 'release_id': selected['release_id'], 'approved_target_ids': sorted(targets)}) + except (ValueError, KeyError, TypeError, OSError, RuntimeError, yaml.YAMLError, + zipfile.BadZipFile, subprocess.SubprocessError): + held.append({'challenge_slug': slug, 'reason': 'SOURCE_POLICY_OR_SCOPE_NOT_CONFIRMED'}) + require(api('git/ref/heads/main')['object']['sha'] == main_sha) + require(plans or held) + tasks = [] + for node in nodes: + images = sorted({item['image'] for plan in plans if plan['architecture'] == node['architecture'] + for item in plan['containers']}) + if not images: + held.append({'node_id': node['node_id'], 'reason': 'NO_APPROVED_IMAGES_FOR_NODE'}) + tasks.append({'node_id': node['node_id'], 'images': images}) + plan = {'schema_version': 1, 'scope': 'PROVIDED_ACTIVE_RELEASES_AND_NODES_ONLY', + 'repository': REPOSITORY, 'source_main_sha': main_sha, 'observed_at': scope['observed_at'], + 'valid_until': scope['valid_until'], 'complete': not held, 'releases': plans, + 'nodes': nodes, 'tasks': tasks, 'held': held, 'excluded': excluded, + 'backend_active_independently_queried': False, + 'preflight_sha256': hashlib.sha256(Path(__file__).with_name('vm_cache_preflight.py').read_bytes()).hexdigest()} + plan['plan_sha256'] = plan_hash(plan) + return plan + + +def plan_hash(plan): + return hashlib.sha256(json.dumps({key: value for key, value in plan.items() if key != 'plan_sha256'}, + sort_keys=True, separators=(',', ':')).encode()).hexdigest() + + +def bounded_command(args, payload, timeout=130): + environment = {key: os.environ[key] for key in ('PATH', 'LANG', 'LC_ALL') if key in os.environ} + process = subprocess.Popen(args, stdin=subprocess.PIPE, stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, start_new_session=True, env=environment) + output, selector = bytearray(), selectors.DefaultSelector() + try: + process.stdin.write(json.dumps(payload, separators=(',', ':')).encode()) + process.stdin.close() + selector.register(process.stdout, selectors.EVENT_READ) + deadline = time.monotonic() + timeout + while selector.get_map(): + require(time.monotonic() < deadline) + for key, _ in selector.select(min(0.5, max(0, deadline - time.monotonic()))): + data = os.read(key.fd, 8192) + if not data: + selector.unregister(key.fileobj) + else: + output.extend(data) + require(len(output) <= 64 * 1024) + require(process.wait(timeout=max(0.1, deadline - time.monotonic())) == 0) + return load_json(output) + finally: + selector.close() + if process.poll() is None: + os.killpg(process.pid, signal.SIGKILL) + process.wait() + if not process.stdin.closed: + process.stdin.close() + process.stdout.close() + + +class SSHAgent: + def __init__(self, key_file, runner=bounded_command): + self.key_file = Path(key_file) + require(not self.key_file.is_symlink() and self.key_file.is_file()) + require(1 <= self.key_file.stat().st_size <= 16 * 1024) + require(stat.S_IMODE(self.key_file.stat().st_mode) & 0o077 == 0) + self.runner = runner + + def __call__(self, node, command, payload): + require(command in (AGENT_COMMAND, PREFLIGHT_COMMAND)) + with tempfile.TemporaryDirectory() as temporary: + known = Path(temporary) / 'known_hosts' + host = node['ssh_host'] if node['ssh_port'] == 22 else f"[{node['ssh_host']}]:{node['ssh_port']}" + known.write_text(f"{host} {node['ssh_host_key']}\n") + known.chmod(0o600) + args = ['ssh', '-F', '/dev/null', '-i', str(self.key_file), '-p', str(node['ssh_port']), + '-oBatchMode=yes', '-oIdentitiesOnly=yes', '-oIdentityAgent=none', + '-oForwardAgent=no', '-oClearAllForwardings=yes', '-oPermitLocalCommand=no', + '-oProxyCommand=none', '-oProxyJump=none', '-oPasswordAuthentication=no', + '-oKbdInteractiveAuthentication=no', '-oStrictHostKeyChecking=yes', + '-oGlobalKnownHostsFile=/dev/null', f'-oUserKnownHostsFile={known}', + '-oConnectTimeout=10', '-oConnectionAttempts=1', '-T', '--', + f"{node['ssh_user']}@{node['ssh_host']}", command] + return self.runner(args, payload) + + +def check_host(node, plan, reply, initial_free=None, workspace=0): + require(isinstance(reply, dict) and reply.get('schema_version') == 1) + for field in ('node_uid', 'machine_id', 'boot_id', 'architecture', 'image_fs_path', 'agent_sha256'): + require(reply.get(field) == node[field]) + require(reply.get('os') == 'linux' and reply.get('preflight_sha256') == plan['preflight_sha256']) + available, total = reply.get('available_bytes'), reply.get('total_bytes') + require(type(available) is int and positive(total) and 0 <= available <= total) + reserve = max(node['reserve_bytes'], (total + 4) // 5) + require(available >= reserve + workspace) + if initial_free is not None: + require(max(0, initial_free - available) <= node['max_additional_bytes']) + return available + + +def checked_evidence(node, image, reply): + require(isinstance(reply, dict) and set(reply) == {'evidence'}) + data = reply['evidence'] + require(isinstance(data, dict)) + for field in ('node_uid', 'machine_id', 'boot_id', 'architecture'): + require(data.get(field) == node[field]) + require(data.get('os') == 'linux' and data.get('image') == image and data.get('complete') is True) + require(isinstance(data.get('resolved_manifest_digest'), str) and DIGEST.fullmatch(data['resolved_manifest_digest'])) + return {'resolved_manifest_digest': data['resolved_manifest_digest'], 'complete': True} + + +def execute(plan, expected_sha, call, *, now=None, on_result=None): + require(plan['complete'] is True and isinstance(expected_sha, str) and SHA.fullmatch(expected_sha) + and expected_sha == plan['plan_sha256'] == plan_hash(plan)) + require((now or datetime.now(timezone.utc)) < timestamp(plan['valid_until'])) + report = {'schema_version': 1, 'plan_sha256': expected_sha, 'status': 'FAILED', 'results': [], + 'runtime_readiness_registered': False, 'backend_operation_registered': False, + 'pod_started': False, 'active_changed': False, 'cold_pull_proven': False} + baseline = {} + context = {'stage': 'PREFLIGHT'} + try: + # Check every configured VM before allowing the first cache mutation. + for node in plan['nodes']: + context = {'node_id': node['node_id'], 'stage': 'PREFLIGHT'} + response = call(node, PREFLIGHT_COMMAND, {}) + baseline[node['node_id']] = check_host(node, plan, response, + workspace=node['max_additional_bytes']) + for node, task in zip(plan['nodes'], plan['tasks']): + for image in task['images']: + context = {'node_id': node['node_id'], 'image': image, 'stage': 'CACHE_CHECK'} + require((now or datetime.now(timezone.utc)) < timestamp(plan['valid_until'])) + check_host(node, plan, call(node, PREFLIGHT_COMMAND, {}), baseline[node['node_id']]) + request = {'image': image, 'architecture': node['architecture'], 'pull': False} + reply = call(node, AGENT_COMMAND, request) + if isinstance(reply, dict) and reply.get('error_code') == 'CONTENT_INCOMPLETE' and set(reply) == {'error_code'}: + check_host(node, plan, call(node, PREFLIGHT_COMMAND, {}), baseline[node['node_id']], + workspace=node['per_image_workspace_bytes']) + request['pull'] = True + context['stage'] = 'PULL_OR_REPAIR' + started = time.monotonic() + reply = call(node, AGENT_COMMAND, request) + elapsed = time.monotonic() - started + mode = 'PULLED_OR_REPAIRED' + else: + elapsed, mode = None, 'REUSED_VERIFIED' + evidence = checked_evidence(node, image, reply) + check_host(node, plan, call(node, PREFLIGHT_COMMAND, {}), baseline[node['node_id']]) + report['results'].append({'node_id': node['node_id'], 'image': image, 'mode': mode, + 'agent_prepare_seconds': elapsed, + 'verified_at': datetime.now(timezone.utc).isoformat(), **evidence}) + if on_result: + on_result(report) + report['status'] = 'SUCCEEDED' + except (OSError, ValueError, TypeError, KeyError, subprocess.SubprocessError): + report['error_code'] = 'PREFLIGHT_SSH_OR_AGENT_CHECK_FAILED' + report['failure'] = context + if on_result: + on_result(report) + return report + + +def save_private(path, data): + temporary = path.with_suffix('.tmp') + descriptor = os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + with os.fdopen(descriptor, 'w') as stream: + json.dump(data, stream, ensure_ascii=True, indent=2, allow_nan=False) + stream.write('\n') + os.replace(temporary, path) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--scope', required=True, type=Path) + parser.add_argument('--nodes', required=True, type=Path) + parser.add_argument('--output-dir', required=True, type=Path) + parser.add_argument('--apply', action='store_true') + parser.add_argument('--expected-plan-sha256', default='') + parser.add_argument('--ssh-key', type=Path) + args = parser.parse_args() + try: + plan = build_plan(read_json(args.scope), read_json(args.nodes)) + args.output_dir.mkdir(mode=0o700, parents=True, exist_ok=False) + save_private(args.output_dir / 'plan.json', plan) + summary = {'plan_sha256': plan['plan_sha256'], 'complete': plan['complete'], + 'release_count': len(plan['releases']), 'node_count': len(plan['nodes']), + 'image_node_pairs': sum(len(row['images']) for row in plan['tasks']), + 'held_count': len(plan['held']), 'excluded_count': len(plan['excluded']), + 'applied': False} + code = 0 if plan['complete'] else 1 + if args.apply: + require(args.ssh_key is not None) + result = execute(plan, args.expected_plan_sha256, SSHAgent(args.ssh_key), + on_result=lambda value: save_private(args.output_dir / 'results.json', value)) + summary.update(applied=True, status=result['status'], checked_pairs=len(result['results'])) + code = 0 if result['status'] == 'SUCCEEDED' else 1 + save_private(args.output_dir / 'summary.json', summary) + print(json.dumps(summary)) + return code + except (OSError, ValueError, TypeError, KeyError, OverflowError, RecursionError, subprocess.SubprocessError): + print('VM image prefetch failed: check approved scope, source, node evidence and output path.', file=sys.stderr) + return 2 + + +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/tests/README.md b/tests/README.md index f57397c..b2fcc06 100644 --- a/tests/README.md +++ b/tests/README.md @@ -5,6 +5,13 @@ ## 테스트 구성 +### `test_vm_image_prefetch.py` + +운영 선택과 원본 checksum·전체 이미지·architecture·노드 범위를 대조하고, 만료· +미승인·변조·SSH 인자 주입·host key·identity·디스크·부분 실패를 검사합니다. +기존 Runtime agent 요청/결과 형식과 캐시 재사용, 실제 Bash main/event 가드도 검사합니다. +실제 VM 접속·pull·Pod·운영 전환을 실행하는 테스트는 아닙니다. + ### `test_collect_node_image_plans.py` 전체 대상과 아키텍처별 취합, 신규·변경·유지 비교, 누락·중복·출처·이전 목록 diff --git a/tests/fixtures/info-valid/prob/for_organizer/web/Dockerfile b/tests/fixtures/info-valid/prob/for_organizer/web/Dockerfile index 6f1bfee..c03e51b 100644 --- a/tests/fixtures/info-valid/prob/for_organizer/web/Dockerfile +++ b/tests/fixtures/info-valid/prob/for_organizer/web/Dockerfile @@ -1 +1,5 @@ FROM nginxinc/nginx-unprivileged:1.29-alpine + +USER root +RUN apk upgrade --no-cache brotli-libs +USER 101 diff --git a/tests/test_vm_image_prefetch.py b/tests/test_vm_image_prefetch.py new file mode 100644 index 0000000..5ef4f0c --- /dev/null +++ b/tests/test_vm_image_prefetch.py @@ -0,0 +1,591 @@ +import base64 +import copy +from datetime import datetime, timezone +import hashlib +import io +import json +import os +from pathlib import Path +import subprocess +import sys +import tempfile +import unittest +from contextlib import redirect_stdout +from types import SimpleNamespace +from unittest.mock import patch + +import yaml + +from scripts import vm_image_prefetch as vm +from scripts import vm_cache_preflight as preflight +from tests.test_collect_node_image_plans import plan as source_plan + +ROOT = Path(__file__).resolve().parents[1] +NOW = datetime(2026, 11, 2, 0, 30, tzinfo=timezone.utc) +KEY = 'ssh-ed25519 ' + base64.b64encode(b'\x00\x00\x00\x0bssh-ed25519\x00\x00\x00\x20' + b'a' * 32).decode() + + +def inputs(): + published = source_plan() + scope = { + 'schema_version': 1, 'selection_mode': 'OPERATOR_APPROVED_ACTIVE_RELEASES', + 'inventory_complete_as_attested': True, + 'observed_at': '2026-11-02T00:00:00Z', 'valid_until': '2026-11-02T01:00:00Z', + 'releases': [{'challenge_slug': 'web-one', + 'release_id': '00000000-0000-4000-8000-000000000001', + 'registry_revision': 1, 'publish_run_id': 100, 'publish_artifact_id': 200, + 'source_sha': '1' * 40, 'artifact_sha256': 'c' * 64, + 'architecture': 'AMD64', 'containers': published['containers'], + 'policy_approved': True, 'approved_target_ids': ['target-one']}], + } + nodes = {'schema_version': 1, 'nodes': [{ + 'node_id': 'vm-one', 'target_id': 'target-one', 'node_uid': 'uid-one', + 'machine_id': 'machine-one', 'boot_id': 'boot-one', 'architecture': 'AMD64', + 'ssh_host': '192.0.2.10', 'ssh_port': 22, 'ssh_user': 'prefetch', + 'ssh_host_key': KEY, 'agent_sha256': 'd' * 64, + 'image_fs_path': '/var/lib/rancher/k3s/agent/containerd', + 'reserve_bytes': 1000, 'max_additional_bytes': 2000, 'per_image_workspace_bytes': 500, + }]} + return scope, nodes + + +def build(scope=None, nodes=None, **kwargs): + default_scope, default_nodes = inputs() + def api(path): + if path == 'git/ref/heads/main': + return {'object': {'sha': '1' * 40}} + if path == 'actions/artifacts/200': + return {'name': 'web-one-100-1-abcdef-publish-bundle'} + if path.startswith('contents/'): + raw = yaml.safe_dump({'deployment': {'runtime_type': 'KUBERNETES'}, + 'flag': 'do-not-copy-author-value'}).encode() + return {'encoding': 'base64', 'size': len(raw), 'content': base64.b64encode(raw).decode()} + raise AssertionError(path) + return vm.build_plan(scope or default_scope, nodes or default_nodes, now=NOW, + api=kwargs.pop('api', api), + fetch=kwargs.pop('fetch', lambda slug, selected: source_plan()), **kwargs) + + +def host_reply(node, plan): + return {**{key: node[key] for key in ('node_uid', 'machine_id', 'boot_id', 'architecture', + 'image_fs_path', 'agent_sha256')}, + 'schema_version': 1, 'os': 'linux', 'available_bytes': 10000, 'total_bytes': 20000, + 'preflight_sha256': plan['preflight_sha256']} + + +def evidence(node, image): + return {'evidence': {**{key: node[key] for key in ('node_uid', 'machine_id', 'boot_id', 'architecture')}, + 'os': 'linux', 'image': image, 'complete': True, + 'resolved_manifest_digest': 'sha256:' + 'a' * 64}} + + +class FakeAgent: + def __init__(self, plan, missing=False, change=None): + self.plan, self.missing, self.change = plan, missing, change + self.calls = [] + + def __call__(self, node, command, request): + self.calls.append((node['node_id'], command, dict(request))) + if command == vm.PREFLIGHT_COMMAND: + reply = host_reply(node, self.plan) + elif self.missing and not request['pull']: + reply = {'error_code': 'CONTENT_INCOMPLETE'} + else: + reply = evidence(node, request['image']) + return self.change(node, command, request, reply) if self.change else reply + + +class VMImagePrefetchTests(unittest.TestCase): + def test_plan_is_deterministic_without_ssh_or_host_changes(self): + one, two = build(), build() + self.assertEqual(one, two) + self.assertTrue(one['complete']) + self.assertEqual(one['plan_sha256'], vm.plan_hash(one)) + self.assertFalse(one['backend_active_independently_queried']) + self.assertNotIn('do-not-copy-author-value', json.dumps(one)) + + def test_expired_future_and_overlong_operator_selection_are_rejected(self): + for key, value in (('valid_until', '2026-11-02T00:30:00Z'), + ('observed_at', '2026-11-02T00:31:00Z'), + ('valid_until', '2026-11-04T00:00:00Z'), + ('observed_at', '2026-11-02T00:00:00')): + scope, nodes = inputs() + scope[key] = value + with self.subTest(key=key, value=value), self.assertRaises(ValueError): + build(scope, nodes) + + def test_inventory_attestation_and_duplicate_releases_are_required(self): + scope, nodes = inputs() + scope['inventory_complete_as_attested'] = False + with self.assertRaises(ValueError): + build(scope, nodes) + scope, nodes = inputs() + scope['releases'] *= 2 + with self.assertRaises(ValueError): + build(scope, nodes) + + def test_policy_approval_is_not_inferred_from_source_success(self): + scope, nodes = inputs() + scope['releases'][0]['policy_approved'] = False + result = build(scope, nodes, fetch=lambda *_: self.fail('must not fetch')) + self.assertFalse(result['complete']) + self.assertEqual(result['held'][0]['reason'], 'POLICY_APPROVAL_NOT_ATTESTED') + + def test_static_docker_and_vm_are_not_silently_preloaded(self): + for kind in ('STATIC', 'DOCKER', 'VM'): + raw = yaml.safe_dump({} if kind == 'STATIC' else {'deployment': {'runtime_type': kind}}).encode() + def api(path): + if path == 'git/ref/heads/main': + return {'object': {'sha': '1' * 40}} + return {'encoding': 'base64', 'size': len(raw), 'content': base64.b64encode(raw).decode()} + with self.subTest(kind=kind): + result = build(api=api, fetch=lambda *_: self.fail('must not fetch')) + self.assertFalse(result['complete']) + self.assertEqual(result['held'][0]['reason'], 'CURRENT_PROBLEM_NOT_KUBERNETES') + + def test_exclusions_are_visible_not_hidden_as_ready(self): + scope, nodes = inputs() + scope['releases'].append({'challenge_slug': 'crypto-reused-seal', 'exclude_reason': 'STATIC_CONVERSION'}) + result = build(scope, nodes) + self.assertTrue(result['complete']) + self.assertEqual(len(result['excluded']), 1) + self.assertEqual(len(result['releases']), 1) + + def test_exact_active_revision_hash_architecture_and_all_containers_match(self): + for change in ({'artifact_sha256': 'f' * 64}, {'architecture': 'ARM64'}, + {'registry_revision': 2}, {'source_sha': '2' * 40}, {'publish_run_id': 101}, + {'containers': [{'name': 'web', 'image': 'ghcr.io/msg-ctf/challenges/web-one/web:latest'}]}, + {'release_id': 'not-a-uuid'}, {'approved_target_ids': ['other-target']}): + scope, nodes = inputs() + scope['releases'][0].update(change) + with self.subTest(change=change): + self.assertFalse(build(scope, nodes)['complete']) + + def test_missing_helper_is_held_not_representative_only(self): + published = source_plan() + published['containers'].append({'name': 'db', 'image': + 'ghcr.io/msg-ctf/challenges/web-one/db@sha256:' + 'b' * 64}) + self.assertFalse(build(fetch=lambda *_: published)['complete']) + + def test_source_failure_or_malformed_yaml_is_redacted(self): + for error in (ValueError('private-marker'), yaml.YAMLError('private-marker'), + OSError('private-marker')): + def fail(*_): + raise error + result = build(fetch=fail) + self.assertFalse(result['complete']) + self.assertNotIn('private-marker', json.dumps(result)) + + def test_mid_audit_main_change_is_rejected(self): + calls = 0 + def api(path): + nonlocal calls + if path == 'git/ref/heads/main': + calls += 1 + return {'object': {'sha': str(calls) * 40}} + if path.startswith('contents/'): + raw = b'deployment: {runtime_type: KUBERNETES}' + return {'encoding': 'base64', 'size': len(raw), 'content': base64.b64encode(raw).decode()} + return {'name': 'web-one-100-1-abcdef-publish-bundle'} + with self.assertRaises(ValueError): + build(api=api) + + def test_all_nodes_of_matching_architecture_are_bound_to_approval(self): + scope, nodes = inputs() + second = dict(nodes['nodes'][0], node_id='vm-two', target_id='target-two', node_uid='uid-two', + machine_id='machine-two', ssh_host='192.0.2.11') + nodes['nodes'].append(second) + self.assertFalse(build(scope, nodes)['complete']) + scope['releases'][0]['approved_target_ids'].append('target-two') + result = build(scope, nodes) + self.assertTrue(result['complete']) + self.assertEqual(len(result['tasks']), 2) + + def test_ssh_injection_duplicate_nodes_and_bad_bounds_are_rejected(self): + for change in ({'ssh_host': '-oProxyCommand=bad'}, {'ssh_host': 'host;bad'}, + {'ssh_host': '127.0.0.1'}, {'ssh_user': 'root;bad'}, + {'ssh_port': True}, {'ssh_port': 65536}, {'agent_sha256': 'bad'}, + {'ssh_host_key': 'ssh-ed25519 not-base64'}, {'image_fs_path': '/tmp/../outside'}, + {'max_additional_bytes': -1}, {'reserve_bytes': True}): + _, nodes = inputs() + nodes['nodes'][0].update(change) + with self.subTest(change=change), self.assertRaises((ValueError, TypeError)): + vm.checked_nodes(nodes) + _, nodes = inputs() + nodes['nodes'] *= 2 + with self.assertRaises(ValueError): + vm.checked_nodes(nodes) + + def test_cache_hit_is_verified_without_any_pull(self): + plan = build() + fake = FakeAgent(plan) + result = vm.execute(plan, plan['plan_sha256'], fake, now=NOW) + self.assertEqual(result['status'], 'SUCCEEDED') + self.assertEqual(result['results'][0]['mode'], 'REUSED_VERIFIED') + self.assertFalse(any(payload.get('pull') for _, _, payload in fake.calls)) + self.assertFalse(result['runtime_readiness_registered']) + self.assertFalse(result['backend_operation_registered']) + self.assertFalse(result['cold_pull_proven']) + + def test_incomplete_content_is_pulled_then_verified_not_claimed_cold(self): + plan = build() + fake = FakeAgent(plan, missing=True) + result = vm.execute(plan, plan['plan_sha256'], fake, now=NOW) + self.assertEqual(result['status'], 'SUCCEEDED') + self.assertEqual(result['results'][0]['mode'], 'PULLED_OR_REPAIRED') + self.assertEqual(sum(payload.get('pull') is True for _, _, payload in fake.calls), 1) + self.assertIn('agent_prepare_seconds', result['results'][0]) + self.assertFalse(result['cold_pull_proven']) + + def test_plan_tampering_wrong_approval_and_expiry_fail_before_ssh(self): + plan = build() + for expected in ('', 'f' * 64): + with self.subTest(expected=expected), self.assertRaises(ValueError): + vm.execute(plan, expected, lambda *_: self.fail('SSH must not start'), now=NOW) + with self.assertRaises(ValueError): + vm.execute(plan, plan['plan_sha256'], lambda *_: self.fail('SSH must not start'), + now=datetime(2026, 11, 3, tzinfo=timezone.utc)) + old = plan['plan_sha256'] + plan['tasks'][0]['images'].append('different-image') + with self.assertRaises(ValueError): + vm.execute(plan, old, lambda *_: self.fail('SSH must not start'), now=NOW) + + def test_held_scope_never_starts_ssh(self): + scope, nodes = inputs() + scope['releases'][0]['policy_approved'] = False + plan = build(scope, nodes) + with self.assertRaises(ValueError): + vm.execute(plan, plan['plan_sha256'], lambda *_: self.fail('SSH must not start'), now=NOW) + + def test_identity_binary_and_disk_mismatch_stop_without_pull(self): + plan = build() + for field, value in (('node_uid', 'wrong'), ('boot_id', 'new-boot'), + ('machine_id', 'wrong'), ('architecture', 'ARM64'), + ('agent_sha256', 'e' * 64), ('preflight_sha256', 'e' * 64), + ('image_fs_path', '/wrong'), ('available_bytes', 1)): + def change(node, command, payload, reply): + if command == vm.PREFLIGHT_COMMAND: + reply[field] = value + return reply + fake = FakeAgent(plan, missing=True, change=change) + with self.subTest(field=field): + result = vm.execute(plan, plan['plan_sha256'], fake, now=NOW) + self.assertEqual(result['status'], 'FAILED') + self.assertFalse(any(payload.get('pull') for _, _, payload in fake.calls)) + + def test_every_vm_is_preflighted_before_the_first_mutation(self): + scope, nodes = inputs() + second = dict(nodes['nodes'][0], node_id='vm-two', node_uid='uid-two', + machine_id='machine-two', ssh_host='192.0.2.11') + nodes['nodes'].append(second) + plan = build(scope, nodes) + def change(node, command, payload, reply): + if node['node_id'] == 'vm-two': + reply['available_bytes'] = 0 + return reply + fake = FakeAgent(plan, missing=True, change=change) + result = vm.execute(plan, plan['plan_sha256'], fake, now=NOW) + self.assertEqual(result['status'], 'FAILED') + self.assertFalse(any(command == vm.AGENT_COMMAND for _, command, _ in fake.calls)) + + def test_agent_policy_auth_and_platform_errors_do_not_trigger_a_pull(self): + plan = build() + for code in ('REGISTRY_UNAUTHORIZED', 'PLATFORM_MISMATCH', 'AGENT_UNAVAILABLE'): + def change(node, command, payload, reply): + return {'error_code': code} if command == vm.AGENT_COMMAND else reply + fake = FakeAgent(plan, change=change) + result = vm.execute(plan, plan['plan_sha256'], fake, now=NOW) + self.assertEqual(result['status'], 'FAILED') + self.assertFalse(any(payload.get('pull') for _, _, payload in fake.calls)) + + def test_wrong_evidence_and_partial_failure_cannot_be_success(self): + plan = build() + for change in ({'complete': False}, {'node_uid': 'wrong'}, {'boot_id': 'wrong'}, + {'architecture': 'ARM64'}, {'image': 'wrong'}, {'resolved_manifest_digest': 'bad'}): + def alter(node, command, payload, reply): + if command == vm.AGENT_COMMAND and 'evidence' in reply: + reply['evidence'].update(change) + return reply + result = vm.execute(plan, plan['plan_sha256'], FakeAgent(plan, change=alter), now=NOW) + self.assertEqual(result['status'], 'FAILED') + self.assertEqual(result['results'], []) + + def test_oci_index_child_digest_is_not_mistaken_for_root_digest_mismatch(self): + plan = build() + def change(node, command, payload, reply): + if command == vm.AGENT_COMMAND: + reply['evidence']['resolved_manifest_digest'] = 'sha256:' + 'b' * 64 + return reply + result = vm.execute(plan, plan['plan_sha256'], FakeAgent(plan, change=change), now=NOW) + self.assertEqual(result['status'], 'SUCCEEDED') + self.assertEqual(result['results'][0]['resolved_manifest_digest'], 'sha256:' + 'b' * 64) + + def test_boot_change_between_inspection_and_pull_stops_mutation(self): + plan, count = build(), 0 + def change(node, command, payload, reply): + nonlocal count + if command == vm.PREFLIGHT_COMMAND: + count += 1 + if count >= 3: + reply['boot_id'] = 'rebooted' + return reply + fake = FakeAgent(plan, missing=True, change=change) + self.assertEqual(vm.execute(plan, plan['plan_sha256'], fake, now=NOW)['status'], 'FAILED') + self.assertFalse(any(payload.get('pull') for _, _, payload in fake.calls)) + + def test_completed_pairs_are_persisted_before_a_later_failure(self): + scope, nodes = inputs() + second = dict(nodes['nodes'][0], node_id='vm-two', node_uid='uid-two', + machine_id='machine-two', ssh_host='192.0.2.11') + nodes['nodes'].append(second) + plan = build(scope, nodes) + def change(node, command, payload, reply): + if node['node_id'] == 'vm-two' and command == vm.AGENT_COMMAND: + raise ValueError('private-server-error') + return reply + saved = [] + result = vm.execute(plan, plan['plan_sha256'], FakeAgent(plan, change=change), now=NOW, + on_result=lambda value: saved.append(copy.deepcopy(value))) + self.assertEqual(result['status'], 'FAILED') + self.assertEqual(len(result['results']), 1) + self.assertEqual(len(saved[0]['results']), 1) + self.assertNotIn('private-server-error', json.dumps(result)) + + def test_ssh_uses_fixed_commands_pinned_hosts_and_no_forwarded_auth(self): + _, inventory = inputs() + node = inventory['nodes'][0] + with tempfile.TemporaryDirectory() as temporary: + key = Path(temporary) / 'key' + key.write_text('private-key-fixture') + key.chmod(0o600) + def runner(args, payload): + self.assertIn('-F', args) + self.assertIn('/dev/null', args) + for option in ('-oStrictHostKeyChecking=yes', '-oForwardAgent=no', + '-oIdentitiesOnly=yes', '-oIdentityAgent=none', '-oProxyCommand=none'): + self.assertIn(option, args) + known = Path(next(value.split('=', 1)[1] for value in args if value.startswith('-oUserKnownHostsFile='))) + self.assertEqual(known.read_text(), node['ssh_host'] + ' ' + KEY + '\n') + self.assertEqual(args[-1], vm.PREFLIGHT_COMMAND) + return {'ok': True} + agent = vm.SSHAgent(key, runner) + self.assertEqual(agent(node, vm.PREFLIGHT_COMMAND, {}), {'ok': True}) + with self.assertRaises(ValueError): + agent(node, 'rm -rf /', {}) + + def test_shared_permissions_empty_key_and_symlink_key_are_rejected(self): + with tempfile.TemporaryDirectory() as temporary: + key = Path(temporary) / 'key' + key.write_text('fixture') + key.chmod(0o644) + with self.assertRaises(ValueError): + vm.SSHAgent(key) + key.chmod(0o600) + link = key.with_name('link') + link.symlink_to(key) + with self.assertRaises(ValueError): + vm.SSHAgent(link) + key.write_text('') + with self.assertRaises(ValueError): + vm.SSHAgent(key) + + def test_bounded_command_rejects_oversized_output_and_timeout(self): + for code, timeout in (('import sys;sys.stdin.read();print("x"*70000)', 2), + ('import time;time.sleep(1)', 0.03)): + with self.subTest(code=code), self.assertRaises((ValueError, subprocess.TimeoutExpired)): + vm.bounded_command([sys.executable, '-c', code], {}, timeout=timeout) + + def test_subprocess_does_not_inherit_tokens(self): + with patch.dict(os.environ, {'GH_TOKEN': 'private-github-token', 'VM_SSH_KEY': 'private-key'}): + reply = vm.bounded_command([sys.executable, '-c', + 'import json,os,sys;sys.stdin.read();print(json.dumps({"clean": "GH_TOKEN" not in os.environ and "VM_SSH_KEY" not in os.environ}))'], {}) + self.assertTrue(reply['clean']) + + def test_invalid_json_and_duplicate_keys_are_rejected(self): + for raw in (b'{"a":1,"a":2}', b'{"x":NaN}', b'x' * (1024 * 1024 + 1)): + with self.subTest(raw=raw[:30]), self.assertRaises(ValueError): + vm.load_json(raw) + + def test_private_reports_are_atomic_and_do_not_overwrite_a_pending_temp(self): + with tempfile.TemporaryDirectory() as temporary: + path = Path(temporary) / 'report.json' + vm.save_private(path, {'ok': True}) + self.assertEqual(path.stat().st_mode & 0o777, 0o600) + vm.save_private(path, {'ok': False}) + self.assertEqual(json.loads(path.read_text()), {'ok': False}) + path.with_suffix('.tmp').write_text('retain') + with self.assertRaises(FileExistsError): + vm.save_private(path, {}) + + def test_missing_ssh_key_does_not_create_an_ssh_process(self): + result = subprocess.run([sys.executable, '-B', '-m', 'scripts.vm_image_prefetch', + '--scope', '/missing', '--nodes', '/missing', + '--output-dir', '/missing', '--apply'], cwd=ROOT, + capture_output=True, text=True, timeout=10) + self.assertEqual(result.returncode, 2) + self.assertNotIn('Traceback', result.stderr) + + def test_cli_plan_keeps_inventory_private_and_never_constructs_ssh(self): + scope, nodes = inputs() + plan = build(scope, nodes) + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + (root / 'scope.json').write_text(json.dumps(scope)) + (root / 'nodes.json').write_text(json.dumps(nodes)) + argv = ['vm_image_prefetch', '--scope', str(root / 'scope.json'), + '--nodes', str(root / 'nodes.json'), '--output-dir', str(root / 'output')] + output = io.StringIO() + with patch.object(sys, 'argv', argv), patch.object(vm, 'build_plan', return_value=plan), \ + patch.object(vm, 'SSHAgent') as ssh, redirect_stdout(output): + self.assertEqual(vm.main(), 0) + ssh.assert_not_called() + self.assertNotIn('192.0.2.10', output.getvalue()) + self.assertNotIn('machine-one', output.getvalue()) + self.assertEqual((root / 'output').stat().st_mode & 0o777, 0o700) + self.assertEqual((root / 'output/plan.json').stat().st_mode & 0o777, 0o600) + self.assertFalse(json.loads((root / 'output/summary.json').read_text())['applied']) + + def test_content_growth_exceeding_budget_is_not_reported_successful(self): + plan, checks = build(), 0 + def change(node, command, request, reply): + nonlocal checks + if command == vm.PREFLIGHT_COMMAND: + checks += 1 + if checks >= 3: + reply['available_bytes'] = 7999 + return reply + result = vm.execute(plan, plan['plan_sha256'], FakeAgent(plan, change=change), now=NOW) + self.assertEqual(result['status'], 'FAILED') + self.assertFalse(result['results']) + + +class VMCachePreflightTests(unittest.TestCase): + def test_snapshot_reads_identity_space_and_binary_hash_without_mutation(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + config, agent_config, binary = root / 'scope.json', root / 'agent.json', root / 'agent' + config.write_text(json.dumps({'image_fs_path': temporary})) + agent_config.write_text(json.dumps({'node_uid': 'node-one', 'k3s_binary': '/usr/local/bin/k3s'})) + binary.write_bytes(b'trusted-agent-fixture') + read_text = Path.read_text + def identity(path, *args, **kwargs): + if str(path) == '/etc/machine-id': + return 'machine-one\n' + if str(path) == '/proc/sys/kernel/random/boot_id': + return 'boot-one\n' + return read_text(path, *args, **kwargs) + with patch.object(preflight, 'CONFIG', config), patch.object(preflight, 'AGENT_CONFIG', agent_config), \ + patch.object(preflight, 'AGENT', binary), patch.object(preflight.os, 'geteuid', return_value=0), \ + patch.object(preflight, 'trusted_file', side_effect=lambda path: path), \ + patch.object(preflight.platform, 'system', return_value='Linux'), \ + patch.object(preflight.platform, 'machine', return_value='x86_64'), \ + patch.object(preflight.os, 'statvfs', return_value=SimpleNamespace(f_bavail=2, f_blocks=4, f_frsize=4096)), \ + patch.object(Path, 'read_text', identity): + reply = preflight.snapshot() + self.assertEqual(reply['available_bytes'], 8192) + self.assertEqual(reply['total_bytes'], 16384) + self.assertEqual(reply['boot_id'], 'boot-one') + self.assertEqual(reply['agent_sha256'], hashlib.sha256(b'trusted-agent-fixture').hexdigest()) + self.assertEqual(reply['node_uid_source'], 'OPERATOR_PINNED_AGENT_CONFIG') + def test_non_root_invocation_fails_before_reading_host_files(self): + with patch.object(preflight.os, 'geteuid', return_value=1000), self.assertRaises(ValueError): + preflight.snapshot() + + def test_symlink_installation_is_not_trusted(self): + with tempfile.TemporaryDirectory() as temporary: + target = Path(temporary) / 'file' + target.write_text('fixture') + link = target.with_name('link') + link.symlink_to(target) + with self.assertRaises(ValueError): + preflight.trusted_file(link) + + def test_host_errors_do_not_disclose_paths_or_config(self): + output = io.StringIO() + with patch.object(preflight, 'snapshot', side_effect=OSError('private-marker')), redirect_stdout(output): + self.assertEqual(preflight.main(), 1) + self.assertEqual(json.loads(output.getvalue()), {'error_code': 'PREFLIGHT_FAILED'}) + self.assertNotIn('private-marker', output.getvalue()) + + +class VMPrefetchWorkflowTests(unittest.TestCase): + def test_native_cms_roundtrip_preserves_private_evidence(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + commands = [ + ['openssl', 'req', '-x509', '-newkey', 'rsa:2048', '-nodes', '-days', '1', + '-subj', '/CN=prefetch-regression', '-keyout', str(root / 'key.pem'), '-out', str(root / 'cert.pem')], + ['openssl', 'x509', '-in', str(root / 'cert.pem'), '-checkend', '10800', '-noout'], + ] + (root / 'evidence.tar').write_bytes(b'private-test-evidence') + commands += [ + ['openssl', 'cms', '-encrypt', '-binary', '-aes-256-cbc', '-outform', 'DER', + '-in', str(root / 'evidence.tar'), '-out', str(root / 'evidence.cms'), str(root / 'cert.pem')], + ['openssl', 'cms', '-decrypt', '-binary', '-inform', 'DER', '-in', str(root / 'evidence.cms'), + '-recip', str(root / 'cert.pem'), '-inkey', str(root / 'key.pem'), '-out', str(root / 'recovered.tar')], + ] + for command in commands: + result = subprocess.run(command, capture_output=True, timeout=20) + self.assertEqual(result.returncode, 0, 'Native CMS operation failed') + self.assertEqual((root / 'recovered.tar').read_bytes(), b'private-test-evidence') + self.assertNotIn(b'private-test-evidence', (root / 'evidence.cms').read_bytes()) + def workflow(self): + return yaml.load((ROOT / '.github/workflows/vm-image-prefetch.yml').read_text(), Loader=yaml.BaseLoader) + + def test_write_is_manual_disabled_by_default_and_requires_approval_environment(self): + workflow = self.workflow() + self.assertEqual(set(workflow['on']), {'workflow_dispatch'}) + self.assertEqual(workflow['on']['workflow_dispatch']['inputs']['apply']['default'], 'false') + self.assertEqual(workflow['permissions'], {'contents': 'read'}) + self.assertEqual(workflow['jobs']['apply']['environment'], 'vm-image-prefetch') + self.assertEqual(workflow['jobs']['apply']['if'], 'inputs.apply') + self.assertEqual(workflow['concurrency']['cancel-in-progress'], 'false') + plan_text = json.dumps(workflow['jobs']['plan']) + self.assertNotIn('VM_IMAGE_PREFETCH_SSH_KEY', plan_text) + self.assertIn('VM_IMAGE_PREFETCH_ENABLED', plan_text) + + def test_guard_actually_rejects_wrong_event_branch_repository_and_disabled_apply(self): + command = self.workflow()['jobs']['plan']['steps'][0]['run'] + base = {**os.environ, 'GITHUB_REPOSITORY': 'MSG-CTF/msgctf-devsecops', + 'GITHUB_EVENT_NAME': 'workflow_dispatch', 'GITHUB_REF': 'refs/heads/main', + 'APPLY': 'false', 'ENABLED': '', 'EXPECTED_SHA': ''} + cases = [({}, 0), ({'GITHUB_EVENT_NAME': 'push'}, 1), + ({'GITHUB_EVENT_NAME': 'schedule'}, 1), ({'GITHUB_REF': 'refs/heads/feature'}, 1), + ({'GITHUB_REPOSITORY': 'other/repo'}, 1), ({'APPLY': 'true'}, 1), + ({'APPLY': 'true', 'ENABLED': 'true', 'EXPECTED_SHA': 'bad'}, 1), + ({'APPLY': 'true', 'ENABLED': 'true', 'EXPECTED_SHA': 'a' * 64}, 0)] + for change, expected in cases: + result = subprocess.run(['bash', '-c', command], env={**base, **change}, + capture_output=True, timeout=5) + with self.subTest(change=change): + self.assertEqual(result.returncode, expected) + + def test_only_aggregate_reports_are_uploaded_and_private_files_are_cleaned(self): + workflow = self.workflow() + for job in workflow['jobs'].values(): + for step in job['steps']: + if 'actions/upload-artifact' in step.get('uses', ''): + self.assertTrue('summary.json' in step['with']['path'] or + 'vm-prefetch-private-results.cms' in step['with']['path']) + self.assertNotIn('plan.json', step['with']['path']) + self.assertNotIn('ssh-key', step['with']['path']) + text = (ROOT / '.github/workflows/vm-image-prefetch.yml').read_text() + self.assertIn('trap ', text) + self.assertNotIn('packages: write', text) + self.assertNotIn('id-token: write', text) + + def test_report_recipient_is_checked_before_any_ssh_execution(self): + step = next(step for step in self.workflow()['jobs']['apply']['steps'] + if 'REPORT_CERT' in step.get('env', {})) + self.assertLess(step['run'].index('openssl x509'), step['run'].index('python3 -B -m scripts.vm_image_prefetch')) + self.assertIn('REPORT_CERT_SHA', step['run']) + self.assertIn('openssl cms -encrypt -binary -aes-256-cbc', step['run']) + + def test_workflow_only_changes_trigger_existing_ci(self): + workflow = yaml.load((ROOT / '.github/workflows/pipeline-self-test.yml').read_text(), Loader=yaml.BaseLoader) + for event in ('push', 'pull_request'): + self.assertIn('.github/workflows/vm-image-prefetch.yml', workflow['on'][event]['paths']) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_workflow_contracts.py b/tests/test_workflow_contracts.py index 6362cdb..4542c8c 100644 --- a/tests/test_workflow_contracts.py +++ b/tests/test_workflow_contracts.py @@ -464,6 +464,7 @@ def test_repository_only_exposes_challenge_validation_workflows(self): "ghcr-cleanup-dry-run.yml", "published-runtime-smoke.yml", "node-image-prefetch-plan.yml", + "vm-image-prefetch.yml", }, ) @@ -506,6 +507,16 @@ def test_pipeline_self_test_calls_reusable_supply_chain(self): workflow["jobs"][job_name].get("if", ""), ) + def test_self_test_server_fixture_updates_brotli_and_keeps_non_root_user(self): + path = ROOT / "tests/fixtures/info-valid/prob/for_organizer/web/Dockerfile" + instructions = [line.strip() for line in path.read_text().splitlines() if line.strip()] + self.assertEqual(instructions, [ + "FROM nginxinc/nginx-unprivileged:1.29-alpine", + "USER root", + "RUN apk upgrade --no-cache brotli-libs", + "USER 101", + ]) + def test_caller_example_separates_branch_validation_from_main_publish(self): path = ROOT / "docs/challenge-caller-example.yml" workflow = load_workflow(path) From 6bba257369194aae6fe47e940a21d2c6e942b090 Mon Sep 17 00:00:00 2001 From: wlsdnd2367 <230291092+wlsdnd2367@users.noreply.github.com> Date: Mon, 2 Nov 2026 10:53:19 +0900 Subject: [PATCH 2/4] =?UTF-8?q?fix:=20=EC=82=AC=EC=A0=84=20=EA=B2=80?= =?UTF-8?q?=EC=82=AC=20=EB=A9=94=EB=AA=A8=EB=A6=AC=20=EC=82=AC=EC=9A=A9?= =?UTF-8?q?=EA=B3=BC=20=EC=9D=BD=EA=B8=B0=20=EA=B6=8C=ED=95=9C=20=EC=95=88?= =?UTF-8?q?=EB=82=B4=20=EB=B3=B4=EC=99=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/vm-image-prefetch.md | 3 ++- scripts/vm_cache_preflight.py | 10 +++++++++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/docs/vm-image-prefetch.md b/docs/vm-image-prefetch.md index e86d137..d9c873f 100644 --- a/docs/vm-image-prefetch.md +++ b/docs/vm-image-prefetch.md @@ -144,7 +144,7 @@ python3 -B -m scripts.vm_image_prefetch --scope /private/scope.json --nodes /pri 필요합니다. 이를 위해 SSH 방화벽을 인터넷 전체에 열지 않습니다. IAP/VPN 어댑터는 포함하지 않습니다. - repo Secrets: VM_IMAGE_PREFETCH_SCOPE, VM_IMAGE_PREFETCH_NODES, - CHALLENGE_ARTIFACT_READ_TOKEN (문제 저장소 Actions 읽기). + CHALLENGE_ARTIFACT_READ_TOKEN (문제 저장소 Contents: Read + Actions: Read). - `vm-image-prefetch` Environment Secret: VM_IMAGE_PREFETCH_SSH_KEY, VM_IMAGE_PREFETCH_REPORT_CERT (결과 수신자의 공개 PEM 인증서, 개인키 아님). - Environment 변수 VM_IMAGE_PREFETCH_REPORT_CERT_SHA256: 인증서 DER SHA-256. @@ -157,3 +157,4 @@ python3 -B -m scripts.vm_image_prefetch --scope /private/scope.json --nodes /pri 확인합니다. 공개 artifact에는 집계와 암호화된 CMS 결과만 올립니다. 상세 plan/results는 수신 인증서로 AES-256-CBC CMS 암호화하고 checksum을 남기며, 원문·키는 러너에서 삭제합니다. 수신 개인키는 러너나 채팅에 전달하지 않습니다. 실제 실행은 별도 정비 일정에 승인합니다. +첫 실제 실행은 승인된 WEB1개·VM1곳으로 확인하고, 성공한 뒤 전체 정비 범위로 확대합니다. diff --git a/scripts/vm_cache_preflight.py b/scripts/vm_cache_preflight.py index cad871e..0d4cbd1 100644 --- a/scripts/vm_cache_preflight.py +++ b/scripts/vm_cache_preflight.py @@ -13,6 +13,14 @@ AGENT = Path('/usr/local/libexec/runtime-image-agent') +def file_digest(path): + digest = hashlib.sha256() + with path.open('rb') as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b''): + digest.update(chunk) + return digest.hexdigest() + + def trusted_file(path): if path.is_symlink(): raise ValueError('UNTRUSTED_INSTALLATION') @@ -56,7 +64,7 @@ def snapshot(): 'image_fs_path': str(image_fs), 'available_bytes': disk.f_bavail * disk.f_frsize, 'total_bytes': disk.f_blocks * disk.f_frsize, - 'agent_sha256': hashlib.sha256(AGENT.read_bytes()).hexdigest(), + 'agent_sha256': file_digest(AGENT), 'preflight_sha256': hashlib.sha256(Path(__file__).read_bytes()).hexdigest(), } From a6289869da464aeba1b95314507800e8c565c25e Mon Sep 17 00:00:00 2001 From: wlsdnd2367 <230291092+wlsdnd2367@users.noreply.github.com> Date: Mon, 2 Nov 2026 10:58:12 +0900 Subject: [PATCH 3/4] =?UTF-8?q?fix:=20=EC=88=98=EB=8F=99=20=EC=A0=95?= =?UTF-8?q?=EB=B9=84=20=EC=9D=B4=EB=AF=B8=EC=A7=80=EC=99=80=20VM=20?= =?UTF-8?q?=EC=A1=B0=ED=95=A9=20=EC=83=81=ED=95=9C=20=EA=B2=80=EC=A6=9D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/vm-image-prefetch.md | 1 + scripts/vm_image_prefetch.py | 2 ++ tests/test_vm_image_prefetch.py | 7 +++++++ 3 files changed, 10 insertions(+) diff --git a/docs/vm-image-prefetch.md b/docs/vm-image-prefetch.md index d9c873f..a91e11f 100644 --- a/docs/vm-image-prefetch.md +++ b/docs/vm-image-prefetch.md @@ -105,6 +105,7 @@ NOT_IN_THIS_MAINTENANCE입니다. 제외를 준비 성공으로 집계하지 않 | per_image_workspace_bytes | 이미지 한 건의 임시·압축 해제 포함 작업 여유 | 동일 architecture의 입력 VM 전체를 각 release의 approved_target_ids가 포함해야 합니다. +한 정비 계획은 최대50개 VM·100개 release·1000개 이미지/VM 쌍까지 허용합니다. VM 주소·Node UID·machine ID 중복이나 누락을 허용하지 않습니다. host key를 현장에서 무인증 `ssh-keyscan` 결과만으로 신뢰하거나 StrictHostKeyChecking을 끄지 않습니다. diff --git a/scripts/vm_image_prefetch.py b/scripts/vm_image_prefetch.py index b6ef150..7aa5fba 100644 --- a/scripts/vm_image_prefetch.py +++ b/scripts/vm_image_prefetch.py @@ -35,6 +35,7 @@ AGENT_COMMAND = 'sudo -n /usr/local/libexec/runtime-image-agent' PREFLIGHT_COMMAND = 'sudo -n /usr/local/libexec/vm-image-cache-preflight' MAX_BYTES = 1024 * 1024 +MAX_PAIRS = 1000 def require(condition): @@ -214,6 +215,7 @@ def build_plan(scope, inventory, *, api=sources.publication.github, fetch=None, if not images: held.append({'node_id': node['node_id'], 'reason': 'NO_APPROVED_IMAGES_FOR_NODE'}) tasks.append({'node_id': node['node_id'], 'images': images}) + require(sum(len(task['images']) for task in tasks) <= MAX_PAIRS) plan = {'schema_version': 1, 'scope': 'PROVIDED_ACTIVE_RELEASES_AND_NODES_ONLY', 'repository': REPOSITORY, 'source_main_sha': main_sha, 'observed_at': scope['observed_at'], 'valid_until': scope['valid_until'], 'complete': not held, 'releases': plans, diff --git a/tests/test_vm_image_prefetch.py b/tests/test_vm_image_prefetch.py index 5ef4f0c..87a57a0 100644 --- a/tests/test_vm_image_prefetch.py +++ b/tests/test_vm_image_prefetch.py @@ -201,6 +201,13 @@ def test_all_nodes_of_matching_architecture_are_bound_to_approval(self): self.assertTrue(result['complete']) self.assertEqual(len(result['tasks']), 2) + def test_oversized_maintenance_is_rejected_before_ssh(self): + scope, nodes = inputs() + nodes['nodes'].append(dict(nodes['nodes'][0], node_id='vm-two', node_uid='uid-two', + machine_id='machine-two', ssh_host='192.0.2.11')) + with patch.object(vm, 'MAX_PAIRS', 1), self.assertRaises(ValueError): + build(scope, nodes) + def test_ssh_injection_duplicate_nodes_and_bad_bounds_are_rejected(self): for change in ({'ssh_host': '-oProxyCommand=bad'}, {'ssh_host': 'host;bad'}, {'ssh_host': '127.0.0.1'}, {'ssh_user': 'root;bad'}, From 543190ae304db27f28cb6a2b390a2bc569d05b68 Mon Sep 17 00:00:00 2001 From: wlsdnd2367 <230291092+wlsdnd2367@users.noreply.github.com> Date: Mon, 2 Nov 2026 11:07:48 +0900 Subject: [PATCH 4/4] Verify every VM cache again before reporting completion --- docs/vm-image-prefetch.md | 3 ++ scripts/vm_image_prefetch.py | 19 +++++++- tests/test_vm_image_prefetch.py | 82 ++++++++++++++++++++++++++++++++- 3 files changed, 102 insertions(+), 2 deletions(-) diff --git a/docs/vm-image-prefetch.md b/docs/vm-image-prefetch.md index a91e11f..508b445 100644 --- a/docs/vm-image-prefetch.md +++ b/docs/vm-image-prefetch.md @@ -128,6 +128,9 @@ python3 -B -m scripts.vm_image_prefetch --scope /private/scope.json --nodes /pri 후자는 cold pull의 증거가 아닙니다. agent_prepare_seconds도 실제 registry 전송 시간만을 측정한 값이 아닙니다. 실패 시 다음 이미지로 넘어가지 않으며, 캐시 삭제·자동 retry는 없습니다. 다시 승인해 실행하면 이미 준비된 이미지를 검증·재사용할 수 있습니다. +마지막에는 전체 이미지/VM 쌍을 pull 없이 다시 검사합니다. 최종 검사에서 GC·내용 누락· +identity 변경이 발견되면 완료로 표시하지 않고 중단합니다. 최종 관측 시각도 각 쌍에 남깁니다. +검사 중 승인이 만료되면 실제 pull을 시작하지 않으며, 만료된 최종 검사를 성공으로 기록하지 않습니다. 새 출력 디렉터리는0700, plan/results/summary는0600입니다. 결과가 부분 실패해도 완료한 쌍과 실패 위치를 남깁니다. Runtime readiness·Backend operation에 결과를 등록하지 않으므로 diff --git a/scripts/vm_image_prefetch.py b/scripts/vm_image_prefetch.py index 7aa5fba..7986bf1 100644 --- a/scripts/vm_image_prefetch.py +++ b/scripts/vm_image_prefetch.py @@ -318,6 +318,7 @@ def execute(plan, expected_sha, call, *, now=None, on_result=None): and expected_sha == plan['plan_sha256'] == plan_hash(plan)) require((now or datetime.now(timezone.utc)) < timestamp(plan['valid_until'])) report = {'schema_version': 1, 'plan_sha256': expected_sha, 'status': 'FAILED', 'results': [], + 'final_sweep_complete': False, 'runtime_readiness_registered': False, 'backend_operation_registered': False, 'pod_started': False, 'active_changed': False, 'cold_pull_proven': False} baseline = {} @@ -339,6 +340,7 @@ def execute(plan, expected_sha, call, *, now=None, on_result=None): if isinstance(reply, dict) and reply.get('error_code') == 'CONTENT_INCOMPLETE' and set(reply) == {'error_code'}: check_host(node, plan, call(node, PREFLIGHT_COMMAND, {}), baseline[node['node_id']], workspace=node['per_image_workspace_bytes']) + require((now or datetime.now(timezone.utc)) < timestamp(plan['valid_until'])) request['pull'] = True context['stage'] = 'PULL_OR_REPAIR' started = time.monotonic() @@ -354,6 +356,20 @@ def execute(plan, expected_sha, call, *, now=None, on_result=None): 'verified_at': datetime.now(timezone.utc).isoformat(), **evidence}) if on_result: on_result(report) + indexed_nodes = {node['node_id']: node for node in plan['nodes']} + for row in report['results']: + node = indexed_nodes[row['node_id']] + context = {'node_id': node['node_id'], 'image': row['image'], 'stage': 'FINAL_CACHE_CHECK'} + require((now or datetime.now(timezone.utc)) < timestamp(plan['valid_until'])) + check_host(node, plan, call(node, PREFLIGHT_COMMAND, {}), baseline[node['node_id']]) + checked_evidence(node, row['image'], call(node, AGENT_COMMAND, + {'image': row['image'], 'architecture': node['architecture'], 'pull': False})) + check_host(node, plan, call(node, PREFLIGHT_COMMAND, {}), baseline[node['node_id']]) + require((now or datetime.now(timezone.utc)) < timestamp(plan['valid_until'])) + row['final_verified_at'] = datetime.now(timezone.utc).isoformat() + if on_result: + on_result(report) + report['final_sweep_complete'] = True report['status'] = 'SUCCEEDED' except (OSError, ValueError, TypeError, KeyError, subprocess.SubprocessError): report['error_code'] = 'PREFLIGHT_SSH_OR_AGENT_CHECK_FAILED' @@ -395,7 +411,8 @@ def main(): require(args.ssh_key is not None) result = execute(plan, args.expected_plan_sha256, SSHAgent(args.ssh_key), on_result=lambda value: save_private(args.output_dir / 'results.json', value)) - summary.update(applied=True, status=result['status'], checked_pairs=len(result['results'])) + summary.update(applied=True, status=result['status'], checked_pairs=len(result['results']), + final_verified_pairs=sum('final_verified_at' in row for row in result['results'])) code = 0 if result['status'] == 'SUCCEEDED' else 1 save_private(args.output_dir / 'summary.json', summary) print(json.dumps(summary)) diff --git a/tests/test_vm_image_prefetch.py b/tests/test_vm_image_prefetch.py index 87a57a0..bd43743 100644 --- a/tests/test_vm_image_prefetch.py +++ b/tests/test_vm_image_prefetch.py @@ -83,14 +83,17 @@ class FakeAgent: def __init__(self, plan, missing=False, change=None): self.plan, self.missing, self.change = plan, missing, change self.calls = [] + self.cached = set() def __call__(self, node, command, request): self.calls.append((node['node_id'], command, dict(request))) if command == vm.PREFLIGHT_COMMAND: reply = host_reply(node, self.plan) - elif self.missing and not request['pull']: + elif self.missing and not request['pull'] and (node['node_id'], request['image']) not in self.cached: reply = {'error_code': 'CONTENT_INCOMPLETE'} else: + if request['pull']: + self.cached.add((node['node_id'], request['image'])) reply = evidence(node, request['image']) return self.change(node, command, request, reply) if self.change else reply @@ -233,6 +236,54 @@ def test_cache_hit_is_verified_without_any_pull(self): self.assertFalse(result['runtime_readiness_registered']) self.assertFalse(result['backend_operation_registered']) self.assertFalse(result['cold_pull_proven']) + self.assertTrue(result['final_sweep_complete']) + self.assertIn('final_verified_at', result['results'][0]) + + def test_eviction_before_final_sweep_is_failure_without_automatic_repull(self): + plan, inspections = build(), 0 + def change(node, command, payload, reply): + nonlocal inspections + if command == vm.AGENT_COMMAND: + inspections += 1 + if inspections == 2: + return {'error_code': 'CONTENT_INCOMPLETE'} + return reply + fake = FakeAgent(plan, change=change) + result = vm.execute(plan, plan['plan_sha256'], fake, now=NOW) + self.assertEqual(result['status'], 'FAILED') + self.assertFalse(result['final_sweep_complete']) + self.assertEqual(result['failure']['stage'], 'FINAL_CACHE_CHECK') + self.assertFalse(any(payload.get('pull') for _, _, payload in fake.calls)) + + def test_identity_drift_during_final_sweep_cannot_be_complete(self): + plan, observations = build(), 0 + def change(node, command, payload, reply): + nonlocal observations + if command == vm.PREFLIGHT_COMMAND: + observations += 1 + if observations >= 4: + reply['boot_id'] = 'changed-after-download' + return reply + result = vm.execute(plan, plan['plan_sha256'], FakeAgent(plan, change=change), now=NOW) + self.assertEqual(result['status'], 'FAILED') + self.assertFalse(result['final_sweep_complete']) + + def test_partial_final_sweep_keeps_completed_checks_and_failure_location(self): + scope, nodes = inputs() + nodes['nodes'].append(dict(nodes['nodes'][0], node_id='vm-two', node_uid='uid-two', + machine_id='machine-two', ssh_host='192.0.2.11')) + plan, inspections = build(scope, nodes), {} + def change(node, command, payload, reply): + if command == vm.AGENT_COMMAND: + count = inspections.get(node['node_id'], 0) + 1 + inspections[node['node_id']] = count + if node['node_id'] == 'vm-two' and count == 2: + return {'error_code': 'CONTENT_INCOMPLETE'} + return reply + result = vm.execute(plan, plan['plan_sha256'], FakeAgent(plan, change=change), now=NOW) + self.assertEqual(result['status'], 'FAILED') + self.assertEqual(sum('final_verified_at' in row for row in result['results']), 1) + self.assertEqual(result['failure']['node_id'], 'vm-two') def test_incomplete_content_is_pulled_then_verified_not_claimed_cold(self): plan = build() @@ -340,6 +391,35 @@ def change(node, command, payload, reply): self.assertEqual(vm.execute(plan, plan['plan_sha256'], fake, now=NOW)['status'], 'FAILED') self.assertFalse(any(payload.get('pull') for _, _, payload in fake.calls)) + def test_approval_expiring_during_inspection_stops_before_pull(self): + plan = build() + clock = SimpleNamespace(now=lambda _: NOW, fromisoformat=datetime.fromisoformat) + def change(node, command, payload, reply): + if command == vm.AGENT_COMMAND: + clock.now = lambda _: datetime(2026, 11, 3, tzinfo=timezone.utc) + return reply + fake = FakeAgent(plan, missing=True, change=change) + with patch.object(vm, 'datetime', clock): + result = vm.execute(plan, plan['plan_sha256'], fake) + self.assertEqual(result['status'], 'FAILED') + self.assertFalse(any(payload.get('pull') for _, _, payload in fake.calls)) + + def test_approval_expiring_during_final_sweep_cannot_report_success(self): + plan, inspections = build(), 0 + clock = SimpleNamespace(now=lambda _: NOW, fromisoformat=datetime.fromisoformat) + def change(node, command, payload, reply): + nonlocal inspections + if command == vm.AGENT_COMMAND: + inspections += 1 + if inspections == 2: + clock.now = lambda _: datetime(2026, 11, 3, tzinfo=timezone.utc) + return reply + with patch.object(vm, 'datetime', clock): + result = vm.execute(plan, plan['plan_sha256'], FakeAgent(plan, change=change)) + self.assertEqual(result['status'], 'FAILED') + self.assertFalse(result['final_sweep_complete']) + self.assertNotIn('final_verified_at', result['results'][0]) + def test_completed_pairs_are_persisted_before_a_later_failure(self): scope, nodes = inputs() second = dict(nodes['nodes'][0], node_id='vm-two', node_uid='uid-two',