From 23f7d5d51c76b5ae3d46d039c3675e44851a6923 Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Wed, 14 Oct 2026 15:20:20 +0200 Subject: [PATCH 1/2] ci(test): separate e2e families from driver target variants Signed-off-by: Evan Lezar --- .agents/skills/watch-github-actions/SKILL.md | 8 + .github/workflows/branch-checks.yml | 3 + .github/workflows/branch-e2e.yml | 240 +++---------- .github/workflows/e2e-conformance.yml | 57 ++++ .github/workflows/e2e-docker-test.yml | 6 + .github/workflows/e2e-drivers.yml | 62 ++++ .github/workflows/e2e-features.yml | 48 +++ .github/workflows/e2e-gpu-test.yaml | 13 + .github/workflows/e2e-kubernetes-test.yml | 12 +- .github/workflows/e2e-podman-test.yml | 6 + .github/workflows/e2e-runtime-tests.yml | 319 ++++++++++++++++++ .github/workflows/e2e-vm-test.yml | 6 + .github/workflows/release-dev.yml | 54 +-- .github/workflows/release-tag.yml | 60 ++-- CI.md | 26 +- TESTING.md | 28 ++ e2e/rust/e2e-docker.sh | 9 +- e2e/rust/e2e-kubernetes.sh | 9 +- e2e/rust/e2e-podman.sh | 11 +- e2e/rust/e2e-vm.sh | 10 +- e2e/support/test-family.sh | 32 ++ tasks/scripts/e2e_lanes_test.py | 299 ++++++++++++++++ .../scripts/generate-qualification-summary.sh | 6 +- tasks/scripts/select-e2e-runtime-jobs.sh | 33 ++ tasks/scripts/test-qualification-summary.sh | 15 + tasks/test.toml | 7 + 26 files changed, 1097 insertions(+), 282 deletions(-) create mode 100644 .github/workflows/e2e-conformance.yml create mode 100644 .github/workflows/e2e-drivers.yml create mode 100644 .github/workflows/e2e-features.yml create mode 100644 .github/workflows/e2e-runtime-tests.yml create mode 100644 e2e/support/test-family.sh create mode 100644 tasks/scripts/e2e_lanes_test.py create mode 100644 tasks/scripts/select-e2e-runtime-jobs.sh diff --git a/.agents/skills/watch-github-actions/SKILL.md b/.agents/skills/watch-github-actions/SKILL.md index 8ba30a8f54..60028ef5fe 100644 --- a/.agents/skills/watch-github-actions/SKILL.md +++ b/.agents/skills/watch-github-actions/SKILL.md @@ -152,6 +152,14 @@ Direct Nix builds and app dependency preparation also retry once; apps run once after preparation succeeds. Skipped dependent E2E suites indicate blocked coverage. +For E2E, inspect the Conformance, Features, and Drivers family calls in +`Branch E2E Checks`, `Release Dev`, or `Release Tag`. Runtime family selection +is logged by `e2e-runtime-tests.yml`; its completion check fails when a selected +job fails, is cancelled, or is skipped. External VM has separate conformance +and runtime jobs. Existing `Core E2E result` and `GPU E2E result` names remain +the PR gate adapters; optional Kubernetes and advisory WSL policy is unchanged. +The manual `Integration Tests` workflow still accepts a custom tmachine matrix. + For `Trivy Changes`, inspect the `Resolve PR baseline` step for the base and head SHAs. PR runs compare the tested merge commit with its first parent; change detection and scans must use the same pair. On reruns, do diff --git a/.github/workflows/branch-checks.yml b/.github/workflows/branch-checks.yml index af6692a7dc..c649246f65 100644 --- a/.github/workflows/branch-checks.yml +++ b/.github/workflows/branch-checks.yml @@ -522,6 +522,9 @@ jobs: bash tasks/scripts/test-gateway-pull-policy.sh bash tasks/scripts/test-gateway-config.sh + - name: Test E2E family selection and coverage + run: mise run test:e2e-lanes + - name: Test release qualification summary run: mise run test:qualification-summary go: diff --git a/.github/workflows/branch-e2e.yml b/.github/workflows/branch-e2e.yml index db4c96b936..fc55b40ea8 100644 --- a/.github/workflows/branch-e2e.yml +++ b/.github/workflows/branch-e2e.yml @@ -240,268 +240,126 @@ jobs: deb-artifact-name: deb-linux-amd64 rpm-artifact-name: rpm-linux-x86_64 - # Run driver-independent conformance tests. - conformance-integration: - needs: prepare-integration + conformance: + needs: [pr_metadata, prepare-integration, build-gateway-plain, build-external-drivers, build-vm-driver] + if: needs.pr_metadata.outputs.run_core_e2e == 'true' permissions: actions: read contents: read packages: read - uses: ./.github/workflows/integration-runner.yml + uses: ./.github/workflows/e2e-conformance.yml with: - category: conformance source-sha: ${{ needs.prepare-integration.outputs.source_sha }} + image-tag: ${{ github.sha }} integration-inputs-artifact-id: ${{ needs.prepare-integration.outputs.integration_inputs_artifact_id }} - test-matrix: >- - [ - {"environment":"ubuntu-docker-rootful","installer":"deb","testsuite":"conformance"}, - {"environment":"ubuntu-k3s","installer":"k3s","testsuite":"conformance"}, - {"environment":"ubuntu-k3s","installer":"k3s-ha","testsuite":"conformance"}, - {"environment":"fedora-podman-rootful","installer":"rpm","testsuite":"conformance"}, - {"environment":"fedora-podman-rootless","installer":"rpm","testsuite":"conformance"} - ] - # Run feature-specific integration tests: - feature-specific-integration: - needs: prepare-integration + features: + needs: [pr_metadata, prepare-integration] + if: needs.pr_metadata.outputs.run_core_e2e == 'true' permissions: actions: read contents: read packages: read - uses: ./.github/workflows/integration-runner.yml + uses: ./.github/workflows/e2e-features.yml with: - category: feature-specific source-sha: ${{ needs.prepare-integration.outputs.source_sha }} + image-tag: ${{ github.sha }} integration-inputs-artifact-id: ${{ needs.prepare-integration.outputs.integration_inputs_artifact_id }} - test-matrix: >- - [ - {"environment":"fedora-podman-rootful","installer":"rpm","testsuite":"provider-refresh"}, - {"environment":"fedora-podman-rootless","installer":"rpm","testsuite":"provider-refresh"}, - {"environment":"ubuntu-docker-rootful","installer":"deb","testsuite":"oci-image"}, - {"environment":"fedora-podman-rootful","installer":"rpm","testsuite":"oci-image"}, - {"environment":"fedora-podman-rootless","installer":"rpm","testsuite":"oci-image"} - ] - # Run driver-specific integration tests: - # These suites require the binary installer's CLI path, local HTTP gateway, - # and system service configuration for their Podman user-namespace fixtures. - driver-specific-integration: - needs: prepare-integration + drivers: + needs: [pr_metadata, prepare-integration, build-gateway-plain, build-external-drivers, build-vm-driver] + if: needs.pr_metadata.outputs.run_core_e2e == 'true' permissions: actions: read contents: read packages: read - uses: ./.github/workflows/integration-runner.yml + uses: ./.github/workflows/e2e-drivers.yml with: - category: driver-specific source-sha: ${{ needs.prepare-integration.outputs.source_sha }} - integration-inputs-artifact-id: ${{ needs.prepare-integration.outputs.integration_inputs_artifact_id }} - test-matrix: >- - [ - {"environment":"fedora-podman-rootful","installer":"binaries","testsuite":"driver-podman"}, - {"environment":"fedora-podman-rootless","installer":"binaries","testsuite":"e2e-podman"}, - {"environment":"fedora-podman-rootless","installer":"binaries","testsuite":"driver-podman"} - ] - - docker-e2e: - needs: [pr_metadata, build-binaries, build-images] - if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true' - permissions: - actions: read - contents: read - packages: read - uses: ./.github/workflows/e2e-docker-test.yml - with: image-tag: ${{ github.sha }} - runner: linux-arm64-cpu8 - - podman-e2e: - needs: [pr_metadata, build-binaries, build-images] - if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true' - permissions: - actions: read - contents: read - packages: read - uses: ./.github/workflows/e2e-podman-test.yml - with: - image-tag: ${{ github.sha }} - suite-matrix: >- - [{"suite":"ci","runner":"ubuntu-26.04","podman_major":"5","podman_package_version":"5.7.0+ds2-3build1","conmon_package_version":"2.1.13+ds1-2","cmd":"mise run --no-deps --skip-deps e2e:podman:ci"}] - - vm-e2e: - needs: [pr_metadata, build-binaries, build-vm-driver] - if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true' - permissions: - actions: read - contents: read - packages: read - uses: ./.github/workflows/e2e-vm-test.yml - - docker-external-driver-e2e: - needs: [pr_metadata, build-binaries, build-gateway-plain, build-external-drivers, build-images] - if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true' - permissions: - actions: read - contents: read - packages: read - uses: ./.github/workflows/e2e-docker-test.yml - with: - image-tag: ${{ github.sha }} - runner: linux-arm64-cpu8 - gateway-artifact: openshell-gateway-plain-aarch64-unknown-linux-gnu - external-driver-binary: openshell-driver-docker - suite-matrix: >- - [{"suite":"external-driver","cmd":"mise run --no-deps --skip-deps e2e:docker:external-driver","apt_packages":"openssh-client","python_proto":false,"mcp":false}] - - podman-external-driver-e2e: - needs: [pr_metadata, build-binaries, build-gateway-plain, build-external-drivers, build-images] - if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true' - permissions: - actions: read - contents: read - packages: read - uses: ./.github/workflows/e2e-podman-test.yml - with: - image-tag: ${{ github.sha }} - gateway-artifact: openshell-gateway-plain-x86_64-unknown-linux-gnu - external-driver-binary: openshell-driver-podman - suite-matrix: >- - [{"suite":"external-driver","runner":"ubuntu-26.04","podman_major":"5","podman_package_version":"5.7.0+ds2-3build1","conmon_package_version":"2.1.13+ds1-2","cmd":"mise run --no-deps --skip-deps e2e:podman:external-driver"}] - - vm-external-driver-e2e: - needs: [pr_metadata, build-binaries, build-gateway-plain, build-vm-driver] - if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true' - permissions: - actions: read - contents: read - packages: read - uses: ./.github/workflows/e2e-vm-test.yml - with: - gateway-artifact: openshell-gateway-plain-x86_64-unknown-linux-gnu - suite-name: external-driver - e2e-task: e2e:vm:external-driver + integration-inputs-artifact-id: ${{ needs.prepare-integration.outputs.integration_inputs_artifact_id }} - gpu-e2e: + gpu-conformance: needs: [pr_metadata, build-binaries, build-images] - if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_gpu_e2e == 'true' + if: needs.pr_metadata.outputs.run_gpu_e2e == 'true' permissions: actions: read contents: read packages: read - uses: ./.github/workflows/e2e-gpu-test.yaml + uses: ./.github/workflows/e2e-conformance.yml with: + profile: gpu + source-sha: ${{ github.sha }} image-tag: ${{ github.sha }} - kubernetes-e2e: + gpu-drivers: needs: [pr_metadata, build-binaries, build-images] - if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true' - strategy: - fail-fast: false - matrix: - include: - - agent_sandbox_api: v1beta1 - agent_sandbox_version: v1.0.3 - - agent_sandbox_api: v1alpha1 - agent_sandbox_version: v0.4.6 + if: needs.pr_metadata.outputs.run_gpu_e2e == 'true' permissions: actions: read contents: read packages: read - uses: ./.github/workflows/e2e-kubernetes-test.yml + uses: ./.github/workflows/e2e-drivers.yml with: + profile: gpu + source-sha: ${{ github.sha }} image-tag: ${{ github.sha }} - job-name: Kubernetes E2E (Rust smoke, Agent Sandbox ${{ matrix.agent_sandbox_api }}) - agent-sandbox-version: ${{ matrix.agent_sandbox_version }} - kubernetes-workspace-managed-e2e: + kubernetes-ha-conformance: needs: [pr_metadata, build-binaries, build-images] - if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true' - permissions: - actions: read - contents: read - packages: read - uses: ./.github/workflows/e2e-kubernetes-test.yml - with: - image-tag: ${{ github.sha }} - job-name: Kubernetes E2E (workspace managed mode) - e2e-task: e2e:kubernetes:workspace-managed - - kubernetes-external-driver-e2e: - needs: [pr_metadata, build-binaries, build-gateway-plain, build-external-drivers, build-images] - if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true' + if: needs.pr_metadata.outputs.run_kubernetes_ha_e2e == 'true' permissions: actions: read contents: read packages: read - uses: ./.github/workflows/e2e-kubernetes-test.yml + uses: ./.github/workflows/e2e-conformance.yml with: + profile: kubernetes-ha + source-sha: ${{ github.sha }} image-tag: ${{ github.sha }} - job-name: Kubernetes E2E (external compute driver) - e2e-task: e2e:kubernetes:external-driver - gateway-artifact: openshell-gateway-plain-x86_64-unknown-linux-gnu - external-driver-binary: openshell-driver-kubernetes - cluster-images: sandbox supervisor - kubernetes-workspace-operator-e2e: + kubernetes-ha-drivers: needs: [pr_metadata, build-binaries, build-images] - if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true' + if: needs.pr_metadata.outputs.run_kubernetes_ha_e2e == 'true' permissions: actions: read contents: read packages: read - uses: ./.github/workflows/e2e-kubernetes-test.yml + uses: ./.github/workflows/e2e-drivers.yml with: + profile: kubernetes-ha + source-sha: ${{ github.sha }} image-tag: ${{ github.sha }} - job-name: Kubernetes E2E (workspace operator mode) - e2e-task: e2e:kubernetes:workspace-operator - kubernetes-ha-e2e: + kubernetes-credentials-conformance: needs: [pr_metadata, build-binaries, build-images] - if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_kubernetes_ha_e2e == 'true' + if: needs.pr_metadata.outputs.run_kubernetes_credential_drivers_e2e == 'true' permissions: actions: read contents: read packages: read - uses: ./.github/workflows/e2e-kubernetes-test.yml + uses: ./.github/workflows/e2e-conformance.yml with: + profile: kubernetes-credentials + source-sha: ${{ github.sha }} image-tag: ${{ github.sha }} - job-name: Kubernetes HA E2E (Rust smoke) - extra-helm-values: deploy/helm/openshell/ci/values-high-availability.yaml - external-postgres-secret: openshell-ha-pg - test-name: kubernetes_ha_rebalancing - kubernetes-features: e2e,e2e-host-gateway,e2e-kubernetes,e2e-kubernetes-ha - use-envoy-gateway: true - kubernetes-credential-drivers-e2e: + kubernetes-credentials-drivers: needs: [pr_metadata, build-binaries, build-images] - if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_kubernetes_credential_drivers_e2e == 'true' + if: needs.pr_metadata.outputs.run_kubernetes_credential_drivers_e2e == 'true' permissions: actions: read contents: read packages: read - uses: ./.github/workflows/e2e-kubernetes-test.yml + uses: ./.github/workflows/e2e-drivers.yml with: + profile: kubernetes-credentials + source-sha: ${{ github.sha }} image-tag: ${{ github.sha }} - job-name: Kubernetes Credential Drivers E2E - e2e-task: e2e:kubernetes:credential-drivers core-e2e-result: name: Core E2E result - needs: - - pr_metadata - - conformance-integration - - feature-specific-integration - - driver-specific-integration - - docker-e2e - - podman-e2e - - vm-e2e - - docker-external-driver-e2e - - podman-external-driver-e2e - - vm-external-driver-e2e - - kubernetes-e2e - - kubernetes-external-driver-e2e - - kubernetes-workspace-managed-e2e - - kubernetes-workspace-operator-e2e + needs: [pr_metadata, conformance, features, drivers] if: always() && needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true' runs-on: ubuntu-latest steps: @@ -514,7 +372,7 @@ jobs: gpu-e2e-result: name: GPU E2E result - needs: [pr_metadata, gpu-e2e] + needs: [pr_metadata, gpu-conformance, gpu-drivers] if: always() && needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_gpu_e2e == 'true' runs-on: ubuntu-latest steps: @@ -527,7 +385,7 @@ jobs: kubernetes-ha-e2e-result: name: Kubernetes HA E2E result - needs: [pr_metadata, kubernetes-ha-e2e] + needs: [pr_metadata, kubernetes-ha-conformance, kubernetes-ha-drivers] if: always() && needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_kubernetes_ha_e2e == 'true' runs-on: ubuntu-latest steps: @@ -540,7 +398,7 @@ jobs: kubernetes-credential-drivers-e2e-result: name: Kubernetes Credential Drivers E2E result - needs: [pr_metadata, kubernetes-credential-drivers-e2e] + needs: [pr_metadata, kubernetes-credentials-conformance, kubernetes-credentials-drivers] if: always() && needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_kubernetes_credential_drivers_e2e == 'true' runs-on: ubuntu-latest steps: diff --git a/.github/workflows/e2e-conformance.yml b/.github/workflows/e2e-conformance.yml new file mode 100644 index 0000000000..c33bee194f --- /dev/null +++ b/.github/workflows/e2e-conformance.yml @@ -0,0 +1,57 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +name: E2E Conformance Tests + +on: + workflow_call: + inputs: + profile: + description: core, release, gpu, kubernetes-ha, or kubernetes-credentials + required: false + type: string + default: core + source-sha: + required: true + type: string + image-tag: + required: true + type: string + integration-inputs-artifact-id: + required: false + type: string + default: "" + +permissions: + actions: read + contents: read + packages: read + +jobs: + conformance-integration: + if: inputs.profile == 'core' || inputs.profile == 'release' + permissions: + actions: read + contents: read + packages: read + uses: ./.github/workflows/integration-runner.yml + with: + category: conformance + source-sha: ${{ inputs.source-sha }} + integration-inputs-artifact-id: ${{ inputs.integration-inputs-artifact-id }} + test-matrix: >- + [ + {"environment":"ubuntu-docker-rootful","installer":"deb","testsuite":"conformance"}, + {"environment":"ubuntu-k3s","installer":"k3s","testsuite":"conformance"}, + {"environment":"ubuntu-k3s","installer":"k3s-ha","testsuite":"conformance"}, + {"environment":"fedora-podman-rootful","installer":"rpm","testsuite":"conformance"}, + {"environment":"fedora-podman-rootless","installer":"rpm","testsuite":"conformance"} + ] + + runtime: + uses: ./.github/workflows/e2e-runtime-tests.yml + with: + test-family: conformance + profile: ${{ inputs.profile }} + source-sha: ${{ inputs.source-sha }} + image-tag: ${{ inputs.image-tag }} diff --git a/.github/workflows/e2e-docker-test.yml b/.github/workflows/e2e-docker-test.yml index c7f21684a1..3cf374849d 100644 --- a/.github/workflows/e2e-docker-test.yml +++ b/.github/workflows/e2e-docker-test.yml @@ -6,6 +6,11 @@ name: Docker E2E Test on: workflow_call: inputs: + test-family: + description: Suite family (all, conformance, or drivers) + required: false + type: string + default: all image-tag: required: true type: string @@ -55,6 +60,7 @@ jobs: - /var/run/docker.sock:/var/run/docker.sock - /home/runner/_work:/home/runner/_work env: + OPENSHELL_E2E_TEST_FAMILY: ${{ inputs.test-family }} MISE_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} IMAGE_TAG: ${{ inputs.image-tag }} OPENSHELL_REGISTRY: ghcr.io/nvidia/openshell diff --git a/.github/workflows/e2e-drivers.yml b/.github/workflows/e2e-drivers.yml new file mode 100644 index 0000000000..b345da6d57 --- /dev/null +++ b/.github/workflows/e2e-drivers.yml @@ -0,0 +1,62 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +name: E2E Drivers Tests + +on: + workflow_call: + outputs: + docker-result: + description: Docker runtime result for existing release summaries + value: ${{ jobs.runtime.outputs.docker-result }} + vm-result: + description: VM runtime result for existing release summaries + value: ${{ jobs.runtime.outputs.vm-result }} + inputs: + profile: + description: core, release, gpu, kubernetes-ha, or kubernetes-credentials + required: false + type: string + default: core + source-sha: + required: true + type: string + image-tag: + required: true + type: string + integration-inputs-artifact-id: + required: false + type: string + default: "" + +permissions: + actions: read + contents: read + packages: read + +jobs: + driver-specific-integration: + if: inputs.profile == 'core' + permissions: + actions: read + contents: read + packages: read + uses: ./.github/workflows/integration-runner.yml + with: + category: driver-specific + source-sha: ${{ inputs.source-sha }} + integration-inputs-artifact-id: ${{ inputs.integration-inputs-artifact-id }} + test-matrix: >- + [ + {"environment":"fedora-podman-rootful","installer":"binaries","testsuite":"driver-podman"}, + {"environment":"fedora-podman-rootless","installer":"binaries","testsuite":"e2e-podman"}, + {"environment":"fedora-podman-rootless","installer":"binaries","testsuite":"driver-podman"} + ] + + runtime: + uses: ./.github/workflows/e2e-runtime-tests.yml + with: + test-family: drivers + profile: ${{ inputs.profile }} + source-sha: ${{ inputs.source-sha }} + image-tag: ${{ inputs.image-tag }} diff --git a/.github/workflows/e2e-features.yml b/.github/workflows/e2e-features.yml new file mode 100644 index 0000000000..1642d1b0a2 --- /dev/null +++ b/.github/workflows/e2e-features.yml @@ -0,0 +1,48 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +name: E2E Features Tests + +on: + workflow_call: + inputs: + profile: + description: Existing core/release feature coverage (identical matrices) + required: false + type: string + default: core + source-sha: + required: true + type: string + image-tag: + required: true + type: string + integration-inputs-artifact-id: + required: false + type: string + default: "" + +permissions: + actions: read + contents: read + packages: read + +jobs: + feature-specific-integration: + permissions: + actions: read + contents: read + packages: read + uses: ./.github/workflows/integration-runner.yml + with: + category: feature-specific + source-sha: ${{ inputs.source-sha }} + integration-inputs-artifact-id: ${{ inputs.integration-inputs-artifact-id }} + test-matrix: >- + [ + {"environment":"fedora-podman-rootful","installer":"rpm","testsuite":"provider-refresh"}, + {"environment":"fedora-podman-rootless","installer":"rpm","testsuite":"provider-refresh"}, + {"environment":"ubuntu-docker-rootful","installer":"deb","testsuite":"oci-image"}, + {"environment":"fedora-podman-rootful","installer":"rpm","testsuite":"oci-image"}, + {"environment":"fedora-podman-rootless","installer":"rpm","testsuite":"oci-image"} + ] diff --git a/.github/workflows/e2e-gpu-test.yaml b/.github/workflows/e2e-gpu-test.yaml index d332ae41f7..d94b42e929 100644 --- a/.github/workflows/e2e-gpu-test.yaml +++ b/.github/workflows/e2e-gpu-test.yaml @@ -3,6 +3,15 @@ name: GPU E2E Test on: workflow_call: inputs: + checkout-ref: + required: false + type: string + default: "" + test-family: + description: Suite family (all, conformance, or drivers) + required: false + type: string + default: all image-tag: description: "Image tag to test (typically the commit SHA)" required: true @@ -42,6 +51,7 @@ jobs: - /var/run/docker.sock:/var/run/docker.sock - /home/runner/_work:/home/runner/_work env: + OPENSHELL_E2E_TEST_FAMILY: ${{ inputs.test-family }} MISE_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} IMAGE_TAG: ${{ inputs.image-tag }} OPENSHELL_REGISTRY: ghcr.io/nvidia/openshell @@ -58,6 +68,9 @@ jobs: OPENSHELL_E2E_GPU_PROBE_IMAGE: "nvcr.io/nvidia/base/ubuntu:noble-20251013" steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ inputs.checkout-ref || github.sha }} + persist-credentials: false - name: Use prebuilt OpenShell CLI uses: ./.github/actions/setup-e2e-cli diff --git a/.github/workflows/e2e-kubernetes-test.yml b/.github/workflows/e2e-kubernetes-test.yml index 449f3c25f0..b0955f4839 100644 --- a/.github/workflows/e2e-kubernetes-test.yml +++ b/.github/workflows/e2e-kubernetes-test.yml @@ -3,6 +3,11 @@ name: Kubernetes E2E Test on: workflow_call: inputs: + test-family: + description: Suite family (all, conformance, or drivers) + required: false + type: string + default: all image-tag: description: "Image tag to test (typically the commit SHA)" required: true @@ -96,6 +101,7 @@ jobs: contents: read packages: read env: + OPENSHELL_E2E_TEST_FAMILY: ${{ inputs.test-family }} MISE_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # Keep bare-runner installs aligned with the project CI image. MISE_VERSION: ${{ inputs.mise-version }} @@ -142,4 +148,8 @@ jobs: IMAGE_TAG: ${{ inputs.image-tag }} OPENSHELL_REGISTRY: ghcr.io/nvidia/openshell E2E_TASK: ${{ inputs.e2e-task }} - run: mise run --no-deps --skip-deps "$E2E_TASK" + run: | + if [ -z "${OPENSHELL_E2E_KUBERNETES_FEATURES}" ]; then + unset OPENSHELL_E2E_KUBERNETES_FEATURES + fi + mise run --no-deps --skip-deps "$E2E_TASK" diff --git a/.github/workflows/e2e-podman-test.yml b/.github/workflows/e2e-podman-test.yml index d70ad6c60c..2bab5744a9 100644 --- a/.github/workflows/e2e-podman-test.yml +++ b/.github/workflows/e2e-podman-test.yml @@ -6,6 +6,11 @@ name: Podman E2E Test on: workflow_call: inputs: + test-family: + description: Suite family (all, conformance, or drivers) + required: false + type: string + default: all image-tag: required: true type: string @@ -40,6 +45,7 @@ jobs: matrix: include: ${{ fromJSON(inputs.suite-matrix) }} env: + OPENSHELL_E2E_TEST_FAMILY: ${{ inputs.test-family }} IMAGE_TAG: ${{ inputs.image-tag }} MISE_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} OPENSHELL_REGISTRY: ghcr.io/nvidia/openshell diff --git a/.github/workflows/e2e-runtime-tests.yml b/.github/workflows/e2e-runtime-tests.yml new file mode 100644 index 0000000000..a75ee2ee7b --- /dev/null +++ b/.github/workflows/e2e-runtime-tests.yml @@ -0,0 +1,319 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +name: E2E Runtime Tests + +on: + workflow_call: + outputs: + docker-result: + description: Docker runtime result for existing release summaries + value: ${{ jobs.result.outputs.docker-result }} + vm-result: + description: VM runtime result for existing release summaries + value: ${{ jobs.result.outputs.vm-result }} + inputs: + test-family: + description: conformance or drivers + required: true + type: string + profile: + description: core, release, gpu, kubernetes-ha, or kubernetes-credentials + required: true + type: string + source-sha: + required: true + type: string + image-tag: + required: true + type: string + +permissions: + actions: read + contents: read + packages: read + +jobs: + selection: + runs-on: ubuntu-latest + outputs: + jobs: ${{ steps.select.outputs.jobs }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ inputs.source-sha }} + persist-credentials: false + - id: select + env: + TEST_FAMILY: ${{ inputs.test-family }} + PROFILE: ${{ inputs.profile }} + SOURCE_SHA: ${{ inputs.source-sha }} + run: | + set -euo pipefail + jobs=$(bash tasks/scripts/select-e2e-runtime-jobs.sh "$TEST_FAMILY" "$PROFILE") + echo "jobs=$jobs" >> "$GITHUB_OUTPUT" + { + echo "### E2E selection: $TEST_FAMILY / $PROFILE" + echo "Source: $SOURCE_SHA" + jq -r '.[] | "- " + .' <<< "$jobs" + } >> "$GITHUB_STEP_SUMMARY" + + docker-e2e: + needs: selection + if: contains(fromJSON(needs.selection.outputs.jobs), 'docker-e2e') + permissions: + actions: read + contents: read + packages: read + uses: ./.github/workflows/e2e-docker-test.yml + with: + checkout-ref: ${{ inputs.source-sha }} + test-family: ${{ inputs.test-family }} + image-tag: ${{ inputs.image-tag }} + runner: linux-arm64-cpu8 + + suite-matrix: >- + ${{ inputs.test-family == 'conformance' && '[{"suite":"conformance","cmd":"mise run --no-deps --skip-deps e2e:rust","apt_packages":"openssh-client","python_proto":false,"mcp":false}]' || '[{"suite":"python","cmd":"mise run --no-deps --skip-deps e2e:python","apt_packages":"","python_proto":true,"mcp":false},{"suite":"oidc-python","cmd":"mise run --no-deps --skip-deps e2e:oidc-python:docker","apt_packages":"","python_proto":true,"mcp":false},{"suite":"oidc-pkce-docker","cmd":"mise run --no-deps --skip-deps e2e:oidc-pkce:docker","apt_packages":"openssh-client","python_proto":false,"mcp":false},{"suite":"rust-docker","cmd":"mise run --no-deps --skip-deps e2e:rust","apt_packages":"openssh-client","python_proto":false,"mcp":false},{"suite":"oci-image","cmd":"mise run --no-deps --skip-deps e2e:oci-image","apt_packages":"openssh-client","python_proto":false,"mcp":false},{"suite":"mcp","cmd":"mise run --no-deps --skip-deps e2e:mcp","apt_packages":"","python_proto":false,"mcp":true}]' }} + + + podman-e2e: + needs: selection + if: contains(fromJSON(needs.selection.outputs.jobs), 'podman-e2e') + permissions: + actions: read + contents: read + packages: read + uses: ./.github/workflows/e2e-podman-test.yml + with: + checkout-ref: ${{ inputs.source-sha }} + test-family: ${{ inputs.test-family }} + image-tag: ${{ inputs.image-tag }} + suite-matrix: >- + [{"suite":"ci","runner":"ubuntu-26.04","podman_major":"5","podman_package_version":"5.7.0+ds2-3build1","conmon_package_version":"2.1.13+ds1-2","cmd":"mise run --no-deps --skip-deps e2e:podman:ci"}] + + + vm-e2e: + needs: selection + if: contains(fromJSON(needs.selection.outputs.jobs), 'vm-e2e') + permissions: + actions: read + contents: read + packages: read + uses: ./.github/workflows/e2e-vm-test.yml + with: + checkout-ref: ${{ inputs.source-sha }} + test-family: ${{ inputs.test-family }} + + docker-external-driver-e2e: + needs: selection + if: contains(fromJSON(needs.selection.outputs.jobs), 'docker-external-driver-e2e') + permissions: + actions: read + contents: read + packages: read + uses: ./.github/workflows/e2e-docker-test.yml + with: + checkout-ref: ${{ inputs.source-sha }} + test-family: ${{ inputs.test-family }} + image-tag: ${{ inputs.image-tag }} + runner: linux-arm64-cpu8 + gateway-artifact: openshell-gateway-plain-aarch64-unknown-linux-gnu + external-driver-binary: openshell-driver-docker + suite-matrix: >- + [{"suite":"external-driver","cmd":"mise run --no-deps --skip-deps e2e:docker:external-driver","apt_packages":"openssh-client","python_proto":false,"mcp":false}] + + + podman-external-driver-e2e: + needs: selection + if: contains(fromJSON(needs.selection.outputs.jobs), 'podman-external-driver-e2e') + permissions: + actions: read + contents: read + packages: read + uses: ./.github/workflows/e2e-podman-test.yml + with: + checkout-ref: ${{ inputs.source-sha }} + test-family: ${{ inputs.test-family }} + image-tag: ${{ inputs.image-tag }} + gateway-artifact: openshell-gateway-plain-x86_64-unknown-linux-gnu + external-driver-binary: openshell-driver-podman + suite-matrix: >- + [{"suite":"external-driver","runner":"ubuntu-26.04","podman_major":"5","podman_package_version":"5.7.0+ds2-3build1","conmon_package_version":"2.1.13+ds1-2","cmd":"mise run --no-deps --skip-deps e2e:podman:external-driver"}] + + + vm-external-driver-e2e: + needs: selection + if: contains(fromJSON(needs.selection.outputs.jobs), 'vm-external-driver-e2e') + permissions: + actions: read + contents: read + packages: read + uses: ./.github/workflows/e2e-vm-test.yml + with: + checkout-ref: ${{ inputs.source-sha }} + test-family: ${{ inputs.test-family }} + gateway-artifact: openshell-gateway-plain-x86_64-unknown-linux-gnu + suite-name: external-driver + e2e-task: e2e:vm:external-driver + + + gpu-e2e: + needs: selection + if: contains(fromJSON(needs.selection.outputs.jobs), 'gpu-e2e') + permissions: + actions: read + contents: read + packages: read + uses: ./.github/workflows/e2e-gpu-test.yaml + with: + checkout-ref: ${{ inputs.source-sha }} + test-family: ${{ inputs.test-family }} + image-tag: ${{ inputs.image-tag }} + + + kubernetes-e2e: + needs: selection + if: contains(fromJSON(needs.selection.outputs.jobs), 'kubernetes-e2e') + strategy: + fail-fast: false + matrix: + include: + - agent_sandbox_api: v1beta1 + agent_sandbox_version: v1.0.3 + - agent_sandbox_api: v1alpha1 + agent_sandbox_version: v0.4.6 + permissions: + actions: read + contents: read + packages: read + uses: ./.github/workflows/e2e-kubernetes-test.yml + with: + checkout-ref: ${{ inputs.source-sha }} + test-family: ${{ inputs.test-family }} + image-tag: ${{ inputs.image-tag }} + job-name: Kubernetes E2E (Rust smoke, Agent Sandbox ${{ matrix.agent_sandbox_api }}) + agent-sandbox-version: ${{ matrix.agent_sandbox_version }} + + + kubernetes-workspace-managed-e2e: + needs: selection + if: contains(fromJSON(needs.selection.outputs.jobs), 'kubernetes-workspace-managed-e2e') + permissions: + actions: read + contents: read + packages: read + uses: ./.github/workflows/e2e-kubernetes-test.yml + with: + checkout-ref: ${{ inputs.source-sha }} + test-family: ${{ inputs.test-family }} + image-tag: ${{ inputs.image-tag }} + job-name: Kubernetes E2E (workspace managed mode) + e2e-task: e2e:kubernetes:workspace-managed + + + kubernetes-external-driver-e2e: + needs: selection + if: contains(fromJSON(needs.selection.outputs.jobs), 'kubernetes-external-driver-e2e') + permissions: + actions: read + contents: read + packages: read + uses: ./.github/workflows/e2e-kubernetes-test.yml + with: + checkout-ref: ${{ inputs.source-sha }} + test-family: ${{ inputs.test-family }} + image-tag: ${{ inputs.image-tag }} + job-name: Kubernetes E2E (external compute driver) + e2e-task: e2e:kubernetes:external-driver + gateway-artifact: openshell-gateway-plain-x86_64-unknown-linux-gnu + external-driver-binary: openshell-driver-kubernetes + cluster-images: sandbox supervisor + + + kubernetes-workspace-operator-e2e: + needs: selection + if: contains(fromJSON(needs.selection.outputs.jobs), 'kubernetes-workspace-operator-e2e') + permissions: + actions: read + contents: read + packages: read + uses: ./.github/workflows/e2e-kubernetes-test.yml + with: + checkout-ref: ${{ inputs.source-sha }} + test-family: ${{ inputs.test-family }} + image-tag: ${{ inputs.image-tag }} + job-name: Kubernetes E2E (workspace operator mode) + e2e-task: e2e:kubernetes:workspace-operator + + + kubernetes-ha-e2e: + needs: selection + if: contains(fromJSON(needs.selection.outputs.jobs), 'kubernetes-ha-e2e') + permissions: + actions: read + contents: read + packages: read + uses: ./.github/workflows/e2e-kubernetes-test.yml + with: + checkout-ref: ${{ inputs.source-sha }} + test-family: ${{ inputs.test-family }} + image-tag: ${{ inputs.image-tag }} + job-name: Kubernetes HA E2E (Rust smoke) + extra-helm-values: deploy/helm/openshell/ci/values-high-availability.yaml + external-postgres-secret: openshell-ha-pg + test-name: kubernetes_ha_rebalancing + kubernetes-features: e2e,e2e-host-gateway,e2e-kubernetes,e2e-kubernetes-ha + use-envoy-gateway: true + + + kubernetes-credential-drivers-e2e: + needs: selection + if: contains(fromJSON(needs.selection.outputs.jobs), 'kubernetes-credential-drivers-e2e') + permissions: + actions: read + contents: read + packages: read + uses: ./.github/workflows/e2e-kubernetes-test.yml + with: + checkout-ref: ${{ inputs.source-sha }} + test-family: ${{ inputs.test-family }} + image-tag: ${{ inputs.image-tag }} + job-name: Kubernetes Credential Drivers E2E + e2e-task: e2e:kubernetes:credential-drivers + + + result: + name: Runtime family result + if: always() + outputs: + docker-result: ${{ needs.docker-e2e.result }} + vm-result: ${{ needs.vm-e2e.result }} + needs: + - selection + - docker-e2e + - podman-e2e + - vm-e2e + - docker-external-driver-e2e + - podman-external-driver-e2e + - vm-external-driver-e2e + - gpu-e2e + - kubernetes-e2e + - kubernetes-workspace-managed-e2e + - kubernetes-external-driver-e2e + - kubernetes-workspace-operator-e2e + - kubernetes-ha-e2e + - kubernetes-credential-drivers-e2e + runs-on: ubuntu-latest + steps: + - name: Require every selected runtime job to succeed + env: + JOB_RESULTS: ${{ toJSON(needs) }} + SELECTED_JOBS: ${{ needs.selection.outputs.jobs }} + run: | + set -euo pipefail + jq -e --argjson selected "$SELECTED_JOBS" ' + . as $results + | .selection.result == "success" + and ($selected | length > 0) + and all($selected[]; $results[.].result == "success") + ' <<< "$JOB_RESULTS" diff --git a/.github/workflows/e2e-vm-test.yml b/.github/workflows/e2e-vm-test.yml index 57e04c32b3..7142162b2a 100644 --- a/.github/workflows/e2e-vm-test.yml +++ b/.github/workflows/e2e-vm-test.yml @@ -6,6 +6,11 @@ name: VM E2E Test on: workflow_call: inputs: + test-family: + description: Suite family (all, conformance, or drivers) + required: false + type: string + default: all checkout-ref: required: false type: string @@ -34,6 +39,7 @@ jobs: runs-on: ubuntu-24.04 timeout-minutes: 90 env: + OPENSHELL_E2E_TEST_FAMILY: ${{ inputs.test-family }} GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} MISE_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} steps: diff --git a/.github/workflows/release-dev.yml b/.github/workflows/release-dev.yml index c97a48e11c..33fa5b4493 100644 --- a/.github/workflows/release-dev.yml +++ b/.github/workflows/release-dev.yml @@ -113,24 +113,17 @@ jobs: rpm-artifact-name: rpm-linux-x86_64 conformance-integration: - needs: prepare-integration + needs: [prepare-integration, build-vm-driver] permissions: actions: read contents: read packages: read - uses: ./.github/workflows/integration-runner.yml + uses: ./.github/workflows/e2e-conformance.yml with: - category: conformance + profile: release source-sha: ${{ needs.prepare-integration.outputs.source_sha }} + image-tag: ${{ github.sha }} integration-inputs-artifact-id: ${{ needs.prepare-integration.outputs.integration_inputs_artifact_id }} - test-matrix: >- - [ - {"environment":"ubuntu-docker-rootful","installer":"deb","testsuite":"conformance"}, - {"environment":"ubuntu-k3s","installer":"k3s","testsuite":"conformance"}, - {"environment":"ubuntu-k3s","installer":"k3s-ha","testsuite":"conformance"}, - {"environment":"fedora-podman-rootful","installer":"rpm","testsuite":"conformance"}, - {"environment":"fedora-podman-rootless","installer":"rpm","testsuite":"conformance"} - ] feature-specific-integration: needs: prepare-integration @@ -138,41 +131,25 @@ jobs: actions: read contents: read packages: read - uses: ./.github/workflows/integration-runner.yml + uses: ./.github/workflows/e2e-features.yml with: - category: feature-specific + profile: release source-sha: ${{ needs.prepare-integration.outputs.source_sha }} - integration-inputs-artifact-id: ${{ needs.prepare-integration.outputs.integration_inputs_artifact_id }} - test-matrix: >- - [ - {"environment":"fedora-podman-rootful","installer":"rpm","testsuite":"provider-refresh"}, - {"environment":"fedora-podman-rootless","installer":"rpm","testsuite":"provider-refresh"}, - {"environment":"ubuntu-docker-rootful","installer":"deb","testsuite":"oci-image"}, - {"environment":"fedora-podman-rootful","installer":"rpm","testsuite":"oci-image"}, - {"environment":"fedora-podman-rootless","installer":"rpm","testsuite":"oci-image"} - ] - - docker-e2e: - needs: [build-binaries, build-images] - permissions: - actions: read - contents: read - packages: read - uses: ./.github/workflows/e2e-docker-test.yml - with: image-tag: ${{ github.sha }} - checkout-ref: ${{ github.sha }} - runner: linux-arm64-cpu8 + integration-inputs-artifact-id: ${{ needs.prepare-integration.outputs.integration_inputs_artifact_id }} - vm-e2e: - needs: [build-binaries, build-vm-driver] + driver-specific: + needs: [prepare-integration, build-vm-driver] permissions: actions: read contents: read packages: read - uses: ./.github/workflows/e2e-vm-test.yml + uses: ./.github/workflows/e2e-drivers.yml with: - checkout-ref: ${{ github.sha }} + profile: release + source-sha: ${{ needs.prepare-integration.outputs.source_sha }} + image-tag: ${{ github.sha }} + integration-inputs-artifact-id: ${{ needs.prepare-integration.outputs.integration_inputs_artifact_id }} tag-ghcr-dev: name: Tag GHCR Images as Dev @@ -283,8 +260,7 @@ jobs: - build-python-wheel - conformance-integration - feature-specific-integration - - docker-e2e - - vm-e2e + - driver-specific - build-deb - build-rpm - publish-snap diff --git a/.github/workflows/release-tag.yml b/.github/workflows/release-tag.yml index f60ec045e9..751e0bf45c 100644 --- a/.github/workflows/release-tag.yml +++ b/.github/workflows/release-tag.yml @@ -161,24 +161,17 @@ jobs: rpm-artifact-name: rpm-linux-x86_64 conformance-integration: - needs: prepare-integration + needs: [prepare-integration, build-vm-driver] permissions: actions: read contents: read packages: read - uses: ./.github/workflows/integration-runner.yml + uses: ./.github/workflows/e2e-conformance.yml with: - category: conformance + profile: release source-sha: ${{ needs.prepare-integration.outputs.source_sha }} + image-tag: ${{ needs.prepare-integration.outputs.source_sha }} integration-inputs-artifact-id: ${{ needs.prepare-integration.outputs.integration_inputs_artifact_id }} - test-matrix: >- - [ - {"environment":"ubuntu-docker-rootful","installer":"deb","testsuite":"conformance"}, - {"environment":"ubuntu-k3s","installer":"k3s","testsuite":"conformance"}, - {"environment":"ubuntu-k3s","installer":"k3s-ha","testsuite":"conformance"}, - {"environment":"fedora-podman-rootful","installer":"rpm","testsuite":"conformance"}, - {"environment":"fedora-podman-rootless","installer":"rpm","testsuite":"conformance"} - ] feature-specific-integration: needs: prepare-integration @@ -186,41 +179,25 @@ jobs: actions: read contents: read packages: read - uses: ./.github/workflows/integration-runner.yml + uses: ./.github/workflows/e2e-features.yml with: - category: feature-specific + profile: release source-sha: ${{ needs.prepare-integration.outputs.source_sha }} + image-tag: ${{ needs.prepare-integration.outputs.source_sha }} integration-inputs-artifact-id: ${{ needs.prepare-integration.outputs.integration_inputs_artifact_id }} - test-matrix: >- - [ - {"environment":"fedora-podman-rootful","installer":"rpm","testsuite":"provider-refresh"}, - {"environment":"fedora-podman-rootless","installer":"rpm","testsuite":"provider-refresh"}, - {"environment":"ubuntu-docker-rootful","installer":"deb","testsuite":"oci-image"}, - {"environment":"fedora-podman-rootful","installer":"rpm","testsuite":"oci-image"}, - {"environment":"fedora-podman-rootless","installer":"rpm","testsuite":"oci-image"} - ] - - docker-e2e: - needs: [compute-versions, build-binaries, build-images] - permissions: - actions: read - contents: read - packages: read - uses: ./.github/workflows/e2e-docker-test.yml - with: - image-tag: ${{ needs.compute-versions.outputs.source_sha }} - checkout-ref: ${{ needs.compute-versions.outputs.source_sha }} - runner: linux-arm64-cpu8 - vm-e2e: - needs: [compute-versions, build-binaries, build-vm-driver] + driver-specific: + needs: [prepare-integration, build-vm-driver] permissions: actions: read contents: read packages: read - uses: ./.github/workflows/e2e-vm-test.yml + uses: ./.github/workflows/e2e-drivers.yml with: - checkout-ref: ${{ needs.compute-versions.outputs.source_sha }} + profile: release + source-sha: ${{ needs.prepare-integration.outputs.source_sha }} + image-tag: ${{ needs.prepare-integration.outputs.source_sha }} + integration-inputs-artifact-id: ${{ needs.prepare-integration.outputs.integration_inputs_artifact_id }} protobuf-compatibility: name: Protobuf Compatibility @@ -260,8 +237,7 @@ jobs: - security - conformance-integration - feature-specific-integration - - docker-e2e - - vm-e2e + - driver-specific runs-on: ubuntu-latest timeout-minutes: 5 permissions: @@ -279,13 +255,14 @@ jobs: id: summary env: CONFORMANCE_RESULT: ${{ needs.conformance-integration.result }} - DOCKER_E2E_RESULT: ${{ needs.docker-e2e.result }} + DRIVER_FAMILY_RESULT: ${{ needs.driver-specific.result }} + DOCKER_E2E_RESULT: ${{ needs.driver-specific.outputs.docker-result || needs.driver-specific.result }} FEATURE_INTEGRATION_RESULT: ${{ needs.feature-specific-integration.result }} IS_PRERELEASE: ${{ needs.compute-versions.outputs.is_prerelease }} PROTO_COMPATIBILITY_RESULT: ${{ needs.protobuf-compatibility.result }} SECURITY_RESULT: ${{ needs.security.result }} SOURCE_SHA: ${{ needs.compute-versions.outputs.source_sha }} - VM_E2E_RESULT: ${{ needs.vm-e2e.result }} + VM_E2E_RESULT: ${{ needs.driver-specific.outputs.vm-result || needs.driver-specific.result }} run: | set -euo pipefail current_profile_passed=$(tasks/scripts/generate-qualification-summary.sh qualification-summary.json) @@ -311,6 +288,7 @@ jobs: echo "| Security | ${SECURITY_RESULT} |" echo "| Conformance integration | ${CONFORMANCE_RESULT} |" echo "| Feature integration | ${FEATURE_INTEGRATION_RESULT} |" + echo "| Driver family | ${DRIVER_FAMILY_RESULT} |" echo "| Docker E2E | ${DOCKER_E2E_RESULT} |" echo "| VM E2E | ${VM_E2E_RESULT} |" echo diff --git a/CI.md b/CI.md index fc9547720f..c4dc73f9ad 100644 --- a/CI.md +++ b/CI.md @@ -150,6 +150,28 @@ retain the binary installer because their fixtures configure its system service, local HTTP gateway, and CLI path. The manual Integration Tests workflow defaults to the package installers and downloads the packages selected by its matrix. +Branch and release workflows call the conformance, features, and drivers family +workflows. Conformance and drivers select runtime jobs through +`e2e-runtime-tests.yml`; the same recorded job list drives execution and the +completion check. A selected job that fails, is cancelled, or is skipped fails +its family. The existing core/GPU required result names and optional Kubernetes +policy remain unchanged, including advisory WSL GPU execution. + +The `core` profile retains the branch target matrix. The `release` profile +retains Docker and VM runtime coverage plus the existing conformance and feature +integration matrices; it does not add the rest of the branch driver matrix to +release qualification. GPU, Kubernetes HA, and credential-driver profiles retain +their existing label selections. Bundled/external is a target dimension: +external VM runs both conformance and runtime suites; the other external lanes +currently run conformance only. The operator workspace lane retains its +conformance exclusion. Mixed Python/OIDC/MCP and other legacy suites remain +under Drivers until their assertions are extracted into the appropriate family. + +Each selected family provisions its own targets using the shared runtime setup. +Splitting the previously combined runtime jobs therefore adds provisioning work; +it also lets conformance and driver failures be reported independently. The +manual `Integration Tests` entrypoint retains its custom tmachine matrix interface. + Three opt-in labels enable the long-running E2E suites: - `test:e2e` runs the Docker, rootless Podman, Kubernetes, and VM E2E suites @@ -545,7 +567,9 @@ The bot's full administrator documentation is internal to NVIDIA. The only comma | `.github/workflows/build-binaries.yml`, `build-vm-driver.yml` | Shared binary matrices used by branch and release workflows. The VM driver remains separate because its build consumes the runtime binaries. | | `.github/workflows/build-images.yml` | Builds and pushes multi-platform images, then uploads the same OCI images as workflow artifacts. | | `.github/workflows/package-release-binaries.yml` | Packages raw build artifacts into release tarballs without rebuilding them. | -| `.github/workflows/e2e-docker-test.yml`, `e2e-podman-test.yml`, `e2e-vm-test.yml`, `e2e-kubernetes-test.yml` | Reusable runtime lanes called directly by branch and release workflows. Callers select suites and declare only the artifacts each runtime consumes. | +| `.github/workflows/e2e-conformance.yml`, `e2e-features.yml`, `e2e-drivers.yml` | Test-family entrypoints shared by branch and release workflows. Retain the current integration matrices and runtime profiles. | +| `.github/workflows/e2e-runtime-tests.yml` | Selects runtime jobs by family and execution profile and requires every selected job to succeed. | +| `.github/workflows/e2e-docker-test.yml`, `e2e-podman-test.yml`, `e2e-vm-test.yml`, `e2e-kubernetes-test.yml`, `e2e-gpu-test.yaml` | Reusable target preparation and execution adapters. Accept an explicit test family while keeping combined execution as their default. | | `.github/actions/setup-e2e-*` | Shared artifact, Podman, KVM, and kind setup used by the runtime lanes. | | `.github/workflows/helm-lint.yml` | Helm chart validation. PR mirror pushes skip lint jobs unless Helm inputs changed; merge groups always validate Helm because they represent the final integration state. | | `.github/actions/setup-nix/action.yml` | Installs Nix and configures the OpenShell Cachix cache, using read-only cache access when no authentication token is available. | diff --git a/TESTING.md b/TESTING.md index 316cbbbfa0..1109b8bfd8 100644 --- a/TESTING.md +++ b/TESTING.md @@ -125,6 +125,34 @@ OPENSHELL_GATEWAY_ENDPOINT=http://127.0.0.1:18080 mise run e2e Raw endpoint mode is HTTP-only. Use a named gateway config when a gateway requires mTLS. +CI groups e2e execution into conformance, features, and drivers. Conformance +uses the shared package in `tests/suites/conformance/`; features retains the +provider-refresh and OCI-image suites. Drivers collects the existing runtime +and host-specific suites, including mixed legacy tests pending extraction. +Bundled versus external compute drivers are target variants across families. + +The Docker, Podman, Kubernetes, and VM Rust harnesses accept +`OPENSHELL_E2E_TEST_FAMILY=conformance` or `drivers` to execute each part +independently. The default, `all`, retains combined local execution. For example: + +```shell +OPENSHELL_E2E_TEST_FAMILY=conformance mise run e2e:docker +OPENSHELL_E2E_TEST_FAMILY=drivers mise run e2e:docker +OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER=1 OPENSHELL_E2E_TEST_FAMILY=drivers mise run e2e:docker +``` + +This selector controls the driver harnesses; it does not filter standalone +Python, OIDC, OCI-image, or MCP entrypoints. Existing external-driver tasks for +Docker, Podman, and Kubernetes retain their conformance-only feature selection. +Use the normal driver task with the external-driver flag, as above, to run its +runtime suite against an external driver. CI preserves its previous target +coverage: external VM runs in both families, while the other external variants +currently run only in conformance. + +`mise run test:e2e-lanes` checks suite dispatch, target matrices, and completion +failure handling without provisioning runtimes. Live CI execution is still +required to validate the prepared targets. + `mise run test:gateway-config` validates generated gateway TOML without a live runtime. It covers the current schema and the Podman in-tree versus external driver configuration boundary. diff --git a/e2e/rust/e2e-docker.sh b/e2e/rust/e2e-docker.sh index 62a9cc8b61..cee3e40a70 100755 --- a/e2e/rust/e2e-docker.sh +++ b/e2e/rust/e2e-docker.sh @@ -16,11 +16,16 @@ DEFAULT_WORKLOAD_MANIFEST="${ROOT}/e2e/gpu/images/.build/workloads.yaml" RUN_WITH_GATEWAY_COMMAND="__openshell_run_docker_e2e" # shellcheck source=e2e/support/conformance.sh source "${ROOT}/e2e/support/conformance.sh" +# shellcheck source=e2e/support/test-family.sh +source "${ROOT}/e2e/support/test-family.sh" +e2e_select_test_family "${E2E_FEATURES}" run_e2e() { - e2e_run_openshell_conformance "Docker" + if [ "${E2E_RUN_CONFORMANCE}" = "1" ]; then + e2e_run_openshell_conformance "Docker" + fi - if [ -z "${E2E_FEATURES}" ]; then + if [ "${E2E_RUN_DRIVER_TESTS}" = "0" ]; then return 0 fi diff --git a/e2e/rust/e2e-kubernetes.sh b/e2e/rust/e2e-kubernetes.sh index f2b90dc583..0b2471d6da 100755 --- a/e2e/rust/e2e-kubernetes.sh +++ b/e2e/rust/e2e-kubernetes.sh @@ -25,8 +25,11 @@ ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" RUN_WITH_GATEWAY_COMMAND="__openshell_run_kubernetes_e2e" # shellcheck source=e2e/support/conformance.sh source "${ROOT}/e2e/support/conformance.sh" +# shellcheck source=e2e/support/test-family.sh +source "${ROOT}/e2e/support/test-family.sh" E2E_FEATURES="${OPENSHELL_E2E_KUBERNETES_FEATURES-e2e,e2e-host-gateway,e2e-kubernetes}" +e2e_select_test_family "${E2E_FEATURES}" # Fixed output path of the `e2e-kubernetes` nextest profile (`.config/nextest.toml`). JUNIT_XML="${ROOT}/results/e2e-kubernetes.xml" @@ -89,8 +92,10 @@ run_suite() { } run_e2e() { - run_conformance - if [ -z "${E2E_FEATURES}" ]; then + if [ "${E2E_RUN_CONFORMANCE}" = "1" ]; then + run_conformance + fi + if [ "${E2E_RUN_DRIVER_TESTS}" = "0" ]; then return 0 fi diff --git a/e2e/rust/e2e-podman.sh b/e2e/rust/e2e-podman.sh index cd8b52e695..7c109383d2 100755 --- a/e2e/rust/e2e-podman.sh +++ b/e2e/rust/e2e-podman.sh @@ -16,6 +16,9 @@ DEFAULT_WORKLOAD_MANIFEST="${ROOT}/e2e/gpu/images/.build/workloads.yaml" RUN_WITH_GATEWAY_COMMAND="__openshell_run_podman_e2e" # shellcheck source=e2e/support/conformance.sh source "${ROOT}/e2e/support/conformance.sh" +# shellcheck source=e2e/support/test-family.sh +source "${ROOT}/e2e/support/test-family.sh" +e2e_select_test_family "${E2E_FEATURES}" # Keep required branch CI on the Podman targets that are known to pass. The # unfiltered e2e:podman task remains available while the omitted targets are @@ -71,8 +74,10 @@ if [ "${E2E_TEST}" = "gpu" ] && [ -z "${OPENSHELL_E2E_WORKLOAD_MANIFEST:-}" ] && fi if [ "${1:-}" = "${RUN_WITH_GATEWAY_COMMAND}" ]; then - e2e_run_openshell_conformance "Podman" - if [ -z "${E2E_FEATURES}" ]; then + if [ "${E2E_RUN_CONFORMANCE}" = "1" ]; then + e2e_run_openshell_conformance "Podman" + fi + if [ "${E2E_RUN_DRIVER_TESTS}" = "0" ]; then exit 0 fi @@ -80,7 +85,7 @@ if [ "${1:-}" = "${RUN_WITH_GATEWAY_COMMAND}" ]; then cargo test --manifest-path "${ROOT}/e2e/rust/Cargo.toml" --features "${E2E_FEATURES}" ) - for test_target in "${SELECTED_TESTS[@]}"; do + for test_target in ${SELECTED_TESTS[@]+"${SELECTED_TESTS[@]}"}; do TEST_ARGS+=(--test "${test_target}") done TEST_ARGS+=(-- --nocapture) diff --git a/e2e/rust/e2e-vm.sh b/e2e/rust/e2e-vm.sh index ebb160c987..3b85b2378d 100755 --- a/e2e/rust/e2e-vm.sh +++ b/e2e/rust/e2e-vm.sh @@ -44,6 +44,8 @@ ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" source "${ROOT}/e2e/support/gateway-common.sh" # shellcheck source=e2e/support/conformance.sh source "${ROOT}/e2e/support/conformance.sh" +# shellcheck source=e2e/support/test-family.sh +source "${ROOT}/e2e/support/test-family.sh" COMPRESSED_DIR="${ROOT}/target/vm-runtime-compressed" GATEWAY_BIN="${OPENSHELL_GATEWAY_BIN:-${ROOT}/target/debug/openshell-gateway}" @@ -51,6 +53,7 @@ DRIVER_BIN="${OPENSHELL_VM_DRIVER_BIN:-${ROOT}/target/debug/openshell-driver-vm} CLI_BIN="${OPENSHELL_BIN:-${ROOT}/target/debug/openshell}" E2E_TEST_OVERRIDE="${OPENSHELL_E2E_VM_TEST:-}" E2E_FEATURES="${OPENSHELL_E2E_VM_FEATURES-e2e-vm}" +e2e_select_test_family "${E2E_FEATURES}" SANDBOX_IMAGE="${OPENSHELL_SANDBOX_IMAGE:-nvcr.io/nvidia/base/ubuntu:24.04}" # The VM driver places `compute-driver.sock` under `[openshell.drivers.vm].state_dir`. @@ -382,7 +385,12 @@ e2e_export_gateway_restart_metadata \ # preparation; allow 180s for slower CI runners. export OPENSHELL_PROVISION_TIMEOUT="${SANDBOX_PROVISION_TIMEOUT}" -e2e_run_openshell_conformance "VM" +if [ "${E2E_RUN_CONFORMANCE}" = "1" ]; then + e2e_run_openshell_conformance "VM" +fi +if [ "${E2E_RUN_DRIVER_TESTS}" = "0" ]; then + exit 0 +fi # Seed the catalog once for the whole lane. The profiles live in the gateway # for as long as it runs, and the import is create-only: a second import of diff --git a/e2e/support/test-family.sh b/e2e/support/test-family.sh new file mode 100644 index 0000000000..7aab8d13e2 --- /dev/null +++ b/e2e/support/test-family.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# shellcheck disable=SC2034 # Selection variables are consumed by sourcing harnesses. + +# Select execution independently of compute-driver linkage and Cargo features. +# Keep the combined local commands compatible while CI reports families apart. +e2e_select_test_family() { + local features="$1" + E2E_RUN_CONFORMANCE=0 + E2E_RUN_DRIVER_TESTS=0 + case "${OPENSHELL_E2E_TEST_FAMILY:-all}" in + all) + E2E_RUN_CONFORMANCE=1 + if [ -n "${features}" ]; then E2E_RUN_DRIVER_TESTS=1; fi + ;; + conformance) + E2E_RUN_CONFORMANCE=1 + ;; + drivers) + if [ -z "${features}" ]; then + echo "error: the drivers family requires a nonempty Cargo feature selection" >&2 + return 2 + fi + E2E_RUN_DRIVER_TESTS=1 + ;; + *) + echo "error: unknown OPENSHELL_E2E_TEST_FAMILY '${OPENSHELL_E2E_TEST_FAMILY}' (expected all, conformance, or drivers)" >&2 + return 2 + ;; + esac +} diff --git a/tasks/scripts/e2e_lanes_test.py b/tasks/scripts/e2e_lanes_test.py new file mode 100644 index 0000000000..6a0377dd34 --- /dev/null +++ b/tasks/scripts/e2e_lanes_test.py @@ -0,0 +1,299 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Exercise family dispatch and guard the existing CI target/coverage contract.""" + +import json +import os +import subprocess +from pathlib import Path + +import pytest +import yaml + +ROOT = Path(__file__).resolve().parents[2] +WORKFLOWS = ROOT / ".github/workflows" + + +def workflow(name): + # BaseLoader leaves the Actions `on` key and expressions as strings. + return yaml.load((WORKFLOWS / name).read_text(), Loader=yaml.BaseLoader) + + +def selected(family, profile): + result = subprocess.run( + ["bash", "tasks/scripts/select-e2e-runtime-jobs.sh", family, profile], + cwd=ROOT, + text=True, + capture_output=True, + check=True, + ) + return json.loads(result.stdout) + + +@pytest.fixture +def cargo_stub(tmp_path): + cargo = tmp_path / "cargo" + cargo.write_text( + "#!/bin/bash\n" + 'printf "%s\\n" "$*" >> "$TEST_CARGO_LOG"\n' + 'if [[ "$*" == *"suites/conformance"* ]]; then\n' + ' exit "${TEST_CONFORMANCE_EXIT:-0}"\n' + "fi\n" + 'exit "${TEST_DRIVER_EXIT:-0}"\n' + ) + cargo.chmod(0o755) + log = tmp_path / "cargo.log" + env = {k: v for k, v in os.environ.items() if not k.startswith("OPENSHELL_")} + env.update( + PATH=f"{tmp_path}:{os.environ['PATH']}", + TEST_CARGO_LOG=str(log), + OPENSHELL_BIN="/usr/bin/true", + ) + return env, log + + +def run_harness(driver, family, cargo_stub, **overrides): + env, log = cargo_stub + env = env | {"OPENSHELL_E2E_TEST_FAMILY": family} | overrides + if driver == "vm": + # Execute the actual test phase, after provisioning, with fake Cargo and + # provider setup. VM process startup itself requires a Linux/KVM runner. + script = (ROOT / "e2e/rust/e2e-vm.sh").read_text() + tests = script[script.index('if [ "${E2E_RUN_CONFORMANCE}" = "1" ]; then') :] + command = [ + "bash", + "-c", + "set -euo pipefail\n" + f'ROOT="{ROOT}"\n' + 'CLI_BIN="$OPENSHELL_BIN"\n' + 'source "$ROOT/e2e/support/conformance.sh"\n' + 'source "$ROOT/e2e/support/test-family.sh"\n' + 'E2E_FEATURES="${OPENSHELL_E2E_VM_FEATURES-e2e-vm}"\n' + 'E2E_TEST_OVERRIDE="${OPENSHELL_E2E_VM_TEST:-}"\n' + "OPENSHELL_GATEWAY_ENDPOINT=http://fixture\n" + 'e2e_select_test_family "$E2E_FEATURES"\n' + "e2e_import_example_provider_profiles() { return 0; }\n" + tests, + ] + else: + command = [ + "bash", + f"e2e/rust/e2e-{driver}.sh", + f"__openshell_run_{driver}_e2e", + ] + result = subprocess.run(command, cwd=ROOT, env=env, text=True, capture_output=True) + calls = log.read_text().splitlines() if log.exists() else [] + return result, calls + + +@pytest.mark.parametrize("driver", ["docker", "podman", "kubernetes", "vm"]) +@pytest.mark.parametrize("family", ["all", "conformance", "drivers"]) +def test_harness_runs_only_selected_suites(driver, family, cargo_stub): + result, calls = run_harness(driver, family, cargo_stub) + assert result.returncode == 0, result.stderr + conformance = [call for call in calls if "suites/conformance" in call] + runtime = [call for call in calls if "e2e/rust/Cargo.toml" in call] + assert len(conformance) == (0 if family == "drivers" else 1) + assert len(runtime) == ( + 0 if family == "conformance" else 5 if driver == "vm" else 1 + ) + if runtime and driver == "vm": + assert [call.split("--test ")[1].split()[0] for call in runtime] == [ + "ephemeral_cleanup", + "host_gateway_alias", + "vm_overlay", + "vm_gateway_start", + "vm_corporate_proxy", + ] + + +@pytest.mark.parametrize("driver", ["docker", "podman", "kubernetes", "vm"]) +@pytest.mark.parametrize( + "family,override", + [ + ("all", {"TEST_CONFORMANCE_EXIT": "19"}), + ("drivers", {"TEST_DRIVER_EXIT": "23"}), + ], +) +def test_suite_failures_propagate(driver, family, override, cargo_stub): + result, calls = run_harness(driver, family, cargo_stub, **override) + assert result.returncode != 0 + if family == "all": + assert len(calls) == 1, "failed conformance must not be masked by a later pass" + + +@pytest.mark.parametrize("driver", ["docker", "podman", "kubernetes"]) +def test_legacy_empty_features_run_conformance_only(driver, cargo_stub): + key = f"OPENSHELL_E2E_{driver.upper()}_FEATURES" + result, calls = run_harness(driver, "all", cargo_stub, **{key: ""}) + assert result.returncode == 0, result.stderr + assert len(calls) == 1 and "suites/conformance" in calls[0] + + +@pytest.mark.parametrize("family", ["typo", "drivers"]) +def test_invalid_or_empty_driver_selection_fails_before_execution(family, cargo_stub): + result, calls = run_harness( + "docker", + family, + cargo_stub, + OPENSHELL_E2E_DOCKER_FEATURES="", + ) + assert result.returncode != 0 + assert not calls + + +def test_operator_workspace_conformance_exclusion_is_preserved(cargo_stub): + result, calls = run_harness( + "kubernetes", + "all", + cargo_stub, + OPENSHELL_E2E_KUBERNETES_FEATURES="e2e-kubernetes-workspace-operator", + ) + assert result.returncode == 0, result.stderr + assert len(calls) == 1 and "suites/conformance" not in calls[0] + + +def test_runtime_profiles_preserve_existing_targets(): + # Frozen from the pre-migration branch workflow: conformance on every + # runtime except operator mode; driver tests on external VM as well. + core = { + "docker-e2e", + "podman-e2e", + "vm-e2e", + "kubernetes-e2e", + "kubernetes-workspace-managed-e2e", + "vm-external-driver-e2e", + } + assert set(selected("conformance", "core")) == core | { + "docker-external-driver-e2e", + "podman-external-driver-e2e", + "kubernetes-external-driver-e2e", + } + assert set(selected("drivers", "core")) == core | { + "kubernetes-workspace-operator-e2e" + } + for family in ("conformance", "drivers"): + assert set(selected(family, "release")) == {"docker-e2e", "vm-e2e"} + assert selected(family, "gpu") == ["gpu-e2e"] + assert selected(family, "kubernetes-ha") == ["kubernetes-ha-e2e"] + assert selected(family, "kubernetes-credentials") == [ + "kubernetes-credential-drivers-e2e" + ] + + +def test_docker_split_retains_all_six_legacy_suites(): + runtime = workflow("e2e-runtime-tests.yml")["jobs"]["docker-e2e"]["with"] + # Inspect the two literal branches of the Actions family expression. + conformance, drivers = runtime["suite-matrix"].split(" && '")[1].split("' || '") + rows = json.loads(drivers.removesuffix("' }}\n").removesuffix("' }}")) + legacy = workflow("e2e-docker-test.yml")["on"]["workflow_call"]["inputs"][ + "suite-matrix" + ]["default"] + assert rows == json.loads(legacy) + assert [row["suite"] for row in rows] == [ + "python", + "oidc-python", + "oidc-pkce-docker", + "rust-docker", + "oci-image", + "mcp", + ] + assert [row["cmd"] for row in json.loads(conformance)] == [ + "mise run --no-deps --skip-deps e2e:rust" + ] + + +@pytest.mark.parametrize("family,profile", [("features", "core"), ("drivers", "typo")]) +def test_unknown_runtime_requests_are_rejected(family, profile): + result = subprocess.run( + ["bash", "tasks/scripts/select-e2e-runtime-jobs.sh", family, profile], + cwd=ROOT, + capture_output=True, + ) + assert result.returncode != 0 and not result.stdout + + +@pytest.mark.parametrize("status", ["success", "failure", "skipped", "cancelled", None]) +def test_runtime_completion_requires_every_selected_job(status): + step = workflow("e2e-runtime-tests.yml")["jobs"]["result"]["steps"][0] + results = {"selection": {"result": "success"}, "unselected": {"result": "skipped"}} + if status is not None: + results["selected"] = {"result": status} + result = subprocess.run( + ["bash", "-c", step["run"]], + env=os.environ + | {"JOB_RESULTS": json.dumps(results), "SELECTED_JOBS": '["selected"]'}, + capture_output=True, + ) + assert (result.returncode == 0) == (status == "success") + + +def test_existing_matrices_and_artifact_variants(): + conf = workflow("e2e-conformance.yml")["jobs"]["conformance-integration"] + rows = json.loads(conf["with"]["test-matrix"]) + assert {(r["environment"], r["installer"]) for r in rows} == { + ("ubuntu-docker-rootful", "deb"), + ("ubuntu-k3s", "k3s"), + ("ubuntu-k3s", "k3s-ha"), + ("fedora-podman-rootful", "rpm"), + ("fedora-podman-rootless", "rpm"), + } + features = workflow("e2e-features.yml")["jobs"]["feature-specific-integration"] + rows = json.loads(features["with"]["test-matrix"]) + assert len(rows) == 5 + assert ( + sorted(r["testsuite"] for r in rows) + == ["oci-image"] * 3 + ["provider-refresh"] * 2 + ) + drivers = workflow("e2e-drivers.yml")["jobs"]["driver-specific-integration"] + rows = json.loads(drivers["with"]["test-matrix"]) + assert {(r["environment"], r["installer"], r["testsuite"]) for r in rows} == { + ("fedora-podman-rootful", "binaries", "driver-podman"), + ("fedora-podman-rootless", "binaries", "e2e-podman"), + ("fedora-podman-rootless", "binaries", "driver-podman"), + } + runtime = workflow("e2e-runtime-tests.yml")["jobs"] + assert runtime["kubernetes-e2e"]["strategy"]["matrix"]["include"] == [ + {"agent_sandbox_api": "v1beta1", "agent_sandbox_version": "v1.0.3"}, + {"agent_sandbox_api": "v1alpha1", "agent_sandbox_version": "v0.4.6"}, + ] + for driver, arch in [ + ("docker", "aarch64"), + ("podman", "x86_64"), + ("kubernetes", "x86_64"), + ("vm", "x86_64"), + ]: + lane = runtime[f"{driver}-external-driver-e2e"]["with"] + assert ( + lane["gateway-artifact"] + == f"openshell-gateway-plain-{arch}-unknown-linux-gnu" + ) + gpu = workflow("e2e-gpu-test.yaml")["jobs"]["e2e-gpu"] + assert [ + (r["name"], r["experimental"]) for r in gpu["strategy"]["matrix"]["include"] + ] == [ + ("linux-arm64", "false"), + ("linux-amd64", "false"), + ("wsl-amd64", "true"), + ] + + +def test_release_and_required_gate_compatibility(): + branch = workflow("branch-e2e.yml")["jobs"] + assert branch["core-e2e-result"]["name"] == "Core E2E result" + assert branch["core-e2e-result"]["needs"] == [ + "pr_metadata", + "conformance", + "features", + "drivers", + ] + assert branch["gpu-e2e-result"]["name"] == "GPU E2E result" + for name in ("dev", "tag"): + jobs = workflow(f"release-{name}.yml")["jobs"] + for key, family in [ + ("conformance-integration", "conformance"), + ("feature-specific-integration", "features"), + ("driver-specific", "drivers"), + ]: + assert jobs[key]["uses"] == f"./.github/workflows/e2e-{family}.yml" + assert jobs[key]["with"]["profile"] == "release" diff --git a/tasks/scripts/generate-qualification-summary.sh b/tasks/scripts/generate-qualification-summary.sh index 78f55c239e..1272579650 100755 --- a/tasks/scripts/generate-qualification-summary.sh +++ b/tasks/scripts/generate-qualification-summary.sh @@ -26,13 +26,15 @@ current_profile_passed=true require_env \ RELEASE_TAG SOURCE_SHA IS_PRERELEASE GITHUB_RUN_ID GITHUB_RUN_ATTEMPT \ GITHUB_SERVER_URL GITHUB_REPOSITORY SECURITY_RESULT CONFORMANCE_RESULT \ - FEATURE_INTEGRATION_RESULT DOCKER_E2E_RESULT VM_E2E_RESULT PROTO_COMPATIBILITY_RESULT + FEATURE_INTEGRATION_RESULT DOCKER_E2E_RESULT VM_E2E_RESULT PROTO_COMPATIBILITY_RESULT \ + DRIVER_FAMILY_RESULT for result in \ "${PROTO_COMPATIBILITY_RESULT}" \ "${SECURITY_RESULT}" \ "${CONFORMANCE_RESULT}" \ "${FEATURE_INTEGRATION_RESULT}" \ + "${DRIVER_FAMILY_RESULT}" \ "${DOCKER_E2E_RESULT}" \ "${VM_E2E_RESULT}"; do if [[ "${result}" != "success" ]]; then @@ -55,6 +57,7 @@ jq -n \ --arg protobuf_compatibility "${PROTO_COMPATIBILITY_RESULT}" \ --arg conformance "${CONFORMANCE_RESULT}" \ --arg feature_integration "${FEATURE_INTEGRATION_RESULT}" \ + --arg driver_family "${DRIVER_FAMILY_RESULT}" \ --arg docker_e2e "${DOCKER_E2E_RESULT}" \ --arg vm_e2e "${VM_E2E_RESULT}" \ '{ @@ -91,6 +94,7 @@ jq -n \ security: $security, conformance_integration: $conformance, feature_integration: $feature_integration, + driver_family: $driver_family, docker_e2e: $docker_e2e, vm_e2e: $vm_e2e } diff --git a/tasks/scripts/select-e2e-runtime-jobs.sh b/tasks/scripts/select-e2e-runtime-jobs.sh new file mode 100644 index 0000000000..c72e91ae66 --- /dev/null +++ b/tasks/scripts/select-e2e-runtime-jobs.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# List the runtime jobs required by one family/profile. The workflow uses this +# same list for execution and completion, so skipped selected jobs fail the gate. +set -euo pipefail + +family="${1:-}" +profile="${2:-}" +case "${family}" in + conformance|drivers) ;; + *) echo "error: expected conformance or drivers, got '${family}'" >&2; exit 2 ;; +esac + +case "${profile}" in + core) + jobs=(docker-e2e podman-e2e vm-e2e kubernetes-e2e kubernetes-workspace-managed-e2e vm-external-driver-e2e) + if [ "${family}" = conformance ]; then + jobs+=(docker-external-driver-e2e podman-external-driver-e2e kubernetes-external-driver-e2e) + else + # Operator mode currently excludes conformance; see #2971. + jobs+=(kubernetes-workspace-operator-e2e) + fi + ;; + release) jobs=(docker-e2e vm-e2e) ;; + gpu) jobs=(gpu-e2e) ;; + kubernetes-ha) jobs=(kubernetes-ha-e2e) ;; + kubernetes-credentials) jobs=(kubernetes-credential-drivers-e2e) ;; + *) echo "error: unknown runtime profile '${profile}'" >&2; exit 2 ;; +esac + +jq -cn --args '$ARGS.positional' "${jobs[@]}" diff --git a/tasks/scripts/test-qualification-summary.sh b/tasks/scripts/test-qualification-summary.sh index dcae7d4574..5e3eac9a96 100755 --- a/tasks/scripts/test-qualification-summary.sh +++ b/tasks/scripts/test-qualification-summary.sh @@ -23,6 +23,7 @@ export FEATURE_INTEGRATION_RESULT=success export DOCKER_E2E_RESULT=success export VM_E2E_RESULT=success export PROTO_COMPATIBILITY_RESULT=success +export DRIVER_FAMILY_RESULT=success current_profile_passed=$("${GENERATOR}" "${TEST_DIR}/qualification-summary.json") [[ "${current_profile_passed}" == "false" ]] @@ -52,4 +53,18 @@ jq -e ' and (.profile.missing_suites | index("SDK/configuration compatibility and migration review") != null) ' "${TEST_DIR}/qualified-summary.json" >/dev/null +# A family failure must block publication even if individual runtime outputs +# are present and successful (for example, completion validation failed). +for status in failure cancelled skipped; do + export DRIVER_FAMILY_RESULT="${status}" + current_profile_passed=$("${GENERATOR}" "${TEST_DIR}/driver-${status}.json") + [[ "${current_profile_passed}" == "false" ]] + jq -e --arg status "${status}" ' + .suites.driver_family == $status + and .suites.docker_e2e == "success" + and .suites.vm_e2e == "success" + and .current_profile_passed == false + ' "${TEST_DIR}/driver-${status}.json" >/dev/null +done + echo "qualification summary tests passed" diff --git a/tasks/test.toml b/tasks/test.toml index c1df20262a..69087c0a48 100644 --- a/tasks/test.toml +++ b/tasks/test.toml @@ -15,6 +15,7 @@ depends = [ "test:postgres-runner", "test:gateway-pull-policy", "test:e2e-image-overrides", + "test:e2e-lanes", "test:gateway-config", "test:packaging-assets", "test:qualification-summary", @@ -312,3 +313,9 @@ description = "Test generated local gateway TOML without starting a runtime" run = "bash tasks/scripts/test-gateway-config.sh" run_windows = "echo Skipping test:gateway-config: Unix gateway scripts do not apply on Windows." hide = true + +["test:e2e-lanes"] +description = "Verify e2e family selection, preserved target matrices, and failure gates without provisioning runtimes" +run = "uv run --no-project --with pytest --with pytest-asyncio --with pyyaml pytest tasks/scripts/e2e_lanes_test.py" +run_windows = "echo Skipping test:e2e-lanes: E2E harnesses execute on Unix hosts." +hide = true From e21c1a8d921317d2c7df7b0326ed6750acbd9b19 Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Wed, 14 Oct 2026 20:46:02 +0200 Subject: [PATCH 2/2] ci(test): flatten family workflows into driver matrices Signed-off-by: Evan Lezar --- .agents/skills/watch-github-actions/SKILL.md | 10 +- .github/workflows/e2e-conformance.yml | 152 ++++++++- .github/workflows/e2e-drivers.yml | 176 +++++++++- .github/workflows/e2e-features.yml | 27 +- .github/workflows/e2e-runtime-tests.yml | 319 ------------------- CI.md | 14 +- TESTING.md | 5 + tasks/scripts/e2e_lanes_test.py | 183 +++++++---- tasks/scripts/select-e2e-runtime-jobs.sh | 33 -- 9 files changed, 479 insertions(+), 440 deletions(-) delete mode 100644 .github/workflows/e2e-runtime-tests.yml delete mode 100644 tasks/scripts/select-e2e-runtime-jobs.sh diff --git a/.agents/skills/watch-github-actions/SKILL.md b/.agents/skills/watch-github-actions/SKILL.md index 60028ef5fe..bf99b6219a 100644 --- a/.agents/skills/watch-github-actions/SKILL.md +++ b/.agents/skills/watch-github-actions/SKILL.md @@ -153,10 +153,12 @@ once after preparation succeeds. Skipped dependent E2E suites indicate blocked coverage. For E2E, inspect the Conformance, Features, and Drivers family calls in -`Branch E2E Checks`, `Release Dev`, or `Release Tag`. Runtime family selection -is logged by `e2e-runtime-tests.yml`; its completion check fails when a selected -job fails, is cancelled, or is skipped. External VM has separate conformance -and runtime jobs. Existing `Core E2E result` and `GPU E2E result` names remain +`Branch E2E Checks`, `Release Dev`, or `Release Tag`. Each family calls driver +workflows directly; inspect their variant/configuration matrices and nested +scenario matrices. Features has separate provider-refresh and OCI-image calls. +Each family result fails when a required job fails, is cancelled, is missing, +or is skipped. Regular and external VM have separate conformance and runtime +execution. Existing `Core E2E result` and `GPU E2E result` names remain the PR gate adapters; optional Kubernetes and advisory WSL policy is unchanged. The manual `Integration Tests` workflow still accepts a custom tmachine matrix. diff --git a/.github/workflows/e2e-conformance.yml b/.github/workflows/e2e-conformance.yml index c33bee194f..94e60e30e4 100644 --- a/.github/workflows/e2e-conformance.yml +++ b/.github/workflows/e2e-conformance.yml @@ -48,10 +48,154 @@ jobs: {"environment":"fedora-podman-rootless","installer":"rpm","testsuite":"conformance"} ] - runtime: - uses: ./.github/workflows/e2e-runtime-tests.yml + docker: + name: Docker (${{ matrix.variant }}) + if: inputs.profile == 'core' || inputs.profile == 'release' + strategy: + fail-fast: false + matrix: + include: >- + ${{ fromJSON(inputs.profile == 'core' && '[ + {"variant":"regular","gateway_artifact":"","external_driver_binary":"","suites":[{"suite":"conformance","cmd":"mise run --no-deps --skip-deps e2e:rust","apt_packages":"openssh-client","python_proto":false,"mcp":false}]}, + {"variant":"external","gateway_artifact":"openshell-gateway-plain-aarch64-unknown-linux-gnu","external_driver_binary":"openshell-driver-docker","suites":[{"suite":"external-driver","cmd":"mise run --no-deps --skip-deps e2e:docker:external-driver","apt_packages":"openssh-client","python_proto":false,"mcp":false}]} + ]' || + '[ + {"variant":"regular","gateway_artifact":"","external_driver_binary":"","suites":[{"suite":"conformance","cmd":"mise run --no-deps --skip-deps e2e:rust","apt_packages":"openssh-client","python_proto":false,"mcp":false}]} + ]') }} + uses: ./.github/workflows/e2e-docker-test.yml with: + checkout-ref: ${{ inputs.source-sha }} + test-family: conformance + image-tag: ${{ inputs.image-tag }} + runner: linux-arm64-cpu8 + gateway-artifact: ${{ matrix.gateway_artifact }} + external-driver-binary: ${{ matrix.external_driver_binary }} + suite-matrix: ${{ toJSON(matrix.suites) }} + + podman: + name: Podman (${{ matrix.variant }}) + if: inputs.profile == 'core' + strategy: + fail-fast: false + matrix: + include: + - variant: regular + gateway_artifact: '' + external_driver_binary: '' + suites: + - suite: ci + runner: ubuntu-26.04 + podman_major: '5' + podman_package_version: 5.7.0+ds2-3build1 + conmon_package_version: 2.1.13+ds1-2 + cmd: mise run --no-deps --skip-deps e2e:podman:ci + - variant: external + gateway_artifact: openshell-gateway-plain-x86_64-unknown-linux-gnu + external_driver_binary: openshell-driver-podman + suites: + - suite: external-driver + runner: ubuntu-26.04 + podman_major: '5' + podman_package_version: 5.7.0+ds2-3build1 + conmon_package_version: 2.1.13+ds1-2 + cmd: mise run --no-deps --skip-deps e2e:podman:external-driver + uses: ./.github/workflows/e2e-podman-test.yml + with: + checkout-ref: ${{ inputs.source-sha }} test-family: conformance - profile: ${{ inputs.profile }} - source-sha: ${{ inputs.source-sha }} image-tag: ${{ inputs.image-tag }} + gateway-artifact: ${{ matrix.gateway_artifact }} + external-driver-binary: ${{ matrix.external_driver_binary }} + suite-matrix: ${{ toJSON(matrix.suites) }} + + vm: + name: VM (${{ matrix.variant }}) + if: inputs.profile == 'core' || inputs.profile == 'release' + strategy: + fail-fast: false + matrix: + include: >- + ${{ fromJSON(inputs.profile == 'core' && '[ + {"variant":"regular","gateway_artifact":"","suite_name":"managed","e2e_task":"e2e:vm"}, + {"variant":"external","gateway_artifact":"openshell-gateway-plain-x86_64-unknown-linux-gnu","suite_name":"external-driver","e2e_task":"e2e:vm:external-driver"} + ]' || + '[ + {"variant":"regular","gateway_artifact":"","suite_name":"managed","e2e_task":"e2e:vm"} + ]') }} + uses: ./.github/workflows/e2e-vm-test.yml + with: + checkout-ref: ${{ inputs.source-sha }} + test-family: conformance + gateway-artifact: ${{ matrix.gateway_artifact }} + suite-name: ${{ matrix.suite_name }} + e2e-task: ${{ matrix.e2e_task }} + + kubernetes: + name: ${{ matrix.job-name }} + if: inputs.profile == 'core' || inputs.profile == 'kubernetes-ha' || inputs.profile == 'kubernetes-credentials' + strategy: + fail-fast: false + matrix: + include: >- + ${{ fromJSON(inputs.profile == 'core' && '[ + {"job-name":"Kubernetes E2E (Rust smoke, Agent Sandbox v1beta1)","agent-sandbox-version":"v1.0.3","e2e-task":"e2e:kubernetes","gateway-artifact":"","external-driver-binary":"","cluster-images":"gateway sandbox supervisor","extra-helm-values":"","external-postgres-secret":"","test-name":"","kubernetes-features":"","use-envoy-gateway":false}, + {"job-name":"Kubernetes E2E (Rust smoke, Agent Sandbox v1alpha1)","agent-sandbox-version":"v0.4.6","e2e-task":"e2e:kubernetes","gateway-artifact":"","external-driver-binary":"","cluster-images":"gateway sandbox supervisor","extra-helm-values":"","external-postgres-secret":"","test-name":"","kubernetes-features":"","use-envoy-gateway":false}, + {"job-name":"Kubernetes E2E (workspace managed mode)","e2e-task":"e2e:kubernetes:workspace-managed","agent-sandbox-version":"v1.0.3","gateway-artifact":"","external-driver-binary":"","cluster-images":"gateway sandbox supervisor","extra-helm-values":"","external-postgres-secret":"","test-name":"","kubernetes-features":"","use-envoy-gateway":false}, + {"job-name":"Kubernetes E2E (external compute driver)","e2e-task":"e2e:kubernetes:external-driver","gateway-artifact":"openshell-gateway-plain-x86_64-unknown-linux-gnu","external-driver-binary":"openshell-driver-kubernetes","cluster-images":"sandbox supervisor","agent-sandbox-version":"v1.0.3","extra-helm-values":"","external-postgres-secret":"","test-name":"","kubernetes-features":"","use-envoy-gateway":false} + ]' || + inputs.profile == 'kubernetes-ha' && '[ + {"job-name":"Kubernetes HA E2E (Rust smoke)","extra-helm-values":"deploy/helm/openshell/ci/values-high-availability.yaml","external-postgres-secret":"openshell-ha-pg","test-name":"kubernetes_ha_rebalancing","kubernetes-features":"e2e,e2e-host-gateway,e2e-kubernetes,e2e-kubernetes-ha","use-envoy-gateway":true,"agent-sandbox-version":"v1.0.3","e2e-task":"e2e:kubernetes","gateway-artifact":"","external-driver-binary":"","cluster-images":"gateway sandbox supervisor"} + ]' || + '[ + {"job-name":"Kubernetes Credential Drivers E2E","e2e-task":"e2e:kubernetes:credential-drivers","agent-sandbox-version":"v1.0.3","gateway-artifact":"","external-driver-binary":"","cluster-images":"gateway sandbox supervisor","extra-helm-values":"","external-postgres-secret":"","test-name":"","kubernetes-features":"","use-envoy-gateway":false} + ]') }} + uses: ./.github/workflows/e2e-kubernetes-test.yml + with: + checkout-ref: ${{ inputs.source-sha }} + test-family: conformance + image-tag: ${{ inputs.image-tag }} + job-name: ${{ matrix.job-name }} + agent-sandbox-version: ${{ matrix.agent-sandbox-version }} + e2e-task: ${{ matrix.e2e-task }} + gateway-artifact: ${{ matrix.gateway-artifact }} + external-driver-binary: ${{ matrix.external-driver-binary }} + cluster-images: ${{ matrix.cluster-images }} + extra-helm-values: ${{ matrix.extra-helm-values }} + external-postgres-secret: ${{ matrix.external-postgres-secret }} + test-name: ${{ matrix.test-name }} + kubernetes-features: ${{ matrix.kubernetes-features }} + use-envoy-gateway: ${{ matrix.use-envoy-gateway }} + + gpu: + name: GPU + if: inputs.profile == 'gpu' + uses: ./.github/workflows/e2e-gpu-test.yaml + with: + checkout-ref: ${{ inputs.source-sha }} + test-family: conformance + image-tag: ${{ inputs.image-tag }} + + result: + name: Family result + if: always() + needs: [conformance-integration, docker, podman, vm, kubernetes, gpu] + runs-on: ubuntu-latest + steps: + - name: Require every selected driver and integration job to succeed + env: + JOB_RESULTS: ${{ toJSON(needs) }} + PROFILE: ${{ inputs.profile }} + run: | + set -euo pipefail + case "$PROFILE" in + core) expected='["conformance-integration","docker","podman","vm","kubernetes"]' ;; + release) expected='["docker","vm","conformance-integration"]' ;; + gpu) expected='["gpu"]' ;; + kubernetes-ha) expected='["kubernetes"]' ;; + kubernetes-credentials) expected='["kubernetes"]' ;; + *) echo "::error::Unknown E2E profile: $PROFILE"; exit 2 ;; + esac + jq -e --argjson expected "$expected" ' + . as $results + | all($expected[]; $results[.].result == "success") + ' <<< "$JOB_RESULTS" diff --git a/.github/workflows/e2e-drivers.yml b/.github/workflows/e2e-drivers.yml index b345da6d57..b76cc0d03a 100644 --- a/.github/workflows/e2e-drivers.yml +++ b/.github/workflows/e2e-drivers.yml @@ -8,10 +8,10 @@ on: outputs: docker-result: description: Docker runtime result for existing release summaries - value: ${{ jobs.runtime.outputs.docker-result }} + value: ${{ jobs.result.outputs.docker-result }} vm-result: description: VM runtime result for existing release summaries - value: ${{ jobs.runtime.outputs.vm-result }} + value: ${{ jobs.result.outputs.vm-result }} inputs: profile: description: core, release, gpu, kubernetes-ha, or kubernetes-credentials @@ -53,10 +53,174 @@ jobs: {"environment":"fedora-podman-rootless","installer":"binaries","testsuite":"driver-podman"} ] - runtime: - uses: ./.github/workflows/e2e-runtime-tests.yml + docker: + name: Docker (${{ matrix.variant }}) + if: inputs.profile == 'core' || inputs.profile == 'release' + strategy: + fail-fast: false + matrix: + include: + - variant: regular + gateway_artifact: '' + external_driver_binary: '' + suites: + - suite: python + cmd: mise run --no-deps --skip-deps e2e:python + apt_packages: '' + python_proto: true + mcp: false + - suite: oidc-python + cmd: mise run --no-deps --skip-deps e2e:oidc-python:docker + apt_packages: '' + python_proto: true + mcp: false + - suite: oidc-pkce-docker + cmd: mise run --no-deps --skip-deps e2e:oidc-pkce:docker + apt_packages: openssh-client + python_proto: false + mcp: false + - suite: rust-docker + cmd: mise run --no-deps --skip-deps e2e:rust + apt_packages: openssh-client + python_proto: false + mcp: false + - suite: oci-image + cmd: mise run --no-deps --skip-deps e2e:oci-image + apt_packages: openssh-client + python_proto: false + mcp: false + - suite: mcp + cmd: mise run --no-deps --skip-deps e2e:mcp + apt_packages: '' + python_proto: false + mcp: true + uses: ./.github/workflows/e2e-docker-test.yml with: + checkout-ref: ${{ inputs.source-sha }} + test-family: drivers + image-tag: ${{ inputs.image-tag }} + runner: linux-arm64-cpu8 + gateway-artifact: ${{ matrix.gateway_artifact }} + external-driver-binary: ${{ matrix.external_driver_binary }} + suite-matrix: ${{ toJSON(matrix.suites) }} + + podman: + name: Podman (${{ matrix.variant }}) + if: inputs.profile == 'core' + strategy: + fail-fast: false + matrix: + include: + - variant: regular + gateway_artifact: '' + external_driver_binary: '' + suites: + - suite: ci + runner: ubuntu-26.04 + podman_major: '5' + podman_package_version: 5.7.0+ds2-3build1 + conmon_package_version: 2.1.13+ds1-2 + cmd: mise run --no-deps --skip-deps e2e:podman:ci + uses: ./.github/workflows/e2e-podman-test.yml + with: + checkout-ref: ${{ inputs.source-sha }} test-family: drivers - profile: ${{ inputs.profile }} - source-sha: ${{ inputs.source-sha }} image-tag: ${{ inputs.image-tag }} + gateway-artifact: ${{ matrix.gateway_artifact }} + external-driver-binary: ${{ matrix.external_driver_binary }} + suite-matrix: ${{ toJSON(matrix.suites) }} + + vm: + name: VM (${{ matrix.variant }}) + if: inputs.profile == 'core' || inputs.profile == 'release' + strategy: + fail-fast: false + matrix: + include: >- + ${{ fromJSON(inputs.profile == 'core' && '[ + {"variant":"regular","gateway_artifact":"","suite_name":"managed","e2e_task":"e2e:vm"}, + {"variant":"external","gateway_artifact":"openshell-gateway-plain-x86_64-unknown-linux-gnu","suite_name":"external-driver","e2e_task":"e2e:vm:external-driver"} + ]' || + '[ + {"variant":"regular","gateway_artifact":"","suite_name":"managed","e2e_task":"e2e:vm"} + ]') }} + uses: ./.github/workflows/e2e-vm-test.yml + with: + checkout-ref: ${{ inputs.source-sha }} + test-family: drivers + gateway-artifact: ${{ matrix.gateway_artifact }} + suite-name: ${{ matrix.suite_name }} + e2e-task: ${{ matrix.e2e_task }} + + kubernetes: + name: ${{ matrix.job-name }} + if: inputs.profile == 'core' || inputs.profile == 'kubernetes-ha' || inputs.profile == 'kubernetes-credentials' + strategy: + fail-fast: false + matrix: + include: >- + ${{ fromJSON(inputs.profile == 'core' && '[ + {"job-name":"Kubernetes E2E (Rust smoke, Agent Sandbox v1beta1)","agent-sandbox-version":"v1.0.3","e2e-task":"e2e:kubernetes","gateway-artifact":"","external-driver-binary":"","cluster-images":"gateway sandbox supervisor","extra-helm-values":"","external-postgres-secret":"","test-name":"","kubernetes-features":"","use-envoy-gateway":false}, + {"job-name":"Kubernetes E2E (Rust smoke, Agent Sandbox v1alpha1)","agent-sandbox-version":"v0.4.6","e2e-task":"e2e:kubernetes","gateway-artifact":"","external-driver-binary":"","cluster-images":"gateway sandbox supervisor","extra-helm-values":"","external-postgres-secret":"","test-name":"","kubernetes-features":"","use-envoy-gateway":false}, + {"job-name":"Kubernetes E2E (workspace managed mode)","e2e-task":"e2e:kubernetes:workspace-managed","agent-sandbox-version":"v1.0.3","gateway-artifact":"","external-driver-binary":"","cluster-images":"gateway sandbox supervisor","extra-helm-values":"","external-postgres-secret":"","test-name":"","kubernetes-features":"","use-envoy-gateway":false}, + {"job-name":"Kubernetes E2E (workspace operator mode)","e2e-task":"e2e:kubernetes:workspace-operator","agent-sandbox-version":"v1.0.3","gateway-artifact":"","external-driver-binary":"","cluster-images":"gateway sandbox supervisor","extra-helm-values":"","external-postgres-secret":"","test-name":"","kubernetes-features":"","use-envoy-gateway":false} + ]' || + inputs.profile == 'kubernetes-ha' && '[ + {"job-name":"Kubernetes HA E2E (Rust smoke)","extra-helm-values":"deploy/helm/openshell/ci/values-high-availability.yaml","external-postgres-secret":"openshell-ha-pg","test-name":"kubernetes_ha_rebalancing","kubernetes-features":"e2e,e2e-host-gateway,e2e-kubernetes,e2e-kubernetes-ha","use-envoy-gateway":true,"agent-sandbox-version":"v1.0.3","e2e-task":"e2e:kubernetes","gateway-artifact":"","external-driver-binary":"","cluster-images":"gateway sandbox supervisor"} + ]' || + '[ + {"job-name":"Kubernetes Credential Drivers E2E","e2e-task":"e2e:kubernetes:credential-drivers","agent-sandbox-version":"v1.0.3","gateway-artifact":"","external-driver-binary":"","cluster-images":"gateway sandbox supervisor","extra-helm-values":"","external-postgres-secret":"","test-name":"","kubernetes-features":"","use-envoy-gateway":false} + ]') }} + uses: ./.github/workflows/e2e-kubernetes-test.yml + with: + checkout-ref: ${{ inputs.source-sha }} + test-family: drivers + image-tag: ${{ inputs.image-tag }} + job-name: ${{ matrix.job-name }} + agent-sandbox-version: ${{ matrix.agent-sandbox-version }} + e2e-task: ${{ matrix.e2e-task }} + gateway-artifact: ${{ matrix.gateway-artifact }} + external-driver-binary: ${{ matrix.external-driver-binary }} + cluster-images: ${{ matrix.cluster-images }} + extra-helm-values: ${{ matrix.extra-helm-values }} + external-postgres-secret: ${{ matrix.external-postgres-secret }} + test-name: ${{ matrix.test-name }} + kubernetes-features: ${{ matrix.kubernetes-features }} + use-envoy-gateway: ${{ matrix.use-envoy-gateway }} + + gpu: + name: GPU + if: inputs.profile == 'gpu' + uses: ./.github/workflows/e2e-gpu-test.yaml + with: + checkout-ref: ${{ inputs.source-sha }} + test-family: drivers + image-tag: ${{ inputs.image-tag }} + + result: + name: Family result + if: always() + outputs: + docker-result: ${{ needs.docker.result }} + vm-result: ${{ needs.vm.result }} + needs: [driver-specific-integration, docker, podman, vm, kubernetes, gpu] + runs-on: ubuntu-latest + steps: + - name: Require every selected driver and integration job to succeed + env: + JOB_RESULTS: ${{ toJSON(needs) }} + PROFILE: ${{ inputs.profile }} + run: | + set -euo pipefail + case "$PROFILE" in + core) expected='["driver-specific-integration","docker","podman","vm","kubernetes"]' ;; + release) expected='["docker","vm"]' ;; + gpu) expected='["gpu"]' ;; + kubernetes-ha) expected='["kubernetes"]' ;; + kubernetes-credentials) expected='["kubernetes"]' ;; + *) echo "::error::Unknown E2E profile: $PROFILE"; exit 2 ;; + esac + jq -e --argjson expected "$expected" ' + . as $results + | all($expected[]; $results[.].result == "success") + ' <<< "$JOB_RESULTS" diff --git a/.github/workflows/e2e-features.yml b/.github/workflows/e2e-features.yml index 1642d1b0a2..76a2baf2bd 100644 --- a/.github/workflows/e2e-features.yml +++ b/.github/workflows/e2e-features.yml @@ -28,21 +28,22 @@ permissions: packages: read jobs: - feature-specific-integration: - permissions: - actions: read - contents: read - packages: read + provider-refresh: + name: provider-refresh uses: ./.github/workflows/integration-runner.yml with: - category: feature-specific + category: feature-specific-provider-refresh source-sha: ${{ inputs.source-sha }} integration-inputs-artifact-id: ${{ inputs.integration-inputs-artifact-id }} test-matrix: >- - [ - {"environment":"fedora-podman-rootful","installer":"rpm","testsuite":"provider-refresh"}, - {"environment":"fedora-podman-rootless","installer":"rpm","testsuite":"provider-refresh"}, - {"environment":"ubuntu-docker-rootful","installer":"deb","testsuite":"oci-image"}, - {"environment":"fedora-podman-rootful","installer":"rpm","testsuite":"oci-image"}, - {"environment":"fedora-podman-rootless","installer":"rpm","testsuite":"oci-image"} - ] + [{"environment":"fedora-podman-rootful","installer":"rpm","testsuite":"provider-refresh"},{"environment":"fedora-podman-rootless","installer":"rpm","testsuite":"provider-refresh"}] + + oci-image: + name: oci-image + uses: ./.github/workflows/integration-runner.yml + with: + category: feature-specific-oci-image + source-sha: ${{ inputs.source-sha }} + integration-inputs-artifact-id: ${{ inputs.integration-inputs-artifact-id }} + test-matrix: >- + [{"environment":"ubuntu-docker-rootful","installer":"deb","testsuite":"oci-image"},{"environment":"fedora-podman-rootful","installer":"rpm","testsuite":"oci-image"},{"environment":"fedora-podman-rootless","installer":"rpm","testsuite":"oci-image"}] diff --git a/.github/workflows/e2e-runtime-tests.yml b/.github/workflows/e2e-runtime-tests.yml deleted file mode 100644 index a75ee2ee7b..0000000000 --- a/.github/workflows/e2e-runtime-tests.yml +++ /dev/null @@ -1,319 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -name: E2E Runtime Tests - -on: - workflow_call: - outputs: - docker-result: - description: Docker runtime result for existing release summaries - value: ${{ jobs.result.outputs.docker-result }} - vm-result: - description: VM runtime result for existing release summaries - value: ${{ jobs.result.outputs.vm-result }} - inputs: - test-family: - description: conformance or drivers - required: true - type: string - profile: - description: core, release, gpu, kubernetes-ha, or kubernetes-credentials - required: true - type: string - source-sha: - required: true - type: string - image-tag: - required: true - type: string - -permissions: - actions: read - contents: read - packages: read - -jobs: - selection: - runs-on: ubuntu-latest - outputs: - jobs: ${{ steps.select.outputs.jobs }} - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - ref: ${{ inputs.source-sha }} - persist-credentials: false - - id: select - env: - TEST_FAMILY: ${{ inputs.test-family }} - PROFILE: ${{ inputs.profile }} - SOURCE_SHA: ${{ inputs.source-sha }} - run: | - set -euo pipefail - jobs=$(bash tasks/scripts/select-e2e-runtime-jobs.sh "$TEST_FAMILY" "$PROFILE") - echo "jobs=$jobs" >> "$GITHUB_OUTPUT" - { - echo "### E2E selection: $TEST_FAMILY / $PROFILE" - echo "Source: $SOURCE_SHA" - jq -r '.[] | "- " + .' <<< "$jobs" - } >> "$GITHUB_STEP_SUMMARY" - - docker-e2e: - needs: selection - if: contains(fromJSON(needs.selection.outputs.jobs), 'docker-e2e') - permissions: - actions: read - contents: read - packages: read - uses: ./.github/workflows/e2e-docker-test.yml - with: - checkout-ref: ${{ inputs.source-sha }} - test-family: ${{ inputs.test-family }} - image-tag: ${{ inputs.image-tag }} - runner: linux-arm64-cpu8 - - suite-matrix: >- - ${{ inputs.test-family == 'conformance' && '[{"suite":"conformance","cmd":"mise run --no-deps --skip-deps e2e:rust","apt_packages":"openssh-client","python_proto":false,"mcp":false}]' || '[{"suite":"python","cmd":"mise run --no-deps --skip-deps e2e:python","apt_packages":"","python_proto":true,"mcp":false},{"suite":"oidc-python","cmd":"mise run --no-deps --skip-deps e2e:oidc-python:docker","apt_packages":"","python_proto":true,"mcp":false},{"suite":"oidc-pkce-docker","cmd":"mise run --no-deps --skip-deps e2e:oidc-pkce:docker","apt_packages":"openssh-client","python_proto":false,"mcp":false},{"suite":"rust-docker","cmd":"mise run --no-deps --skip-deps e2e:rust","apt_packages":"openssh-client","python_proto":false,"mcp":false},{"suite":"oci-image","cmd":"mise run --no-deps --skip-deps e2e:oci-image","apt_packages":"openssh-client","python_proto":false,"mcp":false},{"suite":"mcp","cmd":"mise run --no-deps --skip-deps e2e:mcp","apt_packages":"","python_proto":false,"mcp":true}]' }} - - - podman-e2e: - needs: selection - if: contains(fromJSON(needs.selection.outputs.jobs), 'podman-e2e') - permissions: - actions: read - contents: read - packages: read - uses: ./.github/workflows/e2e-podman-test.yml - with: - checkout-ref: ${{ inputs.source-sha }} - test-family: ${{ inputs.test-family }} - image-tag: ${{ inputs.image-tag }} - suite-matrix: >- - [{"suite":"ci","runner":"ubuntu-26.04","podman_major":"5","podman_package_version":"5.7.0+ds2-3build1","conmon_package_version":"2.1.13+ds1-2","cmd":"mise run --no-deps --skip-deps e2e:podman:ci"}] - - - vm-e2e: - needs: selection - if: contains(fromJSON(needs.selection.outputs.jobs), 'vm-e2e') - permissions: - actions: read - contents: read - packages: read - uses: ./.github/workflows/e2e-vm-test.yml - with: - checkout-ref: ${{ inputs.source-sha }} - test-family: ${{ inputs.test-family }} - - docker-external-driver-e2e: - needs: selection - if: contains(fromJSON(needs.selection.outputs.jobs), 'docker-external-driver-e2e') - permissions: - actions: read - contents: read - packages: read - uses: ./.github/workflows/e2e-docker-test.yml - with: - checkout-ref: ${{ inputs.source-sha }} - test-family: ${{ inputs.test-family }} - image-tag: ${{ inputs.image-tag }} - runner: linux-arm64-cpu8 - gateway-artifact: openshell-gateway-plain-aarch64-unknown-linux-gnu - external-driver-binary: openshell-driver-docker - suite-matrix: >- - [{"suite":"external-driver","cmd":"mise run --no-deps --skip-deps e2e:docker:external-driver","apt_packages":"openssh-client","python_proto":false,"mcp":false}] - - - podman-external-driver-e2e: - needs: selection - if: contains(fromJSON(needs.selection.outputs.jobs), 'podman-external-driver-e2e') - permissions: - actions: read - contents: read - packages: read - uses: ./.github/workflows/e2e-podman-test.yml - with: - checkout-ref: ${{ inputs.source-sha }} - test-family: ${{ inputs.test-family }} - image-tag: ${{ inputs.image-tag }} - gateway-artifact: openshell-gateway-plain-x86_64-unknown-linux-gnu - external-driver-binary: openshell-driver-podman - suite-matrix: >- - [{"suite":"external-driver","runner":"ubuntu-26.04","podman_major":"5","podman_package_version":"5.7.0+ds2-3build1","conmon_package_version":"2.1.13+ds1-2","cmd":"mise run --no-deps --skip-deps e2e:podman:external-driver"}] - - - vm-external-driver-e2e: - needs: selection - if: contains(fromJSON(needs.selection.outputs.jobs), 'vm-external-driver-e2e') - permissions: - actions: read - contents: read - packages: read - uses: ./.github/workflows/e2e-vm-test.yml - with: - checkout-ref: ${{ inputs.source-sha }} - test-family: ${{ inputs.test-family }} - gateway-artifact: openshell-gateway-plain-x86_64-unknown-linux-gnu - suite-name: external-driver - e2e-task: e2e:vm:external-driver - - - gpu-e2e: - needs: selection - if: contains(fromJSON(needs.selection.outputs.jobs), 'gpu-e2e') - permissions: - actions: read - contents: read - packages: read - uses: ./.github/workflows/e2e-gpu-test.yaml - with: - checkout-ref: ${{ inputs.source-sha }} - test-family: ${{ inputs.test-family }} - image-tag: ${{ inputs.image-tag }} - - - kubernetes-e2e: - needs: selection - if: contains(fromJSON(needs.selection.outputs.jobs), 'kubernetes-e2e') - strategy: - fail-fast: false - matrix: - include: - - agent_sandbox_api: v1beta1 - agent_sandbox_version: v1.0.3 - - agent_sandbox_api: v1alpha1 - agent_sandbox_version: v0.4.6 - permissions: - actions: read - contents: read - packages: read - uses: ./.github/workflows/e2e-kubernetes-test.yml - with: - checkout-ref: ${{ inputs.source-sha }} - test-family: ${{ inputs.test-family }} - image-tag: ${{ inputs.image-tag }} - job-name: Kubernetes E2E (Rust smoke, Agent Sandbox ${{ matrix.agent_sandbox_api }}) - agent-sandbox-version: ${{ matrix.agent_sandbox_version }} - - - kubernetes-workspace-managed-e2e: - needs: selection - if: contains(fromJSON(needs.selection.outputs.jobs), 'kubernetes-workspace-managed-e2e') - permissions: - actions: read - contents: read - packages: read - uses: ./.github/workflows/e2e-kubernetes-test.yml - with: - checkout-ref: ${{ inputs.source-sha }} - test-family: ${{ inputs.test-family }} - image-tag: ${{ inputs.image-tag }} - job-name: Kubernetes E2E (workspace managed mode) - e2e-task: e2e:kubernetes:workspace-managed - - - kubernetes-external-driver-e2e: - needs: selection - if: contains(fromJSON(needs.selection.outputs.jobs), 'kubernetes-external-driver-e2e') - permissions: - actions: read - contents: read - packages: read - uses: ./.github/workflows/e2e-kubernetes-test.yml - with: - checkout-ref: ${{ inputs.source-sha }} - test-family: ${{ inputs.test-family }} - image-tag: ${{ inputs.image-tag }} - job-name: Kubernetes E2E (external compute driver) - e2e-task: e2e:kubernetes:external-driver - gateway-artifact: openshell-gateway-plain-x86_64-unknown-linux-gnu - external-driver-binary: openshell-driver-kubernetes - cluster-images: sandbox supervisor - - - kubernetes-workspace-operator-e2e: - needs: selection - if: contains(fromJSON(needs.selection.outputs.jobs), 'kubernetes-workspace-operator-e2e') - permissions: - actions: read - contents: read - packages: read - uses: ./.github/workflows/e2e-kubernetes-test.yml - with: - checkout-ref: ${{ inputs.source-sha }} - test-family: ${{ inputs.test-family }} - image-tag: ${{ inputs.image-tag }} - job-name: Kubernetes E2E (workspace operator mode) - e2e-task: e2e:kubernetes:workspace-operator - - - kubernetes-ha-e2e: - needs: selection - if: contains(fromJSON(needs.selection.outputs.jobs), 'kubernetes-ha-e2e') - permissions: - actions: read - contents: read - packages: read - uses: ./.github/workflows/e2e-kubernetes-test.yml - with: - checkout-ref: ${{ inputs.source-sha }} - test-family: ${{ inputs.test-family }} - image-tag: ${{ inputs.image-tag }} - job-name: Kubernetes HA E2E (Rust smoke) - extra-helm-values: deploy/helm/openshell/ci/values-high-availability.yaml - external-postgres-secret: openshell-ha-pg - test-name: kubernetes_ha_rebalancing - kubernetes-features: e2e,e2e-host-gateway,e2e-kubernetes,e2e-kubernetes-ha - use-envoy-gateway: true - - - kubernetes-credential-drivers-e2e: - needs: selection - if: contains(fromJSON(needs.selection.outputs.jobs), 'kubernetes-credential-drivers-e2e') - permissions: - actions: read - contents: read - packages: read - uses: ./.github/workflows/e2e-kubernetes-test.yml - with: - checkout-ref: ${{ inputs.source-sha }} - test-family: ${{ inputs.test-family }} - image-tag: ${{ inputs.image-tag }} - job-name: Kubernetes Credential Drivers E2E - e2e-task: e2e:kubernetes:credential-drivers - - - result: - name: Runtime family result - if: always() - outputs: - docker-result: ${{ needs.docker-e2e.result }} - vm-result: ${{ needs.vm-e2e.result }} - needs: - - selection - - docker-e2e - - podman-e2e - - vm-e2e - - docker-external-driver-e2e - - podman-external-driver-e2e - - vm-external-driver-e2e - - gpu-e2e - - kubernetes-e2e - - kubernetes-workspace-managed-e2e - - kubernetes-external-driver-e2e - - kubernetes-workspace-operator-e2e - - kubernetes-ha-e2e - - kubernetes-credential-drivers-e2e - runs-on: ubuntu-latest - steps: - - name: Require every selected runtime job to succeed - env: - JOB_RESULTS: ${{ toJSON(needs) }} - SELECTED_JOBS: ${{ needs.selection.outputs.jobs }} - run: | - set -euo pipefail - jq -e --argjson selected "$SELECTED_JOBS" ' - . as $results - | .selection.result == "success" - and ($selected | length > 0) - and all($selected[]; $results[.].result == "success") - ' <<< "$JOB_RESULTS" diff --git a/CI.md b/CI.md index c4dc73f9ad..9a73c327e7 100644 --- a/CI.md +++ b/CI.md @@ -151,10 +151,13 @@ local HTTP gateway, and CLI path. The manual Integration Tests workflow defaults to the package installers and downloads the packages selected by its matrix. Branch and release workflows call the conformance, features, and drivers family -workflows. Conformance and drivers select runtime jobs through -`e2e-runtime-tests.yml`; the same recorded job list drives execution and the -completion check. A selected job that fails, is cancelled, or is skipped fails -its family. The existing core/GPU required result names and optional Kubernetes +workflows within the same Actions run. Each family calls the reusable driver +workflows directly. Driver matrices expand regular/external variants and +Kubernetes configurations; driver workflows can expand their own scenario +matrices. Features exposes provider-refresh and OCI-image separately. Each +family checks that every job required by its profile succeeds, so a failed, +cancelled, missing, or skipped required job fails its family. The existing +core/GPU required result names and optional Kubernetes policy remain unchanged, including advisory WSL GPU execution. The `core` profile retains the branch target matrix. The `release` profile @@ -567,8 +570,7 @@ The bot's full administrator documentation is internal to NVIDIA. The only comma | `.github/workflows/build-binaries.yml`, `build-vm-driver.yml` | Shared binary matrices used by branch and release workflows. The VM driver remains separate because its build consumes the runtime binaries. | | `.github/workflows/build-images.yml` | Builds and pushes multi-platform images, then uploads the same OCI images as workflow artifacts. | | `.github/workflows/package-release-binaries.yml` | Packages raw build artifacts into release tarballs without rebuilding them. | -| `.github/workflows/e2e-conformance.yml`, `e2e-features.yml`, `e2e-drivers.yml` | Test-family entrypoints shared by branch and release workflows. Retain the current integration matrices and runtime profiles. | -| `.github/workflows/e2e-runtime-tests.yml` | Selects runtime jobs by family and execution profile and requires every selected job to succeed. | +| `.github/workflows/e2e-conformance.yml`, `e2e-features.yml`, `e2e-drivers.yml` | Test-family entrypoints shared by branch and release workflows. Call driver adapters directly with variant/configuration matrices and require every selected job to succeed. Retain the current integration matrices and runtime profiles. | | `.github/workflows/e2e-docker-test.yml`, `e2e-podman-test.yml`, `e2e-vm-test.yml`, `e2e-kubernetes-test.yml`, `e2e-gpu-test.yaml` | Reusable target preparation and execution adapters. Accept an explicit test family while keeping combined execution as their default. | | `.github/actions/setup-e2e-*` | Shared artifact, Podman, KVM, and kind setup used by the runtime lanes. | | `.github/workflows/helm-lint.yml` | Helm chart validation. PR mirror pushes skip lint jobs unless Helm inputs changed; merge groups always validate Helm because they represent the final integration state. | diff --git a/TESTING.md b/TESTING.md index 1109b8bfd8..3da4992417 100644 --- a/TESTING.md +++ b/TESTING.md @@ -130,6 +130,11 @@ uses the shared package in `tests/suites/conformance/`; features retains the provider-refresh and OCI-image suites. Drivers collects the existing runtime and host-specific suites, including mixed legacy tests pending extraction. Bundled versus external compute drivers are target variants across families. +Each family calls driver workflows directly, with matrices for applicable +variants and configurations. Regular and external VM targets run in both +conformance and drivers. Features exposes provider-refresh and OCI-image as +separate calls with their existing target matrices. All families share the +artifacts built by the parent workflow and remain in the same Actions run. The Docker, Podman, Kubernetes, and VM Rust harnesses accept `OPENSHELL_E2E_TEST_FAMILY=conformance` or `drivers` to execute each part diff --git a/tasks/scripts/e2e_lanes_test.py b/tasks/scripts/e2e_lanes_test.py index 6a0377dd34..6c9758f12b 100644 --- a/tasks/scripts/e2e_lanes_test.py +++ b/tasks/scripts/e2e_lanes_test.py @@ -5,6 +5,7 @@ import json import os +import re import subprocess from pathlib import Path @@ -20,15 +21,50 @@ def workflow(name): return yaml.load((WORKFLOWS / name).read_text(), Loader=yaml.BaseLoader) -def selected(family, profile): - result = subprocess.run( - ["bash", "tasks/scripts/select-e2e-runtime-jobs.sh", family, profile], - cwd=ROOT, - text=True, - capture_output=True, - check=True, - ) - return json.loads(result.stdout) +def matrix_rows(job, profile="core"): + expression = job["strategy"]["matrix"]["include"] + if isinstance(expression, list): + return expression + if "inputs.profile" in expression: + for candidate, rows in re.findall( + r"inputs.profile == '([^']+)' && '([^']+)'", expression + ): + if profile == candidate: + return json.loads(rows) + return json.loads(re.search(r"\|\|\s+'([^']+)'\) }}$", expression)[1]) + return json.loads(re.search(r"fromJSON\('([^']+)'\)", expression)[1]) + + +def family_targets(family, profile): + """Expand the actual family matrices into identifiable coverage targets.""" + targets = set() + for driver, job in workflow(f"e2e-{family}.yml")["jobs"].items(): + if not job.get("uses", "").startswith("./.github/workflows/e2e-"): + continue + profiles = re.findall(r"inputs.profile == '([^']+)'", job["if"]) + if profile not in profiles: + continue + if driver == "gpu": + targets.add("gpu-e2e") + continue + for row in matrix_rows(job, profile): + if driver == "kubernetes": + task = row["e2e-task"] + if task == "e2e:kubernetes": + key = ( + "kubernetes-ha-e2e" + if profile == "kubernetes-ha" + else f"kubernetes-{row['agent-sandbox-version']}-e2e" + ) + else: + key = task.removeprefix("e2e:").replace(":", "-") + "-e2e" + key = key.replace("credential-drivers", "credentials") + else: + key = driver + ( + "-external-driver-e2e" if row["variant"] == "external" else "-e2e" + ) + targets.add(key) + return targets @pytest.fixture @@ -160,32 +196,41 @@ def test_runtime_profiles_preserve_existing_targets(): "docker-e2e", "podman-e2e", "vm-e2e", - "kubernetes-e2e", + "kubernetes-v1.0.3-e2e", + "kubernetes-v0.4.6-e2e", "kubernetes-workspace-managed-e2e", "vm-external-driver-e2e", } - assert set(selected("conformance", "core")) == core | { + assert set(family_targets("conformance", "core")) == core | { "docker-external-driver-e2e", "podman-external-driver-e2e", "kubernetes-external-driver-e2e", } - assert set(selected("drivers", "core")) == core | { + assert set(family_targets("drivers", "core")) == core | { "kubernetes-workspace-operator-e2e" } for family in ("conformance", "drivers"): - assert set(selected(family, "release")) == {"docker-e2e", "vm-e2e"} - assert selected(family, "gpu") == ["gpu-e2e"] - assert selected(family, "kubernetes-ha") == ["kubernetes-ha-e2e"] - assert selected(family, "kubernetes-credentials") == [ - "kubernetes-credential-drivers-e2e" - ] + assert set(family_targets(family, "release")) == {"docker-e2e", "vm-e2e"} + assert family_targets(family, "gpu") == {"gpu-e2e"} + assert family_targets(family, "kubernetes-ha") == {"kubernetes-ha-e2e"} + assert family_targets(family, "kubernetes-credentials") == { + "kubernetes-credentials-e2e" + } def test_docker_split_retains_all_six_legacy_suites(): - runtime = workflow("e2e-runtime-tests.yml")["jobs"]["docker-e2e"]["with"] - # Inspect the two literal branches of the Actions family expression. - conformance, drivers = runtime["suite-matrix"].split(" && '")[1].split("' || '") - rows = json.loads(drivers.removesuffix("' }}\n").removesuffix("' }}")) + conformance = matrix_rows(workflow("e2e-conformance.yml")["jobs"]["docker"])[0] + drivers = matrix_rows(workflow("e2e-drivers.yml")["jobs"]["docker"])[0] + rows = drivers["suites"] + # BaseLoader keeps YAML scalar values as strings; JSON suite defaults + # contain native booleans. Compare the equivalent execution inputs. + rows = [ + { + key: value == "true" if key in ("python_proto", "mcp") else value + for key, value in row.items() + } + for row in rows + ] legacy = workflow("e2e-docker-test.yml")["on"]["workflow_call"]["inputs"][ "suite-matrix" ]["default"] @@ -198,34 +243,54 @@ def test_docker_split_retains_all_six_legacy_suites(): "oci-image", "mcp", ] - assert [row["cmd"] for row in json.loads(conformance)] == [ + assert [row["cmd"] for row in conformance["suites"]] == [ "mise run --no-deps --skip-deps e2e:rust" ] -@pytest.mark.parametrize("family,profile", [("features", "core"), ("drivers", "typo")]) -def test_unknown_runtime_requests_are_rejected(family, profile): - result = subprocess.run( - ["bash", "tasks/scripts/select-e2e-runtime-jobs.sh", family, profile], - cwd=ROOT, - capture_output=True, - ) - assert result.returncode != 0 and not result.stdout +@pytest.mark.parametrize("family", ["conformance", "drivers"]) +@pytest.mark.parametrize( + "profile", + ["core", "release", "gpu", "kubernetes-ha", "kubernetes-credentials", "typo"], +) +@pytest.mark.parametrize("status", ["success", "failure", "skipped", "cancelled", None]) +def test_family_completion_requires_every_selected_job(family, profile, status): + jobs = workflow(f"e2e-{family}.yml")["jobs"] + result_job = jobs["result"] + expected = [ + key + for key in result_job["needs"] + if profile in re.findall(r"inputs.profile == '([^']+)'", jobs[key]["if"]) + ] + assert expected or profile == "typo" + for selected in expected or ["unknown"]: + results = { + key: {"result": "success" if key in expected else "skipped"} + for key in result_job["needs"] + } + if status is None: + results.pop(selected, None) + else: + results[selected] = {"result": status} + result = subprocess.run( + ["bash", "-c", result_job["steps"][0]["run"]], + env=os.environ | {"JOB_RESULTS": json.dumps(results), "PROFILE": profile}, + capture_output=True, + ) + assert (result.returncode == 0) == (status == "success" and profile != "typo") -@pytest.mark.parametrize("status", ["success", "failure", "skipped", "cancelled", None]) -def test_runtime_completion_requires_every_selected_job(status): - step = workflow("e2e-runtime-tests.yml")["jobs"]["result"]["steps"][0] - results = {"selection": {"result": "success"}, "unselected": {"result": "skipped"}} - if status is not None: - results["selected"] = {"result": status} - result = subprocess.run( - ["bash", "-c", step["run"]], - env=os.environ - | {"JOB_RESULTS": json.dumps(results), "SELECTED_JOBS": '["selected"]'}, - capture_output=True, - ) - assert (result.returncode == 0) == (status == "success") +def test_families_call_driver_workflows_directly(): + assert not (WORKFLOWS / "e2e-runtime-tests.yml").exists() + for family in ("conformance", "drivers"): + jobs = workflow(f"e2e-{family}.yml")["jobs"] + for driver in ("docker", "podman", "vm", "kubernetes", "gpu"): + job = jobs[driver] + suffix = "yaml" if driver == "gpu" else "yml" + assert job["uses"] == f"./.github/workflows/e2e-{driver}-test.{suffix}" + assert job["with"]["test-family"] == family + if driver != "gpu": + assert job["strategy"]["fail-fast"] == "false" def test_existing_matrices_and_artifact_variants(): @@ -238,13 +303,18 @@ def test_existing_matrices_and_artifact_variants(): ("fedora-podman-rootful", "rpm"), ("fedora-podman-rootless", "rpm"), } - features = workflow("e2e-features.yml")["jobs"]["feature-specific-integration"] - rows = json.loads(features["with"]["test-matrix"]) + features = workflow("e2e-features.yml")["jobs"] + rows = [ + row + for job in features.values() + for row in json.loads(job["with"]["test-matrix"]) + ] assert len(rows) == 5 assert ( sorted(r["testsuite"] for r in rows) == ["oci-image"] * 3 + ["provider-refresh"] * 2 ) + assert len({job["with"]["category"] for job in features.values()}) == 2 drivers = workflow("e2e-drivers.yml")["jobs"]["driver-specific-integration"] rows = json.loads(drivers["with"]["test-matrix"]) assert {(r["environment"], r["installer"], r["testsuite"]) for r in rows} == { @@ -252,10 +322,11 @@ def test_existing_matrices_and_artifact_variants(): ("fedora-podman-rootless", "binaries", "e2e-podman"), ("fedora-podman-rootless", "binaries", "driver-podman"), } - runtime = workflow("e2e-runtime-tests.yml")["jobs"] - assert runtime["kubernetes-e2e"]["strategy"]["matrix"]["include"] == [ - {"agent_sandbox_api": "v1beta1", "agent_sandbox_version": "v1.0.3"}, - {"agent_sandbox_api": "v1alpha1", "agent_sandbox_version": "v0.4.6"}, + runtime = workflow("e2e-conformance.yml")["jobs"] + kube_rows = matrix_rows(runtime["kubernetes"]) + assert [(r["job-name"], r["agent-sandbox-version"]) for r in kube_rows[:2]] == [ + ("Kubernetes E2E (Rust smoke, Agent Sandbox v1beta1)", "v1.0.3"), + ("Kubernetes E2E (Rust smoke, Agent Sandbox v1alpha1)", "v0.4.6"), ] for driver, arch in [ ("docker", "aarch64"), @@ -263,11 +334,13 @@ def test_existing_matrices_and_artifact_variants(): ("kubernetes", "x86_64"), ("vm", "x86_64"), ]: - lane = runtime[f"{driver}-external-driver-e2e"]["with"] - assert ( - lane["gateway-artifact"] - == f"openshell-gateway-plain-{arch}-unknown-linux-gnu" - ) + for row in matrix_rows(runtime[driver]): + artifact = row.get("gateway_artifact", row.get("gateway-artifact", "")) + if artifact: + assert artifact == f"openshell-gateway-plain-{arch}-unknown-linux-gnu" + break + else: + pytest.fail(f"Missing external {driver} coverage") gpu = workflow("e2e-gpu-test.yaml")["jobs"]["e2e-gpu"] assert [ (r["name"], r["experimental"]) for r in gpu["strategy"]["matrix"]["include"] diff --git a/tasks/scripts/select-e2e-runtime-jobs.sh b/tasks/scripts/select-e2e-runtime-jobs.sh deleted file mode 100644 index c72e91ae66..0000000000 --- a/tasks/scripts/select-e2e-runtime-jobs.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/usr/bin/env bash -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -# List the runtime jobs required by one family/profile. The workflow uses this -# same list for execution and completion, so skipped selected jobs fail the gate. -set -euo pipefail - -family="${1:-}" -profile="${2:-}" -case "${family}" in - conformance|drivers) ;; - *) echo "error: expected conformance or drivers, got '${family}'" >&2; exit 2 ;; -esac - -case "${profile}" in - core) - jobs=(docker-e2e podman-e2e vm-e2e kubernetes-e2e kubernetes-workspace-managed-e2e vm-external-driver-e2e) - if [ "${family}" = conformance ]; then - jobs+=(docker-external-driver-e2e podman-external-driver-e2e kubernetes-external-driver-e2e) - else - # Operator mode currently excludes conformance; see #2971. - jobs+=(kubernetes-workspace-operator-e2e) - fi - ;; - release) jobs=(docker-e2e vm-e2e) ;; - gpu) jobs=(gpu-e2e) ;; - kubernetes-ha) jobs=(kubernetes-ha-e2e) ;; - kubernetes-credentials) jobs=(kubernetes-credential-drivers-e2e) ;; - *) echo "error: unknown runtime profile '${profile}'" >&2; exit 2 ;; -esac - -jq -cn --args '$ARGS.positional' "${jobs[@]}"