feat(runner): add the test runner, plans and tmachine Session API, and run a plan on CI

.agents/skills/watch-github-actions/SKILL.md

@@ -127,6 +127,8 @@ gh run list --json databaseId,status,headBranch,url --jq '.[] | {id: .databaseId
 
 For tmachine Kubernetes conformance failures, inspect the `tmachine-diagnostics-*` artifact. The K3s report includes Agent Sandbox controller state, leader leases, events, and current and previous controller logs. The conformance playbook checks reconciliation after guest boot with a temporary scheduling-gated Sandbox probe before running tests. Distinguish a failed controller probe from a failed conformance scenario; gateway connectivity and controller Deployment availability do not prove reconciliation works.
 
+For the `e2e-suites` lane (`.github/workflows/e2e-suites.yml`), the last lines of the job log give the runner's result: PASS, FAIL with the failed tests, or ERROR with the step that broke. Then inspect the `e2e-results-<family>-<package>-<environment>-<installer>` artifact, which holds `junit.xml`, `nextest.log` and, after a failure, the guest's `diagnostics/`.
+
 `Pre-release` temporarily omits Codex Security from its `Security Scan` job.
 `Release` uses the passing prerelease qualification result. Other scanners and
 qualification gates still run. Use the standalone

.github/workflows/branch-checks.yml

@@ -222,6 +222,9 @@ jobs:
           cargo fmt --manifest-path examples/supervisor-middleware-content-guard/Cargo.toml --all -- --check
           cargo fmt --manifest-path examples/isolation-backend-v8/Cargo.toml --all -- --check
           cargo fmt --manifest-path tests/testkit/Cargo.toml --all -- --check
+          cargo fmt --manifest-path tests/tmachine/Cargo.toml --all -- --check
+          cargo fmt --manifest-path tests/runner/Cargo.toml --all -- --check
+          cargo fmt --manifest-path tests/suites/conformance/Cargo.toml -p sandbox -- --check
 
   cargo-lockfiles:
     name: Cargo lockfiles
@@ -239,7 +242,9 @@ jobs:
 
       - name: Verify Cargo lockfiles
         shell: nix develop -c bash -euo pipefail {0}
-        run: tasks/scripts/check-cargo-lockfiles.sh
+        run: |
+          tasks/scripts/check-cargo-lockfiles.sh
+          uv run tasks/scripts/check_cargo_lockfile_alignment.py
 
   rust-lint:
     name: Rust lint (${{ matrix.system }})
@@ -279,6 +284,9 @@ jobs:
           cargo clippy --locked --manifest-path examples/supervisor-middleware-content-guard/Cargo.toml --all-targets -- -D warnings
           cargo clippy --locked --manifest-path examples/isolation-backend-v8/Cargo.toml --all-targets -- -D warnings
           cargo clippy --locked --manifest-path tests/testkit/Cargo.toml --all-targets -- -D warnings
+          cargo clippy --locked --manifest-path tests/tmachine/Cargo.toml --all-targets -- -D warnings
+          cargo clippy --locked --manifest-path tests/runner/Cargo.toml --all-targets -- -D warnings
+          cargo clippy --locked --manifest-path tests/suites/conformance/Cargo.toml -p sandbox --all-targets -- -D warnings
 
   rust-test:
     name: Rust test (${{ matrix.system }})
@@ -328,6 +336,8 @@ jobs:
           cargo nextest run --locked --profile ci --workspace --features openshell-server/test-support
           cargo nextest run --locked --config-file .config/nextest.toml --profile ci --manifest-path examples/supervisor-middleware-content-guard/Cargo.toml
           cargo nextest run --locked --config-file .config/nextest.toml --profile ci --manifest-path tests/testkit/Cargo.toml
+          cargo nextest run --locked --config-file .config/nextest.toml --profile ci --manifest-path tests/tmachine/Cargo.toml
+          cargo nextest run --locked --config-file .config/nextest.toml --profile ci --manifest-path tests/runner/Cargo.toml
 
   rust-postgres-test:
     name: Rust PostgreSQL tests

.github/workflows/branch-e2e.yml

@@ -303,6 +303,21 @@ jobs:
           {"environment":"fedora-podman-rootless","installer":"binaries","testsuite":"driver-podman"}
         ]
 
+  # Run the testkit suites in tests/suites/ on tmachine. The environments and
+  # suites are listed in e2e-suites.yml. Not yet in core-e2e-result.needs: it
+  # starts out advisory.
+  e2e-suites:
+    needs: [pr_metadata, prepare-integration]
+    if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true'
+    permissions:
+      actions: read
+      contents: read
+      packages: read
+    uses: ./.github/workflows/e2e-suites.yml
+    with:
+      source-sha: ${{ needs.prepare-integration.outputs.source_sha }}
+      integration-inputs-artifact-id: ${{ needs.prepare-integration.outputs.integration_inputs_artifact_id }}
+
   docker-e2e:
     needs: [pr_metadata, build-binaries, build-images]
     if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true'

.github/workflows/e2e-suites.yml

@@ -0,0 +1,129 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+# Runs testkit suites in tmachine environments on CI. Each matrix row names a
+# tmachine environment and installer, the family (a workspace under
+# tests/suites/) and the package in it, and the runner label for the
+# environment's architecture. `openshell-test-runner` builds the tests, boots
+# the environment, runs them inside it and checks the results.
+#
+# Modeled on the E2E Suites workflow in NVIDIA/OpenShell#4629 by Evan Lezar.
+
+name: E2E Suites
+
+on:
+  workflow_call:
+    inputs:
+      source-sha:
+        description: Source revision of the prepared candidate artifacts
+        required: true
+        type: string
+      integration-inputs-artifact-id:
+        description: Immutable ID of the prepared integration inputs artifact
+        required: true
+        type: string
+
+permissions:
+  actions: read
+  contents: read
+  packages: read
+
+# github.workflow is the caller's name here, so scope the group to this
+# workflow to avoid colliding with (and cancelling) the caller's other jobs.
+concurrency:
+  group: ${{ github.workflow }}-e2e-suites-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  suites:
+    name: E2E suites (${{ matrix.family }}/${{ matrix.package }}, ${{ matrix.environment }}, ${{ matrix.installer }})
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          # Only amd64 for now: CI has no arm64 runner with KVM for tmachine,
+          # and the caller only builds amd64 inputs.
+          - environment: ubuntu-docker-rootful
+            installer: deb
+            family: conformance
+            package: sandbox
+            runner: ubuntu-24.04
+    runs-on: ${{ matrix.runner }}
+    timeout-minutes: 90
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          ref: ${{ inputs['source-sha'] }}
+          persist-credentials: false
+
+      - name: Enable KVM access
+        run: |
+          set -euo pipefail
+          if [[ ! -c /dev/kvm ]]; then
+            echo "::error::The runner did not expose /dev/kvm"
+            exit 1
+          fi
+          sudo chmod 0666 /dev/kvm
+          exec 3<>/dev/kvm
+          exec 3>&-
+
+      - uses: ./.github/actions/setup-nix
+
+      - name: Cache tmachine disks
+        uses: actions/cache@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0
+        with:
+          path: ~/.cache/tmachine
+          # tmachine keys its disk layers on a hash of the Ansible tree.
+          # actions/cache doesn't save after an exact key hit, so a fixed key
+          # would rebuild the changed layers on every run. Key on the playbooks
+          # and fall back to the newest entry for the same runner.
+          key: tmachine-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('tests/tmachine/ansible/**', 'tests/tmachine/images.nix') }}
+          restore-keys: |
+            tmachine-${{ runner.os }}-${{ runner.arch }}-
+
+      - name: Download integration inputs
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          artifact-ids: ${{ inputs['integration-inputs-artifact-id'] }}
+          path: artifacts
+
+      - name: Run the suites in this environment
+        env:
+          ENVIRONMENT: ${{ matrix.environment }}
+          INSTALLER: ${{ matrix.installer }}
+          FAMILY: ${{ matrix.family }}
+          PACKAGE: ${{ matrix.package }}
+        run: |
+          # Retry dependency preparation, then run the suites once. The runner
+          # builds the suites for the guest, so it needs no prepared archives.
+          nix build --no-link .#openshell-test-runner || nix build --no-link .#openshell-test-runner
+          nix run .#openshell-test-runner -- \
+            --environment "${ENVIRONMENT}" \
+            --installer "${INSTALLER}" \
+            --family "${FAMILY}" \
+            --package "${PACKAGE}" \
+            --results artifacts/test-results
+
+      # The results directory holds the JUnit report (junit.xml), the nextest
+      # log, and the guest's diagnostics when the run failed.
+      - name: Upload results
+        if: always()
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
+        with:
+          name: e2e-results-${{ matrix.family }}-${{ matrix.package }}-${{ matrix.environment }}-${{ matrix.installer }}
+          path: artifacts/test-results
+          if-no-files-found: ignore
+          retention-days: 7
+
+  result:
+    name: E2E suites result
+    needs: [suites]
+    if: always()
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+      - uses: ./.github/actions/check-job-results
+        with:
+          results: ${{ toJSON(needs) }}

AGENTS.md

@@ -63,6 +63,9 @@ Do not rely on this file for a full inventory. The detailed public and contribut
 | `sdk/typescript/` | TypeScript SDK | Native Connect client, curated sandbox API, and generated protobuf types |
 | `proto/` | Protobuf definitions | gRPC service contracts |
 | `tests/testkit/` | E2E testkit | Support library and `#[testkit::test]` attribute for SDK-driven E2E suites |
+| `tests/suites/` | E2E suites | One Cargo workspace per test family; `conformance/sandbox` is the first testkit suite |
+| `tests/tmachine/` | Test machines | QEMU machines provisioned with Ansible, and the `Session` API that drives them |
+| `tests/runner/` | Test runner | `openshell-test-runner`: builds a family's tests, runs them in one tmachine environment, checks the JUnit results |
 | `deploy/` | Docker, Helm, K8s | Dockerfiles, Helm chart, manifests |
 | `docs/` | Published docs | MDX pages, navigation, and content assets |
 | `fern/` | Docs site config | Fern site config, components, and theme assets |

CONTRIBUTING.md

@@ -345,6 +345,9 @@ These are the primary `mise` tasks for day-to-day development:
 | `proto/`        | Protocol buffer definitions and [public API conventions](proto/README.md) |
 | `tasks/`        | `mise` task definitions and build scripts     |
 | `tests/testkit/` | Support library for SDK-driven end-to-end suites |
+| `tests/suites/` | End-to-end suites, one Cargo workspace per family |
+| `tests/tmachine/` | QEMU test machines provisioned with Ansible, and the `Session` API that drives them |
+| `tests/runner/` | `openshell-test-runner`: runs a test family in a tmachine environment and checks the results |
 | `deploy/`       | Dockerfiles, Helm chart, Kubernetes manifests |
 | `docs/`         | Published Fern docs source, navigation, and content assets |
 | `fern/`         | Fern site config, components, and theme assets |

TESTING.md

@@ -17,6 +17,9 @@ python/openshell/      # Python unit tests (*_test.py suffix)
 e2e/python/            # Python E2E tests (test_*.py prefix)
 e2e/rust/              # Rust CLI E2E tests
 tests/testkit/         # Support library for SDK-driven E2E suites
+tests/suites/conformance/  # Testkit suites (the `sandbox` crate) beside the older CLI conformance crate
+tests/tmachine/        # QEMU test machines, Ansible playbooks, and the Session API
+tests/runner/          # openshell-test-runner: runs a test family in a tmachine environment
 ```
 
 ## Rust Tests
@@ -140,6 +143,71 @@ by `OPENSHELL_BIN`. See
 [tests/testkit/README.md](tests/testkit/README.md) for how to write and run
 tests.
 
+Suites live under `tests/suites/<family>/`. Each family is a Cargo workspace
+with its own lockfile. `tests/suites/conformance/sandbox` is the first testkit
+suite; it has one smoke test. Run it against a local Docker gateway with:
+
+```shell
+mise run e2e:testkit
+```
+
+That task runs `e2e/with-docker-gateway.sh`, which starts a gateway and exports
+`OPENSHELL_GATEWAY` and `OPENSHELL_BIN`, so the suite runs the same way the
+runner runs it in a guest.
+
+Workspaces that build root crates through path dependencies must lock shared
+crates at the versions in the root `Cargo.lock`; otherwise CI can build against
+dependencies the Nix toolchain doesn't support. `mise run rust:lockfiles:check`
+verifies this, and `uv run tasks/scripts/check_cargo_lockfile_alignment.py --fix`
+re-locks a workspace from the root lockfile. Add new testkit workspaces to
+`ALIGNED_WORKSPACES` in that script.
+
+### Running suites in a tmachine environment (`tests/runner/`)
+
+`openshell-test-runner` runs a test family inside a tmachine VM, against the
+OpenShell that the VM's installer set up. It is adapted from the runner in
+NVIDIA/OpenShell#4629 by Evan Lezar.
+
+```shell
+nix run .#openshell-test-runner -- \
+  --environment ubuntu-docker-rootful --installer deb \
+  --family conformance --package sandbox
+```
+
+It:
+
+1. builds the family's packages on the host as a nextest archive for the
+   guest's musl target, with `testkit-ready`;
+2. boots the tmachine environment with the installer (this needs KVM, and
+   tmachine only boots guests of the host's architecture);
+3. copies the archive in, waits for the gateway with `testkit-ready`, and runs
+   the tests with the guest's `cargo-nextest`, pointing them at the installed
+   CLI (`OPENSHELL_BIN`) and the gateway the installer registered
+   (`OPENSHELL_GATEWAY`, `--gateway`, default `openshell`);
+4. writes `junit.xml`, `nextest.log` and, after a failure, the guest's
+   diagnostics to `artifacts/test-results/` (`--results`);
+5. shuts the VM down, also on failure or SIGTERM.
+
+A run passes if nextest succeeds and its JUnit report shows at least one test
+and no failures. `--filter <filterset>` narrows the tests with a nextest
+filterset, and `--keep` leaves the VM running until you press Enter. Every SSH
+and nextest step has a timeout. Two runs on one host need different
+`TMACHINE_SSH_PORT` values (default 2222). The installer inputs, such as the
+Debian package and the images, come from `artifacts/`, built by
+`nix run .#build-artifacts` or downloaded by CI.
+
+A flaky or broken test is marked `#[ignore = "<issue link>"]`, which takes it
+out of every environment until it's fixed.
+
+**CI.** `.github/workflows/e2e-suites.yml` lists the environments and suites as
+an explicit matrix and runs the runner once per row. Branch E2E calls it when
+the `test:e2e` label is set. It is advisory for now. To run a suite in another
+environment on CI, add a row.
+
+The older conformance, feature and driver lanes in `.github/workflows/` still
+run through the tmachine testsuites in `tests/tmachine/config.nix`. They move
+to the runner as their suites move to testkit.
+
 ### Python E2E (`e2e/python/`)
 
 `mise run e2e:python` builds `openshell/e2e-python:dev` from

flake.nix

@@ -127,7 +127,7 @@
           firmwarePkgs = testGuestPkgs;
         };
         testMachines = import ./tests/tmachine/config.nix {
-          inherit pkgs toolchains;
+          inherit pkgs toolchains rustToolchain;
           qemuPkgs = tmachineRuntimePkgs;
           firmwarePkgs = tmachineRuntimePkgs;
         };
@@ -203,6 +203,7 @@
           tmachine = testMachines.package;
           tmachine-config = testMachines.config;
           tmachine-unwrapped = testMachines.unwrapped;
+          openshell-test-runner = testMachines.testRunner;
         };
 
         devShells = {

tasks/rust.toml

@@ -18,8 +18,11 @@ run_windows = "powershell -NoProfile -ExecutionPolicy Bypass -File tasks/scripts
 hide = true
 
 ["rust:lockfiles:check"]
-description = "Verify all tracked Cargo lockfiles are current"
-run = "tasks/scripts/check-cargo-lockfiles.sh"
+description = "Verify all tracked Cargo lockfiles are current and aligned with the root"
+run = [
+  "tasks/scripts/check-cargo-lockfiles.sh",
+  "uv run tasks/scripts/check_cargo_lockfile_alignment.py",
+]
 run_windows = "powershell -NoProfile -ExecutionPolicy Bypass -File tasks/scripts/check-cargo-lockfiles.ps1"
 hide = true
 
@@ -34,6 +37,9 @@ run = [
   "cargo clippy --manifest-path examples/supervisor-middleware-content-guard/Cargo.toml --all-targets -- -D warnings",
   "cargo clippy --manifest-path examples/isolation-backend-v8/Cargo.toml --all-targets -- -D warnings",
   "cargo clippy --manifest-path tests/testkit/Cargo.toml --all-targets -- -D warnings",
+  "cargo clippy --manifest-path tests/tmachine/Cargo.toml --all-targets -- -D warnings",
+  "cargo clippy --manifest-path tests/runner/Cargo.toml --all-targets -- -D warnings",
+  "cargo clippy --manifest-path tests/suites/conformance/Cargo.toml -p sandbox --all-targets -- -D warnings",
 ]
 run_windows = "powershell -NoProfile -ExecutionPolicy Bypass -File tasks/scripts/windows-msvc.ps1 lint native"
 hide = true
@@ -47,6 +53,9 @@ run = [
   "cargo fmt --manifest-path examples/supervisor-middleware-content-guard/Cargo.toml --all",
   "cargo fmt --manifest-path examples/isolation-backend-v8/Cargo.toml --all",
   "cargo fmt --manifest-path tests/testkit/Cargo.toml --all",
+  "cargo fmt --manifest-path tests/tmachine/Cargo.toml --all",
+  "cargo fmt --manifest-path tests/runner/Cargo.toml --all",
+  "cargo fmt --manifest-path tests/suites/conformance/Cargo.toml -p sandbox",
 ]
 hide = true
 
@@ -59,6 +68,9 @@ run = [
   "cargo fmt --manifest-path examples/supervisor-middleware-content-guard/Cargo.toml --all -- --check",
   "cargo fmt --manifest-path examples/isolation-backend-v8/Cargo.toml --all -- --check",
   "cargo fmt --manifest-path tests/testkit/Cargo.toml --all -- --check",
+  "cargo fmt --manifest-path tests/tmachine/Cargo.toml --all -- --check",
+  "cargo fmt --manifest-path tests/runner/Cargo.toml --all -- --check",
+  "cargo fmt --manifest-path tests/suites/conformance/Cargo.toml -p sandbox -- --check",
 ]
 hide = true