From 78813fc9dab328072646bad5658f05d3c3577886 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Wed, 21 Oct 2026 22:07:03 -0700 Subject: [PATCH 1/6] feat(testkit): track sandboxes that tests create through the CLI Add cx.track and cx.adopt, so a test that creates a sandbox through the CLI still cleans it up after a failure: track the name before the command, adopt it afterwards for an SDK handle. Signed-off-by: Matthew Grossman --- tests/testkit/README.md | 4 +++ tests/testkit/testkit/src/context.rs | 25 +++++++++++++++++++ ...s-ready.yaml => k3s-controller-ready.yaml} | 0 3 files changed, 29 insertions(+) rename tests/tmachine/ansible/playbooks/{conformance/k3s-ready.yaml => k3s-controller-ready.yaml} (100%) diff --git a/tests/testkit/README.md b/tests/testkit/README.md index 0c1d7f2f9d..3dcfb54798 100644 --- a/tests/testkit/README.md +++ b/tests/testkit/README.md @@ -64,6 +64,10 @@ test creates, such as providers, in the test itself. - `cx.sandbox(suffix)` to build and create a sandbox with a run-unique name. The returned `Sandbox` can wait for ready or finished, stop, start, exec, read its config and draft policy, set a setting, and delete. +- `cx.track(name)` to register a sandbox created some other way, such as + through the CLI, for cleanup. Call it before the + create, so a create that fails after the gateway accepted it is still cleaned + up. `cx.adopt(name)` returns a `Sandbox` handle for one the CLI created. - `cx.expect_listed` and `cx.expect_absent` to check a sandbox in list and get. - `cx.cli([...])` to run the installed CLI against the same gateway, with `success()`, `failure()`, and `json()` on the output. diff --git a/tests/testkit/testkit/src/context.rs b/tests/testkit/testkit/src/context.rs index 24e3f0a37a..9d4b4a0ff5 100644 --- a/tests/testkit/testkit/src/context.rs +++ b/tests/testkit/testkit/src/context.rs @@ -101,6 +101,31 @@ impl Context { } } + /// Track a sandbox the test is about to create through the CLI, so cleanup + /// deletes it even if the test fails after the gateway accepted the create. + /// Call this before the CLI command; use [`Context::adopt`] afterwards. + pub fn track(&self, name: &str) { + lock(&self.sandboxes).insert(name.to_string()); + } + + /// A handle to a sandbox created outside the builder, e.g. by the CLI. + /// It is tracked for cleanup like a built one. + pub async fn adopt(&self, name: &str) -> Result { + self.track(name); + let found = self + .client + .get_sandbox(name) + .await + .context(format!("get sandbox {name}"))?; + Ok(Sandbox { + client: self.client.clone(), + grpc: self.grpc.clone(), + sandboxes: Arc::clone(&self.sandboxes), + name: name.to_string(), + id: found.id, + }) + } + /// Fail unless the sandbox is gone from both get and list. pub async fn expect_absent(&self, sandbox: &Sandbox) -> Result { let name = sandbox.name(); diff --git a/tests/tmachine/ansible/playbooks/conformance/k3s-ready.yaml b/tests/tmachine/ansible/playbooks/k3s-controller-ready.yaml similarity index 100% rename from tests/tmachine/ansible/playbooks/conformance/k3s-ready.yaml rename to tests/tmachine/ansible/playbooks/k3s-controller-ready.yaml From 46ce7097604c3d6aa867fd95360f90565f279d52 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Wed, 21 Oct 2026 22:15:37 -0700 Subject: [PATCH 2/6] test(conformance): port the CLI, lifecycle, file-transfer and persistent-workspace suites to testkit Add the lifecycle, cli, persistent_workspace and file_transfer test files to the sandbox crate. The lifecycle, persistent-workspace and file-transfer tests are adapted from NVIDIA/OpenShell#4629. persistent_workspace_survives_stop_start stays in conformance: every conformant driver keeps the workspace across stop and start, and the one that can't yet (MXC) isn't conformant until it can. The policy-advisor suite needs no port: it is a filter over the existing conformance tests that no workflow runs. The OCI image, Keycloak and Podman userns suites are ported separately, in the features and drivers families. The old conformance/cli crate stays until the release lanes move to plans. Co-authored-by: Evan Lezar Signed-off-by: Matthew Grossman --- tests/suites/conformance/Cargo.lock | 3 + tests/suites/conformance/sandbox/Cargo.toml | 3 + tests/suites/conformance/sandbox/README.md | 18 +- tests/suites/conformance/sandbox/tests/cli.rs | 170 +++++++ .../sandbox/tests/file_transfer.rs | 420 ++++++++++++++++++ .../conformance/sandbox/tests/lifecycle.rs | 165 +++++++ .../sandbox/tests/persistent_workspace.rs | 63 +++ 7 files changed, 837 insertions(+), 5 deletions(-) create mode 100644 tests/suites/conformance/sandbox/tests/cli.rs create mode 100644 tests/suites/conformance/sandbox/tests/file_transfer.rs create mode 100644 tests/suites/conformance/sandbox/tests/lifecycle.rs create mode 100644 tests/suites/conformance/sandbox/tests/persistent_workspace.rs diff --git a/tests/suites/conformance/Cargo.lock b/tests/suites/conformance/Cargo.lock index 8e7d2d564d..b6f46bf9d4 100644 --- a/tests/suites/conformance/Cargo.lock +++ b/tests/suites/conformance/Cargo.lock @@ -2685,6 +2685,9 @@ name = "sandbox" version = "0.0.0" dependencies = [ "openshell-testkit", + "serde_json", + "tempfile", + "tokio", ] [[package]] diff --git a/tests/suites/conformance/sandbox/Cargo.toml b/tests/suites/conformance/sandbox/Cargo.toml index 52ec79cbba..d213c6b23a 100644 --- a/tests/suites/conformance/sandbox/Cargo.toml +++ b/tests/suites/conformance/sandbox/Cargo.toml @@ -13,3 +13,6 @@ publish = false [dependencies] openshell-testkit = { path = "../../../testkit/testkit" } +serde_json = "1" +tempfile = "3" +tokio = { version = "1", features = ["time"] } diff --git a/tests/suites/conformance/sandbox/README.md b/tests/suites/conformance/sandbox/README.md index 5e73fb2207..a39eb1f5f8 100644 --- a/tests/suites/conformance/sandbox/README.md +++ b/tests/suites/conformance/sandbox/README.md @@ -8,10 +8,17 @@ the file with `_` replaced by `-` (`file_transfer.rs` is `file-transfer`). | Suite | Covers | |---|---| | `smoke` | Create a sandbox, run `echo` in it, delete it | +| `lifecycle` | SDK: long-running and run-to-completion sandboxes, the default command, restart of a stopped sandbox, deleting a stopped sandbox | +| `cli` | The same lifecycle through the `openshell` CLI: status, create, get, list, exec, stop, start, delete | +| `persistent-workspace` | Files in `/sandbox` survive stop and start | +| `file-transfer` | `sandbox upload` and `download`: round trips, `.gitignore` filtering, workspace-escape and option-looking names | -The suite is small on purpose. It proves that a plan can run a real test through -the runner on CI. The older suites move to testkit by adding suites here (or a -new family under `tests/suites/`) and listing them in a plan. +`smoke` is the smallest end-to-end check that a plan can run a real test through +the runner on CI. The other suites are ports of the old `conformance/cli` +scenarios and of Evan Lezar's suites in NVIDIA/OpenShell#4629 (credited in each +file). `file-transfer` runs `git` on the machine that runs the tests, so a test +environment needs it installed (`tests/tmachine/ansible/playbooks/test-tools.yaml` +does that for the tmachine environments). ## Running it @@ -47,8 +54,9 @@ leave driver behavior alone. published docs. - Assert `phase` and `exit_code`. Don't assert condition reasons. - Check that something did not happen by reading state, not by sleeping. -- Every environment must pass every test here. If a behavior is left to the - driver, the test belongs in a `features` or `drivers` suite instead. +- Don't test behavior that only some drivers have. A test that can't pass on + every environment isn't conformance; put it in `tests/suites/features` or + `tests/suites/drivers`. - Use the CLI for upload, download, connect and forward. Rely on its exit code and `--output json`. - Add a testkit helper instead of calling raw gRPC from a test. diff --git a/tests/suites/conformance/sandbox/tests/cli.rs b/tests/suites/conformance/sandbox/tests/cli.rs new file mode 100644 index 0000000000..d26eed9bb5 --- /dev/null +++ b/tests/suites/conformance/sandbox/tests/cli.rs @@ -0,0 +1,170 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Sandbox lifecycle through the `openshell` CLI. +//! +//! Ported from the `smoke` and `sandbox-lifecycle` scenarios of the old +//! `conformance/cli` suite. The other lifecycle tests act through the SDK; these +//! check that the CLI commands for create, get, list, exec, stop, start, and +//! delete work against the same gateway. + +use serde_json::Value; +use testkit::Context; + +const PAGE_SIZE: &str = "1000"; +/// More pages than any test gateway should have; stops a repeated token from +/// looping forever. +const MAX_PAGES: usize = 1000; + +/// `openshell status` succeeds, and a sandbox created with the CLI is +/// retrievable and Ready, appears in `sandbox list`, runs `sandbox exec`, and is +/// gone from `sandbox list` after `sandbox delete`. +#[testkit::test] +async fn cli_create_exec_list_delete(cx: &mut Context) -> testkit::Result { + cx.cli(["status"]).await?.success()?; + + let name = cx.name("cs"); + cx.track(&name); + cx.cli(["sandbox", "create", "--name", &name, "--detach"]) + .await? + .success()?; + + let sandbox = cx.adopt(&name).await?; + let got: Value = cx + .cli(["sandbox", "get", &name, "--output", "json"]) + .await? + .success()? + .json()?; + testkit::ensure!( + got["name"] == name.as_str(), + "sandbox get returned {:?}; expected {name:?}", + got["name"] + ); + testkit::ensure!( + got["phase"] == "Ready", + "sandbox {name} is in phase {}; expected Ready", + got["phase"] + ); + + let listed = find_listed(cx, &name).await?; + testkit::ensure!(listed.is_some(), "sandbox {name} is not in sandbox list"); + + let marker = format!("openshell-e2e-{}", cx.run_id()); + let exec = cx + .cli([ + "sandbox", "exec", "--name", &name, "--no-tty", "--", "echo", &marker, + ]) + .await? + .success()?; + testkit::ensure!( + exec.stdout == format!("{marker}\n"), + "sandbox exec stdout was {:?}; expected {:?}", + exec.stdout, + format!("{marker}\n") + ); + + cx.cli(["sandbox", "delete", &name]).await?.success()?; + wait_unlisted(cx, &name).await?; + cx.expect_absent(&sandbox).await +} + +/// A sandbox created with the CLI can be stopped, rejects `sandbox exec` while +/// stopped, starts again, and can be stopped and deleted. +#[testkit::test] +async fn cli_stop_start_delete(cx: &mut Context) -> testkit::Result { + let name = cx.name("cl"); + cx.track(&name); + cx.cli([ + "sandbox", + "create", + "--name", + &name, + "--detach", + "--no-tty", + "--", + "sh", + "-c", + "exec sleep infinity", + ]) + .await? + .success()?; + let sandbox = cx.adopt(&name).await?; + sandbox.wait_ready().await?; + + cx.cli(["sandbox", "stop", &name]).await?.success()?; + sandbox + .wait_phase(testkit::sdk::SandboxPhase::Stopped, testkit::STOP_TIMEOUT) + .await?; + cx.cli(["sandbox", "exec", "--name", &name, "--no-tty", "--", "true"]) + .await? + .failure()?; + + cx.cli(["sandbox", "start", &name]).await?.success()?; + sandbox.wait_ready().await?; + cx.cli(["sandbox", "stop", &name]).await?.success()?; + sandbox + .wait_phase(testkit::sdk::SandboxPhase::Stopped, testkit::STOP_TIMEOUT) + .await?; + + cx.cli(["sandbox", "delete", &name]).await?.success()?; + wait_unlisted(cx, &name).await?; + cx.expect_absent(&sandbox).await +} + +/// Wait until `sandbox list` no longer shows the sandbox. +async fn wait_unlisted(cx: &Context, name: &str) -> testkit::Result { + testkit::poll_until( + &format!("sandbox {name} to leave sandbox list"), + testkit::DELETE_TIMEOUT, + std::time::Duration::from_secs(1), + async || { + Ok(match find_listed(cx, name).await? { + None => testkit::Poll::Ready(()), + Some(phase) => testkit::Poll::Pending(format!("still listed in phase {phase}")), + }) + }, + ) + .await +} + +/// The phase of the sandbox in `sandbox list`, if it is listed. Follows every +/// page. +async fn find_listed(cx: &Context, name: &str) -> testkit::Result> { + let mut token = String::new(); + for _ in 0..MAX_PAGES { + let page: Value = cx + .cli([ + "sandbox", + "list", + "--page-size", + PAGE_SIZE, + "--page-token", + &token, + "--output", + "json", + ]) + .await? + .success()? + .json()?; + if let Some(entry) = page["sandboxes"] + .as_array() + .into_iter() + .flatten() + .find(|entry| entry["name"] == name) + { + return Ok(Some( + entry["phase"].as_str().unwrap_or_default().to_string(), + )); + } + token = page["next_page_token"] + .as_str() + .unwrap_or_default() + .to_string(); + if token.is_empty() { + return Ok(None); + } + } + Err(testkit::Error::msg(format!( + "sandbox list has more than {MAX_PAGES} pages" + ))) +} diff --git a/tests/suites/conformance/sandbox/tests/file_transfer.rs b/tests/suites/conformance/sandbox/tests/file_transfer.rs new file mode 100644 index 0000000000..5bd24563c0 --- /dev/null +++ b/tests/suites/conformance/sandbox/tests/file_transfer.rs @@ -0,0 +1,420 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Sandbox file-transfer suite. +//! +//! Adapted from the `file-transfer` suite in NVIDIA/OpenShell#4629 by Evan Lezar. +//! +//! Upload and download exist only in the CLI, which moves files over the +//! sandbox's SSH session, so these tests act through `cx.cli`. They depend on +//! the supervisor, the sandbox image, and the driver's filesystem, which is +//! why they run in every environment. + +use std::fs; +use std::path::{Path, PathBuf}; + +use tempfile::TempDir; +use testkit::{CliOutput, Context, Error, Sandbox}; + +const LARGE_FILE_SIZE: usize = 512 * 1024; + +/// Files and directories round-trip through upload and download unchanged, +/// uploads into existing directories land inside them, and downloads of +/// single files and directories reproduce them locally. +#[testkit::test] +async fn upload_download_round_trip(cx: &mut Context) -> testkit::Result { + let ft = Transfer::new(cx, "fr").await?; + ft.upload_into_existing_directories().await?; + ft.round_trip().await?; + ft.download_file().await?; + ft.download_directory().await?; + ft.sandbox.delete().await +} + +/// Uploads from a Git work tree skip ignored files, a single file uploads +/// alone, and an upload whose selection is entirely ignored is rejected +/// unless `--no-git-ignore` is given. +#[testkit::test] +async fn upload_respects_gitignore(cx: &mut Context) -> testkit::Result { + let ft = Transfer::new(cx, "fg").await?; + ft.gitignore_filtering().await?; + ft.single_file_from_git_repo().await?; + ft.ignored_directory_requires_override().await?; + ft.sandbox.delete().await +} + +/// Downloads refuse sandbox paths that resolve outside the workspace, and +/// file names that look like command-line options download as plain files. +#[testkit::test] +async fn download_stays_inside_workspace(cx: &mut Context) -> testkit::Result { + let ft = Transfer::new(cx, "fs").await?; + ft.reject_workspace_escape().await?; + ft.download_dash_leading_name().await?; + ft.sandbox.delete().await +} + +/// A sandbox, a directory in its workspace, and a local scratch directory. +struct Transfer<'a> { + cx: &'a Context, + sandbox: Sandbox, + remote_root: String, + local: TempDir, +} + +impl<'a> Transfer<'a> { + async fn new(cx: &'a Context, suffix: &str) -> testkit::Result { + let sandbox = cx + .sandbox(suffix) + .command(["sleep", "infinity"]) + .create() + .await?; + sandbox.wait_ready().await?; + let remote_root = format!("/sandbox/file-transfer-{}-{suffix}", cx.run_id()); + let local = tempfile::tempdir() + .map_err(|error| Error::msg(format!("create a local scratch directory: {error}")))?; + let ft = Self { + cx, + sandbox, + remote_root, + local, + }; + ft.sh(&format!("mkdir -p '{}'", ft.remote_root)).await?; + Ok(ft) + } + + fn local(&self, name: &str) -> PathBuf { + self.local.path().join(name) + } + + /// Run a shell script in the sandbox and require it to succeed. + async fn sh(&self, script: &str) -> testkit::Result { + let output = self.sandbox.exec(["sh", "-c", script]).await?; + testkit::ensure!( + output.exit_code == 0, + "`{script}` exited with {}; stderr: {}", + output.exit_code, + output.stderr + ); + Ok(()) + } + + async fn upload( + &self, + source: &Path, + destination: &str, + no_git_ignore: bool, + ) -> testkit::Result { + let mut args = vec![ + "sandbox".to_string(), + "upload".to_string(), + self.sandbox.name().to_string(), + source.display().to_string(), + destination.to_string(), + ]; + if no_git_ignore { + args.push("--no-git-ignore".to_string()); + } + self.cx.cli(args).await + } + + async fn download(&self, source: &str, destination: &Path) -> testkit::Result { + self.cx + .cli([ + "sandbox", + "download", + self.sandbox.name(), + source, + &destination.display().to_string(), + ]) + .await + } + + /// Upload into existing directories, including a symlink to one and one + /// named with a trailing slash; the file must land inside each. Check the + /// destination's type, not only the exit status: tar can succeed after + /// replacing an empty directory with a file. + async fn upload_into_existing_directories(&self) -> testkit::Result { + let source = self.local("config.json"); + write(&source, "upload-destination-payload")?; + let root = format!("{}/upload destinations", self.remote_root); + self.sh(&format!( + "mkdir -p '{root}/empty' '{root}/populated' '{root}/target' '{root}/slash' && \ + printf keep > '{root}/populated/keep.txt' && \ + ln -s target '{root}/link'" + )) + .await?; + + for directory in ["empty", "populated", "link", "slash/"] { + let destination = format!("{root}/{directory}"); + self.upload(&source, &destination, true).await?.success()?; + self.sh(&format!( + "test -d '{destination}' && \ + test \"$(cat '{destination}/config.json')\" = upload-destination-payload && \ + test \"$(cat '{root}/populated/keep.txt')\" = keep && \ + test -L '{root}/link'" + )) + .await?; + } + + // A missing destination is a rename; uploading again must replace + // that file rather than treat it as a directory. + let renamed = format!("{root}/renamed.json"); + for _ in ["rename", "overwrite"] { + self.upload(&source, &renamed, true).await?.success()?; + self.sh(&format!( + "test -f '{renamed}' && test \"$(cat '{renamed}')\" = upload-destination-payload" + )) + .await?; + } + Ok(()) + } + + /// A directory with a nested file and a large binary file, and a single + /// file, survive upload then download. + async fn round_trip(&self) -> testkit::Result { + let source = self.local("roundtrip-upload"); + create_dir_all(&source.join("subdir"))?; + write(&source.join("greeting.txt"), "hello-from-local")?; + write(&source.join("subdir/nested.txt"), "nested-content")?; + let large: Vec = (0u8..=250).cycle().take(LARGE_FILE_SIZE).collect(); + fs::write(source.join("large.bin"), &large) + .map_err(|error| Error::msg(format!("write large.bin: {error}")))?; + + let remote = format!("{}/roundtrip", self.remote_root); + let uploaded = self.upload(&source, &remote, false).await?.success()?; + let warning = message(&uploaded); + testkit::ensure!( + warning.contains("outside a Git work tree") + && warning.contains(".gitignore rules are not applied"), + "upload outside a Git work tree did not warn that filtering is off: {warning}" + ); + + let destination = self.local("roundtrip-download"); + create_dir_all(&destination)?; + self.download(&format!("{remote}/roundtrip-upload"), &destination) + .await? + .success()?; + require_text(&destination.join("greeting.txt"), "hello-from-local")?; + require_text(&destination.join("subdir/nested.txt"), "nested-content")?; + let actual = fs::read(destination.join("large.bin")) + .map_err(|error| Error::msg(format!("read large.bin: {error}")))?; + testkit::ensure!( + actual == large, + "large file changed in the round trip: sent {} bytes, received {}", + large.len(), + actual.len() + ); + + let single = self.local("single.txt"); + write(&single, "single-file-payload")?; + let remote_single = format!("{}/single.txt", self.remote_root); + self.upload(&single, &remote_single, false) + .await? + .success()?; + let single_destination = self.local("single-download"); + create_dir_all(&single_destination)?; + self.download(&remote_single, &single_destination) + .await? + .success()?; + require_text( + &single_destination.join("single.txt"), + "single-file-payload", + ) + } + + async fn download_file(&self) -> testkit::Result { + let remote = format!("{}/download-file.txt", self.remote_root); + self.sh(&format!("printf greeting-payload > '{remote}'")) + .await?; + let destination = self.local("download-file"); + create_dir_all(&destination)?; + self.download(&remote, &destination).await?.success()?; + require_text(&destination.join("download-file.txt"), "greeting-payload") + } + + async fn download_directory(&self) -> testkit::Result { + let remote = format!("{}/tree", self.remote_root); + self.sh(&format!( + "mkdir -p '{remote}/sub' && printf top-level > '{remote}/root.txt' && \ + printf nested > '{remote}/sub/child.txt'" + )) + .await?; + let destination = self.local("download-directory"); + create_dir_all(&destination)?; + self.download(&remote, &destination).await?.success()?; + require_text(&destination.join("root.txt"), "top-level")?; + require_text(&destination.join("sub/child.txt"), "nested") + } + + async fn gitignore_filtering(&self) -> testkit::Result { + let repository = self.local("filter-repo"); + create_dir_all(&repository.join("build"))?; + git(&repository, &["init", "--quiet"])?; + write(&repository.join(".gitignore"), "*.log\nbuild/\n")?; + write(&repository.join("tracked.txt"), "i-am-tracked")?; + write(&repository.join("ignored.log"), "i-should-be-filtered")?; + write(&repository.join("build/output.bin"), "build-artifact")?; + git(&repository, &["add", "."])?; + + let remote = format!("{}/filtered", self.remote_root); + self.upload(&repository, &remote, false).await?.success()?; + let destination = self.local("filter-download"); + create_dir_all(&destination)?; + self.download(&remote, &destination).await?.success()?; + let uploaded = destination.join("filter-repo"); + require_text(&uploaded.join("tracked.txt"), "i-am-tracked")?; + require_exists(&uploaded.join(".gitignore"))?; + require_absent(&uploaded.join("ignored.log"))?; + require_absent(&uploaded.join("build")) + } + + async fn single_file_from_git_repo(&self) -> testkit::Result { + let repository = self.local("single-repo"); + create_dir_all(&repository.join("nested"))?; + git(&repository, &["init", "--quiet"])?; + write(&repository.join(".gitignore"), "*.log\n")?; + write( + &repository.join("nested/config.txt"), + "single-file-from-repo", + )?; + write(&repository.join("tracked.txt"), "should-not-upload")?; + write(&repository.join("ignored.log"), "ignored")?; + + let remote = format!("{}/single-from-repo", self.remote_root); + self.upload(&repository.join("nested/config.txt"), &remote, false) + .await? + .success()?; + let destination = self.local("single-repo-download"); + create_dir_all(&destination)?; + self.download(&remote, &destination).await?.success()?; + require_text(&destination.join("config.txt"), "single-file-from-repo")?; + require_absent(&destination.join("tracked.txt"))?; + require_absent(&destination.join("ignored.log")) + } + + async fn ignored_directory_requires_override(&self) -> testkit::Result { + let remote_seed = format!("{}/runs/test.json", self.remote_root); + self.sh(&format!( + "mkdir -p '{}/runs' && printf downloaded-payload > '{remote_seed}'", + self.remote_root + )) + .await?; + + let repository = self.local("override-repo"); + let runs = repository.join("runs"); + create_dir_all(&runs)?; + git(&repository, &["init", "--quiet"])?; + write(&repository.join(".gitignore"), "runs/\n")?; + self.download(&remote_seed, &runs).await?.success()?; + require_exists(&runs.join("test.json"))?; + + let remote = format!("{}/reuploaded", self.remote_root); + let rejected = self.upload(&runs, &remote, false).await?.failure()?; + let output = message(&rejected); + testkit::ensure!( + output.contains("filtering selected no files") && output.contains("--no-git-ignore"), + "the rejected upload did not explain the --no-git-ignore override: {output}" + ); + self.sh(&format!("test ! -e '{remote}'")).await?; + + self.upload(&runs, &remote, true).await?.success()?; + let destination = self.local("override-download"); + create_dir_all(&destination)?; + self.download(&remote, &destination).await?.success()?; + require_text(&destination.join("runs/test.json"), "downloaded-payload") + } + + async fn reject_workspace_escape(&self) -> testkit::Result { + let etc_link = format!("{}/etc-link", self.remote_root); + let passwd_link = format!("{}/passwd-link", self.remote_root); + self.sh(&format!( + "ln -s /etc '{etc_link}' && ln -s /etc/passwd '{passwd_link}'" + )) + .await?; + let destination = self.local("workspace-escape"); + create_dir_all(&destination)?; + + for source in [etc_link.clone(), passwd_link, format!("{etc_link}/passwd")] { + let rejected = self.download(&source, &destination).await?.failure()?; + let output = message(&rejected); + testkit::ensure!( + output.contains("resolves to") + && output.contains("outside the") + && output.contains("sandbox workspace"), + "download of {source:?} was not rejected as outside the workspace: {output}" + ); + } + require_absent(&destination.join("passwd"))?; + require_absent(&destination.join("etc-link")) + } + + async fn download_dash_leading_name(&self) -> testkit::Result { + let remote = format!("{}/--checkpoint-action=evil", self.remote_root); + self.sh(&format!("printf dash-payload > '{remote}'")) + .await?; + let destination = self.local("dash-leading"); + create_dir_all(&destination)?; + self.download(&remote, &destination).await?.success()?; + require_text( + &destination.join("--checkpoint-action=evil"), + "dash-payload", + ) + } +} + +/// The CLI's stdout and stderr as one line of words. Error reports wrap to +/// the terminal width and draw a gutter, so match against this instead. +fn message(output: &CliOutput) -> String { + format!("{} {}", output.stdout, output.stderr) + .replace(['│', '×', '╰', '─', '╭', '├'], " ") + .split_whitespace() + .collect::>() + .join(" ") +} + +fn git(repository: &Path, args: &[&str]) -> testkit::Result { + let output = std::process::Command::new("git") + .args(args) + .current_dir(repository) + .output() + .map_err(|error| Error::msg(format!("run git {}: {error}", args.join(" "))))?; + testkit::ensure!( + output.status.success(), + "git {} failed in {}: {}", + args.join(" "), + repository.display(), + String::from_utf8_lossy(&output.stderr) + ); + Ok(()) +} + +fn write(path: &Path, contents: &str) -> testkit::Result { + fs::write(path, contents) + .map_err(|error| Error::msg(format!("write {}: {error}", path.display()))) +} + +fn create_dir_all(path: &Path) -> testkit::Result { + fs::create_dir_all(path) + .map_err(|error| Error::msg(format!("create {}: {error}", path.display()))) +} + +fn require_text(path: &Path, expected: &str) -> testkit::Result { + let actual = fs::read_to_string(path) + .map_err(|error| Error::msg(format!("read {}: {error}", path.display())))?; + testkit::ensure!( + actual == expected, + "{} contains {actual:?}; expected {expected:?}", + path.display() + ); + Ok(()) +} + +fn require_exists(path: &Path) -> testkit::Result { + testkit::ensure!(path.exists(), "{} does not exist", path.display()); + Ok(()) +} + +fn require_absent(path: &Path) -> testkit::Result { + testkit::ensure!(!path.exists(), "{} exists but should not", path.display()); + Ok(()) +} diff --git a/tests/suites/conformance/sandbox/tests/lifecycle.rs b/tests/suites/conformance/sandbox/tests/lifecycle.rs new file mode 100644 index 0000000000..02e4f6f117 --- /dev/null +++ b/tests/suites/conformance/sandbox/tests/lifecycle.rs @@ -0,0 +1,165 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Sandbox lifecycle suite. +//! +//! These tests are adapted from the `sandbox-lifecycle` suite in +//! NVIDIA/OpenShell#4629 by Evan Lezar. Tests that need a driver with a +//! persistent workspace live in `persistent_workspace.rs`. + +use testkit::Context; +use testkit::sdk::{SandboxPhase, SandboxRestartPolicy}; + +/// A long-running sandbox becomes ready, runs a command with exec, and is gone +/// from get and list after delete. +#[testkit::test] +async fn long_running_exec_delete(cx: &mut Context) -> testkit::Result { + let sandbox = cx + .sandbox("lr") + .command(["sleep", "infinity"]) + .create() + .await?; + sandbox.wait_ready().await?; + + let marker = format!("openshell-e2e-{}", cx.run_id()); + let output = sandbox.exec(["echo", marker.as_str()]).await?; + testkit::ensure!( + output.exit_code == 0, + "exec exited with {}; stderr: {}", + output.exit_code, + output.stderr + ); + testkit::ensure!( + output.stdout == format!("{marker}\n"), + "exec stdout was {:?}; expected {:?}", + output.stdout, + format!("{marker}\n") + ); + + sandbox.delete().await?; + cx.expect_absent(&sandbox).await +} + +/// A sandbox whose main command exits 0, with restart policy Never, reaches +/// Completed with exit code 0 and is not restarted. It is gone from get and +/// list after delete. +#[testkit::test] +async fn run_to_completion_exit_delete(cx: &mut Context) -> testkit::Result { + let sandbox = cx + .sandbox("rtc") + .command(["sh", "-c", "exit 0"]) + .restart_policy(SandboxRestartPolicy::Never) + .create() + .await?; + + let finished = sandbox.wait_finished().await?; + testkit::ensure!( + finished.phase == SandboxPhase::Completed, + "sandbox finished in phase {:?} with exit code {:?}; expected Completed", + finished.phase, + finished.exit_code + ); + testkit::ensure!( + finished.exit_code == Some(0), + "exit code was {:?}; expected Some(0)", + finished.exit_code + ); + testkit::ensure!( + finished.restart_count == 0, + "restart count was {}; expected 0", + finished.restart_count + ); + + // Completed is final under Never: a later read must not show a restart. + let again = sandbox.get().await?; + testkit::ensure!( + again.phase == SandboxPhase::Completed && again.restart_count == 0, + "sandbox left Completed: phase {:?}, restart count {}", + again.phase, + again.restart_count + ); + + sandbox.delete().await?; + cx.expect_absent(&sandbox).await +} + +/// A sandbox created with the gateway's default command becomes Ready, is +/// listed as Ready, runs a command with exec, and is gone after delete. +#[testkit::test] +async fn default_command_sandbox_is_listed_ready(cx: &mut Context) -> testkit::Result { + let sandbox = cx.sandbox("dc").create().await?; + let ready = sandbox.wait_ready().await?; + testkit::ensure!( + ready.name == sandbox.name(), + "get returned sandbox {:?}; expected {:?}", + ready.name, + sandbox.name() + ); + cx.expect_listed(&sandbox, SandboxPhase::Ready).await?; + + let marker = format!("openshell-e2e-{}", cx.run_id()); + let output = sandbox.exec(["echo", marker.as_str()]).await?; + testkit::ensure!( + output.exit_code == 0 && output.stdout == format!("{marker}\n"), + "exec exited {} with stdout {:?}; stderr: {}", + output.exit_code, + output.stdout, + output.stderr + ); + + sandbox.delete().await?; + cx.expect_absent(&sandbox).await +} + +/// A stopped sandbox rejects exec, accepts a second stop, and returns to +/// Ready when started. +#[testkit::test] +async fn restart_stopped_sandbox(cx: &mut Context) -> testkit::Result { + let sandbox = cx + .sandbox("rs") + .command(["sleep", "infinity"]) + .create() + .await?; + sandbox.wait_ready().await?; + + sandbox.stop().await?; + match sandbox.exec(["true"]).await { + Err(_) => {} + Ok(output) => testkit::ensure!( + output.exit_code != 0, + "exec succeeded while the sandbox was stopped" + ), + } + // Stop is idempotent. + let again = sandbox.stop().await?; + testkit::ensure!( + again.phase == SandboxPhase::Stopped, + "a second stop left the sandbox {:?}", + again.phase + ); + + sandbox.start().await?; + let output = sandbox.exec(["echo", "started"]).await?; + testkit::ensure!( + output.exit_code == 0 && output.stdout == "started\n", + "exec after start exited {} with stdout {:?}; stderr: {}", + output.exit_code, + output.stdout, + output.stderr + ); + Ok(()) +} + +/// A stopped sandbox can be deleted, and is then gone from get and list. +#[testkit::test] +async fn stopped_sandbox_can_be_deleted(cx: &mut Context) -> testkit::Result { + let sandbox = cx + .sandbox("sd") + .command(["sleep", "infinity"]) + .create() + .await?; + sandbox.wait_ready().await?; + sandbox.stop().await?; + sandbox.delete().await?; + cx.expect_absent(&sandbox).await +} diff --git a/tests/suites/conformance/sandbox/tests/persistent_workspace.rs b/tests/suites/conformance/sandbox/tests/persistent_workspace.rs new file mode 100644 index 0000000000..101ea6d6ff --- /dev/null +++ b/tests/suites/conformance/sandbox/tests/persistent_workspace.rs @@ -0,0 +1,63 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Lifecycle tests for a compute driver whose workspace is persistent. +//! +//! Adapted from the `sandbox-lifecycle` suite in NVIDIA/OpenShell#4629 by Evan +//! Lezar. Docker, Podman, Kubernetes, and MicroVM keep `/sandbox` across stop +//! and start, and so must every conformant driver. The Microsoft MXC driver +//! cannot start a stopped sandbox yet, so it is not conformant until it can. + +use testkit::Context; + +/// Files in the workspace survive stop and start, and start runs the main +/// command again. +#[testkit::test] +async fn persistent_workspace_survives_stop_start(cx: &mut Context) -> testkit::Result { + const SENTINEL: &str = "/sandbox/.openshell-stop-start-sentinel"; + const RUN_COUNT: &str = "/sandbox/.openshell-main-run-count"; + let main = format!( + "count=0; test ! -f '{RUN_COUNT}' || count=$(cat '{RUN_COUNT}'); \ + count=$((count + 1)); printf '%s\\n' \"$count\" > '{RUN_COUNT}'; \ + exec sleep infinity" + ); + let sandbox = cx + .sandbox("kw") + .command(["sh", "-c", main.as_str()]) + .create() + .await?; + sandbox.wait_ready().await?; + + let sentinel = format!("openshell-stop-start-{}", cx.run_id()); + let write = sandbox + .exec([ + "sh", + "-c", + &format!("printf '%s\\n' '{sentinel}' > '{SENTINEL}' && sync"), + ]) + .await?; + testkit::ensure!( + write.exit_code == 0, + "writing the sentinel exited {}; stderr: {}", + write.exit_code, + write.stderr + ); + + sandbox.stop().await?; + sandbox.start().await?; + + let read = sandbox.exec(["cat", SENTINEL]).await?; + testkit::ensure!( + read.stdout == format!("{sentinel}\n"), + "sentinel after restart is {:?}; stderr: {}", + read.stdout, + read.stderr + ); + let runs = sandbox.exec(["cat", RUN_COUNT]).await?; + testkit::ensure!( + runs.stdout == "2\n", + "main command ran {:?} times; expected 2", + runs.stdout.trim() + ); + Ok(()) +} From 32c85f6b431a3acd4eb16d6a6a81d640c4875096 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Wed, 21 Oct 2026 22:15:42 -0700 Subject: [PATCH 3/6] test(tmachine): install Git in every test environment The file-transfer suite builds a Git repository on the machine that runs the tests, and the old conformance playbook installed Git itself. Signed-off-by: Matthew Grossman --- .../ansible/playbooks/test-tools.yaml | 19 +++++++++++++++++++ tests/tmachine/config.nix | 4 ++++ 2 files changed, 23 insertions(+) create mode 100644 tests/tmachine/ansible/playbooks/test-tools.yaml diff --git a/tests/tmachine/ansible/playbooks/test-tools.yaml b/tests/tmachine/ansible/playbooks/test-tools.yaml new file mode 100644 index 0000000000..394810f8f2 --- /dev/null +++ b/tests/tmachine/ansible/playbooks/test-tools.yaml @@ -0,0 +1,19 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +--- +# Tools the testkit suites run on the machine itself, next to cargo-nextest. +# The file-transfer suite builds a Git repository there to check that upload +# honors .gitignore. +- name: Install test tools + hosts: all + become: true + gather_facts: false + tasks: + - name: Wait for SSH + ansible.builtin.wait_for_connection: + + - name: Install Git + ansible.builtin.package: + name: git + state: present diff --git a/tests/tmachine/config.nix b/tests/tmachine/config.nix index ee9f70943e..98d0c41519 100644 --- a/tests/tmachine/config.nix +++ b/tests/tmachine/config.nix @@ -48,6 +48,7 @@ let use_galaxy = true; playbooks = [ "ansible/playbooks/nextest.yaml" + "ansible/playbooks/test-tools.yaml" "ansible/playbooks/docker.yaml" ]; }; @@ -59,6 +60,7 @@ let use_galaxy = false; playbooks = [ "ansible/playbooks/nextest.yaml" + "ansible/playbooks/test-tools.yaml" "ansible/playbooks/k3s.yaml" ]; }; @@ -70,6 +72,7 @@ let use_galaxy = false; playbooks = [ "ansible/playbooks/nextest.yaml" + "ansible/playbooks/test-tools.yaml" "ansible/playbooks/selinux.yaml" "ansible/playbooks/podman-rootful.yaml" ]; @@ -82,6 +85,7 @@ let use_galaxy = false; playbooks = [ "ansible/playbooks/nextest.yaml" + "ansible/playbooks/test-tools.yaml" "ansible/playbooks/selinux.yaml" "ansible/playbooks/podman-rootless.yaml" ]; From 4eddc7ba01ab075e9f14dcc6aac1b2f613ab7e2f Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Thu, 22 Oct 2026 17:00:39 -0700 Subject: [PATCH 4/6] test(tmachine): check Agent Sandbox reconciliation in the k3s installer The k3s installer now waits until the Agent Sandbox controller reconciles a gated probe Sandbox, so a test that starts right after boot doesn't race it. The check moves out of the old conformance playbook into tests/tmachine/ansible/playbooks/k3s-controller-ready.yaml, which the installer and the old playbook both include. Tests themselves don't wait for the controller. Signed-off-by: Matthew Grossman --- tests/tmachine/ansible/playbooks/conformance/cli.yaml | 2 +- tests/tmachine/ansible/playbooks/openshell-k3s.yaml | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/tmachine/ansible/playbooks/conformance/cli.yaml b/tests/tmachine/ansible/playbooks/conformance/cli.yaml index e841a11442..071672167d 100644 --- a/tests/tmachine/ansible/playbooks/conformance/cli.yaml +++ b/tests/tmachine/ansible/playbooks/conformance/cli.yaml @@ -83,7 +83,7 @@ when: gateway_status is succeeded and conformance_k3s.stat.exists block: - name: Verify Agent Sandbox reconciliation - ansible.builtin.include_tasks: k3s-ready.yaml + ansible.builtin.include_tasks: ../k3s-controller-ready.yaml rescue: - name: Record controller readiness failure ansible.builtin.set_fact: diff --git a/tests/tmachine/ansible/playbooks/openshell-k3s.yaml b/tests/tmachine/ansible/playbooks/openshell-k3s.yaml index e39a9f3523..aca8ec958f 100644 --- a/tests/tmachine/ansible/playbooks/openshell-k3s.yaml +++ b/tests/tmachine/ansible/playbooks/openshell-k3s.yaml @@ -167,6 +167,11 @@ - k3s_gateway_address.stdout is match('^[0-9.]+:[0-9]+$') fail_msg: "Unexpected gateway Service address: {{ k3s_gateway_address.stdout }}" + # Deployment readiness doesn't establish that the controller reconciles + # Sandboxes, so the first test would otherwise race it. Tests don't wait. + - name: Verify Agent Sandbox reconciliation + ansible.builtin.include_tasks: k3s-controller-ready.yaml + - name: Register OpenShell gateway for test client hosts: all gather_facts: false From a714c815679d3e09010e38db078effcf5630822c Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Thu, 22 Oct 2026 17:00:39 -0700 Subject: [PATCH 5/6] feat(runner): require conformance to select the same suites everywhere A plan can't run part of the conformance family in one environment and a different part in another, or conformance would mean different things in different places. The runner refuses such a plan and names the environments that differ. A test that can't pass everywhere belongs in another family. Signed-off-by: Matthew Grossman --- TESTING.md | 4 ++- tests/runner/src/jobs.rs | 71 +++++++++++++++++++++++++++++++++++++++- 2 files changed, 73 insertions(+), 2 deletions(-) diff --git a/TESTING.md b/TESTING.md index 1b6799a8d6..746d444dd5 100644 --- a/TESTING.md +++ b/TESTING.md @@ -174,7 +174,9 @@ what runs where: CI reads it, so no workflow or playbook repeats it. - **Plans** (`tests/plans/*.yaml`) pair a family (a workspace under `tests/suites/`, such as `conformance`) and optionally some of its suites with - environment IDs. See [tests/plans/README.md](tests/plans/README.md). + environment IDs. See [tests/plans/README.md](tests/plans/README.md). A `conformance` entry must + select the same suites in every environment; the runner refuses a plan that + doesn't. - **Environments** (`tests/environments/*.yaml`) name the tmachine environment and installer that provision them and the gateway name the installer registers. The runner points the tests at that gateway and the installed CLI with diff --git a/tests/runner/src/jobs.rs b/tests/runner/src/jobs.rs index df95ce2034..a9f82d1f2c 100644 --- a/tests/runner/src/jobs.rs +++ b/tests/runner/src/jobs.rs @@ -91,6 +91,8 @@ pub fn expand( } } + check_conformance_is_uniform(&selected)?; + for id in &narrowing.environments { if !selected.contains_key(id) { bail!("--env {id:?} is not in the plan"); @@ -139,6 +141,44 @@ pub fn expand( Ok(jobs) } +/// The family every driver and install must pass. A plan can't run part of it +/// somewhere, or conformance would mean different things in different places. +const CONFORMANCE: &str = "conformance"; + +/// Every environment the plan runs conformance in must select the same suites. +fn check_conformance_is_uniform( + selected: &BTreeMap>>, +) -> Result<()> { + let mut by_suites: BTreeMap, Vec<&str>> = BTreeMap::new(); + for (id, families) in selected { + if let Some(suites) = families.get(CONFORMANCE) { + by_suites + .entry(suites.iter().map(String::as_str).collect()) + .or_default() + .push(id); + } + } + if by_suites.len() > 1 { + let groups: Vec = by_suites + .iter() + .map(|(suites, ids)| { + format!( + "{} select {}", + ids.join(", "), + suites.iter().copied().collect::>().join(", ") + ) + }) + .collect(); + bail!( + "the plan selects different {CONFORMANCE} suites in different environments, so \ + {CONFORMANCE} would not mean the same thing everywhere; use the same suites \ + everywhere, or move the suite that can't run everywhere to another family:\n {}", + groups.join("\n ") + ); + } + Ok(()) +} + /// What strict mode (CI) refuses to run: nothing selected at all, an /// environment this host can't provision, and a family with no tests. Locally these are reported and skipped, because a /// laptop can't boot every environment; in CI each one means a lane that would @@ -339,7 +379,7 @@ mod tests { fn groups_entries_by_environment() { let jobs = expand_plan( &format!( - "matrix:\n - type: conformance\n suites: [providers]\n environments: [{AMD64}, {ARM64}]\n - type: conformance\n suites: [providers, sandbox-lifecycle]\n environments: [{AMD64}]\n" + "matrix:\n - type: conformance\n suites: [providers]\n environments: [{AMD64}, {ARM64}]\n - type: conformance\n suites: [sandbox-lifecycle]\n environments: [{AMD64}, {ARM64}]\n" ), &Narrowing::default(), ) @@ -421,6 +461,35 @@ mod tests { ); } + #[test] + fn conformance_must_select_the_same_suites_in_every_environment() { + let err = expand_plan( + &format!( + "matrix:\n - type: conformance\n environments: [{AMD64}]\n - type: conformance\n suites: [providers]\n environments: [{ARM64}]\n" + ), + &Narrowing::default(), + ) + .unwrap_err(); + let message = err.to_string(); + assert!( + message.contains("different conformance suites"), + "{message}" + ); + assert!( + message.contains(AMD64) && message.contains(ARM64), + "{message}" + ); + + // The same suites listed differently are the same selection. + expand_plan( + &format!( + "matrix:\n - type: conformance\n suites: [providers, sandbox-lifecycle]\n environments: [{AMD64}]\n - type: conformance\n environments: [{ARM64}]\n" + ), + &Narrowing::default(), + ) + .unwrap(); + } + #[test] fn expected_tests_are_the_ones_in_the_suite() { let jobs = expand_plan( From 919f767b0915acd3040ea5898b25dacf1fb605ac Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Thu, 22 Oct 2026 17:00:39 -0700 Subject: [PATCH 6/6] test(plans): run the ported conformance suites on Docker, Podman and k3s Add the Podman (rootful and rootless, RPM) and k3s and HA k3s environments and put them, with Docker, in the cpu-tests plan, which now runs the whole conformance family (smoke, lifecycle, cli, persistent-workspace and file-transfer) on all five. The tmachine disk cache in e2e-suites.yml is keyed per environment; the k3s environments skip it as integration-runner.yml does. The old suites and lanes still run unchanged. arm64 is not defined. Signed-off-by: Matthew Grossman --- .github/workflows/e2e-suites.yml | 9 ++++++--- tests/environments/k3s.yaml | 21 +++++++++++++++++++++ tests/environments/podman.yaml | 20 ++++++++++++++++++++ tests/plans/README.md | 6 +++++- tests/plans/cpu-tests.yaml | 4 ++++ 5 files changed, 56 insertions(+), 4 deletions(-) create mode 100644 tests/environments/k3s.yaml create mode 100644 tests/environments/podman.yaml diff --git a/.github/workflows/e2e-suites.yml b/.github/workflows/e2e-suites.yml index 670918ca6c..0826b5cc73 100644 --- a/.github/workflows/e2e-suites.yml +++ b/.github/workflows/e2e-suites.yml @@ -94,17 +94,20 @@ jobs: - uses: ./.github/actions/setup-nix + # integration-runner.yml skips the disk cache for ubuntu-k3s and doesn't say + # why, so the k3s environments skip it here too until that is understood. - name: Cache tmachine disks + if: ${{ !contains(matrix.environment, '.k3s.') }} uses: actions/cache@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0 with: path: ~/.cache/tmachine # tmachine keys its disk layers on a hash of the Ansible tree. # actions/cache doesn't save after an exact key hit, so a fixed key # would rebuild the changed layers on every run. Key on the playbooks - # and fall back to the newest entry for the same runner. - key: tmachine-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('tests/tmachine/ansible/**', 'tests/tmachine/images.nix') }} + # and fall back to the newest entry for the same environment and runner. + key: tmachine-${{ matrix.environment }}-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('tests/tmachine/ansible/**', 'tests/tmachine/images.nix') }} restore-keys: | - tmachine-${{ runner.os }}-${{ runner.arch }}- + tmachine-${{ matrix.environment }}-${{ runner.os }}-${{ runner.arch }}- - name: Download integration inputs uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 diff --git a/tests/environments/k3s.yaml b/tests/environments/k3s.yaml new file mode 100644 index 0000000000..87ea66462c --- /dev/null +++ b/tests/environments/k3s.yaml @@ -0,0 +1,21 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Kubernetes (k3s) environments, by Evan Lezar (NVIDIA/OpenShell#4629). See +# docker.yaml for the fields and for adding arm64. The k3s installers register the gateway as +# `tmachine`. +environments: + - id: tmachine.ubuntu2404-amd64.k3s.k3s + backend: tmachine + machine: { os: ubuntu, version: "24.04", arch: amd64 } + runtime: k3s + installer: k3s + tmachine: { environment: ubuntu-k3s, installer: k3s } + gateway: tmachine + - id: tmachine.ubuntu2404-amd64.k3s.k3s-ha + backend: tmachine + machine: { os: ubuntu, version: "24.04", arch: amd64 } + runtime: k3s + installer: k3s-ha + tmachine: { environment: ubuntu-k3s, installer: k3s-ha } + gateway: tmachine diff --git a/tests/environments/podman.yaml b/tests/environments/podman.yaml new file mode 100644 index 0000000000..458b9e3701 --- /dev/null +++ b/tests/environments/podman.yaml @@ -0,0 +1,20 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Podman-backed environments, by Evan Lezar (NVIDIA/OpenShell#4629). See +# docker.yaml for the fields and for adding arm64. +environments: + - id: tmachine.fedora44-amd64.podman-rootful.rpm + backend: tmachine + machine: { os: fedora, version: "44", arch: amd64 } + runtime: podman-rootful + installer: rpm + tmachine: { environment: fedora-podman-rootful, installer: rpm } + gateway: openshell + - id: tmachine.fedora44-amd64.podman-rootless.rpm + backend: tmachine + machine: { os: fedora, version: "44", arch: amd64 } + runtime: podman-rootless + installer: rpm + tmachine: { environment: fedora-podman-rootless, installer: rpm } + gateway: openshell diff --git a/tests/plans/README.md b/tests/plans/README.md index ba1fd6436e..f0326bd4eb 100644 --- a/tests/plans/README.md +++ b/tests/plans/README.md @@ -12,6 +12,10 @@ matrix: - tmachine.ubuntu2404-amd64.docker-rootful.deb # IDs from tests/environments/ ``` +A `conformance` entry must select the same suites in every environment: the +runner refuses a plan that doesn't, so conformance means the same thing +everywhere. A test that can't pass everywhere belongs in another family. + A family is a Cargo workspace directly under `tests/suites/` with at least one testkit suite. A suite is a test file in a crate's `tests/` directory containing `#[testkit::test]` functions. Its name is the file name without `.rs`, with `_` @@ -19,7 +23,7 @@ replaced by `-`: `tests/file_transfer.rs` is the suite `file-transfer`. | Plan | Runs | |---|---| -| `cpu-tests.yaml` | The `conformance` family on Ubuntu 24.04 amd64 with rootful Docker and the Debian package | +| `cpu-tests.yaml` | The `conformance` family on rootful Docker (Debian package), rootful and rootless Podman (RPM), and k3s and HA k3s, all amd64 | ## How a plan runs on CI diff --git a/tests/plans/cpu-tests.yaml b/tests/plans/cpu-tests.yaml index 0ac90f2ff3..ee2a67f6aa 100644 --- a/tests/plans/cpu-tests.yaml +++ b/tests/plans/cpu-tests.yaml @@ -16,3 +16,7 @@ matrix: - type: conformance environments: - tmachine.ubuntu2404-amd64.docker-rootful.deb + - tmachine.fedora44-amd64.podman-rootful.rpm + - tmachine.fedora44-amd64.podman-rootless.rpm + - tmachine.ubuntu2404-amd64.k3s.k3s + - tmachine.ubuntu2404-amd64.k3s.k3s-ha