diff --git a/.github/workflows/e2e-suites.yml b/.github/workflows/e2e-suites.yml index 670918ca6c..0826b5cc73 100644 --- a/.github/workflows/e2e-suites.yml +++ b/.github/workflows/e2e-suites.yml @@ -94,17 +94,20 @@ jobs: - uses: ./.github/actions/setup-nix + # integration-runner.yml skips the disk cache for ubuntu-k3s and doesn't say + # why, so the k3s environments skip it here too until that is understood. - name: Cache tmachine disks + if: ${{ !contains(matrix.environment, '.k3s.') }} uses: actions/cache@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0 with: path: ~/.cache/tmachine # tmachine keys its disk layers on a hash of the Ansible tree. # actions/cache doesn't save after an exact key hit, so a fixed key # would rebuild the changed layers on every run. Key on the playbooks - # and fall back to the newest entry for the same runner. - key: tmachine-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('tests/tmachine/ansible/**', 'tests/tmachine/images.nix') }} + # and fall back to the newest entry for the same environment and runner. + key: tmachine-${{ matrix.environment }}-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('tests/tmachine/ansible/**', 'tests/tmachine/images.nix') }} restore-keys: | - tmachine-${{ runner.os }}-${{ runner.arch }}- + tmachine-${{ matrix.environment }}-${{ runner.os }}-${{ runner.arch }}- - name: Download integration inputs uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 diff --git a/TESTING.md b/TESTING.md index 1b6799a8d6..746d444dd5 100644 --- a/TESTING.md +++ b/TESTING.md @@ -174,7 +174,9 @@ what runs where: CI reads it, so no workflow or playbook repeats it. - **Plans** (`tests/plans/*.yaml`) pair a family (a workspace under `tests/suites/`, such as `conformance`) and optionally some of its suites with - environment IDs. See [tests/plans/README.md](tests/plans/README.md). + environment IDs. See [tests/plans/README.md](tests/plans/README.md). A `conformance` entry must + select the same suites in every environment; the runner refuses a plan that + doesn't. - **Environments** (`tests/environments/*.yaml`) name the tmachine environment and installer that provision them and the gateway name the installer registers. The runner points the tests at that gateway and the installed CLI with diff --git a/tests/environments/k3s.yaml b/tests/environments/k3s.yaml new file mode 100644 index 0000000000..87ea66462c --- /dev/null +++ b/tests/environments/k3s.yaml @@ -0,0 +1,21 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Kubernetes (k3s) environments, by Evan Lezar (NVIDIA/OpenShell#4629). See +# docker.yaml for the fields and for adding arm64. The k3s installers register the gateway as +# `tmachine`. +environments: + - id: tmachine.ubuntu2404-amd64.k3s.k3s + backend: tmachine + machine: { os: ubuntu, version: "24.04", arch: amd64 } + runtime: k3s + installer: k3s + tmachine: { environment: ubuntu-k3s, installer: k3s } + gateway: tmachine + - id: tmachine.ubuntu2404-amd64.k3s.k3s-ha + backend: tmachine + machine: { os: ubuntu, version: "24.04", arch: amd64 } + runtime: k3s + installer: k3s-ha + tmachine: { environment: ubuntu-k3s, installer: k3s-ha } + gateway: tmachine diff --git a/tests/environments/podman.yaml b/tests/environments/podman.yaml new file mode 100644 index 0000000000..458b9e3701 --- /dev/null +++ b/tests/environments/podman.yaml @@ -0,0 +1,20 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Podman-backed environments, by Evan Lezar (NVIDIA/OpenShell#4629). See +# docker.yaml for the fields and for adding arm64. +environments: + - id: tmachine.fedora44-amd64.podman-rootful.rpm + backend: tmachine + machine: { os: fedora, version: "44", arch: amd64 } + runtime: podman-rootful + installer: rpm + tmachine: { environment: fedora-podman-rootful, installer: rpm } + gateway: openshell + - id: tmachine.fedora44-amd64.podman-rootless.rpm + backend: tmachine + machine: { os: fedora, version: "44", arch: amd64 } + runtime: podman-rootless + installer: rpm + tmachine: { environment: fedora-podman-rootless, installer: rpm } + gateway: openshell diff --git a/tests/plans/README.md b/tests/plans/README.md index ba1fd6436e..f0326bd4eb 100644 --- a/tests/plans/README.md +++ b/tests/plans/README.md @@ -12,6 +12,10 @@ matrix: - tmachine.ubuntu2404-amd64.docker-rootful.deb # IDs from tests/environments/ ``` +A `conformance` entry must select the same suites in every environment: the +runner refuses a plan that doesn't, so conformance means the same thing +everywhere. A test that can't pass everywhere belongs in another family. + A family is a Cargo workspace directly under `tests/suites/` with at least one testkit suite. A suite is a test file in a crate's `tests/` directory containing `#[testkit::test]` functions. Its name is the file name without `.rs`, with `_` @@ -19,7 +23,7 @@ replaced by `-`: `tests/file_transfer.rs` is the suite `file-transfer`. | Plan | Runs | |---|---| -| `cpu-tests.yaml` | The `conformance` family on Ubuntu 24.04 amd64 with rootful Docker and the Debian package | +| `cpu-tests.yaml` | The `conformance` family on rootful Docker (Debian package), rootful and rootless Podman (RPM), and k3s and HA k3s, all amd64 | ## How a plan runs on CI diff --git a/tests/plans/cpu-tests.yaml b/tests/plans/cpu-tests.yaml index 0ac90f2ff3..ee2a67f6aa 100644 --- a/tests/plans/cpu-tests.yaml +++ b/tests/plans/cpu-tests.yaml @@ -16,3 +16,7 @@ matrix: - type: conformance environments: - tmachine.ubuntu2404-amd64.docker-rootful.deb + - tmachine.fedora44-amd64.podman-rootful.rpm + - tmachine.fedora44-amd64.podman-rootless.rpm + - tmachine.ubuntu2404-amd64.k3s.k3s + - tmachine.ubuntu2404-amd64.k3s.k3s-ha diff --git a/tests/runner/src/jobs.rs b/tests/runner/src/jobs.rs index df95ce2034..a9f82d1f2c 100644 --- a/tests/runner/src/jobs.rs +++ b/tests/runner/src/jobs.rs @@ -91,6 +91,8 @@ pub fn expand( } } + check_conformance_is_uniform(&selected)?; + for id in &narrowing.environments { if !selected.contains_key(id) { bail!("--env {id:?} is not in the plan"); @@ -139,6 +141,44 @@ pub fn expand( Ok(jobs) } +/// The family every driver and install must pass. A plan can't run part of it +/// somewhere, or conformance would mean different things in different places. +const CONFORMANCE: &str = "conformance"; + +/// Every environment the plan runs conformance in must select the same suites. +fn check_conformance_is_uniform( + selected: &BTreeMap>>, +) -> Result<()> { + let mut by_suites: BTreeMap, Vec<&str>> = BTreeMap::new(); + for (id, families) in selected { + if let Some(suites) = families.get(CONFORMANCE) { + by_suites + .entry(suites.iter().map(String::as_str).collect()) + .or_default() + .push(id); + } + } + if by_suites.len() > 1 { + let groups: Vec = by_suites + .iter() + .map(|(suites, ids)| { + format!( + "{} select {}", + ids.join(", "), + suites.iter().copied().collect::>().join(", ") + ) + }) + .collect(); + bail!( + "the plan selects different {CONFORMANCE} suites in different environments, so \ + {CONFORMANCE} would not mean the same thing everywhere; use the same suites \ + everywhere, or move the suite that can't run everywhere to another family:\n {}", + groups.join("\n ") + ); + } + Ok(()) +} + /// What strict mode (CI) refuses to run: nothing selected at all, an /// environment this host can't provision, and a family with no tests. Locally these are reported and skipped, because a /// laptop can't boot every environment; in CI each one means a lane that would @@ -339,7 +379,7 @@ mod tests { fn groups_entries_by_environment() { let jobs = expand_plan( &format!( - "matrix:\n - type: conformance\n suites: [providers]\n environments: [{AMD64}, {ARM64}]\n - type: conformance\n suites: [providers, sandbox-lifecycle]\n environments: [{AMD64}]\n" + "matrix:\n - type: conformance\n suites: [providers]\n environments: [{AMD64}, {ARM64}]\n - type: conformance\n suites: [sandbox-lifecycle]\n environments: [{AMD64}, {ARM64}]\n" ), &Narrowing::default(), ) @@ -421,6 +461,35 @@ mod tests { ); } + #[test] + fn conformance_must_select_the_same_suites_in_every_environment() { + let err = expand_plan( + &format!( + "matrix:\n - type: conformance\n environments: [{AMD64}]\n - type: conformance\n suites: [providers]\n environments: [{ARM64}]\n" + ), + &Narrowing::default(), + ) + .unwrap_err(); + let message = err.to_string(); + assert!( + message.contains("different conformance suites"), + "{message}" + ); + assert!( + message.contains(AMD64) && message.contains(ARM64), + "{message}" + ); + + // The same suites listed differently are the same selection. + expand_plan( + &format!( + "matrix:\n - type: conformance\n suites: [providers, sandbox-lifecycle]\n environments: [{AMD64}]\n - type: conformance\n environments: [{ARM64}]\n" + ), + &Narrowing::default(), + ) + .unwrap(); + } + #[test] fn expected_tests_are_the_ones_in_the_suite() { let jobs = expand_plan( diff --git a/tests/suites/conformance/Cargo.lock b/tests/suites/conformance/Cargo.lock index 8e7d2d564d..b6f46bf9d4 100644 --- a/tests/suites/conformance/Cargo.lock +++ b/tests/suites/conformance/Cargo.lock @@ -2685,6 +2685,9 @@ name = "sandbox" version = "0.0.0" dependencies = [ "openshell-testkit", + "serde_json", + "tempfile", + "tokio", ] [[package]] diff --git a/tests/suites/conformance/sandbox/Cargo.toml b/tests/suites/conformance/sandbox/Cargo.toml index 52ec79cbba..d213c6b23a 100644 --- a/tests/suites/conformance/sandbox/Cargo.toml +++ b/tests/suites/conformance/sandbox/Cargo.toml @@ -13,3 +13,6 @@ publish = false [dependencies] openshell-testkit = { path = "../../../testkit/testkit" } +serde_json = "1" +tempfile = "3" +tokio = { version = "1", features = ["time"] } diff --git a/tests/suites/conformance/sandbox/README.md b/tests/suites/conformance/sandbox/README.md index 5e73fb2207..a39eb1f5f8 100644 --- a/tests/suites/conformance/sandbox/README.md +++ b/tests/suites/conformance/sandbox/README.md @@ -8,10 +8,17 @@ the file with `_` replaced by `-` (`file_transfer.rs` is `file-transfer`). | Suite | Covers | |---|---| | `smoke` | Create a sandbox, run `echo` in it, delete it | +| `lifecycle` | SDK: long-running and run-to-completion sandboxes, the default command, restart of a stopped sandbox, deleting a stopped sandbox | +| `cli` | The same lifecycle through the `openshell` CLI: status, create, get, list, exec, stop, start, delete | +| `persistent-workspace` | Files in `/sandbox` survive stop and start | +| `file-transfer` | `sandbox upload` and `download`: round trips, `.gitignore` filtering, workspace-escape and option-looking names | -The suite is small on purpose. It proves that a plan can run a real test through -the runner on CI. The older suites move to testkit by adding suites here (or a -new family under `tests/suites/`) and listing them in a plan. +`smoke` is the smallest end-to-end check that a plan can run a real test through +the runner on CI. The other suites are ports of the old `conformance/cli` +scenarios and of Evan Lezar's suites in NVIDIA/OpenShell#4629 (credited in each +file). `file-transfer` runs `git` on the machine that runs the tests, so a test +environment needs it installed (`tests/tmachine/ansible/playbooks/test-tools.yaml` +does that for the tmachine environments). ## Running it @@ -47,8 +54,9 @@ leave driver behavior alone. published docs. - Assert `phase` and `exit_code`. Don't assert condition reasons. - Check that something did not happen by reading state, not by sleeping. -- Every environment must pass every test here. If a behavior is left to the - driver, the test belongs in a `features` or `drivers` suite instead. +- Don't test behavior that only some drivers have. A test that can't pass on + every environment isn't conformance; put it in `tests/suites/features` or + `tests/suites/drivers`. - Use the CLI for upload, download, connect and forward. Rely on its exit code and `--output json`. - Add a testkit helper instead of calling raw gRPC from a test. diff --git a/tests/suites/conformance/sandbox/tests/cli.rs b/tests/suites/conformance/sandbox/tests/cli.rs new file mode 100644 index 0000000000..d26eed9bb5 --- /dev/null +++ b/tests/suites/conformance/sandbox/tests/cli.rs @@ -0,0 +1,170 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Sandbox lifecycle through the `openshell` CLI. +//! +//! Ported from the `smoke` and `sandbox-lifecycle` scenarios of the old +//! `conformance/cli` suite. The other lifecycle tests act through the SDK; these +//! check that the CLI commands for create, get, list, exec, stop, start, and +//! delete work against the same gateway. + +use serde_json::Value; +use testkit::Context; + +const PAGE_SIZE: &str = "1000"; +/// More pages than any test gateway should have; stops a repeated token from +/// looping forever. +const MAX_PAGES: usize = 1000; + +/// `openshell status` succeeds, and a sandbox created with the CLI is +/// retrievable and Ready, appears in `sandbox list`, runs `sandbox exec`, and is +/// gone from `sandbox list` after `sandbox delete`. +#[testkit::test] +async fn cli_create_exec_list_delete(cx: &mut Context) -> testkit::Result { + cx.cli(["status"]).await?.success()?; + + let name = cx.name("cs"); + cx.track(&name); + cx.cli(["sandbox", "create", "--name", &name, "--detach"]) + .await? + .success()?; + + let sandbox = cx.adopt(&name).await?; + let got: Value = cx + .cli(["sandbox", "get", &name, "--output", "json"]) + .await? + .success()? + .json()?; + testkit::ensure!( + got["name"] == name.as_str(), + "sandbox get returned {:?}; expected {name:?}", + got["name"] + ); + testkit::ensure!( + got["phase"] == "Ready", + "sandbox {name} is in phase {}; expected Ready", + got["phase"] + ); + + let listed = find_listed(cx, &name).await?; + testkit::ensure!(listed.is_some(), "sandbox {name} is not in sandbox list"); + + let marker = format!("openshell-e2e-{}", cx.run_id()); + let exec = cx + .cli([ + "sandbox", "exec", "--name", &name, "--no-tty", "--", "echo", &marker, + ]) + .await? + .success()?; + testkit::ensure!( + exec.stdout == format!("{marker}\n"), + "sandbox exec stdout was {:?}; expected {:?}", + exec.stdout, + format!("{marker}\n") + ); + + cx.cli(["sandbox", "delete", &name]).await?.success()?; + wait_unlisted(cx, &name).await?; + cx.expect_absent(&sandbox).await +} + +/// A sandbox created with the CLI can be stopped, rejects `sandbox exec` while +/// stopped, starts again, and can be stopped and deleted. +#[testkit::test] +async fn cli_stop_start_delete(cx: &mut Context) -> testkit::Result { + let name = cx.name("cl"); + cx.track(&name); + cx.cli([ + "sandbox", + "create", + "--name", + &name, + "--detach", + "--no-tty", + "--", + "sh", + "-c", + "exec sleep infinity", + ]) + .await? + .success()?; + let sandbox = cx.adopt(&name).await?; + sandbox.wait_ready().await?; + + cx.cli(["sandbox", "stop", &name]).await?.success()?; + sandbox + .wait_phase(testkit::sdk::SandboxPhase::Stopped, testkit::STOP_TIMEOUT) + .await?; + cx.cli(["sandbox", "exec", "--name", &name, "--no-tty", "--", "true"]) + .await? + .failure()?; + + cx.cli(["sandbox", "start", &name]).await?.success()?; + sandbox.wait_ready().await?; + cx.cli(["sandbox", "stop", &name]).await?.success()?; + sandbox + .wait_phase(testkit::sdk::SandboxPhase::Stopped, testkit::STOP_TIMEOUT) + .await?; + + cx.cli(["sandbox", "delete", &name]).await?.success()?; + wait_unlisted(cx, &name).await?; + cx.expect_absent(&sandbox).await +} + +/// Wait until `sandbox list` no longer shows the sandbox. +async fn wait_unlisted(cx: &Context, name: &str) -> testkit::Result { + testkit::poll_until( + &format!("sandbox {name} to leave sandbox list"), + testkit::DELETE_TIMEOUT, + std::time::Duration::from_secs(1), + async || { + Ok(match find_listed(cx, name).await? { + None => testkit::Poll::Ready(()), + Some(phase) => testkit::Poll::Pending(format!("still listed in phase {phase}")), + }) + }, + ) + .await +} + +/// The phase of the sandbox in `sandbox list`, if it is listed. Follows every +/// page. +async fn find_listed(cx: &Context, name: &str) -> testkit::Result> { + let mut token = String::new(); + for _ in 0..MAX_PAGES { + let page: Value = cx + .cli([ + "sandbox", + "list", + "--page-size", + PAGE_SIZE, + "--page-token", + &token, + "--output", + "json", + ]) + .await? + .success()? + .json()?; + if let Some(entry) = page["sandboxes"] + .as_array() + .into_iter() + .flatten() + .find(|entry| entry["name"] == name) + { + return Ok(Some( + entry["phase"].as_str().unwrap_or_default().to_string(), + )); + } + token = page["next_page_token"] + .as_str() + .unwrap_or_default() + .to_string(); + if token.is_empty() { + return Ok(None); + } + } + Err(testkit::Error::msg(format!( + "sandbox list has more than {MAX_PAGES} pages" + ))) +} diff --git a/tests/suites/conformance/sandbox/tests/file_transfer.rs b/tests/suites/conformance/sandbox/tests/file_transfer.rs new file mode 100644 index 0000000000..5bd24563c0 --- /dev/null +++ b/tests/suites/conformance/sandbox/tests/file_transfer.rs @@ -0,0 +1,420 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Sandbox file-transfer suite. +//! +//! Adapted from the `file-transfer` suite in NVIDIA/OpenShell#4629 by Evan Lezar. +//! +//! Upload and download exist only in the CLI, which moves files over the +//! sandbox's SSH session, so these tests act through `cx.cli`. They depend on +//! the supervisor, the sandbox image, and the driver's filesystem, which is +//! why they run in every environment. + +use std::fs; +use std::path::{Path, PathBuf}; + +use tempfile::TempDir; +use testkit::{CliOutput, Context, Error, Sandbox}; + +const LARGE_FILE_SIZE: usize = 512 * 1024; + +/// Files and directories round-trip through upload and download unchanged, +/// uploads into existing directories land inside them, and downloads of +/// single files and directories reproduce them locally. +#[testkit::test] +async fn upload_download_round_trip(cx: &mut Context) -> testkit::Result { + let ft = Transfer::new(cx, "fr").await?; + ft.upload_into_existing_directories().await?; + ft.round_trip().await?; + ft.download_file().await?; + ft.download_directory().await?; + ft.sandbox.delete().await +} + +/// Uploads from a Git work tree skip ignored files, a single file uploads +/// alone, and an upload whose selection is entirely ignored is rejected +/// unless `--no-git-ignore` is given. +#[testkit::test] +async fn upload_respects_gitignore(cx: &mut Context) -> testkit::Result { + let ft = Transfer::new(cx, "fg").await?; + ft.gitignore_filtering().await?; + ft.single_file_from_git_repo().await?; + ft.ignored_directory_requires_override().await?; + ft.sandbox.delete().await +} + +/// Downloads refuse sandbox paths that resolve outside the workspace, and +/// file names that look like command-line options download as plain files. +#[testkit::test] +async fn download_stays_inside_workspace(cx: &mut Context) -> testkit::Result { + let ft = Transfer::new(cx, "fs").await?; + ft.reject_workspace_escape().await?; + ft.download_dash_leading_name().await?; + ft.sandbox.delete().await +} + +/// A sandbox, a directory in its workspace, and a local scratch directory. +struct Transfer<'a> { + cx: &'a Context, + sandbox: Sandbox, + remote_root: String, + local: TempDir, +} + +impl<'a> Transfer<'a> { + async fn new(cx: &'a Context, suffix: &str) -> testkit::Result { + let sandbox = cx + .sandbox(suffix) + .command(["sleep", "infinity"]) + .create() + .await?; + sandbox.wait_ready().await?; + let remote_root = format!("/sandbox/file-transfer-{}-{suffix}", cx.run_id()); + let local = tempfile::tempdir() + .map_err(|error| Error::msg(format!("create a local scratch directory: {error}")))?; + let ft = Self { + cx, + sandbox, + remote_root, + local, + }; + ft.sh(&format!("mkdir -p '{}'", ft.remote_root)).await?; + Ok(ft) + } + + fn local(&self, name: &str) -> PathBuf { + self.local.path().join(name) + } + + /// Run a shell script in the sandbox and require it to succeed. + async fn sh(&self, script: &str) -> testkit::Result { + let output = self.sandbox.exec(["sh", "-c", script]).await?; + testkit::ensure!( + output.exit_code == 0, + "`{script}` exited with {}; stderr: {}", + output.exit_code, + output.stderr + ); + Ok(()) + } + + async fn upload( + &self, + source: &Path, + destination: &str, + no_git_ignore: bool, + ) -> testkit::Result { + let mut args = vec![ + "sandbox".to_string(), + "upload".to_string(), + self.sandbox.name().to_string(), + source.display().to_string(), + destination.to_string(), + ]; + if no_git_ignore { + args.push("--no-git-ignore".to_string()); + } + self.cx.cli(args).await + } + + async fn download(&self, source: &str, destination: &Path) -> testkit::Result { + self.cx + .cli([ + "sandbox", + "download", + self.sandbox.name(), + source, + &destination.display().to_string(), + ]) + .await + } + + /// Upload into existing directories, including a symlink to one and one + /// named with a trailing slash; the file must land inside each. Check the + /// destination's type, not only the exit status: tar can succeed after + /// replacing an empty directory with a file. + async fn upload_into_existing_directories(&self) -> testkit::Result { + let source = self.local("config.json"); + write(&source, "upload-destination-payload")?; + let root = format!("{}/upload destinations", self.remote_root); + self.sh(&format!( + "mkdir -p '{root}/empty' '{root}/populated' '{root}/target' '{root}/slash' && \ + printf keep > '{root}/populated/keep.txt' && \ + ln -s target '{root}/link'" + )) + .await?; + + for directory in ["empty", "populated", "link", "slash/"] { + let destination = format!("{root}/{directory}"); + self.upload(&source, &destination, true).await?.success()?; + self.sh(&format!( + "test -d '{destination}' && \ + test \"$(cat '{destination}/config.json')\" = upload-destination-payload && \ + test \"$(cat '{root}/populated/keep.txt')\" = keep && \ + test -L '{root}/link'" + )) + .await?; + } + + // A missing destination is a rename; uploading again must replace + // that file rather than treat it as a directory. + let renamed = format!("{root}/renamed.json"); + for _ in ["rename", "overwrite"] { + self.upload(&source, &renamed, true).await?.success()?; + self.sh(&format!( + "test -f '{renamed}' && test \"$(cat '{renamed}')\" = upload-destination-payload" + )) + .await?; + } + Ok(()) + } + + /// A directory with a nested file and a large binary file, and a single + /// file, survive upload then download. + async fn round_trip(&self) -> testkit::Result { + let source = self.local("roundtrip-upload"); + create_dir_all(&source.join("subdir"))?; + write(&source.join("greeting.txt"), "hello-from-local")?; + write(&source.join("subdir/nested.txt"), "nested-content")?; + let large: Vec = (0u8..=250).cycle().take(LARGE_FILE_SIZE).collect(); + fs::write(source.join("large.bin"), &large) + .map_err(|error| Error::msg(format!("write large.bin: {error}")))?; + + let remote = format!("{}/roundtrip", self.remote_root); + let uploaded = self.upload(&source, &remote, false).await?.success()?; + let warning = message(&uploaded); + testkit::ensure!( + warning.contains("outside a Git work tree") + && warning.contains(".gitignore rules are not applied"), + "upload outside a Git work tree did not warn that filtering is off: {warning}" + ); + + let destination = self.local("roundtrip-download"); + create_dir_all(&destination)?; + self.download(&format!("{remote}/roundtrip-upload"), &destination) + .await? + .success()?; + require_text(&destination.join("greeting.txt"), "hello-from-local")?; + require_text(&destination.join("subdir/nested.txt"), "nested-content")?; + let actual = fs::read(destination.join("large.bin")) + .map_err(|error| Error::msg(format!("read large.bin: {error}")))?; + testkit::ensure!( + actual == large, + "large file changed in the round trip: sent {} bytes, received {}", + large.len(), + actual.len() + ); + + let single = self.local("single.txt"); + write(&single, "single-file-payload")?; + let remote_single = format!("{}/single.txt", self.remote_root); + self.upload(&single, &remote_single, false) + .await? + .success()?; + let single_destination = self.local("single-download"); + create_dir_all(&single_destination)?; + self.download(&remote_single, &single_destination) + .await? + .success()?; + require_text( + &single_destination.join("single.txt"), + "single-file-payload", + ) + } + + async fn download_file(&self) -> testkit::Result { + let remote = format!("{}/download-file.txt", self.remote_root); + self.sh(&format!("printf greeting-payload > '{remote}'")) + .await?; + let destination = self.local("download-file"); + create_dir_all(&destination)?; + self.download(&remote, &destination).await?.success()?; + require_text(&destination.join("download-file.txt"), "greeting-payload") + } + + async fn download_directory(&self) -> testkit::Result { + let remote = format!("{}/tree", self.remote_root); + self.sh(&format!( + "mkdir -p '{remote}/sub' && printf top-level > '{remote}/root.txt' && \ + printf nested > '{remote}/sub/child.txt'" + )) + .await?; + let destination = self.local("download-directory"); + create_dir_all(&destination)?; + self.download(&remote, &destination).await?.success()?; + require_text(&destination.join("root.txt"), "top-level")?; + require_text(&destination.join("sub/child.txt"), "nested") + } + + async fn gitignore_filtering(&self) -> testkit::Result { + let repository = self.local("filter-repo"); + create_dir_all(&repository.join("build"))?; + git(&repository, &["init", "--quiet"])?; + write(&repository.join(".gitignore"), "*.log\nbuild/\n")?; + write(&repository.join("tracked.txt"), "i-am-tracked")?; + write(&repository.join("ignored.log"), "i-should-be-filtered")?; + write(&repository.join("build/output.bin"), "build-artifact")?; + git(&repository, &["add", "."])?; + + let remote = format!("{}/filtered", self.remote_root); + self.upload(&repository, &remote, false).await?.success()?; + let destination = self.local("filter-download"); + create_dir_all(&destination)?; + self.download(&remote, &destination).await?.success()?; + let uploaded = destination.join("filter-repo"); + require_text(&uploaded.join("tracked.txt"), "i-am-tracked")?; + require_exists(&uploaded.join(".gitignore"))?; + require_absent(&uploaded.join("ignored.log"))?; + require_absent(&uploaded.join("build")) + } + + async fn single_file_from_git_repo(&self) -> testkit::Result { + let repository = self.local("single-repo"); + create_dir_all(&repository.join("nested"))?; + git(&repository, &["init", "--quiet"])?; + write(&repository.join(".gitignore"), "*.log\n")?; + write( + &repository.join("nested/config.txt"), + "single-file-from-repo", + )?; + write(&repository.join("tracked.txt"), "should-not-upload")?; + write(&repository.join("ignored.log"), "ignored")?; + + let remote = format!("{}/single-from-repo", self.remote_root); + self.upload(&repository.join("nested/config.txt"), &remote, false) + .await? + .success()?; + let destination = self.local("single-repo-download"); + create_dir_all(&destination)?; + self.download(&remote, &destination).await?.success()?; + require_text(&destination.join("config.txt"), "single-file-from-repo")?; + require_absent(&destination.join("tracked.txt"))?; + require_absent(&destination.join("ignored.log")) + } + + async fn ignored_directory_requires_override(&self) -> testkit::Result { + let remote_seed = format!("{}/runs/test.json", self.remote_root); + self.sh(&format!( + "mkdir -p '{}/runs' && printf downloaded-payload > '{remote_seed}'", + self.remote_root + )) + .await?; + + let repository = self.local("override-repo"); + let runs = repository.join("runs"); + create_dir_all(&runs)?; + git(&repository, &["init", "--quiet"])?; + write(&repository.join(".gitignore"), "runs/\n")?; + self.download(&remote_seed, &runs).await?.success()?; + require_exists(&runs.join("test.json"))?; + + let remote = format!("{}/reuploaded", self.remote_root); + let rejected = self.upload(&runs, &remote, false).await?.failure()?; + let output = message(&rejected); + testkit::ensure!( + output.contains("filtering selected no files") && output.contains("--no-git-ignore"), + "the rejected upload did not explain the --no-git-ignore override: {output}" + ); + self.sh(&format!("test ! -e '{remote}'")).await?; + + self.upload(&runs, &remote, true).await?.success()?; + let destination = self.local("override-download"); + create_dir_all(&destination)?; + self.download(&remote, &destination).await?.success()?; + require_text(&destination.join("runs/test.json"), "downloaded-payload") + } + + async fn reject_workspace_escape(&self) -> testkit::Result { + let etc_link = format!("{}/etc-link", self.remote_root); + let passwd_link = format!("{}/passwd-link", self.remote_root); + self.sh(&format!( + "ln -s /etc '{etc_link}' && ln -s /etc/passwd '{passwd_link}'" + )) + .await?; + let destination = self.local("workspace-escape"); + create_dir_all(&destination)?; + + for source in [etc_link.clone(), passwd_link, format!("{etc_link}/passwd")] { + let rejected = self.download(&source, &destination).await?.failure()?; + let output = message(&rejected); + testkit::ensure!( + output.contains("resolves to") + && output.contains("outside the") + && output.contains("sandbox workspace"), + "download of {source:?} was not rejected as outside the workspace: {output}" + ); + } + require_absent(&destination.join("passwd"))?; + require_absent(&destination.join("etc-link")) + } + + async fn download_dash_leading_name(&self) -> testkit::Result { + let remote = format!("{}/--checkpoint-action=evil", self.remote_root); + self.sh(&format!("printf dash-payload > '{remote}'")) + .await?; + let destination = self.local("dash-leading"); + create_dir_all(&destination)?; + self.download(&remote, &destination).await?.success()?; + require_text( + &destination.join("--checkpoint-action=evil"), + "dash-payload", + ) + } +} + +/// The CLI's stdout and stderr as one line of words. Error reports wrap to +/// the terminal width and draw a gutter, so match against this instead. +fn message(output: &CliOutput) -> String { + format!("{} {}", output.stdout, output.stderr) + .replace(['│', '×', '╰', '─', '╭', '├'], " ") + .split_whitespace() + .collect::>() + .join(" ") +} + +fn git(repository: &Path, args: &[&str]) -> testkit::Result { + let output = std::process::Command::new("git") + .args(args) + .current_dir(repository) + .output() + .map_err(|error| Error::msg(format!("run git {}: {error}", args.join(" "))))?; + testkit::ensure!( + output.status.success(), + "git {} failed in {}: {}", + args.join(" "), + repository.display(), + String::from_utf8_lossy(&output.stderr) + ); + Ok(()) +} + +fn write(path: &Path, contents: &str) -> testkit::Result { + fs::write(path, contents) + .map_err(|error| Error::msg(format!("write {}: {error}", path.display()))) +} + +fn create_dir_all(path: &Path) -> testkit::Result { + fs::create_dir_all(path) + .map_err(|error| Error::msg(format!("create {}: {error}", path.display()))) +} + +fn require_text(path: &Path, expected: &str) -> testkit::Result { + let actual = fs::read_to_string(path) + .map_err(|error| Error::msg(format!("read {}: {error}", path.display())))?; + testkit::ensure!( + actual == expected, + "{} contains {actual:?}; expected {expected:?}", + path.display() + ); + Ok(()) +} + +fn require_exists(path: &Path) -> testkit::Result { + testkit::ensure!(path.exists(), "{} does not exist", path.display()); + Ok(()) +} + +fn require_absent(path: &Path) -> testkit::Result { + testkit::ensure!(!path.exists(), "{} exists but should not", path.display()); + Ok(()) +} diff --git a/tests/suites/conformance/sandbox/tests/lifecycle.rs b/tests/suites/conformance/sandbox/tests/lifecycle.rs new file mode 100644 index 0000000000..02e4f6f117 --- /dev/null +++ b/tests/suites/conformance/sandbox/tests/lifecycle.rs @@ -0,0 +1,165 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Sandbox lifecycle suite. +//! +//! These tests are adapted from the `sandbox-lifecycle` suite in +//! NVIDIA/OpenShell#4629 by Evan Lezar. Tests that need a driver with a +//! persistent workspace live in `persistent_workspace.rs`. + +use testkit::Context; +use testkit::sdk::{SandboxPhase, SandboxRestartPolicy}; + +/// A long-running sandbox becomes ready, runs a command with exec, and is gone +/// from get and list after delete. +#[testkit::test] +async fn long_running_exec_delete(cx: &mut Context) -> testkit::Result { + let sandbox = cx + .sandbox("lr") + .command(["sleep", "infinity"]) + .create() + .await?; + sandbox.wait_ready().await?; + + let marker = format!("openshell-e2e-{}", cx.run_id()); + let output = sandbox.exec(["echo", marker.as_str()]).await?; + testkit::ensure!( + output.exit_code == 0, + "exec exited with {}; stderr: {}", + output.exit_code, + output.stderr + ); + testkit::ensure!( + output.stdout == format!("{marker}\n"), + "exec stdout was {:?}; expected {:?}", + output.stdout, + format!("{marker}\n") + ); + + sandbox.delete().await?; + cx.expect_absent(&sandbox).await +} + +/// A sandbox whose main command exits 0, with restart policy Never, reaches +/// Completed with exit code 0 and is not restarted. It is gone from get and +/// list after delete. +#[testkit::test] +async fn run_to_completion_exit_delete(cx: &mut Context) -> testkit::Result { + let sandbox = cx + .sandbox("rtc") + .command(["sh", "-c", "exit 0"]) + .restart_policy(SandboxRestartPolicy::Never) + .create() + .await?; + + let finished = sandbox.wait_finished().await?; + testkit::ensure!( + finished.phase == SandboxPhase::Completed, + "sandbox finished in phase {:?} with exit code {:?}; expected Completed", + finished.phase, + finished.exit_code + ); + testkit::ensure!( + finished.exit_code == Some(0), + "exit code was {:?}; expected Some(0)", + finished.exit_code + ); + testkit::ensure!( + finished.restart_count == 0, + "restart count was {}; expected 0", + finished.restart_count + ); + + // Completed is final under Never: a later read must not show a restart. + let again = sandbox.get().await?; + testkit::ensure!( + again.phase == SandboxPhase::Completed && again.restart_count == 0, + "sandbox left Completed: phase {:?}, restart count {}", + again.phase, + again.restart_count + ); + + sandbox.delete().await?; + cx.expect_absent(&sandbox).await +} + +/// A sandbox created with the gateway's default command becomes Ready, is +/// listed as Ready, runs a command with exec, and is gone after delete. +#[testkit::test] +async fn default_command_sandbox_is_listed_ready(cx: &mut Context) -> testkit::Result { + let sandbox = cx.sandbox("dc").create().await?; + let ready = sandbox.wait_ready().await?; + testkit::ensure!( + ready.name == sandbox.name(), + "get returned sandbox {:?}; expected {:?}", + ready.name, + sandbox.name() + ); + cx.expect_listed(&sandbox, SandboxPhase::Ready).await?; + + let marker = format!("openshell-e2e-{}", cx.run_id()); + let output = sandbox.exec(["echo", marker.as_str()]).await?; + testkit::ensure!( + output.exit_code == 0 && output.stdout == format!("{marker}\n"), + "exec exited {} with stdout {:?}; stderr: {}", + output.exit_code, + output.stdout, + output.stderr + ); + + sandbox.delete().await?; + cx.expect_absent(&sandbox).await +} + +/// A stopped sandbox rejects exec, accepts a second stop, and returns to +/// Ready when started. +#[testkit::test] +async fn restart_stopped_sandbox(cx: &mut Context) -> testkit::Result { + let sandbox = cx + .sandbox("rs") + .command(["sleep", "infinity"]) + .create() + .await?; + sandbox.wait_ready().await?; + + sandbox.stop().await?; + match sandbox.exec(["true"]).await { + Err(_) => {} + Ok(output) => testkit::ensure!( + output.exit_code != 0, + "exec succeeded while the sandbox was stopped" + ), + } + // Stop is idempotent. + let again = sandbox.stop().await?; + testkit::ensure!( + again.phase == SandboxPhase::Stopped, + "a second stop left the sandbox {:?}", + again.phase + ); + + sandbox.start().await?; + let output = sandbox.exec(["echo", "started"]).await?; + testkit::ensure!( + output.exit_code == 0 && output.stdout == "started\n", + "exec after start exited {} with stdout {:?}; stderr: {}", + output.exit_code, + output.stdout, + output.stderr + ); + Ok(()) +} + +/// A stopped sandbox can be deleted, and is then gone from get and list. +#[testkit::test] +async fn stopped_sandbox_can_be_deleted(cx: &mut Context) -> testkit::Result { + let sandbox = cx + .sandbox("sd") + .command(["sleep", "infinity"]) + .create() + .await?; + sandbox.wait_ready().await?; + sandbox.stop().await?; + sandbox.delete().await?; + cx.expect_absent(&sandbox).await +} diff --git a/tests/suites/conformance/sandbox/tests/persistent_workspace.rs b/tests/suites/conformance/sandbox/tests/persistent_workspace.rs new file mode 100644 index 0000000000..101ea6d6ff --- /dev/null +++ b/tests/suites/conformance/sandbox/tests/persistent_workspace.rs @@ -0,0 +1,63 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Lifecycle tests for a compute driver whose workspace is persistent. +//! +//! Adapted from the `sandbox-lifecycle` suite in NVIDIA/OpenShell#4629 by Evan +//! Lezar. Docker, Podman, Kubernetes, and MicroVM keep `/sandbox` across stop +//! and start, and so must every conformant driver. The Microsoft MXC driver +//! cannot start a stopped sandbox yet, so it is not conformant until it can. + +use testkit::Context; + +/// Files in the workspace survive stop and start, and start runs the main +/// command again. +#[testkit::test] +async fn persistent_workspace_survives_stop_start(cx: &mut Context) -> testkit::Result { + const SENTINEL: &str = "/sandbox/.openshell-stop-start-sentinel"; + const RUN_COUNT: &str = "/sandbox/.openshell-main-run-count"; + let main = format!( + "count=0; test ! -f '{RUN_COUNT}' || count=$(cat '{RUN_COUNT}'); \ + count=$((count + 1)); printf '%s\\n' \"$count\" > '{RUN_COUNT}'; \ + exec sleep infinity" + ); + let sandbox = cx + .sandbox("kw") + .command(["sh", "-c", main.as_str()]) + .create() + .await?; + sandbox.wait_ready().await?; + + let sentinel = format!("openshell-stop-start-{}", cx.run_id()); + let write = sandbox + .exec([ + "sh", + "-c", + &format!("printf '%s\\n' '{sentinel}' > '{SENTINEL}' && sync"), + ]) + .await?; + testkit::ensure!( + write.exit_code == 0, + "writing the sentinel exited {}; stderr: {}", + write.exit_code, + write.stderr + ); + + sandbox.stop().await?; + sandbox.start().await?; + + let read = sandbox.exec(["cat", SENTINEL]).await?; + testkit::ensure!( + read.stdout == format!("{sentinel}\n"), + "sentinel after restart is {:?}; stderr: {}", + read.stdout, + read.stderr + ); + let runs = sandbox.exec(["cat", RUN_COUNT]).await?; + testkit::ensure!( + runs.stdout == "2\n", + "main command ran {:?} times; expected 2", + runs.stdout.trim() + ); + Ok(()) +} diff --git a/tests/testkit/README.md b/tests/testkit/README.md index 0c1d7f2f9d..3dcfb54798 100644 --- a/tests/testkit/README.md +++ b/tests/testkit/README.md @@ -64,6 +64,10 @@ test creates, such as providers, in the test itself. - `cx.sandbox(suffix)` to build and create a sandbox with a run-unique name. The returned `Sandbox` can wait for ready or finished, stop, start, exec, read its config and draft policy, set a setting, and delete. +- `cx.track(name)` to register a sandbox created some other way, such as + through the CLI, for cleanup. Call it before the + create, so a create that fails after the gateway accepted it is still cleaned + up. `cx.adopt(name)` returns a `Sandbox` handle for one the CLI created. - `cx.expect_listed` and `cx.expect_absent` to check a sandbox in list and get. - `cx.cli([...])` to run the installed CLI against the same gateway, with `success()`, `failure()`, and `json()` on the output. diff --git a/tests/testkit/testkit/src/context.rs b/tests/testkit/testkit/src/context.rs index 24e3f0a37a..9d4b4a0ff5 100644 --- a/tests/testkit/testkit/src/context.rs +++ b/tests/testkit/testkit/src/context.rs @@ -101,6 +101,31 @@ impl Context { } } + /// Track a sandbox the test is about to create through the CLI, so cleanup + /// deletes it even if the test fails after the gateway accepted the create. + /// Call this before the CLI command; use [`Context::adopt`] afterwards. + pub fn track(&self, name: &str) { + lock(&self.sandboxes).insert(name.to_string()); + } + + /// A handle to a sandbox created outside the builder, e.g. by the CLI. + /// It is tracked for cleanup like a built one. + pub async fn adopt(&self, name: &str) -> Result { + self.track(name); + let found = self + .client + .get_sandbox(name) + .await + .context(format!("get sandbox {name}"))?; + Ok(Sandbox { + client: self.client.clone(), + grpc: self.grpc.clone(), + sandboxes: Arc::clone(&self.sandboxes), + name: name.to_string(), + id: found.id, + }) + } + /// Fail unless the sandbox is gone from both get and list. pub async fn expect_absent(&self, sandbox: &Sandbox) -> Result { let name = sandbox.name(); diff --git a/tests/tmachine/ansible/playbooks/conformance/cli.yaml b/tests/tmachine/ansible/playbooks/conformance/cli.yaml index e841a11442..071672167d 100644 --- a/tests/tmachine/ansible/playbooks/conformance/cli.yaml +++ b/tests/tmachine/ansible/playbooks/conformance/cli.yaml @@ -83,7 +83,7 @@ when: gateway_status is succeeded and conformance_k3s.stat.exists block: - name: Verify Agent Sandbox reconciliation - ansible.builtin.include_tasks: k3s-ready.yaml + ansible.builtin.include_tasks: ../k3s-controller-ready.yaml rescue: - name: Record controller readiness failure ansible.builtin.set_fact: diff --git a/tests/tmachine/ansible/playbooks/conformance/k3s-ready.yaml b/tests/tmachine/ansible/playbooks/k3s-controller-ready.yaml similarity index 100% rename from tests/tmachine/ansible/playbooks/conformance/k3s-ready.yaml rename to tests/tmachine/ansible/playbooks/k3s-controller-ready.yaml diff --git a/tests/tmachine/ansible/playbooks/openshell-k3s.yaml b/tests/tmachine/ansible/playbooks/openshell-k3s.yaml index e39a9f3523..aca8ec958f 100644 --- a/tests/tmachine/ansible/playbooks/openshell-k3s.yaml +++ b/tests/tmachine/ansible/playbooks/openshell-k3s.yaml @@ -167,6 +167,11 @@ - k3s_gateway_address.stdout is match('^[0-9.]+:[0-9]+$') fail_msg: "Unexpected gateway Service address: {{ k3s_gateway_address.stdout }}" + # Deployment readiness doesn't establish that the controller reconciles + # Sandboxes, so the first test would otherwise race it. Tests don't wait. + - name: Verify Agent Sandbox reconciliation + ansible.builtin.include_tasks: k3s-controller-ready.yaml + - name: Register OpenShell gateway for test client hosts: all gather_facts: false diff --git a/tests/tmachine/ansible/playbooks/test-tools.yaml b/tests/tmachine/ansible/playbooks/test-tools.yaml new file mode 100644 index 0000000000..394810f8f2 --- /dev/null +++ b/tests/tmachine/ansible/playbooks/test-tools.yaml @@ -0,0 +1,19 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +--- +# Tools the testkit suites run on the machine itself, next to cargo-nextest. +# The file-transfer suite builds a Git repository there to check that upload +# honors .gitignore. +- name: Install test tools + hosts: all + become: true + gather_facts: false + tasks: + - name: Wait for SSH + ansible.builtin.wait_for_connection: + + - name: Install Git + ansible.builtin.package: + name: git + state: present diff --git a/tests/tmachine/config.nix b/tests/tmachine/config.nix index ee9f70943e..98d0c41519 100644 --- a/tests/tmachine/config.nix +++ b/tests/tmachine/config.nix @@ -48,6 +48,7 @@ let use_galaxy = true; playbooks = [ "ansible/playbooks/nextest.yaml" + "ansible/playbooks/test-tools.yaml" "ansible/playbooks/docker.yaml" ]; }; @@ -59,6 +60,7 @@ let use_galaxy = false; playbooks = [ "ansible/playbooks/nextest.yaml" + "ansible/playbooks/test-tools.yaml" "ansible/playbooks/k3s.yaml" ]; }; @@ -70,6 +72,7 @@ let use_galaxy = false; playbooks = [ "ansible/playbooks/nextest.yaml" + "ansible/playbooks/test-tools.yaml" "ansible/playbooks/selinux.yaml" "ansible/playbooks/podman-rootful.yaml" ]; @@ -82,6 +85,7 @@ let use_galaxy = false; playbooks = [ "ansible/playbooks/nextest.yaml" + "ansible/playbooks/test-tools.yaml" "ansible/playbooks/selinux.yaml" "ansible/playbooks/podman-rootless.yaml" ];