Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

3 changes: 3 additions & 0 deletions crates/openshell-cli/src/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1842,6 +1842,7 @@ enum SandboxCommands {
/// Connect to a sandbox.
///
/// When no name is given, reconnects to the last-used sandbox.
/// A sandbox the gateway stopped while idle is started first.
/// Press Ctrl-D or Ctrl-P Ctrl-Q to disconnect without terminating the main process.
#[command(help_template = LEAF_HELP_TEMPLATE, next_help_heading = "FLAGS")]
Connect {
Expand Down Expand Up @@ -3800,6 +3801,8 @@ async fn run_async() -> Result<()> {
}
SandboxCommands::Connect { name, editor } => {
let name = resolve_sandbox_name(name, &ctx.name, &cli.workspace)?;
run::start_if_idle_stopped(endpoint, &name, &cli.workspace, &tls)
.await?;
if let Some(editor) = editor.map(Into::into) {
run::sandbox_connect_editor(
endpoint,
Expand Down
97 changes: 90 additions & 7 deletions crates/openshell-cli/src/run.rs
Original file line number Diff line number Diff line change
Expand Up @@ -4172,6 +4172,53 @@ pub async fn sandbox_start(
Ok(())
}

/// Start a sandbox that the gateway stopped because its program sat idle, so
/// a connect can proceed. Any other phase is left for the connect to report.
pub async fn start_if_idle_stopped(
server: &str,
name: &str,
workspace: &str,
tls: &TlsOptions,
) -> Result<()> {
let mut client = grpc_client(server, tls).await?;
let sandbox = client
.get_sandbox(GetSandboxRequest {
name: name.to_string(),
workspace_scope: Some(openshell_core::proto::workspace_selector(
workspace.to_string(),
)),
})
.await
.into_diagnostic()?
.into_inner()
.sandbox;
let Some(mut sandbox) = sandbox else {
return Ok(());
};
if sandbox.idle_stopping() {
eprintln!("Sandbox {name} is stopping while idle; starting it once it stops");
// The stop can roll back to Ready, or another client can restart the
// sandbox before this watch opens, so wait for it to leave Stopping
// rather than for Stopped.
sandbox = wait_while_lifecycle_phase(&mut client, sandbox, SandboxPhase::Stopping).await?;
match SandboxPhase::try_from(sandbox.phase()).unwrap_or(SandboxPhase::Unknown) {
SandboxPhase::Ready => return Ok(()),
SandboxPhase::Starting => {
wait_for_lifecycle_phase(&mut client, sandbox, SandboxPhase::Ready).await?;
return Ok(());
}
_ if sandbox.idle_stopped() => {}
// Any other phase is for connect to report.
_ => return Ok(()),
}
} else if sandbox.idle_stopped() {
eprintln!("Sandbox {name} was stopped while idle; starting it");
} else {
return Ok(());
}
sandbox_start(server, name, workspace, tls).await
}

/// Suspend a ready sandbox, retaining its memory state, and wait until it is suspended.
pub async fn sandbox_suspend(
server: &str,
Expand Down Expand Up @@ -4288,7 +4335,46 @@ async fn wait_for_lifecycle_phase(
if let Some(err) = lifecycle_phase_error(&sandbox, target, current) {
return Err(err);
}
watch_until(
client,
&sandbox,
&format!("reach {target:?}"),
|sandbox, phase| {
if phase == target {
return Ok(true);
}
lifecycle_phase_error(sandbox, target, phase).map_or(Ok(false), Err)
},
)
.await
}

/// Wait until the sandbox leaves `phase`, returning its first other phase.
async fn wait_while_lifecycle_phase(
client: &mut crate::tls::GrpcClient,
sandbox: Sandbox,
phase: SandboxPhase,
) -> Result<Sandbox> {
if sandbox.phase() != phase as i32 {
return Ok(sandbox);
}
watch_until(
client,
&sandbox,
&format!("leave {phase:?}"),
|_, current| Ok(current != phase),
)
.await
}

/// Watch `sandbox` until `done` accepts a snapshot, within the lifecycle
/// timeout.
async fn watch_until(
client: &mut crate::tls::GrpcClient,
sandbox: &Sandbox,
goal: &str,
mut done: impl FnMut(&Sandbox, SandboxPhase) -> Result<bool>,
) -> Result<Sandbox> {
let timeout = Duration::from_secs(
std::env::var("OPENSHELL_LIFECYCLE_TIMEOUT")
.ok()
Expand Down Expand Up @@ -4321,30 +4407,27 @@ async fn wait_for_lifecycle_phase(
let remaining = deadline.saturating_duration_since(Instant::now());
if remaining.is_zero() {
return Err(miette!(
"timed out after {}s waiting for sandbox to reach {target:?}",
"timed out after {}s waiting for sandbox to {goal}",
timeout.as_secs()
));
}
let event = tokio::time::timeout(remaining, stream.next())
.await
.map_err(|_| {
miette!(
"timed out after {}s waiting for sandbox to reach {target:?}",
"timed out after {}s waiting for sandbox to {goal}",
timeout.as_secs()
)
})?
.ok_or_else(|| miette!("sandbox watch ended before reaching {target:?}"))?
.ok_or_else(|| miette!("sandbox watch ended before the sandbox could {goal}"))?
.into_diagnostic()?;
if let Some(openshell_core::proto::sandbox_stream_event::Payload::Sandbox(sandbox)) =
event.payload
{
let phase = SandboxPhase::try_from(sandbox.phase()).unwrap_or(SandboxPhase::Unknown);
if phase == target {
if done(&sandbox, phase)? {
return Ok(sandbox);
}
if let Some(err) = lifecycle_phase_error(&sandbox, target, phase) {
return Err(err);
}
}
}
}
Expand Down
130 changes: 130 additions & 0 deletions crates/openshell-cli/tests/sandbox_create_lifecycle_integration.rs
Original file line number Diff line number Diff line change
Expand Up @@ -91,6 +91,11 @@ struct SandboxState {
template_get_requests: Arc<Mutex<Vec<GetSandboxTemplateRequest>>>,
template_list_requests: Arc<Mutex<Vec<ListSandboxTemplatesRequest>>>,
template_delete_requests: Arc<Mutex<Vec<DeleteSandboxTemplateRequest>>>,
/// Replace the default Ready `GetSandbox` response.
get_sandbox_response: Arc<Mutex<Option<Sandbox>>>,
/// Replace every `WatchSandbox` stream with these status snapshots.
watch_snapshots: Arc<Mutex<Option<Vec<Sandbox>>>>,
start_requests: Arc<AtomicUsize>,
}

#[derive(Clone, Default)]
Expand Down Expand Up @@ -173,6 +178,17 @@ impl OpenShell for TestOpenShell {
}))
}

async fn start_sandbox(
&self,
_request: tonic::Request<openshell_core::proto::StartSandboxRequest>,
) -> Result<Response<SandboxResponse>, Status> {
self.state.start_requests.fetch_add(1, Ordering::SeqCst);
Ok(Response::new(SandboxResponse {
sandbox: Some(lifecycle_sandbox(SandboxPhase::Starting, "Starting")),
..Default::default()
}))
}

async fn stop_sandbox(
&self,
_request: tonic::Request<openshell_core::proto::StopSandboxRequest>,
Expand All @@ -194,6 +210,13 @@ impl OpenShell for TestOpenShell {
&self,
request: tonic::Request<GetSandboxRequest>,
) -> Result<Response<SandboxResponse>, Status> {
let response = self.state.get_sandbox_response.lock().await.clone();
if let Some(sandbox) = response {
return Ok(Response::new(SandboxResponse {
sandbox: Some(sandbox),
..Default::default()
}));
}
let request = request.into_inner();
let name = request.name;
let mut sandbox = Sandbox {
Expand Down Expand Up @@ -547,6 +570,26 @@ impl OpenShell for TestOpenShell {
&self,
request: tonic::Request<WatchSandboxRequest>,
) -> Result<Response<tonic::codegen::BoxStream<SandboxStreamEvent>>, Status> {
let snapshots = self.state.watch_snapshots.lock().await.clone();
if let Some(snapshots) = snapshots {
let (tx, rx) = mpsc::channel(snapshots.len().max(1));
tokio::spawn(async move {
for sandbox in snapshots {
let event = SandboxStreamEvent {
payload: Some(sandbox_stream_event::Payload::Sandbox(sandbox)),
..Default::default()
};
if tx.send(Ok(event)).await.is_err() {
return;
}
}
// Keep the stream open, as a live watch would.
tx.closed().await;
});
return Ok(Response::new(Box::pin(
tokio_stream::wrappers::ReceiverStream::new(rx),
)));
}
let request = request.into_inner();
let sandbox_id = format!("id-{}", request.sandbox);
let (tx, rx) = mpsc::channel(4);
Expand Down Expand Up @@ -1321,6 +1364,93 @@ fn test_config() -> run::SandboxCreateConfig<'static> {
}
}

/// A sandbox named "idle-wake" in `phase` with Ready condition `reason`.
fn lifecycle_sandbox(phase: SandboxPhase, reason: &str) -> Sandbox {
Sandbox {
metadata: Some(openshell_core::proto::datamodel::v1::ObjectMeta {
id: "id-idle-wake".into(),
name: "idle-wake".into(),
workspace: "default".into(),
..Default::default()
}),
status: Some(SandboxStatus {
phase: phase as i32,
conditions: vec![SandboxCondition {
r#type: "Ready".into(),
status: if phase == SandboxPhase::Ready {
"True".into()
} else {
"False".into()
},
reason: reason.into(),
..Default::default()
}],
..Default::default()
}),
..Default::default()
}
}

/// Connect during an in-flight idle stop waits for the sandbox to leave
/// `Stopping`. It connects to a sandbox that came back `Ready` or `Starting`
/// without another start, and starts it only once it is `IdleStopped`.
#[tokio::test]
async fn connect_during_an_idle_stop_waits_for_the_stop_to_settle() {
let server = run_server().await;
let tls = test_tls(&server);
let config_dir = helpers::tempdir().unwrap();
let stopping = lifecycle_sandbox(SandboxPhase::Stopping, "IdleStopping");
for (snapshots, starts) in [
// The stop rolled back, or another client already restarted it.
(
vec![lifecycle_sandbox(SandboxPhase::Ready, "DependenciesReady")],
0,
),
(
vec![
lifecycle_sandbox(SandboxPhase::Starting, "Starting"),
lifecycle_sandbox(SandboxPhase::Ready, "DependenciesReady"),
],
0,
),
// The idle stop settled; connect starts the sandbox once.
(
vec![
lifecycle_sandbox(SandboxPhase::Stopped, "IdleStopped"),
lifecycle_sandbox(SandboxPhase::Ready, "DependenciesReady"),
],
1,
),
] {
server
.openshell
.state
.start_requests
.store(0, Ordering::SeqCst);
*server.openshell.state.get_sandbox_response.lock().await = Some(stopping.clone());
*server.openshell.state.watch_snapshots.lock().await = Some(snapshots);
Environment::new()
.set("XDG_CONFIG_HOME", config_dir.path())
.run_async(async {
tokio::time::timeout(
Duration::from_secs(5),
run::start_if_idle_stopped(&server.endpoint, "idle-wake", "default", &tls),
)
.await
.expect("connect must not wait for a phase the sandbox already left")
.expect("connect proceeds");
assert_eq!(
server.openshell.state.start_requests.load(Ordering::SeqCst),
starts,
"starts issued for this case"
);
})
.await;
}
*server.openshell.state.get_sandbox_response.lock().await = None;
*server.openshell.state.watch_snapshots.lock().await = None;
}

#[tokio::test]
async fn sandbox_stop_accepts_terminal_results_without_waiting_for_stopped() {
let server = run_server().await;
Expand Down
Loading
Loading