diff --git a/Cargo.lock b/Cargo.lock index 1f0f2b414e..6a185657f5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5508,6 +5508,7 @@ dependencies = [ "openshell-ocsf", "openshell-ocsf-validate", "openshell-sandbox-backend", + "prost-types", "rand 0.10.2", "russh", "rustix 1.1.6", diff --git a/crates/openshell-cli/src/main.rs b/crates/openshell-cli/src/main.rs index 8fc1a33116..243a384482 100644 --- a/crates/openshell-cli/src/main.rs +++ b/crates/openshell-cli/src/main.rs @@ -1842,6 +1842,7 @@ enum SandboxCommands { /// Connect to a sandbox. /// /// When no name is given, reconnects to the last-used sandbox. + /// A sandbox the gateway stopped while idle is started first. /// Press Ctrl-D or Ctrl-P Ctrl-Q to disconnect without terminating the main process. #[command(help_template = LEAF_HELP_TEMPLATE, next_help_heading = "FLAGS")] Connect { @@ -3800,6 +3801,8 @@ async fn run_async() -> Result<()> { } SandboxCommands::Connect { name, editor } => { let name = resolve_sandbox_name(name, &ctx.name, &cli.workspace)?; + run::start_if_idle_stopped(endpoint, &name, &cli.workspace, &tls) + .await?; if let Some(editor) = editor.map(Into::into) { run::sandbox_connect_editor( endpoint, diff --git a/crates/openshell-cli/src/run.rs b/crates/openshell-cli/src/run.rs index dafc10efba..d6af3cb4f4 100644 --- a/crates/openshell-cli/src/run.rs +++ b/crates/openshell-cli/src/run.rs @@ -4172,6 +4172,53 @@ pub async fn sandbox_start( Ok(()) } +/// Start a sandbox that the gateway stopped because its program sat idle, so +/// a connect can proceed. Any other phase is left for the connect to report. +pub async fn start_if_idle_stopped( + server: &str, + name: &str, + workspace: &str, + tls: &TlsOptions, +) -> Result<()> { + let mut client = grpc_client(server, tls).await?; + let sandbox = client + .get_sandbox(GetSandboxRequest { + name: name.to_string(), + workspace_scope: Some(openshell_core::proto::workspace_selector( + workspace.to_string(), + )), + }) + .await + .into_diagnostic()? + .into_inner() + .sandbox; + let Some(mut sandbox) = sandbox else { + return Ok(()); + }; + if sandbox.idle_stopping() { + eprintln!("Sandbox {name} is stopping while idle; starting it once it stops"); + // The stop can roll back to Ready, or another client can restart the + // sandbox before this watch opens, so wait for it to leave Stopping + // rather than for Stopped. + sandbox = wait_while_lifecycle_phase(&mut client, sandbox, SandboxPhase::Stopping).await?; + match SandboxPhase::try_from(sandbox.phase()).unwrap_or(SandboxPhase::Unknown) { + SandboxPhase::Ready => return Ok(()), + SandboxPhase::Starting => { + wait_for_lifecycle_phase(&mut client, sandbox, SandboxPhase::Ready).await?; + return Ok(()); + } + _ if sandbox.idle_stopped() => {} + // Any other phase is for connect to report. + _ => return Ok(()), + } + } else if sandbox.idle_stopped() { + eprintln!("Sandbox {name} was stopped while idle; starting it"); + } else { + return Ok(()); + } + sandbox_start(server, name, workspace, tls).await +} + /// Suspend a ready sandbox, retaining its memory state, and wait until it is suspended. pub async fn sandbox_suspend( server: &str, @@ -4288,7 +4335,46 @@ async fn wait_for_lifecycle_phase( if let Some(err) = lifecycle_phase_error(&sandbox, target, current) { return Err(err); } + watch_until( + client, + &sandbox, + &format!("reach {target:?}"), + |sandbox, phase| { + if phase == target { + return Ok(true); + } + lifecycle_phase_error(sandbox, target, phase).map_or(Ok(false), Err) + }, + ) + .await +} +/// Wait until the sandbox leaves `phase`, returning its first other phase. +async fn wait_while_lifecycle_phase( + client: &mut crate::tls::GrpcClient, + sandbox: Sandbox, + phase: SandboxPhase, +) -> Result { + if sandbox.phase() != phase as i32 { + return Ok(sandbox); + } + watch_until( + client, + &sandbox, + &format!("leave {phase:?}"), + |_, current| Ok(current != phase), + ) + .await +} + +/// Watch `sandbox` until `done` accepts a snapshot, within the lifecycle +/// timeout. +async fn watch_until( + client: &mut crate::tls::GrpcClient, + sandbox: &Sandbox, + goal: &str, + mut done: impl FnMut(&Sandbox, SandboxPhase) -> Result, +) -> Result { let timeout = Duration::from_secs( std::env::var("OPENSHELL_LIFECYCLE_TIMEOUT") .ok() @@ -4321,7 +4407,7 @@ async fn wait_for_lifecycle_phase( let remaining = deadline.saturating_duration_since(Instant::now()); if remaining.is_zero() { return Err(miette!( - "timed out after {}s waiting for sandbox to reach {target:?}", + "timed out after {}s waiting for sandbox to {goal}", timeout.as_secs() )); } @@ -4329,22 +4415,19 @@ async fn wait_for_lifecycle_phase( .await .map_err(|_| { miette!( - "timed out after {}s waiting for sandbox to reach {target:?}", + "timed out after {}s waiting for sandbox to {goal}", timeout.as_secs() ) })? - .ok_or_else(|| miette!("sandbox watch ended before reaching {target:?}"))? + .ok_or_else(|| miette!("sandbox watch ended before the sandbox could {goal}"))? .into_diagnostic()?; if let Some(openshell_core::proto::sandbox_stream_event::Payload::Sandbox(sandbox)) = event.payload { let phase = SandboxPhase::try_from(sandbox.phase()).unwrap_or(SandboxPhase::Unknown); - if phase == target { + if done(&sandbox, phase)? { return Ok(sandbox); } - if let Some(err) = lifecycle_phase_error(&sandbox, target, phase) { - return Err(err); - } } } } diff --git a/crates/openshell-cli/tests/sandbox_create_lifecycle_integration.rs b/crates/openshell-cli/tests/sandbox_create_lifecycle_integration.rs index 6608e54b96..87f911a0e0 100644 --- a/crates/openshell-cli/tests/sandbox_create_lifecycle_integration.rs +++ b/crates/openshell-cli/tests/sandbox_create_lifecycle_integration.rs @@ -91,6 +91,11 @@ struct SandboxState { template_get_requests: Arc>>, template_list_requests: Arc>>, template_delete_requests: Arc>>, + /// Replace the default Ready `GetSandbox` response. + get_sandbox_response: Arc>>, + /// Replace every `WatchSandbox` stream with these status snapshots. + watch_snapshots: Arc>>>, + start_requests: Arc, } #[derive(Clone, Default)] @@ -173,6 +178,17 @@ impl OpenShell for TestOpenShell { })) } + async fn start_sandbox( + &self, + _request: tonic::Request, + ) -> Result, Status> { + self.state.start_requests.fetch_add(1, Ordering::SeqCst); + Ok(Response::new(SandboxResponse { + sandbox: Some(lifecycle_sandbox(SandboxPhase::Starting, "Starting")), + ..Default::default() + })) + } + async fn stop_sandbox( &self, _request: tonic::Request, @@ -194,6 +210,13 @@ impl OpenShell for TestOpenShell { &self, request: tonic::Request, ) -> Result, Status> { + let response = self.state.get_sandbox_response.lock().await.clone(); + if let Some(sandbox) = response { + return Ok(Response::new(SandboxResponse { + sandbox: Some(sandbox), + ..Default::default() + })); + } let request = request.into_inner(); let name = request.name; let mut sandbox = Sandbox { @@ -547,6 +570,26 @@ impl OpenShell for TestOpenShell { &self, request: tonic::Request, ) -> Result>, Status> { + let snapshots = self.state.watch_snapshots.lock().await.clone(); + if let Some(snapshots) = snapshots { + let (tx, rx) = mpsc::channel(snapshots.len().max(1)); + tokio::spawn(async move { + for sandbox in snapshots { + let event = SandboxStreamEvent { + payload: Some(sandbox_stream_event::Payload::Sandbox(sandbox)), + ..Default::default() + }; + if tx.send(Ok(event)).await.is_err() { + return; + } + } + // Keep the stream open, as a live watch would. + tx.closed().await; + }); + return Ok(Response::new(Box::pin( + tokio_stream::wrappers::ReceiverStream::new(rx), + ))); + } let request = request.into_inner(); let sandbox_id = format!("id-{}", request.sandbox); let (tx, rx) = mpsc::channel(4); @@ -1321,6 +1364,93 @@ fn test_config() -> run::SandboxCreateConfig<'static> { } } +/// A sandbox named "idle-wake" in `phase` with Ready condition `reason`. +fn lifecycle_sandbox(phase: SandboxPhase, reason: &str) -> Sandbox { + Sandbox { + metadata: Some(openshell_core::proto::datamodel::v1::ObjectMeta { + id: "id-idle-wake".into(), + name: "idle-wake".into(), + workspace: "default".into(), + ..Default::default() + }), + status: Some(SandboxStatus { + phase: phase as i32, + conditions: vec![SandboxCondition { + r#type: "Ready".into(), + status: if phase == SandboxPhase::Ready { + "True".into() + } else { + "False".into() + }, + reason: reason.into(), + ..Default::default() + }], + ..Default::default() + }), + ..Default::default() + } +} + +/// Connect during an in-flight idle stop waits for the sandbox to leave +/// `Stopping`. It connects to a sandbox that came back `Ready` or `Starting` +/// without another start, and starts it only once it is `IdleStopped`. +#[tokio::test] +async fn connect_during_an_idle_stop_waits_for_the_stop_to_settle() { + let server = run_server().await; + let tls = test_tls(&server); + let config_dir = helpers::tempdir().unwrap(); + let stopping = lifecycle_sandbox(SandboxPhase::Stopping, "IdleStopping"); + for (snapshots, starts) in [ + // The stop rolled back, or another client already restarted it. + ( + vec![lifecycle_sandbox(SandboxPhase::Ready, "DependenciesReady")], + 0, + ), + ( + vec![ + lifecycle_sandbox(SandboxPhase::Starting, "Starting"), + lifecycle_sandbox(SandboxPhase::Ready, "DependenciesReady"), + ], + 0, + ), + // The idle stop settled; connect starts the sandbox once. + ( + vec![ + lifecycle_sandbox(SandboxPhase::Stopped, "IdleStopped"), + lifecycle_sandbox(SandboxPhase::Ready, "DependenciesReady"), + ], + 1, + ), + ] { + server + .openshell + .state + .start_requests + .store(0, Ordering::SeqCst); + *server.openshell.state.get_sandbox_response.lock().await = Some(stopping.clone()); + *server.openshell.state.watch_snapshots.lock().await = Some(snapshots); + Environment::new() + .set("XDG_CONFIG_HOME", config_dir.path()) + .run_async(async { + tokio::time::timeout( + Duration::from_secs(5), + run::start_if_idle_stopped(&server.endpoint, "idle-wake", "default", &tls), + ) + .await + .expect("connect must not wait for a phase the sandbox already left") + .expect("connect proceeds"); + assert_eq!( + server.openshell.state.start_requests.load(Ordering::SeqCst), + starts, + "starts issued for this case" + ); + }) + .await; + } + *server.openshell.state.get_sandbox_response.lock().await = None; + *server.openshell.state.watch_snapshots.lock().await = None; +} + #[tokio::test] async fn sandbox_stop_accepts_terminal_results_without_waiting_for_stopped() { let server = run_server().await; diff --git a/crates/openshell-core/src/metadata.rs b/crates/openshell-core/src/metadata.rs index 7bc4624ae3..63256b4132 100644 --- a/crates/openshell-core/src/metadata.rs +++ b/crates/openshell-core/src/metadata.rs @@ -84,13 +84,59 @@ impl ObjectWorkspace for Sandbox { } } +/// Ready condition reason of a sandbox stopped because its terminal program +/// waited on the user longer than the sandbox's idle timeout. +pub const IDLE_STOPPED_REASON: &str = "IdleStopped"; + +/// Ready condition reason of a sandbox whose idle stop is still in flight. +pub const IDLE_STOPPING_REASON: &str = "IdleStopping"; + +/// Condition type for the program status workload terminals declare via OSC 7501. +/// +/// Its reason is the state (`Idle`, `Working`, `Done`, `Blocked`, or `Error`), +/// and its transition time is when the program began that state or a client +/// last typed into a workload terminal, whichever is later. +pub const PROGRAM_STATUS_CONDITION: &str = "ProgramStatus"; + impl Sandbox { + /// Whether the gateway stopped this sandbox for idleness, so connecting + /// may start it again. + pub fn idle_stopped(&self) -> bool { + self.status.as_ref().is_some_and(|status| { + status.phase == crate::proto::SandboxPhase::Stopped as i32 + && status + .conditions + .iter() + .any(|c| c.r#type == "Ready" && c.reason == IDLE_STOPPED_REASON) + }) + } + + /// Whether an idle stop of this sandbox is still in flight, so a connect + /// should wait for it to finish and then start the sandbox. + pub fn idle_stopping(&self) -> bool { + self.status.as_ref().is_some_and(|status| { + status.phase == crate::proto::SandboxPhase::Stopping as i32 + && status + .conditions + .iter() + .any(|c| c.r#type == "Ready" && c.reason == IDLE_STOPPING_REASON) + }) + } + pub fn phase(&self) -> i32 { self.status.as_ref().map_or(0, |s| s.phase) } + /// Set the lifecycle phase. A phase change ends the program whose status + /// was reported, so its condition is removed with it. pub fn set_phase(&mut self, phase: i32) { - self.status.get_or_insert_with(SandboxStatus::default).phase = phase; + let status = self.status.get_or_insert_with(SandboxStatus::default); + if status.phase != phase { + status + .conditions + .retain(|condition| condition.r#type != PROGRAM_STATUS_CONDITION); + } + status.phase = phase; } pub fn current_policy_version(&self) -> u32 { @@ -418,3 +464,54 @@ impl ObjectWorkspace for ObjectForTest { false } } + +#[cfg(test)] +mod tests { + use super::{IDLE_STOPPED_REASON, IDLE_STOPPING_REASON}; + use crate::proto::{Sandbox, SandboxCondition, SandboxPhase, SandboxStatus}; + + fn sandbox(phase: SandboxPhase, reason: &str) -> Sandbox { + Sandbox { + status: Some(SandboxStatus { + phase: phase.into(), + conditions: vec![SandboxCondition { + r#type: "Ready".to_string(), + status: "False".to_string(), + reason: reason.to_string(), + ..Default::default() + }], + ..Default::default() + }), + ..Default::default() + } + } + + #[test] + fn idle_stop_predicates_need_both_phase_and_reason() { + assert!(sandbox(SandboxPhase::Stopping, IDLE_STOPPING_REASON).idle_stopping()); + assert!(sandbox(SandboxPhase::Stopped, IDLE_STOPPED_REASON).idle_stopped()); + assert!(!sandbox(SandboxPhase::Stopping, "Stopping").idle_stopping()); + assert!(!sandbox(SandboxPhase::Stopped, "Stopped").idle_stopped()); + assert!(!sandbox(SandboxPhase::Stopped, IDLE_STOPPING_REASON).idle_stopping()); + assert!(!sandbox(SandboxPhase::Ready, IDLE_STOPPED_REASON).idle_stopped()); + } + + #[test] + fn a_phase_change_removes_the_program_status_condition() { + let mut sandbox = sandbox(SandboxPhase::Ready, "DependenciesReady"); + sandbox + .status + .as_mut() + .unwrap() + .conditions + .push(SandboxCondition { + r#type: super::PROGRAM_STATUS_CONDITION.to_string(), + reason: "Done".to_string(), + ..Default::default() + }); + sandbox.set_phase(SandboxPhase::Ready as i32); + assert_eq!(sandbox.status.as_ref().unwrap().conditions.len(), 2); + sandbox.set_phase(SandboxPhase::Stopping as i32); + assert_eq!(sandbox.status.as_ref().unwrap().conditions.len(), 1); + } +} diff --git a/crates/openshell-core/src/settings.rs b/crates/openshell-core/src/settings.rs index f819d8ab38..f750f6cd2f 100644 --- a/crates/openshell-core/src/settings.rs +++ b/crates/openshell-core/src/settings.rs @@ -106,6 +106,17 @@ pub const PROPOSAL_APPROVAL_MODE_KEY: &str = "proposal_approval_mode"; /// fail-closes on unknown persisted values for defense in depth. pub const PROPOSAL_APPROVAL_MODE_VALUES: &[&str] = &["manual", "auto"]; +/// Seconds a sandbox's terminal program may wait on the user before it stops. +/// +/// Only programs that report OSC 7501 program status, such as Claude Code, can +/// be judged idle. Unset, zero, or negative disables idle stop; values above +/// [`MAX_IDLE_STOP_AFTER_SECONDS`] are capped. Gateway scope wins over sandbox +/// scope. +pub const IDLE_STOP_AFTER_SECONDS_KEY: &str = "idle_stop_after_seconds"; + +/// Longest effective idle stop timeout: 30 days. +pub const MAX_IDLE_STOP_AFTER_SECONDS: i64 = 30 * 24 * 60 * 60; + pub const REGISTERED_SETTINGS: &[RegisteredSetting] = &[ // When true the supervisor emits OCSF-JSON records to stderr and writes // JSONL to `/var/log/openshell-ocsf*.jsonl` when file logging is available. @@ -136,6 +147,12 @@ pub const REGISTERED_SETTINGS: &[RegisteredSetting] = &[ kind: SettingValueKind::String, allowed_string_values: Some(PROPOSAL_APPROVAL_MODE_VALUES), }, + // Idle stop timeout. See IDLE_STOP_AFTER_SECONDS_KEY. Unset disables it. + RegisteredSetting { + key: IDLE_STOP_AFTER_SECONDS_KEY, + kind: SettingValueKind::Int, + allowed_string_values: None, + }, ]; /// Resolve a setting descriptor from the registry by key. @@ -171,6 +188,13 @@ mod tests { RegisteredSetting, SettingValueKind, parse_bool_like, registered_keys_csv, setting_for_key, }; + #[test] + fn idle_stop_after_seconds_is_an_int_setting() { + let setting = setting_for_key(super::IDLE_STOP_AFTER_SECONDS_KEY) + .expect("idle_stop_after_seconds should be registered"); + assert_eq!(setting.kind, SettingValueKind::Int); + } + #[test] fn setting_for_key_returns_none_for_unknown() { assert!(setting_for_key("nonexistent_key").is_none()); diff --git a/crates/openshell-server/README.md b/crates/openshell-server/README.md index 61e458a23f..f36a9d3492 100644 --- a/crates/openshell-server/README.md +++ b/crates/openshell-server/README.md @@ -167,6 +167,64 @@ A provisioning timeout can be repaired by Stop after its original driver work settles and cleanup completes. Stop exposes Stopped without discarding the timeout diagnosis or storage. This lifecycle behavior also applies when updates are disabled. +## Idle stop + +The replica that holds a sandbox's supervisor session keeps that session's +latest `SupervisorWorkloadActivity` report in memory, in gateway time, and only +that report authorizes an idle stop. After a gateway restart or a session +change, nothing stops until the current session reports. The report also +becomes the `ProgramStatus` condition for display. The condition is written, +with conflict retries, only when its state changes or input moves its time +forward by a minute. It is fenced to the running main-process instance, +survives driver snapshots, and is removed on every phase change. + +Every replica sweeps its own sessions every 30 seconds. A sandbox becomes a +candidate when it is Ready, its program is idle, done, blocked, or in error, and +`idle_stop_after_seconds` (gateway scope, then sandbox scope) plus a 15-second +input report lag has elapsed. The lag runs from the later of the report's time +and the main-process start. + +A sweep starts at most one idle stop per sandbox at a time. Before taking any +lifecycle lock, the stop task does three things: + +1. Requires a fresh shared owner record naming the session. +2. Sends a `WorkloadActivityRequest`. +3. Waits up to 5 seconds for a fresh report. + +Supervisors that do not advertise `supports_workload_activity_requests` are +never idle-stopped. Under the lifecycle lock, each `Stopping` write attempt then +runs a final check. It requires: + +- a confirmation at most 2 seconds old; +- the re-read setting; +- a still-due live report from the confirming session; +- a fresh owner record that still names that session. + +The final check bounds every wait by the confirmation's deadline and holds that +sandbox's activity gate through the write. The deadline is checked again just +before the write is issued, and an attempt that passes it is abandoned. Every +live report update takes the same gate. A report that takes the gate before +the final check counts. One that reaches the handler while the stop holds the +gate waits behind that write. A conflicting write releases the gate and checks +again. The confirmation, with its session and reply arrival time, travels with +that one stop attempt. + +A report is recorded only while its session is still current, checked under the +gate, so a superseded session's late report never displaces its replacement's. + +Live reports carry the supervisor's summary sequence, and an older summary from +the same session is dropped. One background writer per sandbox persists the +newest report's display condition, coalescing bursts, so the session never +waits on the sandbox lock to read its next message. Each write attempt rereads +the newest report and requires a fresh owner record naming its session, so a +former owner cannot overwrite the current owner's status after a handoff. + +The stop records the `internal.openshell.ai/idle-stop` annotation with +`IdleStopping`. That marker survives gateway recovery and settles the stop as +`IdleStopped`. Any other lifecycle write removes it. An explicit stop, whether +the idle stop is settled or still in flight on another replica, rewrites it as +an ordinary stop. + ## Static credential delivery Existing polling and configuration-push environments keep their original wire diff --git a/crates/openshell-server/src/compute/idle_stop.rs b/crates/openshell-server/src/compute/idle_stop.rs new file mode 100644 index 0000000000..770cec519f --- /dev/null +++ b/crates/openshell-server/src/compute/idle_stop.rs @@ -0,0 +1,479 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Idle stop: stop a sandbox whose terminal program has waited on the user +//! longer than its `idle_stop_after_seconds` setting. +//! +//! The supervisor reports OSC 7501 program status with relative ages. The +//! replica that holds a sandbox's supervisor session keeps the latest report +//! of that session in memory, with exact input time, and decides idleness +//! from it alone. The stored `ProgramStatus` condition is for display: it is +//! written only on state changes or coarse input progress, can lag, and can +//! outlive the session that reported it, so it never authorizes a stop. A +//! sandbox with no report on its current session is never idle. + +use std::time::{Duration, SystemTime}; + +use openshell_core::metadata::PROGRAM_STATUS_CONDITION; +use openshell_core::proto::{ + Sandbox, SandboxCondition, SandboxPhase, SupervisorWorkloadActivity, WorkloadBlockedKind, + WorkloadProgramState, +}; + +pub use openshell_core::metadata::IDLE_STOPPED_REASON; + +/// How often each replica looks for idle sandboxes among its sessions. +pub const SWEEP_INTERVAL: Duration = Duration::from_secs(30); +/// Longest delay before the supervisor reports new input: one check interval +/// plus the minimum spacing of input reports. The sweep waits this much past +/// the timeout before asking the supervisor to confirm, which avoids +/// confirmation requests that pushed input would have answered anyway. +pub const INPUT_REPORT_LAG: Duration = Duration::from_secs(15); +/// A confirmation authorizes the `Stopping` write only this long after its +/// reply arrived. A stop that cannot commit by then, such as one waiting on +/// the lifecycle lock, is abandoned and confirmed again on a later sweep. +pub const CONFIRM_FRESHNESS: Duration = Duration::from_secs(2); +/// How long an idle stop waits for the supervisor to confirm. No answer, as +/// from a replaced or dead session, cancels the stop. +#[cfg(not(test))] +pub const CONFIRM_TIMEOUT: Duration = Duration::from_secs(5); +#[cfg(test)] +pub const CONFIRM_TIMEOUT: Duration = Duration::from_millis(200); +/// Input progress smaller than this does not rewrite the displayed condition. +pub const INPUT_WRITE_GRANULARITY: Duration = Duration::from_mins(1); +/// Durable marker that the in-flight stop was started by the idle sweep. It +/// survives gateway restarts and driver snapshots that rewrite conditions. +pub const IDLE_STOP_ANNOTATION: &str = "internal.openshell.ai/idle-stop"; +pub use openshell_core::metadata::IDLE_STOPPING_REASON; +pub const IDLE_STOPPING_MESSAGE: &str = + "Sandbox stop requested after its program waited on the user past its idle timeout"; +pub const IDLE_STOPPED_MESSAGE: &str = "Sandbox stopped after its program waited on the user past its idle timeout; connect to start it"; +/// Reported app names are bounded by OSC 7501; bound them again here. +const MAX_APP: usize = 32; + +fn timestamp(time: SystemTime) -> Option { + openshell_core::time::timestamp_from_system_time(time).ok() +} + +fn system_time(timestamp: Option<&prost_types::Timestamp>) -> Option { + openshell_core::time::system_time_from_timestamp(timestamp?).ok() +} + +fn before(now: SystemTime, age: Option<&prost_types::Duration>) -> Option { + let age = openshell_core::time::duration_to_std(age?).ok()?; + now.checked_sub(age) +} + +fn state_reason(state: WorkloadProgramState) -> Option<&'static str> { + Some(match state { + WorkloadProgramState::Unspecified => return None, + WorkloadProgramState::Idle => "Idle", + WorkloadProgramState::Working => "Working", + WorkloadProgramState::Done => "Done", + WorkloadProgramState::Blocked => "Blocked", + WorkloadProgramState::Error => "Error", + }) +} + +fn condition_message(report: &SupervisorWorkloadActivity, reason: &str) -> String { + let app: String = report + .app + .chars() + .filter(|c| c.is_ascii_alphanumeric() || "_.+-".contains(*c)) + .take(MAX_APP) + .collect(); + let program = if app.is_empty() { + "program".to_string() + } else { + app + }; + let waits_for = match WorkloadBlockedKind::try_from(report.blocked_kind) { + Ok(WorkloadBlockedKind::Permission) => " on permission", + Ok(WorkloadBlockedKind::Question) => " on a question", + Ok(WorkloadBlockedKind::Auth) => " on authentication", + _ => "", + }; + format!("{program} is {}{waits_for}", reason.to_ascii_lowercase()) +} + +/// The latest report of a supervisor session, in gateway time. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct LiveActivity { + pub session_id: String, + pub instance_id: String, + /// The session's supervisor answers activity requests, so it can confirm + /// idleness before a stop. Sessions that cannot are never idle-stopped. + pub confirms: bool, + pub state: WorkloadProgramState, + /// The later of the state change and the latest client input. + pub since: SystemTime, + /// The supervisor's summary sequence; older summaries are dropped. + pub sequence: u64, + /// The report itself and when it arrived, for the display condition. + pub report: SupervisorWorkloadActivity, + pub received: SystemTime, +} + +impl LiveActivity { + /// Whether `report` from `session_id` is older than this one. A new + /// session always replaces the entry. + #[must_use] + pub fn is_newer_than(&self, session_id: &str, report: &SupervisorWorkloadActivity) -> bool { + self.session_id == session_id && report.sequence != 0 && report.sequence < self.sequence + } +} + +/// Live activity for a report received at `now` on `session_id`. +pub fn live_from_report( + report: &SupervisorWorkloadActivity, + session_id: &str, + instance_id: &str, + confirms: bool, + now: SystemTime, +) -> LiveActivity { + let state_since = before(now, report.state_age.as_ref()).unwrap_or(now); + let since = + before(now, report.input_age.as_ref()).map_or(state_since, |input| input.max(state_since)); + LiveActivity { + session_id: session_id.to_string(), + instance_id: instance_id.to_string(), + confirms, + state: WorkloadProgramState::try_from(report.state) + .unwrap_or(WorkloadProgramState::Unspecified), + since, + sequence: report.sequence, + report: report.clone(), + received: now, + } +} + +/// The `ProgramStatus` condition for a supervisor report received at `now`, +/// or `None` when no program currently reports. +pub fn condition_from_report( + report: &SupervisorWorkloadActivity, + now: SystemTime, +) -> Option { + let state = WorkloadProgramState::try_from(report.state).ok()?; + let reason = state_reason(state)?; + let state_since = before(now, report.state_age.as_ref()).unwrap_or(now); + let since = + before(now, report.input_age.as_ref()).map_or(state_since, |input| input.max(state_since)); + Some(SandboxCondition { + r#type: PROGRAM_STATUS_CONDITION.to_string(), + status: "True".to_string(), + reason: reason.to_string(), + message: condition_message(report, reason), + transition_time: timestamp(since), + }) +} + +/// The stored program condition of a sandbox. +pub fn program_condition(sandbox: &Sandbox) -> Option<&SandboxCondition> { + sandbox + .status + .as_ref()? + .conditions + .iter() + .find(|condition| condition.r#type == PROGRAM_STATUS_CONDITION) +} + +/// How to update the stored program condition. +#[derive(Debug, PartialEq, Eq)] +pub enum ConditionWrite { + /// The stored condition is close enough. + Keep, + Set(SandboxCondition), + Remove, +} + +/// How to update the stored condition for the next report. +/// +/// A repeated state keeps its stored time unless input moved it forward by +/// the write granularity, so frequent reports do not churn the record and a +/// reconnecting supervisor cannot postpone an idle stop. +pub fn merge_condition( + stored: Option<&SandboxCondition>, + next: Option, +) -> ConditionWrite { + match (stored, next) { + (None, None) => ConditionWrite::Keep, + (Some(_), None) => ConditionWrite::Remove, + (None, Some(next)) => ConditionWrite::Set(next), + (Some(stored), Some(next)) => { + if stored.reason != next.reason || stored.message != next.message { + return ConditionWrite::Set(next); + } + let advanced = match ( + system_time(stored.transition_time.as_ref()), + system_time(next.transition_time.as_ref()), + ) { + (Some(stored), Some(next)) => next + .duration_since(stored) + .is_ok_and(|advance| advance >= INPUT_WRITE_GRANULARITY), + (None, Some(_)) => true, + (_, None) => false, + }; + if advanced { + ConditionWrite::Set(next) + } else { + ConditionWrite::Keep + } + } + } +} + +/// The effective idle timeout for a raw setting value, or `None` when idle +/// stop is disabled. +pub fn timeout_from_setting(seconds: Option) -> Option { + let seconds = seconds.filter(|seconds| *seconds > 0)?; + let seconds = seconds.min(openshell_core::settings::MAX_IDLE_STOP_AFTER_SECONDS); + Some(Duration::from_secs(u64::try_from(seconds).ok()?)) +} + +/// When the sandbox began waiting on the user, or `None` while it is not. +/// +/// The program waits while the current session last reported idle, done, +/// blocked, or error. The later of that report's time (state change or client +/// input) and the main-process start marks the beginning. A report from +/// another main-process instance does not count. +pub fn idle_since(sandbox: &Sandbox, live: &LiveActivity) -> Option { + let status = sandbox.status.as_ref()?; + if status.phase != SandboxPhase::Ready as i32 + || (!status.main_process_instance_id.is_empty() + && status.main_process_instance_id != live.instance_id) + { + return None; + } + if !matches!( + live.state, + WorkloadProgramState::Idle + | WorkloadProgramState::Done + | WorkloadProgramState::Blocked + | WorkloadProgramState::Error + ) { + return None; + } + let started = system_time(status.main_process_started_time.as_ref()); + Some(started.map_or(live.since, |started| started.max(live.since))) +} + +/// How long the sandbox has waited, when that reaches `timeout` plus the +/// input report lag. +pub fn idle_stop_due( + sandbox: &Sandbox, + live: &LiveActivity, + timeout: Duration, + now: SystemTime, +) -> Option { + let idle = now.duration_since(idle_since(sandbox, live)?).ok()?; + (idle >= timeout + INPUT_REPORT_LAG).then_some(idle) +} + +/// The Ready condition of an idle-stopped sandbox. Driver snapshots of the +/// stopped runtime rewrite conditions, so the gateway reapplies it. +pub fn idle_stopped_condition() -> SandboxCondition { + SandboxCondition { + r#type: "Ready".to_string(), + status: "False".to_string(), + reason: IDLE_STOPPED_REASON.to_string(), + message: IDLE_STOPPED_MESSAGE.to_string(), + transition_time: None, + } +} + +/// The Ready condition of a sandbox whose idle stop is in flight, reapplied +/// over driver snapshots so a connect can tell it apart from other stops. +pub fn idle_stopping_condition() -> SandboxCondition { + SandboxCondition { + r#type: "Ready".to_string(), + status: "False".to_string(), + reason: IDLE_STOPPING_REASON.to_string(), + message: IDLE_STOPPING_MESSAGE.to_string(), + transition_time: None, + } +} + +/// Whether the stop in flight was started by the idle sweep. +pub fn idle_stop_marked(sandbox: &Sandbox) -> bool { + sandbox + .metadata + .as_ref() + .is_some_and(|metadata| metadata.annotations.contains_key(IDLE_STOP_ANNOTATION)) +} + +#[cfg(test)] +mod tests { + use super::*; + use openshell_core::proto::SandboxStatus; + + fn secs(seconds: u64) -> Duration { + Duration::from_secs(seconds) + } + + fn proto_secs(seconds: i64) -> prost_types::Duration { + prost_types::Duration { seconds, nanos: 0 } + } + + fn at(seconds: u64) -> SystemTime { + SystemTime::UNIX_EPOCH + secs(1_000_000 + seconds) + } + + fn report( + state: WorkloadProgramState, + state_age: i64, + input_age: Option, + ) -> SupervisorWorkloadActivity { + SupervisorWorkloadActivity { + state: state.into(), + blocked_kind: WorkloadBlockedKind::Unspecified.into(), + app: "claude-code".into(), + state_age: Some(proto_secs(state_age)), + input_age: input_age.map(proto_secs), + reporting_terminals: 1, + request_id: String::new(), + sequence: 0, + } + } + + fn condition(state: WorkloadProgramState, received: u64) -> SandboxCondition { + condition_from_report(&report(state, 0, None), at(received)).unwrap() + } + + fn ready_sandbox(started: u64) -> Sandbox { + Sandbox { + status: Some(SandboxStatus { + phase: SandboxPhase::Ready.into(), + main_process_instance_id: "instance-1".into(), + main_process_started_time: timestamp(at(started)), + ..Default::default() + }), + ..Default::default() + } + } + + fn live(state: WorkloadProgramState, received: u64, input_age: Option) -> LiveActivity { + live_from_report( + &report(state, 0, input_age), + "session-1", + "instance-1", + true, + at(received), + ) + } + + #[test] + fn report_becomes_condition_at_the_later_of_state_and_input() { + let done = + condition_from_report(&report(WorkloadProgramState::Done, 30, Some(90)), at(100)) + .unwrap(); + assert_eq!(done.reason, "Done"); + assert_eq!(done.message, "claude-code is done"); + assert_eq!(system_time(done.transition_time.as_ref()), Some(at(70))); + let typed = + condition_from_report(&report(WorkloadProgramState::Done, 30, Some(5)), at(100)) + .unwrap(); + assert_eq!(system_time(typed.transition_time.as_ref()), Some(at(95))); + + let mut blocked = report(WorkloadProgramState::Blocked, 0, None); + blocked.blocked_kind = WorkloadBlockedKind::Permission.into(); + blocked.app = "bad app\u{1b}".into(); + let blocked = condition_from_report(&blocked, at(0)).unwrap(); + assert_eq!(blocked.message, "badapp is blocked on permission"); + + assert!( + condition_from_report(&report(WorkloadProgramState::Unspecified, 0, None), at(0)) + .is_none() + ); + } + + #[test] + fn merge_writes_changes_and_coarse_progress_only() { + let stored = condition(WorkloadProgramState::Done, 0); + assert_eq!( + merge_condition( + Some(&stored), + Some(condition(WorkloadProgramState::Done, 20)) + ), + ConditionWrite::Keep + ); + let later = condition(WorkloadProgramState::Done, 90); + assert_eq!( + merge_condition(Some(&stored), Some(later.clone())), + ConditionWrite::Set(later) + ); + let working = condition(WorkloadProgramState::Working, 5); + assert_eq!( + merge_condition(Some(&stored), Some(working.clone())), + ConditionWrite::Set(working) + ); + assert_eq!(merge_condition(Some(&stored), None), ConditionWrite::Remove); + assert_eq!(merge_condition(None, None), ConditionWrite::Keep); + } + + #[test] + fn setting_values_map_to_timeouts() { + assert_eq!(timeout_from_setting(None), None); + assert_eq!(timeout_from_setting(Some(0)), None); + assert_eq!(timeout_from_setting(Some(-5)), None); + assert_eq!(timeout_from_setting(Some(600)), Some(secs(600))); + assert_eq!( + timeout_from_setting(Some(i64::MAX)), + Some(secs(30 * 24 * 60 * 60)) + ); + } + + #[test] + fn waiting_programs_become_due_after_the_timeout_and_report_lag() { + let sandbox = ready_sandbox(0); + let done = live(WorkloadProgramState::Done, 100, None); + assert_eq!(idle_since(&sandbox, &done), Some(at(100))); + let lag = INPUT_REPORT_LAG.as_secs(); + assert_eq!( + idle_stop_due(&sandbox, &done, secs(600), at(699 + lag)), + None + ); + assert_eq!( + idle_stop_due(&sandbox, &done, secs(600), at(700 + lag)), + Some(secs(600 + lag)) + ); + // A later main-process start restarts the wait. + assert_eq!(idle_since(&ready_sandbox(500), &done), Some(at(500))); + } + + #[test] + fn input_moves_the_wait_forward_exactly() { + // Blocked at 0; input at 45 reported at 50. The timeout counts from 45. + let sandbox = ready_sandbox(0); + let blocked = live_from_report( + &report(WorkloadProgramState::Blocked, 50, Some(5)), + "session-1", + "instance-1", + true, + at(50), + ); + assert_eq!(idle_since(&sandbox, &blocked), Some(at(45))); + assert_eq!(idle_stop_due(&sandbox, &blocked, secs(60), at(60)), None); + } + + #[test] + fn working_other_instances_and_non_ready_sandboxes_are_never_idle() { + let sandbox = ready_sandbox(0); + assert_eq!( + idle_since(&sandbox, &live(WorkloadProgramState::Working, 0, None)), + None + ); + assert_eq!( + idle_since(&sandbox, &live(WorkloadProgramState::Unspecified, 0, None)), + None + ); + let mut stale = live(WorkloadProgramState::Done, 0, None); + stale.instance_id = "instance-old".into(); + assert_eq!(idle_since(&sandbox, &stale), None); + let mut stopping = ready_sandbox(0); + stopping.status.as_mut().unwrap().phase = SandboxPhase::Stopping.into(); + assert_eq!( + idle_since(&stopping, &live(WorkloadProgramState::Done, 0, None)), + None + ); + } +} diff --git a/crates/openshell-server/src/compute/mod.rs b/crates/openshell-server/src/compute/mod.rs index 32641d0937..76fc63cd0c 100644 --- a/crates/openshell-server/src/compute/mod.rs +++ b/crates/openshell-server/src/compute/mod.rs @@ -4,6 +4,7 @@ //! Gateway-owned compute orchestration over a pluggable compute backend. pub mod driver_config; +mod idle_stop; mod image_update; pub mod lease; mod mutation_guard; @@ -59,10 +60,10 @@ use std::future::Future; use std::path::{Path, PathBuf}; use std::pin::Pin; use std::sync::{Arc, Mutex as StdMutex, OnceLock, Weak}; -use std::time::{Duration, Instant}; +use std::time::{Duration, Instant, SystemTime}; #[cfg(unix)] use tokio::net::UnixStream; -use tokio::sync::{Mutex, Notify, watch}; +use tokio::sync::{Mutex, Notify, oneshot, watch}; use tonic::transport::Channel; #[cfg(unix)] use tonic::transport::Endpoint; @@ -389,6 +390,56 @@ enum QuiesceOperation { Suspend, } +/// Who asked for a quiesce operation. +#[derive(Debug, Clone, PartialEq, Eq)] +enum QuiesceCause { + /// A client request or gateway recovery. + Requested, + /// The idle sweep found the sandbox's program waiting past its timeout, + /// and its supervisor confirmed that for this stop attempt. + Idle(IdleConfirmation), +} + +/// A supervisor's confirmation that its sandbox is idle, owned by the one +/// stop attempt that requested it. +#[derive(Debug, Clone, PartialEq, Eq)] +struct IdleConfirmation { + /// The session that answered. + session_id: String, + /// When the reply reached the report handler. + arrived: Instant, +} + +/// Marks one sandbox's idle stop as in flight until dropped. +struct IdleStopInFlight { + set: Arc>>, + sandbox_id: String, +} + +impl IdleStopInFlight { + fn claim( + set: &Arc>>, + sandbox_id: &str, + ) -> Option { + set.lock() + .expect("idle stop lock poisoned") + .insert(sandbox_id.to_string()) + .then(|| Self { + set: Arc::clone(set), + sandbox_id: sandbox_id.to_string(), + }) + } +} + +impl Drop for IdleStopInFlight { + fn drop(&mut self) { + self.set + .lock() + .expect("idle stop lock poisoned") + .remove(&self.sandbox_id); + } +} + /// What an explicit quiesce request does with the current durable phase. enum QuiesceAdmission { /// The requested state is already durable. @@ -421,14 +472,18 @@ impl QuiesceOperation { } } - const fn requested_condition(self) -> (&'static str, &'static str) { - match self { - Self::Stop => ("Stopping", "Sandbox stop requested"), - Self::Suspend => ("Suspending", "Sandbox suspend requested"), + const fn requested_condition(self, cause: &QuiesceCause) -> (&'static str, &'static str) { + match (self, cause) { + (Self::Stop, QuiesceCause::Idle(_)) => ( + idle_stop::IDLE_STOPPING_REASON, + idle_stop::IDLE_STOPPING_MESSAGE, + ), + (Self::Stop, QuiesceCause::Requested) => ("Stopping", "Sandbox stop requested"), + (Self::Suspend, _) => ("Suspending", "Sandbox suspend requested"), } } - const fn settled_condition(self) -> (&'static str, &'static str) { + const fn settled_condition_explicit(self) -> (&'static str, &'static str) { match self { Self::Stop => ("Stopped", "Sandbox compute is stopped"), Self::Suspend => ( @@ -438,6 +493,22 @@ impl QuiesceOperation { } } + /// The settled condition for `sandbox`, whose durable marker records + /// whether the idle sweep started the stop. + fn settled_condition(self, sandbox: &Sandbox) -> (&'static str, &'static str) { + match self { + Self::Stop if idle_stop::idle_stop_marked(sandbox) => ( + idle_stop::IDLE_STOPPED_REASON, + idle_stop::IDLE_STOPPED_MESSAGE, + ), + Self::Stop => self.settled_condition_explicit(), + Self::Suspend => ( + "Suspended", + "Sandbox is suspended; its memory state is retained for resume", + ), + } + } + fn admit(self, sandbox: &Sandbox) -> Result { let phase = SandboxPhase::try_from(sandbox.phase()).unwrap_or(SandboxPhase::Unknown); match self { @@ -938,6 +1009,22 @@ pub struct ComputeRuntime { default_image: String, image_preparation_timeout_seconds: u32, provisioning_repair_window_ms: i64, + /// Latest program status per sandbox from this replica's supervisor + /// sessions. Only these reports authorize an idle stop. + live_activity: Arc>>, + /// Per-sandbox gates that serialize live report updates with the final + /// idle check and the `Stopping` write, so a report received before the + /// write is never overtaken by a stop decided from an older one. + activity_gates: Arc>>>>, + /// Sandboxes with an idle stop in flight on this replica. A later sweep + /// skips them instead of starting a second confirmation and stop. + idle_stops_in_flight: Arc>>, + /// Sandboxes with a display writer running, and whether a newer report + /// arrived since it started. One writer per sandbox persists the newest. + display_writers: Arc>>, + /// Idle stops waiting for their supervisor to confirm, by request ID. The + /// reply's arrival time at the report handler is sent to the waiter. + activity_requests: Arc>>>, sandbox_updates_enabled: bool, update_recovery_sweep: Arc, update_recovery_backoff: Arc>>, @@ -1061,6 +1148,11 @@ impl ComputeRuntime { default_image, image_preparation_timeout_seconds: 1800, provisioning_repair_window_ms: 300_000, + live_activity: Arc::default(), + activity_gates: Arc::default(), + idle_stops_in_flight: Arc::default(), + display_writers: Arc::default(), + activity_requests: Arc::default(), sandbox_updates_enabled: false, update_recovery_sweep: Arc::new(tokio::sync::Semaphore::new(1)), update_recovery_backoff: Arc::default(), @@ -2079,8 +2171,13 @@ impl ComputeRuntime { workspace: &str, name: &str, ) -> Result { - self.quiesce_sandbox(QuiesceOperation::Stop, workspace, name) - .await + self.quiesce_sandbox( + QuiesceOperation::Stop, + &QuiesceCause::Requested, + workspace, + name, + ) + .await } /// Whether the configured compute driver retains sandbox memory state @@ -2106,13 +2203,19 @@ impl ComputeRuntime { name: &str, ) -> Result { self.require_suspend_resume()?; - self.quiesce_sandbox(QuiesceOperation::Suspend, workspace, name) - .await + self.quiesce_sandbox( + QuiesceOperation::Suspend, + &QuiesceCause::Requested, + workspace, + name, + ) + .await } async fn quiesce_sandbox( &self, operation: QuiesceOperation, + cause: &QuiesceCause, workspace: &str, name: &str, ) -> Result { @@ -2144,6 +2247,44 @@ impl ComputeRuntime { provisioning_operation::ensure_operation_settled(¤t)?; match operation.admit(¤t)? { QuiesceAdmission::Settled => { + // An explicit stop overrides an idle stop, so connecting + // no longer starts the sandbox. + if operation == QuiesceOperation::Stop + && *cause == QuiesceCause::Requested + && idle_stop::idle_stop_marked(¤t) + { + let (reason, message) = QuiesceOperation::Stop.settled_condition_explicit(); + match self + .write_lifecycle_phase(¤t, SandboxPhase::Stopped, reason, message) + .await + { + Ok(stopped) => { + self.sandbox_index.update_from_sandbox(&stopped); + self.sandbox_watch_bus.notify(&sandbox_id); + current = stopped; + } + Err(crate::persistence::PersistenceError::Conflict { .. }) + if attempt < STOP_PHASE_CAS_RETRY_LIMIT => + { + attempt += 1; + current = self + .store + .get_message::(&sandbox_id) + .await + .map_err(|e| { + Status::internal(format!("fetch sandbox failed: {e}")) + })? + .ok_or_else(|| Status::not_found("sandbox not found"))?; + continue; + } + Err(error) => { + return Err(crate::grpc::persistence_error_to_status( + error, + "update sandbox lifecycle", + )); + } + } + } self.cleanup_stopped_sandbox_sessions(¤t) .await .map_err(Status::internal)?; @@ -2152,9 +2293,65 @@ impl ComputeRuntime { // Acquiring the lifecycle gate proves that no local worker // still owns this transition. Retry the idempotent driver // operation. - QuiesceAdmission::Retry => break (current.clone(), current), + QuiesceAdmission::Retry => { + // An explicit stop replaces an idle stop still in flight on + // another replica, so it cannot settle as `IdleStopped`. + if operation == QuiesceOperation::Stop + && *cause == QuiesceCause::Requested + && idle_stop::idle_stop_marked(¤t) + { + let (reason, message) = operation.requested_condition(cause); + match self + .write_lifecycle_phase( + ¤t, + SandboxPhase::Stopping, + reason, + message, + ) + .await + { + Ok(explicit) => { + self.sandbox_index.update_from_sandbox(&explicit); + self.sandbox_watch_bus.notify(&sandbox_id); + current = explicit; + } + Err(crate::persistence::PersistenceError::Conflict { .. }) + if attempt < STOP_PHASE_CAS_RETRY_LIMIT => + { + attempt += 1; + current = self + .store + .get_message::(&sandbox_id) + .await + .map_err(|e| { + Status::internal(format!("fetch sandbox failed: {e}")) + })? + .ok_or_else(|| Status::not_found("sandbox not found"))?; + continue; + } + Err(error) => { + return Err(crate::grpc::persistence_error_to_status( + error, + "update sandbox lifecycle", + )); + } + } + } + break (current.clone(), current); + } QuiesceAdmission::Begin => {} } + // The final idle check holds this sandbox's activity gate through + // this attempt's `Stopping` write, so a report received before + // the write counts. A retry releases it and checks again. + let idle_admission = if let QuiesceCause::Idle(confirmation) = cause { + match self.final_idle_check(¤t, confirmation).await { + Some(admission) => Some(admission), + None => return Err(Status::aborted("sandbox is no longer idle")), + } + } else { + None + }; let reclaimed_timeout = operation == QuiesceOperation::Stop && current.phase() == SandboxPhase::Error as i32 @@ -2166,9 +2363,17 @@ impl ComputeRuntime { "Timed-out compute has been reclaimed; sandbox stopped for repair", ) } else { - let (reason, message) = operation.requested_condition(); + let (reason, message) = operation.requested_condition(cause); (operation.in_flight_phase(), reason, message) }; + // The confirmation authorizes only a write issued before its + // deadline; an attempt that waited past it is abandoned. + if idle_admission + .as_ref() + .is_some_and(|(_, deadline)| Instant::now() >= *deadline) + { + return Err(Status::aborted("idle confirmation expired before the stop")); + } match self .write_lifecycle_phase(¤t, next_phase, reason, message) .await @@ -3227,7 +3432,7 @@ impl ComputeRuntime { operation.verb() ))); } - let (reason, message) = operation.settled_condition(); + let (reason, message) = operation.settled_condition(&latest); match self .write_lifecycle_phase(&latest, operation.settled_phase(), reason, message) .await @@ -4075,6 +4280,16 @@ impl ComputeRuntime { ) { let _ = self.restart_authority.set(authority); let runtime = Arc::new(self.clone()); + // Every replica decides idleness for the supervisor sessions it holds. + let idle_runtime = runtime.clone(); + let idle_shutdown = shutdown_rx.clone(); + let idle_startup = startup_rx.clone(); + tokio::spawn(async move { + if !wait_for_startup(idle_startup, idle_shutdown.clone()).await { + return; + } + idle_runtime.idle_stop_loop(idle_shutdown).await; + }); if self.store.is_single_replica() { let deadline_runtime = runtime.clone(); let deadline_shutdown = shutdown_rx.clone(); @@ -5064,287 +5279,768 @@ impl ComputeRuntime { Ok(()) } - async fn restart_sandbox_runtime(&self, sandbox_id: &str) -> Result<(), String> { - use openshell_core::proto::compute::v1::StartSandboxResponse; + async fn idle_stop_loop(self: Arc, mut cancel: watch::Receiver) { + loop { + if let Err(err) = Arc::clone(&self).stop_idle_sandboxes().await { + warn!(error = %err, "Sandbox idle sweep failed"); + } + tokio::select! { + () = tokio::time::sleep(idle_stop::SWEEP_INTERVAL) => {} + _ = cancel.changed() => return, + } + } + } - enum RestartOutcome { - Started(StartSandboxResponse), - Superseded, - DriverError(&'static str, Status), + /// The live report for `sandbox_id`, if its session is still current. + /// Reports of ended sessions are dropped. + fn current_live_activity(&self, sandbox_id: &str) -> Option { + let mut live = self + .live_activity + .lock() + .expect("live activity lock poisoned"); + let entry = live.get(sandbox_id)?.clone(); + if self + .supervisor_sessions + .is_current_session(sandbox_id, &entry.session_id) + { + Some(entry) + } else { + live.remove(sandbox_id); + drop(live); + self.activity_gates + .lock() + .expect("activity gate lock poisoned") + .remove(sandbox_id); + None } + } - let lifecycle_guard = self.lifecycle_gates.lock_for(sandbox_id).await; - let sandbox_guard = self.lock_sandbox_for_lifecycle(&lifecycle_guard).await; - let Some(current) = self - .store - .get_message::(sandbox_id) + fn activity_gate(&self, sandbox_id: &str) -> Arc> { + Arc::clone( + self.activity_gates + .lock() + .expect("activity gate lock poisoned") + .entry(sandbox_id.to_string()) + .or_default(), + ) + } + + /// Whether the shared owner record is fresh and names `session_id`. + async fn owned_by_session(&self, sandbox_id: &str, session_id: &str) -> bool { + SupervisorOwnerIndex::new(self.store.clone(), OWNER_TTL) + .read(sandbox_id) .await - .map_err(|err| err.to_string())? + .ok() + .flatten() + .is_some_and(|owner| owner.is_fresh(OWNER_TTL) && owner.session_id == session_id) + } + + /// Confirm with the supervisor, then stop `sandbox` if it is still idle. + /// + /// The round trip runs before the stop takes any lifecycle lock, so the + /// session keeps reading messages meanwhile. Returns `None` when the + /// sandbox is not stopped. + async fn run_idle_stop(&self, sandbox: &Sandbox) -> Result, Status> { + let sandbox_id = sandbox.object_id(); + let Some(_in_flight) = IdleStopInFlight::claim(&self.idle_stops_in_flight, sandbox_id) else { - return Ok(()); + return Ok(None); }; - let phase = SandboxPhase::try_from(current.phase()).unwrap_or(SandboxPhase::Unknown); - let due = current - .status - .as_ref() - .is_some_and(|status| restart_is_due(status, openshell_core::time::now_ms())); - if phase != SandboxPhase::Starting - || !is_automatic_restart_transition(¤t) - || !due - || provisioning_deadline::driver_operation_pending(¤t) - || provisioning_deadline::timed_out(¤t) + let Some(live) = self.current_live_activity(sandbox_id) else { + return Ok(None); + }; + if !live.confirms + || !self.owned_by_session(sandbox_id, &live.session_id).await + || self.idle_stop_due_now(sandbox).await.is_none() { - // Main-process exit may schedule restart while CREATE is pending. - // Keep that schedule intact so settlement makes it eligible again. - return Ok(()); + return Ok(None); } - - let sandbox_name = current.object_name().to_string(); - self.validate_isolation_backend_binding(¤t) - .map_err(|error| error.to_string())?; - let already_claimed = current - .status - .as_ref() - .is_some_and(|status| next_restart_at_ms(status) == 0); - let next_identity = if already_claimed { - None - } else { - Some(next_runtime_identity(¤t).map_err(|status| status.to_string())?) + // The deadline starts when the reply reached the report handler, so + // scheduling delay before this task resumes counts against it. + let Some(arrived) = self.confirm_activity(sandbox_id, &live.session_id).await else { + return Ok(None); }; - let claimed = if already_claimed && sandbox_provisioning_attempt_id(¤t).is_none() { - current.clone() - } else { - // Claim the restart schedule and driver ownership in one CAS. A - // pending check followed by an unclaimed STOP races another replica. - match self - .store - .update_message_cas::( - sandbox_id, - sandbox_resource_version(¤t), - |sandbox| { - if let (Some(identity), Some(metadata)) = - (next_identity.as_ref(), sandbox.metadata.as_mut()) - { - identity.write(&mut metadata.annotations); - } - provisioning_operation::claim_record(sandbox); - let status = sandbox.status.get_or_insert_with(Default::default); - set_next_restart_at_ms(status, 0); - upsert_ready_condition( - &mut sandbox.status, - SandboxCondition { - r#type: "Ready".to_string(), - status: "False".to_string(), - reason: "SandboxRestarting".to_string(), - message: - "Replacing sandbox runtime after canonical main process exit" - .to_string(), - transition_time: None, - }, - ); - }, - ) - .await + let cause = QuiesceCause::Idle(IdleConfirmation { + session_id: live.session_id, + arrived, + }); + let result = self + .quiesce_sandbox( + QuiesceOperation::Stop, + &cause, + sandbox.object_workspace(), + sandbox.object_name(), + ) + .await; + match result { + Ok(stopped) => Ok(Some(stopped)), + Err(status) if status.code() == Code::Aborted => Ok(None), + Err(status) => Err(status), + } + } + + /// The last check before an idle `Stopping` write, run under the + /// lifecycle lock. Returns the sandbox's activity gate, which the caller + /// holds through the write, and the confirmation's deadline, which the + /// caller checks again just before issuing it. + /// + /// It requires a confirmation whose deadline has not passed, the + /// still-enabled setting, a live report from the confirming session that + /// is still due, and a fresh shared owner record naming that session. + /// Every wait here is bounded by the deadline. + async fn final_idle_check( + &self, + sandbox: &Sandbox, + confirmation: &IdleConfirmation, + ) -> Option<(tokio::sync::OwnedMutexGuard<()>, Instant)> { + let sandbox_id = sandbox.object_id(); + let session_id = &confirmation.session_id; + let deadline = confirmation.arrived + idle_stop::CONFIRM_FRESHNESS; + let check = async { + let timeout = self.idle_stop_timeout(sandbox).await?; + let gate = self.activity_gate(sandbox_id).lock_owned().await; + let live = self.current_live_activity(sandbox_id)?; + if live.session_id != *session_id + || idle_stop::idle_stop_due(sandbox, &live, timeout, SystemTime::now()).is_none() + || !self.owned_by_session(sandbox_id, session_id).await { - Ok(claimed) => claimed, - Err(crate::persistence::PersistenceError::Conflict { .. }) => return Ok(()), - Err(error) => return Err(error.to_string()), + return None; } + Some(gate) }; - self.sandbox_index.update_from_sandbox(&claimed); - self.sandbox_watch_bus.notify(sandbox_id); - drop(sandbox_guard); + let gate = tokio::time::timeout_at(deadline.into(), check) + .await + .ok() + .flatten()?; + (Instant::now() < deadline).then_some((gate, deadline)) + } - let runtime = self.clone(); - let owned = claimed.clone(); - let operation_id = sandbox_id.to_string(); - let operation_name = sandbox_name.clone(); - let expected_runtime_identity = sandbox_compute_runtime_identity(¤t); - // One detached worker owns STOP and START. Caller cancellation cannot - // strand a raw START, and the monitor may release the local gate while - // the durable pending flag still protects the unresolved operation. - let result = Box::pin( - self.await_claimed_provisioning_operation(&claimed, async move { - let request_id = operation_id.clone(); - let request_name = operation_name.clone(); - let stop = runtime - .driver - .call_owned( - openshell_otel::rpc::STOP_SANDBOX, - Some(&operation_id), - move |driver| async move { - driver - .stop_sandbox(Request::new(StopSandboxRequest { - sandbox_id: request_id, - name: request_name, - })) - .await + /// Ask the session's supervisor for a fresh report and wait for it. + /// Returns when the reply reached the report handler. + async fn confirm_activity(&self, sandbox_id: &str, session_id: &str) -> Option { + let request_id = uuid::Uuid::new_v4().to_string(); + let (tx, rx) = oneshot::channel(); + self.activity_requests + .lock() + .expect("activity request lock poisoned") + .insert(request_id.clone(), tx); + let sent = self.supervisor_sessions.send_if_current( + sandbox_id, + session_id, + openshell_core::proto::GatewayMessage { + payload: Some( + openshell_core::proto::gateway_message::Payload::WorkloadActivityRequest( + openshell_core::proto::WorkloadActivityRequest { + request_id: request_id.clone(), }, - ) - .await; - if let Err(status) = stop { - return Ok(RestartOutcome::DriverError("stop", status)); - } - runtime - .cleanup_stopped_sandbox_sessions(&owned) - .await - .map_err(Status::internal)?; - let Some(armed) = runtime - .arm_restart_readiness_watchdog(&owned) - .await - .map_err(Status::internal)? - else { - return Ok(RestartOutcome::Superseded); - }; - let authority = runtime.restart_authority.get().and_then(Option::as_deref); - let launch_authentication = runtime - .encode_persisted_launch_authentication(authority, &armed) - .await?; - let request = runtime.driver_start_request( - &armed, - launch_authentication, - expected_runtime_identity, - )?; - let start = - runtime - .driver - .call_owned( - openshell_otel::rpc::START_SANDBOX, - Some(&operation_id), - move |driver| async move { - driver.start_sandbox(Request::new(request)).await - }, - ) - .await; - Ok(match start { - Ok(response) => RestartOutcome::Started(response.into_inner()), - Err(status) => RestartOutcome::DriverError("start", status), - }) - }), + ), + ), + }, + ); + let answered = if sent { + tokio::time::timeout(idle_stop::CONFIRM_TIMEOUT, rx) + .await + .ok() + .and_then(Result::ok) + } else { + None + }; + self.activity_requests + .lock() + .expect("activity request lock poisoned") + .remove(&request_id); + answered + } + + /// The effective idle timeout for `sandbox`, or `None` when disabled. + async fn idle_stop_timeout(&self, sandbox: &Sandbox) -> Option { + let global = crate::grpc::policy::global_idle_stop_after_seconds(&self.store) + .await + .ok()?; + self.idle_stop_timeout_with_global(sandbox, global).await + } + + /// [`Self::idle_stop_timeout`] with a gateway setting a sweep already + /// loaded for many sandboxes. + async fn idle_stop_timeout_with_global( + &self, + sandbox: &Sandbox, + global: Option, + ) -> Option { + let setting = crate::grpc::policy::idle_stop_after_seconds( + &self.store, + global, + sandbox.object_workspace(), + sandbox.object_name(), ) .await - .map_err(|error| error.to_string())?; + .ok()?; + idle_stop::timeout_from_setting(setting) + } - let (outcome, settled) = result; - let response = match outcome { - RestartOutcome::Started(response) => response, - RestartOutcome::Superseded => return Ok(()), - RestartOutcome::DriverError(stage, status) => { - return self - .record_restart_driver_failure(&lifecycle_guard, &settled, stage, status) - .await; + /// How long `sandbox` has waited past its idle timeout, if it is due now. + async fn idle_stop_due_now(&self, sandbox: &Sandbox) -> Option { + let global = crate::grpc::policy::global_idle_stop_after_seconds(&self.store) + .await + .ok()?; + self.idle_stop_due_with_global(sandbox, global).await + } + + /// [`Self::idle_stop_due_now`] with a gateway setting a sweep already + /// loaded for many sandboxes. + async fn idle_stop_due_with_global( + &self, + sandbox: &Sandbox, + global: Option, + ) -> Option { + let live = self.current_live_activity(sandbox.object_id())?; + idle_stop::idle_since(sandbox, &live)?; + let timeout = self.idle_stop_timeout_with_global(sandbox, global).await?; + idle_stop::idle_stop_due(sandbox, &live, timeout, SystemTime::now()) + } + + /// Start an idle stop for every sandbox of this replica's supervisor + /// sessions whose program has waited on the user past its timeout. Each + /// stop runs on its own task so a slow driver does not delay the sweep. + async fn stop_idle_sandboxes(self: Arc) -> Result<(), String> { + let candidates: Vec = self + .live_activity + .lock() + .expect("live activity lock poisoned") + .keys() + .cloned() + .collect(); + if candidates.is_empty() { + return Ok(()); + } + let global = crate::grpc::policy::global_idle_stop_after_seconds(&self.store) + .await + .map_err(|status| status.message().to_string())?; + for sandbox_id in candidates { + if self.current_live_activity(&sandbox_id).is_none() { + continue; } - }; - if self.supports_sandbox_authentication() { - if response.runtime_identity.is_empty() { - return Err( - "compute driver returned an empty runtime identity after restart".into(), + let Ok(Some(sandbox)) = self.store.get_message::(&sandbox_id).await else { + continue; + }; + let Some(idle) = self.idle_stop_due_with_global(&sandbox, global).await else { + continue; + }; + let runtime = Arc::clone(&self); + tokio::spawn(async move { + let program = idle_stop::program_condition(&sandbox) + .map(|condition| condition.message.clone()) + .unwrap_or_default(); + match runtime.run_idle_stop(&sandbox).await { + Ok(Some(_)) => info!( + sandbox_id = %sandbox.object_id(), + sandbox_name = %sandbox.object_name(), + program = %program, + idle_seconds = idle.as_secs(), + "Stopped idle sandbox" + ), + Ok(None) => debug!( + sandbox_id = %sandbox.object_id(), + "Skipped idle stop; the supervisor did not confirm it" + ), + Err(status) => warn!( + sandbox_id = %sandbox.object_id(), + sandbox_name = %sandbox.object_name(), + error = %status.message(), + "Idle sandbox stop failed" + ), + } + }); + } + Ok(()) + } + + /// Record the program status a supervisor session reported. + /// + /// The live report, which alone authorizes an idle stop, is kept in + /// memory exactly; a summary older than the one held is dropped. The + /// displayed `ProgramStatus` condition is persisted on a background task, + /// so the session never waits on the sandbox lock to read its next + /// message. + pub async fn record_workload_activity( + &self, + sandbox_id: &str, + session_id: &str, + instance_id: &str, + confirms: bool, + report: &openshell_core::proto::SupervisorWorkloadActivity, + ) -> Result<(), String> { + self.record_workload_activity_at( + sandbox_id, + session_id, + instance_id, + confirms, + report, + Instant::now(), + ) + .await + } + + /// [`Self::record_workload_activity`] for a report that reached the + /// handler at `arrived`. + async fn record_workload_activity_at( + &self, + sandbox_id: &str, + session_id: &str, + instance_id: &str, + confirms: bool, + report: &openshell_core::proto::SupervisorWorkloadActivity, + arrived: Instant, + ) -> Result<(), String> { + let received = SystemTime::now(); + let (current, accepted) = { + let gate = self.activity_gate(sandbox_id); + let _gate = gate.lock().await; + let mut live = self + .live_activity + .lock() + .expect("live activity lock poisoned"); + // Re-check the session here: a report that passed the session + // loop's check before a replacement registered must not displace + // the replacement's report. + let current = self + .supervisor_sessions + .is_current_session(sandbox_id, session_id); + let accepted = current + && !live + .get(sandbox_id) + .is_some_and(|held| held.is_newer_than(session_id, report)); + if accepted { + live.insert( + sandbox_id.to_string(), + idle_stop::live_from_report( + report, + session_id, + instance_id, + confirms, + received, + ), ); } - self.persist_runtime_binding( - sandbox_id, - &settled, - self.configured_driver_name(), - &response.runtime_identity, - &[SandboxPhase::Starting, SandboxPhase::Ready], - ) - .await?; + (current, accepted) + }; + // A reply from a replaced session confirms nothing. An older reply + // from the current session still answers: the newer report it lost + // to is at least as fresh. + if current + && !report.request_id.is_empty() + && let Some(waiter) = self + .activity_requests + .lock() + .expect("activity request lock poisoned") + .remove(&report.request_id) + { + let _ = waiter.send(arrived); + } + if accepted { + self.schedule_display_write(sandbox_id); } - self.enforce_lifecycle_after_restart_start(&settled).await?; - info!( - sandbox_id, - sandbox_name, "Sandbox runtime restarted; waiting for replacement supervisor" - ); Ok(()) } - /// Arm the replacement readiness watchdog immediately before starting - /// compute. The zero deadline written by the restart claim fences other - /// gateway replicas without charging slow driver shutdown time against - /// the replacement supervisor's readiness window. - async fn arm_restart_readiness_watchdog( - &self, - claimed: &Sandbox, - ) -> Result, String> { - let sandbox_id = claimed.object_id().to_string(); - let mut current = claimed.clone(); - for _ in 0..DELETE_PHASE_CAS_RETRY_LIMIT { - let phase = SandboxPhase::try_from(current.phase()).unwrap_or(SandboxPhase::Unknown); - let claim_owned = phase == SandboxPhase::Starting - && provisioning_operation::same_operation(¤t, claimed) - && sandbox_runtime_generation(¤t) == sandbox_runtime_generation(claimed) - && !provisioning_deadline::timed_out(¤t) - && is_automatic_restart_transition(¤t) - && current - .status - .as_ref() - .is_some_and(|status| next_restart_at_ms(status) == 0); - if !claim_owned { - return Ok(None); + /// Persist the newest report's display condition, coalescing a burst of + /// reports behind at most one running writer per sandbox. + fn schedule_display_write(&self, sandbox_id: &str) { + let start = { + let mut writers = self + .display_writers + .lock() + .expect("display writer lock poisoned"); + match writers.entry(sandbox_id.to_string()) { + std::collections::hash_map::Entry::Occupied(mut running) => { + *running.get_mut() = true; + false + } + std::collections::hash_map::Entry::Vacant(idle) => { + idle.insert(false); + true + } + } + }; + if !start { + return; + } + let runtime = self.clone(); + let sandbox_id = sandbox_id.to_string(); + tokio::spawn(async move { + loop { + if let Err(error) = runtime.persist_program_condition(&sandbox_id).await { + debug!(sandbox_id = %sandbox_id, %error, "failed to persist program status"); + } + let mut writers = runtime + .display_writers + .lock() + .expect("display writer lock poisoned"); + match writers.get_mut(&sandbox_id) { + Some(dirty) if *dirty => *dirty = false, + _ => { + writers.remove(&sandbox_id); + return; + } + } } + }); + } - let readiness_deadline = - openshell_core::time::now_ms().saturating_add(RESTART_READINESS_TIMEOUT_MS); + /// Write the latest live report as the displayed `ProgramStatus` + /// condition, only on state changes or coarse input progress, retrying + /// concurrent writes, and only for the running main-process instance. + async fn persist_program_condition(&self, sandbox_id: &str) -> Result<(), String> { + let _guard = self.lock_sandbox_local(sandbox_id).await; + let mut attempt = 1; + loop { + // Read the versioned sandbox first and check ownership second. A + // takeover before the check fails it; a newer status write after + // the read fails the conditional write and the retry checks + // ownership again, so a former owner never overwrites it. + let Some(existing) = self + .store + .get_message::(sandbox_id) + .await + .map_err(|error| error.to_string())? + else { + return Ok(()); + }; + let Some(live) = self.current_live_activity(sandbox_id) else { + return Ok(()); + }; + if !self.owned_by_session(sandbox_id, &live.session_id).await { + return Ok(()); + } + let Some(status) = existing.status.as_ref() else { + return Ok(()); + }; + if status.phase != SandboxPhase::Ready as i32 + || (!status.main_process_instance_id.is_empty() + && status.main_process_instance_id != live.instance_id) + { + return Ok(()); + } + let next = idle_stop::condition_from_report(&live.report, live.received); + let next = + match idle_stop::merge_condition(idle_stop::program_condition(&existing), next) { + idle_stop::ConditionWrite::Keep => return Ok(()), + idle_stop::ConditionWrite::Set(condition) => Some(condition), + idle_stop::ConditionWrite::Remove => None, + }; match self .store .update_message_cas::( - &sandbox_id, - sandbox_resource_version(¤t), + sandbox_id, + sandbox_resource_version(&existing), |sandbox| { - let status = sandbox.status.get_or_insert_with(Default::default); - set_next_restart_at_ms(status, readiness_deadline); + if let Some(status) = sandbox.status.as_mut() { + status.conditions.retain(|condition| { + condition.r#type + != openshell_core::metadata::PROGRAM_STATUS_CONDITION + }); + status.conditions.extend(next.clone()); + } }, ) .await { - Ok(armed) => { - self.sandbox_index.update_from_sandbox(&armed); - self.sandbox_watch_bus.notify(&sandbox_id); - return Ok(Some(armed)); + Ok(sandbox) => { + self.sandbox_index.update_from_sandbox(&sandbox); + self.sandbox_watch_bus.notify(sandbox_id); + return Ok(()); } - Err(crate::persistence::PersistenceError::Conflict { .. }) => { - let Some(latest) = self - .store - .get_message::(&sandbox_id) - .await - .map_err(|err| err.to_string())? - else { - return Ok(None); - }; - current = latest; + Err(crate::persistence::PersistenceError::Conflict { .. }) + if attempt < STOP_PHASE_CAS_RETRY_LIMIT => + { + attempt += 1; } - Err(err) => return Err(err.to_string()), + Err(error) => return Err(error.to_string()), } } - - Ok(None) } - /// Reassert a concurrent stop/delete after the replacement start call. - /// The durable phase is authoritative across gateway replicas; the - /// compensating driver operation prevents a late start from leaving - /// running compute behind a stopped row or an orphan after deletion. - async fn enforce_lifecycle_after_restart_start(&self, armed: &Sandbox) -> Result<(), String> { - let sandbox_id = armed.object_id().to_string(); - let sandbox_name = armed.object_name().to_string(); - let latest = self + async fn restart_sandbox_runtime(&self, sandbox_id: &str) -> Result<(), String> { + use openshell_core::proto::compute::v1::StartSandboxResponse; + + enum RestartOutcome { + Started(StartSandboxResponse), + Superseded, + DriverError(&'static str, Status), + } + + let lifecycle_guard = self.lifecycle_gates.lock_for(sandbox_id).await; + let sandbox_guard = self.lock_sandbox_for_lifecycle(&lifecycle_guard).await; + let Some(current) = self .store - .get_message::(&sandbox_id) + .get_message::(sandbox_id) .await - .map_err(|err| err.to_string())?; - if sandbox_provisioning_attempt_id(armed).is_some() - && latest.as_ref().is_none_or(|current| { - !provisioning_operation::same_operation(current, armed) - || sandbox_runtime_generation(current) != sandbox_runtime_generation(armed) - || provisioning_deadline::timed_out(current) - }) + .map_err(|err| err.to_string())? + else { + return Ok(()); + }; + let phase = SandboxPhase::try_from(current.phase()).unwrap_or(SandboxPhase::Unknown); + let due = current + .status + .as_ref() + .is_some_and(|status| restart_is_due(status, openshell_core::time::now_ms())); + if phase != SandboxPhase::Starting + || !is_automatic_restart_transition(¤t) + || !due + || provisioning_deadline::driver_operation_pending(¤t) + || provisioning_deadline::timed_out(¤t) { - // Newer operations and timeout reclamation own their own cleanup. + // Main-process exit may schedule restart while CREATE is pending. + // Keep that schedule intact so settlement makes it eligible again. return Ok(()); } - let phase = latest.as_ref().map(|sandbox| { - SandboxPhase::try_from(sandbox.phase()).unwrap_or(SandboxPhase::Unknown) - }); + + let sandbox_name = current.object_name().to_string(); + self.validate_isolation_backend_binding(¤t) + .map_err(|error| error.to_string())?; + let already_claimed = current + .status + .as_ref() + .is_some_and(|status| next_restart_at_ms(status) == 0); + let next_identity = if already_claimed { + None + } else { + Some(next_runtime_identity(¤t).map_err(|status| status.to_string())?) + }; + let claimed = if already_claimed && sandbox_provisioning_attempt_id(¤t).is_none() { + current.clone() + } else { + // Claim the restart schedule and driver ownership in one CAS. A + // pending check followed by an unclaimed STOP races another replica. + match self + .store + .update_message_cas::( + sandbox_id, + sandbox_resource_version(¤t), + |sandbox| { + if let (Some(identity), Some(metadata)) = + (next_identity.as_ref(), sandbox.metadata.as_mut()) + { + identity.write(&mut metadata.annotations); + } + provisioning_operation::claim_record(sandbox); + let status = sandbox.status.get_or_insert_with(Default::default); + set_next_restart_at_ms(status, 0); + upsert_ready_condition( + &mut sandbox.status, + SandboxCondition { + r#type: "Ready".to_string(), + status: "False".to_string(), + reason: "SandboxRestarting".to_string(), + message: + "Replacing sandbox runtime after canonical main process exit" + .to_string(), + transition_time: None, + }, + ); + }, + ) + .await + { + Ok(claimed) => claimed, + Err(crate::persistence::PersistenceError::Conflict { .. }) => return Ok(()), + Err(error) => return Err(error.to_string()), + } + }; + self.sandbox_index.update_from_sandbox(&claimed); + self.sandbox_watch_bus.notify(sandbox_id); + drop(sandbox_guard); + + let runtime = self.clone(); + let owned = claimed.clone(); + let operation_id = sandbox_id.to_string(); + let operation_name = sandbox_name.clone(); + let expected_runtime_identity = sandbox_compute_runtime_identity(¤t); + // One detached worker owns STOP and START. Caller cancellation cannot + // strand a raw START, and the monitor may release the local gate while + // the durable pending flag still protects the unresolved operation. + let result = Box::pin( + self.await_claimed_provisioning_operation(&claimed, async move { + let request_id = operation_id.clone(); + let request_name = operation_name.clone(); + let stop = runtime + .driver + .call_owned( + openshell_otel::rpc::STOP_SANDBOX, + Some(&operation_id), + move |driver| async move { + driver + .stop_sandbox(Request::new(StopSandboxRequest { + sandbox_id: request_id, + name: request_name, + })) + .await + }, + ) + .await; + if let Err(status) = stop { + return Ok(RestartOutcome::DriverError("stop", status)); + } + runtime + .cleanup_stopped_sandbox_sessions(&owned) + .await + .map_err(Status::internal)?; + let Some(armed) = runtime + .arm_restart_readiness_watchdog(&owned) + .await + .map_err(Status::internal)? + else { + return Ok(RestartOutcome::Superseded); + }; + let authority = runtime.restart_authority.get().and_then(Option::as_deref); + let launch_authentication = runtime + .encode_persisted_launch_authentication(authority, &armed) + .await?; + let request = runtime.driver_start_request( + &armed, + launch_authentication, + expected_runtime_identity, + )?; + let start = + runtime + .driver + .call_owned( + openshell_otel::rpc::START_SANDBOX, + Some(&operation_id), + move |driver| async move { + driver.start_sandbox(Request::new(request)).await + }, + ) + .await; + Ok(match start { + Ok(response) => RestartOutcome::Started(response.into_inner()), + Err(status) => RestartOutcome::DriverError("start", status), + }) + }), + ) + .await + .map_err(|error| error.to_string())?; + + let (outcome, settled) = result; + let response = match outcome { + RestartOutcome::Started(response) => response, + RestartOutcome::Superseded => return Ok(()), + RestartOutcome::DriverError(stage, status) => { + return self + .record_restart_driver_failure(&lifecycle_guard, &settled, stage, status) + .await; + } + }; + if self.supports_sandbox_authentication() { + if response.runtime_identity.is_empty() { + return Err( + "compute driver returned an empty runtime identity after restart".into(), + ); + } + self.persist_runtime_binding( + sandbox_id, + &settled, + self.configured_driver_name(), + &response.runtime_identity, + &[SandboxPhase::Starting, SandboxPhase::Ready], + ) + .await?; + } + self.enforce_lifecycle_after_restart_start(&settled).await?; + info!( + sandbox_id, + sandbox_name, "Sandbox runtime restarted; waiting for replacement supervisor" + ); + Ok(()) + } + + /// Arm the replacement readiness watchdog immediately before starting + /// compute. The zero deadline written by the restart claim fences other + /// gateway replicas without charging slow driver shutdown time against + /// the replacement supervisor's readiness window. + async fn arm_restart_readiness_watchdog( + &self, + claimed: &Sandbox, + ) -> Result, String> { + let sandbox_id = claimed.object_id().to_string(); + let mut current = claimed.clone(); + for _ in 0..DELETE_PHASE_CAS_RETRY_LIMIT { + let phase = SandboxPhase::try_from(current.phase()).unwrap_or(SandboxPhase::Unknown); + let claim_owned = phase == SandboxPhase::Starting + && provisioning_operation::same_operation(¤t, claimed) + && sandbox_runtime_generation(¤t) == sandbox_runtime_generation(claimed) + && !provisioning_deadline::timed_out(¤t) + && is_automatic_restart_transition(¤t) + && current + .status + .as_ref() + .is_some_and(|status| next_restart_at_ms(status) == 0); + if !claim_owned { + return Ok(None); + } + + let readiness_deadline = + openshell_core::time::now_ms().saturating_add(RESTART_READINESS_TIMEOUT_MS); + match self + .store + .update_message_cas::( + &sandbox_id, + sandbox_resource_version(¤t), + |sandbox| { + let status = sandbox.status.get_or_insert_with(Default::default); + set_next_restart_at_ms(status, readiness_deadline); + }, + ) + .await + { + Ok(armed) => { + self.sandbox_index.update_from_sandbox(&armed); + self.sandbox_watch_bus.notify(&sandbox_id); + return Ok(Some(armed)); + } + Err(crate::persistence::PersistenceError::Conflict { .. }) => { + let Some(latest) = self + .store + .get_message::(&sandbox_id) + .await + .map_err(|err| err.to_string())? + else { + return Ok(None); + }; + current = latest; + } + Err(err) => return Err(err.to_string()), + } + } + + Ok(None) + } + + /// Reassert a concurrent stop/delete after the replacement start call. + /// The durable phase is authoritative across gateway replicas; the + /// compensating driver operation prevents a late start from leaving + /// running compute behind a stopped row or an orphan after deletion. + async fn enforce_lifecycle_after_restart_start(&self, armed: &Sandbox) -> Result<(), String> { + let sandbox_id = armed.object_id().to_string(); + let sandbox_name = armed.object_name().to_string(); + let latest = self + .store + .get_message::(&sandbox_id) + .await + .map_err(|err| err.to_string())?; + if sandbox_provisioning_attempt_id(armed).is_some() + && latest.as_ref().is_none_or(|current| { + !provisioning_operation::same_operation(current, armed) + || sandbox_runtime_generation(current) != sandbox_runtime_generation(armed) + || provisioning_deadline::timed_out(current) + }) + { + // Newer operations and timeout reclamation own their own cleanup. + return Ok(()); + } + let phase = latest.as_ref().map(|sandbox| { + SandboxPhase::try_from(sandbox.phase()).unwrap_or(SandboxPhase::Unknown) + }); match phase { None | Some(SandboxPhase::Deleting) => { @@ -7872,6 +8568,18 @@ fn apply_driver_snapshot( .clone_from(¤t_status.configuration_admission); status.configuration_activated = current_status.configuration_activated; status.provisioning.clone_from(¤t_status.provisioning); + // Program status is gateway-owned and describes the running program, + // which a phase change ends. + if old_phase == phase + && let Some(program) = current_status.conditions.iter().find(|condition| { + condition.r#type == openshell_core::metadata::PROGRAM_STATUS_CONDITION + }) + { + status.conditions.retain(|condition| { + condition.r#type != openshell_core::metadata::PROGRAM_STATUS_CONDITION + }); + status.conditions.push(program.clone()); + } } if old_phase != phase { info!( @@ -7902,6 +8610,15 @@ fn apply_driver_snapshot( } } + // A snapshot of the stopped runtime reports why its container exited; an + // idle stop keeps its own reason so a connect can start the sandbox. + if idle_stop::idle_stop_marked(sandbox) { + if phase == SandboxPhase::Stopped { + upsert_ready_condition(&mut status, idle_stop::idle_stopped_condition()); + } else if phase == SandboxPhase::Stopping { + upsert_ready_condition(&mut status, idle_stop::idle_stopping_condition()); + } + } if let Some(metadata) = sandbox.metadata.as_mut() { metadata.name.clone_from(sandbox_name); } @@ -8325,6 +9042,17 @@ fn apply_lifecycle_phase( // intent. A resuming start records a new one after this write. if let Some(metadata) = sandbox.metadata.as_mut() { metadata.annotations.remove(RESUME_INTENT_ANNOTATION); + // The idle marker lives from the idle stop request through its + // settled stop. Any other lifecycle write, including an explicit stop + // of the stopped sandbox, removes it. + if reason == idle_stop::IDLE_STOPPING_REASON { + metadata.annotations.insert( + idle_stop::IDLE_STOP_ANNOTATION.to_string(), + "true".to_string(), + ); + } else if reason != idle_stop::IDLE_STOPPED_REASON { + metadata.annotations.remove(idle_stop::IDLE_STOP_ANNOTATION); + } } // Explicit quiesce and start requests cancel any gateway-managed // automatic restart. @@ -8815,6 +9543,11 @@ pub fn new_test_runtime_with_driver( default_image: "openshell/sandbox:test".to_string(), image_preparation_timeout_seconds: 1800, provisioning_repair_window_ms: 300_000, + live_activity: Arc::default(), + activity_gates: Arc::default(), + idle_stops_in_flight: Arc::default(), + display_writers: Arc::default(), + activity_requests: Arc::default(), sandbox_updates_enabled: false, update_recovery_sweep: Arc::new(tokio::sync::Semaphore::new(1)), update_recovery_backoff: Arc::default(), @@ -10114,6 +10847,11 @@ mod tests { default_image: "openshell/sandbox:test".to_string(), image_preparation_timeout_seconds: 1800, provisioning_repair_window_ms: 300_000, + live_activity: Arc::default(), + activity_gates: Arc::default(), + idle_stops_in_flight: Arc::default(), + display_writers: Arc::default(), + activity_requests: Arc::default(), sandbox_updates_enabled: false, update_recovery_sweep: Arc::new(Semaphore::new(1)), update_recovery_backoff: Arc::default(), @@ -22037,165 +22775,1141 @@ endpoint = { kind = "unix", socket_path = "/run/agent-substrate/isolation.sock" } #[tokio::test] - async fn provisioning_worker_uses_policy_commit_time_not_scan_time() { - use crate::policy_store::PolicyStoreExt; + async fn provisioning_worker_uses_policy_commit_time_not_scan_time() { + use crate::policy_store::PolicyStoreExt; + let driver = ControlledDriver::new(); + let runtime = test_runtime(driver).await; + let now = openshell_core::time::now_ms(); + let mut sandbox = sandbox_record("sb-reset", "reset", SandboxPhase::Provisioning); + sandbox.status.as_mut().unwrap().provisioning = + Some(provisioning_deadline::new_record(now - 100_000, 300_000)); + provisioning_deadline::refresh_configuration( + &runtime.store, + &mut sandbox, + now - 100_000, + 300_000, + ) + .await + .unwrap(); + runtime.store.put_message(&sandbox).await.unwrap(); + let policy = openshell_policy::restrictive_default_policy(); + let hash = openshell_core::policy_identity::deterministic_policy_hash(&policy); + runtime + .store + .put_policy_revision( + "new-policy", + "sb-reset", + "default", + 2, + &policy.encode_to_vec(), + &hash, + ) + .await + .unwrap(); + let committed = runtime + .store + .get_latest_policy("sb-reset") + .await + .unwrap() + .unwrap() + .created_at_ms; + runtime + .reconcile_provisioning_deadlines(committed + 60_000) + .await + .unwrap(); + let updated = runtime + .store + .get_message::("sb-reset") + .await + .unwrap() + .unwrap(); + let record = updated + .status + .as_ref() + .unwrap() + .provisioning + .as_ref() + .unwrap(); + assert_eq!( + openshell_core::time::timestamp_to_millis(record.deadline.as_ref().unwrap()).unwrap(), + committed + 300_000 + ); + assert_eq!(updated.phase(), i32::from(SandboxPhase::Provisioning)); + } + #[tokio::test] + async fn provisioning_expiry_does_not_wait_for_busy_lifecycle_driver() { + let driver = ControlledDriver::new(); + let runtime = test_runtime(driver.clone()).await; + let now = openshell_core::time::now_ms(); + let mut sandbox = sandbox_record("sb-busy-ttl", "busy-ttl", SandboxPhase::Starting); + sandbox.status.as_mut().unwrap().provisioning = + Some(provisioning_deadline::new_record(now, 300_000)); + runtime.store.put_message(&sandbox).await.unwrap(); + let gate = runtime.lifecycle_gates.lock_for("sb-busy-ttl").await; + runtime + .reconcile_provisioning_deadlines(now + 300_000) + .await + .unwrap(); + let expired = runtime + .store + .get_message::("sb-busy-ttl") + .await + .unwrap() + .unwrap(); + assert_eq!(expired.phase(), i32::from(SandboxPhase::Error)); + let interrupted = tokio::time::timeout( + Duration::from_secs(3), + runtime.await_provisioning_operation( + &sandbox, + std::future::pending::>(), + ), + ) + .await + .expect("expired startup releases its lifecycle gate") + .unwrap_err(); + assert_eq!(Status::from(interrupted).code(), Code::DeadlineExceeded); + runtime + .mark_sandbox_error(&sandbox, "StartFailed", "late startup failure") + .await; + let retained = runtime + .store + .get_message::("sb-busy-ttl") + .await + .unwrap() + .unwrap(); + assert!(provisioning_deadline::timed_out(&retained)); + assert!( + retained + .status + .as_ref() + .unwrap() + .conditions + .iter() + .any(|condition| condition.reason == "ProvisioningTimedOut") + ); + assert_eq!( + driver.stop_calls(), + 0, + "cleanup waits for the in-flight lifecycle operation" + ); + assert_eq!( + runtime + .start_sandbox("default", "busy-ttl") + .await + .unwrap_err() + .code(), + Code::FailedPrecondition + ); + drop(gate); + runtime + .reconcile_provisioning_deadlines(now + 300_001) + .await + .unwrap(); + tokio::time::timeout(Duration::from_secs(1), driver.stop_finished.notified()) + .await + .unwrap(); + } + + #[tokio::test] + async fn provisioning_retry_recreates_missing_partial_compute() { + let driver = ControlledDriver::new(); + let runtime = test_runtime(driver.clone()).await; + let sandbox = seed_provisioning_attempt(&runtime).await; + let gate = runtime.lifecycle_gates.lock_for("sb-ttl").await; + let sandbox_guard = runtime.lock_sandbox_for_lifecycle(&gate).await; + let expired = runtime + .claim_provisioning_timeout(&sandbox, 300_000) + .await + .unwrap() + .unwrap(); + drop(sandbox_guard); + runtime + .reclaim_provisioning_timeout(&expired, &gate) + .await + .unwrap(); + drop(gate); + driver.set_start_outcome(ControlledLifecycleOutcome::NotFound); + let starting = runtime + .start_sandbox("default", "sandbox-ttl") + .await + .unwrap(); + assert_eq!(starting.object_id(), "sb-ttl"); + assert_eq!(starting.phase(), i32::from(SandboxPhase::Starting)); + } + + fn program_report( + state: openshell_core::proto::WorkloadProgramState, + state_age_secs: i64, + ) -> openshell_core::proto::SupervisorWorkloadActivity { + openshell_core::proto::SupervisorWorkloadActivity { + state: state.into(), + app: "claude-code".into(), + state_age: Some(prost_types::Duration { + seconds: state_age_secs, + nanos: 0, + }), + reporting_terminals: 1, + ..Default::default() + } + } + + fn idle_ready_record(id: &str, name: &str) -> Sandbox { + let mut sandbox = sandbox_record(id, name, SandboxPhase::Ready); + let status = sandbox.status.as_mut().unwrap(); + status.main_process_instance_id = "instance-1".into(); + status.configuration_activated = Some(true); + sandbox + } + + async fn stored_sandbox(runtime: &ComputeRuntime, id: &str) -> Sandbox { + runtime + .store + .get_message::(id) + .await + .unwrap() + .unwrap() + } + + /// Record a pushed report from "session-1" and persist its display + /// condition, which production writes on a background task. + async fn record( + runtime: &ComputeRuntime, + id: &str, + report: &openshell_core::proto::SupervisorWorkloadActivity, + ) { + runtime + .record_workload_activity(id, "session-1", "instance-1", true, report) + .await + .unwrap(); + runtime.persist_program_condition(id).await.unwrap(); + } + + fn still_idle() -> openshell_core::proto::SupervisorWorkloadActivity { + program_report(openshell_core::proto::WorkloadProgramState::Done, 3600) + } + + async fn record_done_an_hour_ago(runtime: &ComputeRuntime, id: &str) { + record(runtime, id, &still_idle()).await; + } + + async fn set_idle_timeout(runtime: &ComputeRuntime, name: &str, seconds: i64) { + crate::grpc::policy::put_int_setting_for_test( + &runtime.store, + Some(("default", name)), + openshell_core::settings::IDLE_STOP_AFTER_SECONDS_KEY, + seconds, + ) + .await; + } + + async fn publish_owner(runtime: &ComputeRuntime, id: &str, session: &str, epoch: u64) { + SupervisorOwnerIndex::new(runtime.store.clone(), OWNER_TTL) + .publish(id, session, "instance-1", epoch, "replica-a", "") + .await + .expect("publish test owner record"); + } + + /// Register "session-1" as the owner, with a supervisor that answers each + /// activity request with `answer`, as the real supervisor answers from + /// its current state. + async fn register_confirming_session( + runtime: &ComputeRuntime, + id: &str, + answer: openshell_core::proto::SupervisorWorkloadActivity, + ) { + // Re-registering the same session keeps its existing owner record. + let _ = SupervisorOwnerIndex::new(runtime.store.clone(), OWNER_TTL) + .publish(id, "session-1", "instance-1", 1, "replica-a", "") + .await; + let (tx, mut rx) = mpsc::channel::(8); + let (shutdown_tx, _shutdown_rx) = oneshot::channel(); + runtime.supervisor_sessions.register( + id.to_string(), + "session-1".to_string(), + tx, + shutdown_tx, + ); + let runtime = runtime.clone(); + let id = id.to_string(); + tokio::spawn(async move { + while let Some(message) = rx.recv().await { + if let Some( + openshell_core::proto::gateway_message::Payload::WorkloadActivityRequest( + request, + ), + ) = message.payload + { + let mut reply = answer.clone(); + reply.request_id = request.request_id; + let _ = runtime + .record_workload_activity(&id, "session-1", "instance-1", true, &reply) + .await; + } + } + }); + } + + /// Idle-stop `name` with a "session-1" confirmation whose reply reached the + /// handler at `arrived`, as `run_idle_stop` does after a round trip. + async fn idle_stop_arrived( + runtime: &ComputeRuntime, + name: &str, + arrived: Instant, + ) -> Result { + let cause = QuiesceCause::Idle(IdleConfirmation { + session_id: "session-1".to_string(), + arrived, + }); + runtime + .quiesce_sandbox(QuiesceOperation::Stop, &cause, "default", name) + .await + } + + async fn idle_stop_directly(runtime: &ComputeRuntime, name: &str) -> Result { + idle_stop_arrived(runtime, name, Instant::now()).await + } + + async fn wait_for_phase(runtime: &ComputeRuntime, id: &str, phase: SandboxPhase) -> Sandbox { + let mut sandbox = stored_sandbox(runtime, id).await; + for _ in 0..200 { + if sandbox.phase() == phase as i32 { + break; + } + tokio::time::sleep(Duration::from_millis(10)).await; + sandbox = stored_sandbox(runtime, id).await; + } + sandbox + } + + /// An idle sandbox with a confirming session, an owner record, a due + /// report, and a 60-second timeout. + async fn due_sandbox(runtime: &ComputeRuntime, id: &str, name: &str) { + runtime + .store + .put_message(&idle_ready_record(id, name)) + .await + .unwrap(); + register_confirming_session(runtime, id, still_idle()).await; + record_done_an_hour_ago(runtime, id).await; + set_idle_timeout(runtime, name, 60).await; + } + + #[tokio::test] + async fn workload_activity_is_recorded_for_the_running_instance_and_survives_snapshots() { + use openshell_core::proto::WorkloadProgramState; + let driver = ControlledDriver::new(); + let runtime = test_runtime(driver.clone()).await; + let sandbox = idle_ready_record("sb-activity", "sandbox-activity"); + runtime.store.put_message(&sandbox).await.unwrap(); + register_test_supervisor_session(&runtime, sandbox.object_id()); + publish_owner(&runtime, "sb-activity", "session-1", 1).await; + + record( + &runtime, + "sb-activity", + &program_report(WorkloadProgramState::Done, 0), + ) + .await; + let recorded = stored_sandbox(&runtime, "sb-activity").await; + let program = idle_stop::program_condition(&recorded).unwrap(); + assert_eq!(program.reason, "Done"); + assert_eq!(program.message, "claude-code is done"); + let version = sandbox_resource_version(&recorded); + + // A stale supervisor instance cannot change the record. + runtime + .record_workload_activity( + "sb-activity", + "session-1", + "instance-old", + true, + &program_report(WorkloadProgramState::Working, 0), + ) + .await + .unwrap(); + runtime + .persist_program_condition("sb-activity") + .await + .unwrap(); + // An unchanged state is not rewritten. + record( + &runtime, + "sb-activity", + &program_report(WorkloadProgramState::Done, 0), + ) + .await; + let unchanged = stored_sandbox(&runtime, "sb-activity").await; + assert_eq!(sandbox_resource_version(&unchanged), version); + assert_eq!( + idle_stop::program_condition(&unchanged).unwrap().reason, + "Done" + ); + + // Driver snapshots rebuild conditions but keep the program status. + runtime + .apply_sandbox_update(ready_driver_sandbox("sb-activity", "sandbox-activity")) + .await + .unwrap(); + let observed = stored_sandbox(&runtime, "sb-activity").await; + assert_eq!(observed.phase(), SandboxPhase::Ready as i32); + assert_eq!( + idle_stop::program_condition(&observed).unwrap().reason, + "Done" + ); + } + + #[tokio::test] + async fn the_session_reader_never_waits_on_the_sandbox_lock() { + let runtime = test_runtime(ControlledDriver::new()).await; + runtime + .store + .put_message(&idle_ready_record("sb-reader", "sandbox-reader")) + .await + .unwrap(); + register_test_supervisor_session(&runtime, "sb-reader"); + let held = runtime.lock_sandbox_local("sb-reader").await; + tokio::time::timeout( + Duration::from_secs(1), + runtime.record_workload_activity( + "sb-reader", + "session-1", + "instance-1", + true, + &still_idle(), + ), + ) + .await + .expect("recording must not wait for the sandbox lock") + .unwrap(); + assert!(runtime.current_live_activity("sb-reader").is_some()); + drop(held); + } + + #[tokio::test] + async fn older_summaries_never_replace_newer_ones_from_the_same_session() { + use openshell_core::proto::WorkloadProgramState; + let runtime = test_runtime(ControlledDriver::new()).await; + register_test_supervisor_session(&runtime, "sb-order"); + let mut done = program_report(WorkloadProgramState::Done, 0); + done.sequence = 5; + let mut stale = program_report(WorkloadProgramState::Working, 0); + stale.sequence = 3; + stale.request_id = "expired-request".into(); + for report in [&done, &stale] { + runtime + .record_workload_activity("sb-order", "session-1", "instance-1", true, report) + .await + .unwrap(); + } + let live = runtime.current_live_activity("sb-order").unwrap(); + assert_eq!(live.state, WorkloadProgramState::Done); + assert_eq!(live.sequence, 5); + + // A new session starts its own sequence. + let (tx, _rx) = mpsc::channel(1); + let (shutdown_tx, _shutdown_rx) = oneshot::channel(); + runtime.supervisor_sessions.register( + "sb-order".to_string(), + "session-2".to_string(), + tx, + shutdown_tx, + ); + stale.request_id.clear(); + stale.sequence = 1; + runtime + .record_workload_activity("sb-order", "session-2", "instance-1", true, &stale) + .await + .unwrap(); + let live = runtime.current_live_activity("sb-order").unwrap(); + assert_eq!(live.session_id, "session-2"); + assert_eq!(live.state, WorkloadProgramState::Working); + } + + #[tokio::test] + async fn idle_sweep_stops_confirmed_waiting_sandboxes_and_start_clears_the_reason() { + use openshell_core::proto::WorkloadProgramState; + let driver = ControlledDriver::new(); + let runtime = Arc::new(test_runtime(driver.clone()).await); + for (id, name, state) in [ + ("sb-idle", "sandbox-idle", WorkloadProgramState::Done), + ("sb-busy", "sandbox-busy", WorkloadProgramState::Working), + ("sb-unset", "sandbox-unset", WorkloadProgramState::Done), + ] { + runtime + .store + .put_message(&idle_ready_record(id, name)) + .await + .unwrap(); + register_confirming_session(&runtime, id, program_report(state, 3600)).await; + record(&runtime, id, &program_report(state, 3600)).await; + } + for name in ["sandbox-idle", "sandbox-busy"] { + set_idle_timeout(&runtime, name, 60).await; + } + + Arc::clone(&runtime).stop_idle_sandboxes().await.unwrap(); + let stopped = wait_for_phase(&runtime, "sb-idle", SandboxPhase::Stopped).await; + assert_eq!(stopped.phase(), SandboxPhase::Stopped as i32); + assert!( + stopped.idle_stopped(), + "idle stop settles with its own reason" + ); + assert!(idle_stop::program_condition(&stopped).is_none()); + assert_eq!( + driver.stop_calls(), + 1, + "only the waiting sandbox with a timeout stops" + ); + for id in ["sb-busy", "sb-unset"] { + assert_eq!( + stored_sandbox(&runtime, id).await.phase(), + SandboxPhase::Ready as i32 + ); + } + + // A snapshot of the exited container keeps the idle reason. + let mut exited = ready_driver_sandbox("sb-idle", "sandbox-idle"); + let condition = &mut exited.status.as_mut().unwrap().conditions[0]; + condition.status = "False".to_string(); + condition.reason = openshell_core::driver_utils::CONDITION_EXITED.to_string(); + runtime.apply_sandbox_update(exited).await.unwrap(); + let observed = stored_sandbox(&runtime, "sb-idle").await; + assert_eq!(observed.phase(), SandboxPhase::Stopped as i32); + assert!( + observed.idle_stopped(), + "driver snapshots keep the idle reason" + ); + + // A start clears the idle reason along with the durable marker. + let starting = runtime + .start_sandbox("default", "sandbox-idle") + .await + .unwrap(); + assert_eq!(starting.phase(), SandboxPhase::Starting as i32); + assert!(!starting.idle_stopped()); + assert!(!idle_stop::idle_stop_marked(&starting)); + } + + #[tokio::test] + async fn input_the_supervisor_confirms_keeps_a_due_sandbox() { + let driver = ControlledDriver::new(); + let runtime = test_runtime(driver.clone()).await; + due_sandbox(&runtime, "sb-typed", "sandbox-typed").await; + // The last pushed report is due, but the user typed just now and the + // confirmation reports that input. + let mut typed = program_report(openshell_core::proto::WorkloadProgramState::Done, 3600); + typed.input_age = Some(prost_types::Duration { + seconds: 0, + nanos: 0, + }); + register_confirming_session(&runtime, "sb-typed", typed).await; + + let sandbox = stored_sandbox(&runtime, "sb-typed").await; + assert!(runtime.idle_stop_due_now(&sandbox).await.is_some()); + assert!(runtime.run_idle_stop(&sandbox).await.unwrap().is_none()); + assert_eq!(driver.stop_calls(), 0); + } + + #[tokio::test] + async fn a_turn_the_supervisor_confirms_keeps_a_due_sandbox() { + let driver = ControlledDriver::new(); + let runtime = test_runtime(driver.clone()).await; + due_sandbox(&runtime, "sb-turn", "sandbox-turn").await; + register_confirming_session( + &runtime, + "sb-turn", + program_report(openshell_core::proto::WorkloadProgramState::Working, 0), + ) + .await; + let sandbox = stored_sandbox(&runtime, "sb-turn").await; + assert!(runtime.idle_stop_due_now(&sandbox).await.is_some()); + assert!(runtime.run_idle_stop(&sandbox).await.unwrap().is_none()); + assert_eq!(driver.stop_calls(), 0); + } + + #[tokio::test] + async fn a_working_report_after_the_confirmation_reply_wins() { + let driver = ControlledDriver::new(); + let runtime = test_runtime(driver.clone()).await; + due_sandbox(&runtime, "sb-race", "sandbox-race").await; + // The supervisor confirmed idle, then its next message on the same + // stream reports a new turn before the stop commits. + runtime + .record_workload_activity( + "sb-race", + "session-1", + "instance-1", + true, + &program_report(openshell_core::proto::WorkloadProgramState::Working, 0), + ) + .await + .unwrap(); + let error = idle_stop_directly(&runtime, "sandbox-race") + .await + .unwrap_err(); + assert_eq!(error.code(), Code::Aborted); + assert_eq!(driver.stop_calls(), 0); + } + + #[tokio::test] + async fn a_takeover_after_confirmation_cancels_the_stop() { + let driver = ControlledDriver::new(); + let runtime = test_runtime(driver.clone()).await; + due_sandbox(&runtime, "sb-takeover", "sandbox-takeover").await; + // Another replica publishes ownership for the supervisor's new + // session before this replica writes Stopping; this replica's local + // registry still lists the old session. + publish_owner(&runtime, "sb-takeover", "session-2", 2).await; + let error = idle_stop_directly(&runtime, "sandbox-takeover") + .await + .unwrap_err(); + assert_eq!(error.code(), Code::Aborted); + assert_eq!(driver.stop_calls(), 0); + } + + #[tokio::test] + async fn sessions_that_do_not_confirm_are_never_stopped() { + let driver = ControlledDriver::new(); + let runtime = test_runtime(driver.clone()).await; + runtime + .store + .put_message(&idle_ready_record("sb-silent", "sandbox-silent")) + .await + .unwrap(); + publish_owner(&runtime, "sb-silent", "session-1", 1).await; + // A stale stream after a handoff: registered, but nothing answers. + let (tx, _rx) = mpsc::channel(8); + let (shutdown_tx, _shutdown_rx) = oneshot::channel(); + runtime.supervisor_sessions.register( + "sb-silent".to_string(), + "session-1".to_string(), + tx, + shutdown_tx, + ); + record_done_an_hour_ago(&runtime, "sb-silent").await; + set_idle_timeout(&runtime, "sandbox-silent", 60).await; + let sandbox = stored_sandbox(&runtime, "sb-silent").await; + assert!(runtime.run_idle_stop(&sandbox).await.unwrap().is_none()); + + // A supervisor that cannot answer requests is never idle-stopped. + runtime + .record_workload_activity("sb-silent", "session-1", "instance-1", false, &still_idle()) + .await + .unwrap(); + assert!(runtime.run_idle_stop(&sandbox).await.unwrap().is_none()); + assert_eq!(driver.stop_calls(), 0); + } + + #[tokio::test] + async fn restored_condition_without_a_live_report_never_stops() { + let driver = ControlledDriver::new(); + let runtime = Arc::new(test_runtime(driver.clone()).await); + due_sandbox(&runtime, "sb-restored", "sandbox-restored").await; + // A gateway restart keeps the stored condition but no live report. + runtime.live_activity.lock().unwrap().clear(); + assert!( + idle_stop::program_condition(&stored_sandbox(&runtime, "sb-restored").await).is_some() + ); + Arc::clone(&runtime).stop_idle_sandboxes().await.unwrap(); + tokio::time::sleep(Duration::from_millis(50)).await; + assert_eq!(driver.stop_calls(), 0); + + // A report from a replaced session does not count either. + record_done_an_hour_ago(&runtime, "sb-restored").await; + let (tx, _rx) = mpsc::channel(1); + let (shutdown_tx, _shutdown_rx) = oneshot::channel(); + runtime.supervisor_sessions.register( + "sb-restored".to_string(), + "session-2".to_string(), + tx, + shutdown_tx, + ); + Arc::clone(&runtime).stop_idle_sandboxes().await.unwrap(); + tokio::time::sleep(Duration::from_millis(50)).await; + assert_eq!(driver.stop_calls(), 0); + } + + #[tokio::test] + async fn idle_stop_rechecks_the_setting_under_the_lifecycle_lock() { + let driver = ControlledDriver::new(); + let runtime = test_runtime(driver.clone()).await; + due_sandbox(&runtime, "sb-disabled", "sandbox-disabled").await; + set_idle_timeout(&runtime, "sandbox-disabled", 0).await; + let error = idle_stop_directly(&runtime, "sandbox-disabled") + .await + .unwrap_err(); + assert_eq!(error.code(), Code::Aborted); + + // A stale confirmation does not authorize the write either. + set_idle_timeout(&runtime, "sandbox-disabled", 60).await; + let stale = Instant::now() + .checked_sub(idle_stop::CONFIRM_FRESHNESS + Duration::from_secs(1)) + .expect("test clock is past the freshness window"); + let error = idle_stop_arrived(&runtime, "sandbox-disabled", stale) + .await + .unwrap_err(); + assert_eq!(error.code(), Code::Aborted); + assert_eq!(driver.stop_calls(), 0); + } + + #[tokio::test] + async fn a_write_conflict_readmits_the_stop_without_stalling_other_sandboxes() { + let directory = tempfile::tempdir().expect("temporary database directory"); + let database_url = format!("sqlite://{}", directory.path().join("gateway.db").display()); + let store = Arc::new(Store::connect(&database_url).await.expect("connect store")); + let pool = sqlx::SqlitePool::connect(&database_url) + .await + .expect("connect conflict injector"); + sqlx::query("CREATE TABLE injected_conflicts (marker TEXT PRIMARY KEY)") + .execute(&pool) + .await + .expect("create conflict marker table"); + // Another replica updates the sandbox once, between the final idle + // check and the idle Stopping write. + sqlx::query( + "CREATE TRIGGER inject_idle_stopping_conflict \ + BEFORE UPDATE OF payload ON objects \ + WHEN instr(NEW.payload, CAST('Sandbox stop requested after its program waited' AS BLOB)) > 0 \ + AND NOT EXISTS (SELECT 1 FROM injected_conflicts WHERE marker = 'idle') \ + BEGIN \ + INSERT INTO injected_conflicts (marker) VALUES ('idle'); \ + UPDATE objects SET resource_version = resource_version + 1 \ + WHERE object_type = OLD.object_type AND id = OLD.id; \ + SELECT RAISE(IGNORE); \ + END", + ) + .execute(&pool) + .await + .expect("install conflict trigger"); + let driver = ControlledDriver::new(); + let mut runtime = test_runtime(driver.clone()).await; + runtime.store = store; + due_sandbox(&runtime, "sb-conflict", "sandbox-conflict").await; + + // While this sandbox's gate is held, another sandbox still records. + let held = runtime.activity_gate("sb-conflict").lock_owned().await; + tokio::time::timeout( + Duration::from_secs(1), + runtime.record_workload_activity( + "sb-other", + "session-1", + "instance-1", + true, + &still_idle(), + ), + ) + .await + .expect("another sandbox's report must not wait on this gate") + .unwrap(); + drop(held); + + let sandbox = stored_sandbox(&runtime, "sb-conflict").await; + let stopped = runtime + .run_idle_stop(&sandbox) + .await + .unwrap() + .expect("the retry readmits the stop"); + assert_eq!(stopped.phase(), SandboxPhase::Stopped as i32); + assert!(stopped.idle_stopped()); + assert_eq!(driver.stop_calls(), 1); + let injected = sqlx::query_scalar::<_, i64>("SELECT COUNT(*) FROM injected_conflicts") + .fetch_one(&pool) + .await + .unwrap(); + assert_eq!( + injected, 1, + "the idle Stopping write hit the injected conflict" + ); + pool.close().await; + } + + #[tokio::test] + async fn explicit_stop_of_an_idle_stopped_sandbox_keeps_it_stopped() { + let driver = ControlledDriver::new(); + let runtime = test_runtime(driver.clone()).await; + due_sandbox(&runtime, "sb-explicit", "sandbox-explicit").await; + let sandbox = stored_sandbox(&runtime, "sb-explicit").await; + let idle_stopped = runtime.run_idle_stop(&sandbox).await.unwrap().unwrap(); + assert!(idle_stopped.idle_stopped()); + + let stopped = runtime + .stop_sandbox("default", "sandbox-explicit") + .await + .unwrap(); + assert_eq!(stopped.phase(), SandboxPhase::Stopped as i32); + assert!( + !stopped.idle_stopped(), + "connect must not wake an explicit stop" + ); + assert!(!idle_stop::idle_stop_marked(&stopped)); + assert_eq!( + driver.stop_calls(), + 1, + "the explicit stop needs no driver call" + ); + } + + #[tokio::test] + async fn explicit_stop_during_an_in_flight_idle_stop_settles_as_explicit() { let driver = ControlledDriver::new(); - let runtime = test_runtime(driver).await; - let now = openshell_core::time::now_ms(); - let mut sandbox = sandbox_record("sb-reset", "reset", SandboxPhase::Provisioning); - sandbox.status.as_mut().unwrap().provisioning = - Some(provisioning_deadline::new_record(now - 100_000, 300_000)); - provisioning_deadline::refresh_configuration( - &runtime.store, + let runtime = test_runtime(driver.clone()).await; + // Another replica's idle stop committed Stopping and is waiting on + // the driver; this replica has no worker for it. + let mut sandbox = idle_ready_record("sb-inflight", "sandbox-inflight"); + apply_lifecycle_phase( &mut sandbox, - now - 100_000, + SandboxPhase::Stopping, + idle_stop::IDLE_STOPPING_REASON, + idle_stop::IDLE_STOPPING_MESSAGE, + 1800, 300_000, + ); + assert!(idle_stop::idle_stop_marked(&sandbox)); + runtime.store.put_message(&sandbox).await.unwrap(); + + let stopped = runtime + .stop_sandbox("default", "sandbox-inflight") + .await + .unwrap(); + assert_eq!(stopped.phase(), SandboxPhase::Stopped as i32); + assert!( + !stopped.idle_stopped(), + "the explicit stop wins over the idle one" + ); + assert!(!idle_stop::idle_stop_marked(&stopped)); + } + + #[tokio::test] + async fn a_confirmation_that_expires_during_the_final_check_is_abandoned() { + let driver = ControlledDriver::new(); + let runtime = test_runtime(driver.clone()).await; + due_sandbox(&runtime, "sb-expiry", "sandbox-expiry").await; + // The confirmation is fresh when the final check starts, but the + // check waits on the gate past its deadline. + let arrived = Instant::now() + .checked_sub(idle_stop::CONFIRM_FRESHNESS.saturating_sub(Duration::from_millis(100))) + .expect("test clock is past the freshness window"); + let held = runtime.activity_gate("sb-expiry").lock_owned().await; + let release = tokio::spawn(async move { + tokio::time::sleep(Duration::from_millis(400)).await; + drop(held); + }); + let error = idle_stop_arrived(&runtime, "sandbox-expiry", arrived) + .await + .unwrap_err(); + assert_eq!(error.code(), Code::Aborted); + assert_eq!(driver.stop_calls(), 0); + release.await.unwrap(); + } + + #[tokio::test] + async fn a_former_owner_never_overwrites_the_new_owners_status() { + use openshell_core::proto::WorkloadProgramState; + let runtime = test_runtime(ControlledDriver::new()).await; + due_sandbox(&runtime, "sb-handoff", "sandbox-handoff").await; + // This replica still holds session-1's `Done`. The supervisor moves to + // session-2 on another replica, which persists `Working`. + publish_owner(&runtime, "sb-handoff", "session-2", 2).await; + let working = idle_stop::condition_from_report( + &program_report(WorkloadProgramState::Working, 0), + SystemTime::now(), ) - .await .unwrap(); - runtime.store.put_message(&sandbox).await.unwrap(); - let policy = openshell_policy::restrictive_default_policy(); - let hash = openshell_core::policy_identity::deterministic_policy_hash(&policy); + let current = stored_sandbox(&runtime, "sb-handoff").await; runtime .store - .put_policy_revision( - "new-policy", - "sb-reset", - "default", - 2, - &policy.encode_to_vec(), - &hash, + .update_message_cas::( + "sb-handoff", + sandbox_resource_version(¤t), + |sandbox| { + let status = sandbox.status.as_mut().unwrap(); + status.conditions.retain(|condition| { + condition.r#type != openshell_core::metadata::PROGRAM_STATUS_CONDITION + }); + status.conditions.push(working.clone()); + }, ) .await .unwrap(); - let committed = runtime - .store - .get_latest_policy("sb-reset") - .await - .unwrap() - .unwrap() - .created_at_ms; + // This replica's delayed writer runs now. runtime - .reconcile_provisioning_deadlines(committed + 60_000) - .await - .unwrap(); - let updated = runtime - .store - .get_message::("sb-reset") + .persist_program_condition("sb-handoff") .await - .unwrap() - .unwrap(); - let record = updated - .status - .as_ref() - .unwrap() - .provisioning - .as_ref() .unwrap(); + let shown = stored_sandbox(&runtime, "sb-handoff").await; assert_eq!( - openshell_core::time::timestamp_to_millis(record.deadline.as_ref().unwrap()).unwrap(), - committed + 300_000 + idle_stop::program_condition(&shown).unwrap().reason, + "Working" ); - assert_eq!(updated.phase(), i32::from(SandboxPhase::Provisioning)); } + #[tokio::test] - async fn provisioning_expiry_does_not_wait_for_busy_lifecycle_driver() { + async fn display_writes_coalesce_behind_one_writer_per_sandbox() { + use openshell_core::proto::WorkloadProgramState; + let runtime = test_runtime(ControlledDriver::new()).await; + due_sandbox(&runtime, "sb-burst", "sandbox-burst").await; + let held = runtime.lock_sandbox_local("sb-burst").await; + for index in 0..50_u64 { + let state = if index % 2 == 0 { + WorkloadProgramState::Working + } else { + WorkloadProgramState::Done + }; + let mut report = program_report(state, 0); + report.sequence = index + 10; + runtime + .record_workload_activity("sb-burst", "session-1", "instance-1", true, &report) + .await + .unwrap(); + assert!( + runtime.display_writers.lock().unwrap().len() <= 1, + "one writer per sandbox, however many reports arrive" + ); + } + drop(held); + for _ in 0..200 { + if runtime.display_writers.lock().unwrap().is_empty() { + break; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + assert!(runtime.display_writers.lock().unwrap().is_empty()); + let shown = stored_sandbox(&runtime, "sb-burst").await; + assert_eq!( + idle_stop::program_condition(&shown).unwrap().reason, + "Done", + "the newest report is persisted" + ); + } + + #[tokio::test] + async fn a_reply_that_reached_the_handler_too_long_ago_never_stops() { let driver = ControlledDriver::new(); let runtime = test_runtime(driver.clone()).await; - let now = openshell_core::time::now_ms(); - let mut sandbox = sandbox_record("sb-busy-ttl", "busy-ttl", SandboxPhase::Starting); - sandbox.status.as_mut().unwrap().provisioning = - Some(provisioning_deadline::new_record(now, 300_000)); - runtime.store.put_message(&sandbox).await.unwrap(); - let gate = runtime.lifecycle_gates.lock_for("sb-busy-ttl").await; - runtime - .reconcile_provisioning_deadlines(now + 300_000) + due_sandbox(&runtime, "sb-late", "sandbox-late").await; + // The reply reached the handler, but the stop task resumed only after + // the freshness window had passed. + let (tx, mut rx) = mpsc::channel::(8); + let (shutdown_tx, _shutdown_rx) = oneshot::channel(); + runtime.supervisor_sessions.register( + "sb-late".to_string(), + "session-1".to_string(), + tx, + shutdown_tx, + ); + let responder = runtime.clone(); + tokio::spawn(async move { + while let Some(message) = rx.recv().await { + if let Some( + openshell_core::proto::gateway_message::Payload::WorkloadActivityRequest( + request, + ), + ) = message.payload + { + let mut reply = still_idle(); + reply.request_id = request.request_id; + let arrived = Instant::now() + .checked_sub(idle_stop::CONFIRM_FRESHNESS + Duration::from_millis(100)) + .expect("test clock is past the freshness window"); + let _ = responder + .record_workload_activity_at( + "sb-late", + "session-1", + "instance-1", + true, + &reply, + arrived, + ) + .await; + } + } + }); + let sandbox = stored_sandbox(&runtime, "sb-late").await; + assert!(runtime.run_idle_stop(&sandbox).await.unwrap().is_none()); + assert_eq!(driver.stop_calls(), 0); + } + + #[tokio::test] + async fn a_takeover_during_a_display_write_never_overwrites_the_new_owner() { + use openshell_core::proto::WorkloadProgramState; + let directory = tempfile::tempdir().expect("temporary database directory"); + let database_url = format!("sqlite://{}", directory.path().join("gateway.db").display()); + let store = Arc::new(Store::connect(&database_url).await.expect("connect store")); + let mut runtime = test_runtime(ControlledDriver::new()).await; + runtime.store = store; + due_sandbox(&runtime, "sb-mid", "sandbox-mid").await; + // The displayed status is `Working`; this replica's live report is an + // older `Done` it has not persisted yet. + record( + &runtime, + "sb-mid", + &program_report(WorkloadProgramState::Working, 0), + ) + .await; + // Let the background writers from setup finish first. + for _ in 0..200 { + if runtime.display_writers.lock().unwrap().is_empty() { + break; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + assert_eq!( + idle_stop::program_condition(&stored_sandbox(&runtime, "sb-mid").await) + .unwrap() + .reason, + "Working" + ); + runtime.live_activity.lock().unwrap().insert( + "sb-mid".to_string(), + idle_stop::live_from_report( + &still_idle(), + "session-1", + "instance-1", + true, + SystemTime::now(), + ), + ); + let pool = sqlx::SqlitePool::connect(&database_url) .await - .unwrap(); - let expired = runtime - .store - .get_message::("sb-busy-ttl") + .expect("connect takeover injector"); + sqlx::query("CREATE TABLE injected_takeovers (marker TEXT PRIMARY KEY)") + .execute(&pool) .await - .unwrap() - .unwrap(); - assert_eq!(expired.phase(), i32::from(SandboxPhase::Error)); - let interrupted = tokio::time::timeout( - Duration::from_secs(3), - runtime.await_provisioning_operation( - &sandbox, - std::future::pending::>(), - ), + .expect("create takeover marker table"); + // Another replica takes ownership and writes the sandbox between this + // replica's ownership check and its conditional write. + sqlx::query( + "CREATE TRIGGER inject_takeover \ + BEFORE UPDATE OF payload ON objects \ + WHEN instr(NEW.payload, CAST('claude-code is done' AS BLOB)) > 0 \ + AND NOT EXISTS (SELECT 1 FROM injected_takeovers WHERE marker = 'takeover') \ + BEGIN \ + INSERT INTO injected_takeovers (marker) VALUES ('takeover'); \ + UPDATE objects SET payload = CAST(replace(CAST(payload AS TEXT), \ + '\"session_id\":\"session-1\"', '\"session_id\":\"session-2\"') AS BLOB) \ + WHERE object_type = 'supervisor_session_owner'; \ + UPDATE objects SET resource_version = resource_version + 1 \ + WHERE object_type = OLD.object_type AND id = OLD.id; \ + SELECT RAISE(IGNORE); \ + END", ) + .execute(&pool) .await - .expect("expired startup releases its lifecycle gate") - .unwrap_err(); - assert_eq!(Status::from(interrupted).code(), Code::DeadlineExceeded); - runtime - .mark_sandbox_error(&sandbox, "StartFailed", "late startup failure") - .await; - let retained = runtime - .store - .get_message::("sb-busy-ttl") + .expect("install takeover trigger"); + + runtime.persist_program_condition("sb-mid").await.unwrap(); + let injected = sqlx::query_scalar::<_, i64>("SELECT COUNT(*) FROM injected_takeovers") + .fetch_one(&pool) .await - .unwrap() .unwrap(); - assert!(provisioning_deadline::timed_out(&retained)); - assert!( - retained - .status - .as_ref() - .unwrap() - .conditions - .iter() - .any(|condition| condition.reason == "ProvisioningTimedOut") - ); + assert_eq!(injected, 1, "the stale write reached the conditional write"); + let shown = stored_sandbox(&runtime, "sb-mid").await; assert_eq!( - driver.stop_calls(), - 0, - "cleanup waits for the in-flight lifecycle operation" + idle_stop::program_condition(&shown).unwrap().reason, + "Working", + "the retry sees the takeover and leaves the new owner's status" ); - assert_eq!( - runtime - .start_sandbox("default", "busy-ttl") - .await - .unwrap_err() - .code(), - Code::FailedPrecondition + pool.close().await; + } + + #[tokio::test] + async fn a_superseded_sessions_late_report_never_displaces_the_replacement() { + use openshell_core::proto::WorkloadProgramState; + let runtime = test_runtime(ControlledDriver::new()).await; + runtime + .store + .put_message(&idle_ready_record("sb-replaced", "sandbox-replaced")) + .await + .unwrap(); + // Session-2 replaced session-1 and reported `Working`. + let (tx, _rx) = mpsc::channel(1); + let (shutdown_tx, _shutdown_rx) = oneshot::channel(); + runtime.supervisor_sessions.register( + "sb-replaced".to_string(), + "session-2".to_string(), + tx, + shutdown_tx, ); - drop(gate); runtime - .reconcile_provisioning_deadlines(now + 300_001) + .record_workload_activity( + "sb-replaced", + "session-2", + "instance-1", + true, + &program_report(WorkloadProgramState::Working, 0), + ) .await .unwrap(); - tokio::time::timeout(Duration::from_secs(1), driver.stop_finished.notified()) + // A `Done` from session-1 that passed the session loop's check + // before session-2 registered arrives late, along with a reply. + let mut late = still_idle(); + late.sequence = 99; + late.request_id = "late-request".into(); + let (waiter, mut answered) = oneshot::channel(); + runtime + .activity_requests + .lock() + .unwrap() + .insert("late-request".to_string(), waiter); + runtime + .record_workload_activity("sb-replaced", "session-1", "instance-1", true, &late) .await .unwrap(); + let live = runtime.current_live_activity("sb-replaced").unwrap(); + assert_eq!(live.session_id, "session-2"); + assert_eq!(live.state, WorkloadProgramState::Working); + assert!( + answered.try_recv().is_err(), + "a replaced session's reply confirms nothing" + ); } #[tokio::test] - async fn provisioning_retry_recreates_missing_partial_compute() { + async fn overlapping_idle_stops_run_one_at_a_time() { let driver = ControlledDriver::new(); let runtime = test_runtime(driver.clone()).await; - let sandbox = seed_provisioning_attempt(&runtime).await; - let gate = runtime.lifecycle_gates.lock_for("sb-ttl").await; - let sandbox_guard = runtime.lock_sandbox_for_lifecycle(&gate).await; - let expired = runtime - .claim_provisioning_timeout(&sandbox, 300_000) - .await - .unwrap() - .unwrap(); - drop(sandbox_guard); + due_sandbox(&runtime, "sb-overlap", "sandbox-overlap").await; + // A stop attempt from an earlier sweep is still in flight. + let earlier = IdleStopInFlight::claim(&runtime.idle_stops_in_flight, "sb-overlap") + .expect("first claim succeeds"); + let sandbox = stored_sandbox(&runtime, "sb-overlap").await; + assert!(runtime.run_idle_stop(&sandbox).await.unwrap().is_none()); + assert_eq!( + driver.stop_calls(), + 0, + "the later sweep does not start a second stop" + ); + drop(earlier); + assert!( + runtime.run_idle_stop(&sandbox).await.unwrap().is_some(), + "once the earlier attempt ends, a later sweep can stop it" + ); + assert_eq!(driver.stop_calls(), 1); + } + + #[tokio::test] + async fn an_in_flight_idle_stop_keeps_its_reason_through_driver_snapshots() { + let runtime = test_runtime(ControlledDriver::new()).await; + let mut sandbox = idle_ready_record("sb-stopping", "sandbox-stopping"); + apply_lifecycle_phase( + &mut sandbox, + SandboxPhase::Stopping, + idle_stop::IDLE_STOPPING_REASON, + idle_stop::IDLE_STOPPING_MESSAGE, + 1800, + 300_000, + ); + runtime.store.put_message(&sandbox).await.unwrap(); + assert!(sandbox.idle_stopping()); runtime - .reclaim_provisioning_timeout(&expired, &gate) - .await - .unwrap(); - drop(gate); - driver.set_start_outcome(ControlledLifecycleOutcome::NotFound); - let starting = runtime - .start_sandbox("default", "sandbox-ttl") + .apply_sandbox_update(ready_driver_sandbox("sb-stopping", "sandbox-stopping")) .await .unwrap(); - assert_eq!(starting.object_id(), "sb-ttl"); - assert_eq!(starting.phase(), i32::from(SandboxPhase::Starting)); + let observed = stored_sandbox(&runtime, "sb-stopping").await; + assert_eq!(observed.phase(), SandboxPhase::Stopping as i32); + assert!( + observed.idle_stopping(), + "connect can still tell it is an idle stop" + ); } } diff --git a/crates/openshell-server/src/config_delivery/session.rs b/crates/openshell-server/src/config_delivery/session.rs index f70a5d1a75..d47d472b69 100644 --- a/crates/openshell-server/src/config_delivery/session.rs +++ b/crates/openshell-server/src/config_delivery/session.rs @@ -1043,6 +1043,7 @@ mod tests { supports_session_redirect: false, pushes_config: true, advertises_services: false, + confirms_activity: false, } } diff --git a/crates/openshell-server/src/grpc/mod.rs b/crates/openshell-server/src/grpc/mod.rs index ba6324f3a1..3224ac6e65 100644 --- a/crates/openshell-server/src/grpc/mod.rs +++ b/crates/openshell-server/src/grpc/mod.rs @@ -1546,6 +1546,7 @@ pub mod test_support { .send(SupervisorMessage { payload: Some(supervisor_message::Payload::Hello(SupervisorHello { supports_service_advertisements: false, + supports_workload_activity_requests: false, sandbox_id: sandbox_id.into(), instance_id: "instance".into(), supports_config_push: features == StreamFeatures::Apply, diff --git a/crates/openshell-server/src/grpc/policy.rs b/crates/openshell-server/src/grpc/policy.rs index 40819e7b37..f95885457a 100644 --- a/crates/openshell-server/src/grpc/policy.rs +++ b/crates/openshell-server/src/grpc/policy.rs @@ -8102,6 +8102,60 @@ fn upsert_setting_value( } } +/// The raw `idle_stop_after_seconds` value for a sandbox, if any scope sets +/// it. Gateway scope wins over sandbox scope, as for every runtime setting. +/// `global` lets a sweep load gateway settings once for many sandboxes. +pub async fn idle_stop_after_seconds( + store: &Store, + global: Option, + workspace: &str, + sandbox_name: &str, +) -> Result, Status> { + if global.is_some() { + return Ok(global); + } + let sandbox = load_sandbox_settings(store, workspace, sandbox_name).await?; + Ok( + match sandbox.settings.get(settings::IDLE_STOP_AFTER_SECONDS_KEY) { + Some(StoredSettingValue::Int(seconds)) => Some(*seconds), + _ => None, + }, + ) +} + +#[cfg(test)] +pub async fn put_int_setting_for_test( + store: &Store, + sandbox: Option<(&str, &str)>, + key: &str, + value: i64, +) { + let mut settings = match sandbox { + Some((workspace, name)) => load_sandbox_settings(store, workspace, name).await, + None => load_global_settings(store).await, + } + .expect("load test settings"); + settings + .settings + .insert(key.to_string(), StoredSettingValue::Int(value)); + match sandbox { + Some((workspace, name)) => save_sandbox_settings(store, workspace, name, &settings).await, + None => save_global_settings(store, &settings).await, + } + .expect("save test settings"); +} + +/// The gateway-scope `idle_stop_after_seconds` value, if set. +pub async fn global_idle_stop_after_seconds(store: &Store) -> Result, Status> { + let global = load_global_settings(store).await?; + Ok( + match global.settings.get(settings::IDLE_STOP_AFTER_SECONDS_KEY) { + Some(StoredSettingValue::Int(seconds)) => Some(*seconds), + _ => None, + }, + ) +} + pub(super) async fn load_global_settings(store: &Store) -> Result { load_settings_record(store, GLOBAL_SETTINGS_OBJECT_TYPE, "", GLOBAL_SETTINGS_NAME).await } diff --git a/crates/openshell-server/src/grpc/policy/endpoint_status_tests.rs b/crates/openshell-server/src/grpc/policy/endpoint_status_tests.rs index 883a69c498..73a1f2a465 100644 --- a/crates/openshell-server/src/grpc/policy/endpoint_status_tests.rs +++ b/crates/openshell-server/src/grpc/policy/endpoint_status_tests.rs @@ -1941,6 +1941,7 @@ async fn replacement_reset_timeout_invalidates_superseded_evidence() { let hello = SupervisorMessage { payload: Some(supervisor_message::Payload::Hello(SupervisorHello { supports_service_advertisements: false, + supports_workload_activity_requests: false, sandbox_id: sandbox_id.to_string(), instance_id: "instance-b".to_string(), connection_epoch: 1, @@ -2016,6 +2017,7 @@ async fn replacement_reset_retry_does_not_hold_gateway_shutdown() { let hello = SupervisorMessage { payload: Some(supervisor_message::Payload::Hello(SupervisorHello { supports_service_advertisements: false, + supports_workload_activity_requests: false, sandbox_id: sandbox_id.to_string(), instance_id: "instance-b".to_string(), connection_epoch: 1, diff --git a/crates/openshell-server/src/grpc/provider_readiness_tests.rs b/crates/openshell-server/src/grpc/provider_readiness_tests.rs index bfe729c044..f74563ba8d 100644 --- a/crates/openshell-server/src/grpc/provider_readiness_tests.rs +++ b/crates/openshell-server/src/grpc/provider_readiness_tests.rs @@ -69,6 +69,7 @@ impl TestRegistry { fn hello() -> SupervisorHello { SupervisorHello { supports_service_advertisements: false, + supports_workload_activity_requests: false, sandbox_id: Uuid::new_v4().to_string(), instance_id: Uuid::new_v4().to_string(), connection_epoch: 0, diff --git a/crates/openshell-server/src/storage_proto.rs b/crates/openshell-server/src/storage_proto.rs index da4b2aa977..9b38899399 100644 --- a/crates/openshell-server/src/storage_proto.rs +++ b/crates/openshell-server/src/storage_proto.rs @@ -227,8 +227,13 @@ mod tests { // Discovery RPCs move to ServiceDiscovery. Application URL maps add two // public-only messages alongside main's public APIs; durable schemas // and the public/durable overlap remain unchanged. + // Workload program status adds SupervisorWorkloadActivity, its two enums, + // and WorkloadActivityRequest to supervisor control traffic, which is + // never stored. The stored + // ProgramStatus condition reuses SandboxCondition, so durable schemas + // remain unchanged. const PUBLIC_RPC_SCHEMA_SHA256: &str = - "ba37ee30d2e8618b947517c67fa6d336a87097978a55cfa82034366ee36a740e"; + "b05c436c6d9133d948bfe8120a2ba1434f1e8603e28bc69eaba3d3fbedff2702"; const DURABLE_SCHEMA_SHA256: &str = "9a025fc84af7c71bce42a52a282d1a96a02f8d71319478bcdb0e0c6367305888"; const PUBLIC_DURABLE_OVERLAP_SHA256: &str = @@ -776,7 +781,7 @@ mod tests { overlap_hash.as_str(), ), ( - (367, 31), + (369, 33), (102, 22), (86, 22), PUBLIC_RPC_SCHEMA_SHA256, diff --git a/crates/openshell-server/src/supervisor_session.rs b/crates/openshell-server/src/supervisor_session.rs index 92a8ed6222..f53bf784a7 100644 --- a/crates/openshell-server/src/supervisor_session.rs +++ b/crates/openshell-server/src/supervisor_session.rs @@ -680,6 +680,24 @@ impl SupervisorSessionRegistry { .map(|session| session.config.is_some()) } + /// Queue `message` on `session_id` if it is still the sandbox's current + /// session. Returns false when the session is gone or its queue is full. + pub fn send_if_current( + &self, + sandbox_id: &str, + session_id: &str, + message: GatewayMessage, + ) -> bool { + let tx = self + .sessions + .lock() + .unwrap() + .get(sandbox_id) + .filter(|session| session.session_id == session_id) + .map(|session| session.tx.clone()); + tx.is_some_and(|tx| tx.try_send(message).is_ok()) + } + pub fn is_current_session(&self, sandbox_id: &str, session_id: &str) -> bool { self.sessions .lock() @@ -2238,6 +2256,7 @@ struct ConfigPushSetup { /// session task or drops with early cleanup. struct SessionSetup { advertises_services: bool, + confirms_activity: bool, state: Arc, sandbox_id: String, instance_id: String, @@ -2257,6 +2276,7 @@ struct SessionSetup { async fn accept_supervisor_session(setup: SessionSetup) -> Result<(), Status> { let SessionSetup { advertises_services, + confirms_activity, state, sandbox_id, instance_id, @@ -2523,6 +2543,7 @@ async fn accept_supervisor_session(setup: SessionSetup) -> Result<(), Status> { supports_session_redirect, pushes_config, advertises_services, + confirms_activity, }; tokio::spawn(async move { let _session_lifetime = session_lifetime; @@ -2660,6 +2681,7 @@ pub async fn handle_connect_supervisor( let setup = SessionSetup { advertises_services: state.config.supervisor_app_listener.enabled && hello.supports_service_advertisements, + confirms_activity: hello.supports_workload_activity_requests, state: Arc::clone(state), sandbox_id, instance_id: hello.instance_id, @@ -2921,6 +2943,8 @@ fn queue_shutdown_redirect( } /// An accepted supervisor session, shared by the handlers for its messages. +// Each bool is an independent capability the supervisor's hello negotiated. +#[allow(clippy::struct_excessive_bools)] pub(crate) struct AcceptedSession { pub(crate) state: Arc, pub(crate) sandbox_id: String, @@ -2935,6 +2959,9 @@ pub(crate) struct AcceptedSession { /// polling. pub(crate) pushes_config: bool, pub(crate) advertises_services: bool, + /// The supervisor answers `WorkloadActivityRequest`, so its sandbox can be + /// idle-stopped after it confirms. + pub(crate) confirms_activity: bool, } async fn run_session_loop( @@ -3157,6 +3184,21 @@ async fn handle_supervisor_message( return false; } } + Some(supervisor_message::Payload::WorkloadActivity(report)) => { + if let Err(error) = state + .compute + .record_workload_activity( + sandbox_id, + session_id, + &session.instance_id, + session.confirms_activity, + &report, + ) + .await + { + debug!(sandbox_id = %sandbox_id, %error, "failed to record workload activity"); + } + } _ => { warn!( sandbox_id = %sandbox_id, @@ -3376,6 +3418,7 @@ mod tests { let mut inbound = futures::stream::pending::>(); let session = AcceptedSession { advertises_services: false, + confirms_activity: false, state: Arc::clone(&state), sandbox_id: "sb-no-peer".into(), session_id: "session-no-peer".into(), @@ -5942,6 +5985,7 @@ mod tests { let mut inbound = futures::stream::iter([Ok(failure)]).chain(futures::stream::pending()); let session = AcceptedSession { advertises_services: false, + confirms_activity: false, state: Arc::clone(&state), sandbox_id: "sb-replaced".into(), session_id: "session-a".into(), @@ -6033,6 +6077,7 @@ mod tests { let mut inbound = futures::stream::iter([Ok(heartbeat)]).chain(futures::stream::pending()); let session = AcceptedSession { advertises_services: false, + confirms_activity: false, state: Arc::clone(&state), sandbox_id: "sb-moved".into(), session_id: "session-replaced".into(), diff --git a/crates/openshell-supervisor-process/Cargo.toml b/crates/openshell-supervisor-process/Cargo.toml index 5e6af570f9..91cee0212a 100644 --- a/crates/openshell-supervisor-process/Cargo.toml +++ b/crates/openshell-supervisor-process/Cargo.toml @@ -22,6 +22,7 @@ base64 = { workspace = true } bytes = { workspace = true } hex = "0.4" miette = { workspace = true } +prost-types = { workspace = true } rand = "0.10" russh = "0.65.2" serde_json = { workspace = true } diff --git a/crates/openshell-supervisor-process/README.md b/crates/openshell-supervisor-process/README.md index 1c2f728b43..10615e5624 100644 --- a/crates/openshell-supervisor-process/README.md +++ b/crates/openshell-supervisor-process/README.md @@ -3,6 +3,36 @@ This crate manages workload processes and SSH sessions through the admitted isolation backend. +## Program status + +`program_status` scans PTY output of the canonical process and of SSH PTY +sessions for OSC 7501 program status reports. It keeps the protocol's +per-terminal records: ids, `clear`, dropping `working` and `blocked` at an OSC +133 `A` prompt, clearing every record on a full reset (RIS), and the record and +sequence limits. Reports, prompts, and resets apply in output order regardless +of how reads split them. A terminal's records end with its process. Report bytes are forwarded +unchanged; `msg` and `title` are validated and then discarded. + +The supervisor answers the support query (`OSC 7501 ; ?`) itself by writing the +reply to the program's input, and removes the query from forwarded output. +Claude Code reports only when that reply arrives before the reply to the DA1 +query it sends next. Only the bytes of a possible query are held back, and a +20 ms pause releases them. + +A sandbox-wide summary takes the most active state across terminals +(working, then blocked, error, done, idle) and the latest client input, which +outlives the terminal that received it. Each gateway session receives the +summary at start, on every change, and within 15 seconds of new input, including +the last keystroke before typing stops. It also answers each +`WorkloadActivityRequest` with the current summary, which the gateway requires +before an idle stop. Every transition, including one undone within the same +read, restarts the state timer and advances a revision. Summaries are published +under the activity lock, so concurrent terminals never republish an older one. +Pushes and request replies take one ordering lock from snapshot to enqueue and +carry increasing sequence numbers, so the gateway can drop an older summary that +arrives late. Ages are relative, so supervisor and gateway clocks need +not agree. + ## Bounded streaming uploads The CLI's built-in uploads use the `openshell-upload-v1` SSH subsystem. Russh diff --git a/crates/openshell-supervisor-process/src/delegated.rs b/crates/openshell-supervisor-process/src/delegated.rs index 4f6edaac11..37bfda20b1 100644 --- a/crates/openshell-supervisor-process/src/delegated.rs +++ b/crates/openshell-supervisor-process/src/delegated.rs @@ -134,6 +134,7 @@ pub async fn start_boundary_access( config_apply_tx, config_push_updates, gateway_home, + workload_activity: Some(main_session.program_activity().clone()), }, ); (Some(task), Some(accepted)) diff --git a/crates/openshell-supervisor-process/src/lib.rs b/crates/openshell-supervisor-process/src/lib.rs index df0f21a519..198ffb8ec0 100644 --- a/crates/openshell-supervisor-process/src/lib.rs +++ b/crates/openshell-supervisor-process/src/lib.rs @@ -11,6 +11,7 @@ pub mod debug_rpc; pub mod delegated; pub mod log_push; pub mod main_session; +pub mod program_status; pub mod skills; #[cfg(unix)] pub mod ssh; diff --git a/crates/openshell-supervisor-process/src/main_session.rs b/crates/openshell-supervisor-process/src/main_session.rs index 5bf4879f04..eda5ecebd7 100644 --- a/crates/openshell-supervisor-process/src/main_session.rs +++ b/crates/openshell-supervisor-process/src/main_session.rs @@ -18,6 +18,8 @@ use tokio::sync::watch; use openshell_isolation_interface::contract::BoundarySignal; +use crate::program_status::{InputMarker, ProgramActivity, TerminalTap}; + const OUTPUT_BUFFER_BYTES: usize = 1024 * 1024; #[derive(Clone, Debug)] @@ -190,6 +192,9 @@ pub struct MainSession { finished: std::sync::atomic::AtomicBool, terminal_attachments: Mutex, terminal_attachments_done: Notify, + activity: Arc, + /// Records client input for the canonical terminal, when it has one. + input_marker: Option, } impl MainSession { @@ -219,6 +224,8 @@ impl MainSession { expectation: AttachmentExpectation::None, }), terminal_attachments_done: Notify::new(), + activity: ProgramActivity::new(), + input_marker: None, }); (session, input_rx) } @@ -239,6 +246,15 @@ impl MainSession { } = attachment; let terminal_mode = terminal.is_some(); let (input, mut input_rx) = tokio::sync::mpsc::channel::>(64); + let activity = ProgramActivity::new(); + let tap = terminal_mode.then(|| { + let reply = input.clone(); + activity.register(move |bytes| { + let _ = reply.try_send(bytes.to_vec()); + }) + }); + let input_marker = tap.as_ref().map(TerminalTap::input_marker); + crate::program_status::spawn_transition_log(&activity); let session = Arc::new(Self { terminal: terminal_mode, input, @@ -256,16 +272,26 @@ impl MainSession { expectation: AttachmentExpectation::None, }), terminal_attachments_done: Notify::new(), + activity, + input_marker, }); let stdout_session = Arc::clone(&session); tokio::spawn(async move { let mut stdout = stdout; let mut buffer = [0u8; 4096]; - loop { - match stdout.read(&mut buffer).await { - Ok(0) | Err(_) => break, - Ok(read) => stdout_session - .publish(MainOutput::Stdout(Bytes::copy_from_slice(&buffer[..read]))), + if let Some(mut tap) = tap { + while let Some(forward) = + crate::program_status::read_observed(&mut stdout, &mut buffer, &mut tap).await + { + stdout_session.publish(MainOutput::Stdout(Bytes::from(forward))); + } + } else { + loop { + match stdout.read(&mut buffer).await { + Ok(0) | Err(_) => break, + Ok(read) => stdout_session + .publish(MainOutput::Stdout(Bytes::copy_from_slice(&buffer[..read]))), + } } } stdout_session.reader_finished(); @@ -305,6 +331,19 @@ impl MainSession { self.output.publish(event); } + /// Program status reported by every workload terminal in this sandbox. + #[must_use] + pub fn program_activity(&self) -> &Arc { + &self.activity + } + + /// Record client input to the canonical terminal. + pub fn mark_input(&self) { + if let Some(marker) = &self.input_marker { + marker.mark(); + } + } + fn reader_finished(&self) { if self.readers_remaining.fetch_sub(1, Ordering::AcqRel) == 1 { self.readers_done.notify_waiters(); diff --git a/crates/openshell-supervisor-process/src/program_status.rs b/crates/openshell-supervisor-process/src/program_status.rs new file mode 100644 index 0000000000..687f041604 --- /dev/null +++ b/crates/openshell-supervisor-process/src/program_status.rs @@ -0,0 +1,1430 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Workload program status from OSC 7501 terminal reports. +//! +//! Agents such as Claude Code declare whether they are working, blocked on the +//! user, or done by writing OSC 7501 reports to their terminal. The supervisor +//! observes workload PTY output, keeps the per-terminal records the protocol +//! defines, and answers the support query itself so reporting does not depend +//! on the terminal of whichever client happens to be attached. +//! +//! Report bytes pass through unchanged. Only the support query is removed from +//! the forwarded stream, because the supervisor answers it and a second answer +//! from a supporting client terminal would reach the program as input. + +use std::collections::BTreeMap; +use std::sync::{Arc, Mutex}; +use std::time::Instant; + +use base64::Engine as _; +use tokio::sync::watch; + +const ESC: u8 = 0x1b; +const BEL: u8 = 0x07; +const CAN: u8 = 0x18; +const SUB: u8 = 0x1a; + +/// Largest legal sequence, OSC through ST. +const MAX_SEQUENCE: usize = 4096; +/// Records one terminal keeps before evicting the least recently updated. +const MAX_RECORDS: usize = 256; +const MAX_KEY: usize = 16; +const MAX_MSG_ENCODED: usize = 2732; +const MAX_TITLE_ENCODED: usize = 256; +const MAX_ID: usize = 128; +const MAX_ID_SEGMENT: usize = 32; +const MAX_ID_DEPTH: usize = 8; +const MAX_APP: usize = 32; + +const REPORT_PREFIX: &[u8] = b"7501;"; +const PROMPT_PREFIX: &[u8] = b"133;A"; +/// Both prefixes are this long, so one bounded buffer decides the sequence. +const PREFIX_LEN: usize = 5; +const QUERY_BODY: &[u8] = b"7501;?"; +const QUERY_ST: &[u8] = b"\x1b]7501;?\x1b\\"; +const QUERY_BEL: &[u8] = b"\x1b]7501;?\x07"; +// The reply lists the supported states and kinds so it never equals the query: +// a program that probes with terminal echo on reads its own echo, and a reply +// identical to the query would be answered again without end. +const REPLY_ST: &[u8] = + b"\x1b]7501;?:states=idle,working,done,blocked,error:kinds=permission,question,auth\x1b\\"; +const REPLY_BEL: &[u8] = + b"\x1b]7501;?:states=idle,working,done,blocked,error:kinds=permission,question,auth\x07"; + +#[derive(Clone, Copy, Debug, Eq, PartialEq, Ord, PartialOrd, Hash)] +pub enum ProgramState { + Idle, + Done, + Error, + Blocked, + Working, +} + +impl ProgramState { + #[must_use] + pub fn as_str(self) -> &'static str { + match self { + Self::Idle => "idle", + Self::Done => "done", + Self::Error => "error", + Self::Blocked => "blocked", + Self::Working => "working", + } + } + + /// Whether the program is waiting on the user rather than running. + #[must_use] + pub fn awaits_user(self) -> bool { + !matches!(self, Self::Working) + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq, Hash)] +pub enum BlockedKind { + Permission, + Question, + Auth, +} + +impl BlockedKind { + #[must_use] + pub fn as_str(self) -> &'static str { + match self { + Self::Permission => "permission", + Self::Question => "question", + Self::Auth => "auth", + } + } +} + +/// One record as the protocol defines it. Free text (`msg`, `title`) is +/// validated but never retained: it is untrusted program output. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct ProgramRecord { + pub state: ProgramState, + pub kind: Option, + pub app: Option, + pub progress: Option, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub enum Report { + /// Replace the record at `id`; the empty id is the root record. + Set { id: String, record: ProgramRecord }, + /// Remove the record at `id` and its descendants, or every record. + Clear { id: Option }, +} + +/// Terminator a query used, so the reply can match it. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum Terminator { + St, + Bel, +} + +/// Something in the output stream that changes a terminal's records. +#[derive(Clone, Debug, Eq, PartialEq)] +pub enum TerminalEvent { + Report(Report), + /// The shell started a new prompt (OSC 133 `A`), ending the program that + /// ran before it. + Prompt, + /// A full terminal reset (RIS, `ESC c`) removes every record. + Reset, +} + +#[derive(Debug, Default, Eq, PartialEq)] +pub struct Scan { + /// Output to forward to attached clients. + pub forward: Vec, + /// Record changes in the order they appeared in the output. + pub events: Vec, + /// Support queries removed from `forward`, in order. + pub queries: Vec, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum ScanState { + Ground, + Escape, + Osc, + OscEscape, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum Tracking { + Undecided, + Report, + Prompt, + Ignore, +} + +/// Streaming OSC scanner. Sequences may span reads; only the support query is +/// held back, and at most one query's worth of bytes at a time. +#[derive(Debug)] +pub struct Scanner { + state: ScanState, + tracking: Tracking, + body: Vec, + sequence_len: usize, + held: Vec, +} + +impl Default for Scanner { + fn default() -> Self { + Self::new() + } +} + +impl Scanner { + #[must_use] + pub fn new() -> Self { + Self { + state: ScanState::Ground, + tracking: Tracking::Ignore, + body: Vec::new(), + sequence_len: 0, + held: Vec::new(), + } + } + + pub fn scan(&mut self, input: &[u8]) -> Scan { + let mut scan = Scan { + forward: Vec::with_capacity(input.len()), + ..Scan::default() + }; + for &byte in input { + self.hold(byte, &mut scan); + self.parse(byte, &mut scan); + } + scan + } + + /// Whether bytes are held back waiting to see if they form a query. + #[must_use] + pub fn has_pending(&self) -> bool { + !self.held.is_empty() + } + + /// Release held bytes when the program stops writing mid-sequence, so a + /// lone ESC is never delayed indefinitely. + pub fn flush(&mut self) -> Vec { + std::mem::take(&mut self.held) + } + + fn hold(&mut self, byte: u8, scan: &mut Scan) { + if self.held.is_empty() { + if byte == ESC { + self.held.push(byte); + } else { + scan.forward.push(byte); + } + return; + } + self.held.push(byte); + if self.held == QUERY_ST || self.held == QUERY_BEL { + scan.queries.push(if byte == BEL { + Terminator::Bel + } else { + Terminator::St + }); + self.held.clear(); + } else if !QUERY_ST.starts_with(&self.held) && !QUERY_BEL.starts_with(&self.held) { + // An ESC that broke the match may begin the next query. + let restart = byte == ESC; + if restart { + self.held.pop(); + } + scan.forward.append(&mut self.held); + if restart { + self.held.push(ESC); + } + } + } + + fn parse(&mut self, byte: u8, scan: &mut Scan) { + match self.state { + ScanState::Ground => { + if byte == ESC { + self.state = ScanState::Escape; + } + } + ScanState::Escape => self.after_escape(byte, scan), + ScanState::Osc => match byte { + BEL => self.finish(scan), + ESC => self.state = ScanState::OscEscape, + CAN | SUB => self.state = ScanState::Ground, + _ => self.accumulate(byte), + }, + ScanState::OscEscape => { + if byte == b'\\' { + self.finish(scan); + } else { + // ESC interrupted the string; it introduces what follows. + self.after_escape(byte, scan); + } + } + } + } + + fn after_escape(&mut self, byte: u8, scan: &mut Scan) { + if byte == b'c' { + scan.events.push(TerminalEvent::Reset); + } + self.state = match byte { + b']' => { + self.tracking = Tracking::Undecided; + self.body.clear(); + // OSC plus the ST that ends it count toward the limit. + self.sequence_len = 4; + ScanState::Osc + } + ESC => ScanState::Escape, + _ => ScanState::Ground, + }; + } + + fn accumulate(&mut self, byte: u8) { + self.sequence_len += 1; + if self.sequence_len > MAX_SEQUENCE { + self.tracking = Tracking::Ignore; + self.body = Vec::new(); + return; + } + match self.tracking { + Tracking::Undecided => { + self.body.push(byte); + if !REPORT_PREFIX.starts_with(&self.body) && !PROMPT_PREFIX.starts_with(&self.body) + { + self.tracking = Tracking::Ignore; + } else if self.body.len() == PREFIX_LEN { + self.tracking = if self.body == REPORT_PREFIX { + Tracking::Report + } else { + Tracking::Prompt + }; + } + } + Tracking::Report => self.body.push(byte), + Tracking::Prompt | Tracking::Ignore => {} + } + } + + fn finish(&mut self, scan: &mut Scan) { + self.state = ScanState::Ground; + match self.tracking { + Tracking::Prompt => scan.events.push(TerminalEvent::Prompt), + Tracking::Report if self.body != QUERY_BODY => { + if let Some(report) = parse_report(&self.body[REPORT_PREFIX.len()..]) { + scan.events.push(TerminalEvent::Report(report)); + } + } + _ => {} + } + self.tracking = Tracking::Ignore; + self.body.clear(); + } +} + +fn is_value_byte(byte: u8) -> bool { + byte.is_ascii_alphanumeric() || b"_.,+/=-".contains(&byte) +} + +fn is_name_byte(byte: u8) -> bool { + byte.is_ascii_alphanumeric() || b"_.+-".contains(&byte) +} + +fn valid_id(id: &str) -> bool { + let segments: Vec<&str> = id.split('/').collect(); + id.len() <= MAX_ID + && segments.len() <= MAX_ID_DEPTH + && segments.iter().all(|segment| { + (1..=MAX_ID_SEGMENT).contains(&segment.len()) && segment.bytes().all(is_name_byte) + }) +} + +/// Validate base64 free text without keeping it. +fn valid_text(value: &str, max_encoded: usize) -> bool { + if value.len() > max_encoded { + return false; + } + let engine = base64::engine::GeneralPurpose::new( + &base64::alphabet::STANDARD, + base64::engine::GeneralPurposeConfig::new() + .with_decode_padding_mode(base64::engine::DecodePaddingMode::Indifferent), + ); + let Ok(decoded) = engine.decode(value) else { + return false; + }; + let Ok(text) = std::str::from_utf8(&decoded) else { + return false; + }; + !text.chars().any(char::is_control) +} + +/// Parse the pairs after `7501;`. `None` means the whole report is discarded. +fn parse_report(pairs: &[u8]) -> Option { + let pairs = std::str::from_utf8(pairs).ok()?; + let mut fields: BTreeMap<&str, &str> = BTreeMap::new(); + for pair in pairs.split(':') { + let Some((key, value)) = pair.split_once('=') else { + continue; + }; + let (key, value) = (key.trim(), value.trim()); + if key.is_empty() + || key.len() > MAX_KEY + || !key.bytes().all(|byte| byte.is_ascii_lowercase()) + || !value.bytes().all(is_value_byte) + { + continue; + } + fields.insert(key, value); + } + + let id = match fields.get("id") { + Some(id) if valid_id(id) => Some((*id).to_string()), + Some(_) => return None, + None => None, + }; + if fields + .get("msg") + .is_some_and(|msg| !valid_text(msg, MAX_MSG_ENCODED)) + || fields + .get("title") + .is_some_and(|title| !valid_text(title, MAX_TITLE_ENCODED)) + { + return None; + } + let state = match *fields.get("state")? { + "clear" => return Some(Report::Clear { id }), + "idle" => ProgramState::Idle, + "working" => ProgramState::Working, + "done" => ProgramState::Done, + "blocked" => ProgramState::Blocked, + "error" => ProgramState::Error, + // A state added later must never read as an older one. + _ => return None, + }; + let kind = (state == ProgramState::Blocked) + .then(|| match *fields.get("kind")? { + "permission" => Some(BlockedKind::Permission), + "question" => Some(BlockedKind::Question), + "auth" => Some(BlockedKind::Auth), + _ => None, + }) + .flatten(); + let app = fields + .get("app") + .filter(|app| (1..=MAX_APP).contains(&app.len()) && app.bytes().all(is_name_byte)) + .map(|app| (*app).to_string()); + let progress = matches!(state, ProgramState::Working | ProgramState::Blocked) + .then(|| { + fields + .get("progress")? + .parse::() + .ok() + .filter(|p| *p <= 100) + }) + .flatten(); + Some(Report::Set { + id: id.unwrap_or_default(), + record: ProgramRecord { + state, + kind, + app, + progress, + }, + }) +} + +/// The records one terminal holds. +#[derive(Debug, Default)] +struct Records { + records: BTreeMap, + next_update: u64, +} + +impl Records { + fn apply(&mut self, report: Report) { + match report { + Report::Set { id, record } => { + let update = self.next_update; + self.next_update += 1; + self.records.insert(id, (record, update)); + while self.records.len() > MAX_RECORDS { + let oldest = self + .records + .iter() + .min_by_key(|(_, (_, update))| *update) + .map(|(id, _)| id.clone()); + if let Some(oldest) = oldest { + self.records.remove(&oldest); + } + } + } + Report::Clear { id: None } => self.records.clear(), + Report::Clear { id: Some(id) } => { + let prefix = format!("{id}/"); + self.records + .retain(|record_id, _| record_id != &id && !record_id.starts_with(&prefix)); + } + } + } + + /// A new prompt or a process exit ends the records that describe a + /// running program. + fn drop_running(&mut self) { + self.records.retain(|_, (record, _)| { + !matches!(record.state, ProgramState::Working | ProgramState::Blocked) + }); + } + + fn app_for(&self, id: &str) -> Option { + let mut current = id; + loop { + if let Some((record, _)) = self.records.get(current) + && let Some(app) = &record.app + { + return Some(app.clone()); + } + if current.is_empty() { + return None; + } + current = current.rsplit_once('/').map_or("", |(parent, _)| parent); + } + } + + /// The most active record: working, then blocked, error, done, idle. + fn dominant(&self) -> Option<(ProgramState, Option, Option)> { + self.records + .iter() + .max_by_key(|(_, (record, update))| (record.state, *update)) + .map(|(id, (record, _))| (record.state, record.kind, self.app_for(id))) + } +} + +#[derive(Debug)] +struct TerminalStatus { + records: Records, + last_output: Instant, +} + +/// Sandbox-wide status that changes only on program state transitions. +#[derive(Clone, Debug, Default, Eq, PartialEq)] +pub struct ActivitySummary { + /// Most active state across terminals; `None` until a program reports. + pub state: Option, + pub kind: Option, + pub app: Option, + /// Open workload terminals. + pub terminals: usize, + /// Terminals with a current program report. + pub reporting_terminals: usize, + /// Increments on every state transition, including one that is undone + /// within the same read, so each transition is a distinct value. + pub revision: u64, +} + +/// Point-in-time activity including timestamps that change on every byte. +#[derive(Clone, Debug)] +pub struct ActivitySnapshot { + pub summary: ActivitySummary, + pub last_output: Option, + pub last_input: Option, + /// When `summary.state` last changed. + pub state_since: Instant, +} + +#[derive(Debug)] +struct Inner { + next_terminal: u64, + terminals: BTreeMap, + state_since: Instant, + /// Latest client input to any workload terminal, kept after the terminal + /// closes so a short session still counts as recent input. + last_input: Option, + revision: u64, +} + +impl Inner { + fn dominant_state(&self) -> Option { + self.terminals + .values() + .filter_map(|terminal| terminal.records.dominant()) + .map(|(state, _, _)| state) + .max() + } + + /// Record a transition when `before` differs from the current state. + fn note_transition(&mut self, before: Option) -> Option { + let after = self.dominant_state(); + if after != before { + self.state_since = Instant::now(); + self.revision += 1; + } + after + } +} + +/// Program status across every workload terminal in one sandbox. +#[derive(Debug)] +pub struct ProgramActivity { + inner: Mutex, + changes: watch::Sender, + /// Held from snapshot to enqueue, so pushes and request replies leave in + /// snapshot order and carry increasing sequence numbers. + report_order: tokio::sync::Mutex, +} + +impl ProgramActivity { + #[must_use] + pub fn new() -> Arc { + let (changes, _) = watch::channel(ActivitySummary::default()); + Arc::new(Self { + inner: Mutex::new(Inner { + next_terminal: 0, + terminals: BTreeMap::new(), + state_since: Instant::now(), + last_input: None, + revision: 0, + }), + changes, + report_order: tokio::sync::Mutex::new(0), + }) + } + + /// Notified only when the summary changes. + #[must_use] + pub fn subscribe(&self) -> watch::Receiver { + self.changes.subscribe() + } + + #[must_use] + pub fn snapshot(&self) -> ActivitySnapshot { + let inner = self.inner.lock().expect("program activity lock poisoned"); + ActivitySnapshot { + summary: summarize(&inner), + last_output: inner.terminals.values().map(|t| t.last_output).max(), + last_input: inner.last_input, + state_since: inner.state_since, + } + } + + /// Track a new workload terminal. `reply` writes to the program's input. + pub fn register( + self: &Arc, + reply: impl Fn(&[u8]) + Send + Sync + 'static, + ) -> TerminalTap { + let id = self.update(|inner| { + let id = inner.next_terminal; + inner.next_terminal += 1; + inner.terminals.insert( + id, + TerminalStatus { + records: Records::default(), + last_output: Instant::now(), + }, + ); + id + }); + TerminalTap { + activity: Arc::clone(self), + id, + scanner: Scanner::new(), + reply: Box::new(reply), + } + } + + fn update(&self, change: impl FnOnce(&mut Inner) -> T) -> T { + let mut inner = self.inner.lock().expect("program activity lock poisoned"); + let before = inner.dominant_state(); + let result = change(&mut inner); + inner.note_transition(before); + self.publish(&inner); + result + } + + /// Publish while holding the lock, so summaries leave in mutation order + /// and an older one never overwrites a newer one. + fn publish(&self, inner: &Inner) { + let after = summarize(inner); + self.changes.send_if_modified(|current| { + if *current == after { + false + } else { + *current = after; + true + } + }); + } + + /// Apply a terminal's output events in order, noting every transition so + /// the timer does not depend on how reads split the output. + fn apply_events(&self, id: u64, events: Vec) { + let mut inner = self.inner.lock().expect("program activity lock poisoned"); + let mut state = inner.dominant_state(); + let Some(terminal) = inner.terminals.get_mut(&id) else { + return; + }; + terminal.last_output = Instant::now(); + for event in events { + let Some(terminal) = inner.terminals.get_mut(&id) else { + break; + }; + match event { + TerminalEvent::Report(report) => terminal.records.apply(report), + TerminalEvent::Prompt => terminal.records.drop_running(), + TerminalEvent::Reset => terminal.records.apply(Report::Clear { id: None }), + } + state = inner.note_transition(state); + } + self.publish(&inner); + } + + fn touch_input(&self, id: u64) { + // Input never changes the summary; skip the change notification. + let mut inner = self.inner.lock().expect("program activity lock poisoned"); + if inner.terminals.contains_key(&id) { + inner.last_input = Some(Instant::now()); + } + } +} + +fn summarize(inner: &Inner) -> ActivitySummary { + let dominant = inner + .terminals + .values() + .filter_map(|terminal| terminal.records.dominant()) + .max_by_key(|(state, _, _)| *state); + ActivitySummary { + state: dominant.as_ref().map(|(state, _, _)| *state), + kind: dominant.as_ref().and_then(|(_, kind, _)| *kind), + app: dominant.and_then(|(_, _, app)| app), + terminals: inner.terminals.len(), + reporting_terminals: inner + .terminals + .values() + .filter(|terminal| !terminal.records.records.is_empty()) + .count(), + revision: inner.revision, + } +} + +/// Per-terminal view of the workload output stream. Dropping it ends the +/// terminal, which removes its records as a process exit would. +pub struct TerminalTap { + activity: Arc, + id: u64, + scanner: Scanner, + reply: ReplyFn, +} + +/// Writes supervisor-generated replies to the program's input. +type ReplyFn = Box; + +impl TerminalTap { + /// Observe program output and return the bytes to forward. + pub fn output(&mut self, data: &[u8]) -> Vec { + let scan = self.scanner.scan(data); + for terminator in &scan.queries { + (self.reply)(match terminator { + Terminator::St => REPLY_ST, + Terminator::Bel => REPLY_BEL, + }); + } + // Wire order matters: a prompt ends only the reports before it. + self.activity.apply_events(self.id, scan.events); + scan.forward + } + + #[must_use] + pub fn has_pending(&self) -> bool { + self.scanner.has_pending() + } + + pub fn flush(&mut self) -> Vec { + self.scanner.flush() + } + + /// A handle that records client input for this terminal. + #[must_use] + pub fn input_marker(&self) -> InputMarker { + InputMarker { + activity: Arc::clone(&self.activity), + id: self.id, + } + } +} + +impl Drop for TerminalTap { + fn drop(&mut self) { + let id = self.id; + self.activity.update(|inner| { + inner.terminals.remove(&id); + }); + } +} + +#[derive(Clone)] +pub struct InputMarker { + activity: Arc, + id: u64, +} + +impl InputMarker { + pub fn mark(&self) { + self.activity.touch_input(self.id); + } +} + +/// How long held query bytes wait for the rest of the sequence. +pub const PENDING_FLUSH: std::time::Duration = std::time::Duration::from_millis(20); + +/// Read the next chunk to forward, observing it through `tap`. Returns `None` +/// at end of stream after releasing any held bytes. +pub async fn read_observed( + reader: &mut R, + buffer: &mut [u8], + tap: &mut TerminalTap, +) -> Option> +where + R: tokio::io::AsyncRead + Unpin, +{ + use tokio::io::AsyncReadExt as _; + loop { + let read = if tap.has_pending() { + match tokio::time::timeout(PENDING_FLUSH, reader.read(buffer)).await { + Ok(read) => read, + Err(_) => return Some(tap.flush()), + } + } else { + reader.read(buffer).await + }; + match read { + Ok(0) | Err(_) => { + let rest = tap.flush(); + return (!rest.is_empty()).then_some(rest); + } + Ok(size) => { + let forward = tap.output(&buffer[..size]); + if !forward.is_empty() { + return Some(forward); + } + } + } + } +} + +/// Log sandbox-wide program state transitions until the activity is dropped. +pub fn spawn_transition_log(activity: &Arc) -> tokio::task::JoinHandle<()> { + let mut changes = activity.subscribe(); + let activity = Arc::downgrade(activity); + tokio::spawn(async move { + let mut last_state = None; + while changes.changed().await.is_ok() { + if activity.upgrade().is_none() { + break; + } + let summary = changes.borrow_and_update().clone(); + if summary.state == last_state { + continue; + } + last_state = summary.state; + tracing::debug!( + state = summary.state.map_or("unknown", ProgramState::as_str), + kind = summary.kind.map(BlockedKind::as_str), + app = summary.app.as_deref(), + terminals = summary.terminals, + reporting_terminals = summary.reporting_terminals, + "workload program status changed" + ); + } + }) +} + +/// How often the reporter checks for input that changed nothing else. +const INPUT_CHECK_INTERVAL: std::time::Duration = std::time::Duration::from_secs(5); +/// Minimum spacing of input-only reports while a client keeps typing. New +/// input is always reported within this interval plus one check, including +/// the last keystroke before the client stops. +const INPUT_REPORT_INTERVAL: std::time::Duration = std::time::Duration::from_secs(10); + +fn proto_state(state: Option) -> openshell_core::proto::WorkloadProgramState { + use openshell_core::proto::WorkloadProgramState as Proto; + match state { + None => Proto::Unspecified, + Some(ProgramState::Idle) => Proto::Idle, + Some(ProgramState::Working) => Proto::Working, + Some(ProgramState::Done) => Proto::Done, + Some(ProgramState::Blocked) => Proto::Blocked, + Some(ProgramState::Error) => Proto::Error, + } +} + +fn proto_kind(kind: Option) -> openshell_core::proto::WorkloadBlockedKind { + use openshell_core::proto::WorkloadBlockedKind as Proto; + match kind { + None => Proto::Unspecified, + Some(BlockedKind::Permission) => Proto::Permission, + Some(BlockedKind::Question) => Proto::Question, + Some(BlockedKind::Auth) => Proto::Auth, + } +} + +fn proto_age(since: Instant) -> Option { + openshell_core::time::duration_from_std(since.elapsed()).ok() +} + +/// The gateway-bound report for a snapshot. Ages are measured now. +#[must_use] +pub fn activity_report( + snapshot: &ActivitySnapshot, +) -> openshell_core::proto::SupervisorWorkloadActivity { + openshell_core::proto::SupervisorWorkloadActivity { + state: proto_state(snapshot.summary.state).into(), + blocked_kind: proto_kind(snapshot.summary.kind).into(), + app: snapshot.summary.app.clone().unwrap_or_default(), + state_age: proto_age(snapshot.state_since), + input_age: snapshot.last_input.and_then(proto_age), + reporting_terminals: u32::try_from(snapshot.summary.reporting_terminals) + .unwrap_or(u32::MAX), + request_id: String::new(), + sequence: 0, + } +} + +impl ProgramActivity { + /// Snapshot the summary and enqueue it on `tx` as one ordered step. + /// `request_id` is set when the summary answers a gateway request. + /// Returns the snapshot that was sent. + /// + /// # Errors + /// + /// Returns an error when the session's outbound channel has closed. + pub async fn send_report( + &self, + tx: &tokio::sync::mpsc::Sender, + request_id: &str, + ) -> Result { + let mut sequence = self.report_order.lock().await; + *sequence += 1; + let snapshot = self.snapshot(); + let mut report = activity_report(&snapshot); + report.request_id = request_id.to_string(); + report.sequence = *sequence; + let message = openshell_core::proto::SupervisorMessage { + payload: Some( + openshell_core::proto::supervisor_message::Payload::WorkloadActivity(report), + ), + }; + tx.send(message).await.map_err(|_| ())?; + Ok(snapshot) + } +} + +/// Aborts the reporter when the session that owns it ends. +pub struct ReporterGuard(tokio::task::JoinHandle<()>); + +impl Drop for ReporterGuard { + fn drop(&mut self) { + self.0.abort(); + } +} + +/// Report activity on one gateway session: once at start, on every summary +/// change, and within seconds of new client input. +pub fn spawn_activity_reporter( + activity: &Arc, + tx: tokio::sync::mpsc::Sender, +) -> ReporterGuard { + let activity = Arc::clone(activity); + let mut changes = activity.subscribe(); + ReporterGuard(tokio::spawn(async move { + let mut input_check = tokio::time::interval(INPUT_CHECK_INTERVAL); + input_check.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); + let mut reported_input: Option = None; + let mut last_send: Option = None; + let mut send_now = true; + loop { + let snapshot = activity.snapshot(); + let input_advanced = snapshot.last_input != reported_input + && last_send.is_none_or(|sent| sent.elapsed() >= INPUT_REPORT_INTERVAL); + if send_now || input_advanced { + let Ok(sent) = activity.send_report(&tx, "").await else { + return; + }; + reported_input = sent.last_input; + last_send = Some(Instant::now()); + } + tokio::select! { + result = changes.changed() => { + if result.is_err() { + return; + } + changes.borrow_and_update(); + send_now = true; + } + _ = input_check.tick() => send_now = false, + } + } + })) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::Mutex as StdMutex; + use tokio::io::AsyncWriteExt as _; + + fn set(state: ProgramState) -> Report { + Report::Set { + id: String::new(), + record: ProgramRecord { + state, + kind: None, + app: None, + progress: None, + }, + } + } + + fn report(body: &str) -> Option { + parse_report(body.as_bytes()) + } + + #[test] + fn parses_blocked_report_with_kind_app_and_message() { + let parsed = + report("state=blocked:kind=permission:app=claude-code:msg=QWxsb3cgcm0/").unwrap(); + assert_eq!( + parsed, + Report::Set { + id: String::new(), + record: ProgramRecord { + state: ProgramState::Blocked, + kind: Some(BlockedKind::Permission), + app: Some("claude-code".into()), + progress: None, + }, + } + ); + } + + #[test] + fn discards_unknown_state_missing_state_and_bad_ids() { + assert_eq!(report("state=sleeping"), None); + assert_eq!(report("app=cargo"), None); + assert_eq!(report("state=idle:id=a//b"), None); + assert_eq!(report("state=idle:id=a/b/c/d/e/f/g/h/i"), None); + } + + #[test] + fn discards_message_with_control_characters_or_bad_base64() { + // "a\nb" + assert_eq!(report("state=done:msg=YQpi"), None); + assert_eq!(report("state=done:msg=A"), None); + // Bytes outside the value alphabet make the pair malformed, not the report. + assert!(report("state=done:msg=@@@").is_some()); + assert!(report("state=done:msg=b2s").is_some()); + } + + #[test] + fn skips_malformed_pairs_and_ignores_irrelevant_fields() { + let parsed = report("bogus: state = working :progress=101:kind=auth:Key=x").unwrap(); + let Report::Set { record, .. } = parsed else { + panic!("expected set"); + }; + assert_eq!(record.state, ProgramState::Working); + assert_eq!(record.progress, None); + assert_eq!(record.kind, None); + let Report::Set { record, .. } = report("state=working:progress=42").unwrap() else { + panic!("expected set"); + }; + assert_eq!(record.progress, Some(42)); + } + + #[test] + fn clear_without_id_clears_everything() { + assert_eq!(report("state=clear"), Some(Report::Clear { id: None })); + assert_eq!( + report("state=clear:id=build"), + Some(Report::Clear { + id: Some("build".into()) + }) + ); + } + + #[test] + fn scanner_passes_reports_through_and_strips_queries() { + let mut scanner = Scanner::new(); + let input = b"hi\x1b]7501;?\x1b\\\x1b[c\x1b]7501;state=working\x07there"; + let scan = scanner.scan(input); + assert_eq!(scan.queries, vec![Terminator::St]); + assert_eq!( + scan.events, + vec![TerminalEvent::Report(set(ProgramState::Working))] + ); + assert_eq!( + scan.forward, + b"hi\x1b[c\x1b]7501;state=working\x07there".to_vec() + ); + assert!(!scanner.has_pending()); + } + + #[test] + fn scanner_handles_sequences_split_across_reads() { + let input = b"\x1b]7501;?\x07\x1b]7501;state=done\x1b\\ok"; + for split in 0..input.len() { + let mut scanner = Scanner::new(); + let first = scanner.scan(&input[..split]); + let second = scanner.scan(&input[split..]); + let mut forward = first.forward; + forward.extend(second.forward); + assert_eq!( + forward, + b"\x1b]7501;state=done\x1b\\ok".to_vec(), + "split {split}" + ); + assert_eq!( + [first.queries, second.queries].concat(), + vec![Terminator::Bel] + ); + assert_eq!( + [first.events, second.events].concat(), + vec![TerminalEvent::Report(set(ProgramState::Done))] + ); + } + } + + #[test] + fn scanner_flushes_lone_escape_and_restarts_on_new_escape() { + let mut scanner = Scanner::new(); + let scan = scanner.scan(b"a\x1b"); + assert_eq!(scan.forward, b"a".to_vec()); + assert!(scanner.has_pending()); + assert_eq!(scanner.flush(), b"\x1b".to_vec()); + + let mut scanner = Scanner::new(); + let scan = scanner.scan(b"\x1b]75\x1b]7501;?\x07"); + assert_eq!(scan.forward, b"\x1b]75".to_vec()); + assert_eq!(scan.queries, vec![Terminator::Bel]); + } + + #[test] + fn scanner_ignores_oversized_and_other_sequences() { + let mut scanner = Scanner::new(); + let mut input = b"\x1b]7501;state=working:msg=".to_vec(); + input.extend(std::iter::repeat_n(b'A', MAX_SEQUENCE)); + input.extend(b"\x1b\\\x1b]0;title\x07\x1b]133;A\x07"); + let scan = scanner.scan(&input); + assert_eq!(scan.events, vec![TerminalEvent::Prompt]); + assert_eq!(scan.forward, input); + } + + #[test] + fn records_clear_descendants_and_inherit_app() { + let mut records = Records::default(); + records.apply(report("state=working:app=cargo:id=build").unwrap()); + records.apply(report("state=done:id=build/test").unwrap()); + records.apply(report("state=idle:id=other").unwrap()); + assert_eq!( + records.dominant(), + Some((ProgramState::Working, None, Some("cargo".into()))) + ); + assert_eq!(records.app_for("build/test"), Some("cargo".into())); + records.apply(Report::Clear { + id: Some("build".into()), + }); + assert_eq!(records.dominant(), Some((ProgramState::Idle, None, None))); + records.drop_running(); + assert_eq!(records.records.len(), 1); + } + + #[test] + fn records_evict_least_recently_updated_over_cap() { + let mut records = Records::default(); + for index in 0..=MAX_RECORDS { + records.apply(report(&format!("state=idle:id=r{index}")).unwrap()); + } + assert_eq!(records.records.len(), MAX_RECORDS); + assert!(!records.records.contains_key("r0")); + } + + #[tokio::test] + async fn activity_answers_queries_and_aggregates_terminals() { + let activity = ProgramActivity::new(); + let mut changes = activity.subscribe(); + let replies = Arc::new(StdMutex::new(Vec::>::new())); + let sink = Arc::clone(&replies); + let mut main = activity.register(move |bytes| { + sink.lock().unwrap().push(bytes.to_vec()); + }); + let mut shell = activity.register(|_| {}); + + let forward = main.output(b"\x1b]7501;?\x1b\\\x1b]7501;state=done:app=claude-code\x1b\\"); + assert_eq!(replies.lock().unwrap().as_slice(), &[REPLY_ST.to_vec()]); + assert_eq!( + forward, + b"\x1b]7501;state=done:app=claude-code\x1b\\".to_vec() + ); + assert!(changes.has_changed().unwrap()); + let summary = changes.borrow_and_update().clone(); + assert_eq!(summary.state, Some(ProgramState::Done)); + assert_eq!(summary.app.as_deref(), Some("claude-code")); + assert_eq!(summary.terminals, 2); + assert_eq!(summary.reporting_terminals, 1); + + shell.output(b"\x1b]7501;state=working\x07"); + assert_eq!( + activity.snapshot().summary.state, + Some(ProgramState::Working) + ); + + shell.input_marker().mark(); + assert!(activity.snapshot().last_input.is_some()); + + // A new prompt ends the shell's running program. + shell.output(b"\x1b]133;A\x07"); + assert_eq!(activity.snapshot().summary.state, Some(ProgramState::Done)); + + drop(main); + drop(shell); + let summary = activity.snapshot().summary; + assert_eq!(summary.state, None); + assert_eq!(summary.terminals, 0); + } + + #[test] + fn prompts_and_reports_apply_in_wire_order_for_every_chunking() { + let input: &[u8] = b"\x1b]7501;state=working\x07\x1b]133;A\x07"; + let finals: Vec> = (0..=input.len()) + .map(|split| { + let activity = ProgramActivity::new(); + let mut tap = activity.register(|_| {}); + tap.output(&input[..split]); + tap.output(&input[split..]); + activity.snapshot().summary.state + }) + .collect(); + assert!( + finals.iter().all(Option::is_none), + "a prompt after a report ends it however the bytes arrive: {finals:?}" + ); + + // A report after the prompt belongs to the next program. + let activity = ProgramActivity::new(); + let mut tap = activity.register(|_| {}); + tap.output(b"\x1b]133;A\x07\x1b]7501;state=blocked\x07"); + assert_eq!( + activity.snapshot().summary.state, + Some(ProgramState::Blocked) + ); + } + + #[tokio::test] + async fn a_turn_within_one_read_restarts_the_timer_and_notifies() { + let activity = ProgramActivity::new(); + let mut tap = activity.register(|_| {}); + tap.output(b"\x1b]7501;state=done\x07"); + let mut changes = activity.subscribe(); + changes.borrow_and_update(); + let before = activity.snapshot(); + tokio::time::sleep(std::time::Duration::from_millis(5)).await; + tap.output(b"\x1b]7501;state=working\x07\x1b]7501;state=done\x07"); + let after = activity.snapshot(); + assert_eq!(after.summary.state, Some(ProgramState::Done)); + assert!( + after.state_since > before.state_since, + "the new turn restarts the wait" + ); + assert_eq!(after.summary.revision, before.summary.revision + 2); + assert!(changes.has_changed().unwrap(), "the reporter is told"); + } + + #[test] + fn concurrent_terminals_publish_every_transition_in_order() { + let activity = ProgramActivity::new(); + let mut changes = activity.subscribe(); + let mut threads = Vec::new(); + for _ in 0..4 { + let activity = Arc::clone(&activity); + threads.push(std::thread::spawn(move || { + let mut tap = activity.register(|_| {}); + for _ in 0..200 { + tap.output(b"\x1b]7501;state=working\x07"); + tap.output(b"\x1b]7501;state=done\x07"); + } + std::mem::forget(tap); + })); + } + for thread in threads { + thread.join().unwrap(); + } + // The last published summary is the final state, never an older one. + assert_eq!(*changes.borrow_and_update(), activity.snapshot().summary); + } + + fn received( + message: openshell_core::proto::SupervisorMessage, + ) -> openshell_core::proto::SupervisorWorkloadActivity { + let Some(openshell_core::proto::supervisor_message::Payload::WorkloadActivity(report)) = + message.payload + else { + panic!("expected workload activity"); + }; + report + } + + #[tokio::test] + async fn replies_carry_the_request_id_and_every_summary_a_new_sequence() { + let activity = ProgramActivity::new(); + let (tx, mut rx) = tokio::sync::mpsc::channel(4); + activity.send_report(&tx, "req-1").await.unwrap(); + activity.send_report(&tx, "").await.unwrap(); + let reply = received(rx.recv().await.unwrap()); + let push = received(rx.recv().await.unwrap()); + assert_eq!(reply.request_id, "req-1"); + assert!(push.request_id.is_empty()); + assert!(push.sequence > reply.sequence); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn replies_and_pushes_leave_in_snapshot_order() { + let activity = ProgramActivity::new(); + let mut tap = activity.register(|_| {}); + let (tx, mut rx) = tokio::sync::mpsc::channel(256); + let mut senders = Vec::new(); + for index in 0..32 { + let activity = Arc::clone(&activity); + let tx = tx.clone(); + senders.push(tokio::spawn(async move { + let request = if index % 2 == 0 { "req" } else { "" }; + activity.send_report(&tx, request).await.unwrap(); + })); + let report: &[u8] = if index % 2 == 0 { + b"\x1b]7501;state=working\x07" + } else { + b"\x1b]7501;state=done\x07" + }; + tap.output(report); + } + for sender in senders { + sender.await.unwrap(); + } + activity.send_report(&tx, "").await.unwrap(); + drop(tx); + let mut last = 0; + let mut final_state = 0; + while let Some(message) = rx.recv().await { + let report = received(message); + assert!( + report.sequence > last, + "sequence {} after {last}", + report.sequence + ); + last = report.sequence; + final_state = report.state; + } + assert_eq!( + final_state, + i32::from(openshell_core::proto::WorkloadProgramState::Done), + "the newest summary is the last one sent" + ); + } + + #[test] + fn full_terminal_reset_clears_records() { + let activity = ProgramActivity::new(); + let mut tap = activity.register(|_| {}); + let forward = tap.output(b"\x1b]7501;state=done\x07\x1bc"); + assert_eq!(forward, b"\x1b]7501;state=done\x07\x1bc".to_vec()); + assert_eq!(activity.snapshot().summary.state, None); + // ESC c interrupting an unterminated OSC also resets. + tap.output(b"\x1b]7501;state=done\x07\x1b]7501;state=wor\x1bc"); + assert_eq!(activity.snapshot().summary.state, None); + } + + #[test] + fn reports_without_a_probe_count_and_input_outlives_its_terminal() { + let activity = ProgramActivity::new(); + let mut probing = activity.register(|_| {}); + probing.output(b"\x1b]7501;?\x07"); + let mut codex = activity.register(|_| {}); + codex.output(b"\x1b]7501;state=working:app=codex\x07"); + assert_eq!(activity.snapshot().summary.reporting_terminals, 1); + + let marker = probing.input_marker(); + marker.mark(); + let typed = activity.snapshot().last_input; + drop(probing); + assert!(typed.is_some()); + assert_eq!(activity.snapshot().last_input, typed); + } + + #[test] + fn echoed_replies_are_not_queries() { + for reply in [REPLY_ST, REPLY_BEL] { + let scan = Scanner::new().scan(reply); + assert!(scan.queries.is_empty()); + assert!(scan.events.is_empty()); + assert_eq!(scan.forward, reply.to_vec()); + } + } + + #[tokio::test] + async fn reporter_sends_on_start_and_on_every_state_change() { + use openshell_core::proto::{WorkloadProgramState, supervisor_message::Payload}; + let activity = ProgramActivity::new(); + let mut tap = activity.register(|_| {}); + let (tx, mut rx) = tokio::sync::mpsc::channel(8); + let guard = spawn_activity_reporter(&activity, tx); + let next_state = async |rx: &mut tokio::sync::mpsc::Receiver<_>| { + let message: openshell_core::proto::SupervisorMessage = rx.recv().await.unwrap(); + let Some(Payload::WorkloadActivity(report)) = message.payload else { + panic!("expected workload activity"); + }; + report.state + }; + assert_eq!( + next_state(&mut rx).await, + i32::from(WorkloadProgramState::Unspecified) + ); + tap.output(b"\x1b]7501;state=working:app=claude-code\x07"); + assert_eq!( + next_state(&mut rx).await, + i32::from(WorkloadProgramState::Working) + ); + tap.output(b"\x1b]7501;state=done:app=claude-code\x07"); + assert_eq!( + next_state(&mut rx).await, + i32::from(WorkloadProgramState::Done) + ); + drop(guard); + tokio::task::yield_now().await; + tap.output(b"\x1b]7501;state=working\x07"); + assert!(rx.recv().await.is_none(), "the guard ends the reporter"); + } + + #[tokio::test] + async fn read_observed_releases_held_bytes_after_a_pause() { + let (mut writer, mut reader) = tokio::io::duplex(64); + let activity = ProgramActivity::new(); + let mut tap = activity.register(|_| {}); + let mut buffer = [0_u8; 16]; + writer.write_all(b"x\x1b").await.unwrap(); + assert_eq!( + read_observed(&mut reader, &mut buffer, &mut tap).await, + Some(b"x".to_vec()) + ); + assert_eq!( + read_observed(&mut reader, &mut buffer, &mut tap).await, + Some(b"\x1b".to_vec()) + ); + drop(writer); + assert_eq!( + read_observed(&mut reader, &mut buffer, &mut tap).await, + None + ); + } +} diff --git a/crates/openshell-supervisor-process/src/ssh.rs b/crates/openshell-supervisor-process/src/ssh.rs index a11eddbd21..9bef74d596 100644 --- a/crates/openshell-supervisor-process/src/ssh.rs +++ b/crates/openshell-supervisor-process/src/ssh.rs @@ -410,6 +410,8 @@ struct ChannelState { main_input_pending: bool, main_detach_prefix_pending: bool, main_output_task: Option, + /// Records client input for a PTY session's program status. + input_marker: Option, } enum InputSender { @@ -977,6 +979,9 @@ impl russh::server::Handler for SshHandler { return Ok(()); }; if !state.main_attached { + if let Some(marker) = &state.input_marker { + marker.mark(); + } // Russh replenishes receive credit before this callback. Waiting // for space here would block signals, close, and output credit on // the same SSH connection. Reject overflow before copying input. @@ -1023,6 +1028,12 @@ impl russh::server::Handler for SshHandler { state.main_input_pending = false; session.extended_data(channel, 1, b"openshell: input enabled\r\n".to_vec())?; } + if !forward.is_empty() + && state.main_input_owner.is_some() + && let Some(main_session) = self.main_session.as_ref() + { + main_session.mark_input(); + } let error = (!forward.is_empty()) .then(|| state.input_sender.as_ref()?.send(forward).err()) .flatten(); @@ -1362,6 +1373,10 @@ impl SshHandler { .exec(spec) .await .map_err(|error| anyhow::anyhow!(error.to_string()))?; + let activity = self + .main_session + .as_ref() + .map(|main_session| main_session.program_activity().clone()); let state = self .channels .get_mut(&channel) @@ -1395,6 +1410,19 @@ impl SshHandler { .input_task .as_ref() .map(input::InputTask::abort_handle); + // Interactive PTY programs may report their status over OSC 7501. + let tap = match (&activity, &state.terminal, &state.input_sender) { + (Some(activity), Some(_), Some(InputSender::Process(sender))) => { + let reply = sender.reply_handle(); + Some(activity.register(move |bytes| { + let _ = reply.send(bytes); + })) + } + _ => None, + }; + state.input_marker = tap + .as_ref() + .map(crate::program_status::TerminalTap::input_marker); // Upload stdout carries only credit, so extraction output uses stderr. let stdout_extended = finisher.is_some(); @@ -1402,6 +1430,14 @@ impl SshHandler { let stdout_handle = handle.clone(); let stdout_task = tokio::spawn(async move { let mut buffer = [0_u8; 4096]; + if let Some(mut tap) = tap { + while let Some(data) = + crate::program_status::read_observed(&mut stdout, &mut buffer, &mut tap).await + { + let _ = stdout_handle.data(channel, data).await; + } + return; + } loop { match stdout.read(&mut buffer).await { Ok(0) | Err(_) => break, diff --git a/crates/openshell-supervisor-process/src/ssh/input.rs b/crates/openshell-supervisor-process/src/ssh/input.rs index 46ca1dfa85..49a409cd82 100644 --- a/crates/openshell-supervisor-process/src/ssh/input.rs +++ b/crates/openshell-supervisor-process/src/ssh/input.rs @@ -4,7 +4,7 @@ //! Bounded stdin delivery without blocking the shared SSH session callback. use std::collections::VecDeque; -use std::sync::{Arc, Mutex}; +use std::sync::{Arc, Mutex, Weak}; use openshell_isolation_interface::contract::BoundaryInput; use tokio::io::AsyncWriteExt; @@ -54,9 +54,9 @@ pub(super) enum SendError { Closed, } -impl InputSender { - pub(super) fn send(&self, bytes: &[u8]) -> Result<(), SendError> { - let mut buffer = self.0.buffer.lock().map_err(|_| SendError::Closed)?; +impl Shared { + fn send(&self, bytes: &[u8]) -> Result<(), SendError> { + let mut buffer = self.buffer.lock().map_err(|_| SendError::Closed)?; if buffer.closed { return Err(SendError::Closed); } @@ -65,11 +65,31 @@ impl InputSender { } buffer.bytes.extend(bytes); drop(buffer); - self.0.ready.notify_one(); + self.ready.notify_one(); Ok(()) } } +impl InputSender { + pub(super) fn send(&self, bytes: &[u8]) -> Result<(), SendError> { + self.0.send(bytes) + } + + /// A sender for supervisor-generated replies. It does not keep stdin open: + /// EOF still follows the client's sender. + pub(super) fn reply_handle(&self) -> ReplyHandle { + ReplyHandle(Arc::downgrade(&self.0)) + } +} + +pub(super) struct ReplyHandle(Weak); + +impl ReplyHandle { + pub(super) fn send(&self, bytes: &[u8]) -> Result<(), SendError> { + self.0.upgrade().ok_or(SendError::Closed)?.send(bytes) + } +} + impl Drop for InputSender { fn drop(&mut self) { if let Ok(mut buffer) = self.0.buffer.lock() { diff --git a/crates/openshell-supervisor-process/src/supervisor_session.rs b/crates/openshell-supervisor-process/src/supervisor_session.rs index a6a05638ad..2dc7eb49ac 100644 --- a/crates/openshell-supervisor-process/src/supervisor_session.rs +++ b/crates/openshell-supervisor-process/src/supervisor_session.rs @@ -136,6 +136,8 @@ pub struct SessionRuntimeContext { /// Peer endpoint of the gateway replica serving this supervisor's shared /// host. When set, sessions dial it instead of the configured address. pub gateway_home: Option>>, + /// Program status from workload terminals, reported to every session. + pub workload_activity: Option>, } /// Parse a gRPC endpoint URI into an OCSF `Endpoint` (host + port). Falls back @@ -432,6 +434,7 @@ pub fn spawn_with_readiness( config_apply_tx: runtime.config_apply_tx, config_push_updates: runtime.config_push_updates, gateway_home: runtime.gateway_home, + workload_activity: runtime.workload_activity, ready_tx, }; (tokio::spawn(run_session_loop(config)), ready_rx) @@ -453,6 +456,7 @@ struct SessionConfig { /// reconnect to a gateway that does not push resumes polling. config_push_updates: watch::Sender, gateway_home: Option>>, + workload_activity: Option>, ready_tx: watch::Sender, } @@ -714,6 +718,7 @@ async fn run_single_session( supports_session_redirect: true, supports_config_push: true, supports_service_advertisements: config.app_installations.is_some(), + supports_workload_activity_requests: config.workload_activity.is_some(), })), }) .await @@ -793,6 +798,12 @@ async fn run_single_session( spawn_config_apply(apply_tx.clone(), bootstrap, tx.clone(), &config.sandbox_id); } + // Each session starts with a full report; the task ends with the session. + let _activity_reporter = config + .workload_activity + .as_ref() + .map(|activity| crate::program_status::spawn_activity_reporter(activity, tx.clone())); + // Main loop: receive gateway messages + send heartbeats. let mut heartbeat_interval = tokio::time::interval(Duration::from_secs(heartbeat_secs)); heartbeat_interval.tick().await; // skip immediate tick @@ -823,6 +834,16 @@ async fn run_single_session( owner_replica_id: r.owner_replica_id.clone(), }); } + if let Some(gateway_message::Payload::WorkloadActivityRequest(request)) = &msg.payload { + // Answer before an idle stop so it sees current input and work. + if let Some(activity) = &config.workload_activity { + activity + .send_report(&tx, &request.request_id) + .await + .map_err(|()| "outbound channel closed")?; + } + continue; + } if let Some(gateway_message::Payload::ServiceRouteLease(lease)) = &msg.payload { if let Some(source) = &config.app_installations { let duration = service_route_lease_remaining(lease); diff --git a/docs/how-it-works/sandboxes/overview.mdx b/docs/how-it-works/sandboxes/overview.mdx index 075447eec2..c8d756311f 100644 --- a/docs/how-it-works/sandboxes/overview.mdx +++ b/docs/how-it-works/sandboxes/overview.mdx @@ -1022,6 +1022,40 @@ If an infrastructure failure leaves a sandbox in `Error`, fix the reported cause For a provisioning timeout, wait for cleanup to finish, then run `sandbox start` directly. +### Stop Idle Sandboxes + +The gateway can stop a sandbox whose agent is waiting for you. Agents that +report their state with the OSC 7501 program status protocol, such as Claude +Code and Codex, tell the supervisor when a turn is working, done, or blocked +on a prompt. Set `idle_stop_after_seconds` to stop a sandbox after its +agent has waited that long with no terminal input: + +```shell +openshell settings set my-sandbox --key idle_stop_after_seconds --value 1800 +openshell settings set --global --key idle_stop_after_seconds --value 3600 +``` + +A global value overrides per-sandbox values. Zero or unset disables idle stop. +A sandbox keeps running while its agent reports `working`, and a sandbox whose +programs never report status is never stopped. `openshell sandbox get` shows the +latest report as the `ProgramStatus` condition. + +A waiting report describes only the program that sent it. Idle stop does not +see other work in the same sandbox, such as a build in another terminal, a +command started with `openshell sandbox exec`, or a command you run in the +agent's shell after the agent exits. If an agent waits past the timeout while +that work runs, the sandbox stops and the work is lost. Enable idle stop only +for sandboxes where the agent is the only long-running work. + +An idle stop moves the sandbox to `Stopped` with the reason `IdleStopped`. +`openshell sandbox connect` starts such a sandbox before connecting. If the stop +is still in progress, connect waits for it to finish first. Running +`openshell sandbox stop` on it makes the stop explicit, and connect then +reports it as stopped instead. Process +memory is not retained: the canonical command starts fresh, so resume the +agent's conversation with its own option, such as `claude --continue`. Workspace +files persist as they do for any stop. + ### Update the Image of a Stopped Sandbox Use `openshell.v1.SandboxUpdates/UpdateSandbox` to change a stopped sandbox's diff --git a/proto/openshell.proto b/proto/openshell.proto index 6ad3e9e5cd..57745bd669 100644 --- a/proto/openshell.proto +++ b/proto/openshell.proto @@ -3463,9 +3463,31 @@ message SupervisorMessage { RelayClose relay_close = 4; ConfigUpdateResult config_update_result = 5; SupervisorServiceAdvertisement service_advertisement = 6; + SupervisorWorkloadActivity workload_activity = 7; } } +// Program status summary across workload terminals. Sent when each session +// starts, when the summary changes, and within seconds of new client input. +// Ages are relative so supervisor and gateway clocks need not agree. +message SupervisorWorkloadActivity { + WorkloadProgramState state = 1; + WorkloadBlockedKind blocked_kind = 2; + string app = 3; + // Time spent in `state`. + google.protobuf.Duration state_age = 4; + // Time since the latest client input to a workload terminal. Absent when no + // client has written to one. + google.protobuf.Duration input_age = 5; + // Workload terminals with a current program report. + uint32 reporting_terminals = 6; + // Set when this summary answers a WorkloadActivityRequest. + string request_id = 7; + // Increases with every summary the supervisor sends, pushes and replies + // alike. The gateway drops a summary older than the one it holds. + uint64 sequence = 8; +} + // Envelope for gateway-to-supervisor messages on the ConnectSupervisor stream. message GatewayMessage { oneof payload { @@ -3477,9 +3499,18 @@ message GatewayMessage { SessionRedirect session_redirect = 6; ConfigUpdate config_update = 7; ServiceRouteLease service_route_lease = 8; + WorkloadActivityRequest workload_activity_request = 9; } } +// Ask the supervisor for a current program status summary before an idle +// stop. Sent only to supervisors that advertise +// `supports_workload_activity_requests`. The reply is a +// SupervisorWorkloadActivity carrying the same `request_id`. +message WorkloadActivityRequest { + string request_id = 1; +} + // Supervisor identifies itself and the sandbox it manages. message SupervisorHello { // Sandbox ID this supervisor manages. @@ -3506,6 +3537,10 @@ message SupervisorHello { // Gateways push configuration only to supervisors that set this. bool supports_config_push = 7; bool supports_service_advertisements = 8; + // The supervisor reports workload program status and answers + // WorkloadActivityRequest. Gateways idle-stop only sandboxes whose current + // supervisor confirms idleness this way. + bool supports_workload_activity_requests = 9; } // Gateway accepts the supervisor session. @@ -4188,6 +4223,28 @@ enum SandboxRestartPolicy { SANDBOX_RESTART_POLICY_ALWAYS = 3; } +// Program state declared by an OSC 7501 report. Kept after existing enums so +// generated enum descriptors remain stable. +enum WorkloadProgramState { + WORKLOAD_PROGRAM_STATE_UNSPECIFIED = 0; + // At rest, waiting for the user's next instruction. + WORKLOAD_PROGRAM_STATE_IDLE = 1; + WORKLOAD_PROGRAM_STATE_WORKING = 2; + // Finished; the result is ready for the user. + WORKLOAD_PROGRAM_STATE_DONE = 3; + // Cannot continue until the user acts. + WORKLOAD_PROGRAM_STATE_BLOCKED = 4; + WORKLOAD_PROGRAM_STATE_ERROR = 5; +} + +// What a blocked program waits for. +enum WorkloadBlockedKind { + WORKLOAD_BLOCKED_KIND_UNSPECIFIED = 0; + WORKLOAD_BLOCKED_KIND_PERMISSION = 1; + WORKLOAD_BLOCKED_KIND_QUESTION = 2; + WORKLOAD_BLOCKED_KIND_AUTH = 3; +} + // Workspace membership record. message WorkspaceMember { openshell.datamodel.v1.ObjectMeta metadata = 1; diff --git a/rfc/0025-workload-idle-stop/README.md b/rfc/0025-workload-idle-stop/README.md new file mode 100644 index 0000000000..ece0aed2e6 --- /dev/null +++ b/rfc/0025-workload-idle-stop/README.md @@ -0,0 +1,480 @@ +--- +authors: + - "@mrunalp" +state: draft +links: + - https://github.com/ai-openshell/openshell/issues/5852 +--- + +# RFC NNNN - Stop Idle Agent Sandboxes + + + +## Summary + +The supervisor reads the program status that coding agents already write to +their terminal with OSC 7501, the Program Status Protocol. It reports a +sandbox-wide activity summary to the gateway, which can then stop a sandbox +whose agent has been waiting on the user for longer than a configured timeout. +A later `openshell sandbox connect` starts the sandbox again before connecting. + +Phase 1 uses the existing stop and start lifecycle. Filesystem state survives, +and the agent resumes its conversation from disk. Phase 2 reuses the same +activity signal to drive the existing `SuspendSandbox` and `ResumeSandbox` +contracts once a backend implements memory checkpoints. + +## Motivation + +Agent sandboxes spend most of their life waiting. A coding agent finishes a +turn, reports its result, and then sits at a prompt until a person comes back, +often hours later or never. The sandbox keeps its CPU and memory reservations +and its node or VM the whole time. Operators clean up by hand or not at all. + +OpenShell cannot tell an idle agent from a busy one today. The only activity +signal is the network connection summary the supervisor sends for telemetry. +An agent waiting on a person and a long build that makes no connections look +the same from outside. Screen scraping, such as matching a spinner in the +window title, is fragile and differs for every agent. + +OSC 7501 ([spec](https://www.superlogical.com/rex/docs/build/program-status), +[announcement](https://mitchellh.com/writing/program-status-osc7501)) gives +programs a standard way to declare `idle`, `working`, `done`, `blocked`, or +`error` on their own terminal. Claude Code (from v2.1.295) and Codex (0.170.0 +verified) report it natively. The workload PTY already passes through the supervisor, so OpenShell +can read these reports without agent-specific integrations or new workload +dependencies. + +The lifecycle side is partly built. `StopSandbox`, `StartSandbox`, +`SuspendSandbox`, and `ResumeSandbox` exist in the public and driver APIs, but +nothing stops a sandbox on its own and no driver implements suspend. + +## Non-goals + +- Memory checkpoints. Phase 1 stops compute and loses process memory. CRIU or + VM snapshots are phase 2 and follow the suspend contract, not this RFC. +- Detecting idleness for programs that do not report OSC 7501. A sandbox with + no reports is never stopped automatically. A waiting report describes only + its own program. Phase 1 does not see other work in the same sandbox: + - a non-reporting terminal; + - a non-PTY `sandbox exec`; + - a command run in a shell after its agent exited. + + Users can rely on idle stop only when the reporting agent is the sandbox's + only long-running work, and the docs say so. +- Waking on inbound service traffic. Only an explicit connect or start wakes a + sandbox in phase 1. +- Interpreting the free-text `msg` and `title` fields. They are validated and + then dropped, never stored or logged. +- Agent-specific resume logic. OpenShell restarts the canonical command; the + agent's own `--continue` or `resume` flags restore its conversation. + +## Proposal + +### Observe program status in the supervisor + +The supervisor scans PTY output for OSC 7501 in two places: + +- the canonical process reader (`openshell-supervisor-process/src/main_session.rs`) +- the SSH exec loop for PTY sessions (`ssh.rs`) + +Both run on plaintext before output reaches any client, and the canonical +reader runs whether or not a client is attached. + +The scanner keeps the per-terminal records the protocol defines: + +- record ids and hierarchy; +- `clear` semantics; +- drop `working` and `blocked` records on process exit or OSC 133 `A`; +- the 256-record cap; +- the 4096-byte sequence limit. + +Report bytes pass through unchanged, so a supporting client terminal still +shows status. + +The supervisor answers the support query (`OSC 7501 ; ?`) itself by writing +the reply to the program's input, and strips the query from forwarded output. +The reply is the full form, `OSC 7501 ; ? : states=... : kinds=... ST`. A +program that probes with terminal echo on reads its own echo, and a reply +identical to the query would be answered again without end. + +Answering the probe is required, not an optimization. A prototype run against +Claude Code 2.1.327 showed: + +- It probes at startup, followed by a DA1 (`CSI c`) query. +- It reports only if the OSC 7501 reply arrives before the DA1 reply. That + follows the spec's detection rule. +- The supervisor's reply reaches the program before any client terminal's. +- Stripping the query keeps a supporting client terminal from answering a + second time into the program's input. + +The supervisor keeps the result as a sandbox-wide summary: the most active +state across terminals (working, then blocked, error, done, idle), plus its +`kind` and `app`. It also tracks when that state began and when any client +last wrote to a workload terminal. + +Observed behavior through the supervisor's scanner: + +| Event | Claude Code 2.1.327 | Codex 0.170.0 | +|-------|---------------------|---------------| +| Support query | Sent at startup | Never sent | +| Startup | `idle` | `idle` | +| Prompt submitted | `working` | `working` | +| Turn finished | `done` | `done` | +| Folder trust or approval prompt | `blocked:kind=permission` | `blocked:kind=permission` | +| Approval denied or interrupted | `clear` | `working`, then `idle` | +| Exit | `clear` | `clear` | + +Neither agent returns to `idle` after a finished turn, so the policy treats +`done` as waiting on the user. Codex reports without querying first, so the +scanner keeps records for every terminal whether or not its program asked. + +### Report activity to the gateway + +A new `ConnectSupervisor` payload carries the summary: + +```proto +message SupervisorMessage { + oneof payload { + // ... existing variants 1-6 ... + SupervisorWorkloadActivity workload_activity = 7; + } +} + +message SupervisorWorkloadActivity { + WorkloadProgramState state = 1; // UNSPECIFIED: no program reports + WorkloadBlockedKind blocked_kind = 2; + string app = 3; // OSC 7501 app grammar + google.protobuf.Duration state_age = 4; // time in `state` + google.protobuf.Duration input_age = 5; // since last client input, if any + uint32 reporting_terminals = 6; + string request_id = 7; // set when answering a request + uint64 sequence = 8; // increases with every summary +} + +message GatewayMessage { + oneof payload { + // ... existing variants 1-8 ... + WorkloadActivityRequest workload_activity_request = 9; + } +} + +message WorkloadActivityRequest { + string request_id = 1; +} +``` + +`SupervisorHello.supports_workload_activity_requests` (field 9) advertises +that the supervisor answers requests. As with every stream payload, the gateway +sends requests only to supervisors that advertise it. + +Ages are relative so supervisor and gateway clocks need not agree. The +supervisor sends the summary when each gateway session starts, on every change, +and within 15 seconds of new client input, including the last keystroke before +typing stops. The latest input time outlives the terminal that received it. +`reporting_terminals` counts terminals with a current report, whether or not +their program sent the support query. + +The supervisor applies reports, OSC 133 `A` prompts, and full resets (RIS) in +output order, however reads split them. A prompt ends only the reports before +it, and a reset clears the terminal's records as the spec requires. Every +transition restarts the state timer and advances a revision, including a +`working` → `done` turn within one read. Summaries are published in mutation +order, so concurrent terminals never hide a transition from the reporter. + +The replica holding the supervisor session keeps that session's latest report +in memory. Only that live report authorizes an idle stop. The gateway also +records the latest report as a `ProgramStatus` condition in the +existing `SandboxStatus.conditions` list. This keeps the public `Sandbox` +messages unchanged: SDK users who build exhaustive `SandboxStatus` literals +still compile, as the `sandbox_status_compatibility` fixture requires. + +| Field | Value | +|-------|-------| +| `reason` | `Idle`, `Working`, `Done`, `Blocked`, or `Error` | +| `message` | For example `claude-code is blocked on permission` | +| `transition_time` | The later of the state change and the latest client input, in gateway time | + +The gateway writes the condition only when its reason or message changes, or +when input moves its time forward by a minute or more. That bounds +resource-version churn for the reconcile sweep. Writes retry on conflict and are +fenced to the running main-process instance. The condition is for display only: +it can lag input by a minute, a failed write can leave it stale, and it outlives +the session that reported it. Driver snapshots keep the condition, and every lifecycle +phase change removes it. `openshell sandbox get` and `WatchSandbox` show it +like any other condition. + +### Stop idle sandboxes + +A sandbox is idle when all of these hold: + +- its phase is `Ready`; +- the live report from its current supervisor session, for the running + main-process instance, is `idle`, `done`, `blocked`, or `error`, so no + terminal is `working`; +- the later of that report's time and `main_process_started_time` is older + than the idle timeout plus a 15-second input report lag; +- at admission, the supervisor confirms it with a fresh report. + +The timeout is a new runtime setting, `idle_stop_after_seconds`, set with +`openshell settings set` at gateway or sandbox scope. As with every runtime +setting, gateway scope wins over sandbox scope. It can be changed on a running +sandbox. Unset, zero, or negative disables idle stop, and values above 30 days +are capped. No new `SandboxSpec` field or gateway TOML key is needed. + +Every replica sweeps the sandboxes of its own supervisor sessions every 30 +seconds and reads settings only for those already waiting. A gateway that +restarts, or a session that moves to another replica, stops nothing until the +new session reports, so a workload that resumed work during an outage is not +stopped on a stale record. Each due sandbox goes through the existing stop path +on its own task, with an idle cause. + +Before taking any lifecycle lock, the stop task: + +1. requires a fresh shared owner record naming the session; +2. sends a `WorkloadActivityRequest`; +3. waits up to 5 seconds for the reply. + +No reply, as from a dead or replaced stream, cancels the stop. Keeping the round +trip outside the lock lets the session keep reading messages throughout. The +gateway persists the display condition through one coalescing writer per +sandbox, never on the message path. Each display write is fenced to the session +named by the owner record, so a former owner cannot overwrite the new owner's +status after a handoff. + +Under the lifecycle lock, each `Stopping` write attempt runs a final check. It +requires: + +- a confirmation at most 2 seconds old; +- the re-read setting; +- a still-due live report from the confirming session; +- a fresh owner record that still names that session. + +The confirmation's deadline, 2 seconds after its reply arrived, bounds every wait +in the final check. It is checked again just before the write is issued, and an +attempt that passes it is abandoned. + +The check holds the sandbox's activity gate through the write, and every live +report update takes the same gate. A report that takes the gate before the +final check, such as `Working`, always counts. A report that reaches the handler +while the stop holds the gate waits behind that write. A conflicting write +releases the gate and checks again. + +Every summary carries an increasing sequence number. The supervisor takes one +lock from snapshot to enqueue for both pushes and replies. The gateway drops an +older summary from the same session, so a late reply cannot overwrite a newer +push. + +The stop boundary is therefore: + +- **Always counted:** everything the supervisor observed before it answered the + confirmation, and every report that takes the activity gate before the final + check does. +- **Can race the stop:** input or a new turn between the supervisor's snapshot + and the write. That covers a report waiting for the gate while the stop + holds it. The write is issued at most 2 seconds after the reply arrives, so + the window is at most one network round trip, plus those 2 seconds, plus the + write's own latency. + +Closing that window would require the supervisor to hold input while the +gateway commits, which this RFC does not propose. A takeover by another replica +in the same window is bounded the same way. The new owner publishes ownership +before it updates the sandbox record, so the final owner check can miss a +takeover only within that window. + +The idle request writes the internal annotation `internal.openshell.ai/idle-stop` +with the reason `IdleStopping`. The annotation survives gateway recovery and +driver snapshots that rewrite conditions. The Ready condition then settles as +`IdleStopped` instead of `Stopped`, so users and the CLI can tell an idle stop +from an explicit one. The gateway reapplies that reason when a driver snapshot +of the stopped runtime rewrites it. Any other lifecycle write removes the +annotation. An explicit `sandbox stop` rewrites an idle stop as an ordinary +stop, so `sandbox connect` no longer starts it. This holds whether the idle +stop has settled or is still waiting on the driver on another replica. The +gateway logs each idle stop with the sandbox, the program, and how long it +waited. + +```mermaid +sequenceDiagram + participant A as Agent (PTY) + participant S as Supervisor + participant G as Gateway + participant D as Compute driver + A->>S: OSC 7501 support query + S->>A: Support reply + A->>S: state=done:app=claude-code + S->>G: SupervisorWorkloadActivity(done, state_age) + G->>G: record ProgramStatus condition + Note over G: idle sweep: done for > idle_stop_after_seconds + G->>S: WorkloadActivityRequest + S->>G: SupervisorWorkloadActivity(done, request_id) + G->>G: final check and Stopping write under the activity gate + G->>D: StopSandbox + G->>G: phase Stopped, reason IdleStopped +``` + +### Wake on connect + +When the target is `Stopped` with reason `IdleStopped`, +`openshell sandbox connect` calls `StartSandbox`, waits for `Ready`, then +connects as usual, printing a one-line notice. When an idle stop is still in +flight (`Stopping` with reason `IdleStopping`, which the gateway keeps through +driver snapshots), connect first waits for the sandbox to leave `Stopping`: + +- `Ready`, as after a rolled-back stop, connects directly; +- `Starting`, as when another client already restarted it, waits for `Ready`; +- `Stopped` with `IdleStopped` starts it. An explicitly stopped +sandbox keeps today's behavior and returns an error. The wake logic lives in the +CLI; the TUI and SDK do not wake sandboxes in phase 1. + +Exec and SSH proxy paths keep their current errors in phase 1, so scripts do +not start sandboxes implicitly. + +### Security and trust + +The workload controls the signal: + +- Falsely reporting `working` only keeps its own sandbox running, which it can + already do by staying busy. +- Falsely reporting `idle` stops its own sandbox. +- Neither crosses a trust boundary. + +A stray report, such as a program printing a log file that contains OSC 7501 +bytes, can make a sandbox look idle. The input-age condition limits the damage +to sandboxes no one is typing into. + +The supervisor writes only the fixed reply bytes to program input, never +report content. `msg` and `title` are validated per the spec, then discarded. + +## Implementation plan + +1. **Supervisor observation.** Add the OSC 7501 scanner, records, probe reply, + and summary in `openshell-supervisor-process`. Wire them into the canonical + reader and SSH PTY sessions, with unit tests for parsing, chunk splits, + limits, and record semantics. A prototype harness has run Claude Code + 2.1.327 and Codex 0.170.0 through the scanner on a host PTY. +2. **Reporting.** Add `SupervisorWorkloadActivity` and the supervisor reporter + task. Add the gateway handler and the throttled `ProgramStatus` condition, + and preserve the condition across driver snapshots. +3. **Policy.** Register the `idle_stop_after_seconds` setting. Add the idle + sweep and the `IdleStopped` reason, and document them on the sandbox + lifecycle page. +4. **Wake on connect.** Auto-start `IdleStopped` sandboxes in + `openshell sandbox connect`. +5. **Validation.** Add the `idle_stop` conformance test, which runs a + scripted OSC 7501 agent on every driver through two sequences. + `IdleStopped`, then `sandbox connect`, starts it again. + `IdleStopped`, then an explicit stop, then `sandbox connect`, leaves it + stopped. Check Claude Code and Codex manually inside Docker + and Kubernetes sandboxes, with provider credentials. +6. **Phase 2 (separate change).** Choose `suspend` instead of `stop` when the + driver advertises `openshell.compute.suspend-resume`, and implement Podman + checkpoint and restore behind that capability. + +Idle stop is off by default, so existing users see no change until an operator +sets a gateway value or a user opts in per sandbox. + +## Risks + +- **Lost in-flight work.** A `blocked` agent loses the pending turn when + stopped. The timeout applies to all waiting states in phase 1. If users find + that too aggressive, a separate blocked timeout is an open question. +- **Agent coverage and drift.** Only agents that implement OSC 7501 benefit, + and their reporting can change between releases, as Claude Code's + probe-before-report rule shows. Unreported sandboxes are never stopped, so + drift fails safe. +- **Multiplexers.** An agent inside tmux in the sandbox probes tmux, not the + supervisor's PTY, and tmux does not forward the reports. Those sandboxes look + unreported and stay running. +- **Conversation resume depends on storage.** The agent's transcript must live + on storage that survives stop: the Docker writable layer or workspace volume, + the Kubernetes PVC, the VM overlay. Agents restarted with their plain command + start a fresh conversation unless the user runs their resume flag. +- **Residual stop race.** Input or a new turn between the supervisor's + confirmation snapshot and the `Stopping` write can still lose the sandbox. + The window is at most one round trip, plus 2 seconds, plus the write's + latency, and only for a sandbox that already waited past its timeout. Closing it fully would need the + supervisor to hold input during the commit. +- **Store churn.** Display writes bump the resource version that the + reconcile sweep compares. The one-minute input threshold and + transition-only state writes limit this. An active agent still writes twice + per turn (`working`, then `done`). Decisions use in-memory reports, so this + throttling never delays an input from counting. + +## Alternatives + +### Screen or title scraping + +Inbox tools such as Herdr detect Claude Code state by matching the Braille +spinner in the window title. That needs a parser per agent and breaks on UI +changes. OSC 7501 is a declared contract with a published grammar. + +### Network or input inactivity only + +Stopping after no egress and no keystrokes needs no agent cooperation. But it +cannot tell a quiet long-running build from a waiting agent, and it would stop +sandboxes mid-task. It remains useful as an extra condition and may become an +opt-in policy for non-reporting workloads later. + +### Agent hooks or SDK callbacks + +Claude Code hooks and similar mechanisms can call out when a turn ends, but +each agent needs its own configuration inside the sandbox image, and the hook +runs as workload code with network access. The terminal channel needs nothing +from the image. + +### Do nothing + +Operators keep deleting or stopping sandboxes by hand, and idle sandboxes keep +holding capacity. + +## Prior art + +- [Program Status Protocol (OSC 7501)](https://www.superlogical.com/rex/docs/build/program-status): + record lifetime and the detection rule shape the scanner directly. The spec + also says the program owns its state, which is why OpenShell treats reports + as untrusted input. +- [Paneflow](https://paneflow.dev/blog/agent-status-osc-7501) answers the probe + on behalf of its sessions and treats an agent's declaration as more reliable + than heuristics. That is the same position the supervisor takes. +- Kubernetes scale-to-zero (KEDA, Knative) separates the activity signal from + the lifecycle action and wakes on demand. The idle sweep and wake on connect + follow that split. +- [RFC 0018](../0018-multi-sandbox-ha-supervisors/README.md) leaves room for + backend-initiated suspend and wake (#4352). This RFC supplies the signal that + decides when. + +## Open questions + +- Should `blocked` get its own, longer timeout, or never stop? It loses the + pending turn in phase 1 but not in phase 2. +- Should an attached client with no input block idle stop? Phase 1 does not + wait for clients to detach. +- Should a sandbox whose terminals all reported `clear` (agent exited, shell + idle at a prompt) count as idle? Phase 1 treats it as unreported. +- Should the wake path also run the agent's resume command, for example via a + `SandboxSpec` resume command, or is restarting the canonical command enough? +- Should exec and SSH proxy also wake an `IdleStopped` sandbox, behind a flag? +- Should idle stop ship behind an `unstable-idle-stop` compile flag until it is + accepted, as RFC 0014 asks of unreleased features? No crate defines such a + flag yet; phase 1 is off unless the setting is set. +- Is gateway-over-sandbox precedence right for this setting? It lets operators + enforce a fleet timeout, but it stops users from choosing a shorter or longer + one or exempting a sandbox, and a global `0` disables per-sandbox opt-in. + Sandbox-over-gateway, or `min(global, sandbox)` with the global as a ceiling, + are alternatives. +- Should other activity hold off an idle stop: a running non-PTY `exec`, an open + terminal with no report, or recent output on one? The supervisor already + tracks output time but does not report it. +- Should a new shell command (OSC 133 `C`) clear a terminal's retained `done`, + `error`, or `idle` record, so a non-reporting command started after an agent + exits is not judged by the agent's last report? The OSC 7501 spec keeps those + records across prompts. +- Should the final check drop the 15-second input report lag? The confirmed + snapshot has no lag, but the sweep would still need it to avoid needless + confirmation requests, so the first request is delayed either way. +- What does `ProgramStatus` display cost fleet-wide? An active agent writes + twice per turn, and each write bumps the resource version and notifies + watchers. Display writes happen even when idle stop is disabled. diff --git a/tests/suites/conformance/src/idle_stop.rs b/tests/suites/conformance/src/idle_stop.rs new file mode 100644 index 0000000000..a064055df1 --- /dev/null +++ b/tests/suites/conformance/src/idle_stop.rs @@ -0,0 +1,146 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Idle stop suite: the gateway stops a sandbox whose terminal program +//! reports over OSC 7501 that it is waiting on the user. + +use std::time::Duration; + +use serde_json::Value; +use testkit::sdk::SandboxPhase; +use testkit::sdk::raw::proto; +use testkit::{Context, Poll, Sandbox, poll_until}; + +const KEY: &str = "idle_stop_after_seconds"; + +/// Probe for support as Claude Code does, report that the turn is done, and +/// wait at the prompt. Echo is on, so the supervisor's reply is echoed back. +const AGENT: &str = r"printf '\033]7501;?\033\\'; sleep 1; printf '\033]7501;state=done:app=e2e-agent\033\\'; exec sleep infinity"; + +/// The supervisor reports within a second; allow for session setup. +const REPORT_DEADLINE: Duration = Duration::from_secs(30); +/// The timeout, the 15 second input report lag, and the 30 second sweep. +const IDLE_STOP_DEADLINE: Duration = Duration::from_secs(90); +/// Connect prints its notice, then starts the sandbox and waits for Ready. +const WAKE_DEADLINE: Duration = Duration::from_secs(60); + +/// A sandbox whose program reports `done` is stopped with reason +/// `IdleStopped` once its idle timeout passes, and `sandbox connect` starts it +/// again. After a later idle stop, an explicit `sandbox stop` keeps it +/// stopped: `sandbox connect` then fails instead of starting it. +#[testkit::test] +async fn idle_stopped_sandbox_wakes_on_connect_until_stopped_explicitly( + cx: &mut Context, +) -> testkit::Result { + let sandbox = cx + .sandbox("idle") + .command(["sh", "-c", AGENT]) + .spec(|spec| spec.tty = true) + .create_ready() + .await?; + expect_program_status(&sandbox, "Done").await?; + set_idle_timeout(&sandbox, 1).await?; + expect_idle_stopped(&sandbox).await?; + + let mut connect = cx.cli_spawn(["sandbox", "connect", sandbox.name()])?; + connect + .stderr_until(WAKE_DEADLINE, |line| line.contains("stopped while idle")) + .await?; + // Connect reports the start only after the sandbox is Ready again. + connect + .stdout_until(WAKE_DEADLINE, |line| line.contains("Started sandbox")) + .await?; + sandbox.wait_ready().await?; + connect.kill().await?; + + // The restarted program reports again and is idle-stopped again. + expect_program_status(&sandbox, "Done").await?; + expect_idle_stopped(&sandbox).await?; + + sandbox.stop().await?; + let reason = ready_reason(&sandbox.get_json().await?); + testkit::ensure!( + reason != "IdleStopped", + "an explicit stop kept the idle reason, so connect would start it" + ); + let connected = cx.cli(["sandbox", "connect", sandbox.name()]).await?; + testkit::ensure!( + connected.exit_code != 0, + "sandbox connect succeeded against an explicitly stopped sandbox" + ); + let phase = sandbox.get().await?.phase; + testkit::ensure!( + phase == SandboxPhase::Stopped, + "sandbox connect changed an explicitly stopped sandbox to {phase:?}" + ); + Ok(()) +} + +/// Wait for an idle stop and check that it kept its own reason. +async fn expect_idle_stopped(sandbox: &Sandbox) -> testkit::Result { + sandbox + .wait_phase(SandboxPhase::Stopped, IDLE_STOP_DEADLINE) + .await?; + let reason = ready_reason(&sandbox.get_json().await?); + testkit::ensure!( + reason == "IdleStopped", + "sandbox {} stopped with reason {reason:?}; expected IdleStopped", + sandbox.name() + ); + Ok(()) +} + +/// Set the sandbox's idle timeout and check the effective value, which a +/// global setting would override. +async fn set_idle_timeout(sandbox: &Sandbox, seconds: i64) -> testkit::Result { + sandbox + .set_setting(KEY, proto::setting_value::Value::IntValue(seconds)) + .await?; + let config = sandbox.config().await?; + let value = config + .settings + .get(KEY) + .and_then(|setting| setting.value.as_ref()) + .and_then(|value| value.value.clone()); + testkit::ensure!( + value == Some(proto::setting_value::Value::IntValue(seconds)), + "effective {KEY} is {value:?}; delete the global value with `openshell settings delete --global --key {KEY}`" + ); + Ok(()) +} + +/// Wait until the sandbox's `ProgramStatus` condition has `reason`. +async fn expect_program_status(sandbox: &Sandbox, reason: &str) -> testkit::Result { + poll_until( + &format!( + "sandbox {} to report program status {reason}", + sandbox.name() + ), + REPORT_DEADLINE, + Duration::from_secs(1), + async || { + let got = sandbox.get_json().await?; + let program = condition(&got, "ProgramStatus"); + Ok(match program { + Some(program) if program["reason"] == reason => Poll::Ready(()), + Some(program) => Poll::Pending(format!("ProgramStatus is {program}")), + None => Poll::Pending("no ProgramStatus condition".to_string()), + }) + }, + ) + .await +} + +fn condition<'a>(sandbox: &'a Value, kind: &str) -> Option<&'a Value> { + sandbox["conditions"] + .as_array()? + .iter() + .find(|condition| condition["type"] == kind) +} + +fn ready_reason(sandbox: &Value) -> String { + condition(sandbox, "Ready") + .and_then(|ready| ready["reason"].as_str()) + .unwrap_or_default() + .to_string() +} diff --git a/tests/suites/conformance/src/lib.rs b/tests/suites/conformance/src/lib.rs index 8ac23f54bc..6f33eae5e3 100644 --- a/tests/suites/conformance/src/lib.rs +++ b/tests/suites/conformance/src/lib.rs @@ -15,6 +15,7 @@ mod credential_gating; mod exec_cli; mod file_transfer; mod filesystem_policy; +mod idle_stop; mod image; mod labels; mod lifecycle;