From d4f701881eb7c68276f09222fbc686ac4563e015 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Thu, 22 Oct 2026 13:37:26 +0530 Subject: [PATCH 1/9] Enforce absolute deadlines for auction response collection --- Cargo.lock | 5 + crates/trusted-server-adapter-axum/Cargo.toml | 1 + .../src/platform.rs | 215 ++++++ .../Cargo.toml | 1 + .../src/platform.rs | 270 ++++++-- .../trusted-server-adapter-fastly/Cargo.toml | 3 + .../src/platform.rs | 10 + .../src/platform/deadline.rs | 649 ++++++++++++++++++ .../tests/deadline_origin.py | 71 ++ crates/trusted-server-adapter-spin/Cargo.toml | 1 + .../src/platform.rs | 235 +++++-- .../src/auction/orchestrator.rs | 493 +++++++++---- .../trusted-server-core/src/platform/error.rs | 3 + .../trusted-server-core/src/platform/http.rs | 154 ++++- .../src/platform/test_support.rs | 61 +- crates/trusted-server-core/src/publisher.rs | 15 + .../trusted-server-core/src/test_support.rs | 12 + docs/guide/architecture.md | 9 +- docs/guide/auction-orchestration.md | 31 +- docs/guide/auction-testing.md | 22 + docs/guide/configuration.md | 15 +- .../2026-10-22-absolute-auction-deadlines.md | 74 ++ 22 files changed, 2058 insertions(+), 292 deletions(-) create mode 100644 crates/trusted-server-adapter-fastly/src/platform/deadline.rs create mode 100644 crates/trusted-server-adapter-fastly/tests/deadline_origin.py create mode 100644 docs/superpowers/plans/2026-10-22-absolute-auction-deadlines.md diff --git a/Cargo.lock b/Cargo.lock index 7ebc0b08ea..757eab75d4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6556,6 +6556,7 @@ dependencies = [ "tokio", "tower 0.4.13", "trusted-server-core", + "web-time", ] [[package]] @@ -6577,6 +6578,7 @@ dependencies = [ "toml", "trusted-server-core", "trusted-server-js", + "web-time", "worker", ] @@ -6594,6 +6596,7 @@ dependencies = [ "error-stack", "esi", "fastly", + "fastly-sys", "fern", "futures", "log", @@ -6604,6 +6607,7 @@ dependencies = [ "trusted-server-core", "url", "urlencoding", + "web-time", ] [[package]] @@ -6626,6 +6630,7 @@ dependencies = [ "tokio", "trusted-server-core", "trusted-server-js", + "web-time", ] [[package]] diff --git a/crates/trusted-server-adapter-axum/Cargo.toml b/crates/trusted-server-adapter-axum/Cargo.toml index 94ad02170c..af707aa24e 100644 --- a/crates/trusted-server-adapter-axum/Cargo.toml +++ b/crates/trusted-server-adapter-axum/Cargo.toml @@ -29,6 +29,7 @@ reqwest = { workspace = true } simple_logger = { workspace = true } tokio = { workspace = true, features = ["rt-multi-thread", "macros", "sync", "time"] } trusted-server-core = { workspace = true } +web-time = { workspace = true } [dev-dependencies] trusted-server-core = { workspace = true, features = ["test-utils"] } diff --git a/crates/trusted-server-adapter-axum/src/platform.rs b/crates/trusted-server-adapter-axum/src/platform.rs index e2713f47f2..fada8a1350 100644 --- a/crates/trusted-server-adapter-axum/src/platform.rs +++ b/crates/trusted-server-adapter-axum/src/platform.rs @@ -446,6 +446,24 @@ impl PlatformHttpClient for AxumPlatformHttpClient { Ok(PlatformPendingRequest::new(pending).with_backend_name(backend_name)) } + async fn select_until( + &self, + pending_requests: Vec, + deadline: web_time::Instant, + ) -> Result, Report> { + let now = web_time::Instant::now(); + if now >= deadline { + return Ok(None); + } + + // Selection owns abort-on-drop task handles and buffers complete bodies. + // Dropping its future on expiration also stops outstanding body reads. + match tokio::time::timeout(deadline - now, self.select(pending_requests)).await { + Ok(result) if web_time::Instant::now() < deadline => result.map(Some), + Ok(_) | Err(_) => Ok(None), + } + } + async fn select( &self, pending_requests: Vec, @@ -839,6 +857,203 @@ mod tests { ); } + async fn deadline_pending( + client: &AxumPlatformHttpClient, + url: String, + backend: &str, + ) -> PlatformPendingRequest { + let request = edgezero_core::http::request_builder() + .uri(url) + .body(EdgeBody::empty()) + .expect("should build deadline request"); + client + .send_async(PlatformHttpRequest::new(request, backend)) + .await + .expect("should dispatch deadline request") + } + + async fn serve_slow_response(trickle_body: bool) -> (String, tokio::task::JoinHandle<()>) { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("should bind deadline server"); + let address = listener + .local_addr() + .expect("should read deadline server address"); + let task = tokio::spawn(async move { + let (mut stream, _) = listener + .accept() + .await + .expect("should accept deadline request"); + let mut request = [0; 1024]; + let bytes_read = stream + .read(&mut request) + .await + .expect("should read deadline request"); + assert!(bytes_read > 0, "should receive a deadline request"); + if trickle_body { + if stream + .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 100\r\n\r\n") + .await + .is_err() + { + return; + } + for _ in 0..100 { + if stream.write_all(b"x").await.is_err() { + return; + } + tokio::time::sleep(Duration::from_millis(50)).await; + } + } else { + tokio::time::sleep(Duration::from_secs(5)).await; + let _ = stream + .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nok") + .await; + } + }); + (format!("http://{address}/"), task) + } + + #[tokio::test] + async fn select_until_bounds_delayed_headers_and_trickling_bodies() { + let client = AxumPlatformHttpClient::new(); + for trickle_body in [false, true] { + let (url, server) = serve_slow_response(trickle_body).await; + let pending = deadline_pending(&client, url, "slow").await; + let start = web_time::Instant::now(); + let result = tokio::time::timeout( + Duration::from_secs(2), + client.select_until(vec![pending], start + Duration::from_millis(200)), + ) + .await + .expect("should finish before the slow response") + .expect("should enforce the deadline"); + server.abort(); + assert!( + result.is_none(), + "should time out before the entire response completes" + ); + assert!( + start.elapsed() < Duration::from_secs(2), + "should bound the complete response wait" + ); + } + } + + #[tokio::test] + async fn select_until_preserves_fast_success_before_slow_body_expires() { + let client = AxumPlatformHttpClient::new(); + let (slow_url, server) = serve_slow_response(true).await; + let fast_url = serve_raw_response(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nok").await; + let slow = deadline_pending(&client, slow_url, "slow").await; + let fast = deadline_pending(&client, fast_url, "fast").await; + let deadline = web_time::Instant::now() + Duration::from_millis(500); + let result = client + .select_until(vec![slow, fast], deadline) + .await + .expect("should collect fast response") + .expect("should complete before the deadline"); + assert_eq!( + result + .ready + .expect("should succeed") + .backend_name + .as_deref(), + Some("fast"), + "should collect the fast complete body first" + ); + assert_eq!( + result.remaining.len(), + 1, + "should preserve the slow body handle" + ); + let result = client + .select_until(result.remaining, deadline) + .await + .expect("should bound slow response"); + server.abort(); + assert!(result.is_none(), "should expire the remaining slow body"); + } + + #[tokio::test] + async fn select_until_expired_deadline_rejects_ready_response() { + let client = AxumPlatformHttpClient::new(); + let handle = tokio::spawn(async { Ok((200, Vec::new(), b"ok".to_vec())) }); + while !handle.is_finished() { + tokio::task::yield_now().await; + } + let pending = PlatformPendingRequest::new(AxumPendingHandle { + backend_name: "ready".to_owned(), + handle, + }); + let result = client + .select_until(vec![pending], web_time::Instant::now()) + .await + .expect("should recognize expired deadline"); + assert!( + result.is_none(), + "should reject an already ready response after cutoff" + ); + } + + #[tokio::test] + async fn select_until_aborts_task_on_expiration() { + let (started_sender, started_receiver) = tokio::sync::oneshot::channel(); + let (dropped_sender, dropped_receiver) = tokio::sync::oneshot::channel::<()>(); + let handle = tokio::spawn(async move { + let _drop_signal = dropped_sender; + started_sender.send(()).expect("should signal task start"); + futures::future::pending::().await + }); + started_receiver + .await + .expect("should start task before selection"); + let pending = PlatformPendingRequest::new(AxumPendingHandle { + backend_name: "slow".to_owned(), + handle, + }); + let result = AxumPlatformHttpClient::new() + .select_until( + vec![pending], + web_time::Instant::now() + Duration::from_millis(50), + ) + .await + .expect("should expire task"); + assert!(result.is_none(), "should report deadline expiration"); + assert!( + tokio::time::timeout(Duration::from_secs(2), dropped_receiver) + .await + .expect("should promptly drop aborted task state") + .is_err(), + "should drop task resources on expiration" + ); + } + + #[tokio::test] + async fn select_until_preserves_failed_backend_attribution() { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("should bind probe"); + let address = listener.local_addr().expect("should read probe address"); + drop(listener); + let client = AxumPlatformHttpClient::new(); + let pending = deadline_pending(&client, format!("http://{address}/"), "failed").await; + let result = client + .select_until( + vec![pending], + web_time::Instant::now() + Duration::from_secs(2), + ) + .await + .expect("should collect transport failure") + .expect("should fail before deadline"); + assert!(result.ready.is_err(), "should preserve transport failure"); + assert_eq!( + result.failed_backend_name.as_deref(), + Some("failed"), + "should attribute failure to its backend" + ); + } + async fn serve_raw_response(response: &'static [u8]) -> String { let listener = tokio::net::TcpListener::bind("127.0.0.1:0") .await diff --git a/crates/trusted-server-adapter-cloudflare/Cargo.toml b/crates/trusted-server-adapter-cloudflare/Cargo.toml index 2ce969ef79..03b342f030 100644 --- a/crates/trusted-server-adapter-cloudflare/Cargo.toml +++ b/crates/trusted-server-adapter-cloudflare/Cargo.toml @@ -29,6 +29,7 @@ edgezero-core = { workspace = true } error-stack = { workspace = true } futures = { workspace = true } log = { workspace = true } +web-time = { workspace = true } serde_json = { workspace = true } trusted-server-core = { workspace = true } trusted-server-js = { workspace = true } diff --git a/crates/trusted-server-adapter-cloudflare/src/platform.rs b/crates/trusted-server-adapter-cloudflare/src/platform.rs index 94616b2035..3a0dc7b039 100644 --- a/crates/trusted-server-adapter-cloudflare/src/platform.rs +++ b/crates/trusted-server-adapter-cloudflare/src/platform.rs @@ -162,33 +162,21 @@ impl KvStore for KvHandleAdapter { // CloudflareHttpClient — WASM target only // --------------------------------------------------------------------------- -/// Carries a completed response through `send_async` → `select`. -/// -/// Same pattern as `AxumPendingResponse`: stores raw parts because `Body::Stream` -/// is `!Send`, which is incompatible with `Box` inside -/// [`PlatformPendingRequest`]. +/// Owns the outbound operation until selection completes or drops it. #[cfg(target_arch = "wasm32")] struct CloudflarePendingResponse { - backend_name: String, - status: u16, - headers: Vec<(String, Vec)>, - body: Vec, + response: + futures::future::LocalBoxFuture<'static, Result>>, } /// [`worker::Fetch`]-backed HTTP client for the Cloudflare Workers runtime. /// /// # Multi-provider auction limitation /// -/// `send_async` eagerly awaits each request before returning, so -/// [`PlatformHttpClient::supports_concurrent_fanout`] reports `false` and the -/// auction orchestrator rejects multi-provider configurations before any -/// request launches. `select` keeps a defense-in-depth rejection for more -/// than one pending request. Configure a single auction provider for -/// Cloudflare Workers, or use the Fastly adapter for parallel DSP fan-out. -/// -/// Per-provider timeouts baked into the backend name are not enforced at the -/// fetch layer; the Workers runtime's global CPU budget (~30 s on paid plans) -/// is the only implicit deadline. +/// Selection supports one provider at a time, so +/// [`PlatformHttpClient::supports_concurrent_fanout`] reports `false`. +/// Deadline selection bounds fetch and body collection together and aborts +/// the Workers fetch when its pending operation is dropped. #[cfg(target_arch = "wasm32")] pub struct CloudflareHttpClient; @@ -283,10 +271,21 @@ fn outbound_request_init(method: worker::Method, headers: worker::Headers) -> wo init } +#[cfg(target_arch = "wasm32")] +struct AbortOnDrop(Option); + +#[cfg(target_arch = "wasm32")] +impl Drop for AbortOnDrop { + fn drop(&mut self) { + if let Some(controller) = self.0.take() { + controller.abort(); + } + } +} + #[cfg(target_arch = "wasm32")] impl CloudflareHttpClient { async fn execute( - &self, request: PlatformHttpRequest, ) -> Result> { use worker::{CacheMode, Fetch, Headers, Method, Request}; @@ -366,8 +365,14 @@ impl CloudflareHttpClient { let worker_req = Request::new_with_init(&uri, &init).change_context(PlatformError::HttpClient)?; + let abort = AbortOnDrop(Some(worker::AbortController::default())); + let signal = abort + .0 + .as_ref() + .expect("should retain abort controller") + .signal(); let mut resp = Fetch::Request(worker_req) - .send() + .send_with_signal(&signal) .await .change_context(PlatformError::HttpClient) .attach_with(|| format!("outbound request to {uri} failed"))?; @@ -449,13 +454,12 @@ impl PlatformHttpClient for CloudflareHttpClient { &self, request: PlatformHttpRequest, ) -> Result> { - self.execute(request).await + Self::execute(request).await } fn supports_concurrent_fanout(&self) -> bool { - // `send_async` executes each request eagerly, so multiple pending - // requests run sequentially. The auction orchestrator checks this - // before launching more than one provider request. + // Keep the existing single-provider restriction: selection polls one + // outbound request at a time. false } @@ -464,33 +468,8 @@ impl PlatformHttpClient for CloudflareHttpClient { request: PlatformHttpRequest, ) -> Result> { let backend_name = request.backend_name.clone(); - let response = self.execute(request).await?; - - let status = response.response.status().as_u16(); - let headers: Vec<(String, Vec)> = response - .response - .headers() - .iter() - .map(|(n, v)| (n.to_string(), v.as_bytes().to_vec())) - .collect(); - let body_bytes = match response.response.into_body() { - edgezero_core::body::Body::Once(bytes) => bytes.to_vec(), - // execute() always buffers via resp.bytes().await → Body::Once. - // Return a typed error rather than panicking in the request path - // in case that edgezero implementation detail ever changes. - edgezero_core::body::Body::Stream(_) => { - return Err(Report::new(PlatformError::HttpClient).attach( - "unexpected streaming body from CloudflareHttpClient::execute \ - — expected a buffered Body::Once", - )); - } - }; - let pending = CloudflarePendingResponse { - backend_name: backend_name.clone(), - status, - headers, - body: body_bytes, + response: Box::pin(Self::execute(request)), }; Ok(PlatformPendingRequest::new(pending).with_backend_name(backend_name)) } @@ -514,21 +493,67 @@ impl PlatformHttpClient for CloudflareHttpClient { .attach("unexpected inner type in CloudflareHttpClient::select") })?; - let mut builder = edgezero_core::http::response_builder().status(pending.status); - for (name, value) in &pending.headers { - builder = builder.header(name.as_str(), value.as_slice()); - } - let edge_resp = builder - .body(edgezero_core::body::Body::from(pending.body)) - .change_context(PlatformError::HttpClient)?; - - let ready = Ok(PlatformResponse::new(edge_resp).with_backend_name(pending.backend_name)); + let ready = pending.response.await; Ok(PlatformSelectResult { ready, remaining: pending_requests, failed_backend_name: None, }) } + + async fn select_until( + &self, + mut pending_requests: Vec, + deadline: web_time::Instant, + ) -> Result, Report> { + if pending_requests.is_empty() { + return Err(Report::new(PlatformError::HttpClient) + .attach("select_until called with an empty pending_requests list")); + } + if pending_requests.len() > 1 { + return Err(Report::new(PlatformError::HttpClient) + .attach("multi-provider fan-out is not supported on this adapter")); + } + let remaining = deadline.saturating_duration_since(web_time::Instant::now()); + if remaining.is_zero() { + return Ok(None); + } + let platform_pending = pending_requests.remove(0); + let backend_name = platform_pending.backend_name().map(str::to_owned); + let pending = platform_pending + .downcast::() + .map_err(|_| { + Report::new(PlatformError::HttpClient) + .attach("unexpected pending request type in deadline selection") + })?; + let result = complete_before_deadline( + pending.response, + async { + // Workers timers use whole signed milliseconds. Recheck the + // absolute deadline after each bounded timer interval. + loop { + let remaining = deadline.saturating_duration_since(web_time::Instant::now()); + if remaining.is_zero() { + break; + } + worker::Delay::from( + remaining + .max(Duration::from_millis(1)) + .min(Duration::from_millis(i32::MAX as u64)), + ) + .await; + } + }, + deadline, + web_time::Instant::now, + ) + .await; + Ok(result.map(|ready| PlatformSelectResult { + failed_backend_name: ready.as_ref().err().and(backend_name), + ready, + remaining: pending_requests, + })) + } } // --------------------------------------------------------------------------- @@ -948,3 +973,128 @@ mod tests { ); } } + +#[cfg(any(test, target_arch = "wasm32"))] +async fn complete_before_deadline( + operation: F, + timer: T, + deadline: web_time::Instant, + now: N, +) -> Option +where + F: std::future::Future, + T: std::future::Future, + N: Fn() -> web_time::Instant, +{ + if now() >= deadline { + return None; + } + match futures::future::select(Box::pin(operation), Box::pin(timer)).await { + futures::future::Either::Left((response, _)) if now() < deadline => Some(response), + _ => None, + } +} + +#[cfg(test)] +mod deadline_tests { + use std::cell::Cell; + use std::rc::Rc; + use std::time::Duration; + + use futures::executor::block_on; + use futures::future::{pending, ready}; + use web_time::Instant; + + use super::complete_before_deadline; + + #[test] + fn expired_deadline_does_not_poll_request() { + let deadline = Instant::now(); + let polled = Cell::new(false); + let operation = async { + polled.set(true); + pending::<()>().await + }; + let result: Option<()> = block_on(complete_before_deadline( + operation, + pending(), + deadline, + || deadline, + )); + assert_eq!(result, None, "should reject before polling expired request"); + assert!(!polled.get(), "should not poll an expired request"); + } + + #[test] + fn timer_cuts_off_pending_operation_and_drops_it() { + struct DropProbe(Rc>); + impl Drop for DropProbe { + fn drop(&mut self) { + self.0.set(true); + } + } + let dropped = Rc::new(Cell::new(false)); + let probe = DropProbe(Rc::clone(&dropped)); + let operation = async move { + let _probe = probe; + pending::<()>().await + }; + let now = Instant::now(); + let result = block_on(complete_before_deadline( + operation, + ready(()), + now + Duration::from_secs(1), + || now, + )); + assert_eq!(result, None, "should cut off a pending request"); + assert!(dropped.get(), "should drop the pending operation at cutoff"); + } + + #[test] + fn accepts_complete_response_before_deadline() { + let now = Instant::now(); + let result = block_on(complete_before_deadline( + ready(42), + pending(), + now + Duration::from_secs(1), + || now, + )); + assert_eq!(result, Some(42), "should accept response before cutoff"); + } + + #[test] + fn rejects_response_that_becomes_ready_at_deadline() { + let now = Instant::now(); + let deadline = now + Duration::from_secs(1); + let clock = Cell::new(now); + let operation = async { + clock.set(deadline); + 42 + }; + let result = block_on(complete_before_deadline( + operation, + pending(), + deadline, + || clock.get(), + )); + assert_eq!(result, None, "should reject a late ready response"); + } + + #[test] + fn timer_bounds_body_after_headers_complete() { + let now = Instant::now(); + let headers_received = Cell::new(false); + let operation = async { + headers_received.set(true); + pending::<()>().await + }; + let result = block_on(complete_before_deadline( + operation, + ready(()), + now + Duration::from_secs(1), + || now, + )); + assert!(headers_received.get(), "should reach response body read"); + assert_eq!(result, None, "should cut off pending body collection"); + } +} diff --git a/crates/trusted-server-adapter-fastly/Cargo.toml b/crates/trusted-server-adapter-fastly/Cargo.toml index 710523766e..b10be8105a 100644 --- a/crates/trusted-server-adapter-fastly/Cargo.toml +++ b/crates/trusted-server-adapter-fastly/Cargo.toml @@ -28,6 +28,8 @@ edgezero-core = { workspace = true } error-stack = { workspace = true } esi = { workspace = true } fastly = { workspace = true } +# The SDK keeps timed body readiness private; auction deadlines use the ABI. +fastly-sys = "0.13.1" fern = { workspace = true } futures = { workspace = true } log = { workspace = true } @@ -37,6 +39,7 @@ serde_json = { workspace = true } trusted-server-core = { workspace = true } url = { workspace = true } urlencoding = { workspace = true } +web-time = { workspace = true } [dev-dependencies] bytes = { workspace = true } diff --git a/crates/trusted-server-adapter-fastly/src/platform.rs b/crates/trusted-server-adapter-fastly/src/platform.rs index d0f4ee9d0e..1e844111b6 100644 --- a/crates/trusted-server-adapter-fastly/src/platform.rs +++ b/crates/trusted-server-adapter-fastly/src/platform.rs @@ -1,6 +1,8 @@ //! Fastly-backed implementations of the platform traits defined in //! `trusted-server-core::platform`. +mod deadline; + use bytes::Bytes; use error_stack::{Report, ResultExt}; use fastly::geo::{Geo, geo_lookup}; @@ -622,6 +624,14 @@ impl PlatformHttpClient for FastlyPlatformHttpClient { }) } + async fn select_until( + &self, + pending_requests: Vec, + deadline: web_time::Instant, + ) -> Result, Report> { + deadline::select_until(pending_requests, deadline) + } + async fn wait( &self, pending: PlatformPendingRequest, diff --git a/crates/trusted-server-adapter-fastly/src/platform/deadline.rs b/crates/trusted-server-adapter-fastly/src/platform/deadline.rs new file mode 100644 index 0000000000..d6da1cb0ec --- /dev/null +++ b/crates/trusted-server-adapter-fastly/src/platform/deadline.rs @@ -0,0 +1,649 @@ +//! Deadline-bounded collection of Fastly auction responses. + +use std::io::Read as _; +use std::time::Duration; + +use error_stack::{Report, ResultExt as _}; +use fastly::handle::BodyHandle; +use fastly::http::request::{PendingRequest, PollResult}; +use trusted_server_core::platform::{ + PlatformError, PlatformPendingRequest, PlatformResponse, PlatformSelectResult, +}; +use web_time::Instant; + +use super::{MAX_PLATFORM_RESPONSE_BODY_BYTES, fastly_response_to_platform, response_carries_body}; + +// Bound payload retained across all partial bodies, as well as each response. +const MAX_TOTAL_BUFFERED_BODY_BYTES: usize = MAX_PLATFORM_RESPONSE_BODY_BYTES; + +// Keep partially buffered responses alongside requests still awaiting headers so +// an early header with a slow body cannot hold up a completed sibling response. +struct DeadlinePending { + backend_name: String, + request_method: Option, + state: Option, +} + +enum PendingState { + Headers(Box>), + Body { + response: Box, + body: BodyHandle, + bytes: Vec, + }, +} + +impl DeadlinePending { + fn from_platform(pending: PlatformPendingRequest) -> Result> { + if pending.stream_response() { + return Err(Report::new(PlatformError::HttpClient) + .attach("stream-marked pending request requires direct wait")); + } + let request_method = pending.request_method().cloned(); + let pending = match pending.downcast::() { + Ok(mut pending) => { + pending.request_method = request_method.or(pending.request_method); + return Ok(pending); + } + Err(pending) => pending, + }; + let backend_name = pending.backend_name().unwrap_or("").to_owned(); + let pending = pending.downcast::().map_err(|_| { + Report::new(PlatformError::HttpClient).attach(format!( + "invalid Fastly deadline pending handle for backend '{backend_name}'" + )) + })?; + Ok(Self { + backend_name, + request_method, + state: Some(PendingState::Headers(Box::new(Some(pending)))), + }) + } + + fn into_platform(self) -> PlatformPendingRequest { + let backend_name = self.backend_name.clone(); + let request_method = self.request_method.clone(); + let pending = PlatformPendingRequest::new(self).with_backend_name(backend_name); + match request_method { + Some(method) => pending.with_response_handling(false, method), + None => pending, + } + } + + fn request_is_head(&self, response: &fastly::Response) -> Result> { + if let Some(method) = &self.request_method { + return Ok(method == edgezero_core::http::Method::HEAD); + } + response + .get_backend_request() + .map(|request| request.get_method() == fastly::http::Method::HEAD) + .ok_or_else(|| { + Report::new(PlatformError::HttpClient).attach(format!( + "deadline response for backend '{}' has no originating request or method hint", + self.backend_name + )) + }) + } + + fn buffered_bytes(&self) -> usize { + match &self.state { + Some(PendingState::Body { bytes, .. }) => bytes.len(), + _ => 0, + } + } + + fn poll( + &mut self, + deadline: Instant, + available_buffer_bytes: usize, + ) -> Result, Report> { + let state = self + .state + .take() + .expect("should retain pending collection state"); + let state = match state { + PendingState::Headers(mut slot) => { + match slot.take().expect("should retain a pending request").poll() { + PollResult::Pending(pending) => { + *slot = Some(pending); + self.state = Some(PendingState::Headers(slot)); + return Ok(None); + } + PollResult::Done(Err(error)) => { + self.backend_name = error.backend_name().to_owned(); + return Err( + Report::new(PlatformError::HttpClient).attach(error.to_string()) + ); + } + PollResult::Done(Ok(mut response)) => { + if let Some(backend_name) = response.get_backend_name() { + self.backend_name = backend_name.to_owned(); + } + let request_is_head = self.request_is_head(&response)?; + if !response_carries_body(request_is_head, response.get_status()) { + return fastly_response_to_platform( + response, + self.backend_name.clone(), + false, + request_is_head, + ) + .map(Some); + } + if response + .get_header("content-length") + .and_then(|value| value.to_str().ok()) + .and_then(|value| value.trim().parse::().ok()) + .is_some_and(|length| length > MAX_PLATFORM_RESPONSE_BODY_BYTES as u64) + { + return Err(Report::new(PlatformError::HttpClient) + .attach("origin Content-Length exceeds response body limit")); + } + let body = response.take_body().into_handle(); + PendingState::Body { + response: Box::new(response), + body, + bytes: Vec::new(), + } + } + } + } + state => state, + }; + let PendingState::Body { + mut response, + mut body, + mut bytes, + } = state + else { + unreachable!("headers are polled before buffering bodies"); + }; + if Instant::now() >= deadline || !body_is_ready(&body)? || Instant::now() >= deadline { + self.state = Some(PendingState::Body { + response, + body, + bytes, + }); + return Ok(None); + } + let mut chunk = [0_u8; 8192]; + let read = body + .read(&mut chunk) + .change_context(PlatformError::HttpClient)?; + if read == 0 { + response.set_body(bytes); + return fastly_response_to_platform(*response, self.backend_name.clone(), false, false) + .map(Some); + } + if bytes.len() + read > MAX_PLATFORM_RESPONSE_BODY_BYTES { + return Err(Report::new(PlatformError::HttpClient) + .attach("origin response body exceeds response body limit")); + } + if read > available_buffer_bytes { + return Err(Report::new(PlatformError::HttpClient) + .attach("origin response body exceeds aggregate auction buffer capacity")); + } + bytes.extend_from_slice(&chunk[..read]); + self.state = Some(PendingState::Body { + response, + body, + bytes, + }); + Ok(None) + } +} + +fn body_is_ready(body: &BodyHandle) -> Result> { + let mut ready = 0; + // SAFETY: the live owned body retains the handle while this hostcall borrows it. + unsafe { fastly_sys::fastly_async_io::is_ready(body.as_u32(), &mut ready) } + .result() + .map_err(|status| { + Report::new(PlatformError::HttpClient) + .attach(format!("body readiness hostcall failed: {status:?}")) + })?; + Ok(ready != 0) +} + +/// Collect one fully buffered response before the absolute deadline. +/// +/// Dropping response bodies closes their host handles. Fastly has no pending +/// request cancellation ABI: unfinished header requests are abandoned and +/// never waited on past the auction deadline. +/// +/// Partial bodies share a 10 MiB payload budget. A provider whose next chunk +/// would exceed that capacity fails with an attributed HTTP client error; +/// its buffered bytes are released and its siblings remain collectible. +/// Completed response payload plus retained partial bodies fits the same budget. +/// +/// # Errors +/// +/// Returns an error for invalid handles or a failure of the timed wait hostcall. +/// Provider errors are attributed in the returned collection result. +pub(super) fn select_until( + pending_requests: Vec, + deadline: Instant, +) -> Result, Report> { + if Instant::now() >= deadline { + return Ok(None); + } + if pending_requests.is_empty() { + return Err(Report::new(PlatformError::HttpClient) + .attach("select_until called with an empty pending_requests list")); + } + let mut pending = pending_requests + .into_iter() + .map(DeadlinePending::from_platform) + .collect::, _>>()?; + let mut buffered_bytes: usize = pending.iter().map(DeadlinePending::buffered_bytes).sum(); + if buffered_bytes > MAX_TOTAL_BUFFERED_BODY_BYTES { + return Err(Report::new(PlatformError::HttpClient) + .attach("resumed auction responses exceed aggregate buffer capacity")); + } + loop { + for index in 0..pending.len() { + if Instant::now() >= deadline { + return Ok(None); + } + let previous_bytes = pending[index].buffered_bytes(); + let result = + pending[index].poll(deadline, MAX_TOTAL_BUFFERED_BODY_BYTES - buffered_bytes); + buffered_bytes = buffered_bytes - previous_bytes + pending[index].buffered_bytes(); + // A host event arriving at the deadline must not make a late bid eligible. + if Instant::now() >= deadline { + return Ok(None); + } + let ready = match result { + Ok(None) => continue, + Ok(Some(response)) => Ok(response), + Err(error) => Err(error), + }; + let completed = pending.swap_remove(index); + let failed_backend_name = ready.as_ref().err().map(|_| completed.backend_name); + return Ok(Some(PlatformSelectResult { + ready, + remaining: pending + .into_iter() + .map(DeadlinePending::into_platform) + .collect(), + failed_backend_name, + })); + } + wait_for_progress(&pending, deadline)?; + } +} + +fn wait_for_progress( + pending: &[DeadlinePending], + deadline: Instant, +) -> Result<(), Report> { + let Some(mut remaining) = deadline.checked_duration_since(Instant::now()) else { + return Ok(()); + }; + if remaining.is_zero() { + return Ok(()); + } + let mut bodies = Vec::new(); + for pending in pending { + match &pending.state { + Some(PendingState::Headers(_)) => { + // High-level PendingRequest keeps its host handle private. Poll + // at most 1ms apart without spinning while headers are pending. + remaining = remaining.min(Duration::from_millis(1)); + } + Some(PendingState::Body { body, .. }) => { + // SAFETY: these handles remain owned by pending throughout select. + bodies.push(unsafe { body.as_u32() }); + } + None => {} + } + } + let timeout_ms = u32::try_from(remaining.as_millis()) + .unwrap_or(u32::MAX) + .max(1); + let mut ready_index = 0; + // SAFETY: bodies is a live slice, and ready_index is writable. A positive + // timeout is essential: the host interprets zero as an unbounded wait. + unsafe { + fastly_sys::fastly_async_io::select( + bodies.as_ptr(), + bodies.len(), + timeout_ms, + &mut ready_index, + ) + } + .result() + .map_err(|status| { + Report::new(PlatformError::HttpClient) + .attach(format!("timed select hostcall failed: {status:?}")) + })?; + Ok(()) +} + +#[cfg(test)] +mod tests { + use std::time::Duration; + + use super::super::{FastlyPlatformHttpClient, MAX_PLATFORM_RESPONSE_BODY_BYTES}; + use super::{DeadlinePending, PendingState}; + use edgezero_core::body::Body; + use trusted_server_core::platform::{ + PlatformHttpClient, PlatformHttpRequest, PlatformPendingRequest, + }; + use web_time::Instant; + + #[test] + fn expired_deadline_does_not_accept_a_pending_result() { + let result = futures::executor::block_on(FastlyPlatformHttpClient.select_until( + vec![PlatformPendingRequest::new(42_u32).with_backend_name("example")], + Instant::now(), + )) + .expect("should report expiration before inspecting handles"); + assert!(result.is_none(), "should discard expired pending requests"); + } + + #[test] + fn deadline_collection_rejects_malformed_handles_with_backend_context() { + let result = futures::executor::block_on(FastlyPlatformHttpClient.select_until( + vec![PlatformPendingRequest::new(42_u32).with_backend_name("example")], + Instant::now() + Duration::from_secs(1), + )); + let error = result.expect_err("should reject a malformed pending handle"); + assert!( + format!("{error:?}").contains("example"), + "should identify the malformed handle's backend" + ); + } + + fn buffered_pending(bytes: Vec, backend_name: &str) -> PlatformPendingRequest { + let mut response = fastly::Response::from_body(bytes); + let body = response.take_body().into_handle(); + DeadlinePending { + backend_name: backend_name.to_owned(), + request_method: Some(edgezero_core::http::Method::GET), + state: Some(PendingState::Body { + response: Box::new(response), + body, + bytes: Vec::new(), + }), + } + .into_platform() + } + + #[test] + fn expired_deadline_discards_an_already_ready_body() { + let pending = buffered_pending(b"ready".to_vec(), "example"); + let result = futures::executor::block_on( + FastlyPlatformHttpClient.select_until(vec![pending], Instant::now()), + ) + .expect("should discard an expired ready response"); + assert!( + result.is_none(), + "should never accept a ready response after expiration" + ); + } + + #[test] + fn remaining_handle_retains_the_original_request_method_hint() { + let pending = buffered_pending(b"body".to_vec(), "example") + .with_response_handling(false, edgezero_core::http::Method::HEAD); + let resumed = DeadlinePending::from_platform(pending) + .expect("should resume a deadline pending request"); + let remaining = resumed.into_platform(); + assert_eq!( + remaining.request_method(), + Some(&edgezero_core::http::Method::HEAD), + "should retain the HEAD method through remaining-handle reconstruction" + ); + let resumed = + DeadlinePending::from_platform(remaining).expect("should resume the remaining handle"); + let response = fastly::Response::from_body("advisory body"); + assert!( + response.get_backend_request().is_none(), + "should exercise a response without SDK originating-request metadata" + ); + assert!( + resumed + .request_is_head(&response) + .expect("should use the preserved method hint"), + "should retain HEAD body semantics after resuming without response origin metadata" + ); + } + + #[test] + fn missing_request_method_and_response_origin_returns_typed_error() { + let pending = buffered_pending(b"body".to_vec(), "example"); + let mut pending = DeadlinePending::from_platform(pending) + .expect("should inspect a buffered pending request"); + pending.request_method = None; + let response = fastly::Response::new(); + let error = pending + .request_is_head(&response) + .expect_err("should reject a response whose request method is unknown"); + assert!( + matches!( + error.current_context(), + trusted_server_core::platform::PlatformError::HttpClient + ), + "should return a typed HTTP client error" + ); + assert!( + format!("{error:?}").contains("example"), + "should identify the backend whose originating method is missing" + ); + } + + #[test] + fn body_collection_is_fair_and_preserves_partial_siblings() { + let large = vec![b'x'; 32768]; + let pending = vec![ + buffered_pending(large.clone(), "large"), + buffered_pending(b"fast".to_vec(), "fast"), + ]; + let deadline = Instant::now() + Duration::from_secs(10); + let result = + futures::executor::block_on(FastlyPlatformHttpClient.select_until(pending, deadline)) + .expect("should collect one response") + .expect("should complete before deadline"); + let response = result + .ready + .expect("should collect the short response first"); + assert_eq!( + response.backend_name.as_deref(), + Some("fast"), + "should give siblings a fair opportunity to complete" + ); + assert_eq!( + result.remaining.len(), + 1, + "should retain the partial response" + ); + let result = futures::executor::block_on( + FastlyPlatformHttpClient.select_until(result.remaining, deadline), + ) + .expect("should resume body collection") + .expect("should finish the remaining body"); + let response = result.ready.expect("should finish the partial body"); + let (_, body) = response.response.into_parts(); + let bytes = body + .into_bytes() + .expect("should read the complete buffered body"); + assert_eq!( + bytes.as_ref(), + large.as_slice(), + "should preserve all bytes across collector calls" + ); + } + + #[test] + fn incremental_body_limit_reports_the_failed_backend() { + let pending = buffered_pending(vec![b'x'; MAX_PLATFORM_RESPONSE_BODY_BYTES + 1], "large"); + let result = futures::executor::block_on( + FastlyPlatformHttpClient + .select_until(vec![pending], Instant::now() + Duration::from_secs(30)), + ) + .expect("should return a provider failure") + .expect("should reject the body before deadline"); + assert!( + result.ready.is_err(), + "should enforce the incremental body cap" + ); + assert_eq!( + result.failed_backend_name.as_deref(), + Some("large"), + "should attribute body conversion failures" + ); + } + + #[test] + fn aggregate_body_limit_rejects_growth_and_preserves_partial_siblings() { + let half_budget = MAX_PLATFORM_RESPONSE_BODY_BYTES / 2; + let partial = |unread: Vec, backend: &str| { + let mut pending = DeadlinePending::from_platform(buffered_pending(unread, backend)) + .expect("should construct a partially buffered response"); + if let Some(PendingState::Body { bytes, .. }) = &mut pending.state { + *bytes = vec![b'x'; half_budget]; + } + pending.into_platform() + }; + let pending = vec![ + partial(vec![b'x'], "growing"), + partial(Vec::new(), "sibling"), + ]; + let deadline = Instant::now() + Duration::from_secs(10); + let result = + futures::executor::block_on(FastlyPlatformHttpClient.select_until(pending, deadline)) + .expect("should return an attributed capacity failure") + .expect("should reject excess buffer growth before deadline"); + assert!( + result.ready.is_err(), + "should reject growth beyond aggregate buffer capacity" + ); + assert_eq!( + result.failed_backend_name.as_deref(), + Some("growing"), + "should attribute the capacity failure to the growing provider" + ); + assert_eq!( + result.remaining.len(), + 1, + "should preserve the partial sibling response" + ); + let result = futures::executor::block_on( + FastlyPlatformHttpClient.select_until(result.remaining, deadline), + ) + .expect("should resume the preserved sibling") + .expect("should complete its already buffered body"); + let response = result.ready.expect("should retain the successful sibling"); + assert_eq!( + response.backend_name.as_deref(), + Some("sibling"), + "should preserve backend correlation" + ); + let (_, body) = response.response.into_parts(); + let bytes = body + .into_bytes() + .expect("should retain a buffered response body"); + assert_eq!( + bytes.len(), + half_budget, + "should preserve all previously buffered sibling bytes" + ); + } + + fn network_pending(path: &str, method: &str) -> PlatformPendingRequest { + let request = edgezero_core::http::request_builder() + .method(method) + .uri(format!("http://example.com/{path}")) + .body(Body::empty()) + .expect("should build fixture request"); + futures::executor::block_on( + FastlyPlatformHttpClient.send_async( + PlatformHttpRequest::new(request, "deadline_origin").with_cache_bypass(), + ), + ) + .expect("should launch fixture request") + } + + #[test] + #[ignore = "requires the local origin started by tests/deadline_origin.py"] + fn network_stalled_headers_stop_at_deadline() { + let pending = network_pending("stalled", "GET"); + let started = Instant::now(); + let result = futures::executor::block_on( + FastlyPlatformHttpClient + .select_until(vec![pending], started + Duration::from_millis(80)), + ) + .expect("should stop waiting for stalled headers"); + assert!( + result.is_none(), + "should expire while headers remain pending" + ); + assert!( + started.elapsed() < Duration::from_millis(500), + "should bound the synchronous header wait" + ); + } + + #[test] + #[ignore = "requires the local origin started by tests/deadline_origin.py"] + fn network_trickling_body_stops_at_deadline() { + let pending = network_pending("trickle", "GET"); + let started = Instant::now(); + let result = futures::executor::block_on( + FastlyPlatformHttpClient + .select_until(vec![pending], started + Duration::from_millis(80)), + ) + .expect("should stop buffering a trickling body"); + assert!( + result.is_none(), + "should expire before the trickling body completes" + ); + assert!( + started.elapsed() < Duration::from_millis(500), + "should bound body reads at the auction deadline" + ); + } + + #[test] + #[ignore = "requires the local origin started by tests/deadline_origin.py"] + fn network_fast_sibling_completes_while_another_body_trickles() { + let slow = network_pending("trickle", "GET"); + let fast = network_pending("fast", "GET"); + let started = Instant::now(); + let result = futures::executor::block_on( + FastlyPlatformHttpClient + .select_until(vec![slow, fast], started + Duration::from_millis(200)), + ) + .expect("should collect the completed sibling") + .expect("should finish before deadline"); + let response = result.ready.expect("should collect the fast body"); + let (_, body) = response.response.into_parts(); + let bytes = body.into_bytes().expect("should read the fast body"); + assert_eq!( + bytes.as_ref(), + b"fast", + "should avoid starvation by a trickling sibling" + ); + assert_eq!( + result.remaining.len(), + 1, + "should retain the incomplete sibling" + ); + } + + #[test] + #[ignore = "requires the local origin started by tests/deadline_origin.py"] + fn network_head_skips_the_body_and_its_declared_length_limit() { + let pending = network_pending("head", "HEAD"); + let result = futures::executor::block_on( + FastlyPlatformHttpClient + .select_until(vec![pending], Instant::now() + Duration::from_secs(1)), + ) + .expect("should collect HEAD response") + .expect("should finish before deadline"); + assert!( + result.ready.is_ok(), + "should accept HEAD with an advisory large content-length" + ); + } +} diff --git a/crates/trusted-server-adapter-fastly/tests/deadline_origin.py b/crates/trusted-server-adapter-fastly/tests/deadline_origin.py new file mode 100644 index 0000000000..570c04f7d7 --- /dev/null +++ b/crates/trusted-server-adapter-fastly/tests/deadline_origin.py @@ -0,0 +1,71 @@ +"""Run ignored Fastly deadline tests against a local, deliberately slow origin. + +Usage: python3 tests/deadline_origin.py /path/to/viceroy /path/to/test-binary.wasm +""" + +import http.server +import os +import pathlib +import subprocess +import sys +import tempfile +import threading +import time + + +class Origin(http.server.BaseHTTPRequestHandler): + def do_HEAD(self): + self.send_response(200) + self.send_header("Content-Length", str(20 * 1024 * 1024)) + self.end_headers() + + def do_GET(self): + try: + if self.path == "/stalled": + time.sleep(2) + if self.path == "/trickle": + self.send_response(200) + self.send_header("Content-Length", "100") + self.end_headers() + for _ in range(100): + self.wfile.write(b"x") + self.wfile.flush() + time.sleep(0.01) + return + time.sleep(0.03) + self.send_response(200) + self.send_header("Content-Length", "4") + self.end_headers() + self.wfile.write(b"fast") + except (BrokenPipeError, ConnectionResetError): + pass + + def log_message(self, *_args): + pass + + +def main(): + with http.server.ThreadingHTTPServer(("127.0.0.1", 0), Origin) as server: + server.daemon_threads = True + threading.Thread(target=server.serve_forever, daemon=True).start() + with tempfile.TemporaryDirectory(prefix="fastly-deadline-") as directory: + config = pathlib.Path(directory) / "fastly.toml" + config.write_text( + '[local_server.backends.deadline_origin]\n' + f'url = "http://127.0.0.1:{server.server_port}"\n' + ) + environment = os.environ.copy() + if pathlib.Path("/etc/ssl/cert.pem").exists(): + environment.setdefault("SSL_CERT_FILE", "/etc/ssl/cert.pem") + result = subprocess.run( + [sys.argv[1], "run", "-C", str(config), sys.argv[2], + "platform::deadline::tests::network_", "--ignored", "--nocapture"], + env=environment, + check=False, + ) + server.shutdown() + return result.returncode + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/crates/trusted-server-adapter-spin/Cargo.toml b/crates/trusted-server-adapter-spin/Cargo.toml index 81eeb94307..b5c81acd8e 100644 --- a/crates/trusted-server-adapter-spin/Cargo.toml +++ b/crates/trusted-server-adapter-spin/Cargo.toml @@ -31,6 +31,7 @@ flate2 = { workspace = true } futures = { workspace = true } http-body-util = { workspace = true } log = { workspace = true } +web-time = { workspace = true } trusted-server-core = { workspace = true } trusted-server-js = { workspace = true } diff --git a/crates/trusted-server-adapter-spin/src/platform.rs b/crates/trusted-server-adapter-spin/src/platform.rs index b1ced0f70a..27468fe6b8 100644 --- a/crates/trusted-server-adapter-spin/src/platform.rs +++ b/crates/trusted-server-adapter-spin/src/platform.rs @@ -427,22 +427,19 @@ fn is_wasi_forbidden_outbound_header(name: &str) -> bool { || name.eq_ignore_ascii_case("proxy-connection") } -/// Carries a completed response through `send_async` → `select`. +/// Owns the outbound operation until selection completes or drops it. #[cfg(all(feature = "spin", target_arch = "wasm32"))] struct SpinPendingResponse { - backend_name: String, - status: u16, - headers: HeaderPairs, - body: Vec, + response: + futures::future::LocalBoxFuture<'static, Result>>, } /// `spin_sdk::http::send`-backed HTTP client for the Spin runtime. /// -/// `send_async` eagerly executes each request before returning, so -/// [`PlatformHttpClient::supports_concurrent_fanout`] reports `false` and the -/// auction orchestrator rejects multi-provider configurations before any -/// request launches. `select` keeps a defense-in-depth rejection for more -/// than one pending request, matching the Cloudflare adapter behavior. +/// Selection supports one provider at a time, so +/// [`PlatformHttpClient::supports_concurrent_fanout`] reports `false`. +/// Deadline selection bounds the full send and response body collection. +/// Dropping the pending operation cancels its WASI asynchronous operations. /// /// Spin's WASI HTTP API sends one request and returns the original response; it /// exposes no redirect-follow policy. Each trait call therefore reaches exactly @@ -451,20 +448,14 @@ struct SpinPendingResponse { /// `StubHttpClient` driver records the one-send 3xx behavior while adapter tests /// cover request/response policy around that single boundary. /// -/// # Known MVP limits -/// -/// **No configurable outbound timeout.** `spin_sdk::http::send` does not -/// expose per-request timeout control, and [`PlatformBackendSpec::first_byte_timeout`] -/// is ignored by [`NoopBackend`]. A slow or hung origin will block the Spin -/// invocation for whatever default the Spin runtime imposes. Operators requiring -/// deterministic timeout behaviour should use the Fastly adapter. +/// Backend phase timeouts remain unsupported. Callers requiring an absolute +/// cutoff use [`PlatformHttpClient::select_until`]. #[cfg(all(feature = "spin", target_arch = "wasm32"))] pub struct SpinPlatformHttpClient; #[cfg(all(feature = "spin", target_arch = "wasm32"))] impl SpinPlatformHttpClient { async fn execute( - &self, request: PlatformHttpRequest, ) -> Result> { reject_unsupported_request_contracts(&request)?; @@ -554,9 +545,8 @@ impl SpinPlatformHttpClient { #[async_trait::async_trait(?Send)] impl PlatformHttpClient for SpinPlatformHttpClient { fn supports_concurrent_fanout(&self) -> bool { - // `send_async` executes each request eagerly, so multiple pending - // requests run sequentially. The auction orchestrator checks this - // before launching more than one provider request. + // Keep the existing single-provider restriction: selection polls one + // outbound request at a time. false } @@ -564,7 +554,7 @@ impl PlatformHttpClient for SpinPlatformHttpClient { &self, request: PlatformHttpRequest, ) -> Result> { - self.execute(request).await + Self::execute(request).await } async fn send_async( @@ -572,28 +562,8 @@ impl PlatformHttpClient for SpinPlatformHttpClient { request: PlatformHttpRequest, ) -> Result> { let backend_name = request.backend_name.clone(); - let response = self.execute(request).await?; - - let status = response.response.status().as_u16(); - let headers: HeaderPairs = response - .response - .headers() - .iter() - .map(|(n, v)| (n.to_string(), v.as_bytes().to_vec())) - .collect(); - let body_bytes = match response.response.into_body() { - edgezero_core::body::Body::Once(bytes) => bytes.to_vec(), - edgezero_core::body::Body::Stream(_) => { - return Err(Report::new(PlatformError::HttpClient) - .attach("SpinPlatformHttpClient::execute returned a streaming body")); - } - }; - let pending = SpinPendingResponse { - backend_name: backend_name.clone(), - status, - headers, - body: body_bytes, + response: Box::pin(Self::execute(request)), }; Ok(PlatformPendingRequest::new(pending).with_backend_name(backend_name)) } @@ -624,21 +594,57 @@ impl PlatformHttpClient for SpinPlatformHttpClient { .attach("unexpected inner type in SpinPlatformHttpClient::select") })?; - let mut builder = edgezero_core::http::response_builder().status(pending.status); - for (name, value) in &pending.headers { - builder = builder.header(name.as_str(), value.as_slice()); - } - let edge_resp = builder - .body(edgezero_core::body::Body::from(pending.body)) - .change_context(PlatformError::HttpClient)?; - - let ready = Ok(PlatformResponse::new(edge_resp).with_backend_name(pending.backend_name)); + let ready = pending.response.await; Ok(PlatformSelectResult { ready, remaining: pending_requests, failed_backend_name: None, }) } + + async fn select_until( + &self, + mut pending_requests: Vec, + deadline: web_time::Instant, + ) -> Result, Report> { + if pending_requests.is_empty() { + return Err(Report::new(PlatformError::HttpClient) + .attach("select_until called with an empty pending_requests list")); + } + if pending_requests.len() > 1 { + return Err(Report::new(PlatformError::HttpClient) + .attach("multi-provider fan-out is not supported on this adapter")); + } + let remaining = deadline.saturating_duration_since(web_time::Instant::now()); + if remaining.is_zero() { + return Ok(None); + } + let platform_pending = pending_requests.remove(0); + let backend_name = platform_pending.backend_name().map(str::to_owned); + let pending = platform_pending + .downcast::() + .map_err(|_| { + Report::new(PlatformError::HttpClient) + .attach("unexpected pending request type in deadline selection") + })?; + let result = complete_before_deadline( + pending.response, + async { + let remaining = deadline.saturating_duration_since(web_time::Instant::now()); + if !remaining.is_zero() { + spin_sdk::time::sleep(remaining).await; + } + }, + deadline, + web_time::Instant::now, + ) + .await; + Ok(result.map(|ready| PlatformSelectResult { + failed_backend_name: ready.as_ref().err().and(backend_name), + ready, + remaining: pending_requests, + })) + } } #[cfg(all(feature = "spin", target_arch = "wasm32"))] @@ -1398,3 +1404,128 @@ mod tests { } } } + +#[cfg(any(test, all(feature = "spin", target_arch = "wasm32")))] +async fn complete_before_deadline( + operation: F, + timer: T, + deadline: web_time::Instant, + now: N, +) -> Option +where + F: std::future::Future, + T: std::future::Future, + N: Fn() -> web_time::Instant, +{ + if now() >= deadline { + return None; + } + match futures::future::select(Box::pin(operation), Box::pin(timer)).await { + futures::future::Either::Left((response, _)) if now() < deadline => Some(response), + _ => None, + } +} + +#[cfg(test)] +mod deadline_tests { + use std::cell::Cell; + use std::rc::Rc; + use std::time::Duration; + + use futures::executor::block_on; + use futures::future::{pending, ready}; + use web_time::Instant; + + use super::complete_before_deadline; + + #[test] + fn expired_deadline_does_not_poll_request() { + let deadline = Instant::now(); + let polled = Cell::new(false); + let operation = async { + polled.set(true); + pending::<()>().await + }; + let result: Option<()> = block_on(complete_before_deadline( + operation, + pending(), + deadline, + || deadline, + )); + assert_eq!(result, None, "should reject before polling expired request"); + assert!(!polled.get(), "should not poll an expired request"); + } + + #[test] + fn timer_cuts_off_pending_operation_and_drops_it() { + struct DropProbe(Rc>); + impl Drop for DropProbe { + fn drop(&mut self) { + self.0.set(true); + } + } + let dropped = Rc::new(Cell::new(false)); + let probe = DropProbe(Rc::clone(&dropped)); + let operation = async move { + let _probe = probe; + pending::<()>().await + }; + let now = Instant::now(); + let result = block_on(complete_before_deadline( + operation, + ready(()), + now + Duration::from_secs(1), + || now, + )); + assert_eq!(result, None, "should cut off a pending request"); + assert!(dropped.get(), "should drop the pending operation at cutoff"); + } + + #[test] + fn accepts_complete_response_before_deadline() { + let now = Instant::now(); + let result = block_on(complete_before_deadline( + ready(42), + pending(), + now + Duration::from_secs(1), + || now, + )); + assert_eq!(result, Some(42), "should accept response before cutoff"); + } + + #[test] + fn rejects_response_that_becomes_ready_at_deadline() { + let now = Instant::now(); + let deadline = now + Duration::from_secs(1); + let clock = Cell::new(now); + let operation = async { + clock.set(deadline); + 42 + }; + let result = block_on(complete_before_deadline( + operation, + pending(), + deadline, + || clock.get(), + )); + assert_eq!(result, None, "should reject a late ready response"); + } + + #[test] + fn timer_bounds_body_after_headers_complete() { + let now = Instant::now(); + let headers_received = Cell::new(false); + let operation = async { + headers_received.set(true); + pending::<()>().await + }; + let result = block_on(complete_before_deadline( + operation, + ready(()), + now + Duration::from_secs(1), + || now, + )); + assert!(headers_received.get(), "should reach response body read"); + assert_eq!(result, None, "should cut off pending body collection"); + } +} diff --git a/crates/trusted-server-core/src/auction/orchestrator.rs b/crates/trusted-server-core/src/auction/orchestrator.rs index 204201e605..1cad8390af 100644 --- a/crates/trusted-server-core/src/auction/orchestrator.rs +++ b/crates/trusted-server-core/src/auction/orchestrator.rs @@ -5,6 +5,7 @@ use error_stack::{Report, ResultExt}; use http::Request; use std::collections::{HashMap, HashSet, hash_map::Entry}; use std::sync::Arc; +use std::time::Duration; use web_time::Instant; use crate::error::TrustedServerError; @@ -232,27 +233,9 @@ fn remaining_budget_ms(start: Instant, timeout_ms: u32) -> u32 { timeout_ms.saturating_sub(elapsed) } -/// Runtime policy for classifying responses that complete after the logical auction budget. -/// -/// Current adapters do not expose an enforceable total-request deadline. They -/// therefore drain already-launched work and accept completed late responses. -#[derive(Debug, Clone, Copy, Default)] -struct AuctionDeadlinePolicy { - enforceable_total_request_deadline: bool, -} - -impl AuctionDeadlinePolicy { - fn rejects_late_completion(self, start: Instant, timeout_ms: u32) -> bool { - self.enforceable_total_request_deadline && remaining_budget_ms(start, timeout_ms) == 0 - } - - fn for_runtime(services: &RuntimeServices) -> Self { - Self { - enforceable_total_request_deadline: services - .http_client() - .has_enforceable_total_request_deadline(), - } - } +/// Returns the original absolute cutoff without rounding transport budgets. +fn collection_deadline(start: Instant, timeout_ms: u32) -> Instant { + start + Duration::from_millis(u64::from(timeout_ms)) } fn routing_metadata(unroutable_bidder_count: u32) -> HashMap { @@ -530,8 +513,17 @@ impl AuctionOrchestratorHarness { } while !pending.is_empty() { - let select_result = match context.services.http_client().select(pending).await { - Ok(result) => result, + let select_result = match context + .services + .http_client() + .select_until( + pending, + collection_deadline(auction_start, context.timeout_ms), + ) + .await + { + Ok(Some(result)) => result, + Ok(None) => break, Err(error) => { log::warn!("Planned provider select failed: {:?}", error); let mut transport_failures = launches @@ -558,10 +550,7 @@ impl AuctionOrchestratorHarness { .to_string(); if let Some(state) = launches.remove(&backend_name) { let elapsed_ms = state.started_at.elapsed().as_millis() as u64; - let deadline_policy = AuctionDeadlinePolicy::for_runtime(context.services); - if deadline_policy - .rejects_late_completion(auction_start, context.timeout_ms) - { + if remaining_budget_ms(auction_start, context.timeout_ms) == 0 { responses.push(provider_timeout_response( state.provider.provider_name(), elapsed_ms, @@ -685,12 +674,19 @@ impl AuctionOrchestratorHarness { Ok(ProviderRequestOutcome::Pending { request: pending, parse_state, - }) => match context.services.http_client().wait(pending).await { + }) => match context + .services + .http_client() + .wait_until( + pending, + collection_deadline(auction_start, context.timeout_ms) + .min(collection_deadline(mediator_start, logical_budget_ms)), + ) + .await + { Ok(platform_response) => { let response_time_ms = mediator_start.elapsed().as_millis() as u64; - if AuctionDeadlinePolicy::for_runtime(context.services) - .rejects_late_completion(auction_start, context.timeout_ms) - { + if remaining_budget_ms(auction_start, context.timeout_ms) == 0 { log::warn!( "Planned mediator '{}' completed after the hard auction deadline; using local ranking ({}ms)", mediator.provider_name(), @@ -961,10 +957,11 @@ impl AuctionOrchestrator { // consumed part of it. Canonicalize the transport timeout so the // backend name remains stable across equivalent budget values. let remaining_ms = remaining_budget_ms(mediation_start, context.timeout_ms); + let logical_budget_ms = remaining_ms.min(mediator.timeout_ms()); let mediator_timeout = context .services .backend() - .canonicalize_transport_timeout_ms(remaining_ms, mediator.timeout_ms()); + .canonicalize_transport_timeout_ms(logical_budget_ms, mediator.timeout_ms()); if mediator_timeout == 0 { log::warn!("Auction timeout exhausted during bidding phase; skipping mediator"); @@ -981,7 +978,7 @@ impl AuctionOrchestrator { let mediator_context = AuctionContext { settings: context.settings, request: context.request, - timeout_ms: mediator_timeout, + timeout_ms: logical_budget_ms, transport_timeout_ms: mediator_timeout, provider_responses: Some(&provider_responses), services: context.services, @@ -1002,18 +999,37 @@ impl AuctionOrchestrator { let platform_resp = mediator_context .services .http_client() - .wait(pending) - .await - .change_context(TrustedServerError::Auction { + .wait_until( + pending, + collection_deadline(mediation_start, context.timeout_ms) + .min(collection_deadline(start_time, logical_budget_ms)), + ) + .await; + if platform_resp.as_ref().is_err_and(|error| { + matches!( + error.current_context(), + crate::platform::PlatformError::DeadlineExceeded + ) + }) { + let winning_bids = + self.select_winning_bids(&provider_responses, &floor_prices); + return Ok(OrchestrationResult { + provider_responses, + mediator_response: None, + winning_bids, + total_time_ms: mediation_start.elapsed().as_millis() as u64, + metadata: HashMap::new(), + }); + } + let platform_resp = + platform_resp.change_context(TrustedServerError::Auction { message: format!( "Mediator {} request failed", mediator.provider_name() ), })?; let response_time_ms = start_time.elapsed().as_millis() as u64; - if AuctionDeadlinePolicy::for_runtime(context.services) - .rejects_late_completion(mediation_start, context.timeout_ms) - { + if remaining_budget_ms(mediation_start, context.timeout_ms) == 0 { log::warn!( "Mediator '{}' completed after the hard auction deadline; using local ranking ({}ms)", mediator.provider_name(), @@ -1313,30 +1329,39 @@ impl AuctionOrchestrator { ), })); } - - let deadline_policy = AuctionDeadlinePolicy::for_runtime(context.services); log::info!( "Launched {} concurrent provider request(s); waiting for responses", pending_requests.len() ); - // Phase 2: Wait for responses using select() to process as they become ready. - // Enforce the auction deadline: after each select() returns, check - // elapsed time and drop remaining requests if the timeout is exceeded. - // - // NOTE: `select()` blocks until at least one backend responds and, on - // some adapters, buffers the selected response body before returning. - // Backend first-byte and between-bytes timeouts are capped to the - // remaining auction budget in Phase 1. They are transport timers, not - // absolute wall-clock limits, so connection setup and byte-trickling - // remain bounded operational risks rather than strict deadline proof. + // Collection bounds header waits and body buffering by the original cutoff. let mut remaining = pending_requests; while !remaining.is_empty() { - let platform_result = match context.services.http_client().select(remaining).await { - Ok(r) => r, + let platform_result = match context + .services + .http_client() + .select_until( + remaining, + collection_deadline(auction_start, context.timeout_ms), + ) + .await + { + Ok(Some(r)) => r, + Ok(None) => break, Err(e) => { log::warn!("select() failed: {:?}", e); + let mut failures = backend_to_provider + .drain() + .map(|(_, state)| { + provider_transport_failed_response( + &state.provider_name, + state.started_at.elapsed().as_millis() as u64, + ) + }) + .collect::>(); + failures.sort_by(|left, right| left.provider.cmp(&right.provider)); + responses.extend(failures); break; } }; @@ -1358,9 +1383,7 @@ impl AuctionOrchestrator { if let Some(state) = backend_to_provider.remove(&backend_name) { let response_time_ms = state.started_at.elapsed().as_millis() as u64; - if deadline_policy - .rejects_late_completion(auction_start, context.timeout_ms) - { + if remaining_budget_ms(auction_start, context.timeout_ms) == 0 { responses.push(provider_timeout_response( &state.provider_name, response_time_ms, @@ -1447,11 +1470,6 @@ impl AuctionOrchestrator { } } } - - // Current adapters cannot enforce a hard total-request deadline, so - // drain already-launched handles and retain completed late responses. - // A future adapter that explicitly claims the capability classifies - // each late completion as a timeout instead. } for state in backend_to_provider.into_values() { @@ -2122,17 +2140,17 @@ impl AuctionOrchestrator { let mut responses: Vec = completed_responses; let mut remaining = pending_requests; - let deadline_policy = AuctionDeadlinePolicy::for_runtime(services); while !remaining.is_empty() { let select_result = match services .http_client() - .select(remaining) + .select_until(remaining, collection_deadline(auction_start, timeout_ms)) .await .change_context(TrustedServerError::Auction { message: "HTTP select failed".to_string(), }) { - Ok(r) => r, + Ok(Some(r)) => r, + Ok(None) => break, Err(e) => { log::warn!("select() failed during auction collection: {:?}", e); // An outer select failure means the platform could not poll @@ -2176,7 +2194,7 @@ impl AuctionOrchestrator { let backend_name = platform_response.backend_name.clone().unwrap_or_default(); if let Some(state) = backend_to_provider.remove(&backend_name) { let response_time_ms = state.started_at.elapsed().as_millis() as u64; - if deadline_policy.rejects_late_completion(auction_start, timeout_ms) { + if remaining_budget_ms(auction_start, timeout_ms) == 0 { responses.push(provider_timeout_response( &state.provider_name, response_time_ms, @@ -2212,7 +2230,7 @@ impl AuctionOrchestrator { } } else if let Some(state) = planned_backend_to_provider.remove(&backend_name) { let response_time_ms = state.started_at.elapsed().as_millis() as u64; - if deadline_policy.rejects_late_completion(auction_start, timeout_ms) { + if remaining_budget_ms(auction_start, timeout_ms) == 0 { responses.push(provider_timeout_response( state.provider.provider_name(), response_time_ms, @@ -2286,14 +2304,6 @@ impl AuctionOrchestrator { } } } - - // Drain every dispatched request. Each backend was capped with - // first-byte and between-bytes timeouts at dispatch time, so by the - // collect phase the remaining handles may already be ready even if - // wall-clock time elapsed while the origin was slow. Dropping them - // here would discard SSP responses that already arrived. The - // mediator launch below still observes A_deadline via - // `remaining_budget_ms`. } for state in backend_to_provider.values() { @@ -2417,17 +2427,23 @@ impl AuctionOrchestrator { Ok(ProviderRequestOutcome::Pending { request: pending, parse_state, - }) => match services.http_client().wait(pending).await.change_context( - TrustedServerError::Auction { + }) => match services + .http_client() + .wait_until( + pending, + collection_deadline(auction_start, timeout_ms) + .min(collection_deadline(mediator_start, logical_budget_ms)), + ) + .await + .change_context(TrustedServerError::Auction { message: format!( "Mediator {} request failed", mediator.provider_name() ), - }, - ) { + }) { Ok(platform_resp) => { let response_time_ms = mediator_start.elapsed().as_millis() as u64; - if deadline_policy.rejects_late_completion(auction_start, timeout_ms) { + if remaining_budget_ms(auction_start, timeout_ms) == 0 { log::warn!( "Mediator '{}' completed after the hard auction deadline; using local ranking ({}ms)", mediator.provider_name(), @@ -3282,6 +3298,14 @@ mod tests { self.inner.send_async(request).await } + async fn select_until( + &self, + pending_requests: Vec, + _deadline: Instant, + ) -> Result, Report> { + self.select(pending_requests).await.map(Some) + } + async fn select( &self, pending_requests: Vec, @@ -3459,7 +3483,7 @@ mod tests { budgets: Option, } - struct PendingDeadlineMediator; + struct PendingDeadlineMediator(u32); #[async_trait::async_trait(?Send)] impl AuctionProvider for PendingDeadlineMediator { @@ -3504,7 +3528,7 @@ mod tests { } fn timeout_ms(&self) -> u32 { - 1_000 + self.0 } fn backend_name(&self, _services: &RuntimeServices, _timeout_ms: u32) -> Option { @@ -4025,7 +4049,7 @@ mod tests { stub.set_enforceable_total_request_deadline(enforceable_total_request_deadline); stub.push_response(200, b"{}".to_vec()); stub.push_response(200, b"{}".to_vec()); - stub.push_select_delay(Duration::from_millis(50)); + stub.push_select_delay(Duration::from_millis(200)); let services = build_services_with_http_client(Arc::clone(&stub) as Arc<_>); let config = AuctionConfig { enabled: true, @@ -4072,39 +4096,45 @@ mod tests { } #[tokio::test] - async fn current_adapter_deadline_drains_late_responses_in_both_paths() { + async fn current_adapter_deadline_stops_collection_in_both_paths() { for split in [false, true] { let result = collect_deadline_test_result(split, false).await; - assert_eq!(result.provider_responses.len(), 2); - assert_eq!(result.provider_responses[0].provider, "late-one"); - assert_eq!(result.provider_responses[0].status, BidStatus::Success); - assert_eq!(result.provider_responses[1].provider, "late-two"); - assert_eq!(result.provider_responses[1].status, BidStatus::Success); + assert_eq!( + result.provider_responses.len(), + 2, + "should account for every provider" + ); assert!( - result - .provider_responses - .iter() - .all(|response| response.response_time_ms >= 50), - "late response times should retain actual elapsed duration" + result.provider_responses.iter().all(|response| { + response.status == BidStatus::Error + && response.metadata["error_type"] == ERROR_TYPE_TIMEOUT + }), + "should time out outstanding providers on every adapter" ); - assert_eq!( - result.winning_bids["slot-1"].bidder, "late-one", - "a completed response remains eligible after the logical deadline" + assert!( + result.total_time_ms < 150, + "should stop before the delayed provider completes: {}ms", + result.total_time_ms + ); + assert!( + result.winning_bids.is_empty(), + "should not accept late bids" ); } } #[tokio::test] - async fn synthetic_hard_deadline_classifies_late_responses_in_both_paths() { + async fn advertised_transport_deadline_still_bounds_collection_in_both_paths() { for split in [false, true] { let result = collect_deadline_test_result(split, true).await; - assert_eq!(result.provider_responses.len(), 2); - assert!(result.provider_responses.iter().all(|response| { - response.status == BidStatus::Error - && response.metadata["error_type"] == ERROR_TYPE_TIMEOUT - && response.response_time_ms >= 50 - })); - assert!(result.winning_bids.is_empty()); + assert!( + result + .provider_responses + .iter() + .all(|response| response.metadata["error_type"] == ERROR_TYPE_TIMEOUT), + "should classify both outstanding requests as timeouts" + ); + assert!(result.winning_bids.is_empty(), "should discard late bids"); } } @@ -4117,7 +4147,7 @@ mod tests { stub.push_response(200, b"{}".to_vec()); stub.push_response(200, b"{}".to_vec()); stub.push_select_delay(Duration::ZERO); - stub.push_wait_delay(Duration::from_millis(50)); + stub.push_wait_delay(Duration::from_millis(200)); let services = build_services_with_http_client(Arc::clone(&stub) as Arc<_>); let config = AuctionConfig { enabled: true, @@ -4131,7 +4161,7 @@ mod tests { name: "local", backend: "local-backend", })); - orchestrator.register_provider(Arc::new(PendingDeadlineMediator)); + orchestrator.register_provider(Arc::new(PendingDeadlineMediator(1_000))); let request = create_test_auction_request(); let settings = create_test_settings(); let downstream = http::Request::new(edgezero_core::body::Body::empty()); @@ -4162,31 +4192,29 @@ mod tests { } #[tokio::test] - async fn pending_mediator_late_completion_policy_is_equivalent_in_sync_and_split_paths() { + async fn pending_mediator_is_bounded_in_sync_and_split_paths() { for split in [false, true] { - let current = pending_mediator_deadline_test_result(split, false).await; - let current_mediator = current - .mediator_response - .as_ref() - .expect("current adapters should accept completed late mediator responses"); - assert!( - current_mediator.response_time_ms >= 50, - "mediator timing should preserve actual elapsed duration" - ); - assert_eq!(current.winning_bids["slot-1"].bidder, "mediated"); - - let hard = pending_mediator_deadline_test_result(split, true).await; - assert!(hard.mediator_response.is_none()); - assert_eq!(hard.winning_bids["slot-1"].bidder, "local"); - assert!( - hard.total_time_ms >= 50, - "discarding a late mediator must retain actual total elapsed time" - ); + for advertised_deadline in [false, true] { + let result = + pending_mediator_deadline_test_result(split, advertised_deadline).await; + assert!( + result.mediator_response.is_none(), + "should abandon a late mediator" + ); + assert_eq!( + result.winning_bids["slot-1"].bidder, "local", + "should retain provider winner" + ); + assert!( + result.total_time_ms < 150, + "should not wait for the delayed mediator" + ); + } } } #[tokio::test] - async fn split_deadline_skips_mediator_and_falls_back_to_provider_winner() { + async fn split_deadline_skips_mediator_and_discards_late_provider() { let stub = Arc::new(StubHttpClient::new()); stub.push_response(200, b"{}".to_vec()); stub.push_select_delay(Duration::from_millis(50)); @@ -4230,11 +4258,14 @@ mod tests { assert_eq!(launches.load(Ordering::Relaxed), 0); assert!(result.mediator_response.is_none()); - assert_eq!(result.winning_bids["slot-1"].bidder, "late-one"); + assert!( + result.winning_bids.is_empty(), + "should not retain a late provider bid" + ); } #[tokio::test] - async fn synchronous_deadline_skips_mediator_and_falls_back_to_provider_winner() { + async fn synchronous_deadline_skips_mediator_and_discards_late_provider() { let stub = Arc::new(StubHttpClient::new()); stub.push_response(200, b"{}".to_vec()); stub.push_select_delay(Duration::from_millis(50)); @@ -4274,7 +4305,10 @@ mod tests { assert_eq!(launches.load(Ordering::Relaxed), 0); assert!(result.mediator_response.is_none()); - assert_eq!(result.winning_bids["slot-1"].bidder, "late-one"); + assert!( + result.winning_bids.is_empty(), + "should not retain a late provider bid" + ); } fn create_test_auction_request() -> AuctionRequest { @@ -5800,8 +5834,141 @@ mod tests { ); } + #[tokio::test] + async fn planned_collection_preserves_on_time_bid_and_times_out_pending_sibling() { + for split in [false, true] { + let http = Arc::new(StubHttpClient::new()); + http.push_response( + 200, + serde_json::to_vec(&serde_json::json!({ + "seatbid": [{"seat": "example", "bid": [{ + "id":"on-time", "impid":"fictional-slot", "price":2.0, + "adm":"
fictional
", "w":300, "h":250 + }]}] + })) + .expect("should serialize on-time bid"), + ); + http.push_response(204, Vec::new()); + http.push_select_delay(Duration::ZERO); + http.push_select_delay(Duration::from_millis(250)); + let services = build_services_with_backend_and_http_client( + Arc::new(NamingBackend::new(BackendNamingPolicy::Axum)), + Arc::clone(&http) as Arc<_>, + ); + let plan = AuctionPlan::compile(planned_config( + &[ + ("provider-a", RoutingMode::AllEligible), + ("provider-b", RoutingMode::AllEligible), + ], + false, + )) + .expect("should compile two providers"); + let orchestrator = AuctionOrchestrator::from_plan(Arc::new(plan), None); + let request = planned_request(); + let settings = create_test_settings(); + let inbound = http::Request::new(edgezero_core::body::Body::empty()); + let context = AuctionContext { + settings: &settings, + request: &inbound, + timeout_ms: 30, + transport_timeout_ms: 30, + provider_responses: None, + services: &services, + }; + let result = if split { + let DispatchAuctionOutcome::Dispatched(dispatched) = + orchestrator.dispatch_auction(&request, &context).await + else { + panic!("should dispatch providers"); + }; + orchestrator + .collect_dispatched_auction(dispatched, &services, &context) + .await + } else { + orchestrator + .run_auction(&request, &context) + .await + .expect("should collect partial results") + }; + assert_eq!( + result.provider_responses.len(), + 2, + "should retain one result per provider" + ); + assert_eq!( + result.provider_responses[0].provider, "provider-a", + "should retain configured ordering" + ); + assert_eq!( + result.provider_responses[1].provider, "provider-b", + "should attribute timeout to outstanding sibling" + ); + assert_eq!( + result.provider_responses[1].metadata["error_type"], ERROR_TYPE_TIMEOUT, + "should report deadline expiry as timeout" + ); + assert_eq!( + result.winning_bids["fictional-slot"].bid_id.as_deref(), + Some("on-time"), + "should preserve the collected winner" + ); + assert!( + result.total_time_ms < 200, + "should not drain the delayed provider" + ); + } + } + + #[tokio::test] + async fn planned_split_collection_uses_dispatch_deadline_after_slow_origin() { + let http = Arc::new(StubHttpClient::new()); + http.push_response(204, Vec::new()); + let services = build_services_with_backend_and_http_client( + Arc::new(NamingBackend::new(BackendNamingPolicy::Axum)), + http, + ); + let plan = AuctionPlan::compile(planned_config( + &[("provider-a", RoutingMode::AllEligible)], + false, + )) + .expect("should compile provider"); + let orchestrator = AuctionOrchestrator::from_plan(Arc::new(plan), None); + let request = planned_request(); + let settings = create_test_settings(); + let inbound = http::Request::new(edgezero_core::body::Body::empty()); + let context = AuctionContext { + settings: &settings, + request: &inbound, + timeout_ms: 10, + transport_timeout_ms: 10, + provider_responses: None, + services: &services, + }; + let DispatchAuctionOutcome::Dispatched(dispatched) = + orchestrator.dispatch_auction(&request, &context).await + else { + panic!("should dispatch provider"); + }; + std::thread::sleep(Duration::from_millis(25)); + let started = Instant::now(); + let result = orchestrator + .collect_dispatched_auction(dispatched, &services, &context) + .await; + assert_eq!( + result.provider_responses[0].metadata["error_type"], ERROR_TYPE_TIMEOUT, + "should not restart the auction clock at collection" + ); + assert!( + started.elapsed() < Duration::from_millis(100), + "should return without waiting again" + ); + } + async fn planned_pending_mediator_deadline_result( enforceable_total_request_deadline: bool, + auction_timeout_ms: u32, + mediator_timeout_ms: u32, + policy: BackendNamingPolicy, ) -> OrchestrationResult { let http = Arc::new(StubHttpClient::new()); http.set_enforceable_total_request_deadline(enforceable_total_request_deadline); @@ -5817,8 +5984,8 @@ mod tests { ); http.push_response(200, b"{}".to_vec()); http.push_select_delay(Duration::ZERO); - http.push_wait_delay(Duration::from_millis(50)); - let backend = Arc::new(NamingBackend::new(BackendNamingPolicy::Axum)); + http.push_wait_delay(Duration::from_millis(200)); + let backend = Arc::new(NamingBackend::new(policy)); let services = build_services_with_backend_and_http_client( Arc::clone(&backend) as Arc<_>, Arc::clone(&http) as Arc<_>, @@ -5828,16 +5995,19 @@ mod tests { false, )) .expect("should compile planned auction"); - let orchestrator = - AuctionOrchestratorHarness::new(plan, Some(Arc::new(PendingDeadlineMediator))); + let orchestrator = AuctionOrchestratorHarness::new( + plan, + Some(Arc::new(PendingDeadlineMediator(mediator_timeout_ms))), + ); let request = planned_request(); let settings = create_test_settings(); let inbound = http::Request::new(edgezero_core::body::Body::empty()); let context = AuctionContext { settings: &settings, request: &inbound, - timeout_ms: 20, - transport_timeout_ms: 20, + timeout_ms: auction_timeout_ms, + transport_timeout_ms: policy + .canonicalize_transport_timeout_ms(auction_timeout_ms, 1_000), provider_responses: None, services: &services, }; @@ -5849,22 +6019,49 @@ mod tests { } #[tokio::test] - async fn planned_pending_mediator_applies_explicit_hard_deadline_policy() { - let current = planned_pending_mediator_deadline_result(false).await; - let current_mediator = current - .mediator_response - .as_ref() - .expect("current adapters should accept completed late mediator responses"); - assert!(current_mediator.response_time_ms >= 50); - assert_eq!(current.winning_bids["slot-1"].bidder, "mediated"); + async fn planned_pending_mediator_observes_absolute_deadline() { + for advertised_deadline in [false, true] { + let result = planned_pending_mediator_deadline_result( + advertised_deadline, + 20, + 1_000, + BackendNamingPolicy::Axum, + ) + .await; + assert!( + result.mediator_response.is_none(), + "should stop waiting for mediator" + ); + assert_eq!( + result.winning_bids["fictional-slot"].bid_id.as_deref(), + Some("provider"), + "should preserve on-time provider bids" + ); + assert!( + result.total_time_ms < 150, + "should bound full mediator response wait" + ); + } + } - let hard = planned_pending_mediator_deadline_result(true).await; - assert!(hard.mediator_response.is_none()); + #[tokio::test] + async fn planned_pending_mediator_observes_own_budget_before_auction_deadline() { + let result = + planned_pending_mediator_deadline_result(false, 1_000, 20, BackendNamingPolicy::Fastly) + .await; + assert!( + result.mediator_response.is_none(), + "should stop at the mediator budget before the auction deadline" + ); assert_eq!( - hard.winning_bids["fictional-slot"].bid_id.as_deref(), - Some("provider") + result.winning_bids["fictional-slot"].bid_id.as_deref(), + Some("provider"), + "should preserve on-time provider bids" + ); + assert!( + result.total_time_ms < 150, + "should avoid waiting for the rounded transport timeout" ); - assert!(hard.total_time_ms >= 50); } #[tokio::test] diff --git a/crates/trusted-server-core/src/platform/error.rs b/crates/trusted-server-core/src/platform/error.rs index 699eb7d483..12d0c39c69 100644 --- a/crates/trusted-server-core/src/platform/error.rs +++ b/crates/trusted-server-core/src/platform/error.rs @@ -20,6 +20,9 @@ pub enum PlatformError { /// HTTP client request failed. #[display("http client error")] HttpClient, + /// The absolute HTTP collection deadline elapsed. + #[display("http collection deadline exceeded")] + DeadlineExceeded, /// Geo lookup failed. #[display("geo lookup error")] Geo, diff --git a/crates/trusted-server-core/src/platform/http.rs b/crates/trusted-server-core/src/platform/http.rs index b47abaebbb..7a52ceae3c 100644 --- a/crates/trusted-server-core/src/platform/http.rs +++ b/crates/trusted-server-core/src/platform/http.rs @@ -3,6 +3,7 @@ use std::fmt; use edgezero_core::http::{Method, Request as EdgeRequest, Response as EdgeResponse}; use error_stack::Report; +use web_time::Instant; use super::PlatformError; use super::image_optimizer::PlatformImageOptimizerOptions; @@ -348,7 +349,10 @@ pub trait PlatformHttpClient: Send + Sync { request: PlatformHttpRequest, ) -> Result>; - /// Start an upstream request without waiting for it to complete. + /// Prepares an upstream request without waiting for its response. + /// + /// Fastly and Axum start network I/O at dispatch. Worker runtimes may + /// defer execution until collection so a deadline can bound all I/O. /// /// # Errors /// @@ -359,15 +363,11 @@ pub trait PlatformHttpClient: Send + Sync { request: PlatformHttpRequest, ) -> Result>; - /// Whether [`send_async`](Self::send_async) defers execution so multiple - /// pending requests progress concurrently and [`select`](Self::select) - /// races them. + /// Whether multiple pending requests can progress concurrently. /// - /// Platforms where `send_async` executes each request eagerly before - /// returning (e.g. Cloudflare Workers) return `false`. On such platforms - /// multi-request fan-out runs sequentially and accrues the sum of the - /// individual latencies, so callers with a latency budget (the auction - /// orchestrator) must check this before launching more than one request. + /// Fastly and Axum provide concurrent fan-out. Cloudflare and Spin retain + /// a single-provider contract and return `false`; the auction orchestrator + /// checks this capability before launching more than one request. fn supports_concurrent_fanout(&self) -> bool { true } @@ -376,8 +376,8 @@ pub trait PlatformHttpClient: Send + Sync { /// /// First-byte and between-byte timers do not qualify: connection setup or a /// byte-trickling response can still overrun the logical auction budget. The - /// auction collector uses this explicit capability to decide whether an - /// already-completed late response remains eligible. + /// capability describes unbounded sends, independently of the stricter + /// [`Self::select_until`] contract used for auction collection. fn has_enforceable_total_request_deadline(&self) -> bool { false } @@ -415,6 +415,80 @@ pub trait PlatformHttpClient: Send + Sync { pending_requests: Vec, ) -> Result>; + /// Collects one complete buffered response before an absolute deadline. + /// + /// Returns `Ok(None)` on expiry, relinquishing every supplied pending + /// request. Implementations must bound both header waits and body reads, + /// reject completions observed after the cutoff, and cancel outstanding + /// work where the runtime supports cancellation. Fastly can abandon pending + /// backend requests but cannot cancel them with the installed SDK. + /// Stream-marked requests require the separate streaming wait contract. + /// + /// # Performance + /// + /// Complete response bodies are buffered within adapter-specific limits. + /// Deadline expiry drops retained partial responses and pending handles. + /// + /// # Examples + /// + /// ```no_run + /// # use trusted_server_core::platform::{PlatformHttpClient, PlatformPendingRequest, PlatformSelectResult, PlatformError}; + /// # use error_stack::Report; + /// # async fn collect(client: &dyn PlatformHttpClient, pending: Vec) -> Result, Report> { + /// let deadline = web_time::Instant::now() + std::time::Duration::from_millis(100); + /// client.select_until(pending, deadline).await + /// # } + /// ``` + /// + /// # Errors + /// + /// Returns [`PlatformError::Unsupported`] unless the adapter implements + /// bounded collection, or an HTTP error when polling itself fails. + async fn select_until( + &self, + pending_requests: Vec, + deadline: Instant, + ) -> Result, Report> { + drop(pending_requests); + if Instant::now() >= deadline { + Ok(None) + } else { + Err(Report::new(PlatformError::Unsupported) + .attach("absolute-deadline HTTP collection is unavailable")) + } + } + + /// Waits for a complete buffered response before an absolute deadline. + /// + /// Uses [`Self::select_until`] so expiry also relinquishes the pending + /// request and never falls back to an unbounded wait. + /// + /// # Examples + /// + /// ```no_run + /// # use trusted_server_core::platform::{PlatformHttpClient, PlatformPendingRequest, PlatformResponse, PlatformError}; + /// # use error_stack::Report; + /// # async fn collect(client: &dyn PlatformHttpClient, pending: PlatformPendingRequest) -> Result> { + /// let deadline = web_time::Instant::now() + std::time::Duration::from_millis(100); + /// client.wait_until(pending, deadline).await + /// # } + /// ``` + /// + /// # Errors + /// + /// Returns [`PlatformError::DeadlineExceeded`] on expiry, an adapter polling + /// error, or the attributed upstream error in the selected result. + async fn wait_until( + &self, + pending: PlatformPendingRequest, + deadline: Instant, + ) -> Result> { + self.select_until(vec![pending], deadline) + .await? + .ok_or_else(|| Report::new(PlatformError::DeadlineExceeded))? + .ready + } + /// Wait for a single in-flight request to complete. /// /// This is a convenience wrapper around [`select`](Self::select) for the @@ -434,11 +508,69 @@ pub trait PlatformHttpClient: Send + Sync { #[cfg(test)] mod tests { + use std::sync::{ + Arc, + atomic::{AtomicUsize, Ordering}, + }; + use std::time::Duration; + use edgezero_core::body::Body; use edgezero_core::http::request_builder; use super::*; + struct DropProbe(Arc); + + impl Drop for DropProbe { + fn drop(&mut self) { + self.0.fetch_add(1, Ordering::SeqCst); + } + } + + #[test] + fn bounded_collection_default_relinquishes_expired_handles() { + let dropped = Arc::new(AtomicUsize::new(0)); + let response = futures::executor::block_on(UnavailableHttpClient.select_until( + vec![PlatformPendingRequest::new(DropProbe(Arc::clone(&dropped)))], + Instant::now(), + )) + .expect("should classify expiry without polling"); + assert!( + response.is_none(), + "should expire before accepting ready input" + ); + assert_eq!( + dropped.load(Ordering::SeqCst), + 1, + "should drop abandoned state" + ); + } + + #[test] + fn bounded_collection_default_fails_closed_instead_of_unbounded_polling() { + let error = futures::executor::block_on(UnavailableHttpClient.select_until( + vec![PlatformPendingRequest::new(())], + Instant::now() + Duration::from_secs(1), + )) + .expect_err("should reject unsupported bounded collection"); + assert!( + matches!(error.current_context(), PlatformError::Unsupported), + "should not delegate to unbounded select" + ); + } + + #[test] + fn bounded_wait_reports_deadline_exceeded() { + let error = futures::executor::block_on( + UnavailableHttpClient.wait_until(PlatformPendingRequest::new(()), Instant::now()), + ) + .expect_err("should report an expired wait"); + assert!( + matches!(error.current_context(), PlatformError::DeadlineExceeded), + "should distinguish expiry from transport failure" + ); + } + #[test] fn platform_http_request_cache_bypass_defaults_to_false() { let request = PlatformHttpRequest::new( diff --git a/crates/trusted-server-core/src/platform/test_support.rs b/crates/trusted-server-core/src/platform/test_support.rs index 17813c77dc..20413a0ef2 100644 --- a/crates/trusted-server-core/src/platform/test_support.rs +++ b/crates/trusted-server-core/src/platform/test_support.rs @@ -7,6 +7,7 @@ use base64::{Engine as _, engine::general_purpose}; use ed25519_dalek::SigningKey; use error_stack::{Report, ResultExt as _}; use rand::rngs::OsRng; +use web_time::Instant; use super::{ ClientInfo, GeoInfo, PlatformBackend, PlatformBackendSpec, PlatformCacheIntent, @@ -245,10 +246,10 @@ pub(crate) struct StubHttpClient { // Test-only wall-clock delays applied before each select result is returned. select_delays: Mutex>, // Reported by supports_concurrent_fanout(); set false to emulate - // platforms whose send_async executes eagerly (e.g. Cloudflare Workers). + // platforms whose collection contract permits only a single provider. concurrent_fanout: std::sync::atomic::AtomicBool, // Reported by has_enforceable_total_request_deadline(); set true to emulate - // a future adapter with a hard total request deadline. + // unbounded request transport with a hard total request deadline. enforceable_total_request_deadline: std::sync::atomic::AtomicBool, // Reported by supports_streaming_responses(); set true to emulate Fastly's // streaming response support. @@ -718,6 +719,62 @@ impl PlatformHttpClient for StubHttpClient { }) } + async fn select_until( + &self, + pending_requests: Vec, + deadline: Instant, + ) -> Result, Report> { + let remaining = deadline.saturating_duration_since(Instant::now()); + if remaining.is_zero() { + return Ok(None); + } + let delay = self + .select_delays + .lock() + .expect("should lock select delays") + .front() + .copied() + .unwrap_or_default(); + if delay >= remaining { + self.select_delays + .lock() + .expect("should lock select delays") + .pop_front(); + std::thread::sleep(remaining); + return Ok(None); + } + let response = self.select(pending_requests).await?; + Ok((Instant::now() < deadline).then_some(response)) + } + + async fn wait_until( + &self, + pending: PlatformPendingRequest, + deadline: Instant, + ) -> Result> { + let remaining = deadline.saturating_duration_since(Instant::now()); + let delay = self + .wait_delays + .lock() + .expect("should lock wait delays") + .front() + .copied() + .unwrap_or_default(); + if remaining.is_zero() || delay >= remaining { + self.wait_delays + .lock() + .expect("should lock wait delays") + .pop_front(); + std::thread::sleep(remaining); + return Err(Report::new(PlatformError::DeadlineExceeded)); + } + let response = self.wait(pending).await?; + if Instant::now() >= deadline { + return Err(Report::new(PlatformError::DeadlineExceeded)); + } + Ok(response) + } + /// Always marks the first pending request in the input as ready (FIFO order). /// /// This differs from Fastly's production `select()`, which returns whichever diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index 4dfe4e5a25..4279b5dad1 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -21146,6 +21146,21 @@ mod tests { self.inner.send_async(request).await } + async fn select_until( + &self, + pending: Vec, + deadline: web_time::Instant, + ) -> Result< + Option, + Report, + > { + if web_time::Instant::now() >= deadline { + return Ok(None); + } + let result = self.select(pending).await?; + Ok((web_time::Instant::now() < deadline).then_some(result)) + } + async fn select( &self, pending: Vec, diff --git a/crates/trusted-server-core/src/test_support.rs b/crates/trusted-server-core/src/test_support.rs index 37b8a9e657..ef7391ce20 100644 --- a/crates/trusted-server-core/src/test_support.rs +++ b/crates/trusted-server-core/src/test_support.rs @@ -294,6 +294,18 @@ pub mod nextjs_auction { Ok(PlatformPendingRequest::new(request).with_backend_name(backend)) } + async fn select_until( + &self, + pending_requests: Vec, + deadline: web_time::Instant, + ) -> Result, Report> { + if web_time::Instant::now() >= deadline { + return Ok(None); + } + let response = self.select(pending_requests).await?; + Ok((web_time::Instant::now() < deadline).then_some(response)) + } + async fn select( &self, mut pending_requests: Vec, diff --git a/docs/guide/architecture.md b/docs/guide/architecture.md index a6145f80d3..a2ac81a0e6 100644 --- a/docs/guide/architecture.md +++ b/docs/guide/architecture.md @@ -170,9 +170,12 @@ The workspace has multiple WASM runtimes with runtime-specific SDKs. Use target- Fastly and Axum support concurrent auction provider fan-out. Cloudflare and Spin currently accept at most one provider in an enabled auction. Every adapter runs -target-aware fan-out and backend-name checks at startup. No current adapter -claims an abortable provider-wide total-request deadline, so configured auction -and provider timeouts are logical budgets rather than hard wall-clock ceilings. +target-aware fan-out and backend-name checks at startup. Auction collection uses +an absolute cutoff across header waits and complete response bodies. Fastly +abandons pending backend requests because its SDK cannot cancel them; other +adapters cancel their owned operations at cutoff. See +[Auction timeout behavior](/guide/auction-orchestration#timeout-behavior) for +scheduling, worker-runtime dispatch, and local processing limitations. ## Next Steps diff --git a/docs/guide/auction-orchestration.md b/docs/guide/auction-orchestration.md index edbc384719..b80d27eae9 100644 --- a/docs/guide/auction-orchestration.md +++ b/docs/guide/auction-orchestration.md @@ -814,15 +814,28 @@ Until that callback lands, startup remains the mandatory target-aware gate. ### Timeout behavior -For each provider, Trusted Server uses the smaller of its resolved timeout and -the remaining auction budget for launch decisions and OpenRTB `tmax`. The -mediator is not launched after the logical auction budget is exhausted. - -No current adapter claims an abortable provider-wide total-request deadline. -Already-launched work may complete after the logical budget, and a completed -late response can remain eligible. Local decision and delivery also finish -after network launch closes, so `timeout_ms` is not a hard wall-clock ceiling -and an auction can exceed it. +Provider response collection uses one absolute deadline measured from auction +dispatch. It includes waiting for headers and buffering the complete upstream +body. When the deadline expires, outstanding providers are recorded as timeouts; +completed bids collected before the cutoff remain available for local ranking. +Collection after a slow origin response does not restart the clock, and late +responses are not admitted. A mediator starts only with remaining budget and +uses the earlier of the auction cutoff and its own logical timeout. + +Fastly limits retained response payloads to 10 MiB across the collector. A provider +whose next chunk would exceed that limit fails while its siblings remain eligible. +Fastly abandons unfinished backend requests and closes acquired response bodies; +the installed SDK has no pending-request cancellation API. Axum aborts outstanding +request tasks. Cloudflare aborts its fetch through an AbortController, and Spin +drops cancelable WASI operations. Cloudflare and Spin retain their single-provider +restriction and defer outbound request execution until collection so the entire +request can be bounded. + +Backend first-byte and between-byte timeouts remain transport safeguards rather +than total deadlines. Timer granularity and runtime scheduling can add a small +overshoot. Parsing, ranking, HTML processing, and delivery are local work outside +network collection, so this cutoff is not a hard ceiling on the whole publisher +response time. Browser Prebid `timeout_ms` and `debug` stay under `[integrations.prebid]` and are independent of all server provider values. Server endpoint, timeout, diff --git a/docs/guide/auction-testing.md b/docs/guide/auction-testing.md index b92b78b0fc..f15a26789d 100644 --- a/docs/guide/auction-testing.md +++ b/docs/guide/auction-testing.md @@ -68,3 +68,25 @@ are part of the contract rather than skipped coverage. See [Auction orchestration](/guide/auction-orchestration) for configuration and response details. + +## Absolute deadline regressions + +Core tests cover split dispatch/collection and synchronous execution, pending +provider attribution, preserved on-time bids, late collection after an origin +wait, and exhausted-budget mediation. Axum loopback tests separately delay +headers and trickle bodies, and verify outstanding task cancellation. Worker +adapter tests use controlled futures and timers, with WASM compilation checking +the actual timer and cancellation APIs. Fastly also has Viceroy network fixtures +for stalled headers, trickling bodies, bodyless responses, and competing bidders; +run its local origin harness after building the adapter test binary: + +```bash +cargo test -p trusted-server-adapter-fastly --target wasm32-wasip1 --no-run +python3 crates/trusted-server-adapter-fastly/tests/deadline_origin.py \ + "$(command -v viceroy)" \ + target/wasm32-wasip1/debug/deps/trusted_server_adapter_fastly-.wasm +``` + +Use the adapter test artifact printed by Cargo in place of ``. The harness +binds a local ephemeral port and runs the four network tests excluded from the +standard suite. diff --git a/docs/guide/configuration.md b/docs/guide/configuration.md index 4355b9e7ea..343fbf1225 100644 --- a/docs/guide/configuration.md +++ b/docs/guide/configuration.md @@ -2239,13 +2239,14 @@ push as proof that a Cloudflare or Spin multi-provider plan can start. ### Deadline behavior -Configured timeouts are logical budgets, not hard wall-clock guarantees. No -current adapter claims an abortable provider-wide total-request deadline. -Already-launched work may complete after the logical budget and a completed late -response can remain eligible. Once the logical auction budget is exhausted, -Trusted Server starts no additional provider or mediator network work, then -finishes local decision and delivery. An auction can therefore exceed its -configured wall-clock timeout. +Auction response collection stops at an absolute deadline measured from dispatch, +including header waits and complete response bodies. Outstanding providers become +timeouts, previously collected bids remain eligible, and mediation cannot start +with an exhausted budget. Backend timeout rounding does not extend that cutoff. +Fastly abandons pending requests; Axum, Cloudflare, and Spin cancel outstanding +operations where supported. Local processing and delivery can finish after the +network cutoff. See [Timeout behavior](/guide/auction-orchestration#timeout-behavior) +for runtime semantics and limitations. Creative sanitization is opt-in. `sanitize_creatives = true` strips executable markup before delivery. `rewrite_creatives = false` skips first-party URL diff --git a/docs/superpowers/plans/2026-10-22-absolute-auction-deadlines.md b/docs/superpowers/plans/2026-10-22-absolute-auction-deadlines.md new file mode 100644 index 0000000000..eb8505e661 --- /dev/null +++ b/docs/superpowers/plans/2026-10-22-absolute-auction-deadlines.md @@ -0,0 +1,74 @@ +# Absolute auction deadline implementation plan + +> **For agentic workers:** Use superpowers:test-driven-development and verification-before-completion. Implement independent adapter scopes in parallel; root owns the collection contract and orchestrator. Independently review the draft PR after creation. + +**Goal:** Resolve #1015 by ending provider collection at the deadline measured from dispatch, recording outstanding providers as timeouts and avoiding mediation after budget exhaustion. + +**Architecture:** Add `PlatformHttpClient::select_until(pending, web_time::Instant)` returning `Result, Report>`. `None` means deadline expiration and relinquishes all pending state. The default fails closed as unsupported; it must never silently delegate to unbounded selection. Add `wait_until` for buffered mediator requests. Preserve existing unbounded APIs for unrelated publisher/proxy traffic. + +**Tech stack:** Rust, web_time, platform runtime timers, Fastly nonblocking pending polls and readiness-gated body reads. + +## Design decisions + +- Relative backend timeouts alone cannot enforce a total budget against trickling bodies. An outer generic async timer cannot interrupt Fastly's synchronous hostcalls. Use adapter-specific bounded collection of complete bodies. +- Axum races its existing whole-body task selection against a tokio timer; timeout drops its abort-on-drop task handles. +- Cloudflare and Spin retain their single-provider capability restrictions, but return owned pending response futures instead of eagerly awaiting complete responses at dispatch. Bounded selection races the entire request/body future against worker Delay or Spin SDK 7's WASI p3 sleep. Cloudflare aborts its fetch through an AbortController on drop; Spin drops cancelable component subtasks and body readers. +- Fastly maintains pending-header and buffered-body states together. Poll headers without blocking; read at most one ready body chunk per provider per sweep. Never use a network-backed read_to_end. Bound readiness waits by the original deadline; preserve remaining collector states between completions. The SDK does not expose high-level pending wait handles, so brief bounded polling may be needed for header readiness. Use the locked low-level ABI only for readiness-gated body I/O, with safety comments and a justified direct dependency. +- Fastly cannot cancel a still-pending backend operation with the locked public API. On expiration, stop waiting and drop guest state; close acquired bodies. Do not claim network cancellation that the SDK cannot provide. +- Apply bounded collection to split dispatch/collect, synchronous planned auctions, legacy compatibility tests, and mediator waits. Recheck the absolute clock before accepting a completion. Retain successful responses collected before the cutoff and attribute every outstanding launch exactly once. +- No expansion of multi-provider support for worker runtimes, no unrelated proxy timeout changes, and no new operator settings. + +## Tasks + +- [x] Run the existing deadline test baseline in the isolated worktree. +- [x] Add failing regressions for expired collection, delayed/trickling responses, timeout attribution, partial success, and exhausted-budget mediation. +- [x] Implement and test the platform bounded-collection contract and safe unsupported default. +- [x] Implement each adapter's complete-response deadline and cancellation/abandonment behavior; run meaningful adapter tests. +- [x] Integrate all auction collection and mediator paths, replacing the former capability-dependent acceptance of late responses. +- [x] Document semantics and platform limitations, including deferred worker-runtime requests and Fastly abandonment. +- [x] Run every AGENTS.md CI gate, native core tests, and the optimized Fastly build; record exact outcomes and environmental limitations. +- [ ] Create a draft PR with .claude/agents/pr-creator.md and Closes #1015. +- [ ] Independently review the PR, address validated findings, rerun affected checks, and push follow-up fixes. + +## Plan review + +Independent review approved the architecture with two requirements: mediator cutoffs use the earlier of the original auction deadline and the mediator's own logical budget; deadline expiry and transport/polling failure retain distinct attribution. Adapter regressions must cover expired-but-ready inputs, full-body timeouts, fairness, cancellation/abandonment, body limits, and bodyless HTTP semantics. + +The original deadline baseline passed. A new regression failed against the original collector because current adapters accepted delayed provider responses after the auction deadline. + +## Implementation review + +A separate requirements review found no required production corrections. It +suggested a tighter mediator-budget regression, which now verifies the mediator +cutoff against a longer auction budget and rounded Fastly transport timeout. + +Review also found two Fastly collector edges before PR creation: a partially +collected response could lose its original HEAD method hint, and concurrent +partial bodies could multiply the old single-response memory allowance. Both +have red/green regressions. Collector state now retains method metadata and +shares a 10 MiB payload budget; exceeding providers fail with attribution while +siblings remain collectible. + +## Verification before PR + +All AGENTS.md CI gates passed: formatting, eight target-specific clippy aliases, +five adapter test aliases (including reusable Fastly sandboxes), 17 parity +tests, native build-digest tests/lint, JS tests/build/format, documentation and +outside-doc Markdown formatting, WASM clock lint, and documentation proposal +script checks. The native core suite passed 2,986 unit tests; Viceroy passed +2,980 core tests and 229 Fastly adapter tests. Four explicitly excluded Fastly +network tests passed separately against the local delayed/trickling origin. +The optimized Fastly WASM build passed. + +A stricter mediator test initially supplied an inconsistent transport timeout +in its fixture; after matching the backend's canonical timeout, both native +and Viceroy full core suites passed. Axum's new fixture also now checks its +request read count to satisfy clippy. + +Verification used Rust 1.98.1 and Viceroy 0.21.0. Loopback fixtures required +local socket access. Cloudflare and Spin timer/cancellation paths were checked +for their WASM targets; deterministic native tests exercise their actual +arbitration helpers, rather than a deployed worker runtime. + +The GitHub token cannot read project-board fields (`read:project` is absent), +so the issue's board status remains unchanged. From 62ad0d95c550d911618a42277fe725bd3a8f3f03 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Thu, 22 Oct 2026 13:41:53 +0530 Subject: [PATCH 2/9] Record deadline verification and independent PR review --- .../2026-10-22-absolute-auction-deadlines.md | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/docs/superpowers/plans/2026-10-22-absolute-auction-deadlines.md b/docs/superpowers/plans/2026-10-22-absolute-auction-deadlines.md index eb8505e661..7c9c45bcaf 100644 --- a/docs/superpowers/plans/2026-10-22-absolute-auction-deadlines.md +++ b/docs/superpowers/plans/2026-10-22-absolute-auction-deadlines.md @@ -27,8 +27,8 @@ - [x] Integrate all auction collection and mediator paths, replacing the former capability-dependent acceptance of late responses. - [x] Document semantics and platform limitations, including deferred worker-runtime requests and Fastly abandonment. - [x] Run every AGENTS.md CI gate, native core tests, and the optimized Fastly build; record exact outcomes and environmental limitations. -- [ ] Create a draft PR with .claude/agents/pr-creator.md and Closes #1015. -- [ ] Independently review the PR, address validated findings, rerun affected checks, and push follow-up fixes. +- [x] Create a draft PR with .claude/agents/pr-creator.md and Closes #1015. +- [x] Independently review the PR, address validated findings, rerun affected checks, and push follow-up fixes. ## Plan review @@ -72,3 +72,16 @@ arbitration helpers, rather than a deployed worker runtime. The GitHub token cannot read project-board fields (`read:project` is absent), so the issue's board status remains unchanged. + +## Draft PR and independent review + +Draft PR #1433 targets main, is assigned to the author, and closes #1015. A fresh +independent reviewer inspected the complete `f9281917..d4f7018` implementation, +including locked SDK behavior, all four adapters, core collection and mediation, +Fastly buffering/readiness, the network harness, and documentation. It found no +correctness or regression issues requiring changes and independently ran all ten +Cloudflare/Spin deadline tests successfully. + +The reviewer confirmed the documented coverage gap: actual deployed worker timer +and cancellation behavior has not received runtime integration coverage. The +review does not establish a production defect. No review findings remain open. From 23b5b4cd024f676e67f09b410dae38ca2beb206f Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Thu, 22 Oct 2026 13:51:04 +0530 Subject: [PATCH 3/9] Clarify worker collection and expected deadline disconnects --- crates/trusted-server-adapter-cloudflare/src/platform.rs | 5 ++--- .../trusted-server-adapter-fastly/tests/deadline_origin.py | 2 ++ 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/crates/trusted-server-adapter-cloudflare/src/platform.rs b/crates/trusted-server-adapter-cloudflare/src/platform.rs index 3a0dc7b039..3db60792ca 100644 --- a/crates/trusted-server-adapter-cloudflare/src/platform.rs +++ b/crates/trusted-server-adapter-cloudflare/src/platform.rs @@ -755,9 +755,8 @@ fn extract_client_ip(ctx: &edgezero_core::context::RequestContext) -> Option Date: Thu, 22 Oct 2026 19:18:13 +0530 Subject: [PATCH 4/9] Preserve dispatched bids and verify runtime deadline behavior --- .github/workflows/test.yml | 9 +- .../workflows/worker-auction-deadlines.yml | 73 ++++ CHANGELOG.md | 6 + .../src/platform.rs | 221 ++++++++--- .../Cargo.toml | 6 + .../src/platform.rs | 193 +++++++-- .../tests/runtime/deadline.rs | 70 ++++ .../src/platform.rs | 46 ++- .../src/platform/deadline.rs | 371 +++++++++++++----- .../tests/deadline_origin.py | 26 +- crates/trusted-server-adapter-spin/Cargo.toml | 6 + .../src/platform.rs | 188 +++++++-- .../tests/runtime/deadline.rs | 65 +++ .../src/auction/orchestrator.rs | 152 ++++--- .../src/auction/provider.rs | 4 +- .../src/integrations/adserver_mock.rs | 2 +- .../src/integrations/aps.rs | 2 +- .../src/integrations/prebid.rs | 2 +- .../trusted-server-core/src/platform/http.rs | 25 +- .../src/platform/test_support.rs | 9 +- crates/trusted-server-core/src/publisher.rs | 4 +- docs/guide/ad-serving.md | 7 +- docs/guide/architecture.md | 3 +- docs/guide/auction-orchestration.md | 40 +- docs/guide/auction-testing.md | 9 +- docs/guide/configuration.md | 27 +- .../2026-10-22-absolute-auction-deadlines.md | 141 +++---- scripts/test-worker-auction-deadlines.py | 126 ++++++ 28 files changed, 1431 insertions(+), 402 deletions(-) create mode 100644 .github/workflows/worker-auction-deadlines.yml create mode 100644 crates/trusted-server-adapter-cloudflare/tests/runtime/deadline.rs create mode 100644 crates/trusted-server-adapter-spin/tests/runtime/deadline.rs create mode 100644 scripts/test-worker-auction-deadlines.py diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b27173dd66..c885b66d4e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -20,12 +20,12 @@ jobs: id: viceroy-version # `.tool-versions` is the single source of truth so this workflow and # `.github/actions/setup-integration-test-env/action.yml` can't drift. - run: echo "viceroy-version=$(grep '^viceroy ' .tool-versions | awk '{print $2}')" >> $GITHUB_OUTPUT + run: echo "viceroy-version=$(grep '^viceroy ' .tool-versions | awk '{print $2}')" >> "$GITHUB_OUTPUT" shell: bash - name: Retrieve Node.js version id: node-version - run: echo "node-version=$(grep '^nodejs ' .tool-versions | awk '{print $2}')" >> $GITHUB_OUTPUT + run: echo "node-version=$(grep '^nodejs ' .tool-versions | awk '{print $2}')" >> "$GITHUB_OUTPUT" shell: bash - name: Set up Rust toolchain @@ -53,6 +53,9 @@ jobs: - name: Run tests run: cargo test-fastly + - name: Run Fastly absolute deadline slow-network tests + run: python3 crates/trusted-server-adapter-fastly/tests/deadline_origin.py "$(command -v viceroy)" + - name: Run tests (reusable-sandbox feature) run: cargo test-fastly-reuse @@ -268,7 +271,7 @@ jobs: - name: Retrieve Node.js version id: node-version working-directory: . - run: echo "node-version=$(grep '^nodejs ' .tool-versions | awk '{print $2}')" >> $GITHUB_OUTPUT + run: echo "node-version=$(grep '^nodejs ' .tool-versions | awk '{print $2}')" >> "$GITHUB_OUTPUT" shell: bash - name: Use Node.js diff --git a/.github/workflows/worker-auction-deadlines.yml b/.github/workflows/worker-auction-deadlines.yml new file mode 100644 index 0000000000..5af9f7d220 --- /dev/null +++ b/.github/workflows/worker-auction-deadlines.yml @@ -0,0 +1,73 @@ +name: Worker auction deadline runtime regressions + +on: + pull_request: + push: + branches: [main] + +permissions: + contents: read + +jobs: + worker-deadlines: + runs-on: ubuntu-latest + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + adapter: [cloudflare, spin] + steps: + - uses: actions/checkout@v4 + - name: Read tool versions + id: versions + shell: bash + run: | + { + echo "node=$(awk '$1 == "nodejs" { print $2 }' .tool-versions)" + echo "spin=$(awk '$1 == "spin" { print $2 }' .tool-versions)" + echo "bindgen=$(sed -n '/^name = "wasm-bindgen"$/ { n; s/version = "\(.*\)"/\1/p; }' Cargo.lock)" + } >> "$GITHUB_OUTPUT" + - uses: actions/setup-node@v4 + with: + node-version: ${{ steps.versions.outputs.node }} + - name: Set up Rust + uses: ./.github/actions/setup-rust + with: + target: ${{ matrix.adapter == 'cloudflare' && 'wasm32-unknown-unknown' || 'wasm32-wasip1' }} + cache-key: worker-deadline-${{ matrix.adapter }} + - name: Install JS dependencies + working-directory: crates/trusted-server-js/lib + run: npm ci + - name: Build actual adapter HTTP fixture + shell: bash + run: | + if [ "${{ matrix.adapter }}" = cloudflare ]; then + cargo build --locked -p trusted-server-adapter-cloudflare --example deadline-runtime --target wasm32-unknown-unknown --features cloudflare + else + cargo build --locked -p trusted-server-adapter-spin --example deadline-runtime --target wasm32-wasip1 --features spin + fi + - name: Install local Cloudflare tools + if: matrix.adapter == 'cloudflare' + shell: bash + run: | + npm install --global wrangler@4.83.0 + mkdir -p "$RUNNER_TEMP/bindgen" + curl --fail --location --silent --show-error "https://github.com/wasm-bindgen/wasm-bindgen/releases/download/${{ steps.versions.outputs.bindgen }}/wasm-bindgen-${{ steps.versions.outputs.bindgen }}-x86_64-unknown-linux-musl.tar.gz" | tar -xz --strip-components=1 -C "$RUNNER_TEMP/bindgen" + echo "$RUNNER_TEMP/bindgen" >> "$GITHUB_PATH" + - name: Install pinned Spin runtime + if: matrix.adapter == 'spin' + shell: bash + run: | + mkdir -p "$RUNNER_TEMP/spin" + curl --fail --location --silent --show-error "https://github.com/spinframework/spin/releases/download/v${{ steps.versions.outputs.spin }}/spin-v${{ steps.versions.outputs.spin }}-linux-amd64.tar.gz" | tar -xz -C "$RUNNER_TEMP/spin" + echo "$RUNNER_TEMP/spin" >> "$GITHUB_PATH" + - name: Exercise delayed headers, trickling bodies, and slow publisher work + shell: bash + env: + WRANGLER_SEND_METRICS: "false" + run: | + if [ "${{ matrix.adapter }}" = cloudflare ]; then + python3 scripts/test-worker-auction-deadlines.py cloudflare --runtime "$(command -v wrangler)" + else + python3 scripts/test-worker-auction-deadlines.py spin --runtime "$(command -v spin)" + fi diff --git a/CHANGELOG.md b/CHANGELOG.md index f691c6a005..671aec1cf1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed +- Auction response collection now bounds network waiting by `[auction].timeout_ms` + measured from dispatch, including delayed headers and trickling bodies on all + adapters. Complete available responses survive a slow publisher origin; + unfinished providers time out and mediation is skipped after budget exhaustion. + Fastly abandons unfinished requests; Axum, Cloudflare, and Spin cancel owned work. + - **Breaking:** `x-ts-version` now reports the deployed git version — the release tag, else the branch, else the first 6 characters of the commit — compiled in from the build-time `TRUSTED_SERVER__GIT_VERSION` (set by the deploy pipeline) or local git, and is sent by every adapter, including on the Fastly `GET /health` probe. The Fastly service version it previously carried is now `x-ts-fastly-version`; update dashboards, monitors, and scripts that read `x-ts-version` as the Fastly version number. Builds with neither the override nor git omit the header. - **Breaking:** `ts prebid bundle` is now `ts prebid client`, alongside the new `ts prebid server` namespace. Update scripts and runbooks to use `ts prebid client` with the same arguments. The old `bundle` spelling is no longer accepted and has no compatibility alias. - The S2S `/_ts/api/v1/batch-sync` endpoint now validates the full batch and calls the CAS-protected update path once per distinct normalized EC ID. The last valid UID wins within a group, and infrastructure failures reject the failing and each unprocessed group, so accepted and `kv_unavailable` input indexes may interleave. diff --git a/crates/trusted-server-adapter-axum/src/platform.rs b/crates/trusted-server-adapter-axum/src/platform.rs index fada8a1350..7529044693 100644 --- a/crates/trusted-server-adapter-axum/src/platform.rs +++ b/crates/trusted-server-adapter-axum/src/platform.rs @@ -451,17 +451,36 @@ impl PlatformHttpClient for AxumPlatformHttpClient { pending_requests: Vec, deadline: web_time::Instant, ) -> Result, Report> { - let now = web_time::Instant::now(); - if now >= deadline { - return Ok(None); + let mut handles = recover_pending_handles(pending_requests)?; + if handles.is_empty() { + return Err(Report::new(PlatformError::HttpClient) + .attach("select_until called with no pending requests")); } - - // Selection owns abort-on-drop task handles and buffers complete bodies. - // Dropping its future on expiration also stops outstanding body reads. - match tokio::time::timeout(deadline - now, self.select(pending_requests)).await { - Ok(result) if web_time::Instant::now() < deadline => result.map(Some), - Ok(_) | Err(_) => Ok(None), + let remaining = deadline.saturating_duration_since(web_time::Instant::now()); + if !remaining.is_zero() { + // Borrow the handles so timer expiry cannot drop completed siblings + // before the final nonblocking snapshot inspects them. + let selected = + tokio::time::timeout(remaining, futures::future::select_all(handles.iter_mut())) + .await; + if let Ok(((backend_name, result), index, _)) = selected { + drop(handles.swap_remove(index)); + return Ok(Some(completed_selection(backend_name, result, handles))); + } + } + // Snapshot once: only completed tasks survive into subsequent calls. + // An unfinished task is aborted on drop and cannot become a late bid. + handles.retain(|pending| pending.handle.is_finished()); + if handles.is_empty() { + return Ok(None); } + let ((backend_name, result), _, remaining_handles) = + futures::future::select_all(handles).await; + Ok(Some(completed_selection( + backend_name, + result, + remaining_handles, + ))) } async fn select( @@ -473,15 +492,7 @@ impl PlatformHttpClient for AxumPlatformHttpClient { .attach("select called with an empty pending_requests list")); } - let handles: Vec = pending_requests - .into_iter() - .map(|pr| { - pr.downcast::().map_err(|_| { - Report::new(PlatformError::HttpClient) - .attach("unexpected inner type in AxumPlatformHttpClient::select") - }) - }) - .collect::, _>>()?; + let handles = recover_pending_handles(pending_requests)?; // Each AxumPendingHandle resolves to (backend_name, result), so the // remaining handles keep their own backend names — no positional @@ -490,48 +501,68 @@ impl PlatformHttpClient for AxumPlatformHttpClient { let ((backend_name, result), _ready_idx, remaining_handles) = futures::future::select_all(handles).await; - let remaining: Vec = remaining_handles - .into_iter() - .map(|handle| { - let backend_name = handle.backend_name.clone(); - PlatformPendingRequest::new(handle).with_backend_name(backend_name) - }) - .collect(); + Ok(completed_selection(backend_name, result, remaining_handles)) + } +} - // Map join panics and per-request errors into ready: Err(...) so that the - // auction orchestrator can log the failure and continue with remaining providers - // rather than treating one bad provider as a fatal select() failure. - let ready = result - .map_err(|e| { +fn completed_selection( + backend_name: String, + result: Result, + remaining_handles: Vec, +) -> PlatformSelectResult { + let remaining: Vec = remaining_handles + .into_iter() + .map(|handle| { + let backend_name = handle.backend_name.clone(); + PlatformPendingRequest::new(handle).with_backend_name(backend_name) + }) + .collect(); + + // Map join panics and per-request errors into ready: Err(...) so that the + // auction orchestrator can log the failure and continue with remaining providers + // rather than treating one bad provider as a fatal select() failure. + let ready = result + .map_err(|e| { + Report::new(PlatformError::HttpClient) + .attach(format!("auction request task panicked: {e}")) + }) + .and_then(|inner| inner) + .and_then(|(status, headers, body)| { + let mut builder = edgezero_core::http::response_builder().status(status); + for (name, value) in &headers { + builder = builder.header(name.as_str(), value.as_slice()); + } + builder + .body(edgezero_core::body::Body::from(body)) + .change_context(PlatformError::HttpClient) + }) + .map(|edge_resp| PlatformResponse::new(edge_resp).with_backend_name(backend_name.clone())); + + // Attribute the failure to its backend so the orchestrator can remove + // the provider and record a BidStatus::Error, matching the Fastly + // adapter. Without this, a failed provider silently vanishes through + // the orchestrator's "backend not identified" branch. + let failed_backend_name = ready.as_ref().err().map(|_| backend_name); + + PlatformSelectResult { + ready, + remaining, + failed_backend_name, + } +} + +fn recover_pending_handles( + pending_requests: Vec, +) -> Result, Report> { + pending_requests + .into_iter() + .map(|pending| { + pending.downcast::().map_err(|_| { Report::new(PlatformError::HttpClient) - .attach(format!("auction request task panicked: {e}")) - }) - .and_then(|inner| inner) - .and_then(|(status, headers, body)| { - let mut builder = edgezero_core::http::response_builder().status(status); - for (name, value) in &headers { - builder = builder.header(name.as_str(), value.as_slice()); - } - builder - .body(edgezero_core::body::Body::from(body)) - .change_context(PlatformError::HttpClient) + .attach("unexpected pending type in Axum selection") }) - .map(|edge_resp| { - PlatformResponse::new(edge_resp).with_backend_name(backend_name.clone()) - }); - - // Attribute the failure to its backend so the orchestrator can remove - // the provider and record a BidStatus::Error, matching the Fastly - // adapter. Without this, a failed provider silently vanishes through - // the orchestrator's "backend not identified" branch. - let failed_backend_name = ready.as_ref().err().map(|_| backend_name); - - Ok(PlatformSelectResult { - ready, - remaining, - failed_backend_name, }) - } + .collect() } // --------------------------------------------------------------------------- @@ -976,7 +1007,7 @@ mod tests { } #[tokio::test] - async fn select_until_expired_deadline_rejects_ready_response() { + async fn select_until_expired_deadline_preserves_buffered_response() { let client = AxumPlatformHttpClient::new(); let handle = tokio::spawn(async { Ok((200, Vec::new(), b"ok".to_vec())) }); while !handle.is_finished() { @@ -991,8 +1022,80 @@ mod tests { .await .expect("should recognize expired deadline"); assert!( - result.is_none(), - "should reject an already ready response after cutoff" + result.is_some(), + "should preserve a complete buffered response without waiting after cutoff" + ); + } + + #[tokio::test] + async fn expired_snapshot_preserves_finished_siblings_and_aborts_unfinished_task() { + let mut tasks = Vec::new(); + for name in ["ready-a", "ready-b"] { + tasks.push(AxumPendingHandle { + backend_name: name.to_owned(), + handle: tokio::spawn(async { Ok((200, Vec::new(), b"ok".to_vec())) }), + }); + } + while tasks.iter().any(|pending| !pending.handle.is_finished()) { + tokio::task::yield_now().await; + } + let (started_sender, started_receiver) = tokio::sync::oneshot::channel(); + let (drop_sender, drop_receiver) = tokio::sync::oneshot::channel::<()>(); + tasks.push(AxumPendingHandle { + backend_name: "unfinished".to_owned(), + handle: tokio::spawn(async move { + let _drop_signal = drop_sender; + started_sender + .send(()) + .expect("should signal unfinished task start"); + futures::future::pending::().await + }), + }); + started_receiver + .await + .expect("should start unfinished task"); + let pending = tasks + .into_iter() + .map(|task| { + let name = task.backend_name.clone(); + PlatformPendingRequest::new(task).with_backend_name(name) + }) + .collect(); + let client = AxumPlatformHttpClient::new(); + let deadline = web_time::Instant::now(); + let first = client + .select_until(pending, deadline) + .await + .expect("should snapshot ready responses") + .expect("should retain first response"); + assert!( + first.ready.is_ok(), + "should preserve first completed result" + ); + assert_eq!( + first.remaining.len(), + 1, + "should retain only the other completed response" + ); + let second = client + .select_until(first.remaining, deadline) + .await + .expect("should drain the completed snapshot") + .expect("should retain second response"); + assert!( + second.ready.is_ok(), + "should preserve the second completed result" + ); + assert!( + second.remaining.is_empty(), + "should leave no unfinished handles" + ); + assert!( + tokio::time::timeout(Duration::from_secs(1), drop_receiver) + .await + .expect("should promptly abort unfinished work") + .is_err(), + "should release unfinished task resources" ); } diff --git a/crates/trusted-server-adapter-cloudflare/Cargo.toml b/crates/trusted-server-adapter-cloudflare/Cargo.toml index 03b342f030..b9bfe3fafa 100644 --- a/crates/trusted-server-adapter-cloudflare/Cargo.toml +++ b/crates/trusted-server-adapter-cloudflare/Cargo.toml @@ -45,4 +45,10 @@ trusted-server-core = { workspace = true, features = ["test-utils"] } base64 = { workspace = true } edgezero-core = { workspace = true } toml = { workspace = true } +[target.'cfg(not(target_arch = "wasm32"))'.dev-dependencies] tokio = { workspace = true, features = ["rt-multi-thread", "macros"] } + +[[example]] +name = "deadline-runtime" +path = "tests/runtime/deadline.rs" +crate-type = ["cdylib"] diff --git a/crates/trusted-server-adapter-cloudflare/src/platform.rs b/crates/trusted-server-adapter-cloudflare/src/platform.rs index 3db60792ca..a2c6f592bc 100644 --- a/crates/trusted-server-adapter-cloudflare/src/platform.rs +++ b/crates/trusted-server-adapter-cloudflare/src/platform.rs @@ -162,20 +162,45 @@ impl KvStore for KvHandleAdapter { // CloudflareHttpClient — WASM target only // --------------------------------------------------------------------------- -/// Owns the outbound operation until selection completes or drops it. -#[cfg(target_arch = "wasm32")] -struct CloudflarePendingResponse { - response: - futures::future::LocalBoxFuture<'static, Result>>, +/// Receives a dispatched operation and cancels it when its owner is dropped. +#[cfg(any(test, target_arch = "wasm32"))] +struct CloudflarePendingResponse { + response: futures::channel::oneshot::Receiver, + abort: futures::future::AbortHandle, +} + +#[cfg(any(test, target_arch = "wasm32"))] +impl CloudflarePendingResponse { + fn start( + operation: impl std::future::Future + 'static, + spawn: impl FnOnce(futures::future::LocalBoxFuture<'static, ()>), + ) -> Self { + let (sender, response) = futures::channel::oneshot::channel(); + let (abort, registration) = futures::future::AbortHandle::new_pair(); + spawn(Box::pin(async move { + if let Ok(result) = futures::future::Abortable::new(operation, registration).await { + let _ = sender.send(result); + } + })); + Self { response, abort } + } +} + +#[cfg(any(test, target_arch = "wasm32"))] +impl Drop for CloudflarePendingResponse { + fn drop(&mut self) { + self.abort.abort(); + } } /// [`worker::Fetch`]-backed HTTP client for the Cloudflare Workers runtime. /// /// # Multi-provider auction limitation /// -/// Selection supports one provider at a time, so +/// Auction configuration supports one provider at a time, so /// [`PlatformHttpClient::supports_concurrent_fanout`] reports `false`. -/// Deadline selection bounds fetch and body collection together and aborts +/// Requests start in a runtime task at dispatch, overlapping publisher-origin work. +/// Deadline selection bounds waiting for fetch and body collection and aborts /// the Workers fetch when its pending operation is dropped. #[cfg(target_arch = "wasm32")] pub struct CloudflareHttpClient; @@ -458,8 +483,8 @@ impl PlatformHttpClient for CloudflareHttpClient { } fn supports_concurrent_fanout(&self) -> bool { - // Keep the existing single-provider restriction: selection polls one - // outbound request at a time. + // Retain the existing auction capability restriction independently of + // runtime task scheduling. false } @@ -468,9 +493,10 @@ impl PlatformHttpClient for CloudflareHttpClient { request: PlatformHttpRequest, ) -> Result> { let backend_name = request.backend_name.clone(); - let pending = CloudflarePendingResponse { - response: Box::pin(Self::execute(request)), - }; + let pending = CloudflarePendingResponse::start( + Self::execute(request), + worker::wasm_bindgen_futures::spawn_local, + ); Ok(PlatformPendingRequest::new(pending).with_backend_name(backend_name)) } @@ -486,14 +512,18 @@ impl PlatformHttpClient for CloudflareHttpClient { reject_multi_provider_fanout(pending_requests.len())?; let ready_platform = pending_requests.remove(0); - let pending = ready_platform - .downcast::() + let mut pending = ready_platform + .downcast::>>>( + ) .map_err(|_| { Report::new(PlatformError::HttpClient) .attach("unexpected inner type in CloudflareHttpClient::select") })?; - let ready = pending.response.await; + let ready = (&mut pending.response).await.unwrap_or_else(|error| { + Err(Report::new(PlatformError::HttpClient) + .attach(format!("dispatched response task stopped: {error}"))) + }); Ok(PlatformSelectResult { ready, remaining: pending_requests, @@ -514,20 +544,34 @@ impl PlatformHttpClient for CloudflareHttpClient { return Err(Report::new(PlatformError::HttpClient) .attach("multi-provider fan-out is not supported on this adapter")); } - let remaining = deadline.saturating_duration_since(web_time::Instant::now()); - if remaining.is_zero() { - return Ok(None); - } let platform_pending = pending_requests.remove(0); let backend_name = platform_pending.backend_name().map(str::to_owned); - let pending = platform_pending - .downcast::() + let mut pending = platform_pending + .downcast::>>>( + ) .map_err(|_| { Report::new(PlatformError::HttpClient) .attach("unexpected pending request type in deadline selection") })?; + // Probe only the task's buffered result. This never polls transport I/O + // and preserves a completed bid when publisher-origin work ran long. + if let Some(ready) = pending.response.try_recv().map_err(|error| { + Report::new(PlatformError::HttpClient) + .attach(format!("dispatched response task stopped: {error}")) + })? { + return Ok(Some(PlatformSelectResult { + failed_backend_name: ready.as_ref().err().and(backend_name), + ready, + remaining: pending_requests, + })); + } let result = complete_before_deadline( - pending.response, + async { + (&mut pending.response).await.unwrap_or_else(|error| { + Err(Report::new(PlatformError::HttpClient) + .attach(format!("dispatched response task stopped: {error}"))) + }) + }, async { // Workers timers use whole signed milliseconds. Recheck the // absolute deadline after each bounded timer interval. @@ -1000,11 +1044,112 @@ mod deadline_tests { use std::rc::Rc; use std::time::Duration; - use futures::executor::block_on; + use futures::executor::{LocalPool, block_on}; use futures::future::{pending, ready}; + use futures::task::LocalSpawnExt as _; use web_time::Instant; - use super::complete_before_deadline; + use super::{CloudflarePendingResponse, complete_before_deadline}; + + #[test] + fn dispatched_task_starts_before_collection_and_buffers_complete_body() { + let mut executor = LocalPool::new(); + let spawner = executor.spawner(); + let started = Rc::new(Cell::new(false)); + let started_operation = Rc::clone(&started); + let (finish_body, body) = futures::channel::oneshot::channel(); + let mut pending = CloudflarePendingResponse::start( + async move { + started_operation.set(true); + body.await.expect("should receive complete response body") + }, + |task| { + spawner + .spawn_local(task) + .expect("should spawn outbound task") + }, + ); + + executor.run_until_stalled(); + assert!(started.get(), "should start outbound work before selection"); + assert_eq!( + pending.response.try_recv().expect("should poll result"), + None, + "should not expose headers before response body completes" + ); + finish_body.send(42).expect("should finish response body"); + executor.run_until_stalled(); + let expired = web_time::Instant::now(); + assert!( + web_time::Instant::now() >= expired, + "should collect after cutoff" + ); + assert_eq!( + pending + .response + .try_recv() + .expect("should poll buffered result"), + Some(42), + "should preserve a complete buffered response without polling transport" + ); + } + + #[test] + fn dropping_dispatched_pending_cancels_started_body_operation() { + struct DropProbe(Rc>); + impl Drop for DropProbe { + fn drop(&mut self) { + self.0.set(true); + } + } + let mut executor = LocalPool::new(); + let spawner = executor.spawner(); + let dropped = Rc::new(Cell::new(false)); + let probe = DropProbe(Rc::clone(&dropped)); + let pending = CloudflarePendingResponse::start( + async move { + let _probe = probe; + futures::future::pending::<()>().await; + }, + |task| { + spawner + .spawn_local(task) + .expect("should spawn outbound task") + }, + ); + executor.run_until_stalled(); + assert!(!dropped.get(), "should retain an active body operation"); + drop(pending); + executor.run_until_stalled(); + assert!( + dropped.get(), + "should drop spawned operation after cancellation" + ); + } + + #[test] + fn dropping_queued_dispatch_prevents_transport_poll() { + let mut executor = LocalPool::new(); + let spawner = executor.spawner(); + let started = Rc::new(Cell::new(false)); + let operation_started = Rc::clone(&started); + let pending = CloudflarePendingResponse::start( + async move { + operation_started.set(true); + }, + |task| { + spawner + .spawn_local(task) + .expect("should spawn outbound task") + }, + ); + drop(pending); + executor.run_until_stalled(); + assert!( + !started.get(), + "should cancel queued work before transport starts" + ); + } #[test] fn expired_deadline_does_not_poll_request() { diff --git a/crates/trusted-server-adapter-cloudflare/tests/runtime/deadline.rs b/crates/trusted-server-adapter-cloudflare/tests/runtime/deadline.rs new file mode 100644 index 0000000000..b3faec2fc1 --- /dev/null +++ b/crates/trusted-server-adapter-cloudflare/tests/runtime/deadline.rs @@ -0,0 +1,70 @@ +//! Local runtime fixture exercising the production HTTP client implementation. + +#[cfg(target_arch = "wasm32")] +#[allow(dead_code)] +#[path = "../../src/platform.rs"] +mod platform; + +#[cfg(target_arch = "wasm32")] +use std::time::Duration; +#[cfg(target_arch = "wasm32")] +use trusted_server_core::platform::{PlatformHttpClient as _, PlatformHttpRequest}; + +#[cfg(target_arch = "wasm32")] +#[worker::event(fetch)] +async fn main( + request: worker::Request, + _: worker::Env, + _: worker::Context, +) -> worker::Result { + let uri = request.url()?.to_string(); + let result = exercise(&uri) + .await + .map_err(|error| worker::Error::RustError(format!("{error:?}")))?; + worker::Response::ok(result) +} + +#[cfg(target_arch = "wasm32")] +async fn exercise( + uri: &str, +) -> Result> { + let parameter = |name: &str| { + uri.split_once('?').and_then(|(_, query)| { + query + .split('&') + .filter_map(|pair| pair.split_once('=')) + .find(|(key, _)| *key == name) + .map(|(_, value)| value) + }) + }; + let upstream = parameter("upstream").expect("should supply local fixture upstream"); + let budget_ms: u64 = parameter("budget_ms") + .expect("should supply budget") + .parse() + .expect("should parse budget"); + let origin_ms: u64 = parameter("origin_ms") + .expect("should supply origin delay") + .parse() + .expect("should parse origin delay"); + let client = platform::CloudflareHttpClient; + let request = edgezero_core::http::request_builder() + .uri(upstream) + .body(edgezero_core::body::Body::empty()) + .expect("should construct fixture request"); + let started = web_time::Instant::now(); + let pending = client + .send_async(PlatformHttpRequest::new(request, "fixture")) + .await?; + worker::Delay::from(Duration::from_millis(origin_ms)).await; + let collected = client + .select_until(vec![pending], started + Duration::from_millis(budget_ms)) + .await?; + let status = match collected { + Some(result) => { + result.ready?; + "complete" + } + None => "timeout", + }; + Ok(format!("{status}:{}", started.elapsed().as_millis())) +} diff --git a/crates/trusted-server-adapter-fastly/src/platform.rs b/crates/trusted-server-adapter-fastly/src/platform.rs index 1e844111b6..c5a916bec7 100644 --- a/crates/trusted-server-adapter-fastly/src/platform.rs +++ b/crates/trusted-server-adapter-fastly/src/platform.rs @@ -22,6 +22,19 @@ use trusted_server_core::platform::{ }; use trusted_server_core::settings::TrustedClientIpConfig; +// Cache-pass requests always have a direct SDK pending handle. Keep this proof +// with the pending request until the deadline collector consumes it. +struct CacheBypassPending(fastly::http::request::PendingRequest); + +fn unwrap_pending( + pending: PlatformPendingRequest, +) -> Result { + match pending.downcast::() { + Ok(pending) => Ok(pending.0), + Err(pending) => pending.downcast(), + } +} + // --------------------------------------------------------------------------- // FastlyPlatformConfigStore // --------------------------------------------------------------------------- @@ -185,6 +198,21 @@ impl PlatformBackend for FastlyPlatformBackend { } } +fn buffered_response_to_platform( + response: &fastly::Response, + backend_name: String, + bytes: Vec, +) -> Result> { + let mut builder = edgezero_core::http::response_builder().status(response.get_status()); + for (name, value) in response.get_headers() { + builder = builder.header(name.as_str(), value.as_bytes()); + } + let response = builder + .body(edgezero_core::body::Body::from(bytes)) + .change_context(PlatformError::HttpClient)?; + Ok(PlatformResponse::new(response).with_backend_name(backend_name)) +} + // --------------------------------------------------------------------------- // FastlyPlatformHttpClient — helpers // --------------------------------------------------------------------------- @@ -494,6 +522,11 @@ fn apply_fastly_cache_intent(request: &mut fastly::Request, intent: &PlatformCac /// environment cannot register a real Fastly backend, so the common /// `StubHttpClient` driver test records the one-send 3xx behavior while adapter /// tests cover request conversion and the single-send boundary. +/// +/// Deadline collection requires [`PlatformCacheIntent::Bypass`], which all +/// auction requests use. This guarantees a direct SDK pending handle that can +/// join body handles in one timed host wait. Shared-cache requests retain the +/// SDK caching state machine through the unbounded send/select/wait APIs. pub struct FastlyPlatformHttpClient; #[async_trait::async_trait(?Send)] @@ -543,7 +576,12 @@ impl PlatformHttpClient for FastlyPlatformHttpClient { let pending = fastly_req .send_async(&backend_name) .change_context(PlatformError::HttpClient)?; - Ok(PlatformPendingRequest::new(pending) + let pending = if cache_intent.is_bypass() { + PlatformPendingRequest::new(CacheBypassPending(pending)) + } else { + PlatformPendingRequest::new(pending) + }; + Ok(pending .with_backend_name(backend_name) .with_response_handling(stream_response, request_method)) } @@ -570,7 +608,7 @@ impl PlatformHttpClient for FastlyPlatformHttpClient { let mut fastly_pending: Vec = Vec::with_capacity(pending_requests.len()); for platform_req in pending_requests { - let inner = platform_req.downcast::().map_err(|platform_req| { + let inner = unwrap_pending(platform_req).map_err(|platform_req| { let backend_name = platform_req.backend_name().unwrap_or(""); Report::new(PlatformError::HttpClient).attach(format!( "PlatformPendingRequest inner type is not fastly::PendingRequest for backend '{backend_name}'" @@ -636,12 +674,10 @@ impl PlatformHttpClient for FastlyPlatformHttpClient { &self, pending: PlatformPendingRequest, ) -> Result> { - use fastly::http::request::PendingRequest; - let backend_hint = pending.backend_name().map(str::to_owned); let stream_response = pending.stream_response(); let request_is_head = pending.request_method() == Some(&edgezero_core::http::Method::HEAD); - let pending = pending.downcast::().map_err(|pending| { + let pending = unwrap_pending(pending).map_err(|pending| { let backend_name = pending.backend_name().unwrap_or(""); Report::new(PlatformError::HttpClient).attach(format!( "PlatformPendingRequest inner type is not fastly::PendingRequest for backend '{backend_name}'" diff --git a/crates/trusted-server-adapter-fastly/src/platform/deadline.rs b/crates/trusted-server-adapter-fastly/src/platform/deadline.rs index d6da1cb0ec..bf3081bb77 100644 --- a/crates/trusted-server-adapter-fastly/src/platform/deadline.rs +++ b/crates/trusted-server-adapter-fastly/src/platform/deadline.rs @@ -1,17 +1,17 @@ //! Deadline-bounded collection of Fastly auction responses. -use std::io::Read as _; -use std::time::Duration; - use error_stack::{Report, ResultExt as _}; -use fastly::handle::BodyHandle; -use fastly::http::request::{PendingRequest, PollResult}; +use fastly::handle::{BodyHandle, PendingRequestHandle, PollHandleResult}; +use std::io::Read as _; use trusted_server_core::platform::{ PlatformError, PlatformPendingRequest, PlatformResponse, PlatformSelectResult, }; use web_time::Instant; -use super::{MAX_PLATFORM_RESPONSE_BODY_BYTES, fastly_response_to_platform, response_carries_body}; +use super::{ + CacheBypassPending, MAX_PLATFORM_RESPONSE_BODY_BYTES, buffered_response_to_platform, + fastly_response_to_platform, response_carries_body, +}; // Bound payload retained across all partial bodies, as well as each response. const MAX_TOTAL_BUFFERED_BODY_BYTES: usize = MAX_PLATFORM_RESPONSE_BODY_BYTES; @@ -25,7 +25,8 @@ struct DeadlinePending { } enum PendingState { - Headers(Box>), + Headers(Option), + Ready(Result>), Body { response: Box, body: BodyHandle, @@ -48,15 +49,19 @@ impl DeadlinePending { Err(pending) => pending, }; let backend_name = pending.backend_name().unwrap_or("").to_owned(); - let pending = pending.downcast::().map_err(|_| { - Report::new(PlatformError::HttpClient).attach(format!( - "invalid Fastly deadline pending handle for backend '{backend_name}'" + // The marker is constructed only after set_pass(true), which selects + // HostCaching in fastly 0.13.1's Request::get_caching_mode. Its direct + // handle conversion therefore cannot encounter a guest caching state. + let pending = pending.downcast::().map_err(|_| { + Report::new(PlatformError::Unsupported).attach(format!( + "deadline collection requires a cache-bypass request for backend '{backend_name}'" )) })?; + let pending = PendingRequestHandle::from(pending.0); Ok(Self { backend_name, request_method, - state: Some(PendingState::Headers(Box::new(Some(pending)))), + state: Some(PendingState::Headers(Some(pending))), }) } @@ -88,37 +93,35 @@ impl DeadlinePending { fn buffered_bytes(&self) -> usize { match &self.state { Some(PendingState::Body { bytes, .. }) => bytes.len(), + Some(PendingState::Ready(Ok(response))) => { + response.response.body().as_bytes().map_or(0, <[u8]>::len) + } _ => 0, } } - fn poll( - &mut self, - deadline: Instant, - available_buffer_bytes: usize, - ) -> Result, Report> { + fn poll(&mut self) -> Result, Report> { let state = self .state .take() .expect("should retain pending collection state"); let state = match state { + PendingState::Ready(result) => return result.map(Some), PendingState::Headers(mut slot) => { match slot.take().expect("should retain a pending request").poll() { - PollResult::Pending(pending) => { - *slot = Some(pending); + PollHandleResult::Pending(pending) => { + slot = Some(pending); self.state = Some(PendingState::Headers(slot)); return Ok(None); } - PollResult::Done(Err(error)) => { - self.backend_name = error.backend_name().to_owned(); + PollHandleResult::Done(Err(error)) => { return Err( Report::new(PlatformError::HttpClient).attach(error.to_string()) ); } - PollResult::Done(Ok(mut response)) => { - if let Some(backend_name) = response.get_backend_name() { - self.backend_name = backend_name.to_owned(); - } + PollHandleResult::Done(Ok((response_handle, body_handle))) => { + let mut response = + fastly::Response::from_handles(response_handle, body_handle); let request_is_head = self.request_is_head(&response)?; if !response_carries_body(request_is_head, response.get_status()) { return fastly_response_to_platform( @@ -150,14 +153,14 @@ impl DeadlinePending { state => state, }; let PendingState::Body { - mut response, + response, mut body, mut bytes, } = state else { unreachable!("headers are polled before buffering bodies"); }; - if Instant::now() >= deadline || !body_is_ready(&body)? || Instant::now() >= deadline { + if !body_is_ready(&body)? { self.state = Some(PendingState::Body { response, body, @@ -170,17 +173,20 @@ impl DeadlinePending { .read(&mut chunk) .change_context(PlatformError::HttpClient)?; if read == 0 { - response.set_body(bytes); - return fastly_response_to_platform(*response, self.backend_name.clone(), false, false) + return buffered_response_to_platform(&response, self.backend_name.clone(), bytes) .map(Some); } if bytes.len() + read > MAX_PLATFORM_RESPONSE_BODY_BYTES { return Err(Report::new(PlatformError::HttpClient) .attach("origin response body exceeds response body limit")); } - if read > available_buffer_bytes { - return Err(Report::new(PlatformError::HttpClient) - .attach("origin response body exceeds aggregate auction buffer capacity")); + if bytes.len() + read > bytes.capacity() { + let capacity = bytes + .capacity() + .max(chunk.len()) + .saturating_mul(2) + .min(MAX_PLATFORM_RESPONSE_BODY_BYTES); + bytes.reserve_exact(capacity - bytes.len()); } bytes.extend_from_slice(&chunk[..read]); self.state = Some(PendingState::Body { @@ -204,15 +210,16 @@ fn body_is_ready(body: &BodyHandle) -> Result> { Ok(ready != 0) } -/// Collect one fully buffered response before the absolute deadline. +/// Collect one fully buffered response without waiting past the absolute deadline. /// /// Dropping response bodies closes their host handles. Fastly has no pending /// request cancellation ABI: unfinished header requests are abandoned and -/// never waited on past the auction deadline. +/// never waited on past the auction deadline. If collection starts after the +/// deadline, one nonblocking snapshot preserves already available responses. /// -/// Partial bodies share a 10 MiB payload budget. A provider whose next chunk -/// would exceed that capacity fails with an attributed HTTP client error; -/// its buffered bytes are released and its siblings remain collectible. +/// Partial bodies share a 10 MiB payload budget. If a chunk exceeds that +/// capacity, the largest partial response fails with an attributed HTTP client +/// error; its buffered bytes are released and smaller siblings remain collectible. /// Completed response payload plus retained partial bodies fits the same budget. /// /// # Errors @@ -223,9 +230,6 @@ pub(super) fn select_until( pending_requests: Vec, deadline: Instant, ) -> Result, Report> { - if Instant::now() >= deadline { - return Ok(None); - } if pending_requests.is_empty() { return Err(Report::new(PlatformError::HttpClient) .attach("select_until called with an empty pending_requests list")); @@ -239,25 +243,71 @@ pub(super) fn select_until( return Err(Report::new(PlatformError::HttpClient) .attach("resumed auction responses exceed aggregate buffer capacity")); } + let expired_at_entry = Instant::now() >= deadline; + if expired_at_entry { + // Take one nonblocking snapshot. Fully buffered results are cached; + // responses that would require a wait are abandoned, never retried. + for index in 0..pending.len() { + loop { + let previous = pending[index].buffered_bytes(); + let result = pending[index].poll(); + let progressed = + matches!(result, Ok(None)) && pending[index].buffered_bytes() > previous; + match result { + Ok(None) if progressed => {} + Ok(None) => { + pending[index].state = None; + } + result => { + pending[index].state = Some(PendingState::Ready(match result { + Ok(Some(response)) => Ok(response), + Err(error) => Err(error), + Ok(None) => unreachable!("pending results handled above"), + })); + } + } + buffered_bytes = buffered_bytes - previous + pending[index].buffered_bytes(); + enforce_aggregate_budget(&mut pending, &mut buffered_bytes); + if !progressed || !matches!(pending[index].state, Some(PendingState::Body { .. })) { + break; + } + } + } + pending.retain(|item| item.state.is_some()); + } loop { for index in 0..pending.len() { - if Instant::now() >= deadline { - return Ok(None); + if !expired_at_entry && Instant::now() >= deadline { + // A timed host wait may wake with a ready event at the cutoff. + // Finish with the same nonblocking snapshot as expired entry. + return select_until( + pending + .into_iter() + .map(DeadlinePending::into_platform) + .collect(), + deadline, + ); } let previous_bytes = pending[index].buffered_bytes(); - let result = - pending[index].poll(deadline, MAX_TOTAL_BUFFERED_BODY_BYTES - buffered_bytes); - buffered_bytes = buffered_bytes - previous_bytes + pending[index].buffered_bytes(); - // A host event arriving at the deadline must not make a late bid eligible. - if Instant::now() >= deadline { - return Ok(None); + let result = pending[index].poll(); + if let Ok(Some(response)) = result { + pending[index].state = Some(PendingState::Ready(Ok(response))); + } else if let Err(error) = result { + pending[index].state = Some(PendingState::Ready(Err(error))); } - let ready = match result { - Ok(None) => continue, - Ok(Some(response)) => Ok(response), - Err(error) => Err(error), + buffered_bytes = buffered_bytes - previous_bytes + pending[index].buffered_bytes(); + enforce_aggregate_budget(&mut pending, &mut buffered_bytes); + // Capacity eviction may have failed an earlier large sibling. + let ready_index = pending + .iter() + .position(|item| matches!(item.state, Some(PendingState::Ready(_)))); + let Some(ready_index) = ready_index else { + continue; + }; + let mut completed = pending.swap_remove(ready_index); + let Some(PendingState::Ready(ready)) = completed.state.take() else { + unreachable!("should select a ready result"); }; - let completed = pending.swap_remove(index); let failed_backend_name = ready.as_ref().err().map(|_| completed.backend_name); return Ok(Some(PlatformSelectResult { ready, @@ -268,15 +318,44 @@ pub(super) fn select_until( failed_backend_name, })); } + if expired_at_entry || pending.is_empty() { + return Ok(None); + } wait_for_progress(&pending, deadline)?; } } +fn enforce_aggregate_budget(pending: &mut [DeadlinePending], buffered_bytes: &mut usize) { + if *buffered_bytes <= MAX_TOTAL_BUFFERED_BODY_BYTES { + return; + } + // The overflow is at most one read chunk. Evict the largest partial body + // first, preserving completed responses and healthy smaller siblings. + let largest = pending + .iter() + .enumerate() + .filter(|(_, item)| matches!(item.state, Some(PendingState::Body { .. }))) + .max_by_key(|(_, item)| item.buffered_bytes()) + .or_else(|| { + pending + .iter() + .enumerate() + .max_by_key(|(_, item)| item.buffered_bytes()) + }) + .map(|(index, _)| index) + .expect("should retain the response exceeding buffer capacity"); + *buffered_bytes -= pending[largest].buffered_bytes(); + pending[largest].state = Some(PendingState::Ready(Err(Report::new( + PlatformError::HttpClient, + ) + .attach("largest partial response exceeds aggregate auction buffer capacity")))); +} + fn wait_for_progress( pending: &[DeadlinePending], deadline: Instant, ) -> Result<(), Report> { - let Some(mut remaining) = deadline.checked_duration_since(Instant::now()) else { + let Some(remaining) = deadline.checked_duration_since(Instant::now()) else { return Ok(()); }; if remaining.is_zero() { @@ -285,18 +364,17 @@ fn wait_for_progress( let mut bodies = Vec::new(); for pending in pending { match &pending.state { - Some(PendingState::Headers(_)) => { - // High-level PendingRequest keeps its host handle private. Poll - // at most 1ms apart without spinning while headers are pending. - remaining = remaining.min(Duration::from_millis(1)); - } + Some(PendingState::Headers(Some(handle))) => bodies.push(handle.as_u32()), Some(PendingState::Body { body, .. }) => { // SAFETY: these handles remain owned by pending throughout select. bodies.push(unsafe { body.as_u32() }); } - None => {} + _ => {} } } + if bodies.is_empty() { + return Ok(()); + } let timeout_ms = u32::try_from(remaining.as_millis()) .unwrap_or(u32::MAX) .max(1); @@ -332,13 +410,19 @@ mod tests { use web_time::Instant; #[test] - fn expired_deadline_does_not_accept_a_pending_result() { + fn expired_deadline_rejects_an_invalid_pending_handle() { let result = futures::executor::block_on(FastlyPlatformHttpClient.select_until( vec![PlatformPendingRequest::new(42_u32).with_backend_name("example")], Instant::now(), )) - .expect("should report expiration before inspecting handles"); - assert!(result.is_none(), "should discard expired pending requests"); + .expect_err("should reject invalid handles even in the nonblocking snapshot"); + assert!( + matches!( + result.current_context(), + trusted_server_core::platform::PlatformError::Unsupported + ), + "should reject unsupported deadline handles without panicking" + ); } #[test] @@ -370,15 +454,15 @@ mod tests { } #[test] - fn expired_deadline_discards_an_already_ready_body() { + fn expired_deadline_preserves_an_already_ready_body() { let pending = buffered_pending(b"ready".to_vec(), "example"); let result = futures::executor::block_on( FastlyPlatformHttpClient.select_until(vec![pending], Instant::now()), ) .expect("should discard an expired ready response"); assert!( - result.is_none(), - "should never accept a ready response after expiration" + result.is_some(), + "should preserve an already complete response without waiting" ); } @@ -432,6 +516,28 @@ mod tests { ); } + #[test] + fn completed_body_moves_into_platform_without_a_payload_copy() { + let bytes = vec![b'x'; 32768]; + let pointer = bytes.as_ptr(); + let response = super::buffered_response_to_platform( + &fastly::Response::new(), + "example".to_owned(), + bytes, + ) + .expect("should move a buffered response into the platform"); + assert_eq!( + response + .response + .body() + .as_bytes() + .expect("should retain a buffered body") + .as_ptr(), + pointer, + "should retain the original allocation without a Fastly host-body roundtrip" + ); + } + #[test] fn body_collection_is_fair_and_preserves_partial_siblings() { let large = vec![b'x'; 32768]; @@ -474,6 +580,30 @@ mod tests { ); } + #[test] + fn body_at_the_exact_limit_remains_eligible() { + let pending = buffered_pending(vec![b'x'; MAX_PLATFORM_RESPONSE_BODY_BYTES], "exact"); + let result = futures::executor::block_on( + FastlyPlatformHttpClient + .select_until(vec![pending], Instant::now() + Duration::from_secs(10)), + ) + .expect("should collect an exactly sized body") + .expect("should complete before the deadline"); + let response = result + .ready + .expect("should accept the configured body-size boundary"); + assert_eq!( + response + .response + .body() + .as_bytes() + .expect("should retain buffered bytes") + .len(), + MAX_PLATFORM_RESPONSE_BODY_BYTES, + "should accept the exact limit without evicting its EOF probe" + ); + } + #[test] fn incremental_body_limit_reports_the_failed_backend() { let pending = buffered_pending(vec![b'x'; MAX_PLATFORM_RESPONSE_BODY_BYTES + 1], "large"); @@ -495,58 +625,70 @@ mod tests { } #[test] - fn aggregate_body_limit_rejects_growth_and_preserves_partial_siblings() { - let half_budget = MAX_PLATFORM_RESPONSE_BODY_BYTES / 2; - let partial = |unread: Vec, backend: &str| { - let mut pending = DeadlinePending::from_platform(buffered_pending(unread, backend)) - .expect("should construct a partially buffered response"); - if let Some(PendingState::Body { bytes, .. }) = &mut pending.state { - *bytes = vec![b'x'; half_budget]; - } - pending.into_platform() - }; + fn aggregate_body_pressure_evicts_largest_partial_provider() { + let mut large = + DeadlinePending::from_platform(buffered_pending(vec![b'x'; 16384], "large")) + .expect("should construct a large partial response"); + if let Some(PendingState::Body { bytes, .. }) = &mut large.state { + *bytes = vec![b'x'; MAX_PLATFORM_RESPONSE_BODY_BYTES - 1]; + } let pending = vec![ - partial(vec![b'x'], "growing"), - partial(Vec::new(), "sibling"), + buffered_pending(b"healthy".to_vec(), "healthy"), + large.into_platform(), ]; let deadline = Instant::now() + Duration::from_secs(10); let result = futures::executor::block_on(FastlyPlatformHttpClient.select_until(pending, deadline)) - .expect("should return an attributed capacity failure") - .expect("should reject excess buffer growth before deadline"); + .expect("should return a capacity result") + .expect("should reject excess growth"); assert!( result.ready.is_err(), - "should reject growth beyond aggregate buffer capacity" + "should reject excess aggregate buffering" ); assert_eq!( result.failed_backend_name.as_deref(), - Some("growing"), - "should attribute the capacity failure to the growing provider" - ); - assert_eq!( - result.remaining.len(), - 1, - "should preserve the partial sibling response" + Some("large"), + "should attribute capacity pressure to the largest partial response" ); let result = futures::executor::block_on( FastlyPlatformHttpClient.select_until(result.remaining, deadline), ) - .expect("should resume the preserved sibling") - .expect("should complete its already buffered body"); - let response = result.ready.expect("should retain the successful sibling"); + .expect("should retain the healthy provider") + .expect("should complete the healthy provider"); assert_eq!( - response.backend_name.as_deref(), - Some("sibling"), - "should preserve backend correlation" + result + .ready + .expect("should preserve the healthy bid") + .backend_name + .as_deref(), + Some("healthy"), + "should avoid blaming the healthy provider for another provider's large body" ); - let (_, body) = response.response.into_parts(); - let bytes = body - .into_bytes() - .expect("should retain a buffered response body"); - assert_eq!( - bytes.len(), - half_budget, - "should preserve all previously buffered sibling bytes" + } + + #[test] + fn expired_snapshot_retains_only_ready_results_across_calls() { + let pending = vec![ + buffered_pending(b"one".to_vec(), "one"), + buffered_pending(b"two".to_vec(), "two"), + ]; + let deadline = Instant::now(); + let result = + futures::executor::block_on(FastlyPlatformHttpClient.select_until(pending, deadline)) + .expect("should snapshot ready results") + .expect("should return first ready result"); + assert!( + result.ready.is_ok(), + "should preserve a complete ready response" + ); + let result = futures::executor::block_on( + FastlyPlatformHttpClient.select_until(result.remaining, deadline), + ) + .expect("should use cached ready results") + .expect("should return second ready result"); + assert!( + result.ready.is_ok(), + "should preserve the second complete response" ); } @@ -564,6 +706,33 @@ mod tests { .expect("should launch fixture request") } + #[test] + #[ignore = "requires the local origin started by tests/deadline_origin.py"] + fn network_slow_origin_preserves_ready_bid_without_waiting_for_stalled_sibling() { + let fast = network_pending("fast", "GET"); + let stalled = network_pending("stalled", "GET"); + let deadline = Instant::now() + Duration::from_millis(40); + std::thread::sleep(Duration::from_millis(100)); + let started = Instant::now(); + let result = futures::executor::block_on( + FastlyPlatformHttpClient.select_until(vec![stalled, fast], deadline), + ) + .expect("should snapshot already ready bids") + .expect("should retain the complete fast response"); + assert!( + result.ready.is_ok(), + "should preserve a bid available after a slow publisher origin" + ); + assert!( + result.remaining.is_empty(), + "should abandon the stalled sibling in the nonblocking snapshot" + ); + assert!( + started.elapsed() < Duration::from_millis(100), + "should never wait for the stalled sibling after expiration" + ); + } + #[test] #[ignore = "requires the local origin started by tests/deadline_origin.py"] fn network_stalled_headers_stop_at_deadline() { diff --git a/crates/trusted-server-adapter-fastly/tests/deadline_origin.py b/crates/trusted-server-adapter-fastly/tests/deadline_origin.py index 8af17cfc2d..9cf7c9c5a1 100644 --- a/crates/trusted-server-adapter-fastly/tests/deadline_origin.py +++ b/crates/trusted-server-adapter-fastly/tests/deadline_origin.py @@ -1,9 +1,12 @@ """Run ignored Fastly deadline tests against a local, deliberately slow origin. -Usage: python3 tests/deadline_origin.py /path/to/viceroy /path/to/test-binary.wasm +Usage: python3 tests/deadline_origin.py /path/to/viceroy [test-binary.wasm] + +Without an artifact argument, build and locate the current binary using Cargo JSON. """ import http.server +import json import os import pathlib import subprocess @@ -47,6 +50,24 @@ def log_message(self, *_args): def main(): + if len(sys.argv) == 2: + build = subprocess.run( + ["cargo", "test", "-p", "trusted-server-adapter-fastly", + "--target", "wasm32-wasip1", "--bin", "trusted-server-adapter-fastly", "--no-run", + "--message-format=json"], + text=True, stdout=subprocess.PIPE, check=True, + ) + artifacts = [json.loads(line) for line in build.stdout.splitlines() if line.startswith("{")] + binaries = [item["executable"] for item in artifacts + if item.get("reason") == "compiler-artifact" + and item.get("profile", {}).get("test") + and item.get("target", {}).get("name") == "trusted-server-adapter-fastly" + and item.get("executable")] + if len(binaries) != 1: + raise RuntimeError(f"Expected one Fastly binary test artifact, found {binaries!r}") + binary = binaries[0] + else: + binary = sys.argv[2] with http.server.ThreadingHTTPServer(("127.0.0.1", 0), Origin) as server: server.daemon_threads = True threading.Thread(target=server.serve_forever, daemon=True).start() @@ -60,10 +81,11 @@ def main(): if pathlib.Path("/etc/ssl/cert.pem").exists(): environment.setdefault("SSL_CERT_FILE", "/etc/ssl/cert.pem") result = subprocess.run( - [sys.argv[1], "run", "-C", str(config), sys.argv[2], + [sys.argv[1], "run", "-C", str(config), binary, "platform::deadline::tests::network_", "--ignored", "--nocapture"], env=environment, check=False, + timeout=120, ) server.shutdown() return result.returncode diff --git a/crates/trusted-server-adapter-spin/Cargo.toml b/crates/trusted-server-adapter-spin/Cargo.toml index b5c81acd8e..734370bb0c 100644 --- a/crates/trusted-server-adapter-spin/Cargo.toml +++ b/crates/trusted-server-adapter-spin/Cargo.toml @@ -42,4 +42,10 @@ spin-sdk = { workspace = true } trusted-server-core = { workspace = true, features = ["test-utils"] } base64 = { workspace = true } edgezero-core = { workspace = true } +[target.'cfg(not(target_arch = "wasm32"))'.dev-dependencies] tokio = { workspace = true, features = ["rt-multi-thread", "macros"] } + +[[example]] +name = "deadline-runtime" +path = "tests/runtime/deadline.rs" +crate-type = ["cdylib"] diff --git a/crates/trusted-server-adapter-spin/src/platform.rs b/crates/trusted-server-adapter-spin/src/platform.rs index 27468fe6b8..19b4cc3022 100644 --- a/crates/trusted-server-adapter-spin/src/platform.rs +++ b/crates/trusted-server-adapter-spin/src/platform.rs @@ -427,18 +427,43 @@ fn is_wasi_forbidden_outbound_header(name: &str) -> bool { || name.eq_ignore_ascii_case("proxy-connection") } -/// Owns the outbound operation until selection completes or drops it. -#[cfg(all(feature = "spin", target_arch = "wasm32"))] -struct SpinPendingResponse { - response: - futures::future::LocalBoxFuture<'static, Result>>, +/// Receives a dispatched operation and cancels it when its owner is dropped. +#[cfg(any(test, all(feature = "spin", target_arch = "wasm32")))] +struct SpinPendingResponse { + response: futures::channel::oneshot::Receiver, + abort: futures::future::AbortHandle, +} + +#[cfg(any(test, all(feature = "spin", target_arch = "wasm32")))] +impl SpinPendingResponse { + fn start( + operation: impl std::future::Future + 'static, + spawn: impl FnOnce(futures::future::LocalBoxFuture<'static, ()>), + ) -> Self { + let (sender, response) = futures::channel::oneshot::channel(); + let (abort, registration) = futures::future::AbortHandle::new_pair(); + spawn(Box::pin(async move { + if let Ok(result) = futures::future::Abortable::new(operation, registration).await { + let _ = sender.send(result); + } + })); + Self { response, abort } + } +} + +#[cfg(any(test, all(feature = "spin", target_arch = "wasm32")))] +impl Drop for SpinPendingResponse { + fn drop(&mut self) { + self.abort.abort(); + } } /// `spin_sdk::http::send`-backed HTTP client for the Spin runtime. /// -/// Selection supports one provider at a time, so +/// Auction configuration supports one provider at a time, so /// [`PlatformHttpClient::supports_concurrent_fanout`] reports `false`. -/// Deadline selection bounds the full send and response body collection. +/// Requests start in a runtime task at dispatch, overlapping publisher-origin work. +/// Deadline selection bounds waiting for the full send and response body collection. /// Dropping the pending operation cancels its WASI asynchronous operations. /// /// Spin's WASI HTTP API sends one request and returns the original response; it @@ -545,8 +570,8 @@ impl SpinPlatformHttpClient { #[async_trait::async_trait(?Send)] impl PlatformHttpClient for SpinPlatformHttpClient { fn supports_concurrent_fanout(&self) -> bool { - // Keep the existing single-provider restriction: selection polls one - // outbound request at a time. + // Retain the existing auction capability restriction independently of + // runtime task scheduling. false } @@ -562,9 +587,7 @@ impl PlatformHttpClient for SpinPlatformHttpClient { request: PlatformHttpRequest, ) -> Result> { let backend_name = request.backend_name.clone(); - let pending = SpinPendingResponse { - response: Box::pin(Self::execute(request)), - }; + let pending = SpinPendingResponse::start(Self::execute(request), spin_sdk::wasip3::spawn); Ok(PlatformPendingRequest::new(pending).with_backend_name(backend_name)) } @@ -587,14 +610,17 @@ impl PlatformHttpClient for SpinPlatformHttpClient { } let ready_platform = pending_requests.remove(0); - let pending = ready_platform - .downcast::() + let mut pending = ready_platform + .downcast::>>>() .map_err(|_| { Report::new(PlatformError::HttpClient) .attach("unexpected inner type in SpinPlatformHttpClient::select") })?; - let ready = pending.response.await; + let ready = (&mut pending.response).await.unwrap_or_else(|error| { + Err(Report::new(PlatformError::HttpClient) + .attach(format!("dispatched response task stopped: {error}"))) + }); Ok(PlatformSelectResult { ready, remaining: pending_requests, @@ -615,20 +641,33 @@ impl PlatformHttpClient for SpinPlatformHttpClient { return Err(Report::new(PlatformError::HttpClient) .attach("multi-provider fan-out is not supported on this adapter")); } - let remaining = deadline.saturating_duration_since(web_time::Instant::now()); - if remaining.is_zero() { - return Ok(None); - } let platform_pending = pending_requests.remove(0); let backend_name = platform_pending.backend_name().map(str::to_owned); - let pending = platform_pending - .downcast::() + let mut pending = platform_pending + .downcast::>>>() .map_err(|_| { Report::new(PlatformError::HttpClient) .attach("unexpected pending request type in deadline selection") })?; + // Probe only the task's buffered result. This never polls transport I/O + // and preserves a completed bid when publisher-origin work ran long. + if let Some(ready) = pending.response.try_recv().map_err(|error| { + Report::new(PlatformError::HttpClient) + .attach(format!("dispatched response task stopped: {error}")) + })? { + return Ok(Some(PlatformSelectResult { + failed_backend_name: ready.as_ref().err().and(backend_name), + ready, + remaining: pending_requests, + })); + } let result = complete_before_deadline( - pending.response, + async { + (&mut pending.response).await.unwrap_or_else(|error| { + Err(Report::new(PlatformError::HttpClient) + .attach(format!("dispatched response task stopped: {error}"))) + }) + }, async { let remaining = deadline.saturating_duration_since(web_time::Instant::now()); if !remaining.is_zero() { @@ -1432,11 +1471,112 @@ mod deadline_tests { use std::rc::Rc; use std::time::Duration; - use futures::executor::block_on; + use futures::executor::{LocalPool, block_on}; use futures::future::{pending, ready}; + use futures::task::LocalSpawnExt as _; use web_time::Instant; - use super::complete_before_deadline; + use super::{SpinPendingResponse, complete_before_deadline}; + + #[test] + fn dispatched_task_starts_before_collection_and_buffers_complete_body() { + let mut executor = LocalPool::new(); + let spawner = executor.spawner(); + let started = Rc::new(Cell::new(false)); + let started_operation = Rc::clone(&started); + let (finish_body, body) = futures::channel::oneshot::channel(); + let mut pending = SpinPendingResponse::start( + async move { + started_operation.set(true); + body.await.expect("should receive complete response body") + }, + |task| { + spawner + .spawn_local(task) + .expect("should spawn outbound task") + }, + ); + + executor.run_until_stalled(); + assert!(started.get(), "should start outbound work before selection"); + assert_eq!( + pending.response.try_recv().expect("should poll result"), + None, + "should not expose headers before response body completes" + ); + finish_body.send(42).expect("should finish response body"); + executor.run_until_stalled(); + let expired = web_time::Instant::now(); + assert!( + web_time::Instant::now() >= expired, + "should collect after cutoff" + ); + assert_eq!( + pending + .response + .try_recv() + .expect("should poll buffered result"), + Some(42), + "should preserve a complete buffered response without polling transport" + ); + } + + #[test] + fn dropping_dispatched_pending_cancels_started_body_operation() { + struct DropProbe(Rc>); + impl Drop for DropProbe { + fn drop(&mut self) { + self.0.set(true); + } + } + let mut executor = LocalPool::new(); + let spawner = executor.spawner(); + let dropped = Rc::new(Cell::new(false)); + let probe = DropProbe(Rc::clone(&dropped)); + let pending = SpinPendingResponse::start( + async move { + let _probe = probe; + futures::future::pending::<()>().await; + }, + |task| { + spawner + .spawn_local(task) + .expect("should spawn outbound task") + }, + ); + executor.run_until_stalled(); + assert!(!dropped.get(), "should retain an active body operation"); + drop(pending); + executor.run_until_stalled(); + assert!( + dropped.get(), + "should drop spawned operation after cancellation" + ); + } + + #[test] + fn dropping_queued_dispatch_prevents_transport_poll() { + let mut executor = LocalPool::new(); + let spawner = executor.spawner(); + let started = Rc::new(Cell::new(false)); + let operation_started = Rc::clone(&started); + let pending = SpinPendingResponse::start( + async move { + operation_started.set(true); + }, + |task| { + spawner + .spawn_local(task) + .expect("should spawn outbound task") + }, + ); + drop(pending); + executor.run_until_stalled(); + assert!( + !started.get(), + "should cancel queued work before transport starts" + ); + } #[test] fn expired_deadline_does_not_poll_request() { diff --git a/crates/trusted-server-adapter-spin/tests/runtime/deadline.rs b/crates/trusted-server-adapter-spin/tests/runtime/deadline.rs new file mode 100644 index 0000000000..b8b13dbeb4 --- /dev/null +++ b/crates/trusted-server-adapter-spin/tests/runtime/deadline.rs @@ -0,0 +1,65 @@ +//! Local runtime fixture exercising the production HTTP client implementation. + +#[cfg(all(feature = "spin", target_arch = "wasm32"))] +#[allow(dead_code)] +#[path = "../../src/platform.rs"] +mod platform; + +#[cfg(all(feature = "spin", target_arch = "wasm32"))] +use std::time::Duration; +#[cfg(all(feature = "spin", target_arch = "wasm32"))] +use trusted_server_core::platform::{PlatformHttpClient as _, PlatformHttpRequest}; + +#[cfg(all(feature = "spin", target_arch = "wasm32"))] +#[spin_sdk::http_service] +async fn handle(request: spin_sdk::http::Request) -> Result { + let result = exercise(&request.uri().to_string()) + .await + .map_err(|error| format!("{error:?}"))?; + Ok(result) +} + +#[cfg(all(feature = "spin", target_arch = "wasm32"))] +async fn exercise( + uri: &str, +) -> Result> { + let parameter = |name: &str| { + uri.split_once('?').and_then(|(_, query)| { + query + .split('&') + .filter_map(|pair| pair.split_once('=')) + .find(|(key, _)| *key == name) + .map(|(_, value)| value) + }) + }; + let upstream = parameter("upstream").expect("should supply local fixture upstream"); + let budget_ms: u64 = parameter("budget_ms") + .expect("should supply budget") + .parse() + .expect("should parse budget"); + let origin_ms: u64 = parameter("origin_ms") + .expect("should supply origin delay") + .parse() + .expect("should parse origin delay"); + let client = platform::SpinPlatformHttpClient; + let request = edgezero_core::http::request_builder() + .uri(upstream) + .body(edgezero_core::body::Body::empty()) + .expect("should construct fixture request"); + let started = web_time::Instant::now(); + let pending = client + .send_async(PlatformHttpRequest::new(request, "fixture")) + .await?; + spin_sdk::time::sleep(Duration::from_millis(origin_ms)).await; + let collected = client + .select_until(vec![pending], started + Duration::from_millis(budget_ms)) + .await?; + let status = match collected { + Some(result) => { + result.ready?; + "complete" + } + None => "timeout", + }; + Ok(format!("{status}:{}", started.elapsed().as_millis())) +} diff --git a/crates/trusted-server-core/src/auction/orchestrator.rs b/crates/trusted-server-core/src/auction/orchestrator.rs index 1cad8390af..2368eddf81 100644 --- a/crates/trusted-server-core/src/auction/orchestrator.rs +++ b/crates/trusted-server-core/src/auction/orchestrator.rs @@ -29,9 +29,9 @@ use crate::request_signing::RequestSigner; /// /// Created by [`AuctionOrchestrator::dispatch_auction`] and consumed by /// [`AuctionOrchestrator::collect_dispatched_auction`]. Carrying this handle -/// across `pending_origin.wait()` lets origin response and SSP HTTP requests -/// race in Fastly's native layer, enabling TTFB ≈ origin latency rather than -/// TTFB ≈ auction timeout. +/// across the publisher origin wait lets provider operations progress while +/// the page loads. Collection retains available responses and never restarts +/// the original auction budget. pub struct DispatchedAuction { pending_requests: Vec, backend_to_provider: HashMap, @@ -550,13 +550,6 @@ impl AuctionOrchestratorHarness { .to_string(); if let Some(state) = launches.remove(&backend_name) { let elapsed_ms = state.started_at.elapsed().as_millis() as u64; - if remaining_budget_ms(auction_start, context.timeout_ms) == 0 { - responses.push(provider_timeout_response( - state.provider.provider_name(), - elapsed_ms, - )); - continue; - } match state .provider .parse_response_with_state( @@ -1140,10 +1133,8 @@ impl AuctionOrchestrator { })); } - // Reject multi-provider fan-out before any request launches when the - // platform executes `send_async` eagerly (e.g. Cloudflare Workers): - // sequential execution would accrue the sum of provider latencies and - // blow the auction budget before a later `select` could reject it. + // Enforce each adapter's supported provider count before dispatch. + // Worker adapters retain their single-provider collection contract. if provider_names.len() > 1 && !context.services.http_client().supports_concurrent_fanout() { return Err(Report::new(TrustedServerError::Auction { @@ -1383,13 +1374,6 @@ impl AuctionOrchestrator { if let Some(state) = backend_to_provider.remove(&backend_name) { let response_time_ms = state.started_at.elapsed().as_millis() as u64; - if remaining_budget_ms(auction_start, context.timeout_ms) == 0 { - responses.push(provider_timeout_response( - &state.provider_name, - response_time_ms, - )); - continue; - } let provider_context = AuctionContext { settings: context.settings, request: context.request, @@ -1886,11 +1870,8 @@ impl AuctionOrchestrator { return DispatchAuctionOutcome::NotStarted; } - // Mirror run_providers_parallel: reject multi-provider fan-out before - // any request launches when the platform executes `send_async` eagerly - // (e.g. Cloudflare Workers, Spin). Sequential execution would accrue - // the sum of provider latencies before the origin fetch and then fail - // collection with empty bids. + // Mirror the adapter capability guard before dispatch. Worker adapters + // start requests eagerly but retain single-provider collection support. #[cfg(test)] if provider_names.len() > 1 && !context.services.http_client().supports_concurrent_fanout() { @@ -2194,13 +2175,6 @@ impl AuctionOrchestrator { let backend_name = platform_response.backend_name.clone().unwrap_or_default(); if let Some(state) = backend_to_provider.remove(&backend_name) { let response_time_ms = state.started_at.elapsed().as_millis() as u64; - if remaining_budget_ms(auction_start, timeout_ms) == 0 { - responses.push(provider_timeout_response( - &state.provider_name, - response_time_ms, - )); - continue; - } let provider_context = AuctionContext { settings: context.settings, request: &provider_request_context, @@ -2230,13 +2204,6 @@ impl AuctionOrchestrator { } } else if let Some(state) = planned_backend_to_provider.remove(&backend_name) { let response_time_ms = state.started_at.elapsed().as_millis() as u64; - if remaining_budget_ms(auction_start, timeout_ms) == 0 { - responses.push(provider_timeout_response( - state.provider.provider_name(), - response_time_ms, - )); - continue; - } match state .provider .parse_response_with_state( @@ -5564,7 +5531,7 @@ mod tests { fn rejects_multi_provider_fanout_before_launch_on_sequential_platform() { futures::executor::block_on(async { // Arrange: two configured providers on a platform whose HTTP - // client executes send_async eagerly (no concurrent fan-out). + // client is configured without multi-provider collection support. let stub = Arc::new(StubHttpClient::new()); stub.set_concurrent_fanout(false); let stub_for_assertion = Arc::clone(&stub); @@ -5627,7 +5594,7 @@ mod tests { fn dispatch_auction_skips_multi_provider_fanout_on_sequential_platform() { futures::executor::block_on(async { // Arrange: two configured providers on a platform whose HTTP - // client executes send_async eagerly (no concurrent fan-out). + // client is configured without multi-provider collection support. // The initial-page dispatch path must apply the same guard as // run_providers_parallel or the summed provider latency lands // before the origin fetch. @@ -5890,6 +5857,12 @@ mod tests { .await .expect("should collect partial results") }; + assert!( + http.recorded_cache_intents() + .iter() + .all(crate::platform::PlatformCacheIntent::is_bypass), + "should explicitly bypass intermediary caches for personalized auctions" + ); assert_eq!( result.provider_responses.len(), 2, @@ -5919,10 +5892,101 @@ mod tests { } } + #[tokio::test] + async fn planned_split_collection_drains_ready_bid_after_slow_origin_without_mediation() { + let http = Arc::new(StubHttpClient::new()); + http.push_response( + 200, + serde_json::to_vec(&serde_json::json!({ + "seatbid": [{"seat": "example", "bid": [{ + "id": "on-time", "impid": "fictional-slot", "price": 2.0, + "adm": "
fictional
", "w": 300, "h": 250 + }]}] + })) + .expect("should serialize buffered bid"), + ); + http.push_response(204, Vec::new()); + http.push_select_delay(Duration::ZERO); + http.push_select_delay(Duration::from_millis(250)); + let services = build_services_with_backend_and_http_client( + Arc::new(NamingBackend::new(BackendNamingPolicy::Axum)), + Arc::clone(&http) as Arc<_>, + ); + let plan = AuctionPlan::compile(planned_config( + &[ + ("provider-a", RoutingMode::AllEligible), + ("provider-b", RoutingMode::AllEligible), + ], + false, + )) + .expect("should compile providers"); + let launches = Arc::new(AtomicUsize::new(0)); + let mediator = Arc::new(DeadlineRecordingMediator { + launches: Arc::clone(&launches), + budgets: None, + }); + let orchestrator = AuctionOrchestrator::from_plan(Arc::new(plan), Some(mediator)); + let request = planned_request(); + let settings = create_test_settings(); + let inbound = http::Request::new(edgezero_core::body::Body::empty()); + let context = AuctionContext { + settings: &settings, + request: &inbound, + timeout_ms: 10, + transport_timeout_ms: 10, + provider_responses: None, + services: &services, + }; + let DispatchAuctionOutcome::Dispatched(dispatched) = + orchestrator.dispatch_auction(&request, &context).await + else { + panic!("should dispatch providers"); + }; + std::thread::sleep(Duration::from_millis(25)); + let started = Instant::now(); + let result = orchestrator + .collect_dispatched_auction(dispatched, &services, &context) + .await; + assert_eq!( + result.provider_responses.len(), + 2, + "should retain one result for every provider" + ); + assert_eq!( + result + .winning_bids + .get("fictional-slot") + .and_then(|bid| bid.bid_id.as_deref()), + Some("on-time"), + "should retain the available bid after the origin wait" + ); + assert_eq!( + result.provider_responses[1].metadata["error_type"], ERROR_TYPE_TIMEOUT, + "should time out the unfinished sibling" + ); + assert!( + result + .provider_responses + .iter() + .all(|response| response.response_time_ms >= 10), + "should report elapsed dispatch-to-collection time" + ); + assert_eq!( + launches.load(Ordering::Relaxed), + 0, + "should skip mediation after budget exhaustion" + ); + assert!( + started.elapsed() < Duration::from_millis(100), + "should never wait again for an unfinished response" + ); + } + #[tokio::test] async fn planned_split_collection_uses_dispatch_deadline_after_slow_origin() { let http = Arc::new(StubHttpClient::new()); http.push_response(204, Vec::new()); + http.push_select_delay(Duration::from_millis(250)); let services = build_services_with_backend_and_http_client( Arc::new(NamingBackend::new(BackendNamingPolicy::Axum)), http, @@ -5995,8 +6059,8 @@ mod tests { false, )) .expect("should compile planned auction"); - let orchestrator = AuctionOrchestratorHarness::new( - plan, + let orchestrator = AuctionOrchestrator::from_plan( + Arc::new(plan), Some(Arc::new(PendingDeadlineMediator(mediator_timeout_ms))), ); let request = planned_request(); diff --git a/crates/trusted-server-core/src/auction/provider.rs b/crates/trusted-server-core/src/auction/provider.rs index 62b2ddc8fc..887cde906f 100644 --- a/crates/trusted-server-core/src/auction/provider.rs +++ b/crates/trusted-server-core/src/auction/provider.rs @@ -381,7 +381,9 @@ impl GenericOpenRtbProvider { } let pending = services .http_client() - .send_async(PlatformHttpRequest::new(outbound, backend_name.clone())) + .send_async( + PlatformHttpRequest::new(outbound, backend_name.clone()).with_cache_bypass(), + ) .await .change_context(TrustedServerError::Auction { message: format!("Provider {} request launch failed", self.provider_name()), diff --git a/crates/trusted-server-core/src/integrations/adserver_mock.rs b/crates/trusted-server-core/src/integrations/adserver_mock.rs index 640574e031..efc789f3aa 100644 --- a/crates/trusted-server-core/src/integrations/adserver_mock.rs +++ b/crates/trusted-server-core/src/integrations/adserver_mock.rs @@ -514,7 +514,7 @@ impl AuctionProvider for AdServerMockProvider { let pending = context .services .http_client() - .send_async(PlatformHttpRequest::new(req, backend_name)) + .send_async(PlatformHttpRequest::new(req, backend_name).with_cache_bypass()) .await .change_context(TrustedServerError::Auction { message: "Failed to send mediation request".to_string(), diff --git a/crates/trusted-server-core/src/integrations/aps.rs b/crates/trusted-server-core/src/integrations/aps.rs index 4f45fbeb9c..4ef306e0b0 100644 --- a/crates/trusted-server-core/src/integrations/aps.rs +++ b/crates/trusted-server-core/src/integrations/aps.rs @@ -1708,7 +1708,7 @@ impl AuctionProvider for ApsAuctionProvider { let pending = context .services .http_client() - .send_async(PlatformHttpRequest::new(outbound_request, backend)) + .send_async(PlatformHttpRequest::new(outbound_request, backend).with_cache_bypass()) .await .change_context(TrustedServerError::Auction { message: "Failed to send APS request".to_string(), diff --git a/crates/trusted-server-core/src/integrations/prebid.rs b/crates/trusted-server-core/src/integrations/prebid.rs index f1d1d0b509..9cb51799c7 100644 --- a/crates/trusted-server-core/src/integrations/prebid.rs +++ b/crates/trusted-server-core/src/integrations/prebid.rs @@ -3480,7 +3480,7 @@ impl AuctionProvider for PrebidAuctionProvider { let pending = context .services .http_client() - .send_async(PlatformHttpRequest::new(pbs_req, backend_name)) + .send_async(PlatformHttpRequest::new(pbs_req, backend_name).with_cache_bypass()) .await .change_context(TrustedServerError::Prebid { message: "Failed to send async request to Prebid Server".to_string(), diff --git a/crates/trusted-server-core/src/platform/http.rs b/crates/trusted-server-core/src/platform/http.rs index 7a52ceae3c..1d5b9c338a 100644 --- a/crates/trusted-server-core/src/platform/http.rs +++ b/crates/trusted-server-core/src/platform/http.rs @@ -349,10 +349,11 @@ pub trait PlatformHttpClient: Send + Sync { request: PlatformHttpRequest, ) -> Result>; - /// Prepares an upstream request without waiting for its response. + /// Starts an upstream request without waiting for its response. /// - /// Fastly and Axum start network I/O at dispatch. Worker runtimes may - /// defer execution until collection so a deadline can bound all I/O. + /// Adapters retain ownership of the request operation so it can progress + /// while the publisher origin is fetched. Dropping its pending handle + /// cancels or abandons unfinished work according to the runtime contract. /// /// # Errors /// @@ -415,19 +416,23 @@ pub trait PlatformHttpClient: Send + Sync { pending_requests: Vec, ) -> Result>; - /// Collects one complete buffered response before an absolute deadline. + /// Collects an available response, waiting only until an absolute deadline. /// - /// Returns `Ok(None)` on expiry, relinquishing every supplied pending - /// request. Implementations must bound both header waits and body reads, - /// reject completions observed after the cutoff, and cancel outstanding - /// work where the runtime supports cancellation. Fastly can abandon pending - /// backend requests but cannot cancel them with the installed SDK. + /// On expiry, implementations take one nonblocking snapshot of complete + /// available responses before relinquishing unfinished requests. This + /// preserves buffered bids when a publisher origin delays collection. + /// Subsequent calls may drain only the completed snapshot. The deadline + /// bounds network waiting, not eligibility by an upstream arrival timestamp. + /// Returns `Ok(None)` when no complete response remains. Both header waits + /// and body reads must be bounded; unfinished work is canceled or abandoned. + /// Fastly cannot cancel backend requests with the installed SDK. /// Stream-marked requests require the separate streaming wait contract. /// /// # Performance /// /// Complete response bodies are buffered within adapter-specific limits. - /// Deadline expiry drops retained partial responses and pending handles. + /// Deadline expiry drops incomplete bodies and pending handles after the + /// nonblocking snapshot; complete buffered responses remain collectible. /// /// # Examples /// diff --git a/crates/trusted-server-core/src/platform/test_support.rs b/crates/trusted-server-core/src/platform/test_support.rs index 20413a0ef2..1b74b2267f 100644 --- a/crates/trusted-server-core/src/platform/test_support.rs +++ b/crates/trusted-server-core/src/platform/test_support.rs @@ -725,9 +725,6 @@ impl PlatformHttpClient for StubHttpClient { deadline: Instant, ) -> Result, Report> { let remaining = deadline.saturating_duration_since(Instant::now()); - if remaining.is_zero() { - return Ok(None); - } let delay = self .select_delays .lock() @@ -735,6 +732,12 @@ impl PlatformHttpClient for StubHttpClient { .front() .copied() .unwrap_or_default(); + if delay.is_zero() { + return self.select(pending_requests).await.map(Some); + } + if remaining.is_zero() { + return Ok(None); + } if delay >= remaining { self.select_delays .lock() diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index 4279b5dad1..e5ac30009e 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -5146,8 +5146,8 @@ pub async fn handle_publisher_request( } } - // SSP requests are already racing through the platform HTTP client, so - // origin TTFB tracks origin latency rather than the auction timeout. + // Provider operations are already running while the origin loads. Collection + // retains buffered results, then waits only for the original remaining budget. let origin_result = if let Some(pending) = pending_origin { services.http_client().wait(pending).await } else { diff --git a/docs/guide/ad-serving.md b/docs/guide/ad-serving.md index 171764bc18..7cd3b03b78 100644 --- a/docs/guide/ad-serving.md +++ b/docs/guide/ad-serving.md @@ -68,8 +68,11 @@ assets at runtime. end-to-end checks. - Confirm provider authorization and test inventory with each upstream before enabling traffic. -- Treat provider timeout values as logical budgets; no adapter currently - promises an abortable provider-wide wall-clock deadline. +- Collection stops network waiting at the auction deadline measured from dispatch. + Complete responses already available remain eligible; unfinished providers time + out and mediation is skipped when no budget remains. Fastly abandons unfinished + requests; other adapters cancel owned request tasks. See + [Timeout behavior](/guide/auction-orchestration#timeout-behavior). See also [Creative processing](/guide/creative-processing) and the [Integrations overview](/guide/integrations-overview). diff --git a/docs/guide/architecture.md b/docs/guide/architecture.md index a2ac81a0e6..fb02fe2e68 100644 --- a/docs/guide/architecture.md +++ b/docs/guide/architecture.md @@ -171,7 +171,8 @@ The workspace has multiple WASM runtimes with runtime-specific SDKs. Use target- Fastly and Axum support concurrent auction provider fan-out. Cloudflare and Spin currently accept at most one provider in an enabled auction. Every adapter runs target-aware fan-out and backend-name checks at startup. Auction collection uses -an absolute cutoff across header waits and complete response bodies. Fastly +an absolute cutoff for network waiting across headers and complete response bodies. +A nonblocking snapshot retains complete results after a slow origin. Fastly abandons pending backend requests because its SDK cannot cancel them; other adapters cancel their owned operations at cutoff. See [Auction timeout behavior](/guide/auction-orchestration#timeout-behavior) for diff --git a/docs/guide/auction-orchestration.md b/docs/guide/auction-orchestration.md index b80d27eae9..e220428061 100644 --- a/docs/guide/auction-orchestration.md +++ b/docs/guide/auction-orchestration.md @@ -814,22 +814,32 @@ Until that callback lands, startup remains the mandatory target-aware gate. ### Timeout behavior -Provider response collection uses one absolute deadline measured from auction -dispatch. It includes waiting for headers and buffering the complete upstream -body. When the deadline expires, outstanding providers are recorded as timeouts; -completed bids collected before the cutoff remain available for local ranking. -Collection after a slow origin response does not restart the clock, and late -responses are not admitted. A mediator starts only with remaining budget and -uses the earlier of the auction cutoff and its own logical timeout. - -Fastly limits retained response payloads to 10 MiB across the collector. A provider -whose next chunk would exceed that limit fails while its siblings remain eligible. -Fastly abandons unfinished backend requests and closes acquired response bodies; +Auction requests start at dispatch and progress while the publisher origin loads. +Response collection waits for headers and complete upstream bodies only until the +original absolute deadline. At expiry, one nonblocking snapshot preserves complete +responses already available; unfinished providers become timeouts. A slow origin +never restarts the budget or causes an already-buffered bid to be discarded. +Fastly has no upstream arrival timestamp, so this is a bound on network waiting, +not bid eligibility based on an exact server-side arrival time. A mediator starts +only with remaining budget and uses the earlier of the auction cutoff and its own +logical timeout. + +Personalized auction and mediator requests explicitly bypass intermediary caches. +Fastly uses the SDK's direct pending-request handles and body readiness handles in +a timed wait; it neither polls headers every millisecond nor waits on an empty +handle list. Retained partial payloads share a 10 MiB budget. Under memory pressure, +the largest partial response is rejected with its own backend attribution so a +large stalled provider cannot make a small healthy sibling take the failure. +Completed payloads move directly into the platform body without a host-buffer +round trip. This cap covers retained payloads, not every allocator or host buffer. + +On expiry Fastly closes acquired response bodies and abandons unfinished requests; the installed SDK has no pending-request cancellation API. Axum aborts outstanding -request tasks. Cloudflare aborts its fetch through an AbortController, and Spin -drops cancelable WASI operations. Cloudflare and Spin retain their single-provider -restriction and defer outbound request execution until collection so the entire -request can be bounded. +request tasks. Cloudflare and Spin start owned background tasks at dispatch and +retain abort handles until response collection completes. Cloudflare also aborts +its fetch through an AbortController. Both workers retain their single-provider +restriction. Dropping a pending handle cancels queued or running guest work; +upstream side effects that already happened cannot be undone. Backend first-byte and between-byte timeouts remain transport safeguards rather than total deadlines. Timer granularity and runtime scheduling can add a small diff --git a/docs/guide/auction-testing.md b/docs/guide/auction-testing.md index f15a26789d..87a0eb6895 100644 --- a/docs/guide/auction-testing.md +++ b/docs/guide/auction-testing.md @@ -75,8 +75,11 @@ Core tests cover split dispatch/collection and synchronous execution, pending provider attribution, preserved on-time bids, late collection after an origin wait, and exhausted-budget mediation. Axum loopback tests separately delay headers and trickle bodies, and verify outstanding task cancellation. Worker -adapter tests use controlled futures and timers, with WASM compilation checking -the actual timer and cancellation APIs. Fastly also has Viceroy network fixtures +adapter tests use controlled futures and timers. CI also runs the actual worker +HTTP clients under local workerd and Spin 4.1, covering eager dispatch, buffered +bids after a slow origin, delayed headers, and trickling bodies. Run +`scripts/test-worker-auction-deadlines.py --help` for the same local fixtures. +Fastly also has Viceroy network fixtures for stalled headers, trickling bodies, bodyless responses, and competing bidders; run its local origin harness after building the adapter test binary: @@ -89,4 +92,4 @@ python3 crates/trusted-server-adapter-fastly/tests/deadline_origin.py \ Use the adapter test artifact printed by Cargo in place of ``. The harness binds a local ephemeral port and runs the four network tests excluded from the -standard suite. +standard suite; CI runs the harness automatically. diff --git a/docs/guide/configuration.md b/docs/guide/configuration.md index 343fbf1225..af9e82dd4f 100644 --- a/docs/guide/configuration.md +++ b/docs/guide/configuration.md @@ -2007,15 +2007,15 @@ provider or bidder route. ### `[auction]` -| Field | Type | Default | Description | -| ---------------------- | ------- | ------------------ | -------------------------------------------------------------- | -| `enabled` | Boolean | `false` | Enable the auction orchestrator | -| `sanitize_creatives` | Boolean | `false` | Strip executable markup from winning-bid `adm` before delivery | -| `rewrite_creatives` | Boolean | `true` | Rewrite winning-bid `adm` through first-party endpoints | -| `timeout_ms` | Integer | `2000` | Logical auction budget in milliseconds | -| `mediator` | String | `None` | Optional separate `adserver_mock` mediator | -| `creative_store` | String | `"creative_store"` | Deprecated; creatives are delivered inline | -| `allowed_context_keys` | Array | `[]` | Request context keys admitted into the auction | +| Field | Type | Default | Description | +| ---------------------- | ------- | ------------------ | ----------------------------------------------------------------------- | +| `enabled` | Boolean | `false` | Enable the auction orchestrator | +| `sanitize_creatives` | Boolean | `false` | Strip executable markup from winning-bid `adm` before delivery | +| `rewrite_creatives` | Boolean | `true` | Rewrite winning-bid `adm` through first-party endpoints | +| `timeout_ms` | Integer | `2000` | Deadline for additional auction network waiting, measured from dispatch | +| `mediator` | String | `None` | Optional separate `adserver_mock` mediator | +| `creative_store` | String | `"creative_store"` | Deprecated; creatives are delivered inline | +| `allowed_context_keys` | Array | `[]` | Request context keys admitted into the auction | Creative markup delivered by `POST /auction` and the publisher SSAT/page-bids path is processed by two independent passes. With `sanitize_creatives = true` @@ -2239,10 +2239,11 @@ push as proof that a Cloudflare or Spin multi-provider plan can start. ### Deadline behavior -Auction response collection stops at an absolute deadline measured from dispatch, -including header waits and complete response bodies. Outstanding providers become -timeouts, previously collected bids remain eligible, and mediation cannot start -with an exhausted budget. Backend timeout rounding does not extend that cutoff. +Auction response collection waits only until an absolute deadline measured from +dispatch, including headers and complete response bodies. One nonblocking snapshot +retains complete available responses after a slow origin; unfinished providers +become timeouts and mediation cannot start with an exhausted budget. Backend +timeout rounding does not extend network waiting. Fastly abandons pending requests; Axum, Cloudflare, and Spin cancel outstanding operations where supported. Local processing and delivery can finish after the network cutoff. See [Timeout behavior](/guide/auction-orchestration#timeout-behavior) diff --git a/docs/superpowers/plans/2026-10-22-absolute-auction-deadlines.md b/docs/superpowers/plans/2026-10-22-absolute-auction-deadlines.md index 7c9c45bcaf..2e3a15dd1e 100644 --- a/docs/superpowers/plans/2026-10-22-absolute-auction-deadlines.md +++ b/docs/superpowers/plans/2026-10-22-absolute-auction-deadlines.md @@ -1,87 +1,54 @@ -# Absolute auction deadline implementation plan - -> **For agentic workers:** Use superpowers:test-driven-development and verification-before-completion. Implement independent adapter scopes in parallel; root owns the collection contract and orchestrator. Independently review the draft PR after creation. - -**Goal:** Resolve #1015 by ending provider collection at the deadline measured from dispatch, recording outstanding providers as timeouts and avoiding mediation after budget exhaustion. - -**Architecture:** Add `PlatformHttpClient::select_until(pending, web_time::Instant)` returning `Result, Report>`. `None` means deadline expiration and relinquishes all pending state. The default fails closed as unsupported; it must never silently delegate to unbounded selection. Add `wait_until` for buffered mediator requests. Preserve existing unbounded APIs for unrelated publisher/proxy traffic. - -**Tech stack:** Rust, web_time, platform runtime timers, Fastly nonblocking pending polls and readiness-gated body reads. - -## Design decisions - -- Relative backend timeouts alone cannot enforce a total budget against trickling bodies. An outer generic async timer cannot interrupt Fastly's synchronous hostcalls. Use adapter-specific bounded collection of complete bodies. -- Axum races its existing whole-body task selection against a tokio timer; timeout drops its abort-on-drop task handles. -- Cloudflare and Spin retain their single-provider capability restrictions, but return owned pending response futures instead of eagerly awaiting complete responses at dispatch. Bounded selection races the entire request/body future against worker Delay or Spin SDK 7's WASI p3 sleep. Cloudflare aborts its fetch through an AbortController on drop; Spin drops cancelable component subtasks and body readers. -- Fastly maintains pending-header and buffered-body states together. Poll headers without blocking; read at most one ready body chunk per provider per sweep. Never use a network-backed read_to_end. Bound readiness waits by the original deadline; preserve remaining collector states between completions. The SDK does not expose high-level pending wait handles, so brief bounded polling may be needed for header readiness. Use the locked low-level ABI only for readiness-gated body I/O, with safety comments and a justified direct dependency. -- Fastly cannot cancel a still-pending backend operation with the locked public API. On expiration, stop waiting and drop guest state; close acquired bodies. Do not claim network cancellation that the SDK cannot provide. -- Apply bounded collection to split dispatch/collect, synchronous planned auctions, legacy compatibility tests, and mediator waits. Recheck the absolute clock before accepting a completion. Retain successful responses collected before the cutoff and attribute every outstanding launch exactly once. -- No expansion of multi-provider support for worker runtimes, no unrelated proxy timeout changes, and no new operator settings. - -## Tasks - -- [x] Run the existing deadline test baseline in the isolated worktree. -- [x] Add failing regressions for expired collection, delayed/trickling responses, timeout attribution, partial success, and exhausted-budget mediation. -- [x] Implement and test the platform bounded-collection contract and safe unsupported default. -- [x] Implement each adapter's complete-response deadline and cancellation/abandonment behavior; run meaningful adapter tests. -- [x] Integrate all auction collection and mediator paths, replacing the former capability-dependent acceptance of late responses. -- [x] Document semantics and platform limitations, including deferred worker-runtime requests and Fastly abandonment. -- [x] Run every AGENTS.md CI gate, native core tests, and the optimized Fastly build; record exact outcomes and environmental limitations. -- [x] Create a draft PR with .claude/agents/pr-creator.md and Closes #1015. -- [x] Independently review the PR, address validated findings, rerun affected checks, and push follow-up fixes. - -## Plan review - -Independent review approved the architecture with two requirements: mediator cutoffs use the earlier of the original auction deadline and the mediator's own logical budget; deadline expiry and transport/polling failure retain distinct attribution. Adapter regressions must cover expired-but-ready inputs, full-body timeouts, fairness, cancellation/abandonment, body limits, and bodyless HTTP semantics. - -The original deadline baseline passed. A new regression failed against the original collector because current adapters accepted delayed provider responses after the auction deadline. - -## Implementation review - -A separate requirements review found no required production corrections. It -suggested a tighter mediator-budget regression, which now verifies the mediator -cutoff against a longer auction budget and rounded Fastly transport timeout. - -Review also found two Fastly collector edges before PR creation: a partially -collected response could lose its original HEAD method hint, and concurrent -partial bodies could multiply the old single-response memory allowance. Both -have red/green regressions. Collector state now retains method metadata and -shares a 10 MiB payload budget; exceeding providers fail with attribution while -siblings remain collectible. - -## Verification before PR - -All AGENTS.md CI gates passed: formatting, eight target-specific clippy aliases, -five adapter test aliases (including reusable Fastly sandboxes), 17 parity -tests, native build-digest tests/lint, JS tests/build/format, documentation and -outside-doc Markdown formatting, WASM clock lint, and documentation proposal -script checks. The native core suite passed 2,986 unit tests; Viceroy passed -2,980 core tests and 229 Fastly adapter tests. Four explicitly excluded Fastly -network tests passed separately against the local delayed/trickling origin. -The optimized Fastly WASM build passed. - -A stricter mediator test initially supplied an inconsistent transport timeout -in its fixture; after matching the backend's canonical timeout, both native -and Viceroy full core suites passed. Axum's new fixture also now checks its -request read count to satisfy clippy. - -Verification used Rust 1.98.1 and Viceroy 0.21.0. Loopback fixtures required -local socket access. Cloudflare and Spin timer/cancellation paths were checked -for their WASM targets; deterministic native tests exercise their actual -arbitration helpers, rather than a deployed worker runtime. - -The GitHub token cannot read project-board fields (`read:project` is absent), -so the issue's board status remains unchanged. - -## Draft PR and independent review - -Draft PR #1433 targets main, is assigned to the author, and closes #1015. A fresh -independent reviewer inspected the complete `f9281917..d4f7018` implementation, -including locked SDK behavior, all four adapters, core collection and mediation, -Fastly buffering/readiness, the network harness, and documentation. It found no -correctness or regression issues requiring changes and independently ran all ten -Cloudflare/Spin deadline tests successfully. - -The reviewer confirmed the documented coverage gap: actual deployed worker timer -and cancellation behavior has not received runtime integration coverage. The -review does not establish a production defect. No review findings remain open. +# Absolute auction collection deadlines + +## Contract + +`PlatformHttpClient::select_until` limits additional network waiting to one +absolute deadline measured from auction dispatch. At expiry the adapter takes +one nonblocking snapshot of complete available responses, drops unfinished +operations, and permits the caller to drain only that completed snapshot. +This retains buffered bids when the publisher origin delayed collection. It +is not an upstream arrival-timestamp admission rule. + +Every configured provider receives exactly one attributed outcome. Completed +bids remain eligible for local ranking. Unfinished responses become timeouts; +polling or transport errors remain transport failures. Mediation cannot launch +without budget and its wait uses the earlier of the auction deadline and its +own logical budget, independently of rounded transport timeouts. + +## Adapter implementation + +- Axum starts whole-response tasks at dispatch. Selection races their completion + against the remaining deadline. The expired snapshot retains only finished + tasks; dropping other handles aborts them. +- Cloudflare and Spin spawn cancelable full-response operations at dispatch and + retain buffered results in channels. Selection probes the channel before + waiting against the runtime timer. Pending ownership includes an abort handle; + Cloudflare's fetch also owns an AbortController. Single-provider restrictions + remain unchanged. +- Fastly explicitly bypasses caching for personalized auction requests, allowing + conversion to the SDK's direct pending-request handle. Timed readiness waits + contain real request/body handles. Header polling and body reads are + nonblocking; ready bodies advance fairly. Expired collection caches one + nonblocking snapshot and never waits for more bytes. The SDK cannot cancel + pending backend requests, so expiration abandons them and closes acquired bodies. +- Fastly partial response payloads share a 10 MiB limit. Capacity pressure rejects + the largest partial response with backend attribution. Completed vectors move + directly into platform bodies. The limit does not include allocator overhead + or runtime-owned host buffers. + +## Validation + +Regression coverage must exercise production split dispatch/collection, complete +buffered bids after a slow origin, provider count/order/elapsed time and timeout +attribution, exhausted-budget mediation, and the mediator's independent budget. +Adapters additionally cover delayed headers, trickling bodies, cancellation or +abandonment, bodyless response semantics, fair collection, and memory pressure. + +The Fastly slow-origin harness runs automatically in CI. Worker runtime fixtures +use the actual HTTP adapters under local workerd and Spin to verify eager dispatch, +complete buffering, deadline waits, and cancellation. Native helper tests and +WASM compilation supplement these runtime cases. + +Local parsing, ranking, HTML transformation, and delivery remain outside the +network-wait cutoff. Runtime scheduling and timer granularity can add a small +overshoot; already-triggered upstream side effects cannot be undone. diff --git a/scripts/test-worker-auction-deadlines.py b/scripts/test-worker-auction-deadlines.py new file mode 100644 index 0000000000..9f53cdbb49 --- /dev/null +++ b/scripts/test-worker-auction-deadlines.py @@ -0,0 +1,126 @@ +#!/usr/bin/env python3 +"""Run production HTTP clients against delayed local origins in worker runtimes.""" + +import argparse +import http.server +import pathlib +import socket +import subprocess +import tempfile +import threading +import time +import urllib.request + + +class Origin(http.server.BaseHTTPRequestHandler): + events = {} + + def do_GET(self): + self.events.setdefault(self.path, threading.Event()).set() + if self.path == "/headers": + time.sleep(0.7) + self.send_response(200) + self.send_header("Content-Length", "4") + self.end_headers() + try: + if self.path == "/body": + self.wfile.write(b"a") + self.wfile.flush() + time.sleep(0.7) + self.wfile.write(b"bcd") + else: + self.wfile.write(b"okay") + except (BrokenPipeError, ConnectionResetError): + # Deadline cancellation deliberately disconnects this slow origin. + pass + + def log_message(self, *_): + pass + + +def free_port(): + with socket.socket() as connection: + connection.bind(("127.0.0.1", 0)) + return connection.getsockname()[1] + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("adapter", choices=["cloudflare", "spin"]) + parser.add_argument("--runtime", required=True) + parser.add_argument("--wasm-bindgen", default="wasm-bindgen") + parser.add_argument("--target-dir", type=pathlib.Path) + args = parser.parse_args() + root = pathlib.Path(__file__).resolve().parent.parent + artifact = (args.target_dir or root / "target") / ("wasm32-unknown-unknown" if args.adapter == "cloudflare" else "wasm32-wasip1") / "debug/examples/deadline_runtime.wasm" + origin = http.server.ThreadingHTTPServer(("127.0.0.1", 0), Origin) + thread = threading.Thread(target=origin.serve_forever, daemon=True) + thread.start() + port = free_port() + with tempfile.TemporaryDirectory(prefix="ts-worker-deadline-") as temporary: + workspace = pathlib.Path(temporary) + if args.adapter == "cloudflare": + subprocess.run([args.wasm_bindgen, str(artifact), "--target", "web", "--out-dir", str(workspace), "--out-name", "fixture"], check=True) + (workspace / "index.js").write_text('import module from "./fixture_bg.wasm";\nimport {initSync, fetch} from "./fixture.js";\ninitSync({module});\nexport default {fetch};\n') + (workspace / "wrangler.toml").write_text('name = "deadline-fixture"\nmain = "index.js"\ncompatibility_date = "2026-04-01"\n') + command = [args.runtime, "dev", "--local", "--port", str(port), "--ip", "127.0.0.1", "--config", str(workspace / "wrangler.toml")] + else: + (workspace / "spin.toml").write_text(f'''spin_manifest_version = 2 +[application] +name = "deadline-fixture" +version = "0.1.0" +[[trigger.http]] +route = "/..." +component = "fixture" +[component.fixture] +source = "{artifact}" +allowed_outbound_hosts = ["http://127.0.0.1:*"] +''') + command = [args.runtime, "up", "--from", str(workspace / "spin.toml"), "--listen", f"127.0.0.1:{port}"] + with (workspace / "runtime.log").open("w+") as log: + process = subprocess.Popen(command, cwd=workspace, stdout=log, stderr=subprocess.STDOUT) + try: + for _ in range(300): + if process.poll() is not None: + raise RuntimeError("worker exited during startup") + try: + with socket.create_connection(("127.0.0.1", port), timeout=0.1): + break + except OSError: + time.sleep(0.1) + else: + raise RuntimeError("worker did not listen") + cases = [ + ("/fast", 80, 200, "complete", 500), + ("/headers", 80, 0, "timeout", 350), + ("/body", 80, 0, "timeout", 350), + ("/headers", 80, 200, "timeout", 500), + ] + for path, budget, origin_delay, expected, maximum in cases: + Origin.events[path] = threading.Event() + upstream = f"http://127.0.0.1:{origin.server_port}{path}" + uri = f"http://127.0.0.1:{port}/?upstream={upstream}&budget_ms={budget}&origin_ms={origin_delay}" + with urllib.request.urlopen(uri, timeout=5) as response: + result = response.read().decode() + status, elapsed = result.split(":") + assert status == expected, (path, expected, result) + assert int(elapsed) < maximum, (path, maximum, result) + assert Origin.events[path].is_set(), (path, "request should start at dispatch") + print(f"PASS {args.adapter} {path} budget={budget} origin={origin_delay}: {result}", flush=True) + except BaseException: + log.flush() + log.seek(0) + print(log.read()) + raise + finally: + process.terminate() + try: + process.wait(timeout=10) + except subprocess.TimeoutExpired: + process.kill() + process.wait() + origin.shutdown() + + +if __name__ == "__main__": + main() From a0b592cd4b5b9e265725f3c532d206ba557b3500 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Thu, 22 Oct 2026 19:29:53 +0530 Subject: [PATCH 5/9] Run deadline regression harnesses with Node.js --- .github/workflows/test.yml | 2 +- .../workflows/worker-auction-deadlines.yml | 6 +- .../src/platform/deadline.rs | 10 +- .../tests/deadline-origin.mjs | 160 ++++++++++ .../tests/deadline_origin.py | 95 ------ docs/guide/auction-testing.md | 6 +- scripts/test-worker-auction-deadlines.mjs | 297 ++++++++++++++++++ scripts/test-worker-auction-deadlines.py | 126 -------- 8 files changed, 469 insertions(+), 233 deletions(-) create mode 100644 crates/trusted-server-adapter-fastly/tests/deadline-origin.mjs delete mode 100644 crates/trusted-server-adapter-fastly/tests/deadline_origin.py create mode 100644 scripts/test-worker-auction-deadlines.mjs delete mode 100644 scripts/test-worker-auction-deadlines.py diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index c885b66d4e..61d88feef1 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -54,7 +54,7 @@ jobs: run: cargo test-fastly - name: Run Fastly absolute deadline slow-network tests - run: python3 crates/trusted-server-adapter-fastly/tests/deadline_origin.py "$(command -v viceroy)" + run: node crates/trusted-server-adapter-fastly/tests/deadline-origin.mjs "$(command -v viceroy)" - name: Run tests (reusable-sandbox feature) run: cargo test-fastly-reuse diff --git a/.github/workflows/worker-auction-deadlines.yml b/.github/workflows/worker-auction-deadlines.yml index 5af9f7d220..41b459230e 100644 --- a/.github/workflows/worker-auction-deadlines.yml +++ b/.github/workflows/worker-auction-deadlines.yml @@ -64,10 +64,10 @@ jobs: - name: Exercise delayed headers, trickling bodies, and slow publisher work shell: bash env: - WRANGLER_SEND_METRICS: "false" + WRANGLER_SEND_METRICS: 'false' run: | if [ "${{ matrix.adapter }}" = cloudflare ]; then - python3 scripts/test-worker-auction-deadlines.py cloudflare --runtime "$(command -v wrangler)" + node scripts/test-worker-auction-deadlines.mjs cloudflare --runtime "$(command -v wrangler)" else - python3 scripts/test-worker-auction-deadlines.py spin --runtime "$(command -v spin)" + node scripts/test-worker-auction-deadlines.mjs spin --runtime "$(command -v spin)" fi diff --git a/crates/trusted-server-adapter-fastly/src/platform/deadline.rs b/crates/trusted-server-adapter-fastly/src/platform/deadline.rs index bf3081bb77..de0af14b60 100644 --- a/crates/trusted-server-adapter-fastly/src/platform/deadline.rs +++ b/crates/trusted-server-adapter-fastly/src/platform/deadline.rs @@ -707,7 +707,7 @@ mod tests { } #[test] - #[ignore = "requires the local origin started by tests/deadline_origin.py"] + #[ignore = "requires the local origin started by tests/deadline-origin.mjs"] fn network_slow_origin_preserves_ready_bid_without_waiting_for_stalled_sibling() { let fast = network_pending("fast", "GET"); let stalled = network_pending("stalled", "GET"); @@ -734,7 +734,7 @@ mod tests { } #[test] - #[ignore = "requires the local origin started by tests/deadline_origin.py"] + #[ignore = "requires the local origin started by tests/deadline-origin.mjs"] fn network_stalled_headers_stop_at_deadline() { let pending = network_pending("stalled", "GET"); let started = Instant::now(); @@ -754,7 +754,7 @@ mod tests { } #[test] - #[ignore = "requires the local origin started by tests/deadline_origin.py"] + #[ignore = "requires the local origin started by tests/deadline-origin.mjs"] fn network_trickling_body_stops_at_deadline() { let pending = network_pending("trickle", "GET"); let started = Instant::now(); @@ -774,7 +774,7 @@ mod tests { } #[test] - #[ignore = "requires the local origin started by tests/deadline_origin.py"] + #[ignore = "requires the local origin started by tests/deadline-origin.mjs"] fn network_fast_sibling_completes_while_another_body_trickles() { let slow = network_pending("trickle", "GET"); let fast = network_pending("fast", "GET"); @@ -801,7 +801,7 @@ mod tests { } #[test] - #[ignore = "requires the local origin started by tests/deadline_origin.py"] + #[ignore = "requires the local origin started by tests/deadline-origin.mjs"] fn network_head_skips_the_body_and_its_declared_length_limit() { let pending = network_pending("head", "HEAD"); let result = futures::executor::block_on( diff --git a/crates/trusted-server-adapter-fastly/tests/deadline-origin.mjs b/crates/trusted-server-adapter-fastly/tests/deadline-origin.mjs new file mode 100644 index 0000000000..97a27005d4 --- /dev/null +++ b/crates/trusted-server-adapter-fastly/tests/deadline-origin.mjs @@ -0,0 +1,160 @@ +// Run the actual Fastly deadline tests against a deliberately slow local origin. +// Usage: node tests/deadline-origin.mjs /path/to/viceroy [test-binary.wasm] +import assert from 'node:assert/strict' +import { execFile, spawn } from 'node:child_process' +import { once } from 'node:events' +import { existsSync } from 'node:fs' +import { mkdtemp, rm, writeFile } from 'node:fs/promises' +import { createServer } from 'node:http' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { setTimeout as delay } from 'node:timers/promises' +import { promisify } from 'node:util' + +const execute = promisify(execFile) +const interruption = new AbortController() +process.once('SIGINT', () => interruption.abort()) +process.once('SIGTERM', () => interruption.abort()) + +async function testArtifact() { + const { stdout } = await execute( + 'cargo', + [ + 'test', + '-p', + 'trusted-server-adapter-fastly', + '--target', + 'wasm32-wasip1', + '--bin', + 'trusted-server-adapter-fastly', + '--no-run', + '--message-format=json', + ], + { + maxBuffer: 16 * 1024 * 1024, + signal: interruption.signal, + killSignal: 'SIGKILL', + } + ) + const binaries = stdout + .split('\n') + .filter((line) => line.startsWith('{')) + .map((line) => JSON.parse(line)) + .filter( + (item) => + item.reason === 'compiler-artifact' && + item.profile?.test && + item.target?.name === 'trusted-server-adapter-fastly' && + item.executable + ) + .map((item) => item.executable) + assert.equal( + binaries.length, + 1, + 'should discover exactly one Fastly test binary' + ) + return binaries[0] +} + +async function serveOrigin(request, response) { + const cancellation = new AbortController() + response.once('close', () => cancellation.abort()) + const pause = (milliseconds) => + delay(milliseconds, undefined, { signal: cancellation.signal }) + try { + if (request.method === 'HEAD') { + response.writeHead(200, { 'Content-Length': 20 * 1024 * 1024 }) + response.end() + return + } + if (request.url === '/stalled') await pause(2000) + if (request.url === '/trickle') { + response.writeHead(200, { 'Content-Length': 100 }) + for (let index = 0; index < 100; index += 1) { + response.write('x') + await pause(10) + } + response.end() + return + } + await pause(30) + response.writeHead(200, { 'Content-Length': 4 }) + response.end('fast') + } catch (error) { + // Deadline expiry deliberately disconnects the slow origin and cancels its timer. + if (error.name !== 'AbortError') throw error + } +} + +async function main() { + const [viceroy, suppliedBinary, ...extra] = process.argv.slice(2) + assert( + viceroy && extra.length === 0, + 'usage: node deadline-origin.mjs /path/to/viceroy [test-binary.wasm]' + ) + const binary = suppliedBinary ?? (await testArtifact()) + let originError + const server = createServer((request, response) => { + void serveOrigin(request, response).catch((error) => { + originError = error + response.destroy() + }) + }) + server.listen(0, '127.0.0.1') + await once(server, 'listening') + const directory = await mkdtemp(join(tmpdir(), 'fastly-deadline-')) + try { + const config = join(directory, 'fastly.toml') + await writeFile( + config, + '[local_server.backends.deadline_origin]\n' + + `url = "http://127.0.0.1:${server.address().port}"\n` + ) + const environment = { ...process.env } + if (existsSync('/etc/ssl/cert.pem')) + environment.SSL_CERT_FILE ??= '/etc/ssl/cert.pem' + const child = spawn( + viceroy, + [ + 'run', + '-C', + config, + binary, + 'platform::deadline::tests::network_', + '--ignored', + '--nocapture', + ], + { + env: environment, + stdio: 'inherit', + signal: interruption.signal, + killSignal: 'SIGKILL', + } + ) + // Clear the watchdog even if spawning fails: Node's built-in child timeout + // can otherwise keep the harness alive after an ENOENT startup error. + const watchdog = setTimeout(() => child.kill('SIGKILL'), 120_000) + watchdog.unref() + try { + const [code] = await once(child, 'close') + if (originError) throw originError + return code ?? 1 + } finally { + clearTimeout(watchdog) + } + } finally { + const closed = new Promise((resolve) => server.close(resolve)) + server.closeAllConnections() + await closed + await rm(directory, { recursive: true, force: true }) + } +} + +main() + .then((code) => { + process.exitCode = code + }) + .catch((error) => { + console.error(error) + process.exitCode = 1 + }) diff --git a/crates/trusted-server-adapter-fastly/tests/deadline_origin.py b/crates/trusted-server-adapter-fastly/tests/deadline_origin.py deleted file mode 100644 index 9cf7c9c5a1..0000000000 --- a/crates/trusted-server-adapter-fastly/tests/deadline_origin.py +++ /dev/null @@ -1,95 +0,0 @@ -"""Run ignored Fastly deadline tests against a local, deliberately slow origin. - -Usage: python3 tests/deadline_origin.py /path/to/viceroy [test-binary.wasm] - -Without an artifact argument, build and locate the current binary using Cargo JSON. -""" - -import http.server -import json -import os -import pathlib -import subprocess -import sys -import tempfile -import threading -import time - - -class Origin(http.server.BaseHTTPRequestHandler): - def do_HEAD(self): - self.send_response(200) - self.send_header("Content-Length", str(20 * 1024 * 1024)) - self.end_headers() - - def do_GET(self): - try: - if self.path == "/stalled": - time.sleep(2) - if self.path == "/trickle": - self.send_response(200) - self.send_header("Content-Length", "100") - self.end_headers() - for _ in range(100): - self.wfile.write(b"x") - self.wfile.flush() - time.sleep(0.01) - return - time.sleep(0.03) - self.send_response(200) - self.send_header("Content-Length", "4") - self.end_headers() - self.wfile.write(b"fast") - except (BrokenPipeError, ConnectionResetError): - # Deadline expiry deliberately closes the response before this slow - # origin finishes writing, so a disconnected client is expected. - pass - - def log_message(self, *_args): - pass - - -def main(): - if len(sys.argv) == 2: - build = subprocess.run( - ["cargo", "test", "-p", "trusted-server-adapter-fastly", - "--target", "wasm32-wasip1", "--bin", "trusted-server-adapter-fastly", "--no-run", - "--message-format=json"], - text=True, stdout=subprocess.PIPE, check=True, - ) - artifacts = [json.loads(line) for line in build.stdout.splitlines() if line.startswith("{")] - binaries = [item["executable"] for item in artifacts - if item.get("reason") == "compiler-artifact" - and item.get("profile", {}).get("test") - and item.get("target", {}).get("name") == "trusted-server-adapter-fastly" - and item.get("executable")] - if len(binaries) != 1: - raise RuntimeError(f"Expected one Fastly binary test artifact, found {binaries!r}") - binary = binaries[0] - else: - binary = sys.argv[2] - with http.server.ThreadingHTTPServer(("127.0.0.1", 0), Origin) as server: - server.daemon_threads = True - threading.Thread(target=server.serve_forever, daemon=True).start() - with tempfile.TemporaryDirectory(prefix="fastly-deadline-") as directory: - config = pathlib.Path(directory) / "fastly.toml" - config.write_text( - '[local_server.backends.deadline_origin]\n' - f'url = "http://127.0.0.1:{server.server_port}"\n' - ) - environment = os.environ.copy() - if pathlib.Path("/etc/ssl/cert.pem").exists(): - environment.setdefault("SSL_CERT_FILE", "/etc/ssl/cert.pem") - result = subprocess.run( - [sys.argv[1], "run", "-C", str(config), binary, - "platform::deadline::tests::network_", "--ignored", "--nocapture"], - env=environment, - check=False, - timeout=120, - ) - server.shutdown() - return result.returncode - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/docs/guide/auction-testing.md b/docs/guide/auction-testing.md index 87a0eb6895..7903a42700 100644 --- a/docs/guide/auction-testing.md +++ b/docs/guide/auction-testing.md @@ -78,18 +78,18 @@ headers and trickle bodies, and verify outstanding task cancellation. Worker adapter tests use controlled futures and timers. CI also runs the actual worker HTTP clients under local workerd and Spin 4.1, covering eager dispatch, buffered bids after a slow origin, delayed headers, and trickling bodies. Run -`scripts/test-worker-auction-deadlines.py --help` for the same local fixtures. +`node scripts/test-worker-auction-deadlines.mjs --help` for the same local fixtures. Fastly also has Viceroy network fixtures for stalled headers, trickling bodies, bodyless responses, and competing bidders; run its local origin harness after building the adapter test binary: ```bash cargo test -p trusted-server-adapter-fastly --target wasm32-wasip1 --no-run -python3 crates/trusted-server-adapter-fastly/tests/deadline_origin.py \ +node crates/trusted-server-adapter-fastly/tests/deadline-origin.mjs \ "$(command -v viceroy)" \ target/wasm32-wasip1/debug/deps/trusted_server_adapter_fastly-.wasm ``` Use the adapter test artifact printed by Cargo in place of ``. The harness -binds a local ephemeral port and runs the four network tests excluded from the +binds a local ephemeral port and runs the five network tests excluded from the standard suite; CI runs the harness automatically. diff --git a/scripts/test-worker-auction-deadlines.mjs b/scripts/test-worker-auction-deadlines.mjs new file mode 100644 index 0000000000..a81ee71b62 --- /dev/null +++ b/scripts/test-worker-auction-deadlines.mjs @@ -0,0 +1,297 @@ +#!/usr/bin/env node +// Exercise production HTTP clients against delayed local origins in worker runtimes. +import assert from 'node:assert/strict' +import { execFile, spawn } from 'node:child_process' +import { open, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises' +import http from 'node:http' +import net from 'node:net' +import os from 'node:os' +import path from 'node:path' +import { setTimeout as delay } from 'node:timers/promises' +import { fileURLToPath } from 'node:url' +import { parseArgs, promisify } from 'node:util' + +const { values, positionals } = parseArgs({ + allowPositionals: true, + options: { + help: { type: 'boolean' }, + runtime: { type: 'string' }, + 'wasm-bindgen': { type: 'string', default: 'wasm-bindgen' }, + 'target-dir': { type: 'string' }, + }, +}) +if (values.help) { + console.log( + 'Usage: node scripts/test-worker-auction-deadlines.mjs --runtime [--wasm-bindgen ] [--target-dir ]' + ) + process.exit(0) +} +const [adapter] = positionals +assert( + positionals.length === 1 && ['cloudflare', 'spin'].includes(adapter), + 'Supply cloudflare or spin as the adapter' +) +assert( + values.runtime, + 'Supply --runtime with the local worker runtime executable' +) +const interrupted = new AbortController() +const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..') +const artifact = path.resolve( + values['target-dir'] ?? path.join(root, 'target'), + adapter === 'cloudflare' ? 'wasm32-unknown-unknown' : 'wasm32-wasip1', + 'debug/examples/deadline_runtime.wasm' +) + +async function listen(server) { + await new Promise((resolve, reject) => { + server.once('error', reject) + server.listen(0, '127.0.0.1', resolve) + }) + return server.address().port +} + +async function closeServer(server) { + server.closeAllConnections?.() + await new Promise((resolve) => server.close(resolve)) +} + +async function availablePort() { + const server = net.createServer() + const port = await listen(server) + await closeServer(server) + return port +} + +function connects(port) { + return new Promise((resolve) => { + const socket = net.createConnection({ host: '127.0.0.1', port }) + const done = (result) => { + socket.destroy() + resolve(result) + } + socket.setTimeout(100) + socket.once('connect', () => done(true)) + socket.once('error', () => done(false)) + socket.once('timeout', () => done(false)) + }) +} + +function get(uri) { + return new Promise((resolve, reject) => { + const request = http.get( + uri, + { signal: interrupted.signal }, + (response) => { + const chunks = [] + response.on('data', (chunk) => chunks.push(chunk)) + response.on('error', reject) + response.on('end', () => { + if (response.statusCode !== 200) { + reject( + new Error( + `Fixture returned ${response.statusCode}: ${Buffer.concat(chunks)}` + ) + ) + } else { + resolve(Buffer.concat(chunks).toString()) + } + }) + } + ) + request.on('error', reject) + request.setTimeout(5000, () => + request.destroy(new Error('Fixture request timed out')) + ) + }) +} + +const observed = new Set() +const origin = http.createServer(async (request, response) => { + const cancellation = new AbortController() + response.once('close', () => cancellation.abort()) + response.on('error', () => {}) + observed.add(request.url) + try { + if (request.url === '/headers') { + await delay(700, undefined, { signal: cancellation.signal }) + } + response.writeHead(200, { 'Content-Length': '4' }) + if (request.url === '/body') { + response.write('a') + await delay(700, undefined, { signal: cancellation.signal }) + response.end('bcd') + } else { + response.end('okay') + } + } catch (error) { + if (!cancellation.signal.aborted) response.destroy(error) + } +}) + +let workspace +let log +let child +let childStopped +let childError +const grouped = process.platform !== 'win32' +function signalRuntime(signal) { + if (!child?.pid) return + try { + if (grouped) process.kill(-child.pid, signal) + else child.kill(signal) + } catch (error) { + if (error.code !== 'ESRCH') throw error + } +} + +const interrupt = (signal) => { + interrupted.abort(new Error(`Harness interrupted by ${signal}`)) + signalRuntime('SIGTERM') +} +process.on('SIGINT', interrupt) +process.on('SIGTERM', interrupt) + +try { + const originPort = await listen(origin) + const port = await availablePort() + workspace = await mkdtemp(path.join(os.tmpdir(), 'ts-worker-deadline-')) + let argumentsForRuntime + if (adapter === 'cloudflare') { + await promisify(execFile)( + values['wasm-bindgen'], + [ + artifact, + '--target', + 'web', + '--out-dir', + workspace, + '--out-name', + 'fixture', + ], + { timeout: 60_000, signal: interrupted.signal } + ) + await writeFile( + path.join(workspace, 'index.js'), + 'import module from "./fixture_bg.wasm";\nimport {initSync, fetch} from "./fixture.js";\ninitSync({module});\nexport default {fetch};\n' + ) + const configuration = path.join(workspace, 'wrangler.toml') + await writeFile( + configuration, + 'name = "deadline-fixture"\nmain = "index.js"\ncompatibility_date = "2026-04-01"\n' + ) + argumentsForRuntime = [ + 'dev', + '--local', + '--port', + String(port), + '--ip', + '127.0.0.1', + '--config', + configuration, + ] + } else { + const configuration = path.join(workspace, 'spin.toml') + await writeFile( + configuration, + `spin_manifest_version = 2 +[application] +name = "deadline-fixture" +version = "0.1.0" +[[trigger.http]] +route = "/..." +component = "fixture" +[component.fixture] +source = ${JSON.stringify(artifact)} +allowed_outbound_hosts = ["http://127.0.0.1:*"] +` + ) + argumentsForRuntime = [ + 'up', + '--from', + configuration, + '--listen', + `127.0.0.1:${port}`, + ] + } + log = await open(path.join(workspace, 'runtime.log'), 'w') + child = spawn(values.runtime, argumentsForRuntime, { + cwd: workspace, + stdio: ['ignore', log.fd, log.fd], + detached: grouped, + }) + childStopped = new Promise((resolve) => { + child.once('exit', resolve) + child.once('error', (error) => { + childError = error + resolve() + }) + }) + let ready = false + for (let attempt = 0; attempt < 300; attempt++) { + interrupted.signal.throwIfAborted() + if (childError) throw childError + assert( + child.exitCode === null && child.signalCode === null, + 'Worker exited during startup' + ) + if (await connects(port)) { + ready = true + break + } + await delay(100, undefined, { signal: interrupted.signal }) + } + assert(ready, 'Worker did not listen') + const cases = [ + ['/fast', 80, 200, 'complete', 500], + ['/headers', 80, 0, 'timeout', 350], + ['/body', 80, 0, 'timeout', 350], + ['/headers', 80, 200, 'timeout', 500], + ] + for (const [endpoint, budget, originDelay, expected, maximum] of cases) { + observed.delete(endpoint) + const uri = `http://127.0.0.1:${port}/?upstream=http://127.0.0.1:${originPort}${endpoint}&budget_ms=${budget}&origin_ms=${originDelay}` + const result = await get(uri) + const [status, elapsed] = result.split(':') + assert.equal( + status, + expected, + `${endpoint} should return ${expected}: ${result}` + ) + assert( + Number.isFinite(Number(elapsed)) && Number(elapsed) < maximum, + `${endpoint} should complete within ${maximum} ms: ${result}` + ) + assert( + observed.has(endpoint), + `${endpoint} request should start at dispatch` + ) + console.log( + `PASS ${adapter} ${endpoint} budget=${budget} origin=${originDelay}: ${result}` + ) + } +} catch (error) { + if (log) + console.error(await readFile(path.join(workspace, 'runtime.log'), 'utf8')) + throw error +} finally { + try { + if (child?.pid) { + signalRuntime('SIGTERM') + const timeout = new AbortController() + await Promise.race([ + childStopped, + delay(10_000, undefined, { signal: timeout.signal }), + ]) + timeout.abort() + signalRuntime('SIGKILL') + await childStopped + } + } finally { + await log?.close() + if (workspace) await rm(workspace, { recursive: true, force: true }) + await closeServer(origin) + process.off('SIGINT', interrupt) + process.off('SIGTERM', interrupt) + } +} diff --git a/scripts/test-worker-auction-deadlines.py b/scripts/test-worker-auction-deadlines.py deleted file mode 100644 index 9f53cdbb49..0000000000 --- a/scripts/test-worker-auction-deadlines.py +++ /dev/null @@ -1,126 +0,0 @@ -#!/usr/bin/env python3 -"""Run production HTTP clients against delayed local origins in worker runtimes.""" - -import argparse -import http.server -import pathlib -import socket -import subprocess -import tempfile -import threading -import time -import urllib.request - - -class Origin(http.server.BaseHTTPRequestHandler): - events = {} - - def do_GET(self): - self.events.setdefault(self.path, threading.Event()).set() - if self.path == "/headers": - time.sleep(0.7) - self.send_response(200) - self.send_header("Content-Length", "4") - self.end_headers() - try: - if self.path == "/body": - self.wfile.write(b"a") - self.wfile.flush() - time.sleep(0.7) - self.wfile.write(b"bcd") - else: - self.wfile.write(b"okay") - except (BrokenPipeError, ConnectionResetError): - # Deadline cancellation deliberately disconnects this slow origin. - pass - - def log_message(self, *_): - pass - - -def free_port(): - with socket.socket() as connection: - connection.bind(("127.0.0.1", 0)) - return connection.getsockname()[1] - - -def main(): - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("adapter", choices=["cloudflare", "spin"]) - parser.add_argument("--runtime", required=True) - parser.add_argument("--wasm-bindgen", default="wasm-bindgen") - parser.add_argument("--target-dir", type=pathlib.Path) - args = parser.parse_args() - root = pathlib.Path(__file__).resolve().parent.parent - artifact = (args.target_dir or root / "target") / ("wasm32-unknown-unknown" if args.adapter == "cloudflare" else "wasm32-wasip1") / "debug/examples/deadline_runtime.wasm" - origin = http.server.ThreadingHTTPServer(("127.0.0.1", 0), Origin) - thread = threading.Thread(target=origin.serve_forever, daemon=True) - thread.start() - port = free_port() - with tempfile.TemporaryDirectory(prefix="ts-worker-deadline-") as temporary: - workspace = pathlib.Path(temporary) - if args.adapter == "cloudflare": - subprocess.run([args.wasm_bindgen, str(artifact), "--target", "web", "--out-dir", str(workspace), "--out-name", "fixture"], check=True) - (workspace / "index.js").write_text('import module from "./fixture_bg.wasm";\nimport {initSync, fetch} from "./fixture.js";\ninitSync({module});\nexport default {fetch};\n') - (workspace / "wrangler.toml").write_text('name = "deadline-fixture"\nmain = "index.js"\ncompatibility_date = "2026-04-01"\n') - command = [args.runtime, "dev", "--local", "--port", str(port), "--ip", "127.0.0.1", "--config", str(workspace / "wrangler.toml")] - else: - (workspace / "spin.toml").write_text(f'''spin_manifest_version = 2 -[application] -name = "deadline-fixture" -version = "0.1.0" -[[trigger.http]] -route = "/..." -component = "fixture" -[component.fixture] -source = "{artifact}" -allowed_outbound_hosts = ["http://127.0.0.1:*"] -''') - command = [args.runtime, "up", "--from", str(workspace / "spin.toml"), "--listen", f"127.0.0.1:{port}"] - with (workspace / "runtime.log").open("w+") as log: - process = subprocess.Popen(command, cwd=workspace, stdout=log, stderr=subprocess.STDOUT) - try: - for _ in range(300): - if process.poll() is not None: - raise RuntimeError("worker exited during startup") - try: - with socket.create_connection(("127.0.0.1", port), timeout=0.1): - break - except OSError: - time.sleep(0.1) - else: - raise RuntimeError("worker did not listen") - cases = [ - ("/fast", 80, 200, "complete", 500), - ("/headers", 80, 0, "timeout", 350), - ("/body", 80, 0, "timeout", 350), - ("/headers", 80, 200, "timeout", 500), - ] - for path, budget, origin_delay, expected, maximum in cases: - Origin.events[path] = threading.Event() - upstream = f"http://127.0.0.1:{origin.server_port}{path}" - uri = f"http://127.0.0.1:{port}/?upstream={upstream}&budget_ms={budget}&origin_ms={origin_delay}" - with urllib.request.urlopen(uri, timeout=5) as response: - result = response.read().decode() - status, elapsed = result.split(":") - assert status == expected, (path, expected, result) - assert int(elapsed) < maximum, (path, maximum, result) - assert Origin.events[path].is_set(), (path, "request should start at dispatch") - print(f"PASS {args.adapter} {path} budget={budget} origin={origin_delay}: {result}", flush=True) - except BaseException: - log.flush() - log.seek(0) - print(log.read()) - raise - finally: - process.terminate() - try: - process.wait(timeout=10) - except subprocess.TimeoutExpired: - process.kill() - process.wait() - origin.shutdown() - - -if __name__ == "__main__": - main() From 3453298ec13d30c80da671c7e6f339b2442ca540 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Thu, 22 Oct 2026 20:50:04 +0530 Subject: [PATCH 6/9] Harden auction deadline isolation and cutoff enforcement --- .../workflows/worker-auction-deadlines.yml | 23 +- CHANGELOG.md | 1 + Cargo.toml | 1 + .../src/platform.rs | 5 - .../Cargo.toml | 1 + .../src/platform.rs | 170 +++---- .../tests/runtime/deadline.rs | 7 +- .../trusted-server-adapter-fastly/Cargo.toml | 2 +- .../src/platform.rs | 5 - .../src/platform/deadline.rs | 130 ++++- .../tests/deadline-origin.mjs | 14 +- crates/trusted-server-adapter-spin/Cargo.toml | 1 + .../src/platform.rs | 210 ++++---- .../tests/runtime/deadline.rs | 23 +- .../src/auction/orchestrator.rs | 469 ++++++++++++++---- .../src/platform/backend_naming.rs | 20 - .../trusted-server-core/src/platform/http.rs | 38 +- .../src/platform/test_support.rs | 41 +- crates/trusted-server-core/src/publisher.rs | 22 +- .../trusted-server-core/src/test_support.rs | 9 +- docs/guide/auction-orchestration.md | 4 +- docs/guide/configuration.md | 2 + scripts/test-worker-auction-deadlines.mjs | 12 + 23 files changed, 809 insertions(+), 401 deletions(-) diff --git a/.github/workflows/worker-auction-deadlines.yml b/.github/workflows/worker-auction-deadlines.yml index 41b459230e..cd1f4faa8b 100644 --- a/.github/workflows/worker-auction-deadlines.yml +++ b/.github/workflows/worker-auction-deadlines.yml @@ -46,20 +46,39 @@ jobs: else cargo build --locked -p trusted-server-adapter-spin --example deadline-runtime --target wasm32-wasip1 --features spin fi + - name: Lint runtime fixture + shell: bash + run: | + if [ "${{ matrix.adapter }}" = cloudflare ]; then + cargo clippy --locked -p trusted-server-adapter-cloudflare --example deadline-runtime --target wasm32-unknown-unknown --features cloudflare -- -D warnings + else + cargo clippy --locked -p trusted-server-adapter-spin --example deadline-runtime --target wasm32-wasip1 --features spin -- -D warnings + fi - name: Install local Cloudflare tools if: matrix.adapter == 'cloudflare' shell: bash run: | npm install --global wrangler@4.83.0 mkdir -p "$RUNNER_TEMP/bindgen" - curl --fail --location --silent --show-error "https://github.com/wasm-bindgen/wasm-bindgen/releases/download/${{ steps.versions.outputs.bindgen }}/wasm-bindgen-${{ steps.versions.outputs.bindgen }}-x86_64-unknown-linux-musl.tar.gz" | tar -xz --strip-components=1 -C "$RUNNER_TEMP/bindgen" + # Digests are published with the upstream GitHub release assets. + # Update the digest alongside the pinned version. + test "${{ steps.versions.outputs.bindgen }}" = 0.2.126 + archive="$RUNNER_TEMP/wasm-bindgen.tar.gz" + curl --fail --location --silent --show-error "https://github.com/wasm-bindgen/wasm-bindgen/releases/download/${{ steps.versions.outputs.bindgen }}/wasm-bindgen-${{ steps.versions.outputs.bindgen }}-x86_64-unknown-linux-musl.tar.gz" --output "$archive" + echo "064948d58e2d6c0a745216477a639ba696216d6309aaa902939d1b865b1d869d $archive" | sha256sum --check + tar -xz --strip-components=1 -C "$RUNNER_TEMP/bindgen" -f "$archive" echo "$RUNNER_TEMP/bindgen" >> "$GITHUB_PATH" - name: Install pinned Spin runtime if: matrix.adapter == 'spin' shell: bash run: | mkdir -p "$RUNNER_TEMP/spin" - curl --fail --location --silent --show-error "https://github.com/spinframework/spin/releases/download/v${{ steps.versions.outputs.spin }}/spin-v${{ steps.versions.outputs.spin }}-linux-amd64.tar.gz" | tar -xz -C "$RUNNER_TEMP/spin" + # Update the published upstream digest alongside the pinned version. + test "${{ steps.versions.outputs.spin }}" = 4.1.0 + archive="$RUNNER_TEMP/spin.tar.gz" + curl --fail --location --silent --show-error "https://github.com/spinframework/spin/releases/download/v${{ steps.versions.outputs.spin }}/spin-v${{ steps.versions.outputs.spin }}-linux-amd64.tar.gz" --output "$archive" + echo "aeeb892f94cf861f5302546faa0bdaa82cc4be30539621299746a6dc011f3b5a $archive" | sha256sum --check + tar -xz -C "$RUNNER_TEMP/spin" -f "$archive" echo "$RUNNER_TEMP/spin" >> "$GITHUB_PATH" - name: Exercise delayed headers, trickling bodies, and slow publisher work shell: bash diff --git a/CHANGELOG.md b/CHANGELOG.md index 671aec1cf1..0cc180471f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 measured from dispatch, including delayed headers and trickling bodies on all adapters. Complete available responses survive a slow publisher origin; unfinished providers time out and mediation is skipped after budget exhaustion. + Provider bids complete at the final nonblocking snapshot remain eligible even after `tmax`; mediation starts only with remaining budget and rejects responses collected after its cutoff. Fastly abandons unfinished requests; Axum, Cloudflare, and Spin cancel owned work. - **Breaking:** `x-ts-version` now reports the deployed git version — the release tag, else the branch, else the first 6 characters of the commit — compiled in from the build-time `TRUSTED_SERVER__GIT_VERSION` (set by the deploy pipeline) or local git, and is sent by every adapter, including on the Fastly `GET /health` probe. The Fastly service version it previously carried is now `x-ts-fastly-version`; update dashboards, monitors, and scripts that read `x-ts-version` as the Fastly version number. Builds with neither the override nor git omit the header. diff --git a/Cargo.toml b/Cargo.toml index 72fecdf6b5..ed2c001130 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -66,6 +66,7 @@ env_logger = "0.11" error-stack = "0.6" esi = "0.7.2" fastly = "0.13" +fastly-sys = "0.13.1" fern = "0.7.1" flate2 = "1.1" futures = "0.3" diff --git a/crates/trusted-server-adapter-axum/src/platform.rs b/crates/trusted-server-adapter-axum/src/platform.rs index 7529044693..fed792c888 100644 --- a/crates/trusted-server-adapter-axum/src/platform.rs +++ b/crates/trusted-server-adapter-axum/src/platform.rs @@ -649,11 +649,6 @@ mod tests { .capabilities(); assert!(client.supports_concurrent_fanout()); assert!(capabilities.supports_concurrent_provider_fanout()); - assert!(!client.has_enforceable_total_request_deadline()); - assert!( - !capabilities.has_enforceable_total_request_deadline(), - "reqwest's transport timeout is not an adapter-enforced auction deadline" - ); } #[test] diff --git a/crates/trusted-server-adapter-cloudflare/Cargo.toml b/crates/trusted-server-adapter-cloudflare/Cargo.toml index b9bfe3fafa..9f9a2e738a 100644 --- a/crates/trusted-server-adapter-cloudflare/Cargo.toml +++ b/crates/trusted-server-adapter-cloudflare/Cargo.toml @@ -52,3 +52,4 @@ tokio = { workspace = true, features = ["rt-multi-thread", "macros"] } name = "deadline-runtime" path = "tests/runtime/deadline.rs" crate-type = ["cdylib"] +required-features = ["cloudflare"] diff --git a/crates/trusted-server-adapter-cloudflare/src/platform.rs b/crates/trusted-server-adapter-cloudflare/src/platform.rs index a2c6f592bc..60eec14e06 100644 --- a/crates/trusted-server-adapter-cloudflare/src/platform.rs +++ b/crates/trusted-server-adapter-cloudflare/src/platform.rs @@ -553,25 +553,8 @@ impl PlatformHttpClient for CloudflareHttpClient { Report::new(PlatformError::HttpClient) .attach("unexpected pending request type in deadline selection") })?; - // Probe only the task's buffered result. This never polls transport I/O - // and preserves a completed bid when publisher-origin work ran long. - if let Some(ready) = pending.response.try_recv().map_err(|error| { - Report::new(PlatformError::HttpClient) - .attach(format!("dispatched response task stopped: {error}")) - })? { - return Ok(Some(PlatformSelectResult { - failed_backend_name: ready.as_ref().err().and(backend_name), - ready, - remaining: pending_requests, - })); - } let result = complete_before_deadline( - async { - (&mut pending.response).await.unwrap_or_else(|error| { - Err(Report::new(PlatformError::HttpClient) - .attach(format!("dispatched response task stopped: {error}"))) - }) - }, + &mut pending.response, async { // Workers timers use whole signed milliseconds. Recheck the // absolute deadline after each bounded timer interval. @@ -592,10 +575,16 @@ impl PlatformHttpClient for CloudflareHttpClient { web_time::Instant::now, ) .await; - Ok(result.map(|ready| PlatformSelectResult { - failed_backend_name: ready.as_ref().err().and(backend_name), - ready, - remaining: pending_requests, + Ok(result.map(|result| { + let ready = result.unwrap_or_else(|error| { + Err(Report::new(PlatformError::HttpClient) + .attach(format!("dispatched response task stopped: {error}"))) + }); + PlatformSelectResult { + failed_backend_name: ready.as_ref().err().and(backend_name), + ready, + remaining: pending_requests, + } })) } } @@ -840,10 +829,6 @@ mod tests { .descriptor() .capabilities(); assert!(!capabilities.supports_concurrent_provider_fanout()); - assert!( - !capabilities.has_enforceable_total_request_deadline(), - "Workers fetch does not expose an enforceable hard total request deadline" - ); } use edgezero_core::http::{HeaderValue, request_builder}; use edgezero_core::params::PathParams; @@ -1018,23 +1003,31 @@ mod tests { } #[cfg(any(test, target_arch = "wasm32"))] -async fn complete_before_deadline( - operation: F, +async fn complete_before_deadline( + response: &mut futures::channel::oneshot::Receiver, timer: T, deadline: web_time::Instant, now: N, -) -> Option +) -> Option> where - F: std::future::Future, T: std::future::Future, N: Fn() -> web_time::Instant, { + // Inspect only the task's buffered result, including when collection starts + // after publisher-origin work has exhausted the auction budget. + match response.try_recv() { + Ok(Some(ready)) => return Some(Ok(ready)), + Err(error) => return Some(Err(error)), + Ok(None) => {} + } if now() >= deadline { return None; } - match futures::future::select(Box::pin(operation), Box::pin(timer)).await { - futures::future::Either::Left((response, _)) if now() < deadline => Some(response), - _ => None, + match futures::future::select(response, Box::pin(timer)).await { + futures::future::Either::Left((ready, _)) => Some(ready), + // The timer and task can become ready in the same executor turn. A final + // channel probe retains buffered results without polling transport I/O. + futures::future::Either::Right(((), response)) => response.try_recv().transpose(), } } @@ -1079,11 +1072,6 @@ mod deadline_tests { ); finish_body.send(42).expect("should finish response body"); executor.run_until_stalled(); - let expired = web_time::Instant::now(); - assert!( - web_time::Instant::now() >= expired, - "should collect after cutoff" - ); assert_eq!( pending .response @@ -1152,93 +1140,91 @@ mod deadline_tests { } #[test] - fn expired_deadline_does_not_poll_request() { + fn expired_deadline_preserves_buffered_response() { + let (sender, mut response) = futures::channel::oneshot::channel(); + sender.send(42).expect("should buffer response"); let deadline = Instant::now(); - let polled = Cell::new(false); - let operation = async { - polled.set(true); - pending::<()>().await - }; - let result: Option<()> = block_on(complete_before_deadline( - operation, + let result = block_on(complete_before_deadline( + &mut response, pending(), deadline, || deadline, )); - assert_eq!(result, None, "should reject before polling expired request"); - assert!(!polled.get(), "should not poll an expired request"); + assert_eq!( + result, + Some(Ok(42)), + "should retain a buffered response after cutoff" + ); } #[test] - fn timer_cuts_off_pending_operation_and_drops_it() { - struct DropProbe(Rc>); - impl Drop for DropProbe { - fn drop(&mut self) { - self.0.set(true); - } - } - let dropped = Rc::new(Cell::new(false)); - let probe = DropProbe(Rc::clone(&dropped)); - let operation = async move { - let _probe = probe; - pending::<()>().await - }; + fn expired_deadline_returns_without_waiting_for_response() { + let (_sender, mut response) = futures::channel::oneshot::channel::<()>(); + let deadline = Instant::now(); + let result = block_on(complete_before_deadline( + &mut response, + pending(), + deadline, + || deadline, + )); + assert_eq!(result, None, "should stop waiting at an expired deadline"); + } + + #[test] + fn timer_cuts_off_pending_response() { + let (_sender, mut response) = futures::channel::oneshot::channel::<()>(); let now = Instant::now(); let result = block_on(complete_before_deadline( - operation, + &mut response, ready(()), now + Duration::from_secs(1), || now, )); - assert_eq!(result, None, "should cut off a pending request"); - assert!(dropped.get(), "should drop the pending operation at cutoff"); + assert_eq!(result, None, "should cut off an unbuffered response"); } #[test] - fn accepts_complete_response_before_deadline() { + fn closed_response_channel_reports_failure() { + let (sender, mut response) = futures::channel::oneshot::channel::<()>(); + drop(sender); let now = Instant::now(); let result = block_on(complete_before_deadline( - ready(42), + &mut response, pending(), now + Duration::from_secs(1), || now, )); - assert_eq!(result, Some(42), "should accept response before cutoff"); + assert!( + matches!(result, Some(Err(_))), + "should preserve a dispatched task failure" + ); } #[test] - fn rejects_response_that_becomes_ready_at_deadline() { + fn timer_rechecks_result_published_during_cutoff() { + let (sender, mut response) = futures::channel::oneshot::channel(); let now = Instant::now(); - let deadline = now + Duration::from_secs(1); let clock = Cell::new(now); - let operation = async { + let deadline = now + Duration::from_secs(1); + let timer = async { clock.set(deadline); - 42 + sender.send(42).expect("should publish at cutoff"); }; let result = block_on(complete_before_deadline( - operation, - pending(), + &mut response, + timer, deadline, || clock.get(), )); - assert_eq!(result, None, "should reject a late ready response"); - } - - #[test] - fn timer_bounds_body_after_headers_complete() { - let now = Instant::now(); - let headers_received = Cell::new(false); - let operation = async { - headers_received.set(true); - pending::<()>().await - }; - let result = block_on(complete_before_deadline( - operation, - ready(()), - now + Duration::from_secs(1), - || now, - )); - assert!(headers_received.get(), "should reach response body read"); - assert_eq!(result, None, "should cut off pending body collection"); + assert_eq!( + result, + Some(Ok(42)), + "should retain result buffered when timer wins" + ); + assert_eq!( + clock.get(), + deadline, + "should exercise the deadline boundary" + ); } } diff --git a/crates/trusted-server-adapter-cloudflare/tests/runtime/deadline.rs b/crates/trusted-server-adapter-cloudflare/tests/runtime/deadline.rs index b3faec2fc1..d3875d76b1 100644 --- a/crates/trusted-server-adapter-cloudflare/tests/runtime/deadline.rs +++ b/crates/trusted-server-adapter-cloudflare/tests/runtime/deadline.rs @@ -61,7 +61,12 @@ async fn exercise( .await?; let status = match collected { Some(result) => { - result.ready?; + let response = result.ready?; + assert_eq!( + response.response.body().as_bytes(), + Some(b"okay".as_slice()), + "should retain the complete nonempty upstream body" + ); "complete" } None => "timeout", diff --git a/crates/trusted-server-adapter-fastly/Cargo.toml b/crates/trusted-server-adapter-fastly/Cargo.toml index b10be8105a..6c7121d0d6 100644 --- a/crates/trusted-server-adapter-fastly/Cargo.toml +++ b/crates/trusted-server-adapter-fastly/Cargo.toml @@ -29,7 +29,7 @@ error-stack = { workspace = true } esi = { workspace = true } fastly = { workspace = true } # The SDK keeps timed body readiness private; auction deadlines use the ABI. -fastly-sys = "0.13.1" +fastly-sys = { workspace = true } fern = { workspace = true } futures = { workspace = true } log = { workspace = true } diff --git a/crates/trusted-server-adapter-fastly/src/platform.rs b/crates/trusted-server-adapter-fastly/src/platform.rs index c5a916bec7..ea2193d090 100644 --- a/crates/trusted-server-adapter-fastly/src/platform.rs +++ b/crates/trusted-server-adapter-fastly/src/platform.rs @@ -1237,11 +1237,6 @@ mod tests { .capabilities(); assert!(client.supports_concurrent_fanout()); assert!(capabilities.supports_concurrent_provider_fanout()); - assert!(!client.has_enforceable_total_request_deadline()); - assert!( - !capabilities.has_enforceable_total_request_deadline(), - "first-byte and between-byte timers are not a hard total request deadline" - ); } #[test] diff --git a/crates/trusted-server-adapter-fastly/src/platform/deadline.rs b/crates/trusted-server-adapter-fastly/src/platform/deadline.rs index de0af14b60..8cff874d99 100644 --- a/crates/trusted-server-adapter-fastly/src/platform/deadline.rs +++ b/crates/trusted-server-adapter-fastly/src/platform/deadline.rs @@ -224,8 +224,9 @@ fn body_is_ready(body: &BodyHandle) -> Result> { /// /// # Errors /// -/// Returns an error for invalid handles or a failure of the timed wait hostcall. -/// Provider errors are attributed in the returned collection result. +/// Returns an error for an empty request list, invalid aggregate buffer state, +/// or a failure of the timed wait hostcall. Individual provider errors, including +/// unsupported pending handles, are attributed in the returned collection result. pub(super) fn select_until( pending_requests: Vec, deadline: Instant, @@ -236,8 +237,19 @@ pub(super) fn select_until( } let mut pending = pending_requests .into_iter() - .map(DeadlinePending::from_platform) - .collect::, _>>()?; + .map(|pending| { + let backend_name = pending.backend_name().unwrap_or("").to_owned(); + let request_method = pending.request_method().cloned(); + match DeadlinePending::from_platform(pending) { + Ok(pending) => pending, + Err(error) => DeadlinePending { + backend_name, + request_method, + state: Some(PendingState::Ready(Err(error))), + }, + } + }) + .collect::>(); let mut buffered_bytes: usize = pending.iter().map(DeadlinePending::buffered_bytes).sum(); if buffered_bytes > MAX_TOTAL_BUFFERED_BODY_BYTES { return Err(Report::new(PlatformError::HttpClient) @@ -410,15 +422,24 @@ mod tests { use web_time::Instant; #[test] - fn expired_deadline_rejects_an_invalid_pending_handle() { + fn expired_deadline_attributes_an_invalid_pending_handle() { let result = futures::executor::block_on(FastlyPlatformHttpClient.select_until( vec![PlatformPendingRequest::new(42_u32).with_backend_name("example")], Instant::now(), )) - .expect_err("should reject invalid handles even in the nonblocking snapshot"); + .expect("should isolate invalid handles even in the nonblocking snapshot") + .expect("should report the invalid pending handle"); + assert_eq!( + result.failed_backend_name.as_deref(), + Some("example"), + "should attribute the invalid pending handle" + ); + let error = result + .ready + .expect_err("should reject the invalid pending handle"); assert!( matches!( - result.current_context(), + error.current_context(), trusted_server_core::platform::PlatformError::Unsupported ), "should reject unsupported deadline handles without panicking" @@ -431,7 +452,17 @@ mod tests { vec![PlatformPendingRequest::new(42_u32).with_backend_name("example")], Instant::now() + Duration::from_secs(1), )); - let error = result.expect_err("should reject a malformed pending handle"); + let result = result + .expect("should isolate malformed pending handles") + .expect("should return the malformed provider's failure"); + assert_eq!( + result.failed_backend_name.as_deref(), + Some("example"), + "should attribute the malformed handle" + ); + let error = result + .ready + .expect_err("should reject a malformed pending handle"); assert!( format!("{error:?}").contains("example"), "should identify the malformed handle's backend" @@ -453,16 +484,58 @@ mod tests { .into_platform() } + #[test] + fn unsupported_provider_does_not_discard_a_healthy_sibling() { + let deadline = Instant::now() + Duration::from_secs(1); + let failed = PlatformPendingRequest::new(42_u32).with_backend_name("unsupported"); + let healthy = buffered_pending(b"healthy".to_vec(), "healthy"); + let result = futures::executor::block_on( + FastlyPlatformHttpClient.select_until(vec![failed, healthy], deadline), + ) + .expect("should isolate a provider with an unsupported pending handle") + .expect("should report the failed provider"); + assert_eq!( + result.failed_backend_name.as_deref(), + Some("unsupported"), + "should attribute the conversion error to its provider" + ); + assert!( + result.ready.is_err(), + "should report the unsupported handle" + ); + let result = futures::executor::block_on( + FastlyPlatformHttpClient.select_until(result.remaining, deadline), + ) + .expect("should continue collecting the healthy sibling") + .expect("should complete the healthy sibling"); + let response = result.ready.expect("should retain the healthy response"); + assert_eq!( + response.backend_name.as_deref(), + Some("healthy"), + "should preserve the unaffected provider's bid" + ); + } + #[test] fn expired_deadline_preserves_an_already_ready_body() { let pending = buffered_pending(b"ready".to_vec(), "example"); let result = futures::executor::block_on( FastlyPlatformHttpClient.select_until(vec![pending], Instant::now()), ) - .expect("should discard an expired ready response"); - assert!( - result.is_some(), - "should preserve an already complete response without waiting" + .expect("should snapshot an expired ready response") + .expect("should preserve an already complete response without waiting"); + let response = result + .ready + .expect("should retain a successful buffered response"); + assert_eq!( + response.backend_name.as_deref(), + Some("example"), + "should retain the ready response's backend" + ); + assert_eq!( + response.response.body().as_bytes(), + Some(b"ready".as_slice()), + "should retain the complete ready payload" ); } @@ -778,10 +851,39 @@ mod tests { fn network_fast_sibling_completes_while_another_body_trickles() { let slow = network_pending("trickle", "GET"); let fast = network_pending("fast", "GET"); + let request = edgezero_core::http::request_builder() + .uri("http://example.com/fast") + .body(Body::empty()) + .expect("should build the request missing cache bypass"); + let unsupported = futures::executor::block_on( + FastlyPlatformHttpClient + .send_async(PlatformHttpRequest::new(request, "deadline_origin")), + ) + .expect("should dispatch the request missing cache bypass"); let started = Instant::now(); + let deadline = started + Duration::from_millis(200); let result = futures::executor::block_on( - FastlyPlatformHttpClient - .select_until(vec![slow, fast], started + Duration::from_millis(200)), + FastlyPlatformHttpClient.select_until(vec![unsupported, slow, fast], deadline), + ) + .expect("should isolate a request missing cache bypass") + .expect("should return the unsupported provider's failure"); + assert_eq!( + result.failed_backend_name.as_deref(), + Some("deadline_origin"), + "should attribute missing cache bypass to that request" + ); + let error = result + .ready + .expect_err("should reject the unsupported handle"); + assert!( + matches!( + error.current_context(), + trusted_server_core::platform::PlatformError::Unsupported + ), + "should reject SDK guest-caching handles before direct handle conversion" + ); + let result = futures::executor::block_on( + FastlyPlatformHttpClient.select_until(result.remaining, deadline), ) .expect("should collect the completed sibling") .expect("should finish before deadline"); diff --git a/crates/trusted-server-adapter-fastly/tests/deadline-origin.mjs b/crates/trusted-server-adapter-fastly/tests/deadline-origin.mjs index 97a27005d4..c985097677 100644 --- a/crates/trusted-server-adapter-fastly/tests/deadline-origin.mjs +++ b/crates/trusted-server-adapter-fastly/tests/deadline-origin.mjs @@ -126,11 +126,17 @@ async function main() { ], { env: environment, - stdio: 'inherit', + stdio: ['inherit', 'pipe', 'inherit'], signal: interruption.signal, killSignal: 'SIGKILL', } ) + let output = '' + child.stdout.setEncoding('utf8') + child.stdout.on('data', (chunk) => { + output += chunk + process.stdout.write(chunk) + }) // Clear the watchdog even if spawning fails: Node's built-in child timeout // can otherwise keep the harness alive after an ENOENT startup error. const watchdog = setTimeout(() => child.kill('SIGKILL'), 120_000) @@ -138,6 +144,12 @@ async function main() { try { const [code] = await once(child, 'close') if (originError) throw originError + assert.equal(code, 0, 'should run the network tests successfully') + assert.match( + output, + /^test result: ok\. 5 passed; 0 failed; 0 ignored; 0 measured;/m, + 'should execute all five network tests, including when the filter changes' + ) return code ?? 1 } finally { clearTimeout(watchdog) diff --git a/crates/trusted-server-adapter-spin/Cargo.toml b/crates/trusted-server-adapter-spin/Cargo.toml index 734370bb0c..4cf16fc781 100644 --- a/crates/trusted-server-adapter-spin/Cargo.toml +++ b/crates/trusted-server-adapter-spin/Cargo.toml @@ -49,3 +49,4 @@ tokio = { workspace = true, features = ["rt-multi-thread", "macros"] } name = "deadline-runtime" path = "tests/runtime/deadline.rs" crate-type = ["cdylib"] +required-features = ["spin"] diff --git a/crates/trusted-server-adapter-spin/src/platform.rs b/crates/trusted-server-adapter-spin/src/platform.rs index 19b4cc3022..27d3612417 100644 --- a/crates/trusted-server-adapter-spin/src/platform.rs +++ b/crates/trusted-server-adapter-spin/src/platform.rs @@ -26,9 +26,9 @@ use trusted_server_core::platform::{ PlatformPendingRequest, PlatformResponse, PlatformSelectResult, }; -// 8 MiB ceiling: conservative for ad-server responses while leaving headroom in -// the Spin WASM component heap. Larger responses from misbehaving origins are -// rejected with a typed error rather than OOMing the component. +// Limit both streamed bytes and decompressed bytes to 8 MiB to preserve +// headroom in the Spin WASM component heap. The streaming limit rejects an +// oversized frame before retaining it in the collected response body. #[cfg(any(test, all(feature = "spin", target_arch = "wasm32")))] const MAX_DECOMPRESSED_SIZE: usize = 8 * 1024 * 1024; @@ -541,8 +541,7 @@ impl SpinPlatformHttpClient { .iter() .map(|(name, value)| (name.to_string(), value.as_bytes().to_vec())) .collect(); - let body = spin_response - .into_body() + let body = http_body_util::Limited::new(spin_response.into_body(), MAX_DECOMPRESSED_SIZE) .collect() .await .map_err(|error| { @@ -649,25 +648,8 @@ impl PlatformHttpClient for SpinPlatformHttpClient { Report::new(PlatformError::HttpClient) .attach("unexpected pending request type in deadline selection") })?; - // Probe only the task's buffered result. This never polls transport I/O - // and preserves a completed bid when publisher-origin work ran long. - if let Some(ready) = pending.response.try_recv().map_err(|error| { - Report::new(PlatformError::HttpClient) - .attach(format!("dispatched response task stopped: {error}")) - })? { - return Ok(Some(PlatformSelectResult { - failed_backend_name: ready.as_ref().err().and(backend_name), - ready, - remaining: pending_requests, - })); - } let result = complete_before_deadline( - async { - (&mut pending.response).await.unwrap_or_else(|error| { - Err(Report::new(PlatformError::HttpClient) - .attach(format!("dispatched response task stopped: {error}"))) - }) - }, + &mut pending.response, async { let remaining = deadline.saturating_duration_since(web_time::Instant::now()); if !remaining.is_zero() { @@ -678,10 +660,16 @@ impl PlatformHttpClient for SpinPlatformHttpClient { web_time::Instant::now, ) .await; - Ok(result.map(|ready| PlatformSelectResult { - failed_backend_name: ready.as_ref().err().and(backend_name), - ready, - remaining: pending_requests, + Ok(result.map(|result| { + let ready = result.unwrap_or_else(|error| { + Err(Report::new(PlatformError::HttpClient) + .attach(format!("dispatched response task stopped: {error}"))) + }); + PlatformSelectResult { + failed_backend_name: ready.as_ref().err().and(backend_name), + ready, + remaining: pending_requests, + } })) } } @@ -835,10 +823,6 @@ mod tests { fn auction_http_capabilities_are_explicit() { let capabilities = AuctionTargetId::Spin.descriptor().capabilities(); assert!(!capabilities.supports_concurrent_provider_fanout()); - assert!( - !capabilities.has_enforceable_total_request_deadline(), - "Spin outbound HTTP does not expose an enforceable hard total request deadline" - ); } use edgezero_core::context::RequestContext; use edgezero_core::http::request_builder; @@ -1445,23 +1429,31 @@ mod tests { } #[cfg(any(test, all(feature = "spin", target_arch = "wasm32")))] -async fn complete_before_deadline( - operation: F, +async fn complete_before_deadline( + response: &mut futures::channel::oneshot::Receiver, timer: T, deadline: web_time::Instant, now: N, -) -> Option +) -> Option> where - F: std::future::Future, T: std::future::Future, N: Fn() -> web_time::Instant, { + // Inspect only the task's buffered result, including when collection starts + // after publisher-origin work has exhausted the auction budget. + match response.try_recv() { + Ok(Some(ready)) => return Some(Ok(ready)), + Err(error) => return Some(Err(error)), + Ok(None) => {} + } if now() >= deadline { return None; } - match futures::future::select(Box::pin(operation), Box::pin(timer)).await { - futures::future::Either::Left((response, _)) if now() < deadline => Some(response), - _ => None, + match futures::future::select(response, Box::pin(timer)).await { + futures::future::Either::Left((ready, _)) => Some(ready), + // The timer and task can become ready in the same executor turn. A final + // channel probe retains buffered results without polling transport I/O. + futures::future::Either::Right(((), response)) => response.try_recv().transpose(), } } @@ -1506,11 +1498,6 @@ mod deadline_tests { ); finish_body.send(42).expect("should finish response body"); executor.run_until_stalled(); - let expired = web_time::Instant::now(); - assert!( - web_time::Instant::now() >= expired, - "should collect after cutoff" - ); assert_eq!( pending .response @@ -1579,93 +1566,122 @@ mod deadline_tests { } #[test] - fn expired_deadline_does_not_poll_request() { + fn spin_body_collection_rejects_oversized_stream() { + let body = http_body_util::Full::new(bytes::Bytes::from(vec![ + 0; + super::MAX_DECOMPRESSED_SIZE + + 1 + ])); + let result = block_on(http_body_util::BodyExt::collect( + http_body_util::Limited::new(body, super::MAX_DECOMPRESSED_SIZE), + )); + assert!( + result.is_err(), + "should reject a response exceeding the streamed body cap" + ); + } + + #[test] + fn spin_body_collection_accepts_body_at_limit() { + let body = + http_body_util::Full::new(bytes::Bytes::from(vec![0; super::MAX_DECOMPRESSED_SIZE])); + let result = block_on(http_body_util::BodyExt::collect( + http_body_util::Limited::new(body, super::MAX_DECOMPRESSED_SIZE), + )) + .expect("should collect a body at the limit"); + assert_eq!( + result.to_bytes().len(), + super::MAX_DECOMPRESSED_SIZE, + "should retain the complete permitted body" + ); + } + + #[test] + fn expired_deadline_preserves_buffered_response() { + let (sender, mut response) = futures::channel::oneshot::channel(); + sender.send(42).expect("should buffer response"); let deadline = Instant::now(); - let polled = Cell::new(false); - let operation = async { - polled.set(true); - pending::<()>().await - }; - let result: Option<()> = block_on(complete_before_deadline( - operation, + let result = block_on(complete_before_deadline( + &mut response, pending(), deadline, || deadline, )); - assert_eq!(result, None, "should reject before polling expired request"); - assert!(!polled.get(), "should not poll an expired request"); + assert_eq!( + result, + Some(Ok(42)), + "should retain a buffered response after cutoff" + ); } #[test] - fn timer_cuts_off_pending_operation_and_drops_it() { - struct DropProbe(Rc>); - impl Drop for DropProbe { - fn drop(&mut self) { - self.0.set(true); - } - } - let dropped = Rc::new(Cell::new(false)); - let probe = DropProbe(Rc::clone(&dropped)); - let operation = async move { - let _probe = probe; - pending::<()>().await - }; + fn expired_deadline_returns_without_waiting_for_response() { + let (_sender, mut response) = futures::channel::oneshot::channel::<()>(); + let deadline = Instant::now(); + let result = block_on(complete_before_deadline( + &mut response, + pending(), + deadline, + || deadline, + )); + assert_eq!(result, None, "should stop waiting at an expired deadline"); + } + + #[test] + fn timer_cuts_off_pending_response() { + let (_sender, mut response) = futures::channel::oneshot::channel::<()>(); let now = Instant::now(); let result = block_on(complete_before_deadline( - operation, + &mut response, ready(()), now + Duration::from_secs(1), || now, )); - assert_eq!(result, None, "should cut off a pending request"); - assert!(dropped.get(), "should drop the pending operation at cutoff"); + assert_eq!(result, None, "should cut off an unbuffered response"); } #[test] - fn accepts_complete_response_before_deadline() { + fn closed_response_channel_reports_failure() { + let (sender, mut response) = futures::channel::oneshot::channel::<()>(); + drop(sender); let now = Instant::now(); let result = block_on(complete_before_deadline( - ready(42), + &mut response, pending(), now + Duration::from_secs(1), || now, )); - assert_eq!(result, Some(42), "should accept response before cutoff"); + assert!( + matches!(result, Some(Err(_))), + "should preserve a dispatched task failure" + ); } #[test] - fn rejects_response_that_becomes_ready_at_deadline() { + fn timer_rechecks_result_published_during_cutoff() { + let (sender, mut response) = futures::channel::oneshot::channel(); let now = Instant::now(); - let deadline = now + Duration::from_secs(1); let clock = Cell::new(now); - let operation = async { + let deadline = now + Duration::from_secs(1); + let timer = async { clock.set(deadline); - 42 + sender.send(42).expect("should publish at cutoff"); }; let result = block_on(complete_before_deadline( - operation, - pending(), + &mut response, + timer, deadline, || clock.get(), )); - assert_eq!(result, None, "should reject a late ready response"); - } - - #[test] - fn timer_bounds_body_after_headers_complete() { - let now = Instant::now(); - let headers_received = Cell::new(false); - let operation = async { - headers_received.set(true); - pending::<()>().await - }; - let result = block_on(complete_before_deadline( - operation, - ready(()), - now + Duration::from_secs(1), - || now, - )); - assert!(headers_received.get(), "should reach response body read"); - assert_eq!(result, None, "should cut off pending body collection"); + assert_eq!( + result, + Some(Ok(42)), + "should retain result buffered when timer wins" + ); + assert_eq!( + clock.get(), + deadline, + "should exercise the deadline boundary" + ); } } diff --git a/crates/trusted-server-adapter-spin/tests/runtime/deadline.rs b/crates/trusted-server-adapter-spin/tests/runtime/deadline.rs index b8b13dbeb4..95cd0013da 100644 --- a/crates/trusted-server-adapter-spin/tests/runtime/deadline.rs +++ b/crates/trusted-server-adapter-spin/tests/runtime/deadline.rs @@ -55,10 +55,25 @@ async fn exercise( .select_until(vec![pending], started + Duration::from_millis(budget_ms)) .await?; let status = match collected { - Some(result) => { - result.ready?; - "complete" - } + Some(result) => match result.ready { + Ok(response) => { + assert_eq!( + response.response.body().as_bytes(), + Some(b"okay".as_slice()), + "should retain the complete nonempty upstream body" + ); + "complete" + } + Err(error) if format!("{error:?}").contains("length limit exceeded") => { + assert_eq!( + result.failed_backend_name.as_deref(), + Some("fixture"), + "should attribute oversized response to its provider" + ); + "oversized" + } + Err(error) => return Err(error), + }, None => "timeout", }; Ok(format!("{status}:{}", started.elapsed().as_millis())) diff --git a/crates/trusted-server-core/src/auction/orchestrator.rs b/crates/trusted-server-core/src/auction/orchestrator.rs index 2368eddf81..befddc8c16 100644 --- a/crates/trusted-server-core/src/auction/orchestrator.rs +++ b/crates/trusted-server-core/src/auction/orchestrator.rs @@ -436,6 +436,10 @@ impl AuctionOrchestratorHarness { let remaining_ms = remaining_budget_ms(auction_start, context.timeout_ms); let logical_budget_ms = remaining_ms.min(provider.timeout_ms()); if logical_budget_ms == 0 { + log::debug!( + "Planned provider '{}' has no remaining auction budget before launch", + provider.provider_name() + ); responses.push(provider_timeout_response(provider.provider_name(), 0)); continue; } @@ -444,6 +448,10 @@ impl AuctionOrchestratorHarness { .backend() .canonicalize_transport_timeout_ms(logical_budget_ms, provider.timeout_ms()); if transport_timeout_ms == 0 { + log::debug!( + "Planned provider '{}' has no remaining transport budget before launch", + provider.provider_name() + ); responses.push(provider_timeout_response(provider.provider_name(), 0)); continue; } @@ -588,6 +596,10 @@ impl AuctionOrchestratorHarness { } } for state in launches.into_values() { + log::debug!( + "Planned provider '{}' exhausted its auction collection budget", + state.provider.provider_name() + ); responses.push(provider_timeout_response( state.provider.provider_name(), state.started_at.elapsed().as_millis() as u64, @@ -659,10 +671,19 @@ impl AuctionOrchestratorHarness { services: context.services, }; let mediator_start = Instant::now(); + let mediator_deadline = collection_deadline(auction_start, context.timeout_ms) + .min(collection_deadline(mediator_start, logical_budget_ms)); let mediated = match mediator .request_bids(original_request, &mediator_context) .await { + Ok(_) if Instant::now() >= mediator_deadline => { + log::debug!( + "Mediator '{}' exhausted its response budget during launch; using local ranking", + mediator.provider_name() + ); + None + } Ok(ProviderRequestOutcome::Immediate(response)) => Some(response), Ok(ProviderRequestOutcome::Pending { request: pending, @@ -670,18 +691,14 @@ impl AuctionOrchestratorHarness { }) => match context .services .http_client() - .wait_until( - pending, - collection_deadline(auction_start, context.timeout_ms) - .min(collection_deadline(mediator_start, logical_budget_ms)), - ) + .wait_until(pending, mediator_deadline) .await { Ok(platform_response) => { let response_time_ms = mediator_start.elapsed().as_millis() as u64; - if remaining_budget_ms(auction_start, context.timeout_ms) == 0 { - log::warn!( - "Planned mediator '{}' completed after the hard auction deadline; using local ranking ({}ms)", + if Instant::now() >= mediator_deadline { + log::debug!( + "Planned mediator '{}' completed after its response deadline; using local ranking ({}ms)", mediator.provider_name(), response_time_ms ); @@ -707,11 +724,21 @@ impl AuctionOrchestratorHarness { } } Err(error) => { - log::warn!( - "Planned mediator '{}' request failed: {:?}", - mediator.provider_name(), - error - ); + if matches!( + error.current_context(), + crate::platform::PlatformError::DeadlineExceeded + ) { + log::debug!( + "Planned mediator '{}' exhausted its response budget; using local ranking", + mediator.provider_name() + ); + } else { + log::warn!( + "Planned mediator '{}' request failed: {:?}", + mediator.provider_name(), + error + ); + } None } }, @@ -978,12 +1005,29 @@ impl AuctionOrchestrator { }; let start_time = Instant::now(); - let mediator_resp = match mediator + let mediator_deadline = collection_deadline(mediation_start, context.timeout_ms) + .min(collection_deadline(start_time, logical_budget_ms)); + let mediator_outcome = mediator .request_bids(request, &mediator_context) .await .change_context(TrustedServerError::Auction { message: format!("Mediator {} failed to launch", mediator.provider_name()), - })? { + })?; + if Instant::now() >= mediator_deadline { + log::debug!( + "Mediator '{}' exhausted its response budget during launch; using local ranking", + mediator.provider_name() + ); + let winning_bids = self.select_winning_bids(&provider_responses, &floor_prices); + return Ok(OrchestrationResult { + provider_responses, + mediator_response: None, + winning_bids, + total_time_ms: mediation_start.elapsed().as_millis() as u64, + metadata: HashMap::new(), + }); + } + let mediator_resp = match mediator_outcome { ProviderRequestOutcome::Immediate(response) => response, ProviderRequestOutcome::Pending { request: pending, @@ -992,11 +1036,7 @@ impl AuctionOrchestrator { let platform_resp = mediator_context .services .http_client() - .wait_until( - pending, - collection_deadline(mediation_start, context.timeout_ms) - .min(collection_deadline(start_time, logical_budget_ms)), - ) + .wait_until(pending, mediator_deadline) .await; if platform_resp.as_ref().is_err_and(|error| { matches!( @@ -1022,9 +1062,9 @@ impl AuctionOrchestrator { ), })?; let response_time_ms = start_time.elapsed().as_millis() as u64; - if remaining_budget_ms(mediation_start, context.timeout_ms) == 0 { - log::warn!( - "Mediator '{}' completed after the hard auction deadline; using local ranking ({}ms)", + if Instant::now() >= mediator_deadline { + log::debug!( + "Mediator '{}' completed after its response deadline; using local ranking ({}ms)", mediator.provider_name(), response_time_ms ); @@ -1033,7 +1073,7 @@ impl AuctionOrchestrator { provider_responses, mediator_response: None, winning_bids: winning, - total_time_ms: 0, + total_time_ms: mediation_start.elapsed().as_millis() as u64, metadata: HashMap::new(), }); } @@ -1714,6 +1754,10 @@ impl AuctionOrchestrator { let logical_budget_ms = remaining_budget_ms(auction_start, context.timeout_ms).min(provider.timeout_ms()); if logical_budget_ms == 0 { + log::debug!( + "Planned provider '{}' has no remaining auction budget before launch", + provider.provider_name() + ); completed_responses.push(provider_timeout_response(provider.provider_name(), 0)); continue; } @@ -1722,6 +1766,10 @@ impl AuctionOrchestrator { .backend() .canonicalize_transport_timeout_ms(logical_budget_ms, provider.timeout_ms()); if transport_timeout_ms == 0 { + log::debug!( + "Planned provider '{}' has no remaining transport budget before launch", + provider.provider_name() + ); completed_responses.push(provider_timeout_response(provider.provider_name(), 0)); continue; } @@ -2286,6 +2334,10 @@ impl AuctionOrchestrator { } backend_to_provider.clear(); for state in planned_backend_to_provider.into_values() { + log::debug!( + "Planned provider '{}' exhausted its dispatched auction collection budget", + state.provider.provider_name() + ); responses.push(provider_timeout_response( state.provider.provider_name(), state.started_at.elapsed().as_millis() as u64, @@ -2386,33 +2438,33 @@ impl AuctionOrchestrator { provider_responses: Some(&responses), services: context.services, }; + let mediator_deadline = collection_deadline(auction_start, timeout_ms) + .min(collection_deadline(mediator_start, logical_budget_ms)); let mediator_response = match mediator .request_bids(&request, &mediator_context) .await { + Ok(_) if Instant::now() >= mediator_deadline => { + log::debug!( + "Mediator '{}' exhausted its response budget during launch; using local ranking", + mediator.provider_name() + ); + None + } Ok(ProviderRequestOutcome::Immediate(response)) => Some(response), Ok(ProviderRequestOutcome::Pending { request: pending, parse_state, }) => match services .http_client() - .wait_until( - pending, - collection_deadline(auction_start, timeout_ms) - .min(collection_deadline(mediator_start, logical_budget_ms)), - ) + .wait_until(pending, mediator_deadline) .await - .change_context(TrustedServerError::Auction { - message: format!( - "Mediator {} request failed", - mediator.provider_name() - ), - }) { + { Ok(platform_resp) => { let response_time_ms = mediator_start.elapsed().as_millis() as u64; - if remaining_budget_ms(auction_start, timeout_ms) == 0 { - log::warn!( - "Mediator '{}' completed after the hard auction deadline; using local ranking ({}ms)", + if Instant::now() >= mediator_deadline { + log::debug!( + "Mediator '{}' completed after its response deadline; using local ranking ({}ms)", mediator.provider_name(), response_time_ms ); @@ -2441,7 +2493,17 @@ impl AuctionOrchestrator { } } Err(error) => { - log::warn!("Mediator request failed: {:?}", error); + if matches!( + error.current_context(), + crate::platform::PlatformError::DeadlineExceeded + ) { + log::debug!( + "Mediator '{}' exhausted its response budget; using local ranking", + mediator.provider_name() + ); + } else { + log::warn!("Mediator request failed: {:?}", error); + } None } }, @@ -3283,6 +3345,55 @@ mod tests { } } + struct LateReadyMediatorHttpClient { + inner: Arc, + waits: Arc, + } + + #[async_trait::async_trait(?Send)] + impl PlatformHttpClient for LateReadyMediatorHttpClient { + async fn send( + &self, + request: PlatformHttpRequest, + ) -> Result> { + self.inner.send(request).await + } + + async fn send_async( + &self, + request: PlatformHttpRequest, + ) -> Result> { + self.inner.send_async(request).await + } + + async fn select( + &self, + pending: Vec, + ) -> Result> { + self.inner.select(pending).await + } + + async fn select_until( + &self, + pending: Vec, + deadline: Instant, + ) -> Result, Report> { + self.inner.select_until(pending, deadline).await + } + + async fn wait_until( + &self, + pending: PlatformPendingRequest, + _deadline: Instant, + ) -> Result> { + self.waits.fetch_add(1, Ordering::Relaxed); + // Model an executor waking after the mediator cutoff with a complete + // result returned by the adapter's final nonblocking snapshot. + std::thread::sleep(Duration::from_millis(40)); + self.inner.wait(pending).await + } + } + struct UnusedSecretStore; impl PlatformSecretStore for UnusedSecretStore { @@ -3503,6 +3614,62 @@ mod tests { } } + struct SlowLaunchMediator { + immediate: bool, + slow_launch: bool, + parses: Arc, + } + + #[async_trait::async_trait(?Send)] + impl AuctionProvider for SlowLaunchMediator { + fn provider_name(&self) -> &str { + "pending-deadline-mediator" + } + + async fn request_bids( + &self, + request: &AuctionRequest, + context: &AuctionContext<'_>, + ) -> Result> { + let outcome = if self.immediate { + ProviderRequestOutcome::Immediate(AuctionResponse::success( + self.provider_name(), + vec![auction_bid("mediated", 9.0)], + 0, + )) + } else { + PendingDeadlineMediator(20) + .request_bids(request, context) + .await? + }; + // Simulate launch work that outlives the mediator's own budget while + // a complete transport result is already buffered. + if self.slow_launch { + std::thread::sleep(Duration::from_millis(40)); + } + Ok(outcome) + } + + async fn parse_response( + &self, + response: PlatformResponse, + response_time_ms: u64, + ) -> Result> { + self.parses.fetch_add(1, Ordering::Relaxed); + PendingDeadlineMediator(20) + .parse_response(response, response_time_ms) + .await + } + + fn timeout_ms(&self) -> u32 { + 20 + } + + fn backend_name(&self, _services: &RuntimeServices, _timeout_ms: u32) -> Option { + Some("pending-mediator-backend".to_string()) + } + } + #[async_trait::async_trait(?Send)] impl AuctionProvider for DeadlineRecordingMediator { fn provider_name(&self) -> &str { @@ -4008,12 +4175,8 @@ mod tests { } } - async fn collect_deadline_test_result( - split: bool, - enforceable_total_request_deadline: bool, - ) -> OrchestrationResult { + async fn collect_deadline_test_result(split: bool) -> OrchestrationResult { let stub = Arc::new(StubHttpClient::new()); - stub.set_enforceable_total_request_deadline(enforceable_total_request_deadline); stub.push_response(200, b"{}".to_vec()); stub.push_response(200, b"{}".to_vec()); stub.push_select_delay(Duration::from_millis(200)); @@ -4065,7 +4228,7 @@ mod tests { #[tokio::test] async fn current_adapter_deadline_stops_collection_in_both_paths() { for split in [false, true] { - let result = collect_deadline_test_result(split, false).await; + let result = collect_deadline_test_result(split).await; assert_eq!( result.provider_responses.len(), 2, @@ -4090,27 +4253,8 @@ mod tests { } } - #[tokio::test] - async fn advertised_transport_deadline_still_bounds_collection_in_both_paths() { - for split in [false, true] { - let result = collect_deadline_test_result(split, true).await; - assert!( - result - .provider_responses - .iter() - .all(|response| response.metadata["error_type"] == ERROR_TYPE_TIMEOUT), - "should classify both outstanding requests as timeouts" - ); - assert!(result.winning_bids.is_empty(), "should discard late bids"); - } - } - - async fn pending_mediator_deadline_test_result( - split: bool, - enforceable_total_request_deadline: bool, - ) -> OrchestrationResult { + async fn pending_mediator_deadline_test_result(split: bool) -> OrchestrationResult { let stub = Arc::new(StubHttpClient::new()); - stub.set_enforceable_total_request_deadline(enforceable_total_request_deadline); stub.push_response(200, b"{}".to_vec()); stub.push_response(200, b"{}".to_vec()); stub.push_select_delay(Duration::ZERO); @@ -4161,22 +4305,19 @@ mod tests { #[tokio::test] async fn pending_mediator_is_bounded_in_sync_and_split_paths() { for split in [false, true] { - for advertised_deadline in [false, true] { - let result = - pending_mediator_deadline_test_result(split, advertised_deadline).await; - assert!( - result.mediator_response.is_none(), - "should abandon a late mediator" - ); - assert_eq!( - result.winning_bids["slot-1"].bidder, "local", - "should retain provider winner" - ); - assert!( - result.total_time_ms < 150, - "should not wait for the delayed mediator" - ); - } + let result = pending_mediator_deadline_test_result(split).await; + assert!( + result.mediator_response.is_none(), + "should abandon a late mediator" + ); + assert_eq!( + result.winning_bids["slot-1"].bidder, "local", + "should retain provider winner" + ); + assert!( + result.total_time_ms < 150, + "should not wait for the delayed mediator" + ); } } @@ -6029,13 +6170,11 @@ mod tests { } async fn planned_pending_mediator_deadline_result( - enforceable_total_request_deadline: bool, auction_timeout_ms: u32, mediator_timeout_ms: u32, policy: BackendNamingPolicy, ) -> OrchestrationResult { let http = Arc::new(StubHttpClient::new()); - http.set_enforceable_total_request_deadline(enforceable_total_request_deadline); http.push_response( 200, serde_json::to_vec(&serde_json::json!({ @@ -6084,35 +6223,27 @@ mod tests { #[tokio::test] async fn planned_pending_mediator_observes_absolute_deadline() { - for advertised_deadline in [false, true] { - let result = planned_pending_mediator_deadline_result( - advertised_deadline, - 20, - 1_000, - BackendNamingPolicy::Axum, - ) - .await; - assert!( - result.mediator_response.is_none(), - "should stop waiting for mediator" - ); - assert_eq!( - result.winning_bids["fictional-slot"].bid_id.as_deref(), - Some("provider"), - "should preserve on-time provider bids" - ); - assert!( - result.total_time_ms < 150, - "should bound full mediator response wait" - ); - } + let result = + planned_pending_mediator_deadline_result(20, 1_000, BackendNamingPolicy::Axum).await; + assert!( + result.mediator_response.is_none(), + "should stop waiting for mediator" + ); + assert_eq!( + result.winning_bids["fictional-slot"].bid_id.as_deref(), + Some("provider"), + "should preserve on-time provider bids" + ); + assert!( + result.total_time_ms < 150, + "should bound full mediator response wait" + ); } #[tokio::test] async fn planned_pending_mediator_observes_own_budget_before_auction_deadline() { let result = - planned_pending_mediator_deadline_result(false, 1_000, 20, BackendNamingPolicy::Fastly) - .await; + planned_pending_mediator_deadline_result(1_000, 20, BackendNamingPolicy::Fastly).await; assert!( result.mediator_response.is_none(), "should stop at the mediator budget before the auction deadline" @@ -6128,6 +6259,130 @@ mod tests { ); } + #[tokio::test] + async fn mediator_own_deadline_rejects_ready_results_after_launch_or_wait() { + for planned in [false, true] { + for split in [false, true] { + for (immediate, slow_launch) in [(false, true), (true, true), (false, false)] { + let http = Arc::new(StubHttpClient::new()); + http.push_response( + 200, + serde_json::to_vec(&serde_json::json!({ + "seatbid": [{"seat": "provider-seat", "bid": [{ + "id": "provider", "impid": "fictional-slot", "price": 2.0, + "adm": "
provider
", "w": 300, "h": 250 + }]}] + })) + .expect("should serialize provider response"), + ); + if !immediate { + http.push_response(200, b"{}".to_vec()); + } + let waits = Arc::new(AtomicUsize::new(0)); + let client: Arc = if slow_launch { + http + } else { + Arc::new(LateReadyMediatorHttpClient { + inner: http, + waits: Arc::clone(&waits), + }) + }; + let services = build_services_with_backend_and_http_client( + Arc::new(NamingBackend::new(BackendNamingPolicy::Axum)), + client, + ); + let parses = Arc::new(AtomicUsize::new(0)); + let mediator = Arc::new(SlowLaunchMediator { + immediate, + slow_launch, + parses: Arc::clone(&parses), + }); + let orchestrator = if planned { + let plan = AuctionPlan::compile(planned_config( + &[("provider-a", RoutingMode::AllEligible)], + false, + )) + .expect("should compile planned auction"); + AuctionOrchestrator::from_plan(Arc::new(plan), Some(mediator)) + } else { + let mut orchestrator = AuctionOrchestrator::new(AuctionConfig { + enabled: true, + providers: AuctionConfig::legacy_provider_map(&["local"]), + mediator: Some("pending-deadline-mediator".to_string()), + timeout_ms: 1_000, + ..Default::default() + }); + orchestrator.register_provider(Arc::new(DeadlineBidProvider { + name: "local", + backend: "local-backend", + })); + orchestrator.register_provider(mediator); + orchestrator + }; + let request = if planned { + planned_request() + } else { + create_test_auction_request() + }; + let settings = create_test_settings(); + let inbound = http::Request::new(edgezero_core::body::Body::empty()); + let context = AuctionContext { + settings: &settings, + request: &inbound, + timeout_ms: 1_000, + transport_timeout_ms: 1_000, + provider_responses: None, + services: &services, + }; + let result = if split { + let DispatchAuctionOutcome::Dispatched(dispatched) = + orchestrator.dispatch_auction(&request, &context).await + else { + panic!("should dispatch provider"); + }; + orchestrator + .collect_dispatched_auction(dispatched, &services, &context) + .await + } else { + orchestrator + .run_auction(&request, &context) + .await + .expect("should complete auction") + }; + assert_eq!( + result.provider_responses.len(), + 1, + "should retain the on-time provider result" + ); + assert!( + result.mediator_response.is_none(), + "should reject expired mediator: planned={planned} split={split} immediate={immediate}" + ); + assert_eq!( + parses.load(Ordering::Relaxed), + 0, + "should reject expiry before parsing" + ); + assert_eq!( + waits.load(Ordering::Relaxed), + usize::from(!slow_launch), + "should exercise the intended launch or wait boundary" + ); + assert!( + result.total_time_ms >= 40, + "should report elapsed auction time after mediator expiry" + ); + let slot = if planned { "fictional-slot" } else { "slot-1" }; + assert_eq!( + result.winning_bids[slot].price, + Some(if planned { 2.0 } else { 3.0 }), + "should preserve the local winner" + ); + } + } + } + } + #[tokio::test] async fn planned_executor_mediator_transport_failure_falls_back_locally() { let http = Arc::new(StubHttpClient::new()); diff --git a/crates/trusted-server-core/src/platform/backend_naming.rs b/crates/trusted-server-core/src/platform/backend_naming.rs index 54fc176b1b..52487def53 100644 --- a/crates/trusted-server-core/src/platform/backend_naming.rs +++ b/crates/trusted-server-core/src/platform/backend_naming.rs @@ -189,7 +189,6 @@ impl AuctionTargetId { naming_policy: BackendNamingPolicy::Fastly, capabilities: AuctionTargetCapabilities { concurrent_provider_fanout: true, - enforceable_total_request_deadline: false, }, }, Self::Axum => AuctionTargetDescriptor { @@ -197,7 +196,6 @@ impl AuctionTargetId { naming_policy: BackendNamingPolicy::Axum, capabilities: AuctionTargetCapabilities { concurrent_provider_fanout: true, - enforceable_total_request_deadline: false, }, }, Self::Cloudflare => AuctionTargetDescriptor { @@ -205,7 +203,6 @@ impl AuctionTargetId { naming_policy: BackendNamingPolicy::Cloudflare, capabilities: AuctionTargetCapabilities { concurrent_provider_fanout: false, - enforceable_total_request_deadline: false, }, }, Self::Spin => AuctionTargetDescriptor { @@ -213,7 +210,6 @@ impl AuctionTargetId { naming_policy: BackendNamingPolicy::Spin, capabilities: AuctionTargetCapabilities { concurrent_provider_fanout: false, - enforceable_total_request_deadline: false, }, }, } @@ -224,7 +220,6 @@ impl AuctionTargetId { #[derive(Debug, Clone, Copy, Eq, PartialEq)] pub struct AuctionTargetCapabilities { concurrent_provider_fanout: bool, - enforceable_total_request_deadline: bool, } impl AuctionTargetCapabilities { @@ -233,15 +228,6 @@ impl AuctionTargetCapabilities { pub const fn supports_concurrent_provider_fanout(self) -> bool { self.concurrent_provider_fanout } - - /// Return whether the adapter enforces a total request deadline per provider. - /// - /// Transport first-byte or between-byte timers do not satisfy this - /// capability because they do not cap the complete request lifetime. - #[must_use] - pub const fn has_enforceable_total_request_deadline(self) -> bool { - self.enforceable_total_request_deadline - } } /// Shared target descriptor used by plan validation. @@ -618,12 +604,6 @@ mod tests { "should declare fanout accurately for {}", id.adapter_id() ); - assert!( - !descriptor - .capabilities() - .has_enforceable_total_request_deadline(), - "no current adapter should claim a total request deadline" - ); assert_eq!(AuctionTargetId::from_adapter_id(id.adapter_id()), Some(id)); } assert_eq!(AuctionTargetId::from_adapter_id("FASTLY"), None); diff --git a/crates/trusted-server-core/src/platform/http.rs b/crates/trusted-server-core/src/platform/http.rs index 1d5b9c338a..cdc23f8cb0 100644 --- a/crates/trusted-server-core/src/platform/http.rs +++ b/crates/trusted-server-core/src/platform/http.rs @@ -373,16 +373,6 @@ pub trait PlatformHttpClient: Send + Sync { true } - /// Whether the adapter enforces a hard total deadline for each request. - /// - /// First-byte and between-byte timers do not qualify: connection setup or a - /// byte-trickling response can still overrun the logical auction budget. The - /// capability describes unbounded sends, independently of the stricter - /// [`Self::select_until`] contract used for auction collection. - fn has_enforceable_total_request_deadline(&self) -> bool { - false - } - /// Whether [`send`](Self::send) can preserve upstream response bodies as /// [`Body::Stream`](edgezero_core::body::Body::Stream) when requested via /// [`PlatformHttpRequest::with_stream_response`]. @@ -452,15 +442,11 @@ pub trait PlatformHttpClient: Send + Sync { async fn select_until( &self, pending_requests: Vec, - deadline: Instant, + _deadline: Instant, ) -> Result, Report> { drop(pending_requests); - if Instant::now() >= deadline { - Ok(None) - } else { - Err(Report::new(PlatformError::Unsupported) - .attach("absolute-deadline HTTP collection is unavailable")) - } + Err(Report::new(PlatformError::Unsupported) + .attach("absolute-deadline HTTP collection is unavailable")) } /// Waits for a complete buffered response before an absolute deadline. @@ -539,11 +525,11 @@ mod tests { vec![PlatformPendingRequest::new(DropProbe(Arc::clone(&dropped)))], Instant::now(), )) - .expect("should classify expiry without polling"); - assert!( - response.is_none(), - "should expire before accepting ready input" - ); + .expect_err("should reject unsupported collection even after expiry"); + assert!(matches!( + response.current_context(), + PlatformError::Unsupported + )); assert_eq!( dropped.load(Ordering::SeqCst), 1, @@ -565,14 +551,14 @@ mod tests { } #[test] - fn bounded_wait_reports_deadline_exceeded() { + fn unsupported_bounded_wait_reports_unsupported() { let error = futures::executor::block_on( UnavailableHttpClient.wait_until(PlatformPendingRequest::new(()), Instant::now()), ) - .expect_err("should report an expired wait"); + .expect_err("should reject an unsupported wait"); assert!( - matches!(error.current_context(), PlatformError::DeadlineExceeded), - "should distinguish expiry from transport failure" + matches!(error.current_context(), PlatformError::Unsupported), + "should preserve the unsupported collection contract" ); } diff --git a/crates/trusted-server-core/src/platform/test_support.rs b/crates/trusted-server-core/src/platform/test_support.rs index 1b74b2267f..40326473eb 100644 --- a/crates/trusted-server-core/src/platform/test_support.rs +++ b/crates/trusted-server-core/src/platform/test_support.rs @@ -248,9 +248,6 @@ pub(crate) struct StubHttpClient { // Reported by supports_concurrent_fanout(); set false to emulate // platforms whose collection contract permits only a single provider. concurrent_fanout: std::sync::atomic::AtomicBool, - // Reported by has_enforceable_total_request_deadline(); set true to emulate - // unbounded request transport with a hard total request deadline. - enforceable_total_request_deadline: std::sync::atomic::AtomicBool, // Reported by supports_streaming_responses(); set true to emulate Fastly's // streaming response support. streaming_responses_supported: std::sync::atomic::AtomicBool, @@ -284,7 +281,6 @@ impl StubHttpClient { pending_backend_name_overrides: Mutex::new(VecDeque::new()), select_delays: Mutex::new(VecDeque::new()), concurrent_fanout: std::sync::atomic::AtomicBool::new(true), - enforceable_total_request_deadline: std::sync::atomic::AtomicBool::new(false), streaming_responses_supported: std::sync::atomic::AtomicBool::new(false), pending_streaming_responses_supported: std::sync::atomic::AtomicBool::new(false), image_optimizer_options: Mutex::new(Vec::new()), @@ -302,12 +298,6 @@ impl StubHttpClient { .store(supported, std::sync::atomic::Ordering::Relaxed); } - /// Make `has_enforceable_total_request_deadline()` report the given value. - pub(crate) fn set_enforceable_total_request_deadline(&self, supported: bool) { - self.enforceable_total_request_deadline - .store(supported, std::sync::atomic::Ordering::Relaxed); - } - /// Make `supports_streaming_responses()` report the given value. pub fn set_streaming_responses_supported(&self, supported: bool) { self.streaming_responses_supported @@ -500,11 +490,6 @@ impl PlatformHttpClient for StubHttpClient { .load(std::sync::atomic::Ordering::Relaxed) } - fn has_enforceable_total_request_deadline(&self) -> bool { - self.enforceable_total_request_deadline - .load(std::sync::atomic::Ordering::Relaxed) - } - fn supports_streaming_responses(&self) -> bool { self.streaming_responses_supported .load(std::sync::atomic::Ordering::Relaxed) @@ -763,6 +748,9 @@ impl PlatformHttpClient for StubHttpClient { .front() .copied() .unwrap_or_default(); + if delay.is_zero() { + return self.wait(pending).await; + } if remaining.is_zero() || delay >= remaining { self.wait_delays .lock() @@ -1151,6 +1139,29 @@ mod tests { use super::*; + #[test] + fn bounded_stub_wait_preserves_an_already_ready_response_after_expiry() { + let stub = StubHttpClient::new(); + stub.push_response(200, b"ready".to_vec()); + let pending = futures::executor::block_on( + stub.send_async(PlatformHttpRequest::new( + request_builder() + .uri("https://example.com/ready") + .body(Body::empty()) + .expect("should build request"), + "ready-backend", + )), + ) + .expect("should dispatch ready response"); + let response = futures::executor::block_on(stub.wait_until(pending, Instant::now())) + .expect("should retain the ready response after expiry"); + assert_eq!( + response.response.status().as_u16(), + 200, + "should preserve ready status" + ); + } + #[test] fn stub_http_client_records_send_calls_and_returns_canned_response() { let stub = StubHttpClient::new(); diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index e5ac30009e..c4025566f2 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -21122,6 +21122,8 @@ mod tests { ); } + // Manually polled fixture for publisher injection order, not timer scheduling. + // Adapter runtime tests cover autonomous deadline wakeups and cancellation. struct GatedAuctionHttpClient { inner: StubHttpClient, released: std::sync::atomic::AtomicBool, @@ -21154,11 +21156,23 @@ mod tests { Option, Report, > { - if web_time::Instant::now() >= deadline { - return Ok(None); + self.collections.fetch_add(1, Ordering::SeqCst); + let ready = futures::future::poll_fn(|_| { + // The gate models completion, so probe it before checking expiry. + if self.released.load(Ordering::SeqCst) { + std::task::Poll::Ready(true) + } else if web_time::Instant::now() >= deadline { + std::task::Poll::Ready(false) + } else { + std::task::Poll::Pending + } + }) + .await; + if ready { + self.inner.select_until(pending, deadline).await + } else { + Ok(None) } - let result = self.select(pending).await?; - Ok((web_time::Instant::now() < deadline).then_some(result)) } async fn select( diff --git a/crates/trusted-server-core/src/test_support.rs b/crates/trusted-server-core/src/test_support.rs index ef7391ce20..02afbccb1b 100644 --- a/crates/trusted-server-core/src/test_support.rs +++ b/crates/trusted-server-core/src/test_support.rs @@ -297,13 +297,10 @@ pub mod nextjs_auction { async fn select_until( &self, pending_requests: Vec, - deadline: web_time::Instant, + _deadline: web_time::Instant, ) -> Result, Report> { - if web_time::Instant::now() >= deadline { - return Ok(None); - } - let response = self.select(pending_requests).await?; - Ok((web_time::Instant::now() < deadline).then_some(response)) + // All fixture responses are deterministic and already available. + self.select(pending_requests).await.map(Some) } async fn select( diff --git a/docs/guide/auction-orchestration.md b/docs/guide/auction-orchestration.md index e220428061..bdbb8789bc 100644 --- a/docs/guide/auction-orchestration.md +++ b/docs/guide/auction-orchestration.md @@ -824,6 +824,8 @@ not bid eligibility based on an exact server-side arrival time. A mediator start only with remaining budget and uses the earlier of the auction cutoff and its own logical timeout. +Provider bids complete at the final nonblocking snapshot remain eligible even after `tmax`; mediation starts only with remaining budget and rejects responses collected after its cutoff. + Personalized auction and mediator requests explicitly bypass intermediary caches. Fastly uses the SDK's direct pending-request handles and body readiness handles in a timed wait; it neither polls headers every millisecond nor waits on an empty @@ -831,7 +833,7 @@ handle list. Retained partial payloads share a 10 MiB budget. Under memory press the largest partial response is rejected with its own backend attribution so a large stalled provider cannot make a small healthy sibling take the failure. Completed payloads move directly into the platform body without a host-buffer -round trip. This cap covers retained payloads, not every allocator or host buffer. +round trip. This cap covers retained payloads, not every allocator or host buffer. The final nonblocking body drain is bounded by this payload limit; its local CPU work can extend beyond the network cutoff. On expiry Fastly closes acquired response bodies and abandons unfinished requests; the installed SDK has no pending-request cancellation API. Axum aborts outstanding diff --git a/docs/guide/configuration.md b/docs/guide/configuration.md index af9e82dd4f..237ec714b2 100644 --- a/docs/guide/configuration.md +++ b/docs/guide/configuration.md @@ -2244,6 +2244,8 @@ dispatch, including headers and complete response bodies. One nonblocking snapsh retains complete available responses after a slow origin; unfinished providers become timeouts and mediation cannot start with an exhausted budget. Backend timeout rounding does not extend network waiting. +Provider bids complete at the final nonblocking snapshot remain eligible even after `tmax`; mediation starts only with remaining budget and rejects responses collected after its cutoff. + Fastly abandons pending requests; Axum, Cloudflare, and Spin cancel outstanding operations where supported. Local processing and delivery can finish after the network cutoff. See [Timeout behavior](/guide/auction-orchestration#timeout-behavior) diff --git a/scripts/test-worker-auction-deadlines.mjs b/scripts/test-worker-auction-deadlines.mjs index a81ee71b62..f3cbad36ab 100644 --- a/scripts/test-worker-auction-deadlines.mjs +++ b/scripts/test-worker-auction-deadlines.mjs @@ -116,6 +116,12 @@ const origin = http.createServer(async (request, response) => { if (request.url === '/headers') { await delay(700, undefined, { signal: cancellation.signal }) } + if (request.url === '/large') { + const body = Buffer.alloc(8 * 1024 * 1024 + 1) + response.writeHead(200, { 'Content-Length': String(body.length) }) + response.end(body) + return + } response.writeHead(200, { 'Content-Length': '4' }) if (request.url === '/body') { response.write('a') @@ -248,6 +254,12 @@ allowed_outbound_hosts = ["http://127.0.0.1:*"] ['/body', 80, 0, 'timeout', 350], ['/headers', 80, 200, 'timeout', 500], ] + if (adapter === 'spin') cases.push(['/large', 2000, 0, 'oversized', 2500]) + assert.equal( + cases.length, + adapter === 'spin' ? 5 : 4, + 'should exercise every runtime case' + ) for (const [endpoint, budget, originDelay, expected, maximum] of cases) { observed.delete(endpoint) const uri = `http://127.0.0.1:${port}/?upstream=http://127.0.0.1:${originPort}${endpoint}&budget_ms=${budget}&origin_ms=${originDelay}` From 879693b0e13d0e1a996125990f70536a2367d9c5 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Thu, 22 Oct 2026 20:55:04 +0530 Subject: [PATCH 7/9] Install Clippy for worker runtime fixture checks --- .github/workflows/worker-auction-deadlines.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/worker-auction-deadlines.yml b/.github/workflows/worker-auction-deadlines.yml index cd1f4faa8b..dda7e26e8d 100644 --- a/.github/workflows/worker-auction-deadlines.yml +++ b/.github/workflows/worker-auction-deadlines.yml @@ -34,6 +34,7 @@ jobs: uses: ./.github/actions/setup-rust with: target: ${{ matrix.adapter == 'cloudflare' && 'wasm32-unknown-unknown' || 'wasm32-wasip1' }} + components: clippy cache-key: worker-deadline-${{ matrix.adapter }} - name: Install JS dependencies working-directory: crates/trusted-server-js/lib From 5728378981619994b6575899c75cbe2f5f14c3b4 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Thu, 29 Oct 2026 22:21:00 +0530 Subject: [PATCH 8/9] Share worker deadlines and align auction tests Share pending-operation ownership and deadline channel collection between Cloudflare and Spin while retaining runtime timers and cancellation. Require cache bypass in bounded HTTP test collection like Fastly, preserve healthy siblings, and update auction test providers. Replace four elapsed wall-clock assertions with checks that unbounded work was skipped. Clarify bid eligibility and coordinated Fastly SDK updates. --- CHANGELOG.md | 4 +- CONTRIBUTING.md | 8 + .../src/platform.rs | 314 ++--------------- .../src/platform.rs | 290 ++-------------- .../src/auction/endpoints.rs | 8 +- .../src/auction/orchestrator.rs | 59 ++-- .../src/auction/provider.rs | 3 + .../trusted-server-core/src/platform/mod.rs | 2 + .../src/platform/pending_response.rs | 323 ++++++++++++++++++ .../src/platform/test_support.rs | 180 +++++++++- crates/trusted-server-core/src/publisher.rs | 23 +- docs/guide/auction-orchestration.md | 7 +- docs/guide/configuration.md | 4 +- 13 files changed, 612 insertions(+), 613 deletions(-) create mode 100644 crates/trusted-server-core/src/platform/pending_response.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 0762d86cb4..9cfb4089a3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,7 +13,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 measured from dispatch, including delayed headers and trickling bodies on all adapters. Complete available responses survive a slow publisher origin; unfinished providers time out and mediation is skipped after budget exhaustion. - Provider bids complete at the final nonblocking snapshot remain eligible even after `tmax`; mediation starts only with remaining budget and rejects responses collected after its cutoff. + Provider bids that are already complete at the final nonblocking snapshot remain + eligible even after `tmax`; mediation starts only with remaining budget and + rejects responses collected after its cutoff. Fastly abandons unfinished requests; Axum, Cloudflare, and Spin cancel owned work. - **Breaking:** `x-ts-version` now reports the deployed git version — the release tag, else the branch, else the first 6 characters of the commit — compiled in from the build-time `TRUSTED_SERVER__GIT_VERSION` (set by the deploy pipeline) or local git, and is sent by every adapter, including on the Fastly `GET /health` probe. The Fastly service version it previously carried is now `x-ts-fastly-version`; update dashboards, monitors, and scripts that read `x-ts-version` as the Fastly version number. Builds with neither the override nor git omit the header. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index cde75feaca..f2de4800c9 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -146,6 +146,14 @@ link it in the description, and mark it ready for review again. - Update the canonical guide or crate README when a user-visible contract changes; do not copy volatile matrices into multiple documents. +## Fastly dependency updates + +Update the workspace `fastly` and `fastly-sys` dependencies together. The +Fastly deadline collector uses raw asynchronous I/O readiness calls from +`fastly-sys` with handles created by `fastly`, so both must use a compatible +SDK ABI. Check the resolved versions in `Cargo.lock` and run the Fastly +adapter tests, including the real deadline network fixture, after an SDK update. + ## Verify Run the target-matched checks in [AGENTS.md](AGENTS.md#ci-gates), the diff --git a/crates/trusted-server-adapter-cloudflare/src/platform.rs b/crates/trusted-server-adapter-cloudflare/src/platform.rs index 639248ed68..7b3890d130 100644 --- a/crates/trusted-server-adapter-cloudflare/src/platform.rs +++ b/crates/trusted-server-adapter-cloudflare/src/platform.rs @@ -19,7 +19,8 @@ use trusted_server_core::platform::UnavailableHttpClient; #[cfg(target_arch = "wasm32")] use trusted_server_core::platform::{ - PlatformHttpRequest, PlatformPendingRequest, PlatformResponse, PlatformSelectResult, + PlatformHttpRequest, PlatformPendingRequest, PlatformPendingResponse, PlatformResponse, + PlatformSelectResult, }; // --------------------------------------------------------------------------- @@ -164,37 +165,6 @@ impl KvStore for KvHandleAdapter { // CloudflareHttpClient — WASM target only // --------------------------------------------------------------------------- -/// Receives a dispatched operation and cancels it when its owner is dropped. -#[cfg(any(test, target_arch = "wasm32"))] -struct CloudflarePendingResponse { - response: futures::channel::oneshot::Receiver, - abort: futures::future::AbortHandle, -} - -#[cfg(any(test, target_arch = "wasm32"))] -impl CloudflarePendingResponse { - fn start( - operation: impl std::future::Future + 'static, - spawn: impl FnOnce(futures::future::LocalBoxFuture<'static, ()>), - ) -> Self { - let (sender, response) = futures::channel::oneshot::channel(); - let (abort, registration) = futures::future::AbortHandle::new_pair(); - spawn(Box::pin(async move { - if let Ok(result) = futures::future::Abortable::new(operation, registration).await { - let _ = sender.send(result); - } - })); - Self { response, abort } - } -} - -#[cfg(any(test, target_arch = "wasm32"))] -impl Drop for CloudflarePendingResponse { - fn drop(&mut self) { - self.abort.abort(); - } -} - /// [`worker::Fetch`]-backed HTTP client for the Cloudflare Workers runtime. /// /// # Multi-provider auction limitation @@ -495,7 +465,7 @@ impl PlatformHttpClient for CloudflareHttpClient { request: PlatformHttpRequest, ) -> Result> { let backend_name = request.backend_name.clone(); - let pending = CloudflarePendingResponse::start( + let pending = PlatformPendingResponse::start( Self::execute(request), worker::wasm_bindgen_futures::spawn_local, ); @@ -515,14 +485,13 @@ impl PlatformHttpClient for CloudflareHttpClient { let ready_platform = pending_requests.remove(0); let mut pending = ready_platform - .downcast::>>>( - ) + .downcast::>>>() .map_err(|_| { Report::new(PlatformError::HttpClient) .attach("unexpected inner type in CloudflareHttpClient::select") })?; - let ready = (&mut pending.response).await.unwrap_or_else(|error| { + let ready = pending.wait().await.unwrap_or_else(|error| { Err(Report::new(PlatformError::HttpClient) .attach(format!("dispatched response task stopped: {error}"))) }); @@ -549,34 +518,34 @@ impl PlatformHttpClient for CloudflareHttpClient { let platform_pending = pending_requests.remove(0); let backend_name = platform_pending.backend_name().map(str::to_owned); let mut pending = platform_pending - .downcast::>>>( - ) + .downcast::>>>() .map_err(|_| { Report::new(PlatformError::HttpClient) .attach("unexpected pending request type in deadline selection") })?; - let result = complete_before_deadline( - &mut pending.response, - async { - // Workers timers use whole signed milliseconds. Recheck the - // absolute deadline after each bounded timer interval. - loop { - let remaining = deadline.saturating_duration_since(web_time::Instant::now()); - if remaining.is_zero() { - break; + let result = pending + .complete_before_deadline( + async { + // Workers timers use whole signed milliseconds. Recheck the + // absolute deadline after each bounded timer interval. + loop { + let remaining = + deadline.saturating_duration_since(web_time::Instant::now()); + if remaining.is_zero() { + break; + } + worker::Delay::from( + remaining + .max(Duration::from_millis(1)) + .min(Duration::from_millis(i32::MAX as u64)), + ) + .await; } - worker::Delay::from( - remaining - .max(Duration::from_millis(1)) - .min(Duration::from_millis(i32::MAX as u64)), - ) - .await; - } - }, - deadline, - web_time::Instant::now, - ) - .await; + }, + deadline, + web_time::Instant::now, + ) + .await; Ok(result.map(|result| { let ready = result.unwrap_or_else(|error| { Err(Report::new(PlatformError::HttpClient) @@ -1122,230 +1091,3 @@ mod tests { ); } } - -#[cfg(any(test, target_arch = "wasm32"))] -async fn complete_before_deadline( - response: &mut futures::channel::oneshot::Receiver, - timer: T, - deadline: web_time::Instant, - now: N, -) -> Option> -where - T: std::future::Future, - N: Fn() -> web_time::Instant, -{ - // Inspect only the task's buffered result, including when collection starts - // after publisher-origin work has exhausted the auction budget. - match response.try_recv() { - Ok(Some(ready)) => return Some(Ok(ready)), - Err(error) => return Some(Err(error)), - Ok(None) => {} - } - if now() >= deadline { - return None; - } - match futures::future::select(response, Box::pin(timer)).await { - futures::future::Either::Left((ready, _)) => Some(ready), - // The timer and task can become ready in the same executor turn. A final - // channel probe retains buffered results without polling transport I/O. - futures::future::Either::Right(((), response)) => response.try_recv().transpose(), - } -} - -#[cfg(test)] -mod deadline_tests { - use std::cell::Cell; - use std::rc::Rc; - use std::time::Duration; - - use futures::executor::{LocalPool, block_on}; - use futures::future::{pending, ready}; - use futures::task::LocalSpawnExt as _; - use web_time::Instant; - - use super::{CloudflarePendingResponse, complete_before_deadline}; - - #[test] - fn dispatched_task_starts_before_collection_and_buffers_complete_body() { - let mut executor = LocalPool::new(); - let spawner = executor.spawner(); - let started = Rc::new(Cell::new(false)); - let started_operation = Rc::clone(&started); - let (finish_body, body) = futures::channel::oneshot::channel(); - let mut pending = CloudflarePendingResponse::start( - async move { - started_operation.set(true); - body.await.expect("should receive complete response body") - }, - |task| { - spawner - .spawn_local(task) - .expect("should spawn outbound task") - }, - ); - - executor.run_until_stalled(); - assert!(started.get(), "should start outbound work before selection"); - assert_eq!( - pending.response.try_recv().expect("should poll result"), - None, - "should not expose headers before response body completes" - ); - finish_body.send(42).expect("should finish response body"); - executor.run_until_stalled(); - assert_eq!( - pending - .response - .try_recv() - .expect("should poll buffered result"), - Some(42), - "should preserve a complete buffered response without polling transport" - ); - } - - #[test] - fn dropping_dispatched_pending_cancels_started_body_operation() { - struct DropProbe(Rc>); - impl Drop for DropProbe { - fn drop(&mut self) { - self.0.set(true); - } - } - let mut executor = LocalPool::new(); - let spawner = executor.spawner(); - let dropped = Rc::new(Cell::new(false)); - let probe = DropProbe(Rc::clone(&dropped)); - let pending = CloudflarePendingResponse::start( - async move { - let _probe = probe; - futures::future::pending::<()>().await; - }, - |task| { - spawner - .spawn_local(task) - .expect("should spawn outbound task") - }, - ); - executor.run_until_stalled(); - assert!(!dropped.get(), "should retain an active body operation"); - drop(pending); - executor.run_until_stalled(); - assert!( - dropped.get(), - "should drop spawned operation after cancellation" - ); - } - - #[test] - fn dropping_queued_dispatch_prevents_transport_poll() { - let mut executor = LocalPool::new(); - let spawner = executor.spawner(); - let started = Rc::new(Cell::new(false)); - let operation_started = Rc::clone(&started); - let pending = CloudflarePendingResponse::start( - async move { - operation_started.set(true); - }, - |task| { - spawner - .spawn_local(task) - .expect("should spawn outbound task") - }, - ); - drop(pending); - executor.run_until_stalled(); - assert!( - !started.get(), - "should cancel queued work before transport starts" - ); - } - - #[test] - fn expired_deadline_preserves_buffered_response() { - let (sender, mut response) = futures::channel::oneshot::channel(); - sender.send(42).expect("should buffer response"); - let deadline = Instant::now(); - let result = block_on(complete_before_deadline( - &mut response, - pending(), - deadline, - || deadline, - )); - assert_eq!( - result, - Some(Ok(42)), - "should retain a buffered response after cutoff" - ); - } - - #[test] - fn expired_deadline_returns_without_waiting_for_response() { - let (_sender, mut response) = futures::channel::oneshot::channel::<()>(); - let deadline = Instant::now(); - let result = block_on(complete_before_deadline( - &mut response, - pending(), - deadline, - || deadline, - )); - assert_eq!(result, None, "should stop waiting at an expired deadline"); - } - - #[test] - fn timer_cuts_off_pending_response() { - let (_sender, mut response) = futures::channel::oneshot::channel::<()>(); - let now = Instant::now(); - let result = block_on(complete_before_deadline( - &mut response, - ready(()), - now + Duration::from_secs(1), - || now, - )); - assert_eq!(result, None, "should cut off an unbuffered response"); - } - - #[test] - fn closed_response_channel_reports_failure() { - let (sender, mut response) = futures::channel::oneshot::channel::<()>(); - drop(sender); - let now = Instant::now(); - let result = block_on(complete_before_deadline( - &mut response, - pending(), - now + Duration::from_secs(1), - || now, - )); - assert!( - matches!(result, Some(Err(_))), - "should preserve a dispatched task failure" - ); - } - - #[test] - fn timer_rechecks_result_published_during_cutoff() { - let (sender, mut response) = futures::channel::oneshot::channel(); - let now = Instant::now(); - let clock = Cell::new(now); - let deadline = now + Duration::from_secs(1); - let timer = async { - clock.set(deadline); - sender.send(42).expect("should publish at cutoff"); - }; - let result = block_on(complete_before_deadline( - &mut response, - timer, - deadline, - || clock.get(), - )); - assert_eq!( - result, - Some(Ok(42)), - "should retain result buffered when timer wins" - ); - assert_eq!( - clock.get(), - deadline, - "should exercise the deadline boundary" - ); - } -} diff --git a/crates/trusted-server-adapter-spin/src/platform.rs b/crates/trusted-server-adapter-spin/src/platform.rs index 92b940b484..58a6744b68 100644 --- a/crates/trusted-server-adapter-spin/src/platform.rs +++ b/crates/trusted-server-adapter-spin/src/platform.rs @@ -23,7 +23,7 @@ use std::io::Read as _; use trusted_server_core::platform::PlatformHttpRequest; #[cfg(all(feature = "spin", target_arch = "wasm32"))] use trusted_server_core::platform::{ - PlatformPendingRequest, PlatformResponse, PlatformSelectResult, + PlatformPendingRequest, PlatformPendingResponse, PlatformResponse, PlatformSelectResult, }; // Limit both streamed bytes and decompressed bytes to 8 MiB to preserve @@ -427,37 +427,6 @@ fn is_wasi_forbidden_outbound_header(name: &str) -> bool { || name.eq_ignore_ascii_case("proxy-connection") } -/// Receives a dispatched operation and cancels it when its owner is dropped. -#[cfg(any(test, all(feature = "spin", target_arch = "wasm32")))] -struct SpinPendingResponse { - response: futures::channel::oneshot::Receiver, - abort: futures::future::AbortHandle, -} - -#[cfg(any(test, all(feature = "spin", target_arch = "wasm32")))] -impl SpinPendingResponse { - fn start( - operation: impl std::future::Future + 'static, - spawn: impl FnOnce(futures::future::LocalBoxFuture<'static, ()>), - ) -> Self { - let (sender, response) = futures::channel::oneshot::channel(); - let (abort, registration) = futures::future::AbortHandle::new_pair(); - spawn(Box::pin(async move { - if let Ok(result) = futures::future::Abortable::new(operation, registration).await { - let _ = sender.send(result); - } - })); - Self { response, abort } - } -} - -#[cfg(any(test, all(feature = "spin", target_arch = "wasm32")))] -impl Drop for SpinPendingResponse { - fn drop(&mut self) { - self.abort.abort(); - } -} - /// `spin_sdk::http::send`-backed HTTP client for the Spin runtime. /// /// Auction configuration supports one provider at a time, so @@ -586,7 +555,8 @@ impl PlatformHttpClient for SpinPlatformHttpClient { request: PlatformHttpRequest, ) -> Result> { let backend_name = request.backend_name.clone(); - let pending = SpinPendingResponse::start(Self::execute(request), spin_sdk::wasip3::spawn); + let pending = + PlatformPendingResponse::start(Self::execute(request), spin_sdk::wasip3::spawn); Ok(PlatformPendingRequest::new(pending).with_backend_name(backend_name)) } @@ -610,13 +580,13 @@ impl PlatformHttpClient for SpinPlatformHttpClient { let ready_platform = pending_requests.remove(0); let mut pending = ready_platform - .downcast::>>>() + .downcast::>>>() .map_err(|_| { Report::new(PlatformError::HttpClient) .attach("unexpected inner type in SpinPlatformHttpClient::select") })?; - let ready = (&mut pending.response).await.unwrap_or_else(|error| { + let ready = pending.wait().await.unwrap_or_else(|error| { Err(Report::new(PlatformError::HttpClient) .attach(format!("dispatched response task stopped: {error}"))) }); @@ -643,23 +613,23 @@ impl PlatformHttpClient for SpinPlatformHttpClient { let platform_pending = pending_requests.remove(0); let backend_name = platform_pending.backend_name().map(str::to_owned); let mut pending = platform_pending - .downcast::>>>() + .downcast::>>>() .map_err(|_| { Report::new(PlatformError::HttpClient) .attach("unexpected pending request type in deadline selection") })?; - let result = complete_before_deadline( - &mut pending.response, - async { - let remaining = deadline.saturating_duration_since(web_time::Instant::now()); - if !remaining.is_zero() { - spin_sdk::time::sleep(remaining).await; - } - }, - deadline, - web_time::Instant::now, - ) - .await; + let result = pending + .complete_before_deadline( + async { + let remaining = deadline.saturating_duration_since(web_time::Instant::now()); + if !remaining.is_zero() { + spin_sdk::time::sleep(remaining).await; + } + }, + deadline, + web_time::Instant::now, + ) + .await; Ok(result.map(|result| { let ready = result.unwrap_or_else(|error| { Err(Report::new(PlatformError::HttpClient) @@ -1445,142 +1415,9 @@ mod tests { } } -#[cfg(any(test, all(feature = "spin", target_arch = "wasm32")))] -async fn complete_before_deadline( - response: &mut futures::channel::oneshot::Receiver, - timer: T, - deadline: web_time::Instant, - now: N, -) -> Option> -where - T: std::future::Future, - N: Fn() -> web_time::Instant, -{ - // Inspect only the task's buffered result, including when collection starts - // after publisher-origin work has exhausted the auction budget. - match response.try_recv() { - Ok(Some(ready)) => return Some(Ok(ready)), - Err(error) => return Some(Err(error)), - Ok(None) => {} - } - if now() >= deadline { - return None; - } - match futures::future::select(response, Box::pin(timer)).await { - futures::future::Either::Left((ready, _)) => Some(ready), - // The timer and task can become ready in the same executor turn. A final - // channel probe retains buffered results without polling transport I/O. - futures::future::Either::Right(((), response)) => response.try_recv().transpose(), - } -} - #[cfg(test)] mod deadline_tests { - use std::cell::Cell; - use std::rc::Rc; - use std::time::Duration; - - use futures::executor::{LocalPool, block_on}; - use futures::future::{pending, ready}; - use futures::task::LocalSpawnExt as _; - use web_time::Instant; - - use super::{SpinPendingResponse, complete_before_deadline}; - - #[test] - fn dispatched_task_starts_before_collection_and_buffers_complete_body() { - let mut executor = LocalPool::new(); - let spawner = executor.spawner(); - let started = Rc::new(Cell::new(false)); - let started_operation = Rc::clone(&started); - let (finish_body, body) = futures::channel::oneshot::channel(); - let mut pending = SpinPendingResponse::start( - async move { - started_operation.set(true); - body.await.expect("should receive complete response body") - }, - |task| { - spawner - .spawn_local(task) - .expect("should spawn outbound task") - }, - ); - - executor.run_until_stalled(); - assert!(started.get(), "should start outbound work before selection"); - assert_eq!( - pending.response.try_recv().expect("should poll result"), - None, - "should not expose headers before response body completes" - ); - finish_body.send(42).expect("should finish response body"); - executor.run_until_stalled(); - assert_eq!( - pending - .response - .try_recv() - .expect("should poll buffered result"), - Some(42), - "should preserve a complete buffered response without polling transport" - ); - } - - #[test] - fn dropping_dispatched_pending_cancels_started_body_operation() { - struct DropProbe(Rc>); - impl Drop for DropProbe { - fn drop(&mut self) { - self.0.set(true); - } - } - let mut executor = LocalPool::new(); - let spawner = executor.spawner(); - let dropped = Rc::new(Cell::new(false)); - let probe = DropProbe(Rc::clone(&dropped)); - let pending = SpinPendingResponse::start( - async move { - let _probe = probe; - futures::future::pending::<()>().await; - }, - |task| { - spawner - .spawn_local(task) - .expect("should spawn outbound task") - }, - ); - executor.run_until_stalled(); - assert!(!dropped.get(), "should retain an active body operation"); - drop(pending); - executor.run_until_stalled(); - assert!( - dropped.get(), - "should drop spawned operation after cancellation" - ); - } - - #[test] - fn dropping_queued_dispatch_prevents_transport_poll() { - let mut executor = LocalPool::new(); - let spawner = executor.spawner(); - let started = Rc::new(Cell::new(false)); - let operation_started = Rc::clone(&started); - let pending = SpinPendingResponse::start( - async move { - operation_started.set(true); - }, - |task| { - spawner - .spawn_local(task) - .expect("should spawn outbound task") - }, - ); - drop(pending); - executor.run_until_stalled(); - assert!( - !started.get(), - "should cancel queued work before transport starts" - ); - } + use futures::executor::block_on; #[test] fn spin_body_collection_rejects_oversized_stream() { @@ -1612,93 +1449,4 @@ mod deadline_tests { "should retain the complete permitted body" ); } - - #[test] - fn expired_deadline_preserves_buffered_response() { - let (sender, mut response) = futures::channel::oneshot::channel(); - sender.send(42).expect("should buffer response"); - let deadline = Instant::now(); - let result = block_on(complete_before_deadline( - &mut response, - pending(), - deadline, - || deadline, - )); - assert_eq!( - result, - Some(Ok(42)), - "should retain a buffered response after cutoff" - ); - } - - #[test] - fn expired_deadline_returns_without_waiting_for_response() { - let (_sender, mut response) = futures::channel::oneshot::channel::<()>(); - let deadline = Instant::now(); - let result = block_on(complete_before_deadline( - &mut response, - pending(), - deadline, - || deadline, - )); - assert_eq!(result, None, "should stop waiting at an expired deadline"); - } - - #[test] - fn timer_cuts_off_pending_response() { - let (_sender, mut response) = futures::channel::oneshot::channel::<()>(); - let now = Instant::now(); - let result = block_on(complete_before_deadline( - &mut response, - ready(()), - now + Duration::from_secs(1), - || now, - )); - assert_eq!(result, None, "should cut off an unbuffered response"); - } - - #[test] - fn closed_response_channel_reports_failure() { - let (sender, mut response) = futures::channel::oneshot::channel::<()>(); - drop(sender); - let now = Instant::now(); - let result = block_on(complete_before_deadline( - &mut response, - pending(), - now + Duration::from_secs(1), - || now, - )); - assert!( - matches!(result, Some(Err(_))), - "should preserve a dispatched task failure" - ); - } - - #[test] - fn timer_rechecks_result_published_during_cutoff() { - let (sender, mut response) = futures::channel::oneshot::channel(); - let now = Instant::now(); - let clock = Cell::new(now); - let deadline = now + Duration::from_secs(1); - let timer = async { - clock.set(deadline); - sender.send(42).expect("should publish at cutoff"); - }; - let result = block_on(complete_before_deadline( - &mut response, - timer, - deadline, - || clock.get(), - )); - assert_eq!( - result, - Some(Ok(42)), - "should retain result buffered when timer wins" - ); - assert_eq!( - clock.get(), - deadline, - "should exercise the deadline boundary" - ); - } } diff --git a/crates/trusted-server-core/src/auction/endpoints.rs b/crates/trusted-server-core/src/auction/endpoints.rs index f364464561..937e282cb0 100644 --- a/crates/trusted-server-core/src/auction/endpoints.rs +++ b/crates/trusted-server-core/src/auction/endpoints.rs @@ -957,10 +957,10 @@ mod tests { context .services .http_client() - .send_async(PlatformHttpRequest::new( - request, - "template-switch-probe-backend", - )) + .send_async( + PlatformHttpRequest::new(request, "template-switch-probe-backend") + .with_cache_bypass(), + ) .await .change_context(TrustedServerError::Auction { message: "probe provider launch failed".to_string(), diff --git a/crates/trusted-server-core/src/auction/orchestrator.rs b/crates/trusted-server-core/src/auction/orchestrator.rs index befddc8c16..c63f3f6628 100644 --- a/crates/trusted-server-core/src/auction/orchestrator.rs +++ b/crates/trusted-server-core/src/auction/orchestrator.rs @@ -3450,7 +3450,7 @@ mod tests { context .services .http_client() - .send_async(req) + .send_async(req.with_cache_bypass()) .await .change_context(TrustedServerError::Auction { message: "stub launch failed".to_string(), @@ -3525,7 +3525,7 @@ mod tests { context .services .http_client() - .send_async(request) + .send_async(request.with_cache_bypass()) .await .change_context(TrustedServerError::Auction { message: "deadline test provider launch failed".to_string(), @@ -3585,7 +3585,7 @@ mod tests { context .services .http_client() - .send_async(request) + .send_async(request.with_cache_bypass()) .await .change_context(TrustedServerError::Auction { message: "pending mediator launch failed".to_string(), @@ -3741,7 +3741,7 @@ mod tests { context .services .http_client() - .send_async(request) + .send_async(request.with_cache_bypass()) .await .change_context(TrustedServerError::Auction { message: "recording launch failed".to_string(), @@ -3802,7 +3802,7 @@ mod tests { context .services .http_client() - .send_async(request) + .send_async(request.with_cache_bypass()) .await .change_context(TrustedServerError::Auction { message: "divergent launch failed".to_string(), @@ -3967,7 +3967,7 @@ mod tests { context .services .http_client() - .send_async(req) + .send_async(req.with_cache_bypass()) .await .change_context(TrustedServerError::Auction { message: "mediator launch failed".to_string(), @@ -4208,7 +4208,7 @@ mod tests { services: &services, }; - if split { + let result = if split { let DispatchAuctionOutcome::Dispatched(dispatched) = orchestrator.dispatch_auction(&request, &context).await else { @@ -4222,7 +4222,13 @@ mod tests { .run_auction(&request, &context) .await .expect("deadline test auction should complete") - } + }; + assert_eq!( + stub.recorded_select_count(), + 0, + "should skip the delayed unbounded collection path" + ); + result } #[tokio::test] @@ -4241,11 +4247,6 @@ mod tests { }), "should time out outstanding providers on every adapter" ); - assert!( - result.total_time_ms < 150, - "should stop before the delayed provider completes: {}ms", - result.total_time_ms - ); assert!( result.winning_bids.is_empty(), "should not accept late bids" @@ -4285,7 +4286,7 @@ mod tests { services: &services, }; - if split { + let result = if split { let DispatchAuctionOutcome::Dispatched(dispatched) = orchestrator.dispatch_auction(&request, &context).await else { @@ -4299,7 +4300,13 @@ mod tests { .run_auction(&request, &context) .await .expect("deadline test auction should complete") - } + }; + assert_eq!( + stub.recorded_wait_count(), + 0, + "should skip the delayed unbounded collection path" + ); + result } #[tokio::test] @@ -4314,10 +4321,6 @@ mod tests { result.winning_bids["slot-1"].bidder, "local", "should retain provider winner" ); - assert!( - result.total_time_ms < 150, - "should not wait for the delayed mediator" - ); } } @@ -6215,10 +6218,16 @@ mod tests { services: &services, }; - orchestrator + let result = orchestrator .run_auction(&request, &context) .await - .expect("should execute planned pending mediator") + .expect("should execute planned pending mediator"); + assert_eq!( + http.recorded_wait_count(), + 0, + "should skip the delayed unbounded mediator wait" + ); + result } #[tokio::test] @@ -6234,10 +6243,6 @@ mod tests { Some("provider"), "should preserve on-time provider bids" ); - assert!( - result.total_time_ms < 150, - "should bound full mediator response wait" - ); } #[tokio::test] @@ -6253,10 +6258,6 @@ mod tests { Some("provider"), "should preserve on-time provider bids" ); - assert!( - result.total_time_ms < 150, - "should avoid waiting for the rounded transport timeout" - ); } #[tokio::test] diff --git a/crates/trusted-server-core/src/auction/provider.rs b/crates/trusted-server-core/src/auction/provider.rs index 887cde906f..d68ebde4d8 100644 --- a/crates/trusted-server-core/src/auction/provider.rs +++ b/crates/trusted-server-core/src/auction/provider.rs @@ -93,6 +93,9 @@ pub trait AuctionProvider: Send + Sync { /// Implementations normally return a pending upstream request, but may return /// an immediate response for a legitimate outcome that requires no HTTP call. /// The orchestrator handles waiting for and parsing pending responses. + /// Outbound auction and mediator requests must use + /// [`PlatformHttpRequest::with_cache_bypass`]: they carry personalized data, + /// and Fastly requires this marker for deadline-bounded collection. /// /// # Errors /// diff --git a/crates/trusted-server-core/src/platform/mod.rs b/crates/trusted-server-core/src/platform/mod.rs index 71258e9c54..e4733106e3 100644 --- a/crates/trusted-server-core/src/platform/mod.rs +++ b/crates/trusted-server-core/src/platform/mod.rs @@ -39,6 +39,7 @@ mod error; mod http; mod image_optimizer; mod kv; +mod pending_response; mod template_assembly; mod template_cache; #[cfg(test)] @@ -62,6 +63,7 @@ pub use image_optimizer::{ PlatformImageOptimizerParams, PlatformImageOptimizerRegion, }; pub use kv::UnavailableKvStore; +pub use pending_response::PlatformPendingResponse; pub use template_assembly::{ PlatformTemplateAssembler, TemplateAssemblyError, UnavailableTemplateAssembler, contains_publisher_esi_directive, diff --git a/crates/trusted-server-core/src/platform/pending_response.rs b/crates/trusted-server-core/src/platform/pending_response.rs new file mode 100644 index 0000000000..9590578df5 --- /dev/null +++ b/crates/trusted-server-core/src/platform/pending_response.rs @@ -0,0 +1,323 @@ +//! Shared ownership and deadline collection for dispatched runtime operations. + +/// Receives a dispatched operation and cancels it when its owner is dropped. +pub struct PlatformPendingResponse { + response: futures::channel::oneshot::Receiver, + abort: futures::future::AbortHandle, +} + +impl PlatformPendingResponse { + /// Starts an operation with the caller's runtime spawner. + /// + /// The operation publishes only its complete result. Dropping this owner + /// aborts queued or running guest work when the executor next polls it. + /// Runtime-specific transport cancellation remains the caller's responsibility. + /// + /// # Examples + /// + /// ```no_run + /// use futures::task::LocalSpawnExt as _; + /// use trusted_server_core::platform::PlatformPendingResponse; + /// let pool = futures::executor::LocalPool::new(); + /// let pending = PlatformPendingResponse::start(async { 42 }, |task| { + /// pool.spawner().spawn_local(task).expect("should spawn operation"); + /// }); + /// ``` + pub fn start( + operation: impl std::future::Future + 'static, + spawn: impl FnOnce(futures::future::LocalBoxFuture<'static, ()>), + ) -> Self { + let (sender, response) = futures::channel::oneshot::channel(); + let (abort, registration) = futures::future::AbortHandle::new_pair(); + spawn(Box::pin(async move { + if let Ok(result) = futures::future::Abortable::new(operation, registration).await { + let _ = sender.send(result); + } + })); + Self { response, abort } + } + + /// Waits for the complete result while retaining cancellation ownership. + /// + /// # Errors + /// + /// Returns a channel error if the dispatched operation stops without a result. + /// + /// # Examples + /// + /// ```no_run + /// # use trusted_server_core::platform::PlatformPendingResponse; + /// # async fn collect(pending: &mut PlatformPendingResponse) -> Result { + /// let result = pending.wait().await?; + /// # Ok(result) + /// # } + /// ``` + pub async fn wait(&mut self) -> Result { + (&mut self.response).await + } + + /// Collects a buffered result, waiting at most until the supplied timer fires. + /// + /// An initial nonblocking probe retains results after an expired deadline. + /// A final probe handles a result published in the same turn as timer expiry. + /// The caller supplies the runtime timer and a monotonic clock; no transport + /// is polled directly here. Returns `None` when the deadline expires without + /// a complete buffered result. The owner remains alive throughout the wait. + /// + /// # Errors + /// + /// Returns a channel error inside `Some` if the operation stops without a result. + /// + /// # Examples + /// + /// ```no_run + /// # use trusted_server_core::platform::PlatformPendingResponse; + /// # async fn collect(pending: &mut PlatformPendingResponse, timer: impl std::future::Future, deadline: web_time::Instant) { + /// let result = pending.complete_before_deadline(timer, deadline, web_time::Instant::now).await; + /// # } + /// ``` + pub async fn complete_before_deadline( + &mut self, + timer: F, + deadline: web_time::Instant, + now: N, + ) -> Option> + where + F: std::future::Future, + N: Fn() -> web_time::Instant, + { + complete_before_deadline(&mut self.response, timer, deadline, now).await + } +} + +impl Drop for PlatformPendingResponse { + fn drop(&mut self) { + self.abort.abort(); + } +} + +async fn complete_before_deadline( + response: &mut futures::channel::oneshot::Receiver, + timer: T, + deadline: web_time::Instant, + now: N, +) -> Option> +where + T: std::future::Future, + N: Fn() -> web_time::Instant, +{ + // Inspect only the task's buffered result, including when collection starts + // after publisher-origin work has exhausted the auction budget. + match response.try_recv() { + Ok(Some(ready)) => return Some(Ok(ready)), + Err(error) => return Some(Err(error)), + Ok(None) => {} + } + if now() >= deadline { + return None; + } + match futures::future::select(response, Box::pin(timer)).await { + futures::future::Either::Left((ready, _)) => Some(ready), + // The timer and task can become ready in the same executor turn. A final + // channel probe retains buffered results without polling transport I/O. + futures::future::Either::Right(((), response)) => response.try_recv().transpose(), + } +} + +#[cfg(test)] +mod deadline_tests { + use std::cell::Cell; + use std::rc::Rc; + use std::time::Duration; + + use futures::executor::{LocalPool, block_on}; + use futures::future::{pending, ready}; + use futures::task::LocalSpawnExt as _; + use web_time::Instant; + + use super::{PlatformPendingResponse, complete_before_deadline}; + + #[test] + fn dispatched_task_starts_before_collection_and_buffers_complete_body() { + let mut executor = LocalPool::new(); + let spawner = executor.spawner(); + let started = Rc::new(Cell::new(false)); + let started_operation = Rc::clone(&started); + let (finish_body, body) = futures::channel::oneshot::channel(); + let mut pending = PlatformPendingResponse::start( + async move { + started_operation.set(true); + body.await.expect("should receive complete response body") + }, + |task| { + spawner + .spawn_local(task) + .expect("should spawn outbound task") + }, + ); + + executor.run_until_stalled(); + assert!(started.get(), "should start outbound work before selection"); + assert_eq!( + pending.response.try_recv().expect("should poll result"), + None, + "should not expose headers before response body completes" + ); + finish_body.send(42).expect("should finish response body"); + executor.run_until_stalled(); + assert_eq!( + pending + .response + .try_recv() + .expect("should poll buffered result"), + Some(42), + "should preserve a complete buffered response without polling transport" + ); + } + + #[test] + fn dropping_dispatched_pending_cancels_started_body_operation() { + struct DropProbe(Rc>); + impl Drop for DropProbe { + fn drop(&mut self) { + self.0.set(true); + } + } + let mut executor = LocalPool::new(); + let spawner = executor.spawner(); + let dropped = Rc::new(Cell::new(false)); + let probe = DropProbe(Rc::clone(&dropped)); + let pending = PlatformPendingResponse::start( + async move { + let _probe = probe; + futures::future::pending::<()>().await; + }, + |task| { + spawner + .spawn_local(task) + .expect("should spawn outbound task") + }, + ); + executor.run_until_stalled(); + assert!(!dropped.get(), "should retain an active body operation"); + drop(pending); + executor.run_until_stalled(); + assert!( + dropped.get(), + "should drop spawned operation after cancellation" + ); + } + + #[test] + fn dropping_queued_dispatch_prevents_transport_poll() { + let mut executor = LocalPool::new(); + let spawner = executor.spawner(); + let started = Rc::new(Cell::new(false)); + let operation_started = Rc::clone(&started); + let pending = PlatformPendingResponse::start( + async move { + operation_started.set(true); + }, + |task| { + spawner + .spawn_local(task) + .expect("should spawn outbound task") + }, + ); + drop(pending); + executor.run_until_stalled(); + assert!( + !started.get(), + "should cancel queued work before transport starts" + ); + } + + #[test] + fn expired_deadline_preserves_buffered_response() { + let (sender, mut response) = futures::channel::oneshot::channel(); + sender.send(42).expect("should buffer response"); + let deadline = Instant::now(); + let result = block_on(complete_before_deadline( + &mut response, + pending(), + deadline, + || deadline, + )); + assert_eq!( + result, + Some(Ok(42)), + "should retain a buffered response after cutoff" + ); + } + + #[test] + fn expired_deadline_returns_without_waiting_for_response() { + let (_sender, mut response) = futures::channel::oneshot::channel::<()>(); + let deadline = Instant::now(); + let result = block_on(complete_before_deadline( + &mut response, + pending(), + deadline, + || deadline, + )); + assert_eq!(result, None, "should stop waiting at an expired deadline"); + } + + #[test] + fn timer_cuts_off_pending_response() { + let (_sender, mut response) = futures::channel::oneshot::channel::<()>(); + let now = Instant::now(); + let result = block_on(complete_before_deadline( + &mut response, + ready(()), + now + Duration::from_secs(1), + || now, + )); + assert_eq!(result, None, "should cut off an unbuffered response"); + } + + #[test] + fn closed_response_channel_reports_failure() { + let (sender, mut response) = futures::channel::oneshot::channel::<()>(); + drop(sender); + let now = Instant::now(); + let result = block_on(complete_before_deadline( + &mut response, + pending(), + now + Duration::from_secs(1), + || now, + )); + assert!( + matches!(result, Some(Err(_))), + "should preserve a dispatched task failure" + ); + } + + #[test] + fn timer_rechecks_result_published_during_cutoff() { + let (sender, mut response) = futures::channel::oneshot::channel(); + let now = Instant::now(); + let clock = Cell::new(now); + let deadline = now + Duration::from_secs(1); + let timer = async { + clock.set(deadline); + sender.send(42).expect("should publish at cutoff"); + }; + let result = block_on(complete_before_deadline( + &mut response, + timer, + deadline, + || clock.get(), + )); + assert_eq!( + result, + Some(Ok(42)), + "should retain result buffered when timer wins" + ); + assert_eq!( + clock.get(), + deadline, + "should exercise the deadline boundary" + ); + } +} diff --git a/crates/trusted-server-core/src/platform/test_support.rs b/crates/trusted-server-core/src/platform/test_support.rs index 40326473eb..be0b3feb8f 100644 --- a/crates/trusted-server-core/src/platform/test_support.rs +++ b/crates/trusted-server-core/src/platform/test_support.rs @@ -206,6 +206,7 @@ impl PlatformBackend for StubBackend { /// and resolved by [`StubHttpClient::select`]. struct StubPendingResponse { backend_name: String, + cache_intent: PlatformCacheIntent, status: u16, body: Vec, headers: Vec<(String, String)>, @@ -245,6 +246,8 @@ pub(crate) struct StubHttpClient { pending_backend_name_overrides: Mutex>>, // Test-only wall-clock delays applied before each select result is returned. select_delays: Mutex>, + select_count: std::sync::atomic::AtomicUsize, + wait_count: std::sync::atomic::AtomicUsize, // Reported by supports_concurrent_fanout(); set false to emulate // platforms whose collection contract permits only a single provider. concurrent_fanout: std::sync::atomic::AtomicBool, @@ -269,6 +272,16 @@ struct StubHttpResponse { } impl StubHttpClient { + /// Number of calls to the unbounded selection method. + pub(crate) fn recorded_select_count(&self) -> usize { + self.select_count.load(std::sync::atomic::Ordering::Relaxed) + } + + /// Number of calls to the unbounded single-request wait method. + pub(crate) fn recorded_wait_count(&self) -> usize { + self.wait_count.load(std::sync::atomic::Ordering::Relaxed) + } + pub fn new() -> Self { Self { calls: Mutex::new(Vec::new()), @@ -280,6 +293,8 @@ impl StubHttpClient { wait_delays: Mutex::new(VecDeque::new()), pending_backend_name_overrides: Mutex::new(VecDeque::new()), select_delays: Mutex::new(VecDeque::new()), + select_count: std::sync::atomic::AtomicUsize::new(0), + wait_count: std::sync::atomic::AtomicUsize::new(0), concurrent_fanout: std::sync::atomic::AtomicBool::new(true), streaming_responses_supported: std::sync::atomic::AtomicBool::new(false), pending_streaming_responses_supported: std::sync::atomic::AtomicBool::new(false), @@ -583,6 +598,7 @@ impl PlatformHttpClient for StubHttpClient { let edge_response = build_stub_pending_response( StubPendingResponse { backend_name: request.backend_name, + cache_intent: request.cache_intent, status: response.status, body: response.body, headers: response.headers, @@ -686,6 +702,7 @@ impl PlatformHttpClient for StubHttpClient { let pending = StubPendingResponse { backend_name: backend_name.clone(), + cache_intent: request.cache_intent, status: response.status, body: response.body, headers: response.headers, @@ -706,9 +723,26 @@ impl PlatformHttpClient for StubHttpClient { async fn select_until( &self, - pending_requests: Vec, + mut pending_requests: Vec, deadline: Instant, ) -> Result, Report> { + // Match Fastly's per-provider rejection before waiting on network work. + let mut unchecked = pending_requests.into_iter(); + let mut checked = Vec::with_capacity(unchecked.len()); + while let Some(pending) = unchecked.next() { + match validate_deadline_cache_bypass(pending) { + Ok(pending) => checked.push(pending), + Err((backend_name, error)) => { + checked.extend(unchecked); + return Ok(Some(PlatformSelectResult { + ready: Err(error), + remaining: checked, + failed_backend_name: backend_name, + })); + } + } + } + pending_requests = checked; let remaining = deadline.saturating_duration_since(Instant::now()); let delay = self .select_delays @@ -740,6 +774,7 @@ impl PlatformHttpClient for StubHttpClient { pending: PlatformPendingRequest, deadline: Instant, ) -> Result> { + let pending = validate_deadline_cache_bypass(pending).map_err(|(_, error)| error)?; let remaining = deadline.saturating_duration_since(Instant::now()); let delay = self .wait_delays @@ -776,6 +811,8 @@ impl PlatformHttpClient for StubHttpClient { &self, mut pending_requests: Vec, ) -> Result> { + self.select_count + .fetch_add(1, std::sync::atomic::Ordering::Relaxed); if pending_requests.is_empty() { return Err(Report::new(PlatformError::HttpClient) .attach("select called with empty pending_requests list")); @@ -851,6 +888,8 @@ impl PlatformHttpClient for StubHttpClient { &self, pending: PlatformPendingRequest, ) -> Result> { + self.wait_count + .fetch_add(1, std::sync::atomic::Ordering::Relaxed); let delay = self .wait_delays .lock() @@ -883,6 +922,37 @@ impl PlatformHttpClient for StubHttpClient { } } +// Require the same request marker as Fastly while retaining wrapper metadata. +fn validate_deadline_cache_bypass( + pending: PlatformPendingRequest, +) -> Result, Report)> { + let backend_name = pending.backend_name().map(str::to_owned); + let stream_response = pending.stream_response(); + let request_method = pending.request_method().cloned(); + let stub = pending.downcast::().map_err(|_| { + ( + backend_name.clone(), + Report::new(PlatformError::HttpClient) + .attach("unexpected pending type in bounded stub collection"), + ) + })?; + if !stub.cache_intent.is_bypass() { + return Err(( + Some(stub.backend_name), + Report::new(PlatformError::Unsupported) + .attach("deadline collection requires a cache-bypass request"), + )); + } + let mut pending = PlatformPendingRequest::new(stub); + if let Some(name) = backend_name { + pending = pending.with_backend_name(name); + } + if let Some(method) = request_method { + pending = pending.with_response_handling(stream_response, method); + } + Ok(pending) +} + fn build_stub_pending_response( stub: StubPendingResponse, stream_response: bool, @@ -1139,18 +1209,112 @@ mod tests { use super::*; + fn dispatch_cache_test_request( + stub: &StubHttpClient, + backend: &str, + bypass: bool, + ) -> PlatformPendingRequest { + stub.push_response(200, b"ready".to_vec()); + let mut request = PlatformHttpRequest::new( + request_builder() + .uri("https://example.com/bid") + .body(Body::empty()) + .expect("should build request"), + backend, + ); + if bypass { + request = request.with_cache_bypass(); + } + futures::executor::block_on(stub.send_async(request)).expect("should dispatch request") + } + + #[test] + fn bounded_stub_collection_rejects_missing_bypass_without_losing_siblings() { + for unsupported_first in [false, true] { + let stub = StubHttpClient::new(); + let unsupported = dispatch_cache_test_request(&stub, "unsupported", false); + let healthy = dispatch_cache_test_request(&stub, "healthy", true); + let selected = futures::executor::block_on(stub.select_until( + if unsupported_first { + vec![unsupported, healthy] + } else { + vec![healthy, unsupported] + }, + Instant::now(), + )) + .expect("should attribute unsupported request") + .expect("should return unsupported result"); + assert!( + matches!( + selected + .ready + .expect_err("should reject missing bypass") + .current_context(), + PlatformError::Unsupported + ), + "should match Fastly's cache bypass requirement" + ); + assert_eq!( + selected.failed_backend_name.as_deref(), + Some("unsupported"), + "should identify unsupported provider" + ); + let selected = + futures::executor::block_on(stub.select_until(selected.remaining, Instant::now())) + .expect("should collect sibling") + .expect("should retain ready sibling"); + assert_eq!( + selected + .ready + .expect("should accept bypassed request") + .backend_name + .as_deref(), + Some("healthy"), + "should preserve healthy provider" + ); + } + } + + #[test] + fn bounded_stub_wait_rejects_missing_bypass() { + let stub = StubHttpClient::new(); + let pending = dispatch_cache_test_request(&stub, "unsupported", false); + let error = futures::executor::block_on(stub.wait_until(pending, Instant::now())) + .expect_err("should reject missing bypass"); + assert!( + matches!(error.current_context(), PlatformError::Unsupported), + "should reject unsupported deadline wait" + ); + } + + #[test] + fn unbounded_stub_wait_accepts_default_cache_intent() { + let stub = StubHttpClient::new(); + let pending = dispatch_cache_test_request(&stub, "ordinary", false); + let response = futures::executor::block_on(stub.wait(pending)) + .expect("should preserve ordinary waits"); + assert_eq!( + response.backend_name.as_deref(), + Some("ordinary"), + "should preserve backend" + ); + } + #[test] fn bounded_stub_wait_preserves_an_already_ready_response_after_expiry() { let stub = StubHttpClient::new(); stub.push_response(200, b"ready".to_vec()); let pending = futures::executor::block_on( - stub.send_async(PlatformHttpRequest::new( - request_builder() - .uri("https://example.com/ready") - .body(Body::empty()) - .expect("should build request"), - "ready-backend", - )), + stub.send_async( + PlatformHttpRequest::new( + request_builder() + .uri("https://example.com/ready") + .body(Body::empty()) + .expect("should build request"), + "ready-backend", + ) + .with_cache_bypass(), + ), ) .expect("should dispatch ready response"); let response = futures::executor::block_on(stub.wait_until(pending, Instant::now())) diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index 42a8dd96b4..edb8e9e777 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -10796,7 +10796,7 @@ mod tests { context .services .http_client() - .send_async(request) + .send_async(request.with_cache_bypass()) .await .change_context(TrustedServerError::Auction { message: "stub bid launch failed".to_string(), @@ -17621,7 +17621,7 @@ mod tests { context .services .http_client() - .send_async(request) + .send_async(request.with_cache_bypass()) .await .change_context(TrustedServerError::Auction { message: "test provider launch failed".to_string(), @@ -24019,13 +24019,16 @@ mod tests { context .services .http_client() - .send_async(crate::platform::PlatformHttpRequest::new( - Request::builder() - .uri("https://bidder.example.com/bid") - .body(EdgeBody::empty()) - .expect("should build test bid request"), - "seam-test", - )) + .send_async( + crate::platform::PlatformHttpRequest::new( + Request::builder() + .uri("https://bidder.example.com/bid") + .body(EdgeBody::empty()) + .expect("should build test bid request"), + "seam-test", + ) + .with_cache_bypass(), + ) .await .change_context(TrustedServerError::Auction { message: "Failed to dispatch test auction".to_string(), @@ -26684,7 +26687,7 @@ mod tests { context .services .http_client() - .send_async(request) + .send_async(request.with_cache_bypass()) .await .change_context(TrustedServerError::Auction { message: "test bidder launch failed".to_string(), diff --git a/docs/guide/auction-orchestration.md b/docs/guide/auction-orchestration.md index bdbb8789bc..583d7f8780 100644 --- a/docs/guide/auction-orchestration.md +++ b/docs/guide/auction-orchestration.md @@ -824,12 +824,13 @@ not bid eligibility based on an exact server-side arrival time. A mediator start only with remaining budget and uses the earlier of the auction cutoff and its own logical timeout. -Provider bids complete at the final nonblocking snapshot remain eligible even after `tmax`; mediation starts only with remaining budget and rejects responses collected after its cutoff. +Provider bids that are already complete at the final nonblocking snapshot remain +eligible even after `tmax`; mediation starts only with remaining budget and +rejects responses collected after its cutoff. Personalized auction and mediator requests explicitly bypass intermediary caches. Fastly uses the SDK's direct pending-request handles and body readiness handles in -a timed wait; it neither polls headers every millisecond nor waits on an empty -handle list. Retained partial payloads share a 10 MiB budget. Under memory pressure, +a timed wait. Retained partial payloads share a 10 MiB budget. Under memory pressure, the largest partial response is rejected with its own backend attribution so a large stalled provider cannot make a small healthy sibling take the failure. Completed payloads move directly into the platform body without a host-buffer diff --git a/docs/guide/configuration.md b/docs/guide/configuration.md index d707ba4867..d0302cd2ca 100644 --- a/docs/guide/configuration.md +++ b/docs/guide/configuration.md @@ -2301,7 +2301,9 @@ dispatch, including headers and complete response bodies. One nonblocking snapsh retains complete available responses after a slow origin; unfinished providers become timeouts and mediation cannot start with an exhausted budget. Backend timeout rounding does not extend network waiting. -Provider bids complete at the final nonblocking snapshot remain eligible even after `tmax`; mediation starts only with remaining budget and rejects responses collected after its cutoff. +Provider bids that are already complete at the final nonblocking snapshot remain +eligible even after `tmax`; mediation starts only with remaining budget and +rejects responses collected after its cutoff. Fastly abandons pending requests; Axum, Cloudflare, and Spin cancel outstanding operations where supported. Local processing and delivery can finish after the From e1a34e978039bb8012d190b2bd1028aaffb92b7a Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Fri, 30 Oct 2026 18:46:51 +0530 Subject: [PATCH 9/9] Tighten deadline regressions and clarify worker adapter limits Shorten the split-collection select delays below the auction budget so a collection clock restarted after the origin wait collects the response and fails the test. Extract the Fastly timed-select rounding and the Spin bounded body collection into helpers that the production paths call, so their tests exercise real code. Explain the worker single-provider limit, the block_on hazard on worker executors, the self-building Fastly harness, and the new deadline CI gates. --- AGENTS.md | 2 + .../src/platform.rs | 10 ++-- .../src/platform/deadline.rs | 35 +++++++++-- .../src/platform.rs | 60 ++++++++++++------- .../src/auction/orchestrator.rs | 8 ++- .../trusted-server-core/src/platform/http.rs | 6 ++ docs/guide/auction-testing.md | 17 +++--- 7 files changed, 96 insertions(+), 42 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 7688c1829f..d90eda1f2d 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -375,6 +375,8 @@ Every PR must pass: 11. Documentation proposal scripts: `shellcheck scripts/docs-proposal/*.sh && node --test scripts/docs-proposal/hunks.test.mjs && scripts/docs-proposal/test.sh` 12. URL-host linter on lines the PR adds (`lint-domains` job, PRs only); run it locally with `cargo run --manifest-path crates/trusted-server-cli/Cargo.toml --target --bin ts -- dev lint domains --changed-vs origin/main`, using the PR's base branch in place of `main` when it targets another branch 13. CI Node pin guard: `node --test scripts/check-ci-node-pin.test.mjs && node scripts/check-ci-node-pin.mjs` with the `.tool-versions` Node (every CI job that compiles Rust sets up the `.tool-versions` Node before its first cargo step) +14. Fastly absolute-deadline network fixtures under Viceroy (the harness builds the adapter test binary itself): `node crates/trusted-server-adapter-fastly/tests/deadline-origin.mjs "$(command -v viceroy)"` +15. Worker auction-deadline runtime fixtures (`worker-auction-deadlines.yml`): build and lint the runtime fixture, then run it under the local runtime. For Cloudflare, `cargo build --locked -p trusted-server-adapter-cloudflare --example deadline-runtime --target wasm32-unknown-unknown --features cloudflare` and `node scripts/test-worker-auction-deadlines.mjs cloudflare --runtime "$(command -v wrangler)"` (requires wrangler and the `Cargo.lock` wasm-bindgen CLI). For Spin, `cargo build --locked -p trusted-server-adapter-spin --example deadline-runtime --target wasm32-wasip1 --features spin` and `node scripts/test-worker-auction-deadlines.mjs spin --runtime "$(command -v spin)"`. CI lints each fixture with the same `cargo clippy` arguments plus `-- -D warnings` CI also runs these HTML EOF checks. Run them locally only when changing EOF finalization, publisher or HTML-processor code: diff --git a/crates/trusted-server-adapter-cloudflare/src/platform.rs b/crates/trusted-server-adapter-cloudflare/src/platform.rs index 7b3890d130..40cfc7b3a1 100644 --- a/crates/trusted-server-adapter-cloudflare/src/platform.rs +++ b/crates/trusted-server-adapter-cloudflare/src/platform.rs @@ -455,8 +455,9 @@ impl PlatformHttpClient for CloudflareHttpClient { } fn supports_concurrent_fanout(&self) -> bool { - // Retain the existing auction capability restriction independently of - // runtime task scheduling. + // Requests start concurrently at dispatch, but `select` and + // `select_until` collect only one response, so multi-provider fan-out + // stays unsupported. false } @@ -764,8 +765,9 @@ fn extract_client_ip(ctx: &edgezero_core::context::RequestContext) -> Option u32 { + u32::try_from(remaining.as_millis()) + .unwrap_or(u32::MAX) + .max(1) +} + #[cfg(test)] mod tests { use std::time::Duration; use super::super::{FastlyPlatformHttpClient, MAX_PLATFORM_RESPONSE_BODY_BYTES}; - use super::{DeadlinePending, PendingState}; + use super::{DeadlinePending, PendingState, host_select_timeout_ms}; + + #[test] + fn sub_millisecond_budget_never_requests_an_unbounded_host_wait() { + assert_eq!( + host_select_timeout_ms(Duration::from_micros(500)), + 1, + "should round a sub-millisecond budget up instead of passing 0" + ); + assert_eq!( + host_select_timeout_ms(Duration::from_millis(25)), + 25, + "should pass whole-millisecond budgets through unchanged" + ); + assert_eq!( + host_select_timeout_ms(Duration::from_secs(u64::MAX)), + u32::MAX, + "should saturate budgets beyond the hostcall range" + ); + } use edgezero_core::body::Body; use trusted_server_core::platform::{ PlatformHttpClient, PlatformHttpRequest, PlatformPendingRequest, diff --git a/crates/trusted-server-adapter-spin/src/platform.rs b/crates/trusted-server-adapter-spin/src/platform.rs index 58a6744b68..53b6386373 100644 --- a/crates/trusted-server-adapter-spin/src/platform.rs +++ b/crates/trusted-server-adapter-spin/src/platform.rs @@ -6,8 +6,8 @@ use bytes::Bytes; use edgezero_core::config_store::ConfigStoreHandle; use edgezero_core::key_value_store::{KvHandle, KvPage, KvStore}; use error_stack::{Report, ResultExt as _}; -#[cfg(all(feature = "spin", target_arch = "wasm32"))] -use http_body_util::BodyExt as _; +#[cfg(any(test, all(feature = "spin", target_arch = "wasm32")))] +use http_body_util::BodyExt; use trusted_server_core::platform::{ BackendNamingPolicy, ClientInfo, GeoInfo, KvError, PlatformBackend, PlatformBackendSpec, PlatformConfigStore, PlatformError, PlatformGeo, PlatformHttpClient, PlatformKvStore, @@ -37,6 +37,34 @@ type HeaderPairs = Vec<(String, Vec)>; #[cfg(any(test, all(feature = "spin", target_arch = "wasm32")))] type BufferedResponseParts = (HeaderPairs, Vec); +/// Collect an outbound response body, rejecting it once it exceeds +/// [`MAX_DECOMPRESSED_SIZE`] streamed bytes. +/// +/// The limit applies frame by frame, so an oversized body fails before it is +/// fully retained in the component heap. +/// +/// # Errors +/// +/// Returns `PlatformError::HttpClient` when the body exceeds the limit or a +/// frame cannot be read. +#[cfg(any(test, all(feature = "spin", target_arch = "wasm32")))] +async fn collect_limited_body(body: B) -> Result, Report> +where + B: BodyExt, + B::Data: bytes::Buf, + B::Error: Into>, +{ + let collected = http_body_util::Limited::new(body, MAX_DECOMPRESSED_SIZE) + .collect() + .await + .map_err(|error| { + Report::new(PlatformError::HttpClient).attach(format!( + "failed to read Spin outbound response body: {error}" + )) + })?; + Ok(collected.to_bytes().to_vec()) +} + #[cfg(any(test, all(feature = "spin", target_arch = "wasm32")))] const SPIN_VARIABLE_HEX: &[u8; 16] = b"0123456789abcdef"; @@ -510,16 +538,7 @@ impl SpinPlatformHttpClient { .iter() .map(|(name, value)| (name.to_string(), value.as_bytes().to_vec())) .collect(); - let body = http_body_util::Limited::new(spin_response.into_body(), MAX_DECOMPRESSED_SIZE) - .collect() - .await - .map_err(|error| { - Report::new(PlatformError::HttpClient).attach(format!( - "failed to read Spin outbound response body: {error}" - )) - })? - .to_bytes() - .to_vec(); + let body = collect_limited_body(spin_response.into_body()).await?; let (headers, body) = apply_spin_response_policy(&method, status, headers, body)?; let mut edge_builder = edgezero_core::http::response_builder().status(status); @@ -538,8 +557,9 @@ impl SpinPlatformHttpClient { #[async_trait::async_trait(?Send)] impl PlatformHttpClient for SpinPlatformHttpClient { fn supports_concurrent_fanout(&self) -> bool { - // Retain the existing auction capability restriction independently of - // runtime task scheduling. + // Requests start concurrently at dispatch, but `select` and + // `select_until` collect only one response, so multi-provider fan-out + // stays unsupported. false } @@ -1426,9 +1446,7 @@ mod deadline_tests { super::MAX_DECOMPRESSED_SIZE + 1 ])); - let result = block_on(http_body_util::BodyExt::collect( - http_body_util::Limited::new(body, super::MAX_DECOMPRESSED_SIZE), - )); + let result = block_on(super::collect_limited_body(body)); assert!( result.is_err(), "should reject a response exceeding the streamed body cap" @@ -1439,12 +1457,10 @@ mod deadline_tests { fn spin_body_collection_accepts_body_at_limit() { let body = http_body_util::Full::new(bytes::Bytes::from(vec![0; super::MAX_DECOMPRESSED_SIZE])); - let result = block_on(http_body_util::BodyExt::collect( - http_body_util::Limited::new(body, super::MAX_DECOMPRESSED_SIZE), - )) - .expect("should collect a body at the limit"); + let result = block_on(super::collect_limited_body(body)) + .expect("should collect a body at the limit"); assert_eq!( - result.to_bytes().len(), + result.len(), super::MAX_DECOMPRESSED_SIZE, "should retain the complete permitted body" ); diff --git a/crates/trusted-server-core/src/auction/orchestrator.rs b/crates/trusted-server-core/src/auction/orchestrator.rs index 317581c25f..888bade15e 100644 --- a/crates/trusted-server-core/src/auction/orchestrator.rs +++ b/crates/trusted-server-core/src/auction/orchestrator.rs @@ -6807,7 +6807,9 @@ mod tests { ); http.push_response(204, Vec::new()); http.push_select_delay(Duration::ZERO); - http.push_select_delay(Duration::from_millis(250)); + // Shorter than the 10 ms budget, so a collection clock restarted after + // the origin wait would collect this 204 instead of timing it out. + http.push_select_delay(Duration::from_millis(1)); let services = build_services_with_backend_and_http_client( Arc::new(NamingBackend::new(BackendNamingPolicy::Axum)), Arc::clone(&http) as Arc<_>, @@ -6886,7 +6888,9 @@ mod tests { async fn planned_split_collection_uses_dispatch_deadline_after_slow_origin() { let http = Arc::new(StubHttpClient::new()); http.push_response(204, Vec::new()); - http.push_select_delay(Duration::from_millis(250)); + // Shorter than the 10 ms budget, so a collection clock restarted after + // the origin wait would collect this 204 instead of timing it out. + http.push_select_delay(Duration::from_millis(1)); let services = build_services_with_backend_and_http_client( Arc::new(NamingBackend::new(BackendNamingPolicy::Axum)), http, diff --git a/crates/trusted-server-core/src/platform/http.rs b/crates/trusted-server-core/src/platform/http.rs index cdc23f8cb0..e1fe8ff1f2 100644 --- a/crates/trusted-server-core/src/platform/http.rs +++ b/crates/trusted-server-core/src/platform/http.rs @@ -355,6 +355,12 @@ pub trait PlatformHttpClient: Send + Sync { /// while the publisher origin is fetched. Dropping its pending handle /// cancels or abandons unfinished work according to the runtime contract. /// + /// On Cloudflare and Spin the request runs as a task on the runtime + /// executor, so [`Self::wait`] and [`Self::select_until`] must be awaited + /// on that executor. Driving them through a blocking executor such as + /// `futures::executor::block_on` blocks the only thread the task can run + /// on and hangs. + /// /// # Errors /// /// Returns `PlatformError::HttpClient` when the request cannot be diff --git a/docs/guide/auction-testing.md b/docs/guide/auction-testing.md index 7903a42700..b317c584d3 100644 --- a/docs/guide/auction-testing.md +++ b/docs/guide/auction-testing.md @@ -79,17 +79,14 @@ adapter tests use controlled futures and timers. CI also runs the actual worker HTTP clients under local workerd and Spin 4.1, covering eager dispatch, buffered bids after a slow origin, delayed headers, and trickling bodies. Run `node scripts/test-worker-auction-deadlines.mjs --help` for the same local fixtures. -Fastly also has Viceroy network fixtures -for stalled headers, trickling bodies, bodyless responses, and competing bidders; -run its local origin harness after building the adapter test binary: +Fastly also has Viceroy network fixtures for stalled headers, trickling bodies, +bodyless responses, and competing bidders. Its local origin harness builds the +adapter test binary itself: ```bash -cargo test -p trusted-server-adapter-fastly --target wasm32-wasip1 --no-run -node crates/trusted-server-adapter-fastly/tests/deadline-origin.mjs \ - "$(command -v viceroy)" \ - target/wasm32-wasip1/debug/deps/trusted_server_adapter_fastly-.wasm +node crates/trusted-server-adapter-fastly/tests/deadline-origin.mjs "$(command -v viceroy)" ``` -Use the adapter test artifact printed by Cargo in place of ``. The harness -binds a local ephemeral port and runs the five network tests excluded from the -standard suite; CI runs the harness automatically. +The harness binds a local ephemeral port and runs the five network tests +excluded from the standard suite; CI runs the same command. To reuse a prebuilt +test artifact, pass its `.wasm` path as a second argument.