diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 1cdf4e905d..60af0bb5d1 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -55,6 +55,9 @@ jobs: - name: Run tests run: cargo test-fastly + - name: Run Fastly absolute deadline slow-network tests + run: node crates/trusted-server-adapter-fastly/tests/deadline-origin.mjs "$(command -v viceroy)" + - name: Run trace emitted-script regressions run: | cargo test -p trusted-server-core --target x86_64-unknown-linux-gnu --lib trace_document_skipped_emitted_script_leaves_ad_state_untouched -- --ignored diff --git a/.github/workflows/worker-auction-deadlines.yml b/.github/workflows/worker-auction-deadlines.yml new file mode 100644 index 0000000000..dda7e26e8d --- /dev/null +++ b/.github/workflows/worker-auction-deadlines.yml @@ -0,0 +1,93 @@ +name: Worker auction deadline runtime regressions + +on: + pull_request: + push: + branches: [main] + +permissions: + contents: read + +jobs: + worker-deadlines: + runs-on: ubuntu-latest + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + adapter: [cloudflare, spin] + steps: + - uses: actions/checkout@v4 + - name: Read tool versions + id: versions + shell: bash + run: | + { + echo "node=$(awk '$1 == "nodejs" { print $2 }' .tool-versions)" + echo "spin=$(awk '$1 == "spin" { print $2 }' .tool-versions)" + echo "bindgen=$(sed -n '/^name = "wasm-bindgen"$/ { n; s/version = "\(.*\)"/\1/p; }' Cargo.lock)" + } >> "$GITHUB_OUTPUT" + - uses: actions/setup-node@v4 + with: + node-version: ${{ steps.versions.outputs.node }} + - name: Set up Rust + uses: ./.github/actions/setup-rust + with: + target: ${{ matrix.adapter == 'cloudflare' && 'wasm32-unknown-unknown' || 'wasm32-wasip1' }} + components: clippy + cache-key: worker-deadline-${{ matrix.adapter }} + - name: Install JS dependencies + working-directory: crates/trusted-server-js/lib + run: npm ci + - name: Build actual adapter HTTP fixture + shell: bash + run: | + if [ "${{ matrix.adapter }}" = cloudflare ]; then + cargo build --locked -p trusted-server-adapter-cloudflare --example deadline-runtime --target wasm32-unknown-unknown --features cloudflare + else + cargo build --locked -p trusted-server-adapter-spin --example deadline-runtime --target wasm32-wasip1 --features spin + fi + - name: Lint runtime fixture + shell: bash + run: | + if [ "${{ matrix.adapter }}" = cloudflare ]; then + cargo clippy --locked -p trusted-server-adapter-cloudflare --example deadline-runtime --target wasm32-unknown-unknown --features cloudflare -- -D warnings + else + cargo clippy --locked -p trusted-server-adapter-spin --example deadline-runtime --target wasm32-wasip1 --features spin -- -D warnings + fi + - name: Install local Cloudflare tools + if: matrix.adapter == 'cloudflare' + shell: bash + run: | + npm install --global wrangler@4.83.0 + mkdir -p "$RUNNER_TEMP/bindgen" + # Digests are published with the upstream GitHub release assets. + # Update the digest alongside the pinned version. + test "${{ steps.versions.outputs.bindgen }}" = 0.2.126 + archive="$RUNNER_TEMP/wasm-bindgen.tar.gz" + curl --fail --location --silent --show-error "https://github.com/wasm-bindgen/wasm-bindgen/releases/download/${{ steps.versions.outputs.bindgen }}/wasm-bindgen-${{ steps.versions.outputs.bindgen }}-x86_64-unknown-linux-musl.tar.gz" --output "$archive" + echo "064948d58e2d6c0a745216477a639ba696216d6309aaa902939d1b865b1d869d $archive" | sha256sum --check + tar -xz --strip-components=1 -C "$RUNNER_TEMP/bindgen" -f "$archive" + echo "$RUNNER_TEMP/bindgen" >> "$GITHUB_PATH" + - name: Install pinned Spin runtime + if: matrix.adapter == 'spin' + shell: bash + run: | + mkdir -p "$RUNNER_TEMP/spin" + # Update the published upstream digest alongside the pinned version. + test "${{ steps.versions.outputs.spin }}" = 4.1.0 + archive="$RUNNER_TEMP/spin.tar.gz" + curl --fail --location --silent --show-error "https://github.com/spinframework/spin/releases/download/v${{ steps.versions.outputs.spin }}/spin-v${{ steps.versions.outputs.spin }}-linux-amd64.tar.gz" --output "$archive" + echo "aeeb892f94cf861f5302546faa0bdaa82cc4be30539621299746a6dc011f3b5a $archive" | sha256sum --check + tar -xz -C "$RUNNER_TEMP/spin" -f "$archive" + echo "$RUNNER_TEMP/spin" >> "$GITHUB_PATH" + - name: Exercise delayed headers, trickling bodies, and slow publisher work + shell: bash + env: + WRANGLER_SEND_METRICS: 'false' + run: | + if [ "${{ matrix.adapter }}" = cloudflare ]; then + node scripts/test-worker-auction-deadlines.mjs cloudflare --runtime "$(command -v wrangler)" + else + node scripts/test-worker-auction-deadlines.mjs spin --runtime "$(command -v spin)" + fi diff --git a/AGENTS.md b/AGENTS.md index 7688c1829f..d90eda1f2d 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -375,6 +375,8 @@ Every PR must pass: 11. Documentation proposal scripts: `shellcheck scripts/docs-proposal/*.sh && node --test scripts/docs-proposal/hunks.test.mjs && scripts/docs-proposal/test.sh` 12. URL-host linter on lines the PR adds (`lint-domains` job, PRs only); run it locally with `cargo run --manifest-path crates/trusted-server-cli/Cargo.toml --target --bin ts -- dev lint domains --changed-vs origin/main`, using the PR's base branch in place of `main` when it targets another branch 13. CI Node pin guard: `node --test scripts/check-ci-node-pin.test.mjs && node scripts/check-ci-node-pin.mjs` with the `.tool-versions` Node (every CI job that compiles Rust sets up the `.tool-versions` Node before its first cargo step) +14. Fastly absolute-deadline network fixtures under Viceroy (the harness builds the adapter test binary itself): `node crates/trusted-server-adapter-fastly/tests/deadline-origin.mjs "$(command -v viceroy)"` +15. Worker auction-deadline runtime fixtures (`worker-auction-deadlines.yml`): build and lint the runtime fixture, then run it under the local runtime. For Cloudflare, `cargo build --locked -p trusted-server-adapter-cloudflare --example deadline-runtime --target wasm32-unknown-unknown --features cloudflare` and `node scripts/test-worker-auction-deadlines.mjs cloudflare --runtime "$(command -v wrangler)"` (requires wrangler and the `Cargo.lock` wasm-bindgen CLI). For Spin, `cargo build --locked -p trusted-server-adapter-spin --example deadline-runtime --target wasm32-wasip1 --features spin` and `node scripts/test-worker-auction-deadlines.mjs spin --runtime "$(command -v spin)"`. CI lints each fixture with the same `cargo clippy` arguments plus `-- -D warnings` CI also runs these HTML EOF checks. Run them locally only when changing EOF finalization, publisher or HTML-processor code: diff --git a/CHANGELOG.md b/CHANGELOG.md index 338ca6a7b7..228837b0f3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed +- Auction response collection now bounds network waiting by `[auction].timeout_ms` + measured from dispatch, including delayed headers and trickling bodies on all + adapters. Complete available responses survive a slow publisher origin; + unfinished providers time out and mediation is skipped after budget exhaustion. + Provider bids that are already complete at the final nonblocking snapshot remain + eligible even after `tmax`; mediation starts only with remaining budget and + rejects responses collected after its cutoff. + Fastly abandons unfinished requests; Axum, Cloudflare, and Spin cancel owned work. - Explicit TS Console activation with `?ts_console=1` now sets a 30-minute diagnostics cookie instead of a browser-session cookie, independently of the mobile trace flag. Ordinary requests do not refresh it; end and explicitly enable diagnostics again to adopt the lifetime for an existing session. - Publisher documents with diagnostics active now use private, no-store responses and strip conditional and range request headers before fetching the origin. An unexpected origin `304` becomes a private `502` because it cannot supply an instrumented document, including when mobile tracing is disabled. - **Breaking:** `x-ts-version` now reports the deployed git version — the release tag, else the branch, else the first 6 characters of the commit — compiled in from the build-time `TRUSTED_SERVER__GIT_VERSION` (set by the deploy pipeline) or local git, and is sent by every adapter, including on the Fastly `GET /health` probe. The Fastly service version it previously carried is now `x-ts-fastly-version`; update dashboards, monitors, and scripts that read `x-ts-version` as the Fastly version number. Builds with neither the override nor git omit the header. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 84b1d58a07..f52d2c4bde 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -146,6 +146,14 @@ link it in the description, and mark it ready for review again. - Update the canonical guide or crate README when a user-visible contract changes; do not copy volatile matrices into multiple documents. +## Fastly dependency updates + +Update the workspace `fastly` and `fastly-sys` dependencies together. The +Fastly deadline collector uses raw asynchronous I/O readiness calls from +`fastly-sys` with handles created by `fastly`, so both must use a compatible +SDK ABI. Check the resolved versions in `Cargo.lock` and run the Fastly +adapter tests, including the real deadline network fixture, after an SDK update. + ## Verify Run the target-matched checks in [AGENTS.md](AGENTS.md#ci-gates), the diff --git a/Cargo.lock b/Cargo.lock index 9e360f9b57..70fcd650a8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6556,6 +6556,7 @@ dependencies = [ "tokio", "tower 0.4.13", "trusted-server-core", + "web-time", ] [[package]] @@ -6576,6 +6577,7 @@ dependencies = [ "tokio", "toml", "trusted-server-core", + "web-time", "worker", ] @@ -6593,6 +6595,7 @@ dependencies = [ "error-stack", "esi", "fastly", + "fastly-sys", "fern", "futures", "log", @@ -6603,6 +6606,7 @@ dependencies = [ "trusted-server-core", "url", "urlencoding", + "web-time", ] [[package]] @@ -6624,6 +6628,7 @@ dependencies = [ "spin-sdk", "tokio", "trusted-server-core", + "web-time", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index d33a1d5e8b..139cf4f610 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -66,6 +66,7 @@ env_logger = "0.11" error-stack = "0.6" esi = "0.7.2" fastly = "0.13" +fastly-sys = "0.13.1" fern = "0.7.1" flate2 = "1.1" futures = "0.3" diff --git a/crates/trusted-server-adapter-axum/Cargo.toml b/crates/trusted-server-adapter-axum/Cargo.toml index 94ad02170c..af707aa24e 100644 --- a/crates/trusted-server-adapter-axum/Cargo.toml +++ b/crates/trusted-server-adapter-axum/Cargo.toml @@ -29,6 +29,7 @@ reqwest = { workspace = true } simple_logger = { workspace = true } tokio = { workspace = true, features = ["rt-multi-thread", "macros", "sync", "time"] } trusted-server-core = { workspace = true } +web-time = { workspace = true } [dev-dependencies] trusted-server-core = { workspace = true, features = ["test-utils"] } diff --git a/crates/trusted-server-adapter-axum/src/platform.rs b/crates/trusted-server-adapter-axum/src/platform.rs index f3f9a4e11b..229bb902f7 100644 --- a/crates/trusted-server-adapter-axum/src/platform.rs +++ b/crates/trusted-server-adapter-axum/src/platform.rs @@ -446,6 +446,43 @@ impl PlatformHttpClient for AxumPlatformHttpClient { Ok(PlatformPendingRequest::new(pending).with_backend_name(backend_name)) } + async fn select_until( + &self, + pending_requests: Vec, + deadline: web_time::Instant, + ) -> Result, Report> { + let mut handles = recover_pending_handles(pending_requests)?; + if handles.is_empty() { + return Err(Report::new(PlatformError::HttpClient) + .attach("select_until called with no pending requests")); + } + let remaining = deadline.saturating_duration_since(web_time::Instant::now()); + if !remaining.is_zero() { + // Borrow the handles so timer expiry cannot drop completed siblings + // before the final nonblocking snapshot inspects them. + let selected = + tokio::time::timeout(remaining, futures::future::select_all(handles.iter_mut())) + .await; + if let Ok(((backend_name, result), index, _)) = selected { + drop(handles.swap_remove(index)); + return Ok(Some(completed_selection(backend_name, result, handles))); + } + } + // Snapshot once: only completed tasks survive into subsequent calls. + // An unfinished task is aborted on drop and cannot become a late bid. + handles.retain(|pending| pending.handle.is_finished()); + if handles.is_empty() { + return Ok(None); + } + let ((backend_name, result), _, remaining_handles) = + futures::future::select_all(handles).await; + Ok(Some(completed_selection( + backend_name, + result, + remaining_handles, + ))) + } + async fn select( &self, pending_requests: Vec, @@ -455,15 +492,7 @@ impl PlatformHttpClient for AxumPlatformHttpClient { .attach("select called with an empty pending_requests list")); } - let handles: Vec = pending_requests - .into_iter() - .map(|pr| { - pr.downcast::().map_err(|_| { - Report::new(PlatformError::HttpClient) - .attach("unexpected inner type in AxumPlatformHttpClient::select") - }) - }) - .collect::, _>>()?; + let handles = recover_pending_handles(pending_requests)?; // Each AxumPendingHandle resolves to (backend_name, result), so the // remaining handles keep their own backend names — no positional @@ -472,48 +501,68 @@ impl PlatformHttpClient for AxumPlatformHttpClient { let ((backend_name, result), _ready_idx, remaining_handles) = futures::future::select_all(handles).await; - let remaining: Vec = remaining_handles - .into_iter() - .map(|handle| { - let backend_name = handle.backend_name.clone(); - PlatformPendingRequest::new(handle).with_backend_name(backend_name) - }) - .collect(); + Ok(completed_selection(backend_name, result, remaining_handles)) + } +} + +fn completed_selection( + backend_name: String, + result: Result, + remaining_handles: Vec, +) -> PlatformSelectResult { + let remaining: Vec = remaining_handles + .into_iter() + .map(|handle| { + let backend_name = handle.backend_name.clone(); + PlatformPendingRequest::new(handle).with_backend_name(backend_name) + }) + .collect(); + + // Map join panics and per-request errors into ready: Err(...) so that the + // auction orchestrator can log the failure and continue with remaining providers + // rather than treating one bad provider as a fatal select() failure. + let ready = result + .map_err(|e| { + Report::new(PlatformError::HttpClient) + .attach(format!("auction request task panicked: {e}")) + }) + .and_then(|inner| inner) + .and_then(|(status, headers, body)| { + let mut builder = edgezero_core::http::response_builder().status(status); + for (name, value) in &headers { + builder = builder.header(name.as_str(), value.as_slice()); + } + builder + .body(edgezero_core::body::Body::from(body)) + .change_context(PlatformError::HttpClient) + }) + .map(|edge_resp| PlatformResponse::new(edge_resp).with_backend_name(backend_name.clone())); + + // Attribute the failure to its backend so the orchestrator can remove + // the provider and record a BidStatus::Error, matching the Fastly + // adapter. Without this, a failed provider silently vanishes through + // the orchestrator's "backend not identified" branch. + let failed_backend_name = ready.as_ref().err().map(|_| backend_name); + + PlatformSelectResult { + ready, + remaining, + failed_backend_name, + } +} - // Map join panics and per-request errors into ready: Err(...) so that the - // auction orchestrator can log the failure and continue with remaining providers - // rather than treating one bad provider as a fatal select() failure. - let ready = result - .map_err(|e| { +fn recover_pending_handles( + pending_requests: Vec, +) -> Result, Report> { + pending_requests + .into_iter() + .map(|pending| { + pending.downcast::().map_err(|_| { Report::new(PlatformError::HttpClient) - .attach(format!("auction request task panicked: {e}")) - }) - .and_then(|inner| inner) - .and_then(|(status, headers, body)| { - let mut builder = edgezero_core::http::response_builder().status(status); - for (name, value) in &headers { - builder = builder.header(name.as_str(), value.as_slice()); - } - builder - .body(edgezero_core::body::Body::from(body)) - .change_context(PlatformError::HttpClient) + .attach("unexpected pending type in Axum selection") }) - .map(|edge_resp| { - PlatformResponse::new(edge_resp).with_backend_name(backend_name.clone()) - }); - - // Attribute the failure to its backend so the orchestrator can remove - // the provider and record a BidStatus::Error, matching the Fastly - // adapter. Without this, a failed provider silently vanishes through - // the orchestrator's "backend not identified" branch. - let failed_backend_name = ready.as_ref().err().map(|_| backend_name); - - Ok(PlatformSelectResult { - ready, - remaining, - failed_backend_name, }) - } + .collect() } // --------------------------------------------------------------------------- @@ -635,11 +684,6 @@ mod tests { .capabilities(); assert!(client.supports_concurrent_fanout()); assert!(capabilities.supports_concurrent_provider_fanout()); - assert!(!client.has_enforceable_total_request_deadline()); - assert!( - !capabilities.has_enforceable_total_request_deadline(), - "reqwest's transport timeout is not an adapter-enforced auction deadline" - ); } #[test] @@ -874,6 +918,275 @@ mod tests { ); } + async fn deadline_pending( + client: &AxumPlatformHttpClient, + url: String, + backend: &str, + ) -> PlatformPendingRequest { + let request = edgezero_core::http::request_builder() + .uri(url) + .body(EdgeBody::empty()) + .expect("should build deadline request"); + client + .send_async(PlatformHttpRequest::new(request, backend)) + .await + .expect("should dispatch deadline request") + } + + async fn serve_slow_response(trickle_body: bool) -> (String, tokio::task::JoinHandle<()>) { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("should bind deadline server"); + let address = listener + .local_addr() + .expect("should read deadline server address"); + let task = tokio::spawn(async move { + let (mut stream, _) = listener + .accept() + .await + .expect("should accept deadline request"); + let mut request = [0; 1024]; + let bytes_read = stream + .read(&mut request) + .await + .expect("should read deadline request"); + assert!(bytes_read > 0, "should receive a deadline request"); + if trickle_body { + if stream + .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 100\r\n\r\n") + .await + .is_err() + { + return; + } + for _ in 0..100 { + if stream.write_all(b"x").await.is_err() { + return; + } + tokio::time::sleep(Duration::from_millis(50)).await; + } + } else { + tokio::time::sleep(Duration::from_secs(5)).await; + let _ = stream + .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nok") + .await; + } + }); + (format!("http://{address}/"), task) + } + + #[tokio::test] + async fn select_until_bounds_delayed_headers_and_trickling_bodies() { + let client = AxumPlatformHttpClient::new(); + for trickle_body in [false, true] { + let (url, server) = serve_slow_response(trickle_body).await; + let pending = deadline_pending(&client, url, "slow").await; + let start = web_time::Instant::now(); + let result = tokio::time::timeout( + Duration::from_secs(2), + client.select_until(vec![pending], start + Duration::from_millis(200)), + ) + .await + .expect("should finish before the slow response") + .expect("should enforce the deadline"); + server.abort(); + assert!( + result.is_none(), + "should time out before the entire response completes" + ); + assert!( + start.elapsed() < Duration::from_secs(2), + "should bound the complete response wait" + ); + } + } + + #[tokio::test] + async fn select_until_preserves_fast_success_before_slow_body_expires() { + let client = AxumPlatformHttpClient::new(); + let (slow_url, server) = serve_slow_response(true).await; + let fast_url = serve_raw_response(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nok").await; + let slow = deadline_pending(&client, slow_url, "slow").await; + let fast = deadline_pending(&client, fast_url, "fast").await; + let deadline = web_time::Instant::now() + Duration::from_millis(500); + let result = client + .select_until(vec![slow, fast], deadline) + .await + .expect("should collect fast response") + .expect("should complete before the deadline"); + assert_eq!( + result + .ready + .expect("should succeed") + .backend_name + .as_deref(), + Some("fast"), + "should collect the fast complete body first" + ); + assert_eq!( + result.remaining.len(), + 1, + "should preserve the slow body handle" + ); + let result = client + .select_until(result.remaining, deadline) + .await + .expect("should bound slow response"); + server.abort(); + assert!(result.is_none(), "should expire the remaining slow body"); + } + + #[tokio::test] + async fn select_until_expired_deadline_preserves_buffered_response() { + let client = AxumPlatformHttpClient::new(); + let handle = tokio::spawn(async { Ok((200, Vec::new(), b"ok".to_vec())) }); + while !handle.is_finished() { + tokio::task::yield_now().await; + } + let pending = PlatformPendingRequest::new(AxumPendingHandle { + backend_name: "ready".to_owned(), + handle, + }); + let result = client + .select_until(vec![pending], web_time::Instant::now()) + .await + .expect("should recognize expired deadline"); + assert!( + result.is_some(), + "should preserve a complete buffered response without waiting after cutoff" + ); + } + + #[tokio::test] + async fn expired_snapshot_preserves_finished_siblings_and_aborts_unfinished_task() { + let mut tasks = Vec::new(); + for name in ["ready-a", "ready-b"] { + tasks.push(AxumPendingHandle { + backend_name: name.to_owned(), + handle: tokio::spawn(async { Ok((200, Vec::new(), b"ok".to_vec())) }), + }); + } + while tasks.iter().any(|pending| !pending.handle.is_finished()) { + tokio::task::yield_now().await; + } + let (started_sender, started_receiver) = tokio::sync::oneshot::channel(); + let (drop_sender, drop_receiver) = tokio::sync::oneshot::channel::<()>(); + tasks.push(AxumPendingHandle { + backend_name: "unfinished".to_owned(), + handle: tokio::spawn(async move { + let _drop_signal = drop_sender; + started_sender + .send(()) + .expect("should signal unfinished task start"); + futures::future::pending::().await + }), + }); + started_receiver + .await + .expect("should start unfinished task"); + let pending = tasks + .into_iter() + .map(|task| { + let name = task.backend_name.clone(); + PlatformPendingRequest::new(task).with_backend_name(name) + }) + .collect(); + let client = AxumPlatformHttpClient::new(); + let deadline = web_time::Instant::now(); + let first = client + .select_until(pending, deadline) + .await + .expect("should snapshot ready responses") + .expect("should retain first response"); + assert!( + first.ready.is_ok(), + "should preserve first completed result" + ); + assert_eq!( + first.remaining.len(), + 1, + "should retain only the other completed response" + ); + let second = client + .select_until(first.remaining, deadline) + .await + .expect("should drain the completed snapshot") + .expect("should retain second response"); + assert!( + second.ready.is_ok(), + "should preserve the second completed result" + ); + assert!( + second.remaining.is_empty(), + "should leave no unfinished handles" + ); + assert!( + tokio::time::timeout(Duration::from_secs(1), drop_receiver) + .await + .expect("should promptly abort unfinished work") + .is_err(), + "should release unfinished task resources" + ); + } + + #[tokio::test] + async fn select_until_aborts_task_on_expiration() { + let (started_sender, started_receiver) = tokio::sync::oneshot::channel(); + let (dropped_sender, dropped_receiver) = tokio::sync::oneshot::channel::<()>(); + let handle = tokio::spawn(async move { + let _drop_signal = dropped_sender; + started_sender.send(()).expect("should signal task start"); + futures::future::pending::().await + }); + started_receiver + .await + .expect("should start task before selection"); + let pending = PlatformPendingRequest::new(AxumPendingHandle { + backend_name: "slow".to_owned(), + handle, + }); + let result = AxumPlatformHttpClient::new() + .select_until( + vec![pending], + web_time::Instant::now() + Duration::from_millis(50), + ) + .await + .expect("should expire task"); + assert!(result.is_none(), "should report deadline expiration"); + assert!( + tokio::time::timeout(Duration::from_secs(2), dropped_receiver) + .await + .expect("should promptly drop aborted task state") + .is_err(), + "should drop task resources on expiration" + ); + } + + #[tokio::test] + async fn select_until_preserves_failed_backend_attribution() { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("should bind probe"); + let address = listener.local_addr().expect("should read probe address"); + drop(listener); + let client = AxumPlatformHttpClient::new(); + let pending = deadline_pending(&client, format!("http://{address}/"), "failed").await; + let result = client + .select_until( + vec![pending], + web_time::Instant::now() + Duration::from_secs(2), + ) + .await + .expect("should collect transport failure") + .expect("should fail before deadline"); + assert!(result.ready.is_err(), "should preserve transport failure"); + assert_eq!( + result.failed_backend_name.as_deref(), + Some("failed"), + "should attribute failure to its backend" + ); + } + async fn serve_raw_response(response: &'static [u8]) -> String { let listener = tokio::net::TcpListener::bind("127.0.0.1:0") .await diff --git a/crates/trusted-server-adapter-cloudflare/Cargo.toml b/crates/trusted-server-adapter-cloudflare/Cargo.toml index 0a243bc2a5..36a9041641 100644 --- a/crates/trusted-server-adapter-cloudflare/Cargo.toml +++ b/crates/trusted-server-adapter-cloudflare/Cargo.toml @@ -31,6 +31,7 @@ edgezero-core = { workspace = true } error-stack = { workspace = true } futures = { workspace = true } log = { workspace = true } +web-time = { workspace = true } serde_json = { workspace = true } trusted-server-core = { workspace = true } worker = { workspace = true, optional = true } @@ -45,4 +46,11 @@ trusted-server-core = { workspace = true, features = ["test-utils"] } base64 = { workspace = true } edgezero-core = { workspace = true } toml = { workspace = true } +[target.'cfg(not(target_arch = "wasm32"))'.dev-dependencies] tokio = { workspace = true, features = ["rt-multi-thread", "macros"] } + +[[example]] +name = "deadline-runtime" +path = "tests/runtime/deadline.rs" +crate-type = ["cdylib"] +required-features = ["cloudflare"] diff --git a/crates/trusted-server-adapter-cloudflare/src/platform.rs b/crates/trusted-server-adapter-cloudflare/src/platform.rs index 3b798bd78d..40cfc7b3a1 100644 --- a/crates/trusted-server-adapter-cloudflare/src/platform.rs +++ b/crates/trusted-server-adapter-cloudflare/src/platform.rs @@ -19,7 +19,8 @@ use trusted_server_core::platform::UnavailableHttpClient; #[cfg(target_arch = "wasm32")] use trusted_server_core::platform::{ - PlatformHttpRequest, PlatformPendingRequest, PlatformResponse, PlatformSelectResult, + PlatformHttpRequest, PlatformPendingRequest, PlatformPendingResponse, PlatformResponse, + PlatformSelectResult, }; // --------------------------------------------------------------------------- @@ -164,33 +165,15 @@ impl KvStore for KvHandleAdapter { // CloudflareHttpClient — WASM target only // --------------------------------------------------------------------------- -/// Carries a completed response through `send_async` → `select`. -/// -/// Same pattern as `AxumPendingResponse`: stores raw parts because `Body::Stream` -/// is `!Send`, which is incompatible with `Box` inside -/// [`PlatformPendingRequest`]. -#[cfg(target_arch = "wasm32")] -struct CloudflarePendingResponse { - backend_name: String, - status: u16, - headers: Vec<(String, Vec)>, - body: Vec, -} - /// [`worker::Fetch`]-backed HTTP client for the Cloudflare Workers runtime. /// /// # Multi-provider auction limitation /// -/// `send_async` eagerly awaits each request before returning, so -/// [`PlatformHttpClient::supports_concurrent_fanout`] reports `false` and the -/// auction orchestrator rejects multi-provider configurations before any -/// request launches. `select` keeps a defense-in-depth rejection for more -/// than one pending request. Configure a single auction provider for -/// Cloudflare Workers, or use the Fastly adapter for parallel DSP fan-out. -/// -/// Per-provider timeouts baked into the backend name are not enforced at the -/// fetch layer; the Workers runtime's global CPU budget (~30 s on paid plans) -/// is the only implicit deadline. +/// Auction configuration supports one provider at a time, so +/// [`PlatformHttpClient::supports_concurrent_fanout`] reports `false`. +/// Requests start in a runtime task at dispatch, overlapping publisher-origin work. +/// Deadline selection bounds waiting for fetch and body collection and aborts +/// the Workers fetch when its pending operation is dropped. #[cfg(target_arch = "wasm32")] pub struct CloudflareHttpClient; @@ -285,10 +268,21 @@ fn outbound_request_init(method: worker::Method, headers: worker::Headers) -> wo init } +#[cfg(target_arch = "wasm32")] +struct AbortOnDrop(Option); + +#[cfg(target_arch = "wasm32")] +impl Drop for AbortOnDrop { + fn drop(&mut self) { + if let Some(controller) = self.0.take() { + controller.abort(); + } + } +} + #[cfg(target_arch = "wasm32")] impl CloudflareHttpClient { async fn execute( - &self, request: PlatformHttpRequest, ) -> Result> { use worker::{CacheMode, Fetch, Headers, Method, Request}; @@ -368,8 +362,14 @@ impl CloudflareHttpClient { let worker_req = Request::new_with_init(&uri, &init).change_context(PlatformError::HttpClient)?; + let abort = AbortOnDrop(Some(worker::AbortController::default())); + let signal = abort + .0 + .as_ref() + .expect("should retain abort controller") + .signal(); let mut resp = Fetch::Request(worker_req) - .send() + .send_with_signal(&signal) .await .change_context(PlatformError::HttpClient) .attach_with(|| format!("outbound request to {uri} failed"))?; @@ -451,13 +451,13 @@ impl PlatformHttpClient for CloudflareHttpClient { &self, request: PlatformHttpRequest, ) -> Result> { - self.execute(request).await + Self::execute(request).await } fn supports_concurrent_fanout(&self) -> bool { - // `send_async` executes each request eagerly, so multiple pending - // requests run sequentially. The auction orchestrator checks this - // before launching more than one provider request. + // Requests start concurrently at dispatch, but `select` and + // `select_until` collect only one response, so multi-provider fan-out + // stays unsupported. false } @@ -466,34 +466,10 @@ impl PlatformHttpClient for CloudflareHttpClient { request: PlatformHttpRequest, ) -> Result> { let backend_name = request.backend_name.clone(); - let response = self.execute(request).await?; - - let status = response.response.status().as_u16(); - let headers: Vec<(String, Vec)> = response - .response - .headers() - .iter() - .map(|(n, v)| (n.to_string(), v.as_bytes().to_vec())) - .collect(); - let body_bytes = match response.response.into_body() { - edgezero_core::body::Body::Once(bytes) => bytes.to_vec(), - // execute() always buffers via resp.bytes().await → Body::Once. - // Return a typed error rather than panicking in the request path - // in case that edgezero implementation detail ever changes. - edgezero_core::body::Body::Stream(_) => { - return Err(Report::new(PlatformError::HttpClient).attach( - "unexpected streaming body from CloudflareHttpClient::execute \ - — expected a buffered Body::Once", - )); - } - }; - - let pending = CloudflarePendingResponse { - backend_name: backend_name.clone(), - status, - headers, - body: body_bytes, - }; + let pending = PlatformPendingResponse::start( + Self::execute(request), + worker::wasm_bindgen_futures::spawn_local, + ); Ok(PlatformPendingRequest::new(pending).with_backend_name(backend_name)) } @@ -509,28 +485,80 @@ impl PlatformHttpClient for CloudflareHttpClient { reject_multi_provider_fanout(pending_requests.len())?; let ready_platform = pending_requests.remove(0); - let pending = ready_platform - .downcast::() + let mut pending = ready_platform + .downcast::>>>() .map_err(|_| { Report::new(PlatformError::HttpClient) .attach("unexpected inner type in CloudflareHttpClient::select") })?; - let mut builder = edgezero_core::http::response_builder().status(pending.status); - for (name, value) in &pending.headers { - builder = builder.header(name.as_str(), value.as_slice()); - } - let edge_resp = builder - .body(edgezero_core::body::Body::from(pending.body)) - .change_context(PlatformError::HttpClient)?; - - let ready = Ok(PlatformResponse::new(edge_resp).with_backend_name(pending.backend_name)); + let ready = pending.wait().await.unwrap_or_else(|error| { + Err(Report::new(PlatformError::HttpClient) + .attach(format!("dispatched response task stopped: {error}"))) + }); Ok(PlatformSelectResult { ready, remaining: pending_requests, failed_backend_name: None, }) } + + async fn select_until( + &self, + mut pending_requests: Vec, + deadline: web_time::Instant, + ) -> Result, Report> { + if pending_requests.is_empty() { + return Err(Report::new(PlatformError::HttpClient) + .attach("select_until called with an empty pending_requests list")); + } + if pending_requests.len() > 1 { + return Err(Report::new(PlatformError::HttpClient) + .attach("multi-provider fan-out is not supported on this adapter")); + } + let platform_pending = pending_requests.remove(0); + let backend_name = platform_pending.backend_name().map(str::to_owned); + let mut pending = platform_pending + .downcast::>>>() + .map_err(|_| { + Report::new(PlatformError::HttpClient) + .attach("unexpected pending request type in deadline selection") + })?; + let result = pending + .complete_before_deadline( + async { + // Workers timers use whole signed milliseconds. Recheck the + // absolute deadline after each bounded timer interval. + loop { + let remaining = + deadline.saturating_duration_since(web_time::Instant::now()); + if remaining.is_zero() { + break; + } + worker::Delay::from( + remaining + .max(Duration::from_millis(1)) + .min(Duration::from_millis(i32::MAX as u64)), + ) + .await; + } + }, + deadline, + web_time::Instant::now, + ) + .await; + Ok(result.map(|result| { + let ready = result.unwrap_or_else(|error| { + Err(Report::new(PlatformError::HttpClient) + .attach(format!("dispatched response task stopped: {error}"))) + }); + PlatformSelectResult { + failed_backend_name: ready.as_ref().err().and(backend_name), + ready, + remaining: pending_requests, + } + })) + } } // --------------------------------------------------------------------------- @@ -737,9 +765,9 @@ fn extract_client_ip(ctx: &edgezero_core::context::RequestContext) -> Option worker::Result { + let uri = request.url()?.to_string(); + let result = exercise(&uri) + .await + .map_err(|error| worker::Error::RustError(format!("{error:?}")))?; + worker::Response::ok(result) +} + +#[cfg(target_arch = "wasm32")] +async fn exercise( + uri: &str, +) -> Result> { + let parameter = |name: &str| { + uri.split_once('?').and_then(|(_, query)| { + query + .split('&') + .filter_map(|pair| pair.split_once('=')) + .find(|(key, _)| *key == name) + .map(|(_, value)| value) + }) + }; + let upstream = parameter("upstream").expect("should supply local fixture upstream"); + let budget_ms: u64 = parameter("budget_ms") + .expect("should supply budget") + .parse() + .expect("should parse budget"); + let origin_ms: u64 = parameter("origin_ms") + .expect("should supply origin delay") + .parse() + .expect("should parse origin delay"); + let client = platform::CloudflareHttpClient; + let request = edgezero_core::http::request_builder() + .uri(upstream) + .body(edgezero_core::body::Body::empty()) + .expect("should construct fixture request"); + let started = web_time::Instant::now(); + let pending = client + .send_async(PlatformHttpRequest::new(request, "fixture")) + .await?; + worker::Delay::from(Duration::from_millis(origin_ms)).await; + let collected = client + .select_until(vec![pending], started + Duration::from_millis(budget_ms)) + .await?; + let status = match collected { + Some(result) => { + let response = result.ready?; + assert_eq!( + response.response.body().as_bytes(), + Some(b"okay".as_slice()), + "should retain the complete nonempty upstream body" + ); + "complete" + } + None => "timeout", + }; + Ok(format!("{status}:{}", started.elapsed().as_millis())) +} diff --git a/crates/trusted-server-adapter-fastly/Cargo.toml b/crates/trusted-server-adapter-fastly/Cargo.toml index 710523766e..6c7121d0d6 100644 --- a/crates/trusted-server-adapter-fastly/Cargo.toml +++ b/crates/trusted-server-adapter-fastly/Cargo.toml @@ -28,6 +28,8 @@ edgezero-core = { workspace = true } error-stack = { workspace = true } esi = { workspace = true } fastly = { workspace = true } +# The SDK keeps timed body readiness private; auction deadlines use the ABI. +fastly-sys = { workspace = true } fern = { workspace = true } futures = { workspace = true } log = { workspace = true } @@ -37,6 +39,7 @@ serde_json = { workspace = true } trusted-server-core = { workspace = true } url = { workspace = true } urlencoding = { workspace = true } +web-time = { workspace = true } [dev-dependencies] bytes = { workspace = true } diff --git a/crates/trusted-server-adapter-fastly/src/platform.rs b/crates/trusted-server-adapter-fastly/src/platform.rs index d0f4ee9d0e..ea2193d090 100644 --- a/crates/trusted-server-adapter-fastly/src/platform.rs +++ b/crates/trusted-server-adapter-fastly/src/platform.rs @@ -1,6 +1,8 @@ //! Fastly-backed implementations of the platform traits defined in //! `trusted-server-core::platform`. +mod deadline; + use bytes::Bytes; use error_stack::{Report, ResultExt}; use fastly::geo::{Geo, geo_lookup}; @@ -20,6 +22,19 @@ use trusted_server_core::platform::{ }; use trusted_server_core::settings::TrustedClientIpConfig; +// Cache-pass requests always have a direct SDK pending handle. Keep this proof +// with the pending request until the deadline collector consumes it. +struct CacheBypassPending(fastly::http::request::PendingRequest); + +fn unwrap_pending( + pending: PlatformPendingRequest, +) -> Result { + match pending.downcast::() { + Ok(pending) => Ok(pending.0), + Err(pending) => pending.downcast(), + } +} + // --------------------------------------------------------------------------- // FastlyPlatformConfigStore // --------------------------------------------------------------------------- @@ -183,6 +198,21 @@ impl PlatformBackend for FastlyPlatformBackend { } } +fn buffered_response_to_platform( + response: &fastly::Response, + backend_name: String, + bytes: Vec, +) -> Result> { + let mut builder = edgezero_core::http::response_builder().status(response.get_status()); + for (name, value) in response.get_headers() { + builder = builder.header(name.as_str(), value.as_bytes()); + } + let response = builder + .body(edgezero_core::body::Body::from(bytes)) + .change_context(PlatformError::HttpClient)?; + Ok(PlatformResponse::new(response).with_backend_name(backend_name)) +} + // --------------------------------------------------------------------------- // FastlyPlatformHttpClient — helpers // --------------------------------------------------------------------------- @@ -492,6 +522,11 @@ fn apply_fastly_cache_intent(request: &mut fastly::Request, intent: &PlatformCac /// environment cannot register a real Fastly backend, so the common /// `StubHttpClient` driver test records the one-send 3xx behavior while adapter /// tests cover request conversion and the single-send boundary. +/// +/// Deadline collection requires [`PlatformCacheIntent::Bypass`], which all +/// auction requests use. This guarantees a direct SDK pending handle that can +/// join body handles in one timed host wait. Shared-cache requests retain the +/// SDK caching state machine through the unbounded send/select/wait APIs. pub struct FastlyPlatformHttpClient; #[async_trait::async_trait(?Send)] @@ -541,7 +576,12 @@ impl PlatformHttpClient for FastlyPlatformHttpClient { let pending = fastly_req .send_async(&backend_name) .change_context(PlatformError::HttpClient)?; - Ok(PlatformPendingRequest::new(pending) + let pending = if cache_intent.is_bypass() { + PlatformPendingRequest::new(CacheBypassPending(pending)) + } else { + PlatformPendingRequest::new(pending) + }; + Ok(pending .with_backend_name(backend_name) .with_response_handling(stream_response, request_method)) } @@ -568,7 +608,7 @@ impl PlatformHttpClient for FastlyPlatformHttpClient { let mut fastly_pending: Vec = Vec::with_capacity(pending_requests.len()); for platform_req in pending_requests { - let inner = platform_req.downcast::().map_err(|platform_req| { + let inner = unwrap_pending(platform_req).map_err(|platform_req| { let backend_name = platform_req.backend_name().unwrap_or(""); Report::new(PlatformError::HttpClient).attach(format!( "PlatformPendingRequest inner type is not fastly::PendingRequest for backend '{backend_name}'" @@ -622,16 +662,22 @@ impl PlatformHttpClient for FastlyPlatformHttpClient { }) } + async fn select_until( + &self, + pending_requests: Vec, + deadline: web_time::Instant, + ) -> Result, Report> { + deadline::select_until(pending_requests, deadline) + } + async fn wait( &self, pending: PlatformPendingRequest, ) -> Result> { - use fastly::http::request::PendingRequest; - let backend_hint = pending.backend_name().map(str::to_owned); let stream_response = pending.stream_response(); let request_is_head = pending.request_method() == Some(&edgezero_core::http::Method::HEAD); - let pending = pending.downcast::().map_err(|pending| { + let pending = unwrap_pending(pending).map_err(|pending| { let backend_name = pending.backend_name().unwrap_or(""); Report::new(PlatformError::HttpClient).attach(format!( "PlatformPendingRequest inner type is not fastly::PendingRequest for backend '{backend_name}'" @@ -1191,11 +1237,6 @@ mod tests { .capabilities(); assert!(client.supports_concurrent_fanout()); assert!(capabilities.supports_concurrent_provider_fanout()); - assert!(!client.has_enforceable_total_request_deadline()); - assert!( - !capabilities.has_enforceable_total_request_deadline(), - "first-byte and between-byte timers are not a hard total request deadline" - ); } #[test] diff --git a/crates/trusted-server-adapter-fastly/src/platform/deadline.rs b/crates/trusted-server-adapter-fastly/src/platform/deadline.rs new file mode 100644 index 0000000000..146f189dff --- /dev/null +++ b/crates/trusted-server-adapter-fastly/src/platform/deadline.rs @@ -0,0 +1,947 @@ +//! Deadline-bounded collection of Fastly auction responses. + +use error_stack::{Report, ResultExt as _}; +use fastly::handle::{BodyHandle, PendingRequestHandle, PollHandleResult}; +use std::io::Read as _; +use trusted_server_core::platform::{ + PlatformError, PlatformPendingRequest, PlatformResponse, PlatformSelectResult, +}; +use web_time::Instant; + +use super::{ + CacheBypassPending, MAX_PLATFORM_RESPONSE_BODY_BYTES, buffered_response_to_platform, + fastly_response_to_platform, response_carries_body, +}; + +// Bound payload retained across all partial bodies, as well as each response. +const MAX_TOTAL_BUFFERED_BODY_BYTES: usize = MAX_PLATFORM_RESPONSE_BODY_BYTES; + +// Keep partially buffered responses alongside requests still awaiting headers so +// an early header with a slow body cannot hold up a completed sibling response. +struct DeadlinePending { + backend_name: String, + request_method: Option, + state: Option, +} + +enum PendingState { + Headers(Option), + Ready(Result>), + Body { + response: Box, + body: BodyHandle, + bytes: Vec, + }, +} + +impl DeadlinePending { + fn from_platform(pending: PlatformPendingRequest) -> Result> { + if pending.stream_response() { + return Err(Report::new(PlatformError::HttpClient) + .attach("stream-marked pending request requires direct wait")); + } + let request_method = pending.request_method().cloned(); + let pending = match pending.downcast::() { + Ok(mut pending) => { + pending.request_method = request_method.or(pending.request_method); + return Ok(pending); + } + Err(pending) => pending, + }; + let backend_name = pending.backend_name().unwrap_or("").to_owned(); + // The marker is constructed only after set_pass(true), which selects + // HostCaching in fastly 0.13.1's Request::get_caching_mode. Its direct + // handle conversion therefore cannot encounter a guest caching state. + let pending = pending.downcast::().map_err(|_| { + Report::new(PlatformError::Unsupported).attach(format!( + "deadline collection requires a cache-bypass request for backend '{backend_name}'" + )) + })?; + let pending = PendingRequestHandle::from(pending.0); + Ok(Self { + backend_name, + request_method, + state: Some(PendingState::Headers(Some(pending))), + }) + } + + fn into_platform(self) -> PlatformPendingRequest { + let backend_name = self.backend_name.clone(); + let request_method = self.request_method.clone(); + let pending = PlatformPendingRequest::new(self).with_backend_name(backend_name); + match request_method { + Some(method) => pending.with_response_handling(false, method), + None => pending, + } + } + + fn request_is_head(&self, response: &fastly::Response) -> Result> { + if let Some(method) = &self.request_method { + return Ok(method == edgezero_core::http::Method::HEAD); + } + response + .get_backend_request() + .map(|request| request.get_method() == fastly::http::Method::HEAD) + .ok_or_else(|| { + Report::new(PlatformError::HttpClient).attach(format!( + "deadline response for backend '{}' has no originating request or method hint", + self.backend_name + )) + }) + } + + fn buffered_bytes(&self) -> usize { + match &self.state { + Some(PendingState::Body { bytes, .. }) => bytes.len(), + Some(PendingState::Ready(Ok(response))) => { + response.response.body().as_bytes().map_or(0, <[u8]>::len) + } + _ => 0, + } + } + + fn poll(&mut self) -> Result, Report> { + let state = self + .state + .take() + .expect("should retain pending collection state"); + let state = match state { + PendingState::Ready(result) => return result.map(Some), + PendingState::Headers(mut slot) => { + match slot.take().expect("should retain a pending request").poll() { + PollHandleResult::Pending(pending) => { + slot = Some(pending); + self.state = Some(PendingState::Headers(slot)); + return Ok(None); + } + PollHandleResult::Done(Err(error)) => { + return Err( + Report::new(PlatformError::HttpClient).attach(error.to_string()) + ); + } + PollHandleResult::Done(Ok((response_handle, body_handle))) => { + let mut response = + fastly::Response::from_handles(response_handle, body_handle); + let request_is_head = self.request_is_head(&response)?; + if !response_carries_body(request_is_head, response.get_status()) { + return fastly_response_to_platform( + response, + self.backend_name.clone(), + false, + request_is_head, + ) + .map(Some); + } + if response + .get_header("content-length") + .and_then(|value| value.to_str().ok()) + .and_then(|value| value.trim().parse::().ok()) + .is_some_and(|length| length > MAX_PLATFORM_RESPONSE_BODY_BYTES as u64) + { + return Err(Report::new(PlatformError::HttpClient) + .attach("origin Content-Length exceeds response body limit")); + } + let body = response.take_body().into_handle(); + PendingState::Body { + response: Box::new(response), + body, + bytes: Vec::new(), + } + } + } + } + state => state, + }; + let PendingState::Body { + response, + mut body, + mut bytes, + } = state + else { + unreachable!("headers are polled before buffering bodies"); + }; + if !body_is_ready(&body)? { + self.state = Some(PendingState::Body { + response, + body, + bytes, + }); + return Ok(None); + } + let mut chunk = [0_u8; 8192]; + let read = body + .read(&mut chunk) + .change_context(PlatformError::HttpClient)?; + if read == 0 { + return buffered_response_to_platform(&response, self.backend_name.clone(), bytes) + .map(Some); + } + if bytes.len() + read > MAX_PLATFORM_RESPONSE_BODY_BYTES { + return Err(Report::new(PlatformError::HttpClient) + .attach("origin response body exceeds response body limit")); + } + if bytes.len() + read > bytes.capacity() { + let capacity = bytes + .capacity() + .max(chunk.len()) + .saturating_mul(2) + .min(MAX_PLATFORM_RESPONSE_BODY_BYTES); + bytes.reserve_exact(capacity - bytes.len()); + } + bytes.extend_from_slice(&chunk[..read]); + self.state = Some(PendingState::Body { + response, + body, + bytes, + }); + Ok(None) + } +} + +fn body_is_ready(body: &BodyHandle) -> Result> { + let mut ready = 0; + // SAFETY: the live owned body retains the handle while this hostcall borrows it. + unsafe { fastly_sys::fastly_async_io::is_ready(body.as_u32(), &mut ready) } + .result() + .map_err(|status| { + Report::new(PlatformError::HttpClient) + .attach(format!("body readiness hostcall failed: {status:?}")) + })?; + Ok(ready != 0) +} + +/// Collect one fully buffered response without waiting past the absolute deadline. +/// +/// Dropping response bodies closes their host handles. Fastly has no pending +/// request cancellation ABI: unfinished header requests are abandoned and +/// never waited on past the auction deadline. If collection starts after the +/// deadline, one nonblocking snapshot preserves already available responses. +/// +/// Partial bodies share a 10 MiB payload budget. If a chunk exceeds that +/// capacity, the largest partial response fails with an attributed HTTP client +/// error; its buffered bytes are released and smaller siblings remain collectible. +/// Completed response payload plus retained partial bodies fits the same budget. +/// +/// # Errors +/// +/// Returns an error for an empty request list, invalid aggregate buffer state, +/// or a failure of the timed wait hostcall. Individual provider errors, including +/// unsupported pending handles, are attributed in the returned collection result. +pub(super) fn select_until( + pending_requests: Vec, + deadline: Instant, +) -> Result, Report> { + if pending_requests.is_empty() { + return Err(Report::new(PlatformError::HttpClient) + .attach("select_until called with an empty pending_requests list")); + } + let mut pending = pending_requests + .into_iter() + .map(|pending| { + let backend_name = pending.backend_name().unwrap_or("").to_owned(); + let request_method = pending.request_method().cloned(); + match DeadlinePending::from_platform(pending) { + Ok(pending) => pending, + Err(error) => DeadlinePending { + backend_name, + request_method, + state: Some(PendingState::Ready(Err(error))), + }, + } + }) + .collect::>(); + let mut buffered_bytes: usize = pending.iter().map(DeadlinePending::buffered_bytes).sum(); + if buffered_bytes > MAX_TOTAL_BUFFERED_BODY_BYTES { + return Err(Report::new(PlatformError::HttpClient) + .attach("resumed auction responses exceed aggregate buffer capacity")); + } + let expired_at_entry = Instant::now() >= deadline; + if expired_at_entry { + // Take one nonblocking snapshot. Fully buffered results are cached; + // responses that would require a wait are abandoned, never retried. + for index in 0..pending.len() { + loop { + let previous = pending[index].buffered_bytes(); + let result = pending[index].poll(); + let progressed = + matches!(result, Ok(None)) && pending[index].buffered_bytes() > previous; + match result { + Ok(None) if progressed => {} + Ok(None) => { + pending[index].state = None; + } + result => { + pending[index].state = Some(PendingState::Ready(match result { + Ok(Some(response)) => Ok(response), + Err(error) => Err(error), + Ok(None) => unreachable!("pending results handled above"), + })); + } + } + buffered_bytes = buffered_bytes - previous + pending[index].buffered_bytes(); + enforce_aggregate_budget(&mut pending, &mut buffered_bytes); + if !progressed || !matches!(pending[index].state, Some(PendingState::Body { .. })) { + break; + } + } + } + pending.retain(|item| item.state.is_some()); + } + loop { + for index in 0..pending.len() { + if !expired_at_entry && Instant::now() >= deadline { + // A timed host wait may wake with a ready event at the cutoff. + // Finish with the same nonblocking snapshot as expired entry. + return select_until( + pending + .into_iter() + .map(DeadlinePending::into_platform) + .collect(), + deadline, + ); + } + let previous_bytes = pending[index].buffered_bytes(); + let result = pending[index].poll(); + if let Ok(Some(response)) = result { + pending[index].state = Some(PendingState::Ready(Ok(response))); + } else if let Err(error) = result { + pending[index].state = Some(PendingState::Ready(Err(error))); + } + buffered_bytes = buffered_bytes - previous_bytes + pending[index].buffered_bytes(); + enforce_aggregate_budget(&mut pending, &mut buffered_bytes); + // Capacity eviction may have failed an earlier large sibling. + let ready_index = pending + .iter() + .position(|item| matches!(item.state, Some(PendingState::Ready(_)))); + let Some(ready_index) = ready_index else { + continue; + }; + let mut completed = pending.swap_remove(ready_index); + let Some(PendingState::Ready(ready)) = completed.state.take() else { + unreachable!("should select a ready result"); + }; + let failed_backend_name = ready.as_ref().err().map(|_| completed.backend_name); + return Ok(Some(PlatformSelectResult { + ready, + remaining: pending + .into_iter() + .map(DeadlinePending::into_platform) + .collect(), + failed_backend_name, + })); + } + if expired_at_entry || pending.is_empty() { + return Ok(None); + } + wait_for_progress(&pending, deadline)?; + } +} + +fn enforce_aggregate_budget(pending: &mut [DeadlinePending], buffered_bytes: &mut usize) { + if *buffered_bytes <= MAX_TOTAL_BUFFERED_BODY_BYTES { + return; + } + // The overflow is at most one read chunk. Evict the largest partial body + // first, preserving completed responses and healthy smaller siblings. + let largest = pending + .iter() + .enumerate() + .filter(|(_, item)| matches!(item.state, Some(PendingState::Body { .. }))) + .max_by_key(|(_, item)| item.buffered_bytes()) + .or_else(|| { + pending + .iter() + .enumerate() + .max_by_key(|(_, item)| item.buffered_bytes()) + }) + .map(|(index, _)| index) + .expect("should retain the response exceeding buffer capacity"); + *buffered_bytes -= pending[largest].buffered_bytes(); + pending[largest].state = Some(PendingState::Ready(Err(Report::new( + PlatformError::HttpClient, + ) + .attach("largest partial response exceeds aggregate auction buffer capacity")))); +} + +fn wait_for_progress( + pending: &[DeadlinePending], + deadline: Instant, +) -> Result<(), Report> { + let Some(remaining) = deadline.checked_duration_since(Instant::now()) else { + return Ok(()); + }; + if remaining.is_zero() { + return Ok(()); + } + let mut bodies = Vec::new(); + for pending in pending { + match &pending.state { + Some(PendingState::Headers(Some(handle))) => bodies.push(handle.as_u32()), + Some(PendingState::Body { body, .. }) => { + // SAFETY: these handles remain owned by pending throughout select. + bodies.push(unsafe { body.as_u32() }); + } + _ => {} + } + } + if bodies.is_empty() { + return Ok(()); + } + let timeout_ms = host_select_timeout_ms(remaining); + let mut ready_index = 0; + // SAFETY: bodies is a live slice, and ready_index is writable. A positive + // timeout is essential: the host interprets zero as an unbounded wait. + unsafe { + fastly_sys::fastly_async_io::select( + bodies.as_ptr(), + bodies.len(), + timeout_ms, + &mut ready_index, + ) + } + .result() + .map_err(|status| { + Report::new(PlatformError::HttpClient) + .attach(format!("timed select hostcall failed: {status:?}")) + })?; + Ok(()) +} + +/// Convert a remaining budget into the timed-select hostcall argument. +/// +/// The host treats a zero timeout as unbounded, so sub-millisecond budgets +/// round up to one millisecond instead. +fn host_select_timeout_ms(remaining: std::time::Duration) -> u32 { + u32::try_from(remaining.as_millis()) + .unwrap_or(u32::MAX) + .max(1) +} + +#[cfg(test)] +mod tests { + use std::time::Duration; + + use super::super::{FastlyPlatformHttpClient, MAX_PLATFORM_RESPONSE_BODY_BYTES}; + use super::{DeadlinePending, PendingState, host_select_timeout_ms}; + + #[test] + fn sub_millisecond_budget_never_requests_an_unbounded_host_wait() { + assert_eq!( + host_select_timeout_ms(Duration::from_micros(500)), + 1, + "should round a sub-millisecond budget up instead of passing 0" + ); + assert_eq!( + host_select_timeout_ms(Duration::from_millis(25)), + 25, + "should pass whole-millisecond budgets through unchanged" + ); + assert_eq!( + host_select_timeout_ms(Duration::from_secs(u64::MAX)), + u32::MAX, + "should saturate budgets beyond the hostcall range" + ); + } + use edgezero_core::body::Body; + use trusted_server_core::platform::{ + PlatformHttpClient, PlatformHttpRequest, PlatformPendingRequest, + }; + use web_time::Instant; + + #[test] + fn expired_deadline_attributes_an_invalid_pending_handle() { + let result = futures::executor::block_on(FastlyPlatformHttpClient.select_until( + vec![PlatformPendingRequest::new(42_u32).with_backend_name("example")], + Instant::now(), + )) + .expect("should isolate invalid handles even in the nonblocking snapshot") + .expect("should report the invalid pending handle"); + assert_eq!( + result.failed_backend_name.as_deref(), + Some("example"), + "should attribute the invalid pending handle" + ); + let error = result + .ready + .expect_err("should reject the invalid pending handle"); + assert!( + matches!( + error.current_context(), + trusted_server_core::platform::PlatformError::Unsupported + ), + "should reject unsupported deadline handles without panicking" + ); + } + + #[test] + fn deadline_collection_rejects_malformed_handles_with_backend_context() { + let result = futures::executor::block_on(FastlyPlatformHttpClient.select_until( + vec![PlatformPendingRequest::new(42_u32).with_backend_name("example")], + Instant::now() + Duration::from_secs(1), + )); + let result = result + .expect("should isolate malformed pending handles") + .expect("should return the malformed provider's failure"); + assert_eq!( + result.failed_backend_name.as_deref(), + Some("example"), + "should attribute the malformed handle" + ); + let error = result + .ready + .expect_err("should reject a malformed pending handle"); + assert!( + format!("{error:?}").contains("example"), + "should identify the malformed handle's backend" + ); + } + + fn buffered_pending(bytes: Vec, backend_name: &str) -> PlatformPendingRequest { + let mut response = fastly::Response::from_body(bytes); + let body = response.take_body().into_handle(); + DeadlinePending { + backend_name: backend_name.to_owned(), + request_method: Some(edgezero_core::http::Method::GET), + state: Some(PendingState::Body { + response: Box::new(response), + body, + bytes: Vec::new(), + }), + } + .into_platform() + } + + #[test] + fn unsupported_provider_does_not_discard_a_healthy_sibling() { + let deadline = Instant::now() + Duration::from_secs(1); + let failed = PlatformPendingRequest::new(42_u32).with_backend_name("unsupported"); + let healthy = buffered_pending(b"healthy".to_vec(), "healthy"); + let result = futures::executor::block_on( + FastlyPlatformHttpClient.select_until(vec![failed, healthy], deadline), + ) + .expect("should isolate a provider with an unsupported pending handle") + .expect("should report the failed provider"); + assert_eq!( + result.failed_backend_name.as_deref(), + Some("unsupported"), + "should attribute the conversion error to its provider" + ); + assert!( + result.ready.is_err(), + "should report the unsupported handle" + ); + let result = futures::executor::block_on( + FastlyPlatformHttpClient.select_until(result.remaining, deadline), + ) + .expect("should continue collecting the healthy sibling") + .expect("should complete the healthy sibling"); + let response = result.ready.expect("should retain the healthy response"); + assert_eq!( + response.backend_name.as_deref(), + Some("healthy"), + "should preserve the unaffected provider's bid" + ); + } + + #[test] + fn expired_deadline_preserves_an_already_ready_body() { + let pending = buffered_pending(b"ready".to_vec(), "example"); + let result = futures::executor::block_on( + FastlyPlatformHttpClient.select_until(vec![pending], Instant::now()), + ) + .expect("should snapshot an expired ready response") + .expect("should preserve an already complete response without waiting"); + let response = result + .ready + .expect("should retain a successful buffered response"); + assert_eq!( + response.backend_name.as_deref(), + Some("example"), + "should retain the ready response's backend" + ); + assert_eq!( + response.response.body().as_bytes(), + Some(b"ready".as_slice()), + "should retain the complete ready payload" + ); + } + + #[test] + fn remaining_handle_retains_the_original_request_method_hint() { + let pending = buffered_pending(b"body".to_vec(), "example") + .with_response_handling(false, edgezero_core::http::Method::HEAD); + let resumed = DeadlinePending::from_platform(pending) + .expect("should resume a deadline pending request"); + let remaining = resumed.into_platform(); + assert_eq!( + remaining.request_method(), + Some(&edgezero_core::http::Method::HEAD), + "should retain the HEAD method through remaining-handle reconstruction" + ); + let resumed = + DeadlinePending::from_platform(remaining).expect("should resume the remaining handle"); + let response = fastly::Response::from_body("advisory body"); + assert!( + response.get_backend_request().is_none(), + "should exercise a response without SDK originating-request metadata" + ); + assert!( + resumed + .request_is_head(&response) + .expect("should use the preserved method hint"), + "should retain HEAD body semantics after resuming without response origin metadata" + ); + } + + #[test] + fn missing_request_method_and_response_origin_returns_typed_error() { + let pending = buffered_pending(b"body".to_vec(), "example"); + let mut pending = DeadlinePending::from_platform(pending) + .expect("should inspect a buffered pending request"); + pending.request_method = None; + let response = fastly::Response::new(); + let error = pending + .request_is_head(&response) + .expect_err("should reject a response whose request method is unknown"); + assert!( + matches!( + error.current_context(), + trusted_server_core::platform::PlatformError::HttpClient + ), + "should return a typed HTTP client error" + ); + assert!( + format!("{error:?}").contains("example"), + "should identify the backend whose originating method is missing" + ); + } + + #[test] + fn completed_body_moves_into_platform_without_a_payload_copy() { + let bytes = vec![b'x'; 32768]; + let pointer = bytes.as_ptr(); + let response = super::buffered_response_to_platform( + &fastly::Response::new(), + "example".to_owned(), + bytes, + ) + .expect("should move a buffered response into the platform"); + assert_eq!( + response + .response + .body() + .as_bytes() + .expect("should retain a buffered body") + .as_ptr(), + pointer, + "should retain the original allocation without a Fastly host-body roundtrip" + ); + } + + #[test] + fn body_collection_is_fair_and_preserves_partial_siblings() { + let large = vec![b'x'; 32768]; + let pending = vec![ + buffered_pending(large.clone(), "large"), + buffered_pending(b"fast".to_vec(), "fast"), + ]; + let deadline = Instant::now() + Duration::from_secs(10); + let result = + futures::executor::block_on(FastlyPlatformHttpClient.select_until(pending, deadline)) + .expect("should collect one response") + .expect("should complete before deadline"); + let response = result + .ready + .expect("should collect the short response first"); + assert_eq!( + response.backend_name.as_deref(), + Some("fast"), + "should give siblings a fair opportunity to complete" + ); + assert_eq!( + result.remaining.len(), + 1, + "should retain the partial response" + ); + let result = futures::executor::block_on( + FastlyPlatformHttpClient.select_until(result.remaining, deadline), + ) + .expect("should resume body collection") + .expect("should finish the remaining body"); + let response = result.ready.expect("should finish the partial body"); + let (_, body) = response.response.into_parts(); + let bytes = body + .into_bytes() + .expect("should read the complete buffered body"); + assert_eq!( + bytes.as_ref(), + large.as_slice(), + "should preserve all bytes across collector calls" + ); + } + + #[test] + fn body_at_the_exact_limit_remains_eligible() { + let pending = buffered_pending(vec![b'x'; MAX_PLATFORM_RESPONSE_BODY_BYTES], "exact"); + let result = futures::executor::block_on( + FastlyPlatformHttpClient + .select_until(vec![pending], Instant::now() + Duration::from_secs(10)), + ) + .expect("should collect an exactly sized body") + .expect("should complete before the deadline"); + let response = result + .ready + .expect("should accept the configured body-size boundary"); + assert_eq!( + response + .response + .body() + .as_bytes() + .expect("should retain buffered bytes") + .len(), + MAX_PLATFORM_RESPONSE_BODY_BYTES, + "should accept the exact limit without evicting its EOF probe" + ); + } + + #[test] + fn incremental_body_limit_reports_the_failed_backend() { + let pending = buffered_pending(vec![b'x'; MAX_PLATFORM_RESPONSE_BODY_BYTES + 1], "large"); + let result = futures::executor::block_on( + FastlyPlatformHttpClient + .select_until(vec![pending], Instant::now() + Duration::from_secs(30)), + ) + .expect("should return a provider failure") + .expect("should reject the body before deadline"); + assert!( + result.ready.is_err(), + "should enforce the incremental body cap" + ); + assert_eq!( + result.failed_backend_name.as_deref(), + Some("large"), + "should attribute body conversion failures" + ); + } + + #[test] + fn aggregate_body_pressure_evicts_largest_partial_provider() { + let mut large = + DeadlinePending::from_platform(buffered_pending(vec![b'x'; 16384], "large")) + .expect("should construct a large partial response"); + if let Some(PendingState::Body { bytes, .. }) = &mut large.state { + *bytes = vec![b'x'; MAX_PLATFORM_RESPONSE_BODY_BYTES - 1]; + } + let pending = vec![ + buffered_pending(b"healthy".to_vec(), "healthy"), + large.into_platform(), + ]; + let deadline = Instant::now() + Duration::from_secs(10); + let result = + futures::executor::block_on(FastlyPlatformHttpClient.select_until(pending, deadline)) + .expect("should return a capacity result") + .expect("should reject excess growth"); + assert!( + result.ready.is_err(), + "should reject excess aggregate buffering" + ); + assert_eq!( + result.failed_backend_name.as_deref(), + Some("large"), + "should attribute capacity pressure to the largest partial response" + ); + let result = futures::executor::block_on( + FastlyPlatformHttpClient.select_until(result.remaining, deadline), + ) + .expect("should retain the healthy provider") + .expect("should complete the healthy provider"); + assert_eq!( + result + .ready + .expect("should preserve the healthy bid") + .backend_name + .as_deref(), + Some("healthy"), + "should avoid blaming the healthy provider for another provider's large body" + ); + } + + #[test] + fn expired_snapshot_retains_only_ready_results_across_calls() { + let pending = vec![ + buffered_pending(b"one".to_vec(), "one"), + buffered_pending(b"two".to_vec(), "two"), + ]; + let deadline = Instant::now(); + let result = + futures::executor::block_on(FastlyPlatformHttpClient.select_until(pending, deadline)) + .expect("should snapshot ready results") + .expect("should return first ready result"); + assert!( + result.ready.is_ok(), + "should preserve a complete ready response" + ); + let result = futures::executor::block_on( + FastlyPlatformHttpClient.select_until(result.remaining, deadline), + ) + .expect("should use cached ready results") + .expect("should return second ready result"); + assert!( + result.ready.is_ok(), + "should preserve the second complete response" + ); + } + + fn network_pending(path: &str, method: &str) -> PlatformPendingRequest { + let request = edgezero_core::http::request_builder() + .method(method) + .uri(format!("http://example.com/{path}")) + .body(Body::empty()) + .expect("should build fixture request"); + futures::executor::block_on( + FastlyPlatformHttpClient.send_async( + PlatformHttpRequest::new(request, "deadline_origin").with_cache_bypass(), + ), + ) + .expect("should launch fixture request") + } + + #[test] + #[ignore = "requires the local origin started by tests/deadline-origin.mjs"] + fn network_slow_origin_preserves_ready_bid_without_waiting_for_stalled_sibling() { + let fast = network_pending("fast", "GET"); + let stalled = network_pending("stalled", "GET"); + let deadline = Instant::now() + Duration::from_millis(40); + std::thread::sleep(Duration::from_millis(100)); + let started = Instant::now(); + let result = futures::executor::block_on( + FastlyPlatformHttpClient.select_until(vec![stalled, fast], deadline), + ) + .expect("should snapshot already ready bids") + .expect("should retain the complete fast response"); + assert!( + result.ready.is_ok(), + "should preserve a bid available after a slow publisher origin" + ); + assert!( + result.remaining.is_empty(), + "should abandon the stalled sibling in the nonblocking snapshot" + ); + assert!( + started.elapsed() < Duration::from_millis(100), + "should never wait for the stalled sibling after expiration" + ); + } + + #[test] + #[ignore = "requires the local origin started by tests/deadline-origin.mjs"] + fn network_stalled_headers_stop_at_deadline() { + let pending = network_pending("stalled", "GET"); + let started = Instant::now(); + let result = futures::executor::block_on( + FastlyPlatformHttpClient + .select_until(vec![pending], started + Duration::from_millis(80)), + ) + .expect("should stop waiting for stalled headers"); + assert!( + result.is_none(), + "should expire while headers remain pending" + ); + assert!( + started.elapsed() < Duration::from_millis(500), + "should bound the synchronous header wait" + ); + } + + #[test] + #[ignore = "requires the local origin started by tests/deadline-origin.mjs"] + fn network_trickling_body_stops_at_deadline() { + let pending = network_pending("trickle", "GET"); + let started = Instant::now(); + let result = futures::executor::block_on( + FastlyPlatformHttpClient + .select_until(vec![pending], started + Duration::from_millis(80)), + ) + .expect("should stop buffering a trickling body"); + assert!( + result.is_none(), + "should expire before the trickling body completes" + ); + assert!( + started.elapsed() < Duration::from_millis(500), + "should bound body reads at the auction deadline" + ); + } + + #[test] + #[ignore = "requires the local origin started by tests/deadline-origin.mjs"] + fn network_fast_sibling_completes_while_another_body_trickles() { + let slow = network_pending("trickle", "GET"); + let fast = network_pending("fast", "GET"); + let request = edgezero_core::http::request_builder() + .uri("http://example.com/fast") + .body(Body::empty()) + .expect("should build the request missing cache bypass"); + let unsupported = futures::executor::block_on( + FastlyPlatformHttpClient + .send_async(PlatformHttpRequest::new(request, "deadline_origin")), + ) + .expect("should dispatch the request missing cache bypass"); + let started = Instant::now(); + let deadline = started + Duration::from_millis(200); + let result = futures::executor::block_on( + FastlyPlatformHttpClient.select_until(vec![unsupported, slow, fast], deadline), + ) + .expect("should isolate a request missing cache bypass") + .expect("should return the unsupported provider's failure"); + assert_eq!( + result.failed_backend_name.as_deref(), + Some("deadline_origin"), + "should attribute missing cache bypass to that request" + ); + let error = result + .ready + .expect_err("should reject the unsupported handle"); + assert!( + matches!( + error.current_context(), + trusted_server_core::platform::PlatformError::Unsupported + ), + "should reject SDK guest-caching handles before direct handle conversion" + ); + let result = futures::executor::block_on( + FastlyPlatformHttpClient.select_until(result.remaining, deadline), + ) + .expect("should collect the completed sibling") + .expect("should finish before deadline"); + let response = result.ready.expect("should collect the fast body"); + let (_, body) = response.response.into_parts(); + let bytes = body.into_bytes().expect("should read the fast body"); + assert_eq!( + bytes.as_ref(), + b"fast", + "should avoid starvation by a trickling sibling" + ); + assert_eq!( + result.remaining.len(), + 1, + "should retain the incomplete sibling" + ); + } + + #[test] + #[ignore = "requires the local origin started by tests/deadline-origin.mjs"] + fn network_head_skips_the_body_and_its_declared_length_limit() { + let pending = network_pending("head", "HEAD"); + let result = futures::executor::block_on( + FastlyPlatformHttpClient + .select_until(vec![pending], Instant::now() + Duration::from_secs(1)), + ) + .expect("should collect HEAD response") + .expect("should finish before deadline"); + assert!( + result.ready.is_ok(), + "should accept HEAD with an advisory large content-length" + ); + } +} diff --git a/crates/trusted-server-adapter-fastly/tests/deadline-origin.mjs b/crates/trusted-server-adapter-fastly/tests/deadline-origin.mjs new file mode 100644 index 0000000000..c985097677 --- /dev/null +++ b/crates/trusted-server-adapter-fastly/tests/deadline-origin.mjs @@ -0,0 +1,172 @@ +// Run the actual Fastly deadline tests against a deliberately slow local origin. +// Usage: node tests/deadline-origin.mjs /path/to/viceroy [test-binary.wasm] +import assert from 'node:assert/strict' +import { execFile, spawn } from 'node:child_process' +import { once } from 'node:events' +import { existsSync } from 'node:fs' +import { mkdtemp, rm, writeFile } from 'node:fs/promises' +import { createServer } from 'node:http' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { setTimeout as delay } from 'node:timers/promises' +import { promisify } from 'node:util' + +const execute = promisify(execFile) +const interruption = new AbortController() +process.once('SIGINT', () => interruption.abort()) +process.once('SIGTERM', () => interruption.abort()) + +async function testArtifact() { + const { stdout } = await execute( + 'cargo', + [ + 'test', + '-p', + 'trusted-server-adapter-fastly', + '--target', + 'wasm32-wasip1', + '--bin', + 'trusted-server-adapter-fastly', + '--no-run', + '--message-format=json', + ], + { + maxBuffer: 16 * 1024 * 1024, + signal: interruption.signal, + killSignal: 'SIGKILL', + } + ) + const binaries = stdout + .split('\n') + .filter((line) => line.startsWith('{')) + .map((line) => JSON.parse(line)) + .filter( + (item) => + item.reason === 'compiler-artifact' && + item.profile?.test && + item.target?.name === 'trusted-server-adapter-fastly' && + item.executable + ) + .map((item) => item.executable) + assert.equal( + binaries.length, + 1, + 'should discover exactly one Fastly test binary' + ) + return binaries[0] +} + +async function serveOrigin(request, response) { + const cancellation = new AbortController() + response.once('close', () => cancellation.abort()) + const pause = (milliseconds) => + delay(milliseconds, undefined, { signal: cancellation.signal }) + try { + if (request.method === 'HEAD') { + response.writeHead(200, { 'Content-Length': 20 * 1024 * 1024 }) + response.end() + return + } + if (request.url === '/stalled') await pause(2000) + if (request.url === '/trickle') { + response.writeHead(200, { 'Content-Length': 100 }) + for (let index = 0; index < 100; index += 1) { + response.write('x') + await pause(10) + } + response.end() + return + } + await pause(30) + response.writeHead(200, { 'Content-Length': 4 }) + response.end('fast') + } catch (error) { + // Deadline expiry deliberately disconnects the slow origin and cancels its timer. + if (error.name !== 'AbortError') throw error + } +} + +async function main() { + const [viceroy, suppliedBinary, ...extra] = process.argv.slice(2) + assert( + viceroy && extra.length === 0, + 'usage: node deadline-origin.mjs /path/to/viceroy [test-binary.wasm]' + ) + const binary = suppliedBinary ?? (await testArtifact()) + let originError + const server = createServer((request, response) => { + void serveOrigin(request, response).catch((error) => { + originError = error + response.destroy() + }) + }) + server.listen(0, '127.0.0.1') + await once(server, 'listening') + const directory = await mkdtemp(join(tmpdir(), 'fastly-deadline-')) + try { + const config = join(directory, 'fastly.toml') + await writeFile( + config, + '[local_server.backends.deadline_origin]\n' + + `url = "http://127.0.0.1:${server.address().port}"\n` + ) + const environment = { ...process.env } + if (existsSync('/etc/ssl/cert.pem')) + environment.SSL_CERT_FILE ??= '/etc/ssl/cert.pem' + const child = spawn( + viceroy, + [ + 'run', + '-C', + config, + binary, + 'platform::deadline::tests::network_', + '--ignored', + '--nocapture', + ], + { + env: environment, + stdio: ['inherit', 'pipe', 'inherit'], + signal: interruption.signal, + killSignal: 'SIGKILL', + } + ) + let output = '' + child.stdout.setEncoding('utf8') + child.stdout.on('data', (chunk) => { + output += chunk + process.stdout.write(chunk) + }) + // Clear the watchdog even if spawning fails: Node's built-in child timeout + // can otherwise keep the harness alive after an ENOENT startup error. + const watchdog = setTimeout(() => child.kill('SIGKILL'), 120_000) + watchdog.unref() + try { + const [code] = await once(child, 'close') + if (originError) throw originError + assert.equal(code, 0, 'should run the network tests successfully') + assert.match( + output, + /^test result: ok\. 5 passed; 0 failed; 0 ignored; 0 measured;/m, + 'should execute all five network tests, including when the filter changes' + ) + return code ?? 1 + } finally { + clearTimeout(watchdog) + } + } finally { + const closed = new Promise((resolve) => server.close(resolve)) + server.closeAllConnections() + await closed + await rm(directory, { recursive: true, force: true }) + } +} + +main() + .then((code) => { + process.exitCode = code + }) + .catch((error) => { + console.error(error) + process.exitCode = 1 + }) diff --git a/crates/trusted-server-adapter-spin/Cargo.toml b/crates/trusted-server-adapter-spin/Cargo.toml index a169ead538..47cf7b589c 100644 --- a/crates/trusted-server-adapter-spin/Cargo.toml +++ b/crates/trusted-server-adapter-spin/Cargo.toml @@ -33,6 +33,7 @@ flate2 = { workspace = true } futures = { workspace = true } http-body-util = { workspace = true } log = { workspace = true } +web-time = { workspace = true } trusted-server-core = { workspace = true } [target.'cfg(target_arch = "wasm32")'.dependencies] @@ -42,4 +43,11 @@ spin-sdk = { workspace = true } trusted-server-core = { workspace = true, features = ["test-utils"] } base64 = { workspace = true } edgezero-core = { workspace = true } +[target.'cfg(not(target_arch = "wasm32"))'.dev-dependencies] tokio = { workspace = true, features = ["rt-multi-thread", "macros"] } + +[[example]] +name = "deadline-runtime" +path = "tests/runtime/deadline.rs" +crate-type = ["cdylib"] +required-features = ["spin"] diff --git a/crates/trusted-server-adapter-spin/src/platform.rs b/crates/trusted-server-adapter-spin/src/platform.rs index 9d6b25571c..53b6386373 100644 --- a/crates/trusted-server-adapter-spin/src/platform.rs +++ b/crates/trusted-server-adapter-spin/src/platform.rs @@ -6,8 +6,8 @@ use bytes::Bytes; use edgezero_core::config_store::ConfigStoreHandle; use edgezero_core::key_value_store::{KvHandle, KvPage, KvStore}; use error_stack::{Report, ResultExt as _}; -#[cfg(all(feature = "spin", target_arch = "wasm32"))] -use http_body_util::BodyExt as _; +#[cfg(any(test, all(feature = "spin", target_arch = "wasm32")))] +use http_body_util::BodyExt; use trusted_server_core::platform::{ BackendNamingPolicy, ClientInfo, GeoInfo, KvError, PlatformBackend, PlatformBackendSpec, PlatformConfigStore, PlatformError, PlatformGeo, PlatformHttpClient, PlatformKvStore, @@ -23,12 +23,12 @@ use std::io::Read as _; use trusted_server_core::platform::PlatformHttpRequest; #[cfg(all(feature = "spin", target_arch = "wasm32"))] use trusted_server_core::platform::{ - PlatformPendingRequest, PlatformResponse, PlatformSelectResult, + PlatformPendingRequest, PlatformPendingResponse, PlatformResponse, PlatformSelectResult, }; -// 8 MiB ceiling: conservative for ad-server responses while leaving headroom in -// the Spin WASM component heap. Larger responses from misbehaving origins are -// rejected with a typed error rather than OOMing the component. +// Limit both streamed bytes and decompressed bytes to 8 MiB to preserve +// headroom in the Spin WASM component heap. The streaming limit rejects an +// oversized frame before retaining it in the collected response body. #[cfg(any(test, all(feature = "spin", target_arch = "wasm32")))] const MAX_DECOMPRESSED_SIZE: usize = 8 * 1024 * 1024; @@ -37,6 +37,34 @@ type HeaderPairs = Vec<(String, Vec)>; #[cfg(any(test, all(feature = "spin", target_arch = "wasm32")))] type BufferedResponseParts = (HeaderPairs, Vec); +/// Collect an outbound response body, rejecting it once it exceeds +/// [`MAX_DECOMPRESSED_SIZE`] streamed bytes. +/// +/// The limit applies frame by frame, so an oversized body fails before it is +/// fully retained in the component heap. +/// +/// # Errors +/// +/// Returns `PlatformError::HttpClient` when the body exceeds the limit or a +/// frame cannot be read. +#[cfg(any(test, all(feature = "spin", target_arch = "wasm32")))] +async fn collect_limited_body(body: B) -> Result, Report> +where + B: BodyExt, + B::Data: bytes::Buf, + B::Error: Into>, +{ + let collected = http_body_util::Limited::new(body, MAX_DECOMPRESSED_SIZE) + .collect() + .await + .map_err(|error| { + Report::new(PlatformError::HttpClient).attach(format!( + "failed to read Spin outbound response body: {error}" + )) + })?; + Ok(collected.to_bytes().to_vec()) +} + #[cfg(any(test, all(feature = "spin", target_arch = "wasm32")))] const SPIN_VARIABLE_HEX: &[u8; 16] = b"0123456789abcdef"; @@ -427,22 +455,13 @@ fn is_wasi_forbidden_outbound_header(name: &str) -> bool { || name.eq_ignore_ascii_case("proxy-connection") } -/// Carries a completed response through `send_async` → `select`. -#[cfg(all(feature = "spin", target_arch = "wasm32"))] -struct SpinPendingResponse { - backend_name: String, - status: u16, - headers: HeaderPairs, - body: Vec, -} - /// `spin_sdk::http::send`-backed HTTP client for the Spin runtime. /// -/// `send_async` eagerly executes each request before returning, so -/// [`PlatformHttpClient::supports_concurrent_fanout`] reports `false` and the -/// auction orchestrator rejects multi-provider configurations before any -/// request launches. `select` keeps a defense-in-depth rejection for more -/// than one pending request, matching the Cloudflare adapter behavior. +/// Auction configuration supports one provider at a time, so +/// [`PlatformHttpClient::supports_concurrent_fanout`] reports `false`. +/// Requests start in a runtime task at dispatch, overlapping publisher-origin work. +/// Deadline selection bounds waiting for the full send and response body collection. +/// Dropping the pending operation cancels its WASI asynchronous operations. /// /// Spin's WASI HTTP API sends one request and returns the original response; it /// exposes no redirect-follow policy. Each trait call therefore reaches exactly @@ -451,20 +470,14 @@ struct SpinPendingResponse { /// `StubHttpClient` driver records the one-send 3xx behavior while adapter tests /// cover request/response policy around that single boundary. /// -/// # Known MVP limits -/// -/// **No configurable outbound timeout.** `spin_sdk::http::send` does not -/// expose per-request timeout control, and [`PlatformBackendSpec::first_byte_timeout`] -/// is ignored by [`NoopBackend`]. A slow or hung origin will block the Spin -/// invocation for whatever default the Spin runtime imposes. Operators requiring -/// deterministic timeout behaviour should use the Fastly adapter. +/// Backend phase timeouts remain unsupported. Callers requiring an absolute +/// cutoff use [`PlatformHttpClient::select_until`]. #[cfg(all(feature = "spin", target_arch = "wasm32"))] pub struct SpinPlatformHttpClient; #[cfg(all(feature = "spin", target_arch = "wasm32"))] impl SpinPlatformHttpClient { async fn execute( - &self, request: PlatformHttpRequest, ) -> Result> { reject_unsupported_request_contracts(&request)?; @@ -525,17 +538,7 @@ impl SpinPlatformHttpClient { .iter() .map(|(name, value)| (name.to_string(), value.as_bytes().to_vec())) .collect(); - let body = spin_response - .into_body() - .collect() - .await - .map_err(|error| { - Report::new(PlatformError::HttpClient).attach(format!( - "failed to read Spin outbound response body: {error}" - )) - })? - .to_bytes() - .to_vec(); + let body = collect_limited_body(spin_response.into_body()).await?; let (headers, body) = apply_spin_response_policy(&method, status, headers, body)?; let mut edge_builder = edgezero_core::http::response_builder().status(status); @@ -554,9 +557,9 @@ impl SpinPlatformHttpClient { #[async_trait::async_trait(?Send)] impl PlatformHttpClient for SpinPlatformHttpClient { fn supports_concurrent_fanout(&self) -> bool { - // `send_async` executes each request eagerly, so multiple pending - // requests run sequentially. The auction orchestrator checks this - // before launching more than one provider request. + // Requests start concurrently at dispatch, but `select` and + // `select_until` collect only one response, so multi-provider fan-out + // stays unsupported. false } @@ -564,7 +567,7 @@ impl PlatformHttpClient for SpinPlatformHttpClient { &self, request: PlatformHttpRequest, ) -> Result> { - self.execute(request).await + Self::execute(request).await } async fn send_async( @@ -572,29 +575,8 @@ impl PlatformHttpClient for SpinPlatformHttpClient { request: PlatformHttpRequest, ) -> Result> { let backend_name = request.backend_name.clone(); - let response = self.execute(request).await?; - - let status = response.response.status().as_u16(); - let headers: HeaderPairs = response - .response - .headers() - .iter() - .map(|(n, v)| (n.to_string(), v.as_bytes().to_vec())) - .collect(); - let body_bytes = match response.response.into_body() { - edgezero_core::body::Body::Once(bytes) => bytes.to_vec(), - edgezero_core::body::Body::Stream(_) => { - return Err(Report::new(PlatformError::HttpClient) - .attach("SpinPlatformHttpClient::execute returned a streaming body")); - } - }; - - let pending = SpinPendingResponse { - backend_name: backend_name.clone(), - status, - headers, - body: body_bytes, - }; + let pending = + PlatformPendingResponse::start(Self::execute(request), spin_sdk::wasip3::spawn); Ok(PlatformPendingRequest::new(pending).with_backend_name(backend_name)) } @@ -617,28 +599,69 @@ impl PlatformHttpClient for SpinPlatformHttpClient { } let ready_platform = pending_requests.remove(0); - let pending = ready_platform - .downcast::() + let mut pending = ready_platform + .downcast::>>>() .map_err(|_| { Report::new(PlatformError::HttpClient) .attach("unexpected inner type in SpinPlatformHttpClient::select") })?; - let mut builder = edgezero_core::http::response_builder().status(pending.status); - for (name, value) in &pending.headers { - builder = builder.header(name.as_str(), value.as_slice()); - } - let edge_resp = builder - .body(edgezero_core::body::Body::from(pending.body)) - .change_context(PlatformError::HttpClient)?; - - let ready = Ok(PlatformResponse::new(edge_resp).with_backend_name(pending.backend_name)); + let ready = pending.wait().await.unwrap_or_else(|error| { + Err(Report::new(PlatformError::HttpClient) + .attach(format!("dispatched response task stopped: {error}"))) + }); Ok(PlatformSelectResult { ready, remaining: pending_requests, failed_backend_name: None, }) } + + async fn select_until( + &self, + mut pending_requests: Vec, + deadline: web_time::Instant, + ) -> Result, Report> { + if pending_requests.is_empty() { + return Err(Report::new(PlatformError::HttpClient) + .attach("select_until called with an empty pending_requests list")); + } + if pending_requests.len() > 1 { + return Err(Report::new(PlatformError::HttpClient) + .attach("multi-provider fan-out is not supported on this adapter")); + } + let platform_pending = pending_requests.remove(0); + let backend_name = platform_pending.backend_name().map(str::to_owned); + let mut pending = platform_pending + .downcast::>>>() + .map_err(|_| { + Report::new(PlatformError::HttpClient) + .attach("unexpected pending request type in deadline selection") + })?; + let result = pending + .complete_before_deadline( + async { + let remaining = deadline.saturating_duration_since(web_time::Instant::now()); + if !remaining.is_zero() { + spin_sdk::time::sleep(remaining).await; + } + }, + deadline, + web_time::Instant::now, + ) + .await; + Ok(result.map(|result| { + let ready = result.unwrap_or_else(|error| { + Err(Report::new(PlatformError::HttpClient) + .attach(format!("dispatched response task stopped: {error}"))) + }); + PlatformSelectResult { + failed_backend_name: ready.as_ref().err().and(backend_name), + ready, + remaining: pending_requests, + } + })) + } } #[cfg(all(feature = "spin", target_arch = "wasm32"))] @@ -807,10 +830,6 @@ mod tests { fn auction_http_capabilities_are_explicit() { let capabilities = AuctionTargetId::Spin.descriptor().capabilities(); assert!(!capabilities.supports_concurrent_provider_fanout()); - assert!( - !capabilities.has_enforceable_total_request_deadline(), - "Spin outbound HTTP does not expose an enforceable hard total request deadline" - ); } use edgezero_core::context::RequestContext; use edgezero_core::http::request_builder; @@ -1415,3 +1434,35 @@ mod tests { } } } + +#[cfg(test)] +mod deadline_tests { + use futures::executor::block_on; + + #[test] + fn spin_body_collection_rejects_oversized_stream() { + let body = http_body_util::Full::new(bytes::Bytes::from(vec![ + 0; + super::MAX_DECOMPRESSED_SIZE + + 1 + ])); + let result = block_on(super::collect_limited_body(body)); + assert!( + result.is_err(), + "should reject a response exceeding the streamed body cap" + ); + } + + #[test] + fn spin_body_collection_accepts_body_at_limit() { + let body = + http_body_util::Full::new(bytes::Bytes::from(vec![0; super::MAX_DECOMPRESSED_SIZE])); + let result = block_on(super::collect_limited_body(body)) + .expect("should collect a body at the limit"); + assert_eq!( + result.len(), + super::MAX_DECOMPRESSED_SIZE, + "should retain the complete permitted body" + ); + } +} diff --git a/crates/trusted-server-adapter-spin/tests/runtime/deadline.rs b/crates/trusted-server-adapter-spin/tests/runtime/deadline.rs new file mode 100644 index 0000000000..95cd0013da --- /dev/null +++ b/crates/trusted-server-adapter-spin/tests/runtime/deadline.rs @@ -0,0 +1,80 @@ +//! Local runtime fixture exercising the production HTTP client implementation. + +#[cfg(all(feature = "spin", target_arch = "wasm32"))] +#[allow(dead_code)] +#[path = "../../src/platform.rs"] +mod platform; + +#[cfg(all(feature = "spin", target_arch = "wasm32"))] +use std::time::Duration; +#[cfg(all(feature = "spin", target_arch = "wasm32"))] +use trusted_server_core::platform::{PlatformHttpClient as _, PlatformHttpRequest}; + +#[cfg(all(feature = "spin", target_arch = "wasm32"))] +#[spin_sdk::http_service] +async fn handle(request: spin_sdk::http::Request) -> Result { + let result = exercise(&request.uri().to_string()) + .await + .map_err(|error| format!("{error:?}"))?; + Ok(result) +} + +#[cfg(all(feature = "spin", target_arch = "wasm32"))] +async fn exercise( + uri: &str, +) -> Result> { + let parameter = |name: &str| { + uri.split_once('?').and_then(|(_, query)| { + query + .split('&') + .filter_map(|pair| pair.split_once('=')) + .find(|(key, _)| *key == name) + .map(|(_, value)| value) + }) + }; + let upstream = parameter("upstream").expect("should supply local fixture upstream"); + let budget_ms: u64 = parameter("budget_ms") + .expect("should supply budget") + .parse() + .expect("should parse budget"); + let origin_ms: u64 = parameter("origin_ms") + .expect("should supply origin delay") + .parse() + .expect("should parse origin delay"); + let client = platform::SpinPlatformHttpClient; + let request = edgezero_core::http::request_builder() + .uri(upstream) + .body(edgezero_core::body::Body::empty()) + .expect("should construct fixture request"); + let started = web_time::Instant::now(); + let pending = client + .send_async(PlatformHttpRequest::new(request, "fixture")) + .await?; + spin_sdk::time::sleep(Duration::from_millis(origin_ms)).await; + let collected = client + .select_until(vec![pending], started + Duration::from_millis(budget_ms)) + .await?; + let status = match collected { + Some(result) => match result.ready { + Ok(response) => { + assert_eq!( + response.response.body().as_bytes(), + Some(b"okay".as_slice()), + "should retain the complete nonempty upstream body" + ); + "complete" + } + Err(error) if format!("{error:?}").contains("length limit exceeded") => { + assert_eq!( + result.failed_backend_name.as_deref(), + Some("fixture"), + "should attribute oversized response to its provider" + ); + "oversized" + } + Err(error) => return Err(error), + }, + None => "timeout", + }; + Ok(format!("{status}:{}", started.elapsed().as_millis())) +} diff --git a/crates/trusted-server-core/src/auction/endpoints.rs b/crates/trusted-server-core/src/auction/endpoints.rs index a089f52298..086515fa20 100644 --- a/crates/trusted-server-core/src/auction/endpoints.rs +++ b/crates/trusted-server-core/src/auction/endpoints.rs @@ -1030,10 +1030,10 @@ mod tests { context .services .http_client() - .send_async(PlatformHttpRequest::new( - request, - "template-switch-probe-backend", - )) + .send_async( + PlatformHttpRequest::new(request, "template-switch-probe-backend") + .with_cache_bypass(), + ) .await .change_context(TrustedServerError::Auction { message: "probe provider launch failed".to_string(), diff --git a/crates/trusted-server-core/src/auction/orchestrator.rs b/crates/trusted-server-core/src/auction/orchestrator.rs index 95b8549863..888bade15e 100644 --- a/crates/trusted-server-core/src/auction/orchestrator.rs +++ b/crates/trusted-server-core/src/auction/orchestrator.rs @@ -5,6 +5,7 @@ use error_stack::{Report, ResultExt}; use http::Request; use std::collections::{HashMap, HashSet, hash_map::Entry}; use std::sync::Arc; +use std::time::Duration; use web_time::Instant; use crate::error::TrustedServerError; @@ -32,9 +33,9 @@ use crate::trace::{ /// /// Created by [`AuctionOrchestrator::dispatch_auction`] and consumed by /// [`AuctionOrchestrator::collect_dispatched_auction`]. Carrying this handle -/// across `pending_origin.wait()` lets origin response and SSP HTTP requests -/// race in Fastly's native layer, enabling TTFB ≈ origin latency rather than -/// TTFB ≈ auction timeout. +/// across the publisher origin wait lets provider operations progress while +/// the page loads. Collection retains available responses and never restarts +/// the original auction budget. pub struct DispatchedAuction { pending_requests: Vec, backend_to_provider: HashMap, @@ -253,27 +254,9 @@ fn remaining_budget_ms(start: Instant, timeout_ms: u32) -> u32 { timeout_ms.saturating_sub(elapsed) } -/// Runtime policy for classifying responses that complete after the logical auction budget. -/// -/// Current adapters do not expose an enforceable total-request deadline. They -/// therefore drain already-launched work and accept completed late responses. -#[derive(Debug, Clone, Copy, Default)] -struct AuctionDeadlinePolicy { - enforceable_total_request_deadline: bool, -} - -impl AuctionDeadlinePolicy { - fn rejects_late_completion(self, start: Instant, timeout_ms: u32) -> bool { - self.enforceable_total_request_deadline && remaining_budget_ms(start, timeout_ms) == 0 - } - - fn for_runtime(services: &RuntimeServices) -> Self { - Self { - enforceable_total_request_deadline: services - .http_client() - .has_enforceable_total_request_deadline(), - } - } +/// Returns the original absolute cutoff without rounding transport budgets. +fn collection_deadline(start: Instant, timeout_ms: u32) -> Instant { + start + Duration::from_millis(u64::from(timeout_ms)) } fn routing_metadata(unroutable_bidder_count: u32) -> HashMap { @@ -493,6 +476,10 @@ impl AuctionOrchestratorHarness { let remaining_ms = remaining_budget_ms(auction_start, context.timeout_ms); let logical_budget_ms = remaining_ms.min(provider.timeout_ms()); if logical_budget_ms == 0 { + log::debug!( + "Planned provider '{}' has no remaining auction budget before launch", + provider.provider_name() + ); responses.push(provider_timeout_response(provider.provider_name(), 0)); continue; } @@ -501,6 +488,10 @@ impl AuctionOrchestratorHarness { .backend() .canonicalize_transport_timeout_ms(logical_budget_ms, provider.timeout_ms()); if transport_timeout_ms == 0 { + log::debug!( + "Planned provider '{}' has no remaining transport budget before launch", + provider.provider_name() + ); responses.push(provider_timeout_response(provider.provider_name(), 0)); continue; } @@ -571,8 +562,17 @@ impl AuctionOrchestratorHarness { } while !pending.is_empty() { - let select_result = match context.services.http_client().select(pending).await { - Ok(result) => result, + let select_result = match context + .services + .http_client() + .select_until( + pending, + collection_deadline(auction_start, context.timeout_ms), + ) + .await + { + Ok(Some(result)) => result, + Ok(None) => break, Err(error) => { log::warn!("Planned provider select failed: {:?}", error); let mut transport_failures = launches @@ -599,16 +599,6 @@ impl AuctionOrchestratorHarness { .to_string(); if let Some(state) = launches.remove(&backend_name) { let elapsed_ms = state.started_at.elapsed().as_millis() as u64; - let deadline_policy = AuctionDeadlinePolicy::for_runtime(context.services); - if deadline_policy - .rejects_late_completion(auction_start, context.timeout_ms) - { - responses.push(provider_timeout_response( - state.provider.provider_name(), - elapsed_ms, - )); - continue; - } match state .provider .parse_response_with_state( @@ -647,6 +637,10 @@ impl AuctionOrchestratorHarness { } } for state in launches.into_values() { + log::debug!( + "Planned provider '{}' exhausted its auction collection budget", + state.provider.provider_name() + ); responses.push(provider_timeout_response( state.provider.provider_name(), state.started_at.elapsed().as_millis() as u64, @@ -718,22 +712,34 @@ impl AuctionOrchestratorHarness { services: context.services, }; let mediator_start = Instant::now(); + let mediator_deadline = collection_deadline(auction_start, context.timeout_ms) + .min(collection_deadline(mediator_start, logical_budget_ms)); let mediated = match mediator .request_bids(original_request, &mediator_context) .await { + Ok(_) if Instant::now() >= mediator_deadline => { + log::debug!( + "Mediator '{}' exhausted its response budget during launch; using local ranking", + mediator.provider_name() + ); + None + } Ok(ProviderRequestOutcome::Immediate(response)) => Some(response), Ok(ProviderRequestOutcome::Pending { request: pending, parse_state, - }) => match context.services.http_client().wait(pending).await { + }) => match context + .services + .http_client() + .wait_until(pending, mediator_deadline) + .await + { Ok(platform_response) => { let response_time_ms = mediator_start.elapsed().as_millis() as u64; - if AuctionDeadlinePolicy::for_runtime(context.services) - .rejects_late_completion(auction_start, context.timeout_ms) - { - log::warn!( - "Planned mediator '{}' completed after the hard auction deadline; using local ranking ({}ms)", + if Instant::now() >= mediator_deadline { + log::debug!( + "Planned mediator '{}' completed after its response deadline; using local ranking ({}ms)", mediator.provider_name(), response_time_ms ); @@ -759,11 +765,21 @@ impl AuctionOrchestratorHarness { } } Err(error) => { - log::warn!( - "Planned mediator '{}' request failed: {:?}", - mediator.provider_name(), - error - ); + if matches!( + error.current_context(), + crate::platform::PlatformError::DeadlineExceeded + ) { + log::debug!( + "Planned mediator '{}' exhausted its response budget; using local ranking", + mediator.provider_name() + ); + } else { + log::warn!( + "Planned mediator '{}' request failed: {:?}", + mediator.provider_name(), + error + ); + } None } }, @@ -1019,10 +1035,11 @@ impl AuctionOrchestrator { // consumed part of it. Canonicalize the transport timeout so the // backend name remains stable across equivalent budget values. let remaining_ms = remaining_budget_ms(mediation_start, context.timeout_ms); + let logical_budget_ms = remaining_ms.min(mediator.timeout_ms()); let mediator_timeout = context .services .backend() - .canonicalize_transport_timeout_ms(remaining_ms, mediator.timeout_ms()); + .canonicalize_transport_timeout_ms(logical_budget_ms, mediator.timeout_ms()); if mediator_timeout == 0 { log::warn!("Auction timeout exhausted during bidding phase; skipping mediator"); @@ -1039,19 +1056,36 @@ impl AuctionOrchestrator { let mediator_context = AuctionContext { settings: context.settings, request: context.request, - timeout_ms: mediator_timeout, + timeout_ms: logical_budget_ms, transport_timeout_ms: mediator_timeout, provider_responses: Some(&provider_responses), services: context.services, }; let start_time = Instant::now(); - let mediator_resp = match mediator + let mediator_deadline = collection_deadline(mediation_start, context.timeout_ms) + .min(collection_deadline(start_time, logical_budget_ms)); + let mediator_outcome = mediator .request_bids(request, &mediator_context) .await .change_context(TrustedServerError::Auction { message: format!("Mediator {} failed to launch", mediator.provider_name()), - })? { + })?; + if Instant::now() >= mediator_deadline { + log::debug!( + "Mediator '{}' exhausted its response budget during launch; using local ranking", + mediator.provider_name() + ); + let winning_bids = self.select_winning_bids(&provider_responses, &floor_prices); + return Ok(OrchestrationResult { + provider_responses, + mediator_response: None, + winning_bids, + total_time_ms: mediation_start.elapsed().as_millis() as u64, + metadata: HashMap::new(), + }); + } + let mediator_resp = match mediator_outcome { ProviderRequestOutcome::Immediate(response) => response, ProviderRequestOutcome::Pending { request: pending, @@ -1060,20 +1094,35 @@ impl AuctionOrchestrator { let platform_resp = mediator_context .services .http_client() - .wait(pending) - .await - .change_context(TrustedServerError::Auction { + .wait_until(pending, mediator_deadline) + .await; + if platform_resp.as_ref().is_err_and(|error| { + matches!( + error.current_context(), + crate::platform::PlatformError::DeadlineExceeded + ) + }) { + let winning_bids = + self.select_winning_bids(&provider_responses, &floor_prices); + return Ok(OrchestrationResult { + provider_responses, + mediator_response: None, + winning_bids, + total_time_ms: mediation_start.elapsed().as_millis() as u64, + metadata: HashMap::new(), + }); + } + let platform_resp = + platform_resp.change_context(TrustedServerError::Auction { message: format!( "Mediator {} request failed", mediator.provider_name() ), })?; let response_time_ms = start_time.elapsed().as_millis() as u64; - if AuctionDeadlinePolicy::for_runtime(context.services) - .rejects_late_completion(mediation_start, context.timeout_ms) - { - log::warn!( - "Mediator '{}' completed after the hard auction deadline; using local ranking ({}ms)", + if Instant::now() >= mediator_deadline { + log::debug!( + "Mediator '{}' completed after its response deadline; using local ranking ({}ms)", mediator.provider_name(), response_time_ms ); @@ -1082,7 +1131,7 @@ impl AuctionOrchestrator { provider_responses, mediator_response: None, winning_bids: winning, - total_time_ms: 0, + total_time_ms: mediation_start.elapsed().as_millis() as u64, metadata: HashMap::new(), }); } @@ -1182,10 +1231,8 @@ impl AuctionOrchestrator { })); } - // Reject multi-provider fan-out before any request launches when the - // platform executes `send_async` eagerly (e.g. Cloudflare Workers): - // sequential execution would accrue the sum of provider latencies and - // blow the auction budget before a later `select` could reject it. + // Enforce each adapter's supported provider count before dispatch. + // Worker adapters retain their single-provider collection contract. if provider_names.len() > 1 && !context.services.http_client().supports_concurrent_fanout() { return Err(Report::new(TrustedServerError::Auction { @@ -1371,30 +1418,39 @@ impl AuctionOrchestrator { ), })); } - - let deadline_policy = AuctionDeadlinePolicy::for_runtime(context.services); log::info!( "Launched {} concurrent provider request(s); waiting for responses", pending_requests.len() ); - // Phase 2: Wait for responses using select() to process as they become ready. - // Enforce the auction deadline: after each select() returns, check - // elapsed time and drop remaining requests if the timeout is exceeded. - // - // NOTE: `select()` blocks until at least one backend responds and, on - // some adapters, buffers the selected response body before returning. - // Backend first-byte and between-bytes timeouts are capped to the - // remaining auction budget in Phase 1. They are transport timers, not - // absolute wall-clock limits, so connection setup and byte-trickling - // remain bounded operational risks rather than strict deadline proof. + // Collection bounds header waits and body buffering by the original cutoff. let mut remaining = pending_requests; while !remaining.is_empty() { - let platform_result = match context.services.http_client().select(remaining).await { - Ok(r) => r, + let platform_result = match context + .services + .http_client() + .select_until( + remaining, + collection_deadline(auction_start, context.timeout_ms), + ) + .await + { + Ok(Some(r)) => r, + Ok(None) => break, Err(e) => { log::warn!("select() failed: {:?}", e); + let mut failures = backend_to_provider + .drain() + .map(|(_, state)| { + provider_transport_failed_response( + &state.provider_name, + state.started_at.elapsed().as_millis() as u64, + ) + }) + .collect::>(); + failures.sort_by(|left, right| left.provider.cmp(&right.provider)); + responses.extend(failures); break; } }; @@ -1416,15 +1472,6 @@ impl AuctionOrchestrator { if let Some(state) = backend_to_provider.remove(&backend_name) { let response_time_ms = state.started_at.elapsed().as_millis() as u64; - if deadline_policy - .rejects_late_completion(auction_start, context.timeout_ms) - { - responses.push(provider_timeout_response( - &state.provider_name, - response_time_ms, - )); - continue; - } let provider_context = AuctionContext { settings: context.settings, request: context.request, @@ -1505,11 +1552,6 @@ impl AuctionOrchestrator { } } } - - // Current adapters cannot enforce a hard total-request deadline, so - // drain already-launched handles and retain completed late responses. - // A future adapter that explicitly claims the capability classifies - // each late completion as a timeout instead. } for state in backend_to_provider.into_values() { @@ -1775,6 +1817,10 @@ impl AuctionOrchestrator { let logical_budget_ms = remaining_budget_ms(auction_start, context.timeout_ms).min(provider.timeout_ms()); if logical_budget_ms == 0 { + log::debug!( + "Planned provider '{}' has no remaining auction budget before launch", + provider.provider_name() + ); completed_responses.push(provider_timeout_response(provider.provider_name(), 0)); continue; } @@ -1783,6 +1829,10 @@ impl AuctionOrchestrator { .backend() .canonicalize_transport_timeout_ms(logical_budget_ms, provider.timeout_ms()); if transport_timeout_ms == 0 { + log::debug!( + "Planned provider '{}' has no remaining transport budget before launch", + provider.provider_name() + ); completed_responses.push(provider_timeout_response(provider.provider_name(), 0)); continue; } @@ -1974,11 +2024,8 @@ impl AuctionOrchestrator { return DispatchAuctionOutcome::NotStarted; } - // Mirror run_providers_parallel: reject multi-provider fan-out before - // any request launches when the platform executes `send_async` eagerly - // (e.g. Cloudflare Workers, Spin). Sequential execution would accrue - // the sum of provider latencies before the origin fetch and then fail - // collection with empty bids. + // Mirror the adapter capability guard before dispatch. Worker adapters + // start requests eagerly but retain single-provider collection support. #[cfg(test)] if provider_names.len() > 1 && !context.services.http_client().supports_concurrent_fanout() { @@ -2235,17 +2282,17 @@ impl AuctionOrchestrator { let mut responses: Vec = completed_responses; let mut remaining = pending_requests; - let deadline_policy = AuctionDeadlinePolicy::for_runtime(services); while !remaining.is_empty() { let select_result = match services .http_client() - .select(remaining) + .select_until(remaining, collection_deadline(auction_start, timeout_ms)) .await .change_context(TrustedServerError::Auction { message: "HTTP select failed".to_string(), }) { - Ok(r) => r, + Ok(Some(r)) => r, + Ok(None) => break, Err(e) => { if let Some(trace) = &trace { trace.observe_collection_failure(); @@ -2297,13 +2344,6 @@ impl AuctionOrchestrator { let backend_name = platform_response.backend_name.clone().unwrap_or_default(); if let Some(state) = backend_to_provider.remove(&backend_name) { let response_time_ms = state.started_at.elapsed().as_millis() as u64; - if deadline_policy.rejects_late_completion(auction_start, timeout_ms) { - responses.push(provider_timeout_response( - &state.provider_name, - response_time_ms, - )); - continue; - } let provider_context = AuctionContext { settings: context.settings, request: &provider_request_context, @@ -2333,18 +2373,6 @@ impl AuctionOrchestrator { } } else if let Some(state) = planned_backend_to_provider.remove(&backend_name) { let response_time_ms = state.started_at.elapsed().as_millis() as u64; - if deadline_policy.rejects_late_completion(auction_start, timeout_ms) { - if let (Some(trace), Some(observation)) = - (&trace, state.trace_observation) - { - trace.observe_failure(observation); - } - responses.push(provider_timeout_response( - state.provider.provider_name(), - response_time_ms, - )); - continue; - } match state .provider .parse_response_with_state( @@ -2431,14 +2459,6 @@ impl AuctionOrchestrator { } } } - - // Drain every dispatched request. Each backend was capped with - // first-byte and between-bytes timeouts at dispatch time, so by the - // collect phase the remaining handles may already be ready even if - // wall-clock time elapsed while the origin was slow. Dropping them - // here would discard SSP responses that already arrived. The - // mediator launch below still observes A_deadline via - // `remaining_budget_ms`. } for state in backend_to_provider.values() { @@ -2454,6 +2474,10 @@ impl AuctionOrchestrator { } backend_to_provider.clear(); for state in planned_backend_to_provider.into_values() { + log::debug!( + "Planned provider '{}' exhausted its dispatched auction collection budget", + state.provider.provider_name() + ); if let (Some(trace), Some(observation)) = (&trace, state.trace_observation) { trace.observe_failure(observation); } @@ -2563,6 +2587,8 @@ impl AuctionOrchestrator { provider_responses: Some(&responses), services: context.services, }; + let mediator_deadline = collection_deadline(auction_start, timeout_ms) + .min(collection_deadline(mediator_start, logical_budget_ms)); let mut trace_observation = trace .as_ref() .map(|trace| trace.launch_provider(TraceProviderRole::Mediator)); @@ -2570,6 +2596,13 @@ impl AuctionOrchestrator { .request_bids(&request, &mediator_context) .await { + Ok(_) if Instant::now() >= mediator_deadline => { + log::debug!( + "Mediator '{}' exhausted its response budget during launch; using local ranking", + mediator.provider_name() + ); + None + } Ok(ProviderRequestOutcome::Immediate(response)) => { if let (Some(trace), Some(observation)) = (&trace, trace_observation.take()) { @@ -2580,19 +2613,16 @@ impl AuctionOrchestrator { Ok(ProviderRequestOutcome::Pending { request: pending, parse_state, - }) => match services.http_client().wait(pending).await.change_context( - TrustedServerError::Auction { - message: format!( - "Mediator {} request failed", - mediator.provider_name() - ), - }, - ) { + }) => match services + .http_client() + .wait_until(pending, mediator_deadline) + .await + { Ok(platform_resp) => { let response_time_ms = mediator_start.elapsed().as_millis() as u64; - if deadline_policy.rejects_late_completion(auction_start, timeout_ms) { - log::warn!( - "Mediator '{}' completed after the hard auction deadline; using local ranking ({}ms)", + if Instant::now() >= mediator_deadline { + log::debug!( + "Mediator '{}' completed after its response deadline; using local ranking ({}ms)", mediator.provider_name(), response_time_ms ); @@ -2628,7 +2658,17 @@ impl AuctionOrchestrator { } } Err(error) => { - log::warn!("Mediator request failed: {:?}", error); + if matches!( + error.current_context(), + crate::platform::PlatformError::DeadlineExceeded + ) { + log::debug!( + "Mediator '{}' exhausted its response budget; using local ranking", + mediator.provider_name() + ); + } else { + log::warn!("Mediator request failed: {:?}", error); + } None } }, @@ -3887,6 +3927,14 @@ mod tests { self.inner.send_async(request).await } + async fn select_until( + &self, + pending_requests: Vec, + _deadline: Instant, + ) -> Result, Report> { + self.select(pending_requests).await.map(Some) + } + async fn select( &self, pending_requests: Vec, @@ -3897,6 +3945,55 @@ mod tests { } } + struct LateReadyMediatorHttpClient { + inner: Arc, + waits: Arc, + } + + #[async_trait::async_trait(?Send)] + impl PlatformHttpClient for LateReadyMediatorHttpClient { + async fn send( + &self, + request: PlatformHttpRequest, + ) -> Result> { + self.inner.send(request).await + } + + async fn send_async( + &self, + request: PlatformHttpRequest, + ) -> Result> { + self.inner.send_async(request).await + } + + async fn select( + &self, + pending: Vec, + ) -> Result> { + self.inner.select(pending).await + } + + async fn select_until( + &self, + pending: Vec, + deadline: Instant, + ) -> Result, Report> { + self.inner.select_until(pending, deadline).await + } + + async fn wait_until( + &self, + pending: PlatformPendingRequest, + _deadline: Instant, + ) -> Result> { + self.waits.fetch_add(1, Ordering::Relaxed); + // Model an executor waking after the mediator cutoff with a complete + // result returned by the adapter's final nonblocking snapshot. + std::thread::sleep(Duration::from_millis(40)); + self.inner.wait(pending).await + } + } + struct UnusedSecretStore; impl PlatformSecretStore for UnusedSecretStore { @@ -3953,7 +4050,7 @@ mod tests { context .services .http_client() - .send_async(req) + .send_async(req.with_cache_bypass()) .await .change_context(TrustedServerError::Auction { message: "stub launch failed".to_string(), @@ -4028,7 +4125,7 @@ mod tests { context .services .http_client() - .send_async(request) + .send_async(request.with_cache_bypass()) .await .change_context(TrustedServerError::Auction { message: "deadline test provider launch failed".to_string(), @@ -4064,7 +4161,7 @@ mod tests { budgets: Option, } - struct PendingDeadlineMediator; + struct PendingDeadlineMediator(u32); #[async_trait::async_trait(?Send)] impl AuctionProvider for PendingDeadlineMediator { @@ -4088,7 +4185,7 @@ mod tests { context .services .http_client() - .send_async(request) + .send_async(request.with_cache_bypass()) .await .change_context(TrustedServerError::Auction { message: "pending mediator launch failed".to_string(), @@ -4109,7 +4206,63 @@ mod tests { } fn timeout_ms(&self) -> u32 { - 1_000 + self.0 + } + + fn backend_name(&self, _services: &RuntimeServices, _timeout_ms: u32) -> Option { + Some("pending-mediator-backend".to_string()) + } + } + + struct SlowLaunchMediator { + immediate: bool, + slow_launch: bool, + parses: Arc, + } + + #[async_trait::async_trait(?Send)] + impl AuctionProvider for SlowLaunchMediator { + fn provider_name(&self) -> &str { + "pending-deadline-mediator" + } + + async fn request_bids( + &self, + request: &AuctionRequest, + context: &AuctionContext<'_>, + ) -> Result> { + let outcome = if self.immediate { + ProviderRequestOutcome::Immediate(AuctionResponse::success( + self.provider_name(), + vec![auction_bid("mediated", 9.0)], + 0, + )) + } else { + PendingDeadlineMediator(20) + .request_bids(request, context) + .await? + }; + // Simulate launch work that outlives the mediator's own budget while + // a complete transport result is already buffered. + if self.slow_launch { + std::thread::sleep(Duration::from_millis(40)); + } + Ok(outcome) + } + + async fn parse_response( + &self, + response: PlatformResponse, + response_time_ms: u64, + ) -> Result> { + self.parses.fetch_add(1, Ordering::Relaxed); + PendingDeadlineMediator(20) + .parse_response(response, response_time_ms) + .await + } + + fn timeout_ms(&self) -> u32 { + 20 } fn backend_name(&self, _services: &RuntimeServices, _timeout_ms: u32) -> Option { @@ -4188,7 +4341,7 @@ mod tests { context .services .http_client() - .send_async(request) + .send_async(request.with_cache_bypass()) .await .change_context(TrustedServerError::Auction { message: "recording launch failed".to_string(), @@ -4249,7 +4402,7 @@ mod tests { context .services .http_client() - .send_async(request) + .send_async(request.with_cache_bypass()) .await .change_context(TrustedServerError::Auction { message: "divergent launch failed".to_string(), @@ -4414,7 +4567,7 @@ mod tests { context .services .http_client() - .send_async(req) + .send_async(req.with_cache_bypass()) .await .change_context(TrustedServerError::Auction { message: "mediator launch failed".to_string(), @@ -4622,15 +4775,11 @@ mod tests { } } - async fn collect_deadline_test_result( - split: bool, - enforceable_total_request_deadline: bool, - ) -> OrchestrationResult { + async fn collect_deadline_test_result(split: bool) -> OrchestrationResult { let stub = Arc::new(StubHttpClient::new()); - stub.set_enforceable_total_request_deadline(enforceable_total_request_deadline); stub.push_response(200, b"{}".to_vec()); stub.push_response(200, b"{}".to_vec()); - stub.push_select_delay(Duration::from_millis(50)); + stub.push_select_delay(Duration::from_millis(200)); let services = build_services_with_http_client(Arc::clone(&stub) as Arc<_>); let config = AuctionConfig { enabled: true, @@ -4659,7 +4808,7 @@ mod tests { services: &services, }; - if split { + let result = if split { let DispatchAuctionOutcome::Dispatched(dispatched) = orchestrator.dispatch_auction(&request, &context).await else { @@ -4673,56 +4822,44 @@ mod tests { .run_auction(&request, &context) .await .expect("deadline test auction should complete") - } + }; + assert_eq!( + stub.recorded_select_count(), + 0, + "should skip the delayed unbounded collection path" + ); + result } #[tokio::test] - async fn current_adapter_deadline_drains_late_responses_in_both_paths() { + async fn current_adapter_deadline_stops_collection_in_both_paths() { for split in [false, true] { - let result = collect_deadline_test_result(split, false).await; - assert_eq!(result.provider_responses.len(), 2); - assert_eq!(result.provider_responses[0].provider, "late-one"); - assert_eq!(result.provider_responses[0].status, BidStatus::Success); - assert_eq!(result.provider_responses[1].provider, "late-two"); - assert_eq!(result.provider_responses[1].status, BidStatus::Success); + let result = collect_deadline_test_result(split).await; + assert_eq!( + result.provider_responses.len(), + 2, + "should account for every provider" + ); assert!( - result - .provider_responses - .iter() - .all(|response| response.response_time_ms >= 50), - "late response times should retain actual elapsed duration" + result.provider_responses.iter().all(|response| { + response.status == BidStatus::Error + && response.metadata["error_type"] == ERROR_TYPE_TIMEOUT + }), + "should time out outstanding providers on every adapter" ); - assert_eq!( - result.winning_bids["slot-1"].bidder, "late-one", - "a completed response remains eligible after the logical deadline" + assert!( + result.winning_bids.is_empty(), + "should not accept late bids" ); } } - #[tokio::test] - async fn synthetic_hard_deadline_classifies_late_responses_in_both_paths() { - for split in [false, true] { - let result = collect_deadline_test_result(split, true).await; - assert_eq!(result.provider_responses.len(), 2); - assert!(result.provider_responses.iter().all(|response| { - response.status == BidStatus::Error - && response.metadata["error_type"] == ERROR_TYPE_TIMEOUT - && response.response_time_ms >= 50 - })); - assert!(result.winning_bids.is_empty()); - } - } - - async fn pending_mediator_deadline_test_result( - split: bool, - enforceable_total_request_deadline: bool, - ) -> OrchestrationResult { + async fn pending_mediator_deadline_test_result(split: bool) -> OrchestrationResult { let stub = Arc::new(StubHttpClient::new()); - stub.set_enforceable_total_request_deadline(enforceable_total_request_deadline); stub.push_response(200, b"{}".to_vec()); stub.push_response(200, b"{}".to_vec()); stub.push_select_delay(Duration::ZERO); - stub.push_wait_delay(Duration::from_millis(50)); + stub.push_wait_delay(Duration::from_millis(200)); let services = build_services_with_http_client(Arc::clone(&stub) as Arc<_>); let config = AuctionConfig { enabled: true, @@ -4736,7 +4873,7 @@ mod tests { name: "local", backend: "local-backend", })); - orchestrator.register_provider(Arc::new(PendingDeadlineMediator)); + orchestrator.register_provider(Arc::new(PendingDeadlineMediator(1_000))); let request = create_test_auction_request(); let settings = create_test_settings(); let downstream = http::Request::new(edgezero_core::body::Body::empty()); @@ -4749,7 +4886,7 @@ mod tests { services: &services, }; - if split { + let result = if split { let DispatchAuctionOutcome::Dispatched(dispatched) = orchestrator.dispatch_auction(&request, &context).await else { @@ -4763,35 +4900,32 @@ mod tests { .run_auction(&request, &context) .await .expect("deadline test auction should complete") - } + }; + assert_eq!( + stub.recorded_wait_count(), + 0, + "should skip the delayed unbounded collection path" + ); + result } #[tokio::test] - async fn pending_mediator_late_completion_policy_is_equivalent_in_sync_and_split_paths() { + async fn pending_mediator_is_bounded_in_sync_and_split_paths() { for split in [false, true] { - let current = pending_mediator_deadline_test_result(split, false).await; - let current_mediator = current - .mediator_response - .as_ref() - .expect("current adapters should accept completed late mediator responses"); + let result = pending_mediator_deadline_test_result(split).await; assert!( - current_mediator.response_time_ms >= 50, - "mediator timing should preserve actual elapsed duration" + result.mediator_response.is_none(), + "should abandon a late mediator" ); - assert_eq!(current.winning_bids["slot-1"].bidder, "mediated"); - - let hard = pending_mediator_deadline_test_result(split, true).await; - assert!(hard.mediator_response.is_none()); - assert_eq!(hard.winning_bids["slot-1"].bidder, "local"); - assert!( - hard.total_time_ms >= 50, - "discarding a late mediator must retain actual total elapsed time" + assert_eq!( + result.winning_bids["slot-1"].bidder, "local", + "should retain provider winner" ); } } #[tokio::test] - async fn split_deadline_skips_mediator_and_falls_back_to_provider_winner() { + async fn split_deadline_skips_mediator_and_discards_late_provider() { let stub = Arc::new(StubHttpClient::new()); stub.push_response(200, b"{}".to_vec()); stub.push_select_delay(Duration::from_millis(50)); @@ -4835,11 +4969,14 @@ mod tests { assert_eq!(launches.load(Ordering::Relaxed), 0); assert!(result.mediator_response.is_none()); - assert_eq!(result.winning_bids["slot-1"].bidder, "late-one"); + assert!( + result.winning_bids.is_empty(), + "should not retain a late provider bid" + ); } #[tokio::test] - async fn synchronous_deadline_skips_mediator_and_falls_back_to_provider_winner() { + async fn synchronous_deadline_skips_mediator_and_discards_late_provider() { let stub = Arc::new(StubHttpClient::new()); stub.push_response(200, b"{}".to_vec()); stub.push_select_delay(Duration::from_millis(50)); @@ -4879,7 +5016,10 @@ mod tests { assert_eq!(launches.load(Ordering::Relaxed), 0); assert!(result.mediator_response.is_none()); - assert_eq!(result.winning_bids["slot-1"].bidder, "late-one"); + assert!( + result.winning_bids.is_empty(), + "should not retain a late provider bid" + ); } fn create_test_auction_request() -> AuctionRequest { @@ -6291,7 +6431,7 @@ mod tests { fn rejects_multi_provider_fanout_before_launch_on_sequential_platform() { futures::executor::block_on(async { // Arrange: two configured providers on a platform whose HTTP - // client executes send_async eagerly (no concurrent fan-out). + // client is configured without multi-provider collection support. let stub = Arc::new(StubHttpClient::new()); stub.set_concurrent_fanout(false); let stub_for_assertion = Arc::clone(&stub); @@ -6354,7 +6494,7 @@ mod tests { fn dispatch_auction_skips_multi_provider_fanout_on_sequential_platform() { futures::executor::block_on(async { // Arrange: two configured providers on a platform whose HTTP - // client executes send_async eagerly (no concurrent fan-out). + // client is configured without multi-provider collection support. // The initial-page dispatch path must apply the same guard as // run_providers_parallel or the summed provider latency lands // before the origin fetch. @@ -6561,11 +6701,243 @@ mod tests { ); } + #[tokio::test] + async fn planned_collection_preserves_on_time_bid_and_times_out_pending_sibling() { + for split in [false, true] { + let http = Arc::new(StubHttpClient::new()); + http.push_response( + 200, + serde_json::to_vec(&serde_json::json!({ + "seatbid": [{"seat": "example", "bid": [{ + "id":"on-time", "impid":"fictional-slot", "price":2.0, + "adm":"
fictional
", "w":300, "h":250 + }]}] + })) + .expect("should serialize on-time bid"), + ); + http.push_response(204, Vec::new()); + http.push_select_delay(Duration::ZERO); + http.push_select_delay(Duration::from_millis(250)); + let services = build_services_with_backend_and_http_client( + Arc::new(NamingBackend::new(BackendNamingPolicy::Axum)), + Arc::clone(&http) as Arc<_>, + ); + let plan = AuctionPlan::compile(planned_config( + &[ + ("provider-a", RoutingMode::AllEligible), + ("provider-b", RoutingMode::AllEligible), + ], + false, + )) + .expect("should compile two providers"); + let orchestrator = AuctionOrchestrator::from_plan(Arc::new(plan), None); + let request = planned_request(); + let settings = create_test_settings(); + let inbound = http::Request::new(edgezero_core::body::Body::empty()); + let context = AuctionContext { + settings: &settings, + request: &inbound, + timeout_ms: 30, + transport_timeout_ms: 30, + provider_responses: None, + services: &services, + }; + let result = if split { + let DispatchAuctionOutcome::Dispatched(dispatched) = + orchestrator.dispatch_auction(&request, &context).await + else { + panic!("should dispatch providers"); + }; + orchestrator + .collect_dispatched_auction(dispatched, &services, &context) + .await + } else { + orchestrator + .run_auction(&request, &context) + .await + .expect("should collect partial results") + }; + assert!( + http.recorded_cache_intents() + .iter() + .all(crate::platform::PlatformCacheIntent::is_bypass), + "should explicitly bypass intermediary caches for personalized auctions" + ); + assert_eq!( + result.provider_responses.len(), + 2, + "should retain one result per provider" + ); + assert_eq!( + result.provider_responses[0].provider, "provider-a", + "should retain configured ordering" + ); + assert_eq!( + result.provider_responses[1].provider, "provider-b", + "should attribute timeout to outstanding sibling" + ); + assert_eq!( + result.provider_responses[1].metadata["error_type"], ERROR_TYPE_TIMEOUT, + "should report deadline expiry as timeout" + ); + assert_eq!( + result.winning_bids["fictional-slot"].bid_id.as_deref(), + Some("on-time"), + "should preserve the collected winner" + ); + assert!( + result.total_time_ms < 200, + "should not drain the delayed provider" + ); + } + } + + #[tokio::test] + async fn planned_split_collection_drains_ready_bid_after_slow_origin_without_mediation() { + let http = Arc::new(StubHttpClient::new()); + http.push_response( + 200, + serde_json::to_vec(&serde_json::json!({ + "seatbid": [{"seat": "example", "bid": [{ + "id": "on-time", "impid": "fictional-slot", "price": 2.0, + "adm": "
fictional
", "w": 300, "h": 250 + }]}] + })) + .expect("should serialize buffered bid"), + ); + http.push_response(204, Vec::new()); + http.push_select_delay(Duration::ZERO); + // Shorter than the 10 ms budget, so a collection clock restarted after + // the origin wait would collect this 204 instead of timing it out. + http.push_select_delay(Duration::from_millis(1)); + let services = build_services_with_backend_and_http_client( + Arc::new(NamingBackend::new(BackendNamingPolicy::Axum)), + Arc::clone(&http) as Arc<_>, + ); + let plan = AuctionPlan::compile(planned_config( + &[ + ("provider-a", RoutingMode::AllEligible), + ("provider-b", RoutingMode::AllEligible), + ], + false, + )) + .expect("should compile providers"); + let launches = Arc::new(AtomicUsize::new(0)); + let mediator = Arc::new(DeadlineRecordingMediator { + launches: Arc::clone(&launches), + budgets: None, + }); + let orchestrator = AuctionOrchestrator::from_plan(Arc::new(plan), Some(mediator)); + let request = planned_request(); + let settings = create_test_settings(); + let inbound = http::Request::new(edgezero_core::body::Body::empty()); + let context = AuctionContext { + settings: &settings, + request: &inbound, + timeout_ms: 10, + transport_timeout_ms: 10, + provider_responses: None, + services: &services, + }; + let DispatchAuctionOutcome::Dispatched(dispatched) = + orchestrator.dispatch_auction(&request, &context).await + else { + panic!("should dispatch providers"); + }; + std::thread::sleep(Duration::from_millis(25)); + let started = Instant::now(); + let result = orchestrator + .collect_dispatched_auction(dispatched, &services, &context) + .await; + assert_eq!( + result.provider_responses.len(), + 2, + "should retain one result for every provider" + ); + assert_eq!( + result + .winning_bids + .get("fictional-slot") + .and_then(|bid| bid.bid_id.as_deref()), + Some("on-time"), + "should retain the available bid after the origin wait" + ); + assert_eq!( + result.provider_responses[1].metadata["error_type"], ERROR_TYPE_TIMEOUT, + "should time out the unfinished sibling" + ); + assert!( + result + .provider_responses + .iter() + .all(|response| response.response_time_ms >= 10), + "should report elapsed dispatch-to-collection time" + ); + assert_eq!( + launches.load(Ordering::Relaxed), + 0, + "should skip mediation after budget exhaustion" + ); + assert!( + started.elapsed() < Duration::from_millis(100), + "should never wait again for an unfinished response" + ); + } + + #[tokio::test] + async fn planned_split_collection_uses_dispatch_deadline_after_slow_origin() { + let http = Arc::new(StubHttpClient::new()); + http.push_response(204, Vec::new()); + // Shorter than the 10 ms budget, so a collection clock restarted after + // the origin wait would collect this 204 instead of timing it out. + http.push_select_delay(Duration::from_millis(1)); + let services = build_services_with_backend_and_http_client( + Arc::new(NamingBackend::new(BackendNamingPolicy::Axum)), + http, + ); + let plan = AuctionPlan::compile(planned_config( + &[("provider-a", RoutingMode::AllEligible)], + false, + )) + .expect("should compile provider"); + let orchestrator = AuctionOrchestrator::from_plan(Arc::new(plan), None); + let request = planned_request(); + let settings = create_test_settings(); + let inbound = http::Request::new(edgezero_core::body::Body::empty()); + let context = AuctionContext { + settings: &settings, + request: &inbound, + timeout_ms: 10, + transport_timeout_ms: 10, + provider_responses: None, + services: &services, + }; + let DispatchAuctionOutcome::Dispatched(dispatched) = + orchestrator.dispatch_auction(&request, &context).await + else { + panic!("should dispatch provider"); + }; + std::thread::sleep(Duration::from_millis(25)); + let started = Instant::now(); + let result = orchestrator + .collect_dispatched_auction(dispatched, &services, &context) + .await; + assert_eq!( + result.provider_responses[0].metadata["error_type"], ERROR_TYPE_TIMEOUT, + "should not restart the auction clock at collection" + ); + assert!( + started.elapsed() < Duration::from_millis(100), + "should return without waiting again" + ); + } + async fn planned_pending_mediator_deadline_result( - enforceable_total_request_deadline: bool, + auction_timeout_ms: u32, + mediator_timeout_ms: u32, + policy: BackendNamingPolicy, ) -> OrchestrationResult { let http = Arc::new(StubHttpClient::new()); - http.set_enforceable_total_request_deadline(enforceable_total_request_deadline); http.push_response( 200, serde_json::to_vec(&serde_json::json!({ @@ -6578,8 +6950,8 @@ mod tests { ); http.push_response(200, b"{}".to_vec()); http.push_select_delay(Duration::ZERO); - http.push_wait_delay(Duration::from_millis(50)); - let backend = Arc::new(NamingBackend::new(BackendNamingPolicy::Axum)); + http.push_wait_delay(Duration::from_millis(200)); + let backend = Arc::new(NamingBackend::new(policy)); let services = build_services_with_backend_and_http_client( Arc::clone(&backend) as Arc<_>, Arc::clone(&http) as Arc<_>, @@ -6589,43 +6961,187 @@ mod tests { false, )) .expect("should compile planned auction"); - let orchestrator = - AuctionOrchestratorHarness::new(plan, Some(Arc::new(PendingDeadlineMediator))); + let orchestrator = AuctionOrchestrator::from_plan( + Arc::new(plan), + Some(Arc::new(PendingDeadlineMediator(mediator_timeout_ms))), + ); let request = planned_request(); let settings = create_test_settings(); let inbound = http::Request::new(edgezero_core::body::Body::empty()); let context = AuctionContext { settings: &settings, request: &inbound, - timeout_ms: 20, - transport_timeout_ms: 20, + timeout_ms: auction_timeout_ms, + transport_timeout_ms: policy + .canonicalize_transport_timeout_ms(auction_timeout_ms, 1_000), provider_responses: None, services: &services, }; - orchestrator + let result = orchestrator .run_auction(&request, &context) .await - .expect("should execute planned pending mediator") + .expect("should execute planned pending mediator"); + assert_eq!( + http.recorded_wait_count(), + 0, + "should skip the delayed unbounded mediator wait" + ); + result } #[tokio::test] - async fn planned_pending_mediator_applies_explicit_hard_deadline_policy() { - let current = planned_pending_mediator_deadline_result(false).await; - let current_mediator = current - .mediator_response - .as_ref() - .expect("current adapters should accept completed late mediator responses"); - assert!(current_mediator.response_time_ms >= 50); - assert_eq!(current.winning_bids["slot-1"].bidder, "mediated"); + async fn planned_pending_mediator_observes_absolute_deadline() { + let result = + planned_pending_mediator_deadline_result(20, 1_000, BackendNamingPolicy::Axum).await; + assert!( + result.mediator_response.is_none(), + "should stop waiting for mediator" + ); + assert_eq!( + result.winning_bids["fictional-slot"].bid_id.as_deref(), + Some("provider"), + "should preserve on-time provider bids" + ); + } - let hard = planned_pending_mediator_deadline_result(true).await; - assert!(hard.mediator_response.is_none()); + #[tokio::test] + async fn planned_pending_mediator_observes_own_budget_before_auction_deadline() { + let result = + planned_pending_mediator_deadline_result(1_000, 20, BackendNamingPolicy::Fastly).await; + assert!( + result.mediator_response.is_none(), + "should stop at the mediator budget before the auction deadline" + ); assert_eq!( - hard.winning_bids["fictional-slot"].bid_id.as_deref(), - Some("provider") + result.winning_bids["fictional-slot"].bid_id.as_deref(), + Some("provider"), + "should preserve on-time provider bids" ); - assert!(hard.total_time_ms >= 50); + } + + #[tokio::test] + async fn mediator_own_deadline_rejects_ready_results_after_launch_or_wait() { + for planned in [false, true] { + for split in [false, true] { + for (immediate, slow_launch) in [(false, true), (true, true), (false, false)] { + let http = Arc::new(StubHttpClient::new()); + http.push_response( + 200, + serde_json::to_vec(&serde_json::json!({ + "seatbid": [{"seat": "provider-seat", "bid": [{ + "id": "provider", "impid": "fictional-slot", "price": 2.0, + "adm": "
provider
", "w": 300, "h": 250 + }]}] + })) + .expect("should serialize provider response"), + ); + if !immediate { + http.push_response(200, b"{}".to_vec()); + } + let waits = Arc::new(AtomicUsize::new(0)); + let client: Arc = if slow_launch { + http + } else { + Arc::new(LateReadyMediatorHttpClient { + inner: http, + waits: Arc::clone(&waits), + }) + }; + let services = build_services_with_backend_and_http_client( + Arc::new(NamingBackend::new(BackendNamingPolicy::Axum)), + client, + ); + let parses = Arc::new(AtomicUsize::new(0)); + let mediator = Arc::new(SlowLaunchMediator { + immediate, + slow_launch, + parses: Arc::clone(&parses), + }); + let orchestrator = if planned { + let plan = AuctionPlan::compile(planned_config( + &[("provider-a", RoutingMode::AllEligible)], + false, + )) + .expect("should compile planned auction"); + AuctionOrchestrator::from_plan(Arc::new(plan), Some(mediator)) + } else { + let mut orchestrator = AuctionOrchestrator::new(AuctionConfig { + enabled: true, + providers: AuctionConfig::legacy_provider_map(&["local"]), + mediator: Some("pending-deadline-mediator".to_string()), + timeout_ms: 1_000, + ..Default::default() + }); + orchestrator.register_provider(Arc::new(DeadlineBidProvider { + name: "local", + backend: "local-backend", + })); + orchestrator.register_provider(mediator); + orchestrator + }; + let request = if planned { + planned_request() + } else { + create_test_auction_request() + }; + let settings = create_test_settings(); + let inbound = http::Request::new(edgezero_core::body::Body::empty()); + let context = AuctionContext { + settings: &settings, + request: &inbound, + timeout_ms: 1_000, + transport_timeout_ms: 1_000, + provider_responses: None, + services: &services, + }; + let result = if split { + let DispatchAuctionOutcome::Dispatched(dispatched) = + orchestrator.dispatch_auction(&request, &context).await + else { + panic!("should dispatch provider"); + }; + orchestrator + .collect_dispatched_auction(dispatched, &services, &context) + .await + } else { + orchestrator + .run_auction(&request, &context) + .await + .expect("should complete auction") + }; + assert_eq!( + result.provider_responses.len(), + 1, + "should retain the on-time provider result" + ); + assert!( + result.mediator_response.is_none(), + "should reject expired mediator: planned={planned} split={split} immediate={immediate}" + ); + assert_eq!( + parses.load(Ordering::Relaxed), + 0, + "should reject expiry before parsing" + ); + assert_eq!( + waits.load(Ordering::Relaxed), + usize::from(!slow_launch), + "should exercise the intended launch or wait boundary" + ); + assert!( + result.total_time_ms >= 40, + "should report elapsed auction time after mediator expiry" + ); + let slot = if planned { "fictional-slot" } else { "slot-1" }; + assert_eq!( + result.winning_bids[slot].price, + Some(if planned { 2.0 } else { 3.0 }), + "should preserve the local winner" + ); + } + } + } } #[tokio::test] diff --git a/crates/trusted-server-core/src/auction/provider.rs b/crates/trusted-server-core/src/auction/provider.rs index 62b2ddc8fc..d68ebde4d8 100644 --- a/crates/trusted-server-core/src/auction/provider.rs +++ b/crates/trusted-server-core/src/auction/provider.rs @@ -93,6 +93,9 @@ pub trait AuctionProvider: Send + Sync { /// Implementations normally return a pending upstream request, but may return /// an immediate response for a legitimate outcome that requires no HTTP call. /// The orchestrator handles waiting for and parsing pending responses. + /// Outbound auction and mediator requests must use + /// [`PlatformHttpRequest::with_cache_bypass`]: they carry personalized data, + /// and Fastly requires this marker for deadline-bounded collection. /// /// # Errors /// @@ -381,7 +384,9 @@ impl GenericOpenRtbProvider { } let pending = services .http_client() - .send_async(PlatformHttpRequest::new(outbound, backend_name.clone())) + .send_async( + PlatformHttpRequest::new(outbound, backend_name.clone()).with_cache_bypass(), + ) .await .change_context(TrustedServerError::Auction { message: format!("Provider {} request launch failed", self.provider_name()), diff --git a/crates/trusted-server-core/src/integrations/adserver_mock.rs b/crates/trusted-server-core/src/integrations/adserver_mock.rs index 640574e031..efc789f3aa 100644 --- a/crates/trusted-server-core/src/integrations/adserver_mock.rs +++ b/crates/trusted-server-core/src/integrations/adserver_mock.rs @@ -514,7 +514,7 @@ impl AuctionProvider for AdServerMockProvider { let pending = context .services .http_client() - .send_async(PlatformHttpRequest::new(req, backend_name)) + .send_async(PlatformHttpRequest::new(req, backend_name).with_cache_bypass()) .await .change_context(TrustedServerError::Auction { message: "Failed to send mediation request".to_string(), diff --git a/crates/trusted-server-core/src/integrations/aps.rs b/crates/trusted-server-core/src/integrations/aps.rs index 2c8cacd32d..62f8c8f068 100644 --- a/crates/trusted-server-core/src/integrations/aps.rs +++ b/crates/trusted-server-core/src/integrations/aps.rs @@ -1720,7 +1720,7 @@ impl AuctionProvider for ApsAuctionProvider { let pending = context .services .http_client() - .send_async(PlatformHttpRequest::new(outbound_request, backend)) + .send_async(PlatformHttpRequest::new(outbound_request, backend).with_cache_bypass()) .await .change_context(TrustedServerError::Auction { message: "Failed to send APS request".to_string(), diff --git a/crates/trusted-server-core/src/integrations/prebid.rs b/crates/trusted-server-core/src/integrations/prebid.rs index 82c5d808f3..b3381911a5 100644 --- a/crates/trusted-server-core/src/integrations/prebid.rs +++ b/crates/trusted-server-core/src/integrations/prebid.rs @@ -3480,7 +3480,7 @@ impl AuctionProvider for PrebidAuctionProvider { let pending = context .services .http_client() - .send_async(PlatformHttpRequest::new(pbs_req, backend_name)) + .send_async(PlatformHttpRequest::new(pbs_req, backend_name).with_cache_bypass()) .await .change_context(TrustedServerError::Prebid { message: "Failed to send async request to Prebid Server".to_string(), diff --git a/crates/trusted-server-core/src/platform/backend_naming.rs b/crates/trusted-server-core/src/platform/backend_naming.rs index 54fc176b1b..52487def53 100644 --- a/crates/trusted-server-core/src/platform/backend_naming.rs +++ b/crates/trusted-server-core/src/platform/backend_naming.rs @@ -189,7 +189,6 @@ impl AuctionTargetId { naming_policy: BackendNamingPolicy::Fastly, capabilities: AuctionTargetCapabilities { concurrent_provider_fanout: true, - enforceable_total_request_deadline: false, }, }, Self::Axum => AuctionTargetDescriptor { @@ -197,7 +196,6 @@ impl AuctionTargetId { naming_policy: BackendNamingPolicy::Axum, capabilities: AuctionTargetCapabilities { concurrent_provider_fanout: true, - enforceable_total_request_deadline: false, }, }, Self::Cloudflare => AuctionTargetDescriptor { @@ -205,7 +203,6 @@ impl AuctionTargetId { naming_policy: BackendNamingPolicy::Cloudflare, capabilities: AuctionTargetCapabilities { concurrent_provider_fanout: false, - enforceable_total_request_deadline: false, }, }, Self::Spin => AuctionTargetDescriptor { @@ -213,7 +210,6 @@ impl AuctionTargetId { naming_policy: BackendNamingPolicy::Spin, capabilities: AuctionTargetCapabilities { concurrent_provider_fanout: false, - enforceable_total_request_deadline: false, }, }, } @@ -224,7 +220,6 @@ impl AuctionTargetId { #[derive(Debug, Clone, Copy, Eq, PartialEq)] pub struct AuctionTargetCapabilities { concurrent_provider_fanout: bool, - enforceable_total_request_deadline: bool, } impl AuctionTargetCapabilities { @@ -233,15 +228,6 @@ impl AuctionTargetCapabilities { pub const fn supports_concurrent_provider_fanout(self) -> bool { self.concurrent_provider_fanout } - - /// Return whether the adapter enforces a total request deadline per provider. - /// - /// Transport first-byte or between-byte timers do not satisfy this - /// capability because they do not cap the complete request lifetime. - #[must_use] - pub const fn has_enforceable_total_request_deadline(self) -> bool { - self.enforceable_total_request_deadline - } } /// Shared target descriptor used by plan validation. @@ -618,12 +604,6 @@ mod tests { "should declare fanout accurately for {}", id.adapter_id() ); - assert!( - !descriptor - .capabilities() - .has_enforceable_total_request_deadline(), - "no current adapter should claim a total request deadline" - ); assert_eq!(AuctionTargetId::from_adapter_id(id.adapter_id()), Some(id)); } assert_eq!(AuctionTargetId::from_adapter_id("FASTLY"), None); diff --git a/crates/trusted-server-core/src/platform/error.rs b/crates/trusted-server-core/src/platform/error.rs index 699eb7d483..12d0c39c69 100644 --- a/crates/trusted-server-core/src/platform/error.rs +++ b/crates/trusted-server-core/src/platform/error.rs @@ -20,6 +20,9 @@ pub enum PlatformError { /// HTTP client request failed. #[display("http client error")] HttpClient, + /// The absolute HTTP collection deadline elapsed. + #[display("http collection deadline exceeded")] + DeadlineExceeded, /// Geo lookup failed. #[display("geo lookup error")] Geo, diff --git a/crates/trusted-server-core/src/platform/http.rs b/crates/trusted-server-core/src/platform/http.rs index b47abaebbb..e1fe8ff1f2 100644 --- a/crates/trusted-server-core/src/platform/http.rs +++ b/crates/trusted-server-core/src/platform/http.rs @@ -3,6 +3,7 @@ use std::fmt; use edgezero_core::http::{Method, Request as EdgeRequest, Response as EdgeResponse}; use error_stack::Report; +use web_time::Instant; use super::PlatformError; use super::image_optimizer::PlatformImageOptimizerOptions; @@ -348,7 +349,17 @@ pub trait PlatformHttpClient: Send + Sync { request: PlatformHttpRequest, ) -> Result>; - /// Start an upstream request without waiting for it to complete. + /// Starts an upstream request without waiting for its response. + /// + /// Adapters retain ownership of the request operation so it can progress + /// while the publisher origin is fetched. Dropping its pending handle + /// cancels or abandons unfinished work according to the runtime contract. + /// + /// On Cloudflare and Spin the request runs as a task on the runtime + /// executor, so [`Self::wait`] and [`Self::select_until`] must be awaited + /// on that executor. Driving them through a blocking executor such as + /// `futures::executor::block_on` blocks the only thread the task can run + /// on and hangs. /// /// # Errors /// @@ -359,29 +370,15 @@ pub trait PlatformHttpClient: Send + Sync { request: PlatformHttpRequest, ) -> Result>; - /// Whether [`send_async`](Self::send_async) defers execution so multiple - /// pending requests progress concurrently and [`select`](Self::select) - /// races them. + /// Whether multiple pending requests can progress concurrently. /// - /// Platforms where `send_async` executes each request eagerly before - /// returning (e.g. Cloudflare Workers) return `false`. On such platforms - /// multi-request fan-out runs sequentially and accrues the sum of the - /// individual latencies, so callers with a latency budget (the auction - /// orchestrator) must check this before launching more than one request. + /// Fastly and Axum provide concurrent fan-out. Cloudflare and Spin retain + /// a single-provider contract and return `false`; the auction orchestrator + /// checks this capability before launching more than one request. fn supports_concurrent_fanout(&self) -> bool { true } - /// Whether the adapter enforces a hard total deadline for each request. - /// - /// First-byte and between-byte timers do not qualify: connection setup or a - /// byte-trickling response can still overrun the logical auction budget. The - /// auction collector uses this explicit capability to decide whether an - /// already-completed late response remains eligible. - fn has_enforceable_total_request_deadline(&self) -> bool { - false - } - /// Whether [`send`](Self::send) can preserve upstream response bodies as /// [`Body::Stream`](edgezero_core::body::Body::Stream) when requested via /// [`PlatformHttpRequest::with_stream_response`]. @@ -415,6 +412,80 @@ pub trait PlatformHttpClient: Send + Sync { pending_requests: Vec, ) -> Result>; + /// Collects an available response, waiting only until an absolute deadline. + /// + /// On expiry, implementations take one nonblocking snapshot of complete + /// available responses before relinquishing unfinished requests. This + /// preserves buffered bids when a publisher origin delays collection. + /// Subsequent calls may drain only the completed snapshot. The deadline + /// bounds network waiting, not eligibility by an upstream arrival timestamp. + /// Returns `Ok(None)` when no complete response remains. Both header waits + /// and body reads must be bounded; unfinished work is canceled or abandoned. + /// Fastly cannot cancel backend requests with the installed SDK. + /// Stream-marked requests require the separate streaming wait contract. + /// + /// # Performance + /// + /// Complete response bodies are buffered within adapter-specific limits. + /// Deadline expiry drops incomplete bodies and pending handles after the + /// nonblocking snapshot; complete buffered responses remain collectible. + /// + /// # Examples + /// + /// ```no_run + /// # use trusted_server_core::platform::{PlatformHttpClient, PlatformPendingRequest, PlatformSelectResult, PlatformError}; + /// # use error_stack::Report; + /// # async fn collect(client: &dyn PlatformHttpClient, pending: Vec) -> Result, Report> { + /// let deadline = web_time::Instant::now() + std::time::Duration::from_millis(100); + /// client.select_until(pending, deadline).await + /// # } + /// ``` + /// + /// # Errors + /// + /// Returns [`PlatformError::Unsupported`] unless the adapter implements + /// bounded collection, or an HTTP error when polling itself fails. + async fn select_until( + &self, + pending_requests: Vec, + _deadline: Instant, + ) -> Result, Report> { + drop(pending_requests); + Err(Report::new(PlatformError::Unsupported) + .attach("absolute-deadline HTTP collection is unavailable")) + } + + /// Waits for a complete buffered response before an absolute deadline. + /// + /// Uses [`Self::select_until`] so expiry also relinquishes the pending + /// request and never falls back to an unbounded wait. + /// + /// # Examples + /// + /// ```no_run + /// # use trusted_server_core::platform::{PlatformHttpClient, PlatformPendingRequest, PlatformResponse, PlatformError}; + /// # use error_stack::Report; + /// # async fn collect(client: &dyn PlatformHttpClient, pending: PlatformPendingRequest) -> Result> { + /// let deadline = web_time::Instant::now() + std::time::Duration::from_millis(100); + /// client.wait_until(pending, deadline).await + /// # } + /// ``` + /// + /// # Errors + /// + /// Returns [`PlatformError::DeadlineExceeded`] on expiry, an adapter polling + /// error, or the attributed upstream error in the selected result. + async fn wait_until( + &self, + pending: PlatformPendingRequest, + deadline: Instant, + ) -> Result> { + self.select_until(vec![pending], deadline) + .await? + .ok_or_else(|| Report::new(PlatformError::DeadlineExceeded))? + .ready + } + /// Wait for a single in-flight request to complete. /// /// This is a convenience wrapper around [`select`](Self::select) for the @@ -434,11 +505,69 @@ pub trait PlatformHttpClient: Send + Sync { #[cfg(test)] mod tests { + use std::sync::{ + Arc, + atomic::{AtomicUsize, Ordering}, + }; + use std::time::Duration; + use edgezero_core::body::Body; use edgezero_core::http::request_builder; use super::*; + struct DropProbe(Arc); + + impl Drop for DropProbe { + fn drop(&mut self) { + self.0.fetch_add(1, Ordering::SeqCst); + } + } + + #[test] + fn bounded_collection_default_relinquishes_expired_handles() { + let dropped = Arc::new(AtomicUsize::new(0)); + let response = futures::executor::block_on(UnavailableHttpClient.select_until( + vec![PlatformPendingRequest::new(DropProbe(Arc::clone(&dropped)))], + Instant::now(), + )) + .expect_err("should reject unsupported collection even after expiry"); + assert!(matches!( + response.current_context(), + PlatformError::Unsupported + )); + assert_eq!( + dropped.load(Ordering::SeqCst), + 1, + "should drop abandoned state" + ); + } + + #[test] + fn bounded_collection_default_fails_closed_instead_of_unbounded_polling() { + let error = futures::executor::block_on(UnavailableHttpClient.select_until( + vec![PlatformPendingRequest::new(())], + Instant::now() + Duration::from_secs(1), + )) + .expect_err("should reject unsupported bounded collection"); + assert!( + matches!(error.current_context(), PlatformError::Unsupported), + "should not delegate to unbounded select" + ); + } + + #[test] + fn unsupported_bounded_wait_reports_unsupported() { + let error = futures::executor::block_on( + UnavailableHttpClient.wait_until(PlatformPendingRequest::new(()), Instant::now()), + ) + .expect_err("should reject an unsupported wait"); + assert!( + matches!(error.current_context(), PlatformError::Unsupported), + "should preserve the unsupported collection contract" + ); + } + #[test] fn platform_http_request_cache_bypass_defaults_to_false() { let request = PlatformHttpRequest::new( diff --git a/crates/trusted-server-core/src/platform/mod.rs b/crates/trusted-server-core/src/platform/mod.rs index 71258e9c54..e4733106e3 100644 --- a/crates/trusted-server-core/src/platform/mod.rs +++ b/crates/trusted-server-core/src/platform/mod.rs @@ -39,6 +39,7 @@ mod error; mod http; mod image_optimizer; mod kv; +mod pending_response; mod template_assembly; mod template_cache; #[cfg(test)] @@ -62,6 +63,7 @@ pub use image_optimizer::{ PlatformImageOptimizerParams, PlatformImageOptimizerRegion, }; pub use kv::UnavailableKvStore; +pub use pending_response::PlatformPendingResponse; pub use template_assembly::{ PlatformTemplateAssembler, TemplateAssemblyError, UnavailableTemplateAssembler, contains_publisher_esi_directive, diff --git a/crates/trusted-server-core/src/platform/pending_response.rs b/crates/trusted-server-core/src/platform/pending_response.rs new file mode 100644 index 0000000000..9590578df5 --- /dev/null +++ b/crates/trusted-server-core/src/platform/pending_response.rs @@ -0,0 +1,323 @@ +//! Shared ownership and deadline collection for dispatched runtime operations. + +/// Receives a dispatched operation and cancels it when its owner is dropped. +pub struct PlatformPendingResponse { + response: futures::channel::oneshot::Receiver, + abort: futures::future::AbortHandle, +} + +impl PlatformPendingResponse { + /// Starts an operation with the caller's runtime spawner. + /// + /// The operation publishes only its complete result. Dropping this owner + /// aborts queued or running guest work when the executor next polls it. + /// Runtime-specific transport cancellation remains the caller's responsibility. + /// + /// # Examples + /// + /// ```no_run + /// use futures::task::LocalSpawnExt as _; + /// use trusted_server_core::platform::PlatformPendingResponse; + /// let pool = futures::executor::LocalPool::new(); + /// let pending = PlatformPendingResponse::start(async { 42 }, |task| { + /// pool.spawner().spawn_local(task).expect("should spawn operation"); + /// }); + /// ``` + pub fn start( + operation: impl std::future::Future + 'static, + spawn: impl FnOnce(futures::future::LocalBoxFuture<'static, ()>), + ) -> Self { + let (sender, response) = futures::channel::oneshot::channel(); + let (abort, registration) = futures::future::AbortHandle::new_pair(); + spawn(Box::pin(async move { + if let Ok(result) = futures::future::Abortable::new(operation, registration).await { + let _ = sender.send(result); + } + })); + Self { response, abort } + } + + /// Waits for the complete result while retaining cancellation ownership. + /// + /// # Errors + /// + /// Returns a channel error if the dispatched operation stops without a result. + /// + /// # Examples + /// + /// ```no_run + /// # use trusted_server_core::platform::PlatformPendingResponse; + /// # async fn collect(pending: &mut PlatformPendingResponse) -> Result { + /// let result = pending.wait().await?; + /// # Ok(result) + /// # } + /// ``` + pub async fn wait(&mut self) -> Result { + (&mut self.response).await + } + + /// Collects a buffered result, waiting at most until the supplied timer fires. + /// + /// An initial nonblocking probe retains results after an expired deadline. + /// A final probe handles a result published in the same turn as timer expiry. + /// The caller supplies the runtime timer and a monotonic clock; no transport + /// is polled directly here. Returns `None` when the deadline expires without + /// a complete buffered result. The owner remains alive throughout the wait. + /// + /// # Errors + /// + /// Returns a channel error inside `Some` if the operation stops without a result. + /// + /// # Examples + /// + /// ```no_run + /// # use trusted_server_core::platform::PlatformPendingResponse; + /// # async fn collect(pending: &mut PlatformPendingResponse, timer: impl std::future::Future, deadline: web_time::Instant) { + /// let result = pending.complete_before_deadline(timer, deadline, web_time::Instant::now).await; + /// # } + /// ``` + pub async fn complete_before_deadline( + &mut self, + timer: F, + deadline: web_time::Instant, + now: N, + ) -> Option> + where + F: std::future::Future, + N: Fn() -> web_time::Instant, + { + complete_before_deadline(&mut self.response, timer, deadline, now).await + } +} + +impl Drop for PlatformPendingResponse { + fn drop(&mut self) { + self.abort.abort(); + } +} + +async fn complete_before_deadline( + response: &mut futures::channel::oneshot::Receiver, + timer: T, + deadline: web_time::Instant, + now: N, +) -> Option> +where + T: std::future::Future, + N: Fn() -> web_time::Instant, +{ + // Inspect only the task's buffered result, including when collection starts + // after publisher-origin work has exhausted the auction budget. + match response.try_recv() { + Ok(Some(ready)) => return Some(Ok(ready)), + Err(error) => return Some(Err(error)), + Ok(None) => {} + } + if now() >= deadline { + return None; + } + match futures::future::select(response, Box::pin(timer)).await { + futures::future::Either::Left((ready, _)) => Some(ready), + // The timer and task can become ready in the same executor turn. A final + // channel probe retains buffered results without polling transport I/O. + futures::future::Either::Right(((), response)) => response.try_recv().transpose(), + } +} + +#[cfg(test)] +mod deadline_tests { + use std::cell::Cell; + use std::rc::Rc; + use std::time::Duration; + + use futures::executor::{LocalPool, block_on}; + use futures::future::{pending, ready}; + use futures::task::LocalSpawnExt as _; + use web_time::Instant; + + use super::{PlatformPendingResponse, complete_before_deadline}; + + #[test] + fn dispatched_task_starts_before_collection_and_buffers_complete_body() { + let mut executor = LocalPool::new(); + let spawner = executor.spawner(); + let started = Rc::new(Cell::new(false)); + let started_operation = Rc::clone(&started); + let (finish_body, body) = futures::channel::oneshot::channel(); + let mut pending = PlatformPendingResponse::start( + async move { + started_operation.set(true); + body.await.expect("should receive complete response body") + }, + |task| { + spawner + .spawn_local(task) + .expect("should spawn outbound task") + }, + ); + + executor.run_until_stalled(); + assert!(started.get(), "should start outbound work before selection"); + assert_eq!( + pending.response.try_recv().expect("should poll result"), + None, + "should not expose headers before response body completes" + ); + finish_body.send(42).expect("should finish response body"); + executor.run_until_stalled(); + assert_eq!( + pending + .response + .try_recv() + .expect("should poll buffered result"), + Some(42), + "should preserve a complete buffered response without polling transport" + ); + } + + #[test] + fn dropping_dispatched_pending_cancels_started_body_operation() { + struct DropProbe(Rc>); + impl Drop for DropProbe { + fn drop(&mut self) { + self.0.set(true); + } + } + let mut executor = LocalPool::new(); + let spawner = executor.spawner(); + let dropped = Rc::new(Cell::new(false)); + let probe = DropProbe(Rc::clone(&dropped)); + let pending = PlatformPendingResponse::start( + async move { + let _probe = probe; + futures::future::pending::<()>().await; + }, + |task| { + spawner + .spawn_local(task) + .expect("should spawn outbound task") + }, + ); + executor.run_until_stalled(); + assert!(!dropped.get(), "should retain an active body operation"); + drop(pending); + executor.run_until_stalled(); + assert!( + dropped.get(), + "should drop spawned operation after cancellation" + ); + } + + #[test] + fn dropping_queued_dispatch_prevents_transport_poll() { + let mut executor = LocalPool::new(); + let spawner = executor.spawner(); + let started = Rc::new(Cell::new(false)); + let operation_started = Rc::clone(&started); + let pending = PlatformPendingResponse::start( + async move { + operation_started.set(true); + }, + |task| { + spawner + .spawn_local(task) + .expect("should spawn outbound task") + }, + ); + drop(pending); + executor.run_until_stalled(); + assert!( + !started.get(), + "should cancel queued work before transport starts" + ); + } + + #[test] + fn expired_deadline_preserves_buffered_response() { + let (sender, mut response) = futures::channel::oneshot::channel(); + sender.send(42).expect("should buffer response"); + let deadline = Instant::now(); + let result = block_on(complete_before_deadline( + &mut response, + pending(), + deadline, + || deadline, + )); + assert_eq!( + result, + Some(Ok(42)), + "should retain a buffered response after cutoff" + ); + } + + #[test] + fn expired_deadline_returns_without_waiting_for_response() { + let (_sender, mut response) = futures::channel::oneshot::channel::<()>(); + let deadline = Instant::now(); + let result = block_on(complete_before_deadline( + &mut response, + pending(), + deadline, + || deadline, + )); + assert_eq!(result, None, "should stop waiting at an expired deadline"); + } + + #[test] + fn timer_cuts_off_pending_response() { + let (_sender, mut response) = futures::channel::oneshot::channel::<()>(); + let now = Instant::now(); + let result = block_on(complete_before_deadline( + &mut response, + ready(()), + now + Duration::from_secs(1), + || now, + )); + assert_eq!(result, None, "should cut off an unbuffered response"); + } + + #[test] + fn closed_response_channel_reports_failure() { + let (sender, mut response) = futures::channel::oneshot::channel::<()>(); + drop(sender); + let now = Instant::now(); + let result = block_on(complete_before_deadline( + &mut response, + pending(), + now + Duration::from_secs(1), + || now, + )); + assert!( + matches!(result, Some(Err(_))), + "should preserve a dispatched task failure" + ); + } + + #[test] + fn timer_rechecks_result_published_during_cutoff() { + let (sender, mut response) = futures::channel::oneshot::channel(); + let now = Instant::now(); + let clock = Cell::new(now); + let deadline = now + Duration::from_secs(1); + let timer = async { + clock.set(deadline); + sender.send(42).expect("should publish at cutoff"); + }; + let result = block_on(complete_before_deadline( + &mut response, + timer, + deadline, + || clock.get(), + )); + assert_eq!( + result, + Some(Ok(42)), + "should retain result buffered when timer wins" + ); + assert_eq!( + clock.get(), + deadline, + "should exercise the deadline boundary" + ); + } +} diff --git a/crates/trusted-server-core/src/platform/test_support.rs b/crates/trusted-server-core/src/platform/test_support.rs index 17813c77dc..be0b3feb8f 100644 --- a/crates/trusted-server-core/src/platform/test_support.rs +++ b/crates/trusted-server-core/src/platform/test_support.rs @@ -7,6 +7,7 @@ use base64::{Engine as _, engine::general_purpose}; use ed25519_dalek::SigningKey; use error_stack::{Report, ResultExt as _}; use rand::rngs::OsRng; +use web_time::Instant; use super::{ ClientInfo, GeoInfo, PlatformBackend, PlatformBackendSpec, PlatformCacheIntent, @@ -205,6 +206,7 @@ impl PlatformBackend for StubBackend { /// and resolved by [`StubHttpClient::select`]. struct StubPendingResponse { backend_name: String, + cache_intent: PlatformCacheIntent, status: u16, body: Vec, headers: Vec<(String, String)>, @@ -244,12 +246,11 @@ pub(crate) struct StubHttpClient { pending_backend_name_overrides: Mutex>>, // Test-only wall-clock delays applied before each select result is returned. select_delays: Mutex>, + select_count: std::sync::atomic::AtomicUsize, + wait_count: std::sync::atomic::AtomicUsize, // Reported by supports_concurrent_fanout(); set false to emulate - // platforms whose send_async executes eagerly (e.g. Cloudflare Workers). + // platforms whose collection contract permits only a single provider. concurrent_fanout: std::sync::atomic::AtomicBool, - // Reported by has_enforceable_total_request_deadline(); set true to emulate - // a future adapter with a hard total request deadline. - enforceable_total_request_deadline: std::sync::atomic::AtomicBool, // Reported by supports_streaming_responses(); set true to emulate Fastly's // streaming response support. streaming_responses_supported: std::sync::atomic::AtomicBool, @@ -271,6 +272,16 @@ struct StubHttpResponse { } impl StubHttpClient { + /// Number of calls to the unbounded selection method. + pub(crate) fn recorded_select_count(&self) -> usize { + self.select_count.load(std::sync::atomic::Ordering::Relaxed) + } + + /// Number of calls to the unbounded single-request wait method. + pub(crate) fn recorded_wait_count(&self) -> usize { + self.wait_count.load(std::sync::atomic::Ordering::Relaxed) + } + pub fn new() -> Self { Self { calls: Mutex::new(Vec::new()), @@ -282,8 +293,9 @@ impl StubHttpClient { wait_delays: Mutex::new(VecDeque::new()), pending_backend_name_overrides: Mutex::new(VecDeque::new()), select_delays: Mutex::new(VecDeque::new()), + select_count: std::sync::atomic::AtomicUsize::new(0), + wait_count: std::sync::atomic::AtomicUsize::new(0), concurrent_fanout: std::sync::atomic::AtomicBool::new(true), - enforceable_total_request_deadline: std::sync::atomic::AtomicBool::new(false), streaming_responses_supported: std::sync::atomic::AtomicBool::new(false), pending_streaming_responses_supported: std::sync::atomic::AtomicBool::new(false), image_optimizer_options: Mutex::new(Vec::new()), @@ -301,12 +313,6 @@ impl StubHttpClient { .store(supported, std::sync::atomic::Ordering::Relaxed); } - /// Make `has_enforceable_total_request_deadline()` report the given value. - pub(crate) fn set_enforceable_total_request_deadline(&self, supported: bool) { - self.enforceable_total_request_deadline - .store(supported, std::sync::atomic::Ordering::Relaxed); - } - /// Make `supports_streaming_responses()` report the given value. pub fn set_streaming_responses_supported(&self, supported: bool) { self.streaming_responses_supported @@ -499,11 +505,6 @@ impl PlatformHttpClient for StubHttpClient { .load(std::sync::atomic::Ordering::Relaxed) } - fn has_enforceable_total_request_deadline(&self) -> bool { - self.enforceable_total_request_deadline - .load(std::sync::atomic::Ordering::Relaxed) - } - fn supports_streaming_responses(&self) -> bool { self.streaming_responses_supported .load(std::sync::atomic::Ordering::Relaxed) @@ -597,6 +598,7 @@ impl PlatformHttpClient for StubHttpClient { let edge_response = build_stub_pending_response( StubPendingResponse { backend_name: request.backend_name, + cache_intent: request.cache_intent, status: response.status, body: response.body, headers: response.headers, @@ -700,6 +702,7 @@ impl PlatformHttpClient for StubHttpClient { let pending = StubPendingResponse { backend_name: backend_name.clone(), + cache_intent: request.cache_intent, status: response.status, body: response.body, headers: response.headers, @@ -718,6 +721,86 @@ impl PlatformHttpClient for StubHttpClient { }) } + async fn select_until( + &self, + mut pending_requests: Vec, + deadline: Instant, + ) -> Result, Report> { + // Match Fastly's per-provider rejection before waiting on network work. + let mut unchecked = pending_requests.into_iter(); + let mut checked = Vec::with_capacity(unchecked.len()); + while let Some(pending) = unchecked.next() { + match validate_deadline_cache_bypass(pending) { + Ok(pending) => checked.push(pending), + Err((backend_name, error)) => { + checked.extend(unchecked); + return Ok(Some(PlatformSelectResult { + ready: Err(error), + remaining: checked, + failed_backend_name: backend_name, + })); + } + } + } + pending_requests = checked; + let remaining = deadline.saturating_duration_since(Instant::now()); + let delay = self + .select_delays + .lock() + .expect("should lock select delays") + .front() + .copied() + .unwrap_or_default(); + if delay.is_zero() { + return self.select(pending_requests).await.map(Some); + } + if remaining.is_zero() { + return Ok(None); + } + if delay >= remaining { + self.select_delays + .lock() + .expect("should lock select delays") + .pop_front(); + std::thread::sleep(remaining); + return Ok(None); + } + let response = self.select(pending_requests).await?; + Ok((Instant::now() < deadline).then_some(response)) + } + + async fn wait_until( + &self, + pending: PlatformPendingRequest, + deadline: Instant, + ) -> Result> { + let pending = validate_deadline_cache_bypass(pending).map_err(|(_, error)| error)?; + let remaining = deadline.saturating_duration_since(Instant::now()); + let delay = self + .wait_delays + .lock() + .expect("should lock wait delays") + .front() + .copied() + .unwrap_or_default(); + if delay.is_zero() { + return self.wait(pending).await; + } + if remaining.is_zero() || delay >= remaining { + self.wait_delays + .lock() + .expect("should lock wait delays") + .pop_front(); + std::thread::sleep(remaining); + return Err(Report::new(PlatformError::DeadlineExceeded)); + } + let response = self.wait(pending).await?; + if Instant::now() >= deadline { + return Err(Report::new(PlatformError::DeadlineExceeded)); + } + Ok(response) + } + /// Always marks the first pending request in the input as ready (FIFO order). /// /// This differs from Fastly's production `select()`, which returns whichever @@ -728,6 +811,8 @@ impl PlatformHttpClient for StubHttpClient { &self, mut pending_requests: Vec, ) -> Result> { + self.select_count + .fetch_add(1, std::sync::atomic::Ordering::Relaxed); if pending_requests.is_empty() { return Err(Report::new(PlatformError::HttpClient) .attach("select called with empty pending_requests list")); @@ -803,6 +888,8 @@ impl PlatformHttpClient for StubHttpClient { &self, pending: PlatformPendingRequest, ) -> Result> { + self.wait_count + .fetch_add(1, std::sync::atomic::Ordering::Relaxed); let delay = self .wait_delays .lock() @@ -835,6 +922,37 @@ impl PlatformHttpClient for StubHttpClient { } } +// Require the same request marker as Fastly while retaining wrapper metadata. +fn validate_deadline_cache_bypass( + pending: PlatformPendingRequest, +) -> Result, Report)> { + let backend_name = pending.backend_name().map(str::to_owned); + let stream_response = pending.stream_response(); + let request_method = pending.request_method().cloned(); + let stub = pending.downcast::().map_err(|_| { + ( + backend_name.clone(), + Report::new(PlatformError::HttpClient) + .attach("unexpected pending type in bounded stub collection"), + ) + })?; + if !stub.cache_intent.is_bypass() { + return Err(( + Some(stub.backend_name), + Report::new(PlatformError::Unsupported) + .attach("deadline collection requires a cache-bypass request"), + )); + } + let mut pending = PlatformPendingRequest::new(stub); + if let Some(name) = backend_name { + pending = pending.with_backend_name(name); + } + if let Some(method) = request_method { + pending = pending.with_response_handling(stream_response, method); + } + Ok(pending) +} + fn build_stub_pending_response( stub: StubPendingResponse, stream_response: bool, @@ -1091,6 +1209,123 @@ mod tests { use super::*; + fn dispatch_cache_test_request( + stub: &StubHttpClient, + backend: &str, + bypass: bool, + ) -> PlatformPendingRequest { + stub.push_response(200, b"ready".to_vec()); + let mut request = PlatformHttpRequest::new( + request_builder() + .uri("https://example.com/bid") + .body(Body::empty()) + .expect("should build request"), + backend, + ); + if bypass { + request = request.with_cache_bypass(); + } + futures::executor::block_on(stub.send_async(request)).expect("should dispatch request") + } + + #[test] + fn bounded_stub_collection_rejects_missing_bypass_without_losing_siblings() { + for unsupported_first in [false, true] { + let stub = StubHttpClient::new(); + let unsupported = dispatch_cache_test_request(&stub, "unsupported", false); + let healthy = dispatch_cache_test_request(&stub, "healthy", true); + let selected = futures::executor::block_on(stub.select_until( + if unsupported_first { + vec![unsupported, healthy] + } else { + vec![healthy, unsupported] + }, + Instant::now(), + )) + .expect("should attribute unsupported request") + .expect("should return unsupported result"); + assert!( + matches!( + selected + .ready + .expect_err("should reject missing bypass") + .current_context(), + PlatformError::Unsupported + ), + "should match Fastly's cache bypass requirement" + ); + assert_eq!( + selected.failed_backend_name.as_deref(), + Some("unsupported"), + "should identify unsupported provider" + ); + let selected = + futures::executor::block_on(stub.select_until(selected.remaining, Instant::now())) + .expect("should collect sibling") + .expect("should retain ready sibling"); + assert_eq!( + selected + .ready + .expect("should accept bypassed request") + .backend_name + .as_deref(), + Some("healthy"), + "should preserve healthy provider" + ); + } + } + + #[test] + fn bounded_stub_wait_rejects_missing_bypass() { + let stub = StubHttpClient::new(); + let pending = dispatch_cache_test_request(&stub, "unsupported", false); + let error = futures::executor::block_on(stub.wait_until(pending, Instant::now())) + .expect_err("should reject missing bypass"); + assert!( + matches!(error.current_context(), PlatformError::Unsupported), + "should reject unsupported deadline wait" + ); + } + + #[test] + fn unbounded_stub_wait_accepts_default_cache_intent() { + let stub = StubHttpClient::new(); + let pending = dispatch_cache_test_request(&stub, "ordinary", false); + let response = futures::executor::block_on(stub.wait(pending)) + .expect("should preserve ordinary waits"); + assert_eq!( + response.backend_name.as_deref(), + Some("ordinary"), + "should preserve backend" + ); + } + + #[test] + fn bounded_stub_wait_preserves_an_already_ready_response_after_expiry() { + let stub = StubHttpClient::new(); + stub.push_response(200, b"ready".to_vec()); + let pending = futures::executor::block_on( + stub.send_async( + PlatformHttpRequest::new( + request_builder() + .uri("https://example.com/ready") + .body(Body::empty()) + .expect("should build request"), + "ready-backend", + ) + .with_cache_bypass(), + ), + ) + .expect("should dispatch ready response"); + let response = futures::executor::block_on(stub.wait_until(pending, Instant::now())) + .expect("should retain the ready response after expiry"); + assert_eq!( + response.response.status().as_u16(), + 200, + "should preserve ready status" + ); + } + #[test] fn stub_http_client_records_send_calls_and_returns_canned_response() { let stub = StubHttpClient::new(); diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index e534b50a90..a670d31416 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -4655,13 +4655,14 @@ async fn collect_non_html_auction( .and_then(|_| diagnostics_auction_id(settings)) }); let placeholder = mediator_placeholder_request(); - let result = orchestrator - .collect_dispatched_auction( - dispatched, - services, - &make_collect_context(settings, services, &placeholder), - ) - .await; + // Auction collection nests deadline and trace futures. Keep its future off + // the WASI call stack. + let result = Box::pin(orchestrator.collect_dispatched_auction( + dispatched, + services, + &make_collect_context(settings, services, &placeholder), + )) + .await; let delivered_winner_slots = write_bids_to_state( &result.winning_bids, params.price_granularity, @@ -4716,9 +4717,10 @@ async fn collect_stream_auction( log::info!("body_close_hold_loop: collecting dispatched auction before held body tail"); let placeholder = mediator_placeholder_request(); let collect_ctx = make_collect_context(settings, services, &placeholder); - let result = orchestrator - .collect_dispatched_auction(dispatched, services, &collect_ctx) - .await; + // Auction collection nests deadline and trace futures. Keep its future off + // the WASI call stack. + let result = + Box::pin(orchestrator.collect_dispatched_auction(dispatched, services, &collect_ctx)).await; log::info!( "body_close_hold_loop: collect complete - {} winning bid(s)", result.winning_bids.len() @@ -5668,8 +5670,8 @@ pub async fn handle_publisher_request( } } - // SSP requests are already racing through the platform HTTP client, so - // origin TTFB tracks origin latency rather than the auction timeout. + // Provider operations are already running while the origin loads. Collection + // retains buffered results, then waits only for the original remaining budget. let origin_result = if let Some(pending) = pending_origin { services.http_client().wait(pending).await } else { @@ -12011,7 +12013,7 @@ mod tests { context .services .http_client() - .send_async(request) + .send_async(request.with_cache_bypass()) .await .change_context(TrustedServerError::Auction { message: "stub bid launch failed".to_string(), @@ -19776,7 +19778,7 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { context .services .http_client() - .send_async(request) + .send_async(request.with_cache_bypass()) .await .change_context(TrustedServerError::Auction { message: "test provider launch failed".to_string(), @@ -26225,6 +26227,8 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { ); } + // Manually polled fixture for publisher injection order, not timer scheduling. + // Adapter runtime tests cover autonomous deadline wakeups and cancellation. struct GatedAuctionHttpClient { inner: StubHttpClient, released: std::sync::atomic::AtomicBool, @@ -26249,6 +26253,33 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { self.inner.send_async(request).await } + async fn select_until( + &self, + pending: Vec, + deadline: web_time::Instant, + ) -> Result< + Option, + Report, + > { + self.collections.fetch_add(1, Ordering::SeqCst); + let ready = futures::future::poll_fn(|_| { + // The gate models completion, so probe it before checking expiry. + if self.released.load(Ordering::SeqCst) { + std::task::Poll::Ready(true) + } else if web_time::Instant::now() >= deadline { + std::task::Poll::Ready(false) + } else { + std::task::Poll::Pending + } + }) + .await; + if ready { + self.inner.select_until(pending, deadline).await + } else { + Ok(None) + } + } + async fn select( &self, pending: Vec, @@ -26283,13 +26314,16 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { context .services .http_client() - .send_async(crate::platform::PlatformHttpRequest::new( - Request::builder() - .uri("https://bidder.example.com/bid") - .body(EdgeBody::empty()) - .expect("should build test bid request"), - "seam-test", - )) + .send_async( + crate::platform::PlatformHttpRequest::new( + Request::builder() + .uri("https://bidder.example.com/bid") + .body(EdgeBody::empty()) + .expect("should build test bid request"), + "seam-test", + ) + .with_cache_bypass(), + ) .await .change_context(TrustedServerError::Auction { message: "Failed to dispatch test auction".to_string(), @@ -28999,7 +29033,7 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { context .services .http_client() - .send_async(request) + .send_async(request.with_cache_bypass()) .await .change_context(TrustedServerError::Auction { message: "test bidder launch failed".to_string(), diff --git a/crates/trusted-server-core/src/test_support.rs b/crates/trusted-server-core/src/test_support.rs index cb2eda85c2..37d54614c0 100644 --- a/crates/trusted-server-core/src/test_support.rs +++ b/crates/trusted-server-core/src/test_support.rs @@ -471,6 +471,15 @@ pub mod nextjs_auction { Ok(PlatformPendingRequest::new(request).with_backend_name(backend)) } + async fn select_until( + &self, + pending_requests: Vec, + _deadline: web_time::Instant, + ) -> Result, Report> { + // All fixture responses are deterministic and already available. + self.select(pending_requests).await.map(Some) + } + async fn select( &self, mut pending_requests: Vec, diff --git a/docs/guide/ad-serving.md b/docs/guide/ad-serving.md index 171764bc18..7cd3b03b78 100644 --- a/docs/guide/ad-serving.md +++ b/docs/guide/ad-serving.md @@ -68,8 +68,11 @@ assets at runtime. end-to-end checks. - Confirm provider authorization and test inventory with each upstream before enabling traffic. -- Treat provider timeout values as logical budgets; no adapter currently - promises an abortable provider-wide wall-clock deadline. +- Collection stops network waiting at the auction deadline measured from dispatch. + Complete responses already available remain eligible; unfinished providers time + out and mediation is skipped when no budget remains. Fastly abandons unfinished + requests; other adapters cancel owned request tasks. See + [Timeout behavior](/guide/auction-orchestration#timeout-behavior). See also [Creative processing](/guide/creative-processing) and the [Integrations overview](/guide/integrations-overview). diff --git a/docs/guide/architecture.md b/docs/guide/architecture.md index a6145f80d3..fb02fe2e68 100644 --- a/docs/guide/architecture.md +++ b/docs/guide/architecture.md @@ -170,9 +170,13 @@ The workspace has multiple WASM runtimes with runtime-specific SDKs. Use target- Fastly and Axum support concurrent auction provider fan-out. Cloudflare and Spin currently accept at most one provider in an enabled auction. Every adapter runs -target-aware fan-out and backend-name checks at startup. No current adapter -claims an abortable provider-wide total-request deadline, so configured auction -and provider timeouts are logical budgets rather than hard wall-clock ceilings. +target-aware fan-out and backend-name checks at startup. Auction collection uses +an absolute cutoff for network waiting across headers and complete response bodies. +A nonblocking snapshot retains complete results after a slow origin. Fastly +abandons pending backend requests because its SDK cannot cancel them; other +adapters cancel their owned operations at cutoff. See +[Auction timeout behavior](/guide/auction-orchestration#timeout-behavior) for +scheduling, worker-runtime dispatch, and local processing limitations. ## Next Steps diff --git a/docs/guide/auction-orchestration.md b/docs/guide/auction-orchestration.md index afde126b71..c377157ce5 100644 --- a/docs/guide/auction-orchestration.md +++ b/docs/guide/auction-orchestration.md @@ -826,15 +826,41 @@ Until that callback lands, startup remains the mandatory target-aware gate. ### Timeout behavior -For each provider, Trusted Server uses the smaller of its resolved timeout and -the remaining auction budget for launch decisions and OpenRTB `tmax`. The -mediator is not launched after the logical auction budget is exhausted. - -No current adapter claims an abortable provider-wide total-request deadline. -Already-launched work may complete after the logical budget, and a completed -late response can remain eligible. Local decision and delivery also finish -after network launch closes, so `timeout_ms` is not a hard wall-clock ceiling -and an auction can exceed it. +Auction requests start at dispatch and progress while the publisher origin loads. +Response collection waits for headers and complete upstream bodies only until the +original absolute deadline. At expiry, one nonblocking snapshot preserves complete +responses already available; unfinished providers become timeouts. A slow origin +never restarts the budget or causes an already-buffered bid to be discarded. +Fastly has no upstream arrival timestamp, so this is a bound on network waiting, +not bid eligibility based on an exact server-side arrival time. A mediator starts +only with remaining budget and uses the earlier of the auction cutoff and its own +logical timeout. + +Provider bids that are already complete at the final nonblocking snapshot remain +eligible even after `tmax`; mediation starts only with remaining budget and +rejects responses collected after its cutoff. + +Personalized auction and mediator requests explicitly bypass intermediary caches. +Fastly uses the SDK's direct pending-request handles and body readiness handles in +a timed wait. Retained partial payloads share a 10 MiB budget. Under memory pressure, +the largest partial response is rejected with its own backend attribution so a +large stalled provider cannot make a small healthy sibling take the failure. +Completed payloads move directly into the platform body without a host-buffer +round trip. This cap covers retained payloads, not every allocator or host buffer. The final nonblocking body drain is bounded by this payload limit; its local CPU work can extend beyond the network cutoff. + +On expiry Fastly closes acquired response bodies and abandons unfinished requests; +the installed SDK has no pending-request cancellation API. Axum aborts outstanding +request tasks. Cloudflare and Spin start owned background tasks at dispatch and +retain abort handles until response collection completes. Cloudflare also aborts +its fetch through an AbortController. Both workers retain their single-provider +restriction. Dropping a pending handle cancels queued or running guest work; +upstream side effects that already happened cannot be undone. + +Backend first-byte and between-byte timeouts remain transport safeguards rather +than total deadlines. Timer granularity and runtime scheduling can add a small +overshoot. Parsing, ranking, HTML processing, and delivery are local work outside +network collection, so this cutoff is not a hard ceiling on the whole publisher +response time. Browser Prebid `timeout_ms` and `debug` stay under `[integrations.prebid]` and are independent of all server provider values. Server endpoint, timeout, diff --git a/docs/guide/auction-testing.md b/docs/guide/auction-testing.md index b92b78b0fc..b317c584d3 100644 --- a/docs/guide/auction-testing.md +++ b/docs/guide/auction-testing.md @@ -68,3 +68,25 @@ are part of the contract rather than skipped coverage. See [Auction orchestration](/guide/auction-orchestration) for configuration and response details. + +## Absolute deadline regressions + +Core tests cover split dispatch/collection and synchronous execution, pending +provider attribution, preserved on-time bids, late collection after an origin +wait, and exhausted-budget mediation. Axum loopback tests separately delay +headers and trickle bodies, and verify outstanding task cancellation. Worker +adapter tests use controlled futures and timers. CI also runs the actual worker +HTTP clients under local workerd and Spin 4.1, covering eager dispatch, buffered +bids after a slow origin, delayed headers, and trickling bodies. Run +`node scripts/test-worker-auction-deadlines.mjs --help` for the same local fixtures. +Fastly also has Viceroy network fixtures for stalled headers, trickling bodies, +bodyless responses, and competing bidders. Its local origin harness builds the +adapter test binary itself: + +```bash +node crates/trusted-server-adapter-fastly/tests/deadline-origin.mjs "$(command -v viceroy)" +``` + +The harness binds a local ephemeral port and runs the five network tests +excluded from the standard suite; CI runs the same command. To reuse a prebuilt +test artifact, pass its `.wasm` path as a second argument. diff --git a/docs/guide/configuration.md b/docs/guide/configuration.md index 7a164260fd..1a875e02ed 100644 --- a/docs/guide/configuration.md +++ b/docs/guide/configuration.md @@ -2182,7 +2182,7 @@ provider or bidder route. | `sanitize_creatives` | Boolean | `false` | Strip executable markup from winning-bid `adm` before delivery | | `rewrite_creatives` | Boolean | `true` | Rewrite winning-bid asset URLs through first-party endpoints | | `rewrite_clicks` | Boolean | unset | Wrap creative links in `/first-party/click`; unset follows `rewrite_creatives` (proxied HTML keeps wrapping) | -| `timeout_ms` | Integer | `2000` | Logical auction budget in milliseconds | +| `timeout_ms` | Integer | `2000` | Deadline for additional auction network waiting, measured from dispatch | | `mediator` | String | `None` | Optional separate `adserver_mock` mediator | | `creative_store` | String | `"creative_store"` | Deprecated; creatives are delivered inline | | `allowed_context_keys` | Array | `[]` | Request context keys admitted into the auction | @@ -2430,13 +2430,19 @@ push as proof that a Cloudflare or Spin multi-provider plan can start. ### Deadline behavior -Configured timeouts are logical budgets, not hard wall-clock guarantees. No -current adapter claims an abortable provider-wide total-request deadline. -Already-launched work may complete after the logical budget and a completed late -response can remain eligible. Once the logical auction budget is exhausted, -Trusted Server starts no additional provider or mediator network work, then -finishes local decision and delivery. An auction can therefore exceed its -configured wall-clock timeout. +Auction response collection waits only until an absolute deadline measured from +dispatch, including headers and complete response bodies. One nonblocking snapshot +retains complete available responses after a slow origin; unfinished providers +become timeouts and mediation cannot start with an exhausted budget. Backend +timeout rounding does not extend network waiting. +Provider bids that are already complete at the final nonblocking snapshot remain +eligible even after `tmax`; mediation starts only with remaining budget and +rejects responses collected after its cutoff. + +Fastly abandons pending requests; Axum, Cloudflare, and Spin cancel outstanding +operations where supported. Local processing and delivery can finish after the +network cutoff. See [Timeout behavior](/guide/auction-orchestration#timeout-behavior) +for runtime semantics and limitations. Creative sanitization is opt-in. `sanitize_creatives = true` strips executable markup before delivery. `rewrite_creatives = false` leaves asset URLs diff --git a/docs/superpowers/plans/2026-10-22-absolute-auction-deadlines.md b/docs/superpowers/plans/2026-10-22-absolute-auction-deadlines.md new file mode 100644 index 0000000000..2e3a15dd1e --- /dev/null +++ b/docs/superpowers/plans/2026-10-22-absolute-auction-deadlines.md @@ -0,0 +1,54 @@ +# Absolute auction collection deadlines + +## Contract + +`PlatformHttpClient::select_until` limits additional network waiting to one +absolute deadline measured from auction dispatch. At expiry the adapter takes +one nonblocking snapshot of complete available responses, drops unfinished +operations, and permits the caller to drain only that completed snapshot. +This retains buffered bids when the publisher origin delayed collection. It +is not an upstream arrival-timestamp admission rule. + +Every configured provider receives exactly one attributed outcome. Completed +bids remain eligible for local ranking. Unfinished responses become timeouts; +polling or transport errors remain transport failures. Mediation cannot launch +without budget and its wait uses the earlier of the auction deadline and its +own logical budget, independently of rounded transport timeouts. + +## Adapter implementation + +- Axum starts whole-response tasks at dispatch. Selection races their completion + against the remaining deadline. The expired snapshot retains only finished + tasks; dropping other handles aborts them. +- Cloudflare and Spin spawn cancelable full-response operations at dispatch and + retain buffered results in channels. Selection probes the channel before + waiting against the runtime timer. Pending ownership includes an abort handle; + Cloudflare's fetch also owns an AbortController. Single-provider restrictions + remain unchanged. +- Fastly explicitly bypasses caching for personalized auction requests, allowing + conversion to the SDK's direct pending-request handle. Timed readiness waits + contain real request/body handles. Header polling and body reads are + nonblocking; ready bodies advance fairly. Expired collection caches one + nonblocking snapshot and never waits for more bytes. The SDK cannot cancel + pending backend requests, so expiration abandons them and closes acquired bodies. +- Fastly partial response payloads share a 10 MiB limit. Capacity pressure rejects + the largest partial response with backend attribution. Completed vectors move + directly into platform bodies. The limit does not include allocator overhead + or runtime-owned host buffers. + +## Validation + +Regression coverage must exercise production split dispatch/collection, complete +buffered bids after a slow origin, provider count/order/elapsed time and timeout +attribution, exhausted-budget mediation, and the mediator's independent budget. +Adapters additionally cover delayed headers, trickling bodies, cancellation or +abandonment, bodyless response semantics, fair collection, and memory pressure. + +The Fastly slow-origin harness runs automatically in CI. Worker runtime fixtures +use the actual HTTP adapters under local workerd and Spin to verify eager dispatch, +complete buffering, deadline waits, and cancellation. Native helper tests and +WASM compilation supplement these runtime cases. + +Local parsing, ranking, HTML transformation, and delivery remain outside the +network-wait cutoff. Runtime scheduling and timer granularity can add a small +overshoot; already-triggered upstream side effects cannot be undone. diff --git a/scripts/test-worker-auction-deadlines.mjs b/scripts/test-worker-auction-deadlines.mjs new file mode 100644 index 0000000000..f3cbad36ab --- /dev/null +++ b/scripts/test-worker-auction-deadlines.mjs @@ -0,0 +1,309 @@ +#!/usr/bin/env node +// Exercise production HTTP clients against delayed local origins in worker runtimes. +import assert from 'node:assert/strict' +import { execFile, spawn } from 'node:child_process' +import { open, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises' +import http from 'node:http' +import net from 'node:net' +import os from 'node:os' +import path from 'node:path' +import { setTimeout as delay } from 'node:timers/promises' +import { fileURLToPath } from 'node:url' +import { parseArgs, promisify } from 'node:util' + +const { values, positionals } = parseArgs({ + allowPositionals: true, + options: { + help: { type: 'boolean' }, + runtime: { type: 'string' }, + 'wasm-bindgen': { type: 'string', default: 'wasm-bindgen' }, + 'target-dir': { type: 'string' }, + }, +}) +if (values.help) { + console.log( + 'Usage: node scripts/test-worker-auction-deadlines.mjs --runtime [--wasm-bindgen ] [--target-dir ]' + ) + process.exit(0) +} +const [adapter] = positionals +assert( + positionals.length === 1 && ['cloudflare', 'spin'].includes(adapter), + 'Supply cloudflare or spin as the adapter' +) +assert( + values.runtime, + 'Supply --runtime with the local worker runtime executable' +) +const interrupted = new AbortController() +const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..') +const artifact = path.resolve( + values['target-dir'] ?? path.join(root, 'target'), + adapter === 'cloudflare' ? 'wasm32-unknown-unknown' : 'wasm32-wasip1', + 'debug/examples/deadline_runtime.wasm' +) + +async function listen(server) { + await new Promise((resolve, reject) => { + server.once('error', reject) + server.listen(0, '127.0.0.1', resolve) + }) + return server.address().port +} + +async function closeServer(server) { + server.closeAllConnections?.() + await new Promise((resolve) => server.close(resolve)) +} + +async function availablePort() { + const server = net.createServer() + const port = await listen(server) + await closeServer(server) + return port +} + +function connects(port) { + return new Promise((resolve) => { + const socket = net.createConnection({ host: '127.0.0.1', port }) + const done = (result) => { + socket.destroy() + resolve(result) + } + socket.setTimeout(100) + socket.once('connect', () => done(true)) + socket.once('error', () => done(false)) + socket.once('timeout', () => done(false)) + }) +} + +function get(uri) { + return new Promise((resolve, reject) => { + const request = http.get( + uri, + { signal: interrupted.signal }, + (response) => { + const chunks = [] + response.on('data', (chunk) => chunks.push(chunk)) + response.on('error', reject) + response.on('end', () => { + if (response.statusCode !== 200) { + reject( + new Error( + `Fixture returned ${response.statusCode}: ${Buffer.concat(chunks)}` + ) + ) + } else { + resolve(Buffer.concat(chunks).toString()) + } + }) + } + ) + request.on('error', reject) + request.setTimeout(5000, () => + request.destroy(new Error('Fixture request timed out')) + ) + }) +} + +const observed = new Set() +const origin = http.createServer(async (request, response) => { + const cancellation = new AbortController() + response.once('close', () => cancellation.abort()) + response.on('error', () => {}) + observed.add(request.url) + try { + if (request.url === '/headers') { + await delay(700, undefined, { signal: cancellation.signal }) + } + if (request.url === '/large') { + const body = Buffer.alloc(8 * 1024 * 1024 + 1) + response.writeHead(200, { 'Content-Length': String(body.length) }) + response.end(body) + return + } + response.writeHead(200, { 'Content-Length': '4' }) + if (request.url === '/body') { + response.write('a') + await delay(700, undefined, { signal: cancellation.signal }) + response.end('bcd') + } else { + response.end('okay') + } + } catch (error) { + if (!cancellation.signal.aborted) response.destroy(error) + } +}) + +let workspace +let log +let child +let childStopped +let childError +const grouped = process.platform !== 'win32' +function signalRuntime(signal) { + if (!child?.pid) return + try { + if (grouped) process.kill(-child.pid, signal) + else child.kill(signal) + } catch (error) { + if (error.code !== 'ESRCH') throw error + } +} + +const interrupt = (signal) => { + interrupted.abort(new Error(`Harness interrupted by ${signal}`)) + signalRuntime('SIGTERM') +} +process.on('SIGINT', interrupt) +process.on('SIGTERM', interrupt) + +try { + const originPort = await listen(origin) + const port = await availablePort() + workspace = await mkdtemp(path.join(os.tmpdir(), 'ts-worker-deadline-')) + let argumentsForRuntime + if (adapter === 'cloudflare') { + await promisify(execFile)( + values['wasm-bindgen'], + [ + artifact, + '--target', + 'web', + '--out-dir', + workspace, + '--out-name', + 'fixture', + ], + { timeout: 60_000, signal: interrupted.signal } + ) + await writeFile( + path.join(workspace, 'index.js'), + 'import module from "./fixture_bg.wasm";\nimport {initSync, fetch} from "./fixture.js";\ninitSync({module});\nexport default {fetch};\n' + ) + const configuration = path.join(workspace, 'wrangler.toml') + await writeFile( + configuration, + 'name = "deadline-fixture"\nmain = "index.js"\ncompatibility_date = "2026-04-01"\n' + ) + argumentsForRuntime = [ + 'dev', + '--local', + '--port', + String(port), + '--ip', + '127.0.0.1', + '--config', + configuration, + ] + } else { + const configuration = path.join(workspace, 'spin.toml') + await writeFile( + configuration, + `spin_manifest_version = 2 +[application] +name = "deadline-fixture" +version = "0.1.0" +[[trigger.http]] +route = "/..." +component = "fixture" +[component.fixture] +source = ${JSON.stringify(artifact)} +allowed_outbound_hosts = ["http://127.0.0.1:*"] +` + ) + argumentsForRuntime = [ + 'up', + '--from', + configuration, + '--listen', + `127.0.0.1:${port}`, + ] + } + log = await open(path.join(workspace, 'runtime.log'), 'w') + child = spawn(values.runtime, argumentsForRuntime, { + cwd: workspace, + stdio: ['ignore', log.fd, log.fd], + detached: grouped, + }) + childStopped = new Promise((resolve) => { + child.once('exit', resolve) + child.once('error', (error) => { + childError = error + resolve() + }) + }) + let ready = false + for (let attempt = 0; attempt < 300; attempt++) { + interrupted.signal.throwIfAborted() + if (childError) throw childError + assert( + child.exitCode === null && child.signalCode === null, + 'Worker exited during startup' + ) + if (await connects(port)) { + ready = true + break + } + await delay(100, undefined, { signal: interrupted.signal }) + } + assert(ready, 'Worker did not listen') + const cases = [ + ['/fast', 80, 200, 'complete', 500], + ['/headers', 80, 0, 'timeout', 350], + ['/body', 80, 0, 'timeout', 350], + ['/headers', 80, 200, 'timeout', 500], + ] + if (adapter === 'spin') cases.push(['/large', 2000, 0, 'oversized', 2500]) + assert.equal( + cases.length, + adapter === 'spin' ? 5 : 4, + 'should exercise every runtime case' + ) + for (const [endpoint, budget, originDelay, expected, maximum] of cases) { + observed.delete(endpoint) + const uri = `http://127.0.0.1:${port}/?upstream=http://127.0.0.1:${originPort}${endpoint}&budget_ms=${budget}&origin_ms=${originDelay}` + const result = await get(uri) + const [status, elapsed] = result.split(':') + assert.equal( + status, + expected, + `${endpoint} should return ${expected}: ${result}` + ) + assert( + Number.isFinite(Number(elapsed)) && Number(elapsed) < maximum, + `${endpoint} should complete within ${maximum} ms: ${result}` + ) + assert( + observed.has(endpoint), + `${endpoint} request should start at dispatch` + ) + console.log( + `PASS ${adapter} ${endpoint} budget=${budget} origin=${originDelay}: ${result}` + ) + } +} catch (error) { + if (log) + console.error(await readFile(path.join(workspace, 'runtime.log'), 'utf8')) + throw error +} finally { + try { + if (child?.pid) { + signalRuntime('SIGTERM') + const timeout = new AbortController() + await Promise.race([ + childStopped, + delay(10_000, undefined, { signal: timeout.signal }), + ]) + timeout.abort() + signalRuntime('SIGKILL') + await childStopped + } + } finally { + await log?.close() + if (workspace) await rm(workspace, { recursive: true, force: true }) + await closeServer(origin) + process.off('SIGINT', interrupt) + process.off('SIGTERM', interrupt) + } +}