diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index ca9d35ad41..8500af3508 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -1,4 +1,4 @@ -name: "Integration Tests" +name: 'Integration Tests' permissions: contents: read @@ -30,11 +30,15 @@ jobs: uses: ./.github/actions/setup-integration-test-env with: origin-port: ${{ env.ORIGIN_PORT }} - install-viceroy: "false" - build-cloudflare: "true" + install-viceroy: 'false' + build-cloudflare: 'true' - name: Generate integration Viceroy configs - run: ./scripts/generate-integration-viceroy-configs.sh + run: | + ./scripts/generate-integration-viceroy-configs.sh + trace_bidder_origin=http://127.0.0.1 + INTEGRATION_APP_CONFIG_PATH=crates/trusted-server-integration-tests/fixtures/configs/trusted-server.trace.toml INTEGRATION_BIDDER_ORIGIN_URL="$trace_bidder_origin:$ORIGIN_PORT" ARTIFACTS_DIR="$ARTIFACTS_DIR/trace" ./scripts/generate-integration-viceroy-configs.sh + INTEGRATION_APP_CONFIG_PATH=crates/trusted-server-integration-tests/fixtures/configs/trusted-server.trace-auth.toml ARTIFACTS_DIR="$ARTIFACTS_DIR/trace-auth" ./scripts/generate-integration-viceroy-configs.sh env: INTEGRATION_ORIGIN_PORT: ${{ env.ORIGIN_PORT }} @@ -68,10 +72,10 @@ jobs: uses: ./.github/actions/setup-integration-test-env with: origin-port: ${{ env.ORIGIN_PORT }} - install-viceroy: "true" - build-wasm: "false" - build-axum: "false" - build-test-images: "false" + install-viceroy: 'true' + build-wasm: 'false' + build-axum: 'false' + build-test-images: 'false' - name: Download integration test artifacts uses: actions/download-artifact@v4 @@ -100,6 +104,7 @@ jobs: --target x86_64-unknown-linux-gnu -- --include-ignored --skip test_wordpress_fastly --skip test_nextjs_fastly + --skip trace_browser_workflow --skip trace_runtime_boundary --test-threads=1 env: WASM_BINARY_PATH: ${{ env.WASM_ARTIFACT_PATH }} @@ -122,9 +127,9 @@ jobs: uses: ./.github/actions/setup-integration-test-env with: origin-port: ${{ env.ORIGIN_PORT }} - install-viceroy: "true" - build-wasm: "false" - build-test-images: "false" + install-viceroy: 'true' + build-wasm: 'false' + build-test-images: 'false' - name: Download integration test artifacts uses: actions/download-artifact@v4 @@ -147,6 +152,97 @@ jobs: VICEROY_CONFIG_PATH: ${{ env.ARTIFACTS_DIR }}/configs/viceroy.toml RUST_LOG: info + trace-runtime-tests: + name: trace runtime and browser acceptance + needs: prepare-artifacts + # Pinned for the same reason as the browser integration tests job below: + # Playwright cannot install chromium on the ubuntu-latest 26.04 image. + runs-on: ubuntu-24.04 + timeout-minutes: 45 + steps: + - uses: actions/checkout@v4 + + - name: Set up trace test runtime + id: shared-setup + uses: ./.github/actions/setup-integration-test-env + with: + origin-port: ${{ env.ORIGIN_PORT }} + install-viceroy: 'true' + build-wasm: 'false' + build-axum: 'false' + build-test-images: 'false' + + - name: Download integration test artifacts + uses: actions/download-artifact@v4 + with: + name: integration-test-artifacts + path: ${{ env.ARTIFACTS_DIR }} + + - name: Restore runtime binaries and framework images + run: | + chmod +x "$AXUM_ARTIFACT_PATH" + mkdir -p crates/trusted-server-adapter-cloudflare/build + cp -r "$CF_BUILD_ARTIFACT_PATH/." crates/trusted-server-adapter-cloudflare/build/ + docker load --input "$DOCKER_ARTIFACT_PATH" + + - name: Set up Node.js + uses: actions/setup-node@v4 + with: + node-version: ${{ steps.shared-setup.outputs.node-version }} + cache: npm + cache-dependency-path: | + crates/trusted-server-integration-tests/browser/package-lock.json + crates/trusted-server-js/lib/package-lock.json + + - name: Install browser dependencies + run: | + npm ci --prefix crates/trusted-server-js/lib + npm ci --prefix crates/trusted-server-integration-tests/browser + cd crates/trusted-server-integration-tests/browser + npx playwright install --with-deps chromium + npm install -g wrangler@4.83.0 + + - name: Install the verified Spin runtime + run: | + spin_version=$(awk '$1 == "spin" { print $2; exit }' .tool-versions) + case "$spin_version" in + 4.1.0) spin_sha256=aeeb892f94cf861f5302546faa0bdaa82cc4be30539621299746a6dc011f3b5a ;; + *) echo "::error::No verified Spin checksum for pin '$spin_version'; update the installer with .tool-versions"; exit 1 ;; + esac + trace_spin_dir="$RUNNER_TEMP/trace-spin" + mkdir -p "$trace_spin_dir" + curl --fail --location --silent --show-error "https://github.com/spinframework/spin/releases/download/v${spin_version}/spin-v${spin_version}-linux-amd64.tar.gz" --output "$trace_spin_dir/spin.tar.gz" + echo "$spin_sha256 $trace_spin_dir/spin.tar.gz" | sha256sum --check --strict + tar -xzf "$trace_spin_dir/spin.tar.gz" -C "$trace_spin_dir" spin + installed_version=$("$trace_spin_dir/spin" --version) + if [[ "$installed_version" != "spin $spin_version "* ]]; then + echo "::error::Expected Spin $spin_version from .tool-versions; found $installed_version" + exit 1 + fi + echo "$installed_version" + echo "$trace_spin_dir" >> "$GITHUB_PATH" + + - name: Build the production Spin component + run: cargo build --package trusted-server-adapter-spin --target wasm32-wasip1 --features spin --release + + - name: Run trace acceptance against prepared runtimes + run: | + cargo test --manifest-path crates/trusted-server-integration-tests/Cargo.toml --target x86_64-unknown-linux-gnu --test integration trace_runtime_boundary -- --ignored --test-threads=1 --nocapture + cargo test --manifest-path crates/trusted-server-integration-tests/Cargo.toml --target x86_64-unknown-linux-gnu --test integration trace_browser_workflow -- --ignored --test-threads=1 --nocapture + env: + WASM_BINARY_PATH: ${{ env.WASM_ARTIFACT_PATH }} + AXUM_BINARY_PATH: ${{ env.AXUM_ARTIFACT_PATH }} + CLOUDFLARE_WRANGLER_DIR: ${{ github.workspace }}/crates/trusted-server-adapter-cloudflare + INTEGRATION_ORIGIN_PORT: ${{ env.ORIGIN_PORT }} + + - name: Upload trace browser evidence + uses: actions/upload-artifact@v4 + if: always() + with: + name: trace-runtime-browser-evidence + path: crates/trusted-server-integration-tests/browser/test-results/trace-runtime-*/ + retention-days: 7 + browser-tests: name: browser integration tests needs: prepare-artifacts @@ -166,9 +262,9 @@ jobs: uses: ./.github/actions/setup-integration-test-env with: origin-port: ${{ env.ORIGIN_PORT }} - install-viceroy: "true" - build-wasm: "false" - build-test-images: "false" + install-viceroy: 'true' + build-wasm: 'false' + build-test-images: 'false' - name: Download integration test artifacts uses: actions/download-artifact@v4 @@ -205,6 +301,10 @@ jobs: working-directory: crates/trusted-server-integration-tests/browser run: node --test initial-render/pages.test.cjs + - name: Test trace fixture cleanup + working-directory: crates/trusted-server-integration-tests/browser + run: node --test trace-fixture.test.cjs + - name: Test initial render ownership working-directory: crates/trusted-server-integration-tests/browser # Let the runner build its Rubicon + Shared ID artifact; the bidder-only @@ -226,6 +326,8 @@ jobs: WASM_BINARY_PATH: ${{ env.WASM_ARTIFACT_PATH }} INTEGRATION_ORIGIN_PORT: ${{ env.ORIGIN_PORT }} VICEROY_CONFIG_PATH: ${{ env.ARTIFACTS_DIR }}/configs/viceroy.toml + TRACE_VICEROY_CONFIG_PATH: ${{ env.ARTIFACTS_DIR }}/trace/configs/viceroy.toml + TRACE_AUTH_VICEROY_CONFIG_PATH: ${{ env.ARTIFACTS_DIR }}/trace-auth/configs/viceroy.toml TEST_FRAMEWORK: nextjs PLAYWRIGHT_HTML_REPORT: playwright-report-nextjs run: npx playwright test @@ -245,6 +347,8 @@ jobs: WASM_BINARY_PATH: ${{ env.WASM_ARTIFACT_PATH }} INTEGRATION_ORIGIN_PORT: ${{ env.ORIGIN_PORT }} VICEROY_CONFIG_PATH: ${{ env.ARTIFACTS_DIR }}/configs/viceroy.toml + TRACE_VICEROY_CONFIG_PATH: ${{ env.ARTIFACTS_DIR }}/trace/configs/viceroy.toml + TRACE_AUTH_VICEROY_CONFIG_PATH: ${{ env.ARTIFACTS_DIR }}/trace-auth/configs/viceroy.toml TEST_FRAMEWORK: wordpress PLAYWRIGHT_HTML_REPORT: playwright-report-wordpress run: npx playwright test diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 0a956a9fc6..1cdf4e905d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -55,6 +55,11 @@ jobs: - name: Run tests run: cargo test-fastly + - name: Run trace emitted-script regressions + run: | + cargo test -p trusted-server-core --target x86_64-unknown-linux-gnu --lib trace_document_skipped_emitted_script_leaves_ad_state_untouched -- --ignored + cargo test -p trusted-server-core --target x86_64-unknown-linux-gnu --lib trace_document_emitted_script_deep_freezes_owned_context_in_node -- --ignored + - name: Run tests (reusable-sandbox feature) run: cargo test-fastly-reuse @@ -146,6 +151,14 @@ jobs: - name: Build Axum adapter run: cargo build -p trusted-server-adapter-axum + - name: Verify prebuilt trace source digest parity + shell: bash + run: | + bundle_dir="$(mktemp -d)" + trap 'rm -rf "$bundle_dir"' EXIT + node crates/trusted-server-js/lib/build-all.mjs --out-dir "$bundle_dir" + TSJS_PREBUILT_DIR="$bundle_dir" cargo check-axum --locked + - name: Run Axum adapter tests run: cargo test-axum diff --git a/CHANGELOG.md b/CHANGELOG.md index fa9bf084c4..c9715a38f4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed +- Explicit TS Console activation with `?ts_console=1` now sets a 30-minute diagnostics cookie instead of a browser-session cookie, independently of the mobile trace flag. Ordinary requests do not refresh it; end and explicitly enable diagnostics again to adopt the lifetime for an existing session. +- Publisher documents with diagnostics active now use private, no-store responses and strip conditional and range request headers before fetching the origin. An unexpected origin `304` becomes a private `502` because it cannot supply an instrumented document, including when mobile tracing is disabled. - **Breaking:** `x-ts-version` now reports the deployed git version — the release tag, else the branch, else the first 6 characters of the commit — compiled in from the build-time `TRUSTED_SERVER__GIT_VERSION` (set by the deploy pipeline) or local git, and is sent by every adapter, including on the Fastly `GET /health` probe. The Fastly service version it previously carried is now `x-ts-fastly-version`; update dashboards, monitors, and scripts that read `x-ts-version` as the Fastly version number. Builds with neither the override nor git omit the header. - **Breaking:** `ts prebid bundle` is now `ts prebid client`, alongside the new `ts prebid server` namespace. Update scripts and runbooks to use `ts prebid client` with the same arguments. The old `bundle` spelling is no longer accepted and has no compatibility alias. - The S2S `/_ts/api/v1/batch-sync` endpoint now validates the full batch and calls the CAS-protected update path once per distinct normalized EC ID. The last valid UID wins within a group, and infrastructure failures reject the failing and each unprocessed group, so accepted and `kv_unavailable` input indexes may interleave. @@ -30,14 +32,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Didomi SDK requests now derive `X-Geo-Country`, `X-Geo-Region`, and `CloudFront-Viewer-Country` only from trusted platform geo (#1151). Caller-supplied `FastlyGeo-CountryCode` and `FastlyGeo-Region` headers previously let callers choose the geography sent to Didomi. **Breaking** for deployments that injected those headers: configure CDN-fronted client IP instead. Without a complete trusted country/region pair (currently always on Cloudflare, Axum, and Spin), the geo header group is omitted. Noncanonical loaders and SDK POSTs bypass the outbound cache and return `no-store, private`; other SDK GETs retain Didomi's cache policy. Purge cached SDK entries before rollout. Geo-less loaders risk Didomi throttling and disablement; production Fastly deployments should validate geo coverage and enable `geo_query_parameters = true`. Other adapters currently have no compliant loader-caching mode through this proxy; use direct SDK delivery instead. Audit-generated configurations leave detected Didomi disabled for manual review. - `ts prebid client` now keeps the config file's mode bits when it writes `external_bundle_sha256` and `external_bundle_sri`; previously a `0600` config was reset to the umask default (usually `0644`). Configs already rewritten by an earlier version keep the widened mode, so restrict them again (for example `chmod 600 trusted-server.toml`). The rewrite is fsynced and leaves no temporary copy behind on failure. TOML parse errors from `ts prebid client` and `ts audit generate` now name the file, line and column without quoting the offending config line (#1202). -- Publisher and page-bids processing now reject malformed effective request hosts with 400 before inserting request-derived bytes into rewrite output. `/first-party/sign` also returns 400 for a malformed effective host when the request URI has no scheme and request metadata is needed to determine it. Host selection remains `Forwarded` (`host=`), then `X-Forwarded-Host`, then `Host`; a malformed or explicitly empty selected host is rejected without fallback. Valid local hostnames, underscore labels, ports, and bracketed IPv6 remain supported. Forwarded-header trust policy is unchanged (#1206, #1207, #1208). +- Reserve the application-visible `/_ts/trace*` prefix locally on every adapter, including when mobile tracing is disabled. Existing authentication runs first; disabled or unknown trace routes then return a local `404` instead of forwarding to the publisher. +- Stop deriving the public host and scheme from unauthenticated `Forwarded`, `X-Forwarded-Host` and `X-Forwarded-Proto` fields on every adapter, and strip those fields before routing. Fastly and Spin already removed them; Axum and Cloudflare previously let a client-supplied value steer first-party URL rewriting. A deployment behind a TLS-terminating or Host-rewriting proxy, including `ts dev proxy` against Axum, must now configure the opt-in `[trusted_forwarder]` section and send its matching credential to preserve the public origin. The section authenticates one publisher-domain-bounded `X-Forwarded-Host`/`X-Forwarded-Proto` pair with a secret-store-resolved shared secret; without it, first-party URLs use the transport host and scheme. Axum uses `http://` even when the proxy's browser-facing connection uses HTTPS. +- Publisher and page-bids processing now reject malformed effective request hosts with 400 before inserting request-derived bytes into rewrite output. `/first-party/sign` also returns 400 for a malformed effective host when the request URI has no scheme and request metadata is needed to determine it. Host selection is the authenticated trusted forwarder's public origin, then ingress or adapter runtime origin, then a nonempty `Host`, then the URI authority without userinfo; every selected host is validated, and a malformed selected host is rejected without fallback. Valid local hostnames, underscore labels, ports, and bracketed IPv6 remain supported (#1206, #1207, #1208). - `/first-party/sign` now rejects valid targets outside `proxy.allowed_domains` before minting a proxy token. The creative runtime keeps image and iframe assignments blocked after this `403` policy response instead of loading the rejected URL directly; fetch-time checks still cover the initial target and every redirect. - Reserved the complete admin namespace at the publisher-fallback boundary. Percent-encoded separators (`/_ts/admin%2Fec`, `%2f`, and double-encoded forms) matched the `^/_ts/admin` Basic-auth handler but escaped the literal-slash namespace check, so an authenticated request fell through to publisher fallback and forwarded its `Authorization` header and body to the publisher origin. The reservation now spans the whole `/_ts/admin` prefix plus the retired `/admin/keys` aliases — including trailing, descendant, and encoded-separator forms — evaluated on the raw path and on each of its bounded percent-decodings, so multi-encoded separators such as `/admin%252Fkeys/rotate` cannot survive to fallback for a proxy or origin to decode again, and applies to every adapter. - Validate synthetic ID format on inbound values from the `x-synthetic-id` header and `synthetic_id` cookie; values that do not match the expected format (`64-hex-hmac.6-alphanumeric-suffix`) are discarded and a fresh ID is generated rather than forwarded to response headers, cookies, or third-party APIs ### Fixed -- `rewrite.exclude_domains` matching is now case-insensitive and ignores surrounding whitespace. Entries written with uppercase letters previously never matched and now take effect, so those hosts stop being proxied and click-wrapped, and `/first-party/sign` now rejects them with `502`; it used to sign them, or return `403` when the host was also outside `proxy.allowed_domains`. Empty and bare `"*"` `exclude_domains` entries, which never matched a host, are dropped at load with a warning, and other entries that can never match a host are logged with a warning. Absolute `http(s)` creative URLs that cannot be parsed (for example, a host containing a space) are now left untouched; previously the trimmed value was written back re-quoted (with `https:` prepended to `//` input, so `src=" //exa mple.example/x "` became `https://exa mple.example/x`), an unparseable quoted URL in `