diff --git a/.github/scripts/apt-get-prefetch.sh b/.github/scripts/apt-get-prefetch.sh new file mode 100755 index 000000000..455e84564 --- /dev/null +++ b/.github/scripts/apt-get-prefetch.sh @@ -0,0 +1,73 @@ +#!/usr/bin/env bash +# Stands in for apt-get on PATH while `playwright install-deps` runs (see +# install-playwright.sh). For `apt-get install` it first downloads the +# packages apt would fetch, several at a time and from fallback Ubuntu +# mirrors when the runner's mirror is slow, into apt's archive directory; +# then it runs the real apt-get, which finds them already downloaded. +# Anything the prefetch misses, the real apt-get still downloads itself. +set -uo pipefail + +real=/usr/bin/apt-get + +if [ "${1:-}" = install ]; then + archives=/var/cache/apt/archives + mirrors=(${PREFETCH_MIRRORS:-http://archive.ubuntu.com/ubuntu http://azure.archive.ubuntu.com/ubuntu}) + + fetch() { + local url="${1//\'/}" file="$2" hash="$3" + local algo="${hash%%:*}" sum="${hash#*:}" path tmp candidate rest list checker + local -a bases + [ -s "$archives/$file" ] && return 0 + case "$url" in + # The hosted images point apt at a mirror list: + # mirror+file:/etc/apt/apt-mirrors.txt/pool/main/... + mirror+file:*) + rest="${url#mirror+file:}" + list="${rest%%.txt/*}.txt" + path="${rest#*.txt/}" + mapfile -t bases < <(awk '/^[a-z]+:\/\//{sub(/\/$/, "", $1); print $1}' "$list") + ;; + *) + path="${url#*/ubuntu/}" + bases=("${url%%/ubuntu/*}/ubuntu") + ;; + esac + # apt prints MD5Sum:, SHA1:, SHA256: or SHA512: before the hash. + case "$algo" in + MD5Sum) checker=md5sum ;; + SHA1) checker=sha1sum ;; + SHA256) checker=sha256sum ;; + *) checker=sha512sum ;; + esac + tmp="$(mktemp -p "$archives/partial")" + for candidate in "${bases[@]/%//$path}" "${mirrors[@]/%//$path}"; do + # Give up on a connection slower than 50 kB/s for 15 s and move on to + # the next mirror. + if curl -fsL --connect-timeout 10 --speed-limit 51200 --speed-time 15 \ + --max-time 300 -o "$tmp" "$candidate" && + echo "$sum $tmp" | "$checker" --status -c; then + chmod 644 "$tmp" + mv "$tmp" "$archives/$file" + return 0 + fi + done + rm -f "$tmp" + echo "prefetch: could not download $file; apt-get will retry it" >&2 + } + export -f fetch + export archives + export PREFETCH_MIRROR_LIST="${mirrors[*]}" + + started=$SECONDS + uris="$("$real" "$@" --print-uris -qq 2> /dev/null | grep "^'" || true)" + count="$(printf '%s' "$uris" | grep -c . || true)" + if [ "$count" -gt 0 ]; then + echo "prefetch: downloading $count package(s), 8 at a time" + printf '%s\n' "$uris" | + xargs -P 8 -L 1 bash -c 'mirrors=($PREFETCH_MIRROR_LIST); fetch "$0" "$1" "$3"' + missing="$("$real" "$@" --print-uris -qq 2> /dev/null | grep -c "^'" || true)" + echo "prefetch: $((count - missing)) of $count package(s) downloaded in $((SECONDS - started))s" + fi +fi + +exec "$real" "$@" diff --git a/.github/scripts/install-playwright.sh b/.github/scripts/install-playwright.sh new file mode 100755 index 000000000..4e2dd5655 --- /dev/null +++ b/.github/scripts/install-playwright.sh @@ -0,0 +1,100 @@ +#!/usr/bin/env bash +# Installs Playwright browsers and their system packages on a GitHub-hosted +# Ubuntu runner without letting a slow apt mirror hold the job hostage. +# +# install-playwright.sh ... +# +# Environment: +# PLAYWRIGHT_CLI the Playwright CLI to run (default: "bunx playwright") +# ATTEMPT_TIMEOUT the time limit of one attempt (default: 4m) +# ATTEMPTS how many attempts to make (default: 3) +# PLAYWRIGHT_APT_CACHE a directory of .deb files kept between runs by +# actions/cache; seeded into apt before installing and +# refreshed after a successful install (optional) +# +# `playwright install --with-deps` runs `apt-get update` and +# `apt-get install` for the packages the browsers need. Firefox and WebKit +# need about 185 packages (140 MB) the image does not ship, and the Ubuntu +# mirror of the runner sometimes serves them at a few kB/s, so one install +# could run until the job timed out. Here the packages are downloaded +# several at a time with fallback mirrors, each attempt is time-limited, and +# packages downloaded before a timeout stay in /var/cache/apt/archives, so +# the next attempt only fetches the rest. +set -euo pipefail + +if [ "$#" -eq 0 ]; then + echo "usage: $0 ..." >&2 + exit 2 +fi + +read -r -a cli <<< "${PLAYWRIGHT_CLI:-bunx playwright}" + +attempt_timeout="${ATTEMPT_TIMEOUT:-4m}" +attempts="${ATTEMPTS:-3}" +apt_cache="${PLAYWRIGHT_APT_CACHE:-}" +apt_cache="${apt_cache/#\~/$HOME}" +archives=/var/cache/apt/archives + +# Keep downloaded packages, retry failed requests and drop stalled +# connections instead of waiting on them, and wait for a dpkg lock left by +# an interrupted attempt rather than failing on it. +sudo tee /etc/apt/apt.conf.d/99-playwright-ci > /dev/null << 'EOF' +APT::Keep-Downloaded-Packages "true"; +Binary::apt::APT::Keep-Downloaded-Packages "true"; +Acquire::Retries "3"; +Acquire::http::Timeout "30"; +Acquire::https::Timeout "30"; +DPkg::Lock::Timeout "120"; +EOF + +if [ -n "$apt_cache" ] && compgen -G "$apt_cache/*.deb" > /dev/null; then + echo "Seeding apt with $(find "$apt_cache" -name '*.deb' | wc -l) cached package(s)" + sudo cp -n "$apt_cache"/*.deb "$archives/" +fi + +# `install-deps` runs as root so Playwright calls apt-get without sudo, which +# would reset PATH; the apt-get on PATH is then apt-get-prefetch.sh, which +# downloads the packages in parallel (with fallback mirrors) before handing +# over to the real apt-get. The browsers install as the runner user. +shim="$(mktemp -d)" +ln -s "$(cd "$(dirname "$0")" && pwd)/apt-get-prefetch.sh" "$shim/apt-get" + +# One attempt. `timeout` runs inside sudo: sudo starts the command in a +# session of its own, so a `timeout` outside it would not reach apt-get and +# a timed-out attempt would keep holding the dpkg lock. +install_once() { + sudo timeout --kill-after=20s "$attempt_timeout" \ + env "PATH=$shim:$PATH" "${cli[@]}" install-deps "$@" && + timeout --kill-after=20s "$attempt_timeout" "${cli[@]}" install "$@" +} + +attempt=1 +until install_once "$@"; do + status=$? + if [ "$attempt" -ge "$attempts" ]; then + echo "::error::Playwright install failed after $attempt attempt(s) (last exit status $status)" + exit "$status" + fi + echo "::warning::Playwright install attempt $attempt exited with status $status (124 = timed out after $attempt_timeout); retrying" + # A killed attempt can leave apt-get or its downloads running, or dpkg + # half-configured. + sudo pkill -KILL -x apt-get || true + sudo pkill -KILL -x curl || true + sudo dpkg --configure -a || true + echo "Packages already downloaded: $(find "$archives" -maxdepth 1 -name '*.deb' | wc -l)" + attempt=$((attempt + 1)) +done +echo "Playwright installed on attempt $attempt" + +if [ -n "$apt_cache" ]; then + # Keep only packages apt can still use, so the cache does not grow with + # every Ubuntu update. + sudo apt-get autoclean -qq + mkdir -p "$apt_cache" + find "$apt_cache" -name '*.deb' -delete + if compgen -G "$archives/*.deb" > /dev/null; then + sudo cp "$archives"/*.deb "$apt_cache/" + sudo chown "$(id -u):$(id -g)" "$apt_cache"/*.deb + fi + echo "Kept $(find "$apt_cache" -name '*.deb' | wc -l) package(s) for the apt cache" +fi diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 428d15200..56cfac0c7 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -639,9 +639,14 @@ jobs: with: path: ~/.cache/ms-playwright key: playwright-chromium-${{ runner.os }}-${{ env.RUNNER_IMAGE }}-${{ env.PLAYWRIGHT_REVISIONS }} + # Each attempt is time-limited and retried, so a stalled apt mirror or + # browser download costs minutes instead of the whole job. - &install-chromium name: Install Playwright Browsers - run: bunx playwright install chromium --with-deps + timeout-minutes: 14 + env: + ATTEMPT_TIMEOUT: 3m + run: bash .github/scripts/install-playwright.sh chromium # A run only compares against the screenshots checked into the repository: # it never creates or commits them. A new visual test needs its baselines # generated by a maintainer (`bun run test:e2e:update`) and committed. @@ -954,8 +959,31 @@ jobs: with: path: ~/.cache/ms-playwright key: playwright-release-compat-${{ runner.os }}-${{ env.RUNNER_IMAGE }}-${{ env.PLAYWRIGHT_REVISIONS }} + # Firefox and WebKit need about 140 MB of Ubuntu packages the image does + # not ship. Keeping the .deb files between runs spares the runner's + # Ubuntu mirror, which has served them slowly enough to time the job out. + - name: Record the week for the apt package cache key + run: echo "CACHE_WEEK=$(date -u +%G-W%V)" >> "$GITHUB_ENV" + - name: Restore Playwright system packages + id: restore-playwright-apt + uses: actions/cache/restore@v6 + with: + path: ~/.cache/playwright-apt + key: playwright-apt-release-compat-${{ runner.os }}-${{ env.RUNNER_IMAGE }}-${{ env.PLAYWRIGHT_REVISIONS }}-${{ env.CACHE_WEEK }} + restore-keys: | + playwright-apt-release-compat-${{ runner.os }}-${{ env.RUNNER_IMAGE }}- - name: Install Playwright browsers - run: bunx playwright install chromium firefox webkit --with-deps + timeout-minutes: 14 + env: + ATTEMPT_TIMEOUT: 4m + PLAYWRIGHT_APT_CACHE: ~/.cache/playwright-apt + run: bash .github/scripts/install-playwright.sh chromium firefox webkit + - name: Save Playwright system packages + if: steps.restore-playwright-apt.outputs.cache-hit != 'true' + uses: actions/cache/save@v6 + with: + path: ~/.cache/playwright-apt + key: playwright-apt-release-compat-${{ runner.os }}-${{ env.RUNNER_IMAGE }}-${{ env.PLAYWRIGHT_REVISIONS }}-${{ env.CACHE_WEEK }} - name: Build packed consumers and check production computed styles run: bun run test:release - name: Upload release compatibility evidence @@ -997,12 +1025,18 @@ jobs: # The `build` job's fresh WASM and packages of this commit. - *download-build - *unpack-build - - name: Install latest private Playwright and Chromium + - name: Install latest private Playwright run: | mkdir -p "$DEVUP_PLAYWRIGHT_ROOT" "$DEVUP_EXTENDS_RESULTS" printf '{"private":true,"type":"module"}\n' > "$DEVUP_PLAYWRIGHT_ROOT/package.json" bun add --cwd "$DEVUP_PLAYWRIGHT_ROOT" @playwright/test@latest - node "$DEVUP_PLAYWRIGHT_ROOT/node_modules/@playwright/test/cli.js" install chromium --with-deps + - name: Install Chromium for the private Playwright + timeout-minutes: 14 + env: + ATTEMPT_TIMEOUT: 4m + run: | + PLAYWRIGHT_CLI="node $DEVUP_PLAYWRIGHT_ROOT/node_modules/@playwright/test/cli.js" \ + bash .github/scripts/install-playwright.sh chromium - name: Installed formats and production theme delivery regressions run: | set -euo pipefail