Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
73 changes: 73 additions & 0 deletions .github/scripts/apt-get-prefetch.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,73 @@
#!/usr/bin/env bash
# Stands in for apt-get on PATH while `playwright install-deps` runs (see
# install-playwright.sh). For `apt-get install` it first downloads the
# packages apt would fetch, several at a time and from fallback Ubuntu
# mirrors when the runner's mirror is slow, into apt's archive directory;
# then it runs the real apt-get, which finds them already downloaded.
# Anything the prefetch misses, the real apt-get still downloads itself.
set -uo pipefail

real=/usr/bin/apt-get

if [ "${1:-}" = install ]; then
archives=/var/cache/apt/archives
mirrors=(${PREFETCH_MIRRORS:-http://archive.ubuntu.com/ubuntu http://azure.archive.ubuntu.com/ubuntu})

fetch() {
local url="${1//\'/}" file="$2" hash="$3"
local algo="${hash%%:*}" sum="${hash#*:}" path tmp candidate rest list checker
local -a bases
[ -s "$archives/$file" ] && return 0
case "$url" in
# The hosted images point apt at a mirror list:
# mirror+file:/etc/apt/apt-mirrors.txt/pool/main/...
mirror+file:*)
rest="${url#mirror+file:}"
list="${rest%%.txt/*}.txt"
path="${rest#*.txt/}"
mapfile -t bases < <(awk '/^[a-z]+:\/\//{sub(/\/$/, "", $1); print $1}' "$list")
;;
*)
path="${url#*/ubuntu/}"
bases=("${url%%/ubuntu/*}/ubuntu")
;;
esac
# apt prints MD5Sum:, SHA1:, SHA256: or SHA512: before the hash.
case "$algo" in
MD5Sum) checker=md5sum ;;
SHA1) checker=sha1sum ;;
SHA256) checker=sha256sum ;;
*) checker=sha512sum ;;
esac
tmp="$(mktemp -p "$archives/partial")"
for candidate in "${bases[@]/%//$path}" "${mirrors[@]/%//$path}"; do
# Give up on a connection slower than 50 kB/s for 15 s and move on to
# the next mirror.
if curl -fsL --connect-timeout 10 --speed-limit 51200 --speed-time 15 \
--max-time 300 -o "$tmp" "$candidate" &&
echo "$sum $tmp" | "$checker" --status -c; then
chmod 644 "$tmp"
mv "$tmp" "$archives/$file"
return 0
fi
done
rm -f "$tmp"
echo "prefetch: could not download $file; apt-get will retry it" >&2
}
export -f fetch
export archives
export PREFETCH_MIRROR_LIST="${mirrors[*]}"

started=$SECONDS
uris="$("$real" "$@" --print-uris -qq 2> /dev/null | grep "^'" || true)"
count="$(printf '%s' "$uris" | grep -c . || true)"
if [ "$count" -gt 0 ]; then
echo "prefetch: downloading $count package(s), 8 at a time"
printf '%s\n' "$uris" |
xargs -P 8 -L 1 bash -c 'mirrors=($PREFETCH_MIRROR_LIST); fetch "$0" "$1" "$3"'
missing="$("$real" "$@" --print-uris -qq 2> /dev/null | grep -c "^'" || true)"
echo "prefetch: $((count - missing)) of $count package(s) downloaded in $((SECONDS - started))s"
fi
fi

exec "$real" "$@"
100 changes: 100 additions & 0 deletions .github/scripts/install-playwright.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,100 @@
#!/usr/bin/env bash
# Installs Playwright browsers and their system packages on a GitHub-hosted
# Ubuntu runner without letting a slow apt mirror hold the job hostage.
#
# install-playwright.sh <browser>...
#
# Environment:
# PLAYWRIGHT_CLI the Playwright CLI to run (default: "bunx playwright")
# ATTEMPT_TIMEOUT the time limit of one attempt (default: 4m)
# ATTEMPTS how many attempts to make (default: 3)
# PLAYWRIGHT_APT_CACHE a directory of .deb files kept between runs by
# actions/cache; seeded into apt before installing and
# refreshed after a successful install (optional)
#
# `playwright install --with-deps` runs `apt-get update` and
# `apt-get install` for the packages the browsers need. Firefox and WebKit
# need about 185 packages (140 MB) the image does not ship, and the Ubuntu
# mirror of the runner sometimes serves them at a few kB/s, so one install
# could run until the job timed out. Here the packages are downloaded
# several at a time with fallback mirrors, each attempt is time-limited, and
# packages downloaded before a timeout stay in /var/cache/apt/archives, so
# the next attempt only fetches the rest.
set -euo pipefail

if [ "$#" -eq 0 ]; then
echo "usage: $0 <browser>..." >&2
exit 2
fi

read -r -a cli <<< "${PLAYWRIGHT_CLI:-bunx playwright}"

attempt_timeout="${ATTEMPT_TIMEOUT:-4m}"
attempts="${ATTEMPTS:-3}"
apt_cache="${PLAYWRIGHT_APT_CACHE:-}"
apt_cache="${apt_cache/#\~/$HOME}"
archives=/var/cache/apt/archives

# Keep downloaded packages, retry failed requests and drop stalled
# connections instead of waiting on them, and wait for a dpkg lock left by
# an interrupted attempt rather than failing on it.
sudo tee /etc/apt/apt.conf.d/99-playwright-ci > /dev/null << 'EOF'
APT::Keep-Downloaded-Packages "true";
Binary::apt::APT::Keep-Downloaded-Packages "true";
Acquire::Retries "3";
Acquire::http::Timeout "30";
Acquire::https::Timeout "30";
DPkg::Lock::Timeout "120";
EOF

if [ -n "$apt_cache" ] && compgen -G "$apt_cache/*.deb" > /dev/null; then
echo "Seeding apt with $(find "$apt_cache" -name '*.deb' | wc -l) cached package(s)"
sudo cp -n "$apt_cache"/*.deb "$archives/"
fi

# `install-deps` runs as root so Playwright calls apt-get without sudo, which
# would reset PATH; the apt-get on PATH is then apt-get-prefetch.sh, which
# downloads the packages in parallel (with fallback mirrors) before handing
# over to the real apt-get. The browsers install as the runner user.
shim="$(mktemp -d)"
ln -s "$(cd "$(dirname "$0")" && pwd)/apt-get-prefetch.sh" "$shim/apt-get"

# One attempt. `timeout` runs inside sudo: sudo starts the command in a
# session of its own, so a `timeout` outside it would not reach apt-get and
# a timed-out attempt would keep holding the dpkg lock.
install_once() {
sudo timeout --kill-after=20s "$attempt_timeout" \
env "PATH=$shim:$PATH" "${cli[@]}" install-deps "$@" &&
timeout --kill-after=20s "$attempt_timeout" "${cli[@]}" install "$@"
}

attempt=1
until install_once "$@"; do
status=$?
if [ "$attempt" -ge "$attempts" ]; then
echo "::error::Playwright install failed after $attempt attempt(s) (last exit status $status)"
exit "$status"
fi
echo "::warning::Playwright install attempt $attempt exited with status $status (124 = timed out after $attempt_timeout); retrying"
# A killed attempt can leave apt-get or its downloads running, or dpkg
# half-configured.
sudo pkill -KILL -x apt-get || true
sudo pkill -KILL -x curl || true
sudo dpkg --configure -a || true
echo "Packages already downloaded: $(find "$archives" -maxdepth 1 -name '*.deb' | wc -l)"
attempt=$((attempt + 1))
done
echo "Playwright installed on attempt $attempt"

if [ -n "$apt_cache" ]; then
# Keep only packages apt can still use, so the cache does not grow with
# every Ubuntu update.
sudo apt-get autoclean -qq
mkdir -p "$apt_cache"
find "$apt_cache" -name '*.deb' -delete
if compgen -G "$archives/*.deb" > /dev/null; then
sudo cp "$archives"/*.deb "$apt_cache/"
sudo chown "$(id -u):$(id -g)" "$apt_cache"/*.deb
fi
echo "Kept $(find "$apt_cache" -name '*.deb' | wc -l) package(s) for the apt cache"
fi
42 changes: 38 additions & 4 deletions .github/workflows/publish.yml
Original file line number Diff line number Diff line change
Expand Up @@ -639,9 +639,14 @@ jobs:
with:
path: ~/.cache/ms-playwright
key: playwright-chromium-${{ runner.os }}-${{ env.RUNNER_IMAGE }}-${{ env.PLAYWRIGHT_REVISIONS }}
# Each attempt is time-limited and retried, so a stalled apt mirror or
# browser download costs minutes instead of the whole job.
- &install-chromium
name: Install Playwright Browsers
run: bunx playwright install chromium --with-deps
timeout-minutes: 14
env:
ATTEMPT_TIMEOUT: 3m
run: bash .github/scripts/install-playwright.sh chromium
# A run only compares against the screenshots checked into the repository:
# it never creates or commits them. A new visual test needs its baselines
# generated by a maintainer (`bun run test:e2e:update`) and committed.
Expand Down Expand Up @@ -954,8 +959,31 @@ jobs:
with:
path: ~/.cache/ms-playwright
key: playwright-release-compat-${{ runner.os }}-${{ env.RUNNER_IMAGE }}-${{ env.PLAYWRIGHT_REVISIONS }}
# Firefox and WebKit need about 140 MB of Ubuntu packages the image does
# not ship. Keeping the .deb files between runs spares the runner's
# Ubuntu mirror, which has served them slowly enough to time the job out.
- name: Record the week for the apt package cache key
run: echo "CACHE_WEEK=$(date -u +%G-W%V)" >> "$GITHUB_ENV"
- name: Restore Playwright system packages
id: restore-playwright-apt
uses: actions/cache/restore@v6
with:
path: ~/.cache/playwright-apt
key: playwright-apt-release-compat-${{ runner.os }}-${{ env.RUNNER_IMAGE }}-${{ env.PLAYWRIGHT_REVISIONS }}-${{ env.CACHE_WEEK }}
restore-keys: |
playwright-apt-release-compat-${{ runner.os }}-${{ env.RUNNER_IMAGE }}-
- name: Install Playwright browsers
run: bunx playwright install chromium firefox webkit --with-deps
timeout-minutes: 14
env:
ATTEMPT_TIMEOUT: 4m
PLAYWRIGHT_APT_CACHE: ~/.cache/playwright-apt
run: bash .github/scripts/install-playwright.sh chromium firefox webkit
- name: Save Playwright system packages
if: steps.restore-playwright-apt.outputs.cache-hit != 'true'
uses: actions/cache/save@v6
with:
path: ~/.cache/playwright-apt
key: playwright-apt-release-compat-${{ runner.os }}-${{ env.RUNNER_IMAGE }}-${{ env.PLAYWRIGHT_REVISIONS }}-${{ env.CACHE_WEEK }}
- name: Build packed consumers and check production computed styles
run: bun run test:release
- name: Upload release compatibility evidence
Expand Down Expand Up @@ -997,12 +1025,18 @@ jobs:
# The `build` job's fresh WASM and packages of this commit.
- *download-build
- *unpack-build
- name: Install latest private Playwright and Chromium
- name: Install latest private Playwright
run: |
mkdir -p "$DEVUP_PLAYWRIGHT_ROOT" "$DEVUP_EXTENDS_RESULTS"
printf '{"private":true,"type":"module"}\n' > "$DEVUP_PLAYWRIGHT_ROOT/package.json"
bun add --cwd "$DEVUP_PLAYWRIGHT_ROOT" @playwright/test@latest
node "$DEVUP_PLAYWRIGHT_ROOT/node_modules/@playwright/test/cli.js" install chromium --with-deps
- name: Install Chromium for the private Playwright
timeout-minutes: 14
env:
ATTEMPT_TIMEOUT: 4m
run: |
PLAYWRIGHT_CLI="node $DEVUP_PLAYWRIGHT_ROOT/node_modules/@playwright/test/cli.js" \
bash .github/scripts/install-playwright.sh chromium
- name: Installed formats and production theme delivery regressions
run: |
set -euo pipefail
Expand Down