From a68b976d2433d32272854aa985a666a6855a5fd1 Mon Sep 17 00:00:00 2001 From: Andrei Borza Date: Thu, 29 Oct 2026 17:11:52 +0100 Subject: [PATCH] ci: Split the E2E test matrix into two jobs Co-Authored-By: Claude Opus 5.5 --- .github/workflows/build.yml | 44 +++++++++++++++----- dev-packages/e2e-tests/lib/getTestMatrix.mjs | 33 +++++++++++++-- 2 files changed, 62 insertions(+), 15 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 8766de594d0c..a546292a32ef 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -140,7 +140,7 @@ jobs: - name: Determine which test applications should be run id: matrix run: - pnpm --silent run ci:build-matrix --base=${{ (github.event_name == 'pull_request' && + pnpm --silent run ci:build-matrix --shards=2 --base=${{ (github.event_name == 'pull_request' && needs.job_get_metadata.outputs.changed_ci != 'true' && github.event.pull_request.base.sha) || '' }} >> $GITHUB_OUTPUT working-directory: dev-packages/e2e-tests @@ -184,15 +184,17 @@ jobs: changed_aws_serverless: ${{ needs.job_get_metadata.outputs.changed_ci == 'true' || contains(steps.checkForAffected.outputs.affected, '@sentry/aws-serverless') }} - e2e-matrix: ${{ steps.matrix.outputs.matrix }} + e2e-matrix-1: ${{ steps.matrix.outputs.matrix-1 }} + e2e-matrix-2: ${{ steps.matrix.outputs.matrix-2 }} e2e-matrix-optional: ${{ steps.matrix-optional.outputs.matrix }} job_build_layer: name: Build Lambda layer needs: [job_get_metadata, job_build] if: - needs.job_build.outputs.changed_aws_serverless == 'true' || contains(needs.job_build.outputs.e2e-matrix, - 'aws-serverless') || github.event_name != 'pull_request' + needs.job_build.outputs.changed_aws_serverless == 'true' || contains(needs.job_build.outputs.e2e-matrix-1, + 'aws-serverless') || contains(needs.job_build.outputs.e2e-matrix-2, 'aws-serverless') || github.event_name != + 'pull_request' timeout-minutes: 10 runs-on: namespace-profile-sentry-javascript;overrides.cache-tag=sentry-javascript${{ github.event_name == 'pull_request' @@ -1021,20 +1023,23 @@ jobs: compression-level: 6 overwrite: true - job_e2e_tests: + # GitHub Actions allows at most 256 jobs in one matrix, so the E2E tests are split into two jobs that share `env` + # and `steps`. + job_e2e_tests_1: name: E2E ${{ matrix.label || matrix.test-application }} Test # We need to add the `always()` check here because the previous step has this as well :( # See: https://github.com/actions/runner/issues/2205 if: - always() && needs.job_build_tarballs.result == 'success' && needs.job_build.outputs.e2e-matrix !='{"include":[]}' + always() && needs.job_build_tarballs.result == 'success' && needs.job_build.outputs.e2e-matrix-1 != + '{"include":[]}' needs: [job_get_metadata, job_build, job_build_layer, job_build_tarballs] # Most apps finish in under a minute on 4 vCPUs, so they run on 2. Apps that take longer set `sentryTest.runner`. runs-on: ${{ matrix.runner || 'namespace-profile-sentry-javascript-small' }};overrides.cache-tag=sentry-javascript-e2e${{ github.event_name == 'pull_request' && '-pr' || '' }} timeout-minutes: 15 - env: - # We just use a dummy DSN here, only send to the tunnel anyhow + # We just use a dummy DSN here, only send to the tunnel anyhow + env: &e2e-env E2E_TEST_DSN: 'https://username@domain/123' # Needed because some apps expect a certain prefix NEXT_PUBLIC_E2E_TEST_DSN: 'https://username@domain/123' @@ -1048,8 +1053,8 @@ jobs: pnpm_config_verify_deps_before_run: 'false' strategy: fail-fast: false - matrix: ${{ fromJson(needs.job_build.outputs.e2e-matrix) }} - steps: + matrix: ${{ fromJson(needs.job_build.outputs.e2e-matrix-1) }} + steps: &e2e-steps - name: Check out current commit (${{ needs.job_get_metadata.outputs.commit_label }}) uses: actions/checkout@v7 with: @@ -1115,6 +1120,22 @@ jobs: overwrite: true retention-days: 7 + job_e2e_tests_2: + name: E2E ${{ matrix.label || matrix.test-application }} Test + if: + always() && needs.job_build_tarballs.result == 'success' && needs.job_build.outputs.e2e-matrix-2 != + '{"include":[]}' + needs: [job_get_metadata, job_build, job_build_layer, job_build_tarballs] + runs-on: + ${{ matrix.runner || 'namespace-profile-sentry-javascript-small' }};overrides.cache-tag=sentry-javascript-e2e${{ + github.event_name == 'pull_request' && '-pr' || '' }} + timeout-minutes: 15 + env: *e2e-env + strategy: + fail-fast: false + matrix: ${{ fromJson(needs.job_build.outputs.e2e-matrix-2) }} + steps: *e2e-steps + # - We skip optional tests on release branches # - The tests run in optional-e2e.yml, so their failures do not mark the PR as failed. This job only waits for # them. optional-e2e.yml reports the results in a PR comment. @@ -1175,7 +1196,8 @@ jobs: job_deno_integration_tests, job_browser_playwright_tests, job_browser_loader_tests, - job_e2e_tests, + job_e2e_tests_1, + job_e2e_tests_2, job_artifacts, job_lint, job_check_lockfile, diff --git a/dev-packages/e2e-tests/lib/getTestMatrix.mjs b/dev-packages/e2e-tests/lib/getTestMatrix.mjs index 0ec456f391a2..6d409d4e9411 100644 --- a/dev-packages/e2e-tests/lib/getTestMatrix.mjs +++ b/dev-packages/e2e-tests/lib/getTestMatrix.mjs @@ -6,6 +6,9 @@ import { parseArgs } from 'node:util'; const __dirname = path.dirname(fileURLToPath(import.meta.url)); +// GitHub Actions fails a job whose matrix has more jobs than this, and runs none of them. +const MAX_MATRIX_JOBS = 256; + const VARIANT_FIELDS = { required: 'variants', optional: 'optionalVariants', @@ -22,6 +25,9 @@ const VARIANT_FIELDS = { * - not set: the required test applications and their `variants` * - `optional`: the optional test applications and the `optionalVariants` of all test applications * - `canary`: only the `canaryVariants` of all test applications + * + * `--shards=N` splits the matrix into N parts (`matrix-1=...` to `matrix-N=...`), because GitHub Actions allows at + * most 256 jobs in one matrix. */ function run() { const { values } = parseArgs({ @@ -30,10 +36,11 @@ function run() { base: { type: 'string' }, head: { type: 'string' }, variant: { type: 'string', default: 'required' }, + shards: { type: 'string' }, }, }); - const { base, head = 'HEAD', variant } = values; + const { base, head = 'HEAD', variant, shards } = values; // For GitHub Action debugging // eslint-disable-next-line no-console @@ -61,9 +68,27 @@ function run() { addIncludesForTestApp(testApp, includes, { variant }); }); - // We print this to the output, so the GHA can use it for the matrix - // eslint-disable-next-line no-console - console.log(`matrix=${JSON.stringify({ include: includes })}`); + if (!shards) { + // We print this to the output, so the GHA can use it for the matrix + // eslint-disable-next-line no-console + console.log(`matrix=${JSON.stringify({ include: includes })}`); + return; + } + + const shardCount = Number(shards); + const shardSize = Math.ceil(includes.length / shardCount); + + if (shardSize > MAX_MATRIX_JOBS) { + throw new Error( + `${includes.length} jobs do not fit into ${shardCount} matrices of at most ${MAX_MATRIX_JOBS} jobs, increase --shards`, + ); + } + + for (let i = 0; i < shardCount; i++) { + const shardIncludes = includes.slice(i * shardSize, (i + 1) * shardSize); + // eslint-disable-next-line no-console + console.log(`matrix-${i + 1}=${JSON.stringify({ include: shardIncludes })}`); + } } /** Direct children of `test-applications/` that contain a `package.json` (replaces glob one-segment + package.json). */