Skip to content

ci: Split the E2E test matrix into two jobs #25913

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
msonnb merged 1 commit into from
Oct 29, 2026
Merged

ci: Split the E2E test matrix into two jobs #25913

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
44 changes: 33 additions & 11 deletions .github/workflows/build.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -140,7 +140,7 @@ jobs:
- name: Determine which test applications should be run
id: matrix
run:
pnpm --silent run ci:build-matrix --base=${{ (github.event_name == 'pull_request' &&
pnpm --silent run ci:build-matrix --shards=2 --base=${{ (github.event_name == 'pull_request' &&
needs.job_get_metadata.outputs.changed_ci != 'true' && github.event.pull_request.base.sha) || '' }} >>
$GITHUB_OUTPUT
working-directory: dev-packages/e2e-tests
Expand Down Expand Up @@ -184,15 +184,17 @@ jobs:
changed_aws_serverless:
${{ needs.job_get_metadata.outputs.changed_ci == 'true' || contains(steps.checkForAffected.outputs.affected,
'@sentry/aws-serverless') }}
e2e-matrix: ${{ steps.matrix.outputs.matrix }}
e2e-matrix-1: ${{ steps.matrix.outputs.matrix-1 }}
e2e-matrix-2: ${{ steps.matrix.outputs.matrix-2 }}
e2e-matrix-optional: ${{ steps.matrix-optional.outputs.matrix }}

job_build_layer:
name: Build Lambda layer
needs: [job_get_metadata, job_build]
if:
needs.job_build.outputs.changed_aws_serverless == 'true' || contains(needs.job_build.outputs.e2e-matrix,
'aws-serverless') || github.event_name != 'pull_request'
needs.job_build.outputs.changed_aws_serverless == 'true' || contains(needs.job_build.outputs.e2e-matrix-1,
'aws-serverless') || contains(needs.job_build.outputs.e2e-matrix-2, 'aws-serverless') || github.event_name !=
'pull_request'
timeout-minutes: 10
runs-on:
namespace-profile-sentry-javascript;overrides.cache-tag=sentry-javascript${{ github.event_name == 'pull_request'
Expand Down Expand Up @@ -1021,20 +1023,23 @@ jobs:
compression-level: 6
overwrite: true

job_e2e_tests:
# GitHub Actions allows at most 256 jobs in one matrix, so the E2E tests are split into two jobs that share `env`
# and `steps`.
job_e2e_tests_1:
name: E2E ${{ matrix.label || matrix.test-application }} Test
# We need to add the `always()` check here because the previous step has this as well :(
# See: https://github.com/actions/runner/issues/2205
if:
always() && needs.job_build_tarballs.result == 'success' && needs.job_build.outputs.e2e-matrix !='{"include":[]}'
always() && needs.job_build_tarballs.result == 'success' && needs.job_build.outputs.e2e-matrix-1 !=
'{"include":[]}'
needs: [job_get_metadata, job_build, job_build_layer, job_build_tarballs]
# Most apps finish in under a minute on 4 vCPUs, so they run on 2. Apps that take longer set `sentryTest.runner`.
runs-on:
${{ matrix.runner || 'namespace-profile-sentry-javascript-small' }};overrides.cache-tag=sentry-javascript-e2e${{
github.event_name == 'pull_request' && '-pr' || '' }}
timeout-minutes: 15
env:
# We just use a dummy DSN here, only send to the tunnel anyhow
# We just use a dummy DSN here, only send to the tunnel anyhow
env: &e2e-env
E2E_TEST_DSN: 'https://username@domain/123'
# Needed because some apps expect a certain prefix
NEXT_PUBLIC_E2E_TEST_DSN: 'https://username@domain/123'
Expand All @@ -1048,8 +1053,8 @@ jobs:
pnpm_config_verify_deps_before_run: 'false'
strategy:
fail-fast: false
matrix: ${{ fromJson(needs.job_build.outputs.e2e-matrix) }}
steps:
matrix: ${{ fromJson(needs.job_build.outputs.e2e-matrix-1) }}
steps: &e2e-steps
- name: Check out current commit (${{ needs.job_get_metadata.outputs.commit_label }})
uses: actions/checkout@v7
with:
Expand Down Expand Up @@ -1115,6 +1120,22 @@ jobs:
overwrite: true
retention-days: 7

job_e2e_tests_2:
name: E2E ${{ matrix.label || matrix.test-application }} Test
if:
always() && needs.job_build_tarballs.result == 'success' && needs.job_build.outputs.e2e-matrix-2 !=
'{"include":[]}'
needs: [job_get_metadata, job_build, job_build_layer, job_build_tarballs]
runs-on:
${{ matrix.runner || 'namespace-profile-sentry-javascript-small' }};overrides.cache-tag=sentry-javascript-e2e${{
github.event_name == 'pull_request' && '-pr' || '' }}
timeout-minutes: 15
env: *e2e-env
strategy:
fail-fast: false
matrix: ${{ fromJson(needs.job_build.outputs.e2e-matrix-2) }}
steps: *e2e-steps

# - We skip optional tests on release branches
# - The tests run in optional-e2e.yml, so their failures do not mark the PR as failed. This job only waits for
# them. optional-e2e.yml reports the results in a PR comment.
Expand Down Expand Up @@ -1175,7 +1196,8 @@ jobs:
job_deno_integration_tests,
job_browser_playwright_tests,
job_browser_loader_tests,
job_e2e_tests,
job_e2e_tests_1,
job_e2e_tests_2,
job_artifacts,
job_lint,
job_check_lockfile,
Expand Down
33 changes: 29 additions & 4 deletions dev-packages/e2e-tests/lib/getTestMatrix.mjs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,9 @@ import { parseArgs } from 'node:util';

const __dirname = path.dirname(fileURLToPath(import.meta.url));

// GitHub Actions fails a job whose matrix has more jobs than this, and runs none of them.
const MAX_MATRIX_JOBS = 256;

const VARIANT_FIELDS = {
required: 'variants',
optional: 'optionalVariants',
Expand All @@ -22,6 +25,9 @@ const VARIANT_FIELDS = {
* - not set: the required test applications and their `variants`
* - `optional`: the optional test applications and the `optionalVariants` of all test applications
* - `canary`: only the `canaryVariants` of all test applications
*
* `--shards=N` splits the matrix into N parts (`matrix-1=...` to `matrix-N=...`), because GitHub Actions allows at
* most 256 jobs in one matrix.
*/
function run() {
const { values } = parseArgs({
Expand All @@ -30,10 +36,11 @@ function run() {
base: { type: 'string' },
head: { type: 'string' },
variant: { type: 'string', default: 'required' },
shards: { type: 'string' },
},
});

const { base, head = 'HEAD', variant } = values;
const { base, head = 'HEAD', variant, shards } = values;

// For GitHub Action debugging
// eslint-disable-next-line no-console
Expand Down Expand Up @@ -61,9 +68,27 @@ function run() {
addIncludesForTestApp(testApp, includes, { variant });
});

// We print this to the output, so the GHA can use it for the matrix
// eslint-disable-next-line no-console
console.log(`matrix=${JSON.stringify({ include: includes })}`);
if (!shards) {
// We print this to the output, so the GHA can use it for the matrix
// eslint-disable-next-line no-console
console.log(`matrix=${JSON.stringify({ include: includes })}`);
return;
}

const shardCount = Number(shards);
const shardSize = Math.ceil(includes.length / shardCount);

if (shardSize > MAX_MATRIX_JOBS) {
throw new Error(
`${includes.length} jobs do not fit into ${shardCount} matrices of at most ${MAX_MATRIX_JOBS} jobs, increase --shards`,
);
}

for (let i = 0; i < shardCount; i++) {
const shardIncludes = includes.slice(i * shardSize, (i + 1) * shardSize);
// eslint-disable-next-line no-console
console.log(`matrix-${i + 1}=${JSON.stringify({ include: shardIncludes })}`);
}
}

/** Direct children of `test-applications/` that contain a `package.json` (replaces glob one-segment + package.json). */
Expand Down
Loading