WIP: [CI] Run PR builds in contributor forks

.github/workflows/archery.yml

@@ -30,7 +30,9 @@ on:
       - 'dev/archery/**'
       - 'dev/tasks/**'
       - 'compose.yaml'
+  workflow_dispatch:
   pull_request:
+    types: [opened, synchronize, reopened, labeled]
     paths:
       - '.dockerignore'
       - '.github/workflows/archery.yml'
@@ -43,16 +45,25 @@ env:
   ARCHERY_DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}
 
 concurrency:
-  group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }}
+  # Cancel superseded fork pushes; unrelated labels must not cancel running builds.
+  group: "${{ github.workflow }}-${{ github.repository }}-${{ github.event.pull_request.number || (github.repository != 'apache/arrow' && github.ref) || github.sha }}-${{ github.event.action == 'labeled' && github.event.label.name != 'CI: Upstream' }}"
   cancel-in-progress: true
 
 permissions:
   contents: read
 
+# Fork PRs to main run in the contributor's fork; CI: Upstream requests merge-ref validation.
 jobs:
 
   test:
-    if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
+    if: >-
+      !contains(github.event.pull_request.title, 'WIP') &&
+      (github.event.action != 'labeled' || github.event.label.name == 'CI: Upstream') &&
+      (github.event_name != 'pull_request' ||
+       github.repository != 'apache/arrow' ||
+       github.base_ref != 'main' ||
+       github.event.pull_request.head.repo.full_name == github.repository ||
+       contains(github.event.pull_request.labels.*.name, 'CI: Upstream'))
     name: Archery Unittests and Crossbow Check Config
     runs-on: ubuntu-latest
     timeout-minutes: 15

.github/workflows/check_labels.yml

@@ -60,5 +60,20 @@ jobs:
                 echo "LABELS"
               } >> "${GITHUB_OUTPUT}"
               ;;
+            # A push to a contributor fork can pick up labels on its open PR.
+            # Adding a label alone cannot dispatch into another user's fork;
+            # the contributor can push again or run that workflow manually.
+            push)
+              if [ "${GITHUB_REPOSITORY}" != "apache/arrow" ] && [ "${GITHUB_REF_TYPE}" = "branch" ]; then
+                {
+                  echo "ci-extra-labels<<LABELS"
+                  gh pr list --repo apache/arrow --base main --state open \
+                    --head "${GITHUB_REPOSITORY_OWNER}:${GITHUB_REF_NAME}" \
+                    --json labels \
+                    --jq '[.[].labels[].name | select(startswith("CI: Extra"))] | unique'
+                  echo "LABELS"
+                } >> "${GITHUB_OUTPUT}"
+              fi
+              ;;
             # Other event types trigger through specific rules in `*_extra.yml`
           esac

.github/workflows/cpp.yml

@@ -41,7 +41,9 @@ on:
       - 'compose.yaml'
       - 'format/Flight.proto'
       - 'testing'
+  workflow_dispatch:
   pull_request:
+    types: [opened, synchronize, reopened, labeled]
     paths:
       - '.dockerignore'
       - '.github/workflows/cpp.yml'
@@ -61,7 +63,8 @@ on:
       - 'testing'
 
 concurrency:
-  group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }}
+  # Cancel superseded fork pushes; unrelated labels must not cancel running builds.
+  group: "${{ github.workflow }}-${{ github.repository }}-${{ github.event.pull_request.number || (github.repository != 'apache/arrow' && github.ref) || github.sha }}-${{ github.event.action == 'labeled' && github.event.label.name != 'CI: Upstream' }}"
   cancel-in-progress: true
 
 permissions:
@@ -73,11 +76,19 @@ env:
   ARROW_ENABLE_TIMING_TESTS: OFF
   DOCKER_VOLUME_PREFIX: ".docker/"
 
+# Fork PRs to main run in the contributor's fork; CI: Upstream requests merge-ref validation.
 jobs:
   docker:
     name: ${{ matrix.title }}
     runs-on: ${{ matrix.runs-on }}
-    if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
+    if: &run-builds >-
+      !contains(github.event.pull_request.title, 'WIP') &&
+      (github.event.action != 'labeled' || github.event.label.name == 'CI: Upstream') &&
+      (github.event_name != 'pull_request' ||
+       github.repository != 'apache/arrow' ||
+       github.base_ref != 'main' ||
+       github.event.pull_request.head.repo.full_name == github.repository ||
+       contains(github.event.pull_request.labels.*.name, 'CI: Upstream'))
     timeout-minutes: 75
     strategy:
       fail-fast: false
@@ -174,7 +185,7 @@ jobs:
   build-example:
     name: C++ Minimal Build Example
     runs-on: ubuntu-24.04
-    if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
+    if: *run-builds
     timeout-minutes: 45
     steps:
       - name: Checkout Arrow
@@ -195,7 +206,7 @@ jobs:
   macos:
     name: ${{ matrix.architecture }} macOS ${{ matrix.macos-version }} C++
     runs-on: macos-${{ matrix.macos-version }}
-    if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
+    if: *run-builds
     timeout-minutes: 75
     strategy:
       fail-fast: false
@@ -300,7 +311,7 @@ jobs:
           ci/scripts/cpp_test.sh $(pwd) $(pwd)/build
 
   windows:
-    if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
+    if: *run-builds
     name: AMD64 Windows 2022 AVX2 C++
     uses: ./.github/workflows/cpp_windows.yml
     with:
@@ -311,7 +322,7 @@ jobs:
   windows-mingw:
     name: AMD64 Windows MinGW ${{ matrix.msystem_upper }} C++
     runs-on: windows-2022
-    if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
+    if: *run-builds
     # Build may take 1h+ without cache.
     timeout-minutes: 120
     strategy:

.github/workflows/cpp_extra.yml

@@ -44,6 +44,7 @@ on:
       - 'testing'
     tags:
       - '**'
+  workflow_dispatch:
   pull_request:
     # This trigger ensures that the `check_labels` workflow runs on manual label changes
     types:
@@ -56,7 +57,8 @@ on:
         0 0 * * *
 
 concurrency:
-  group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }}
+  # Cancel superseded fork pushes, but keep upstream main commits independent.
+  group: ${{ github.workflow }}-${{ github.repository }}-${{ github.event.pull_request.number || (github.repository != 'apache/arrow' && github.ref) || github.sha }}
   cancel-in-progress: true
 
 permissions:
@@ -84,6 +86,7 @@ jobs:
     steps:
       - id: set_enabled
         if: >-
+          github.event_name == 'workflow_dispatch' ||
           github.ref_type == 'tag' ||
           (github.event_name == 'schedule' && github.repository == 'apache/arrow') ||
           contains(fromJSON(needs.check-labels.outputs.ci-extra-labels || '[]'), 'CI: Extra') ||
@@ -92,7 +95,13 @@ jobs:
 
   docker:
     needs: check-enabled
-    if: needs.check-enabled.outputs.is_enabled == 'true'
+    if: &run-builds >-
+      needs.check-enabled.outputs.is_enabled == 'true' &&
+      (github.event_name != 'pull_request' ||
+       github.repository != 'apache/arrow' ||
+       github.base_ref != 'main' ||
+       github.event.pull_request.head.repo.full_name == github.repository ||
+       contains(github.event.pull_request.labels.*.name, 'CI: Upstream'))
     name: ${{ matrix.title }}
     runs-on: ${{ matrix.runs-on }}
     timeout-minutes: 75
@@ -103,16 +112,6 @@ jobs:
           - image: alpine-linux-cpp
             runs-on: ubuntu-latest
             title: AMD64 Alpine Linux
-          - image: ubuntu-cpp
-            run-options: >-
-              -e ARROW_CTEST_TIMEOUT=2000
-              -e ARROW_C_FLAGS_DEBUG="-O1"
-              -e ARROW_CXX_FLAGS_DEBUG="-O1"
-              -e ARROW_GANDIVA=OFF
-              -e ARROW_LARGE_MEMORY_TESTS=ON
-              -e BUILD_WARNING_LEVEL=PRODUCTION
-            runs-on: "runs-on=${{ github.run_id }}/family=x8i.2xlarge/volume=80gb/spot=capacity-optimized"
-            title: AMD64 Ubuntu Large Memory Tests
           - image: ubuntu-cpp-bundled-offline
             runs-on: ubuntu-latest
             envs:
@@ -138,7 +137,7 @@ jobs:
               -e ARROW_CXXFLAGS=-march=x86-64
             runs-on: ubuntu-latest
             title: AMD64 Debian SIMD Level NONE
-    steps:
+    steps: &docker-steps
       - name: Checkout Arrow
         uses: actions/checkout@v7
         with:
@@ -189,9 +188,34 @@ jobs:
         continue-on-error: true
         run: archery docker push ${{ matrix.image }}
 
+  large-memory:
+    # This hardware is supplied by Arrow, not the shared GitHub runner pool.
+    needs: check-enabled
+    if: needs.check-enabled.outputs.is_enabled == 'true' && github.repository == 'apache/arrow'
+    name: ${{ matrix.title }}
+    runs-on: ${{ matrix.runs-on }}
+    timeout-minutes: 75
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - image: ubuntu-cpp
+            continue-on-error: false
+            envs: []
+            run-options: >-
+              -e ARROW_CTEST_TIMEOUT=2000
+              -e ARROW_C_FLAGS_DEBUG="-O1"
+              -e ARROW_CXX_FLAGS_DEBUG="-O1"
+              -e ARROW_GANDIVA=OFF
+              -e ARROW_LARGE_MEMORY_TESTS=ON
+              -e BUILD_WARNING_LEVEL=PRODUCTION
+            runs-on: "runs-on=${{ github.run_id }}/family=x8i.2xlarge/volume=80gb/spot=capacity-optimized"
+            title: AMD64 Ubuntu Large Memory Tests
+    steps: *docker-steps
+
   msvc-arm64:
     needs: check-enabled
-    if: needs.check-enabled.outputs.is_enabled == 'true'
+    if: *run-builds
     name: ARM64 Windows 11 MSVC
     uses: ./.github/workflows/cpp_windows.yml
     with:
@@ -201,7 +225,7 @@ jobs:
 
   jni-linux:
     needs: check-enabled
-    if: needs.check-enabled.outputs.is_enabled == 'true'
+    if: *run-builds
     name: JNI ${{ matrix.platform.runs-on }} ${{ matrix.platform.arch }}
     runs-on: ${{ matrix.platform.runs-on }}
     timeout-minutes: 240
@@ -270,7 +294,7 @@ jobs:
 
   jni-macos:
     needs: check-enabled
-    if: needs.check-enabled.outputs.is_enabled == 'true'
+    if: *run-builds
     name: JNI macOS
     runs-on: macos-14
     timeout-minutes: 45
@@ -360,7 +384,7 @@ jobs:
 
   jni-windows:
     needs: check-enabled
-    if: needs.check-enabled.outputs.is_enabled == 'true'
+    if: *run-builds
     name: JNI Windows
     runs-on: windows-2022
     timeout-minutes: 240
@@ -489,7 +513,7 @@ jobs:
 
   odbc-linux:
     needs: check-enabled
-    if: needs.check-enabled.outputs.is_enabled == 'true'
+    if: *run-builds
     name: ODBC Linux
     runs-on: ubuntu-latest
     timeout-minutes: 75
@@ -548,7 +572,7 @@ jobs:
 
   odbc-macos:
     needs: check-enabled
-    if: needs.check-enabled.outputs.is_enabled == 'true'
+    if: *run-builds
     name: ODBC ${{ matrix.build-type }} ${{ matrix.architecture }} macOS ${{ matrix.macos-version }}
     runs-on: macos-${{ matrix.macos-version }}
     timeout-minutes: 120
@@ -792,7 +816,7 @@ jobs:
 
   odbc-msvc:
     needs: check-enabled
-    if: needs.check-enabled.outputs.is_enabled == 'true'
+    if: *run-builds
     name: ODBC Windows
     runs-on: windows-2022
     timeout-minutes: 240
@@ -978,6 +1002,7 @@ jobs:
     if: github.event_name == 'schedule' && always()
     needs:
       - docker
+      - large-memory
       - jni-linux
       - jni-macos
       - msvc-arm64

.github/workflows/dev.yml

@@ -25,10 +25,13 @@ on:
       - '!dependabot/**'
     tags:
       - '**'
+  workflow_dispatch:
   pull_request:
+    types: [opened, synchronize, reopened, labeled]
 
 concurrency:
-  group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }}
+  # Cancel superseded fork pushes; unrelated labels must not cancel running builds.
+  group: "${{ github.workflow }}-${{ github.repository }}-${{ github.event.pull_request.number || (github.repository != 'apache/arrow' && github.ref) || github.sha }}-${{ github.event.action == 'labeled' && github.event.label.name != 'CI: Upstream' }}"
   cancel-in-progress: true
 
 permissions:
@@ -37,13 +40,21 @@ permissions:
 env:
   ARCHERY_DEBUG: 1
 
+# Fork PRs to main run in the contributor's fork; CI: Upstream requests merge-ref validation.
 jobs:
 
   lint:
     name: Lint C++, Python, R, Docker, RAT
     # Use Ubuntu 24.04 to ensure working pre-commit on Ubuntu 24.04.
     runs-on: ubuntu-24.04
-    if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
+    if: &run-builds >-
+      !contains(github.event.pull_request.title, 'WIP') &&
+      (github.event.action != 'labeled' || github.event.label.name == 'CI: Upstream') &&
+      (github.event_name != 'pull_request' ||
+       github.repository != 'apache/arrow' ||
+       github.base_ref != 'main' ||
+       github.event.pull_request.head.repo.full_name == github.repository ||
+       contains(github.event.pull_request.labels.*.name, 'CI: Upstream'))
     timeout-minutes: 15
     steps:
       - name: Checkout Arrow
@@ -71,7 +82,7 @@ jobs:
   release:
     name: Source Release and Merge Script on ${{ matrix.runs-on }}
     runs-on: ${{ matrix.runs-on }}
-    if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
+    if: *run-builds
     timeout-minutes: 15
     strategy:
       fail-fast: false
@@ -107,6 +118,7 @@ jobs:
             "cython>=3.1" \
             "dev/archery[release]" \
             pytest \
+            pyyaml \
             requests \
             scikit-build-core \
             setuptools-scm
@@ -118,3 +130,6 @@ jobs:
       - name: Run Merge Script Test
         shell: bash
         run: pytest -v dev/test_merge_arrow_pr.py
+      - name: Test fork CI status and routing
+        shell: bash
+        run: node --test ci/scripts/test_fork_ci_status.cjs