diff --git a/.github/workflows/create_archive_and_notes.sh b/.github/workflows/create_archive_and_notes.sh index cec8d82a97..91df1167e1 100755 --- a/.github/workflows/create_archive_and_notes.sh +++ b/.github/workflows/create_archive_and_notes.sh @@ -19,14 +19,14 @@ set -o errexit set -x -TAG=$1 -if [ -z "$TAG" ]; then - echo "ERROR: TAG env var must be set" - exit 1 +TAG=${1:-} +if [ -n "$TAG" ]; then + # If the workflow checks out one commit, but is releasing another + git fetch origin tag "$TAG" + git checkout "$TAG" +else + TAG=$(git rev-parse HEAD) fi -# If the workflow checks out one commit, but is releasing another -git fetch origin tag "$TAG" -git checkout "$TAG" # A prefix is added to better match the GitHub generated archives. PREFIX="rules_python-${TAG}" diff --git a/.github/workflows/release_build_artifacts.yaml b/.github/workflows/release_build_artifacts.yaml index 744fc0779e..662839f191 100644 --- a/.github/workflows/release_build_artifacts.yaml +++ b/.github/workflows/release_build_artifacts.yaml @@ -1,11 +1,17 @@ -# Builds `//dev/release_artifacts:artifacts_for_release` for the platforms in -# dev/platforms, which cover python-build-standalone's platforms except s390x; -# see dev/release_artifacts/build.sh. Linux x86_64 runners cross-compile the -# files for other Linux CPUs. macOS and Windows files need those OSes' SDKs, so -# they're built on those OSes. +# Builds all deliverables for a release without publishing them: +# - `//dev/release_artifacts:artifacts_for_release` for the platforms in +# dev/platforms, which cover python-build-standalone's platforms except s390x; +# see dev/release_artifacts/build.sh. Linux x86_64 runners cross-compile the +# files for other Linux CPUs. macOS and Windows files need those OSes' SDKs, +# so they're built on those OSes. +# - `SHA256SUMS` and `artifacts_manifest.json` for the built platform binaries. +# - The release source archive (`rules_python-.tar.gz`) and +# `release_notes.txt`. +# - The `bazel_runfiles` wheel (`//python/runfiles:wheel.dist`). name: "Release: Build Artifacts" on: + pull_request: workflow_call: inputs: ref: @@ -20,7 +26,7 @@ on: type: string permissions: - contents: write + contents: read defaults: run: @@ -65,7 +71,7 @@ jobs: - triple: riscv64gc-unknown-linux-musl runner: ubuntu-24.04 env: - REF: ${{ inputs.ref || github.ref_name }} + REF: ${{ inputs.ref || github.sha }} TRIPLE: ${{ matrix.triple }} steps: - name: Checkout @@ -96,8 +102,8 @@ jobs: if: steps.build.outputs.files != '' run: dev/release_artifacts/verify.sh "$TRIPLE" - # Stores the files with this workflow run so the publish job (and users - # viewing the run) can download them. + # Stores the files with this workflow run so the package job, caller + # workflow, and users viewing the run can download them. - name: Upload artifacts if: steps.build.outputs.files != '' uses: actions/upload-artifact@v7 @@ -106,14 +112,49 @@ jobs: path: ${{ steps.build.outputs.files }} if-no-files-found: error - publish: - name: Publish artifacts + build_wheel: + name: Build wheel + runs-on: ubuntu-24.04 + permissions: + contents: read + timeout-minutes: 30 + steps: + - name: Checkout + uses: actions/checkout@v7 + with: + ref: ${{ inputs.ref }} + persist-credentials: false + + - name: Setup Bazel + uses: ./.github/actions/setup-bazel + + - name: Build runfiles wheel + run: >- + bazel build + ${{ inputs.ref && format('--stamp --embed_label={0}', inputs.ref) || '' }} + //python/runfiles:wheel.dist + //python/runfiles:wheel.publish + //tools/publish:twine + + - name: Check runfiles wheel + run: bazel-bin/tools/publish/twine check bazel-bin/python/runfiles/dist/*.whl + + - name: Upload runfiles wheel + uses: actions/upload-artifact@v7 + with: + name: release-wheel + path: bazel-bin/python/runfiles/dist/*.whl + if-no-files-found: error + + package: + name: Package release needs: build runs-on: ubuntu-24.04 permissions: - contents: write + contents: read + timeout-minutes: 15 env: - REF: ${{ inputs.ref || github.ref_name }} + REF: ${{ inputs.ref || github.sha }} steps: - name: Checkout uses: actions/checkout@v7 @@ -141,20 +182,9 @@ jobs: if: >- hashFiles('dev/release_artifacts/generate_artifacts_manifest.py') != '' && hashFiles('release-artifacts/*') != '' - run: | - dev/release_artifacts/generate_artifacts_manifest.py \ - "$REF" release-artifacts artifacts_manifest.json - - - name: Upload artifacts to GitHub release - if: hashFiles('release-artifacts/*') != '' - uses: softprops/action-gh-release@v3 - with: - tag_name: ${{ inputs.ref || github.ref_name }} - draft: true - fail_on_unmatched_files: true - files: | - release-artifacts/* - SHA256SUMS + run: >- + dev/release_artifacts/generate_artifacts_manifest.py + "$REF" release-artifacts artifacts_manifest.json - name: Upload artifacts manifest if: hashFiles('artifacts_manifest.json') != '' @@ -163,3 +193,16 @@ jobs: name: release-artifacts-manifest path: artifacts_manifest.json if-no-files-found: error + + - name: Create release archive and notes + run: .github/workflows/create_archive_and_notes.sh ${{ inputs.ref }} + + - name: Upload release archive and notes + uses: actions/upload-artifact@v7 + with: + name: release-archive + path: | + rules_python-*.tar.gz + release_notes.txt + SHA256SUMS + if-no-files-found: error diff --git a/.github/workflows/release_publish.yaml b/.github/workflows/release_publish.yaml index 22109eb79f..2209e67353 100644 --- a/.github/workflows/release_publish.yaml +++ b/.github/workflows/release_publish.yaml @@ -46,25 +46,14 @@ on: default: true jobs: - create_release: - name: Create draft release - runs-on: ubuntu-latest - steps: - - name: Create draft release - uses: softprops/action-gh-release@v3 - with: - draft: true - prerelease: ${{ contains( (inputs.tag_name || github.ref), '-rc') }} - tag_name: ${{ inputs.tag_name || github.ref_name }} - build_artifacts: - # Builds release artifacts and uploads them to the draft release. A failed - # build only adds a warning to this run, since releases don't require the - # artifacts yet. The run fails if a build hangs past the job timeout, the - # built files fail their checks, or the ref's dev/platforms lacks a job's + # Builds the release archive, release notes, runfiles wheel, and platform + # binaries. A failed platform binary build only adds a warning to this run, + # since releases don't require those binaries yet. The run fails if a build + # hangs past the job timeout, the built files fail their checks, the wheel + # or archive fails to build, or the ref's dev/platforms lacks a job's # platform. name: Build artifacts - needs: create_release uses: ./.github/workflows/release_build_artifacts.yaml with: ref: ${{ inputs.tag_name || github.ref_name }} @@ -74,12 +63,20 @@ jobs: needs: build_artifacts runs-on: ubuntu-latest steps: - - name: Checkout - uses: actions/checkout@v7 + - name: Download release archive and notes + uses: actions/download-artifact@v8 with: - ref: ${{ inputs.tag_name || github.ref_name }} - - name: Create release archive and notes - run: .github/workflows/create_archive_and_notes.sh ${{ inputs.tag_name || github.ref_name }} + name: release-archive + + - name: Download built artifacts + uses: actions/download-artifact@v8 + with: + # Match platform triples (--), excluding + # release-artifacts-manifest. + pattern: release-artifacts-*-* + path: release-artifacts + merge-multiple: true + - name: Release uses: softprops/action-gh-release@v3 with: @@ -89,7 +86,10 @@ jobs: draft: false prerelease: ${{ contains( (inputs.tag_name || github.ref), '-rc') }} fail_on_unmatched_files: true - files: rules_python-*.tar.gz + files: | + rules_python-*.tar.gz + release-artifacts/* + SHA256SUMS tag_name: ${{ inputs.tag_name || github.ref_name }} publish_bcr: diff --git a/tests/workflows/BUILD.bazel b/tests/workflows/BUILD.bazel index 5ddc52046c..f1e56256bd 100644 --- a/tests/workflows/BUILD.bazel +++ b/tests/workflows/BUILD.bazel @@ -42,6 +42,7 @@ pytest_test( name = "release_build_artifacts_test", srcs = ["release_build_artifacts_test.py"], data = [ + "//.github/workflows:create_archive_and_notes.sh", "//.github/workflows:release_build_artifacts.yaml", "//dev/platforms:BUILD.bazel", ], diff --git a/tests/workflows/release_build_artifacts_test.py b/tests/workflows/release_build_artifacts_test.py index 87c2de7e6a..7f89a30ded 100644 --- a/tests/workflows/release_build_artifacts_test.py +++ b/tests/workflows/release_build_artifacts_test.py @@ -2,13 +2,23 @@ from __future__ import annotations +import hashlib import re +import subprocess +import tarfile +from pathlib import Path import yaml from python.runfiles import runfiles +def _load_workflow() -> dict: + root = runfiles.CreateOrRaise().root() / "rules_python" + workflow_path = root / ".github/workflows/release_build_artifacts.yaml" + return yaml.safe_load(workflow_path.read_text(encoding="utf-8")) + + def test_matrix_matches_dev_platforms(): """Each platform in dev/platforms has one matrix job, and vice versa. @@ -16,8 +26,7 @@ def test_matrix_matches_dev_platforms(): its platform is missing. """ root = runfiles.CreateOrRaise().root() / "rules_python" - workflow_path = root / ".github/workflows/release_build_artifacts.yaml" - workflow = yaml.safe_load(workflow_path.read_text(encoding="utf-8")) + workflow = _load_workflow() triples = [ entry["triple"] for entry in workflow["jobs"]["build"]["strategy"]["matrix"]["include"] @@ -34,3 +43,128 @@ def test_matrix_matches_dev_platforms(): assert set(triples) == set(platforms), "Matrix (left) vs. dev/platforms (right)" duplicates = sorted({triple for triple in triples if triples.count(triple) > 1}) assert not duplicates, f"Triples with more than one job: {duplicates}" + + +def test_workflow_is_safe_for_presubmit(): + """Verifies that release_build_artifacts.yaml is read-only and never publishes.""" + workflow = _load_workflow() + + # PyYAML parses unquoted `on:` as boolean True in YAML 1.1. + triggers = workflow.get("on") or workflow.get(True, {}) + assert "pull_request" in triggers + + assert workflow.get("permissions") == {"contents": "read"} + + for job_name, job in workflow["jobs"].items(): + if "permissions" in job: + assert job["permissions"] == {"contents": "read"}, ( + f"Job '{job_name}' must not request write permissions" + ) + uses = job.get("uses", "") + assert "publish" not in uses, f"Job '{job_name}' calls publish workflow: {uses}" + for step in job.get("steps", []): + step_uses = step.get("uses", "") + assert not step_uses.startswith("softprops/action-gh-release"), ( + f"Job '{job_name}' creates/updates a GitHub release" + ) + step_run = step.get("run", "") + assert "wheel.publish" not in step_run or "bazel build" in step_run, ( + f"Job '{job_name}' runs wheel.publish instead of building it" + ) + + +def test_create_archive_and_notes_fetches_and_expands_tag(tmp_path: Path): + """Verifies create_archive_and_notes.sh fetches missing tags and expands export-subst.""" + script = ( + runfiles.CreateOrRaise().root() + / "rules_python/.github/workflows/create_archive_and_notes.sh" + ) + origin = tmp_path / "origin" + origin.mkdir() + + def git(cwd: Path, *args: str) -> None: + subprocess.run( + ["git", *args], + cwd=cwd, + check=True, + capture_output=True, + text=True, + ) + + git(origin, "init") + git(origin, "config", "user.email", "test@example.com") + git(origin, "config", "user.name", "Test User") + + (origin / ".gitattributes").write_text( + "python/features.bzl export-subst\n", encoding="utf-8" + ) + (origin / "python").mkdir() + (origin / "python/features.bzl").write_text( + '_VERSION_PRIVATE = "$Format:%(describe:tags=true)$"\n' + '_COMMIT_PRIVATE = "$Format:%H$"\n', + encoding="utf-8", + ) + git(origin, "add", ".gitattributes", "python/features.bzl") + git(origin, "commit", "-m", "Release commit") + git(origin, "tag", "2.5.0") + release_sha = subprocess.check_output( + ["git", "rev-parse", "HEAD"], cwd=origin, text=True + ).strip() + + (origin / "README.md").write_text("later commit on main\n", encoding="utf-8") + git(origin, "add", "README.md") + git(origin, "commit", "-m", "Later commit") + + # Clone without tags at the later commit (matches actions/checkout default) + clone = tmp_path / "clone" + git(tmp_path, "clone", "--no-tags", str(origin), str(clone)) + + # Git < 2.35 (e.g. Debian 11 / Ubuntu 22.04 on Buildkite) does not support + # %(describe:tags=true) and leaves the literal placeholder unexpanded. + supports_describe_tags = ( + subprocess.check_output( + ["git", "log", "-1", "--format=%(describe:tags=true)"], + cwd=origin, + text=True, + ).strip() + != "%(describe:tags=true)" + ) + + # Presubmit run with no tag argument on an untagged commit uses HEAD SHA + head_sha = subprocess.check_output( + ["git", "rev-parse", "HEAD"], cwd=clone, text=True + ).strip() + subprocess.run([str(script)], cwd=clone, check=True) + presubmit_archive = clone / f"rules_python-{head_sha}.tar.gz" + assert presubmit_archive.is_file() + with tarfile.open(presubmit_archive, "r:gz") as tar: + features_member = tar.extractfile( + f"rules_python-{head_sha}/python/features.bzl" + ) + assert features_member is not None # type assert + features_content = features_member.read().decode("utf-8") + assert f'_COMMIT_PRIVATE = "{head_sha}"' in features_content + if supports_describe_tags: + assert '_VERSION_PRIVATE = ""' in features_content + + # Release run fetches and checks out the tag from origin + subprocess.run([str(script), "2.5.0"], cwd=clone, check=True) + archive = clone / "rules_python-2.5.0.tar.gz" + notes = clone / "release_notes.txt" + assert archive.is_file() + assert notes.is_file() + + expected_sha = hashlib.sha256(archive.read_bytes()).hexdigest() + notes_text = notes.read_text(encoding="utf-8") + assert expected_sha in notes_text + assert 'version = "2.5.0"' in notes_text + + with tarfile.open(archive, "r:gz") as tar: + names = tar.getnames() + assert "rules_python-2.5.0/README.md" not in names + features_member = tar.extractfile("rules_python-2.5.0/python/features.bzl") + assert features_member is not None # type assert + features_content = features_member.read().decode("utf-8") + assert f'_COMMIT_PRIVATE = "{release_sha}"' in features_content + if supports_describe_tags: + assert '_VERSION_PRIVATE = "2.5.0"' in features_content