diff --git a/.github/workflows/create_archive_and_notes.sh b/.github/workflows/create_archive_and_notes.sh index d3e0b583ed..cec8d82a97 100755 --- a/.github/workflows/create_archive_and_notes.sh +++ b/.github/workflows/create_archive_and_notes.sh @@ -31,6 +31,8 @@ git checkout "$TAG" # A prefix is added to better match the GitHub generated archives. PREFIX="rules_python-${TAG}" ARCHIVE="rules_python-$TAG.tar.gz" +# TODO(https://github.com/bazel-contrib/rules_python/issues/4250): Add +# artifacts_manifest.json to the release archive. git archive --format=tar "--prefix=${PREFIX}/" "$TAG" | gzip > "$ARCHIVE" SHA=$(shasum -a 256 "$ARCHIVE" | awk '{print $1}') diff --git a/.github/workflows/release_build_tools.yaml b/.github/workflows/release_build_artifacts.yaml similarity index 61% rename from .github/workflows/release_build_tools.yaml rename to .github/workflows/release_build_artifacts.yaml index 15a651e69e..744fc0779e 100644 --- a/.github/workflows/release_build_tools.yaml +++ b/.github/workflows/release_build_artifacts.yaml @@ -3,10 +3,7 @@ # see dev/release_artifacts/build.sh. Linux x86_64 runners cross-compile the # files for other Linux CPUs. macOS and Windows files need those OSes' SDKs, so # they're built on those OSes. -# -# For now, this only verifies that the artifacts build; they aren't attached to -# releases yet. -name: "Release: Build Tools" +name: "Release: Build Artifacts" on: workflow_call: @@ -23,7 +20,7 @@ on: type: string permissions: - contents: read + contents: write defaults: run: @@ -33,6 +30,8 @@ jobs: build: name: Build ${{ matrix.triple }} runs-on: ${{ matrix.runner }} + permissions: + contents: read # Builds take a few minutes; cancel hung ones instead of holding a runner # for the default 6 hours. timeout-minutes: 30 @@ -97,12 +96,70 @@ jobs: if: steps.build.outputs.files != '' run: dev/release_artifacts/verify.sh "$TRIPLE" - # Stores the files with this workflow run so they can be downloaded from - # the run's page. Nothing is added to the GitHub release. + # Stores the files with this workflow run so the publish job (and users + # viewing the run) can download them. - name: Upload artifacts if: steps.build.outputs.files != '' uses: actions/upload-artifact@v7 with: - name: release-tools-${{ matrix.triple }} + name: release-artifacts-${{ matrix.triple }} path: ${{ steps.build.outputs.files }} if-no-files-found: error + + publish: + name: Publish artifacts + needs: build + runs-on: ubuntu-24.04 + permissions: + contents: write + env: + REF: ${{ inputs.ref || github.ref_name }} + steps: + - name: Checkout + uses: actions/checkout@v7 + with: + ref: ${{ inputs.ref }} + persist-credentials: false + + - name: Download built artifacts + uses: actions/download-artifact@v8 + with: + pattern: release-artifacts-* + path: release-artifacts + merge-multiple: true + + # The hashFiles(...) checks below are for backwards compatibility with + # patch releases of older release branches that predate these scripts and + # artifacts (and while artifact build failures are non-fatal). They can be + # removed once older release branches are no longer supported. + - name: Generate SHA256SUMS + if: hashFiles('release-artifacts/*') != '' + working-directory: release-artifacts + run: sha256sum * > ../SHA256SUMS + + - name: Generate artifacts manifest + if: >- + hashFiles('dev/release_artifacts/generate_artifacts_manifest.py') != '' && + hashFiles('release-artifacts/*') != '' + run: | + dev/release_artifacts/generate_artifacts_manifest.py \ + "$REF" release-artifacts artifacts_manifest.json + + - name: Upload artifacts to GitHub release + if: hashFiles('release-artifacts/*') != '' + uses: softprops/action-gh-release@v3 + with: + tag_name: ${{ inputs.ref || github.ref_name }} + draft: true + fail_on_unmatched_files: true + files: | + release-artifacts/* + SHA256SUMS + + - name: Upload artifacts manifest + if: hashFiles('artifacts_manifest.json') != '' + uses: actions/upload-artifact@v7 + with: + name: release-artifacts-manifest + path: artifacts_manifest.json + if-no-files-found: error diff --git a/.github/workflows/release_publish.yaml b/.github/workflows/release_publish.yaml index 9b5d647c3d..22109eb79f 100644 --- a/.github/workflows/release_publish.yaml +++ b/.github/workflows/release_publish.yaml @@ -46,19 +46,32 @@ on: default: true jobs: - build_tools: - # For now, this only verifies that the tools build. They aren't attached - # to the release, so no other job waits on this one. A failed build only - # adds a warning to this run. The run fails if a build hangs past the job - # timeout, the built files fail their checks, or the ref's dev/platforms - # lacks a job's platform. - name: Build tools - uses: ./.github/workflows/release_build_tools.yaml + create_release: + name: Create draft release + runs-on: ubuntu-latest + steps: + - name: Create draft release + uses: softprops/action-gh-release@v3 + with: + draft: true + prerelease: ${{ contains( (inputs.tag_name || github.ref), '-rc') }} + tag_name: ${{ inputs.tag_name || github.ref_name }} + + build_artifacts: + # Builds release artifacts and uploads them to the draft release. A failed + # build only adds a warning to this run, since releases don't require the + # artifacts yet. The run fails if a build hangs past the job timeout, the + # built files fail their checks, or the ref's dev/platforms lacks a job's + # platform. + name: Build artifacts + needs: create_release + uses: ./.github/workflows/release_build_artifacts.yaml with: ref: ${{ inputs.tag_name || github.ref_name }} release: name: Release + needs: build_artifacts runs-on: ubuntu-latest steps: - name: Checkout @@ -73,6 +86,7 @@ jobs: # Use GH feature to populate the changelog automatically generate_release_notes: true body_path: release_notes.txt + draft: false prerelease: ${{ contains( (inputs.tag_name || github.ref), '-rc') }} fail_on_unmatched_files: true files: rules_python-*.tar.gz diff --git a/RELEASING.md b/RELEASING.md index 2bf5e59fd0..a52bf47d8a 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -67,11 +67,12 @@ you can set the `publish_to_pypi` input to `false`: gh workflow run release_publish.yaml --ref -f publish_to_pypi=false ``` -### Prebuilt tools +### Release artifacts The release workflow also builds `//dev/release_artifacts:artifacts_for_release` -for each platform to verify it builds. For now, the files aren't attached to -the release, so a failed "Build tools" job doesn't block it. +for each platform and uploads the built artifacts and `SHA256SUMS` to the GitHub +release. For now, build failures only emit a warning rather than failing the +release. Presubmit builds it for every platform too: natively on each CI host, and through `//dev/release_artifacts:cross_compiled_artifacts` for the others. diff --git a/dev/release_artifacts/BUILD.bazel b/dev/release_artifacts/BUILD.bazel index d81b5294b1..8f5f0c673e 100644 --- a/dev/release_artifacts/BUILD.bazel +++ b/dev/release_artifacts/BUILD.bazel @@ -115,3 +115,7 @@ bzl_library( "@rules_cc//cc:find_cc_toolchain_bzl", ], ) + +exports_files([ + "generate_artifacts_manifest.py", +]) diff --git a/dev/release_artifacts/generate_artifacts_manifest.py b/dev/release_artifacts/generate_artifacts_manifest.py new file mode 100755 index 0000000000..c6c5af236d --- /dev/null +++ b/dev/release_artifacts/generate_artifacts_manifest.py @@ -0,0 +1,122 @@ +#!/usr/bin/env python3 +"""Generates an artifacts_manifest.json file for built release artifacts.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import re +import sys +from pathlib import Path +from typing import TypedDict + + +class ArtifactEntry(TypedDict): + exec_compatible_with: list[str] + filename: str + sha256: str + urls: list[str] + + +_CPU_TO_CONSTRAINT = { + "aarch64": "@platforms//cpu:aarch64", + "armv7": "@platforms//cpu:armv7", + "i686": "@platforms//cpu:x86_32", + "powerpc64le": "@platforms//cpu:ppc64le", + "riscv64gc": "@platforms//cpu:riscv64", + "x86_64": "@platforms//cpu:x86_64", +} + +_FILENAME_RE = re.compile(r"^(.+)-((?:" + "|".join(_CPU_TO_CONSTRAINT) + r")-.+)$") + + +def _cpu_constraint(triple: str) -> str: + cpu = triple.split("-", 1)[0] + if cpu in _CPU_TO_CONSTRAINT: + return _CPU_TO_CONSTRAINT[cpu] + raise ValueError(f"Unsupported CPU in triple: {triple}") + + +def _os_constraint(triple: str) -> str: + if triple.endswith("-apple-darwin"): + return "@platforms//os:macos" + if "-pc-windows-" in triple: + return "@platforms//os:windows" + if "-linux-" in triple: + return "@platforms//os:linux" + raise ValueError(f"Unsupported OS in triple: {triple}") + + +def generate_manifest( + tag: str, + artifacts_dir: Path, + repo: str, +) -> dict[str, dict[str, list[ArtifactEntry]]]: + artifacts: dict[str, list[ArtifactEntry]] = {} + for file_path in sorted(artifacts_dir.iterdir()): + filename = file_path.name + base = filename.removesuffix(".exe") + match = _FILENAME_RE.match(base) + if not match: + raise ValueError( + f"Cannot parse artifact and triple from filename: {filename}" + ) + artifact, triple = match.group(1), match.group(2) + + cpu = _cpu_constraint(triple) + os_constraint = _os_constraint(triple) + sha256 = hashlib.sha256(file_path.read_bytes()).hexdigest() + url = f"https://github.com/{repo}/releases/download/{tag}/{filename}" + + artifacts.setdefault(artifact, []).append( + { + "exec_compatible_with": [cpu, os_constraint], + "filename": filename, + "sha256": sha256, + "urls": [url], + } + ) + + for entries in artifacts.values(): + entries.sort(key=lambda entry: entry["filename"]) + + return {"artifacts": dict(sorted(artifacts.items()))} + + +def _main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("tag", help="Release tag name (e.g. 2.4.0).") + parser.add_argument( + "artifacts_dir", + type=Path, + help="Directory containing built -[.exe] files.", + ) + parser.add_argument( + "output_file", + type=Path, + help="Path to the artifacts_manifest.json file to write.", + ) + parser.add_argument( + "repo", + nargs="?", + default=os.environ.get("GITHUB_REPOSITORY") or "bazel-contrib/rules_python", + help="GitHub owner/repository name.", + ) + args = parser.parse_args(argv) + try: + manifest = generate_manifest(args.tag, args.artifacts_dir, args.repo) + except ValueError as err: + print(f"::error::{err}") + return 1 + args.output_file.parent.mkdir(parents=True, exist_ok=True) + args.output_file.write_text( + json.dumps(manifest, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + return 0 + + +if __name__ == "__main__": + sys.exit(_main()) diff --git a/dev/release_artifacts/release_files.bzl b/dev/release_artifacts/release_files.bzl index ae6563c6c4..32a84b3c91 100644 --- a/dev/release_artifacts/release_files.bzl +++ b/dev/release_artifacts/release_files.bzl @@ -42,15 +42,47 @@ _release_transition = transition( ], ) +def _triple(ctx): + cpu = ctx.attr.target_cpu + os = ctx.attr.target_os + if os == "macos": + return "{}-apple-darwin".format(cpu) + if os == "windows": + return "{}-pc-windows-msvc".format(cpu) + if os == "linux": + if cpu == "armv7": + return "{}-unknown-linux-musleabihf".format(cpu) + if ctx.attr.target_libc == "musl": + return "{}-unknown-linux-musl".format(cpu) + return "{}-unknown-linux-gnu".format(cpu) + fail("Unsupported target_os: {}".format(os)) + def _release_files_impl(ctx): + triple = _triple(ctx) + files = [] + for src in ctx.files.srcs: + if src.extension: + stem = src.basename[:-len(src.extension) - 1] + ext = "." + src.extension + else: + stem = src.basename + ext = "" + out = ctx.actions.declare_file("{}-{}{}".format(stem, triple, ext)) + ctx.actions.symlink( + output = out, + target_file = src, + is_executable = True, + ) + files.append(out) + manifest = ctx.actions.declare_file(ctx.label.name + ".txt") ctx.actions.write( output = manifest, # End every line with "\n", including the last: `while read` loops skip # an unterminated last line, and build.sh adds a line after the paths. - content = "".join([file.path + "\n" for file in ctx.files.srcs]), + content = "".join([file.path + "\n" for file in files]), ) - return [DefaultInfo(files = depset([manifest] + ctx.files.srcs))] + return [DefaultInfo(files = depset([manifest] + files))] _release_files = rule( implementation = _release_files_impl, @@ -60,11 +92,16 @@ _release_files = rule( cfg = _release_transition, doc = "The files to build and list.", ), + "target_cpu": attr.string( + doc = "The CPU that the files are built for. Set by the macro.", + mandatory = True, + ), "target_libc": attr.string( doc = "The libc that the files are built for. Set by the macro.", ), "target_os": attr.string( doc = "The OS that the files are built for. Set by the macro.", + mandatory = True, ), }, ) @@ -72,16 +109,16 @@ _release_files = rule( def release_files(name, srcs, **kwargs): """Builds files for a release and writes their paths to `.txt`. - The files are built for the target platform (`--platforms`), with release - settings for its OS and libc: optimized, and linked so that they run on - older OS versions than the build machine's, without extra runtime - libraries. + The files are named `-[.]` and built for the target + platform (`--platforms`), with release settings for its OS and libc: + optimized, and linked so that they run on older OS versions than the build + machine's, without extra runtime libraries. The paths, one per line, are relative to the execroot, e.g. - `bazel-out/k8-opt-ST-1234/bin/foo/foo`. Paths of generated files also - resolve from the workspace root through the `bazel-out` convenience - symlink. This lets scripts find the files without running `bazel cquery`, - which re-analyzes the build. + `bazel-out/k8-opt/bin/dev/release_artifacts/foo-x86_64-unknown-linux-gnu`. + Paths of generated files also resolve from the workspace root through the + `bazel-out` convenience symlink. This lets scripts find the files without + running `bazel cquery`, which re-analyzes the build. Args: name: The target name. @@ -91,14 +128,22 @@ def release_files(name, srcs, **kwargs): _release_files( name = name, srcs = srcs, + target_cpu = select({ + Label("@platforms//cpu:aarch64"): "aarch64", + Label("@platforms//cpu:armv7"): "armv7", + Label("@platforms//cpu:ppc64le"): "powerpc64le", + Label("@platforms//cpu:riscv64"): "riscv64gc", + Label("@platforms//cpu:x86_32"): "i686", + Label("@platforms//cpu:x86_64"): "x86_64", + }), target_libc = select({ "//dev/platforms:musl": "musl", "//conditions:default": "", }), target_os = select({ Label("@platforms//os:linux"): "linux", + labels.PLATFORMS_OS_MACOS: "macos", labels.PLATFORMS_OS_WINDOWS: "windows", - "//conditions:default": "", }), **kwargs ) diff --git a/tests/workflows/BUILD.bazel b/tests/workflows/BUILD.bazel index 76656e5d48..5ddc52046c 100644 --- a/tests/workflows/BUILD.bazel +++ b/tests/workflows/BUILD.bazel @@ -39,10 +39,10 @@ pytest_test( ) pytest_test( - name = "release_build_tools_test", - srcs = ["release_build_tools_test.py"], + name = "release_build_artifacts_test", + srcs = ["release_build_artifacts_test.py"], data = [ - "//.github/workflows:release_build_tools.yaml", + "//.github/workflows:release_build_artifacts.yaml", "//dev/platforms:BUILD.bazel", ], target_compatible_with = NOT_WINDOWS, @@ -62,3 +62,15 @@ pytest_test( "@pypi//pyyaml", ], ) + +pytest_test( + name = "release_artifacts_test", + srcs = ["release_artifacts_test.py"], + data = [ + "//dev/release_artifacts:generate_artifacts_manifest.py", + ], + target_compatible_with = NOT_WINDOWS, + deps = [ + "//python/runfiles", + ], +) diff --git a/tests/workflows/release_artifacts_test.py b/tests/workflows/release_artifacts_test.py new file mode 100644 index 0000000000..c49672b74b --- /dev/null +++ b/tests/workflows/release_artifacts_test.py @@ -0,0 +1,80 @@ +"""Tests for release artifact manifest generation.""" + +from __future__ import annotations + +import hashlib +import json +import subprocess +from pathlib import Path + +from python.runfiles import runfiles + + +def test_generate_artifacts_manifest(tmp_path: Path): + generate_script = ( + runfiles.CreateOrRaise().root() + / "rules_python/dev/release_artifacts/generate_artifacts_manifest.py" + ) + + artifacts_dir = tmp_path / "release-artifacts" + artifacts_dir.mkdir(parents=True) + manifest_file = tmp_path / "artifacts_manifest.json" + + linux_bin = artifacts_dir / "exe_zip_maker-x86_64-unknown-linux-gnu" + linux_bin.write_bytes(b"linux-binary-contents") + linux_sha = hashlib.sha256(b"linux-binary-contents").hexdigest() + + win_bin = artifacts_dir / "exe_zip_maker-x86_64-pc-windows-msvc.exe" + win_bin.write_bytes(b"windows-binary-contents") + win_sha = hashlib.sha256(b"windows-binary-contents").hexdigest() + + mac_bin = artifacts_dir / "exe_zip_maker-aarch64-apple-darwin" + mac_bin.write_bytes(b"mac-binary-contents") + mac_sha = hashlib.sha256(b"mac-binary-contents").hexdigest() + + subprocess.run( + [ + str(generate_script), + "2.4.0", + str(artifacts_dir), + str(manifest_file), + "bazel-contrib/rules_python", + ], + check=True, + ) + + manifest = json.loads(manifest_file.read_text(encoding="utf-8")) + base_url = "https://github.com/bazel-contrib/rules_python/releases/download/2.4.0" + assert manifest == { + "artifacts": { + "exe_zip_maker": [ + { + "exec_compatible_with": [ + "@platforms//cpu:aarch64", + "@platforms//os:macos", + ], + "filename": "exe_zip_maker-aarch64-apple-darwin", + "sha256": mac_sha, + "urls": [f"{base_url}/exe_zip_maker-aarch64-apple-darwin"], + }, + { + "exec_compatible_with": [ + "@platforms//cpu:x86_64", + "@platforms//os:windows", + ], + "filename": "exe_zip_maker-x86_64-pc-windows-msvc.exe", + "sha256": win_sha, + "urls": [f"{base_url}/exe_zip_maker-x86_64-pc-windows-msvc.exe"], + }, + { + "exec_compatible_with": [ + "@platforms//cpu:x86_64", + "@platforms//os:linux", + ], + "filename": "exe_zip_maker-x86_64-unknown-linux-gnu", + "sha256": linux_sha, + "urls": [f"{base_url}/exe_zip_maker-x86_64-unknown-linux-gnu"], + }, + ] + } + } diff --git a/tests/workflows/release_build_tools_test.py b/tests/workflows/release_build_artifacts_test.py similarity index 91% rename from tests/workflows/release_build_tools_test.py rename to tests/workflows/release_build_artifacts_test.py index f9eb9e7250..87c2de7e6a 100644 --- a/tests/workflows/release_build_tools_test.py +++ b/tests/workflows/release_build_artifacts_test.py @@ -1,4 +1,4 @@ -"""Tests for the release_build_tools.yaml workflow.""" +"""Tests for the release_build_artifacts.yaml workflow.""" from __future__ import annotations @@ -16,7 +16,7 @@ def test_matrix_matches_dev_platforms(): its platform is missing. """ root = runfiles.CreateOrRaise().root() / "rules_python" - workflow_path = root / ".github/workflows/release_build_tools.yaml" + workflow_path = root / ".github/workflows/release_build_artifacts.yaml" workflow = yaml.safe_load(workflow_path.read_text(encoding="utf-8")) triples = [ entry["triple"]