From 030851916eb53efde74625786c2ad055d7660b4e Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Fri, 16 Oct 2026 15:26:05 -0700 Subject: [PATCH 1/8] build(release): upload prebuilt tools and manifest to GitHub release Prebuilt release tools were previously built and verified in CI, but never attached to the GitHub release or indexed in a manifest for downstream consumption. Rename built release artifacts to include their target triple, upload them to a draft GitHub release during the tool build workflow, and generate per-binary JSON manifest snippets with SHA-256 digests, platform constraints, and download URLs. The release publish workflow then merges these snippets into `tools_manifest.json` and publishes it alongside the release archive when finalizing the release. Work towards https://github.com/bazel-contrib/rules_python/issues/4216 Work towards https://github.com/bazel-contrib/rules_python/issues/4250 --- .github/workflows/create_archive_and_notes.sh | 2 + .github/workflows/release_build_tools.yaml | 60 +++++++- .github/workflows/release_publish.yaml | 36 ++++- RELEASING.md | 5 +- dev/release_artifacts/BUILD.bazel | 5 + .../generate_tool_manifest_snippet.py | 113 +++++++++++++++ .../merge_manifest_snippets.py | 51 +++++++ dev/release_artifacts/release_files.bzl | 67 +++++++-- tests/workflows/BUILD.bazel | 13 ++ tests/workflows/release_artifacts_test.py | 131 ++++++++++++++++++ 10 files changed, 458 insertions(+), 25 deletions(-) create mode 100755 dev/release_artifacts/generate_tool_manifest_snippet.py create mode 100755 dev/release_artifacts/merge_manifest_snippets.py create mode 100644 tests/workflows/release_artifacts_test.py diff --git a/.github/workflows/create_archive_and_notes.sh b/.github/workflows/create_archive_and_notes.sh index d3e0b583ed..755b29f794 100755 --- a/.github/workflows/create_archive_and_notes.sh +++ b/.github/workflows/create_archive_and_notes.sh @@ -31,6 +31,8 @@ git checkout "$TAG" # A prefix is added to better match the GitHub generated archives. PREFIX="rules_python-${TAG}" ARCHIVE="rules_python-$TAG.tar.gz" +# TODO(https://github.com/bazel-contrib/rules_python/issues/4250): Add +# tools_manifest.json to the release archive. git archive --format=tar "--prefix=${PREFIX}/" "$TAG" | gzip > "$ARCHIVE" SHA=$(shasum -a 256 "$ARCHIVE" | awk '{print $1}') diff --git a/.github/workflows/release_build_tools.yaml b/.github/workflows/release_build_tools.yaml index 15a651e69e..3e2eedcde7 100644 --- a/.github/workflows/release_build_tools.yaml +++ b/.github/workflows/release_build_tools.yaml @@ -3,9 +3,6 @@ # see dev/release_artifacts/build.sh. Linux x86_64 runners cross-compile the # files for other Linux CPUs. macOS and Windows files need those OSes' SDKs, so # they're built on those OSes. -# -# For now, this only verifies that the artifacts build; they aren't attached to -# releases yet. name: "Release: Build Tools" on: @@ -23,7 +20,7 @@ on: type: string permissions: - contents: read + contents: write defaults: run: @@ -33,6 +30,8 @@ jobs: build: name: Build ${{ matrix.triple }} runs-on: ${{ matrix.runner }} + permissions: + contents: read # Builds take a few minutes; cancel hung ones instead of holding a runner # for the default 6 hours. timeout-minutes: 30 @@ -97,8 +96,8 @@ jobs: if: steps.build.outputs.files != '' run: dev/release_artifacts/verify.sh "$TRIPLE" - # Stores the files with this workflow run so they can be downloaded from - # the run's page. Nothing is added to the GitHub release. + # Stores the files with this workflow run so the publish job (and users + # viewing the run) can download them. - name: Upload artifacts if: steps.build.outputs.files != '' uses: actions/upload-artifact@v7 @@ -106,3 +105,52 @@ jobs: name: release-tools-${{ matrix.triple }} path: ${{ steps.build.outputs.files }} if-no-files-found: error + + publish: + name: Publish tools + needs: build + runs-on: ubuntu-24.04 + permissions: + contents: write + env: + REF: ${{ inputs.ref || github.ref_name }} + steps: + - name: Checkout + uses: actions/checkout@v7 + with: + ref: ${{ inputs.ref }} + persist-credentials: false + + - name: Download built tools + uses: actions/download-artifact@v8 + with: + pattern: release-tools-* + path: release-tools + merge-multiple: true + + # The hashFiles(...) checks below are for backwards compatibility with + # patch releases of older release branches that predate these scripts and + # artifacts (and while tool build failures are non-fatal). They can be + # removed once older release branches are no longer supported. + - name: Generate manifest snippets + if: hashFiles('dev/release_artifacts/generate_tool_manifest_snippet.py') != '' + run: | + dev/release_artifacts/generate_tool_manifest_snippet.py \ + "$REF" release-tools manifest-snippets + + - name: Upload tools to GitHub release + if: hashFiles('release-tools/*') != '' + uses: softprops/action-gh-release@v3 + with: + tag_name: ${{ inputs.ref || github.ref_name }} + draft: true + fail_on_unmatched_files: true + files: release-tools/* + + - name: Upload manifest snippets + if: hashFiles('manifest-snippets/*.json') != '' + uses: actions/upload-artifact@v7 + with: + name: release-tools-manifest-snippets + path: manifest-snippets/*.json + if-no-files-found: error diff --git a/.github/workflows/release_publish.yaml b/.github/workflows/release_publish.yaml index 9b5d647c3d..a21b63fccf 100644 --- a/.github/workflows/release_publish.yaml +++ b/.github/workflows/release_publish.yaml @@ -46,25 +46,46 @@ on: default: true jobs: + create_release: + name: Create draft release + runs-on: ubuntu-latest + steps: + - name: Create draft release + uses: softprops/action-gh-release@v3 + with: + draft: true + prerelease: ${{ contains( (inputs.tag_name || github.ref), '-rc') }} + tag_name: ${{ inputs.tag_name || github.ref_name }} + build_tools: - # For now, this only verifies that the tools build. They aren't attached - # to the release, so no other job waits on this one. A failed build only - # adds a warning to this run. The run fails if a build hangs past the job - # timeout, the built files fail their checks, or the ref's dev/platforms - # lacks a job's platform. + # Builds prebuilt tools and uploads them to the draft release. A failed + # build only adds a warning to this run, since releases don't require the + # tools yet. The run fails if a build hangs past the job timeout, the built + # files fail their checks, or the ref's dev/platforms lacks a job's + # platform. name: Build tools + needs: create_release uses: ./.github/workflows/release_build_tools.yaml with: ref: ${{ inputs.tag_name || github.ref_name }} release: name: Release + needs: build_tools runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v7 with: ref: ${{ inputs.tag_name || github.ref_name }} + - name: Download tool manifest snippets + uses: actions/download-artifact@v8 + with: + pattern: release-tools-manifest-snippets + path: manifest-snippets + merge-multiple: true + - name: Merge tool manifest snippets + run: dev/release_artifacts/merge_manifest_snippets.py manifest-snippets tools_manifest.json - name: Create release archive and notes run: .github/workflows/create_archive_and_notes.sh ${{ inputs.tag_name || github.ref_name }} - name: Release @@ -73,9 +94,12 @@ jobs: # Use GH feature to populate the changelog automatically generate_release_notes: true body_path: release_notes.txt + draft: false prerelease: ${{ contains( (inputs.tag_name || github.ref), '-rc') }} fail_on_unmatched_files: true - files: rules_python-*.tar.gz + files: | + rules_python-*.tar.gz + tools_manifest.json tag_name: ${{ inputs.tag_name || github.ref_name }} publish_bcr: diff --git a/RELEASING.md b/RELEASING.md index 2bf5e59fd0..ff0f288f05 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -70,8 +70,9 @@ gh workflow run release_publish.yaml --ref -f publish_to_pypi=false ### Prebuilt tools The release workflow also builds `//dev/release_artifacts:artifacts_for_release` -for each platform to verify it builds. For now, the files aren't attached to -the release, so a failed "Build tools" job doesn't block it. +for each platform and uploads the built binaries and `tools_manifest.json` to +the GitHub release. For now, build failures only emit a warning rather than +failing the release. Presubmit builds it for every platform too: natively on each CI host, and through `//dev/release_artifacts:cross_compiled_artifacts` for the others. diff --git a/dev/release_artifacts/BUILD.bazel b/dev/release_artifacts/BUILD.bazel index d81b5294b1..a8dae9ff6b 100644 --- a/dev/release_artifacts/BUILD.bazel +++ b/dev/release_artifacts/BUILD.bazel @@ -115,3 +115,8 @@ bzl_library( "@rules_cc//cc:find_cc_toolchain_bzl", ], ) + +exports_files([ + "generate_tool_manifest_snippet.py", + "merge_manifest_snippets.py", +]) diff --git a/dev/release_artifacts/generate_tool_manifest_snippet.py b/dev/release_artifacts/generate_tool_manifest_snippet.py new file mode 100755 index 0000000000..096e75ac51 --- /dev/null +++ b/dev/release_artifacts/generate_tool_manifest_snippet.py @@ -0,0 +1,113 @@ +#!/usr/bin/env python3 +"""Generates a JSON manifest snippet for each built release binary.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import re +import sys +from pathlib import Path +from typing import Any + +_CPU_TO_CONSTRAINT = { + "aarch64": "@platforms//cpu:aarch64", + "armv7": "@platforms//cpu:armv7", + "i686": "@platforms//cpu:x86_32", + "powerpc64le": "@platforms//cpu:ppc64le", + "riscv64gc": "@platforms//cpu:riscv64", + "x86_64": "@platforms//cpu:x86_64", +} + +_FILENAME_RE = re.compile(r"^(.+)-((?:" + "|".join(_CPU_TO_CONSTRAINT) + r")-.+)$") + + +def _cpu_constraint(triple: str) -> str: + cpu = triple.split("-", 1)[0] + if cpu in _CPU_TO_CONSTRAINT: + return _CPU_TO_CONSTRAINT[cpu] + raise ValueError(f"Unsupported CPU in triple: {triple}") + + +def _os_constraint(triple: str) -> str: + if triple.endswith("-apple-darwin"): + return "@platforms//os:macos" + if "-pc-windows-" in triple: + return "@platforms//os:windows" + if "-linux-" in triple: + return "@platforms//os:linux" + raise ValueError(f"Unsupported OS in triple: {triple}") + + +def generate_snippets( + tag: str, + tools_dir: Path, + snippets_dir: Path, + repo: str, +) -> None: + snippets_dir.mkdir(parents=True, exist_ok=True) + if not tools_dir.is_dir(): + return + + for file_path in sorted(tools_dir.iterdir()): + if not file_path.is_file(): + continue + filename = file_path.name + base = filename.removesuffix(".exe") + match = _FILENAME_RE.match(base) + if not match: + raise ValueError(f"Cannot parse tool and triple from filename: {filename}") + tool, triple = match.group(1), match.group(2) + + cpu = _cpu_constraint(triple) + os_constraint = _os_constraint(triple) + sha256 = hashlib.sha256(file_path.read_bytes()).hexdigest() + url = f"https://github.com/{repo}/releases/download/{tag}/{filename}" + + snippet: dict[str, Any] = { + "tools": { + tool: [ + { + "exec_compatible_with": [cpu, os_constraint], + "filename": filename, + "sha256": sha256, + "urls": [url], + } + ] + } + } + snippet_path = snippets_dir / f"{base}.json" + snippet_path.write_text( + json.dumps(snippet, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("tag", help="Release tag name (e.g. 2.4.0).") + parser.add_argument( + "tools_dir", + type=Path, + help="Directory containing built -[.exe] files.", + ) + parser.add_argument( + "snippets_dir", + type=Path, + help="Output directory for generated *.json manifest snippets.", + ) + parser.add_argument( + "repo", + nargs="?", + default=os.environ.get("GITHUB_REPOSITORY") or "bazel-contrib/rules_python", + help="GitHub owner/repository name.", + ) + args = parser.parse_args(argv) + generate_snippets(args.tag, args.tools_dir, args.snippets_dir, args.repo) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/dev/release_artifacts/merge_manifest_snippets.py b/dev/release_artifacts/merge_manifest_snippets.py new file mode 100755 index 0000000000..a0b8fa9bfd --- /dev/null +++ b/dev/release_artifacts/merge_manifest_snippets.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python3 +"""Merges JSON manifest snippets into a single tools_manifest.json file.""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path +from typing import Any + + +def merge_snippets(snippets_dir: Path) -> dict[str, Any]: + tools: dict[str, list[dict[str, Any]]] = {} + if snippets_dir.is_dir(): + for snippet_path in sorted(snippets_dir.glob("*.json")): + data = json.loads(snippet_path.read_text(encoding="utf-8")) + for tool, entries in data.get("tools", {}).items(): + tools.setdefault(tool, []).extend(entries) + + for tool, entries in tools.items(): + entries.sort(key=lambda entry: entry["filename"]) + + return {"tools": dict(sorted(tools.items()))} + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "snippets_dir", + type=Path, + help="Directory containing per-binary *.json manifest snippets.", + ) + parser.add_argument( + "output_file", + type=Path, + help="Path to the merged tools_manifest.json file to write.", + ) + args = parser.parse_args(argv) + + manifest = merge_snippets(args.snippets_dir) + args.output_file.parent.mkdir(parents=True, exist_ok=True) + args.output_file.write_text( + json.dumps(manifest, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/dev/release_artifacts/release_files.bzl b/dev/release_artifacts/release_files.bzl index ae6563c6c4..32a84b3c91 100644 --- a/dev/release_artifacts/release_files.bzl +++ b/dev/release_artifacts/release_files.bzl @@ -42,15 +42,47 @@ _release_transition = transition( ], ) +def _triple(ctx): + cpu = ctx.attr.target_cpu + os = ctx.attr.target_os + if os == "macos": + return "{}-apple-darwin".format(cpu) + if os == "windows": + return "{}-pc-windows-msvc".format(cpu) + if os == "linux": + if cpu == "armv7": + return "{}-unknown-linux-musleabihf".format(cpu) + if ctx.attr.target_libc == "musl": + return "{}-unknown-linux-musl".format(cpu) + return "{}-unknown-linux-gnu".format(cpu) + fail("Unsupported target_os: {}".format(os)) + def _release_files_impl(ctx): + triple = _triple(ctx) + files = [] + for src in ctx.files.srcs: + if src.extension: + stem = src.basename[:-len(src.extension) - 1] + ext = "." + src.extension + else: + stem = src.basename + ext = "" + out = ctx.actions.declare_file("{}-{}{}".format(stem, triple, ext)) + ctx.actions.symlink( + output = out, + target_file = src, + is_executable = True, + ) + files.append(out) + manifest = ctx.actions.declare_file(ctx.label.name + ".txt") ctx.actions.write( output = manifest, # End every line with "\n", including the last: `while read` loops skip # an unterminated last line, and build.sh adds a line after the paths. - content = "".join([file.path + "\n" for file in ctx.files.srcs]), + content = "".join([file.path + "\n" for file in files]), ) - return [DefaultInfo(files = depset([manifest] + ctx.files.srcs))] + return [DefaultInfo(files = depset([manifest] + files))] _release_files = rule( implementation = _release_files_impl, @@ -60,11 +92,16 @@ _release_files = rule( cfg = _release_transition, doc = "The files to build and list.", ), + "target_cpu": attr.string( + doc = "The CPU that the files are built for. Set by the macro.", + mandatory = True, + ), "target_libc": attr.string( doc = "The libc that the files are built for. Set by the macro.", ), "target_os": attr.string( doc = "The OS that the files are built for. Set by the macro.", + mandatory = True, ), }, ) @@ -72,16 +109,16 @@ _release_files = rule( def release_files(name, srcs, **kwargs): """Builds files for a release and writes their paths to `.txt`. - The files are built for the target platform (`--platforms`), with release - settings for its OS and libc: optimized, and linked so that they run on - older OS versions than the build machine's, without extra runtime - libraries. + The files are named `-[.]` and built for the target + platform (`--platforms`), with release settings for its OS and libc: + optimized, and linked so that they run on older OS versions than the build + machine's, without extra runtime libraries. The paths, one per line, are relative to the execroot, e.g. - `bazel-out/k8-opt-ST-1234/bin/foo/foo`. Paths of generated files also - resolve from the workspace root through the `bazel-out` convenience - symlink. This lets scripts find the files without running `bazel cquery`, - which re-analyzes the build. + `bazel-out/k8-opt/bin/dev/release_artifacts/foo-x86_64-unknown-linux-gnu`. + Paths of generated files also resolve from the workspace root through the + `bazel-out` convenience symlink. This lets scripts find the files without + running `bazel cquery`, which re-analyzes the build. Args: name: The target name. @@ -91,14 +128,22 @@ def release_files(name, srcs, **kwargs): _release_files( name = name, srcs = srcs, + target_cpu = select({ + Label("@platforms//cpu:aarch64"): "aarch64", + Label("@platforms//cpu:armv7"): "armv7", + Label("@platforms//cpu:ppc64le"): "powerpc64le", + Label("@platforms//cpu:riscv64"): "riscv64gc", + Label("@platforms//cpu:x86_32"): "i686", + Label("@platforms//cpu:x86_64"): "x86_64", + }), target_libc = select({ "//dev/platforms:musl": "musl", "//conditions:default": "", }), target_os = select({ Label("@platforms//os:linux"): "linux", + labels.PLATFORMS_OS_MACOS: "macos", labels.PLATFORMS_OS_WINDOWS: "windows", - "//conditions:default": "", }), **kwargs ) diff --git a/tests/workflows/BUILD.bazel b/tests/workflows/BUILD.bazel index 76656e5d48..a7b6c833b1 100644 --- a/tests/workflows/BUILD.bazel +++ b/tests/workflows/BUILD.bazel @@ -62,3 +62,16 @@ pytest_test( "@pypi//pyyaml", ], ) + +pytest_test( + name = "release_artifacts_test", + srcs = ["release_artifacts_test.py"], + data = [ + "//dev/release_artifacts:generate_tool_manifest_snippet.py", + "//dev/release_artifacts:merge_manifest_snippets.py", + ], + target_compatible_with = NOT_WINDOWS, + deps = [ + "//python/runfiles", + ], +) diff --git a/tests/workflows/release_artifacts_test.py b/tests/workflows/release_artifacts_test.py new file mode 100644 index 0000000000..6f48541424 --- /dev/null +++ b/tests/workflows/release_artifacts_test.py @@ -0,0 +1,131 @@ +"""Tests for release artifact manifest snippet generation and merging.""" + +from __future__ import annotations + +import hashlib +import json +import subprocess +from pathlib import Path + +from python.runfiles import runfiles + + +def _rlocation(relpath: str) -> Path: + rf = runfiles.CreateOrRaise() + path = rf.Rlocation(relpath) + assert path is not None, f"Could not find runfile: {relpath}" + return Path(path) + + +def test_prepare_and_merge_release_files(tmp_path: Path): + prepare_script = _rlocation( + "rules_python/dev/release_artifacts/generate_tool_manifest_snippet.py" + ) + merge_script = _rlocation( + "rules_python/dev/release_artifacts/merge_manifest_snippets.py" + ) + + tools_dir = tmp_path / "release-tools" + tools_dir.mkdir(parents=True) + snippets_dir = tmp_path / "manifest-snippets" + manifest_file = tmp_path / "tools_manifest.json" + + linux_bin = tools_dir / "exe_zip_maker-x86_64-unknown-linux-gnu" + linux_bin.write_bytes(b"linux-binary-contents") + linux_sha = hashlib.sha256(b"linux-binary-contents").hexdigest() + + win_bin = tools_dir / "exe_zip_maker-x86_64-pc-windows-msvc.exe" + win_bin.write_bytes(b"windows-binary-contents") + win_sha = hashlib.sha256(b"windows-binary-contents").hexdigest() + + mac_bin = tools_dir / "exe_zip_maker-aarch64-apple-darwin" + mac_bin.write_bytes(b"mac-binary-contents") + mac_sha = hashlib.sha256(b"mac-binary-contents").hexdigest() + + subprocess.run( + [ + str(prepare_script), + "2.4.0", + str(tools_dir), + str(snippets_dir), + "bazel-contrib/rules_python", + ], + check=True, + ) + + subprocess.run( + [str(merge_script), str(snippets_dir), str(manifest_file)], + check=True, + ) + + manifest = json.loads(manifest_file.read_text(encoding="utf-8")) + assert manifest == { + "tools": { + "exe_zip_maker": [ + { + "exec_compatible_with": [ + "@platforms//cpu:aarch64", + "@platforms//os:macos", + ], + "filename": "exe_zip_maker-aarch64-apple-darwin", + "sha256": mac_sha, + "urls": [ + "https://github.com/bazel-contrib/rules_python/releases/download/2.4.0/exe_zip_maker-aarch64-apple-darwin" + ], + }, + { + "exec_compatible_with": [ + "@platforms//cpu:x86_64", + "@platforms//os:windows", + ], + "filename": "exe_zip_maker-x86_64-pc-windows-msvc.exe", + "sha256": win_sha, + "urls": [ + "https://github.com/bazel-contrib/rules_python/releases/download/2.4.0/exe_zip_maker-x86_64-pc-windows-msvc.exe" + ], + }, + { + "exec_compatible_with": [ + "@platforms//cpu:x86_64", + "@platforms//os:linux", + ], + "filename": "exe_zip_maker-x86_64-unknown-linux-gnu", + "sha256": linux_sha, + "urls": [ + "https://github.com/bazel-contrib/rules_python/releases/download/2.4.0/exe_zip_maker-x86_64-unknown-linux-gnu" + ], + }, + ] + } + } + + +def test_prepare_and_merge_when_no_artifacts(tmp_path: Path): + prepare_script = _rlocation( + "rules_python/dev/release_artifacts/generate_tool_manifest_snippet.py" + ) + merge_script = _rlocation( + "rules_python/dev/release_artifacts/merge_manifest_snippets.py" + ) + + tools_dir = tmp_path / "missing-release-tools" + snippets_dir = tmp_path / "manifest-snippets" + manifest_file = tmp_path / "tools_manifest.json" + + subprocess.run( + [ + str(prepare_script), + "2.4.0", + str(tools_dir), + str(snippets_dir), + ], + check=True, + ) + assert list(snippets_dir.iterdir()) == [] + + subprocess.run( + [str(merge_script), str(snippets_dir), str(manifest_file)], + check=True, + ) + manifest = json.loads(manifest_file.read_text(encoding="utf-8")) + assert manifest == {"tools": {}} From a4425e2d2cf2fb7c19149e610329db57ebd165b6 Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Fri, 16 Oct 2026 19:57:01 -0700 Subject: [PATCH 2/8] build(release): merge and upload tools_manifest.json in build_tools Keep all tool manifest generation and release asset uploads together in the build_tools workflow by merging snippets and uploading tools_manifest.json in the publish job. --- .github/workflows/release_build_tools.yaml | 18 +++++++++++++----- .github/workflows/release_publish.yaml | 12 +----------- 2 files changed, 14 insertions(+), 16 deletions(-) diff --git a/.github/workflows/release_build_tools.yaml b/.github/workflows/release_build_tools.yaml index 3e2eedcde7..b2c97db280 100644 --- a/.github/workflows/release_build_tools.yaml +++ b/.github/workflows/release_build_tools.yaml @@ -138,6 +138,12 @@ jobs: dev/release_artifacts/generate_tool_manifest_snippet.py \ "$REF" release-tools manifest-snippets + - name: Merge tool manifest snippets + if: hashFiles('manifest-snippets/*.json') != '' + run: | + dev/release_artifacts/merge_manifest_snippets.py \ + manifest-snippets tools_manifest.json + - name: Upload tools to GitHub release if: hashFiles('release-tools/*') != '' uses: softprops/action-gh-release@v3 @@ -145,12 +151,14 @@ jobs: tag_name: ${{ inputs.ref || github.ref_name }} draft: true fail_on_unmatched_files: true - files: release-tools/* + files: | + release-tools/* + tools_manifest.json - - name: Upload manifest snippets - if: hashFiles('manifest-snippets/*.json') != '' + - name: Upload tool manifest + if: hashFiles('tools_manifest.json') != '' uses: actions/upload-artifact@v7 with: - name: release-tools-manifest-snippets - path: manifest-snippets/*.json + name: release-tools-manifest + path: tools_manifest.json if-no-files-found: error diff --git a/.github/workflows/release_publish.yaml b/.github/workflows/release_publish.yaml index a21b63fccf..243c02ecbf 100644 --- a/.github/workflows/release_publish.yaml +++ b/.github/workflows/release_publish.yaml @@ -78,14 +78,6 @@ jobs: uses: actions/checkout@v7 with: ref: ${{ inputs.tag_name || github.ref_name }} - - name: Download tool manifest snippets - uses: actions/download-artifact@v8 - with: - pattern: release-tools-manifest-snippets - path: manifest-snippets - merge-multiple: true - - name: Merge tool manifest snippets - run: dev/release_artifacts/merge_manifest_snippets.py manifest-snippets tools_manifest.json - name: Create release archive and notes run: .github/workflows/create_archive_and_notes.sh ${{ inputs.tag_name || github.ref_name }} - name: Release @@ -97,9 +89,7 @@ jobs: draft: false prerelease: ${{ contains( (inputs.tag_name || github.ref), '-rc') }} fail_on_unmatched_files: true - files: | - rules_python-*.tar.gz - tools_manifest.json + files: rules_python-*.tar.gz tag_name: ${{ inputs.tag_name || github.ref_name }} publish_bcr: From 5bfa6a5a7ca1e7a254b105236618e27359888100 Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Fri, 16 Oct 2026 20:13:30 -0700 Subject: [PATCH 3/8] build(release): generate artifacts_manifest.json in one step Replace the intermediate snippet generation and merging scripts with a single generate_artifacts_manifest.py script that writes artifacts_manifest.json directly from the built release artifacts. --- .github/workflows/create_archive_and_notes.sh | 2 +- .github/workflows/release_build_tools.yaml | 26 ++++----- RELEASING.md | 4 +- dev/release_artifacts/BUILD.bazel | 3 +- ...ppet.py => generate_artifacts_manifest.py} | 51 ++++++++---------- .../merge_manifest_snippets.py | 51 ------------------ tests/workflows/BUILD.bazel | 3 +- tests/workflows/release_artifacts_test.py | 54 +++---------------- 8 files changed, 46 insertions(+), 148 deletions(-) rename dev/release_artifacts/{generate_tool_manifest_snippet.py => generate_artifacts_manifest.py} (70%) delete mode 100755 dev/release_artifacts/merge_manifest_snippets.py diff --git a/.github/workflows/create_archive_and_notes.sh b/.github/workflows/create_archive_and_notes.sh index 755b29f794..cec8d82a97 100755 --- a/.github/workflows/create_archive_and_notes.sh +++ b/.github/workflows/create_archive_and_notes.sh @@ -32,7 +32,7 @@ git checkout "$TAG" PREFIX="rules_python-${TAG}" ARCHIVE="rules_python-$TAG.tar.gz" # TODO(https://github.com/bazel-contrib/rules_python/issues/4250): Add -# tools_manifest.json to the release archive. +# artifacts_manifest.json to the release archive. git archive --format=tar "--prefix=${PREFIX}/" "$TAG" | gzip > "$ARCHIVE" SHA=$(shasum -a 256 "$ARCHIVE" | awk '{print $1}') diff --git a/.github/workflows/release_build_tools.yaml b/.github/workflows/release_build_tools.yaml index b2c97db280..fa5eb4568a 100644 --- a/.github/workflows/release_build_tools.yaml +++ b/.github/workflows/release_build_tools.yaml @@ -132,17 +132,13 @@ jobs: # patch releases of older release branches that predate these scripts and # artifacts (and while tool build failures are non-fatal). They can be # removed once older release branches are no longer supported. - - name: Generate manifest snippets - if: hashFiles('dev/release_artifacts/generate_tool_manifest_snippet.py') != '' + - name: Generate artifacts manifest + if: >- + hashFiles('dev/release_artifacts/generate_artifacts_manifest.py') != '' && + hashFiles('release-tools/*') != '' run: | - dev/release_artifacts/generate_tool_manifest_snippet.py \ - "$REF" release-tools manifest-snippets - - - name: Merge tool manifest snippets - if: hashFiles('manifest-snippets/*.json') != '' - run: | - dev/release_artifacts/merge_manifest_snippets.py \ - manifest-snippets tools_manifest.json + dev/release_artifacts/generate_artifacts_manifest.py \ + "$REF" release-tools artifacts_manifest.json - name: Upload tools to GitHub release if: hashFiles('release-tools/*') != '' @@ -153,12 +149,12 @@ jobs: fail_on_unmatched_files: true files: | release-tools/* - tools_manifest.json + artifacts_manifest.json - - name: Upload tool manifest - if: hashFiles('tools_manifest.json') != '' + - name: Upload artifacts manifest + if: hashFiles('artifacts_manifest.json') != '' uses: actions/upload-artifact@v7 with: - name: release-tools-manifest - path: tools_manifest.json + name: release-artifacts-manifest + path: artifacts_manifest.json if-no-files-found: error diff --git a/RELEASING.md b/RELEASING.md index ff0f288f05..fc04135ddd 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -70,8 +70,8 @@ gh workflow run release_publish.yaml --ref -f publish_to_pypi=false ### Prebuilt tools The release workflow also builds `//dev/release_artifacts:artifacts_for_release` -for each platform and uploads the built binaries and `tools_manifest.json` to -the GitHub release. For now, build failures only emit a warning rather than +for each platform and uploads the built binaries and `artifacts_manifest.json` +to the GitHub release. For now, build failures only emit a warning rather than failing the release. Presubmit builds it for every platform too: natively on each CI host, and diff --git a/dev/release_artifacts/BUILD.bazel b/dev/release_artifacts/BUILD.bazel index a8dae9ff6b..8f5f0c673e 100644 --- a/dev/release_artifacts/BUILD.bazel +++ b/dev/release_artifacts/BUILD.bazel @@ -117,6 +117,5 @@ bzl_library( ) exports_files([ - "generate_tool_manifest_snippet.py", - "merge_manifest_snippets.py", + "generate_artifacts_manifest.py", ]) diff --git a/dev/release_artifacts/generate_tool_manifest_snippet.py b/dev/release_artifacts/generate_artifacts_manifest.py similarity index 70% rename from dev/release_artifacts/generate_tool_manifest_snippet.py rename to dev/release_artifacts/generate_artifacts_manifest.py index 096e75ac51..32a5434619 100755 --- a/dev/release_artifacts/generate_tool_manifest_snippet.py +++ b/dev/release_artifacts/generate_artifacts_manifest.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""Generates a JSON manifest snippet for each built release binary.""" +"""Generates an artifacts_manifest.json file for built release binaries.""" from __future__ import annotations @@ -41,19 +41,13 @@ def _os_constraint(triple: str) -> str: raise ValueError(f"Unsupported OS in triple: {triple}") -def generate_snippets( +def generate_manifest( tag: str, tools_dir: Path, - snippets_dir: Path, repo: str, -) -> None: - snippets_dir.mkdir(parents=True, exist_ok=True) - if not tools_dir.is_dir(): - return - +) -> dict[str, Any]: + tools: dict[str, list[dict[str, Any]]] = {} for file_path in sorted(tools_dir.iterdir()): - if not file_path.is_file(): - continue filename = file_path.name base = filename.removesuffix(".exe") match = _FILENAME_RE.match(base) @@ -66,24 +60,20 @@ def generate_snippets( sha256 = hashlib.sha256(file_path.read_bytes()).hexdigest() url = f"https://github.com/{repo}/releases/download/{tag}/{filename}" - snippet: dict[str, Any] = { - "tools": { - tool: [ - { - "exec_compatible_with": [cpu, os_constraint], - "filename": filename, - "sha256": sha256, - "urls": [url], - } - ] + tools.setdefault(tool, []).append( + { + "exec_compatible_with": [cpu, os_constraint], + "filename": filename, + "sha256": sha256, + "urls": [url], } - } - snippet_path = snippets_dir / f"{base}.json" - snippet_path.write_text( - json.dumps(snippet, indent=2, sort_keys=True) + "\n", - encoding="utf-8", ) + for entries in tools.values(): + entries.sort(key=lambda entry: entry["filename"]) + + return {"tools": dict(sorted(tools.items()))} + def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser(description=__doc__) @@ -94,9 +84,9 @@ def main(argv: list[str] | None = None) -> int: help="Directory containing built -[.exe] files.", ) parser.add_argument( - "snippets_dir", + "output_file", type=Path, - help="Output directory for generated *.json manifest snippets.", + help="Path to the artifacts_manifest.json file to write.", ) parser.add_argument( "repo", @@ -105,7 +95,12 @@ def main(argv: list[str] | None = None) -> int: help="GitHub owner/repository name.", ) args = parser.parse_args(argv) - generate_snippets(args.tag, args.tools_dir, args.snippets_dir, args.repo) + manifest = generate_manifest(args.tag, args.tools_dir, args.repo) + args.output_file.parent.mkdir(parents=True, exist_ok=True) + args.output_file.write_text( + json.dumps(manifest, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) return 0 diff --git a/dev/release_artifacts/merge_manifest_snippets.py b/dev/release_artifacts/merge_manifest_snippets.py deleted file mode 100755 index a0b8fa9bfd..0000000000 --- a/dev/release_artifacts/merge_manifest_snippets.py +++ /dev/null @@ -1,51 +0,0 @@ -#!/usr/bin/env python3 -"""Merges JSON manifest snippets into a single tools_manifest.json file.""" - -from __future__ import annotations - -import argparse -import json -import sys -from pathlib import Path -from typing import Any - - -def merge_snippets(snippets_dir: Path) -> dict[str, Any]: - tools: dict[str, list[dict[str, Any]]] = {} - if snippets_dir.is_dir(): - for snippet_path in sorted(snippets_dir.glob("*.json")): - data = json.loads(snippet_path.read_text(encoding="utf-8")) - for tool, entries in data.get("tools", {}).items(): - tools.setdefault(tool, []).extend(entries) - - for tool, entries in tools.items(): - entries.sort(key=lambda entry: entry["filename"]) - - return {"tools": dict(sorted(tools.items()))} - - -def main(argv: list[str] | None = None) -> int: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument( - "snippets_dir", - type=Path, - help="Directory containing per-binary *.json manifest snippets.", - ) - parser.add_argument( - "output_file", - type=Path, - help="Path to the merged tools_manifest.json file to write.", - ) - args = parser.parse_args(argv) - - manifest = merge_snippets(args.snippets_dir) - args.output_file.parent.mkdir(parents=True, exist_ok=True) - args.output_file.write_text( - json.dumps(manifest, indent=2, sort_keys=True) + "\n", - encoding="utf-8", - ) - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/tests/workflows/BUILD.bazel b/tests/workflows/BUILD.bazel index a7b6c833b1..3e0cbf50e1 100644 --- a/tests/workflows/BUILD.bazel +++ b/tests/workflows/BUILD.bazel @@ -67,8 +67,7 @@ pytest_test( name = "release_artifacts_test", srcs = ["release_artifacts_test.py"], data = [ - "//dev/release_artifacts:generate_tool_manifest_snippet.py", - "//dev/release_artifacts:merge_manifest_snippets.py", + "//dev/release_artifacts:generate_artifacts_manifest.py", ], target_compatible_with = NOT_WINDOWS, deps = [ diff --git a/tests/workflows/release_artifacts_test.py b/tests/workflows/release_artifacts_test.py index 6f48541424..151790dcce 100644 --- a/tests/workflows/release_artifacts_test.py +++ b/tests/workflows/release_artifacts_test.py @@ -1,4 +1,4 @@ -"""Tests for release artifact manifest snippet generation and merging.""" +"""Tests for release artifact manifest generation.""" from __future__ import annotations @@ -17,18 +17,14 @@ def _rlocation(relpath: str) -> Path: return Path(path) -def test_prepare_and_merge_release_files(tmp_path: Path): - prepare_script = _rlocation( - "rules_python/dev/release_artifacts/generate_tool_manifest_snippet.py" - ) - merge_script = _rlocation( - "rules_python/dev/release_artifacts/merge_manifest_snippets.py" +def test_generate_artifacts_manifest(tmp_path: Path): + generate_script = _rlocation( + "rules_python/dev/release_artifacts/generate_artifacts_manifest.py" ) tools_dir = tmp_path / "release-tools" tools_dir.mkdir(parents=True) - snippets_dir = tmp_path / "manifest-snippets" - manifest_file = tmp_path / "tools_manifest.json" + manifest_file = tmp_path / "artifacts_manifest.json" linux_bin = tools_dir / "exe_zip_maker-x86_64-unknown-linux-gnu" linux_bin.write_bytes(b"linux-binary-contents") @@ -44,20 +40,15 @@ def test_prepare_and_merge_release_files(tmp_path: Path): subprocess.run( [ - str(prepare_script), + str(generate_script), "2.4.0", str(tools_dir), - str(snippets_dir), + str(manifest_file), "bazel-contrib/rules_python", ], check=True, ) - subprocess.run( - [str(merge_script), str(snippets_dir), str(manifest_file)], - check=True, - ) - manifest = json.loads(manifest_file.read_text(encoding="utf-8")) assert manifest == { "tools": { @@ -98,34 +89,3 @@ def test_prepare_and_merge_release_files(tmp_path: Path): ] } } - - -def test_prepare_and_merge_when_no_artifacts(tmp_path: Path): - prepare_script = _rlocation( - "rules_python/dev/release_artifacts/generate_tool_manifest_snippet.py" - ) - merge_script = _rlocation( - "rules_python/dev/release_artifacts/merge_manifest_snippets.py" - ) - - tools_dir = tmp_path / "missing-release-tools" - snippets_dir = tmp_path / "manifest-snippets" - manifest_file = tmp_path / "tools_manifest.json" - - subprocess.run( - [ - str(prepare_script), - "2.4.0", - str(tools_dir), - str(snippets_dir), - ], - check=True, - ) - assert list(snippets_dir.iterdir()) == [] - - subprocess.run( - [str(merge_script), str(snippets_dir), str(manifest_file)], - check=True, - ) - manifest = json.loads(manifest_file.read_text(encoding="utf-8")) - assert manifest == {"tools": {}} From 96386e94986016d5d63ad934fe8f423c58fa107a Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Fri, 16 Oct 2026 20:21:55 -0700 Subject: [PATCH 4/8] build(release): rename build_tools job to build_artifacts --- .github/workflows/release_publish.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/release_publish.yaml b/.github/workflows/release_publish.yaml index 243c02ecbf..f2028d4b06 100644 --- a/.github/workflows/release_publish.yaml +++ b/.github/workflows/release_publish.yaml @@ -57,13 +57,13 @@ jobs: prerelease: ${{ contains( (inputs.tag_name || github.ref), '-rc') }} tag_name: ${{ inputs.tag_name || github.ref_name }} - build_tools: + build_artifacts: # Builds prebuilt tools and uploads them to the draft release. A failed # build only adds a warning to this run, since releases don't require the # tools yet. The run fails if a build hangs past the job timeout, the built # files fail their checks, or the ref's dev/platforms lacks a job's # platform. - name: Build tools + name: Build artifacts needs: create_release uses: ./.github/workflows/release_build_tools.yaml with: @@ -71,7 +71,7 @@ jobs: release: name: Release - needs: build_tools + needs: build_artifacts runs-on: ubuntu-latest steps: - name: Checkout From bf17fb4fb3efbaecd83acf433eb9125c239739bc Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Fri, 16 Oct 2026 20:38:46 -0700 Subject: [PATCH 5/8] build(release): upload SHA256SUMS instead of artifacts_manifest.json to release --- .github/workflows/release_build_tools.yaml | 7 ++++++- RELEASING.md | 6 +++--- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/.github/workflows/release_build_tools.yaml b/.github/workflows/release_build_tools.yaml index fa5eb4568a..ec49cb9bc1 100644 --- a/.github/workflows/release_build_tools.yaml +++ b/.github/workflows/release_build_tools.yaml @@ -132,6 +132,11 @@ jobs: # patch releases of older release branches that predate these scripts and # artifacts (and while tool build failures are non-fatal). They can be # removed once older release branches are no longer supported. + - name: Generate SHA256SUMS + if: hashFiles('release-tools/*') != '' + working-directory: release-tools + run: sha256sum * > ../SHA256SUMS + - name: Generate artifacts manifest if: >- hashFiles('dev/release_artifacts/generate_artifacts_manifest.py') != '' && @@ -149,7 +154,7 @@ jobs: fail_on_unmatched_files: true files: | release-tools/* - artifacts_manifest.json + SHA256SUMS - name: Upload artifacts manifest if: hashFiles('artifacts_manifest.json') != '' diff --git a/RELEASING.md b/RELEASING.md index fc04135ddd..188464361b 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -70,9 +70,9 @@ gh workflow run release_publish.yaml --ref -f publish_to_pypi=false ### Prebuilt tools The release workflow also builds `//dev/release_artifacts:artifacts_for_release` -for each platform and uploads the built binaries and `artifacts_manifest.json` -to the GitHub release. For now, build failures only emit a warning rather than -failing the release. +for each platform and uploads the built binaries and `SHA256SUMS` to the GitHub +release. For now, build failures only emit a warning rather than failing the +release. Presubmit builds it for every platform too: natively on each CI host, and through `//dev/release_artifacts:cross_compiled_artifacts` for the others. From 0c8aa3191d8c088a2b673405041fa2f65689fcda Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Fri, 16 Oct 2026 20:44:41 -0700 Subject: [PATCH 6/8] build(release): use artifacts terminology instead of tools --- .github/workflows/release_build_tools.yaml | 26 +++++++++---------- .github/workflows/release_publish.yaml | 6 ++--- RELEASING.md | 4 +-- .../generate_artifacts_manifest.py | 26 ++++++++++--------- tests/workflows/release_artifacts_test.py | 14 +++++----- 5 files changed, 39 insertions(+), 37 deletions(-) diff --git a/.github/workflows/release_build_tools.yaml b/.github/workflows/release_build_tools.yaml index ec49cb9bc1..9693eff2dd 100644 --- a/.github/workflows/release_build_tools.yaml +++ b/.github/workflows/release_build_tools.yaml @@ -102,12 +102,12 @@ jobs: if: steps.build.outputs.files != '' uses: actions/upload-artifact@v7 with: - name: release-tools-${{ matrix.triple }} + name: release-artifacts-${{ matrix.triple }} path: ${{ steps.build.outputs.files }} if-no-files-found: error publish: - name: Publish tools + name: Publish artifacts needs: build runs-on: ubuntu-24.04 permissions: @@ -121,39 +121,39 @@ jobs: ref: ${{ inputs.ref }} persist-credentials: false - - name: Download built tools + - name: Download built artifacts uses: actions/download-artifact@v8 with: - pattern: release-tools-* - path: release-tools + pattern: release-artifacts-* + path: release-artifacts merge-multiple: true # The hashFiles(...) checks below are for backwards compatibility with # patch releases of older release branches that predate these scripts and - # artifacts (and while tool build failures are non-fatal). They can be + # artifacts (and while artifact build failures are non-fatal). They can be # removed once older release branches are no longer supported. - name: Generate SHA256SUMS - if: hashFiles('release-tools/*') != '' - working-directory: release-tools + if: hashFiles('release-artifacts/*') != '' + working-directory: release-artifacts run: sha256sum * > ../SHA256SUMS - name: Generate artifacts manifest if: >- hashFiles('dev/release_artifacts/generate_artifacts_manifest.py') != '' && - hashFiles('release-tools/*') != '' + hashFiles('release-artifacts/*') != '' run: | dev/release_artifacts/generate_artifacts_manifest.py \ - "$REF" release-tools artifacts_manifest.json + "$REF" release-artifacts artifacts_manifest.json - - name: Upload tools to GitHub release - if: hashFiles('release-tools/*') != '' + - name: Upload artifacts to GitHub release + if: hashFiles('release-artifacts/*') != '' uses: softprops/action-gh-release@v3 with: tag_name: ${{ inputs.ref || github.ref_name }} draft: true fail_on_unmatched_files: true files: | - release-tools/* + release-artifacts/* SHA256SUMS - name: Upload artifacts manifest diff --git a/.github/workflows/release_publish.yaml b/.github/workflows/release_publish.yaml index f2028d4b06..2bf29cce67 100644 --- a/.github/workflows/release_publish.yaml +++ b/.github/workflows/release_publish.yaml @@ -58,10 +58,10 @@ jobs: tag_name: ${{ inputs.tag_name || github.ref_name }} build_artifacts: - # Builds prebuilt tools and uploads them to the draft release. A failed + # Builds release artifacts and uploads them to the draft release. A failed # build only adds a warning to this run, since releases don't require the - # tools yet. The run fails if a build hangs past the job timeout, the built - # files fail their checks, or the ref's dev/platforms lacks a job's + # artifacts yet. The run fails if a build hangs past the job timeout, the + # built files fail their checks, or the ref's dev/platforms lacks a job's # platform. name: Build artifacts needs: create_release diff --git a/RELEASING.md b/RELEASING.md index 188464361b..a52bf47d8a 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -67,10 +67,10 @@ you can set the `publish_to_pypi` input to `false`: gh workflow run release_publish.yaml --ref -f publish_to_pypi=false ``` -### Prebuilt tools +### Release artifacts The release workflow also builds `//dev/release_artifacts:artifacts_for_release` -for each platform and uploads the built binaries and `SHA256SUMS` to the GitHub +for each platform and uploads the built artifacts and `SHA256SUMS` to the GitHub release. For now, build failures only emit a warning rather than failing the release. diff --git a/dev/release_artifacts/generate_artifacts_manifest.py b/dev/release_artifacts/generate_artifacts_manifest.py index 32a5434619..c0ccb5a8a3 100755 --- a/dev/release_artifacts/generate_artifacts_manifest.py +++ b/dev/release_artifacts/generate_artifacts_manifest.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""Generates an artifacts_manifest.json file for built release binaries.""" +"""Generates an artifacts_manifest.json file for built release artifacts.""" from __future__ import annotations @@ -43,24 +43,26 @@ def _os_constraint(triple: str) -> str: def generate_manifest( tag: str, - tools_dir: Path, + artifacts_dir: Path, repo: str, ) -> dict[str, Any]: - tools: dict[str, list[dict[str, Any]]] = {} - for file_path in sorted(tools_dir.iterdir()): + artifacts: dict[str, list[dict[str, Any]]] = {} + for file_path in sorted(artifacts_dir.iterdir()): filename = file_path.name base = filename.removesuffix(".exe") match = _FILENAME_RE.match(base) if not match: - raise ValueError(f"Cannot parse tool and triple from filename: {filename}") - tool, triple = match.group(1), match.group(2) + raise ValueError( + f"Cannot parse artifact and triple from filename: {filename}" + ) + artifact, triple = match.group(1), match.group(2) cpu = _cpu_constraint(triple) os_constraint = _os_constraint(triple) sha256 = hashlib.sha256(file_path.read_bytes()).hexdigest() url = f"https://github.com/{repo}/releases/download/{tag}/{filename}" - tools.setdefault(tool, []).append( + artifacts.setdefault(artifact, []).append( { "exec_compatible_with": [cpu, os_constraint], "filename": filename, @@ -69,19 +71,19 @@ def generate_manifest( } ) - for entries in tools.values(): + for entries in artifacts.values(): entries.sort(key=lambda entry: entry["filename"]) - return {"tools": dict(sorted(tools.items()))} + return {"artifacts": dict(sorted(artifacts.items()))} def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("tag", help="Release tag name (e.g. 2.4.0).") parser.add_argument( - "tools_dir", + "artifacts_dir", type=Path, - help="Directory containing built -[.exe] files.", + help="Directory containing built -[.exe] files.", ) parser.add_argument( "output_file", @@ -95,7 +97,7 @@ def main(argv: list[str] | None = None) -> int: help="GitHub owner/repository name.", ) args = parser.parse_args(argv) - manifest = generate_manifest(args.tag, args.tools_dir, args.repo) + manifest = generate_manifest(args.tag, args.artifacts_dir, args.repo) args.output_file.parent.mkdir(parents=True, exist_ok=True) args.output_file.write_text( json.dumps(manifest, indent=2, sort_keys=True) + "\n", diff --git a/tests/workflows/release_artifacts_test.py b/tests/workflows/release_artifacts_test.py index 151790dcce..37a93d9049 100644 --- a/tests/workflows/release_artifacts_test.py +++ b/tests/workflows/release_artifacts_test.py @@ -22,19 +22,19 @@ def test_generate_artifacts_manifest(tmp_path: Path): "rules_python/dev/release_artifacts/generate_artifacts_manifest.py" ) - tools_dir = tmp_path / "release-tools" - tools_dir.mkdir(parents=True) + artifacts_dir = tmp_path / "release-artifacts" + artifacts_dir.mkdir(parents=True) manifest_file = tmp_path / "artifacts_manifest.json" - linux_bin = tools_dir / "exe_zip_maker-x86_64-unknown-linux-gnu" + linux_bin = artifacts_dir / "exe_zip_maker-x86_64-unknown-linux-gnu" linux_bin.write_bytes(b"linux-binary-contents") linux_sha = hashlib.sha256(b"linux-binary-contents").hexdigest() - win_bin = tools_dir / "exe_zip_maker-x86_64-pc-windows-msvc.exe" + win_bin = artifacts_dir / "exe_zip_maker-x86_64-pc-windows-msvc.exe" win_bin.write_bytes(b"windows-binary-contents") win_sha = hashlib.sha256(b"windows-binary-contents").hexdigest() - mac_bin = tools_dir / "exe_zip_maker-aarch64-apple-darwin" + mac_bin = artifacts_dir / "exe_zip_maker-aarch64-apple-darwin" mac_bin.write_bytes(b"mac-binary-contents") mac_sha = hashlib.sha256(b"mac-binary-contents").hexdigest() @@ -42,7 +42,7 @@ def test_generate_artifacts_manifest(tmp_path: Path): [ str(generate_script), "2.4.0", - str(tools_dir), + str(artifacts_dir), str(manifest_file), "bazel-contrib/rules_python", ], @@ -51,7 +51,7 @@ def test_generate_artifacts_manifest(tmp_path: Path): manifest = json.loads(manifest_file.read_text(encoding="utf-8")) assert manifest == { - "tools": { + "artifacts": { "exe_zip_maker": [ { "exec_compatible_with": [ From 2033dca6d40d2b71269ec42523dfec16e292c469 Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Fri, 16 Oct 2026 20:55:19 -0700 Subject: [PATCH 7/8] build(release): rename release_build_tools.yaml to release_build_artifacts.yaml --- ...elease_build_tools.yaml => release_build_artifacts.yaml} | 2 +- .github/workflows/release_publish.yaml | 2 +- tests/workflows/BUILD.bazel | 6 +++--- ..._build_tools_test.py => release_build_artifacts_test.py} | 4 ++-- 4 files changed, 7 insertions(+), 7 deletions(-) rename .github/workflows/{release_build_tools.yaml => release_build_artifacts.yaml} (99%) rename tests/workflows/{release_build_tools_test.py => release_build_artifacts_test.py} (91%) diff --git a/.github/workflows/release_build_tools.yaml b/.github/workflows/release_build_artifacts.yaml similarity index 99% rename from .github/workflows/release_build_tools.yaml rename to .github/workflows/release_build_artifacts.yaml index 9693eff2dd..744fc0779e 100644 --- a/.github/workflows/release_build_tools.yaml +++ b/.github/workflows/release_build_artifacts.yaml @@ -3,7 +3,7 @@ # see dev/release_artifacts/build.sh. Linux x86_64 runners cross-compile the # files for other Linux CPUs. macOS and Windows files need those OSes' SDKs, so # they're built on those OSes. -name: "Release: Build Tools" +name: "Release: Build Artifacts" on: workflow_call: diff --git a/.github/workflows/release_publish.yaml b/.github/workflows/release_publish.yaml index 2bf29cce67..22109eb79f 100644 --- a/.github/workflows/release_publish.yaml +++ b/.github/workflows/release_publish.yaml @@ -65,7 +65,7 @@ jobs: # platform. name: Build artifacts needs: create_release - uses: ./.github/workflows/release_build_tools.yaml + uses: ./.github/workflows/release_build_artifacts.yaml with: ref: ${{ inputs.tag_name || github.ref_name }} diff --git a/tests/workflows/BUILD.bazel b/tests/workflows/BUILD.bazel index 3e0cbf50e1..5ddc52046c 100644 --- a/tests/workflows/BUILD.bazel +++ b/tests/workflows/BUILD.bazel @@ -39,10 +39,10 @@ pytest_test( ) pytest_test( - name = "release_build_tools_test", - srcs = ["release_build_tools_test.py"], + name = "release_build_artifacts_test", + srcs = ["release_build_artifacts_test.py"], data = [ - "//.github/workflows:release_build_tools.yaml", + "//.github/workflows:release_build_artifacts.yaml", "//dev/platforms:BUILD.bazel", ], target_compatible_with = NOT_WINDOWS, diff --git a/tests/workflows/release_build_tools_test.py b/tests/workflows/release_build_artifacts_test.py similarity index 91% rename from tests/workflows/release_build_tools_test.py rename to tests/workflows/release_build_artifacts_test.py index f9eb9e7250..87c2de7e6a 100644 --- a/tests/workflows/release_build_tools_test.py +++ b/tests/workflows/release_build_artifacts_test.py @@ -1,4 +1,4 @@ -"""Tests for the release_build_tools.yaml workflow.""" +"""Tests for the release_build_artifacts.yaml workflow.""" from __future__ import annotations @@ -16,7 +16,7 @@ def test_matrix_matches_dev_platforms(): its platform is missing. """ root = runfiles.CreateOrRaise().root() / "rules_python" - workflow_path = root / ".github/workflows/release_build_tools.yaml" + workflow_path = root / ".github/workflows/release_build_artifacts.yaml" workflow = yaml.safe_load(workflow_path.read_text(encoding="utf-8")) triples = [ entry["triple"] From 2dbc0b6a8336572b241264539b15e9737311eab1 Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Fri, 16 Oct 2026 21:15:40 -0700 Subject: [PATCH 8/8] build(release): address code review style findings --- .../generate_artifacts_manifest.py | 24 +++++++++++++----- tests/workflows/release_artifacts_test.py | 25 ++++++------------- 2 files changed, 25 insertions(+), 24 deletions(-) diff --git a/dev/release_artifacts/generate_artifacts_manifest.py b/dev/release_artifacts/generate_artifacts_manifest.py index c0ccb5a8a3..c6c5af236d 100755 --- a/dev/release_artifacts/generate_artifacts_manifest.py +++ b/dev/release_artifacts/generate_artifacts_manifest.py @@ -10,7 +10,15 @@ import re import sys from pathlib import Path -from typing import Any +from typing import TypedDict + + +class ArtifactEntry(TypedDict): + exec_compatible_with: list[str] + filename: str + sha256: str + urls: list[str] + _CPU_TO_CONSTRAINT = { "aarch64": "@platforms//cpu:aarch64", @@ -45,8 +53,8 @@ def generate_manifest( tag: str, artifacts_dir: Path, repo: str, -) -> dict[str, Any]: - artifacts: dict[str, list[dict[str, Any]]] = {} +) -> dict[str, dict[str, list[ArtifactEntry]]]: + artifacts: dict[str, list[ArtifactEntry]] = {} for file_path in sorted(artifacts_dir.iterdir()): filename = file_path.name base = filename.removesuffix(".exe") @@ -77,7 +85,7 @@ def generate_manifest( return {"artifacts": dict(sorted(artifacts.items()))} -def main(argv: list[str] | None = None) -> int: +def _main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("tag", help="Release tag name (e.g. 2.4.0).") parser.add_argument( @@ -97,7 +105,11 @@ def main(argv: list[str] | None = None) -> int: help="GitHub owner/repository name.", ) args = parser.parse_args(argv) - manifest = generate_manifest(args.tag, args.artifacts_dir, args.repo) + try: + manifest = generate_manifest(args.tag, args.artifacts_dir, args.repo) + except ValueError as err: + print(f"::error::{err}") + return 1 args.output_file.parent.mkdir(parents=True, exist_ok=True) args.output_file.write_text( json.dumps(manifest, indent=2, sort_keys=True) + "\n", @@ -107,4 +119,4 @@ def main(argv: list[str] | None = None) -> int: if __name__ == "__main__": - sys.exit(main()) + sys.exit(_main()) diff --git a/tests/workflows/release_artifacts_test.py b/tests/workflows/release_artifacts_test.py index 37a93d9049..c49672b74b 100644 --- a/tests/workflows/release_artifacts_test.py +++ b/tests/workflows/release_artifacts_test.py @@ -10,16 +10,10 @@ from python.runfiles import runfiles -def _rlocation(relpath: str) -> Path: - rf = runfiles.CreateOrRaise() - path = rf.Rlocation(relpath) - assert path is not None, f"Could not find runfile: {relpath}" - return Path(path) - - def test_generate_artifacts_manifest(tmp_path: Path): - generate_script = _rlocation( - "rules_python/dev/release_artifacts/generate_artifacts_manifest.py" + generate_script = ( + runfiles.CreateOrRaise().root() + / "rules_python/dev/release_artifacts/generate_artifacts_manifest.py" ) artifacts_dir = tmp_path / "release-artifacts" @@ -50,6 +44,7 @@ def test_generate_artifacts_manifest(tmp_path: Path): ) manifest = json.loads(manifest_file.read_text(encoding="utf-8")) + base_url = "https://github.com/bazel-contrib/rules_python/releases/download/2.4.0" assert manifest == { "artifacts": { "exe_zip_maker": [ @@ -60,9 +55,7 @@ def test_generate_artifacts_manifest(tmp_path: Path): ], "filename": "exe_zip_maker-aarch64-apple-darwin", "sha256": mac_sha, - "urls": [ - "https://github.com/bazel-contrib/rules_python/releases/download/2.4.0/exe_zip_maker-aarch64-apple-darwin" - ], + "urls": [f"{base_url}/exe_zip_maker-aarch64-apple-darwin"], }, { "exec_compatible_with": [ @@ -71,9 +64,7 @@ def test_generate_artifacts_manifest(tmp_path: Path): ], "filename": "exe_zip_maker-x86_64-pc-windows-msvc.exe", "sha256": win_sha, - "urls": [ - "https://github.com/bazel-contrib/rules_python/releases/download/2.4.0/exe_zip_maker-x86_64-pc-windows-msvc.exe" - ], + "urls": [f"{base_url}/exe_zip_maker-x86_64-pc-windows-msvc.exe"], }, { "exec_compatible_with": [ @@ -82,9 +73,7 @@ def test_generate_artifacts_manifest(tmp_path: Path): ], "filename": "exe_zip_maker-x86_64-unknown-linux-gnu", "sha256": linux_sha, - "urls": [ - "https://github.com/bazel-contrib/rules_python/releases/download/2.4.0/exe_zip_maker-x86_64-unknown-linux-gnu" - ], + "urls": [f"{base_url}/exe_zip_maker-x86_64-unknown-linux-gnu"], }, ] }