From d6ad99d11d5349ae091b603cfe7e9c6c2782f31a Mon Sep 17 00:00:00 2001 From: Piotr Mlocek Date: Thu, 19 Nov 2026 10:30:01 -0800 Subject: [PATCH 1/5] feat(core-sync): share the SDK core-sync scripts and workflows Move the core pin automation that sdk-python and sdk-typescript each carry a copy of into one place: the core-sync scripts (core tags, bump pull requests, the drift check and issue, release tagging) and reusable core-watch and core-drift workflows. SDK repositories pin the scripts as mise remote tasks and provide the language-specific parts as tasks: proto:fetch, proto:gen, proto:buf, check, test and optionally core:uncovered-rpcs. The drift check now runs check before test, so a type error in bindings from core main fails it. Signed-off-by: Piotr Mlocek --- .github/workflows/core-drift.yml | 63 +++++++++ .github/workflows/core-watch.yml | 89 ++++++++++++ .github/workflows/test.yml | 6 + README.md | 5 + core-sync/README.md | 94 +++++++++++++ core-sync/bump-pr-body.md | 16 +++ core-sync/core-tags.sh | 72 ++++++++++ core-sync/drift-check.sh | 153 +++++++++++++++++++++ core-sync/drift-issue.sh | 37 +++++ core-sync/open-bump-pr.sh | 225 +++++++++++++++++++++++++++++++ core-sync/report-failure.sh | 47 +++++++ core-sync/tag-release.sh | 129 ++++++++++++++++++ core-sync/update-drift-issue.sh | 64 +++++++++ 13 files changed, 1000 insertions(+) create mode 100644 .github/workflows/core-drift.yml create mode 100644 .github/workflows/core-watch.yml create mode 100644 core-sync/README.md create mode 100644 core-sync/bump-pr-body.md create mode 100755 core-sync/core-tags.sh create mode 100755 core-sync/drift-check.sh create mode 100644 core-sync/drift-issue.sh create mode 100755 core-sync/open-bump-pr.sh create mode 100755 core-sync/report-failure.sh create mode 100755 core-sync/tag-release.sh create mode 100755 core-sync/update-drift-issue.sh diff --git a/.github/workflows/core-drift.yml b/.github/workflows/core-drift.yml new file mode 100644 index 0000000..455c72e --- /dev/null +++ b/.github/workflows/core-drift.yml @@ -0,0 +1,63 @@ +# SPDX-License-Identifier: Apache-2.0 + +# Reusable: compares core main with the calling SDK repository's core pin and +# keeps its single "Core main drift" issue current (mise run core:drift). See +# core-sync/README.md for the contract the caller implements. +name: core-drift + +on: + workflow_call: + +permissions: + contents: read + +jobs: + drift: + name: drift + # The bot secrets live in the caller's sdk-bot environment, which only + # main can deploy to, so a pushed branch can't mint a bot token. + environment: sdk-bot + runs-on: ubuntu-latest + timeout-minutes: 20 + concurrency: + group: core-drift-${{ github.repository }} + cancel-in-progress: false + steps: + - id: app-token + uses: actions/create-github-app-token@64ab729a6294000f5aa4a4bc0b73d1f32bc45510 # v3.3.0 + with: + client-id: ${{ secrets.SDK_BOT_CLIENT_ID }} + private-key: ${{ secrets.SDK_BOT_PRIVATE_KEY }} + permission-issues: write + + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - uses: jdx/mise-action@b44d12e7d0d8ee7461bb893e1b1993500433185b # v5.2.7 + with: + install: false + + - name: Install tools + run: | + if [ -f mise.lock ]; then mise install --locked; else mise install; fi + + # Exit 1 with a report means drift; the issue records it. No report + # means the check itself broke, which the failure step records. + - name: Check drift against core main + run: | + mise run core:drift -- --out "$RUNNER_TEMP/drift-report.md" || true + test -s "$RUNNER_TEMP/drift-report.md" + cat "$RUNNER_TEMP/drift-report.md" >>"$GITHUB_STEP_SUMMARY" + + - name: Update the drift issue + env: + GH_TOKEN: ${{ steps.app-token.outputs.token }} + run: mise run core:drift-issue -- "$RUNNER_TEMP/drift-report.md" + + - name: Report failure + if: failure() + env: + GH_TOKEN: ${{ steps.app-token.outputs.token }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: mise run core:report-failure -- core-drift "$RUN_URL" diff --git a/.github/workflows/core-watch.yml b/.github/workflows/core-watch.yml new file mode 100644 index 0000000..8611296 --- /dev/null +++ b/.github/workflows/core-watch.yml @@ -0,0 +1,89 @@ +# SPDX-License-Identifier: Apache-2.0 + +# Reusable: opens one update proposal per new core release tag in the calling +# SDK repository (mise run core:bump). See core-sync/README.md for the +# contract the caller implements. +name: core-watch + +on: + workflow_call: + inputs: + tag: + description: Core tag to propose (vX.Y.Z or vX.Y.Z-pre.N); empty proposes every new tag + type: string + required: false + default: "" + +permissions: + contents: read + +jobs: + watch: + name: watch + # The bot secrets live in the caller's sdk-bot environment, which only + # main can deploy to, so a pushed branch can't mint a bot token. + environment: sdk-bot + runs-on: ubuntu-latest + timeout-minutes: 20 + concurrency: + group: core-watch-${{ github.repository }} + cancel-in-progress: false + steps: + - id: app-token + uses: actions/create-github-app-token@64ab729a6294000f5aa4a4bc0b73d1f32bc45510 # v3.3.0 + with: + client-id: ${{ secrets.SDK_BOT_CLIENT_ID }} + private-key: ${{ secrets.SDK_BOT_PRIVATE_KEY }} + permission-contents: write + permission-pull-requests: write + permission-issues: write + + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 0 + token: ${{ steps.app-token.outputs.token }} + persist-credentials: true + + - uses: jdx/mise-action@b44d12e7d0d8ee7461bb893e1b1993500433185b # v5.2.7 + with: + install: false + + - name: Install tools + run: | + if [ -f mise.lock ]; then mise install --locked; else mise install; fi + + - name: Set the bot identity + env: + GH_TOKEN: ${{ steps.app-token.outputs.token }} + APP_SLUG: ${{ steps.app-token.outputs.app-slug }} + run: | + set -euo pipefail + id=$(gh api "users/${APP_SLUG}[bot]" --jq .id) + git config user.name "${APP_SLUG}[bot]" + git config user.email "${id}+${APP_SLUG}[bot]@users.noreply.github.com" + + - name: Propose new core tags + env: + GH_TOKEN: ${{ steps.app-token.outputs.token }} + INPUT_TAG: ${{ inputs.tag }} + run: | + set -euo pipefail + if [ -n "$INPUT_TAG" ]; then + tags=$INPUT_TAG + else + tags=$(mise run --quiet core:tags) + fi + if [ -z "$tags" ]; then + echo "No core tags newer than $(mise run --quiet core:version)." + exit 0 + fi + for t in $tags; do + mise run core:bump -- "$t" + done + + - name: Report failure + if: failure() + env: + GH_TOKEN: ${{ steps.app-token.outputs.token }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: mise run core:report-failure -- core-watch "$RUN_URL" diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index eb81739..86ca28c 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -34,3 +34,9 @@ jobs: - uses: ./spdx-check with: include: "*.mjs" + + core-sync: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - run: shellcheck -x -S warning core-sync/*.sh diff --git a/README.md b/README.md index 6c980c7..6ad64f9 100644 --- a/README.md +++ b/README.md @@ -8,3 +8,8 @@ Reusable GitHub Actions workflows for the ai-openshell organization. `SPDX-License-Identifier: Apache-2.0` header. - [`quay-login`](quay-login/): logs docker, podman or helm in to quay.io as a robot account, using the job's GitHub OIDC token instead of a secret. + +## Workflows + +- [`core-watch.yml`, `core-drift.yml`](core-sync/): keep an SDK repository in + step with OpenShell core, with the core-sync scripts behind mise tasks. diff --git a/core-sync/README.md b/core-sync/README.md new file mode 100644 index 0000000..4436f09 --- /dev/null +++ b/core-sync/README.md @@ -0,0 +1,94 @@ +# core-sync + +Keeps an SDK repository in step with OpenShell core +([NVIDIA/OpenShell](https://github.com/NVIDIA/OpenShell)): one pin, +`OPENSHELL_CORE_REF` in `mise.toml`, a bump pull request for each new core +release, a nightly drift check against core `main`, and a release tag when a +stable pin lands on `main`. + +The language-independent part lives here: the scripts in this directory and +the reusable workflows +[`core-watch.yml`](../.github/workflows/core-watch.yml) and +[`core-drift.yml`](../.github/workflows/core-drift.yml). The SDK repository +provides the language-specific part as mise tasks. + +## Scripts + +| Script | Task | What it does | +| ----------------------- | --------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------- | +| `core-tags.sh` | `core:tags` | Lists core release tags newer than the pin. | +| `open-bump-pr.sh` | `core:bump` | Moves the pin to a tag, regenerates, refreshes copied core files, and opens the `chore(core): track ` pull request. | +| `drift-check.sh` | `core:drift` | Compares core `main` with the pin: `buf breaking`, regenerate then `check` and `test`, uncovered RPCs. Writes a markdown report. | +| `update-drift-issue.sh` | `core:drift-issue` | Creates, updates, or closes the "Core main drift" issue from a report. | +| `report-failure.sh` | `core:report-failure` | Records a failed automation run in the drift issue. | +| `tag-release.sh` | `release:tag` | Tags `vX.Y.Z` and drafts the GitHub release when a stable pin lands on `main`. The repository's own `release.yml` builds and publishes the package. | + +Each script documents its arguments in its header. All of them run from the +root of the SDK clone and use `gh`, `git`, and `mise`. + +## Contract + +The SDK repository's `mise.toml` provides: + +| Name | Kind | Used by | What it does | +| --------------------------- | ---- | ---------------- | -------------------------------------------------------------------------------------------------------- | +| `OPENSHELL_CORE_REF` | env | all | The pin, a core release tag (`vX.Y.Z` or `vX.Y.Z-pre.N`), as `OPENSHELL_CORE_REF = ""` on one line. | +| `OPENSHELL_GENERATED_PATHS` | env | bump, drift | Space-separated paths of the committed generated bindings, for the summaries. | +| `core:version` | task | core-watch | Prints the pin. | +| `proto:fetch` | task | drift | Exports the protos the SDK generates from, at the pin, into `proto/`. | +| `proto:gen` | task | bump, drift | Regenerates the committed bindings from the pin. | +| `proto:buf` | task | drift | Runs `buf` with the given arguments. | +| `check`, `test` | task | drift | The repository's checks (including a type check, if the language has one) and unit tests. | +| `core:uncovered-rpcs` | task | drift (optional) | Prints the RPCs in the bindings that the curated SDK doesn't call, one `Service.Method` per line. | + +Files copied from core, listed as ` ` lines in +`scripts/vendored-core-files.txt`, are refreshed by bump pull requests when +the file exists. + +The repository also needs: + +- The `sdk-bot` environment, deployable only from `main`, with the + `SDK_BOT_CLIENT_ID` and `SDK_BOT_PRIVATE_KEY` secrets of a GitHub App that + can write contents, pull requests, and issues. +- The labels `core-bump`, `stable`, `prerelease`, and `core-drift`. + +## Using it + +Point the core-sync tasks at the scripts with mise +[remote tasks](https://mise.jdx.dev/tasks/toml-tasks.html#remote-tasks), +pinned to a commit of this repository: + +```toml +[tasks."core:tags"] +description = "List core release tags newer than the pin" +file = "git::https://github.com/ai-openshell/workflows.git//core-sync/core-tags.sh?ref=" +``` + +and call the reusable workflows at the same commit: + +```yaml +name: core-watch + +on: + schedule: + - cron: "17 * * * *" + workflow_dispatch: + inputs: + tag: + description: Core tag to propose; empty proposes every new tag + required: false + default: "" + +permissions: + contents: read + +jobs: + watch: + uses: ai-openshell/workflows/.github/workflows/core-watch.yml@ + with: + tag: ${{ inputs.tag }} + secrets: inherit +``` + +The workflow and the scripts are pinned separately, so update both refs +together. diff --git a/core-sync/bump-pr-body.md b/core-sync/bump-pr-body.md new file mode 100644 index 0000000..cba358c --- /dev/null +++ b/core-sync/bump-pr-body.md @@ -0,0 +1,16 @@ +Move the core pin from `{{OLD}}` to `{{NEW}}` ({{KIND}}). + +- Core changes: [{{OLD}}...{{NEW}}]({{COMPARE_URL}}) +- Regenerated bindings: {{GENERATED}} +- E2E runs against `{{IMAGE}}` +- Files copied from core: +{{FIXTURES}} + +## Proto changes + +{{PROTO_DIFF}} + +## Before merging + +- CI and E2E must pass. If the SDK needs changes for this core version, push them to this branch. +- Pre-release pins are not released; a newer pre-release proposal replaces this one. diff --git a/core-sync/core-tags.sh b/core-sync/core-tags.sh new file mode 100755 index 0000000..8bb4de4 --- /dev/null +++ b/core-sync/core-tags.sh @@ -0,0 +1,72 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: Apache-2.0 + +# core-tags.sh: print NVIDIA/OpenShell release tags newer than the pin, +# oldest first. +# +# Usage: core-tags.sh [--all] [--pin ] +# +# Release tags are vX.Y.Z and vX.Y.Z-pre.N; a pre-release sorts below its +# stable version. By default every newer stable tag is printed, plus the +# newest pre-release when it is newer than all of them: one proposal for the +# current release train instead of one per pre-release. --all prints every +# newer tag. --pin overrides the pin, OPENSHELL_CORE_REF in mise.toml. The pin +# must be a release tag; a commit SHA fails. +set -euo pipefail + +repo=${CORE_REPO:-NVIDIA/OpenShell} +all=0 +pin= + +while [ $# -gt 0 ]; do + case $1 in + --all) all=1; shift ;; + --pin) pin=${2:?--pin needs a tag}; shift 2 ;; + *) echo "usage: $0 [--all] [--pin ]" >&2; exit 2 ;; + esac +done + +[ -n "$pin" ] || pin=$(sed -n 's/^OPENSHELL_CORE_REF = "\(.*\)"$/\1/p' mise.toml) +[ -n "$pin" ] || { echo "ERROR: no OPENSHELL_CORE_REF in mise.toml" >&2; exit 1; } + +[[ "$pin" =~ ^v[0-9]+\.[0-9]+\.[0-9]+(-pre\.[0-9]+)?$ ]] || + { echo "ERROR: pin '$pin' is not a release tag; commit SHA pins are not supported by core-sync" >&2; exit 1; } + +# key : "major minor patch rank pre" with rank 1 for stable, 0 for pre. +key() { + awk -v t="$1" 'BEGIN { + s = t; sub(/^v/, "", s) + pre = ""; rank = 1 + if (index(s, "-pre.")) { pre = substr(s, index(s, "-pre.") + 5); s = substr(s, 1, index(s, "-pre.") - 1); rank = 0 } + split(s, v, ".") + printf "%d %d %d %d %d\n", v[1], v[2], v[3], rank, (pre == "" ? 0 : pre) + }' +} + +newer=$( + gh api --paginate "repos/$repo/git/matching-refs/tags/v" --jq '.[].ref' | + sed 's#^refs/tags/##' | + { grep -E '^v[0-9]+\.[0-9]+\.[0-9]+(-pre\.[0-9]+)?$' || true; } | + while read -r t; do echo "$(key "$t") $t"; done | + awk -v p="$(key "$pin")" ' + BEGIN { split(p, k, " ") } + { + for (i = 1; i <= 5; i++) { + if ($i + 0 > k[i] + 0) { print; next } + if ($i + 0 < k[i] + 0) next + } + }' | + sort -n -k1,1 -k2,2 -k3,3 -k4,4 -k5,5 | + awk '{ print $6 }' +) + +[ -n "$newer" ] || exit 0 +if [ "$all" -eq 1 ]; then + echo "$newer" + exit 0 +fi + +# Stable tags, then the last tag if it is a pre-release (newer than every stable). +grep -v -- '-pre\.' <<<"$newer" || true +last=$(tail -1 <<<"$newer") +case $last in *-pre.*) echo "$last" ;; esac diff --git a/core-sync/drift-check.sh b/core-sync/drift-check.sh new file mode 100755 index 0000000..6c9956a --- /dev/null +++ b/core-sync/drift-check.sh @@ -0,0 +1,153 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: Apache-2.0 + +# drift-check.sh: compare core main with the pinned core tag. +# +# Usage: drift-check.sh --out +# +# Run from the root of an SDK clone that implements the core-sync contract +# (README.md). Works in a temporary worktree of HEAD. Three checks: +# 1. buf breaking (FILE rules) between the protos `mise run proto:fetch` +# exports at the pin and at core main, so only the files the SDK +# generates from count +# 2. regenerate and test: `mise run proto:gen` with the pin moved to core +# main, then `mise run check` and `mise run test`. The type check in +# `check` matters: a new enum value can pass buf breaking and the unit +# tests and still break an exhaustive map. +# 3. uncovered RPCs: `mise run core:uncovered-rpcs` on the regenerated +# bindings, when the repository has that task; informational, it does +# not fail the check +# Writes a markdown report with the header +# "Pin: | Core main: | Result: pass|fail" and one section per +# check. Exits 0 when checks 1 and 2 pass, 1 otherwise; when the check itself +# cannot run, it exits 1 without writing a report. +set -euo pipefail + +core_url=${CORE_URL:-https://github.com/NVIDIA/OpenShell.git} +out= +while [ $# -gt 0 ]; do + case $1 in + --out) out=${2:?--out needs a file}; shift 2 ;; + *) echo "usage: $0 --out " >&2; exit 2 ;; + esac +done +[ -n "$out" ] || { echo "usage: $0 --out " >&2; exit 2; } +[ "$(git rev-parse --show-toplevel)" = "$(pwd -P)" ] || { + echo "ERROR: run this from the root of the SDK clone" >&2 + exit 1 +} +case $out in /*) ;; *) out="$(pwd)/$out" ;; esac +rm -f "$out" + +pin_of() { sed -n 's/^OPENSHELL_CORE_REF = "\(.*\)"$/\1/p'; } +pin=$(pin_of &2; exit 1; } +[[ "$pin" =~ ^v[0-9]+\.[0-9]+\.[0-9]+(-pre\.[0-9]+)?$ ]] || { + echo "ERROR: pin '$pin' is not a release tag; commit SHA pins are not supported by core-sync" >&2 + exit 1 +} +main_sha=$(git ls-remote "$core_url" refs/heads/main | cut -f1) +[ -n "$main_sha" ] || { echo "ERROR: cannot resolve core main" >&2; exit 1; } + +work=$(mktemp -d) +cleanup() { git worktree remove --force "$work/wt" >/dev/null 2>&1 || true; rm -rf "$work"; } +trap cleanup EXIT +result=pass +fence() { printf '```text\n%s\n```\n' "$1"; } +has_task() { mise tasks info "$1" >/dev/null 2>&1; } + +git worktree add --quiet --detach "$work/wt" HEAD +cd "$work/wt" +mise trust --quiet mise.toml + +# The protos at the pin, for buf breaking. proto:fetch writes them to proto/. +log=$(mise run --quiet proto:fetch 2>&1) || { + printf '%s\n' "$log" >&2 + echo "ERROR: mise run proto:fetch failed at the pin $pin" >&2 + exit 1 +} +mv proto "$work/pinned" +files=$(cd "$work/pinned" && find . -name '*.proto' | sed 's#^\./##' | sort) + +# 2. Move the pin to core main and regenerate. proto:gen exports main's protos +# into proto/ on the way. +sed -i.bak "s/^OPENSHELL_CORE_REF = \".*\"$/OPENSHELL_CORE_REF = \"$main_sha\"/" mise.toml +rm -f mise.toml.bak +mise trust --quiet mise.toml +# proto:check compares the bindings with the last commit, so commit the +# regenerated ones in the throwaway worktree before running check. +commit_regenerated() { + git add -A && + git -c user.name=core-drift -c user.email=core-drift@localhost -c commit.gpgsign=false \ + commit --quiet --no-verify --allow-empty -m "regenerate from core main" +} +if build=$( + mise run --quiet proto:gen 2>&1 && + commit_regenerated 2>&1 && + mise run --quiet check 2>&1 && + mise run --quiet test 2>&1 +); then + # shellcheck disable=SC2086 # a space-separated list of paths + changed=$(git diff --stat HEAD~1 HEAD -- ${OPENSHELL_GENERATED_PATHS:-.} | tail -1 | sed 's/^ *//') + build_section="Bindings from main pass \`mise run check\` and \`mise run test\` (${changed:-no generated files change})." +else + result=fail + build_section="Regenerating from main, checking or testing against it fails: + +$(fence "$(tail -n 60 <<<"$build")")" +fi +# proto/ exists when proto:gen got as far as exporting main's protos. +generated=no +if [ -d proto ]; then generated=yes; fi + +# 1. buf breaking between the two exports. Without a buf.yaml the directories +# are plain modules, so FILE rules compare file by file. +if [ "$generated" = no ]; then + result=fail + breaking_section="Not run: exporting core main's protos failed (see below)." +elif breaking=$(mise run --quiet proto:buf -- breaking proto --against "$work/pinned" \ + --config '{"version":"v2","breaking":{"use":["FILE"]}}' 2>&1); then + breaking_section="No breaking changes in $(echo $files | sed 's/ /, /g') between \`$pin\` and main." +else + result=fail + breaking_section="Breaking changes between \`$pin\` and main: + +$(fence "$breaking")" +fi + +# 3. RPCs the SDK does not call yet, from the repository's own task: one +# "Service.Method" per line. +if ! has_task core:uncovered-rpcs; then + uncovered_section="Not reported: the repository has no \`core:uncovered-rpcs\` task." +elif [ "$generated" = no ]; then + uncovered_section="Not reported: regenerating from main failed." +elif ! uncovered=$(mise run --quiet core:uncovered-rpcs 2>&1); then + uncovered_section="\`mise run core:uncovered-rpcs\` failed: + +$(fence "$uncovered")" +elif [ -z "$uncovered" ]; then + uncovered_section="Every RPC in the regenerated bindings is called by the SDK." +else + uncovered_section="RPCs in core main that the SDK does not call yet ($(grep -c . <<<"$uncovered")): + +$(sed 's/^/- /' <<<"$uncovered")" +fi + +cat >"$out" < " lines of a body's failures section. +failure_lines() { + awk -v h="$failures_heading" ' + $0 == h { f = 1; next } + f && /^## / { f = 0 } + f && /^- / { print }' <<<"$1" +} + +# compose_body +compose_body() { + printf '%s\n\n%s\n\n' "$1" "$failures_heading" + if [ -n "$2" ]; then + printf '%s\n\n' "$2" + else + printf 'None.\n\n' + fi + printf '%s\n' "Automation appends a dated line here when core-watch or core-drift fails. Delete a line once it is handled; the issue closes when the drift check passes and no lines are left." +} diff --git a/core-sync/open-bump-pr.sh b/core-sync/open-bump-pr.sh new file mode 100755 index 0000000..99eeeea --- /dev/null +++ b/core-sync/open-bump-pr.sh @@ -0,0 +1,225 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: Apache-2.0 + +# open-bump-pr.sh: open the update proposal that moves the core pin to . +# +# Usage: open-bump-pr.sh [--dry-run] [--base ] +# +# Run from the root of an SDK clone. Builds branch core/ from +# origin/main (or --base) in a temporary worktree: pin change +# (OPENSHELL_CORE_REF in mise.toml), regenerated bindings, refreshed copies of +# core files (scripts/vendored-core-files.txt, when present), one commit +# "chore(core): track ". Then pushes it and opens a PR labeled +# core-bump plus stable or prerelease, and closes open pre-release proposals +# for older tags. --dry-run does the local part, prints the PR body, and +# changes nothing on GitHub. +# +# Skips the tag when any PR from branch core/ exists (open, merged, or +# closed), so a closed proposal is never reopened. A branch on origin without +# a PR is the leftover of a run that pushed but failed to open the PR; the +# run rebuilds it and replaces the branch (force-with-lease) so the PR gets +# opened. When the core tag no longer exists, closes an open proposal for it. +# The current pin must be a release tag; a commit SHA pin stops the run. +# Git identity and GH_TOKEN come from the environment (the App in CI). +set -euo pipefail + +core_repo=${CORE_REPO:-NVIDIA/OpenShell} +# The generated bindings, for the summary in the PR body (mise.toml [env]). +generated_paths=${OPENSHELL_GENERATED_PATHS:?set OPENSHELL_GENERATED_PATHS in mise.toml} +# The SDK repository: SDK_REPO, else the repository running the workflow, else +# the one the clone belongs to. +sdk_repo=${SDK_REPO:-${GITHUB_REPOSITORY:-$(gh repo view --json nameWithOwner --jq .nameWithOwner)}} +image=ghcr.io/nvidia/openshell/gateway +diff_cap=400 + +tag= +dry=0 +base=origin/main +while [ $# -gt 0 ]; do + case $1 in + --dry-run) dry=1; shift ;; + --base) base=${2:?--base needs a ref}; shift 2 ;; + -*) echo "usage: $0 [--dry-run] [--base ]" >&2; exit 2 ;; + *) [ -z "$tag" ] || { echo "usage: $0 [--dry-run]" >&2; exit 2; }; tag=$1; shift ;; + esac +done +[[ "$tag" =~ ^v[0-9]+\.[0-9]+\.[0-9]+(-pre\.[0-9]+)?$ ]] || { + echo "ERROR: '$tag' is not a core release tag (vX.Y.Z or vX.Y.Z-pre.N)" >&2 + exit 2 +} + +here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +body_tmpl="$here/bump-pr-body.md" +branch="core/$tag" +case $tag in *-pre.*) kind=prerelease ;; *) kind=stable ;; esac + +die() { echo "ERROR: $*" >&2; exit 1; } + +# older : true when release tag a sorts below b (vX.Y.Z-pre.N < vX.Y.Z). +older() { + awk -v a="$1" -v b="$2" ' + function key(t, s, pre, rank, v) { + s = t; sub(/^v/, "", s); pre = 0; rank = 1 + if (index(s, "-pre.")) { pre = substr(s, index(s, "-pre.") + 5) + 0; s = substr(s, 1, index(s, "-pre.") - 1); rank = 0 } + split(s, v, ".") + return sprintf("%06d%06d%06d%d%06d", v[1], v[2], v[3], rank, pre) + } + BEGIN { exit !(key(a) < key(b)) }' +} +run() { if [ "$dry" -eq 1 ]; then echo "dry-run: $*"; else "$@"; fi; } +pin_of() { sed -n 's/^OPENSHELL_CORE_REF = "\(.*\)"$/\1/p'; } + +[ "$(git rev-parse --show-toplevel)" = "$(pwd -P)" ] || die "run this from the root of the SDK clone" +[ -f "$body_tmpl" ] || die "$body_tmpl is missing" + +open_pr_for_branch() { + gh pr list -R "$sdk_repo" --head "$branch" --state open --json number --jq '.[0].number // empty' +} + +# The core tag may have been deleted (a withdrawn pre-release). Only a 404 +# means that; any other failure (rate limit, 5xx) stops the run. +if ! err=$(gh api "repos/$core_repo/git/ref/tags/$tag" --silent 2>&1); then + grep -q 'HTTP 404' <<<"$err" || die "looking up core tag $tag failed: $err" + n=$(open_pr_for_branch) + if [ -n "$n" ]; then + echo "Core tag $tag is gone; closing #$n." + run gh pr close "$n" -R "$sdk_repo" --delete-branch \ + --comment "Core tag \`$tag\` no longer exists in $core_repo, so this proposal is closed." + else + echo "Core tag $tag does not exist; nothing to do." + fi + exit 0 +fi + +# Any earlier proposal is final: merged, closed by a maintainer, or closed as +# superseded by a newer pre-release. Its branch may be gone, so ask for PRs. +prev=$(gh pr list -R "$sdk_repo" --head "$branch" --state all --json number,state \ + --jq '.[0] | select(.) | "#\(.number) (\(.state | ascii_downcase))"') +if [ -n "$prev" ]; then + echo "Proposal $prev exists for $tag already; skipping." + exit 0 +fi +# No PR, but the branch may exist: an earlier run pushed it and then failed +# to open the PR. Remember its tip so the push replaces exactly that. +stale_tip=$(git ls-remote --heads origin "$branch" | cut -f1) +[ -z "$stale_tip" ] || echo "Branch $branch exists on origin without a PR; rebuilding it for $tag." + +git fetch --quiet origin +git rev-parse --verify --quiet "$base^{commit}" >/dev/null || die "base $base is not a commit" +old=$(git show "$base:mise.toml" | pin_of) +[ -n "$old" ] || die "no OPENSHELL_CORE_REF in mise.toml on $base" +[[ "$old" =~ ^v[0-9]+\.[0-9]+\.[0-9]+(-pre\.[0-9]+)?$ ]] || + die "OPENSHELL_CORE_REF on $base is '$old', not a release tag; the core-sync automation does not support commit SHA pins" +[ "$old" != "$tag" ] || { echo "The pin on $base is $tag already; skipping."; exit 0; } + +work=$(mktemp -d) +cleanup() { git worktree remove --force "$work/wt" >/dev/null 2>&1 || true; rm -rf "$work"; } +trap cleanup EXIT +git worktree add --quiet -b "$branch" "$work/wt" "$base" + +( + cd "$work/wt" + sed -i.bak "s/^OPENSHELL_CORE_REF = \".*\"$/OPENSHELL_CORE_REF = \"$tag\"/" mise.toml && rm -f mise.toml.bak + [ "$(pin_of &2; exit 1; } + mise trust --quiet mise.toml + log=$(mise run --quiet proto:gen 2>&1) || { echo "$log" >&2; exit 1; } + + # Refresh the copies of core files; one not in the tag yet is left as is. + : >"$work/fixtures.txt" + while read -r src local; do + status=$(curl -sS -o "$work/fixture" -w '%{http_code}' \ + "https://raw.githubusercontent.com/$core_repo/$tag/$src") + case $status in + 200) + if ! cmp -s "$work/fixture" "$local"; then + cp "$work/fixture" "$local" + echo "- \`$local\` updated from core \`$tag\`" >>"$work/fixtures.txt" + fi + ;; + 404) echo "- \`$local\` is not in core \`$tag\` yet, left unchanged" >>"$work/fixtures.txt" ;; + *) echo "ERROR: fetching $src at $tag returned HTTP $status" >&2; exit 1 ;; + esac + done < <([ ! -f scripts/vendored-core-files.txt ] || grep -v '^#' scripts/vendored-core-files.txt) + + # Keep the README's "currently vX.Y.Z" in step with stable pins. + if [ "$kind" = stable ]; then + sed -i.bak "s/currently \`${old}\`/currently \`${tag}\`/" README.md && rm -f README.md.bak + fi + + git add -A + git commit --quiet --signoff -m "chore(core): track $tag" -m "Move the core pin from $old to $tag and regenerate the bindings." +) + +# Proto diff between the two core tags. The compare API caps its file list at +# 300 entries, so fetch both tags shallow and blobless and diff locally; only +# the proto/ blobs are downloaded. +compare_url="https://github.com/$core_repo/compare/$old...$tag" +git init --quiet --bare "$work/core.git" +git -C "$work/core.git" fetch --quiet --depth=1 --filter=blob:none --no-tags \ + "https://github.com/$core_repo.git" "refs/tags/$old:refs/tags/$old" "refs/tags/$tag:refs/tags/$tag" +proto_diff=$(git -C "$work/core.git" diff "$old" "$tag" -- proto/) +if [ -z "$proto_diff" ]; then + proto_section="No changes under \`proto/\` between \`$old\` and \`$tag\`." +else + lines=$(wc -l <<<"$proto_diff" | tr -d ' ') + shown=$(head -n "$diff_cap" <<<"$proto_diff") + proto_section=$(printf '```diff\n%s\n```' "$shown") + if [ "$lines" -gt "$diff_cap" ]; then + proto_section=$(printf '%s\n\nShowing %s of %s lines; see the [full compare](%s).' "$proto_section" "$diff_cap" "$lines" "$compare_url") + fi +fi +# shellcheck disable=SC2086 # a space-separated list of paths +generated=$(git diff --stat "$base" "$branch" -- $generated_paths | tail -1 | sed "s/^ *//") +fixtures=$(cat "$work/fixtures.txt") + +body="$work/body.md" +OLD="$old" NEW="$tag" KIND="$kind" COMPARE_URL="$compare_url" IMAGE="$image:${tag#v}" \ +GENERATED="${generated:-no generated files changed}" FIXTURES="${fixtures:- - none changed}" \ +PROTO_DIFF="$proto_section" \ + awk '{ + while (match($0, /\{\{[A-Z_]+\}\}/)) { + key = substr($0, RSTART + 2, RLENGTH - 4) + $0 = substr($0, 1, RSTART - 1) ENVIRON[key] substr($0, RSTART + RLENGTH) + } + print + }' "$body_tmpl" >"$body" + +echo "Proposal for $tag ($kind), pin $old -> $tag:" +git log --format=' %h %an <%ae> %s' "$base..$branch" +git diff --stat "$base" "$branch" | tail -1 | sed 's/^/ /' + +if [ "$dry" -eq 1 ]; then + echo + echo "PR title: chore(core): track $tag" + echo "PR body:" + sed 's/^/ | /' "$body" + cleanup + git branch -D "$branch" >/dev/null + echo + echo "dry-run: nothing pushed; local branch $branch removed." + exit 0 +fi + +if [ -n "$stale_tip" ]; then + git push --quiet --force-with-lease="$branch:$stale_tip" origin "$branch" +else + git push --quiet origin "$branch" +fi +url=$(gh pr create -R "$sdk_repo" --base main --head "$branch" \ + --title "chore(core): track $tag" --body-file "$body" \ + --label core-bump --label "$kind") +echo "Opened $url" +new=${url##*/} + +# Close open pre-release proposals for older tags; this one supersedes them. +gh pr list -R "$sdk_repo" --label core-bump --label prerelease --state open \ + --json number,headRefName --jq '.[] | "\(.number) \(.headRefName)"' | + while read -r n head; do + other=${head#core/} + [ "$n" != "$new" ] && [ "$other" != "$tag" ] || continue + if older "$other" "$tag"; then + echo "Closing #$n ($other), superseded by #$new." + gh pr close "$n" -R "$sdk_repo" --delete-branch \ + --comment "Superseded by #$new, which tracks core \`$tag\`." + fi + done diff --git a/core-sync/report-failure.sh b/core-sync/report-failure.sh new file mode 100755 index 0000000..262d08a --- /dev/null +++ b/core-sync/report-failure.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: Apache-2.0 + +# report-failure.sh: record a failed automation run in the drift issue. +# +# Usage: report-failure.sh [--dry-run] +# +# Appends "- " to the "## Automation failures" section +# of the open "Core main drift" issue, creating the issue when none is open. +# Also writes the line to the job summary. +set -euo pipefail +# shellcheck source=core-sync/drift-issue.sh +. "$(dirname "${BASH_SOURCE[0]}")/drift-issue.sh" + +dry=0 +args=() +for a in "$@"; do + case $a in --dry-run) dry=1 ;; *) args+=("$a") ;; esac +done +[ ${#args[@]} -eq 2 ] || { echo "usage: $0 [--dry-run]" >&2; exit 2; } +line="- $(date -u +%Y-%m-%d) ${args[0]} ${args[1]}" +[ -z "${GITHUB_STEP_SUMMARY:-}" ] || echo "Automation failure: $line" >>"$GITHUB_STEP_SUMMARY" + +n=$(drift_issue_number) +if [ -n "$n" ]; then + current=$(gh issue view "$n" -R "$sdk_repo" --json body --jq .body) + report=$(awk -v h="$failures_heading" '$0 == h { exit } { print }' <<<"$current" | sed -e :a -e '/^\n*$/{$d;N;ba' -e '}') + failures=$(failure_lines "$current") +else + report="No drift report yet; the next core-drift run fills this in." + failures= +fi +failures=$(printf '%s\n%s' "$failures" "$line" | sed '/^$/d') +body=$(compose_body "$report" "$failures") + +if [ "$dry" -eq 1 ]; then + if [ -n "$n" ]; then target="update #$n"; else target="create the issue"; fi + echo "dry-run: would $target with:" + sed 's/^/ | /' <<<"$body" + exit 0 +fi +if [ -n "$n" ]; then + gh issue edit "$n" -R "$sdk_repo" --body "$body" >/dev/null + echo "Recorded the failure in #$n." +else + gh issue create -R "$sdk_repo" --title "$drift_title" --label "$drift_label" --body "$body" +fi diff --git a/core-sync/tag-release.sh b/core-sync/tag-release.sh new file mode 100755 index 0000000..de99aca --- /dev/null +++ b/core-sync/tag-release.sh @@ -0,0 +1,129 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: Apache-2.0 + +# tag-release.sh: start the lockstep SDK release for a stable core pin. +# +# Usage: tag-release.sh [--dry-run] [--force] [--sha ] [--since ] +# +# Reads the pin, OPENSHELL_CORE_REF in mise.toml, at (default: +# $GITHUB_SHA, else HEAD). A pre-release pin creates nothing. Without --force, +# nothing is created either when the pin at equals the pin at --since +# (default: the parent of ) or there is no pin there, so ordinary +# pushes to main and the push that adds mise.toml are no-ops. +# The release workflow passes the commit before the push as --since, which +# covers pushes of several commits. Then, each step only when missing: +# - tag vX.Y.Z (lightweight) at ; a tag at another commit fails +# - a draft GitHub release with the merged PR's body plus +# "Matches OpenShell core vX.Y.Z" +# Prints "tag=vX.Y.Z" when the release still has to be built and published, +# and nothing when there is nothing to release or the release is published +# already. The workflow appends the output to $GITHUB_OUTPUT. +set -euo pipefail + +# The SDK repository: SDK_REPO, else the repository running the workflow, else +# the one the clone belongs to. +sdk_repo=${SDK_REPO:-${GITHUB_REPOSITORY:-$(gh repo view --json nameWithOwner --jq .nameWithOwner)}} +dry=0 +force=0 +sha=${GITHUB_SHA:-} +since= + +usage() { echo "usage: $0 [--dry-run] [--force] [--sha ] [--since ]" >&2; exit 2; } +while [ $# -gt 0 ]; do + case $1 in + --dry-run) dry=1; shift ;; + --force) force=1; shift ;; + --sha) sha=${2:?--sha needs a commit}; shift 2 ;; + --since) since=${2:?--since needs a commit}; shift 2 ;; + *) usage ;; + esac +done +[ -n "$sha" ] || sha=$(git rev-parse HEAD) +sha=$(git rev-parse "$sha^{commit}") +short=$(git rev-parse --short "$sha") + +pin_at() { git show "$1:mise.toml" 2>/dev/null | sed -n 's/^OPENSHELL_CORE_REF = "\(.*\)"$/\1/p'; } + +pin=$(pin_at "$sha") +if ! [[ "$pin" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + echo "Pin $pin is not a stable release; no SDK release." >&2 + exit 0 +fi + +if [ "$force" -eq 0 ]; then + # A new branch push reports an all-zero "before"; fall back to the parent. + if [ -z "$since" ] || ! git rev-parse --verify --quiet "$since^{commit}" >/dev/null; then + since="$sha^" + fi + old=$(pin_at "$since" || true) + if [ "$old" = "$pin" ]; then + echo "The pin is $pin both at $short and at $(git rev-parse --short "$since"); no SDK release." >&2 + exit 0 + fi + # No pin before means the push added mise.toml (the import), not a bump. + if [ -z "$old" ]; then + echo "No pin at $(git rev-parse --short "$since"); no SDK release." >&2 + exit 0 + fi +fi + +# get_or_empty : the value, empty on 404, and fail on any other +# error. gh prints the error body on stdout for a 404, so stdout is only used +# after a successful call. +get_or_empty() { + local out err + if out=$(gh api "$1" --jq "$2" 2>"${tmp}/err"); then + printf '%s' "$out" + else + err=$(cat "${tmp}/err") + grep -q 'HTTP 404' <<<"$err" || { echo "ERROR: gh api $1 failed: $err" >&2; return 1; } + fi +} +tmp=$(mktemp -d) +trap 'rm -rf "$tmp"' EXIT + +existing=$(get_or_empty "repos/$sdk_repo/git/ref/tags/$pin" '.object.sha') +if [ -n "$existing" ] && [ "$existing" != "$sha" ]; then + echo "ERROR: tag $pin exists at $existing, not at $sha. SDK tags never move; fix this by hand." >&2 + exit 1 +fi +# The tag lookup doesn't return drafts, so search the release list. +release=$(gh api --paginate "repos/$sdk_repo/releases" \ + --jq ".[] | select(.tag_name == \"$pin\") | if .draft then \"draft\" else \"published\" end" | head -1) +if [ "$release" = published ]; then + echo "Release $pin is published already; nothing to do." >&2 + exit 0 +fi + +# Release notes: the body of the PR that merged this commit, then the match line. +notes="$tmp/notes.md" +pr_body=$(get_or_empty "repos/$sdk_repo/commits/$sha/pulls" '.[0].body // empty') +{ + [ -z "$pr_body" ] || printf '%s\n\n' "$pr_body" + echo "Matches OpenShell core $pin." +} >"$notes" + +if [ "$dry" -eq 1 ]; then + if [ -n "$existing" ]; then + echo "dry-run: tag $pin exists at $sha" >&2 + else + echo "dry-run: would create tag $pin at $sha" >&2 + fi + if [ "$release" = draft ]; then + echo "dry-run: draft release $pin exists" >&2 + else + echo "dry-run: would create draft release $pin with notes:" >&2 + sed 's/^/ | /' "$notes" >&2 + fi + echo "tag=$pin" + exit 0 +fi + +if [ -z "$existing" ]; then + gh api -X POST "repos/$sdk_repo/git/refs" -f ref="refs/tags/$pin" -f sha="$sha" --silent + echo "Created tag $pin at $sha." >&2 +fi +if [ -z "$release" ]; then + gh release create "$pin" -R "$sdk_repo" --draft --verify-tag --title "$pin" --notes-file "$notes" >&2 +fi +echo "tag=$pin" diff --git a/core-sync/update-drift-issue.sh b/core-sync/update-drift-issue.sh new file mode 100755 index 0000000..492dae5 --- /dev/null +++ b/core-sync/update-drift-issue.sh @@ -0,0 +1,64 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: Apache-2.0 + +# update-drift-issue.sh: keep the single "Core main drift" issue current. +# +# Usage: update-drift-issue.sh [--dry-run] +# +# The report comes from drift-check.sh. The issue body is the report plus the +# "## Automation failures" section, which is carried over unchanged. With +# "Result: fail" or pending failures, the issue is created or its body +# replaced. With "Result: pass" and no pending failures, an open issue is +# closed with a comment, and nothing is created. +set -euo pipefail +# shellcheck source=core-sync/drift-issue.sh +. "$(dirname "${BASH_SOURCE[0]}")/drift-issue.sh" + +report= +dry=0 +for a in "$@"; do + case $a in + --dry-run) dry=1 ;; + -*) echo "usage: $0 [--dry-run]" >&2; exit 2 ;; + *) report=$a ;; + esac +done +[ -f "$report" ] || { echo "usage: $0 [--dry-run]" >&2; exit 2; } + +text=$(cat "$report") +header=$(head -1 <<<"$text") +case $header in +*"Result: pass"*) result=pass ;; +*"Result: fail"*) result=fail ;; +*) echo "ERROR: $report has no 'Result: pass|fail' header" >&2; exit 1 ;; +esac + +n=$(drift_issue_number) +failures= +[ -z "$n" ] || failures=$(failure_lines "$(gh issue view "$n" -R "$sdk_repo" --json body --jq .body)") +body=$(compose_body "$text" "$failures") + +if [ "$result" = pass ] && [ -z "$failures" ]; then + if [ -z "$n" ]; then + echo "Drift check passes and no failures are pending; no issue needed." + exit 0 + fi + echo "Closing #$n: the drift check passes and no failures are pending." + [ "$dry" -eq 1 ] && exit 0 + gh issue edit "$n" -R "$sdk_repo" --body "$body" >/dev/null + gh issue close "$n" -R "$sdk_repo" --comment "The drift check passes against core main ($(sed 's/.*Core main: \([0-9a-f]*\).*/\1/' <<<"$header")) and no automation failures are pending." + exit 0 +fi + +if [ "$dry" -eq 1 ]; then + if [ -n "$n" ]; then target="update #$n"; else target="create the issue"; fi + echo "dry-run: would $target with:" + sed 's/^/ | /' <<<"$body" + exit 0 +fi +if [ -n "$n" ]; then + gh issue edit "$n" -R "$sdk_repo" --body "$body" >/dev/null + echo "Updated #$n." +else + gh issue create -R "$sdk_repo" --title "$drift_title" --label "$drift_label" --body "$body" +fi From 48190aef802bffe8d45feab6c169dd09c24a94a3 Mon Sep 17 00:00:00 2001 From: Piotr Mlocek Date: Thu, 19 Nov 2026 10:31:28 -0800 Subject: [PATCH 2/5] fix(core-sync): compare the proto exports as plain directories An inline v2 config makes the repository root the buf module, so buf refuses proto/ as the input. A v1 config applies to the directory. Signed-off-by: Piotr Mlocek --- core-sync/drift-check.sh | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/core-sync/drift-check.sh b/core-sync/drift-check.sh index 6c9956a..13535bc 100755 --- a/core-sync/drift-check.sh +++ b/core-sync/drift-check.sh @@ -100,13 +100,14 @@ fi generated=no if [ -d proto ]; then generated=yes; fi -# 1. buf breaking between the two exports. Without a buf.yaml the directories -# are plain modules, so FILE rules compare file by file. +# 1. buf breaking between the two exports, file by file (FILE rules). A v1 +# config applies to the input directory itself; a v2 one would make the +# repository root the module. if [ "$generated" = no ]; then result=fail breaking_section="Not run: exporting core main's protos failed (see below)." elif breaking=$(mise run --quiet proto:buf -- breaking proto --against "$work/pinned" \ - --config '{"version":"v2","breaking":{"use":["FILE"]}}' 2>&1); then + --config '{"version":"v1","breaking":{"use":["FILE"]}}' 2>&1); then breaking_section="No breaking changes in $(echo $files | sed 's/ /, /g') between \`$pin\` and main." else result=fail From 63d4b041b9b99fdfa3491573d31a537451a427ab Mon Sep 17 00:00:00 2001 From: Piotr Mlocek Date: Thu, 19 Nov 2026 10:32:49 -0800 Subject: [PATCH 3/5] docs(core-sync): callers pass no secrets The called jobs read the App secrets from the caller's sdk-bot environment, so secrets: inherit isn't needed. Signed-off-by: Piotr Mlocek --- core-sync/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/core-sync/README.md b/core-sync/README.md index 4436f09..8e1393f 100644 --- a/core-sync/README.md +++ b/core-sync/README.md @@ -87,8 +87,8 @@ jobs: uses: ai-openshell/workflows/.github/workflows/core-watch.yml@ with: tag: ${{ inputs.tag }} - secrets: inherit ``` -The workflow and the scripts are pinned separately, so update both refs -together. +The caller passes no secrets: the called jobs declare the `sdk-bot` +environment and read the App secrets from it. The workflow and the scripts +are pinned separately, so update both refs together. From 33ae68eb87030b53b479f4ed8426f458c1d52b2d Mon Sep 17 00:00:00 2001 From: Piotr Mlocek Date: Thu, 19 Nov 2026 10:43:44 -0800 Subject: [PATCH 4/5] feat(core-sync): run the scripts of the workflow's own commit The reusable workflows load core-sync/ as an action with `uses: $/core-sync`, which resolves to this repository at the commit the caller pinned, and the action sets CORE_SYNC_DIR. SDKs keep a copy of core-sync.sh, which runs the scripts from CORE_SYNC_DIR in CI and, locally, downloads the commit that .github/workflows/core-drift.yml calls. The workflow pin is then the only one; mise remote tasks with their own ref are no longer needed. Add release-tag.yml so the release workflow's tag job is shared too. Signed-off-by: Piotr Mlocek --- .github/workflows/core-drift.yml | 4 ++ .github/workflows/core-watch.yml | 4 ++ .github/workflows/release-tag.yml | 65 +++++++++++++++++++++++++++++++ README.md | 2 +- core-sync/README.md | 51 ++++++++++++++---------- core-sync/action.yml | 7 ++++ core-sync/core-sync.sh | 35 +++++++++++++++++ 7 files changed, 146 insertions(+), 22 deletions(-) create mode 100644 .github/workflows/release-tag.yml create mode 100644 core-sync/action.yml create mode 100755 core-sync/core-sync.sh diff --git a/.github/workflows/core-drift.yml b/.github/workflows/core-drift.yml index 455c72e..529582b 100644 --- a/.github/workflows/core-drift.yml +++ b/.github/workflows/core-drift.yml @@ -42,6 +42,10 @@ jobs: run: | if [ -f mise.lock ]; then mise install --locked; else mise install; fi + # The core-sync scripts of the commit this workflow runs from; the + # SDK's scripts/core-sync.sh runs them from CORE_SYNC_DIR. + - uses: $/core-sync + # Exit 1 with a report means drift; the issue records it. No report # means the check itself broke, which the failure step records. - name: Check drift against core main diff --git a/.github/workflows/core-watch.yml b/.github/workflows/core-watch.yml index 8611296..fcbda59 100644 --- a/.github/workflows/core-watch.yml +++ b/.github/workflows/core-watch.yml @@ -52,6 +52,10 @@ jobs: run: | if [ -f mise.lock ]; then mise install --locked; else mise install; fi + # The core-sync scripts of the commit this workflow runs from; the + # SDK's scripts/core-sync.sh runs them from CORE_SYNC_DIR. + - uses: $/core-sync + - name: Set the bot identity env: GH_TOKEN: ${{ steps.app-token.outputs.token }} diff --git a/.github/workflows/release-tag.yml b/.github/workflows/release-tag.yml new file mode 100644 index 0000000..71b9e74 --- /dev/null +++ b/.github/workflows/release-tag.yml @@ -0,0 +1,65 @@ +# SPDX-License-Identifier: Apache-2.0 + +# Reusable: the first job of an SDK's lockstep release. When the core pin on +# main moved to a stable tag vX.Y.Z, tags vX.Y.Z and creates a draft GitHub +# release (core-sync/tag-release.sh), and outputs the tag; the caller's own +# jobs then build, publish, and publish the release. Outputs an empty tag when +# there is nothing to release. The caller job grants contents: write and +# pull-requests: read. +name: release-tag + +on: + workflow_call: + inputs: + sha: + description: Commit on main to release, for workflow_dispatch (default the tip); the pin there must be stable + type: string + required: false + default: "" + outputs: + tag: + description: The tag to release, or empty + value: ${{ jobs.tag.outputs.tag }} + +permissions: + contents: read + +jobs: + tag: + name: tag + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: write # create the tag and the draft release + pull-requests: read # read the merged PR's body for the release notes + outputs: + tag: ${{ steps.tag.outputs.tag }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 0 + persist-credentials: false + + - uses: $/core-sync + + - id: tag + name: Tag the release and create a draft + env: + GH_TOKEN: ${{ github.token }} + EVENT: ${{ github.event_name }} + INPUT_SHA: ${{ inputs.sha }} + BEFORE: ${{ github.event.before }} + run: | + set -euo pipefail + if [ "$EVENT" = workflow_dispatch ]; then + sha=${INPUT_SHA:-$GITHUB_SHA} + git merge-base --is-ancestor "$sha" "$GITHUB_SHA" || { + echo "::error::$sha is not on main" + exit 1 + } + "$CORE_SYNC_DIR/tag-release.sh" --force --sha "$sha" >>"$GITHUB_OUTPUT" + else + # Compare against the commit before the push, so a pin change + # that isn't the pushed tip still counts. + "$CORE_SYNC_DIR/tag-release.sh" --since "$BEFORE" >>"$GITHUB_OUTPUT" + fi diff --git a/README.md b/README.md index 6ad64f9..74724fa 100644 --- a/README.md +++ b/README.md @@ -11,5 +11,5 @@ Reusable GitHub Actions workflows for the ai-openshell organization. ## Workflows -- [`core-watch.yml`, `core-drift.yml`](core-sync/): keep an SDK repository in +- [`core-watch.yml`, `core-drift.yml`, `release-tag.yml`](core-sync/): keep an SDK repository in step with OpenShell core, with the core-sync scripts behind mise tasks. diff --git a/core-sync/README.md b/core-sync/README.md index 8e1393f..77f3386 100644 --- a/core-sync/README.md +++ b/core-sync/README.md @@ -8,10 +8,18 @@ stable pin lands on `main`. The language-independent part lives here: the scripts in this directory and the reusable workflows -[`core-watch.yml`](../.github/workflows/core-watch.yml) and -[`core-drift.yml`](../.github/workflows/core-drift.yml). The SDK repository +[`core-watch.yml`](../.github/workflows/core-watch.yml), +[`core-drift.yml`](../.github/workflows/core-drift.yml), and +[`release-tag.yml`](../.github/workflows/release-tag.yml). The SDK repository provides the language-specific part as mise tasks. +The SDK pins one commit of this repository, in the `uses:` lines of its +workflows. The reusable workflows run the scripts of that same commit: they +load this directory as an action with `uses: $/core-sync`, which resolves to +the workflow's own repository at the commit being run, and the action sets +`CORE_SYNC_DIR`. Locally, the SDK's copy of `core-sync.sh` reads the commit +from `.github/workflows/core-drift.yml` and downloads that commit's scripts. + ## Scripts | Script | Task | What it does | @@ -23,7 +31,7 @@ provides the language-specific part as mise tasks. | `report-failure.sh` | `core:report-failure` | Records a failed automation run in the drift issue. | | `tag-release.sh` | `release:tag` | Tags `vX.Y.Z` and drafts the GitHub release when a stable pin lands on `main`. The repository's own `release.yml` builds and publishes the package. | -Each script documents its arguments in its header. All of them run from the +`core-sync.sh` is the SDK-side entry point that finds the scripts, and `action.yml` sets `CORE_SYNC_DIR` in the reusable workflows. Each script documents its arguments in its header. All of them run from the root of the SDK clone and use `gh`, `git`, and `mise`. ## Contract @@ -54,41 +62,42 @@ The repository also needs: ## Using it -Point the core-sync tasks at the scripts with mise -[remote tasks](https://mise.jdx.dev/tasks/toml-tasks.html#remote-tasks), -pinned to a commit of this repository: +Copy [`core-sync.sh`](core-sync.sh) to `scripts/core-sync.sh` and point the +tasks at it: ```toml [tasks."core:tags"] description = "List core release tags newer than the pin" -file = "git::https://github.com/ai-openshell/workflows.git//core-sync/core-tags.sh?ref=" +run = "scripts/core-sync.sh core-tags.sh" ``` -and call the reusable workflows at the same commit: +The same for `core:bump` (`open-bump-pr.sh`), `core:drift` +(`drift-check.sh`), `core:drift-issue` (`update-drift-issue.sh`), +`core:report-failure` (`report-failure.sh`), and `release:tag` +(`tag-release.sh`, for running it locally). + +Call the reusable workflows, all at the same commit: ```yaml -name: core-watch +name: core-drift on: schedule: - - cron: "17 * * * *" + - cron: "41 4 * * *" workflow_dispatch: - inputs: - tag: - description: Core tag to propose; empty proposes every new tag - required: false - default: "" permissions: contents: read jobs: - watch: - uses: ai-openshell/workflows/.github/workflows/core-watch.yml@ - with: - tag: ${{ inputs.tag }} + drift: + uses: ai-openshell/workflows/.github/workflows/core-drift.yml@ ``` +`core-watch.yml` takes an optional `tag` input. In the release workflow, the +tag job is a call to `release-tag.yml` that grants `contents: write` and +`pull-requests: read` and passes `sha` on `workflow_dispatch`; its `tag` +output gates the build and publish jobs. + The caller passes no secrets: the called jobs declare the `sdk-bot` -environment and read the App secrets from it. The workflow and the scripts -are pinned separately, so update both refs together. +environment and read the App secrets from it. diff --git a/core-sync/action.yml b/core-sync/action.yml new file mode 100644 index 0000000..bfc7406 --- /dev/null +++ b/core-sync/action.yml @@ -0,0 +1,7 @@ +name: core-sync +description: Set CORE_SYNC_DIR to the core-sync scripts of the commit this action runs from, for an SDK's core-sync mise tasks. +runs: + using: composite + steps: + - shell: bash + run: echo "CORE_SYNC_DIR=$GITHUB_ACTION_PATH" >>"$GITHUB_ENV" diff --git a/core-sync/core-sync.sh b/core-sync/core-sync.sh new file mode 100755 index 0000000..5c7505a --- /dev/null +++ b/core-sync/core-sync.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: Apache-2.0 + +# core-sync.sh: run a shared core-sync script from ai-openshell/workflows. +# SDK repositories keep a copy of this file as scripts/core-sync.sh and point +# their core-sync mise tasks at it. +# +# Usage: scripts/core-sync.sh