diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 0000000..8584626 --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,6 @@ +# SPDX-License-Identifier: Apache-2.0 + +# SDK repositories run the core-sync scripts and reusable workflows with their +# bot App's secrets, so changes to them need an infra review. +/core-sync/ @ai-openshell/infra +/.github/ @ai-openshell/infra diff --git a/.github/workflows/core-drift.yml b/.github/workflows/core-drift.yml new file mode 100644 index 0000000..11c4219 --- /dev/null +++ b/.github/workflows/core-drift.yml @@ -0,0 +1,69 @@ +# SPDX-License-Identifier: Apache-2.0 + +# Reusable: compares core main with the calling SDK repository's core pin and +# keeps its single "Core main drift" issue current (mise run core:drift). See +# core-sync/README.md for the contract the caller implements. +name: core-drift + +on: + workflow_call: + +permissions: + contents: read + +jobs: + drift: + name: drift + # The bot secrets live in the caller's sdk-bot environment, which only + # main can deploy to, so a pushed branch can't mint a bot token. + environment: sdk-bot + runs-on: ubuntu-latest + timeout-minutes: 20 + concurrency: + group: core-drift-${{ github.repository }} + cancel-in-progress: false + steps: + - id: app-token + uses: actions/create-github-app-token@64ab729a6294000f5aa4a4bc0b73d1f32bc45510 # v3.3.0 + with: + client-id: ${{ secrets.SDK_BOT_CLIENT_ID }} + private-key: ${{ secrets.SDK_BOT_PRIVATE_KEY }} + permission-issues: write + + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - uses: jdx/mise-action@b44d12e7d0d8ee7461bb893e1b1993500433185b # v5.2.7 + with: + # A fixed mise, not the newest release, for jobs that hold the bot token. + version: 2026.11.6 + install: false + + - name: Install tools + run: | + if [ -f mise.lock ]; then mise install --locked; else mise install; fi + + # The core-sync scripts of the commit this workflow runs from; the + # SDK's scripts/core-sync.sh runs them from CORE_SYNC_DIR. + - uses: $/core-sync + + # Exit 1 with a report means drift; the issue records it. No report + # means the check itself broke, which the failure step records. + - name: Check drift against core main + run: | + mise run core:drift -- --out "$RUNNER_TEMP/drift-report.md" || true + test -s "$RUNNER_TEMP/drift-report.md" + cat "$RUNNER_TEMP/drift-report.md" >>"$GITHUB_STEP_SUMMARY" + + - name: Update the drift issue + env: + GH_TOKEN: ${{ steps.app-token.outputs.token }} + run: mise run core:drift-issue -- "$RUNNER_TEMP/drift-report.md" + + - name: Report failure + if: failure() + env: + GH_TOKEN: ${{ steps.app-token.outputs.token }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: mise run core:report-failure -- core-drift "$RUN_URL" diff --git a/.github/workflows/core-watch.yml b/.github/workflows/core-watch.yml new file mode 100644 index 0000000..12e4b4f --- /dev/null +++ b/.github/workflows/core-watch.yml @@ -0,0 +1,100 @@ +# SPDX-License-Identifier: Apache-2.0 + +# Reusable: opens one update proposal per new core release tag in the calling +# SDK repository (mise run core:bump). See core-sync/README.md for the +# contract the caller implements. +name: core-watch + +on: + workflow_call: + inputs: + tag: + description: Core tag to propose (vX.Y.Z or vX.Y.Z-pre.N); empty proposes every new tag + type: string + required: false + default: "" + +permissions: + contents: read + +jobs: + watch: + name: watch + # The bot secrets live in the caller's sdk-bot environment, which only + # main can deploy to, so a pushed branch can't mint a bot token. + environment: sdk-bot + runs-on: ubuntu-latest + timeout-minutes: 20 + concurrency: + group: core-watch-${{ github.repository }} + cancel-in-progress: false + steps: + - id: app-token + uses: actions/create-github-app-token@64ab729a6294000f5aa4a4bc0b73d1f32bc45510 # v3.3.0 + with: + client-id: ${{ secrets.SDK_BOT_CLIENT_ID }} + private-key: ${{ secrets.SDK_BOT_PRIVATE_KEY }} + permission-contents: write + permission-pull-requests: write + permission-issues: write + + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 0 + # The token stays out of .git/config: core:bump runs the SDK's + # dependency installs (npm, uv), and their install scripts must not + # find it. git push gets it from gh (setup-git below). + persist-credentials: false + + - uses: jdx/mise-action@b44d12e7d0d8ee7461bb893e1b1993500433185b # v5.2.7 + with: + # A fixed mise, not the newest release, for jobs that hold the bot token. + version: 2026.11.6 + install: false + + - name: Install tools + run: | + if [ -f mise.lock ]; then mise install --locked; else mise install; fi + + # The core-sync scripts of the commit this workflow runs from; the + # SDK's scripts/core-sync.sh runs them from CORE_SYNC_DIR. + - uses: $/core-sync + + - name: Set the bot identity + env: + GH_TOKEN: ${{ steps.app-token.outputs.token }} + APP_SLUG: ${{ steps.app-token.outputs.app-slug }} + run: | + set -euo pipefail + id=$(gh api "users/${APP_SLUG}[bot]" --jq .id) + git config user.name "${APP_SLUG}[bot]" + git config user.email "${id}+${APP_SLUG}[bot]@users.noreply.github.com" + + - name: Propose new core tags + env: + GH_TOKEN: ${{ steps.app-token.outputs.token }} + INPUT_TAG: ${{ inputs.tag }} + run: | + set -euo pipefail + # git push authenticates through gh, which reads GH_TOKEN at push + # time. open-bump-pr.sh drops GH_TOKEN while it regenerates. + gh auth setup-git + if [ -n "$INPUT_TAG" ]; then + tags=$INPUT_TAG + else + tags=$(mise run --quiet core:tags) + fi + if [ -z "$tags" ]; then + echo "No core tags newer than $(mise run --quiet core:version)." + exit 0 + fi + for t in $tags; do + mise run core:bump -- "$t" + done + + - name: Report failure + if: failure() + env: + GH_TOKEN: ${{ steps.app-token.outputs.token }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: mise run core:report-failure -- core-watch "$RUN_URL" diff --git a/.github/workflows/release-tag.yml b/.github/workflows/release-tag.yml new file mode 100644 index 0000000..71b9e74 --- /dev/null +++ b/.github/workflows/release-tag.yml @@ -0,0 +1,65 @@ +# SPDX-License-Identifier: Apache-2.0 + +# Reusable: the first job of an SDK's lockstep release. When the core pin on +# main moved to a stable tag vX.Y.Z, tags vX.Y.Z and creates a draft GitHub +# release (core-sync/tag-release.sh), and outputs the tag; the caller's own +# jobs then build, publish, and publish the release. Outputs an empty tag when +# there is nothing to release. The caller job grants contents: write and +# pull-requests: read. +name: release-tag + +on: + workflow_call: + inputs: + sha: + description: Commit on main to release, for workflow_dispatch (default the tip); the pin there must be stable + type: string + required: false + default: "" + outputs: + tag: + description: The tag to release, or empty + value: ${{ jobs.tag.outputs.tag }} + +permissions: + contents: read + +jobs: + tag: + name: tag + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: write # create the tag and the draft release + pull-requests: read # read the merged PR's body for the release notes + outputs: + tag: ${{ steps.tag.outputs.tag }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 0 + persist-credentials: false + + - uses: $/core-sync + + - id: tag + name: Tag the release and create a draft + env: + GH_TOKEN: ${{ github.token }} + EVENT: ${{ github.event_name }} + INPUT_SHA: ${{ inputs.sha }} + BEFORE: ${{ github.event.before }} + run: | + set -euo pipefail + if [ "$EVENT" = workflow_dispatch ]; then + sha=${INPUT_SHA:-$GITHUB_SHA} + git merge-base --is-ancestor "$sha" "$GITHUB_SHA" || { + echo "::error::$sha is not on main" + exit 1 + } + "$CORE_SYNC_DIR/tag-release.sh" --force --sha "$sha" >>"$GITHUB_OUTPUT" + else + # Compare against the commit before the push, so a pin change + # that isn't the pushed tip still counts. + "$CORE_SYNC_DIR/tag-release.sh" --since "$BEFORE" >>"$GITHUB_OUTPUT" + fi diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index eb81739..86ca28c 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -34,3 +34,9 @@ jobs: - uses: ./spdx-check with: include: "*.mjs" + + core-sync: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - run: shellcheck -x -S warning core-sync/*.sh diff --git a/README.md b/README.md index 6c980c7..74724fa 100644 --- a/README.md +++ b/README.md @@ -8,3 +8,8 @@ Reusable GitHub Actions workflows for the ai-openshell organization. `SPDX-License-Identifier: Apache-2.0` header. - [`quay-login`](quay-login/): logs docker, podman or helm in to quay.io as a robot account, using the job's GitHub OIDC token instead of a secret. + +## Workflows + +- [`core-watch.yml`, `core-drift.yml`, `release-tag.yml`](core-sync/): keep an SDK repository in + step with OpenShell core, with the core-sync scripts behind mise tasks. diff --git a/core-sync/README.md b/core-sync/README.md new file mode 100644 index 0000000..10a1fa1 --- /dev/null +++ b/core-sync/README.md @@ -0,0 +1,122 @@ +# core-sync + +Keeps an SDK repository in step with OpenShell core +([NVIDIA/OpenShell](https://github.com/NVIDIA/OpenShell)): one pin, +`OPENSHELL_CORE_REF` in `mise.toml`, a bump pull request for each new core +release, a nightly drift check against core `main`, and a release tag when a +stable pin lands on `main`. + +The language-independent part lives here: the scripts in this directory and +the reusable workflows +[`core-watch.yml`](../.github/workflows/core-watch.yml), +[`core-drift.yml`](../.github/workflows/core-drift.yml), and +[`release-tag.yml`](../.github/workflows/release-tag.yml). The SDK repository +provides the language-specific part as mise tasks. + +The SDK pins one commit of this repository, in the `uses:` lines of its +workflows. The reusable workflows run the scripts of that same commit: they +load this directory as an action with `uses: $/core-sync`, which resolves to +the workflow's own repository at the commit being run, and the action sets +`CORE_SYNC_DIR`. Locally, the SDK's copy of `core-sync.sh` reads the commit +from `.github/workflows/core-drift.yml` and downloads that commit's scripts. + +## Scripts + +| Script | Task | What it does | +| ----------------------- | --------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------- | +| `core-tags.sh` | `core:tags` | Lists core release tags newer than the pin. | +| `open-bump-pr.sh` | `core:bump` | Moves the pin to a tag, regenerates, refreshes copied core files, and opens the `chore(core): track ` pull request. | +| `drift-check.sh` | `core:drift` | Compares core `main` with the pin: `buf breaking`, regenerate then `check` and `test`, uncovered RPCs. Writes a markdown report. | +| `update-drift-issue.sh` | `core:drift-issue` | Creates, updates, or closes the "Core main drift" issue from a report. | +| `report-failure.sh` | `core:report-failure` | Records a failed automation run in the drift issue. | +| `tag-release.sh` | `release:tag` | Tags `vX.Y.Z` and drafts the GitHub release when a stable pin lands on `main`. The repository's own `release.yml` builds and publishes the package. | + +`core-sync.sh` is the SDK-side entry point that finds the scripts, and `action.yml` sets `CORE_SYNC_DIR` in the reusable workflows. Each script documents its arguments in its header. All of them run from the +root of the SDK clone and use `gh`, `git`, and `mise`. + +## Contract + +The SDK repository's `mise.toml` provides: + +| Name | Kind | Used by | What it does | +| --------------------------- | ---- | ---------------- | -------------------------------------------------------------------------------------------------------- | +| `OPENSHELL_CORE_REF` | env | all | The pin, a core release tag (`vX.Y.Z` or `vX.Y.Z-pre.N`), as `OPENSHELL_CORE_REF = ""` on one line. | +| `OPENSHELL_GENERATED_PATHS` | env | bump, drift | Space-separated paths of the committed generated bindings, for the summaries. | +| `core:version` | task | core-watch | Prints the pin. | +| `proto:fetch` | task | drift | Exports the protos the SDK generates from, at the pin, into `proto/`. | +| `proto:gen` | task | bump, drift | Regenerates the committed bindings from the pin. | +| `proto:buf` | task | drift | Runs `buf` with the given arguments. | +| `check`, `test` | task | drift | The repository's checks (including a type check, if the language has one) and unit tests. | +| `core:uncovered-rpcs` | task | drift (optional) | Prints the RPCs in the bindings that the curated SDK doesn't call, one `Service.Method` per line. | + +Files copied from core, listed as ` ` lines in +`scripts/vendored-core-files.txt`, are refreshed by bump pull requests when +the file exists. + +The repository also needs: + +- The `sdk-bot` environment, deployable only from `main`, with the + `SDK_BOT_CLIENT_ID` and `SDK_BOT_PRIVATE_KEY` secrets of a GitHub App that + can write contents, pull requests, and issues. +- The labels `core-bump`, `stable`, `prerelease`, and `core-drift`. + +## Using it + +Copy [`core-sync.sh`](core-sync.sh) to `scripts/core-sync.sh` and point the +tasks at it: + +```toml +[tasks."core:tags"] +description = "List core release tags newer than the pin" +run = "scripts/core-sync.sh core-tags.sh" +``` + +The same for `core:bump` (`open-bump-pr.sh`), `core:drift` +(`drift-check.sh`), `core:drift-issue` (`update-drift-issue.sh`), +`core:report-failure` (`report-failure.sh`), and `release:tag` +(`tag-release.sh`, for running it locally). + +Call the reusable workflows, all at the same commit: + +```yaml +name: core-drift + +on: + schedule: + - cron: "41 4 * * *" + workflow_dispatch: + +permissions: + contents: read + +jobs: + drift: + uses: ai-openshell/workflows/.github/workflows/core-drift.yml@ +``` + +`core-watch.yml` takes an optional `tag` input. In the release workflow, the +tag job is a call to `release-tag.yml` that grants `contents: write` and +`pull-requests: read` and passes `sha` on `workflow_dispatch`; its `tag` +output gates the build and publish jobs. + +The caller passes no secrets: the called jobs declare the `sdk-bot` +environment and read the App secrets from it. + +## Security + +The scripts and workflows here run in SDK repositories with their bot App's +secrets, so: + +- Changes here need an infra review (`.github/CODEOWNERS`). +- SDKs pin a full commit SHA of `main`. Run `scripts/core-sync.sh --check-pin` + in the SDK's CI: it fails when the workflows call the shared workflows at + different refs, or at a commit that isn't on this repository's `main`. A + commit that only exists in a fork is reachable through this repository's + name too, so a pin bump could otherwise point at one. The check runs from + the SDK's own copy of `core-sync.sh`, not from the pinned commit, and runs + again before a local download. +- `core-watch` keeps the App token out of the checkout's git config and out + of the environment while `open-bump-pr.sh` regenerates the bindings, which + installs the SDK's dependencies and runs their install scripts. `git push` + gets the token from gh. +- The workflows that hold the token use a fixed mise version. diff --git a/core-sync/action.yml b/core-sync/action.yml new file mode 100644 index 0000000..bfc7406 --- /dev/null +++ b/core-sync/action.yml @@ -0,0 +1,7 @@ +name: core-sync +description: Set CORE_SYNC_DIR to the core-sync scripts of the commit this action runs from, for an SDK's core-sync mise tasks. +runs: + using: composite + steps: + - shell: bash + run: echo "CORE_SYNC_DIR=$GITHUB_ACTION_PATH" >>"$GITHUB_ENV" diff --git a/core-sync/bump-pr-body.md b/core-sync/bump-pr-body.md new file mode 100644 index 0000000..cba358c --- /dev/null +++ b/core-sync/bump-pr-body.md @@ -0,0 +1,16 @@ +Move the core pin from `{{OLD}}` to `{{NEW}}` ({{KIND}}). + +- Core changes: [{{OLD}}...{{NEW}}]({{COMPARE_URL}}) +- Regenerated bindings: {{GENERATED}} +- E2E runs against `{{IMAGE}}` +- Files copied from core: +{{FIXTURES}} + +## Proto changes + +{{PROTO_DIFF}} + +## Before merging + +- CI and E2E must pass. If the SDK needs changes for this core version, push them to this branch. +- Pre-release pins are not released; a newer pre-release proposal replaces this one. diff --git a/core-sync/core-sync.sh b/core-sync/core-sync.sh new file mode 100755 index 0000000..dc72b81 --- /dev/null +++ b/core-sync/core-sync.sh @@ -0,0 +1,78 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: Apache-2.0 + +# core-sync.sh: run a shared core-sync script from ai-openshell/workflows. +# SDK repositories keep a copy of this file as scripts/core-sync.sh and point +# their core-sync mise tasks at it. +# +# Usage: scripts/core-sync.sh