From 0531f3db5691ea0c6a965b2915bb044fd04cb5e7 Mon Sep 17 00:00:00 2001 From: jasper blues Date: Fri, 30 Oct 2026 00:31:29 +1000 Subject: [PATCH] Skip the release gate when CI has already passed on the same tree Port of liberation-data/drivine#431 (#432). release.yml's new trust job runs scripts/ci_trust.py, which asks the GitHub API, reading only, for a push or pull_request run of ci.yml on the tree being released whose every job passed, waiting for one still going. The gate then runs only when none did; a run on the tree that it waited for and that failed fails the release. ci.yml's first job records the tree it tested, so a pull request's run of its merge ref can be matched. build, publish and github-release check their upstream results explicitly, so a skipped gate doesn't skip them. Closes #254. --- .github/workflows/ci.yml | 27 +- .github/workflows/release.yml | 55 +++- RELEASING.md | 29 +- scripts/ci_trust.py | 296 +++++++++++++++++++ tests/unit/test_ci_trust.py | 515 ++++++++++++++++++++++++++++++++++ 5 files changed, 908 insertions(+), 14 deletions(-) create mode 100644 scripts/ci_trust.py create mode 100644 tests/unit/test_ci_trust.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 89713e3..5d4865e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -4,7 +4,8 @@ on: push: branches: [main] pull_request: - # release.yml runs this as its gate, so that a release passes the same CI as a pull request. + # release.yml runs this as its gate, so that a release passes the same CI as a pull request, unless a push or + # pull_request run of it has already passed on the same tree. workflow_call: # Each night on the images the tests name, without the pin below, so an engine's new release shows up here and not # in a pull request's run. @@ -19,6 +20,30 @@ env: FALKORDB_TEST_IMAGE: ${{ github.event_name != 'schedule' && 'falkordb/falkordb:6.0.3' || '' }} jobs: + # Records the git tree this run tests, as an artifact named ci-tree-, so a release of the same tree can trust + # this run and skip its gate (scripts/ci_trust.py). It matters on a pull request, which tests its merge ref, not the + # head commit the run names. It has no condition, as no job here may: release.yml trusts a run only when every job + # in it passed, and a job skipped by a condition doesn't. + tree: + name: record the tree + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Work out the tree + id: tree + run: | + tree=$(git rev-parse 'HEAD^{tree}') + echo "tree=$tree" >> "$GITHUB_OUTPUT" + echo "$tree $(git rev-parse HEAD)" > ci-tree.txt + echo "This run tests tree $tree, of commit $(git rev-parse HEAD)." >> "$GITHUB_STEP_SUMMARY" + # Overwritten when the run is re-run, as an artifact's name is unique within a run. + - uses: actions/upload-artifact@v4 + with: + name: ci-tree-${{ steps.tree.outputs.tree }} + path: ci-tree.txt + if-no-files-found: error + overwrite: true + # The Cypher spec's copies in tests/cypher are the tag tests/cypher/VERSION names, unchanged, and a later tag is # shown as a warning. The script reads the standard library alone, so nothing is installed for it. cypher-spec: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index ff489e6..b9488e3 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -3,8 +3,8 @@ # - A push to main that raises pyproject.toml's version (the merge of a "Release vX.Y.Z" PR) publishes it, and tags # the commit vX.Y.Z. A push to main that changes pyproject.toml without raising the version publishes nothing. # - A push of a tag vX.Y.Z, by hand, publishes too: the fallback. -# - Either way, the full CI runs, the distributions are built and checked, and those exact files are uploaded to PyPI -# with trusted publishing. +# - Either way, the full CI runs, unless CI has already passed on the same tree; the distributions are built and +# checked, and those exact files are uploaded to PyPI with trusted publishing. # - workflow_dispatch, and a pull request that changes the release machinery, are dry runs: everything but the # publish, with the built files left as the run's artifact to inspect. name: Release @@ -24,6 +24,7 @@ on: - pyproject.toml - scripts/smoke_wheel.py - scripts/release_version.py + - scripts/ci_trust.py permissions: contents: read @@ -94,16 +95,51 @@ jobs: echo "publish=$publish" >> "$GITHUB_OUTPUT" echo "tag=$tag" >> "$GITHUB_OUTPUT" - # The full CI, as ci.yml runs it on a push or a pull request. A push that releases nothing stops at check: ci.yml - # runs on the same push to main by itself. - gate: + # Whether ci.yml has already passed on the git tree this run builds, from a push or a pull request, so the gate + # needn't run it again: the release PR's run, when it was up to date with main and squash merged, or the push run on + # the merge commit, which starts beside this one and is waited for. A run on this tree that fails fails the release. + # scripts/ci_trust.py says how a run's tree is known, and why a pull request's run can be believed; the step summary + # links the run trusted, or says why none was. Skipped, as the gate is, on a push that releases nothing. + trust: needs: check if: needs.check.outputs.publish == 'true' || github.event_name != 'push' + runs-on: ubuntu-latest + timeout-minutes: 70 + permissions: + # The checkout's, as for the workflow. + contents: read + # Reading ci.yml's runs, their jobs and their artifacts. + actions: read + outputs: + trusted: ${{ steps.trust.outputs.trusted }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + # On a pull request the checkout is its merge ref, and ci.yml's runs for it name its head commit. The script + # reads the standard library alone, so the runner's python3 runs it, as check's does. + - name: Look for a run of CI on this tree + id: trust + env: + GH_TOKEN: ${{ github.token }} + PR_HEAD: ${{ github.event.pull_request.head.sha }} + run: | + tree=$(git rev-parse 'HEAD^{tree}') + python3 scripts/ci_trust.py "$GITHUB_REPOSITORY" "$tree" "$(git rev-parse HEAD)" "$PR_HEAD" + + # The full CI, as ci.yml runs it on a push or a pull request, unless trust found it has already passed on this tree. + # A push that releases nothing stops at check: ci.yml runs on the same push to main by itself. + gate: + needs: [check, trust] + if: needs.trust.outputs.trusted != 'true' uses: ./.github/workflows/ci.yml - # A push that releases nothing skips gate, and so this and the jobs after it. + # Runs when the gate passed, or was skipped because trust found CI had passed on this tree. A failed or cancelled + # check, trust or gate stops it, and so does a gate skipped for any other reason (a push that releases nothing). build: - needs: [check, gate] + needs: [check, trust, gate] + if: >- + ${{ !cancelled() && needs.check.result == 'success' && needs.trust.result == 'success' && + (needs.gate.result == 'success' || + (needs.gate.result == 'skipped' && needs.trust.outputs.trusted == 'true')) }} runs-on: ubuntu-latest env: VERSION: ${{ needs.check.outputs.version }} @@ -157,9 +193,11 @@ jobs: path: dist/ if-no-files-found: error + # A skipped gate is still upstream of this job, so its condition checks build's result itself rather than leave it + # to the default, which would skip it. publish: needs: [check, build] - if: needs.check.outputs.publish == 'true' + if: ${{ !cancelled() && needs.build.result == 'success' && needs.check.outputs.publish == 'true' }} runs-on: ubuntu-latest environment: name: release @@ -179,6 +217,7 @@ jobs: github-release: needs: [check, publish] + if: ${{ !cancelled() && needs.publish.result == 'success' }} runs-on: ubuntu-latest permissions: contents: write diff --git a/RELEASING.md b/RELEASING.md index 857be4f..ffbf335 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -14,8 +14,9 @@ three repositories at once. lists the commits since the last tag. 2. **Review and merge the release PR.** It passes CI like any other PR. Merging it is the release point. 3. **CI publishes.** The merge, a push to the default branch that changes the version file, runs - `.github/workflows/release.yml`. It runs the full gate, builds, checks, publishes, and creates the GitHub release - with its tag `vX.Y.Z` on the merge commit. Nothing is built from a laptop. + `.github/workflows/release.yml`. It runs the full gate, unless CI has already passed on the same tree, builds, + checks, publishes, and creates the GitHub release with its tag `vX.Y.Z` on the merge commit. Nothing is built from + a laptop. 4. **Credentials.** Repository secrets or trusted publishing (OIDC), held in a GitHub environment named `release`. Give that environment required reviewers: now that merging publishes, their approval of the run is the human checkpoint before anything goes out. @@ -43,8 +44,24 @@ git push origin vX.Y.Z bump, for example. - On a tag push it fails, saying why, unless the tag is `vX.Y.Z` (or `vX.Y.Z-suffix`), its version equals the build file's, and the tagged commit is on the default branch. -- **`gate`** runs the repository's full CI, `ci.yml`, unchanged. A push to the default branch that releases nothing - skips it, as `ci.yml` runs on that push anyway. +- **`trust`** looks for a `ci.yml` run that has already passed on the git tree being released + (`git rev-parse HEAD^{tree}` of the commit the run checked out), so that `gate` needn't run CI again. Trees are + compared, not commits: a squash merge of a release PR that was up to date with the default branch has the tree its + pull request run tested. `scripts/ci_trust.py` does it, reading the GitHub API with `actions: read`. + - A run counts when it is from a push or a pull request (the nightly, which runs without the pins, doesn't), its + head commit has this tree, and every job of its latest attempt concluded success. A pull request run tests the + pull request's merge ref, not its head, so `ci.yml`'s first job records the tree it checked out as an artifact, + `ci-tree-`, and a pull request run counts only when it recorded this tree as well. A run of a fork's + pull request is believed for the same reason: its head has this tree, so the `ci.yml` it ran is the one being + released. + - When one has passed, `gate` is skipped. When one is still going, as the push run on the merge commit is when + the release starts beside it, `trust` waits for it, up to an hour: if it passes `gate` is skipped, and if it fails + or is cancelled the release fails, linking it. Re-running the release's failed jobs then runs `gate`, unless that + CI run has been re-run too, which `trust` then waits for or trusts. When none counts, or the wait runs out, or + the API can't be asked, `gate` runs as before. + - The run's summary links the run trusted, or says of each run it looked at why it didn't count. +- **`gate`** runs the repository's full CI, `ci.yml`, unchanged, unless `trust` found it has already passed on this + tree. A push to the default branch that releases nothing skips both, as `ci.yml` runs on that push anyway. - **`build`** builds the release artifacts from the checked-out ref, checks them, runs a smoke test, and uploads them as the run's artifact. - **`publish`** runs on a release alone, in the `release` environment. It publishes the exact files `build` made. @@ -59,7 +76,9 @@ failed jobs. A **dry run** is everything but `publish` and `github-release`. Run it from the Actions tab (Release, Run workflow, on any branch or tag), and it runs by itself on a pull request that changes the release machinery, a release PR -included. Only a merged version bump or a pushed tag publishes. +included. Only a merged version bump or a pushed tag publishes. `trust` skips its `gate` as it does a release's: a +manual run on the default branch trusts the push run on its commit, and a pull request's dry run the pull request's +own run, when the pull request is up to date with its base. ## drivine-python: PyPI diff --git a/scripts/ci_trust.py b/scripts/ci_trust.py new file mode 100644 index 0000000..3d3e49a --- /dev/null +++ b/scripts/ci_trust.py @@ -0,0 +1,296 @@ +""" +Whether CI (ci.yml) has already passed on the git tree a release is about to build, so that release.yml can skip its +gate, the full CI run again. See RELEASING.md. + + python scripts/ci_trust.py [] + + is `git rev-parse HEAD^{tree}` of , the commit release.yml checked out. On a pull request, is +its merge ref and the head is the pull request's head commit, which is what ci.yml's runs for it name. It asks the +GitHub API with `gh api`, reading only, and decides one of: + +- trust: a ci.yml run on this tree passed. It writes `trusted=true` to $GITHUB_OUTPUT and the gate is skipped. +- gate: none did, so the gate runs the full CI as before. `trusted=false`. +- fail: a run on this tree that it waited for failed or was cancelled. It exits 1, naming the run. + +Either way it says what it decided, and why, in $GITHUB_STEP_SUMMARY, linking the run it trusted or waited for. + +Trees are compared, not commits: a squash merge of a pull request that is up to date with main has the same tree as +the pull request's merge ref, which its pull_request run tested. A run counts when: + +- it is a ci.yml run in this repository, from a push or a pull request. The nightly (schedule) runs without the + FalkorDB pin, and a workflow_call run is some other workflow's, so neither counts; +- its own commit, head_commit, which GitHub reports and nothing in the run can change, has this tree. A push run + tested that commit, so that is all it needs; +- a pull_request run tested its merge ref instead, which is the same tree only when the pull request was up to date + with its base. ci.yml's first job records the tree it checked out as an artifact named `ci-tree-`, and a + pull_request run counts only when it recorded this tree, besides its head commit having it; +- it passed: concluded success, with every job of its latest attempt concluded success. A job skipped by a condition + would leave part of the tree untested and the run still green, so a skipped job means it doesn't count. + +Why a record a run makes of itself can be believed: a pull_request run runs the ci.yml of its merge ref, which a fork +could change, and its jobs run the fork's code. But a run counts only when its head commit has this tree, so the +head's ci.yml and code are byte for byte the ones being released, and its merge ref holds nothing else but what main +had, which is trusted. When it then records this tree, the ci.yml it ran is the one being released, unchanged, with +every job in it. A fork's own ci.yml can upload `ci-tree-`, but on a head commit of another tree, so it is +never looked at. + +A run on this tree that is still going, as ci.yml's push run on a merge commit is when this starts beside it, is +waited for, up to CI_TRUST_WAIT_MINUTES (60), polling every CI_TRUST_POLL_SECONDS (60): each poll makes three or four +API calls, and GITHUB_TOKEN has 1,000 an hour for the whole repository. If it passes the gate is skipped, and if it +fails or is cancelled the release fails, linking it. A wait that runs out runs the gate. So does an API call that +fails: not trusting a run is always safe, as the gate is what ran before. + +It reads the standard library alone, so nothing is installed for it. +""" + +from __future__ import annotations + +import json +import math +import os +import re +import subprocess +import sys +import time +from collections.abc import Callable, Sequence +from dataclasses import dataclass, field +from typing import Any + +CI = ".github/workflows/ci.yml" +EVENTS = ("push", "pull_request") +RECORD = "ci-tree-" +# They go into the API's paths, so each must be what it says: a repository's name, and git object names. +REPOSITORY = re.compile(r"[\w.-]+/[\w.-]+", re.ASCII) +OBJECT = re.compile(r"[0-9a-f]{40}|[0-9a-f]{64}") +USAGE = "Usage: python scripts/ci_trust.py []" + +Run = dict[str, Any] +Api = Callable[[str], Any] + + +def gh(path: str) -> Any: + """`gh api `, parsed. The token is the workflow's (GH_TOKEN), or gh's own login when run by hand.""" + done = subprocess.run(["gh", "api", path], capture_output=True, text=True) + if done.returncode != 0: + raise RuntimeError(f"gh api {path}: {done.stderr.strip() or f'exit status {done.returncode}'}") + return json.loads(done.stdout) + + +@dataclass +class Gathered: + run: Run + trees: list[str] + + +@dataclass +class Look: + """What one run says about this tree, as `classify` gives it.""" + + run: Run + state: str + reason: str + + +@dataclass +class Decision: + """`trust`, `gate` or `fail`, the run it rests on, and each run looked at with why it did or didn't count.""" + + decision: str + reason: str + run: Run | None = None + looked: list[Look] = field(default_factory=lambda: []) + + +def gather(api: Api, repo: str, tree: str, shas: Sequence[str]) -> list[Gathered]: + """ + The ci.yml runs that might be on this tree: those on the commit or the pull request head given (a push run on the + commit has its tree; a run there that has only just started has recorded nothing yet), and those whose record says + this tree (the pull request's run, when a squash merge is released). Each comes with the trees it recorded, which + are asked for only on a pull_request run whose head commit has this tree, the only runs whose record is read. + """ + runs: dict[int, Run] = {} + for sha in dict.fromkeys(shas): + found = api(f"repos/{repo}/actions/workflows/ci.yml/runs?head_sha={sha}&per_page=30") + for run in found["workflow_runs"]: + runs[run["id"]] = run + records = api(f"repos/{repo}/actions/artifacts?name={RECORD}{tree}&per_page=100") + for artifact in records["artifacts"]: + workflow_run: Run = artifact.get("workflow_run") or {} + run_id = workflow_run.get("id") + if run_id is not None and run_id not in runs: + runs[run_id] = api(f"repos/{repo}/actions/runs/{run_id}") + gathered: list[Gathered] = [] + for run in sorted(runs.values(), key=lambda run: run["id"], reverse=True): + trees: list[str] = [] + if run.get("event") == "pull_request" and _tree_of(run) == tree: + artifacts = api(f"repos/{repo}/actions/runs/{run['id']}/artifacts?per_page=100") + trees = [a["name"][len(RECORD) :] for a in artifacts["artifacts"] if a["name"].startswith(RECORD)] + gathered.append(Gathered(run, trees)) + return gathered + + +def _tree_of(run: Run) -> str | None: + commit: dict[str, Any] = run.get("head_commit") or {} + return commit.get("tree_id") + + +def classify(gathered: Gathered, tree: str) -> Look: + """ + What one run says about this tree: `passed`, `failed` or `running` when it is a run on this tree; `maybe` when it + is a pull_request run whose head has this tree and which hasn't recorded its merge ref's tree yet; `ignored`, with + the reason, when it doesn't count. + """ + run, trees = gathered.run, gathered.trees + + def look(state: str, reason: str) -> Look: + return Look(run, state, reason) + + path = (run.get("path") or "").split("@")[0] + sha = (run.get("head_sha") or "")[:7] + if path != CI: + return look("ignored", f"it is a run of {path or 'another workflow'}, not {CI}") + if run.get("event") not in EVENTS: + return look("ignored", f"it ran on {run.get('event')}, and only push and pull_request runs count") + if _tree_of(run) != tree: + return look("ignored", f"its commit {sha} is another tree") + done = run.get("status") == "completed" + if run.get("event") == "pull_request" and tree not in trees: + if trees: + return look("ignored", f"it tested its merge ref, tree {trees[0]}, as its base had moved on") + if not done: + return look("maybe", "it has not yet recorded the tree of the merge ref it tests") + return look( + "ignored", "it recorded no tree: it ran before ci.yml recorded trees, or its job 'record the tree' failed" + ) + if not done: + return look("running", f"it is {run.get('status')}") + if run.get("conclusion") == "success": + return look("passed", "it passed") + return look("failed", f"it concluded {run.get('conclusion')}") + + +def unpassed_jobs(api: Api, repo: str, run: Run) -> list[str]: + """The jobs of a run's latest attempt that didn't conclude success, as `name (conclusion)`. None means it passed.""" + jobs = api(f"repos/{repo}/actions/runs/{run['id']}/jobs?filter=latest&per_page=100")["jobs"] + if not jobs: + return ["no jobs at all"] + return [f"{job['name']} ({_text(job.get('conclusion'))})" for job in jobs if job.get("conclusion") != "success"] + + +def _text(value: Any) -> str: + """A value from the API as JavaScript's template strings write it, which the summaries were first written with.""" + return "null" if value is None else str(value) + + +def link(run: Run) -> str: + return f"[{run.get('event')} run {run['id']}]({run.get('html_url')}) on {(run.get('head_sha') or '')[:7]}" + + +def assess( + api: Api, + repo: str, + tree: str, + shas: Sequence[str], + *, + wait_seconds: float = 3600, + poll_seconds: float = 60, + now: Callable[[], float] = time.monotonic, + sleep: Callable[[float], None] = time.sleep, + log: Callable[[str], None] = lambda _: None, +) -> Decision: + """Decides, waiting while a run on this tree is going.""" + deadline = now() + wait_seconds + waited: set[int] = set() + while True: + looked = [classify(gathered, tree) for gathered in gather(api, repo, tree, shas)] + for entry in [e for e in looked if e.state == "passed"]: + unpassed = unpassed_jobs(api, repo, entry.run) + if not unpassed: + return Decision("trust", "it passed", entry.run, looked) + entry.state, entry.reason = "ignored", f"not every job passed: {', '.join(unpassed)}" + failed = next((e for e in looked if e.state == "failed" and e.run["id"] in waited), None) + if failed is not None: + return Decision("fail", failed.reason, failed.run, looked) + going = [e for e in looked if e.state in ("running", "maybe")] + if not going: + why = "no run on this tree passed" if looked else "no ci.yml run was found on this tree" + return Decision("gate", why, None, looked) + if now() >= deadline: + # Rounded half up, as JavaScript's Math.round does, rather than to even. + minutes = math.floor(wait_seconds / 60 + 0.5) + return Decision("gate", f"it waited {minutes} minutes and a run was still going", None, looked) + for entry in going: + if entry.state == "running": + waited.add(entry.run["id"]) + log(f"Waiting for {entry.run.get('html_url')}: {entry.reason}.") + sleep(poll_seconds) + + +def summary(result: Decision, tree: str) -> str: + """The step summary: the decision, the run it rests on, and each run looked at with why it did or didn't count.""" + lines = ["### CI gate", ""] + if result.decision == "trust" and result.run is not None: + lines.append(f"Skipped: CI already passed on this tree ({tree}), in {link(result.run)}.") + elif result.decision == "fail" and result.run is not None: + reason = re.sub(r"^it ", "", result.reason) + lines.append(f"Failed: {link(result.run)}, on this tree ({tree}), {reason}. Nothing is released.") + else: + lines.append(f"The gate runs the full CI on this tree ({tree}): {result.reason}.") + others = [entry for entry in result.looked if entry.run is not result.run] + if others: + lines += ["", "Runs looked at:", ""] + lines += [f"- {link(entry.run)}: {entry.reason}." for entry in others] + return "\n".join(lines) + "\n" + + +def setting(text: str | None, otherwise: float) -> float: + """ + A number of minutes or seconds from the environment, or the default when it is unset or isn't a number, so a typo + can't make the wait unbounded or the polling continuous. + """ + try: + value = float(text) if text else math.nan + except ValueError: + return otherwise + return value if math.isfinite(value) and value >= 0 else otherwise + + +def main(args: list[str]) -> int: + if not 3 <= len(args) <= 4: + print(USAGE, file=sys.stderr) + return 2 + repo, tree, *given = args + shas = [sha for sha in given if sha] + if REPOSITORY.fullmatch(repo) is None or not all(OBJECT.fullmatch(name) for name in [tree, *shas]): + print(f"Not a repository, a tree and commits: {' '.join(args)}", file=sys.stderr) + return 2 + env = os.environ + try: + result = assess( + gh, + repo, + tree, + shas, + wait_seconds=setting(env.get("CI_TRUST_WAIT_MINUTES"), 60) * 60, + poll_seconds=setting(env.get("CI_TRUST_POLL_SECONDS"), 60), + log=lambda line: print(line, flush=True), + ) + except Exception as error: # Not trusting a run is always safe: the gate runs, as it did before. + result = Decision("gate", f"the GitHub API couldn't be asked ({str(error).strip()})") + text = summary(result, tree) + print(text) + if env.get("GITHUB_STEP_SUMMARY"): + with open(env["GITHUB_STEP_SUMMARY"], "a") as out: + out.write(text) + url = result.run.get("html_url", "") if result.run is not None and result.decision != "gate" else "" + if env.get("GITHUB_OUTPUT"): + with open(env["GITHUB_OUTPUT"], "a") as out: + out.write(f"trusted={'true' if result.decision == 'trust' else 'false'}\nrun={url}\n") + if result.decision == "fail": + print(f"::error::CI failed on this tree, in {url}: {result.reason}.") + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/tests/unit/test_ci_trust.py b/tests/unit/test_ci_trust.py new file mode 100644 index 0000000..422e642 --- /dev/null +++ b/tests/unit/test_ci_trust.py @@ -0,0 +1,515 @@ +"""scripts/ci_trust.py: whether CI has already passed on the tree a release builds, so release.yml can skip its gate. +The GitHub API is faked, and so is the clock; the script as the workflow runs it gets a fake `gh` on PATH.""" + +from __future__ import annotations + +import importlib.util +import json +import os +import subprocess +import sys +from pathlib import Path +from typing import Any, cast + +import pytest + +_path = Path(__file__).parents[2] / "scripts" / "ci_trust.py" +_spec = importlib.util.spec_from_file_location("ci_trust", _path) +assert _spec is not None and _spec.loader is not None +ci_trust = importlib.util.module_from_spec(_spec) +# Its dataclasses look their module up by name. +sys.modules["ci_trust"] = ci_trust +_spec.loader.exec_module(ci_trust) + +REPO = "liberation-data/drivine-python" +TREE = "a" * 40 +OTHER = "b" * 40 +SHA = "c" * 40 +HEAD = "d" * 40 + + +def run(id: int, *, tree: str = TREE, **fields: Any) -> dict[str, Any]: + """A ci.yml run as the API gives it: by default a push run on SHA, of TREE, that passed.""" + head = fields.get("head_sha", SHA) + return { + "id": id, + "event": "push", + "status": "completed", + "conclusion": "success", + "head_sha": head, + "head_commit": {"id": head, "tree_id": tree}, + "path": ".github/workflows/ci.yml", + "html_url": f"https://github.com/{REPO}/actions/runs/{id}", + **fields, + } + + +def going(id: int, **fields: Any) -> dict[str, Any]: + return run(id, status="in_progress", conclusion=None, **fields) + + +def job(name: str, conclusion: str | None) -> dict[str, Any]: + return {"name": name, "conclusion": conclusion} + + +PASSED_JOBS = {"jobs": [job(name, "success") for name in ["record the tree", "cypher-spec", "test", "test (Oracle)"]]} + + +def runs_on(sha: str) -> str: + return f"repos/{REPO}/actions/workflows/ci.yml/runs?head_sha={sha}&per_page=30" + + +RECORDS = f"repos/{REPO}/actions/artifacts?name=ci-tree-{TREE}&per_page=100" + + +def run_path(id: int) -> str: + return f"repos/{REPO}/actions/runs/{id}" + + +def artifacts_of(id: int) -> str: + return f"repos/{REPO}/actions/runs/{id}/artifacts?per_page=100" + + +def jobs_of(id: int) -> str: + return f"repos/{REPO}/actions/runs/{id}/jobs?filter=latest&per_page=100" + + +def recorded(*trees: str) -> dict[str, Any]: + return {"artifacts": [{"name": f"ci-tree-{tree}"} for tree in trees]} + + +def record_on(*ids: int) -> dict[str, Any]: + return {"artifacts": [{"name": f"ci-tree-{TREE}", "workflow_run": {"id": id}} for id in ids]} + + +def workflow_runs(*runs: dict[str, Any]) -> dict[str, Any]: + return {"workflow_runs": list(runs)} + + +class Api: + """ + The GitHub API, faked: each path gives its answer, or a list of answers, one for each poll in turn and the last + from then on. A path not given fails the test, as the script asked for something unexpected. + """ + + def __init__(self, answers: dict[str, Any]) -> None: + self.answers = answers + self.asked: list[str] = [] + self.polls = 0 + + def get(self, path: str) -> Any: + self.asked.append(path) + if path not in self.answers: + raise AssertionError(f"Not faked: {path}") + answer: Any = self.answers[path] + if isinstance(answer, list): + answers = cast("list[Any]", answer) + return answers[min(self.polls, len(answers) - 1)] + return answer + + +class Decided: + def __init__(self, result: Any, sleeps: list[float], asked: list[str]) -> None: + self.decision: str = result.decision + self.reason: str = result.reason + self.run: dict[str, Any] | None = result.run + self.sleeps = sleeps + self.asked = asked + self.text: str = ci_trust.summary(result, TREE) + + +def decide(answers: dict[str, Any], shas: list[str] | None = None, wait_seconds: float = 3600) -> Decided: + """Runs assess against the fake API, on a fake clock that each poll's wait moves on.""" + api = Api(answers) + clock = [0.0] + sleeps: list[float] = [] + + def sleep(seconds: float) -> None: + sleeps.append(seconds) + clock[0] += seconds + api.polls += 1 + + result = ci_trust.assess( + api.get, + REPO, + TREE, + shas or [SHA], + wait_seconds=wait_seconds, + poll_seconds=30, + now=lambda: clock[0], + sleep=sleep, + ) + return Decided(result, sleeps, api.asked) + + +def test_trusts_a_push_run_on_the_releases_own_commit_that_passed_and_links_it() -> None: + result = decide({runs_on(SHA): workflow_runs(run(1)), RECORDS: record_on(), jobs_of(1): PASSED_JOBS}) + + assert (result.decision, result.run and result.run["id"], result.sleeps) == ("trust", 1, []) + assert f"Skipped: CI already passed on this tree ({TREE}), in [push run 1]" in result.text + + +def test_trusts_a_squash_merged_pull_requests_run_found_by_the_tree_it_recorded_of_its_merge_ref() -> None: + result = decide( + { + runs_on(SHA): workflow_runs(), + RECORDS: record_on(2), + run_path(2): run(2, event="pull_request", head_sha=HEAD), + artifacts_of(2): recorded(TREE), + jobs_of(2): PASSED_JOBS, + } + ) + + assert result.decision == "trust" + assert result.run is not None and (result.run["id"], result.run["event"]) == (2, "pull_request") + + +def test_looks_for_a_pull_requests_runs_by_its_head_commit_as_well_on_a_dry_run_on_the_pull_request() -> None: + result = decide( + { + runs_on(SHA): workflow_runs(), + runs_on(HEAD): workflow_runs(run(3, event="pull_request", head_sha=HEAD)), + RECORDS: record_on(3), + artifacts_of(3): recorded(TREE), + jobs_of(3): PASSED_JOBS, + }, + [SHA, HEAD], + ) + + assert (result.decision, result.run and result.run["id"]) == ("trust", 3) + + +def test_doesnt_trust_a_pull_requests_run_that_tested_another_tree_as_its_base_had_moved_on() -> None: + result = decide( + { + runs_on(SHA): workflow_runs(), + RECORDS: record_on(), + runs_on(HEAD): workflow_runs(run(4, event="pull_request", head_sha=HEAD)), + artifacts_of(4): recorded(OTHER), + }, + [SHA, HEAD], + ) + + assert result.decision == "gate" + assert f"it tested its merge ref, tree {OTHER}, as its base had moved on" in result.text + + +def test_doesnt_believe_a_record_on_a_run_whose_head_commit_is_another_tree_as_a_forks_ci_could_make_it() -> None: + forged = run(5, event="pull_request", head_sha=HEAD, tree=OTHER) + result = decide({runs_on(SHA): workflow_runs(), RECORDS: record_on(5), run_path(5): forged}) + + assert result.decision == "gate" + assert "its commit ddddddd is another tree" in result.text + assert artifacts_of(5) not in result.asked + assert jobs_of(5) not in result.asked + + +def test_doesnt_trust_a_pull_requests_run_that_recorded_no_tree_as_one_before_ci_recorded_trees() -> None: + result = decide( + { + runs_on(SHA): workflow_runs(), + runs_on(HEAD): workflow_runs(run(6, event="pull_request", head_sha=HEAD)), + RECORDS: record_on(), + artifacts_of(6): recorded(), + }, + [SHA, HEAD], + ) + + assert result.decision == "gate" + assert "it recorded no tree" in result.text + + +def test_counts_neither_the_nightly_which_drops_the_falkordb_pin_nor_a_run_of_another_workflow() -> None: + result = decide( + { + runs_on(SHA): workflow_runs(run(7, event="schedule"), run(8, path=".github/workflows/release.yml")), + RECORDS: record_on(), + } + ) + + assert result.decision == "gate" + assert "it ran on schedule, and only push and pull_request runs count" in result.text + assert "it is a run of .github/workflows/release.yml, not .github/workflows/ci.yml" in result.text + + +def test_reads_a_workflows_path_with_the_ref_github_can_append_to_it() -> None: + entry = ci_trust.Gathered(run(9, path=".github/workflows/ci.yml@refs/heads/main"), []) + assert ci_trust.classify(entry, TREE).state == "passed" + + +def test_doesnt_trust_a_green_run_with_a_job_that_was_skipped_which_left_part_of_the_tree_untested() -> None: + result = decide( + { + runs_on(SHA): workflow_runs(run(10)), + RECORDS: record_on(), + jobs_of(10): {"jobs": [job("test", "success"), job("test (Oracle)", "skipped")]}, + } + ) + + assert result.decision == "gate" + assert "not every job passed: test (Oracle) (skipped)" in result.text + + +def test_doesnt_trust_a_green_run_the_api_gives_no_jobs_for_as_nothing_shows_it_tested_anything() -> None: + result = decide({runs_on(SHA): workflow_runs(run(20)), RECORDS: record_on(), jobs_of(20): {"jobs": []}}) + + assert result.decision == "gate" + assert "not every job passed: no jobs at all" in result.text + + +def test_passes_over_a_record_whose_run_the_api_no_longer_names_as_of_a_deleted_run() -> None: + result = decide( + { + runs_on(SHA): workflow_runs(run(21)), + RECORDS: {"artifacts": [{"name": f"ci-tree-{TREE}", "workflow_run": None}]}, + jobs_of(21): PASSED_JOBS, + } + ) + + assert (result.decision, result.run and result.run["id"]) == ("trust", 21) + + +def test_runs_the_gate_after_a_run_on_this_tree_that_had_already_failed_as_a_rerun_might_pass() -> None: + result = decide({runs_on(SHA): workflow_runs(run(11, conclusion="failure")), RECORDS: record_on()}) + + assert (result.decision, result.reason, result.sleeps) == ("gate", "no run on this tree passed", []) + assert "it concluded failure" in result.text + + +def test_runs_the_gate_when_there_is_no_run_at_all_and_says_so() -> None: + result = decide({runs_on(SHA): workflow_runs(), RECORDS: record_on()}) + + assert result.decision == "gate" + assert f"The gate runs the full CI on this tree ({TREE}): no ci.yml run was found on this tree." in result.text + + +def test_waits_for_the_push_run_on_the_merge_commit_started_beside_the_release_and_trusts_it_when_it_passes() -> None: + result = decide( + { + runs_on(SHA): [ + workflow_runs(run(12, status="queued", conclusion=None)), + workflow_runs(going(12)), + workflow_runs(run(12)), + ], + RECORDS: record_on(), + jobs_of(12): PASSED_JOBS, + } + ) + + assert (result.decision, result.run and result.run["id"], result.sleeps) == ("trust", 12, [30, 30]) + + +def test_fails_the_release_when_a_run_on_this_tree_that_it_waited_for_fails_and_links_it() -> None: + result = decide( + { + runs_on(SHA): [workflow_runs(going(13)), workflow_runs(run(13, conclusion="cancelled"))], + RECORDS: record_on(), + } + ) + + assert (result.decision, result.run and result.run["id"]) == ("fail", 13) + assert f"Failed: [push run 13](https://github.com/{REPO}/actions/runs/13)" in result.text + assert "concluded cancelled. Nothing is released." in result.text + + +def test_waits_for_a_pull_requests_run_to_record_its_tree_then_for_it_to_pass() -> None: + pr = going(14, event="pull_request", head_sha=HEAD) + result = decide( + { + runs_on(SHA): workflow_runs(), + runs_on(HEAD): [ + workflow_runs(pr), + workflow_runs(pr), + workflow_runs(run(14, event="pull_request", head_sha=HEAD)), + ], + RECORDS: [record_on(), record_on(14)], + artifacts_of(14): [recorded(), recorded(TREE)], + jobs_of(14): PASSED_JOBS, + }, + [SHA, HEAD], + ) + + assert (result.decision, result.run and result.run["id"], result.sleeps) == ("trust", 14, [30, 30]) + + +def test_runs_the_gate_not_failing_when_a_pull_requests_run_it_waited_on_records_another_tree() -> None: + result = decide( + { + runs_on(SHA): workflow_runs(), + runs_on(HEAD): [ + workflow_runs(going(22, event="pull_request", head_sha=HEAD)), + workflow_runs(run(22, event="pull_request", head_sha=HEAD, conclusion="failure")), + ], + RECORDS: record_on(), + artifacts_of(22): [recorded(), recorded(OTHER)], + }, + [SHA, HEAD], + ) + + assert (result.decision, result.reason, result.sleeps) == ("gate", "no run on this tree passed", [30]) + assert f"it tested its merge ref, tree {OTHER}, as its base had moved on" in result.text + + +def test_runs_the_gate_not_failing_when_a_run_it_waited_for_ends_green_with_a_job_skipped() -> None: + result = decide( + { + runs_on(SHA): [workflow_runs(going(23)), workflow_runs(run(23))], + RECORDS: record_on(), + jobs_of(23): {"jobs": [job("test", "success"), job("test (Oracle)", "skipped")]}, + } + ) + + assert (result.decision, result.reason, result.sleeps) == ("gate", "no run on this tree passed", [30]) + assert "not every job passed: test (Oracle) (skipped)" in result.text + + +def test_runs_the_gate_when_a_run_is_still_going_once_the_wait_is_up() -> None: + result = decide({runs_on(SHA): workflow_runs(going(15)), RECORDS: record_on()}, [SHA], 90) + + assert (result.decision, result.sleeps) == ("gate", [30, 30, 30]) + assert "it waited 2 minutes and a run was still going" in result.text + + +def test_prefers_the_newest_run_that_passed() -> None: + result = decide({runs_on(SHA): workflow_runs(run(16), run(17)), RECORDS: record_on(), jobs_of(17): PASSED_JOBS}) + + assert result.run is not None and result.run["id"] == 17 + assert "Runs looked at:\n\n- [push run 16]" in result.text + + +# The script as the workflow runs it. + + +class Ran: + def __init__(self, done: subprocess.CompletedProcess[str], output: Path, summary: Path) -> None: + self.status = done.returncode + self.stdout = done.stdout + self.stderr = done.stderr + self.output = output.read_text() if output.exists() else "" + self.summary = summary.read_text() if summary.exists() else "" + + +FAKE_GH = """ +import json, os, sys +here = os.path.dirname(os.path.abspath(__file__)) +answers = json.load(open(os.path.join(here, "answers.json"))) +asked = os.path.join(here, "asked") +path = sys.argv[2] +if path not in answers: + print("HTTP 502: Bad gateway (" + path + ")", file=sys.stderr) + sys.exit(1) +times = open(asked).read().split("\\n").count(path) if os.path.exists(asked) else 0 +open(asked, "a").write(path + "\\n") +answer = answers[path] +print(json.dumps(answer[min(times, len(answer) - 1)] if isinstance(answer, list) else answer)) +""" + + +def main(tmp_path: Path, answers: dict[str, Any] | None, **env: str) -> Ran: + """ + Runs the script with a `gh` on PATH that answers from the answers given, a list of answers being one for each time + it is asked in turn and the last from then on, and fails on a path it has no answer for. The environment given is + added to the workflow's, which polls without waiting here. + """ + bin = tmp_path / "bin" + bin.mkdir() + (bin / "answers.json").write_text(json.dumps(answers or {})) + (bin / "fake_gh.py").write_text(FAKE_GH) + gh = bin / "gh" + gh.write_text(f'#!/bin/sh\nexec "{sys.executable}" "{bin / "fake_gh.py"}" "$@"\n') + gh.chmod(0o755) + output, summary = tmp_path / "output", tmp_path / "summary" + done = subprocess.run( + [sys.executable, str(_path), REPO, TREE, SHA, ""], + capture_output=True, + text=True, + env={ + **os.environ, + "PATH": f"{bin}{os.pathsep}{os.environ['PATH']}", + "GITHUB_OUTPUT": str(output), + "GITHUB_STEP_SUMMARY": str(summary), + "CI_TRUST_POLL_SECONDS": "0", + **env, + }, + ) + return Ran(done, output, summary) + + +def test_writes_trusted_true_and_the_run_it_trusted_and_the_summary(tmp_path: Path) -> None: + result = main(tmp_path, {runs_on(SHA): workflow_runs(run(18)), RECORDS: record_on(), jobs_of(18): PASSED_JOBS}) + + assert (result.status, result.output) == (0, f"trusted=true\nrun={run(18)['html_url']}\n") + assert "### CI gate\n\nSkipped: CI already passed on this tree" in result.summary + + +def test_fails_with_an_error_naming_the_run_when_a_run_on_this_tree_it_waited_for_failed(tmp_path: Path) -> None: + result = main( + tmp_path, + { + runs_on(SHA): [workflow_runs(going(19)), workflow_runs(run(19, conclusion="failure"))], + RECORDS: record_on(), + }, + ) + + url = run(19)["html_url"] + assert (result.status, result.output) == (1, f"trusted=false\nrun={url}\n") + assert f"Waiting for {url}: it is in_progress." in result.stdout + assert f"::error::CI failed on this tree, in {url}: it concluded failure." in result.stdout + assert "Failed: [push run 19]" in result.summary + + +ANSWERS_GOING_THEN_PASSED = { + runs_on(SHA): [workflow_runs(going(20)), workflow_runs(run(20))], + RECORDS: record_on(), + jobs_of(20): PASSED_JOBS, +} + + +def test_reads_the_wait_from_ci_trust_wait_minutes(tmp_path: Path) -> None: + result = main(tmp_path, ANSWERS_GOING_THEN_PASSED, CI_TRUST_WAIT_MINUTES="0") + + assert (result.status, result.output) == (0, "trusted=false\nrun=\n") + assert "it waited 0 minutes and a run was still going" in result.summary + + +# Read as a number, each would be no wait at all, or none to speak of, and the run going would run the gate rather +# than be waited for. +@pytest.mark.parametrize("wait", ["", "-5", "nan", "inf", "five"]) +def test_reads_a_wait_that_is_empty_below_zero_or_not_a_number_as_an_hour(tmp_path: Path, wait: str) -> None: + result = main(tmp_path, ANSWERS_GOING_THEN_PASSED, CI_TRUST_WAIT_MINUTES=wait) + + assert (result.status, result.output) == (0, f"trusted=true\nrun={run(20)['html_url']}\n") + + +def test_runs_the_gate_when_the_api_cant_be_asked_as_not_trusting_a_run_is_always_safe(tmp_path: Path) -> None: + result = main(tmp_path, None) + + assert (result.status, result.output) == (0, "trusted=false\nrun=\n") + assert "): the GitHub API couldn't be asked (" in result.summary + assert "HTTP 502: Bad gateway" in result.summary + + +@pytest.mark.parametrize( + "args", + [ + [REPO, "../../x", SHA], + [REPO, TREE, f"{SHA}&per_page=1"], + [REPO, TREE, f"{SHA}\n"], + ["a/b/c", TREE, SHA], + [f"{REPO}\n", TREE, SHA], + ], +) +def test_refuses_a_tree_or_a_commit_that_isnt_a_git_objects_name_as_each_goes_into_the_apis_paths( + args: list[str], +) -> None: + done = subprocess.run([sys.executable, str(_path), *args], capture_output=True, text=True) + + assert done.returncode == 2 + assert "Not a repository" in done.stderr + + +def test_prints_the_usage_when_not_given_a_repository_a_tree_and_a_commit() -> None: + done = subprocess.run([sys.executable, str(_path), REPO, TREE], capture_output=True, text=True) + + assert done.returncode == 2 + assert "Usage: python scripts/ci_trust.py " in done.stderr