Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
182 changes: 182 additions & 0 deletions .github/scripts/connector_notes.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,182 @@
# /// script
# requires-python = ">=3.11"
# dependencies = ["GitPython>=3.1"]
#
# [tool.ty.rules]
# unresolved-import = "ignore"
# ///
"""Print the release notes of timenet-connectors between two refs.

The notes list the connectors that were added, changed or removed, and the changes to the shared
connector code (the "framework"). Core changes that only touched connector files on the way are
left out; the timenet notes list them.

- A connector is a ``datasets/<org>/<name>/`` folder with a ``dataset.yaml`` card, the rule that
``scripts/gen_dataset_docs.py`` uses. Connectors are matched by the card's ``dataset_id``, so a
moved folder that keeps its id is changed, and a new id is one removal and one addition.
- An added connector cites the PR that added it, a removed one the PR that removed it. A changed
connector cites its connector PRs, and is left out when it has none.
- The framework section lists the connector PRs outside ``datasets/``, breaking ones first.
- A connector PR is a ``feat``, a ``fix`` or a breaking change that did not touch
``packages/timenet``. A core PR edits connectors on the way, and the timenet notes list it.

Called by .github/workflows/release.yml. Run it the way the workflow does::

uv run --no-project .github/scripts/connector_notes.py v0.11.0 HEAD \
--release timenet-connectors-v0.12.0
"""

import argparse
import re

from git import Repo


PACKAGE = "packages/timenet-connectors"
DATASETS = f"{PACKAGE}/src/timenet_connectors/datasets"
PR_URL = "https://github.com/OpenTSLM/TimeNet/pull/"
# A squash-merged PR title: type(scope)!: description (#123).
SUBJECT = re.compile(r"^(?P<type>\w+)(?:\((?P<scope>[^)]*)\))?(?P<bang>!)?: (?P<text>.+?)(?: \(#(?P<pr>\d+)\))?$")
# The commits cliff.toml skips: release and development-version commits.
SKIP = re.compile(r"^chore(\(release\)|: start .* development)")


def repo() -> Repo:
"""Open the repository around the current directory.

Returns:
The repository.
"""
return Repo(search_parent_directories=True)


def commits(base: str, head: str, *paths: str) -> list[tuple[str, str]]:
"""List the commits between ``base`` and ``head`` that touched ``paths``, oldest first.

Returns:
``(short hash, subject)`` pairs, without release and development-version commits.
"""
found = repo().iter_commits(f"{base}..{head}", paths=list(paths), reverse=True)
pairs = [(commit.hexsha[:9], str(commit.summary)) for commit in found]
return [(sha, subject) for sha, subject in pairs if not SKIP.match(subject)]


def notable(subject: str) -> bool:
"""Return whether a commit is a feature, a fix or a breaking change."""
match = SUBJECT.match(subject)
return bool(match) and (match["type"] in {"feat", "fix"} or bool(match["bang"]))


def link(sha: str, subject: str) -> str:
"""Link the PR of a commit, or name the commit when it has no PR number.

Returns:
A markdown link, or the short hash.
"""
number = re.search(r"\(#(\d+)\)$", subject)
return f"[#{number[1]}]({PR_URL}{number[1]})" if number else sha


def connectors(ref: str) -> dict[str, tuple[str, str]]:
"""Map each dataset id at ``ref`` to its folder and display name.

Returns:
``{dataset_id: (folder, name)}``.
"""
found = {}
try:
datasets = repo().commit(ref).tree / DATASETS
except KeyError:
return found
for folder in (folder for org in datasets.trees for folder in org.trees):
try:
text = (folder / "dataset.yaml").data_stream.read().decode()
except KeyError:
continue
dataset_id = re.search(r"^dataset_id:\s*(\S+)", text, re.M)
name = re.search(r"^name:\s*(.+)$", text, re.M)
key = dataset_id[1] if dataset_id else "/".join(folder.path.split("/")[-2:])
found[key] = (folder.path, name[1].strip() if name else key)
return found


def connector_sections(base: str, head: str, core: set[str]) -> tuple[dict[str, list[str]], set[str]]:
"""List the connectors that were added, changed or removed.

Returns:
The entries of each section, and the commits that added a connector.
"""
old, new = connectors(base), connectors(head)
sections: dict[str, list[str]] = {"Added connectors": [], "Changed connectors": [], "Removed connectors": []}
added: set[str] = set()
for dataset_id in sorted(old.keys() | new.keys()):
folders = sorted({entry[0] for entry in (old.get(dataset_id), new.get(dataset_id)) if entry})
touched = commits(base, head, *folders)
if dataset_id not in old:
section, cited = "Added connectors", touched[:1]
added.update(sha for sha, _ in cited)
elif dataset_id not in new:
section, cited = "Removed connectors", touched[-1:]
else:
section, cited = "Changed connectors", [c for c in touched if c[0] not in core and notable(c[1])]
if cited:
name = (new.get(dataset_id) or old[dataset_id])[1]
links = ", ".join(dict.fromkeys(link(*c) for c in cited))
sections[section].append(f"- {name} (`{dataset_id}`): {links}")
return sections, added


def framework_entries(base: str, head: str, skip: set[str]) -> list[str]:
"""List the connector PRs outside ``datasets/``, breaking ones first.

Returns:
The entries, without the commits in ``skip``.
"""
entries = []
for sha, subject in commits(base, head, PACKAGE, f":(exclude){DATASETS}"):
match = SUBJECT.match(subject)
if match and sha not in skip and notable(subject):
breaking = "**Breaking:** " if match["bang"] else ""
scope = f"**{match['scope']}:** " if match["scope"] else ""
text = match["text"][:1].upper() + match["text"][1:]
entries.append((not match["bang"], f"- {breaking}{scope}{text} ({link(sha, subject)})"))
return [entry for _, entry in sorted(entries, key=lambda pair: pair[0])]


def notes(base: str, head: str) -> str:
"""Render the notes as markdown.

Returns:
The markdown, or an empty string when nothing in the package changed.
"""
core = {sha for sha, _ in commits(base, head, "packages/timenet")}
sections, added = connector_sections(base, head, core)
sections["Connector framework"] = framework_entries(base, head, core | added)
out = []
for title, entries in sections.items():
if entries:
out += [f"### {title}", "", *entries, ""]
return "\n".join(out)


def badge(release: str) -> str:
"""Render the PyPI badge of a ``<package>-vX.Y.Z`` tag, like the one cliff.toml writes.

Returns:
The markdown badge and a blank line.
"""
package, version = release.rsplit("-v", 1)
label = package.replace("-", "--")
image = f"https://img.shields.io/badge/{label}-{version}-blue?logo=pypi&logoColor=white"
return f"[![{package} {version}]({image})](https://pypi.org/project/{package}/{version}/)\n\n"


if __name__ == "__main__":
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument("base", help="the last release tag")
parser.add_argument("head", help="the commit to release")
parser.add_argument(
"--release", help="the release tag, for example timenet-connectors-v0.12.0, to add a PyPI badge"
)
args = parser.parse_args()
print((badge(args.release) if args.release else "") + notes(args.base, args.head), end="")
113 changes: 67 additions & 46 deletions .github/workflows/publish.yml
Original file line number Diff line number Diff line change
@@ -1,5 +1,9 @@
name: Publish to PyPI

# Uploads the wheel and the sdist that release.yml built and attached to the GitHub release of a
# <package>-vX.Y.Z tag. It does not build. release.yml starts the TestPyPI run, and a successful
# TestPyPI run starts the PyPI run of the same tag.

on:
workflow_dispatch:
inputs:
Expand All @@ -20,8 +24,12 @@ on:
- both
- timenet
- timenet-connectors
ref:
description: Release tag to publish, for example timenet-v0.12.0
required: true
type: string

run-name: Publish ${{ inputs.package }} to ${{ inputs.target }}
run-name: Publish ${{ inputs.package }} ${{ inputs.ref }} to ${{ inputs.target }}

env:
BRANCH_NAME: ${{ github.head_ref || github.ref_name }}
Expand All @@ -39,67 +47,80 @@ jobs:
# tokens are stored as repo secrets.
id-token: write
contents: read
# Start the PyPI run after a TestPyPI run.
actions: write
steps:
- name: Guard PyPI to main
- name: Guard to main
run: |
echo "Branch: $BRANCH_NAME"
if [ "${{ inputs.target }}" = "PyPI" ] && [ "$BRANCH_NAME" != "main" ]; then
echo "::error title=Invalid branch::PyPI releases must come from main."
if [ "$BRANCH_NAME" != "main" ]; then
echo "::error title=Invalid branch::Run publish.yml from main."
exit 1
fi
- name: Checkout
uses: actions/checkout@v5
- name: Set up uv
uses: astral-sh/setup-uv@v7
- name: Build
# build_dists.py records the commit inside each package, so a build can name its code.
with:
ref: ${{ inputs.ref }}
fetch-depth: 0
- name: Guard the release tag
env:
REF: ${{ inputs.ref }}
run: |
if [ "${{ inputs.package }}" = "both" ]; then
uv run scripts/build_dists.py --all-packages
else
uv run scripts/build_dists.py --package "${{ inputs.package }}"
if ! [[ "$REF" =~ ^(timenet|timenet-connectors)-v[0-9]+\.[0-9]+\.[0-9]+$ ]] || ! git merge-base --is-ancestor HEAD origin/main; then
echo "::error title=Invalid ref::Publish only a <package>-vX.Y.Z tag on main, not '$REF'."
exit 1
fi
- name: Smoke test built artifacts
# Import each built wheel and sdist in an isolated environment to catch missing files or
# broken metadata before uploading. Each artifact must also report this commit, clean, as
# the code it was built from. timenet-connectors depends on timenet, so hand it the
# wheel we just built when it is present (a `both` run); otherwise timenet resolves from the
# default index, which requires it to be published already.
- name: Set up uv
uses: astral-sh/setup-uv@v7
- name: Download the release files
# Each published package needs its own tag on this commit, and the files its GitHub
# release holds: one wheel and one sdist at the tagged version, nothing else.
env:
GH_TOKEN: ${{ github.token }}
GH_REPO: ${{ github.repository }}
PACKAGE: ${{ inputs.package }}
run: |
set -euo pipefail
shopt -s nullglob
timenet_whl=(dist/timenet-*.whl)
smoke() {
local module="$1" dist="$2"; shift 2
local args=() artifact
for artifact in "$@"; do args+=(--with "$artifact"); done
echo "::group::import $module"
uv run --isolated --no-project "${args[@]}" \
python -c "import $module; print('smoke ok:', $module.__name__)"
uv run --isolated --no-project "${args[@]}" python -c "
from timenet.manifest import GitSource
from timenet.provenance import PackageSnapshot
commit = PackageSnapshot.read('$dist').source
expected = GitSource(sha='$GITHUB_SHA', contains_local_changes=False)
assert commit == expected, f'$dist records {commit}, expected {expected}'
print('commit ok:', commit)"
echo "::endgroup::"
}
for artifact in dist/*; do
case "$artifact" in
dist/timenet-*.whl | dist/timenet-*.tar.gz)
smoke timenet timenet "$artifact" ;;
dist/timenet_connectors-*.whl | dist/timenet_connectors-*.tar.gz)
smoke timenet_connectors timenet-connectors ${timenet_whl[@]+"${timenet_whl[@]}"} "$artifact" ;;
esac
if [ "$PACKAGE" = both ]; then packages="timenet timenet-connectors"; else packages=$PACKAGE; fi
expected=()
for package in $packages; do
version=$(uv version --package "$package" --short)
tag="$package-v$version"
if [ "$(git rev-parse -q --verify "refs/tags/$tag^{commit}")" != "$(git rev-parse HEAD)" ]; then
echo "::error title=Not released::$package is at $version, but $tag does not point at this commit. Release it first."
exit 1
fi
gh release download "$tag" --dir dist
expected+=("${package//-/_}-$version-py3-none-any.whl" "${package//-/_}-$version.tar.gz")
done
if [ "$(ls dist | sort)" != "$(printf '%s\n' "${expected[@]}" | sort)" ]; then
echo "::error title=Unexpected files::The releases hold $(ls dist | xargs), expected ${expected[*]}."
exit 1
fi
ls -l dist
- name: Publish
run: |
if [ "${{ inputs.target }}" = "TestPyPI" ]; then
uv publish --publish-url https://test.pypi.org/legacy/
uv publish --publish-url https://test.pypi.org/legacy/ dist/*
else
uv publish
uv publish dist/*
fi
- name: Start the PyPI run
# A TestPyPI run starts the PyPI run of the same tag, which waits for approval. So PyPI
# can only be approved after TestPyPI took the same files.
if: inputs.target == 'TestPyPI'
env:
GH_TOKEN: ${{ github.token }}
GH_REPO: ${{ github.repository }}
PACKAGE: ${{ inputs.package }}
REF: ${{ inputs.ref }}
run: |
name="Publish $PACKAGE $REF to PyPI"
if gh run list --workflow publish.yml --event workflow_dispatch --limit 100 --json displayTitle \
--jq '.[].displayTitle' | grep -qxF "$name"; then
echo "Skipping '$name': it was already started."
exit 0
fi
gh workflow run publish.yml --ref main -f target=PyPI -f package="$PACKAGE" -f ref="$REF"
- name: Summary
if: success()
run: |
Expand Down
Loading
Loading