From 2949d429be05aea44b2923e37eb0a28bb1f32c1a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ernesto=20Garc=C3=ADa?= Date: Sun, 25 Oct 2026 12:47:52 -0300 Subject: [PATCH 1/2] Format the Rust code with rustfmt cargo fmt --check failed on 8 spots in ai.rs and lib.rs. This commit only reformats them, so the next one can add the check to pnpm verify. --- src-tauri/src/ai.rs | 25 +++++++++++++++++++------ src-tauri/src/lib.rs | 19 +++++++++++++++---- 2 files changed, 34 insertions(+), 10 deletions(-) diff --git a/src-tauri/src/ai.rs b/src-tauri/src/ai.rs index 8c83e71..0f1dbb0 100644 --- a/src-tauri/src/ai.rs +++ b/src-tauri/src/ai.rs @@ -37,7 +37,11 @@ fn api_key() -> Option { return Some(key); } } - keychain().ok()?.get_password().ok().filter(|k| !k.trim().is_empty()) + keychain() + .ok()? + .get_password() + .ok() + .filter(|k| !k.trim().is_empty()) } #[tauri::command] @@ -157,7 +161,9 @@ async fn send( if status.is_success() { return Ok(Ok(value)); } - let message = value["error"]["message"].as_str().unwrap_or("unknown error"); + let message = value["error"]["message"] + .as_str() + .unwrap_or("unknown error"); Ok(Err(match status.as_u16() { 401 => "no_key: The API key was rejected. Check it and try again.".to_string(), 429 => format!("api: Rate limited. Try again in a moment. ({message})"), @@ -169,10 +175,14 @@ async fn send( fn parse_response(response: &Value) -> Result { match response["stop_reason"].as_str() { Some("refusal") => { - let why = response["stop_details"]["explanation"].as_str().unwrap_or("no reason given"); + let why = response["stop_details"]["explanation"] + .as_str() + .unwrap_or("no reason given"); return Err(format!("refused: Claude declined this request ({why}).")); } - Some("max_tokens") => return Err("api: The answer was cut off. Try a shorter selection.".into()), + Some("max_tokens") => { + return Err("api: The answer was cut off. Try a shorter selection.".into()) + } _ => {} } let text = response["content"] @@ -223,7 +233,10 @@ mod tests { #[test] fn reports_refusals() { - let response = json!({"stop_reason": "refusal", "stop_details": {"explanation": "x"}, "content": []}); - assert!(parse_response(&response).unwrap_err().starts_with("refused:")); + let response = + json!({"stop_reason": "refusal", "stop_details": {"explanation": "x"}, "content": []}); + assert!(parse_response(&response) + .unwrap_err() + .starts_with("refused:")); } } diff --git a/src-tauri/src/lib.rs b/src-tauri/src/lib.rs index 6f68699..2e8c887 100644 --- a/src-tauri/src/lib.rs +++ b/src-tauri/src/lib.rs @@ -9,11 +9,17 @@ const DOCUMENT_EXTENSIONS: [&str; 3] = ["md", "markdown", "txt"]; fn document_path(path: &str) -> Result<&Path, String> { let target = Path::new(path); - let ext = target.extension().and_then(|e| e.to_str()).unwrap_or("").to_lowercase(); + let ext = target + .extension() + .and_then(|e| e.to_str()) + .unwrap_or("") + .to_lowercase(); if DOCUMENT_EXTENSIONS.contains(&ext.as_str()) { Ok(target) } else { - Err(format!("Undertext only opens and saves .md, .markdown, and .txt files: {path}")) + Err(format!( + "Undertext only opens and saves .md, .markdown, and .txt files: {path}" + )) } } @@ -50,7 +56,9 @@ fn write_text(path: String, contents: String, expected: Option) -> Resul /// Only web and mail links, with no spaces or control characters. fn safe_url(url: &str) -> Result<&str, String> { let lower = url.to_lowercase(); - let scheme_ok = ["http://", "https://", "mailto:"].iter().any(|s| lower.starts_with(s)); + let scheme_ok = ["http://", "https://", "mailto:"] + .iter() + .any(|s| lower.starts_with(s)); if scheme_ok && !url.chars().any(|c| c.is_whitespace() || c.is_control()) { Ok(url) } else { @@ -115,7 +123,10 @@ mod tests { assert!(write_text(path.clone(), "edited".into(), Some("mine".into())).is_ok()); // Another app changed it: refused, and the file keeps their text. fs::write(&file, "theirs").unwrap(); - assert_eq!(write_text(path.clone(), "again".into(), Some("edited".into())), Err(CHANGED_ON_DISK.into())); + assert_eq!( + write_text(path.clone(), "again".into(), Some("edited".into())), + Err(CHANGED_ON_DISK.into()) + ); assert_eq!(fs::read_to_string(&file).unwrap(), "theirs"); // Without `expected` (Keep mine), it overwrites. assert!(write_text(path.clone(), "again".into(), None).is_ok()); From 9ad77e78131f3255295cf97baf6f3b79c0a1771c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ernesto=20Garc=C3=ADa?= Date: Sun, 25 Oct 2026 12:47:58 -0300 Subject: [PATCH 2/2] Add pnpm verify, a Definition of Done, and CI pnpm verify runs every check: both TypeScript projects, vitest, the web build, and cargo fmt, clippy, and tests. It takes about 5 seconds warm. It records a hash of the tree in .git/undertext/ only when every check passes and no file changed during the run, so a later hook can tell whether a change was verified. A docs-only change needs no run. A change to CI files alone is never recorded. - CLAUDE.md says what done means and to run pnpm verify. - .github/workflows/verify.yml runs it on macOS for pushes to main and pull requests. The actions are pinned by commit. - docs/agents.md says what is checked, what is only asked, and the known limits. - scripts/verify.test.mjs runs copies of the scripts in throwaway repositories. - .gitignore now covers .claude/settings.local.json. Claude Code writes it, and a change to it would void a pass. --- .github/workflows/verify.yml | 36 ++++++ .gitignore | 1 + CLAUDE.md | 21 +++- README.md | 7 +- docs/agents.md | 90 +++++++++++++++ package.json | 1 + scripts/tree.mjs | 87 +++++++++++++++ scripts/verify-steps.mjs | 43 ++++++++ scripts/verify.mjs | 95 ++++++++++++++++ scripts/verify.test.mjs | 207 +++++++++++++++++++++++++++++++++++ 10 files changed, 582 insertions(+), 6 deletions(-) create mode 100644 .github/workflows/verify.yml create mode 100644 docs/agents.md create mode 100644 scripts/tree.mjs create mode 100644 scripts/verify-steps.mjs create mode 100644 scripts/verify.mjs create mode 100644 scripts/verify.test.mjs diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml new file mode 100644 index 0000000..ab483eb --- /dev/null +++ b/.github/workflows/verify.yml @@ -0,0 +1,36 @@ +# Runs `pnpm verify`, the same checks as on your Mac, on every push to main and every pull request. +# macOS because the app is built and tested there, and Tauri on Linux needs extra WebKit packages. +name: verify + +on: + push: + branches: [main] + pull_request: + +permissions: + contents: read + +# A newer push to the same branch cancels the older run. A cancelled run is not a failure. +concurrency: + group: verify-${{ github.ref }} + cancel-in-progress: true + +jobs: + verify: + runs-on: macos-latest + timeout-minutes: 20 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: pnpm/action-setup@ea17c68df8912ef543352723c149a84f56e3d413 # v6.1.0 + with: + version: 10.28.1 + - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + with: + node-version: 22 + cache: pnpm + - run: rustup component add clippy rustfmt + - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + with: + workspaces: src-tauri + - run: pnpm install --frozen-lockfile + - run: pnpm verify diff --git a/.gitignore b/.gitignore index a547bf3..931e878 100644 --- a/.gitignore +++ b/.gitignore @@ -11,6 +11,7 @@ node_modules dist dist-ssr *.local +.claude/settings.local.json # Editor directories and files .vscode/* diff --git a/CLAUDE.md b/CLAUDE.md index 06a173a..9123672 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -12,13 +12,26 @@ What comes next, and the decisions behind the editor: the "Next" section of pnpm install pnpm dev # the editor in a plain browser: no files, fake AI results pnpm tauri dev # the desktop app, with live reload -pnpm test # editor, format, Lab, and palette tests (vitest, no DOM) -pnpm exec tsc --noEmit -p . # type check -cd src-tauri && cargo test --lib # Rust tests +pnpm verify # every check CI runs, in about 10 seconds +pnpm test # only the vitest tests pnpm tauri build --bundles app # src-tauri/target/release/bundle/macos/Undertext.app ``` -`cargo` must be on `PATH`. With Homebrew's rustup it lives in `/opt/homebrew/opt/rustup/bin`. +`pnpm tauri` needs `cargo` on `PATH`. With Homebrew's rustup it lives in +`/opt/homebrew/opt/rustup/bin`. `pnpm verify` finds it there on its own. + +## Done + +- Done means proven: a test that failed before the change and passes after, or the change seen + in the app. Reading the code or a passing compile is not proof. +- Run `pnpm verify` after each change, and before you call anything done. It records the tree + when every check passes on it, or when only docs changed since its last pass. +- It can't see the app window (WebKit), real AI calls, file dialogs, or the keychain. When a + change depends on them, check it in `pnpm tauri dev`, or say it is not verified. +- Never weaken a check to pass it: no skipped or deleted tests, no loosened assertions. If a + check is wrong, stop and say why. +- Report what ran, what it showed, and what you did not verify. How this works: + `docs/agents.md`. ## How the editor works diff --git a/README.md b/README.md index fe9a822..e16e9f6 100644 --- a/README.md +++ b/README.md @@ -166,11 +166,14 @@ limits. If `claude` isn't in `~/.local/bin`, `/opt/homebrew/bin`, or `/usr/local ## Development ```bash -pnpm test # editor and format tests +pnpm verify # every check CI runs: types, tests, web build, Rust format, lint, and tests +pnpm test # only the vitest tests pnpm icons # rebuild the app icons from brand/ (needs ImageMagick) -cd src-tauri && cargo test ``` +`pnpm verify` is also what coding agents run to prove their work. See +[docs/agents.md](docs/agents.md). + The editor is [CodeMirror](https://codemirror.net) inside a [Tauri](https://tauri.app) app. The page text is plain Markdown; alternatives, ghosts, and versions are ranges kept beside it (`src/editor/layers.ts`). The icon borrows the mark of [Klipstack](brand/README.md), with a handwritten u in place of its clip. diff --git a/docs/agents.md b/docs/agents.md new file mode 100644 index 0000000..99e47dc --- /dev/null +++ b/docs/agents.md @@ -0,0 +1,90 @@ +# Working with coding agents + +Undertext is built with coding agents, mostly Claude Code. This page says what the repository +checks on its own, what it only asks agents to do, and what is still up to you. + +The idea: an agent proves its work before it calls it done, without you asking. A passing check +is proof; "I read the code" is not. + +## The layers so far + +1. **`pnpm verify`** runs every check, in about 10 seconds, and records the tree when they all + pass. +2. **CI** runs `pnpm verify` on GitHub for every push to `main` and every pull request. +3. **`CLAUDE.md`** tells every agent session what done means and to run `pnpm verify`. + +## Rules, and what holds them + +| Rule | What holds it | How sure | +| --- | --- | --- | +| Types, tests, web build, and Rust format, lint, and tests pass | `pnpm verify`, then CI | Checked. CI shows a red run, but nothing blocks a push to `main`. | +| Run `pnpm verify` before calling work done | `CLAUDE.md` | Asked | +| Never weaken a check to pass it | `CLAUDE.md` | Asked | +| Say what was not verified | `CLAUDE.md` | Asked | + +"Asked" means an agent can forget it. Later steps turn some of these into checks. + +## `pnpm verify` + +It runs, in order, and prints one line per check: + +| Check | Command | +| --- | --- | +| types | `tsc --noEmit -p .` | +| config types | `tsc` on `vite.config.ts` | +| tests | `vitest run`, with no `.only` allowed (fails if no test ran) | +| web build | `vite build`, into a scratch folder, so `dist/` stays as it was | +| rust format | `cargo fmt --check` | +| rust lint | `cargo clippy --all-targets --locked -- -D warnings` | +| rust tests | `cargo test --lib --locked` (fails if no test ran) | + +- It never picks a subset of the checks. They are fast, so there is nothing to gain. It runs all + of them, or none in the three cases below. +- On a failure it prints the end of that check's log and the full log's path. Logs live in + `.git/undertext/logs/`, or under `.git/worktrees//` in a linked worktree. +- **It records a pass** only when every check passed and no file changed during the run. The + record is a hash of every file's contents, untracked files included, in + `.git/undertext/verified.json`. Each worktree keeps its own. A failed run deletes the record. +- **Nothing changed** since the last pass: it says so and stops. `pnpm verify --force` runs the + checks anyway, for example after `rustup update`. +- **Only docs changed** since the last pass (`docs/`, `LICENSE`, and `*.md` outside `src/` and + `src-tauri/`): nothing to check. It records the new tree. +- **Only CI files changed** (`.github/`): local checks can't see them. It records nothing, even + with `--force`. Push and read the CI run. +- It finds `cargo` on `PATH`, in `~/.cargo/bin`, or in Homebrew's rustup folder. Without cargo it + fails. It never skips a check it can't run. + +**What it can't see:** the app window, which is WebKit, while `pnpm dev` runs in Chromium; real AI +calls; file dialogs; and the keychain. For those, run `pnpm tauri dev` and try the change. + +**Known limits:** + +- It compares the tree before and after the run. An edit made and undone while it runs goes + unseen. +- It checks the files on disk, not what is staged. When you stage only part of a change, the + commit holds a tree that was never checked. CI checks the commit after you push. +- A stray `.only` fails it, as in CI. A `.skip` does not. + +The script is `scripts/verify.mjs`, the checks are in `scripts/verify-steps.mjs`, and its tests are +in `scripts/verify.test.mjs`. The tests run copies of the scripts in throwaway repositories. + +## CI + +`.github/workflows/verify.yml` runs `pnpm verify` on a GitHub macOS machine. It is free for public +repositories. + +- **Why macOS:** the app is built and tested there. On Linux, Tauri needs extra WebKit packages. +- **It is a signal, not a gate.** `main` has no branch protection, so a red run blocks nothing. + You push to `main` yourself, and that stays as it is. +- **Cancelled runs are not failures.** A newer push to the same branch cancels the older run. A + run that never got a machine or timed out needs a re-run, not a code change. +- **Rust is not pinned.** CI uses the latest stable Rust. A new Clippy lint can fail CI before it + fails on your Mac. Run `rustup update`, then `pnpm verify --force`. +- Each action is pinned to one commit, with its version in a comment, so a changed tag can't + change what runs. + +## Where the ideas come from + +Kent Beck, Simon Willison, Mitchell Hashimoto, Anthropic's and Amp's guidance on coding agents, +OpenAI's harness engineering, Lauren Tan's pstack, Matt Pocock's skills, and Jesse Vincent's +superpowers. diff --git a/package.json b/package.json index 8ab0a40..d6d16a1 100644 --- a/package.json +++ b/package.json @@ -16,6 +16,7 @@ "preview": "vite preview", "tauri": "tauri", "test": "vitest run", + "verify": "node scripts/verify.mjs", "icons": "node scripts/icons.mjs" }, "dependencies": { diff --git a/scripts/tree.mjs b/scripts/tree.mjs new file mode 100644 index 0000000..1f94107 --- /dev/null +++ b/scripts/tree.mjs @@ -0,0 +1,87 @@ +// The checkout's files by content hash, and the record of the last tree that passed `pnpm verify`. +import { execFileSync } from "node:child_process"; +import { createHash } from "node:crypto"; +import { lstatSync, mkdirSync, readFileSync, readlinkSync, rmSync, writeFileSync } from "node:fs"; +import { dirname, join, resolve } from "node:path"; + +/** The environment without git's own variables, so git acts on `root` even when run from a git hook. */ +let cleanEnv; +function gitEnv() { + if (!cleanEnv) { + cleanEnv = { ...process.env }; + for (const name of execFileSync("git", ["rev-parse", "--local-env-vars"], { encoding: "utf8" }).split("\n")) { + if (name) delete cleanEnv[name]; + } + } + return cleanEnv; +} + +export const git = (root, ...args) => + execFileSync("git", ["-C", root, ...args], { env: gitEnv(), encoding: "utf8", stdio: ["ignore", "pipe", "pipe"] }); + +/** + * Every file git sees in `root`: tracked, staged and untracked, but not ignored. Hashes contents, so + * an edit in the same second counts, and writes nothing to git's object store. + */ +export function snapshot(root) { + const paths = new Set(git(root, "ls-files", "-z", "--cached", "--others", "--exclude-standard").split("\0")); + const files = {}; + for (const path of [...paths].filter(Boolean).sort()) { + const hash = hashFile(join(root, path)); + if (hash) files[path] = hash; + } + const fingerprint = createHash("sha256").update(JSON.stringify(files)).digest("hex"); + return { fingerprint, files }; +} + +function hashFile(file) { + let stat; + try { + stat = lstatSync(file); + } catch { + return null; // deleted + } + if (stat.isSymbolicLink()) return `link:${readlinkSync(file)}`; + if (!stat.isFile()) return null; + const mode = stat.mode & 0o111 ? "x" : "-"; + return createHash("sha256").update(mode).update(readFileSync(file)).digest("hex"); +} + +/** Paths that differ between two snapshots' files. */ +export function changedPaths(before, after) { + const paths = new Set([...Object.keys(before), ...Object.keys(after)]); + return [...paths].filter((path) => before[path] !== after[path]).sort(); +} + +/** + * Docs: no check reads them, so a change to them alone needs no verify. Markdown under src/ or + * src-tauri/ may be a fixture the code reads, so it counts as code. + */ +export const isDoc = (path) => + !/^src(-tauri)?\//.test(path) && (path.endsWith(".md") || path.startsWith("docs/") || path === "LICENSE"); + +/** Only CI reads these. Local checks can't vouch for them. */ +export const isCiOnly = (path) => path.startsWith(".github/"); + +/** Per checkout: a linked worktree keeps its own record. */ +export const stateDir = (root) => resolve(root, git(root, "rev-parse", "--git-path", "undertext").trim()); + +const passFile = (root) => join(stateDir(root), "verified.json"); + +/** The last pass, or null when there is none or the record is unreadable. */ +export function readPass(root) { + try { + const pass = JSON.parse(readFileSync(passFile(root), "utf8")); + return typeof pass?.fingerprint === "string" && pass.files && typeof pass.files === "object" ? pass : null; + } catch { + return null; + } +} + +export function recordPass(root, snap) { + const file = passFile(root); + mkdirSync(dirname(file), { recursive: true }); + writeFileSync(file, JSON.stringify({ ...snap, at: new Date().toISOString() })); +} + +export const forgetPass = (root) => rmSync(passFile(root), { force: true }); diff --git a/scripts/verify-steps.mjs b/scripts/verify-steps.mjs new file mode 100644 index 0000000..36a0cb5 --- /dev/null +++ b/scripts/verify-steps.mjs @@ -0,0 +1,43 @@ +// The checks `pnpm verify` runs, in order. CI runs the same list through `pnpm verify`. +import { existsSync } from "node:fs"; +import { homedir } from "node:os"; +import { delimiter, join } from "node:path"; +import { stateDir } from "./tree.mjs"; + +/** The folder with cargo: PATH, rustup's default, or Homebrew's rustup, which fish leaves off PATH. */ +function cargoDir() { + const dirs = [...(process.env.PATH ?? "").split(delimiter), join(homedir(), ".cargo/bin"), "/opt/homebrew/opt/rustup/bin"]; + return dirs.find((dir) => dir && existsSync(join(dir, "cargo"))) ?? null; +} + +/** How many tests passed, from the runner's summary. Zero means the step checked nothing. */ +export const vitestPassed = (log) => Number(/Tests\s+.*?(\d+) passed/.exec(log)?.[1] ?? 0); +export const cargoPassed = (log) => + [...log.matchAll(/test result: \w+\. (\d+) passed/g)].reduce((sum, m) => sum + Number(m[1]), 0); + +const ranTests = (count) => (log) => (count(log) > 0 ? null : "No tests ran."); + +export function steps(root) { + const bin = (name) => join(root, "node_modules/.bin", name); + const rust = join(root, "src-tauri"); + const cargo = cargoDir(); + const cargoStep = (name, args, check) => + cargo + ? { name, cmd: join(cargo, "cargo"), args, cwd: rust, env: { PATH: `${cargo}${delimiter}${process.env.PATH}` }, check } + : { name, missing: "cargo not found. Install Rust (see README), or put cargo on PATH." }; + return [ + { name: "types", cmd: bin("tsc"), args: ["--noEmit", "-p", "."] }, + // composite would write tsconfig.node.tsbuildinfo into the tree. + { name: "config types", cmd: bin("tsc"), args: ["--noEmit", "-p", "tsconfig.node.json", "--composite", "false"] }, + // CI rejects a stray `.only`; so does this, or a local pass would fail there. + { name: "tests", cmd: bin("vitest"), args: ["run", "--allowOnly=false"], check: ranTests(vitestPassed) }, + // Outside the tree, so dist/ stays as the last real build left it. + { name: "web build", cmd: bin("vite"), args: ["build", "--outDir", join(stateDir(root), "web"), "--emptyOutDir"] }, + cargoStep("rust format", ["fmt", "--check"]), + cargoStep("rust lint", ["clippy", "--all-targets", "--locked", "--", "-D", "warnings"]), + cargoStep("rust tests", ["test", "--lib", "--locked"], ranTests(cargoPassed)), + ]; +} + +/** What these checks can't see. */ +export const unseen = "the app window (WebKit), real AI calls, file dialogs, and the keychain"; diff --git a/scripts/verify.mjs b/scripts/verify.mjs new file mode 100644 index 0000000..a40caa1 --- /dev/null +++ b/scripts/verify.mjs @@ -0,0 +1,95 @@ +// pnpm verify: every check CI runs, in seconds. Records the tree only when all of them pass on it. +// `--force` runs the checks even when nothing changed since the last pass. +import { spawnSync } from "node:child_process"; +import { mkdirSync, readFileSync, writeFileSync } from "node:fs"; +import { dirname, join, relative, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import { changedPaths, forgetPass, isCiOnly, isDoc, readPass, recordPass, snapshot, stateDir } from "./tree.mjs"; +import { steps, unseen } from "./verify-steps.mjs"; + +// The checkout this script lives in, wherever it is run from. +const root = resolve(dirname(fileURLToPath(import.meta.url)), ".."); +const force = process.argv.includes("--force"); +const seconds = (ms) => `${(ms / 1000).toFixed(1)}s`; +const list = (paths) => (paths.length > 3 ? `${paths.slice(0, 3).join(", ")} and ${paths.length - 3} more` : paths.join(", ")); + +const before = snapshot(root); +const last = readPass(root); +const changed = last ? changedPaths(last.files, before.files) : null; +// Local checks can't vouch for CI files, so a change to them alone is never recorded, even with --force. +const ciOnly = changed !== null && changed.some(isCiOnly) && changed.every((path) => isDoc(path) || isCiOnly(path)); +const ciNote = () => { + console.log(`Only CI files changed since the last pass (${list(changed.filter(isCiOnly))}).`); + console.log("Local checks can't see them. Push and read the CI run. Nothing recorded."); +}; +if (changed && !force) { + if (changed.length === 0) { + console.log(`Nothing changed since the last pass (${last.at}). Use --force to run the checks anyway.`); + process.exit(0); + } + if (changed.every(isDoc)) { + recordPass(root, before); + console.log(`Only docs changed since the last pass (${list(changed)}). Nothing to check.`); + process.exit(0); + } + if (ciOnly) { + ciNote(); + process.exit(0); + } +} + +const logs = join(stateDir(root), "logs"); +mkdirSync(logs, { recursive: true }); +const started = Date.now(); +const failed = []; + +for (const step of steps(root)) { + const log = join(logs, `${step.name.replaceAll(" ", "-")}.log`); + const begin = Date.now(); + let output; + let problem = null; + if (step.missing) { + output = step.missing; + problem = step.missing; + } else { + const result = spawnSync(step.cmd, step.args, { + cwd: step.cwd ?? root, + env: { ...process.env, ...step.env, FORCE_COLOR: "0", NO_COLOR: "1" }, + encoding: "utf8", + maxBuffer: 64 * 1024 * 1024, + }); + output = `$ ${step.cmd} ${step.args.join(" ")}\n${result.stdout ?? ""}${result.stderr ?? ""}`; + if (result.error) problem = `Could not start: ${result.error.message}. Run pnpm install?`; + else if (result.status !== 0) problem = `Exited with ${result.status ?? result.signal}.`; + else problem = step.check?.(output) ?? null; + } + writeFileSync(log, problem ? `${output}\n${problem}\n` : output); + console.log(`${problem ? "✗" : "✓"} ${step.name.padEnd(13)} ${seconds(Date.now() - begin)}`); + if (problem) { + failed.push(step.name); + const tail = readFileSync(log, "utf8").trimEnd().split("\n").slice(-25); + console.log(tail.map((line) => ` ${line}`).join("\n")); + console.log(` Full log: ${relative(process.cwd(), log) || log}`); + } +} + +const after = snapshot(root); +const moved = changedPaths(before.files, after.files); +// A failed run forgets an earlier pass. +if (failed.length > 0) { + forgetPass(root); + console.log(`Failed: ${failed.join(", ")}. Nothing recorded.`); + process.exit(1); +} +if (moved.length > 0) { + forgetPass(root); + console.log(`Files changed during the run (${list(moved)}). Nothing recorded. Run it again.`); + process.exit(1); +} +if (ciOnly) { + ciNote(); + process.exit(0); +} +recordPass(root, after); +console.log(`Passed in ${seconds(Date.now() - started)}. Recorded ${after.fingerprint.slice(0, 12)}.`); +console.log(`Not checked: ${unseen}.`); diff --git a/scripts/verify.test.mjs b/scripts/verify.test.mjs new file mode 100644 index 0000000..b0c2487 --- /dev/null +++ b/scripts/verify.test.mjs @@ -0,0 +1,207 @@ +// Runs copies of the verify scripts in throwaway repositories, never in this checkout. A fake check +// stands in for the real ones: it counts its runs, fails while a `broken` file exists, and writes a +// file into the tree while a `touch` file exists. +import { spawnSync } from "node:child_process"; +import { chmodSync, copyFileSync, existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { basename, dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { afterEach, describe, expect, it } from "vitest"; +import { git, snapshot } from "./tree.mjs"; +import { cargoPassed, vitestPassed } from "./verify-steps.mjs"; + +const here = dirname(fileURLToPath(import.meta.url)); +const made = []; +afterEach(() => made.splice(0).forEach((dir) => rmSync(dir, { recursive: true, force: true }))); + +const tempDir = () => { + const dir = mkdtempSync(join(tmpdir(), "undertext-verify-test-")); + made.push(dir); + return dir; +}; + +const FAKE_CHECK = `const fs = require("fs"); +fs.appendFileSync(process.env.RUNS, "x"); +if (fs.existsSync("broken")) process.exit(1); +if (fs.existsSync("touch")) fs.writeFileSync("made-during-run.txt", "x");`; + +const fakeSteps = (step = `{ name: "check", cmd: process.execPath, args: ["-e", ${JSON.stringify(FAKE_CHECK)}] }`) => + `export const unseen = "nothing";\nexport const steps = () => [${step}];\n`; + +function makeRepo(steps = fakeSteps()) { + const dir = tempDir(); + mkdirSync(join(dir, "scripts")); + for (const file of ["verify.mjs", "tree.mjs"]) copyFileSync(join(here, file), join(dir, "scripts", file)); + writeFileSync(join(dir, "scripts/verify-steps.mjs"), steps); + writeFileSync(join(dir, "app.js"), "ok\n"); + writeFileSync(join(dir, "README.md"), "# App\n"); + writeFileSync(join(dir, ".gitignore"), "ignored/\n"); + git(dir, "init", "-q"); + git(dir, "add", "."); + git(dir, "-c", "commit.gpgsign=false", "-c", "user.name=Test", "-c", "user.email=test@example.com", "commit", "-qm", "Start"); + return dir; +} + +function verify(dir, args = [], env = {}) { + const runs = join(tempDir(), "runs"); + writeFileSync(runs, ""); + const result = spawnSync(process.execPath, [join(dir, "scripts/verify.mjs"), ...args], { + cwd: tmpdir(), + env: { ...process.env, RUNS: runs, ...env }, + encoding: "utf8", + }); + return { status: result.status, out: result.stdout + result.stderr, runs: readFileSync(runs, "utf8").length }; +} + +const passFile = (dir) => join(dir, ".git/undertext/verified.json"); +const recorded = (dir) => (existsSync(passFile(dir)) ? JSON.parse(readFileSync(passFile(dir), "utf8")).fingerprint : null); + +describe("pnpm verify", () => { + it("records a pass, then skips an unchanged tree unless forced", () => { + const dir = makeRepo(); + const first = verify(dir); + expect(first).toMatchObject({ status: 0, runs: 1 }); + expect(first.out).toContain("✓ check"); + expect(first.out).toContain("Not checked: nothing."); + expect(recorded(dir)).toBe(snapshot(dir).fingerprint); + + const again = verify(dir); + expect(again).toMatchObject({ status: 0, runs: 0 }); + expect(again.out).toContain("Nothing changed since the last pass"); + expect(verify(dir, ["--force"])).toMatchObject({ status: 0, runs: 1 }); + }); + + it("records nothing when a check fails, and forgets the earlier pass", () => { + const dir = makeRepo(); + verify(dir); + writeFileSync(join(dir, "broken"), ""); + const failed = verify(dir); + expect(failed).toMatchObject({ status: 1, runs: 1 }); + expect(failed.out).toContain("✗ check"); + expect(failed.out).toMatch(/Full log: .*check\.log/); + expect(failed.out).toContain("Failed: check. Nothing recorded."); + expect(recorded(dir)).toBeNull(); + }); + + it("records nothing when a check exits cleanly but ran no tests", () => { + const dir = makeRepo(fakeSteps(`{ name: "tests", cmd: process.execPath, args: ["-e", ""], check: () => "No tests ran." }`)); + const result = verify(dir); + expect(result.status).toBe(1); + expect(result.out).toContain("✗ tests"); + expect(result.out).toContain("No tests ran."); + expect(recorded(dir)).toBeNull(); + }); + + it("runs the checks when the record is unreadable", () => { + const dir = makeRepo(); + mkdirSync(dirname(passFile(dir)), { recursive: true }); + writeFileSync(passFile(dir), '{"fingerprint":"abc"}'); + expect(verify(dir)).toMatchObject({ status: 0, runs: 1 }); + expect(recorded(dir)).toBe(snapshot(dir).fingerprint); + }); + + it("records nothing when files change during the run", () => { + const dir = makeRepo(); + writeFileSync(join(dir, "touch"), ""); + const result = verify(dir); + expect(result.status).toBe(1); + expect(result.out).toContain("Files changed during the run (made-during-run.txt)"); + expect(recorded(dir)).toBeNull(); + }); + + it("checks nothing when only docs changed since the last pass, and records that tree", () => { + const dir = makeRepo(); + verify(dir); + writeFileSync(join(dir, "README.md"), "# App, with docs\n"); + const result = verify(dir); + expect(result).toMatchObject({ status: 0, runs: 0 }); + expect(result.out).toContain("Only docs changed since the last pass (README.md)"); + expect(recorded(dir)).toBe(snapshot(dir).fingerprint); + + mkdirSync(join(dir, "src")); + writeFileSync(join(dir, "src/fixture.md"), "Code may read this.\n"); + expect(verify(dir)).toMatchObject({ status: 0, runs: 1 }); + }); + + it("records nothing when only CI files changed, and checks again once code changes", () => { + const dir = makeRepo(); + verify(dir); + const before = recorded(dir); + mkdirSync(join(dir, ".github/workflows"), { recursive: true }); + writeFileSync(join(dir, ".github/workflows/ci.yml"), "on: push\n"); + const ci = verify(dir); + expect(ci).toMatchObject({ status: 0, runs: 0 }); + expect(ci.out).toContain("Only CI files changed since the last pass (.github/workflows/ci.yml)"); + expect(recorded(dir)).toBe(before); + const forced = verify(dir, ["--force"]); + expect(forced).toMatchObject({ status: 0, runs: 1 }); + expect(forced.out).toContain("Nothing recorded."); + expect(recorded(dir)).toBe(before); + + writeFileSync(join(dir, "app.js"), "changed\n"); + expect(verify(dir)).toMatchObject({ status: 0, runs: 1 }); + expect(recorded(dir)).toBe(snapshot(dir).fingerprint); + }); + + it("fails clearly when a check can't start or its tool is missing", () => { + const cantStart = verify(makeRepo(fakeSteps(`{ name: "check", cmd: "/no/such/tool", args: [] }`))); + expect(cantStart.status).toBe(1); + expect(cantStart.out).toContain("Could not start"); + + const missing = verify(makeRepo(fakeSteps(`{ name: "rust", missing: "cargo not found." }`))); + expect(missing.status).toBe(1); + expect(missing.out).toContain("✗ rust"); + expect(missing.out).toContain("cargo not found."); + }); + + it("keeps a separate record in a linked worktree", () => { + const dir = makeRepo(); + const tree = join(tempDir(), "linked"); + git(dir, "worktree", "add", "-q", tree); + expect(verify(tree)).toMatchObject({ status: 0, runs: 1 }); + expect(existsSync(join(dir, ".git/worktrees", basename(tree), "undertext/verified.json"))).toBe(true); + expect(recorded(dir)).toBeNull(); + }); + + it("acts on its own checkout when git's hook variables point elsewhere", () => { + const dir = makeRepo(); + const other = makeRepo(); + const env = { GIT_DIR: join(other, ".git"), GIT_WORK_TREE: other, GIT_INDEX_FILE: join(other, ".git/index") }; + expect(verify(dir, [], env)).toMatchObject({ status: 0, runs: 1 }); + expect(recorded(dir)).toBe(snapshot(dir).fingerprint); + expect(recorded(other)).toBeNull(); + }); +}); + +describe("the tree's fingerprint", () => { + it("sees untracked files, same-second edits and deletions, but not ignored files", () => { + const dir = makeRepo(); + const start = snapshot(dir).fingerprint; + mkdirSync(join(dir, "ignored")); + writeFileSync(join(dir, "ignored/cache"), "x"); + expect(snapshot(dir).fingerprint).toBe(start); + + writeFileSync(join(dir, "new.js"), "x"); + expect(snapshot(dir).fingerprint).not.toBe(start); + rmSync(join(dir, "new.js")); + writeFileSync(join(dir, "app.js"), "ko\n"); // same size, same second + expect(snapshot(dir).fingerprint).not.toBe(start); + rmSync(join(dir, "app.js")); + expect(snapshot(dir).fingerprint).not.toBe(start); + writeFileSync(join(dir, "app.js"), "ok\n"); // an undone edit is the same tree + expect(snapshot(dir).fingerprint).toBe(start); + chmodSync(join(dir, "app.js"), 0o755); + expect(snapshot(dir).fingerprint).not.toBe(start); + }); +}); + +describe("test counts", () => { + // Summary lines copied from real vitest 5 and cargo runs. + it("reads how many tests passed, and zero when none ran", () => { + expect(vitestPassed(" Tests 73 passed (73)")).toBe(73); + expect(vitestPassed(" Tests 4 failed | 6 passed (10)")).toBe(6); + expect(vitestPassed("No test files found, exiting with code 1")).toBe(0); + expect(cargoPassed("test result: ok. 11 passed; 0 failed; 1 ignored; 0 measured; 0 filtered out")).toBe(11); + expect(cargoPassed("test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out")).toBe(0); + }); +});