audit, manifest: cleanup asks again before it removes, and the manifest's length is counted

.github/osv-scanner.toml

@@ -25,3 +25,22 @@
 id = "GHSA-84m7-p3x7-pcfv"
 ignoreUntil = 2026-11-15
 reason = "semgrep pins mcp==1.29.0 in every release, and the advisory is about an MCP HTTP server, which semgrep scan never starts"
+
+# opentelemetry-api 1.37.0, in requirements-semgrep.txt for the scanner and
+# nothing else.
+#
+# semgrep 1.180.0 and 1.181.0 both ask for opentelemetry-api~=1.37.0, read
+# from their wheels on 2026-10-26, so no set pip can install holds the fixed
+# 1.45.1. certifi, found by the same scan, had no such bound and was raised.
+#
+# The advisory is about TraceState.from_header taking quadratic time on an
+# inbound tracestate header - reached through propagate.extract() in the
+# instrumentation of a server receiving requests. semgrep scan sends its
+# telemetry out and receives no request, on a runner that lives for one scan.
+#
+# The same date as the entry above, on purpose: both wait on a semgrep that
+# moves its pins, and both come back on one reminder.
+[[IgnoredVulns]]
+id = "GHSA-xcq7-p3pf-p88h"
+ignoreUntil = 2026-11-15
+reason = "semgrep pins opentelemetry-api~=1.37.0 in every release, and the advisory is about parsing an inbound tracestate header in a server, which semgrep scan never runs"

.github/requirements-semgrep.txt

@@ -25,7 +25,7 @@ boltons==21.0.0
     #   semgrep
 bracex==3.1
     # via wcmatch
-certifi==2026.7.22
+certifi==2026.10.23
     # via
     #   httpcore
     #   httpx

.github/workflows/ci.yml

@@ -26,7 +26,7 @@ env:
   # The exact toolchain used for tests and releases. go.mod declares a
   # minimum - this is the pin. Raising it can change generated bytes, so the
   # byte stability guard has to be green before it moves.
-  GO_VERSION: "1.27.2"
+  GO_VERSION: "1.27.3"
 
 jobs:
   test:

.github/workflows/dev-build.yml

@@ -51,7 +51,7 @@ concurrency:
   cancel-in-progress: true
 
 env:
-  GO_VERSION: "1.27.2"
+  GO_VERSION: "1.27.3"
   # Fourteen days rather than the default ninety. These are throwaway builds
   # handed to one person, and an unsigned binary should not sit for a quarter of
   # a year behind a link somebody can pass on as if it were official.

.github/workflows/mutation.yml

@@ -42,7 +42,7 @@ concurrency:
   cancel-in-progress: true
 
 env:
-  GO_VERSION: "1.27.2"
+  GO_VERSION: "1.27.3"
   # Built as the test job builds on Linux, so a verdict here is about the
   # guards CI actually runs. Entries on files only a cgo build or only
   # Windows compiles come back UNJUDGED, said out loud in the report.

.github/workflows/release.yml

@@ -29,7 +29,7 @@ concurrency:
 env:
   # The same pin CI carries. A release built on a different toolchain than the
   # one the byte stability guards ran under is a release nobody measured.
-  GO_VERSION: "1.27.2"
+  GO_VERSION: "1.27.3"
 
 jobs:
   check:

CHANGELOG.md

@@ -24,6 +24,37 @@ because it turns other people's test suites red.
   `--user "$(id -u):$(id -g)"` to write there. README also shows a step that
   installs `tfg` on Windows and macOS runners in GitHub Actions.
 
+### Fixed
+
+- **`tfg cleanup --yes` keeps a file that changed after it was checked.**
+  Cleanup checks every file before it removes any, and a file changed in that
+  time was removed without `--force`. It now looks again just before it
+  removes each file and keeps one that was written to or replaced, saying so.
+  A write that keeps the length and lands within the same tick of a coarse
+  clock can still go unseen - FAT32 counts in two seconds.
+- **`tfg verify` and `tfg cleanup` stop part way through a large file on
+  Ctrl+C**, never wait on a pipe put where a listed file was, and read no more
+  than the size the manifest records. `verify` now reports a file it could not
+  even look up, for want of permission or a name too long for the system, as
+  `unreadable` with the reason rather than as `missing`.
+- **A run is warned when its manifest will be too large to read back,
+  whatever the format.** The warning came from a fixed cost per file that was
+  low for every format, so a run of about 20 000 small files wrote a manifest
+  `tfg verify` and `tfg cleanup` refuse, with no warning and no
+  `too_large_to_read_back` in it. The manifest's length is now counted, and
+  after the run `generate` corrects what it said before when the two differ.
+  Counting adds about seven seconds to a run or preview of a million files.
+- **The free space check counts the manifest and the instructions.** A disk
+  with room for the files and not for the manifest took the files and then
+  refused the manifest, which left files `tfg cleanup` cannot remove. The
+  refusal now says how much the run needs with its manifest.
+
+### Security
+
+- **Built with Go 1.27.3.** It fixes a vulnerability in `crypto/x509` of the
+  Go standard library, which only the desktop window links. The command line
+  tool was not affected. Building from source now needs Go 1.27.3.
+
 ## [0.5.0] - 2026-10-20
 
 ### Breaking

CONTRIBUTING.md

@@ -22,7 +22,7 @@ reached yet.
 
 ## Building it
 
-Needs Go 1.27.2 or newer, and nothing else for the command line tool.
+Needs Go 1.27.3 or newer, and nothing else for the command line tool.
 
 ```
 git clone https://github.com/donislawdev/TestingFilesGenerator

README.md

@@ -115,7 +115,7 @@ instead of writing files that disappear with the container.
 go install -tags noasm github.com/donislawdev/TestingFilesGenerator/cmd/tfg@latest
 ```
 
-**From source.** Needs Go 1.27.2 or newer, and nothing else:
+**From source.** Needs Go 1.27.3 or newer, and nothing else:
 
 ```
 git clone https://github.com/donislawdev/TestingFilesGenerator

go.mod

@@ -19,7 +19,7 @@ module github.com/donislawdev/TestingFilesGenerator
 // which version it is. Kept honest by TestTheBuildFloorIsThePinnedToolchainAndTheReadmeSaysSo,
 // which also holds the sentence in README.md against this number. See
 // docs/STACK.md.
-go 1.27.2
+go 1.27.3
 
 // How the line above got to the number it carries. There used to be a separate
 // "toolchain" directive here saying which compiler produces our binaries, and
@@ -75,6 +75,12 @@ go 1.27.2
 // guards were run under 1.27.2 the same day - all 104 golden values and every
 // pinned standard library path - and none shifted, so D11 holds and no major
 // version is owed.
+//
+// Raised to 1.27.3 on 2026-10-26 because govulncheck reported GO-2026-6809 in
+// crypto/x509, fixed in 1.27.3, and the known vulnerabilities check of every
+// pull request went red on it. Only the window binary links crypto/x509 (go
+// list -deps, the same day). The byte stability guards were run under 1.27.3
+// before the move, as they were for 1.27.2.
 
 require (
 	fyne.io/fyne/v2 v2.8.2

internal/audit/audit.go

@@ -11,11 +11,7 @@ package audit
 
 import (
 	"context"
-	"crypto/sha256"
-	"encoding/hex"
 	"fmt"
-	"io"
-	"os"
 	"path"
 	"path/filepath"
 	"sort"
@@ -38,9 +34,10 @@ const (
 	WrongSize Kind = "wrong-size"
 	// WrongHash is a file of the right length and the wrong content.
 	WrongHash Kind = "wrong-hash"
-	// Unreadable is a file that is there and could not be read. It is not
-	// agreement and it is not a mismatch, and calling it either would be a
-	// guess.
+	// Unreadable is a file that is there and could not be read, or a name the
+	// system would not even look up - no permission, a name too long for it.
+	// It is not agreement and it is not a mismatch, and calling it either
+	// would be a guess. Got carries the system's own reason.
 	Unreadable Kind = "unreadable"
 	// Leftover is a file this tool started and never finished, from a run that
 	// was killed before it could tidy up.
@@ -294,7 +291,7 @@ func Verify(ctx context.Context, dir string, m *manifest.Manifest, skip string)
 	}
 
 	found, stopped := InOrder(ctx, len(claimed), func(i int, scratch []byte) Difference {
-		return compare(claimed[i], full[i], scratch)
+		return compare(ctx, claimed[i], full[i], scratch)
 	})
 	for _, d := range found {
 		if d.Kind != "" {
@@ -456,31 +453,40 @@ func claimedPaths(b core.Boundary, files []manifest.File) ([]string, error) {
 // agreement on its own. No sentinel anybody has to remember, and no pointer
 // per file - a run of ten thousand would allocate ten thousand of them to say
 // "nothing to report" ten thousand times.
-func compare(f manifest.File, full string, scratch []byte) Difference {
-	info, statErr := os.Stat(full)
-	if statErr != nil {
+func compare(ctx context.Context, f manifest.File, full string, scratch []byte) Difference {
+	got, err := readListed(ctx, full, f.Bytes, scratch)
+	// Only an absent file is missing. Every failure to look used to be called
+	// that, so a file somebody had locked or taken the permissions off was
+	// reported as gone - a sentence about the wrong thing, and the one a
+	// person acts on (docs/STABILITY-REVIEW-2026-10-26.md, R04). cleanup has
+	// always told the two apart.
+	if absent(err) {
 		return Difference{Kind: Missing, Path: f.Path}
 	}
+	if err != nil {
+		return Difference{Kind: Unreadable, Path: f.Path, Got: err.Error()}
+	}
 
 	// Size first. It is free, it catches the common failure, and it names the
-	// cause more precisely than a hash mismatch would.
-	if info.Size() != f.Bytes {
+	// cause more precisely than a hash mismatch would. A file that grew while
+	// it was read is the same answer, reached later: at least as long as what
+	// was read of it, and longer if it went on growing.
+	size := got.last.Size()
+	if got.grew(f.Bytes) {
+		size = max(size, got.read)
+	}
+	if size != f.Bytes {
 		return Difference{
 			Kind: WrongSize, Path: f.Path,
 			Want: fmt.Sprintf("%d B", f.Bytes),
-			Got:  fmt.Sprintf("%d B", info.Size()),
+			Got:  fmt.Sprintf("%d B", size),
 		}
 	}
-
-	sum, hashErr := hashFile(full, scratch)
-	if hashErr != nil {
-		return Difference{Kind: Unreadable, Path: f.Path, Got: hashErr.Error()}
-	}
-	if sum != f.Hashes.SHA256 {
+	if got.sum != f.Hashes.SHA256 {
 		return Difference{
 			Kind: WrongHash, Path: f.Path,
 			Want: f.Hashes.SHA256,
-			Got:  sum,
+			Got:  got.sum,
 		}
 	}
 	return Difference{}
@@ -494,41 +500,3 @@ func compare(f manifest.File, full string, scratch []byte) Difference {
 // told apart, so this is the size that buys it without asking every worker for
 // a megabyte.
 const hashScratch = 256 << 10
-
-// onlyRead hides everything but Read from io.CopyBuffer.
-//
-// This is not a wrapper for its own sake and without it the buffer below does
-// nothing at all. os.File implements io.WriterTo, so io.CopyBuffer hands the
-// whole job to the file and THROWS THE BUFFER AWAY - measured on 2026-09-05,
-// passing 128 KiB, 256 KiB or 1 MiB with a plain file on the other end all take
-// the same 167-172 ms that io.Copy takes, because all four are the same 32 KiB
-// path inside. Hidden behind this, a 256 KiB buffer takes 150-161 ms.
-//
-// The report that suggested the buffer said it would "cut the syscall count 32
-// fold" and would have changed nothing as written. That is the whole reason
-// this type exists rather than a comment saying the buffer is bigger.
-type onlyRead struct{ io.Reader }
-
-// hashFile streams the file rather than reading it in. A run of this tool
-// produces files measured in gigabytes, and verify has to survive its own
-// output.
-//
-// scratch belongs to the caller and is reused across every file that caller
-// answers for. It may be nil, and then this asks for one of its own - which is
-// what a sequential caller wants and what every test that reaches here gets.
-func hashFile(path string, scratch []byte) (string, error) {
-	f, err := os.Open(path)
-	if err != nil {
-		return "", err
-	}
-	defer func() { _ = f.Close() }()
-
-	if scratch == nil {
-		scratch = make([]byte, hashScratch)
-	}
-	h := sha256.New()
-	if _, err := io.CopyBuffer(h, onlyRead{f}, scratch); err != nil {
-		return "", err
-	}
-	return hex.EncodeToString(h.Sum(nil)), nil
-}

internal/audit/cleanup.go

@@ -44,6 +44,12 @@ type Candidate struct {
 	Path        string
 	Disposition Disposition
 	Detail      string
+
+	// seen is the file as it was when its content was found to be ours, from
+	// the handle it was read through. Remove asks the disk again just before
+	// it deletes and leaves the file alone when the answer differs - see
+	// changedSince.
+	seen os.FileInfo
 }
 
 // Removable says whether this candidate is one cleanup will delete.
@@ -80,36 +86,71 @@ func Inspect(ctx context.Context, dir string, m *manifest.Manifest) ([]Candidate
 	// In order, because this list is what cleanup removes from and what it
 	// printed to a person beforehand.
 	return InOrder(ctx, len(claimed), func(i int, scratch []byte) Candidate {
-		return look(claimed[i], full[i], scratch)
+		return look(ctx, claimed[i], full[i], scratch)
 	})
 }
 
 // look is what one claimed file comes to for cleanup: whether it may be
 // removed, and if not, why not in the words a person needs.
-func look(f manifest.File, full string, scratch []byte) Candidate {
-	info, err := os.Stat(full)
-	if errors.Is(err, fs.ErrNotExist) {
+func look(ctx context.Context, f manifest.File, full string, scratch []byte) Candidate {
+	got, err := readListed(ctx, full, f.Bytes, scratch)
+	if absent(err) {
 		return Candidate{Path: f.Path, Disposition: Absent}
 	}
 	if err != nil {
 		return Candidate{Path: f.Path, Disposition: Unreachable, Detail: err.Error()}
 	}
 
-	// Size before hash, so a file of obviously the wrong length does not cost
-	// a read of however many gigabytes it is.
-	if info.Size() != f.Bytes {
+	// Size before hash - readListed reads nothing of a file of the wrong
+	// length, so one of obviously the wrong length does not cost a read of
+	// however many gigabytes it is.
+	if size := got.seen.Size(); size != f.Bytes {
 		return Candidate{Path: f.Path, Disposition: Changed,
-			Detail: fmt.Sprintf("it is %d B and the manifest recorded %d B", info.Size(), f.Bytes)}
+			Detail: fmt.Sprintf("it is %d B and the manifest recorded %d B", size, f.Bytes)}
 	}
-	sum, err := hashFile(full, scratch)
-	if err != nil {
-		return Candidate{Path: f.Path, Disposition: Unreachable, Detail: err.Error()}
+	// Changed under the read is changed, whatever the hash came to. Part of
+	// what was hashed may be what this run wrote and part what somebody wrote
+	// since, and a match on that says nothing about what is there now.
+	if got.grew(f.Bytes) || got.moved() {
+		return Candidate{Path: f.Path, Disposition: Changed,
+			Detail: "it changed while it was being checked"}
 	}
-	if sum != f.Hashes.SHA256 {
+	if got.sum != f.Hashes.SHA256 {
 		return Candidate{Path: f.Path, Disposition: Changed,
 			Detail: "its content is not the content this run wrote"}
 	}
-	return Candidate{Path: f.Path, Disposition: Ready}
+	return Candidate{Path: f.Path, Disposition: Ready, seen: got.last}
+}
+
+// changedSince says whether the file at full is no longer the file seen when
+// it was checked, and why, in the words a person needs.
+//
+// Asked just before the file is deleted, and the reason is R01 of the review
+// of 2026-10-21 (docs/STABILITY-REVIEW-2026-10-26.md). Inspect reads every
+// file before Remove deletes any, so the first file was judged ours and then
+// waited for the hashing of all the others - reproduced there with 257 files
+// and a change 400 ms in: the changed file went, without --force, under exit
+// 0. Asking again closes that wait down to the moment between this look and
+// the deletion, which a file somebody is working on is very unlikely to fall
+// into and which no portable call closes completely.
+//
+// Three questions, and each catches what the others do not. The identity is a
+// file put in its place. The size is a file appended to. The time is a file
+// written in place at the same length, measured on 2026-10-26 on NTFS to move
+// at once, even while the writer still holds the file open. What it cannot
+// see is a write in place that keeps the length and lands within the same tick
+// of a coarse clock - FAT32 counts in two seconds.
+func changedSince(full string, seen os.FileInfo) (string, bool) {
+	now, err := os.Lstat(full)
+	if err != nil {
+		// Gone, or no longer answering. Either way os.Remove would have the
+		// last word, and it is given it - this only stops a deletion.
+		return "", false
+	}
+	if !os.SameFile(seen, now) || now.Size() != seen.Size() || !now.ModTime().Equal(seen.ModTime()) {
+		return "it changed after it was checked, so it may not be ours. Run cleanup again to check it afresh", true
+	}
+	return "", false
 }
 
 // Outcome is what happened to one file.
@@ -199,6 +240,14 @@ func Remove(ctx context.Context, dir string, cands []Candidate, force bool) ([]O
 			})
 			continue
 		}
+		// Only for a file judged ours. --force already covers a changed file,
+		// and a file found changed and forced has nothing to compare against.
+		if c.Disposition == Ready {
+			if why, changed := changedSince(full, c.seen); changed {
+				out = append(out, Outcome{Path: c.Path, Blocked: true, Reason: why})
+				continue
+			}
+		}
 		if err := os.Remove(full); err != nil {
 			if errors.Is(err, fs.ErrNotExist) {
 				// It went away between the two passes. That is the state the