diff --git a/.github/workflows/publish-npm-on-release.yml b/.github/workflows/publish-npm-on-release.yml index 4ca7cf4..68755f1 100644 --- a/.github/workflows/publish-npm-on-release.yml +++ b/.github/workflows/publish-npm-on-release.yml @@ -8,9 +8,9 @@ on: workflow_dispatch: inputs: tag: - description: 'Tag to publish (e.g. v2.3.0)' + description: 'Release tag to publish' required: true - default: 'v2.3.0' + type: string permissions: contents: read diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 8496e23..d518c79 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -17,7 +17,7 @@ jobs: version: 10 - uses: actions/setup-node@v4 with: - node-version: '20' + node-version: '24' cache: 'pnpm' - run: pnpm install --frozen-lockfile @@ -35,6 +35,9 @@ jobs: test: name: Functional Tests runs-on: ubuntu-latest + strategy: + matrix: + node: [22, 24] steps: - uses: actions/checkout@v4 - uses: pnpm/action-setup@v2 @@ -42,7 +45,7 @@ jobs: version: 10 - uses: actions/setup-node@v4 with: - node-version: '20' + node-version: ${{ matrix.node }} cache: 'pnpm' - run: pnpm install --frozen-lockfile diff --git a/CHANGELOG.md b/CHANGELOG.md index 6c7647d..33f0294 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,22 @@ # Changelog +## Product candidate (not published) + +- New projects index and search without embedding or reranker models. Existing index modes are preserved; explicit mode switches rebuild fully. Transformers and LanceDB are optional peers with actionable preflight errors. +- Setup detects supported coding tools and configures the selected tools together. It preserves existing settings, saves version-pinned launches for the selected repository, and leaves indexing to the first search. +- Added `get_code_relationships`: directed import/re-export paths up to three hops, current resolution and source checks, confined endpoints, unresolved/stale diagnostics, and a 4,000-byte response cap. Resolution selects the root's `tsconfig.json` or, when absent, `tsconfig.base.json`, without inheriting an unrelated parent configuration. +- AST chunks preserve contiguous source and static property ownership, including concise descriptor methods, callback bodies and same-line siblings. Generated scope labels remain presentation metadata. Index format 4 rebuilds incompatible indexes even when sources are unchanged, preserves the selected embedding mode, and retains previous artifacts if rebuilding fails. +- Angular TypeScript eligibility uses parsed Angular imports; ordinary `resolve` or `validate` methods, comments, strings and imports from other packages no longer select the Angular analyzer. Other eligible analyzers retain the source and declaration context. The registry order and existing template/style routing are unchanged. +- Compact `signaturePreview` uses a contiguous window of up to three lines around the strongest query-term match in the returned snippet, falling back to its opening lines. Default and explicit exploration searches omit edit preflight while retaining search quality and incomplete-source warnings. Explicit `edit`, `refactor`, and `migrate` intents retain mutation guards. +- Search excludes ordinary stale/unavailable nominees without discarding verified current lower results or triggering an automatic full rebuild. Partial coverage, bounded exhaustion and failed semantic replenishment are explicit; corrupt metadata, build identity, paths and source ranges still fail closed. +- Symbol references scan same-build manifest sources in bounded read batches and skip parsing when a simple ASCII symbol occurs only inside longer identifiers. Unicode or arbitrary query spellings retain raw-text nomination. Coverage keeps raw-text absence separate from parser work and uncertain lexical fallback. Keyword artifact identity uses the shared index validator. Graph traversal verifies edges before consuming output caps, with bounded inspection and honest continuation. +- Watcher shutdown awaits handle closure and cancels pending callbacks. Connected stdio sessions have no default inactivity shutdown; explicit positive idle timeouts remain supported alongside handshake, EOF and parent-death guards. +- Chunk paths are persisted relative to the selected project root, including explicit-root indexing from another working directory. Previously incorrect relative paths can require a full reindex. +- Query advice is limited to grounded returned source, and abstained searches omit global pattern advice. Same-named symbols in different files remain eligible after file-level deduplication; TypeScript type arguments alone no longer select the JSX analyzer. +- The candidate requires Node 22+. CLI dispatch compares canonical entrypoint identity, including npm symlink launchers, while preserving the exported server's initialization on imports. +- Updated the runtime parser and watcher to remove the legacy braces dependency chain. Watcher exclusions use exact directory names; quoted import and export names retain their public symbol names. +- Known limits: model-free conceptual relevance is not guaranteed. Short requests can miss implementations when query words differ from source names; syntactic edges are not runtime calls; workspace/language-specific resolution can remain unresolved. Packed/protocol tests and scoped negative cases do not establish general agent-quality improvement or customer adoption. No npm publication is implied by this entry. + ## [2.3.0](https://github.com/PatrickSys/codebase-context/compare/v2.2.0...v2.3.0) (2026-04-30) diff --git a/README.md b/README.md index 6249c02..d93d577 100644 --- a/README.md +++ b/README.md @@ -2,39 +2,51 @@ [![npm version](https://img.shields.io/npm/v/codebase-context)](https://www.npmjs.com/package/codebase-context) [![license](https://img.shields.io/npm/l/codebase-context)](./LICENSE) [![node](https://img.shields.io/node/v/codebase-context)](https://github.com/PatrickSys/codebase-context/blob/master/package.json) -## Your coding agent doesn't understand your codebase. - -Coding agents can read files, but they still have to discover how your repository is organized, which patterns your team follows, and which examples are worth copying. - -Codebase Context gives an agent a local view of that information through code search, team patterns, strong examples, and project memory. It runs as an MCP server - a local tool that your editor or command-line agent can call while it works - and keeps the index on your machine by default. +Codebase Context gives coding agents a map of your repository through MCP, for developers who want their agent to find relevant code and follow the team's patterns. Search, examples and project memory run locally by default. ## Set up your AI client -Choose your coding tool and run its command once. Use Node.js 22 or newer. These commands use published npm `2.2.0` and do not require a project folder in your configuration: +Use Node.js 22 or newer. These commands use the current npm release of Codebase Context: ```bash # Claude Code -claude mcp add --scope user --transport stdio codebase-context -- npx -y codebase-context@2.2.0 +claude mcp add --scope user --transport stdio codebase-context -- npx -y codebase-context -# Codex CLI -codex mcp add codebase-context -- npx -y codebase-context@2.2.0 +# Codex CLI (this is the Codex setup line) +codex mcp add codebase-context -- npx -y codebase-context # OpenCode 1.x (keep the quoted separator on Windows) -opencode mcp add codebase-context '--' npx -y codebase-context@2.2.0 +opencode mcp add codebase-context '--' npx -y codebase-context ``` +In the unreleased candidate, `--version` prints the installed package version and `map` builds a missing index before printing the map. A low-confidence search includes a hint that weak matches do not establish that code is absent. These additions require the built candidate; the unpinned commands above install published 2.2.0. + +This repository's `init` command detects Claude Code, Codex, Cursor, and OpenCode and writes a project config for each selected tool. The saved launch is the Node binary and the package file from the npx cache. That command is in this source and is not part of published 2.2.0. Until it is published, use the copy-paste lines above. The [client setup guide](./docs/client-setup.md) marks the unpublished command. + Start a new agent session in your project, then ask: -> Use Codebase Context to find [feature] in this repository. Pass this repository's absolute path as project when checking get_indexing_status and searching. Wait for indexing if needed, read codebase://context, then search_codebase and open a returned source file. Show me the relevant files. +> Use Codebase Context to find [feature] in this repository. Pass this repository's absolute path as project, wait for indexing if needed, and open the relevant source files. + +Replace `[feature]` with something you want to find. The first run downloads a local model and indexes the repository; time depends on repository size, CPU and model. + +### Planned for 2.4.0 (not yet published) -Replace `[feature]` with something you want to find. The agent supplies the repository path in its tool calls, so the registration can serve different projects. Initial indexing may need a local model download. The October 6 isolated checks proved these client registrations and published-package project selection/search; they did not establish a full native agent investigation. +The unreleased source candidate starts new indexes without a model download. Semantic embeddings are optional; existing indexes keep their recorded mode. Its setup wizard can configure Claude Code, Codex, Cursor and OpenCode using the Node binary and package file that ran setup. These changes are not in the current npm release. + +- NestJS support from community contributor Aaron Olin. See the [analyzer details](./docs/capabilities.md#analyzers). +- Better React/Next.js detection and a fix for reranker passage scoring. +- Transformers.js v4 for local models. + +Known limits: [Intel Mac indexing crash (#68)](https://github.com/PatrickSys/codebase-context/issues/68), [custom OpenAI endpoints (#70)](https://github.com/PatrickSys/codebase-context/issues/70) and [Ollama (#71)](https://github.com/PatrickSys/codebase-context/issues/71) remain open. + +The existing [benchmark](https://github.com/PatrickSys/codebase-context/blob/master/docs/benchmark.md) reports 25.7% expected-file recovery versus jCodeMunch's 27.1% in a fixed retrieval comparison; it does not establish a winner or coding quality. For Codex Desktop, create or merge `.codex/config.toml` in the project you want to search: ```toml [mcp_servers.codebase-context] command = "npx" -args = ["-y", "codebase-context@2.2.0"] +args = ["-y", "codebase-context"] startup_timeout_sec = 120 ``` @@ -44,13 +56,13 @@ Other clients use their own setup commands: | Client | Shortest current setup | | --------------------------- | ---------------------------------------------------------------------------- | -| Gemini CLI | `gemini mcp add --scope user codebase-context npx -y codebase-context@2.2.0` | +| Gemini CLI | `gemini mcp add --scope user codebase-context npx -y codebase-context` | | Cursor | Add `.cursor/mcp.json` | | VS Code with GitHub Copilot | Add `.vscode/mcp.json` | -| GitHub Copilot CLI | `copilot mcp add codebase-context -- npx -y codebase-context@2.2.0` | +| GitHub Copilot CLI | `copilot mcp add codebase-context -- npx -y codebase-context` | | Windsurf | Add `~/.codeium/windsurf/mcp_config.json` | -Check an existing same-name entry before replacing it. To give the server a default folder, append that folder's absolute path to the `npx` arguments. The [client setup guide](./docs/client-setup.md) covers scopes, optional fixed-folder configuration, verification limits and the unreleased installer. Published `2.2.0`'s interactive `init` has registration bugs; use the commands above. +Check an existing same-name entry before replacing it. To give the server a default folder, append that folder's absolute path to the `npx` arguments. The [client setup guide](./docs/client-setup.md) covers scopes, optional fixed-folder configuration and the interactive setup wizard's limitations. The [client setup guide](./docs/client-setup.md) has the exact commands and config for every client, plus what was checked locally and what still relies on official instructions. @@ -64,32 +76,36 @@ The default connection is `stdio` (standard input/output): your client starts th ### Team patterns and examples -`get_team_patterns` shows the approaches used in the repository and points to representative files. Published `2.2.0` includes dedicated analyzers for Angular, React and Next.js, with a generic analyzer for other stacks. NestJS support belongs to the newer source candidate. +`get_team_patterns` shows the approaches used in the repository and points to representative files. Dedicated analyzers for Angular, React, and Next.js ship in published 2.2.0; NestJS support (along with React and Next.js detection improvements and a reranker fix) is planned for unpublished 2.4.0. ### Project memory `remember` stores a convention, decision, gotcha, or past failure for the project. `get_memory` retrieves relevant entries in later sessions, including when the agent or editor changes. +### Review context + +The `codebase-context-review` CLI creates a bounded context packet from a committed Git diff. It gives a reviewer context; it does not review code or prove review quality. See the [review-context guide](./docs/review-context.md). + ## How it works 1. **Index locally.** Codebase Context scans the project, builds a keyword index, and creates local semantic embeddings - numeric representations used to match code by meaning as well as exact words. 2. **Understand the repository.** The agent can request a compact codebase map with structure, patterns, and representative files. 3. **Find the code for the task.** Search returns ranked files and symbols; the agent reads the selected files before editing. -The same information is available from the terminal. Run these commands from your project root. The first index can take a while because it scans the project and creates local embeddings; once it is ready, inspect the map and search for the code you need: +The same information is available from the terminal. Run these commands from your project root. The first run downloads a local model and indexes the repository; time depends on repository size, CPU and model. Once it is ready, inspect the map and search for the code you need: ```bash # Build or refresh the local index -npx -y codebase-context@2.2.0 reindex +npx -y codebase-context reindex # Repository structure, patterns, and representative files -npx -y codebase-context@2.2.0 map +npx -y codebase-context map # Ranked code search -npx -y codebase-context@2.2.0 search --query "auth middleware" +npx -y codebase-context search --query "auth middleware" # Current team patterns -npx -y codebase-context@2.2.0 patterns +npx -y codebase-context patterns ``` One stdio server can route across several repositories. Supply `project` in tool calls to select the intended repository; a successful selection becomes the default for later calls in that process. Some clients also announce workspace roots: one root can auto-select, while an ambiguous selection asks for a project instead of guessing. MCP deprecated Roots in its July 2026 revision, so explicit project selection is the documented default rather than a dependency on client discovery. @@ -145,8 +161,8 @@ The method and failures are documented so the measurements can be inspected with - Retrieval measurements describe expected-file coverage, file precision, and reported `peakPrivateGb`, not patch correctness or end-to-end coding quality. - The paired token observation covers two frozen investigation tasks and records observed agent behavior; it is not a universal token or time guarantee. - Setup checks differ by client. The detailed guide distinguishes a written config, a config recognized by the client, a local connection, and instructions checked only against official docs. -- Published `2.2.0` has dedicated Angular, React and Next.js analyzers; NestJS support is in the newer source candidate. Other projects use the generic analyzer and the language parsers available for that stack. -- The default searchable-chunk limit is 5,000 per project. Larger repositories can raise it in `.codebase-context/config.json`. +- Dedicated Angular, React, and Next.js analyzers ship in published 2.2.0; NestJS support is planned for unpublished 2.4.0. Other projects use the generic analyzer and the language parsers available for that stack. +- The default searchable-chunk limit is 5,000 per project. Customizing this via `projects[].parsing.maxChunks` in `~/.codebase-context/config.json` requires 2.4.0 (not yet published). - The agent must identify its repository in tool calls when no default or unambiguous client root is available. Concurrent HTTP client isolation is not established by the stdio routing checks. ## Reference diff --git a/docs/capabilities.md b/docs/capabilities.md index 748d5a1..49fc4f9 100644 --- a/docs/capabilities.md +++ b/docs/capabilities.md @@ -1,6 +1,8 @@ # Capabilities Reference -Technical reference for what `codebase-context` ships today. The public product posture is map first, find second: the bounded conventions map is the first-call surface, and search narrows to the right local example after that. For the user-facing overview, see [README.md](../README.md). +Unpinned `npx codebase-context` commands install the published 2.2.0 package. Sections marked as unreleased describe the source candidate and require a built checkout or a local candidate installation. + +Start with the repository map, then search for relevant code. For an overview, see [README.md](../README.md). Features marked as planned for 2.4.0 are not yet available from npm. ## Transport Modes @@ -8,8 +10,8 @@ The server supports two transport modes: | Mode | Command | MCP endpoint | | ------------------- | ------------------------------------------------- | ---------------------------- | -| **stdio** (default) | `npx -y codebase-context@2.2.0` | Spawned process stdin/stdout | -| **HTTP** | `npx -y codebase-context@2.2.0 --http [--port N]` | `http://127.0.0.1:3100/mcp` | +| **stdio** (default) | `npx -y codebase-context` | Spawned process stdin/stdout | +| **HTTP** | `npx -y codebase-context --http [--port N]` | `http://127.0.0.1:3100/mcp` | HTTP defaults to `127.0.0.1:3100`. Override with `--port`, `CODEBASE_CONTEXT_PORT`, or `server.port` in `~/.codebase-context/config.json`. @@ -20,13 +22,14 @@ Per-project config overrides supported today: - `projects[].excludePatterns`: merged with the built-in exclusion set for that project at index time - `projects[].analyzerHints.analyzer`: prefers a registered analyzer by name for that project and falls back safely when the name is missing or invalid - `projects[].analyzerHints.extensions`: adds project-local source extensions for indexing and auto-refresh watching without changing defaults for other projects +- `projects[].parsing.maxChunks`: sets the maximum number of searchable chunks indexed for that project (requires 2.4.0, not yet published) Copy-pasteable client config templates are shipped in the package: - `templates/mcp/stdio/.mcp.json` — stdio setup for `.mcp.json`-style clients - `templates/mcp/http/.mcp.json` — HTTP setup for `.mcp.json`-style clients -Use Node.js 22 or newer with the published 2.2.0 commands shown here. For client registration recipes and their verification limits, see the [client setup guide](./client-setup.md). +Use Node.js 22 or newer with the commands shown here. For setup commands and what was checked, see the [client setup guide](./client-setup.md). ## CLI Reference @@ -49,15 +52,15 @@ For a command gallery with examples, see `docs/cli.md`. | `memory add` | `--type`, `--category`, `--memory`, `--reason` | `remember` | | `memory remove ` | — | — | -Commands that list `--json` above support raw JSON output. For MCP client registration, follow the [published client setup recipes](./client-setup.md); do not use the broken `init` setup command in published 2.2.0. Errors go to stderr with exit code 1. +Commands that list `--json` above support raw JSON output. Follow the [client setup guide](./client-setup.md) to connect your client. The `init` command can set up Claude Code and Codex for the current project; the guide covers the other clients. Errors go to stderr with exit code 1. ```bash # Quick examples -npx -y codebase-context@2.2.0 status -npx -y codebase-context@2.2.0 search --query "auth middleware" --intent edit -npx -y codebase-context@2.2.0 refs --symbol "UserService" --limit 10 -npx -y codebase-context@2.2.0 cycles --scope src/features -npx -y codebase-context@2.2.0 reindex --incremental +npx -y codebase-context status +npx -y codebase-context search --query "auth middleware" --intent edit +npx -y codebase-context refs --symbol "UserService" --limit 10 +npx -y codebase-context cycles --scope src/features +npx -y codebase-context reindex --incremental ``` ## Tool Surface @@ -91,6 +94,10 @@ Shared selector inputs: | `refresh_index` | Full or incremental re-index + git memory extraction | | `get_indexing_status` | Index state, progress, last stats | +### File navigation in the unreleased candidate + +`get_code_relationships` follows verified TypeScript/JavaScript file imports and re-exports. Supply `file`, optional `direction` (`outgoing` or `incoming`), `max_hops` (1 to 3, default 2) and `max_nodes` (1 to 100, default 50). It checks current source snapshots and reports stale or unresolved edges. Python and Go import navigation are not supported. + ## Project Routing Behavior matrix: @@ -187,11 +194,12 @@ Ordered by execution: 12. **Payload budgeting** — final serialized search responses include `searchQuality.tokenEstimate`; warnings only appear above the 4K-token threshold and differ between compact and full mode. 13. **Full-mode chunk metadata** — when available, full-mode results surface chunk-level `imports` (top 5), `exports` (top 5), and cyclomatic `complexity`. -### Defaults +### Defaults in the unreleased candidate - **Chunk size**: 50 lines, 0 overlap - **Reranker trigger**: activates when top-3 results are within 0.08 score of each other -- **Embedding model**: `Xenova/bge-small-en-v1.5` (512 token context, fast, local-first) via `@huggingface/transformers`. Override: `EMBEDDING_MODEL=onnx-community/granite-embedding-small-english-r2-ONNX` for Granite (8192 ctx, slower). +- **Embedding mode**: new indexes are model-free. Existing indexes keep their recorded mode unless explicitly changed. `reindex --embeddings` opts into semantic indexing and requires the optional model/storage packages. +- **Semantic model when enabled**: `Xenova/bge-small-en-v1.5` (512 token context, fast, local-first) via `@huggingface/transformers`. Override: `EMBEDDING_MODEL=onnx-community/granite-embedding-small-english-r2-ONNX` for Granite (8192 ctx, slower). - **Vector DB**: LanceDB with cosine distance ## Decision Card (Edit Intent) @@ -257,13 +265,20 @@ Impact is 2-hop transitive: direct importers (hop 1) and their importers (hop 2) - Git auto-extraction: conventional commits from last 90 days - Surface locations: `search_codebase` results (as `relatedMemories`), `get_team_patterns` responses, preflight analysis -## Indexing +## Indexing in the unreleased candidate -- Initial: full scan → chunking (50 lines, 0 overlap) → embedding → vector DB (LanceDB) + keyword index (Fuse.js) +- Initial: full scan → chunking (50 lines, 0 overlap) → keyword index (Fuse.js). Explicit semantic mode also creates embeddings and a LanceDB vector index. - Incremental: SHA-256 manifest diffing, selective embed/delete, full intelligence regeneration - Auto-refresh (MCP server mode only): chokidar file watcher triggers incremental reindex after a debounce on any source file change — `node_modules/`, `.git/`, `dist/`, and `.codebase-context/` are excluded. One-shot CLI runs skip the watcher entirely. -- Version gating: `index-meta.json` tracks format version; mismatches trigger automatic rebuild -- Crash-safe rebuilds: full rebuilds write to `.staging/` and swap atomically only on success +- Version gating: `index-meta.json` tracks format version; older format-4 caches require a full rebuild in their recorded mode before use with format 5. Unknown mode or occupied recovery slots can require explicit recovery. +- Publication: full and incremental builds write complete `.generations//` directories, close vector handles, then replace the authoritative `index-meta.json`. An interrupted build retains the last committed generation. Each read pins one generation; a later operation sees the latest committed build. +- Producer integrity: chunks must belong to the analyzed file and contain real source from their claimed interval. Independent, overlapping or disjoint component spans remain separate. Invalid producer output aborts publication and preserves the prior generation; reader source/range checks remain strict. +- Search evidence: the source manifest must match the pinned build. Each nominated chunk's full body is checked against its current source interval before it can produce a result; a forged stored prefix is corruption and triggers rebuild rather than being stripped. Valid context metadata stays separate from the exact source snippet. These bounded candidate checks do not audit every stored chunk, and describe source state when checked. +- Relationship freshness: unresolved specifiers are checked again with the current TypeScript/JavaScript resolver. A newly available target is reported as stale until refresh records a verified edge; unchanged importer bytes alone do not prove unchanged resolution. +- Recovery status: a late auto-heal result carries its current index state instead of a preflight ready signal. Failed launches report an error; wait for a confirmed ready state before retrying. +- Retention: at most three retained or in-progress generations, counting legacy fixed files. A refresh is deferred when live readers occupy the available slots. Cleanup failures do not roll back a committed build. +- Recovery: missing root metadata does not make surviving generations a fresh index. Completed or unknown generations are preserved; automatic map/default reindex requires explicit recovery. If all three slots are occupied, first copy and verify the generation you choose to a recovery location outside `.codebase-context` (for example `E:/cbc-index-recovery/my-project/`), then relocate that selected directory out of `.generations` and retry `reindex --embeddings` or `reindex --no-embeddings` with the intended mode. The program does not select a generation for you; mode flags do not override the capacity limit. +- Refresh search: searches can use the previous complete build while a refresh runs, but exclude evidence whose source changed. An initial build still reports `indexing`. - Auto-heal: corrupted index triggers automatic full re-index on next search - Relationships sidecar: `relationships.json` contains file import graph, symbol export index, and per-edge import details (`importDetails`: line number + imported symbol names where available) - Storage: `.codebase-context/` directory (memory.json + generated files) @@ -273,6 +288,7 @@ Impact is 2-hop transitive: direct importers (hop 1) and their importers (hop 2) - **Angular**: signals, standalone components, control flow syntax, lifecycle hooks, DI patterns, component metadata - **React**: function/class components, custom hooks, context usage, memoization, Suspense, ecosystem signal extraction - **Next.js**: App Router and Pages Router detection, route/API classification, route paths, `"use client"`, metadata exports +- **NestJS** (planned for 2.4.0, not yet published): modules, controllers, providers and route handlers, contributed by Aaron Olin. Matching NestJS source is detected automatically. - **Generic**: 30+ have indexing/retrieval coverage including PHP, Ruby, Swift, Scala, Shell, config/markup., 10 languages have full symbol extraction (Tree-sitter: TypeScript, JavaScript, Python, Java, Kotlin, C, C++, C#, Go, Rust). Notes: @@ -301,6 +317,6 @@ Reproducible evaluation is shipped as a CLI entrypoint backed by shared scoring - **Symbol refs are not a call-graph.** `get_symbol_references` counts identifier-node occurrences in the AST (comments/strings excluded via Tree-sitter). It does not distinguish call sites from type annotations, variable assignments, or imports. Full call-site-specific analysis (`call_expression` nodes only) is a roadmap item. - **Impact is 2-hop max.** `computeImpactCandidates` walks direct importers then their importers. Full BFS reachability is on the roadmap. -- **Published 2.2.0 has dedicated Angular, React, and Next.js analyzers.** NestJS support is in a newer source candidate and is not part of published 2.2.0. Other languages use the Generic analyzer (30+ languages, chunking + import graph, no framework-specific signal extraction). -- **Default embedding model is `bge-small-en-v1.5` (512-token context).** Granite (8192 context) is opt-in via `EMBEDDING_MODEL`. OpenAI is opt-in via `EMBEDDING_PROVIDER=openai` — sends code externally. +- **Planned for unpublished 2.4.0:** NestJS analyzer support, React and Next.js detection improvements, and reranker fixes. Dedicated Angular, React, and Next.js analyzers already ship in published 2.2.0. Other languages use the Generic analyzer for indexing and retrieval; available symbol extraction depends on parser support. File import resolution and multi-hop navigation currently support TypeScript and JavaScript. Python and Go import navigation are not supported. +- **The unreleased candidate defaults new indexes to model-free mode. Published 2.2.0 enables embeddings by default.** When semantic mode is enabled, its default model is `bge-small-en-v1.5` (512-token context). Granite (8192 context) is opt-in via `EMBEDDING_MODEL`. OpenAI is opt-in via `EMBEDDING_PROVIDER=openai` — sends code externally. - **Patterns are file-level frequency counts.** Not semantic clustering. Rising/Declining trend is derived from git commit recency for files using each pattern, not from usage semantics. diff --git a/docs/cli.md b/docs/cli.md index dfda760..9ae65f6 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -10,18 +10,20 @@ > Output depends on the repo you run it against. The examples below are illustrative (paths, counts, and detected frameworks will vary). > -> These examples show an earlier CLI run; they do not verify the current package or an MCP connection. +> These commands use the current npm release. The examples are illustrative and were not captured from the current npm package. > -> The commands below use published `codebase-context@2.2.0` and require Node.js 22 or newer. The CLI is intentionally single-project per invocation and targets one root via `CODEBASE_ROOT` or the current working directory. MCP registration and project selection are covered in the [client setup guide](./client-setup.md). +> These output examples come from an earlier CLI run and do not verify the current npm package or an MCP connection. +> +> The commands below use `codebase-context` and require Node.js 22 or newer. The CLI is intentionally single-project per invocation and targets one root via `CODEBASE_ROOT` or the current working directory. MCP registration and project selection are covered in the [client setup guide](./client-setup.md). ## How to run ```bash # CLI only: run from a repo root, or set CODEBASE_ROOT explicitly: -CODEBASE_ROOT=/path/to/repo npx -y codebase-context@2.2.0 status +CODEBASE_ROOT=/path/to/repo npx -y codebase-context status # Commands that show --json in their help support machine output. Human mode is default. -npx -y codebase-context@2.2.0 patterns --json +npx -y codebase-context patterns --json ``` ### ASCII fallback @@ -29,7 +31,7 @@ npx -y codebase-context@2.2.0 patterns --json If your terminal doesn’t render Unicode box-drawing cleanly: ```bash -CODEBASE_CONTEXT_ASCII=1 npx -y codebase-context@2.2.0 patterns +CODEBASE_CONTEXT_ASCII=1 npx -y codebase-context patterns ``` ## Commands @@ -44,18 +46,33 @@ CODEBASE_CONTEXT_ASCII=1 npx -y codebase-context@2.2.0 patterns - `reindex` — rebuild index (full or incremental) - `style-guide` — find style guide sections in docs - `memory list|add|remove` — manage team memory (stored in `.codebase-context/memory.json`) -- MCP client registration — use the published 2.2.0 commands in the [client setup guide](./client-setup.md); published `init` is not a reliable setup route +- MCP client registration — use the recipes in the [client setup guide](./client-setup.md). Published 2.2.0 has the older interactive HTTP wizard; the four-client wizard below requires the unreleased candidate. + +## `init` in the unreleased candidate + +`init` detects Claude Code, Codex, Cursor and OpenCode, then configures the selected tools together. In a terminal, detected tools start selected; use Space to change the selection and Enter to apply it. Without a terminal, it configures all detected tools. + +```powershell +codebase-context init --help +codebase-context init +codebase-context init --yes +codebase-context init --client codex --client cursor --root "C:\path\to\project" +``` + +Use repeatable `--client` options to choose tools explicitly. `--yes` configures every detected tool when no clients are named. `--root` selects another repository; otherwise `CODEBASE_ROOT` or the current directory is used. Setup writes client registrations but does not build an index or change agent instructions. The first search creates the index. The multi-tool flow is not in published npm `2.2.0` yet; see the [main setup guide](../README.md#set-up-your-ai-client) for its publication status and the [client setup guide](./client-setup.md) for current manual recipes. + +The built candidate also supports `--version` and builds a missing index when you run `map`. New indexes use keyword search without a model download; `reindex --embeddings` explicitly enables local semantic search. Preview, reindex, and map apply the same project exclusions and analyzer hints. A low-confidence search result means the matches are weak; it does not establish that code is absent. --- ## First use -Run these CLI commands from the project root. Build or refresh the local index first; the first index may take a while while the project is scanned and local embeddings are created. Then inspect the conventions map and search for the code you need: +Run these CLI commands from the project root. Build or refresh the local index first; the first run downloads a local model and indexes the repository; time depends on repository size, CPU and model. Then inspect the conventions map and search for the code you need: ```bash -npx -y codebase-context@2.2.0 reindex -npx -y codebase-context@2.2.0 map -npx -y codebase-context@2.2.0 search --query "auth middleware" +npx -y codebase-context reindex +npx -y codebase-context map +npx -y codebase-context search --query "auth middleware" ``` This is the CLI preparation flow. It is separate from MCP client registration and does not by itself show that an MCP client connected successfully. @@ -63,8 +80,8 @@ This is the CLI preparation flow. It is separate from MCP client registration an ## `map` ```bash -npx -y codebase-context@2.2.0 reindex -npx -y codebase-context@2.2.0 map +npx -y codebase-context reindex +npx -y codebase-context map ``` The conventions map - run this first on an unfamiliar repo. It shows architecture layers, active patterns with adoption rates and trend direction, and the golden files the team treats as the strongest examples. This is also what the MCP server delivers to AI agents via the `codebase://context` resource on first call, before search narrows to a specific local example. @@ -99,7 +116,7 @@ Example output (truncated): ## `metadata` ```bash -npx -y codebase-context@2.2.0 metadata +npx -y codebase-context metadata ``` Example output: @@ -120,7 +137,7 @@ Example output: ## `patterns` ```bash -npx -y codebase-context@2.2.0 patterns +npx -y codebase-context patterns ``` Example output (truncated): @@ -142,8 +159,8 @@ Example output (truncated): ## `search` ```bash -npx -y codebase-context@2.2.0 reindex -npx -y codebase-context@2.2.0 search --query "file watcher" --intent edit --limit 3 +npx -y codebase-context reindex +npx -y codebase-context search --query "file watcher" --intent edit --limit 3 ``` Example output (truncated): @@ -164,7 +181,7 @@ Example output (truncated): ## `refs` ```bash -npx -y codebase-context@2.2.0 refs --symbol "startFileWatcher" --limit 10 +npx -y codebase-context refs --symbol "startFileWatcher" --limit 10 ``` Example output (truncated): @@ -183,7 +200,7 @@ Example output (truncated): ## `cycles` ```bash -npx -y codebase-context@2.2.0 cycles --scope src +npx -y codebase-context cycles --scope src ``` Example output: @@ -199,7 +216,7 @@ Example output: ## `status` ```bash -npx -y codebase-context@2.2.0 status +npx -y codebase-context status ``` Example output: @@ -217,9 +234,21 @@ Example output: ## `reindex` +In the unreleased model-free candidate, use the locally installed package: + +```bash +CODEBASE_ROOT=/path/to/repository node /path/to/cbc/node_modules/codebase-context/dist/index.js reindex +CODEBASE_ROOT=/path/to/repository node /path/to/cbc/node_modules/codebase-context/dist/index.js reindex --incremental +CODEBASE_ROOT=/path/to/repository node /path/to/cbc/node_modules/codebase-context/dist/index.js reindex --no-embeddings +# Requires the optional Transformers and LanceDB runtimes; may download models. +CODEBASE_ROOT=/path/to/repository node /path/to/cbc/node_modules/codebase-context/dist/index.js reindex --embeddings +``` + +New projects default to model-free. Omit mode flags to preserve the successful mode; explicit changes require a full rebuild even with `--incremental`. The two mode flags are mutually exclusive and index failures return nonzero. The examples below target the older published package. + ```bash -npx -y codebase-context@2.2.0 reindex -npx -y codebase-context@2.2.0 reindex --incremental --reason "changed watcher logic" +npx -y codebase-context reindex +npx -y codebase-context reindex --incremental --reason "changed watcher logic" ``` > **MCP server mode**: if you're running codebase-context as an MCP server (long-running process), the index auto-refreshes via a file watcher — you don't need to call `reindex` between edits. Use `reindex` for one-shot CLI runs or to force a full rebuild. @@ -227,7 +256,7 @@ npx -y codebase-context@2.2.0 reindex --incremental --reason "changed watcher lo ## `style-guide` ```bash -npx -y codebase-context@2.2.0 style-guide --query "naming" +npx -y codebase-context style-guide --query "naming" ``` Example output: @@ -240,14 +269,14 @@ No style guides found. ## `memory` ```bash -npx -y codebase-context@2.2.0 memory list -npx -y codebase-context@2.2.0 memory list --query "watcher" +npx -y codebase-context memory list +npx -y codebase-context memory list --query "watcher" -npx -y codebase-context@2.2.0 memory add \ +npx -y codebase-context memory add \ --type gotcha \ --category tooling \ --memory "Use pnpm, not npm" \ --reason "Workspace support and speed" -npx -y codebase-context@2.2.0 memory remove +npx -y codebase-context memory remove ``` diff --git a/docs/client-setup.md b/docs/client-setup.md index 7770e7f..f43ab54 100644 --- a/docs/client-setup.md +++ b/docs/client-setup.md @@ -1,16 +1,18 @@ # Client Setup -Codebase Context runs as a local MCP server: a tool your AI editor or command-line client can call while it works. Register it without a fixed folder, then let the agent supply its current repository through the `project` tool parameter. These examples use published npm `2.2.0`. +These commands use the current npm release because the package name is unpinned. The current published version is 2.2.0; 2.4.0 is not yet published. -Use Node.js 22 or newer for these recipes. The package declares Node 18+, but native dependency requirements are narrower than that broad declaration; these checks do not establish Node 18.0 compatibility. +Codebase Context runs as a local MCP server: a tool your AI editor or command-line client can call while it works. Register it without a fixed folder, then let the agent supply its current repository through the `project` tool parameter. + +Use Node.js 22 or newer, as required by the package. The copyable registration commands target PowerShell or a POSIX terminal. Windows Command Prompt was not tested; OpenCode's quoted separator is shell-dependent. -## Current published-package recipes (updated 2026-10-06) +## Client recipes Claude Code, Codex CLI and OpenCode registration commands were checked again on October 6 without a project argument. The other clients retain their older config-shape evidence; no-folder native agent use is unverified for them. See the dated notes below. -Each client section says how far its isolated Windows check went: +Each client section says how far its earlier Windows check went. Those checks used published `2.2.0` unless another build is named; they do not verify upcoming unpublished changes: | Status | What it means | | ------------------------------ | ------------------------------------------------------------------------------------------ | @@ -25,7 +27,7 @@ After registration, start a new agent session in your project and ask: > Use Codebase Context to find [feature] in this repository. Pass this repository's absolute path as project when checking get_indexing_status and searching. Wait for indexing if needed, read codebase://context, then search_codebase and open a returned source file. Show me the relevant files. -The first index may take a while and download a local model. You can prepare it separately with `npx -y codebase-context@2.2.0 reindex` from the project folder. Index readiness, client connection and useful agent behavior are separate checks. +The first run downloads a local model and indexes the repository; time depends on repository size, CPU and model. You can prepare it separately with `npx -y codebase-context reindex` from the project folder. Index readiness, client connection and useful agent behavior are separate checks. ## Claude Code @@ -38,13 +40,13 @@ The first index may take a while and download a local model. You can prepare it User setup: ```bash -claude mcp add --scope user --transport stdio codebase-context -- npx -y codebase-context@2.2.0 +claude mcp add --scope user --transport stdio codebase-context -- npx -y codebase-context ``` Optional shared project setup: ```bash -claude mcp add --scope project --transport stdio codebase-context -- npx -y codebase-context@2.2.0 +claude mcp add --scope project --transport stdio codebase-context -- npx -y codebase-context ``` `--scope project` writes shared `.mcp.json` and requires project-server approval. `--scope local` instead stores a private association for the current project in `~/.claude.json`; it does not write `.mcp.json`. @@ -64,7 +66,7 @@ claude mcp add --scope project --transport stdio codebase-context -- npx -y code **Scope:** The command writes user config. A trusted project can use a project `.codex/config.toml` instead. ```bash -codex mcp add codebase-context -- npx -y codebase-context@2.2.0 +codex mcp add codebase-context -- npx -y codebase-context ``` Equivalent project config: @@ -72,7 +74,7 @@ Equivalent project config: ```toml [mcp_servers.codebase-context] command = "npx" -args = ["-y", "codebase-context@2.2.0"] +args = ["-y", "codebase-context"] startup_timeout_sec = 120 ``` @@ -82,58 +84,57 @@ startup_timeout_sec = 120 **Official docs:** [Codex MCP configuration](https://learn.chatgpt.com/docs/extend/mcp?surface=cli). The supported `startup_timeout_sec` option defaults to ten seconds; the project example allows 120 seconds for initial `npx` package preparation. This does not prove Desktop startup or guarantee completion of a cold download. -## Codex Desktop: published 2.2.0 +## Codex Desktop recipe Create or merge `.codex/config.toml` in the project you want to search. Preserve any existing settings: ```toml [mcp_servers.codebase-context] command = "npx" -args = ["-y", "codebase-context@2.2.0"] +args = ["-y", "codebase-context"] startup_timeout_sec = 120 ``` Trust the project if asked, restart Codex and start a new task there. Use the first-use prompt above so the agent selects the current repository. The CLI command in the preceding section writes user-level config; this recipe uses a trusted project's config instead. -**Verification boundary:** isolated Windows CLI and SDK/MCP checks on October 5 indexed a six-file fixture with published `2.2.0`, selected its project, retrieved context and found a source that the host could open. The exact `npx -y codebase-context@2.2.0 ` entrypoint also passed using a warmed isolated npm cache and prepared index. This does not establish a fully cold download or a fresh Codex Desktop agent session. The September Desktop session below used a packed candidate; keep it separate from published-package acceptance. - -On October 6 the exact published `npx` entrypoint was also tested without a folder, environment root or MCP roots, launched from an unrelated directory. Its first tool call returned `selection_required`; an explicit `project` call selected the prepared fixture, search found the expected symbol, context returned its map, and the runner host opened the returned source. A later call without `project` retained that selection. This is SDK/protocol and runner-host evidence with warmed caches and a pre-indexed fixture, not a native Desktop task. - -## Unreleased installer candidate +**Verification boundary:** isolated Windows CLI and SDK/MCP checks on October 5 exercised published `2.2.0` on a six-file fixture; the exact `npx -y codebase-context@2.2.0 ` entrypoint passed with a warmed isolated npm cache and prepared index. This is historical evidence for `2.2.0`, not verification of `2.4.0`, a fully cold download or a fresh Codex Desktop agent session. -The next release is expected to support one project-only command from the target project: +On October 6 the exact published `2.2.0` `npx` entrypoint was tested without a folder, environment root or MCP roots, launched from an unrelated directory. That check does not verify the `2.4.0` package. Its first tool call returned `selection_required`; an explicit `project` call selected the prepared fixture, search found the expected symbol, context returned its map, and the runner host opened the returned source. A later call without `project` retained that selection. This is SDK/protocol and runner-host evidence with warmed caches and a pre-indexed fixture, not a native Desktop task. -The `init --client codex --yes` flow is an unreleased source candidate. Published npm `2.2.0` does not include these flags. Use the published configuration above rather than an `@latest` installer command; candidate verification uses an exact built or installed candidate. +## Setup wizard in the unreleased candidate -A successful `init --client codex --yes` writes the project-scoped `.codex/config.toml` and prepares the index for that root. If config writing or index preparation fails, setup is incomplete and Codex is not ready to use. +The four-client wizard is not in published 2.2.0. From a built checkout or a locally installed candidate, run its actual entrypoint from the project you want to search: -For candidate verification before publication, build the source checkout and run its absolute `dist/index.js` path from the project you want to configure: +```bash +# Detect Claude Code, Codex, Cursor and OpenCode, then configure the detected clients. +node /path/to/codebase-context/dist/index.js init --yes -```powershell -# In the codebase-context source checkout -npm run build +# Configure one client for an explicit project. +node /path/to/codebase-context/dist/index.js init --client codex --root /path/to/project --yes -# In the target project, using the built CLI path -node C:\path\to\codebase-context\dist\index.js init --client codex --yes +# Inspect selected source files and exclusions without writing client config. +node /path/to/codebase-context/dist/index.js init --root /path/to/project --preview ``` -Use `--root C:\path\to\project` when the command is run from elsewhere. Here, `root` means the project folder to configure and index. The candidate writes the project-scoped `/.codex/config.toml`, preserves unrelated Codex settings, prepares the index for that folder, and does not write `AGENTS.md` or global client configuration. After a successful result, trust the project in Codex Desktop if prompted, restart or reopen Codex, start a new task in that project, and ask a small project question. A written config and ready index do not prove that Codex loaded or queried the server. +Replace the paths with the candidate installation and project folders. With an interactive terminal, omit `--yes` to select clients together. `--client` is repeatable. `--root` takes priority over `CODEBASE_ROOT` and the current folder. + +Claude Code uses `.mcp.json`, Codex uses `.codex/config.toml`, Cursor uses `.cursor/mcp.json`, and OpenCode uses `opencode.json`. Generated configs save the running Node binary, the actual package entrypoint and the project folder. If setup ran through `npx`, that entrypoint is in its package cache. Rerun setup if that installation or cache is removed or moved. -Before release, the generated `npx` server command still resolves the published package unless the candidate is installed in the target project's `node_modules`. Our packaged test installs the candidate in a disposable project and checks its file hash. Running the built setup command alone does not prove that Codex will launch that same candidate. +Existing unrelated settings are preserved. A conflicting custom Codebase Context launch stops setup before selected config files are written. To replace a custom launch, remove only its Codebase Context entry from the client config, keep the other settings, and rerun setup. Restart the client and approve the server or project trust if asked. Config generation and SDK launch tests do not establish acceptance in each native client. -**Checked on 2026-09-15:** the packed candidate installed and indexed an isolated Windows project. A fresh Codex Desktop task then called the actual `search_codebase` MCP tool and located the requested chart component with its source file and lines. The normal Desktop task flow established project trust; no trust-file edits or replacement CLI/SDK search were used for that answer. This covers the small prepared project with shared caches, not a cold-machine install or the published npm package. +New candidate indexes default to model-free search and do not initialize the optional model packages. Existing indexes keep their recorded mode; semantic indexing remains an explicit option. ## Optional fixed-folder setup and CLI preparation -To set a default project before the first tool call, append its full path to the server arguments, for example `args = ["-y", "codebase-context@2.2.0", "C:/projects/my-app"]`. Use a quoted literal shell argument in registration commands. This makes the default fixed across repositories when the registration is user-scoped; explicit tool selection can still choose another allowed project. +To set a default project before the first tool call, append its full path to the server arguments, for example `args = ["-y", "codebase-context", "C:/projects/my-app"]`. Use a quoted literal shell argument in registration commands. This makes the default fixed across repositories when the registration is user-scoped; explicit tool selection can still choose another allowed project. To prepare the index separately, run from the project folder: ```powershell -npx -y codebase-context@2.2.0 reindex +npx -y codebase-context reindex ``` -Do not use published `2.2.0`'s interactive `init` as the setup route. October 6 isolated checks found that it ignores `--help`, `--client` and `--yes`; its Claude/Codex registration executes `mcp` instead of the client executable, and its HTTP recipes require a separately started server. It can exit zero after registration fails. The unreleased installer above is a separate implementation. +The older interactive HTTP wizard remains the behavior of published 2.2.0. A successful candidate wizard exit alone does not establish working client registration. ## Gemini CLI @@ -146,13 +147,13 @@ Do not use published `2.2.0`'s interactive `init` as the setup route. October 6 User setup: ```bash -gemini mcp add --scope user codebase-context npx -y codebase-context@2.2.0 +gemini mcp add --scope user codebase-context npx -y codebase-context ``` Project setup: ```bash -gemini mcp add --scope project codebase-context npx -y codebase-context@2.2.0 +gemini mcp add --scope project codebase-context npx -y codebase-context ``` Gemini writes an `mcpServers` entry in its settings. Claude, Codex, and Copilot CLI use `--`; Gemini does not. @@ -178,7 +179,7 @@ Create `.cursor/mcp.json`: "mcpServers": { "codebase-context": { "command": "npx", - "args": ["-y", "codebase-context@2.2.0"] + "args": ["-y", "codebase-context"] } } } @@ -203,7 +204,7 @@ Checked config shape: `.cursor/mcp.json with mcpServers.codebase-context command Official user command: ```powershell -code --add-mcp '{"name":"codebase-context","command":"npx","args":["-y","codebase-context@2.2.0"]}' +code --add-mcp '{"name":"codebase-context","command":"npx","args":["-y","codebase-context"]}' ``` Deterministic workspace config at `.vscode/mcp.json`: @@ -213,7 +214,7 @@ Deterministic workspace config at `.vscode/mcp.json`: "servers": { "codebase-context": { "command": "npx", - "args": ["-y", "codebase-context@2.2.0"] + "args": ["-y", "codebase-context"] } } } @@ -234,7 +235,7 @@ Deterministic workspace config at `.vscode/mcp.json`: **Scope:** User config. ```bash -copilot mcp add codebase-context -- npx -y codebase-context@2.2.0 +copilot mcp add codebase-context -- npx -y codebase-context ``` **What was checked:** The command ran with an isolated `--config-dir` and wrote the expected `mcp-config.json` command, arguments, and tool filter. @@ -254,7 +255,7 @@ copilot mcp add codebase-context -- npx -y codebase-context@2.2.0 Run once: ```bash -opencode mcp add codebase-context '--' npx -y codebase-context@2.2.0 +opencode mcp add codebase-context '--' npx -y codebase-context ``` Keep the quoted `'--'`: the installed Windows PowerShell npm shim consumes an unquoted separator. The quoted form also works in POSIX shells. @@ -267,7 +268,7 @@ For a project-local alternative, merge this into `opencode.json`: "mcp": { "codebase-context": { "type": "local", - "command": ["npx", "-y", "codebase-context@2.2.0"], + "command": ["npx", "-y", "codebase-context"], "enabled": true, "timeout": 120000 } @@ -277,7 +278,7 @@ For a project-local alternative, merge this into `opencode.json`: Checked config shape: `opencode.json with a local command array and timeout 120000`. -The longer timeout is intentional because the isolated cold start took about 38 seconds while `npx` prepared the package and server. +The longer timeout allows for initial `npx` package preparation and server startup; that time varies with the machine, network and cache. **What was checked:** With home, config, data, cache, temporary and workspace paths redirected, the no-folder command wrote user configuration and `opencode mcp list` reported Connected. An August 11 check with OpenCode 1.18.5 also connected using the project config shape and an explicit folder; the no-folder project variant is not a new native acceptance check. @@ -300,7 +301,7 @@ Create `~/.codeium/windsurf/mcp_config.json`: "mcpServers": { "codebase-context": { "command": "npx", - "args": ["-y", "codebase-context@2.2.0"] + "args": ["-y", "codebase-context"] } } } @@ -319,11 +320,11 @@ Checked config shape: `~/.codeium/windsurf/mcp_config.json with mcpServers.codeb Use `stdio` for the simplest setup. HTTP can let several clients share one long-running local process, but support depends on the client and this path is not part of the default setup proof: ```bash -npx -y codebase-context@2.2.0 --http -npx -y codebase-context@2.2.0 --http --port 4000 +npx -y codebase-context --http +npx -y codebase-context --http --port 4000 ``` -The documented default endpoint is `http://127.0.0.1:3100/mcp`. Config-shape templates are available in [`templates/mcp/stdio/.mcp.json`](../templates/mcp/stdio/.mcp.json) and [`templates/mcp/http/.mcp.json`](../templates/mcp/http/.mcp.json); pin the published package as in the recipes above. HTTP does not discover a client's project folder; the agent still has to select its repository. +The documented default endpoint is `http://127.0.0.1:3100/mcp`. Config-shape templates are available in [`templates/mcp/stdio/.mcp.json`](../templates/mcp/stdio/.mcp.json) and [`templates/mcp/http/.mcp.json`](../templates/mcp/http/.mcp.json); the recipes use the current npm release. HTTP does not discover a client's project folder; the agent still has to select its repository. The [MCP July 28, 2026 HTTP revision](https://modelcontextprotocol.io/specification/2026-07-28/basic/transports/streamable-http) removed protocol sessions and the GET stream endpoint. This source checkout uses SDK 1.27.1 and the earlier protocol/session model. The stdio checks do not validate its concurrent HTTP project isolation or compatibility with the new revision. A protocol migration and host testing are separate from these setup recipes. @@ -339,7 +340,7 @@ Some clients also announce workspace roots. One valid root can auto-select; seve [MCP deprecated Roots on July 28, 2026](https://modelcontextprotocol.io/specification/2026-07-28/client/roots), recommending explicit directories/files in tool parameters, resource URIs or server configuration. CBC's older Roots support remains a compatibility aid rather than a required setup step. -October 6 verification: 18 focused current-source routing tests passed, including ambiguous roots, explicit selection, subsequent calls and root changes. The exact published server also switched between two isolated prepared repositories A to B to A in one no-roots stdio process: search paths, context map and subsequent status calls followed each selection. The runner host opened the expected distinct sources. A native agent session across real repositories remains a validation step; stdio process-local selection does not establish concurrent HTTP isolation. +October 6 verification: 18 focused current-source routing tests passed, including ambiguous roots, explicit selection, subsequent calls and root changes. The exact published `2.2.0` server also switched between two isolated prepared repositories A to B to A in one no-roots stdio process; search paths, context map and subsequent status calls followed each selection, and the runner host opened the expected distinct sources. This historical package check does not verify newer source changes or establish concurrent HTTP isolation. A native agent session across real repositories remains a validation step; stdio process-local selection does not establish concurrent HTTP isolation. A follow-up recorded raw `search_codebase` calls containing only `query` and `mode`, with no `project`: they returned A, B and A after those explicit selections. The caller does not automatically add the full path; CBC routes omitted-project calls using its selected project. This state lasts for that stdio process. Select again after a server restart or when changing repositories. diff --git a/docs/demo.md b/docs/demo.md index 8b06f23..407fd82 100644 --- a/docs/demo.md +++ b/docs/demo.md @@ -2,14 +2,16 @@ This walkthrough shows real CLI output captured from the open-source `angular-spotify` repository during a local proof run. That sample checkout is not bundled here. -To try the same CLI flow, use Node.js 22 or newer and open a terminal in the repository you want to inspect. These commands use published `codebase-context@2.2.0`; the CLI uses the current directory as the project root. MCP registration and project selection are covered in the [client setup guide](./client-setup.md). +These commands use the current npm release. The output shown below is historical and does not establish behavior of that release. -The saved output below is from an earlier CLI run, not the published-package commands shown here. The machine-specific repository root is omitted from the returned file path. This capture does not verify the current package or an MCP connection. +To try the same CLI flow, use Node.js 22 or newer and open a terminal in the repository you want to inspect. These commands target `codebase-context`; the CLI uses the current directory as the project root. MCP registration and project selection are covered in the [client setup guide](./client-setup.md). + +The saved output below is from an earlier CLI run, not from the current commands. The machine-specific repository root is omitted from the returned file path. This capture does not verify the current package or an MCP connection. ## 0. Build The Local Index ```bash -npx -y codebase-context@2.2.0 reindex +npx -y codebase-context reindex ``` Run this before the first map or search in a repository. Later runs can use `reindex --incremental` after the code changes. @@ -17,7 +19,7 @@ Run this before the first map or search in a repository. Later runs can use `rei ## 1. Start With The Conventions Map ```bash -npx -y codebase-context@2.2.0 map --json +npx -y codebase-context map --json ``` Captured output excerpt: @@ -50,7 +52,7 @@ What this shows: ## 2. Search With Edit Intent ```bash -npx -y codebase-context@2.2.0 search --query "auth headers" --intent edit --limit 3 --json +npx -y codebase-context search --query "auth headers" --intent edit --limit 3 --json ``` Captured output excerpt: @@ -91,7 +93,7 @@ What this shows: ## 3. Check A Team Pattern Directly ```bash -npx -y codebase-context@2.2.0 patterns --category state --json +npx -y codebase-context patterns --category state --json ``` Captured output excerpt: diff --git a/docs/review-context.md b/docs/review-context.md index e35df6a..964a9b4 100644 --- a/docs/review-context.md +++ b/docs/review-context.md @@ -4,6 +4,8 @@ It is deliberately **not** an AI reviewer. It does not call an LLM, decide whether code is correct, or post comments. Its job is narrower and testable: compile the changed surface, stable diff signals, related codebase context, and current conventions into a versioned input for a reviewer or evaluation harness. +The packaged review CLI requires the unpublished 2.4.0 release (the `codebase-context-review` executable does not exist in published 2.2.0); until then, use the source-build command shown below. + ## Why this exists A PR reviewer needs more than the patch, but dumping an entire repository into a model is expensive and hard to reproduce. The review-context path creates a clean boundary between: @@ -23,16 +25,16 @@ pnpm build node dist/review-bin.js --base origin/main --head HEAD ``` -After package publication, the package also exposes: +Requires 2.4.0 (not yet published): after package publication, the package also exposes: ```bash -npx codebase-context-review --base origin/main --head HEAD +npm exec --yes --package=codebase-context@2.4.0 -- codebase-context-review --base origin/main --head HEAD ``` -Use `--json` for the complete machine-readable packet: +Use `--json` for the complete machine-readable packet (requires 2.4.0, not yet published): ```bash -npx codebase-context-review \ +npm exec --yes --package=codebase-context@2.4.0 -- codebase-context-review \ --base origin/main \ --head HEAD \ --max-queries 8 \ diff --git a/package.json b/package.json index 2b75047..a855889 100644 --- a/package.json +++ b/package.json @@ -60,7 +60,7 @@ ], "packageManager": "pnpm@10.27.0", "engines": { - "node": ">=18.0.0" + "node": ">=22.0.0" }, "keywords": [ "mcp", @@ -118,7 +118,7 @@ "sync:grammars": "node scripts/sync-grammars.mjs", "prebuild": "pnpm run sync:grammars", "build": "tsc", - "prepack": "pnpm run sync:grammars", + "prepack": "pnpm run build", "prepublishOnly": "pnpm run build", "pretest": "pnpm run sync:grammars", "start": "node dist/index.js", @@ -135,23 +135,36 @@ "prepare": "husky" }, "dependencies": { - "@huggingface/transformers": "^4.2.0", "@inquirer/prompts": "^3.0.0", - "@lancedb/lancedb": "^0.4.0", "@modelcontextprotocol/sdk": "^1.27.1", - "@typescript-eslint/typescript-estree": "^7.0.0", - "chokidar": "^3.6.0", + "@typescript-eslint/typescript-estree": "^8.51.0", + "chokidar": "^4.0.3", "fuse.js": "^7.0.0", "glob": "^10.3.10", "hono": "^4.12.12", "ignore": "^5.3.1", + "smol-toml": "1.9.1", "typescript": "^5.3.3", "uuid": "^14.0.0", "web-tree-sitter": "^0.25.10", "zod": "^4.3.4" }, + "peerDependencies": { + "@huggingface/transformers": "^4.2.0", + "@lancedb/lancedb": "^0.4.0" + }, + "peerDependenciesMeta": { + "@huggingface/transformers": { + "optional": true + }, + "@lancedb/lancedb": { + "optional": true + } + }, "devDependencies": { "@eslint/js": "^9.39.2", + "@huggingface/transformers": "^4.2.0", + "@lancedb/lancedb": "^0.4.0", "@types/glob": "^8.1.0", "@types/node": "^20.11.24", "@types/uuid": "^9.0.8", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index b550f49..468d100 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -23,24 +23,18 @@ importers: .: dependencies: - '@huggingface/transformers': - specifier: ^4.2.0 - version: 4.2.0 '@inquirer/prompts': specifier: ^3.0.0 version: 3.3.2 - '@lancedb/lancedb': - specifier: ^0.4.0 - version: 0.4.20(zod@4.3.4) '@modelcontextprotocol/sdk': specifier: ^1.27.1 version: 1.27.1(zod@4.3.4) '@typescript-eslint/typescript-estree': - specifier: ^7.0.0 - version: 7.18.0(typescript@5.9.3) + specifier: ^8.51.0 + version: 8.51.0(typescript@5.9.3) chokidar: - specifier: ^3.6.0 - version: 3.6.0 + specifier: ^4.0.3 + version: 4.0.3 fuse.js: specifier: ^7.0.0 version: 7.1.0 @@ -49,10 +43,13 @@ importers: version: 10.5.0 hono: specifier: ^4.12.12 - version: 4.12.14 + version: 4.12.16 ignore: specifier: ^5.3.1 version: 5.3.2 + smol-toml: + specifier: 1.9.1 + version: 1.9.1 typescript: specifier: ^5.3.3 version: 5.9.3 @@ -69,6 +66,12 @@ importers: '@eslint/js': specifier: ^9.39.2 version: 9.39.2 + '@huggingface/transformers': + specifier: ^4.2.0 + version: 4.2.0 + '@lancedb/lancedb': + specifier: ^0.4.0 + version: 0.4.20(zod@4.3.4) '@types/glob': specifier: ^8.1.0 version: 8.1.0 @@ -380,105 +383,89 @@ packages: resolution: {integrity: sha512-excjX8DfsIcJ10x1Kzr4RcWe1edC9PquDRRPx3YVCvQv+U5p7Yin2s32ftzikXojb1PIFc/9Mt28/y+iRklkrw==} cpu: [arm64] os: [linux] - libc: [glibc] '@img/sharp-libvips-linux-arm@1.2.4': resolution: {integrity: sha512-bFI7xcKFELdiNCVov8e44Ia4u2byA+l3XtsAj+Q8tfCwO6BQ8iDojYdvoPMqsKDkuoOo+X6HZA0s0q11ANMQ8A==} cpu: [arm] os: [linux] - libc: [glibc] '@img/sharp-libvips-linux-ppc64@1.2.4': resolution: {integrity: sha512-FMuvGijLDYG6lW+b/UvyilUWu5Ayu+3r2d1S8notiGCIyYU/76eig1UfMmkZ7vwgOrzKzlQbFSuQfgm7GYUPpA==} cpu: [ppc64] os: [linux] - libc: [glibc] '@img/sharp-libvips-linux-riscv64@1.2.4': resolution: {integrity: sha512-oVDbcR4zUC0ce82teubSm+x6ETixtKZBh/qbREIOcI3cULzDyb18Sr/Wcyx7NRQeQzOiHTNbZFF1UwPS2scyGA==} cpu: [riscv64] os: [linux] - libc: [glibc] '@img/sharp-libvips-linux-s390x@1.2.4': resolution: {integrity: sha512-qmp9VrzgPgMoGZyPvrQHqk02uyjA0/QrTO26Tqk6l4ZV0MPWIW6LTkqOIov+J1yEu7MbFQaDpwdwJKhbJvuRxQ==} cpu: [s390x] os: [linux] - libc: [glibc] '@img/sharp-libvips-linux-x64@1.2.4': resolution: {integrity: sha512-tJxiiLsmHc9Ax1bz3oaOYBURTXGIRDODBqhveVHonrHJ9/+k89qbLl0bcJns+e4t4rvaNBxaEZsFtSfAdquPrw==} cpu: [x64] os: [linux] - libc: [glibc] '@img/sharp-libvips-linuxmusl-arm64@1.2.4': resolution: {integrity: sha512-FVQHuwx1IIuNow9QAbYUzJ+En8KcVm9Lk5+uGUQJHaZmMECZmOlix9HnH7n1TRkXMS0pGxIJokIVB9SuqZGGXw==} cpu: [arm64] os: [linux] - libc: [musl] '@img/sharp-libvips-linuxmusl-x64@1.2.4': resolution: {integrity: sha512-+LpyBk7L44ZIXwz/VYfglaX/okxezESc6UxDSoyo2Ks6Jxc4Y7sGjpgU9s4PMgqgjj1gZCylTieNamqA1MF7Dg==} cpu: [x64] os: [linux] - libc: [musl] '@img/sharp-linux-arm64@0.34.5': resolution: {integrity: sha512-bKQzaJRY/bkPOXyKx5EVup7qkaojECG6NLYswgktOZjaXecSAeCWiZwwiFf3/Y+O1HrauiE3FVsGxFg8c24rZg==} engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} cpu: [arm64] os: [linux] - libc: [glibc] '@img/sharp-linux-arm@0.34.5': resolution: {integrity: sha512-9dLqsvwtg1uuXBGZKsxem9595+ujv0sJ6Vi8wcTANSFpwV/GONat5eCkzQo/1O6zRIkh0m/8+5BjrRr7jDUSZw==} engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} cpu: [arm] os: [linux] - libc: [glibc] '@img/sharp-linux-ppc64@0.34.5': resolution: {integrity: sha512-7zznwNaqW6YtsfrGGDA6BRkISKAAE1Jo0QdpNYXNMHu2+0dTrPflTLNkpc8l7MUP5M16ZJcUvysVWWrMefZquA==} engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} cpu: [ppc64] os: [linux] - libc: [glibc] '@img/sharp-linux-riscv64@0.34.5': resolution: {integrity: sha512-51gJuLPTKa7piYPaVs8GmByo7/U7/7TZOq+cnXJIHZKavIRHAP77e3N2HEl3dgiqdD/w0yUfiJnII77PuDDFdw==} engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} cpu: [riscv64] os: [linux] - libc: [glibc] '@img/sharp-linux-s390x@0.34.5': resolution: {integrity: sha512-nQtCk0PdKfho3eC5MrbQoigJ2gd1CgddUMkabUj+rBevs8tZ2cULOx46E7oyX+04WGfABgIwmMC0VqieTiR4jg==} engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} cpu: [s390x] os: [linux] - libc: [glibc] '@img/sharp-linux-x64@0.34.5': resolution: {integrity: sha512-MEzd8HPKxVxVenwAa+JRPwEC7QFjoPWuS5NZnBt6B3pu7EG2Ge0id1oLHZpPJdn3OQK+BQDiw9zStiHBTJQQQQ==} engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} cpu: [x64] os: [linux] - libc: [glibc] '@img/sharp-linuxmusl-arm64@0.34.5': resolution: {integrity: sha512-fprJR6GtRsMt6Kyfq44IsChVZeGN97gTD331weR1ex1c1rypDEABN6Tm2xa1wE6lYb5DdEnk03NZPqA7Id21yg==} engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} cpu: [arm64] os: [linux] - libc: [musl] '@img/sharp-linuxmusl-x64@0.34.5': resolution: {integrity: sha512-Jg8wNT1MUzIvhBFxViqrEhWDGzqymo3sV7z7ZsaWbZNDLXRJZoRGrjulp60YYtV4wfY8VIKcWidjojlLcWrd8Q==} engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} cpu: [x64] os: [linux] - libc: [musl] '@img/sharp-wasm32@0.34.5': resolution: {integrity: sha512-OdWTEiVkY2PHwqkbBI8frFxQQFekHaSSkUIJkwzclWZe64O1X4UlUjqqqLaPbUpMOQk6FBu/HtlGXNblIs0huw==} @@ -571,14 +558,12 @@ packages: engines: {node: '>= 18'} cpu: [arm64] os: [linux] - libc: [glibc] '@lancedb/lancedb-linux-x64-gnu@0.4.20': resolution: {integrity: sha512-TVicKBhT3Q73gtWmMdR/0+r4gBve5gw3KbcrvzHP/5bNpo2sffETbLEMgLjulunu/Jp8IgC921WW6IFJzNFwrQ==} engines: {node: '>= 18'} cpu: [x64] os: [linux] - libc: [glibc] '@lancedb/lancedb-win32-x64-msvc@0.4.20': resolution: {integrity: sha512-7RsTJAx93NbVD//9m0/TpgQmPsTHlCBfGNFD2zkdZ0j7iacURdG9+egz+Os0veTBqMgwnr6VfMf3T7S0wuu99g==} @@ -602,18 +587,6 @@ packages: '@cfworker/json-schema': optional: true - '@nodelib/fs.scandir@2.1.5': - resolution: {integrity: sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==} - engines: {node: '>= 8'} - - '@nodelib/fs.stat@2.0.5': - resolution: {integrity: sha512-RkhPPp2zrqDAQA/2jNhnztcPAlv64XdhIp7a7454A5ovI7Bukxgt7MX7udwAu3zg1DcpPU0rz3VV1SeaqvY4+A==} - engines: {node: '>= 8'} - - '@nodelib/fs.walk@1.2.8': - resolution: {integrity: sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg==} - engines: {node: '>= 8'} - '@pkgjs/parseargs@0.11.0': resolution: {integrity: sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==} engines: {node: '>=14'} @@ -682,79 +655,66 @@ packages: resolution: {integrity: sha512-t4ONHboXi/3E0rT6OZl1pKbl2Vgxf9vJfWgmUoCEVQVxhW6Cw/c8I6hbbu7DAvgp82RKiH7TpLwxnJeKv2pbsw==} cpu: [arm] os: [linux] - libc: [glibc] '@rollup/rollup-linux-arm-musleabihf@4.59.0': resolution: {integrity: sha512-CikFT7aYPA2ufMD086cVORBYGHffBo4K8MQ4uPS/ZnY54GKj36i196u8U+aDVT2LX4eSMbyHtyOh7D7Zvk2VvA==} cpu: [arm] os: [linux] - libc: [musl] '@rollup/rollup-linux-arm64-gnu@4.59.0': resolution: {integrity: sha512-jYgUGk5aLd1nUb1CtQ8E+t5JhLc9x5WdBKew9ZgAXg7DBk0ZHErLHdXM24rfX+bKrFe+Xp5YuJo54I5HFjGDAA==} cpu: [arm64] os: [linux] - libc: [glibc] '@rollup/rollup-linux-arm64-musl@4.59.0': resolution: {integrity: sha512-peZRVEdnFWZ5Bh2KeumKG9ty7aCXzzEsHShOZEFiCQlDEepP1dpUl/SrUNXNg13UmZl+gzVDPsiCwnV1uI0RUA==} cpu: [arm64] os: [linux] - libc: [musl] '@rollup/rollup-linux-loong64-gnu@4.59.0': resolution: {integrity: sha512-gbUSW/97f7+r4gHy3Jlup8zDG190AuodsWnNiXErp9mT90iCy9NKKU0Xwx5k8VlRAIV2uU9CsMnEFg/xXaOfXg==} cpu: [loong64] os: [linux] - libc: [glibc] '@rollup/rollup-linux-loong64-musl@4.59.0': resolution: {integrity: sha512-yTRONe79E+o0FWFijasoTjtzG9EBedFXJMl888NBEDCDV9I2wGbFFfJQQe63OijbFCUZqxpHz1GzpbtSFikJ4Q==} cpu: [loong64] os: [linux] - libc: [musl] '@rollup/rollup-linux-ppc64-gnu@4.59.0': resolution: {integrity: sha512-sw1o3tfyk12k3OEpRddF68a1unZ5VCN7zoTNtSn2KndUE+ea3m3ROOKRCZxEpmT9nsGnogpFP9x6mnLTCaoLkA==} cpu: [ppc64] os: [linux] - libc: [glibc] '@rollup/rollup-linux-ppc64-musl@4.59.0': resolution: {integrity: sha512-+2kLtQ4xT3AiIxkzFVFXfsmlZiG5FXYW7ZyIIvGA7Bdeuh9Z0aN4hVyXS/G1E9bTP/vqszNIN/pUKCk/BTHsKA==} cpu: [ppc64] os: [linux] - libc: [musl] '@rollup/rollup-linux-riscv64-gnu@4.59.0': resolution: {integrity: sha512-NDYMpsXYJJaj+I7UdwIuHHNxXZ/b/N2hR15NyH3m2qAtb/hHPA4g4SuuvrdxetTdndfj9b1WOmy73kcPRoERUg==} cpu: [riscv64] os: [linux] - libc: [glibc] '@rollup/rollup-linux-riscv64-musl@4.59.0': resolution: {integrity: sha512-nLckB8WOqHIf1bhymk+oHxvM9D3tyPndZH8i8+35p/1YiVoVswPid2yLzgX7ZJP0KQvnkhM4H6QZ5m0LzbyIAg==} cpu: [riscv64] os: [linux] - libc: [musl] '@rollup/rollup-linux-s390x-gnu@4.59.0': resolution: {integrity: sha512-oF87Ie3uAIvORFBpwnCvUzdeYUqi2wY6jRFWJAy1qus/udHFYIkplYRW+wo+GRUP4sKzYdmE1Y3+rY5Gc4ZO+w==} cpu: [s390x] os: [linux] - libc: [glibc] '@rollup/rollup-linux-x64-gnu@4.59.0': resolution: {integrity: sha512-3AHmtQq/ppNuUspKAlvA8HtLybkDflkMuLK4DPo77DfthRb71V84/c4MlWJXixZz4uruIH4uaa07IqoAkG64fg==} cpu: [x64] os: [linux] - libc: [glibc] '@rollup/rollup-linux-x64-musl@4.59.0': resolution: {integrity: sha512-2UdiwS/9cTAx7qIUZB/fWtToJwvt0Vbo0zmnYt7ED35KPg13Q0ym1g442THLC7VyI6JfYTP4PiSOWyoMdV2/xg==} cpu: [x64] os: [linux] - libc: [musl] '@rollup/rollup-openbsd-x64@4.59.0': resolution: {integrity: sha512-M3bLRAVk6GOwFlPTIxVBSYKUaqfLrn8l0psKinkCFxl4lQvOSz8ZrKDz2gxcBwHFpci0B6rttydI4IpS4IS/jQ==} @@ -878,23 +838,10 @@ packages: eslint: ^8.57.0 || ^9.0.0 typescript: '>=4.8.4 <6.0.0' - '@typescript-eslint/types@7.18.0': - resolution: {integrity: sha512-iZqi+Ds1y4EDYUtlOOC+aUmxnE9xS/yCigkjA7XpTKV6nCBd3Hp/PRGGmdwnfkV2ThMyYldP1wRpm/id99spTQ==} - engines: {node: ^18.18.0 || >=20.0.0} - '@typescript-eslint/types@8.51.0': resolution: {integrity: sha512-TizAvWYFM6sSscmEakjY3sPqGwxZRSywSsPEiuZF6d5GmGD9Gvlsv0f6N8FvAAA0CD06l3rIcWNbsN1e5F/9Ag==} engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} - '@typescript-eslint/typescript-estree@7.18.0': - resolution: {integrity: sha512-aP1v/BSPnnyhMHts8cf1qQ6Q1IFwwRvAQGRvBFkWlo3/lH29OXA3Pts+c10nxRxIBrDnoMqzhgdwVe5f2D6OzA==} - engines: {node: ^18.18.0 || >=20.0.0} - peerDependencies: - typescript: '*' - peerDependenciesMeta: - typescript: - optional: true - '@typescript-eslint/typescript-estree@8.51.0': resolution: {integrity: sha512-1qNjGqFRmlq0VW5iVlcyHBbCjPB7y6SxpBkrbhNWMy/65ZoncXCEPJxkRZL8McrseNH6lFhaxCIaX+vBuFnRng==} engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} @@ -908,10 +855,6 @@ packages: eslint: ^8.57.0 || ^9.0.0 typescript: '>=4.8.4 <6.0.0' - '@typescript-eslint/visitor-keys@7.18.0': - resolution: {integrity: sha512-cDF0/Gf81QpY3xYyJKDV14Zwdmid5+uuENhjH2EqFaF0ni+yAyq/LzMaIJdhNJXZI7uLzwIlA+V7oWoyn6Curg==} - engines: {node: ^18.18.0 || >=20.0.0} - '@typescript-eslint/visitor-keys@8.51.0': resolution: {integrity: sha512-mM/JRQOzhVN1ykejrvwnBRV3+7yTKK8tVANVN3o1O0t0v7o+jqdVu9crPy5Y9dov15TJk/FTIgoUGHrTOVL3Zg==} engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} @@ -1005,10 +948,6 @@ packages: resolution: {integrity: sha512-4Dj6M28JB+oAH8kFkTLUo+a2jwOFkuqb3yucU0CANcRRUbxS0cP0nZYCGjcc3BNXwRIsUVmDGgzawme7zvJHvg==} engines: {node: '>=12'} - anymatch@3.1.3: - resolution: {integrity: sha512-KMReFUr0B4t+D+OBkjR3KYqvocp2XaSzO55UcB6mgQMd3KbcE+mWTyvVV7D/zsdEbNnV6acZUutkiHQXvTr1Rw==} - engines: {node: '>= 8'} - apache-arrow@15.0.2: resolution: {integrity: sha512-RvwlFxLRpO405PLGffx4N2PYLiF7FD86Q1hHl6J2XCWiq+tTCzpb9ngFw0apFDcXZBMpCzMuwAvA7hjyL1/73A==} hasBin: true @@ -1032,10 +971,6 @@ packages: resolution: {integrity: sha512-FmeCCAenzH0KH381SPT5FZmiA/TmpndpcaShhfgEN9eCVjnFBqq3l1xrI42y8+PPLI6hypzou4GXw00WHmPBLQ==} engines: {node: '>= 0.4'} - array-union@2.1.0: - resolution: {integrity: sha512-HGyxoOTYUyCM6stUe6EJgnd4EoewAI7zMdfqO+kGjnlZmBDz/cR5pf8r/cR4Wq60sL/p0IkcjUEEPwS3GFrIyw==} - engines: {node: '>=8'} - array.prototype.findlastindex@1.2.6: resolution: {integrity: sha512-F/TKATkzseUExPlfvmwQKGITM3DGTK+vkAsCZoDc5daVygbJBnjEUCbgkAvVFsgfXfX4YIqZ/27G3k3tdXrTxQ==} engines: {node: '>= 0.4'} @@ -1071,10 +1006,6 @@ packages: resolution: {integrity: sha512-1pHv8LX9CpKut1Zp4EXey7Z8OfH11ONNH6Dhi2WDUt31VVZFXZzKwXcysBgqSumFCmR+0dqjMK5v5JiFHzi0+g==} engines: {node: 20 || >=22} - binary-extensions@2.3.0: - resolution: {integrity: sha512-Ceh+7ox5qe7LJuLHoY0feh3pHuUDHAcRUeyL2VYghZwfpkNIy/+8Ocg0a3UuSoYzavmylwuLWQOf3hl0jjMMIw==} - engines: {node: '>=8'} - body-parser@2.2.2: resolution: {integrity: sha512-oP5VkATKlNwcgvxi0vM0p/D3n2C3EReYVX+DNYs5TjZFn/oQt2j+4sVJtSMr18pdRr8wjTcBl6LoV+FUwzPmNA==} engines: {node: '>=18'} @@ -1087,10 +1018,6 @@ packages: resolution: {integrity: sha512-VZznLgtwhn+Mact9tfiwx64fA9erHH/MCXEUfB/0bX/6Fz6ny5EGTXYltMocqg4xFAQZtnO3DHWWXi8RiuN7cQ==} engines: {node: 18 || 20 || >=22} - braces@3.0.3: - resolution: {integrity: sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==} - engines: {node: '>=8'} - bytes@3.1.2: resolution: {integrity: sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==} engines: {node: '>= 0.8'} @@ -1126,9 +1053,9 @@ packages: chardet@0.7.0: resolution: {integrity: sha512-mT8iDcrh03qDGRRmoA2hmBJnxpllMR+0/0qlzjqZES6NdiWDcZkCNAk4rPFZ9Q85r27unkiNNg8ZOiwZXBHwcA==} - chokidar@3.6.0: - resolution: {integrity: sha512-7VT13fmjotKpGipCW9JEQAusEPE+Ei8nl6/g4FBAmIm0GOOLMua9NDDo/DWp0ZAxCr3cPq5ZpBqmPAQgDda2Pw==} - engines: {node: '>= 8.10.0'} + chokidar@4.0.3: + resolution: {integrity: sha512-Qgzu8kfBvo+cA4962jnP1KkS6Dop5NS6g7R5LFYJr4b8Ub94PPQXUksCw9PvXoeXPRRddRNC5C1JQUR2SMGtnA==} + engines: {node: '>= 14.16.0'} cli-spinners@2.9.2: resolution: {integrity: sha512-ywqV+5MmyL4E7ybXgKys4DugZbX0FC6LnwrhjuykIjnK9k8OQacQ7axGKnjDXWNhns0xot3bZI5h55H8yo9cJg==} @@ -1236,10 +1163,6 @@ packages: detect-node@2.1.0: resolution: {integrity: sha512-T0NIuQpnTvFDATNuHN5roPwSBG83rFsuO+MXXH9/3N1eFbn4wcPjttvjMLEPWJ0RGUYgQE7cGgS3tNxbqCGM7g==} - dir-glob@3.0.1: - resolution: {integrity: sha512-WkrWp9GR4KXfKGYzOLmTuGVi1UWFfws377n9cc55/tb6DuqyF6pcQ5AbiHEshaDpY9v6oaSr2XCDidGmMwdzIA==} - engines: {node: '>=8'} - doctrine@2.1.0: resolution: {integrity: sha512-35mSku4ZXK0vfCuHEDAwt55dg2jNajHZ1odvF+8SSr82EsZY4QmXfuWso8oEd8zRhVObSN18aM0CjSdoBX7zIw==} engines: {node: '>=0.10.0'} @@ -1369,6 +1292,7 @@ packages: eslint@9.39.3: resolution: {integrity: sha512-VmQ+sifHUbI/IcSopBCF/HO3YiHQx/AVd3UVyYL6weuwW+HvON9VYn5l6Zl1WZzPWXPNZrSQpxwkkZ/VuvJZzg==} engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + deprecated: This version is no longer supported. Please see https://eslint.org/version-support for other options. hasBin: true peerDependencies: jiti: '*' @@ -1436,10 +1360,6 @@ packages: fast-deep-equal@3.1.3: resolution: {integrity: sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==} - fast-glob@3.3.3: - resolution: {integrity: sha512-7MptL8U0cqcFdzIzwOTHoilX9x5BrNqye7Z/LuC7kCMRio1EMSyqRK3BEAUD7sXRq4iT4AzTVuZdhgQ2TCvYLg==} - engines: {node: '>=8.6.0'} - fast-json-stable-stringify@2.1.0: resolution: {integrity: sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==} @@ -1449,9 +1369,6 @@ packages: fast-uri@3.1.0: resolution: {integrity: sha512-iPeeDKJSWf4IEOasVVrknXpaBV0IApz/gp7S2bb7Z4Lljbl2MGJRqInZiUrQwV16cpzw/D3S5j5Julj/gT52AA==} - fastq@1.19.1: - resolution: {integrity: sha512-GwLTyxkCXjXbxqIhTsMI2Nui8huMPtnxg7krajPJAjnEG/iiOS7i+zCtWGZR9G0NBKbXKh6X9m9UIsYX/N6vvQ==} - fdir@6.5.0: resolution: {integrity: sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg==} engines: {node: '>=12.0.0'} @@ -1469,10 +1386,6 @@ packages: resolution: {integrity: sha512-XXTUwCvisa5oacNGRP9SfNtYBNAMi+RPwBFmblZEF7N7swHYQS6/Zfk7SRwx4D5j3CH211YNRco1DEMNVfZCnQ==} engines: {node: '>=16.0.0'} - fill-range@7.1.1: - resolution: {integrity: sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==} - engines: {node: '>=8'} - finalhandler@2.1.1: resolution: {integrity: sha512-S8KoZgRZN+a5rNwqTxlZZePjT/4cnm0ROV70LedRHZ0p8u9fRID0hJUZQpkKLzro8LfmC8sx23bY6tVNxv8pQA==} engines: {node: '>= 18.0.0'} @@ -1563,10 +1476,6 @@ packages: get-tsconfig@4.13.0: resolution: {integrity: sha512-1VKTZJCwBrvbd+Wn3AOgQP/2Av+TfTCOlE4AcRJE72W1ksZXbAx8PPBR9RzgTeSPzlPMHrbANMH3LbltH73wxQ==} - glob-parent@5.1.2: - resolution: {integrity: sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==} - engines: {node: '>= 6'} - glob-parent@6.0.2: resolution: {integrity: sha512-XxwI8EOhVQgWp6iDL+3b0r86f4d6AX6zSU55HfB4ydCEuXLXc5FcYeOu+nnGftS4TEju/11rt4KJPTMgbfmv4A==} engines: {node: '>=10.13.0'} @@ -1592,10 +1501,6 @@ packages: resolution: {integrity: sha512-DpLKbNU4WylpxJykQujfCcwYWiV/Jhm50Goo0wrVILAv5jOr9d+H+UR3PhSCD2rCCEIg0uc+G+muBTwD54JhDQ==} engines: {node: '>= 0.4'} - globby@11.1.0: - resolution: {integrity: sha512-jhIXaOzy1sb8IyocaruWSn1TjmnBVs8Ayhcy83rmxNJ8q2uWKCAj3CnJY+KpGSXCueAPc0i05kVvVKtP1t9S3g==} - engines: {node: '>=10'} - gopd@1.2.0: resolution: {integrity: sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==} engines: {node: '>= 0.4'} @@ -1630,8 +1535,8 @@ packages: resolution: {integrity: sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==} engines: {node: '>= 0.4'} - hono@4.12.14: - resolution: {integrity: sha512-am5zfg3yu6sqn5yjKBNqhnTX7Cv+m00ox+7jbaKkrLMRJ4rAdldd1xPd/JzbBWspqaQv6RSTrgFN95EsfhC+7w==} + hono@4.12.16: + resolution: {integrity: sha512-jN0ZewiNAWSe5khM3EyCmBb250+b40wWbwNILNfEvq84VREWwOIkuUsFONk/3i3nqkz7Oe1PcpM2mwQEK2L9Kg==} engines: {node: '>=16.9.0'} http-errors@2.0.1: @@ -1697,10 +1602,6 @@ packages: resolution: {integrity: sha512-n4ZT37wG78iz03xPRKJrHTdZbe3IicyucEtdRsV5yglwc3GyUfbAfpSeD0FJ41NbUNSt5wbhqfp1fS+BgnvDFQ==} engines: {node: '>= 0.4'} - is-binary-path@2.1.0: - resolution: {integrity: sha512-ZMERYes6pDydyuGidse7OsHxtbI7WVeUEozgR/g7rd0xUimYNlvZRE/K2MgZTjWy725IfelLeVcEM97mmtRGXw==} - engines: {node: '>=8'} - is-boolean-object@1.2.2: resolution: {integrity: sha512-wa56o2/ElJMYqjCjGkXri7it5FbebW5usLw/nPmCMs5DeZ7eziSYZhSmPRn0txqeW4LnAmQQU7FgqLpsEFKM4A==} engines: {node: '>= 0.4'} @@ -1753,10 +1654,6 @@ packages: resolution: {integrity: sha512-lZhclumE1G6VYD8VHe35wFaIif+CTy5SJIi5+3y4psDgWu4wPDoBhF8NxUOinEc7pHgiTsT6MaBb92rKhhD+Xw==} engines: {node: '>= 0.4'} - is-number@7.0.0: - resolution: {integrity: sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==} - engines: {node: '>=0.12.0'} - is-promise@4.0.0: resolution: {integrity: sha512-hvpoI6korhJMnej285dSg6nu1+e6uxs7zG3BYAm5byqDsgJNWwxzM6z6iZiAgQR4TJ30JmBTOwqZUw3WlyH3AQ==} @@ -1880,14 +1777,6 @@ packages: resolution: {integrity: sha512-Snk314V5ayFLhp3fkUREub6WtjBfPdCPY1Ln8/8munuLuiYhsABgBVWsozAG+MWMbVEvcdcpbi9R7ww22l9Q3g==} engines: {node: '>=18'} - merge2@1.4.1: - resolution: {integrity: sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg==} - engines: {node: '>= 8'} - - micromatch@4.0.8: - resolution: {integrity: sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA==} - engines: {node: '>=8.6'} - mime-db@1.52.0: resolution: {integrity: sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==} engines: {node: '>= 0.6'} @@ -1948,10 +1837,6 @@ packages: encoding: optional: true - normalize-path@3.0.0: - resolution: {integrity: sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==} - engines: {node: '>=0.10.0'} - object-assign@4.1.1: resolution: {integrity: sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==} engines: {node: '>=0.10.0'} @@ -2060,20 +1945,12 @@ packages: path-to-regexp@8.4.0: resolution: {integrity: sha512-PuseHIvAnz3bjrM2rGJtSgo1zjgxapTLZ7x2pjhzWwlp4SJQgK3f3iZIQwkpEnBaKz6seKBADpM4B4ySkuYypg==} - path-type@4.0.0: - resolution: {integrity: sha512-gDKb8aZMDeD/tZWs9P6+q0J9Mwkdl6xMV8TjnGP3qJVJ06bdMgkbBlLU8IdfOsIsFz2BW1rNVT3XuNEl8zPAvw==} - engines: {node: '>=8'} - pathe@2.0.3: resolution: {integrity: sha512-WUjGcAqP1gQacoQe+OBJsFA7Ld4DyXuUIjZ5cc75cLHvJ7dtNsTugphxIADwspS+AraAUePCKrSVtPLFj/F88w==} picocolors@1.1.1: resolution: {integrity: sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==} - picomatch@2.3.2: - resolution: {integrity: sha512-V7+vQEJ06Z+c5tSye8S+nHUfI51xoXIXjHQ99cQtKUkQqqO1kO/KCJUfZXuB47h/YBlDhah2H3hdUGXn8ie0oA==} - engines: {node: '>=8.6'} - picomatch@4.0.4: resolution: {integrity: sha512-QP88BAKvMam/3NxH6vj2o21R6MjxZUAd6nlwAS/pnGvN9IVLocLHxGYIzFhg6fUQ+5th6P4dv4eW9jX3DSIj7A==} engines: {node: '>=12'} @@ -2118,9 +1995,6 @@ packages: resolution: {integrity: sha512-V/yCWTTF7VJ9hIh18Ugr2zhJMP01MY7c5kh4J870L7imm6/DIzBsNLTXzMwUA3yZ5b/KBqLx8Kp3uRvd7xSe3Q==} engines: {node: '>=0.6'} - queue-microtask@1.2.3: - resolution: {integrity: sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==} - range-parser@1.2.1: resolution: {integrity: sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg==} engines: {node: '>= 0.6'} @@ -2129,9 +2003,9 @@ packages: resolution: {integrity: sha512-K5zQjDllxWkf7Z5xJdV0/B0WTNqx6vxG70zJE4N0kBs4LovmEYWJzQGxC9bS9RAKu3bgM40lrd5zoLJ12MQ5BA==} engines: {node: '>= 0.10'} - readdirp@3.6.0: - resolution: {integrity: sha512-hOS089on8RduqdbhvQ5Z37A0ESjsqz6qnRcffsMU3495FuTdqSm+7bhJ29JvIOsBDEEnan5DPu9t3To9VRlMzA==} - engines: {node: '>=8.10.0'} + readdirp@4.1.2: + resolution: {integrity: sha512-GDhwkLfywWL2s6vEjyhri+eXmfH6j1L7JE27WhqLeYzoh/A3DBaYGEj2H/HFZCn/kMfim73FXxEJTw06WtxQwg==} + engines: {node: '>= 14.18.0'} reflect.getprototypeof@1.0.10: resolution: {integrity: sha512-00o4I+DVrefhv+nX0ulyi3biSHCPDe+yLv5o/p6d/UVlirijB8E16FtfwSAi4g3tcqrQ4lRAqQSoFEZJehYEcw==} @@ -2157,10 +2031,6 @@ packages: engines: {node: '>= 0.4'} hasBin: true - reusify@1.1.0: - resolution: {integrity: sha512-g6QUff04oZpHs0eG5p83rFLhHeV00ug/Yf9nZM6fLeUrPguBTkTQOdpAWWspMh55TZfVQDPaN3NQJfbVRAxdIw==} - engines: {iojs: '>=1.0.0', node: '>=0.10.0'} - roarr@2.15.4: resolution: {integrity: sha512-CHhPh+UNHD2GTXNYhPWLnU8ONHdI+5DI+4EYIAOaiD63rHeYlZvyh8P+in5999TTSFgUYuKUAjzRI4mdh/p+2A==} engines: {node: '>=8.0'} @@ -2178,9 +2048,6 @@ packages: resolution: {integrity: sha512-540WwVDOMxA6dN6We19EcT9sc3hkXPw5mzRNGM3FkdN/vtE9NFvj5lFAPNwUDmJjXidm3v7TC1cTE7t17Ulm1Q==} engines: {node: '>=0.12.0'} - run-parallel@1.2.0: - resolution: {integrity: sha512-5l4VyZR86LZ/lDxZTR6jqL8AFE2S0IFLMP26AbjsLVADxHdhB/c0GUsH+y39UfCi3dzz8OlQuPmnaJOMoDHQBA==} - safe-array-concat@1.1.3: resolution: {integrity: sha512-AURm5f0jYEOydBj7VQlVvDrjeFgthDdEF5H1dP+6mNpoXOMo1quQqJ4wvJDyRZ9+pO3kGWoOdmV08cSv2aJV6Q==} engines: {node: '>=0.4'} @@ -2270,9 +2137,9 @@ packages: resolution: {integrity: sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==} engines: {node: '>=14'} - slash@3.0.0: - resolution: {integrity: sha512-g9Q1haeby36OSStwb4ntCGGGaKsaVSjQ68fBxoQcutl5fS1vuY18H3wSt3jFyFtrkx+Kz0V1G85A4MyAdDMi2Q==} - engines: {node: '>=8'} + smol-toml@1.9.1: + resolution: {integrity: sha512-j7WyPdf0CJoutMzMoPLF/bVXHtKZIT9Rb4eGeEbWw8X2t5oirjPgpL+OpaENZrILYaXqYwnVH+qgUwVbl+pUJQ==} + engines: {node: '>= 18'} source-map-js@1.2.1: resolution: {integrity: sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==} @@ -2362,10 +2229,6 @@ packages: resolution: {integrity: sha512-UdiSoX6ypifLmrfQ/XfiawN6hkjSBpCjhKxxZcWlUUmoXLaCKQU0bx4HF/tdDK2uzRuchf1txGvrWBzYREssoQ==} engines: {node: '>=14.14'} - to-regex-range@5.0.1: - resolution: {integrity: sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==} - engines: {node: '>=8.0'} - toidentifier@1.0.1: resolution: {integrity: sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==} engines: {node: '>=0.6'} @@ -2376,12 +2239,6 @@ packages: tree-sitter-wasms@0.1.13: resolution: {integrity: sha512-wT+cR6DwaIz80/vho3AvSF0N4txuNx/5bcRKoXouOfClpxh/qqrF4URNLQXbbt8MaAxeksZcZd1j8gcGjc+QxQ==} - ts-api-utils@1.4.3: - resolution: {integrity: sha512-i3eMG77UTMD0hZhgRS562pv83RC6ukSAC2GMNWc+9dieh/+jDM5u5YG+NHX6VNDRHQcHwmsTHctP9LhbC3WxVw==} - engines: {node: '>=16'} - peerDependencies: - typescript: '>=4.2.0' - ts-api-utils@2.3.0: resolution: {integrity: sha512-6eg3Y9SF7SsAvGzRHQvvc1skDAhwI4YQ32ui1scxD1Ccr0G5qIIbUBT3pFTKX8kmWIQClHobtUdNuaBgwdfdWg==} engines: {node: '>=18.12'} @@ -2762,9 +2619,9 @@ snapshots: '@eslint/core': 0.17.0 levn: 0.4.1 - '@hono/node-server@1.19.13(hono@4.12.14)': + '@hono/node-server@1.19.13(hono@4.12.16)': dependencies: - hono: 4.12.14 + hono: 4.12.16 '@huggingface/jinja@0.5.9': {} @@ -3016,7 +2873,7 @@ snapshots: '@modelcontextprotocol/sdk@1.27.1(zod@4.3.4)': dependencies: - '@hono/node-server': 1.19.13(hono@4.12.14) + '@hono/node-server': 1.19.13(hono@4.12.16) ajv: 8.18.0 ajv-formats: 3.0.1(ajv@8.18.0) content-type: 1.0.5 @@ -3026,7 +2883,7 @@ snapshots: eventsource-parser: 3.0.6 express: 5.2.1 express-rate-limit: 8.2.2(express@5.2.1) - hono: 4.12.14 + hono: 4.12.16 jose: 6.1.3 json-schema-typed: 8.0.2 pkce-challenge: 5.0.1 @@ -3036,18 +2893,6 @@ snapshots: transitivePeerDependencies: - supports-color - '@nodelib/fs.scandir@2.1.5': - dependencies: - '@nodelib/fs.stat': 2.0.5 - run-parallel: 1.2.0 - - '@nodelib/fs.stat@2.0.5': {} - - '@nodelib/fs.walk@1.2.8': - dependencies: - '@nodelib/fs.scandir': 2.1.5 - fastq: 1.19.1 - '@pkgjs/parseargs@0.11.0': optional: true @@ -3260,25 +3105,8 @@ snapshots: transitivePeerDependencies: - supports-color - '@typescript-eslint/types@7.18.0': {} - '@typescript-eslint/types@8.51.0': {} - '@typescript-eslint/typescript-estree@7.18.0(typescript@5.9.3)': - dependencies: - '@typescript-eslint/types': 7.18.0 - '@typescript-eslint/visitor-keys': 7.18.0 - debug: 4.4.3 - globby: 11.1.0 - is-glob: 4.0.3 - minimatch: 10.2.3 - semver: 7.7.3 - ts-api-utils: 1.4.3(typescript@5.9.3) - optionalDependencies: - typescript: 5.9.3 - transitivePeerDependencies: - - supports-color - '@typescript-eslint/typescript-estree@8.51.0(typescript@5.9.3)': dependencies: '@typescript-eslint/project-service': 8.51.0(typescript@5.9.3) @@ -3305,11 +3133,6 @@ snapshots: transitivePeerDependencies: - supports-color - '@typescript-eslint/visitor-keys@7.18.0': - dependencies: - '@typescript-eslint/types': 7.18.0 - eslint-visitor-keys: 3.4.3 - '@typescript-eslint/visitor-keys@8.51.0': dependencies: '@typescript-eslint/types': 8.51.0 @@ -3407,11 +3230,6 @@ snapshots: ansi-styles@6.2.3: {} - anymatch@3.1.3: - dependencies: - normalize-path: 3.0.0 - picomatch: 2.3.2 - apache-arrow@15.0.2: dependencies: '@swc/helpers': 0.5.17 @@ -3446,8 +3264,6 @@ snapshots: is-string: 1.1.1 math-intrinsics: 1.1.0 - array-union@2.1.0: {} - array.prototype.findlastindex@1.2.6: dependencies: call-bind: 1.0.8 @@ -3494,8 +3310,6 @@ snapshots: balanced-match@4.0.3: {} - binary-extensions@2.3.0: {} - body-parser@2.2.2: dependencies: bytes: 3.1.2 @@ -3516,10 +3330,6 @@ snapshots: dependencies: balanced-match: 4.0.3 - braces@3.0.3: - dependencies: - fill-range: 7.1.1 - bytes@3.1.2: {} call-bind-apply-helpers@1.0.2: @@ -3554,17 +3364,9 @@ snapshots: chardet@0.7.0: {} - chokidar@3.6.0: + chokidar@4.0.3: dependencies: - anymatch: 3.1.3 - braces: 3.0.3 - glob-parent: 5.1.2 - is-binary-path: 2.1.0 - is-glob: 4.0.3 - normalize-path: 3.0.0 - readdirp: 3.6.0 - optionalDependencies: - fsevents: 2.3.3 + readdirp: 4.1.2 cli-spinners@2.9.2: {} @@ -3661,10 +3463,6 @@ snapshots: detect-node@2.1.0: {} - dir-glob@3.0.1: - dependencies: - path-type: 4.0.0 - doctrine@2.1.0: dependencies: esutils: 2.0.3 @@ -3985,24 +3783,12 @@ snapshots: fast-deep-equal@3.1.3: {} - fast-glob@3.3.3: - dependencies: - '@nodelib/fs.stat': 2.0.5 - '@nodelib/fs.walk': 1.2.8 - glob-parent: 5.1.2 - merge2: 1.4.1 - micromatch: 4.0.8 - fast-json-stable-stringify@2.1.0: {} fast-levenshtein@2.0.6: {} fast-uri@3.1.0: {} - fastq@1.19.1: - dependencies: - reusify: 1.1.0 - fdir@6.5.0(picomatch@4.0.4): optionalDependencies: picomatch: 4.0.4 @@ -4015,10 +3801,6 @@ snapshots: dependencies: flat-cache: 4.0.1 - fill-range@7.1.1: - dependencies: - to-regex-range: 5.0.1 - finalhandler@2.1.1: dependencies: debug: 4.4.3 @@ -4126,10 +3908,6 @@ snapshots: dependencies: resolve-pkg-maps: 1.0.0 - glob-parent@5.1.2: - dependencies: - is-glob: 4.0.3 - glob-parent@6.0.2: dependencies: is-glob: 4.0.3 @@ -4161,15 +3939,6 @@ snapshots: define-properties: 1.2.1 gopd: 1.2.0 - globby@11.1.0: - dependencies: - array-union: 2.1.0 - dir-glob: 3.0.1 - fast-glob: 3.3.3 - ignore: 5.3.2 - merge2: 1.4.1 - slash: 3.0.0 - gopd@1.2.0: {} guid-typescript@1.0.9: {} @@ -4196,7 +3965,7 @@ snapshots: dependencies: function-bind: 1.1.2 - hono@4.12.14: {} + hono@4.12.16: {} http-errors@2.0.1: dependencies: @@ -4261,10 +4030,6 @@ snapshots: dependencies: has-bigints: 1.1.0 - is-binary-path@2.1.0: - dependencies: - binary-extensions: 2.3.0 - is-boolean-object@1.2.2: dependencies: call-bound: 1.0.4 @@ -4316,8 +4081,6 @@ snapshots: call-bound: 1.0.4 has-tostringtag: 1.0.2 - is-number@7.0.0: {} - is-promise@4.0.0: {} is-regex@1.2.1: @@ -4428,13 +4191,6 @@ snapshots: merge-descriptors@2.0.0: {} - merge2@1.4.1: {} - - micromatch@4.0.8: - dependencies: - braces: 3.0.3 - picomatch: 2.3.2 - mime-db@1.52.0: {} mime-db@1.54.0: {} @@ -4471,8 +4227,6 @@ snapshots: dependencies: whatwg-url: 5.0.0 - normalize-path@3.0.0: {} - object-assign@4.1.1: {} object-inspect@1.13.4: {} @@ -4595,14 +4349,10 @@ snapshots: path-to-regexp@8.4.0: {} - path-type@4.0.0: {} - pathe@2.0.3: {} picocolors@1.1.1: {} - picomatch@2.3.2: {} - picomatch@4.0.4: {} pkce-challenge@5.0.1: {} @@ -4647,8 +4397,6 @@ snapshots: dependencies: side-channel: 1.1.0 - queue-microtask@1.2.3: {} - range-parser@1.2.1: {} raw-body@3.0.2: @@ -4658,9 +4406,7 @@ snapshots: iconv-lite: 0.7.2 unpipe: 1.0.0 - readdirp@3.6.0: - dependencies: - picomatch: 2.3.2 + readdirp@4.1.2: {} reflect.getprototypeof@1.0.10: dependencies: @@ -4694,8 +4440,6 @@ snapshots: path-parse: 1.0.7 supports-preserve-symlinks-flag: 1.0.0 - reusify@1.1.0: {} - roarr@2.15.4: dependencies: boolean: 3.2.0 @@ -4748,10 +4492,6 @@ snapshots: run-async@3.0.0: {} - run-parallel@1.2.0: - dependencies: - queue-microtask: 1.2.3 - safe-array-concat@1.1.3: dependencies: call-bind: 1.0.8 @@ -4901,7 +4641,7 @@ snapshots: signal-exit@4.1.0: {} - slash@3.0.0: {} + smol-toml@1.9.1: {} source-map-js@1.2.1: {} @@ -4989,20 +4729,12 @@ snapshots: tmp@0.2.4: {} - to-regex-range@5.0.1: - dependencies: - is-number: 7.0.0 - toidentifier@1.0.1: {} tr46@0.0.3: {} tree-sitter-wasms@0.1.13: {} - ts-api-utils@1.4.3(typescript@5.9.3): - dependencies: - typescript: 5.9.3 - ts-api-utils@2.3.0(typescript@5.9.3): dependencies: typescript: 5.9.3 diff --git a/src/analyzers/angular/index.ts b/src/analyzers/angular/index.ts index aa748ad..5740342 100644 --- a/src/analyzers/angular/index.ts +++ b/src/analyzers/angular/index.ts @@ -1,3 +1,4 @@ +import { resolveIndexArtifact, withAvailableIndexSnapshot } from '../../core/index-generation.js'; /** * Angular Analyzer - Comprehensive Angular-specific code analysis * Understands components, services, directives, pipes, modules, guards, interceptors, etc. @@ -103,9 +104,20 @@ export class AngularAnalyzer implements FrameworkAnalyzer { return false; } - // For TypeScript files, check if it contains Angular decorators + // Eligibility comes from actual framework imports. Method/decorator spellings + // classify confirmed Angular code later; comments and lookalikes are not proof. if (ext === '.ts' && content) { - return Object.values(this.angularPatterns).some((pattern) => pattern.test(content)); + try { + const ast = parse(content); + return ast.body.some( + (node) => + node.type === 'ImportDeclaration' && + typeof node.source.value === 'string' && + node.source.value.startsWith('@angular/') + ); + } catch { + return false; + } } // Angular component templates and styles @@ -171,7 +183,9 @@ export class AngularAnalyzer implements FrameworkAnalyzer { if (s.type === 'ImportDefaultSpecifier') return 'default'; if (s.type === 'ImportNamespaceSpecifier') return '*'; const specifier = s as TSESTree.ImportSpecifier; - return specifier.imported.name || specifier.local.name; + return specifier.imported.type === 'Identifier' + ? specifier.imported.name + : specifier.imported.value; }), isDefault: node.specifiers.some( (s: TSESTree.ImportClause) => s.type === 'ImportDefaultSpecifier' @@ -854,6 +868,10 @@ export class AngularAnalyzer implements FrameworkAnalyzer { } async detectCodebaseMetadata(rootPath: string): Promise { + return withAvailableIndexSnapshot(rootPath, () => this.detectPinnedMetadata(rootPath)); + } + + private async detectPinnedMetadata(rootPath: string): Promise { const metadata: CodebaseMetadata = { name: path.basename(rootPath), rootPath, @@ -984,7 +1002,7 @@ export class AngularAnalyzer implements FrameworkAnalyzer { // Calculate statistics from existing index if available try { - const indexPath = path.join(rootPath, CODEBASE_CONTEXT_DIRNAME, KEYWORD_INDEX_FILENAME); + const indexPath = await resolveIndexArtifact(rootPath, KEYWORD_INDEX_FILENAME); const indexContent = await fs.readFile(indexPath, 'utf-8'); const parsed = JSON.parse(indexContent) as unknown; diff --git a/src/analyzers/generic/index.ts b/src/analyzers/generic/index.ts index 6dcfcaa..78ed320 100644 --- a/src/analyzers/generic/index.ts +++ b/src/analyzers/generic/index.ts @@ -8,6 +8,7 @@ import path from 'path'; import type { TSESTree } from '@typescript-eslint/typescript-estree'; import { FrameworkAnalyzer, + AnalysisOptions, AnalysisResult, CodebaseMetadata, CodeChunk, @@ -99,7 +100,11 @@ export class GenericAnalyzer implements FrameworkAnalyzer { return this.supportedExtensions.includes(ext); } - async analyze(filePath: string, content: string): Promise { + async analyze( + filePath: string, + content: string, + options?: AnalysisOptions + ): Promise { const language = detectLanguage(filePath); const relativePath = path.relative(process.cwd(), filePath); @@ -173,7 +178,8 @@ export class GenericAnalyzer implements FrameworkAnalyzer { filePath, language, framework: 'generic', - componentType: 'module' + componentType: 'module', + includeScopePrefix: options?.includeScopePrefix }); // Enrich AST chunks with the correct relativePath for (const chunk of chunks) { @@ -362,7 +368,9 @@ export class GenericAnalyzer implements FrameworkAnalyzer { if (s.type === 'ImportDefaultSpecifier') return 'default'; if (s.type === 'ImportNamespaceSpecifier') return '*'; const specifier = s as TSESTree.ImportSpecifier; - return specifier.imported.name || specifier.local.name; + return specifier.imported.type === 'Identifier' + ? specifier.imported.name + : specifier.imported.value; }), isDefault: node.specifiers.some( (s: TSESTree.ImportClause) => s.type === 'ImportDefaultSpecifier' @@ -440,7 +448,8 @@ export class GenericAnalyzer implements FrameworkAnalyzer { for (const spec of node.specifiers) { if (spec.type === 'ExportSpecifier') { exports.push({ - name: spec.exported.name, + name: + spec.exported.type === 'Identifier' ? spec.exported.name : spec.exported.value, isDefault: false, type: 'named' }); diff --git a/src/analyzers/react/index.ts b/src/analyzers/react/index.ts index 5838c09..193658f 100644 --- a/src/analyzers/react/index.ts +++ b/src/analyzers/react/index.ts @@ -87,12 +87,21 @@ export class ReactAnalyzer implements FrameworkAnalyzer { return false; } - return ( + const hasReactMarker = /\bfrom\s+['"]react['"]/.test(content) || /\brequire\(\s*['"]react['"]\s*\)/.test(content) || - /\bReact\.createElement\b/.test(content) || - /<[A-Za-z][^>]*>/.test(content) - ); + /\bReact\.createElement\b/.test(content); + if (hasReactMarker) { + return true; + } + + // TypeScript JSX requires .tsx. Plain TypeScript angle brackets also + // start generics and type assertions, so they cannot identify React. + if (extension === '.ts' || extension === '.mts' || extension === '.cts') { + return false; + } + + return /<[A-Za-z][^>]*>/.test(content); } async analyze(filePath: string, content: string): Promise { diff --git a/src/analyzers/shared/metadata.ts b/src/analyzers/shared/metadata.ts index 233274b..5f0f34b 100644 --- a/src/analyzers/shared/metadata.ts +++ b/src/analyzers/shared/metadata.ts @@ -1,3 +1,4 @@ +import { withAvailableIndexSnapshot, resolveIndexArtifact } from '../../core/index-generation.js'; import { promises as fs } from 'fs'; import path from 'path'; import type { CodebaseMetadata } from '../../types/index.js'; @@ -73,9 +74,15 @@ export function createEmptyStatistics(): CodebaseMetadata['statistics'] { export async function loadAnalyzerIndexStatistics( rootPath: string +): Promise { + return withAvailableIndexSnapshot(rootPath, () => loadPinnedAnalyzerIndexStatistics(rootPath)); +} + +async function loadPinnedAnalyzerIndexStatistics( + rootPath: string ): Promise { const statistics = createEmptyStatistics(); - const indexPath = path.join(rootPath, CODEBASE_CONTEXT_DIRNAME, KEYWORD_INDEX_FILENAME); + const indexPath = await resolveIndexArtifact(rootPath, KEYWORD_INDEX_FILENAME); try { const stat = await fs.stat(indexPath); diff --git a/src/cli-formatters.ts b/src/cli-formatters.ts index 3a8049d..c89295f 100644 --- a/src/cli-formatters.ts +++ b/src/cli-formatters.ts @@ -480,7 +480,25 @@ export function formatRefs(data: RefsResponse, rootPath: string): void { if (data.isComplete === false) { lines.push(''); - lines.push(`${g.arrow} results capped at limit — use --limit to see more`); + const coverage = 'coverage' in data ? data.coverage : undefined; + const filesUnavailable = + coverage !== null && + typeof coverage === 'object' && + 'filesUnavailable' in coverage && + typeof coverage.filesUnavailable === 'number' + ? coverage.filesUnavailable + : 0; + if (filesUnavailable > 0) { + const noun = filesUnavailable === 1 ? 'source' : 'sources'; + lines.push(`${g.arrow} incomplete coverage: ${filesUnavailable} indexed ${noun} unavailable`); + } else { + lines.push(`${g.arrow} results capped at limit — use --limit to see more`); + } + } + + if (confidence === 'lexical') { + lines.push(''); + lines.push('Raw-text fallback may match comments or strings.'); } lines.push(''); diff --git a/src/cli-init.ts b/src/cli-init.ts index e9efc76..4641e84 100644 --- a/src/cli-init.ts +++ b/src/cli-init.ts @@ -1,291 +1,597 @@ +import { cliProjectOptions } from './cli-project-options.js'; +import { CodebaseIndexer } from './core/indexer.js'; +import { loadProjectConfig } from './server/config.js'; /** * Interactive setup wizard for codebase-context. - * Handles `codebase-context init` — generates MCP config and instruction block + * Handles `codebase-context init` — detects clients and merges project MCP configs * for the four first-wave AI clients: Claude Code, Cursor, Codex, OpenCode. * - * Depends only on Node built-ins and @inquirer/prompts. - * No imports from the rest of the codebase — keeps this module side-effect-free - * outside the wizard call and easy to unit test. + * Config writes run only after all selected registrations pass preflight. + * File previews share the indexer's selection rules and project configuration. */ import path from 'path'; -import * as fs from 'node:fs/promises'; -import { execFileSync } from 'child_process'; -import { select, confirm } from '@inquirer/prompts'; - -export type Client = 'claude-code' | 'cursor' | 'codex' | 'opencode'; - -export type McpConfigResult = - | { kind: 'file'; path: string; content: string } - | { kind: 'command'; args: string[] }; +import { parse as parseToml } from 'smol-toml'; +import { isDeepStrictEqual } from 'node:util'; +import { promises as fs } from 'node:fs'; +import { atomicWrite } from './utils/atomic-write.js'; +import { checkbox } from '@inquirer/prompts'; +import { homedir } from 'node:os'; +import { constants } from 'node:fs'; + +export type Client = (typeof CLIENTS)[number]['id']; + +export type McpConfigResult = { kind: 'file'; path: string; content: string }; + +export type McpConfigOptions = { + rootPath?: string; + /** Node binary written into the client config. Defaults to the current process. */ + nodeExecutable?: string; + /** + * Package entrypoint written into the client config. + * When init runs through npx this is the file inside the npx cache. + */ + serverEntrypoint?: string; +}; type JsonObject = Record; -export function generateMcpConfig(client: Client): McpConfigResult { - switch (client) { - case 'claude-code': - return { - kind: 'command', - args: ['mcp', 'add', '--transport', 'http', 'codebase-context', 'http://127.0.0.1:3100/mcp'] - }; - case 'cursor': - return { - kind: 'file', - path: '.cursor/mcp.json', - content: JSON.stringify( - { - mcpServers: { - 'codebase-context': { - type: 'http', - url: 'http://127.0.0.1:3100/mcp' - } - } - }, - null, - 2 +export const CLIENTS = [ + { id: 'claude-code', name: 'Claude Code', executable: 'claude', directory: '.claude' }, + { id: 'codex', name: 'Codex', executable: 'codex', directory: '.codex' }, + { id: 'cursor', name: 'Cursor', executable: 'cursor', directory: '.cursor' }, + { id: 'opencode', name: 'OpenCode', executable: 'opencode', directory: '.opencode' } +] as const; + +export async function detectClients( + rootPath: string, + env: NodeJS.ProcessEnv = process.env, + platform: NodeJS.Platform = process.platform +): Promise { + const paths = platform === 'win32' ? path.win32 : path.posix; + const home = env.HOME || env.USERPROFILE || homedir(); + const configHome = env.XDG_CONFIG_HOME || paths.join(home, '.config'); + const searchPath = (env.PATH || env.Path || '').split(platform === 'win32' ? ';' : ':'); + const extensions = platform === 'win32' ? ['', '.exe', '.cmd', '.bat', '.ps1'] : ['']; + const detected: Client[] = []; + for (const client of CLIENTS) { + const userDirectory = + client.id === 'codex' + ? env.CODEX_HOME || paths.join(home, '.codex') + : client.id === 'claude-code' + ? env.CLAUDE_CONFIG_DIR || paths.join(home, '.claude') + : client.id === 'opencode' + ? paths.join(configHome, 'opencode') + : paths.join(home, client.directory); + const directories = [paths.join(rootPath, client.directory), userDirectory]; + const executables = searchPath + .filter(Boolean) + .flatMap((directory) => + extensions.map((extension) => paths.join(directory, client.executable + extension)) + ); + const candidates = [ + ...directories.map((candidate) => ({ path: candidate, directory: true })), + ...executables.map((candidate) => ({ path: candidate, directory: false })) + ]; + for (const candidate of candidates) { + try { + await fs.access( + candidate.path, + candidate.directory || platform === 'win32' ? constants.F_OK : constants.X_OK + ); + const info = await fs.stat(candidate.path); + if (candidate.directory ? !info.isDirectory() : !info.isFile()) continue; + detected.push(client.id); + break; + } catch (error) { + if ( + !isFileNotFound(error) && + !( + typeof error === 'object' && + error !== null && + 'code' in error && + (error.code === 'EACCES' || error.code === 'ENOTDIR') + ) ) - }; - case 'codex': - return { - kind: 'command', - args: ['mcp', 'add', 'codebase-context', 'http://127.0.0.1:3100/mcp'] - }; - case 'opencode': - return { - kind: 'file', - path: 'opencode.json', - content: JSON.stringify( - { + throw error; + } + } + } + return detected; +} + +function resolveServerEntrypoint(explicit?: string): string { + if (explicit) return path.resolve(explicit); + const entrypoint = process.argv[1]; + if (!entrypoint) { + throw new Error('Cannot locate the codebase-context CLI entrypoint.'); + } + return path.resolve(entrypoint); +} + +/** + * Launch command saved for every client: the Node binary that ran init, the + * package entrypoint (the npx cache path when init itself was started with npx), + * and the project folder. Clients then start that file directly. + */ +export function generateMcpConfig(client: Client, options: McpConfigOptions = {}): McpConfigResult { + const rootPath = path.resolve(options.rootPath ?? process.cwd()); + const nodeExecutable = path.resolve(options.nodeExecutable ?? process.execPath); + const serverEntrypoint = resolveServerEntrypoint(options.serverEntrypoint); + const args = [serverEntrypoint, rootPath]; + if (client === 'codex') { + return { + kind: 'file', + path: path.join(rootPath, '.codex', 'config.toml'), + content: + `[mcp_servers.codebase-context]\ncommand = ${JSON.stringify(nodeExecutable)}\n` + + `args = [${args.map((value) => JSON.stringify(value)).join(', ')}]\nstartup_timeout_sec = 120\n` + }; + } + const configPath = + client === 'claude-code' + ? '.mcp.json' + : client === 'cursor' + ? '.cursor/mcp.json' + : 'opencode.json'; + return { + kind: 'file', + path: path.join(rootPath, configPath), + content: JSON.stringify( + client === 'opencode' + ? { $schema: 'https://opencode.ai/config.json', mcp: { + 'codebase-context': { type: 'local', command: [nodeExecutable, ...args] } + } + } + : { + mcpServers: { 'codebase-context': { - type: 'remote', - url: 'http://127.0.0.1:3100/mcp' + ...(client === 'cursor' ? { type: 'stdio' } : {}), + command: nodeExecutable, + args } } }, - null, - 2 - ) - }; - } + null, + 2 + ) + }; } -export function generateInstructionBlock(): string { - return ` -## Codebase Context (MCP) +export function resolveInitRoot( + args: string[], + envRoot: string | undefined, + cwd = process.cwd() +): string { + let explicitRoot: string | undefined; + + for (let index = 0; index < args.length; index++) { + const arg = args[index]; + if (arg === '--root') { + const value = args[index + 1]; + if (!value || value.startsWith('--')) { + throw new Error('Expected a directory path after --root.'); + } + explicitRoot = value; + index++; + } else if (arg?.startsWith('--root=')) { + const value = arg.slice('--root='.length); + if (!value) { + throw new Error('Expected a directory path after --root.'); + } + explicitRoot = value; + } + } -**Start of every task:** Call \`get_memory\` to load team conventions before writing any code. + return path.resolve(explicitRoot ?? (envRoot ? envRoot : cwd)); +} -**Before editing existing code:** Call \`search_codebase\` with \`intent: "edit"\`. If the preflight card says \`ready: false\`, read the listed files before touching anything. +function isJsonObject(value: unknown): value is JsonObject { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} -**Before writing new code:** Call \`get_team_patterns\` to check how the team handles DI, state, testing, and library wrappers — don't introduce a new pattern if one already exists. +function normalizeJson(value: unknown): unknown { + if (Array.isArray(value)) { + return value.map(normalizeJson); + } + if (isJsonObject(value)) { + return Object.fromEntries( + Object.entries(value) + .sort(([left], [right]) => left.localeCompare(right)) + .map(([key, entry]) => [key, normalizeJson(entry)]) + ); + } + return value; +} -**When asked to "remember" or "record" something:** Call \`remember\` immediately, before doing anything else. +function jsonValuesEqual(left: unknown, right: unknown): boolean { + return JSON.stringify(normalizeJson(left)) === JSON.stringify(normalizeJson(right)); +} -**When adding imports that cross module boundaries:** Call \`detect_circular_dependencies\` with the relevant scope after adding the import. - -`; +function isFileNotFound(error: unknown): boolean { + return typeof error === 'object' && error !== null && 'code' in error && error.code === 'ENOENT'; } -export function resolveInstructionFilePath(client: Client, cwd: string): string | null { - switch (client) { - case 'claude-code': - return path.join(cwd, 'CLAUDE.md'); - case 'cursor': - return path.join(cwd, '.cursorrules'); - case 'codex': - return path.join(cwd, 'AGENTS.md'); - case 'opencode': - return null; +async function readExistingConfig(filePath: string): Promise { + try { + return await fs.readFile(filePath, 'utf8'); + } catch (error) { + if (isFileNotFound(error)) return null; + throw error; } } +function isAbsolutePath(value: string): boolean { + return path.posix.isAbsolute(value) || path.win32.isAbsolute(value); +} + +function samePath(left: string, right: string): boolean { + const normalize = (value: string) => value.replace(/\\/g, '/').replace(/\/+$/, '').toLowerCase(); + return normalize(left) === normalize(right); +} + +function isNpxCacheEntrypoint(value: string): boolean { + return /[\\/]node_modules[\\/]codebase-context[\\/]dist[\\/]index\.js$/.test(value); +} + /** - * Appends the instruction block to an existing file, or skips if already present. - * Exported with leading underscore to signal internal/test-only use. + * Recognize only CBC-specific launch shapes written by init. + * A node launch counts when it points at the npx cache copy of this package, + * or at the entrypoint this run is about to save. */ -export async function _appendInstructionBlock(filePath: string): Promise<'written' | 'skipped'> { - let existing = ''; - try { - existing = await fs.readFile(filePath, 'utf8'); - } catch { - // file does not exist — write fresh - await fs.writeFile(filePath, generateInstructionBlock(), 'utf8'); - return 'written'; +function isGeneratedLaunch(command: unknown, args: unknown, entrypoint?: string): boolean { + if ( + typeof command !== 'string' || + !Array.isArray(args) || + !args.every((arg) => typeof arg === 'string') + ) + return false; + const executable = path.basename(command.replace(/\\/g, '/')).toLowerCase(); + if (executable === 'node' || executable === 'node.exe') { + const script = args[0] ?? ''; + return ( + args.length === 2 && + isAbsolutePath(script) && + isAbsolutePath(args[1] ?? '') && + (isNpxCacheEntrypoint(script) || (entrypoint !== undefined && samePath(script, entrypoint))) + ); } + const launch = command === 'npx' || command === 'npx.cmd' ? args : []; + return ( + launch.length === 3 && + (launch[0] === '--yes' || launch[0] === '-y') && + /^codebase-context@\d+\.\d+\.\d+(?:-[\w.-]+)?$/.test(launch[1] ?? '') && + isAbsolutePath(launch[2] ?? '') + ); +} - if (existing.includes('')) { - return 'skipped'; +function generatedEntrypoint(generated: unknown, client: Client): string | undefined { + if (!isJsonObject(generated)) return undefined; + if (client === 'opencode' && Array.isArray(generated.command)) { + return typeof generated.command[1] === 'string' ? generated.command[1] : undefined; } - - await fs.writeFile(filePath, existing + '\n' + generateInstructionBlock(), 'utf8'); - return 'written'; + return Array.isArray(generated.args) && typeof generated.args[0] === 'string' + ? generated.args[0] + : undefined; } -function isJsonObject(value: unknown): value is JsonObject { - return typeof value === 'object' && value !== null && !Array.isArray(value); +function mergeGeneratedServer(existing: unknown, generated: unknown, client: Client): unknown { + if (jsonValuesEqual(existing, generated)) return existing; + if (!isJsonObject(existing) || !isJsonObject(generated) || 'url' in existing) return null; + const local = client === 'opencode'; + const command = local && Array.isArray(existing.command) ? existing.command[0] : existing.command; + const args = local && Array.isArray(existing.command) ? existing.command.slice(1) : existing.args; + if ( + (existing.type !== undefined && existing.type !== (local ? 'local' : 'stdio')) || + !isGeneratedLaunch(command, args, generatedEntrypoint(generated, client)) + ) + return null; + return { + ...existing, + command: generated.command, + ...(!local ? { args: generated.args } : {}), + ...(generated.type ? { type: generated.type } : {}) + }; } /** * Merge generated MCP config into an existing JSON config file when possible. - * Preserves unrelated keys and only updates the codebase-context server entry. + * Preserves unrelated keys and updates only recognized generated launches. */ export async function _buildMergedMcpContent( filePath: string, generatedContent: string, - client: Extract + client: Extract ): Promise<{ content: string; mergedFromExisting: boolean }> { + const existingText = await readExistingConfig(filePath); + if (existingText === null) { + return { content: generatedContent, mergedFromExisting: false }; + } + let existing: unknown; try { - existing = JSON.parse(await fs.readFile(filePath, 'utf8')); - } catch { - return { content: generatedContent, mergedFromExisting: false }; + existing = JSON.parse(existingText) as unknown; + } catch (error) { + throw new Error(`Cannot merge ${client} config at ${filePath}: existing JSON is invalid.`, { + cause: error + }); } const generated = JSON.parse(generatedContent) as unknown; if (!isJsonObject(existing) || !isJsonObject(generated)) { - return { content: generatedContent, mergedFromExisting: false }; + throw new Error(`Cannot merge ${client} config at ${filePath}: expected a JSON object.`); } - if (client === 'cursor') { - const existingServers = isJsonObject(existing.mcpServers) ? existing.mcpServers : {}; - const generatedServers = isJsonObject(generated.mcpServers) ? generated.mcpServers : {}; - return { - content: JSON.stringify( - { - ...existing, - ...generated, - mcpServers: { - ...existingServers, - ...generatedServers - } - }, - null, - 2 - ), - mergedFromExisting: true - }; + const serverKey = client === 'opencode' ? 'mcp' : 'mcpServers'; + const existingServersValue = existing[serverKey]; + if (existingServersValue !== undefined && !isJsonObject(existingServersValue)) { + throw new Error( + `Cannot merge ${client} config at ${filePath}: ${serverKey} must be a JSON object.` + ); } - - const existingMcp = isJsonObject(existing.mcp) ? existing.mcp : {}; - const generatedMcp = isJsonObject(generated.mcp) ? generated.mcp : {}; + const generatedServersValue = generated[serverKey]; + if (!isJsonObject(generatedServersValue)) { + throw new Error(`Invalid generated ${client} MCP config.`); + } + const existingServers = isJsonObject(existingServersValue) ? existingServersValue : {}; + const hasExistingCodebaseContext = Object.prototype.hasOwnProperty.call( + existingServers, + 'codebase-context' + ); + const generatedCodebaseContext = generatedServersValue['codebase-context']; + const mergedServer = hasExistingCodebaseContext + ? mergeGeneratedServer(existingServers['codebase-context'], generatedCodebaseContext, client) + : generatedCodebaseContext; + if (mergedServer === null) { + throw new Error( + `Cannot merge ${client} config at ${filePath}: it already contains a different codebase-context configuration.` + ); + } + const merged = { + ...generated, + ...existing, + [serverKey]: { ...existingServers, 'codebase-context': mergedServer } + }; return { - content: JSON.stringify( - { - ...existing, - ...generated, - mcp: { - ...existingMcp, - ...generatedMcp - } - }, - null, - 2 - ), + content: jsonValuesEqual(existing, merged) ? existingText : JSON.stringify(merged, null, 2), mergedFromExisting: true }; } -export async function handleInitCli(_argv: string[]): Promise { - console.log('\nSet up codebase-context for your AI client\n'); - - const client = await select({ - message: 'Which client?', - choices: [ - { name: 'Claude Code', value: 'claude-code' }, - { name: 'Cursor', value: 'cursor' }, - { name: 'Codex', value: 'codex' }, - { name: 'OpenCode', value: 'opencode' } - ] - }); - - const mcpResult = generateMcpConfig(client); +function normalizeTomlNewlines(value: string): string { + return value.replace(/\r\n/g, '\n'); +} - console.log('\n--- MCP Config Preview ---'); - if (mcpResult.kind === 'file') { - try { - await fs.access(mcpResult.path); - console.log(`Warning: ${mcpResult.path} already exists; existing entries will be preserved.`); - } catch { - // file does not exist - } - console.log(`File: ${mcpResult.path}\n${mcpResult.content}`); - } else { - console.log(`Command to run: ${mcpResult.args[0]} ${mcpResult.args.slice(1).join(' ')}`); +/** Merge only launch values after parsing; candidate byte edits must preserve every other value. */ +export async function _buildMergedCodexContent( + filePath: string, + generatedContent: string +): Promise<{ content: string; mergedFromExisting: boolean }> { + const existing = await readExistingConfig(filePath); + if (existing === null) return { content: generatedContent, mergedFromExisting: false }; + const parse = (text: string) => parseToml(text, { integersAsBigInt: 'asNeeded' }); + const original = parse(existing); + const generated = parse(generatedContent); + const generatedServers = generated.mcp_servers; + const servers = original.mcp_servers; + if (!isJsonObject(generatedServers) || !isJsonObject(generatedServers['codebase-context'])) + throw new Error('Generated Codex configuration has no server entry.'); + const desired = generatedServers['codebase-context']; + const current = isJsonObject(servers) ? servers['codebase-context'] : undefined; + const conflict = () => + new Error( + `Cannot merge Codex config at ${filePath}: it already contains a different codebase-context configuration.` + ); + if (current === undefined) { + const newline = existing.includes('\r\n') ? '\r\n' : '\n'; + const addition = normalizeTomlNewlines(generatedContent).trim().replace(/\n/g, newline); + const content = `${existing}${existing.endsWith(newline) ? newline : newline + newline}${addition}${newline}`; + if (servers === undefined) original.mcp_servers = generatedServers; + else if (isJsonObject(servers)) servers['codebase-context'] = desired; + else throw conflict(); + if (!isDeepStrictEqual(parse(content), original)) throw conflict(); + return { content, mergedFromExisting: true }; } - - const applyMcp = await confirm({ - message: 'Apply MCP config? [y/N]', - default: false - }); - - const instructionPath = resolveInstructionFilePath(client, process.cwd()); - - let applyInstruction = false; - if (instructionPath !== null) { - let fileExists = false; + if (!isJsonObject(current) || !isJsonObject(servers) || 'url' in current) throw conflict(); + if ( + isDeepStrictEqual(current.command, desired.command) && + isDeepStrictEqual(current.args, desired.args) + ) + return { content: existing, mergedFromExisting: true }; + const entrypoint = + Array.isArray(desired.args) && typeof desired.args[0] === 'string' + ? desired.args[0] + : undefined; + if (!isGeneratedLaunch(current.command, current.args, entrypoint)) throw conflict(); + const expected = parse(existing); + const expectedServers = expected.mcp_servers; + if (!isJsonObject(expectedServers) || !isJsonObject(expectedServers['codebase-context'])) + throw conflict(); + expectedServers['codebase-context'].command = desired.command; + expectedServers['codebase-context'].args = desired.args; + const key = (name: string) => `(?:${name}|"${name}"|'${name}')`; + const headers = new RegExp( + `^[ \t]*\\[[ \t]*${key('mcp_servers')}[ \t]*\\.[ \t]*${key('codebase-context')}[ \t]*\\][ \t]*(?:#.*)?$`, + 'gm' + ); + const normalized = normalizeTomlNewlines(existing); + for (const header of normalized.matchAll(headers)) { + const start = header.index; + const bodyStart = start + header[0].length; + const next = /^[ \t]*\[{1,2}[^\r\n]+\]{1,2}[ \t]*(?:#.*)?$/gm; + next.lastIndex = bodyStart; + const end = next.exec(normalized)?.index ?? normalized.length; + const section = normalized.slice(start, end); + const updated = section + .replace( + /^([ \t]*command[ \t]*=[ \t]*)("(?:[^"\\]|\\.)*"|'[^'\r\n]*')([ \t]*(?:#.*)?)$/m, + (_line, prefix: string, _value: string, comment: string) => + prefix + JSON.stringify(desired.command) + comment + ) + .replace( + /^([ \t]*args[ \t]*=[ \t]*)(\[[^\r\n]*\])([ \t]*(?:#.*)?)$/m, + (_line, prefix: string, _value: string, comment: string) => + prefix + + '[' + + (Array.isArray(desired.args) + ? desired.args.map((arg) => JSON.stringify(arg)).join(', ') + : '') + + ']' + + comment + ); + const candidate = normalized.slice(0, start) + updated + normalized.slice(end); try { - await fs.access(instructionPath); - fileExists = true; + if (isDeepStrictEqual(parse(candidate), expected)) + return { + content: existing.includes('\r\n') ? candidate.replace(/\n/g, '\r\n') : candidate, + mergedFromExisting: true + }; } catch { - // does not exist + // A textual match inside a string or an unsupported launch shape is never published. } + } + throw conflict(); +} - if (fileExists) { - console.log(`\n--- Instruction Block Preview (will append to ${instructionPath}) ---`); - } else { - console.log(`\n--- Instruction Block Preview (will create ${instructionPath}) ---`); - } - console.log(generateInstructionBlock()); +type InitOptions = { + clients: Client[]; + rootPath: string; + yes: boolean; + help: boolean; + preview: boolean; +}; + +function parseClient(value: string): Client { + const client = CLIENTS.find((client) => client.id === value); + if (client) return client.id; + throw new Error( + `Unknown client "${value}". Choose ${CLIENTS.map((client) => client.id).join(', ')}.` + ); +} - applyInstruction = await confirm({ - message: 'Write instruction block? [y/N]', - default: false - }); +function parseInitOptions(args: string[]): InitOptions { + if (args.includes('--help') || args.includes('-h')) + return { clients: [], rootPath: process.cwd(), yes: false, help: true, preview: false }; + const clients: Client[] = []; + let yes = false; + let preview = false; + for (let index = 0; index < args.length; index++) { + const arg = args[index]; + if (arg === '--preview') preview = true; + else if (arg === '--yes' || arg === '-y') yes = true; + else if (arg === '--client') { + const value = args[++index]; + if (!value || value.startsWith('--')) throw new Error('Expected a client after --client.'); + clients.push(parseClient(value)); + } else if (arg?.startsWith('--client=')) + clients.push(parseClient(arg.slice('--client='.length))); + else if (arg === '--root') index++; + else if (!arg?.startsWith('--root=')) throw new Error(`Unknown init option "${arg}".`); } + return { + clients: [...new Set(clients)], + rootPath: resolveInitRoot(args, process.env.CODEBASE_ROOT), + yes, + help: false, + preview + }; +} - // Execute confirmed actions +function printInitUsage(): void { + console.log('codebase-context init [options]'); + console.log( + 'Detect and configure coding tools together. With a terminal, Space toggles and Enter applies.' + ); + console.log( + 'Each config saves this Node binary and the package file from the npx cache, plus the project folder.' + ); + console.log('Codex setup line: codex mcp add codebase-context -- npx -y codebase-context'); + console.log( + ' --client Choose a tool (repeatable): ' + + CLIENTS.map((client) => client.id).join(', ') + ); + console.log(' --root Project root (default: CODEBASE_ROOT or cwd)'); + console.log(' --preview Show selected files and exclusions without writing config'); + console.log(' --yes, -y Configure every detected tool without prompts'); + console.log(' --help, -h Show this help'); +} - if (applyMcp) { - if (mcpResult.kind === 'file') { - const dir = path.dirname(mcpResult.path); - if (dir && dir !== '.') { - await fs.mkdir(dir, { recursive: true }); - } - const mergedConfig = - client === 'cursor' || client === 'opencode' - ? await _buildMergedMcpContent(mcpResult.path, mcpResult.content, client) - : { content: mcpResult.content, mergedFromExisting: false }; - await fs.writeFile(mcpResult.path, mergedConfig.content, 'utf8'); - if (mergedConfig.mergedFromExisting) { - console.log(`Merged: ${mcpResult.path} (existing entries preserved)`); - } - console.log(`Written: ${mcpResult.path}`); - } else { - const [cmd, ...rest] = mcpResult.args; - try { - execFileSync(cmd, rest, { stdio: 'inherit' }); - } catch { - console.log( - `Could not run '${cmd}' automatically. Run it yourself:\n ${cmd} ${rest.join(' ')}` - ); +export async function handleInitCli(args: string[]): Promise { + try { + const options = parseInitOptions(args); + if (options.help) { + printInitUsage(); + return; + } + if (!(await fs.stat(options.rootPath)).isDirectory()) + throw new Error(`Project root is not a directory: ${options.rootPath}`); + if (options.preview) { + const config = await loadProjectConfig(options.rootPath); + const preview = await new CodebaseIndexer({ + rootPath: options.rootPath, + ...cliProjectOptions(config, true) + }).previewFiles(); + console.log(JSON.stringify(preview, null, 2)); + return; + } + console.log(`Set up Codebase Context in ${options.rootPath}`); + let clients = options.clients; + if (!clients.length) { + const detected = await detectClients(options.rootPath); + clients = + !options.yes && process.stdin.isTTY && process.stdout.isTTY + ? await checkbox({ + message: 'Choose tools (Space toggles, Enter sets up):', + choices: CLIENTS.map((client) => ({ + name: client.name + (detected.includes(client.id) ? ' (detected)' : ''), + value: client.id, + checked: detected.includes(client.id) + })) + }) + : detected; + } + if (!clients.length) + throw new Error( + 'No coding tools selected or detected. Choose with --client claude-code, --client codex, --client cursor, or --client opencode.' + ); + + const prepared = []; + for (const client of clients) { + const config = generateMcpConfig(client, { rootPath: options.rootPath }); + const merged = + client === 'codex' + ? await _buildMergedCodexContent(config.path, config.content) + : await _buildMergedMcpContent(config.path, config.content, client); + prepared.push({ + ...config, + content: merged.content, + unchanged: (await readExistingConfig(config.path)) === merged.content + }); + } + for (const config of prepared) { + if (!config.unchanged) { + await fs.mkdir(path.dirname(config.path), { recursive: true }); + await atomicWrite(config.path, config.content); } + console.log( + `${config.unchanged ? 'Unchanged' : 'Configured'}: ${path.relative(options.rootPath, config.path)}` + ); } - } - - if (applyInstruction && instructionPath !== null) { - const status = await _appendInstructionBlock(instructionPath); - if (status === 'skipped') { - console.log(`Instruction block already present in ${instructionPath}, skipping.`); - } else { - console.log(`Written: ${instructionPath}`); + console.log('Reopen your tools; approve the server or project trust if asked.'); + console.log("The first search builds this repository's index."); + } catch (error) { + if ( + error instanceof Error && + (error.name === 'ExitPromptError' || + error.name === 'AbortPromptError' || + error.name === 'CancelPromptError') + ) { + console.log('Setup cancelled; no files written.'); + return; } + console.error(`Init failed: ${error instanceof Error ? error.message : String(error)}`); + process.exitCode = 1; } - - console.log( - '\nNext steps:\n' + - ' Run `npx codebase-context map` to see your codebase conventions\n' + - ' Start the HTTP server: npx codebase-context --http\n' - ); } diff --git a/src/cli-map.ts b/src/cli-map.ts index 4479afe..caab63f 100644 --- a/src/cli-map.ts +++ b/src/cli-map.ts @@ -1,3 +1,4 @@ +import { resolveIndexArtifact } from './core/index-generation.js'; /** * CLI handler for `codebase-context map`. * @@ -11,16 +12,15 @@ import path from 'path'; import { promises as fs } from 'fs'; -import { - CODEBASE_CONTEXT_DIRNAME, - INTELLIGENCE_FILENAME, - KEYWORD_INDEX_FILENAME, - VECTOR_DB_DIRNAME, - MEMORY_FILENAME -} from './constants/codebase-context.js'; +import { KEYWORD_INDEX_FILENAME } from './constants/codebase-context.js'; import { createProjectState } from './project-state.js'; import { buildCodebaseMap, renderMapMarkdown, renderMapPretty } from './core/codebase-map.js'; import type { IndexState } from './tools/types.js'; +import { CodebaseIndexer } from './core/indexer.js'; +import { loadProjectConfig } from './server/config.js'; +import { cliProjectOptions } from './cli-project-options.js'; +import { inspectIndexDiskState, permitsFirstBuild } from './core/index-generation.js'; +import { IndexCorruptedError } from './errors/index.js'; function resolveMapRootPath(): string { return path.resolve(process.env.CODEBASE_ROOT ?? process.cwd()); @@ -56,21 +56,25 @@ export async function handleMapCli(args: string[]): Promise { const rootPath = resolveMapRootPath(); // Build a minimal project state — same pattern as the MCP resource path. - const baseDir = path.join(rootPath, CODEBASE_CONTEXT_DIRNAME); - const paths = { - baseDir, - memory: path.join(baseDir, MEMORY_FILENAME), - intelligence: path.join(baseDir, INTELLIGENCE_FILENAME), - keywordIndex: path.join(baseDir, KEYWORD_INDEX_FILENAME), - vectorDb: path.join(baseDir, VECTOR_DB_DIRNAME) - }; - let indexExists = false; try { - await fs.access(paths.keywordIndex); + await fs.access(await resolveIndexArtifact(rootPath, KEYWORD_INDEX_FILENAME)); + indexExists = true; + } catch (error) { + if (!(error instanceof Error && 'code' in error && error.code === 'ENOENT')) throw error; + if (!permitsFirstBuild(await inspectIndexDiskState(rootPath))) + throw new IndexCorruptedError( + 'The keyword index is missing from an existing index. Run reindex explicitly before building the map.' + ); + console.error(`Indexing ${rootPath} before building the map.`); + const config = await loadProjectConfig(rootPath); + await new CodebaseIndexer({ + rootPath, + firstBuildOnly: true, + ...cliProjectOptions(config) + }).index(); + console.error('Index ready for the map.'); indexExists = true; - } catch { - // no index yet } const indexState: IndexState = { status: indexExists ? 'ready' : 'idle' }; diff --git a/src/cli-project-options.ts b/src/cli-project-options.ts new file mode 100644 index 0000000..3b42cf3 --- /dev/null +++ b/src/cli-project-options.ts @@ -0,0 +1,21 @@ +import { EXCLUDED_GLOB_PATTERNS } from './constants/codebase-context.js'; +import type { IndexerOptions } from './core/indexer.js'; +import type { loadProjectConfig } from './server/config.js'; + +/** Adapt persisted project settings without deciding an unspecified embedding mode. */ +export function cliProjectOptions( + project: Awaited>, + skipEmbedding = project?.skipEmbedding +): Pick { + return { + config: { + exclude: [...EXCLUDED_GLOB_PATTERNS, ...(project?.excludePatterns ?? [])], + ...(project?.parsing ? { parsing: project.parsing } : {}), + ...(skipEmbedding !== undefined ? { skipEmbedding } : {}) + }, + projectOptions: { + preferredAnalyzer: project?.analyzerHints?.analyzer, + extraFileExtensions: project?.analyzerHints?.extensions + } + }; +} diff --git a/src/cli.ts b/src/cli.ts index c7a0ed4..345eaec 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -1,3 +1,4 @@ +import { resolveIndexArtifact } from './core/index-generation.js'; /** * CLI subcommands for codebase-context. * Memory list/add/remove — vendor-neutral access without any AI agent. @@ -6,14 +7,8 @@ import path from 'path'; import { promises as fs } from 'fs'; -import { - CODEBASE_CONTEXT_DIRNAME, - MEMORY_FILENAME, - INTELLIGENCE_FILENAME, - HEALTH_FILENAME, - KEYWORD_INDEX_FILENAME, - VECTOR_DB_DIRNAME -} from './constants/codebase-context.js'; +import { KEYWORD_INDEX_FILENAME } from './constants/codebase-context.js'; +import { makePaths } from './project-state.js'; import { CodebaseIndexer } from './core/indexer.js'; import { dispatchTool } from './tools/index.js'; import type { ToolContext } from './tools/index.js'; @@ -30,6 +25,7 @@ export { handleMemoryCli } from './cli-memory.js'; import { handleInitCli } from './cli-init.js'; import { handleMapCli } from './cli-map.js'; import { loadProjectConfig } from './server/config.js'; +import { cliProjectOptions } from './cli-project-options.js'; analyzerRegistry.register(new AngularAnalyzer()); analyzerRegistry.register(new NextJsAnalyzer()); @@ -82,17 +78,21 @@ function printUsage(): void { console.log(' [--limit ] [--lang ] [--framework ] [--layer ]'); console.log(' metadata Project structure, frameworks, deps'); console.log(' status Index state and progress'); - console.log(' reindex [--incremental] [--reason ] Re-index the codebase'); + console.log(' reindex [--incremental] [--embeddings|--no-embeddings] Re-index the codebase'); console.log(' style-guide [--query ] [--category ] Style guide rules'); console.log(' patterns [--category all|di|state|testing|libraries] Team patterns'); console.log(' refs --symbol [--limit ] Symbol references'); console.log(' cycles [--scope ] Circular dependency detection'); - console.log(' init Interactive setup wizard for AI clients'); - console.log(' map [--json] [--pretty] Conventions map'); + console.log(' init [--client ] [--yes] Detect and configure coding tools'); + console.log(' map [--json] [--pretty] Conventions map (indexes first if needed)'); console.log(''); console.log('Global flags:'); - console.log(' --json Output raw JSON (default: human-readable)'); - console.log(' --help Show this help'); + console.log(' --json Output raw JSON (default: human-readable)'); + console.log(' --version Print the package version'); + console.log(' --help Show this help'); + console.log(''); + console.log('Codex setup line:'); + console.log(' codex mcp add codebase-context -- npx -y codebase-context'); console.log(''); console.log('Environment:'); console.log(' CODEBASE_ROOT Project root path (default: cwd for CLI only)'); @@ -108,18 +108,11 @@ async function initToolContext(): Promise { const rootPath = resolveCliRootPath(); const projectConfig = await loadProjectConfig(rootPath); - const paths = { - baseDir: path.join(rootPath, CODEBASE_CONTEXT_DIRNAME), - memory: path.join(rootPath, CODEBASE_CONTEXT_DIRNAME, MEMORY_FILENAME), - intelligence: path.join(rootPath, CODEBASE_CONTEXT_DIRNAME, INTELLIGENCE_FILENAME), - health: path.join(rootPath, CODEBASE_CONTEXT_DIRNAME, HEALTH_FILENAME), - keywordIndex: path.join(rootPath, CODEBASE_CONTEXT_DIRNAME, KEYWORD_INDEX_FILENAME), - vectorDb: path.join(rootPath, CODEBASE_CONTEXT_DIRNAME, VECTOR_DB_DIRNAME) - }; + const paths = makePaths(rootPath); let indexExists = false; try { - await fs.access(paths.keywordIndex); + await fs.access(await resolveIndexArtifact(rootPath, KEYWORD_INDEX_FILENAME)); indexExists = true; } catch { // no index on disk @@ -129,7 +122,11 @@ async function initToolContext(): Promise { status: indexExists ? 'ready' : 'idle' }; - const performIndexing = async (incrementalOnly?: boolean, reason?: string): Promise => { + const performIndexing = async ( + incrementalOnly?: boolean, + reason?: string, + skipEmbedding?: boolean + ): Promise => { indexState.status = 'indexing'; const mode = incrementalOnly ? 'incremental' : 'full'; console.error(`Indexing (${mode})${reason ? ` — ${reason}` : ''}: ${rootPath}`); @@ -138,7 +135,7 @@ async function initToolContext(): Promise { let lastLoggedProgress = { phase: '', percentage: -1 }; const indexer = new CodebaseIndexer({ rootPath, - ...(projectConfig?.parsing ? { config: { parsing: projectConfig.parsing } } : {}), + ...cliProjectOptions(projectConfig, skipEmbedding ?? projectConfig?.skipEmbedding), incrementalOnly, onProgress: (progress) => { const shouldLog = @@ -167,6 +164,7 @@ async function initToolContext(): Promise { indexState.status = 'error'; indexState.error = error instanceof Error ? error.message : String(error); console.error('Indexing failed:', indexState.error); + throw error; } }; @@ -256,7 +254,7 @@ function booleanFlag(flags: Flags, key: string, usage: string): boolean { export async function handleCliCommand(argv: string[]): Promise { const rawCommand = argv[0]; - if (!rawCommand || rawCommand === '--help') { + if (!rawCommand || rawCommand === '--help' || rawCommand === 'help') { printUsage(); return; } @@ -282,6 +280,11 @@ export async function handleCliCommand(argv: string[]): Promise { return handleMapCli(argv.slice(1)); } + if (argv.slice(1).includes('--help')) { + printUsage(); + return; + } + const useJson = argv.includes('--json'); const flags = parseFlags(argv); @@ -359,10 +362,23 @@ export async function handleCliCommand(argv: string[]): Promise { break; } case 'reindex': { - const usage = 'codebase-context reindex [--incremental] [--reason ]'; + const usage = + 'codebase-context reindex [--incremental] [--reason ] [--embeddings|--no-embeddings]'; const reason = optionalStringFlag(flags, 'reason', usage); const incremental = booleanFlag(flags, 'incremental', usage); - await ctx.performIndexing(incremental, reason); + if ('embeddings' in flags && 'no-embeddings' in flags) { + exitWithError( + `Error: --embeddings and --no-embeddings are mutually exclusive\nUsage: ${usage}` + ); + } + for (const modeFlag of ['embeddings', 'no-embeddings']) { + if (modeFlag in flags && flags[modeFlag] !== true) { + exitWithError(`Error: --${modeFlag} takes no value\nUsage: ${usage}`); + } + } + const skipEmbedding = + 'no-embeddings' in flags ? true : 'embeddings' in flags ? false : undefined; + await ctx.performIndexing(incremental, reason, skipEmbedding); const statusResult = await dispatchTool('get_indexing_status', {}, ctx); formatJson(extractText(statusResult), useJson, 'status', ctx.rootPath); return; diff --git a/src/constants/codebase-context.ts b/src/constants/codebase-context.ts index 473748f..e52463d 100644 --- a/src/constants/codebase-context.ts +++ b/src/constants/codebase-context.ts @@ -11,7 +11,7 @@ export const CODEBASE_CONTEXT_DIRNAME = '.codebase-context' as const; * - Embedding input string changes (e.g. scope-prefix content, prepended metadata) * - Required persisted fields/artifact headers change */ -export const INDEX_FORMAT_VERSION = 1 as const; +export const INDEX_FORMAT_VERSION = 5 as const; /** Schema version for `.codebase-context/index-meta.json` itself. */ export const INDEX_META_VERSION = 1 as const; diff --git a/src/core/analyzer-registry.ts b/src/core/analyzer-registry.ts index 6f4df74..d857c2e 100644 --- a/src/core/analyzer-registry.ts +++ b/src/core/analyzer-registry.ts @@ -4,9 +4,9 @@ */ import path from 'path'; -import { FrameworkAnalyzer, AnalysisResult } from '../types/index.js'; +import { FrameworkAnalyzer, AnalysisResult, AnalysisOptions } from '../types/index.js'; -export interface AnalyzerSelectionOptions { +export interface AnalyzerSelectionOptions extends AnalysisOptions { preferredAnalyzer?: string; extraFileExtensions?: string[]; } @@ -118,7 +118,7 @@ export class AnalyzerRegistry { } try { - return await analyzer.analyze(filePath, content); + return await analyzer.analyze(filePath, content, options); } catch (error) { console.error(`Error analyzing ${filePath} with ${analyzer.name}:`, error); return null; diff --git a/src/core/codebase-map.ts b/src/core/codebase-map.ts index 61a4495..87b491f 100644 --- a/src/core/codebase-map.ts +++ b/src/core/codebase-map.ts @@ -1,3 +1,5 @@ +import { withAvailableIndexSnapshot } from './index-generation.js'; +import { resolveProjectPaths } from '../project-state.js'; /** * Codebase map builder and renderers. * @@ -97,6 +99,18 @@ const MAP_EXCLUDED_DIRECTORY_NAMES = new Set( export async function buildCodebaseMap( project: ProjectState, options: BuildCodebaseMapOptions = {} +): Promise { + return withAvailableIndexSnapshot(project.rootPath, async () => + buildPinnedCodebaseMap( + { ...project, paths: await resolveProjectPaths(project.rootPath) }, + options + ) + ); +} + +async function buildPinnedCodebaseMap( + project: ProjectState, + options: BuildCodebaseMapOptions ): Promise { const mode = options.mode ?? 'bounded'; const projectName = path.basename(project.rootPath); @@ -234,8 +248,20 @@ export async function buildCodebaseMap( })); // --- Graph stats --- + // Prefer recorded stats. When they are missing or say 0 files but the index + // already has files, count those files so the map does not report an empty project. + const countedFiles = new Set([ + ...Object.keys(graph.imports ?? {}), + ...Object.keys(graph.importedBy ?? {}), + ...chunks.flatMap((chunk) => + typeof chunk.relativePath === 'string' && chunk.relativePath.length > 0 + ? [chunk.relativePath] + : [] + ) + ]).size; + const recordedFiles = statsSource.files ?? 0; const graphStats = { - files: statsSource.files ?? 0, + files: recordedFiles > 0 ? recordedFiles : countedFiles, edges: statsSource.edges ?? 0, avgDependencies: statsSource.avgDependencies ?? 0 }; diff --git a/src/core/file-watcher.ts b/src/core/file-watcher.ts index cf5542e..8b63cb0 100644 --- a/src/core/file-watcher.ts +++ b/src/core/file-watcher.ts @@ -1,6 +1,6 @@ import chokidar from 'chokidar'; import path from 'path'; -import { EXCLUDED_GLOB_PATTERNS } from '../constants/codebase-context.js'; +import { EXCLUDED_DIRECTORY_NAMES } from '../constants/codebase-context.js'; import { getSupportedExtensions } from '../utils/language-detection.js'; export interface FileWatcherOptions { @@ -27,16 +27,19 @@ function isTrackedSourcePath(filePath: string, trackedExtensions: Set): /** * Watch rootPath for source file changes and call onChanged (debounced). - * Returns a stop() function that cancels the debounce timer and closes the watcher. + * Returns a stop() function that cancels pending work and resolves when the watcher closes. */ -export function startFileWatcher(opts: FileWatcherOptions): () => void { +export function startFileWatcher(opts: FileWatcherOptions): () => Promise { const { rootPath, debounceMs = 2000, extraExtensions, onReady, onChanged } = opts; const trackedExtensions = new Set( getSupportedExtensions(extraExtensions).map((extension) => extension.toLowerCase()) ); let debounceTimer: ReturnType | undefined; + let stopped = false; + let closePromise: Promise | undefined; const trigger = (filePath: string) => { + if (stopped) return; if (!isTrackedSourcePath(filePath, trackedExtensions)) return; if (debounceTimer !== undefined) clearTimeout(debounceTimer); debounceTimer = setTimeout(() => { @@ -46,21 +49,39 @@ export function startFileWatcher(opts: FileWatcherOptions): () => void { }; const watcher = chokidar.watch(rootPath, { - ignored: [...EXCLUDED_GLOB_PATTERNS], + ignored: (filePath) => { + const relative = path.relative(rootPath, filePath); + if (!relative) return false; + if (path.isAbsolute(relative) || relative === '..' || relative.startsWith(`..${path.sep}`)) + return true; + const pathParts = relative.split(/[\\/]+/); + return EXCLUDED_DIRECTORY_NAMES.some((directoryName) => pathParts.includes(directoryName)); + }, persistent: true, ignoreInitial: true, awaitWriteFinish: { stabilityThreshold: 200, pollInterval: 100 } }); watcher - .on('ready', () => onReady?.()) + .on('ready', () => { + if (!stopped) onReady?.(); + }) .on('add', trigger) .on('change', trigger) .on('unlink', trigger) .on('error', (err: unknown) => console.error('[file-watcher] error:', err)); return () => { + if (closePromise) return closePromise; + stopped = true; if (debounceTimer !== undefined) clearTimeout(debounceTimer); - void watcher.close(); + debounceTimer = undefined; + try { + closePromise = watcher.close(); + } catch (error) { + closePromise = Promise.reject(error); + void closePromise.catch(() => undefined); + } + return closePromise; }; } diff --git a/src/core/index-generation.ts b/src/core/index-generation.ts new file mode 100644 index 0000000..af956d8 --- /dev/null +++ b/src/core/index-generation.ts @@ -0,0 +1,871 @@ +import { AsyncLocalStorage } from 'node:async_hooks'; +import { randomUUID } from 'node:crypto'; +import { promises as fs } from 'node:fs'; +import path from 'node:path'; +import { setTimeout as delay } from 'node:timers/promises'; +import { z } from 'zod'; +import { atomicWrite } from '../utils/atomic-write.js'; +import { + CODEBASE_CONTEXT_DIRNAME, + INDEX_META_FILENAME, + KEYWORD_INDEX_FILENAME, + VECTOR_DB_DIRNAME, + INTELLIGENCE_FILENAME, + MANIFEST_FILENAME, + INDEXING_STATS_FILENAME, + RELATIONSHIPS_FILENAME, + HEALTH_FILENAME, + INDEX_FORMAT_VERSION, + INDEX_META_VERSION, + MEMORY_FILENAME +} from '../constants/codebase-context.js'; +import { IndexCorruptedError } from '../errors/index.js'; +import { + IndexMetaSchema, + readIndexMetaFromFile, + validateIndexArtifacts, + type IndexMeta +} from './index-meta.js'; + +export const MAX_INDEX_GENERATIONS = 3; +const GENERATIONS = '.generations'; +const OwnerSchema = z.object({ pid: z.number().int().positive() }); +const TicketSchema = OwnerSchema.extend({ + choosing: z.boolean(), + ticket: z.number().int().nonnegative() +}); +const PinSchema = OwnerSchema.extend({ + buildId: z.string().min(1), + released: z.boolean().default(false) +}); +type Snapshot = { + kind: 'committed'; + root: string; + realRoot: string; + meta: IndexMeta; + directory: string; +}; +type UnavailableSnapshot = { kind: 'legacy' | 'unavailable'; root: string; directory: string }; +type IndexBoundary = Snapshot | UnavailableSnapshot; +const snapshots = new AsyncLocalStorage(); +const pendingPinReleases = new Map(); +const writers = new AsyncLocalStorage(); +const rootPaths = new AsyncLocalStorage<{ root: string; realRoot: string }>(); + +async function withResolvedRoot(root: string, operation: () => Promise): Promise { + const current = rootPaths.getStore(); + if (current && (current.root === root || current.realRoot === root)) return operation(); + return rootPaths.run({ root, realRoot: await fs.realpath(root) }, operation); +} + +function rootKey(rootPath: string): string { + const root = path.resolve(rootPath); + const current = rootPaths.getStore(); + return current && (current.root === root || current.realRoot === root) ? current.realRoot : root; +} + +export class IndexCapacityError extends Error {} + +const GENERATION_ID = /^[a-f0-9]{8}-[a-f0-9]{4}-4[a-f0-9]{3}-[89ab][a-f0-9]{3}-[a-f0-9]{12}$/i; +type RecordedMode = 'keyword' | 'semantic'; +export type IndexDiskState = + | { kind: 'fresh' | 'incomplete-first-build' } + | { kind: 'committed'; mode: RecordedMode } + | { kind: 'legacy'; mode: RecordedMode | 'unknown' } + | { kind: 'recoverable-unpublished'; generations: string[] } + | { kind: 'unknown'; reason: string }; +type GenerationEvidence = { kind: 'complete' | 'incomplete' } | { kind: 'unknown'; reason: string }; +const DERIVED_FILES: readonly string[] = [ + KEYWORD_INDEX_FILENAME, + VECTOR_DB_DIRNAME, + INTELLIGENCE_FILENAME, + MANIFEST_FILENAME, + INDEXING_STATS_FILENAME, + RELATIONSHIPS_FILENAME, + HEALTH_FILENAME +]; + +function bindGenerationMeta(meta: IndexMeta): IndexMeta { + const prefix = `${GENERATIONS}/${meta.buildId}/`; + return IndexMetaSchema.parse({ + ...meta, + artifacts: Object.fromEntries( + Object.entries(meta.artifacts).map(([key, value]) => { + const artifact = z.object({ path: z.string() }).passthrough().parse(value); + return [key, { ...artifact, path: `${prefix}${artifact.path}` }]; + }) + ) + }); +} + +async function optionalJson( + root: string, + relative: string +): Promise<{ kind: 'missing' } | { kind: 'value'; value: unknown } | { kind: 'malformed' }> { + let raw: string; + try { + raw = await fs.readFile(await confinedPath(root, relative), 'utf8'); + } catch (error) { + if (isMissing(error)) return { kind: 'missing' }; + throw error; + } + try { + return { kind: 'value', value: JSON.parse(raw) }; + } catch (error) { + if (error instanceof SyntaxError) return { kind: 'malformed' }; + throw error; + } +} + +/** Admission-held: no lock-taking public resolver or provider is initialized here. */ +async function inspectGeneration(root: string, name: string): Promise { + if (!GENERATION_ID.test(name)) + return { kind: 'unknown', reason: 'Unrecognized generation entry.' }; + const relative = `${GENERATIONS}/${name}`; + try { + const directory = await confinedPath(root, relative); + if (!(await fs.stat(directory)).isDirectory()) + return { kind: 'unknown', reason: 'Generation is not a directory.' }; + await validateIndexArtifactTree(root, directory); + const entries = await fs.readdir(directory); + if (entries.some((entry) => ![...DERIVED_FILES, INDEX_META_FILENAME].includes(entry))) + return { kind: 'unknown', reason: 'Generation contains unknown data.' }; + const local = await optionalJson(root, `${relative}/${INDEX_META_FILENAME}`); + if (local.kind !== 'missing') { + const parsed = local.kind === 'value' ? IndexMetaSchema.safeParse(local.value) : undefined; + if ( + !parsed?.success || + parsed.data.buildId !== name || + parsed.data.metaVersion > INDEX_META_VERSION || + parsed.data.formatVersion > INDEX_FORMAT_VERSION + ) + return { kind: 'unknown', reason: 'Generation completion metadata is untrusted.' }; + // Local completion metadata must name this directory's fixed artifact set. + for (const filename of DERIVED_FILES) artifactRelative(parsed.data, filename); + if (parsed.data.artifacts.keywordIndex.path !== KEYWORD_INDEX_FILENAME) + return { kind: 'unknown', reason: 'Generation metadata is not local.' }; + for (const filename of DERIVED_FILES) + await fs.access(await confinedPath(root, `${relative}/${filename}`)); + await validateIndexArtifacts(root, bindGenerationMeta(parsed.data)); + return { kind: 'complete' }; + } + const writeOrder = [ + VECTOR_DB_DIRNAME, + KEYWORD_INDEX_FILENAME, + INTELLIGENCE_FILENAME, + RELATIONSHIPS_FILENAME, + HEALTH_FILENAME, + MANIFEST_FILENAME, + INDEXING_STATS_FILENAME + ]; + if (writeOrder.slice(0, entries.length).some((filename) => !entries.includes(filename))) + return { kind: 'unknown', reason: 'Artifacts do not form a writer-produced prefix.' }; + // A closed writer-produced prefix is recoverable as a first build only when + // it contains no native payload and every completed JSON write is coherent. + for (const filename of entries) { + if (filename === VECTOR_DB_DIRNAME) { + const native = await fs.readdir(await confinedPath(root, `${relative}/${filename}`)); + if (native.some((entry) => entry !== 'index-build.json')) + return { + kind: 'unknown', + reason: 'Native vectors exist without trusted completion metadata.' + }; + if (entries.length > 1 && !native.includes('index-build.json')) + return { kind: 'unknown', reason: 'Artifacts precede their vector build marker.' }; + for (const entry of native) { + const marker = await optionalJson(root, `${relative}/${filename}/${entry}`); + const parsed = + marker.kind === 'value' + ? z + .object({ + buildId: z.literal(name), + formatVersion: z.literal(INDEX_FORMAT_VERSION) + }) + .safeParse(marker.value) + : undefined; + if (!parsed?.success) + return { kind: 'unknown', reason: 'Incomplete vector marker is untrusted.' }; + } + } else { + const json = await optionalJson(root, `${relative}/${filename}`); + const schema = + filename === INDEXING_STATS_FILENAME + ? z.object({ + indexedFiles: z.number().int().nonnegative(), + totalChunks: z.number().int().nonnegative(), + totalFiles: z.number().int().nonnegative(), + generatedAt: z.string().datetime() + }) + : z.object({ + header: z.object({ + buildId: z.literal(name), + formatVersion: z.literal(INDEX_FORMAT_VERSION) + }) + }); + if (json.kind !== 'value' || !schema.safeParse(json.value).success) + return { kind: 'unknown', reason: 'Incomplete artifact is untrusted.' }; + } + } + return { kind: 'incomplete' }; + } catch (error) { + if (error instanceof IndexCorruptedError || isMissing(error)) + return { kind: 'unknown', reason: 'Generation integrity cannot be established.' }; + throw error; + } +} + +async function inspectIndexDiskStateHeld(root: string): Promise { + const pointer = await optionalJson(root, INDEX_META_FILENAME); + if (pointer.kind !== 'missing') { + const parsed = pointer.kind === 'value' ? IndexMetaSchema.safeParse(pointer.value) : undefined; + if ( + !parsed?.success || + parsed.data.metaVersion > INDEX_META_VERSION || + parsed.data.formatVersion > INDEX_FORMAT_VERSION + ) + return { kind: 'unknown', reason: 'Root index metadata is malformed or unsupported.' }; + const meta = parsed.data; + const mode = meta.artifacts.vectorDb.embeddingEnabled === false ? 'keyword' : 'semantic'; + if ( + meta.artifacts.keywordIndex.path === KEYWORD_INDEX_FILENAME || + meta.formatVersion < INDEX_FORMAT_VERSION + ) + return { kind: 'legacy', mode }; + for (const filename of DERIVED_FILES) + await confinedPath(root, artifactRelative(meta, filename)); + return { kind: 'committed', mode }; + } + const context = await contextDirectory(root); + const entries = await fs.readdir(context); + if ( + entries.some( + (entry) => + ![ + ...DERIVED_FILES, + MEMORY_FILENAME, + '.writers', + '.publication', + '.pins', + GENERATIONS + ].includes(entry) + ) + ) + return { kind: 'unknown', reason: 'Index directory contains unrecognized data.' }; + if (entries.some((entry) => DERIVED_FILES.includes(entry))) + return { kind: 'legacy', mode: 'unknown' }; + const directory = await confinedPath(root, GENERATIONS); + let names: string[]; + try { + names = await fs.readdir(directory); + } catch (error) { + if (isMissing(error)) return { kind: 'fresh' }; + throw error; + } + const completed: string[] = []; + for (const name of names) { + const evidence = await inspectGeneration(root, name); + if (evidence.kind === 'unknown') return { kind: 'unknown', reason: evidence.reason }; + if (evidence.kind === 'complete') completed.push(name); + } + if (completed.length) return { kind: 'recoverable-unpublished', generations: completed }; + return { kind: names.length ? 'incomplete-first-build' : 'fresh' }; +} + +export async function inspectIndexDiskState(rootPath: string): Promise { + const root = path.resolve(rootPath); + return withResolvedRoot(root, () => + withLease(rootKey(root), '.publication', () => inspectIndexDiskStateHeld(rootKey(root))) + ); +} + +/** Only these states authorize automatic first-build indexing. */ +export function permitsFirstBuild(state: IndexDiskState): boolean { + return state.kind === 'fresh' || state.kind === 'incomplete-first-build'; +} + +function isMissing(error: unknown): boolean { + return error instanceof Error && 'code' in error && error.code === 'ENOENT'; +} + +function isAlive(pid: number): boolean { + try { + process.kill(pid, 0); + return true; + } catch (error) { + // EPERM cannot establish that the owner is dead. + return !(error instanceof Error && 'code' in error && error.code === 'ESRCH'); + } +} + +function contained(base: string, target: string): boolean { + const relative = path.relative(base, target); + return !path.isAbsolute(relative) && relative !== '..' && !relative.startsWith(`..${path.sep}`); +} + +/** Reject links at every artifact component, including a linked context directory. */ +async function confinedPath(root: string, relative: string): Promise { + if ( + !relative || + relative.includes('\0') || + path.posix.isAbsolute(relative) || + path.win32.isAbsolute(relative) + ) { + throw new IndexCorruptedError('Invalid index artifact path. Rebuild required.'); + } + const parts = relative.replace(/\\/g, '/').split('/'); + if (parts.some((part) => !part || part === '.' || part === '..')) { + throw new IndexCorruptedError('Escaping index artifact path. Rebuild required.'); + } + const current = rootPaths.getStore(); + const realRoot = + current && (current.root === path.resolve(root) || current.realRoot === path.resolve(root)) + ? current.realRoot + : await fs.realpath(root); + let target = realRoot; + for (const part of [CODEBASE_CONTEXT_DIRNAME, ...parts]) { + target = path.join(target, part); + try { + if ((await fs.lstat(target)).isSymbolicLink()) + throw new IndexCorruptedError('Linked index artifacts are unsafe. Rebuild required.'); + if (!contained(realRoot, await fs.realpath(target))) + throw new IndexCorruptedError('Index artifact escapes project.'); + } catch (error) { + if (!isMissing(error)) throw error; + } + } + return target; +} + +async function contextDirectory(root: string): Promise { + // confinedPath checks the context component even when the lock directory is absent. + const probe = await confinedPath(root, '.writers'); + const directory = path.dirname(probe); + await fs.mkdir(directory, { recursive: true }); + return directory; +} + +async function atomicJson(target: string, value: unknown): Promise { + await atomicWrite(target, JSON.stringify(value)); +} + +function isBlockedTicketReplacement(error: unknown, target: string): boolean { + if ( + process.platform !== 'win32' || + !(error instanceof Error) || + !('code' in error) || + error.code !== 'EPERM' || + !('syscall' in error) || + error.syscall !== 'rename' || + !('dest' in error) || + error.dest !== target || + !('path' in error) || + typeof error.path !== 'string' + ) + return false; + const prefix = `${target}.`; + return ( + error.path.startsWith(prefix) && + error.path.endsWith('.tmp') && + GENERATION_ID.test(error.path.slice(prefix.length, -4)) + ); +} + +async function replaceLeaseTicket( + target: string, + owner: z.infer +): Promise { + const deadline = Date.now() + 250; + for (let attempt = 1; ; attempt++) { + try { + await atomicJson(target, owner); + return; + } catch (error) { + if (!isBlockedTicketReplacement(error, target) || attempt >= 8 || Date.now() + 25 >= deadline) + throw error; + await delay(25); + if (Date.now() >= deadline) throw error; + } + } +} + +/** + * Lamport's bakery lock uses unique, atomic ticket files rather than unlinking a + * shared stale lock. Concurrent dead-owner recovery cannot remove a successor. + * The lock covers baseline selection as well as staging and publication. + * Algorithm: https://lamport.azurewebsites.net/pubs/bakery.pdf + */ +async function withLease( + rootPath: string, + leaseDirectory: string, + operation: () => Promise +): Promise { + const root = path.resolve(rootPath); + await contextDirectory(root); + const directory = await confinedPath(root, leaseDirectory); + await fs.mkdir(directory, { recursive: true }); + const id = `${process.pid}-${randomUUID()}.json`; + const ownPath = path.join(directory, id); + const owner = { pid: process.pid, choosing: true, ticket: 0 }; + const readTickets = async () => { + const tickets: Array<{ id: string; owner: z.infer }> = []; + for (const entry of await fs.readdir(directory)) { + if (!entry.endsWith('.json')) continue; + const filename = await confinedPath(root, `${leaseDirectory}/${entry}`); + let value: unknown; + try { + value = JSON.parse(await fs.readFile(filename, 'utf8')); + } catch (error) { + if (isMissing(error)) continue; + throw error; + } + const ticket = TicketSchema.parse(value); + if (!isAlive(ticket.pid) || (!ticket.choosing && ticket.ticket === 0)) { + await fs.rm(filename, { force: true }).catch(() => undefined); + continue; + } + tickets.push({ id: entry, owner: ticket }); + } + return tickets; + }; + try { + await atomicJson(ownPath, owner); + owner.ticket = 1 + Math.max(0, ...(await readTickets()).map((entry) => entry.owner.ticket)); + owner.choosing = false; + await replaceLeaseTicket(ownPath, owner); + const deadline = Date.now() + 60_000; + while ( + (await readTickets()).some( + (entry) => + entry.id !== id && + (entry.owner.choosing || + entry.owner.ticket < owner.ticket || + (entry.owner.ticket === owner.ticket && entry.id < id)) + ) + ) { + if (Date.now() >= deadline) + throw new Error('Index writer is busy; retry after the current build.'); + await delay(25); + } + return await operation(); + } finally { + // Zero is a released ticket even when file deletion fails. A failed cleanup + // must never turn a successful metadata commit into an indexing failure. + try { + await replaceLeaseTicket(ownPath, { ...owner, choosing: false, ticket: 0 }); + } catch (error) { + console.warn('Index lease release deferred:', error); + } + await fs + .rm(ownPath, { force: true }) + .catch((error: unknown) => console.warn('Index lease cleanup deferred:', error)); + // Keep the empty directory: deleting it races the next ticket creator. + } +} + +export async function withIndexWriter( + rootPath: string, + operation: () => Promise +): Promise { + const root = path.resolve(rootPath); + return withResolvedRoot(root, () => + withLease(rootKey(root), '.writers', () => + writers.run(rootKey(root), () => snapshots.run(undefined, operation)) + ) + ); +} + +function currentIndexBoundary(rootPath: string): IndexBoundary | undefined { + const snapshot = snapshots.getStore(); + return snapshot?.root === rootKey(rootPath) ? snapshot : undefined; +} + +export function currentIndexSnapshot(rootPath: string): Snapshot | undefined { + const boundary = currentIndexBoundary(rootPath); + return boundary?.kind === 'committed' ? boundary : undefined; +} + +export function indexUnavailableForOperation(rootPath: string): boolean { + const boundary = currentIndexBoundary(rootPath); + return boundary !== undefined && boundary.kind !== 'committed'; +} + +async function readPublishedMeta(root: string): Promise { + const filename = await confinedPath(root, INDEX_META_FILENAME); + await fs.access(filename); + return readIndexMetaFromFile(filename); +} + +function artifactRelative(meta: IndexMeta, filename: string): string { + const entries: Record = { + [KEYWORD_INDEX_FILENAME]: meta.artifacts.keywordIndex, + [VECTOR_DB_DIRNAME]: meta.artifacts.vectorDb, + [INTELLIGENCE_FILENAME]: meta.artifacts.intelligence, + [MANIFEST_FILENAME]: meta.artifacts.manifest, + [INDEXING_STATS_FILENAME]: meta.artifacts.indexingStats, + [RELATIONSHIPS_FILENAME]: meta.artifacts.relationships, + [HEALTH_FILENAME]: meta.artifacts.health + }; + const generationPrefix = `${GENERATIONS}/${meta.buildId}/`; + const prefix = + meta.artifacts.keywordIndex.path === `${generationPrefix}${KEYWORD_INDEX_FILENAME}` + ? generationPrefix + : ''; + // A legacy build uses fixed names throughout. A generation uses exactly one + // directory throughout, even when a caller asks for just one optional artifact. + for (const [name, artifact] of Object.entries(entries)) { + if (artifact && artifact.path !== `${prefix}${name}`) + throw new IndexCorruptedError( + `Index artifact ${name} path belongs to a different generation.` + ); + } + const relative = entries[filename]?.path ?? `${prefix}${filename}`; + if (prefix && relative !== `${prefix}${filename}`) + throw new IndexCorruptedError( + `Index artifact ${filename} path belongs to a different generation.` + ); + return relative; +} + +export async function resolveIndexArtifact( + root: string, + filename: string, + meta?: IndexMeta +): Promise { + const boundary = currentIndexBoundary(root); + if (boundary && boundary.kind !== 'committed') { + try { + return await confinedPath(root, filename); + } catch (error) { + if (isMissing(error)) return path.join(root, CODEBASE_CONTEXT_DIRNAME, filename); + throw error; + } + } + if ( + boundary?.kind === 'committed' && + meta && + (meta.buildId !== boundary.meta.buildId || + meta.artifacts.keywordIndex.path !== boundary.meta.artifacts.keywordIndex.path) + ) + throw new IndexCorruptedError('Artifact metadata differs from the pinned generation.'); + let selected = boundary?.kind === 'committed' ? boundary.meta : meta; + if (!selected) { + try { + selected = await readPublishedMeta(root); + } catch (error) { + if (isMissing(error)) return path.join(root, CODEBASE_CONTEXT_DIRNAME, filename); + throw error; + } + } + try { + return await confinedPath(root, artifactRelative(selected, filename)); + } catch (error) { + if (error instanceof IndexCorruptedError) + throw new IndexCorruptedError(`${filename} artifact path: ${error.message}`); + throw error; + } +} + +/** Older metadata-free artifacts and an absent first build are snapshots too. */ +export async function withAvailableIndexSnapshot( + root: string, + operation: () => Promise +): Promise { + if (currentIndexBoundary(root)) return operation(); + try { + await fs.access(path.join(root, CODEBASE_CONTEXT_DIRNAME, INDEX_META_FILENAME)); + } catch (error) { + if (isMissing(error)) return withLegacyOrEmptySnapshot(root, operation); + throw error; + } + let started = false; + try { + return await withIndexSnapshot(root, () => { + started = true; + return operation(); + }); + } catch (error) { + // Preserve the owning tool's auto-heal/error path without letting nested + // readers switch to a publication that arrived halfway through this call. + if (!started && error instanceof IndexCorruptedError) + return withLegacyOrEmptySnapshot(root, operation); + throw error; + } +} + +async function withLegacyOrEmptySnapshot( + rootPath: string, + operation: () => Promise +): Promise { + const root = path.resolve(rootPath); + try { + await fs.access(path.join(root, CODEBASE_CONTEXT_DIRNAME, KEYWORD_INDEX_FILENAME)); + } catch (error) { + if (!isMissing(error)) throw error; + return snapshots.run( + { + kind: 'unavailable', + root: rootKey(root), + directory: path.join(root, CODEBASE_CONTEXT_DIRNAME) + }, + operation + ); + } + return withResolvedRoot(root, () => pinSnapshot(rootKey(root), operation, true)); +} + +/** Pin before validation/loading; nested readers cannot change the selected build. */ +export async function withIndexSnapshot( + rootPath: string, + operation: () => Promise +): Promise { + const boundary = currentIndexBoundary(rootPath); + if (boundary?.kind === 'committed') return operation(); + if (boundary) + throw new IndexCorruptedError( + 'No committed index was available at the start of this operation. Rebuild required.' + ); + const root = path.resolve(rootPath); + return withResolvedRoot(root, () => pinSnapshot(rootKey(root), operation)); +} + +async function releasePin(root: string, pinPath: string, buildId: string): Promise { + pendingPinReleases.set(pinPath, { root, buildId }); + await atomicJson(pinPath, { pid: process.pid, buildId, released: true }).catch((error: unknown) => + console.warn('Index pin release deferred:', error) + ); + try { + await fs.rm(pinPath, { force: true }); + pendingPinReleases.delete(pinPath); + } catch (error) { + console.warn('Index pin cleanup deferred:', error); + } +} + +async function pinSnapshot( + root: string, + operation: () => Promise, + allowLegacy = false +): Promise { + let pinPath = ''; + let pinBuildId = 'legacy-unversioned'; + let snapshot: IndexBoundary; + try { + snapshot = await withLease(root, '.publication', async (): Promise => { + let meta: IndexMeta | null; + try { + meta = await readPublishedMeta(root); + } catch (error) { + if (!allowLegacy || (!isMissing(error) && !(error instanceof IndexCorruptedError))) + throw error; + meta = null; + } + let selected: IndexBoundary; + if (meta) { + const indexPath = await resolveIndexArtifact(root, KEYWORD_INDEX_FILENAME, meta); + for (const name of [ + VECTOR_DB_DIRNAME, + INTELLIGENCE_FILENAME, + MANIFEST_FILENAME, + INDEXING_STATS_FILENAME, + RELATIONSHIPS_FILENAME, + HEALTH_FILENAME + ]) + await resolveIndexArtifact(root, name, meta); + const resolved = rootPaths.getStore(); + if (!resolved) throw new Error('Snapshot root resolution is missing.'); + selected = { + kind: 'committed', + root, + realRoot: resolved.realRoot, + meta, + directory: path.dirname(indexPath) + }; + pinBuildId = meta.buildId; + } else { + selected = { + kind: 'legacy', + root, + directory: path.dirname(await confinedPath(root, KEYWORD_INDEX_FILENAME)) + }; + } + const pinDirectory = await confinedPath(root, '.pins'); + await fs.mkdir(pinDirectory, { recursive: true }); + pinPath = path.join(pinDirectory, `${process.pid}-${randomUUID()}.json`); + await atomicJson(pinPath, { pid: process.pid, buildId: pinBuildId }); + return selected; + }); + } catch (error) { + if (pinPath) await releasePin(root, pinPath, pinBuildId); + throw error; + } + try { + return await snapshots.run(snapshot, operation); + } finally { + await releasePin(root, pinPath, pinBuildId); + } +} + +async function livePins(root: string): Promise> { + // Retry only releases this process knows completed; active operations remain pinned. + for (const [filename, release] of pendingPinReleases) { + if (release.root === rootKey(root)) await releasePin(root, filename, release.buildId); + } + const directory = await confinedPath(root, '.pins'); + const builds = new Set(); + let names: string[]; + try { + names = await fs.readdir(directory); + } catch (error) { + if (isMissing(error)) return builds; + throw error; + } + for (const name of names) { + if (!name.endsWith('.json')) continue; + const filename = await confinedPath(root, `.pins/${name}`); + let pin: z.infer; + try { + pin = PinSchema.parse(JSON.parse(await fs.readFile(filename, 'utf8'))); + } catch (error) { + if (isMissing(error)) continue; + throw error; + } + if (!pin.released && isAlive(pin.pid)) builds.add(pin.buildId); + else await fs.rm(filename, { force: true }).catch(() => undefined); + } + return builds; +} + +/** Called only while holding the writer lease; unknown pins fail closed. */ +async function collect(root: string, reserve: number): Promise { + const directory = await confinedPath(root, GENERATIONS); + await fs.mkdir(directory, { recursive: true }); + let active: string | undefined; + let activeGeneration = false; + try { + const raw: unknown = JSON.parse( + await fs.readFile(await confinedPath(root, INDEX_META_FILENAME), 'utf8') + ); + const parsed = IndexMetaSchema.safeParse(raw); + if ( + parsed.success && + parsed.data.metaVersion <= INDEX_META_VERSION && + parsed.data.formatVersion <= INDEX_FORMAT_VERSION + ) { + active = parsed.data.buildId; + activeGeneration = parsed.data.artifacts.keywordIndex.path.startsWith(`${GENERATIONS}/`); + if (activeGeneration) + for (const filename of DERIVED_FILES) artifactRelative(parsed.data, filename); + } + } catch (error) { + if (!isMissing(error) && !(error instanceof SyntaxError)) throw error; + } + const pinned = await livePins(root); + const names = await fs.readdir(directory); + const legacyIndex = await confinedPath(root, KEYWORD_INDEX_FILENAME); + let hasLegacy = false; + try { + await fs.access(legacyIndex); + hasLegacy = true; + } catch (error) { + if (!isMissing(error)) throw error; + } + // Legacy snapshots use fixed files and never mutate them after migration. Keep + // them while a legacy reader is pinned; they count toward the same ceiling. + if ( + hasLegacy && + activeGeneration && + !Array.from(pinned).some((build) => !names.includes(build)) + ) { + for (const filename of [ + KEYWORD_INDEX_FILENAME, + VECTOR_DB_DIRNAME, + MANIFEST_FILENAME, + INTELLIGENCE_FILENAME, + INDEXING_STATS_FILENAME, + RELATIONSHIPS_FILENAME, + HEALTH_FILENAME + ]) { + await fs.rm(await confinedPath(root, filename), { recursive: true, force: true }); + } + hasLegacy = false; + } + let count = names.length + Number(hasLegacy); + for (const name of names) { + if (count + reserve <= MAX_INDEX_GENERATIONS) break; + if (name === active || pinned.has(name)) continue; + // Unknown entries are preserved and consume capacity, never guessed disposable. + const evidence = await inspectGeneration(root, name); + if (evidence.kind === 'unknown' || (!activeGeneration && evidence.kind !== 'incomplete')) + continue; + const target = await confinedPath(root, `${GENERATIONS}/${name}`); + await fs.rm(target, { recursive: true, force: true }); + count--; + } + if (count + reserve > MAX_INDEX_GENERATIONS) + throw new IndexCapacityError( + activeGeneration + ? 'Index refresh deferred: live readers or unknown data occupy the three-generation capacity. Retry after readers finish; preserve unknown data.' + : 'Index recovery needs space: preserved generations occupy the three-generation capacity. Back up and verify an operator-selected generation, relocate it outside .codebase-context/.generations, then retry an explicit --embeddings or --no-embeddings reindex. No completed or unknown generation was selected or deleted.' + ); +} + +/** Native storage can read nested files, so validate the whole artifact tree. */ +export async function validateIndexArtifactTree(root: string, directory: string): Promise { + const current = rootPaths.getStore(); + const realRoot = + current && (current.root === path.resolve(root) || current.realRoot === path.resolve(root)) + ? current.realRoot + : await fs.realpath(root); + const context = path.join(realRoot, CODEBASE_CONTEXT_DIRNAME); + const inspect = async (target: string): Promise => { + await confinedPath(root, path.relative(context, target)); + for (const entry of await fs.readdir(target, { withFileTypes: true })) { + const filename = path.join(target, entry.name); + await confinedPath(root, path.relative(context, filename)); + if (entry.isDirectory()) await inspect(filename); + } + }; + await inspect(directory); +} + +/** Clone a closed immutable vector baseline without carrying filesystem links. */ +export async function copyIndexVectors( + root: string, + baseline: string, + destination: string +): Promise { + await validateIndexArtifactTree(root, baseline); + await fs.cp(baseline, destination, { + recursive: true, + dereference: false, + force: false, + errorOnExist: true + }); +} + +export async function prepareIndexGeneration(root: string, buildId: string): Promise { + if (writers.getStore() !== rootKey(root)) + throw new Error('Generation preparation requires the writer lease.'); + return withLease(root, '.publication', async () => { + await collect(root, 1); + const directory = await confinedPath(root, `${GENERATIONS}/${buildId}`); + await fs.mkdir(directory); + return directory; + }); +} + +/** Atomic metadata replacement is the sole commit point. Cleanup never rolls it back. */ +export async function publishIndexGeneration(root: string, meta: IndexMeta): Promise { + if (writers.getStore() !== rootKey(root)) + throw new Error('Index publication requires the writer lease.'); + const published = bindGenerationMeta(meta); + await withLease(root, '.publication', async () => { + await validateIndexArtifacts(root, published); + await atomicJson(await confinedPath(root, INDEX_META_FILENAME), published); + try { + await collect(root, 0); + } catch (error) { + console.warn('Index committed; generation cleanup deferred:', error); + } + }); +} diff --git a/src/core/index-meta.ts b/src/core/index-meta.ts index 4994235..db6780e 100644 --- a/src/core/index-meta.ts +++ b/src/core/index-meta.ts @@ -14,12 +14,22 @@ import { VECTOR_DB_DIRNAME } from '../constants/codebase-context.js'; import { IndexCorruptedError } from '../errors/index.js'; - -const ArtifactHeaderSchema = z.object({ +import { + currentIndexSnapshot, + inspectIndexDiskState, + indexUnavailableForOperation, + resolveIndexArtifact, + validateIndexArtifactTree, + withIndexSnapshot +} from './index-generation.js'; + +export const ArtifactHeaderSchema = z.object({ buildId: z.string().min(1), formatVersion: z.number().int().nonnegative() }); +export type ArtifactHeader = z.infer; + const KeywordIndexFileSchema = z.object({ header: ArtifactHeaderSchema, chunks: z.array(z.unknown()) @@ -81,7 +91,8 @@ export const IndexMetaSchema = z.object({ path: z.string().min(1), provider: z.string().min(1), embeddingProvider: z.string().optional(), - embeddingModel: z.string().optional() + embeddingModel: z.string().optional(), + embeddingEnabled: z.boolean().optional() }), intelligence: z .object({ @@ -92,19 +103,64 @@ export const IndexMetaSchema = z.object({ .object({ path: z.string().min(1) }) - .optional() + .optional(), + manifest: z + .object({ + path: z.string().min(1) + }) + .optional(), + indexingStats: z.object({ path: z.string().min(1) }).optional(), + relationships: z.object({ path: z.string().min(1) }).optional() }) .passthrough() }); export type IndexMeta = z.infer; +type KeywordIndexFile = z.infer; + +export function validateArtifactBuildIdentity( + header: ArtifactHeader, + meta: IndexMeta, + artifactLabel: string, + artifactFile: string +): void { + if (header.formatVersion !== meta.formatVersion) { + throw new IndexCorruptedError( + `${artifactLabel} formatVersion mismatch (rebuild required): meta=${meta.formatVersion}, ${artifactFile}=${header.formatVersion}` + ); + } + if (header.buildId !== meta.buildId) { + throw new IndexCorruptedError( + `${artifactLabel} buildId mismatch (rebuild required): meta=${meta.buildId}, ${artifactFile}=${header.buildId}` + ); + } +} + +/** Validate the keyword artifact's shape and identity against the active index metadata. */ +export function validateKeywordIndexArtifact(json: unknown, meta: IndexMeta): KeywordIndexFile { + const parsed = KeywordIndexFileSchema.safeParse(json); + if (!parsed.success) { + throw new IndexCorruptedError( + `Keyword index schema mismatch (rebuild required): ${parsed.error.message}` + ); + } + + validateArtifactBuildIdentity(parsed.data.header, meta, 'Keyword index', 'index.json'); + + return parsed.data; +} + export function checkEmbeddingMismatch( meta: IndexMeta, currentProvider: string, - currentModel: string + currentModel: string, + currentEmbeddingEnabled = true ): boolean { const stored = meta.artifacts.vectorDb; + const storedEmbeddingEnabled = stored.embeddingEnabled ?? true; + if (storedEmbeddingEnabled !== currentEmbeddingEnabled) return true; + if (!currentEmbeddingEnabled) return false; if (!stored.embeddingProvider || !stored.embeddingModel) return false; // legacy index, no info return stored.embeddingProvider !== currentProvider || stored.embeddingModel !== currentModel; } @@ -142,8 +198,17 @@ function asIndexCorrupted(message: string, error: unknown): IndexCorruptedError } export async function readIndexMeta(rootDir: string): Promise { + const snapshot = currentIndexSnapshot(rootDir); + if (snapshot) return snapshot.meta; + if (indexUnavailableForOperation(rootDir)) + throw new IndexCorruptedError( + 'Index meta was unavailable at the start of this operation. Rebuild required.' + ); const metaPath = path.join(rootDir, CODEBASE_CONTEXT_DIRNAME, INDEX_META_FILENAME); + return readIndexMetaFromFile(metaPath); +} +export async function readIndexMetaFromFile(metaPath: string): Promise { let parsed: unknown; try { const raw = await fs.readFile(metaPath, 'utf-8'); @@ -176,39 +241,53 @@ export async function readIndexMeta(rootDir: string): Promise { return meta; } -export async function validateIndexArtifacts(rootDir: string, meta: IndexMeta): Promise { - const contextDir = path.join(rootDir, CODEBASE_CONTEXT_DIRNAME); +/** Keep an existing index's successful mode; use model-free mode only for a fresh folder. */ +export async function resolveSkipEmbedding( + rootDir: string, + explicitChoice?: boolean +): Promise { + if (explicitChoice !== undefined) { + if (typeof explicitChoice !== 'boolean') throw new TypeError('skipEmbedding must be a boolean'); + return explicitChoice; + } - const keywordPath = path.join(contextDir, KEYWORD_INDEX_FILENAME); - const vectorDir = path.join(contextDir, VECTOR_DB_DIRNAME); + const state = await inspectIndexDiskState(rootDir); + switch (state.kind) { + case 'fresh': + case 'incomplete-first-build': + return true; + case 'committed': + return state.mode === 'keyword'; + case 'legacy': + if (state.mode !== 'unknown') return state.mode === 'keyword'; + break; + case 'recoverable-unpublished': + case 'unknown': + break; + default: { + const exhaustive: never = state; + return exhaustive; + } + } + throw new IndexCorruptedError( + 'Cannot recover embedding mode safely. Preserved index data needs explicit recovery; select --embeddings or --no-embeddings after backing up any recoverable generations. Capacity may require relocating an operator-selected backup outside .codebase-context.' + ); +} + +export async function validateIndexArtifacts(rootDir: string, meta: IndexMeta): Promise { + const keywordPath = await resolveIndexArtifact(rootDir, KEYWORD_INDEX_FILENAME, meta); + const vectorDir = await resolveIndexArtifact(rootDir, VECTOR_DB_DIRNAME, meta); const vectorBuildPath = path.join(vectorDir, 'index-build.json'); await requireFile(keywordPath, 'Keyword index'); await requireDirectory(vectorDir, 'Vector DB directory'); + await validateIndexArtifactTree(rootDir, vectorDir); await requireFile(vectorBuildPath, 'Vector DB build marker'); // Keyword index header (required) try { const raw = await fs.readFile(keywordPath, 'utf-8'); - const json = JSON.parse(raw); - const parsed = KeywordIndexFileSchema.safeParse(json); - if (!parsed.success) { - throw new IndexCorruptedError( - `Keyword index schema mismatch (rebuild required): ${parsed.error.message}` - ); - } - - const { buildId, formatVersion } = parsed.data.header; - if (formatVersion !== meta.formatVersion) { - throw new IndexCorruptedError( - `Keyword index formatVersion mismatch (rebuild required): meta=${meta.formatVersion}, index.json=${formatVersion}` - ); - } - if (buildId !== meta.buildId) { - throw new IndexCorruptedError( - `Keyword index buildId mismatch (rebuild required): meta=${meta.buildId}, index.json=${buildId}` - ); - } + validateKeywordIndexArtifact(JSON.parse(raw), meta); } catch (error) { if (error instanceof IndexCorruptedError) throw error; throw asIndexCorrupted('Keyword index corrupted (rebuild required)', error); @@ -242,7 +321,7 @@ export async function validateIndexArtifacts(rootDir: string, meta: IndexMeta): } // Optional intelligence artifact: validate if present, but do not require. - const intelligencePath = path.join(contextDir, INTELLIGENCE_FILENAME); + const intelligencePath = await resolveIndexArtifact(rootDir, INTELLIGENCE_FILENAME, meta); if (await pathExists(intelligencePath)) { try { const raw = await fs.readFile(intelligencePath, 'utf-8'); @@ -272,7 +351,7 @@ export async function validateIndexArtifacts(rootDir: string, meta: IndexMeta): } // Optional relationships sidecar: validate if present, but do not require. - const relationshipsPath = path.join(contextDir, RELATIONSHIPS_FILENAME); + const relationshipsPath = await resolveIndexArtifact(rootDir, RELATIONSHIPS_FILENAME, meta); if (await pathExists(relationshipsPath)) { try { const raw = await fs.readFile(relationshipsPath, 'utf-8'); @@ -302,7 +381,7 @@ export async function validateIndexArtifacts(rootDir: string, meta: IndexMeta): } // Optional health sidecar: validate if present, but do not require. - const healthPath = path.join(contextDir, HEALTH_FILENAME); + const healthPath = await resolveIndexArtifact(rootDir, HEALTH_FILENAME, meta); if (await pathExists(healthPath)) { try { const raw = await fs.readFile(healthPath, 'utf-8'); @@ -331,3 +410,16 @@ export async function validateIndexArtifacts(rootDir: string, meta: IndexMeta): } } } + +/** A refresh may serve the last complete build, including after a failed refresh. */ +export async function hasCommittedIndex(root: string): Promise { + try { + return await withIndexSnapshot(root, async () => { + const meta = await readIndexMeta(root); + await validateIndexArtifacts(root, meta); + return true; + }); + } catch { + return false; + } +} diff --git a/src/core/index-signal.ts b/src/core/index-signal.ts new file mode 100644 index 0000000..1c4c75d --- /dev/null +++ b/src/core/index-signal.ts @@ -0,0 +1,17 @@ +import { z } from 'zod'; + +/** Metadata supplied by an index-consuming tool after its work may supersede preflight. */ +export const IndexSignalSchema = z.object({ + status: z.enum(['ready', 'rebuild-required', 'indexing', 'unknown', 'error']), + confidence: z.enum(['high', 'low']), + action: z.enum([ + 'served', + 'rebuild-started', + 'rebuilt-and-served', + 'rebuild-completed', + 'rebuild-failed', + 'rebuild-requested' + ]), + reason: z.string().optional() +}); +export type IndexSignal = z.infer; diff --git a/src/core/indexer.ts b/src/core/indexer.ts index 93aebbe..6d9ed37 100644 --- a/src/core/indexer.ts +++ b/src/core/indexer.ts @@ -3,9 +3,10 @@ * Scans files, delegates to analyzers, creates embeddings, stores in vector DB */ -import { randomUUID } from 'crypto'; +import { createHash, randomUUID } from 'crypto'; import { promises as fs } from 'fs'; import path from 'path'; +import { IndexCorruptedError, InvalidSourceChunkError } from '../errors/index.js'; import { glob } from 'glob'; import ignore from 'ignore'; import { @@ -24,20 +25,26 @@ import { analyzerRegistry } from './analyzer-registry.js'; import type { AnalyzerSelectionOptions } from './analyzer-registry.js'; import { getSupportedExtensions, isBinaryFile, isCodeFile } from '../utils/language-detection.js'; import { + assertEmbeddingRuntimeAvailable, getEmbeddingProvider, getConfiguredDimensions, DEFAULT_MODEL, parseEmbeddingProviderName } from '../embeddings/index.js'; -import { getStorageProvider, CodeChunkWithEmbedding } from '../storage/index.js'; +import { + assertStorageRuntimeAvailable, + getStorageProvider, + CodeChunkWithEmbedding +} from '../storage/index.js'; import { LibraryUsageTracker, PatternDetector, ImportGraph, InternalFileGraph, - FileExport + FileExport, + ImportEdgeSource } from '../utils/usage-tracker.js'; -import { mergeSmallChunks } from '../utils/chunking.js'; +import { mergeSmallChunks, validateSourceChunks } from '../utils/chunking.js'; import { getFileCommitDates } from '../utils/git-dates.js'; import { CODEBASE_CONTEXT_DIRNAME, @@ -53,21 +60,45 @@ import { RELATIONSHIPS_FILENAME, VECTOR_DB_DIRNAME } from '../constants/codebase-context.js'; +import { isPathWithin } from '../utils/project-discovery.js'; import { deriveCodebaseHealth } from '../health/derive.js'; +import { + extractSourceReexports, + InternalSourceResolver, + loadTsconfigCompilerOptions +} from '../utils/internal-source-resolver.js'; -const STAGING_DIRNAME = '.staging'; -const PREVIOUS_DIRNAME = '.previous'; const DEFAULT_MAX_CHUNKS = 5000; +class SourceSnapshotChangedError extends Error {} + import { computeFileHashes, + hashFileContent, readManifest, writeManifest, diffManifest, type FileManifest, type ManifestDiff } from './manifest.js'; -import { readIndexMeta, checkEmbeddingMismatch } from './index-meta.js'; +import { + readIndexMeta, + readIndexMetaFromFile, + checkEmbeddingMismatch, + resolveSkipEmbedding, + validateIndexArtifacts +} from './index-meta.js'; +import { + withIndexWriter, + withIndexSnapshot, + inspectIndexDiskState, + permitsFirstBuild, + prepareIndexGeneration, + publishIndexGeneration, + resolveIndexArtifact, + copyIndexVectors, + withAvailableIndexSnapshot +} from './index-generation.js'; let cachedToolVersion: string | null = null; @@ -89,126 +120,6 @@ async function getToolVersion(): Promise { return cachedToolVersion; } -/** - * Perform a Windows-safe atomic swap of staging artifacts into active location. - * Strategy: move current active to .previous, then rename staging to active. - * If staging rename fails, restore from .previous. - */ -async function atomicSwapStagingToActive( - contextDir: string, - stagingDir: string, - buildId: string -): Promise { - const previousDir = path.join(contextDir, PREVIOUS_DIRNAME); - const activeMetaPath = path.join(contextDir, INDEX_META_FILENAME); - const activeIndexPath = path.join(contextDir, KEYWORD_INDEX_FILENAME); - const activeIntelligencePath = path.join(contextDir, INTELLIGENCE_FILENAME); - const activeVectorDir = path.join(contextDir, VECTOR_DB_DIRNAME); - const activeManifestPath = path.join(contextDir, MANIFEST_FILENAME); - const activeStatsPath = path.join(contextDir, INDEXING_STATS_FILENAME); - const activeRelationshipsPath = path.join(contextDir, RELATIONSHIPS_FILENAME); - const activeHealthPath = path.join(contextDir, HEALTH_FILENAME); - - const stagingMetaPath = path.join(stagingDir, INDEX_META_FILENAME); - const stagingIndexPath = path.join(stagingDir, KEYWORD_INDEX_FILENAME); - const stagingIntelligencePath = path.join(stagingDir, INTELLIGENCE_FILENAME); - const stagingVectorDir = path.join(stagingDir, VECTOR_DB_DIRNAME); - const stagingManifestPath = path.join(stagingDir, MANIFEST_FILENAME); - const stagingStatsPath = path.join(stagingDir, INDEXING_STATS_FILENAME); - const stagingRelationshipsPath = path.join(stagingDir, RELATIONSHIPS_FILENAME); - const stagingHealthPath = path.join(stagingDir, HEALTH_FILENAME); - - // Step 1: Create .previous directory and move current active there - await fs.mkdir(previousDir, { recursive: true }); - - const moveIfExists = async (src: string, dest: string): Promise => { - try { - await fs.rename(src, dest); - } catch (error) { - const code = (error as NodeJS.ErrnoException).code; - if (code !== 'ENOENT') { - // File doesn't exist is OK, other errors are problems - throw error; - } - } - }; - - const moveDirIfExists = async (src: string, dest: string): Promise => { - try { - const stat = await fs.stat(src); - if (stat.isDirectory()) { - await fs.rename(src, dest); - } - } catch (error) { - const code = (error as NodeJS.ErrnoException).code; - if (code !== 'ENOENT') { - throw error; - } - } - }; - - // Move active artifacts to .previous - await moveIfExists(activeMetaPath, path.join(previousDir, INDEX_META_FILENAME)); - await moveIfExists(activeIndexPath, path.join(previousDir, KEYWORD_INDEX_FILENAME)); - await moveIfExists(activeIntelligencePath, path.join(previousDir, INTELLIGENCE_FILENAME)); - await moveIfExists(activeManifestPath, path.join(previousDir, MANIFEST_FILENAME)); - await moveIfExists(activeStatsPath, path.join(previousDir, INDEXING_STATS_FILENAME)); - await moveIfExists(activeRelationshipsPath, path.join(previousDir, RELATIONSHIPS_FILENAME)); - await moveIfExists(activeHealthPath, path.join(previousDir, HEALTH_FILENAME)); - await moveDirIfExists(activeVectorDir, path.join(previousDir, VECTOR_DB_DIRNAME)); - - // Step 2: Move staging artifacts to active location - try { - await moveIfExists(stagingMetaPath, activeMetaPath); - await moveIfExists(stagingIndexPath, activeIndexPath); - await moveIfExists(stagingIntelligencePath, activeIntelligencePath); - await moveIfExists(stagingManifestPath, activeManifestPath); - await moveIfExists(stagingStatsPath, activeStatsPath); - await moveIfExists(stagingRelationshipsPath, activeRelationshipsPath); - await moveIfExists(stagingHealthPath, activeHealthPath); - await moveDirIfExists(stagingVectorDir, activeVectorDir); - - // Step 3: Clean up .previous and staging directories - await cleanupDirectory(previousDir); - await cleanupDirectory(stagingDir); - - // Also clean up the parent .staging/ directory if empty - const stagingBase = path.join(contextDir, STAGING_DIRNAME); - try { - const remaining = await fs.readdir(stagingBase); - if (remaining.length === 0) { - await fs.rmdir(stagingBase); - } - } catch { - // Directory doesn't exist or not empty - ignore - } - - console.error(`Atomic swap complete: build ${buildId} now active`); - } catch (swapError) { - console.error('Atomic swap failed, attempting rollback:', swapError); - - // Attempt rollback: restore from .previous - try { - await moveIfExists(path.join(previousDir, INDEX_META_FILENAME), activeMetaPath); - await moveIfExists(path.join(previousDir, KEYWORD_INDEX_FILENAME), activeIndexPath); - await moveIfExists(path.join(previousDir, INTELLIGENCE_FILENAME), activeIntelligencePath); - await moveIfExists(path.join(previousDir, MANIFEST_FILENAME), activeManifestPath); - await moveIfExists(path.join(previousDir, INDEXING_STATS_FILENAME), activeStatsPath); - await moveIfExists(path.join(previousDir, RELATIONSHIPS_FILENAME), activeRelationshipsPath); - await moveIfExists(path.join(previousDir, HEALTH_FILENAME), activeHealthPath); - await moveDirIfExists(path.join(previousDir, VECTOR_DB_DIRNAME), activeVectorDir); - console.error('Rollback successful'); - } catch (rollbackError) { - console.error('Rollback also failed:', rollbackError); - } - - throw swapError; - } -} - -/** - * Best-effort cleanup of a directory and its contents. - */ async function cleanupDirectory(dirPath: string): Promise { try { await fs.rm(dirPath, { recursive: true, force: true }); @@ -223,6 +134,8 @@ export interface IndexerOptions { projectOptions?: AnalyzerSelectionOptions; onProgress?: (progress: IndexingProgress) => void; incrementalOnly?: boolean; + /** Recheck first-use eligibility after writer admission; never replace a newly published index. */ + firstBuildOnly?: boolean; } interface PersistedIndexingStats { @@ -232,6 +145,78 @@ interface PersistedIndexingStats { generatedAt: string; } +const MAX_INDEX_FILES = 10000; +const MAX_PREVIEW_SKIP_SAMPLES = 20; + +export interface IndexFileSelection { + selected: string[]; + overLimitCount: number; + overLimitSamples: string[]; +} + +export interface IndexPreview { + rootPath: string; + eligibleFiles: string[]; + skippedFiles: Array<{ path: string; reason: string }>; + skippedFileCount: number; + /** Patterns applied before glob enumeration; matching files are not individually observed. */ + excludedPatterns: string[]; +} + +interface FileScanResult { + files: string[]; + skippedFiles: Array<{ path: string; reason: string }>; + skippedFileCount: number; +} + +type SourcePathResolution = + | { status: 'contained'; path: string } + | { status: 'outside' } + | { status: 'alias' } + | { status: 'unavailable' }; + +function normalizeComparablePath(filePath: string): string { + const normalized = path.normalize(path.resolve(filePath)); + return process.platform === 'win32' ? normalized.toLowerCase() : normalized; +} + +async function resolveContainedSourcePath( + filePath: string, + canonicalRootPath: string +): Promise { + let canonicalFilePath: string; + try { + canonicalFilePath = await fs.realpath(filePath); + } catch { + return { status: 'unavailable' }; + } + if (!isPathWithin(canonicalRootPath, canonicalFilePath)) return { status: 'outside' }; + if (normalizeComparablePath(filePath) !== normalizeComparablePath(canonicalFilePath)) { + return { status: 'alias' }; + } + return { status: 'contained', path: canonicalFilePath }; +} + +function addSkippedFile( + scan: FileScanResult, + filePath: string, + reason: string, + rootPath: string +): void { + scan.skippedFileCount++; + if (scan.skippedFiles.length < MAX_PREVIEW_SKIP_SAMPLES) { + scan.skippedFiles.push({ path: path.relative(rootPath, filePath).replace(/\\/g, '/'), reason }); + } +} + +export function selectFilesForIndex(files: string[]): IndexFileSelection { + return { + selected: files.slice(0, MAX_INDEX_FILES), + overLimitCount: Math.max(0, files.length - MAX_INDEX_FILES), + overLimitSamples: files.slice(MAX_INDEX_FILES, MAX_INDEX_FILES + MAX_PREVIEW_SKIP_SAMPLES) + }; +} + export class CodebaseIndexer { private rootPath: string; private config: CodebaseConfig; @@ -240,10 +225,14 @@ export class CodebaseIndexer { private progress: IndexingProgress; private onProgressCallback?: (progress: IndexingProgress) => void; private incrementalOnly: boolean; + private readonly firstBuildOnly: boolean; + private readonly explicitSkipEmbedding?: boolean; constructor(options: IndexerOptions) { this.rootPath = path.resolve(options.rootPath); + this.firstBuildOnly = options.firstBuildOnly ?? false; this.config = this.mergeConfig(options.config); + this.explicitSkipEmbedding = options.config?.skipEmbedding; this.projectOptions = { preferredAnalyzer: options.projectOptions?.preferredAnalyzer, extraFileExtensions: options.projectOptions?.extraFileExtensions @@ -324,7 +313,7 @@ export class CodebaseIndexer { model: defaultModel, batchSize: 32 }, - skipEmbedding: false, + skipEmbedding: true, storage: { provider: 'lancedb', path: './codebase-index' @@ -385,7 +374,43 @@ export class CodebaseIndexer { return Array.from(new Set([...includePatterns, ...extraPatterns])); } + /** Enumerate the exact files the next index pass would select, without indexing or writing. */ + async previewFiles(): Promise { + const scan = await this.scanFiles(false); + const selection = selectFilesForIndex(scan.files); + const skippedFiles = [...scan.skippedFiles]; + for (const file of selection.overLimitSamples.slice( + 0, + MAX_PREVIEW_SKIP_SAMPLES - skippedFiles.length + )) { + skippedFiles.push({ + path: path.relative(this.rootPath, file).replace(/\\/g, '/'), + reason: 'exceeds the indexer file-count limit' + }); + } + const toRelativePath = (file: string): string => + path.relative(this.rootPath, file).replace(/\\/g, '/'); + const comparePaths = (left: string, right: string): number => + left < right ? -1 : left > right ? 1 : 0; + const eligibleFiles = selection.selected.map(toRelativePath).sort(comparePaths); + skippedFiles.sort( + (left, right) => + comparePaths(left.path, right.path) || comparePaths(left.reason, right.reason) + ); + return { + rootPath: this.rootPath, + eligibleFiles, + skippedFiles, + skippedFileCount: scan.skippedFileCount + selection.overLimitCount, + excludedPatterns: [...(this.config.exclude ?? [])] + }; + } + async index(): Promise { + return withIndexWriter(this.rootPath, () => this.indexLocked()); + } + + private async indexLocked(): Promise { const startTime = Date.now(); const stats: IndexingStats = { totalFiles: 0, @@ -414,6 +439,29 @@ export class CodebaseIndexer { let stagingDir: string | null = null; try { + if (this.firstBuildOnly) { + const state = await inspectIndexDiskState(this.rootPath); + if (state.kind === 'committed' || state.kind === 'legacy') { + await withIndexSnapshot(this.rootPath, async () => { + await validateIndexArtifacts(this.rootPath, await readIndexMeta(this.rootPath)); + }); + stats.duration = Date.now() - startTime; + stats.completedAt = new Date(); + return stats; + } + if (!permitsFirstBuild(state)) + throw new IndexCorruptedError( + 'Existing index data needs explicit recovery before map can build a first index.' + ); + } + this.config.skipEmbedding = await resolveSkipEmbedding( + this.rootPath, + this.explicitSkipEmbedding + ); + if (!this.config.skipEmbedding) { + assertEmbeddingRuntimeAvailable(this.config.embedding); + assertStorageRuntimeAvailable(); + } // Ensure there is at least a generic fallback analyzer registered when the indexer // is used directly (e.g. in tests or standalone scripts). if (analyzerRegistry.getAll().length === 0) { @@ -429,18 +477,18 @@ export class CodebaseIndexer { // Phase 1: Scanning this.updateProgress('scanning', 0); - let files = await this.scanFiles(); + const scan = await this.scanFiles(); + const selection = selectFilesForIndex(scan.files); + const files = selection.selected; // Memory safety: limit total files to prevent heap exhaustion - const MAX_FILES = 10000; - if (files.length > MAX_FILES) { + if (selection.overLimitCount > 0) { console.warn( - `WARNING: Found ${files.length} files, limiting to ${MAX_FILES} to prevent memory issues.` + `WARNING: Found ${scan.files.length} files, limiting to ${MAX_INDEX_FILES} to prevent memory issues.` ); console.warn( `Consider using more specific include patterns or excluding large directories.` ); - files = files.slice(0, MAX_FILES); } stats.totalFiles = files.length; @@ -450,8 +498,14 @@ export class CodebaseIndexer { // Phase 1b: Incremental diff (if incremental mode) const contextDir = path.join(this.rootPath, CODEBASE_CONTEXT_DIRNAME); - const manifestPath = path.join(contextDir, MANIFEST_FILENAME); - const indexingStatsPath = path.join(contextDir, INDEXING_STATS_FILENAME); + const baselineMeta = await readIndexMeta(this.rootPath).catch(() => null); + const baselineDirectory = baselineMeta + ? path.dirname( + await resolveIndexArtifact(this.rootPath, KEYWORD_INDEX_FILENAME, baselineMeta) + ) + : contextDir; + const manifestPath = path.join(baselineDirectory, MANIFEST_FILENAME); + const indexingStatsPath = path.join(baselineDirectory, INDEXING_STATS_FILENAME); let diff: ManifestDiff | null = null; let currentHashes: Record | null = null; let previousManifest: FileManifest | null = null; @@ -482,15 +536,24 @@ export class CodebaseIndexer { const existingMeta = await readIndexMeta(this.rootPath); const currentProvider = this.config.embedding?.provider ?? 'transformers'; const currentModel = this.config.embedding?.model ?? DEFAULT_MODEL; - if (checkEmbeddingMismatch(existingMeta, currentProvider, currentModel)) { + if ( + checkEmbeddingMismatch( + existingMeta, + currentProvider, + currentModel, + !this.config.skipEmbedding + ) + ) { const stored = existingMeta.artifacts.vectorDb; console.error( - `Embedding provider/model changed (stored: ${stored.embeddingProvider}:${stored.embeddingModel}, current: ${currentProvider}:${currentModel}) — forcing full rebuild` + `Embedding configuration changed (stored: ${stored.embeddingEnabled === false ? 'disabled' : `${stored.embeddingProvider ?? 'unknown'}:${stored.embeddingModel ?? 'unknown'}`}, current: ${this.config.skipEmbedding ? 'disabled' : `${currentProvider}:${currentModel}`}); forcing full rebuild` ); diff = null; } } catch { - // No meta yet or legacy index without embedding fields — proceed with incremental + // Chunk semantics and embedding input may have changed even when source hashes did not. + // Incompatible metadata must never reuse chunks from the previous format. + diff = null; } // Short-circuit: nothing changed @@ -532,7 +595,7 @@ export class CodebaseIndexer { } try { - const existingIndexPath = path.join(contextDir, KEYWORD_INDEX_FILENAME); + const existingIndexPath = path.join(baselineDirectory, KEYWORD_INDEX_FILENAME); const existing = JSON.parse(await fs.readFile(existingIndexPath, 'utf-8')) as unknown; const existingObj = existing as { chunks?: unknown }; const existingChunks: unknown = Array.isArray(existing) @@ -585,6 +648,31 @@ export class CodebaseIndexer { // When incremental, track which files need embedding const filesToProcessSet = diff ? new Set(filesToProcess.map((f) => f)) : null; + // Resolve graph edges only to discovered, root-confined source files. + let tsconfigPaths: Record | undefined; + const tsconfigCompilerOptions = loadTsconfigCompilerOptions(this.rootPath); + if (tsconfigCompilerOptions.paths) { + tsconfigPaths = Object.fromEntries( + Object.entries(tsconfigCompilerOptions.paths).map(([alias, targets]) => [ + alias, + [...targets] + ]) + ); + } + const sourceResolver = await InternalSourceResolver.create(this.rootPath, files, { + compilerOptions: tsconfigCompilerOptions + }); + const sourceFingerprints = new Map(); + const analyzedManifestHashes = new Map(); + const pendingResolvedEdges: Array<{ + from: string; + to: string; + target: string; + line: number; + importedSymbols: string[]; + source: Omit; + }> = []; + for (let i = 0; i < files.length; i++) { const file = files[i]; this.progress.currentFile = file; @@ -593,14 +681,53 @@ export class CodebaseIndexer { try { // Normalize line endings to \n for consistent cross-platform output - const rawContent = await fs.readFile(file, 'utf-8'); + const resolution = await resolveContainedSourcePath(file, this.rootPath); + if (resolution.status !== 'contained') { + throw new Error( + resolution.status === 'outside' + ? `Source resolves outside selected root: ${file}` + : resolution.status === 'alias' + ? `Source alias is not indexed: ${file}` + : `Source could not be resolved before reading: ${file}` + ); + } + const resolvedFile = resolution.path; + const rawBytes = await fs.readFile(resolvedFile); + const sourceFingerprint = createHash('sha256').update(rawBytes).digest('hex'); + sourceFingerprints.set(resolvedFile, sourceFingerprint); + const rawContent = rawBytes.toString('utf-8'); + const relativeFile = path.relative(this.rootPath, resolvedFile).replace(/\\/g, '/'); + const analyzedManifestHash = hashFileContent(rawContent); + analyzedManifestHashes.set(relativeFile, analyzedManifestHash); + if ( + currentHashes && + diff?.unchanged.includes(relativeFile) && + currentHashes[relativeFile] !== analyzedManifestHash + ) { + throw new SourceSnapshotChangedError( + `Source changed during incremental scan; retry indexing: ${relativeFile}` + ); + } const content = rawContent.replace(/\r\n/g, '\n'); - const result = await analyzerRegistry.analyzeFile(file, content, resolvedProjectOptions); + const result = await analyzerRegistry.analyzeFile(file, content, { + ...resolvedProjectOptions, + includeScopePrefix: false + }); if (result) { const isFileChanged = !filesToProcessSet || filesToProcessSet.has(file); + // Analyzer paths are derived from process.cwd(); persisted paths belong to this index root. + for (const chunk of result.chunks) { + chunk.relativePath = relativeFile; + } + const mergedChunks = mergeSmallChunks(result.chunks, 15); + validateSourceChunks(mergedChunks, { + filePath: resolvedFile, + relativePath: relativeFile, + content + }); allChunks.push(...mergedChunks); if (isFileChanged) { @@ -614,24 +741,60 @@ export class CodebaseIndexer { libraryTracker.track(imp.source, file); importGraph.trackImport(imp.source, file, imp.line || 1); - // Track internal file-to-file imports (relative paths) - if (imp.source.startsWith('.')) { - // Resolve the relative import to an absolute path - const fileDir = path.dirname(file); - let resolvedPath = path.resolve(fileDir, imp.source); - - // Try common extensions if not already specified - const ext = path.extname(resolvedPath); - if (!ext) { - for (const tryExt of ['.ts', '.tsx', '.js', '.jsx']) { - const withExt = resolvedPath + tryExt; - // We don't check if file exists for performance - just track what's referenced - resolvedPath = withExt; - break; + const resolution = await sourceResolver.resolve(file, imp.source); + if (resolution.status === 'resolved') { + pendingResolvedEdges.push({ + from: file, + to: resolution.path, + target: resolution.target, + line: imp.line || 1, + importedSymbols: imp.imports, + source: { + kind: 'import', + specifier: imp.source, + sourceFingerprint } - } + }); + } else { + internalFileGraph.trackUnresolved( + file, + { + line: imp.line || 1, + specifier: imp.source, + kind: 'import', + sourceFingerprint, + targetFingerprint: '' + }, + resolution + ); + } + } - internalFileGraph.trackImport(file, resolvedPath, imp.line || 1, imp.imports); + // Re-export relationships are separate typed edges and require AST source evidence. + for (const reexport of extractSourceReexports(content, file)) { + const resolution = await sourceResolver.resolve(file, reexport.specifier); + const proof = { + line: reexport.line, + specifier: reexport.specifier, + kind: 'reexport' as const, + sourceFingerprint, + targetFingerprint: '' + }; + if (resolution.status === 'resolved') { + pendingResolvedEdges.push({ + from: file, + to: resolution.path, + target: resolution.target, + line: reexport.line, + importedSymbols: [], + source: { + kind: proof.kind, + specifier: proof.specifier, + sourceFingerprint: proof.sourceFingerprint + } + }); + } else { + internalFileGraph.trackUnresolved(file, proof, resolution); } } @@ -727,6 +890,22 @@ export class CodebaseIndexer { stats.skippedFiles++; } } catch (error) { + if ( + error instanceof SourceSnapshotChangedError || + error instanceof InvalidSourceChunkError + ) + throw error; + const relativeFile = path.relative(this.rootPath, file).replace(/\\/g, '/'); + if ( + diff?.unchanged.includes(relativeFile) && + error instanceof Error && + 'code' in error && + error.code === 'ENOENT' + ) { + throw new SourceSnapshotChangedError( + `Source disappeared during incremental scan; retry indexing: ${relativeFile}` + ); + } stats.skippedFiles++; stats.errors.push({ filePath: file, @@ -741,6 +920,28 @@ export class CodebaseIndexer { } } + // Finish edges only after each endpoint has been read for this build. This binds both + // fingerprints to the exact source snapshots analyzed above, without retaining file bytes. + for (const edge of pendingResolvedEdges) { + const targetFingerprint = sourceFingerprints.get(edge.to); + if (targetFingerprint) { + internalFileGraph.trackImport(edge.from, edge.to, edge.line, edge.importedSymbols, { + ...edge.source, + targetFingerprint + }); + } else { + internalFileGraph.trackUnresolved( + edge.from, + { + ...edge.source, + line: edge.line, + targetFingerprint: '' + }, + { reason: 'analysis-unavailable', target: edge.target } + ); + } + } + stats.totalChunks = allChunks.length; stats.avgChunkSize = allChunks.length > 0 @@ -833,60 +1034,61 @@ export class CodebaseIndexer { // Phase 4: Storing this.updateProgress('storing', 75); - // For full rebuilds, use staging directory for atomic swap - // For incremental, write directly to active location - const isFullRebuild = !diff; - let activeContextDir = contextDir; - - if (isFullRebuild) { - // Create staging directory for atomic swap - const stagingBase = path.join(contextDir, STAGING_DIRNAME); - stagingDir = path.join(stagingBase, buildId); - await fs.mkdir(stagingDir, { recursive: true }); - activeContextDir = stagingDir; - console.error(`Full rebuild: writing to staging ${stagingDir}`); + // Both full and incremental builds write a new, unpublished generation. + stagingDir = await prepareIndexGeneration(this.rootPath, buildId); + const generationDirectory = stagingDir; + if (diff && !this.config.skipEmbedding) { + // The published baseline is immutable and has no writer handles. A physical + // copy preserves unchanged vectors without sharing writable Lance files. + await copyIndexVectors( + this.rootPath, + path.join(baselineDirectory, VECTOR_DB_DIRNAME), + path.join(generationDirectory, VECTOR_DB_DIRNAME) + ); } - await fs.mkdir(activeContextDir, { recursive: true }); - if (!this.config.skipEmbedding) { - const storagePath = path.join(activeContextDir, VECTOR_DB_DIRNAME); + const storagePath = path.join(generationDirectory, VECTOR_DB_DIRNAME); const storageProvider = await getStorageProvider( { path: storagePath }, diff ? { expectedDimensions: getConfiguredDimensions(this.config.embedding) } : undefined ); - if (diff) { - // Incremental: delete old chunks for changed + deleted files, then add new - const filesToDelete = [...diff.changed, ...diff.deleted].map((rel) => - path.join(this.rootPath, rel).replace(/\\/g, '/') - ); - // Also try with OS-native separators for matching - const filePathsForDelete = [...diff.changed, ...diff.deleted].map((rel) => - path.resolve(this.rootPath, rel) - ); - const allDeletePaths = [...new Set([...filesToDelete, ...filePathsForDelete])]; + try { + if (diff) { + // Incremental: delete old chunks for changed + deleted files, then add new + const filesToDelete = [...diff.changed, ...diff.deleted].map((rel) => + path.join(this.rootPath, rel).replace(/\\/g, '/') + ); + // Also try with OS-native separators for matching + const filePathsForDelete = [...diff.changed, ...diff.deleted].map((rel) => + path.resolve(this.rootPath, rel) + ); + const allDeletePaths = [...new Set([...filesToDelete, ...filePathsForDelete])]; - if (allDeletePaths.length > 0) { - await storageProvider.deleteByFilePaths(allDeletePaths); - } - if (chunksWithEmbeddings.length > 0) { + if (allDeletePaths.length > 0) { + await storageProvider.deleteByFilePaths(allDeletePaths); + } + if (chunksWithEmbeddings.length > 0) { + await storageProvider.store(chunksWithEmbeddings); + } + console.error( + `Incremental store: deleted chunks for ${diff.changed.length + diff.deleted.length} files, ` + + `added ${chunksWithEmbeddings.length} new chunks` + ); + } else { + // Full rebuild: store to staging (no clear - fresh directory) + console.error(`Storing ${chunksToEmbed.length} chunks to staging...`); await storageProvider.store(chunksWithEmbeddings); } - console.error( - `Incremental store: deleted chunks for ${diff.changed.length + diff.deleted.length} files, ` + - `added ${chunksWithEmbeddings.length} new chunks` - ); - } else { - // Full rebuild: store to staging (no clear - fresh directory) - console.error(`Storing ${chunksToEmbed.length} chunks to staging...`); - await storageProvider.store(chunksWithEmbeddings); + } finally { + await storageProvider.close?.(); } } // Vector DB build marker (required for version gating) // Write after semantic store step so marker reflects the latest DB state. - const vectorDir = path.join(activeContextDir, VECTOR_DB_DIRNAME); + const vectorDir = path.join(generationDirectory, VECTOR_DB_DIRNAME); await fs.mkdir(vectorDir, { recursive: true }); await fs.writeFile( path.join(vectorDir, 'index-build.json'), @@ -894,7 +1096,7 @@ export class CodebaseIndexer { ); // Keyword index always uses ALL chunks (full regen) - const indexPath = path.join(activeContextDir, KEYWORD_INDEX_FILENAME); + const indexPath = path.join(generationDirectory, KEYWORD_INDEX_FILENAME); // Memory safety: cap keyword index too const keywordChunks = allChunks.length > maxChunks ? allChunks.slice(0, maxChunks) : allChunks; @@ -907,23 +1109,11 @@ export class CodebaseIndexer { ); // Save library usage and pattern stats (always full regen) - const intelligencePath = path.join(activeContextDir, INTELLIGENCE_FILENAME); + const intelligencePath = path.join(generationDirectory, INTELLIGENCE_FILENAME); const libraryStats = libraryTracker.getStats(); - // Extract tsconfig paths for AI to understand import aliases - let tsconfigPaths: Record | undefined; - try { - const tsconfigPath = path.join(this.rootPath, 'tsconfig.json'); - const tsconfigContent = await fs.readFile(tsconfigPath, 'utf-8'); - const tsconfig = JSON.parse(tsconfigContent); - if (tsconfig.compilerOptions?.paths) { - tsconfigPaths = tsconfig.compilerOptions.paths; - console.error( - `Found ${Object.keys(tsconfigPaths!).length} path aliases in tsconfig.json` - ); - } - } catch (_error) { - // No tsconfig.json or no paths defined + if (tsconfigPaths && Object.keys(tsconfigPaths).length > 0) { + console.error(`Found ${Object.keys(tsconfigPaths).length} path aliases in tsconfig.json`); } const intelligence = { @@ -945,7 +1135,7 @@ export class CodebaseIndexer { await fs.writeFile(intelligencePath, JSON.stringify(intelligence, null, 2)); // Write relationships sidecar (versioned, for fast lookup) - const relationshipsPath = path.join(activeContextDir, RELATIONSHIPS_FILENAME); + const relationshipsPath = path.join(generationDirectory, RELATIONSHIPS_FILENAME); const graphData = internalFileGraph.toJSON(); // Build reverse import map (importedBy) @@ -980,6 +1170,7 @@ export class CodebaseIndexer { graph: { imports: graphData.imports || {}, ...(graphData.importDetails ? { importDetails: graphData.importDetails } : {}), + ...(graphData.unresolvedEdges ? { unresolvedEdges: graphData.unresolvedEdges } : {}), importedBy, exports: graphData.exports || {} }, @@ -990,7 +1181,7 @@ export class CodebaseIndexer { }; await fs.writeFile(relationshipsPath, JSON.stringify(relationships, null, 2)); - const healthPath = path.join(activeContextDir, HEALTH_FILENAME); + const healthPath = path.join(generationDirectory, HEALTH_FILENAME); const health = deriveCodebaseHealth({ buildId, formatVersion: INDEX_FORMAT_VERSION, @@ -1001,12 +1192,14 @@ export class CodebaseIndexer { await fs.writeFile(healthPath, JSON.stringify(health, null, 2)); // Write manifest (both full and incremental) - // For full rebuild, write to staging; for incremental, write to active - const activeManifestPath = path.join(activeContextDir, MANIFEST_FILENAME); + // Full and incremental builds both write an immutable staged generation. + const activeManifestPath = path.join(generationDirectory, MANIFEST_FILENAME); + const analyzedHashes = Object.fromEntries(analyzedManifestHashes); const manifest: FileManifest = { version: 1, generatedAt: new Date().toISOString(), - files: currentHashes ?? (await computeFileHashes(files, this.rootPath)) + header: { buildId, formatVersion: INDEX_FORMAT_VERSION }, + files: analyzedHashes }; await writeManifest(activeManifestPath, manifest); @@ -1016,11 +1209,11 @@ export class CodebaseIndexer { totalFiles: stats.totalFiles, generatedAt }; - const activeIndexingStatsPath = path.join(activeContextDir, INDEXING_STATS_FILENAME); + const activeIndexingStatsPath = path.join(generationDirectory, INDEXING_STATS_FILENAME); await fs.writeFile(activeIndexingStatsPath, JSON.stringify(persistedStats, null, 2)); - // Index meta (authoritative) — write last so readers never observe meta pointing to missing artifacts. - const metaPath = path.join(activeContextDir, INDEX_META_FILENAME); + // Complete the staged generation metadata before the separate root-pointer publication. + const metaPath = path.join(generationDirectory, INDEX_META_FILENAME); await fs.writeFile( metaPath, JSON.stringify( @@ -1036,7 +1229,8 @@ export class CodebaseIndexer { path: VECTOR_DB_DIRNAME, provider: 'lancedb', embeddingProvider: this.config.embedding?.provider ?? 'transformers', - embeddingModel: this.config.embedding?.model ?? DEFAULT_MODEL + embeddingModel: this.config.embedding?.model ?? DEFAULT_MODEL, + embeddingEnabled: !this.config.skipEmbedding }, intelligence: { path: INTELLIGENCE_FILENAME }, manifest: { path: MANIFEST_FILENAME }, @@ -1050,11 +1244,11 @@ export class CodebaseIndexer { ) ); - // Atomic swap for full rebuilds: move staging into active location - if (isFullRebuild && stagingDir) { - console.error('Performing atomic swap of staging to active...'); - await atomicSwapStagingToActive(contextDir, stagingDir, buildId); - } + const generationMeta = await readIndexMetaFromFile(metaPath); + // Once publication starts, only retention may remove the generation. Even a + // post-commit housekeeping fault must not enter the staging rollback path. + stagingDir = null; + await publishIndexGeneration(this.rootPath, generationMeta); // Phase 5: Complete this.updateProgress('complete', 100); @@ -1091,9 +1285,10 @@ export class CodebaseIndexer { } } - private async scanFiles(): Promise { - const files: string[] = []; + private async scanFiles(warnOnLargeFiles = true): Promise { + const scan: FileScanResult = { files: [], skippedFiles: [], skippedFileCount: 0 }; const seen = new Set(); + this.rootPath = await fs.realpath(this.rootPath); // Read .gitignore if respecting it let ig: ReturnType | null = null; @@ -1127,33 +1322,55 @@ export class CodebaseIndexer { seen.add(normalizedFile); const relativePath = path.relative(this.rootPath, file); + const sourcePath = await resolveContainedSourcePath(file, this.rootPath); + if (sourcePath.status !== 'contained') { + addSkippedFile( + scan, + file, + sourcePath.status === 'outside' + ? 'source resolves outside selected root' + : sourcePath.status === 'alias' + ? 'source alias is not indexed; use its canonical path' + : 'file could not be inspected', + this.rootPath + ); + continue; + } // Check gitignore if (ig && ig.ignores(relativePath)) { + addSkippedFile(scan, file, 'ignored by .gitignore', this.rootPath); continue; } // Check if it's a code file - if (!isCodeFile(file, this.supportedCodeExtensions) || isBinaryFile(file)) { + if ( + !isCodeFile(sourcePath.path, this.supportedCodeExtensions) || + isBinaryFile(sourcePath.path) + ) { + addSkippedFile(scan, file, 'unsupported or binary file', this.rootPath); continue; } // Check file size try { - const stats = await fs.stat(file); + const stats = await fs.stat(sourcePath.path); if (stats.size > (this.config.parsing?.maxFileSize || 1048576)) { - console.warn(`Skipping large file: ${file} (${stats.size} bytes)`); + if (warnOnLargeFiles) + console.warn(`Skipping large file: ${file} (${stats.size} bytes)`); + addSkippedFile(scan, file, 'exceeds configured maximum file size', this.rootPath); continue; } } catch (_error) { + addSkippedFile(scan, file, 'file could not be inspected', this.rootPath); continue; } - files.push(file); + scan.files.push(sourcePath.path); } } - return files; + return scan; } private updateProgress(phase: IndexingPhase, percentage: number): void { @@ -1166,6 +1383,10 @@ export class CodebaseIndexer { } async detectMetadata(): Promise { + return withAvailableIndexSnapshot(this.rootPath, () => this.detectPinnedMetadata()); + } + + private async detectPinnedMetadata(): Promise { // Get all registered analyzers (sorted by priority, highest first) const analyzers = analyzerRegistry.getAll(); @@ -1229,11 +1450,7 @@ export class CodebaseIndexer { // Load intelligence data if available try { - const intelligencePath = path.join( - this.rootPath, - CODEBASE_CONTEXT_DIRNAME, - INTELLIGENCE_FILENAME - ); + const intelligencePath = await resolveIndexArtifact(this.rootPath, INTELLIGENCE_FILENAME); const intelligenceContent = await fs.readFile(intelligencePath, 'utf-8'); const intelligence = JSON.parse(intelligenceContent) as IntelligenceData; diff --git a/src/core/manifest.ts b/src/core/manifest.ts index 04f7c61..fdadd27 100644 --- a/src/core/manifest.ts +++ b/src/core/manifest.ts @@ -1,3 +1,4 @@ +import { currentIndexSnapshot, resolveIndexArtifact } from './index-generation.js'; /** * File hash manifest for incremental indexing. * Tracks SHA-256 hashes of indexed files to detect changes between runs. @@ -6,13 +7,34 @@ import { createHash } from 'crypto'; import { promises as fs } from 'fs'; import path from 'path'; +import { z } from 'zod'; + +import { CODEBASE_CONTEXT_DIRNAME, MANIFEST_FILENAME } from '../constants/codebase-context.js'; +import { IndexCorruptedError } from '../errors/index.js'; +import { + ArtifactHeaderSchema, + validateArtifactBuildIdentity, + type ArtifactHeader, + type IndexMeta +} from './index-meta.js'; +import { isPathWithin } from '../utils/project-discovery.js'; export interface FileManifest { version: 1; generatedAt: string; files: Record; // relativePath → SHA-256 hash (first 16 hex chars) + header?: ArtifactHeader; } +const ReferenceManifestSchema = z + .object({ + version: z.literal(1), + generatedAt: z.string().datetime(), + header: ArtifactHeaderSchema, + files: z.record(z.string(), z.string().regex(/^[a-f0-9]{16}$/i)) + }) + .strict(); + export interface ManifestDiff { added: string[]; // new files (not in old manifest) changed: string[]; // hash differs @@ -44,6 +66,107 @@ export async function readManifest(manifestPath: string): Promise !segment || segment === '.' || segment === '..')) { + return false; + } + + const resolved = path.resolve(rootPath, relativePath); + if (!isPathWithin(rootPath, resolved)) return false; + return path.relative(rootPath, resolved).replace(/\\/g, '/') === normalized; +} + +/** Read the build-bound source nominator used by current-source reference lookup. */ +export async function readReferenceManifest( + rootPath: string, + meta: IndexMeta +): Promise { + if (!meta.artifacts.manifest?.path) { + throw fullReindexError('Reference manifest path is missing or invalid.'); + } + + let realRoot: string; + let realContext: string; + try { + realRoot = currentIndexSnapshot(rootPath)?.realRoot ?? (await fs.realpath(rootPath)); + realContext = await fs.realpath(path.join(realRoot, CODEBASE_CONTEXT_DIRNAME)); + } catch (error) { + throw fullReindexError( + 'Reference manifest directory is missing or unreadable.', + error instanceof Error ? error.message : String(error) + ); + } + if (!isPathWithin(realRoot, realContext)) { + throw fullReindexError('Reference manifest directory resolves outside the selected project.'); + } + + const expectedManifestPath = await resolveIndexArtifact(rootPath, MANIFEST_FILENAME, meta); + let realManifestPath: string; + try { + realManifestPath = await fs.realpath(expectedManifestPath); + } catch (error) { + throw fullReindexError( + 'Reference manifest is missing or unreadable.', + error instanceof Error ? error.message : String(error) + ); + } + if (!isPathWithin(realContext, realManifestPath)) { + throw fullReindexError('Reference manifest resolves outside the index directory.'); + } + + let json: unknown; + try { + json = JSON.parse(await fs.readFile(realManifestPath, 'utf8')); + } catch (error) { + throw fullReindexError( + 'Reference manifest is malformed or unreadable.', + error instanceof Error ? error.message : String(error) + ); + } + + const parsed = ReferenceManifestSchema.safeParse(json); + if (!parsed.success) { + const details = parsed.error.issues.some((issue) => issue.path[0] === 'header') + ? 'The source manifest is legacy or has an invalid build header.' + : `Manifest schema mismatch: ${parsed.error.message}`; + throw fullReindexError('Reference manifest is invalid.', details); + } + + try { + validateArtifactBuildIdentity( + parsed.data.header, + meta, + 'Reference manifest', + MANIFEST_FILENAME + ); + } catch (error) { + throw fullReindexError( + 'Reference manifest build identity does not match index metadata.', + error instanceof Error ? error.message : String(error) + ); + } + + for (const relativePath of Object.keys(parsed.data.files)) { + if (!isSafeManifestSourcePath(realRoot, relativePath)) { + throw new IndexCorruptedError( + `Reference manifest contains an invalid or escaping source path (${relativePath}). Rebuild required.` + ); + } + } + + return parsed.data; +} + /** * Write a manifest to disk. */ diff --git a/src/core/search-quality.ts b/src/core/search-quality.ts index 49c1938..f7e9e80 100644 --- a/src/core/search-quality.ts +++ b/src/core/search-quality.ts @@ -18,6 +18,9 @@ export function isTestArtifactPath(filePath: string): boolean { ); } +const LOW_CONFIDENCE_HINT = + 'Low confidence: this search is not evidence that the code is missing. Try one concrete symbol, file, or route, or call get_symbol_references.'; + export function assessSearchQuality( query: string, results: SearchResult[] @@ -28,7 +31,7 @@ export function assessSearchQuality( confidence: 0, signals: ['no results returned'], nextSteps: [ - 'Try a narrower query with one concrete symbol, route, or file hint.', + LOW_CONFIDENCE_HINT, 'Apply search filters (framework/language/componentType/layer).', 'Use get_symbol_references to find where a specific symbol is used across the codebase.' ] @@ -74,7 +77,7 @@ export function assessSearchQuality( signals, ...(lowConfidence && { nextSteps: [ - 'Add one or two concrete symbols, routes, or file hints to the query.', + LOW_CONFIDENCE_HINT, 'Apply filters (framework/language/componentType/layer) to narrow candidates.', 'Use get_symbol_references when the question is about where a symbol or function is used.' ] diff --git a/src/core/search.ts b/src/core/search.ts index 4a9be1b..743a3b2 100644 --- a/src/core/search.ts +++ b/src/core/search.ts @@ -1,19 +1,37 @@ +import { + withAvailableIndexSnapshot, + currentIndexSnapshot, + resolveIndexArtifact +} from './index-generation.js'; /** * Hybrid search combining semantic vector search with keyword matching */ -import Fuse from 'fuse.js'; +import Fuse, { type FuseResult, type IFuseOptions } from 'fuse.js'; import path from 'path'; import { promises as fs } from 'fs'; import { CodeChunk, SearchResult, SearchFilters, IntelligenceData } from '../types/index.js'; -import { EmbeddingProvider, EmbeddingConfig, getEmbeddingProvider } from '../embeddings/index.js'; +import { + EmbeddingProvider, + EmbeddingConfig, + getEmbeddingProvider, + parseEmbeddingProviderName +} from '../embeddings/index.js'; import { VectorStorageProvider, getStorageProvider } from '../storage/index.js'; import { analyzerRegistry } from './analyzer-registry.js'; -import { IndexCorruptedError } from '../errors/index.js'; +import { IndexCorruptedError, OptionalDependencyError } from '../errors/index.js'; import { isTestingRelatedQuery } from '../preflight/query-scope.js'; import { assessSearchQuality } from './search-quality.js'; -import { rerank, getRerankerStatus } from './reranker.js'; +import { rerank } from './reranker.js'; import { type IndexMeta, readIndexMeta, validateIndexArtifacts } from './index-meta.js'; +import { hashFileContent, readReferenceManifest, type FileManifest } from './manifest.js'; +import { isPathWithin } from '../utils/project-discovery.js'; +import { isSourceBodyWithinInterval } from '../utils/source-interval.js'; +import { + InternalSourceResolver, + loadTsconfigCompilerOptions +} from '../utils/internal-source-resolver.js'; +import type { ImportEdgeDetail } from '../utils/usage-tracker.js'; import { CODEBASE_CONTEXT_DIRNAME, INTELLIGENCE_FILENAME, @@ -36,6 +54,39 @@ export interface SearchOptions { export type SearchIntentProfile = 'explore' | 'edit' | 'refactor' | 'migrate'; +/** Coverage of nominated sources, not a claim that the entire repository was searched. */ +export interface SearchSourceCoverage { + status: 'current' | 'partial' | 'no_fresh_results'; + checkedFiles: number; + skippedFiles: number; + candidateBudgetExhausted?: true; + candidateRetrievalFailed?: true; + refreshAction?: string; +} + +export interface SearchOutcome { + results: SearchResult[]; + sourceCoverage: SearchSourceCoverage; +} + +// Null records ordinary drift/unavailability; integrity failures still throw. +type CheckedSources = Map; + +interface SourceValidationContext { + sources: CheckedSources; + realRootPath?: Promise; + candidateBudgetExhausted?: true; + candidateRetrievalFailed?: true; +} + +interface SourceInspectionBudget { + limit: number; + inspected: Set; +} + +// Reuse the lexical rescue work ceiling for bounded stale-candidate replenishment. +const MAX_CANDIDATE_WORK = 750; + type QueryIntent = 'EXACT_NAME' | 'CONCEPTUAL' | 'FLOW' | 'CONFIG' | 'WIRING'; interface QueryVariant { @@ -108,6 +159,44 @@ const QUERY_STOP_WORDS = new Set([ 'before' ]); +// Syntax and question scaffolding are weak evidence after strict keyword +// retrieval fails, so they cannot independently support a rescue match. +const NON_DISCRIMINATIVE_FALLBACK_TERMS = new Set([ + 'class', + 'code', + 'const', + 'does', + 'export', + 'file', + 'function', + 'import', + 'return', + 'something', + 'still', + 'thing', + 'value', + 'what', + 'when', + 'where', + 'why' +]); + +const KEYWORD_FUSE_OPTIONS: IFuseOptions = { + keys: [ + { name: 'content', weight: 0.4 }, + { name: 'metadata.componentName', weight: 0.25 }, + { name: 'filePath', weight: 0.15 }, + { name: 'relativePath', weight: 0.15 }, + { name: 'componentType', weight: 0.15 }, + { name: 'layer', weight: 0.1 }, + { name: 'tags', weight: 0.15 } + ], + includeScore: true, + threshold: 0.4, + useExtendedSearch: true, + ignoreLocation: true +}; + export class CodebaseSearcher { private rootPath: string; private storagePath: string; @@ -121,6 +210,7 @@ export class CodebaseSearcher { private storageProvider: VectorStorageProvider | null = null; private initialized = false; + private sourceManifest: FileManifest | null = null; // Pattern intelligence for trend detection private patternIntelligence: { @@ -130,38 +220,98 @@ export class CodebaseSearcher { } | null = null; private importCentrality: Map | null = null; + private indexedImports = new Map>(); constructor(rootPath: string) { this.rootPath = rootPath; this.storagePath = path.join(rootPath, CODEBASE_CONTEXT_DIRNAME, VECTOR_DB_DIRNAME); } + private operationTail: Promise = Promise.resolve(); + + private async runOperation(operation: () => Promise): Promise { + const preceding = this.operationTail; + let release: () => void = () => undefined; + this.operationTail = new Promise((resolve) => { + release = resolve; + }); + await preceding; + try { + return await withAvailableIndexSnapshot(this.rootPath, async () => { + const snapshot = currentIndexSnapshot(this.rootPath); + if (snapshot) this.rootPath = snapshot.realRoot; + if (snapshot && snapshot.meta.buildId !== this.indexMeta?.buildId) { + this.initialized = false; + this.embeddingProvider = null; + this.storageProvider = null; + this.chunks = []; + this.fuseIndex = null; + this.patternIntelligence = null; + this.importCentrality = null; + this.indexedImports.clear(); + } + if ( + this.initialized && + this.indexMeta?.artifacts.vectorDb.embeddingEnabled !== false && + !this.storageProvider && + this.indexMeta + ) { + // Reopening follows the same fail-closed/keyword-fallback policy as first use. + try { + await this.initializeSemanticProviders(); + } catch (error) { + if (error instanceof OptionalDependencyError || error instanceof IndexCorruptedError) + throw error; + console.warn('Partial initialization (keyword search only):', error); + } + } + try { + return await operation(); + } finally { + const storage = this.storageProvider; + if (storage?.close) { + this.storageProvider = null; + try { + await storage.close(); + } catch (error) { + console.warn('Search completed; native handle cleanup failed:', error); + } + } + } + }); + } finally { + release(); + } + } + async initialize(): Promise { + return this.runOperation(() => this.initializePinned()); + } + + private async initializePinned(): Promise { if (this.initialized) return; try { // Fail closed on version mismatch/corruption before serving any results. this.indexMeta = await readIndexMeta(this.rootPath); await validateIndexArtifacts(this.rootPath, this.indexMeta); + this.storagePath = await resolveIndexArtifact( + this.rootPath, + VECTOR_DB_DIRNAME, + this.indexMeta + ); await this.loadKeywordIndex(); + await this.loadSourceManifest(); await this.loadPatternIntelligence(); - // Use the embedding config the index was built with, not the current env-var defaults. - // This ensures query vectors are in the same space as stored vectors. - // Legacy indexes (no embeddingProvider stored) fall back to env-var defaults. - const storedProvider = this.indexMeta.artifacts.vectorDb.embeddingProvider; - const storedModel = this.indexMeta.artifacts.vectorDb.embeddingModel; - const embeddingConfig: Partial = storedProvider - ? { provider: storedProvider as EmbeddingConfig['provider'], model: storedModel } - : {}; - this.embeddingProvider = await getEmbeddingProvider(embeddingConfig); - this.storageProvider = await getStorageProvider({ - path: this.storagePath - }); + await this.initializeSemanticProviders(); this.initialized = true; } catch (error) { + if (error instanceof OptionalDependencyError) { + throw error; + } if (error instanceof IndexCorruptedError) { throw error; // Propagate to handler for auto-heal } @@ -170,9 +320,183 @@ export class CodebaseSearcher { } } + private async initializeSemanticProviders(): Promise { + const stored = this.indexMeta?.artifacts.vectorDb; + if (!stored || stored.embeddingEnabled === false) return; + const provider = parseEmbeddingProviderName(stored.embeddingProvider); + if (stored.embeddingProvider && !provider) + throw new IndexCorruptedError('Recorded embedding provider is invalid. Rebuild required.'); + const embeddingConfig: Partial = provider + ? { provider, model: stored.embeddingModel } + : {}; + // Assign the pair only after both are ready. A new model cannot inherit an old + // query provider when initialization falls back to keyword search. + const embedding = await getEmbeddingProvider(embeddingConfig); + const storage = await getStorageProvider( + { path: this.storagePath }, + { expectExisting: this.chunks.length > 0 } + ); + this.embeddingProvider = embedding; + this.storageProvider = storage; + } + + private getIndexedSourcePath(chunk: CodeChunk): string { + if (typeof chunk.filePath !== 'string' || !chunk.filePath.trim()) { + throw new IndexCorruptedError('Indexed source location is invalid. Full rebuild required.'); + } + + const indexedPath = path.resolve(this.rootPath, chunk.filePath); + if (!isPathWithin(this.rootPath, indexedPath)) { + throw new IndexCorruptedError( + 'Indexed source belongs to a different project location. Full rebuild required.' + ); + } + + if (typeof chunk.relativePath === 'string' && chunk.relativePath.trim()) { + const rawRelativePath = chunk.relativePath.trim(); + if (!path.isAbsolute(rawRelativePath)) { + const relativeSourcePath = path.resolve(this.rootPath, rawRelativePath); + if ( + isPathWithin(this.rootPath, relativeSourcePath) && + path.relative(relativeSourcePath, indexedPath) !== '' + ) { + throw new IndexCorruptedError( + 'Indexed source belongs to a different project location. Full rebuild required.' + ); + } + } + } + + // Some existing analyzers recorded relativePath from process.cwd(). The absolute + // filePath remains authoritative when it is confined to the selected root. + return indexedPath; + } + + private async loadSourceManifest(): Promise { + if (!this.indexMeta) + throw new IndexCorruptedError( + 'Source manifest metadata is unavailable. Full rebuild required.' + ); + const manifest = await readReferenceManifest(this.rootPath, this.indexMeta); + + // Reject copied indexes with foreign absolute source locations before loading models. + for (const chunk of this.chunks) this.getIndexedSourcePath(chunk); + this.sourceManifest = manifest; + } + + private async validateCandidateSources( + chunks: Iterable, + validation: SourceValidationContext, + budget?: SourceInspectionBudget + ): Promise { + const checkedSources = validation.sources; + if (!this.sourceManifest) { + throw new IndexCorruptedError('Source manifest is unavailable. Full rebuild required.'); + } + const snapshot = currentIndexSnapshot(this.rootPath); + validation.realRootPath ??= ( + snapshot ? Promise.resolve(snapshot.realRoot) : fs.realpath(this.rootPath) + ).catch(() => { + throw new IndexCorruptedError('Selected project is unavailable. Full rebuild required.'); + }); + const realRootPath = await validation.realRootPath; + for (const chunk of chunks) { + const sourcePath = this.getIndexedSourcePath(chunk); + const sourceKey = path.resolve(this.rootPath, chunk.filePath); + const relativePath = path.relative(this.rootPath, sourcePath).replace(/\\/g, '/'); + const expectedHash = this.sourceManifest.files[relativePath]; + if (typeof expectedHash !== 'string' || !/^[a-f0-9]{16}$/.test(expectedHash)) { + throw new IndexCorruptedError( + 'Indexed source is absent from its manifest or its hash is invalid. Full rebuild required.' + ); + } + this.validateSourceRange(chunk); + if (!checkedSources.has(sourceKey)) { + if (budget && budget.inspected.size >= budget.limit) { + validation.candidateBudgetExhausted = true; + continue; + } + budget?.inspected.add(sourceKey); + try { + const realSourcePath = await fs.realpath(sourcePath); + if (!isPathWithin(realRootPath, realSourcePath)) { + throw new IndexCorruptedError( + 'Source resolves outside the selected project. Full rebuild required.' + ); + } + if (!(await fs.stat(realSourcePath)).isFile()) { + throw new IndexCorruptedError( + 'Indexed source is not a regular file. Full rebuild required.' + ); + } + const content = await fs.readFile(realSourcePath, 'utf-8'); + checkedSources.set( + sourceKey, + hashFileContent(content) === expectedHash + ? content.replace(/\r\n/g, '\n').split('\n') + : null + ); + } catch (error) { + if (error instanceof IndexCorruptedError) throw error; + // Source reads can fail after a successful stat (delete, permissions, I/O). + // Preserve other candidates and report incomplete coverage without rebuilding. + checkedSources.set(sourceKey, null); + } + } + const lines = checkedSources.get(sourceKey); + if (lines) { + this.validateSourceRange(chunk, lines); + // The source read is cached per file; each nominated body still needs + // its own proof, including siblings sharing the same source lines. + if (!isSourceBodyWithinInterval(chunk, lines)) + throw new IndexCorruptedError( + 'Indexed chunk body does not match its source interval. Full rebuild required.' + ); + } + } + } + + private async selectCurrentCandidates( + candidates: Iterable, + limit: number, + getChunk: (candidate: T) => CodeChunk, + validation: SourceValidationContext, + budget: SourceInspectionBudget = { limit, inspected: new Set() } + ): Promise { + const current: T[] = []; + for (const candidate of candidates) { + if (current.length >= limit) break; + const chunk = getChunk(candidate); + await this.validateCandidateSources([chunk], validation, budget); + const source = validation.sources.get(path.resolve(this.rootPath, chunk.filePath)); + if (source === undefined) break; // Inspection budget ended before this source. + if (source !== null) current.push(candidate); + } + return current; + } + + private validateSourceRange( + range: Pick, + lines?: string[] + ): void { + if ( + !Number.isInteger(range.startLine) || + !Number.isInteger(range.endLine) || + range.startLine < 1 || + range.endLine < range.startLine || + (lines !== undefined && range.endLine > lines.length) + ) { + throw new IndexCorruptedError('Indexed source line range is invalid. Full rebuild required.'); + } + } + private async loadKeywordIndex(): Promise { try { - const indexPath = path.join(this.rootPath, CODEBASE_CONTEXT_DIRNAME, KEYWORD_INDEX_FILENAME); + const indexPath = await resolveIndexArtifact( + this.rootPath, + KEYWORD_INDEX_FILENAME, + this.indexMeta ?? undefined + ); const content = await fs.readFile(indexPath, 'utf-8'); const parsed = JSON.parse(content) as unknown; @@ -193,21 +517,7 @@ export class CodebaseSearcher { this.chunks = chunks; - this.fuseIndex = new Fuse(this.chunks, { - keys: [ - { name: 'content', weight: 0.4 }, - { name: 'metadata.componentName', weight: 0.25 }, - { name: 'filePath', weight: 0.15 }, - { name: 'relativePath', weight: 0.15 }, - { name: 'componentType', weight: 0.15 }, - { name: 'layer', weight: 0.1 }, - { name: 'tags', weight: 0.15 } - ], - includeScore: true, - threshold: 0.4, - useExtendedSearch: true, - ignoreLocation: true - }); + this.fuseIndex = new Fuse(this.chunks, KEYWORD_FUSE_OPTIONS); } catch (error) { if (error instanceof IndexCorruptedError) { throw error; @@ -222,11 +532,12 @@ export class CodebaseSearcher { * Load pattern intelligence for trend detection and warnings */ private async loadPatternIntelligence(): Promise { + this.indexedImports.clear(); try { - const intelligencePath = path.join( + const intelligencePath = await resolveIndexArtifact( this.rootPath, - CODEBASE_CONTEXT_DIRNAME, - INTELLIGENCE_FILENAME + INTELLIGENCE_FILENAME, + this.indexMeta ?? undefined ); const content = await fs.readFile(intelligencePath, 'utf-8'); const intelligence = JSON.parse(content) as IntelligenceData; @@ -272,12 +583,37 @@ export class CodebaseSearcher { // Count how many files import each file (in-degree centrality) const importCounts = new Map(); - for (const [_importingFile, importedFiles] of Object.entries( + const details = intelligence.internalFileGraph.importDetails as + | Record> + | undefined; + for (const [importingFile, importedFiles] of Object.entries( intelligence.internalFileGraph.imports )) { const imports = importedFiles as string[]; for (const imported of imports) { importCounts.set(imported, (importCounts.get(imported) || 0) + 1); + const sources = details?.[importingFile]?.[imported]?.sources; + const sourceHash = this.sourceManifest?.files[importingFile]; + const targetHash = this.sourceManifest?.files[imported]; + if (!sourceHash || !targetHash || !Array.isArray(sources)) continue; + const witnesses = sources.filter( + (source) => + source.kind === 'import' && + source.line > 0 && + !!source.specifier && + /^[a-f0-9]{64}$/.test(source.sourceFingerprint) && + /^[a-f0-9]{64}$/.test(source.targetFingerprint) && + source.sourceFingerprint.slice(0, 16) === sourceHash && + source.targetFingerprint.slice(0, 16) === targetHash + ); + if (witnesses.length === 0) continue; + const key = this.indexedFileKey(importingFile); + const targets = this.indexedImports.get(key) ?? new Map(); + targets.set( + this.indexedFileKey(imported), + witnesses.map((source) => source.specifier) + ); + this.indexedImports.set(key, targets); } } @@ -298,6 +634,7 @@ export class CodebaseSearcher { ); this.patternIntelligence = null; this.importCentrality = null; + this.indexedImports.clear(); } } @@ -352,6 +689,361 @@ export class CodebaseSearcher { .filter((term) => term.length > 2 && !QUERY_STOP_WORDS.has(term)); } + private normalizeLexicalTerm(term: string): string { + const lower = term.toLowerCase(); + if (lower.length > 5 && lower.endsWith('ies')) { + return `${lower.slice(0, -3)}y`; + } + if (lower.length > 4 && lower.endsWith('s') && !/(ss|us)$/.test(lower)) { + return lower.slice(0, -1); + } + return lower; + } + + private lexicalWords(text: string): string[] { + return ( + text + .replace(/([A-Z]+)([A-Z][a-z])/g, '$1 $2') + .replace(/([a-z\d])([A-Z])/g, '$1 $2') + .match(/[a-zA-Z0-9]+/g) ?? [] + ).map((word) => this.normalizeLexicalTerm(word)); + } + + private lexicalAliases(word: string): string[] { + // Keep the original word: conservative suffix aliases improve recall but + // must not erase exact evidence. Query and indexed words use identical rules. + const aliases = new Set([word]); + const stem = word.replace(/(?:ed|ing)$/, ''); + if (stem !== word && stem.length >= 4) { + aliases.add(stem); + aliases.add(`${stem}e`); + if (/([b-df-hj-np-tv-z])\1$/.test(stem)) aliases.add(stem.slice(0, -1)); + } + return [...aliases]; + } + + private lexicalQueryTerms(query: string): Map { + const groups = new Map(); + // An explicit code identifier remains one anchor, governed by the indexed + // edge proof below. Its component words cannot corroborate that anchor. + const naturalTerms = query.replace(/[a-zA-Z_][a-zA-Z0-9_]*/g, (word) => + /[a-z][A-Z]|_/.test(word) ? '' : word + ); + const words = [...new Set(this.lexicalWords(naturalTerms))] + .filter( + (term) => + term.length >= 4 && + !QUERY_STOP_WORDS.has(term) && + !NON_DISCRIMINATIVE_FALLBACK_TERMS.has(term) + ) + .slice(0, 12); + for (const word of words) { + const aliases = new Set(this.lexicalAliases(word)); + let key = word; + // Repeated inflections are one clue, never independent corroboration. + for (const [existing, values] of groups) { + if (!values.some((value) => aliases.has(value))) continue; + key = existing; + for (const value of values) aliases.add(value); + groups.delete(existing); + } + groups.set(key, [...aliases]); + } + return groups; + } + + private lexicalEvidence(text: string): Set { + return new Set(this.lexicalWords(text).flatMap((word) => this.lexicalAliases(word))); + } + + private indexedDeclarationNames(chunk: Pick): string[] { + return [ + chunk.metadata?.symbolName, + chunk.metadata?.componentName, + chunk.metadata?.functionName, + chunk.metadata?.className + ] + .filter((name): name is string => typeof name === 'string') + .flatMap((name) => name.split('+')); + } + + private hasExactQueryDeclaration(chunk: Pick, query: string): boolean { + const identifiers = new Set( + (/^[a-zA-Z_][a-zA-Z0-9_]*$/.test(query.trim()) + ? [query.trim()] + : (query.match(/[a-zA-Z_][a-zA-Z0-9_]*/g) ?? []).filter((word) => /[a-z][A-Z]|_/.test(word)) + ).map((word) => word.toLowerCase()) + ); + return this.indexedDeclarationNames(chunk).some((name) => identifiers.has(name.toLowerCase())); + } + + private matchesSearchFilters(chunk: CodeChunk, filters?: SearchFilters): boolean { + if (!filters) return true; + if (filters.componentType && chunk.componentType !== filters.componentType) return false; + if (filters.layer && chunk.layer !== filters.layer) return false; + if (filters.framework && chunk.framework !== filters.framework) return false; + if (filters.language && chunk.language !== filters.language) return false; + if (filters.tags?.length && !filters.tags.some((tag) => chunk.tags?.includes(tag))) { + return false; + } + return true; + } + + private indexedFileKey(file: string): string { + const absolute = path.resolve(this.rootPath, file).replace(/\\/g, '/'); + return process.platform === 'win32' ? absolute.toLowerCase() : absolute; + } + + /** + * Extended Fuse treats spaces as AND within one chunk. If strict retrieval + * yields nothing, admit files with multiple lexical clues or an exact symbol + * whose indexed import edge connects it to a multi-clue companion. + */ + private async rescueKeywordMatches( + query: string, + limit: number, + filters?: SearchFilters, + validation: SourceValidationContext = { sources: new Map() }, + sourceBudget?: SourceInspectionBudget, + retrievedCandidates?: FuseResult[] + ): Promise[]> { + const checkedSources = validation.sources; + if (!this.fuseIndex || /[|^$!=]/.test(query)) return []; + + const queryTerms = this.lexicalQueryTerms(query); + const terms = [...queryTerms.keys()]; + if (terms.length < 2) return []; + + const symbolAnchors = (query.match(/[a-zA-Z_][a-zA-Z0-9_]*/g) ?? []) + .filter((term) => /[a-z][A-Z]/.test(term) && term.length >= 8) + .map((term) => term.toLowerCase()); + + const candidateLimit = Math.min(MAX_CANDIDATE_WORK, Math.max(256, limit * 80)); + const budget = sourceBudget ?? { limit: candidateLimit, inspected: new Set() }; + const aliases = [...new Set([...queryTerms.values()].flat())]; + // Invert the existing suffix aliases conservatively: policy/policies share + // "polic", and archive/archiving share "archiv". These fragments only nominate; + // the unchanged exact lexical evidence below still decides admission. + const fragments = [...new Set(aliases.map((alias) => alias.replace(/[ey]$/, '')))]; + const lookupTerms = fragments.filter( + (fragment) => !fragments.some((other) => other !== fragment && fragment.includes(other)) + ); + // Keep original chunk ordinals, including ties, and nominate across every + // exact evidence zone. Filters apply before the bounded evidence window. + const rawCandidates = + retrievedCandidates ?? + this.chunks.flatMap((item, refIndex): FuseResult[] => { + if (!this.matchesSearchFilters(item, filters)) return []; + const fields = [ + item.content, + path.basename(item.relativePath), + ...this.indexedDeclarationNames(item), + ...(item.exports ?? []) + ]; + const matches = fields.some((field) => { + const text = field.toLowerCase(); + return ( + lookupTerms.some((term) => text.includes(term)) || + symbolAnchors.some((anchor) => text.includes(anchor)) + ); + }); + return matches ? [{ item, refIndex, score: 0 }] : []; + }); + const availableCandidates = rawCandidates.filter( + ({ item }) => checkedSources.get(path.resolve(this.rootPath, item.filePath)) !== null + ); + if (availableCandidates.length > candidateLimit) validation.candidateBudgetExhausted = true; + const candidates = availableCandidates.slice(0, candidateLimit); + const unavailableBefore = [...checkedSources.values()].filter( + (source) => source === null + ).length; + + const queryAliases = new Map( + terms.map((term) => [term, new Set(queryTerms.get(term)!)] as const) + ); + const evidence = candidates.map((result) => { + const { item } = result; + const bodyWords = this.lexicalWords(item.content); + const bodyLexemes = new Set(); + const nonBodyLexemes = this.lexicalEvidence( + [ + item.metadata?.componentName ?? '', + item.metadata?.symbolName ?? '', + path.basename(item.relativePath) + ].join(' ') + ); + const termFrequencies = new Map(terms.map((term) => [term, 0])); + for (const word of bodyWords) { + const wordAliases = this.lexicalAliases(word); + for (const alias of wordAliases) bodyLexemes.add(alias); + for (const term of terms) { + if (wordAliases.some((alias) => queryAliases.get(term)!.has(alias))) { + termFrequencies.set(term, termFrequencies.get(term)! + 1); + } + } + } + const lexemes = new Set([...bodyLexemes, ...nonBodyLexemes]); + const matching = new Set( + terms.filter((term) => queryTerms.get(term)!.some((alias) => lexemes.has(alias))) + ); + const density = + terms.reduce((total, term) => total + Math.sqrt(termFrequencies.get(term)!), 0) / + Math.sqrt(Math.max(1, bodyWords.length)); + // Export names remain useful lexical/graph clues, but can be re-exports; + // exact definition priority below requires own declaration metadata. + const indexedSymbols = [...this.indexedDeclarationNames(item), ...(item.exports ?? [])]; + const exactAnchor = symbolAnchors.some((symbol) => + indexedSymbols.some( + (indexedSymbol) => + typeof indexedSymbol === 'string' && indexedSymbol.toLowerCase() === symbol + ) + ); + const declarationLexemes = this.lexicalEvidence(indexedSymbols.filter(Boolean).join(' ')); + const declarationMatching = new Set( + terms.filter((term) => queryTerms.get(term)!.some((alias) => declarationLexemes.has(alias))) + ); + return { result, matching, declarationMatching, exactAnchor, density }; + }); + + const supportedCompanions = new Map( + evidence + .filter(({ matching }) => matching.size >= 2) + .map(({ result }) => [this.indexedFileKey(result.item.filePath), result.item] as const) + ); + const anchors = new Map( + evidence + .filter(({ exactAnchor }) => exactAnchor) + .map(({ result }) => [this.indexedFileKey(result.item.filePath), result.item] as const) + ); + const corroboratedFiles = new Set(); + const corroboratingTargets = new Map>(); + const corroboratingSources = new Map(); + // Visit each anchor's indexed adjacency once; do not rescan all chunks or + // infer target extensions from source text. + for (const [key, anchor] of anchors) { + for (const target of this.indexedImports.get(key)?.keys() ?? []) { + const companion = supportedCompanions.get(target); + if (!companion || target === key) continue; + const targets = corroboratingTargets.get(key) ?? new Set(); + targets.add(target); + corroboratingTargets.set(key, targets); + corroboratingSources.set(key, anchor); + corroboratingSources.set(target, companion); + } + } + // A stale endpoint cannot lend confidence to a current anchor, even when + // the companion would otherwise fall outside the returned result limit. + if (corroboratingSources.size > 0) { + await this.validateCandidateSources(corroboratingSources.values(), validation, budget); + // Only candidate endpoints enter the resolver. Re-read compiler options for + // each query: unchanged endpoint bytes do not prove unchanged alias routing. + const resolver = await InternalSourceResolver.create( + this.rootPath, + [...corroboratingSources.values()] + .filter( + (chunk) => checkedSources.get(path.resolve(this.rootPath, chunk.filePath)) != null + ) + .map((chunk) => path.resolve(this.rootPath, chunk.filePath)), + { compilerOptions: loadTsconfigCompilerOptions(this.rootPath) } + ); + for (const [key, targets] of corroboratingTargets) { + const anchor = corroboratingSources.get(key)!; + if (checkedSources.get(path.resolve(this.rootPath, anchor.filePath)) == null) continue; + for (const target of targets) { + const companion = corroboratingSources.get(target)!; + if (checkedSources.get(path.resolve(this.rootPath, companion.filePath)) == null) continue; + for (const specifier of this.indexedImports.get(key)?.get(target) ?? []) { + const current = await resolver.resolve( + path.resolve(this.rootPath, anchor.filePath), + specifier + ); + if (current.status === 'unresolved' && current.reason === 'outside-root') { + throw new IndexCorruptedError( + 'Import resolves outside the selected project. Full rebuild required.' + ); + } + if ( + current.status === 'resolved' && + current.path && + this.indexedFileKey(current.path) === target + ) { + corroboratedFiles.add(key); + break; + } + } + if (corroboratedFiles.has(key)) break; + } + } + } + + const fileEvidence = new Map>(); + for (const { result, matching } of evidence) { + const key = this.indexedFileKey(result.item.filePath); + const termsForFile = fileEvidence.get(key) ?? new Set(); + for (const term of matching) termsForFile.add(term); + fileEvidence.set(key, termsForFile); + } + + const seenFiles = new Set(); + const nominated = evidence + .filter(({ result, exactAnchor }) => { + const key = this.indexedFileKey(result.item.filePath); + return ( + (exactAnchor && corroboratedFiles.has(key)) || (fileEvidence.get(key)?.size ?? 0) >= 2 + ); + }) + .sort((a, b) => { + const keyA = this.indexedFileKey(a.result.item.filePath); + const keyB = this.indexedFileKey(b.result.item.filePath); + const score = (entry: typeof a, key: string) => + ((fileEvidence.get(key)?.size ?? 0) + entry.declarationMatching.size) * 8 + + entry.matching.size * 3 + + (entry.exactAnchor ? 8 : 0) - + (this.isTestFile(entry.result.item.filePath) ? 2 : 0) - + (entry.result.score ?? 0); + const primaryDifference = score(b, keyB) - score(a, keyA); + if (primaryDifference !== 0) return primaryDifference; + return b.density - a.density || a.result.refIndex - b.result.refIndex; + }) + .filter(({ result }) => { + const key = this.indexedFileKey(result.item.filePath); + if (seenFiles.has(key)) return false; + seenFiles.add(key); + return true; + }) + .map(({ result, declarationMatching }) => ({ + ...result, + // Equal document zones: distinct source clues nominate the file; + // indexed declaration clues distinguish its own work from mentions in + // imports or configuration. This remains bounded to [0, 1]. + score: + 1 - + ((fileEvidence.get(this.indexedFileKey(result.item.filePath))?.size ?? 0) + + declarationMatching.size) / + (terms.length * 2) + })); + const current = await this.selectCurrentCandidates( + nominated, + limit, + (result) => result.item, + validation, + budget + ); + const unavailableAfter = [...checkedSources.values()].filter( + (source) => source === null + ).length; + if ( + current.length < limit && + unavailableAfter > unavailableBefore && + candidates.length === candidateLimit + ) { + // Refill the same bounded evidence window after excluding every chunk of + // a rejected source. Fresh-prefix queries retain the original single pass. + return this.rescueKeywordMatches(query, limit, filters, validation, budget, rawCandidates); + } + return current; + } + /** * Classify query intent based on heuristic patterns */ @@ -708,18 +1400,23 @@ export class CodebaseSearcher { }) .sort((a, b) => b.score - a.score); - // SEARCH-01: Definition-first boost for EXACT_NAME intent - // Boost results where symbolName matches query (case-insensitive) - if (intent === 'EXACT_NAME') { - const queryNormalized = query.toLowerCase(); + // Definition first for explicit identifiers, regardless of which + // analyzer populated the existing declaration metadata. Coarse component or + // layer boosts must not move a fuzzy same-suffix mention above that match. + if ( + intent === 'EXACT_NAME' || + scoredResults.some((result) => this.hasExactQueryDeclaration(result, query)) + ) { for (const result of scoredResults) { - const symbolName = result.metadata?.symbolName; - if (symbolName && symbolName.toLowerCase() === queryNormalized) { + if (this.hasExactQueryDeclaration(result, query)) { result.score *= 1.15; // +15% boost for definition } } - // Re-sort after boost - scoredResults.sort((a, b) => b.score - a.score); + scoredResults.sort( + (a, b) => + Number(this.hasExactQueryDeclaration(b, query)) - + Number(this.hasExactQueryDeclaration(a, query)) || b.score - a.score + ); } // File-level deduplication @@ -733,35 +1430,9 @@ export class CodebaseSearcher { if (deduped.length >= limit) break; } - // SEARCH-01: Symbol-level deduplication - // Within each symbol group (symbolPath), keep only the highest-scoring chunk - const seenSymbols = new Map(); - const symbolDeduped: SearchResult[] = []; - for (const result of deduped) { - const symbolPath = result.metadata?.symbolPath; - if (!symbolPath) { - // No symbol info, keep as-is - symbolDeduped.push(result); - continue; - } - - const symbolPathKey = Array.isArray(symbolPath) ? symbolPath.join('.') : String(symbolPath); - const existing = seenSymbols.get(symbolPathKey); - if (!existing || result.score > existing.score) { - if (existing) { - // Replace lower-scoring version - const idx = symbolDeduped.indexOf(existing); - if (idx >= 0) { - symbolDeduped[idx] = result; - } - } else { - symbolDeduped.push(result); - } - seenSymbols.set(symbolPathKey, result); - } - } - - const finalResults = symbolDeduped; + // File-level deduplication above already keeps only one result per file. + // Keep same-named symbols in different files as distinct search results. + const finalResults = deduped; if ( isNonTestQuery && @@ -838,7 +1509,8 @@ export class CodebaseSearcher { useSemanticSearch: boolean, useKeywordSearch: boolean, semanticWeight: number, - keywordWeight: number + keywordWeight: number, + validation: SourceValidationContext ): Promise<{ semantic: Map }>; keyword: Map }>; @@ -856,7 +1528,12 @@ export class CodebaseSearcher { if (useSemanticSearch && this.embeddingProvider && this.storageProvider) { try { for (const variant of queryVariants) { - const vectorResults = await this.semanticSearch(variant.query, candidateLimit, filters); + const vectorResults = await this.semanticSearch( + variant.query, + candidateLimit, + filters, + validation + ); // Assign ranks based on retrieval order (0-indexed) vectorResults.forEach((result, index) => { @@ -886,13 +1563,23 @@ export class CodebaseSearcher { if (useKeywordSearch && this.fuseIndex) { try { for (const variant of queryVariants) { - const keywordResults = await this.keywordSearch(variant.query, candidateLimit, filters); + const keywordResults = await this.keywordSearch( + variant.query, + candidateLimit, + filters, + validation + ); + // Grounded multi-clue rescue needs its evidence distinction retained + // through fusion. Single/exact identifier searches keep their existing + // rank semantics and confidence calibration. + const weightLexicalEvidence = this.lexicalQueryTerms(variant.query).size >= 2; // Assign ranks based on retrieval order (0-indexed) keywordResults.forEach((result, index) => { const id = result.chunk.id; const rank = index; // 0-indexed rank - const weight = keywordWeight * variant.weight; + const weight = + keywordWeight * variant.weight * (weightLexicalEvidence ? result.score : 1); const existing = keywordRanks.get(id); if (existing) { @@ -906,6 +1593,9 @@ export class CodebaseSearcher { }); } } catch (error) { + if (error instanceof IndexCorruptedError) { + throw error; // Preserve the caller's existing auto-heal path. + } console.warn('Keyword search failed:', error); } } @@ -919,14 +1609,36 @@ export class CodebaseSearcher { filters?: SearchFilters, options: SearchOptions = DEFAULT_SEARCH_OPTIONS ): Promise { + return (await this.searchWithCoverage(query, limit, filters, options)).results; + } + + async searchWithCoverage( + query: string, + limit: number = 5, + filters?: SearchFilters, + options: SearchOptions = DEFAULT_SEARCH_OPTIONS + ): Promise { + return this.runOperation(() => this.searchPinned(query, limit, filters, options)); + } + + private async searchPinned( + query: string, + limit: number, + filters: SearchFilters | undefined, + options: SearchOptions + ): Promise { if (!this.initialized) { - await this.initialize(); + await this.initializePinned(); } const merged = { ...DEFAULT_SEARCH_OPTIONS, ...options }; + if (this.indexMeta?.artifacts.vectorDb.embeddingEnabled === false) { + merged.useSemanticSearch = false; + merged.enableReranker = false; + } const { useSemanticSearch, useKeywordSearch, @@ -945,6 +1657,8 @@ export class CodebaseSearcher { const candidateLimit = Math.max(limit * 2, candidateFloor || 30); const primaryVariants = this.buildQueryVariants(query, enableQueryExpansion ? 1 : 0); + const validation: SourceValidationContext = { sources: new Map() }; + const checkedSources = validation.sources; const primaryMatches = await this.collectHybridMatches( primaryVariants, candidateLimit, @@ -952,8 +1666,28 @@ export class CodebaseSearcher { Boolean(useSemanticSearch), Boolean(useKeywordSearch), finalSemanticWeight, - finalKeywordWeight + finalKeywordWeight, + validation + ); + + const keepCurrentMatches = (matches: typeof primaryMatches): void => { + for (const channel of [matches.semantic, matches.keyword]) { + for (const [id, { chunk }] of channel) { + if (checkedSources.get(path.resolve(this.rootPath, chunk.filePath)) == null) + channel.delete(id); + } + } + }; + + // Validate every nominated source before scoring and limiting so stale top + // matches cannot hide a valid lower-ranked current file. + await this.validateCandidateSources( + [...primaryMatches.semantic.values(), ...primaryMatches.keyword.values()].map( + (match) => match.chunk + ), + validation ); + keepCurrentMatches(primaryMatches); const primaryTotalWeight = primaryVariants.reduce((sum, v) => sum + v.weight, 0) * @@ -984,9 +1718,18 @@ export class CodebaseSearcher { Boolean(useSemanticSearch), Boolean(useKeywordSearch), finalSemanticWeight, - finalKeywordWeight + finalKeywordWeight, + validation ); + await this.validateCandidateSources( + [...rescueMatches.semantic.values(), ...rescueMatches.keyword.values()].map( + (match) => match.chunk + ), + validation + ); + keepCurrentMatches(rescueMatches); + const rescueVariantWeights = rescueVariants.map((_, i) => (i === 0 ? 1 : 0.8)); const rescueTotalWeight = rescueVariantWeights.reduce((sum, w) => sum + w, 0) * @@ -1015,7 +1758,38 @@ export class CodebaseSearcher { } } - return bestResults; + const results = bestResults.map((result) => { + const lines = checkedSources.get(path.resolve(this.rootPath, result.filePath)); + if (lines == null) { + throw new IndexCorruptedError('Result source was not validated. Full rebuild required.'); + } + this.validateSourceRange(result, lines); + return result; + }); + const skippedFiles = [...checkedSources.values()].filter((source) => source === null).length; + return { + results, + sourceCoverage: { + status: + skippedFiles === 0 && + !validation.candidateBudgetExhausted && + !validation.candidateRetrievalFailed + ? 'current' + : results.length > 0 + ? 'partial' + : 'no_fresh_results', + checkedFiles: checkedSources.size, + skippedFiles, + ...(validation.candidateBudgetExhausted && { candidateBudgetExhausted: true }), + ...(validation.candidateRetrievalFailed && { candidateRetrievalFailed: true }), + ...((skippedFiles > 0 || + validation.candidateBudgetExhausted || + validation.candidateRetrievalFailed) && { + refreshAction: + 'Run refresh_index with incrementalOnly=true to refresh changed or unavailable sources, then retry.' + }) + } + }; } private generateSummary(chunk: CodeChunk): string { @@ -1081,7 +1855,8 @@ export class CodebaseSearcher { private async semanticSearch( query: string, limit: number, - filters?: SearchFilters + filters?: SearchFilters, + validation: SourceValidationContext = { sources: new Map() } ): Promise<{ chunk: CodeChunk; score: number }[]> { if (!this.embeddingProvider || !this.storageProvider) { return []; @@ -1089,18 +1864,71 @@ export class CodebaseSearcher { const queryVector = await this.embeddingProvider.embed(query); - const results = await this.storageProvider.search(queryVector, limit, filters); - - return results.map((r) => ({ - chunk: r.chunk, - score: r.score - })); + const budget: SourceInspectionBudget = { + limit: Math.min(limit, MAX_CANDIDATE_WORK), + inspected: new Set() + }; + const retainedCount = this.chunks.length; + let requested = Math.min(limit, MAX_CANDIDATE_WORK); + let retrievedWork = 0; + let retained: Array<{ chunk: CodeChunk; score: number }> | undefined; + for (;;) { + const results = await this.storageProvider + .search(queryVector, requested, filters) + .catch((error: unknown) => { + if (error instanceof IndexCorruptedError || retained === undefined) throw error; + validation.candidateRetrievalFailed = true; + console.warn('Semantic refill failed; retaining validated current candidates:', error); + return null; + }); + if (results === null) return retained ?? []; + retrievedWork += results.length; + const current = await this.selectCurrentCandidates( + results, + limit, + (result) => result.chunk, + validation, + budget + ); + retained = current.map(({ chunk, score }) => ({ chunk, score })); + if ( + current.length >= limit || + results.length < requested || + (retainedCount > 0 && requested >= retainedCount) + ) { + return retained; + } + const staleSources = new Set( + [...validation.sources].filter(([, source]) => source === null).map(([file]) => file) + ); + if (staleSources.size === 0) { + if (requested < limit && results.length === requested) + validation.candidateBudgetExhausted = true; + return retained; + } + const rejectedChunks = this.chunks.filter((chunk) => + staleSources.has(path.resolve(this.rootPath, chunk.filePath)) + ).length; + // Widen by known rejected rows, never by an arbitrary multiplier. Sum of + // retrieved rows is bounded, and the query vector is reused for all retries. + const next = Math.min( + retainedCount || requested, + limit + rejectedChunks, + MAX_CANDIDATE_WORK - retrievedWork + ); + if (next <= requested || budget.inspected.size >= budget.limit) { + validation.candidateBudgetExhausted = true; + return retained; + } + requested = next; + } } private async keywordSearch( query: string, limit: number, - filters?: SearchFilters + filters?: SearchFilters, + validation: SourceValidationContext = { sources: new Map() } ): Promise<{ chunk: CodeChunk; score: number }[]> { if (!this.fuseIndex || this.chunks.length === 0) { return []; @@ -1109,68 +1937,63 @@ export class CodebaseSearcher { let fuseResults = this.fuseIndex.search(query); if (filters) { - fuseResults = fuseResults.filter((r) => { - const chunk = r.item; - - if (filters.componentType && chunk.componentType !== filters.componentType) { - return false; - } - if (filters.layer && chunk.layer !== filters.layer) { - return false; - } - if (filters.framework && chunk.framework !== filters.framework) { - return false; - } - if (filters.language && chunk.language !== filters.language) { - return false; - } - if (filters.tags && filters.tags.length > 0) { - const chunkTags = chunk.tags || []; - if (!filters.tags.some((tag) => chunkTags.includes(tag))) { - return false; - } - } - - return true; - }); + fuseResults = fuseResults.filter(({ item }) => this.matchesSearchFilters(item, filters)); } - return fuseResults.slice(0, limit).map((r) => { - const chunk = r.item; - let score = 1 - (r.score || 0); + if (fuseResults.length === 0 || (fuseResults[0].score ?? 0) > KEYWORD_FUSE_OPTIONS.threshold!) { + const rescue = await this.rescueKeywordMatches(query, limit, filters, validation); + const seen = new Set(rescue.map(({ item }) => item.id)); + fuseResults = [...rescue, ...fuseResults.filter(({ item }) => !seen.has(item.id))].sort( + (a, b) => (a.score ?? 0) - (b.score ?? 0) + ); + } + const current = await this.selectCurrentCandidates( + fuseResults, + limit, + (result) => result.item, + validation + ); + return current + .map((r) => { + const chunk = r.item; + let score = 1 - (r.score || 0); - // Boost exact matches on class name or file path - const queryLower = query.toLowerCase(); - const fileName = path.basename(chunk.filePath).toLowerCase(); - const relativePathLower = chunk.relativePath.toLowerCase(); - const componentName = chunk.metadata?.componentName?.toLowerCase() || ''; + // Boost exact matches on class name or file path + const queryLower = query.toLowerCase(); + const fileName = path.basename(chunk.filePath).toLowerCase(); + const relativePathLower = chunk.relativePath.toLowerCase(); - // Exact class name match - if (componentName && queryLower === componentName) { - score = Math.min(1.0, score + 0.3); - } + // Use the same analyzer-neutral declaration evidence as final ranking. + if (this.hasExactQueryDeclaration(chunk, query)) { + score = Math.min(1.0, score + 0.3); + } - // Exact file name match - if ( - fileName === queryLower || - fileName.replace(/\.ts$/, '') === queryLower.replace(/\.ts$/, '') - ) { - score = Math.min(1.0, score + 0.2); - } + // Exact file name match + if ( + fileName === queryLower || + fileName.replace(/\.ts$/, '') === queryLower.replace(/\.ts$/, '') + ) { + score = Math.min(1.0, score + 0.2); + } - // File path contains query - if ( - chunk.filePath.toLowerCase().includes(queryLower) || - relativePathLower.includes(queryLower) - ) { - score = Math.min(1.0, score + 0.1); - } + // File path contains query + if ( + chunk.filePath.toLowerCase().includes(queryLower) || + relativePathLower.includes(queryLower) + ) { + score = Math.min(1.0, score + 0.1); + } - return { - chunk, - score - }; - }); + return { + chunk, + score + }; + }) + .sort( + (a, b) => + Number(this.hasExactQueryDeclaration(b.chunk, query)) - + Number(this.hasExactQueryDeclaration(a.chunk, query)) || b.score - a.score + ); } private generateRelevanceReason(chunk: CodeChunk, query: string): string { @@ -1194,10 +2017,10 @@ export class CodebaseSearcher { } async getChunkCount(): Promise { - if (this.storageProvider) { - return await this.storageProvider.count(); - } - return this.chunks.length; + return this.runOperation(async () => { + if (!this.initialized) await this.initializePinned(); + return this.storageProvider ? this.storageProvider.count() : this.chunks.length; + }); } isReady(): boolean { diff --git a/src/core/symbol-references.ts b/src/core/symbol-references.ts index f4c1986..e49c04b 100644 --- a/src/core/symbol-references.ts +++ b/src/core/symbol-references.ts @@ -1,14 +1,24 @@ +import { + withIndexSnapshot, + currentIndexSnapshot, + resolveIndexArtifact +} from './index-generation.js'; import { promises as fs } from 'fs'; import path from 'path'; import { CODEBASE_CONTEXT_DIRNAME, KEYWORD_INDEX_FILENAME } from '../constants/codebase-context.js'; import { IndexCorruptedError } from '../errors/index.js'; +import { readIndexMeta, validateKeywordIndexArtifact } from './index-meta.js'; +import { readReferenceManifest } from './manifest.js'; import type { UsageLocation } from '../types/index.js'; import { detectLanguage } from '../utils/language-detection.js'; -import { findIdentifierOccurrences } from '../utils/tree-sitter.js'; +import { isPathWithin } from '../utils/project-discovery.js'; +import { + hasIdentifierCandidate, + findIdentifierOccurrences, + findLexicalIdentifierOccurrences +} from '../utils/tree-sitter.js'; interface IndexedChunk { - content?: unknown; - startLine?: unknown; relativePath?: unknown; filePath?: unknown; } @@ -22,8 +32,16 @@ interface SymbolReferencesSuccess { symbol: string; usageCount: number; usages: SymbolUsage[]; - confidence: 'syntactic'; + confidence: 'syntactic' | 'lexical'; isComplete: boolean; + coverage: { + candidateFiles: number; + sourcesRead: number; + filesWithoutText: number; + filesUnavailable: number; + parserCalls: number; + lexicalFallbackFiles: number; + }; } interface SymbolReferencesError { @@ -33,33 +51,16 @@ interface SymbolReferencesError { export type SymbolReferencesResult = SymbolReferencesSuccess | SymbolReferencesError; -function escapeRegex(value: string): string { - return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); -} - -function getUsageFile(rootPath: string, chunk: IndexedChunk): string { - if (typeof chunk.relativePath === 'string' && chunk.relativePath.trim()) { - return chunk.relativePath.replace(/\\/g, '/'); - } +type SourceReadResult = + | { status: 'source'; source: string; realSourcePath: string } + | { status: 'without-text' } + | { status: 'without-candidate' } + | { status: 'unavailable' } + | SymbolReferencesError; - if (typeof chunk.filePath === 'string' && chunk.filePath.trim()) { - const relativePath = path.relative(rootPath, chunk.filePath); - if (!relativePath || relativePath.startsWith('..')) { - return path.basename(chunk.filePath); - } - return relativePath.replace(/\\/g, '/'); - } - - return 'unknown'; -} - -function buildPreview(content: string, lineOffset: number): string { - const lines = content.split('\n'); - const start = Math.max(0, lineOffset - 1); - const end = Math.min(lines.length, lineOffset + 2); - const previewLines = lines.slice(start, end); - return previewLines.join('\n').trim(); -} +// Bound both outstanding source I/O and buffered source contents. Parsing remains +// sequential because Tree-sitter caches a mutable parser for each language. +const SOURCE_READ_BATCH_SIZE = 4; function buildPreviewFromFileLines(lines: string[], line: number): string { const start = Math.max(0, line - 2); @@ -67,12 +68,47 @@ function buildPreviewFromFileLines(lines: string[], line: number): string { return lines.slice(start, end).join('\n').trim(); } +function isOrdinarySourceAvailabilityError(error: unknown): boolean { + if (error === null || typeof error !== 'object' || !('code' in error)) return false; + const code = error.code; + return code === 'ENOENT' || code === 'ENOTDIR' || code === 'EACCES' || code === 'EPERM'; +} + +async function readReferenceSource( + relativePath: string, + absolutePath: string, + realRootPath: string, + symbol: string +): Promise { + try { + const realSourcePath = await fs.realpath(absolutePath); + if (!isPathWithin(realRootPath, realSourcePath)) { + return { + status: 'error', + message: `Indexed source resolves outside the selected project (${relativePath}). Refresh the index before trusting reference results.` + }; + } + if (!(await fs.stat(realSourcePath)).isFile()) throw new Error('not a file'); + const source = (await fs.readFile(realSourcePath, 'utf-8')).replace(/\r\n/g, '\n'); + if (!source.includes(symbol)) return { status: 'without-text' }; + // The boundary screen only suppresses simple ASCII identifier interiors; + // other spellings and non-ASCII adjacency stay on the existing parser path. + return hasIdentifierCandidate(source, symbol) + ? { status: 'source', source, realSourcePath } + : { status: 'without-candidate' }; + } catch (error) { + if (isOrdinarySourceAvailabilityError(error)) return { status: 'unavailable' }; + return { + status: 'error', + message: `Indexed source is missing or unreadable (${relativePath}). Refresh the index before trusting reference results.` + }; + } +} + function resolveAbsoluteChunkPath(rootPath: string, chunk: IndexedChunk): string | null { const resolvedRoot = path.resolve(rootPath); const isWithinRoot = (candidate: string): boolean => { - const resolvedCandidate = path.resolve(candidate); - const relative = path.relative(resolvedRoot, resolvedCandidate); - return Boolean(relative) && !relative.startsWith('..') && !path.isAbsolute(relative); + return isPathWithin(resolvedRoot, candidate); }; if (typeof chunk.filePath === 'string' && chunk.filePath.trim()) { @@ -92,20 +128,20 @@ function resolveAbsoluteChunkPath(rootPath: string, chunk: IndexedChunk): string return null; } -async function fileExists(targetPath: string): Promise { - try { - const stat = await fs.stat(targetPath); - return stat.isFile(); - } catch { - return false; - } -} - export async function findSymbolReferences( rootPath: string, symbol: string, limit = 10 ): Promise { + return withIndexSnapshot(rootPath, () => findPinnedSymbolReferences(rootPath, symbol, limit)); +} + +async function findPinnedSymbolReferences( + rootPath: string, + symbol: string, + limit: number +): Promise { + rootPath = currentIndexSnapshot(rootPath)?.realRoot ?? rootPath; const normalizedSymbol = symbol.trim(); const normalizedLimit = Number.isFinite(limit) && limit > 0 ? Math.floor(limit) : 10; @@ -116,12 +152,14 @@ export async function findSymbolReferences( }; } - const indexPath = path.join(rootPath, CODEBASE_CONTEXT_DIRNAME, KEYWORD_INDEX_FILENAME); + const indexPath = await resolveIndexArtifact(rootPath, KEYWORD_INDEX_FILENAME); + const meta = await readIndexMeta(rootPath); + const manifest = await readReferenceManifest(rootPath, meta); - let chunksRaw: unknown; + let keywordIndexRaw: unknown; try { const content = await fs.readFile(indexPath, 'utf-8'); - chunksRaw = JSON.parse(content); + keywordIndexRaw = JSON.parse(content); } catch (error) { throw new IndexCorruptedError( `Keyword index missing or unreadable (rebuild required): ${ @@ -130,126 +168,141 @@ export async function findSymbolReferences( ); } - if (Array.isArray(chunksRaw)) { + if (Array.isArray(keywordIndexRaw)) { throw new IndexCorruptedError( 'Legacy keyword index format detected (missing header). Rebuild required.' ); } - const chunks = - chunksRaw !== null && - typeof chunksRaw === 'object' && - 'chunks' in chunksRaw && - Array.isArray(chunksRaw.chunks) - ? (chunksRaw.chunks as unknown[]) - : null; - - if (!chunks) { - throw new IndexCorruptedError('Keyword index corrupted: expected { header, chunks }'); - } + const { chunks } = validateKeywordIndexArtifact(keywordIndexRaw, meta); const usages: SymbolUsage[] = []; let usageCount = 0; - - const escapedSymbol = escapeRegex(normalizedSymbol); - const prefilter = new RegExp(`\\b${escapedSymbol}\\b`); - const matcher = new RegExp(`\\b${escapedSymbol}\\b`, 'g'); - - // Prefilter candidate files from the keyword index. We do not trust chunk contents for - // exact reference counting when Tree-sitter is available; chunks only guide which files to scan. - const chunksByFile = new Map< - string, - { relPath: string; absPath: string | null; chunks: IndexedChunk[] } - >(); + let sourcesRead = 0; + let filesWithoutText = 0; + let filesUnavailable = 0; + let parserCalls = 0; + let lexicalFallbackFiles = 0; + + // The manifest nominates every analyzed source; capped keyword chunks cannot define + // the reference scope. Current source bytes determine matches, locations, and previews. + const candidateFiles = new Map(); + + for (const relativePath of Object.keys(manifest.files)) { + const absPath = resolveAbsoluteChunkPath(rootPath, { relativePath }); + if (!absPath) { + throw new IndexCorruptedError( + `Reference manifest contains an invalid source path (${relativePath}). Rebuild required.` + ); + } + const relPath = path.relative(rootPath, absPath).replace(/\\/g, '/'); + candidateFiles.set(relPath, absPath); + } for (const chunkRaw of chunks) { const chunk = chunkRaw as IndexedChunk; - if (typeof chunk.content !== 'string') continue; - if (!prefilter.test(chunk.content)) continue; - - const relPath = getUsageFile(rootPath, chunk); const absPath = resolveAbsoluteChunkPath(rootPath, chunk); - - const entry = chunksByFile.get(relPath); - if (entry) { - entry.chunks.push(chunk); - // Prefer a real absolute path when available - if (!entry.absPath && absPath) { - entry.absPath = absPath; - } - } else { - chunksByFile.set(relPath, { relPath, absPath, chunks: [chunk] }); + if (!absPath) { + return { + status: 'error', + message: + 'Indexed source location is invalid. Refresh the index before trusting reference results.' + }; + } + const relPath = path.relative(rootPath, absPath).replace(/\\/g, '/'); + if (!Object.hasOwn(manifest.files, relPath)) { + throw new IndexCorruptedError( + `Keyword index source is absent from the build-bound manifest (${relPath}). Rebuild required.` + ); } } - for (const entry of chunksByFile.values()) { - const relPath = entry.relPath; - const absPath = entry.absPath; - - // Preferred: Tree-sitter identifier walk on the real file content. - if (absPath && (await fileExists(absPath))) { - try { - const raw = await fs.readFile(absPath, 'utf-8'); - const content = raw.replace(/\r\n/g, '\n'); - const language = detectLanguage(absPath); - const occurrences = await findIdentifierOccurrences(content, language, normalizedSymbol); - - if (occurrences) { - usageCount += occurrences.length; - - if (usages.length < normalizedLimit && occurrences.length > 0) { - const lines = content.split('\n'); - for (const occ of occurrences) { - if (usages.length >= normalizedLimit) break; - usages.push({ - file: relPath, - line: occ.line, - preview: buildPreviewFromFileLines(lines, occ.line) - }); - } - } - - continue; - } - } catch { - // Fall through to chunk-regex fallback (missing grammar, parse failure, etc.) + let realRootPath: string; + try { + realRootPath = await fs.realpath(rootPath); + } catch { + return { status: 'error', message: 'Selected project path is missing or unreadable.' }; + } + const sourceFiles = [...candidateFiles]; + for (let offset = 0; offset < sourceFiles.length; offset += SOURCE_READ_BATCH_SIZE) { + const batch = sourceFiles.slice(offset, offset + SOURCE_READ_BATCH_SIZE); + // Reads return values even on failure. Settle every started read before a fatal + // return, then consume in manifest order so snippets and totals stay deterministic. + const sources = await Promise.all( + batch.map(([relativePath, absolutePath]) => + readReferenceSource(relativePath, absolutePath, realRootPath, normalizedSymbol) + ) + ); + const fatalSource = sources.find((source) => source.status === 'error'); + if (fatalSource?.status === 'error') return fatalSource; + + for (let index = 0; index < batch.length; index += 1) { + const [relativePath] = batch[index]; + const result = sources[index]; + if (result.status === 'error') return result; + if (result.status === 'unavailable') { + filesUnavailable += 1; + continue; } - } - - // Fallback: regex scan inside the matched chunks (legacy behavior). - for (const chunk of entry.chunks) { - if (typeof chunk.content !== 'string') continue; - - const chunkContent = chunk.content; - const startLine = typeof chunk.startLine === 'number' ? chunk.startLine : 1; - matcher.lastIndex = 0; - - let match: RegExpExecArray | null; - while ((match = matcher.exec(chunkContent)) !== null) { - usageCount += 1; - - if (usages.length >= normalizedLimit) { - continue; + sourcesRead += 1; + if (result.status === 'without-text') { + filesWithoutText += 1; + continue; + } + // Raw query text was present but only inside longer ASCII identifiers; + // keep this distinct from filesWithoutText coverage. + if (result.status === 'without-candidate') continue; + const { source, realSourcePath } = result; + const lines = source.split('\n'); + parserCalls += 1; + const occurrences = await findIdentifierOccurrences( + source, + detectLanguage(realSourcePath), + normalizedSymbol + ).catch(() => null); + + if (occurrences) { + usageCount += occurrences.length; + for (const occurrence of occurrences) { + if (usages.length >= normalizedLimit) break; + usages.push({ + file: relativePath, + line: occurrence.line, + preview: buildPreviewFromFileLines(lines, occurrence.line) + }); } + continue; + } - const prefix = chunkContent.slice(0, match.index); - const lineOffset = prefix.split('\n').length - 1; - + lexicalFallbackFiles += 1; + const lexicalOccurrences = findLexicalIdentifierOccurrences(source, normalizedSymbol); + usageCount += lexicalOccurrences.length; + for (const occurrence of lexicalOccurrences) { + if (usages.length >= normalizedLimit) break; usages.push({ - file: relPath, - line: startLine + lineOffset, - preview: buildPreview(chunkContent, lineOffset) + file: relativePath, + line: occurrence.line, + preview: buildPreviewFromFileLines(lines, occurrence.line) }); } } } + const coverageComplete = filesUnavailable === 0; return { status: 'success', symbol: normalizedSymbol, usageCount, usages, - confidence: 'syntactic', - isComplete: usageCount < normalizedLimit + confidence: lexicalFallbackFiles === 0 ? 'syntactic' : 'lexical', + isComplete: coverageComplete && usageCount < normalizedLimit, + coverage: { + candidateFiles: candidateFiles.size, + sourcesRead, + filesWithoutText, + filesUnavailable, + parserCalls, + lexicalFallbackFiles + } }; } diff --git a/src/embeddings/index.ts b/src/embeddings/index.ts index aabb3fa..f8c1718 100644 --- a/src/embeddings/index.ts +++ b/src/embeddings/index.ts @@ -9,6 +9,14 @@ import { parseEmbeddingProviderName } from './types.js'; import { TransformersEmbeddingProvider, MODEL_CONFIGS } from './transformers.js'; +import { assertOptionalDependency } from '../utils/optional-dependency.js'; + +export function assertEmbeddingRuntimeAvailable(config: Partial = {}): void { + const provider = config.provider ?? DEFAULT_EMBEDDING_CONFIG.provider; + if (provider !== 'openai' && provider !== 'custom') { + assertOptionalDependency('@huggingface/transformers', '^4.2.0'); + } +} /** * Returns expected embedding dimensions for a given config without initializing any provider. @@ -32,6 +40,7 @@ let cachedProviderType: string | null = null; export async function getEmbeddingProvider( config: Partial = {} ): Promise { + assertEmbeddingRuntimeAvailable(config); const mergedConfig = { ...DEFAULT_EMBEDDING_CONFIG, ...config }; const providerKey = `${mergedConfig.provider}:${mergedConfig.model}`; diff --git a/src/embeddings/transformers.ts b/src/embeddings/transformers.ts index 7fc62e8..bb358d1 100644 --- a/src/embeddings/transformers.ts +++ b/src/embeddings/transformers.ts @@ -1,6 +1,7 @@ import { EmbeddingProvider, DEFAULT_MODEL } from './types.js'; import type { FeatureExtractionPipeline } from '@huggingface/transformers'; import os from 'os'; +import { assertOptionalDependency } from '../utils/optional-dependency.js'; /** * Returns the number of ONNX intra-op threads to use. @@ -56,6 +57,8 @@ export class TransformersEmbeddingProvider implements EmbeddingProvider { if (this.ready) return; if (this.initPromise) return this.initPromise; + assertOptionalDependency('@huggingface/transformers', '^4.2.0'); + this.initPromise = this._initialize(); return this.initPromise; } diff --git a/src/errors/index.ts b/src/errors/index.ts index 96a9a30..144b447 100644 --- a/src/errors/index.ts +++ b/src/errors/index.ts @@ -8,3 +8,21 @@ export class IndexCorruptedError extends Error { this.name = 'IndexCorruptedError'; } } + +/** Missing opt-in runtime, distinct from damaged index data. */ +export class OptionalDependencyError extends Error { + constructor(packageName: string, versionRange: string) { + super( + `Semantic mode needs ${packageName}. In the environment where CBC is installed, run npm install "${packageName}@${versionRange}", then retry.` + ); + this.name = 'OptionalDependencyError'; + } +} + +/** Producer corruption is fatal to this build, never a skippable source read error. */ +export class InvalidSourceChunkError extends Error { + constructor(message: string) { + super(message); + this.name = 'InvalidSourceChunkError'; + } +} diff --git a/src/index.ts b/src/index.ts index ab539b8..a3bd53b 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,11 +1,12 @@ #!/usr/bin/env node +import { withAvailableIndexSnapshot, resolveIndexArtifact } from './core/index-generation.js'; /** * MCP Server for Codebase Context * Provides codebase indexing and semantic search capabilities */ -import { promises as fs } from 'fs'; +import { promises as fs, realpathSync } from 'fs'; import path from 'path'; import { fileURLToPath } from 'url'; @@ -20,6 +21,8 @@ import { NextJsAnalyzer } from './analyzers/nextjs/index.js'; import { NestJsAnalyzer } from './analyzers/nestjs/index.js'; import { ReactAnalyzer } from './analyzers/react/index.js'; import { GenericAnalyzer } from './analyzers/generic/index.js'; +import { IndexSignalSchema, type IndexSignal } from './core/index-signal.js'; +export type { IndexSignal } from './core/index-signal.js'; import { IndexCorruptedError } from './errors/index.js'; import { appendMemoryFile } from './memory/store.js'; import { handleCliCommand } from './cli.js'; @@ -45,8 +48,9 @@ import { findNearestProjectBoundary, isPathWithin } from './utils/project-discovery.js'; -import { readIndexMeta, validateIndexArtifacts } from './core/index-meta.js'; +import { readIndexMeta, validateIndexArtifacts, hasCommittedIndex } from './core/index-meta.js'; import { TOOLS, dispatchTool, type ToolContext, type ToolResponse } from './tools/index.js'; +import { finalizeRelationshipPayload } from './tools/get-code-relationships.js'; import { finalizeSearchPayloadText } from './tools/search-payload-budget.js'; import type { ProjectDescriptor, ToolPaths } from './tools/types.js'; import { @@ -57,6 +61,8 @@ import { makeLegacyPaths, normalizeRootKey, removeProject, + stopProjectWatcher, + stopAllProjectWatchers, type ProjectRuntimeOverrides, type ProjectState } from './project-state.js'; @@ -81,7 +87,15 @@ let mcpRuntimeReady = false; let mcpRuntimePromise: Promise | undefined; // Flags that are NOT project paths — skip them when resolving the bootstrap root. -const KNOWN_FLAGS = new Set(['--http', '--port', '--help']); +const KNOWN_FLAGS = new Set([ + '--http', + '--port', + '--help', + '--embeddings', + '--no-embeddings', + '--version', + '-v' +]); const CLI_SUBCOMMANDS = [ 'memory', 'search', @@ -93,21 +107,54 @@ const CLI_SUBCOMMANDS = [ 'refs', 'cycles', 'init', - 'map' + 'map', + 'help' ]; -// Check if this module is the entry point. +function canonicalEntrypointPath(filePath: string | undefined): string | undefined { + if (!filePath) return undefined; + try { + const canonicalPath = path.normalize(realpathSync(filePath)); + return process.platform === 'win32' ? canonicalPath.toLowerCase() : canonicalPath; + } catch { + return undefined; + } +} + +// Resolve symlink launchers while keeping imports and unrelated index files inert. +const moduleEntrypointPath = canonicalEntrypointPath(fileURLToPath(import.meta.url)); +const processEntrypointPath = canonicalEntrypointPath(process.argv[1]); const isDirectRun = - process.argv[1]?.replace(/\\/g, '/').endsWith('index.js') || - process.argv[1]?.replace(/\\/g, '/').endsWith('index.ts'); + moduleEntrypointPath !== undefined && moduleEntrypointPath === processEntrypointPath; const directSubcommand = process.argv[2]; const isDirectCliSubcommand = isDirectRun && typeof directSubcommand === 'string' && - (CLI_SUBCOMMANDS.includes(directSubcommand) || directSubcommand === '--help'); + (CLI_SUBCOMMANDS.includes(directSubcommand) || + directSubcommand === '--help' || + directSubcommand === '--version' || + directSubcommand === '-v'); + +if ( + !isDirectCliSubcommand && + process.argv.includes('--embeddings') && + process.argv.includes('--no-embeddings') +) { + console.error('Choose either --embeddings or --no-embeddings, not both.'); + process.exit(1); +} +const bootstrapSkipEmbedding = process.argv.includes('--no-embeddings') + ? true + : process.argv.includes('--embeddings') + ? false + : undefined; // Resolve optional bootstrap root with validation handled later in main(). function resolveRootPath(): string | undefined { + if (isDirectRun && directSubcommand === 'help') { + return undefined; + } + const envPath = process.env.CODEBASE_ROOT; // Walk argv starting at position 2, skip known flags and their values. @@ -154,9 +201,7 @@ const stdioIdleTimeoutEnv = Number.parseInt( 10 ); const STDIO_IDLE_TIMEOUT_MS = - Number.isFinite(stdioIdleTimeoutEnv) && stdioIdleTimeoutEnv >= 0 - ? stdioIdleTimeoutEnv - : 10 * 60 * 1000; + Number.isFinite(stdioIdleTimeoutEnv) && stdioIdleTimeoutEnv >= 0 ? stdioIdleTimeoutEnv : 0; let noteSessionActivity: () => void = () => undefined; let beginTrackedSessionWork: () => void = () => undefined; let endTrackedSessionWork: () => void = () => undefined; @@ -207,7 +252,14 @@ type ProjectResolution = | { ok: true; project: ProjectState } | { ok: false; response: ToolResponse }; -function finalizeJsonTextPayload(payload: Record): string { +export function finalizeJsonTextPayload( + payload: Record, + relationshipResponse = false +): string { + if (relationshipResponse) { + return finalizeRelationshipPayload(payload); + } + const mode = typeof payload.budget === 'object' && payload.budget !== null && @@ -430,7 +482,10 @@ function setActiveProject(rootPath: string): void { rememberProjectPath(resolvedRootPath); } -function syncKnownRoots(rootEntries: Array<{ rootPath: string; label?: string }>): void { +async function syncKnownRoots( + rootEntries: Array<{ rootPath: string; label?: string }> +): Promise { + const watcherClosures: Promise[] = []; const nextRoots = new Map(); const normalizedRoots = rootEntries.length > 0 ? rootEntries : primaryRootPath ? [{ rootPath: primaryRootPath }] : []; @@ -452,7 +507,7 @@ function syncKnownRoots(rootEntries: Array<{ rootPath: string; label?: string }> for (const [rootKey, existingRoot] of knownRoots.entries()) { if (!nextRoots.has(rootKey)) { - removeProject(existingRoot.rootPath); + watcherClosures.push(removeProject(existingRoot.rootPath)); forgetProjectPath(existingRoot.rootPath); } } @@ -462,7 +517,7 @@ function syncKnownRoots(rootEntries: Array<{ rootPath: string; label?: string }> isPathWithin(knownRoot.rootPath, project.rootPath) ); if (!stillAllowed) { - removeProject(project.rootPath); + watcherClosures.push(removeProject(project.rootPath)); forgetProjectPath(project.rootPath); } } @@ -479,6 +534,7 @@ function syncKnownRoots(rootEntries: Array<{ rootPath: string; label?: string }> activeProjectKey = undefined; } } + await Promise.all(watcherClosures); } function parseProjectSelector(value: unknown): string | undefined { @@ -618,7 +674,8 @@ function createToolContext(project: ProjectState): ToolContext { paths: project.paths, rootPath: project.rootPath, project: buildProjectDescriptor(project.rootPath), - performIndexing: (incrementalOnly?: boolean) => performIndexing(project, incrementalOnly), + performIndexing: (incrementalOnly, _reason, skipEmbedding) => + performIndexing(project, incrementalOnly, skipEmbedding), listProjects: () => listProjectDescriptors(), getActiveProject: () => getActiveProjectDescriptor() }; @@ -643,6 +700,7 @@ if (primaryRootPath) { export const INDEX_CONSUMING_TOOL_NAMES = [ 'search_codebase', 'get_symbol_references', + 'get_code_relationships', 'detect_circular_dependencies', 'get_team_patterns', 'get_codebase_metadata', @@ -651,23 +709,30 @@ export const INDEX_CONSUMING_TOOL_NAMES = [ export const INDEX_CONSUMING_RESOURCE_NAMES = ['Codebase Intelligence'] as const; -type IndexStatus = 'ready' | 'rebuild-required' | 'indexing' | 'unknown'; -type IndexConfidence = 'high' | 'low'; -type IndexAction = 'served' | 'rebuild-started' | 'rebuilt-and-served'; - -export type IndexSignal = { - status: IndexStatus; - confidence: IndexConfidence; - action: IndexAction; - reason?: string; -}; +async function requireValidIndex(project: ProjectState): Promise { + const meta = await readIndexMeta(project.rootPath); + await validateIndexArtifacts(project.rootPath, meta); -async function requireValidIndex(rootPath: string, paths: ToolPaths): Promise { - const meta = await readIndexMeta(rootPath); - await validateIndexArtifacts(rootPath, meta); + const requestedSkipEmbedding = project.runtimeOverrides.skipEmbedding ?? bootstrapSkipEmbedding; + if ( + requestedSkipEmbedding !== undefined && + (meta.artifacts.vectorDb.embeddingEnabled ?? true) !== !requestedSkipEmbedding + ) { + const reason = `Index embedding mode differs from explicit configuration; rebuilding ${project.rootPath}`; + console.error(`[Index] ${reason}`); + void performIndexing(project); + return { + status: 'indexing', + confidence: 'low', + action: 'rebuild-started', + reason + }; + } // Optional artifact presence informs confidence. - const hasIntelligence = await fileExists(paths.intelligence); + const hasIntelligence = await fileExists( + await resolveIndexArtifact(project.rootPath, 'intelligence.json', meta) + ); return { status: 'ready', @@ -678,7 +743,7 @@ async function requireValidIndex(rootPath: string, paths: ToolPaths): Promise { - if (project.indexState.status === 'indexing') { + if (project.indexState.status === 'indexing' && !(await hasCommittedIndex(project.rootPath))) { return { status: 'indexing', confidence: 'low', @@ -688,7 +753,7 @@ async function ensureValidIndexOrAutoHeal(project: ProjectState): Promise { + // Gate INDEX_CONSUMING tools on a valid, healthy index + let indexSignal: IndexSignal | undefined; + if ((INDEX_CONSUMING_TOOL_NAMES as readonly string[]).includes(name)) { + if ( + project.indexState.status === 'indexing' && + !(await hasCommittedIndex(project.rootPath)) + ) { + return { + content: [ + { + type: 'text', + text: JSON.stringify({ + status: 'indexing', + message: 'Index build in progress. Check indexing status, then retry.' + }) + } + ] + }; + } + if ( + project.indexState.status === 'error' && + !(await hasCommittedIndex(project.rootPath)) + ) { + return { + content: [ + { + type: 'text', + text: JSON.stringify({ + status: 'error', + message: `Indexer error: ${project.indexState.error}` + }) + } + ] + }; + } + indexSignal = await ensureValidIndexOrAutoHeal(project); + if (indexSignal.action === 'rebuild-started') { + return { + content: [ + { + type: 'text', + text: JSON.stringify({ + status: 'indexing', + message: 'Index rebuild in progress. Check indexing status, then retry.', + index: indexSignal + }) + } + ] + }; + } } - } - const result = await dispatchTool(name, normalizedArgs, createToolContext(project)); - - // Inject routing/index metadata into JSON responses so agents can reuse the resolved project safely. - if (indexSignal !== undefined && result.content?.[0]) { - try { - const parsed = JSON.parse(result.content[0].text); - result.content[0] = { - type: 'text', - text: finalizeJsonTextPayload({ - ...parsed, - index: indexSignal, - project: buildProjectDescriptor(project.rootPath) - }) - }; - } catch { - /* response wasn't JSON, skip injection */ - } - } else if (result.content?.[0]) { - try { - const parsed = JSON.parse(result.content[0].text); - result.content[0] = { - type: 'text', - text: finalizeJsonTextPayload({ - ...parsed, - project: buildProjectDescriptor(project.rootPath) - }) - }; - } catch { - /* response wasn't JSON, skip injection */ + const result = await dispatchTool(name, normalizedArgs, createToolContext(project)); + + // Inject routing/index metadata into JSON responses so agents can reuse the resolved project safely. + if (indexSignal !== undefined && result.content?.[0]) { + try { + const parsed = JSON.parse(result.content[0].text); + const lateSignal = IndexSignalSchema.safeParse(parsed.index); + let finalSignal = + lateSignal.success && lateSignal.data.status === parsed.status + ? lateSignal.data + : indexSignal; + if ( + lateSignal.success && + lateSignal.data.action.startsWith('rebuild-') && + project.indexState.status === 'error' + ) { + parsed.status = 'error'; + parsed.message = `Index recovery failed: ${project.indexState.error ?? 'unknown error'}`; + finalSignal = { + status: 'error', + confidence: 'low', + action: 'rebuild-failed', + reason: parsed.message + }; + result.isError = true; + } + result.content[0] = { + type: 'text', + text: finalizeJsonTextPayload( + { + ...parsed, + index: finalSignal, + project: buildProjectDescriptor(project.rootPath) + }, + name === 'get_code_relationships' + ) + }; + } catch { + /* response wasn't JSON, skip injection */ + } + } else if (result.content?.[0]) { + try { + const parsed = JSON.parse(result.content[0].text); + result.content[0] = { + type: 'text', + text: finalizeJsonTextPayload( + { + ...parsed, + project: buildProjectDescriptor(project.rootPath) + }, + name === 'get_code_relationships' + ) + }; + } catch { + /* response wasn't JSON, skip injection */ + } } - } - return result; + return result; + }); } catch (error) { return { content: [ @@ -1248,21 +1346,19 @@ async function performIndexingOnce( let lastLoggedProgress = { phase: '', percentage: -1 }; const indexer = new CodebaseIndexer({ rootPath: project.rootPath, - ...(project.runtimeOverrides.extraExcludePatterns?.length - ? { - config: { - exclude: [ - ...EXCLUDED_GLOB_PATTERNS, - ...project.runtimeOverrides.extraExcludePatterns - ], - ...(project.runtimeOverrides.maxChunks - ? { parsing: { maxChunks: project.runtimeOverrides.maxChunks } } - : {}) + config: { + ...(project.runtimeOverrides.extraExcludePatterns?.length + ? { + exclude: [...EXCLUDED_GLOB_PATTERNS, ...project.runtimeOverrides.extraExcludePatterns] } - } - : project.runtimeOverrides.maxChunks - ? { config: { parsing: { maxChunks: project.runtimeOverrides.maxChunks } } } : {}), + ...(project.runtimeOverrides.maxChunks + ? { parsing: { maxChunks: project.runtimeOverrides.maxChunks } } + : {}), + ...((project.runtimeOverrides.skipEmbedding ?? bootstrapSkipEmbedding) !== undefined + ? { skipEmbedding: project.runtimeOverrides.skipEmbedding ?? bootstrapSkipEmbedding } + : {}) + }, ...(project.runtimeOverrides.preferredAnalyzer || project.runtimeOverrides.extraSourceExtensions?.length ? { @@ -1317,11 +1413,25 @@ async function performIndexingOnce( } } -async function performIndexing(project: ProjectState, incrementalOnly?: boolean): Promise { +async function performIndexing( + project: ProjectState, + incrementalOnly?: boolean, + skipEmbedding?: boolean +): Promise { + const previousSkipEmbedding = project.runtimeOverrides.skipEmbedding; + if (skipEmbedding !== undefined) project.runtimeOverrides.skipEmbedding = skipEmbedding; let nextMode = incrementalOnly; for (;;) { await performIndexingOnce(project, nextMode); + if (project.indexState.status === 'error' && skipEmbedding !== undefined) { + if (previousSkipEmbedding === undefined) { + delete project.runtimeOverrides.skipEmbedding; + } else { + project.runtimeOverrides.skipEmbedding = previousSkipEmbedding; + } + } + const shouldRunQueuedRefresh = project.autoRefresh.consumeQueuedRefresh( project.indexState.status ); @@ -1336,7 +1446,7 @@ async function performIndexing(project: ProjectState, incrementalOnly?: boolean) async function shouldReindex(paths: ToolPaths): Promise { try { - await fs.access(paths.keywordIndex); + await fs.access(await resolveIndexArtifact(path.dirname(paths.baseDir), 'index.json')); return false; } catch { return true; @@ -1397,10 +1507,10 @@ async function refreshKnownRootsFromClient(): Promise { ); clientRootsEnabled = fileRoots.length > 0; - syncKnownRoots(fileRoots); + await syncKnownRoots(fileRoots); } catch { clientRootsEnabled = false; - syncKnownRoots(primaryRootPath ? [{ rootPath: primaryRootPath }] : []); + await syncKnownRoots(primaryRootPath ? [{ rootPath: primaryRootPath }] : []); } await refreshDiscoveredProjectsForKnownRoots(); @@ -1555,12 +1665,20 @@ async function ensureProjectInitialized(project: ProjectState): Promise { await project.initPromise; } -function ensureProjectWatcher(project: ProjectState, debounceMs: number): void { +async function ensureProjectWatcher(project: ProjectState, debounceMs: number): Promise { if (project.stopWatcher) { touchProject(project.rootPath); return; } + if (project.watcherClosePromise) { + await project.watcherClosePromise; + if (project.stopWatcher) { + touchProject(project.rootPath); + return; + } + } + project.stopWatcher = startFileWatcher({ rootPath: project.rootPath, debounceMs, @@ -1602,8 +1720,7 @@ function ensureProjectWatcher(project: ProjectState, debounceMs: number): void { const projectToEvict = evictionCandidates[0]; if (projectToEvict?.stopWatcher) { - projectToEvict.stopWatcher(); - delete projectToEvict.stopWatcher; + await stopProjectWatcher(projectToEvict); } } @@ -1618,7 +1735,7 @@ async function initProject( touchProject(project.rootPath); if (options.enableWatcher) { - ensureProjectWatcher(project, debounceMs); + await ensureProjectWatcher(project, debounceMs); } } @@ -1642,6 +1759,10 @@ function normalizeRuntimeExtensions(extensions?: string[]): string[] | undefined function buildProjectRuntimeOverrides(projectConfig: ProjectConfig): ProjectRuntimeOverrides { const runtimeOverrides: ProjectRuntimeOverrides = {}; + if ((bootstrapSkipEmbedding ?? projectConfig.skipEmbedding) !== undefined) { + runtimeOverrides.skipEmbedding = bootstrapSkipEmbedding ?? projectConfig.skipEmbedding; + } + if (projectConfig.excludePatterns?.length) { runtimeOverrides.extraExcludePatterns = [...projectConfig.excludePatterns]; } @@ -1754,12 +1875,6 @@ async function main() { // ── Cleanup guards (normal MCP lifecycle) ────────────────────────────────── let shuttingDown = false; - const stopAllWatchers = () => { - for (const project of getAllProjects()) { - project.stopWatcher?.(); - } - }; - let idleTimer: NodeJS.Timeout | undefined; let activeSessionWork = 0; let mcpClientInitialized = false; @@ -1777,8 +1892,13 @@ async function main() { } shuttingDown = true; clearIdleTimer(); - stopAllWatchers(); - process.exit(code); + void stopAllProjectWatchers().then( + () => process.exit(code), + (error: unknown) => { + console.error('[file-watcher] Shutdown cleanup failed:', error); + process.exit(code); + } + ); }; const scheduleIdleShutdown = () => { @@ -1812,7 +1932,11 @@ async function main() { scheduleIdleShutdown(); }; - process.once('exit', stopAllWatchers); + process.once('exit', () => { + void stopAllProjectWatchers().catch((error: unknown) => { + console.error('[file-watcher] Exit cleanup failed:', error); + }); + }); process.once('SIGINT', () => { shutdown(0); }); @@ -1945,22 +2069,32 @@ async function startHttp(explicitPort?: number): Promise { }); // Register cleanup — no parent death guard or stdin listeners in HTTP mode - const stopAllWatchers = () => { - for (const project of getAllProjects()) { - project.stopWatcher?.(); - } - }; - const shutdown = async () => { console.error('[HTTP] Shutting down...'); - stopAllWatchers(); + await stopAllProjectWatchers(); await handle.close(); process.exit(0); }; - process.once('SIGINT', () => void shutdown()); - process.once('SIGTERM', () => void shutdown()); - process.once('exit', stopAllWatchers); + process.once( + 'SIGINT', + () => + void shutdown().catch((error: unknown) => { + console.error('[HTTP] Shutdown failed:', error); + }) + ); + process.once( + 'SIGTERM', + () => + void shutdown().catch((error: unknown) => { + console.error('[HTTP] Shutdown failed:', error); + }) + ); + process.once('exit', () => { + void stopAllProjectWatchers().catch((error: unknown) => { + console.error('[file-watcher] Exit cleanup failed:', error); + }); + }); // If a bootstrap root was provided, auto-init it if (primaryRootPath) { @@ -1980,7 +2114,10 @@ if (!isDirectCliSubcommand) { if (isDirectRun) { const subcommand = process.argv[2]; - if (CLI_SUBCOMMANDS.includes(subcommand) || subcommand === '--help') { + if (subcommand === '--version' || subcommand === '-v') { + console.log(PKG_VERSION); + process.exit(0); + } else if (CLI_SUBCOMMANDS.includes(subcommand) || subcommand === '--help') { handleCliCommand(process.argv.slice(2)).catch((error) => { console.error('Error:', error instanceof Error ? error.message : String(error)); process.exit(1); diff --git a/src/project-state.ts b/src/project-state.ts index deaa5a6..fe7aae9 100644 --- a/src/project-state.ts +++ b/src/project-state.ts @@ -1,3 +1,4 @@ +import { resolveIndexArtifact } from './core/index-generation.js'; import path from 'path'; import { CODEBASE_CONTEXT_DIRNAME, @@ -12,6 +13,8 @@ import type { AutoRefreshController } from './core/auto-refresh.js'; import type { ToolPaths, IndexState } from './tools/types.js'; export interface ProjectRuntimeOverrides { + /** Explicit embedding mode; absent preserves the recorded project mode. */ + skipEmbedding?: boolean; /** Extra glob exclusion patterns merged with the default index-time exclusions. */ extraExcludePatterns?: string[]; /** Maximum chunks retained in semantic and keyword indexes for this project. */ @@ -28,7 +31,8 @@ export interface ProjectState { indexState: IndexState; autoRefresh: AutoRefreshController; initPromise?: Promise; - stopWatcher?: () => void; + stopWatcher?: () => Promise; + watcherClosePromise?: Promise; runtimeOverrides: ProjectRuntimeOverrides; } @@ -43,6 +47,17 @@ export function makePaths(rootPath: string): ToolPaths { }; } +export async function resolveProjectPaths(rootPath: string): Promise { + const legacy = makePaths(rootPath); + const [intelligence, health, keywordIndex, vectorDb] = await Promise.all([ + resolveIndexArtifact(rootPath, INTELLIGENCE_FILENAME), + resolveIndexArtifact(rootPath, HEALTH_FILENAME), + resolveIndexArtifact(rootPath, KEYWORD_INDEX_FILENAME), + resolveIndexArtifact(rootPath, VECTOR_DB_DIRNAME) + ]); + return { ...legacy, intelligence, health, keywordIndex, vectorDb }; +} + export function makeLegacyPaths(rootPath: string) { return { intelligence: path.join(rootPath, '.codebase-intelligence.json'), @@ -94,16 +109,45 @@ export function getAllProjects(): ProjectState[] { return Array.from(projects.values()); } -export function removeProject(rootPath: string): void { +export function stopAllProjectWatchers(): Promise { + const closing = Array.from(projects.values(), stopProjectWatcher); + return Promise.all(closing).then(() => undefined); +} + +export function stopProjectWatcher(project: ProjectState): Promise { + if (project.watcherClosePromise) return project.watcherClosePromise; + const stop = project.stopWatcher; + if (!stop) return Promise.resolve(); + + let closing: Promise; + try { + closing = Promise.resolve(stop()); + } catch (error) { + console.error(`[file-watcher] Failed to close watcher for ${project.rootPath}:`, error); + delete project.stopWatcher; + return Promise.resolve(); + } + + delete project.stopWatcher; + project.watcherClosePromise = closing + .catch((error: unknown) => { + console.error(`[file-watcher] Failed to close watcher for ${project.rootPath}:`, error); + }) + .finally(() => { + delete project.watcherClosePromise; + }); + return project.watcherClosePromise; +} + +export function removeProject(rootPath: string): Promise { const key = normalizeRootKey(rootPath); const project = projects.get(key); - project?.stopWatcher?.(); projects.delete(key); + return project ? stopProjectWatcher(project) : Promise.resolve(); } -export function clearProjects(): void { - for (const project of projects.values()) { - project.stopWatcher?.(); - } +export function clearProjects(): Promise { + const closing = stopAllProjectWatchers(); projects.clear(); + return closing; } diff --git a/src/review-cli.ts b/src/review-cli.ts index e4e08f7..aa9c6c1 100644 --- a/src/review-cli.ts +++ b/src/review-cli.ts @@ -1,4 +1,5 @@ #!/usr/bin/env node +import { resolveIndexArtifact } from './core/index-generation.js'; import { execFileSync } from 'node:child_process'; import { promises as fs } from 'node:fs'; @@ -9,14 +10,8 @@ import { GenericAnalyzer } from './analyzers/generic/index.js'; import { NestJsAnalyzer } from './analyzers/nestjs/index.js'; import { NextJsAnalyzer } from './analyzers/nextjs/index.js'; import { ReactAnalyzer } from './analyzers/react/index.js'; -import { - CODEBASE_CONTEXT_DIRNAME, - HEALTH_FILENAME, - INTELLIGENCE_FILENAME, - KEYWORD_INDEX_FILENAME, - MEMORY_FILENAME, - VECTOR_DB_DIRNAME -} from './constants/codebase-context.js'; +import { KEYWORD_INDEX_FILENAME } from './constants/codebase-context.js'; +import { makePaths } from './project-state.js'; import { analyzerRegistry } from './core/analyzer-registry.js'; import { CodebaseIndexer } from './core/indexer.js'; import { buildReviewContextPacket, parseNameStatus } from './review-context.js'; @@ -177,18 +172,11 @@ function assertReviewSourceState(rootPath: string, headCommit: string): void { async function createToolContext(rootPath: string, noIndex: boolean): Promise { const projectConfig = await loadProjectConfig(rootPath); - const paths = { - baseDir: path.join(rootPath, CODEBASE_CONTEXT_DIRNAME), - memory: path.join(rootPath, CODEBASE_CONTEXT_DIRNAME, MEMORY_FILENAME), - intelligence: path.join(rootPath, CODEBASE_CONTEXT_DIRNAME, INTELLIGENCE_FILENAME), - health: path.join(rootPath, CODEBASE_CONTEXT_DIRNAME, HEALTH_FILENAME), - keywordIndex: path.join(rootPath, CODEBASE_CONTEXT_DIRNAME, KEYWORD_INDEX_FILENAME), - vectorDb: path.join(rootPath, CODEBASE_CONTEXT_DIRNAME, VECTOR_DB_DIRNAME) - }; + const paths = makePaths(rootPath); let indexExists = false; try { - await fs.access(paths.keywordIndex); + await fs.access(await resolveIndexArtifact(rootPath, KEYWORD_INDEX_FILENAME)); indexExists = true; } catch { indexExists = false; diff --git a/src/review-context.ts b/src/review-context.ts index f3b76a7..7d97490 100644 --- a/src/review-context.ts +++ b/src/review-context.ts @@ -116,6 +116,7 @@ export interface ReviewSearchResult { sourceFiles: string[]; searchQuality?: SearchQuality; preflight?: SearchResponse['preflight']; + sourceCoverage?: SearchResponse['sourceCoverage']; results: ReviewRelatedResult[]; error?: string; } @@ -398,8 +399,12 @@ export async function buildReviewContextPacket( sourceFiles: reviewQuery.sourceFiles, searchQuality: response.searchQuality, ...(response.preflight ? { preflight: response.preflight } : {}), + ...(response.sourceCoverage ? { sourceCoverage: response.sourceCoverage } : {}), results: response.results.slice(0, maxResultsPerQuery).map(compactSearchResult) }); + if (response.sourceCoverage && response.sourceCoverage.status !== 'current') { + warnings.push(`Search source coverage is incomplete for query: ${reviewQuery.query}`); + } } catch (error) { searches.push({ query: reviewQuery.query, diff --git a/src/server/config.ts b/src/server/config.ts index ff6c1d6..8a8533f 100644 --- a/src/server/config.ts +++ b/src/server/config.ts @@ -4,6 +4,7 @@ import path from 'node:path'; export interface ProjectConfig { root: string; + skipEmbedding?: boolean; excludePatterns?: string[]; parsing?: { maxChunks?: number; @@ -85,6 +86,13 @@ export async function loadServerConfig(): Promise { const resolvedRoot = path.resolve(expandTilde(rawRoot)); const parsedProject: ProjectConfig = { root: resolvedRoot }; + if (typeof project.skipEmbedding === 'boolean') { + parsedProject.skipEmbedding = project.skipEmbedding; + } else if (project.skipEmbedding !== undefined) { + console.error( + `[config] Ignoring invalid project skipEmbedding: ${project.skipEmbedding}` + ); + } const excludePatterns = parseStringArray(project.excludePatterns); if (excludePatterns) { parsedProject.excludePatterns = excludePatterns; diff --git a/src/storage/index.ts b/src/storage/index.ts index fb9d9e2..d776902 100644 --- a/src/storage/index.ts +++ b/src/storage/index.ts @@ -8,6 +8,11 @@ export * from './lancedb.js'; import { VectorStorageProvider, StorageConfig, DEFAULT_STORAGE_CONFIG } from './types.js'; import { LanceDBStorageProvider } from './lancedb.js'; +import { assertOptionalDependency } from '../utils/optional-dependency.js'; + +export function assertStorageRuntimeAvailable(): void { + assertOptionalDependency('@lancedb/lancedb', '^0.4.0'); +} /** * Get a storage provider based on configuration @@ -16,6 +21,7 @@ export async function getStorageProvider( config: Partial = {}, options?: { expectExisting?: boolean; expectedDimensions?: number } ): Promise { + assertStorageRuntimeAvailable(); const mergedConfig = { ...DEFAULT_STORAGE_CONFIG, ...config }; const provider = new LanceDBStorageProvider(); diff --git a/src/storage/lancedb.ts b/src/storage/lancedb.ts index 9351d94..ec745d8 100644 --- a/src/storage/lancedb.ts +++ b/src/storage/lancedb.ts @@ -8,6 +8,11 @@ import type { Connection, Table } from '@lancedb/lancedb'; import { VectorStorageProvider, CodeChunkWithEmbedding, VectorSearchResult } from './types.js'; import { CodeChunk, SearchFilters } from '../types/index.js'; import { IndexCorruptedError } from '../errors/index.js'; +import { assertOptionalDependency } from '../utils/optional-dependency.js'; + +function sqlString(value: string): string { + return `'${value.replace(/'/g, "''")}'`; +} interface LanceDBRecord { id: string; @@ -48,6 +53,7 @@ export class LanceDBStorageProvider implements VectorStorageProvider { options?: { expectExisting?: boolean; expectedDimensions?: number } ): Promise { if (this.initialized) return; + assertOptionalDependency('@lancedb/lancedb', '^0.4.0'); try { this.storagePath = storagePath; @@ -91,6 +97,11 @@ export class LanceDBStorageProvider implements VectorStorageProvider { this.initialized = true; console.error(`LanceDB initialized at: ${storagePath}`); } catch (error) { + try { + this.close(); + } catch (closeError) { + console.warn('Failed to release partial LanceDB initialization:', closeError); + } if (error instanceof IndexCorruptedError) { throw error; } @@ -176,16 +187,16 @@ export class LanceDBStorageProvider implements VectorStorageProvider { const whereConditions: string[] = []; if (filters.framework) { - whereConditions.push(`framework = '${filters.framework}'`); + whereConditions.push(`framework = ${sqlString(filters.framework)}`); } if (filters.componentType) { - whereConditions.push(`"componentType" = '${filters.componentType}'`); + whereConditions.push(`\`componentType\` = ${sqlString(filters.componentType)}`); } if (filters.layer) { - whereConditions.push(`layer = '${filters.layer}'`); + whereConditions.push(`layer = ${sqlString(filters.layer)}`); } if (filters.language) { - whereConditions.push(`language = '${filters.language}'`); + whereConditions.push(`language = ${sqlString(filters.language)}`); } if (whereConditions.length > 0) { @@ -248,11 +259,10 @@ export class LanceDBStorageProvider implements VectorStorageProvider { try { const countBefore = await this.table.countRows(); - // LanceDB supports SQL-style filter for delete - // Escape single quotes in file paths to prevent SQL injection - const escaped = filePaths.map((p) => p.replace(/'/g, "''")); - const inClause = escaped.map((p) => `'${p}'`).join(', '); - await this.table.delete(`"filePath" IN (${inClause})`); + // The supported Lance parser requires backticks for case-sensitive identifiers; + // double quotes produce a string literal and silently match no rows. + const inClause = filePaths.map(sqlString).join(', '); + await this.table.delete(`\`filePath\` IN (${inClause})`); const countAfter = await this.table.countRows(); const deleted = countBefore - countAfter; @@ -296,6 +306,19 @@ export class LanceDBStorageProvider implements VectorStorageProvider { } } + close(): void { + const table = this.table; + const database = this.db; + this.table = null; + this.db = null; + this.initialized = false; + try { + table?.close(); + } finally { + database?.close(); + } + } + isInitialized(): boolean { return this.initialized; } diff --git a/src/storage/types.ts b/src/storage/types.ts index 75a553f..85665d4 100644 --- a/src/storage/types.ts +++ b/src/storage/types.ts @@ -50,6 +50,9 @@ export interface VectorStorageProvider { * Check if storage is initialized */ isInitialized(): boolean; + + /** Release native handles before publishing or releasing an operation snapshot. */ + close?(): Promise | void; } export interface CodeChunkWithEmbedding extends CodeChunk { diff --git a/src/tools/detect-circular-dependencies.ts b/src/tools/detect-circular-dependencies.ts index 8a9a679..eb443a9 100644 --- a/src/tools/detect-circular-dependencies.ts +++ b/src/tools/detect-circular-dependencies.ts @@ -1,3 +1,4 @@ +import { withToolIndexSnapshot } from './index-snapshot.js'; import type { Tool } from '@modelcontextprotocol/sdk/types.js'; import { promises as fs } from 'fs'; import path from 'path'; @@ -26,6 +27,13 @@ export const definition: Tool = { export async function handle( args: Record, ctx: ToolContext +): Promise { + return withToolIndexSnapshot(ctx, (pinned) => handlePinned(args, pinned)); +} + +async function handlePinned( + args: Record, + ctx: ToolContext ): Promise { const { scope } = args as { scope?: string }; diff --git a/src/tools/get-code-relationships.ts b/src/tools/get-code-relationships.ts new file mode 100644 index 0000000..8897c69 --- /dev/null +++ b/src/tools/get-code-relationships.ts @@ -0,0 +1,418 @@ +import { withToolIndexSnapshot } from './index-snapshot.js'; +import type { Tool } from '@modelcontextprotocol/sdk/types.js'; +import { createHash } from 'crypto'; +import { promises as fs } from 'fs'; +import path from 'path'; +import { z } from 'zod'; +import { RELATIONSHIPS_FILENAME } from '../constants/codebase-context.js'; +import { readIndexMeta } from '../core/index-meta.js'; +import { + traverseFileGraphWithVerification, + type FileGraphEdge, + type UnresolvedImportEdge +} from '../utils/usage-tracker.js'; +import { + InternalSourceResolver, + loadTsconfigCompilerOptions +} from '../utils/internal-source-resolver.js'; +import type { ToolContext, ToolResponse } from './types.js'; + +const RESPONSE_BYTE_BUDGET = 4000; +const MAX_INSPECTIONS = 1000; +const MAX_DIAGNOSTICS = 20; + +function serializeBoundedResponse(value: Record): { + text: string; + truncated: boolean; +} { + const bounded: Record = { ...value }; + const collectionKeys = ['paths', 'unresolved', 'staleEdges', 'staleUnresolvedEdges']; + for (const key of collectionKeys) { + if (Array.isArray(bounded[key])) bounded[key] = [...bounded[key]]; + } + let truncated = value.truncated === true; + const encode = (): string => JSON.stringify(bounded); + let payload = encode(); + const trimOrder = ['staleUnresolvedEdges', 'staleEdges', 'unresolved', 'paths']; + while (Buffer.byteLength(payload, 'utf8') > RESPONSE_BYTE_BUDGET) { + const key = trimOrder.find( + (candidate) => + Array.isArray(bounded[candidate]) && (bounded[candidate] as unknown[]).length > 0 + ); + if (!key) { + return { + text: JSON.stringify({ + status: 'output_limited', + truncated: true, + message: + 'The relationship response exceeded its byte budget; narrow the starting file or hop limit.' + }), + truncated: true + }; + } + (bounded[key] as unknown[]).pop(); + truncated = true; + bounded.truncated = true; + bounded.outputNote = [ + value.outputNote, + 'The serialized response reached its 4000-byte budget; some paths or diagnostics were omitted. Narrow the starting file or hop limit for more.' + ] + .filter(Boolean) + .join(' '); + payload = encode(); + } + return { text: payload, truncated }; +} + +export function finalizeRelationshipPayload(value: Record): string { + return serializeBoundedResponse(value).text; +} + +export const definition: Tool = { + name: 'get_code_relationships', + description: + 'Follow verified import or re-export relationships up to three hops from one indexed file. Returns source lines and freshness diagnostics; imports never imply runtime calls.', + inputSchema: { + type: 'object', + properties: { + file: { type: 'string', description: 'Indexed repository-relative file path.' }, + direction: { type: 'string', enum: ['outgoing', 'incoming'] }, + max_hops: { type: 'number', description: 'Hop limit from 1 to 3; defaults to 2.' }, + max_nodes: { type: 'number', description: 'Output node cap from 1 to 100; defaults to 50.' } + }, + required: ['file'] + } +}; + +function asObject(value: unknown): Record { + return value !== null && typeof value === 'object' && !Array.isArray(value) + ? (value as Record) + : {}; +} + +function safeRelativePath(value: string): string | null { + const normalized = value.replace(/\\/g, '/').replace(/^\.\//, ''); + if ( + !normalized || + path.posix.isAbsolute(normalized) || + /^[A-Za-z]:\//.test(normalized) || + normalized.startsWith('//') || + normalized.split('/').some((part) => part === '..') + ) { + return null; + } + return normalized; +} + +const StoredRelativePath = z + .string() + .refine( + (value) => + safeRelativePath(value) === value && + !path.win32.parse(value).root && + !value.includes('\\') && + !value.includes('\0') && + value.split('/').every((part) => part !== '' && part !== '.'), + 'Expected a canonical repository-relative path.' + ); +const StoredSource = z.object({ + line: z.number().int().min(1), + specifier: z.string(), + kind: z.enum(['import', 'reexport']), + sourceFingerprint: z.string(), + targetFingerprint: z.string() +}); +const UnresolvedSource = z.object({ + from: StoredRelativePath, + kind: z.enum(['import', 'reexport']), + source: StoredSource.extend({ sourceFingerprint: z.string().regex(/^[a-f0-9]{64}$/) }) +}); +const StoredUnresolved = z.union([ + UnresolvedSource.extend({ + reason: z.enum(['external', 'not-found', 'outside-root', 'unsupported-alias']) + }), + UnresolvedSource.extend({ + reason: z.enum(['not-indexed', 'analysis-unavailable']), + target: StoredRelativePath + }) +]); +const StoredGraph = z.object({ + imports: z.record(StoredRelativePath, z.array(StoredRelativePath)), + importDetails: z + .record( + StoredRelativePath, + z.record( + StoredRelativePath, + z.object({ + line: z.number().optional(), + importedSymbols: z.array(z.string()).optional(), + sources: z.array(StoredSource).optional() + }) + ) + ) + .optional(), + unresolvedEdges: z.array(StoredUnresolved).optional() +}); + +function unresolvedDiagnostic(edge: UnresolvedImportEdge) { + return { + from: edge.from, + kind: edge.kind, + source: { line: edge.source.line, specifier: edge.source.specifier }, + reason: edge.reason, + ...('target' in edge ? { target: edge.target } : {}) + }; +} + +function matchesRelativePath(root: string, absolute: string, expected: string): boolean { + const relative = path.relative(root, absolute).replace(/\\/g, '/'); + return process.platform === 'win32' + ? relative.toLowerCase() === expected.replace(/\\/g, '/').toLowerCase() + : relative === expected.replace(/\\/g, '/'); +} + +async function resolveConfined(rootRealPath: string, relativePath: string): Promise { + const safe = safeRelativePath(relativePath); + if (!safe) return null; + try { + const realPath = await fs.realpath(path.resolve(rootRealPath, ...safe.split('/'))); + const relative = path.relative(rootRealPath, realPath); + return relative === '' || + (!relative.startsWith(`..${path.sep}`) && relative !== '..' && !path.isAbsolute(relative)) + ? realPath + : null; + } catch { + return null; + } +} + +async function currentFingerprint(filePath: string): Promise { + try { + const content = await fs.readFile(filePath); + return createHash('sha256').update(content).digest('hex'); + } catch { + return null; + } +} + +async function verifyEdge( + rootRealPath: string, + edge: FileGraphEdge, + resolver: InternalSourceResolver +): Promise { + if ( + !edge.verified || + !edge.source.specifier || + !edge.source.sourceFingerprint || + !edge.source.targetFingerprint + ) { + return false; + } + const [from, to] = await Promise.all([ + resolveConfined(rootRealPath, edge.from), + resolveConfined(rootRealPath, edge.to) + ]); + if (!from || !to) return false; + const currentResolution = await resolver.resolve(from, edge.source.specifier, true); + if (currentResolution.status !== 'resolved' || !currentResolution.path) return false; + const currentTarget = await fs.realpath(currentResolution.path).catch(() => null); + if (!currentTarget || !matchesRelativePath(rootRealPath, currentTarget, edge.to)) { + return false; + } + const [fromFingerprint, toFingerprint] = await Promise.all([ + currentFingerprint(from), + currentFingerprint(to) + ]); + return ( + fromFingerprint === edge.source.sourceFingerprint && + toFingerprint === edge.source.targetFingerprint + ); +} + +export async function handle( + args: Record, + ctx: ToolContext +): Promise { + return withToolIndexSnapshot(ctx, (pinned) => handlePinned(args, pinned)); +} + +async function handlePinned( + args: Record, + ctx: ToolContext +): Promise { + const fileArg = typeof args.file === 'string' ? args.file : ''; + const direction = args.direction === 'incoming' ? 'incoming' : 'outgoing'; + const maxHops = + args.max_hops === 1 || args.max_hops === 2 || args.max_hops === 3 ? args.max_hops : 2; + const maxNodes = + typeof args.max_nodes === 'number' && Number.isFinite(args.max_nodes) + ? Math.max(1, Math.min(100, Math.floor(args.max_nodes))) + : 50; + const reply = (value: Record, isError = false): ToolResponse => ({ + content: [{ type: 'text', text: finalizeRelationshipPayload(value) }], + ...(isError ? { isError: true } : {}) + }); + + try { + const rootRealPath = await fs.realpath(ctx.rootPath); + const start = safeRelativePath(fileArg); + if (!start) + return reply( + { status: 'invalid_path', message: 'Provide a repository-relative file path.' }, + true + ); + const startRealPath = await resolveConfined(rootRealPath, start); + if (!startRealPath) + return reply({ + status: 'not_indexed_or_outside_root', + file: start, + message: 'The requested file is missing or resolves outside this project root.' + }); + + const sidecarPath = path.join(path.dirname(ctx.paths.intelligence), RELATIONSHIPS_FILENAME); + const parsed: unknown = JSON.parse(await fs.readFile(sidecarPath, 'utf8')); + const sidecar = asObject(parsed); + const header = asObject(sidecar.header); + const meta = await readIndexMeta(ctx.rootPath); + if (!header.buildId || header.buildId !== meta.buildId) { + return reply({ + status: 'stale_index', + message: + 'Relationship sidecar and index metadata have different build identities. Refresh the index before relying on graph results.' + }); + } + const graph = StoredGraph.safeParse(sidecar.graph); + if (!graph.success) { + return reply({ + status: 'graph_unavailable', + message: + 'The relationship graph is missing or contains invalid records. Refresh indexing to rebuild it.' + }); + } + + // Every edge is resolved against current confined files with allowUnindexed=true. + // Avoid resolving every stored graph file during setup for a nearby relationship query. + const resolver = await InternalSourceResolver.create(rootRealPath, [], { + compilerOptions: loadTsconfigCompilerOptions(rootRealPath) + }); + const verificationCache = new Map(); + const result = await traverseFileGraphWithVerification( + graph.data, + start, + { direction, maxHops, maxNodes, maxPaths: 40, maxInspections: MAX_INSPECTIONS }, + async (edge) => { + const proofKey = JSON.stringify([ + edge.from, + edge.to, + edge.kind, + edge.source.specifier, + edge.source.line, + edge.source.sourceFingerprint, + edge.source.targetFingerprint + ]); + let verified = verificationCache.get(proofKey); + if (verified === undefined) { + verified = await verifyEdge(rootRealPath, edge, resolver); + verificationCache.set(proofKey, verified); + } + return verified; + } + ); + const staleEdges = result.rejectedEdges.slice(0, MAX_DIAGNOSTICS).map((edge) => ({ + from: edge.from, + to: edge.to, + kind: edge.kind, + line: edge.source.line, + specifier: edge.source.specifier + })); + + const unresolved: UnresolvedImportEdge[] = []; + const staleUnresolvedEdges: UnresolvedImportEdge[] = []; + const unresolvedVerification = new Map>(); + // Traversal already charged each returned unresolved candidate to its shared + // inspection budget. Verify only that bounded set, once per distinct proof. + for (const edge of result.unresolved) { + const key = JSON.stringify([ + edge.from, + edge.source.specifier, + edge.source.sourceFingerprint, + edge.reason, + ...('target' in edge ? [edge.target] : []) + ]); + let verification = unresolvedVerification.get(key); + if (!verification) { + verification = (async () => { + const importer = await resolveConfined(rootRealPath, edge.from); + if (!importer || !edge.source.specifier) return false; + if ((await currentFingerprint(importer)) !== edge.source.sourceFingerprint) return false; + const current = await resolver.resolve(importer, edge.source.specifier, true); + if ('target' in edge) { + if (current.status !== 'resolved') return false; + return process.platform === 'win32' + ? current.target.toLowerCase() === edge.target.toLowerCase() + : current.target === edge.target; + } + return current.status === 'unresolved' && current.reason === edge.reason; + })(); + unresolvedVerification.set(key, verification); + } + if (await verification) unresolved.push(edge); + else staleUnresolvedEdges.push(edge); + } + const stale = result.rejectedEdges.length > 0 || staleUnresolvedEdges.length > 0; + const visiblePaths = result.paths; + const visibleUnresolved = unresolved.slice(0, MAX_DIAGNOSTICS); + const visibleStaleEdges = staleEdges; + const visibleStaleUnresolved = staleUnresolvedEdges.slice(0, MAX_DIAGNOSTICS); + const diagnosticsLimited = + staleEdges.length < result.rejectedEdges.length || + visibleUnresolved.length < unresolved.length || + visibleStaleUnresolved.length < staleUnresolvedEdges.length; + const outputLimited = result.truncated || diagnosticsLimited; + return reply({ + status: stale ? 'stale_edges' : 'success', + file: start, + direction, + maxHops, + paths: visiblePaths.map((entry) => ({ + nodes: entry.nodes, + edges: entry.edges.map((edge) => ({ + from: edge.from, + to: edge.to, + kind: edge.kind, + source: { line: edge.source.line, specifier: edge.source.specifier } + })) + })), + unresolved: visibleUnresolved.map(unresolvedDiagnostic), + ...(visibleStaleEdges.length > 0 ? { staleEdges: visibleStaleEdges } : {}), + ...(visibleStaleUnresolved.length > 0 + ? { + staleUnresolvedEdges: visibleStaleUnresolved.map(unresolvedDiagnostic) + } + : {}), + truncated: outputLimited, + message: stale + ? 'Some stored relationships could not be proven against current confined source files. Omitted those edges; refresh indexing to update this graph.' + : unresolved.some((edge) => edge.reason === 'analysis-unavailable') + ? 'Some selected targets lack source snapshots. Inspect indexing errors and refresh after fixing them; these diagnostics were not followed.' + : unresolved.length > 0 + ? 'Resolved relationships are current. Unresolved source specifiers are diagnostics and were not followed.' + : 'Relationships are limited to verified imports and re-exports; dynamic calls are not inferred.', + ...(outputLimited + ? { + outputNote: result.inspectionLimited + ? 'The relationship inspection budget was reached; additional relationships were not checked. Refresh indexing to remove stale edges, then retry from this file or a returned endpoint.' + : result.truncated + ? 'The accepted node or path cap was reached; narrow the hop limit or continue from a returned endpoint for more.' + : 'Some freshness or unresolved diagnostics were omitted. Refresh indexing to update this graph.' + } + : {}) + }); + } catch (error) { + return reply({ + status: 'graph_unavailable', + message: 'Could not read a current relationship sidecar. Refresh indexing and retry.', + detail: error instanceof Error ? error.message : String(error) + }); + } +} diff --git a/src/tools/get-codebase-health.ts b/src/tools/get-codebase-health.ts index c770eff..233566a 100644 --- a/src/tools/get-codebase-health.ts +++ b/src/tools/get-codebase-health.ts @@ -1,3 +1,4 @@ +import { withToolIndexSnapshot } from './index-snapshot.js'; import type { Tool } from '@modelcontextprotocol/sdk/types.js'; import type { ToolContext, ToolResponse } from './types.js'; import { indexHealthByFile, normalizeHealthLookupKey, readHealthFile } from '../health/store.js'; @@ -30,6 +31,13 @@ export const definition: Tool = { export async function handle( args: Record, ctx: ToolContext +): Promise { + return withToolIndexSnapshot(ctx, (pinned) => handlePinned(args, pinned)); +} + +async function handlePinned( + args: Record, + ctx: ToolContext ): Promise { const file = typeof args.file === 'string' ? args.file.trim() : undefined; const limit = typeof args.limit === 'number' && Number.isFinite(args.limit) ? args.limit : 10; diff --git a/src/tools/get-codebase-metadata.ts b/src/tools/get-codebase-metadata.ts index 93f8a1e..7c98e41 100644 --- a/src/tools/get-codebase-metadata.ts +++ b/src/tools/get-codebase-metadata.ts @@ -1,3 +1,4 @@ +import { withToolIndexSnapshot } from './index-snapshot.js'; import type { Tool } from '@modelcontextprotocol/sdk/types.js'; import { promises as fs } from 'fs'; import type { ToolContext, ToolResponse } from './types.js'; @@ -17,6 +18,13 @@ export const definition: Tool = { export async function handle( _args: Record, ctx: ToolContext +): Promise { + return withToolIndexSnapshot(ctx, (pinned) => handlePinned(_args, pinned)); +} + +async function handlePinned( + _args: Record, + ctx: ToolContext ): Promise { const indexer = new CodebaseIndexer({ rootPath: ctx.rootPath }); const metadata = await indexer.detectMetadata(); diff --git a/src/tools/get-indexing-status.ts b/src/tools/get-indexing-status.ts index 9da86ca..1dc5e59 100644 --- a/src/tools/get-indexing-status.ts +++ b/src/tools/get-indexing-status.ts @@ -1,6 +1,9 @@ +import { fileURLToPath } from 'node:url'; import type { Tool } from '@modelcontextprotocol/sdk/types.js'; import type { ToolContext, ToolResponse } from './types.js'; +const loadedStatusModulePath = fileURLToPath(import.meta.url); + export const definition: Tool = { name: 'get_indexing_status', description: @@ -26,6 +29,7 @@ export async function handle( { status: ctx.indexState.status, rootPath: ctx.rootPath, + loadedStatusModulePath, lastIndexed: ctx.indexState.lastIndexed?.toISOString(), stats: ctx.indexState.stats ? { diff --git a/src/tools/get-style-guide.ts b/src/tools/get-style-guide.ts index f32f2c0..c4bd0ad 100644 --- a/src/tools/get-style-guide.ts +++ b/src/tools/get-style-guide.ts @@ -1,3 +1,4 @@ +import { withToolIndexSnapshot } from './index-snapshot.js'; import type { Tool } from '@modelcontextprotocol/sdk/types.js'; import { promises as fs } from 'fs'; import { glob } from 'glob'; @@ -26,6 +27,13 @@ export const definition: Tool = { export async function handle( args: Record, ctx: ToolContext +): Promise { + return withToolIndexSnapshot(ctx, (pinned) => handlePinned(args, pinned)); +} + +async function handlePinned( + args: Record, + ctx: ToolContext ): Promise { const { query, category } = args as { query?: string; diff --git a/src/tools/get-symbol-references.ts b/src/tools/get-symbol-references.ts index 0536f3c..8867cfd 100644 --- a/src/tools/get-symbol-references.ts +++ b/src/tools/get-symbol-references.ts @@ -1,11 +1,13 @@ +import { withToolIndexSnapshot } from './index-snapshot.js'; import type { Tool } from '@modelcontextprotocol/sdk/types.js'; import type { ToolContext, ToolResponse } from './types.js'; -import { findSymbolReferences } from '../core/symbol-references.js'; +import { findSymbolReferences, type SymbolReferencesResult } from '../core/symbol-references.js'; +import { IndexCorruptedError } from '../errors/index.js'; export const definition: Tool = { name: 'get_symbol_references', description: - 'Find concrete references to a symbol in indexed chunks. Returns total usageCount and top usage snippets.', + 'Find current-source references across the files recorded by the last successful index. Returns observed usageCount, bounded snippets, parser confidence, and coverage.', inputSchema: { type: 'object', properties: { @@ -26,6 +28,13 @@ export const definition: Tool = { export async function handle( args: Record, ctx: ToolContext +): Promise { + return withToolIndexSnapshot(ctx, (pinned) => handlePinned(args, pinned)); +} + +async function handlePinned( + args: Record, + ctx: ToolContext ): Promise { const { symbol, limit } = args as { symbol?: unknown; limit?: unknown }; const normalizedSymbol = typeof symbol === 'string' ? symbol.trim() : ''; @@ -51,7 +60,29 @@ export async function handle( }; } - const result = await findSymbolReferences(ctx.rootPath, normalizedSymbol, normalizedLimit); + let result: SymbolReferencesResult; + try { + result = await findSymbolReferences(ctx.rootPath, normalizedSymbol, normalizedLimit); + } catch (error) { + if (!(error instanceof IndexCorruptedError)) throw error; + return { + content: [ + { + type: 'text', + text: JSON.stringify( + { + status: 'error', + symbol: normalizedSymbol, + message: error.message + }, + null, + 2 + ) + } + ], + isError: true + }; + } if (result.status === 'error') { return { @@ -68,7 +99,8 @@ export async function handle( 2 ) } - ] + ], + isError: true }; } @@ -83,7 +115,8 @@ export async function handle( usageCount: result.usageCount, usages: result.usages, confidence: result.confidence, - isComplete: result.isComplete + isComplete: result.isComplete, + coverage: result.coverage }, null, 2 diff --git a/src/tools/get-team-patterns.ts b/src/tools/get-team-patterns.ts index 1988beb..4953d36 100644 --- a/src/tools/get-team-patterns.ts +++ b/src/tools/get-team-patterns.ts @@ -1,3 +1,4 @@ +import { withToolIndexSnapshot } from './index-snapshot.js'; import type { Tool } from '@modelcontextprotocol/sdk/types.js'; import { promises as fs } from 'fs'; import type { ToolContext, ToolResponse } from './types.js'; @@ -45,6 +46,13 @@ export const definition: Tool = { export async function handle( args: Record, ctx: ToolContext +): Promise { + return withToolIndexSnapshot(ctx, (pinned) => handlePinned(args, pinned)); +} + +async function handlePinned( + args: Record, + ctx: ToolContext ): Promise { const { category } = args as { category?: string }; diff --git a/src/tools/index-snapshot.ts b/src/tools/index-snapshot.ts new file mode 100644 index 0000000..8d72b92 --- /dev/null +++ b/src/tools/index-snapshot.ts @@ -0,0 +1,16 @@ +import { withAvailableIndexSnapshot } from '../core/index-generation.js'; +import { resolveProjectPaths } from '../project-state.js'; +import type { ToolContext } from './types.js'; + +/** Keep derived paths and all nested readers on one operation-scoped build. */ +export function withToolIndexSnapshot( + ctx: ToolContext, + operation: (pinned: ToolContext) => Promise +): Promise { + return withAvailableIndexSnapshot(ctx.rootPath, async () => + operation({ + ...ctx, + paths: await resolveProjectPaths(ctx.rootPath) + }) + ); +} diff --git a/src/tools/index.ts b/src/tools/index.ts index def32ac..7f5ea4b 100644 --- a/src/tools/index.ts +++ b/src/tools/index.ts @@ -13,6 +13,7 @@ import { definition as d8, handle as h8 } from './detect-circular-dependencies.j import { definition as d9, handle as h9 } from './remember.js'; import { definition as d10, handle as h10 } from './get-memory.js'; import { definition as d11, handle as h11 } from './get-codebase-health.js'; +import { definition as d12, handle as h12 } from './get-code-relationships.js'; import type { ToolContext, ToolResponse } from './types.js'; @@ -52,7 +53,7 @@ function withProjectSelector(definition: Tool): Tool { }; } -export const TOOLS: Tool[] = [d1, d2, d3, d4, d5, d6, d7, d8, d9, d10, d11].map( +export const TOOLS: Tool[] = [d1, d2, d3, d4, d5, d6, d7, d8, d9, d10, d11, d12].map( withProjectSelector ); @@ -84,6 +85,8 @@ export async function dispatchTool( return h10(args, ctx); case 'get_codebase_health': return h11(args, ctx); + case 'get_code_relationships': + return h12(args, ctx); default: return { content: [{ type: 'text', text: JSON.stringify({ error: `Unknown tool: ${name}` }) }], diff --git a/src/tools/refresh-index.ts b/src/tools/refresh-index.ts index 3c1b647..b2d1726 100644 --- a/src/tools/refresh-index.ts +++ b/src/tools/refresh-index.ts @@ -1,11 +1,14 @@ import type { Tool } from '@modelcontextprotocol/sdk/types.js'; +import { assertEmbeddingRuntimeAvailable } from '../embeddings/index.js'; +import { OptionalDependencyError } from '../errors/index.js'; +import { assertStorageRuntimeAvailable } from '../storage/index.js'; import type { ToolContext, ToolResponse } from './types.js'; export const definition: Tool = { name: 'refresh_index', description: 'Re-index the codebase. Supports full re-index or incremental mode. ' + - 'Use incrementalOnly=true to only process files changed since last index.', + 'Use incrementalOnly=true to process changed files. Omit skipEmbedding to preserve the last successful mode; true disables embedding and reranker models, false opts into semantic embeddings. Mode changes require a full rebuild.', inputSchema: { type: 'object', properties: { @@ -17,6 +20,11 @@ export const definition: Tool = { type: 'boolean', description: 'If true, only re-index files changed since last full index (faster). Default: false (full re-index)' + }, + skipEmbedding: { + type: 'boolean', + description: + 'True selects model-free indexing; false explicitly opts into semantic embeddings. Omit to preserve the last successful mode.' } } } @@ -26,12 +34,45 @@ export async function handle( args: Record, ctx: ToolContext ): Promise { - const { reason, incrementalOnly } = args as { reason?: string; incrementalOnly?: boolean }; + if (args.skipEmbedding !== undefined && typeof args.skipEmbedding !== 'boolean') { + return { + isError: true, + content: [{ type: 'text', text: 'skipEmbedding must be a boolean' }] + }; + } + if (ctx.indexState.status === 'indexing') { + return { + isError: true, + content: [ + { type: 'text', text: 'An index build is already running; retry after it completes.' } + ] + }; + } + const { reason, incrementalOnly, skipEmbedding } = args as { + reason?: string; + incrementalOnly?: boolean; + skipEmbedding?: boolean; + }; + + if (skipEmbedding === false) { + try { + assertEmbeddingRuntimeAvailable(); + assertStorageRuntimeAvailable(); + } catch (error) { + if (error instanceof OptionalDependencyError) { + return { + isError: true, + content: [{ type: 'text', text: error.message }] + }; + } + throw error; + } + } const mode = incrementalOnly ? 'incremental' : 'full'; console.error(`Refresh requested (${mode}): ${reason || 'Manual trigger'}`); - ctx.performIndexing(incrementalOnly); + void ctx.performIndexing(incrementalOnly, reason, skipEmbedding); return { content: [ @@ -42,7 +83,7 @@ export async function handle( status: 'started', mode, message: incrementalOnly - ? 'Incremental re-indexing started. Only changed files will be re-embedded.' + ? 'Incremental re-indexing started. A changed embedding mode requires a full rebuild.' : 'Full re-indexing started. Check status with get_indexing_status.', reason }, diff --git a/src/tools/search-codebase.ts b/src/tools/search-codebase.ts index 7a4f822..d423ce2 100644 --- a/src/tools/search-codebase.ts +++ b/src/tools/search-codebase.ts @@ -1,18 +1,18 @@ +import { hasCommittedIndex } from '../core/index-meta.js'; +import { withToolIndexSnapshot } from './index-snapshot.js'; import type { Tool } from '@modelcontextprotocol/sdk/types.js'; import { promises as fs } from 'fs'; import path from 'path'; import type { ToolContext, ToolResponse, DecisionCard } from './types.js'; import { CodebaseSearcher } from '../core/search.js'; -import type { SearchIntentProfile } from '../core/search.js'; +import type { SearchIntentProfile, SearchSourceCoverage } from '../core/search.js'; import type { SearchResult, IntelligenceData, PatternsData, - IntelligenceGoldenFile, ChunkMetadata } from '../types/index.js'; import { buildEvidenceLock } from '../preflight/evidence-lock.js'; -import type { EvidenceLock } from '../preflight/evidence-lock.js'; import { shouldIncludePatternConflictCategory } from '../preflight/query-scope.js'; import { isComplementaryPatternConflict, @@ -20,8 +20,9 @@ import { } from '../patterns/semantics.js'; import { assessSearchQuality } from '../core/search-quality.js'; import { getRerankerStatus } from '../core/reranker.js'; +import type { IndexSignal } from '../core/index-signal.js'; import { IndexCorruptedError } from '../errors/index.js'; -import { formatMemoryScopeText, readMemoriesFile, withConfidence } from '../memory/store.js'; +import { readMemoriesFile, withConfidence } from '../memory/store.js'; import type { MemoryWithConfidence } from '../memory/store.js'; import { indexHealthByFile, normalizeHealthLookupKey, readHealthFile } from '../health/store.js'; import { InternalFileGraph } from '../utils/usage-tracker.js'; @@ -104,7 +105,7 @@ export const definition: Tool = { includeSnippets: { type: 'boolean', description: - 'Include code snippets in results (default: false). If you need code, prefer read_file instead.', + 'Include code snippets in results (default: false). Open returned paths with your host file reader; CBC does not expose a file-reading tool.', default: false }, filters: { @@ -143,6 +144,13 @@ export const definition: Tool = { export async function handle( args: Record, ctx: ToolContext +): Promise { + return withToolIndexSnapshot(ctx, (pinned) => handlePinned(args, pinned)); +} + +async function handlePinned( + args: Record, + ctx: ToolContext ): Promise { const { query, limit, filters, intent, includeSnippets, mode } = args as { query?: unknown; @@ -175,7 +183,7 @@ export async function handle( }; } - if (ctx.indexState.status === 'indexing') { + if (ctx.indexState.status === 'indexing' && !(await hasCommittedIndex(ctx.rootPath))) { return { content: [ { @@ -183,7 +191,7 @@ export async function handle( text: JSON.stringify( { status: 'indexing', - message: 'Index is still being built. Retry in a moment.', + message: 'Index build in progress. Check indexing status, then retry.', progress: ctx.indexState.indexer?.getProgress() }, null, @@ -194,7 +202,7 @@ export async function handle( }; } - if (ctx.indexState.status === 'error') { + if (ctx.indexState.status === 'error' && !(await hasCommittedIndex(ctx.rootPath))) { return { content: [ { @@ -214,37 +222,121 @@ export async function handle( const searcher = new CodebaseSearcher(ctx.rootPath); let results: SearchResult[]; + let sourceCoverage: SearchSourceCoverage; const searchProfile = ( intent && ['explore', 'edit', 'refactor', 'migrate'].includes(intent) ? intent : 'explore' ) as SearchIntentProfile; try { - results = await searcher.search(queryStr, limit || 5, filters, { + const outcome = await searcher.searchWithCoverage(queryStr, limit || 5, filters, { profile: searchProfile }); + results = outcome.results; + sourceCoverage = outcome.sourceCoverage; } catch (error) { if (error instanceof IndexCorruptedError) { - console.error('[Auto-Heal] Index corrupted. Triggering background re-index...'); - void ctx.performIndexing(); + console.error('[Auto-Heal] Index corrupted. Requesting background re-index...'); + let finished = false; + if (ctx.indexState.status !== 'indexing') { + const fail = (failure: unknown) => { + ctx.indexState.status = 'error'; + ctx.indexState.error = failure instanceof Error ? failure.message : String(failure); + }; + try { + const task = ctx.performIndexing(); + if (task) + void task.then(() => { + finished = true; + }, fail); + } catch (failure) { + fail(failure); + } + // Observe immediate rejection/state transitions without waiting for a rebuild. + await Promise.resolve(); + } + const state = ctx.indexState.status; + const index: IndexSignal = + state === 'error' + ? { + status: 'error', + confidence: 'low', + action: 'rebuild-failed', + reason: ctx.indexState.error ?? error.message + } + : state === 'indexing' + ? { + status: 'indexing', + confidence: 'low', + action: 'rebuild-started', + reason: error.message + } + : finished && state === 'ready' + ? { + status: 'ready', + confidence: 'low', + action: 'rebuild-completed', + reason: 'Recovery completed; retry the search.' + } + : { + status: 'unknown', + confidence: 'low', + action: 'rebuild-requested', + reason: 'Recovery was requested but has not entered a confirmed indexing state.' + }; return { content: [ { type: 'text', text: JSON.stringify( { - status: 'indexing', - message: 'Index was corrupt. Rebuild started — retry shortly.' + status: index.status, + message: + index.status === 'error' + ? `Index recovery failed: ${index.reason}` + : index.status === 'indexing' + ? 'Index is unavailable or not ready. Indexing has started. Check indexing status, then retry.' + : index.reason, + index }, null, 2 ) } - ] + ], + ...(index.status === 'error' ? { isError: true } : {}) }; } throw error; } + if (sourceCoverage.status === 'no_fresh_results') { + return { + content: [ + { + type: 'text', + text: finalizeSearchPayloadText( + { + status: 'no_fresh_results', + message: sourceCoverage.candidateRetrievalFailed + ? 'No fresh results in the inspected candidates. Semantic refill failed; retry or refresh the index.' + : sourceCoverage.candidateBudgetExhausted + ? 'No fresh results in the bounded candidate window. Some candidates remain uninspected; refresh the index or narrow the query and retry.' + : 'No fresh results: nominated sources changed or could not be read. Refresh the index and retry.', + sourceCoverage, + searchQuality: { status: 'low_confidence', confidence: 0 }, + budget: { mode: mode === 'full' ? 'full' : 'compact', resultCount: 0 }, + ...(searchProfile !== 'explore' && { + preflight: { ready: false, reason: sourceCoverage.refreshAction } + }), + results: [] + }, + { mode: mode === 'full' ? 'full' : 'compact', pretty: true, transportAware: true } + ) + } + ] + }; + } + // Load memories for keyword matching, enriched with confidence const allMemories = await readMemoriesFile(ctx.paths.memory); const allMemoriesWithConf = withConfidence(allMemories); @@ -336,6 +428,38 @@ export async function handle( .filter((value): value is string => value !== null) ); + function canonicalRootRelativePath(filePath: string): string | undefined { + if (!filePath || path.isAbsolute(filePath)) return undefined; + const normalized = filePath.replace(/\\/g, '/'); + const parts = normalized.split('/'); + if ( + /^[a-z]:/i.test(parts[0]) || + parts.some((part) => part === '' || part === '.' || part === '..') + ) { + return undefined; + } + return normalized; + } + + function hasGroundedCanonicalExample(candidate: { + canonicalExample?: { file: string; snippet?: string }; + }): boolean { + const example = candidate.canonicalExample; + const examplePath = example && canonicalRootRelativePath(example.file); + const snippet = example?.snippet; + if (!examplePath || !snippet?.trim()) return false; + + const matches = results.filter((result) => { + const resultPath = canonicalRootRelativePath(normalizeGraphPath(result.filePath)); + return resultPath === examplePath; + }); + return matches.length === 1 && matches[0].snippet.includes(snippet); + } + + function isPatternInQueryScope(category: string): boolean { + return shouldIncludePatternConflictCategory(category, queryStr); + } + function getMemoryScopeBoost(memory: MemoryWithConfidence): number { if (!memory.scope || memory.scope.kind === 'global') return 0; @@ -354,10 +478,13 @@ export async function handle( return 0; } + function getMemoryText(memory: MemoryWithConfidence): string { + return `${memory.memory} ${memory.reason}`.toLowerCase(); + } + function getMemoryTextMatchCount(memory: MemoryWithConfidence): number { - const haystack = - `${memory.memory} ${memory.reason} ${formatMemoryScopeText(memory.scope)}`.toLowerCase(); - return queryTerms.filter((term) => haystack.includes(term)).length; + const text = getMemoryText(memory); + return queryTerms.filter((term) => text.includes(term)).length; } function formatMemoryForOutput(memory: MemoryWithConfidence): string { @@ -628,7 +755,7 @@ export async function handle( if (terms.length === 0) return []; return memories .filter((m) => { - const text = `${m.memory} ${m.reason} ${formatMemoryScopeText(m.scope)}`.toLowerCase(); + const text = getMemoryText(m); const matchCount = terms.filter((t) => text.includes(t)).length; return (matchCount >= 2 || getMemoryScopeBoost(m) >= 2) && m.effectiveConfidence >= 0.5; }) @@ -682,55 +809,35 @@ export async function handle( function buildPatternSummary(): string | undefined { const patterns = intelligence?.patterns; if (!patterns) return undefined; - const entries = Object.values(patterns) - .filter((data) => Boolean(data.primary)) + const entries = Object.entries(patterns) + .filter(([category, data]) => + Boolean( + data.primary && + isPatternInQueryScope(category) && + hasGroundedCanonicalExample(data.primary) + ) + ) .slice(0, 3) - .map((data) => { + .map(([, data]) => { const p = data.primary!; return `${p.name} (${p.frequency}, ${p.trend ?? 'Stable'})`; }); return entries.length > 0 ? entries.join(', ') : undefined; } - // Return the golden file path most relevant to the given search results, using directory - // overlap as a query-aware proximity signal. Falls back to goldenFiles[0] when no overlap. - // Track B: use cross-encoder reranker to score golden files against the query for stronger - // relevance — deferred to avoid reranker latency on compact mode in v2. + // A repository-wide example is not evidence for this query. Recommend it only + // when search independently returned the same file, preserving result order. function getBestExample(topResults: Array<{ filePath: string }>): string | undefined { const goldenFiles = intelligence?.goldenFiles; - if (!goldenFiles?.length) return undefined; - if (!topResults?.length) return goldenFiles[0].file; - - // Extract directory paths from top-5 results - const resultDirs = topResults.slice(0, 5).map((r) => path.dirname(r.filePath)); - - let bestFile = goldenFiles[0]; - let bestScore = -1; - - for (const gf of goldenFiles) { - const gfParts = path.dirname(gf.file).split(path.sep).filter(Boolean); - let maxShared = 0; - - for (const resDir of resultDirs) { - const resParts = resDir.split(path.sep).filter(Boolean); - let shared = 0; - const minLen = Math.min(gfParts.length, resParts.length); - for (let i = 0; i < minLen; i++) { - if (gfParts[i] === resParts[i]) shared++; - else break; - } - if (shared > maxShared) maxShared = shared; - } - - // Use golden file score as tie-breaker - const score = maxShared * 1000 + (gf.score ?? 0); - if (score > bestScore) { - bestScore = score; - bestFile = gf; - } + if (!goldenFiles?.length || !topResults.length) return undefined; + const examples = new Map( + goldenFiles.map((file) => [path.resolve(ctx.rootPath, file.file), file.file]) + ); + for (const result of topResults.slice(0, 5)) { + const example = examples.get(path.resolve(ctx.rootPath, result.filePath)); + if (example) return example; } - - return bestFile.file; + return undefined; } // Build up to 2 dynamic next-hop suggestions (compact mode) @@ -739,74 +846,30 @@ export async function handle( sq: { status: string } ): Array<{ tool: string; args?: Record; why: string }> { const hops: Array<{ tool: string; args?: Record; why: string }> = []; + // Only advertise tools this MCP server actually supplies. Source reads belong + // to the host; returned result paths already identify what to open there. if (compactResults.length > 0) { - hops.push({ - tool: 'read_file', - args: { path: compactResults[0].filePath }, - why: 'Read the top match for implementation details' - }); - } - if (sq.status === 'low_confidence') { - hops.push({ tool: 'get_team_patterns', why: 'Review pattern guidance — confidence is low' }); - } else if (hops.length < 2) { - hops.push({ tool: 'get_team_patterns', why: 'Review team patterns and best examples' }); + const relativeFile = path.relative(ctx.rootPath, compactResults[0].filePath); + if ( + !path.isAbsolute(relativeFile) && + relativeFile !== '..' && + !relativeFile.startsWith(`..${path.sep}`) + ) { + hops.push({ + tool: 'get_code_relationships', + args: { file: relativeFile.replace(/\\/g, '/'), direction: 'outgoing' }, + why: + sq.status === 'low_confidence' + ? 'Follow current import evidence while continuing source discovery' + : 'Follow verified imports to related implementation files' + }); + } } return hops.slice(0, 2); } const searchQuality = assessSearchQuality(queryStr, results); - // Always-on edit preflight (lite): do not require intent and keep payload small. - let editPreflight: - | { mode: string; riskLevel: string; confidence: string; evidenceLock: EvidenceLock } - | undefined = undefined; - if (intelligence && (!intent || intent === 'explore')) { - try { - const resultPaths = results.map((r) => r.filePath); - const impactCandidates = computeImpactCandidates(resultPaths); - - // Use existing pattern intelligence for evidenceLock scoring, but keep the output payload lite. - const preferredPatternsForEvidence: Array<{ pattern: string; example?: string }> = []; - const patterns: PatternsData = intelligence.patterns || {}; - for (const [_, data] of Object.entries(patterns)) { - if (data.primary) { - const p = data.primary; - if (p.trend === 'Rising' || p.trend === 'Stable') { - preferredPatternsForEvidence.push({ - pattern: p.name, - ...(p.canonicalExample && { example: p.canonicalExample.file }) - }); - } - } - } - - let riskLevel: 'low' | 'medium' | 'high' = 'low'; - if (impactCandidates.length > 10) { - riskLevel = 'high'; - } else if (impactCandidates.length > 3) { - riskLevel = 'medium'; - } - - editPreflight = { - mode: 'lite', - riskLevel, - confidence: computeIndexConfidence(), - evidenceLock: buildEvidenceLock({ - results, - preferredPatterns: preferredPatternsForEvidence.slice(0, 5), - relatedMemories, - failureWarnings: [], - patternConflicts: [], - searchQualityStatus: searchQuality.status - // Note: indexFreshness intentionally omitted — explore intent is for navigation, - // not mutation; freshness gating does not apply. - }) - }; - } catch { - // editPreflight is best-effort - never fail search over it - } - } - // Compose preflight card for edit/refactor/migrate intents let preflight: DecisionCard | undefined = undefined; const preflightIntents = ['edit', 'refactor', 'migrate']; @@ -839,7 +902,11 @@ export async function handle( // Primary pattern = preferred if Rising or Stable if (data.primary) { const p = data.primary; - if (p.trend === 'Rising' || p.trend === 'Stable') { + if ( + (p.trend === 'Rising' || p.trend === 'Stable') && + isPatternInQueryScope(category) && + hasGroundedCanonicalExample(p) + ) { preferredPatternsList.push({ pattern: p.name, category, @@ -853,7 +920,11 @@ export async function handle( // Also-detected patterns that are Declining = avoid if (data.alsoDetected) { for (const alt of data.alsoDetected) { - if (alt.trend === 'Declining') { + if ( + alt.trend === 'Declining' && + isPatternInQueryScope(category) && + hasGroundedCanonicalExample(alt) + ) { avoidPatternsList.push({ pattern: alt.name, category, @@ -870,7 +941,7 @@ export async function handle( const resultPaths = results.map((r) => r.filePath); const impactCandidates = computeImpactCandidates(resultPaths); - // PREF-02: Compute impact coverage (callers of result files that appear in results) + // Compute impact coverage (callers of result files that appear in results) const callerFiles = resultPaths.flatMap((p) => { const importers: string[] = []; for (const [dep, importerList] of reverseImports) { @@ -928,14 +999,6 @@ export async function handle( riskLevel = 'medium'; } - // --- Golden files (exemplar code) --- - const goldenFiles = (intelligence.goldenFiles ?? []) - .slice(0, 3) - .map((g: IntelligenceGoldenFile) => ({ - file: g.file, - score: g.score - })); - // --- Confidence (index freshness) --- // TODO: Review this confidence calculation //const confidence = computeIndexConfidence(); @@ -964,9 +1027,11 @@ export async function handle( if (shouldSkipLegacyTestingFrameworkCategory(cat, patterns)) continue; if (!shouldIncludePatternConflictCategory(cat, queryStr)) continue; if (!data.primary || !data.alsoDetected?.length) continue; + if (!hasGroundedCanonicalExample(data.primary)) continue; const primaryFreq = parseFloat(data.primary.frequency) || 100; if (primaryFreq >= 80) continue; for (const alt of data.alsoDetected) { + if (!hasGroundedCanonicalExample(alt)) continue; const altFreq = parseFloat(alt.frequency) || 0; if (altFreq >= 20) { if (isComplementaryPatternConflict(cat, data.primary.name, alt.name)) continue; @@ -991,7 +1056,7 @@ export async function handle( indexFreshness: computeIndexConfidence() }); - // Build clean decision card (PREF-01 to PREF-04) + // Build clean decision card const decisionCard: DecisionCard = { ready: evidenceLock.readyToEdit }; @@ -1014,13 +1079,15 @@ export async function handle( decisionCard.warnings = [...(decisionCard.warnings ?? []), ...freshnessGaps]; } - // Add patterns (do/avoid, capped at 3 each, with adoption %) - const doPatterns = preferredPatternsForOutput - .slice(0, 3) - .map((p) => `${p.pattern} — ${p.adoption ? `${p.adoption} adoption` : ''}`); - const avoidPatterns = avoidPatternsForOutput - .slice(0, 3) - .map((p) => `${p.pattern} — ${p.adoption ? `${p.adoption} adoption` : ''} (declining)`); + // Pattern advice is actionable only when search itself did not abstain. + const suppressPatternAdvice = + evidenceLock.status === 'block' || searchQuality.status === 'low_confidence'; + const doPatterns = suppressPatternAdvice + ? [] + : preferredPatternsForOutput.slice(0, 3).map((p) => p.pattern); + const avoidPatterns = suppressPatternAdvice + ? [] + : avoidPatternsForOutput.slice(0, 3).map((p) => `${p.pattern} (declining)`); if (doPatterns.length > 0 || avoidPatterns.length > 0) { decisionCard.patterns = { ...(doPatterns.length > 0 && { do: doPatterns }), @@ -1091,22 +1158,23 @@ export async function handle( } } - // For edit/refactor/migrate: return clean decision card. - // For explore or lite-only: return lightweight { ready, reason }. + // Discovery returns retrieval quality and source coverage; mutation intents get edit readiness. let preflightPayload: { ready: boolean; reason?: string } | Record | undefined; if (preflight) { // preflight is already a clean decision card (DecisionCard type) preflightPayload = preflight; - } else if (editPreflight) { - // Lite preflight for explore intent - const el = editPreflight.evidenceLock; + } + if (sourceCoverage.status === 'partial' && preflightIntents.includes(searchProfile)) { preflightPayload = { - ready: el?.readyToEdit ?? false, - ...(el && !el.readyToEdit && el.nextAction && { reason: el.nextAction }) + ...preflightPayload, + ready: false, + abstain: true, + reason: sourceCoverage.refreshAction, + nextAction: sourceCoverage.refreshAction }; } - // Helper: Build scope header for symbol-aware chunks (SEARCH-02) + // Helper: Build scope header for symbol-aware chunks function buildScopeHeader(metadata: ChunkMetadata): string | null { // Try symbolPath first (most reliable for AST-based symbols) if (metadata?.symbolPath && Array.isArray(metadata.symbolPath)) { @@ -1168,25 +1236,35 @@ export async function handle( ...(rerankerStatus === 'unavailable' && { rerankerStatus: 'unavailable' }) }; - type SearchResponsePayload = { - status: 'success'; - searchQuality: typeof searchQualityBlock & { - tokenEstimate?: number; - warning?: string; - }; - budget: { mode: 'compact' | 'full'; resultCount: number }; - preflight?: typeof preflightPayload; - patternSummary?: string; - bestExample?: string; - nextHops?: Array<{ tool: string; why: string; args?: Record }>; - results: Array>; - totalResults?: number; - relatedMemories?: string[]; - }; - // Compact mode (default): bounded response with light graph context const isCompact = mode !== 'full'; + function buildSignaturePreview(snippet: string | undefined): string | undefined { + if (!snippet) return undefined; + + const lines = snippet.replace(/^\r?\n+/, '').split('\n'); + const previewTerms = [...new Set(queryTerms.filter((term) => term.length > 3))]; + const strongestMatch = lines + .map((line, index) => ({ + index, + matches: previewTerms.filter((term) => line.toLowerCase().includes(term)).length + })) + .filter((entry) => entry.matches > 0) + .sort((a, b) => b.matches - a.matches || a.index - b.index)[0]; + + let previewLines = lines.slice(0, 3); + if (strongestMatch) { + const windowLength = Math.min(3, lines.length); + const windowStart = Math.max( + 0, + Math.min(strongestMatch.index - 1, lines.length - windowLength) + ); + previewLines = lines.slice(windowStart, windowStart + windowLength); + } + + return previewLines.join('\n').trim() || undefined; + } + if (isCompact) { const compactResults = results.slice(0, 6); const strongMemories = filterStrongMemories(relatedMemories, queryStr); @@ -1196,6 +1274,7 @@ export async function handle( const payloadText = finalizeSearchPayloadText( { status: 'success', + ...(sourceCoverage.status === 'partial' && { sourceCoverage }), searchQuality: searchQualityBlock, budget: { mode: 'compact', resultCount: compactResults.length }, ...(preflightPayload && { preflight: preflightPayload }), @@ -1207,15 +1286,7 @@ export async function handle( const topExports = getTopExports(r.filePath); const scope = buildScopeHeader(r.metadata); const health = getResultHealth(r.filePath); - // First 3 lines of chunk content as a lightweight signature preview - const signaturePreview = r.snippet - ? r.snippet - .replace(/^\r?\n+/, '') - .split('\n') - .slice(0, 3) - .join('\n') - .trim() || undefined - : undefined; + const signaturePreview = buildSignaturePreview(r.snippet); return { file: `${r.filePath}:${r.startLine}-${r.endLine}`, summary: r.summary, @@ -1258,6 +1329,7 @@ export async function handle( const payloadText = finalizeSearchPayloadText( { status: 'success', + ...(sourceCoverage.status === 'partial' && { sourceCoverage }), searchQuality: searchQualityBlock, budget: { mode: 'full', resultCount: results.length }, ...(preflightPayload && { preflight: preflightPayload }), diff --git a/src/tools/types.ts b/src/tools/types.ts index 5de5ee8..43cabb8 100644 --- a/src/tools/types.ts +++ b/src/tools/types.ts @@ -1,5 +1,6 @@ import type { CodebaseIndexer } from '../core/indexer.js'; import type { IndexingStats } from '../types/index.js'; +import type { SearchSourceCoverage } from '../core/search.js'; export interface DecisionCard { ready: boolean; @@ -56,7 +57,11 @@ export interface ToolContext { paths: ToolPaths; rootPath: string; project?: ProjectDescriptor; - performIndexing: (incrementalOnly?: boolean, reason?: string) => void; + performIndexing: ( + incrementalOnly?: boolean, + reason?: string, + skipEmbedding?: boolean + ) => void | Promise; listProjects?: () => ProjectDescriptor[]; getActiveProject?: () => ProjectDescriptor | undefined; } @@ -132,6 +137,7 @@ export interface SearchResultItem { export interface SearchResponse { status: string; + sourceCoverage?: SearchSourceCoverage; searchQuality: SearchQuality; budget?: SearchBudget; preflight?: DecisionCard | SearchLitePreflight; diff --git a/src/types/index.ts b/src/types/index.ts index 9010843..619b5e9 100644 --- a/src/types/index.ts +++ b/src/types/index.ts @@ -21,7 +21,7 @@ export interface FrameworkAnalyzer { canAnalyze(filePath: string, content?: string): boolean; /** Parse a file and extract structured information */ - analyze(filePath: string, content: string): Promise; + analyze(filePath: string, content: string, options?: AnalysisOptions): Promise; /** Detect framework-specific patterns and metadata from the entire codebase */ detectCodebaseMetadata(rootPath: string): Promise; @@ -39,6 +39,10 @@ export interface FrameworkAnalyzer { getSnippetPattern?(category: string, name: string): RegExp | null; } +export interface AnalysisOptions { + includeScopePrefix?: boolean; +} + // ============================================================================ // ANALYSIS RESULTS // ============================================================================ diff --git a/src/utils/ast-chunker.ts b/src/utils/ast-chunker.ts index 9e7e8be..0d245f1 100644 --- a/src/utils/ast-chunker.ts +++ b/src/utils/ast-chunker.ts @@ -56,8 +56,10 @@ export const MAX_AST_CHUNK_FILE_LINES = 10_000; // 10K lines */ export function generateScopePrefix(node: SymbolNode, ancestors: SymbolNode[]): string { // Build scope path: ancestor names joined with " > ", then current node - const pathParts = ancestors.map((a) => a.symbol.name); - pathParts.push(node.symbol.name); + const pathParts = node.symbol.symbolPath ?? [ + ...ancestors.map((a) => a.symbol.name), + node.symbol.name + ]; const scopePath = pathParts.join(' > '); // Build signature hint from the symbol content @@ -69,7 +71,7 @@ export function generateScopePrefix(node: SymbolNode, ancestors: SymbolNode[]): /** * Extract a short signature hint from a symbol's content. */ -function extractSignatureHint(sym: TreeSitterSymbol): string { +function extractSignatureHint(sym: Pick): string { const kind = sym.kind; if (kind === 'class') return 'class'; @@ -132,13 +134,18 @@ export function buildSymbolTree(symbols: TreeSitterSymbol[]): SymbolNode[] { const cs = candidate.symbol; const ns = node.symbol; - // candidate must fully contain node (and not be identical range unless different symbol) - if ( - cs.startLine <= ns.startLine && - cs.endLine >= ns.endLine && - !(cs.startLine === ns.startLine && cs.endLine === ns.endLine) - ) { - const span = cs.endLine - cs.startLine; + // Real parser offsets disambiguate nested and sibling symbols on the same line. + // Synthetic callers without offsets retain the public line-range contract. + const hasOffsets = cs.endIndex > cs.startIndex && ns.endIndex > ns.startIndex; + const contains = hasOffsets + ? cs.startIndex <= ns.startIndex && + cs.endIndex >= ns.endIndex && + (cs.startIndex !== ns.startIndex || cs.endIndex !== ns.endIndex) + : cs.startLine <= ns.startLine && + cs.endLine >= ns.endLine && + !(cs.startLine === ns.startLine && cs.endLine === ns.endLine); + if (contains) { + const span = hasOffsets ? cs.endIndex - cs.startIndex : cs.endLine - cs.startLine; if (span < bestSpan) { bestSpan = span; bestParent = candidate; @@ -172,146 +179,111 @@ export function generateASTChunks( options: ASTChunkOptions ): CodeChunk[] { const lines = content.split('\n'); - const totalLines = lines.length; + if (symbols.length === 0) return [makeFillerChunk(lines, 1, lines.length, options)]; - if (symbols.length === 0) { - // No symbols: entire file is one filler chunk - return [makeFillerChunk(lines, 1, totalLines, options)]; + const lineStarts = [0]; + for (let i = 0; i < content.length; i++) { + if (content[i] === '\n') lineStarts.push(i + 1); } - - const roots = buildSymbolTree(symbols); - - // Sort roots by startLine for ordered processing - roots.sort((a, b) => a.symbol.startLine - b.symbol.startLine); - - const chunks: CodeChunk[] = []; - let cursor = 1; // 1-based line cursor - - const shouldPrefix = options.includeScopePrefix !== false; - - for (const root of roots) { - // Gap before this root - if (root.symbol.startLine > cursor) { - chunks.push(makeFillerChunk(lines, cursor, root.symbol.startLine - 1, options)); - } - // Process the root symbol (recurse for containers) - chunks.push(...processNode(root, lines, options, null, [], shouldPrefix)); - cursor = root.symbol.endLine + 1; - } - - // Suffix after last root - if (cursor <= totalLines) { - chunks.push(makeFillerChunk(lines, cursor, totalLines, options)); - } - - return chunks; -} - -/** Recursively process a SymbolNode into chunks. */ -function processNode( - node: SymbolNode, - lines: string[], - options: ASTChunkOptions, - parentName: string | null, - ancestors: SymbolNode[], - shouldPrefix: boolean -): CodeChunk[] { - const sym = node.symbol; - const symbolPath = parentName ? [parentName, sym.name] : [sym.name]; - - if (node.children.length === 0) { - // Leaf symbol → single chunk - const chunk = makeSymbolChunk(sym, lines, options, symbolPath, parentName); - if (shouldPrefix) { - const prefix = generateScopePrefix(node, ancestors); - chunk.content = prefix + '\n' + chunk.content; - } - if (chunk.metadata) { - (chunk.metadata as Record).symbolPath = symbolPath; + const lineAt = (offset: number): number => { + let low = 0; + let high = lineStarts.length; + while (low + 1 < high) { + const middle = Math.floor((low + high) / 2); + if (lineStarts[middle] <= offset) low = middle; + else high = middle; } - return [chunk]; - } - - // Container symbol — split into header, children, footer - const chunks: CodeChunk[] = []; - const sortedChildren = [...node.children].sort((a, b) => a.symbol.startLine - b.symbol.startLine); - - // Header: from container start to first child start - 1 - const headerEnd = sortedChildren[0].symbol.startLine - 1; - if (headerEnd >= sym.startLine) { - const headerLines = extractLines(lines, sym.startLine, headerEnd); - const nonBlank = headerLines.filter((l) => l.trim().length > 0).length; - if (nonBlank > 2) { - const headerChunk = makeSymbolChunk( - { - ...sym, - name: `${sym.name}:header`, - startLine: sym.startLine, - endLine: headerEnd, - content: headerLines.join('\n') - }, - lines, - options, - symbolPath, - parentName, - true // use provided content - ); - if (shouldPrefix) { - const prefix = generateScopePrefix(node, ancestors); - headerChunk.content = `${prefix}\n${headerChunk.content}`; + return low + 1; + }; + const ranges = new Map(); + for (const symbol of symbols) { + let start = lineStarts[symbol.startLine - 1] ?? content.length; + let end = symbol.endLine < lineStarts.length ? lineStarts[symbol.endLine] - 1 : content.length; + if (symbol.endIndex > symbol.startIndex) { + start = symbol.startIndex; + end = symbol.endIndex; + // web-tree-sitter releases may expose byte offsets or UTF-16 offsets. + // Match the already extracted source rather than guessing from non-ASCII text. + if (content.slice(start, end) !== symbol.content) { + const bytes = Buffer.from(content, 'utf8'); + start = bytes.subarray(0, symbol.startIndex).toString('utf8').length; + end = bytes.subarray(0, symbol.endIndex).toString('utf8').length; } - chunks.push(headerChunk); } + ranges.set(symbol, { start, end }); } - // Children - let childCursor = sortedChildren[0].symbol.startLine; - for (const child of sortedChildren) { - // Gap between children within container - if (child.symbol.startLine > childCursor) { - const gapStart = childCursor; - const gapEnd = child.symbol.startLine - 1; - const gapLines = extractLines(lines, gapStart, gapEnd); - const nonBlank = gapLines.filter((l) => l.trim().length > 0).length; - if (nonBlank > 0) { - chunks.push(makeFillerChunk(lines, gapStart, gapEnd, options)); - } + const chunks: CodeChunk[] = []; + const appendRange = ( + start: number, + end: number, + owner?: SymbolNode, + ancestors: SymbolNode[] = [], + suffix = '' + ): void => { + const originalStart = start; + const fragment = content.slice(start, end); + const first = fragment.search(/\S/); + if (first < 0) return; + const last = fragment.search(/\s*$/); + // Drop only whitespace crossing a line boundary. Same-line separators remain + // exact source; indentation on an adjacent symbol's line is not a body span. + if (fragment.slice(0, first).includes('\n')) start = lineStarts[lineAt(start + first) - 1]; + if (fragment.slice(last).includes('\n')) end = originalStart + last; + const startLine = lineAt(start); + const endLine = lineAt(Math.max(start, end - 1)); + const source = content.slice(start, end); + if (!owner) { + chunks.push({ ...makeFillerChunk(lines, startLine, endLine, options), content: source }); + return; } - chunks.push( - ...processNode(child, lines, options, sym.name, [...ancestors, node], shouldPrefix) + const symbol = owner.symbol; + const symbolPath = symbol.symbolPath ?? [ + ...ancestors.map((ancestor) => ancestor.symbol.name), + symbol.name + ]; + const chunk = makeSymbolChunk( + { ...symbol, name: symbol.name + suffix, startLine, endLine, content: source }, + lines, + options, + symbolPath, + symbolPath.at(-2) ?? null, + true ); - childCursor = child.symbol.endLine + 1; - } - - // Footer: from last child end + 1 to container end - const lastChildEnd = sortedChildren[sortedChildren.length - 1].symbol.endLine; - if (lastChildEnd < sym.endLine) { - const footerStart = lastChildEnd + 1; - const footerLines = extractLines(lines, footerStart, sym.endLine); - const nonBlank = footerLines.filter((l) => l.trim().length > 0).length; - if (nonBlank > 2) { - const footerChunk = makeSymbolChunk( - { - ...sym, - name: `${sym.name}:footer`, - startLine: footerStart, - endLine: sym.endLine, - content: footerLines.join('\n') - }, - lines, - options, - symbolPath, - parentName, - true - ); - if (shouldPrefix) { - const prefix = generateScopePrefix(node, ancestors); - footerChunk.content = `${prefix}\n${footerChunk.content}`; - } - chunks.push(footerChunk); + if (options.includeScopePrefix !== false) + chunk.content = generateScopePrefix(owner, ancestors) + '\n' + chunk.content; + chunks.push(chunk); + }; + const processNode = (node: SymbolNode, ancestors: SymbolNode[]): void => { + const range = ranges.get(node.symbol)!; + const children = [...node.children].sort( + (a, b) => ranges.get(a.symbol)!.start - ranges.get(b.symbol)!.start + ); + if (children.length === 0) { + appendRange(range.start, range.end, node, ancestors); + return; + } + let cursor = range.start; + for (let i = 0; i < children.length; i++) { + const child = children[i]; + const childRange = ranges.get(child.symbol)!; + appendRange(cursor, childRange.start, node, ancestors, i === 0 ? ':header' : ':body'); + processNode(child, [...ancestors, node]); + cursor = childRange.end; } + appendRange(cursor, range.end, node, ancestors, ':footer'); + }; + const roots = buildSymbolTree(symbols).sort( + (a, b) => ranges.get(a.symbol)!.start - ranges.get(b.symbol)!.start + ); + let cursor = 0; + for (const root of roots) { + const range = ranges.get(root.symbol)!; + appendRange(cursor, range.start); + processNode(root, []); + cursor = range.end; } - + appendRange(cursor, content.length); return chunks; } @@ -490,16 +462,43 @@ function findSafeSplitPoint(lines: string[], targetIdx: number): number { /** * Main public API: generate AST-aligned chunks from content + symbols, - * then merge tiny chunks and split oversized ones. + * then split oversized source spans and optionally add presentation context. */ export function createASTAlignedChunks( content: string, symbols: TreeSitterSymbol[], options: ASTChunkOptions ): CodeChunk[] { - const raw = generateASTChunks(content, symbols, options); - const merged = mergeSmallSymbolChunks(raw, options.minChunkLines); - const final = splitOversizedChunks(merged, options.maxChunkLines); + // Partition source before adding presentation context. Persisted callers opt out + // of the final prefix; they receive exactly the same source ranges and bytes. + const raw = generateASTChunks(content, symbols, { ...options, includeScopePrefix: false }); + // Even identical owner labels can describe distinct declarations. The public + // merge helper is retained for callers, but persisted source spans stay separate. + const final = splitOversizedChunks(raw, options.maxChunkLines); + if (options.includeScopePrefix !== false) { + const prefixes = new Map(); + for (const chunk of raw) { + if (!chunk.metadata.symbolAware) continue; + const path = chunk.metadata.symbolPath ?? []; + const key = JSON.stringify(path); + if (!prefixes.has(key)) { + const symbol = symbols.find( + (candidate) => + candidate.name === path.at(-1) && + candidate.startLine <= chunk.startLine && + candidate.endLine >= chunk.endLine + ); + prefixes.set( + key, + `// ${path.join(' > ')} :: ${extractSignatureHint(symbol ?? { kind: chunk.metadata.symbolKind ?? '', content: chunk.content })}` + ); + } + } + for (const chunk of final) { + const prefix = prefixes.get(JSON.stringify(chunk.metadata.symbolPath ?? [])); + if (chunk.metadata.symbolAware && prefix) chunk.content = `${prefix}\n${chunk.content}`; + } + } return final; } @@ -547,9 +546,10 @@ function makeSymbolChunk( parentName: string | null, useProvidedContent = false ): CodeChunk { - const chunkContent = useProvidedContent - ? sym.content - : extractLines(lines, sym.startLine, sym.endLine).join('\n'); + const chunkContent = + useProvidedContent || sym.endIndex > sym.startIndex + ? sym.content + : extractLines(lines, sym.startLine, sym.endLine).join('\n'); return { id: uuidv4(), diff --git a/src/utils/atomic-write.ts b/src/utils/atomic-write.ts new file mode 100644 index 0000000..333b077 --- /dev/null +++ b/src/utils/atomic-write.ts @@ -0,0 +1,21 @@ +import { randomUUID } from 'node:crypto'; +import { promises as fs } from 'node:fs'; + +/** Publish complete text by a same-directory rename; failures leave the target intact. */ +export async function atomicWrite(target: string, content: string): Promise { + const temporary = `${target}.${randomUUID()}.tmp`; + const mode = await fs.stat(target).then( + (stat) => stat.mode, + (error: unknown) => { + if (typeof error === 'object' && error !== null && 'code' in error && error.code === 'ENOENT') + return 0o600; + throw error; + } + ); + try { + await fs.writeFile(temporary, content, { encoding: 'utf8', flag: 'wx', mode }); + await fs.rename(temporary, target); + } finally { + await fs.rm(temporary, { force: true }).catch(() => undefined); + } +} diff --git a/src/utils/chunking.ts b/src/utils/chunking.ts index 66a1bf0..3252d49 100644 --- a/src/utils/chunking.ts +++ b/src/utils/chunking.ts @@ -5,7 +5,11 @@ */ import { v4 as uuidv4 } from 'uuid'; -import { CodeChunk, CodeComponent } from '../types/index.js'; +import path from 'node:path'; +import { hasValidSourceLineSpan, isSourceBodyWithinInterval } from './source-interval.js'; +import { isDeepStrictEqual } from 'node:util'; +import { InvalidSourceChunkError } from '../errors/index.js'; +import type { CodeChunk, CodeComponent } from '../types/index.js'; export interface ChunkingOptions { maxChunkSize?: number; // Max lines per chunk @@ -260,32 +264,83 @@ function generateTags(component: CodeComponent, metadata?: Record): /** * Merge adjacent chunks if they're small */ +/** Validate producer evidence against the normalized source snapshot already analyzed. */ +export function validateSourceChunks( + chunks: CodeChunk[], + source: { filePath: string; relativePath: string; content: string } +): void { + const lines = source.content.split('\n'); + for (const chunk of chunks) { + if ( + typeof chunk.filePath !== 'string' || + path.relative(path.resolve(source.filePath), path.resolve(chunk.filePath)) !== '' || + chunk.relativePath !== source.relativePath + ) + throw new InvalidSourceChunkError( + `Analyzer chunk belongs to another source: ${source.relativePath}` + ); + if (!isSourceBodyWithinInterval(chunk, lines)) + throw new InvalidSourceChunkError( + `Analyzer chunk does not match its source interval: ${source.relativePath}:${chunk.startLine}-${chunk.endLine}` + ); + } +} + +function mergeMetadata(chunk: CodeChunk): Record { + const { complexity: _complexity, merged: _merged, ...metadata } = chunk.metadata; + return metadata; +} + export function mergeSmallChunks(chunks: CodeChunk[], minSize: number = 20): CodeChunk[] { if (chunks.length <= 1) return chunks; - const merged: CodeChunk[] = []; let current = chunks[0]; - + const unowned = (chunk: CodeChunk) => + ![ + 'componentName', + 'className', + 'functionName', + 'symbolName', + 'symbolPath', + 'parentSymbol' + ].some((key) => chunk.metadata[key] !== undefined); for (let i = 1; i < chunks.length; i++) { const next = chunks[i]; - const currentLines = current.content.split('\n').length; - const nextLines = next.content.split('\n').length; - const currentIsSymbolAware = current.metadata?.symbolAware === true; - const nextIsSymbolAware = next.metadata?.symbolAware === true; - if ( - !currentIsSymbolAware && - !nextIsSymbolAware && - currentLines < minSize && - nextLines < minSize + hasValidSourceLineSpan(current) && + hasValidSourceLineSpan(next) && + current.filePath === next.filePath && + current.relativePath === next.relativePath && + next.startLine === current.endLine + 1 && + current.language === next.language && + current.framework === next.framework && + current.componentType === next.componentType && + current.layer === next.layer && + current.embedding === undefined && + next.embedding === undefined && + !current.metadata.symbolAware && + !next.metadata.symbolAware && + unowned(current) && + unowned(next) && + isDeepStrictEqual(mergeMetadata(current), mergeMetadata(next)) && + current.content.split('\n').length < minSize && + next.content.split('\n').length < minSize ) { - // Merge chunks + const content = current.content + '\n' + next.content; current = { ...current, - content: current.content + '\n' + next.content, + content, endLine: next.endLine, + dependencies: [...new Set([...current.dependencies, ...next.dependencies])], + imports: [...new Set([...current.imports, ...next.imports])], + exports: [...new Set([...current.exports, ...next.exports])], + tags: [...new Set([...current.tags, ...next.tags])], metadata: { ...current.metadata, + ...(typeof current.metadata.complexity === 'number' || + typeof next.metadata.complexity === 'number' + ? { complexity: calculateComplexity(content) } + : {}), merged: true } }; @@ -294,7 +349,6 @@ export function mergeSmallChunks(chunks: CodeChunk[], minSize: number = 20): Cod current = next; } } - merged.push(current); return merged; } diff --git a/src/utils/internal-source-resolver.ts b/src/utils/internal-source-resolver.ts new file mode 100644 index 0000000..bb29f7d --- /dev/null +++ b/src/utils/internal-source-resolver.ts @@ -0,0 +1,198 @@ +import { promises as fs } from 'fs'; +import path from 'path'; +import ts from 'typescript'; +import { parse } from '@typescript-eslint/typescript-estree'; + +export interface TsconfigResolverOptions { + baseUrl?: string; + paths?: Record; + compilerOptions?: ts.CompilerOptions; +} + +export type SourceResolution = + | { status: 'resolved'; path: string; target: string } + | { status: 'unresolved'; reason: 'not-indexed'; target: string } + | { + status: 'unresolved'; + reason: 'external' | 'not-found' | 'outside-root' | 'unsupported-alias'; + }; + +function isWithin(root: string, candidate: string): boolean { + const relative = path.relative(root, candidate); + return ( + relative === '' || + (relative !== '..' && !relative.startsWith(`..${path.sep}`) && !path.isAbsolute(relative)) + ); +} + +export function loadTsconfigCompilerOptions(rootPath: string): ts.CompilerOptions { + // Keep discovery within this root and config paths normalized for TypeScript diagnostics. + const configPath = ['tsconfig.json', 'tsconfig.base.json'] + .map((name) => path.join(rootPath, name).split(path.sep).join('/')) + .find((candidate) => ts.sys.fileExists(candidate)); + if (!configPath) { + return { + module: ts.ModuleKind.Preserve, + moduleResolution: ts.ModuleResolutionKind.Bundler + }; + } + const readResult = ts.readConfigFile(configPath, ts.sys.readFile); + if (readResult.error) { + return { + module: ts.ModuleKind.Preserve, + moduleResolution: ts.ModuleResolutionKind.Bundler + }; + } + const parsed = ts.parseJsonConfigFileContent(readResult.config, ts.sys, path.dirname(configPath)); + return { + module: ts.ModuleKind.Preserve, + moduleResolution: ts.ModuleResolutionKind.Bundler, + ...parsed.options + }; +} + +function matchesPathAlias( + specifier: string, + paths: Record +): Array<{ target: string; suffix: string }> { + const matches: Array<{ target: string; suffix: string; specificity: number }> = []; + for (const [pattern, targets] of Object.entries(paths)) { + const star = pattern.indexOf('*'); + if (star < 0) { + if (specifier === pattern) + for (const target of targets) + matches.push({ target, suffix: '', specificity: pattern.length }); + continue; + } + const prefix = pattern.slice(0, star); + const suffix = pattern.slice(star + 1); + if ( + specifier.startsWith(prefix) && + specifier.endsWith(suffix) && + specifier.length >= prefix.length + suffix.length + ) { + const capture = specifier.slice(prefix.length, specifier.length - suffix.length); + for (const target of targets) + matches.push({ + target: target.replace('*', capture), + suffix: '', + specificity: prefix.length + suffix.length + }); + } + } + return matches + .sort((a, b) => b.specificity - a.specificity) + .map(({ target, suffix }) => ({ target, suffix })); +} + +export class InternalSourceResolver { + private readonly rootPath: string; + private readonly rootRealPath: string; + private readonly indexedFiles = new Map(); + private readonly options: TsconfigResolverOptions; + private readonly compilerOptions: ts.CompilerOptions; + + private constructor( + rootPath: string, + rootRealPath: string, + indexedFiles: Map, + options: TsconfigResolverOptions + ) { + this.rootPath = rootPath; + this.rootRealPath = rootRealPath; + this.indexedFiles = indexedFiles; + this.options = options; + this.compilerOptions = options.compilerOptions ?? { + module: ts.ModuleKind.Preserve, + moduleResolution: ts.ModuleResolutionKind.Bundler, + ...(options.baseUrl ? { baseUrl: path.resolve(rootPath, options.baseUrl) } : {}), + ...(options.paths ? { paths: options.paths } : {}) + }; + } + + static async create( + rootPath: string, + files: string[], + options: TsconfigResolverOptions = {} + ): Promise { + const rootRealPath = await fs.realpath(rootPath); + const indexedFiles = new Map(); + for (const file of files) { + const absolute = path.resolve(file); + const relative = path.relative(rootPath, absolute); + if (!isWithin(rootPath, absolute)) continue; + const rel = relative.split(path.sep).join('/'); + indexedFiles.set(process.platform === 'win32' ? rel.toLowerCase() : rel, absolute); + const real = await fs.realpath(absolute).catch(() => null); + if (real && isWithin(rootRealPath, real)) { + const realRelative = path.relative(rootRealPath, real).split(path.sep).join('/'); + indexedFiles.set( + process.platform === 'win32' ? realRelative.toLowerCase() : realRelative, + absolute + ); + } + } + return new InternalSourceResolver(path.resolve(rootPath), rootRealPath, indexedFiles, options); + } + + async resolve( + importer: string, + specifier: string, + allowUnindexed = false + ): Promise { + const importerPath = path.resolve(importer); + const importerRealPath = await fs.realpath(importerPath).catch(() => null); + if (!importerRealPath || !isWithin(this.rootRealPath, importerRealPath)) + return { status: 'unresolved', reason: 'outside-root' }; + const matchingAlias = matchesPathAlias(specifier, this.compilerOptions.paths ?? {}); + const bareSpecifier = !specifier.startsWith('.') && !path.isAbsolute(specifier); + if (bareSpecifier && matchingAlias.length === 0 && !this.compilerOptions.baseUrl) { + return { status: 'unresolved', reason: 'external' }; + } + const resolved = ts.resolveModuleName(specifier, importerPath, this.compilerOptions, ts.sys) + .resolvedModule?.resolvedFileName; + if (!resolved) + return { + status: 'unresolved', + reason: matchingAlias.length > 0 ? 'not-found' : bareSpecifier ? 'external' : 'not-found' + }; + const real = await fs.realpath(resolved).catch(() => null); + if (!real) return { status: 'unresolved', reason: 'not-found' }; + if (!isWithin(this.rootRealPath, real)) { + return { + status: 'unresolved', + reason: bareSpecifier && matchingAlias.length === 0 ? 'external' : 'outside-root' + }; + } + const relative = path.relative(this.rootRealPath, real).split(path.sep).join('/'); + const indexedPath = this.indexedFiles.get( + process.platform === 'win32' ? relative.toLowerCase() : relative + ); + if (indexedPath) return { status: 'resolved', path: indexedPath, target: relative }; + if (allowUnindexed) return { status: 'resolved', path: real, target: relative }; + return { status: 'unresolved', reason: 'not-indexed', target: relative }; + } +} + +export function extractSourceReexports( + source: string, + filePath: string +): Array<{ specifier: string; line: number }> { + if (!/\.[cm]?[jt]sx?$/i.test(filePath)) return []; + if (!/\bexport\s+(?:type\s+)?(?:\*|\{)/.test(source)) return []; + try { + const ast = parse(source, { loc: true, jsx: /\.[jt]sx$/i.test(filePath) }); + const reexports: Array<{ specifier: string; line: number }> = []; + for (const node of ast.body) { + if ( + (node.type === 'ExportNamedDeclaration' || node.type === 'ExportAllDeclaration') && + node.source + ) { + reexports.push({ specifier: node.source.value, line: node.loc.start.line }); + } + } + return reexports; + } catch { + return []; + } +} diff --git a/src/utils/optional-dependency.ts b/src/utils/optional-dependency.ts new file mode 100644 index 0000000..707c503 --- /dev/null +++ b/src/utils/optional-dependency.ts @@ -0,0 +1,17 @@ +import { createRequire } from 'node:module'; +import { OptionalDependencyError } from '../errors/index.js'; + +const resolve = createRequire(import.meta.url).resolve; + +/** Check an opt-in dependency without loading its native runtime or model. */ +export function assertOptionalDependency(packageName: string, versionRange: string): void { + try { + resolve(packageName); + } catch (error) { + const code = (error as NodeJS.ErrnoException).code; + if (code === 'MODULE_NOT_FOUND' || code === 'ERR_MODULE_NOT_FOUND') { + throw new OptionalDependencyError(packageName, versionRange); + } + throw error; + } +} diff --git a/src/utils/source-interval.ts b/src/utils/source-interval.ts new file mode 100644 index 0000000..548b582 --- /dev/null +++ b/src/utils/source-interval.ts @@ -0,0 +1,30 @@ +import type { CodeChunk } from '../types/index.js'; + +type SourceInterval = Pick; + +/** Raw persisted bodies span exactly their claimed normalized source lines. */ +export function hasValidSourceLineSpan(chunk: SourceInterval): boolean { + return ( + Number.isInteger(chunk.startLine) && + Number.isInteger(chunk.endLine) && + chunk.startLine >= 1 && + chunk.endLine >= chunk.startLine && + typeof chunk.content === 'string' && + chunk.content.split('\n').length === chunk.endLine - chunk.startLine + 1 + ); +} + +/** Partial AST bodies may occupy part of a line, but never invent surrounding text. */ +export function isSourceBodyWithinInterval( + chunk: SourceInterval, + lines: readonly string[] +): boolean { + return ( + hasValidSourceLineSpan(chunk) && + chunk.endLine <= lines.length && + lines + .slice(chunk.startLine - 1, chunk.endLine) + .join('\n') + .includes(chunk.content) + ); +} diff --git a/src/utils/tree-sitter.ts b/src/utils/tree-sitter.ts index f24d69c..a268c66 100644 --- a/src/utils/tree-sitter.ts +++ b/src/utils/tree-sitter.ts @@ -15,6 +15,7 @@ export interface TreeSitterSymbol { endIndex: number; content: string; nodeType: string; + symbolPath?: string[]; } export interface TreeSitterSymbolExtraction { @@ -42,6 +43,9 @@ const SYMBOL_CANDIDATE_NODE_TYPES = [ 'method', 'method_declaration', 'method_definition', + 'pair', + 'field_definition', + 'public_field_definition', 'struct_item', 'struct_specifier', 'trait_item', @@ -218,6 +222,10 @@ function maybeGetNameNode(node: Node): Node | null { } function extractNodeName(node: Node): string { + if (node.type === 'pair') return staticPropertyName(node.childForFieldName('key')) ?? 'anonymous'; + if (['method_definition', 'field_definition', 'public_field_definition'].includes(node.type)) { + return staticPropertyName(node.childForFieldName('name')) ?? 'anonymous'; + } const nameNode = maybeGetNameNode(node); if (nameNode?.text) { const normalized = normalizeSymbolName(nameNode.text); @@ -237,6 +245,114 @@ function extractNodeName(node: Node): string { return 'anonymous'; } +/** Read only literal property keys; a computed identifier is not its runtime value. */ +function staticPropertyName(node: Node | null): string | null { + if (!node) return null; + if (node.type === 'computed_property_name') { + const expression = node.namedChildren[0]; + return expression?.type === 'string' || expression?.type === 'number' + ? staticPropertyName(expression) + : null; + } + if (node.type === 'string') { + const text = node.text.slice(1, -1); + return text.replace( + /\\(?:u\{([\da-f]+)\}|u([\da-f]{4})|x([\da-f]{2})|([\s\S]))/gi, + ( + _match: string, + codePoint: string | undefined, + unicode: string | undefined, + hex: string | undefined, + escaped: string | undefined + ) => { + const digits = codePoint ?? unicode ?? hex; + if (digits) return String.fromCodePoint(Number.parseInt(digits, 16)); + const escapes: Record = { + n: '\n', + r: '\r', + t: '\t', + b: '\b', + f: '\f', + v: '\v', + '0': '\0', + '\n': '', + '\r': '' + }; + return escaped === undefined ? '' : (escapes[escaped] ?? escaped); + } + ); + } + return ['identifier', 'property_identifier', 'private_property_identifier', 'number'].includes( + node.type + ) + ? normalizeSymbolName(node.text) + : null; +} + +function isCallableValue(node: Node | null): boolean { + if (node?.type === 'parenthesized_expression') + return isCallableValue(node.namedChildren[0] ?? null); + return ( + node?.type === 'arrow_function' || + node?.type === 'function_expression' || + node?.type === 'generator_function' + ); +} + +/** Property descriptors keep their immediate named owner and implementation together. */ +function isCallableDescriptor(node: Node | null): boolean { + if (node?.type !== 'object') return false; + const fields = node.namedChildren.filter( + (child): child is Node => child !== null && child.type !== 'comment' + ); + const descriptorKeys = new Set(['value', 'get', 'set', 'writable', 'enumerable', 'configurable']); + const fieldKey = (field: Node): string | null => + staticPropertyName(field.childForFieldName(field.type === 'pair' ? 'key' : 'name')); + return ( + fields.length > 0 && + fields.every( + (field) => + (field.type === 'pair' || field.type === 'method_definition') && + descriptorKeys.has(fieldKey(field) ?? '') + ) && + fields.some((field) => { + return ( + ['value', 'get', 'set'].includes(fieldKey(field) ?? '') && + (field.type === 'method_definition' + ? field.childForFieldName('body') !== null + : isCallableValue(field.childForFieldName('value'))) + ); + }) + ); +} + +function getSymbolPath(node: Node): string[] { + const names: string[] = []; + let current: Node | null = node; + while (current) { + if ( + current.type === 'pair' || + current.type === 'variable_declarator' || + ((SYMBOL_CANDIDATE_NODE_TYPES as readonly string[]).includes(current.type) && + !['lexical_declaration', 'type_declaration'].includes(current.type)) + ) { + const name = extractNodeName(current); + // A dynamic property interrupts the static route; omitting it would falsely + // connect the inner declaration directly to an outer owner. + if ( + ['pair', 'method_definition', 'field_definition', 'public_field_definition'].includes( + current.type + ) && + name === 'anonymous' + ) + break; + if (name !== 'anonymous') names.unshift(name); + } + current = current.parent; + } + return names; +} + function isFunctionVariableDeclarator(node: Node): boolean { if (node.type !== 'variable_declarator') { return false; @@ -251,6 +367,20 @@ function isFunctionVariableDeclarator(node: Node): boolean { } function shouldSkipNode(language: string, node: Node): boolean { + if ( + (node.type === 'pair' || node.type === 'method_definition') && + node.parent?.parent?.type === 'pair' && + isCallableDescriptor(node.parent) + ) { + return true; + } + if (node.type === 'pair') { + const value = node.childForFieldName('value'); + return !isCallableValue(value) && !isCallableDescriptor(value); + } + if (node.type === 'field_definition' || node.type === 'public_field_definition') { + return !isCallableValue(node.childForFieldName('value')); + } if (node.type === 'variable_declarator') { return ( !['javascript', 'javascriptreact', 'typescript', 'typescriptreact'].includes(language) || @@ -288,7 +418,8 @@ function buildSymbol(node: Node, content: string): TreeSitterSymbol { startIndex: rangeNode.startIndex, endIndex: rangeNode.endIndex, content: extractNodeContent(rangeNode, content), - nodeType: node.type + nodeType: node.type, + symbolPath: getSymbolPath(node) }; } @@ -346,7 +477,7 @@ export async function extractTreeSitterSymbols( continue; } - const key = `${symbol.kind}:${symbol.name}:${symbol.startLine}:${symbol.endLine}`; + const key = `${symbol.kind}:${symbol.name}:${symbol.startIndex}:${symbol.endIndex}`; if (seen.has(key)) { continue; } @@ -389,6 +520,87 @@ export interface IdentifierOccurrence { nodeType: string; } +const LEXICAL_IDENTIFIER_CONTINUE = /(?:\p{ID_Continue}|[$]|\u200c|\u200d)/u; +const ASCII_IDENTIFIER_CONTINUE = /[A-Za-z0-9_$]/; +const ASCII_IDENTIFIER = /^[A-Za-z_$][A-Za-z0-9_$]*$/; + +/** + * Conservatively nominate source for identifier matching without parsing longer + * ASCII identifiers. Non-ASCII and non-identifier query spellings retain raw + * substring behavior, so grammar-specific and uncertain cases remain included. + */ +export function hasIdentifierCandidate(content: string, symbol: string): boolean { + const normalizedSymbol = symbol.trim(); + if (!normalizedSymbol) return false; + if (!ASCII_IDENTIFIER.test(normalizedSymbol)) return content.includes(normalizedSymbol); + + let searchFrom = 0; + while (searchFrom <= content.length - normalizedSymbol.length) { + const startIndex = content.indexOf(normalizedSymbol, searchFrom); + if (startIndex === -1) return false; + + const endIndex = startIndex + normalizedSymbol.length; + const previous = startIndex > 0 ? content[startIndex - 1] : undefined; + const next = endIndex < content.length ? content[endIndex] : undefined; + const hasLeftBoundary = previous === undefined || !ASCII_IDENTIFIER_CONTINUE.test(previous); + const hasRightBoundary = next === undefined || !ASCII_IDENTIFIER_CONTINUE.test(next); + if (hasLeftBoundary && hasRightBoundary) return true; + + searchFrom = startIndex + Math.max(1, normalizedSymbol.length); + } + return false; +} + +/** + * Conservative language-neutral token matching for files without a usable parser. + * This preserves whole identifier boundaries for Unicode, `_`, and `$` spellings. + */ +export function findLexicalIdentifierOccurrences( + content: string, + symbol: string +): IdentifierOccurrence[] { + const normalizedSymbol = symbol.trim(); + if (!normalizedSymbol) return []; + + const occurrences: IdentifierOccurrence[] = []; + let searchFrom = 0; + let lineCursor = 0; + let line = 1; + while (searchFrom <= content.length - normalizedSymbol.length) { + const startIndex = content.indexOf(normalizedSymbol, searchFrom); + if (startIndex === -1) break; + const endIndex = startIndex + normalizedSymbol.length; + const previousIndex = startIndex > 0 ? startIndex - 1 : -1; + const previousCodePointIndex = + previousIndex >= 0 && + content.charCodeAt(previousIndex) >= 0xdc00 && + content.charCodeAt(previousIndex) <= 0xdfff + ? previousIndex - 1 + : previousIndex; + const previousCodePoint = + previousCodePointIndex >= 0 + ? String.fromCodePoint(content.codePointAt(previousCodePointIndex) ?? 0) + : undefined; + const nextCodePoint = + endIndex < content.length + ? String.fromCodePoint(content.codePointAt(endIndex) ?? 0) + : undefined; + const hasIdentifierBoundary = + (!previousCodePoint || !LEXICAL_IDENTIFIER_CONTINUE.test(previousCodePoint)) && + (!nextCodePoint || !LEXICAL_IDENTIFIER_CONTINUE.test(nextCodePoint)); + + while (lineCursor < startIndex) { + if (content.charCodeAt(lineCursor) === 10) line += 1; + lineCursor += 1; + } + if (hasIdentifierBoundary) { + occurrences.push({ line, startIndex, endIndex, nodeType: 'lexical_identifier' }); + } + searchFrom = startIndex + Math.max(1, normalizedSymbol.length); + } + return occurrences; +} + const IDENTIFIER_NODE_TYPES = [ 'identifier', 'type_identifier', diff --git a/src/utils/usage-tracker.ts b/src/utils/usage-tracker.ts index 23bedcb..7bd9d9f 100644 --- a/src/utils/usage-tracker.ts +++ b/src/utils/usage-tracker.ts @@ -625,6 +625,189 @@ export interface UnusedExport { export interface ImportEdgeDetail { line?: number; importedSymbols?: string[]; + /** Source declarations that produced this adjacency. Legacy details have no proof and stay unverified. */ + sources?: ImportEdgeSource[]; +} + +export type GraphRelationKind = 'import' | 'reexport'; + +export interface ImportEdgeSource { + line: number; + specifier: string; + kind: GraphRelationKind; + sourceFingerprint: string; + targetFingerprint: string; +} + +type UnresolvedSpecifierReason = 'external' | 'not-found' | 'outside-root' | 'unsupported-alias'; + +type UnresolvedImportDiagnostic = + | { reason: UnresolvedSpecifierReason } + | { reason: 'not-indexed' | 'analysis-unavailable'; target: string }; + +export type UnresolvedImportEdge = { + from: string; + kind: GraphRelationKind; + source: ImportEdgeSource; +} & UnresolvedImportDiagnostic; + +export interface FileGraphEdge { + from: string; + to: string; + kind: GraphRelationKind; + source: ImportEdgeSource; + verified: boolean; +} + +export interface FileGraphTraversal { + paths: Array<{ nodes: string[]; edges: FileGraphEdge[] }>; + unresolved: UnresolvedImportEdge[]; + truncated: boolean; +} + +interface TraversableFileGraph { + imports?: Record; + importDetails?: Record>; + unresolvedEdges?: UnresolvedImportEdge[]; +} + +interface FileGraphTraversalOptions { + direction: 'outgoing' | 'incoming'; + maxHops: 1 | 2 | 3; + maxNodes?: number; + maxPaths?: number; + /** Candidate edge attempts and unresolved diagnostics share this work budget. */ + maxInspections?: number; +} + +interface CheckedFileGraphTraversal extends FileGraphTraversal { + rejectedEdges: FileGraphEdge[]; + inspectionLimited: boolean; +} + +/** A shared BFS lets asynchronous freshness checks run before a path consumes output limits. */ +function* fileGraphTraversal( + graph: TraversableFileGraph, + start: string, + options: FileGraphTraversalOptions +): Generator { + const imports = graph.imports ?? {}; + const incoming = new Map(); + if (options.direction === 'incoming') { + // The sidecar stores forward adjacency only. Build its reverse lookup once, without source IO. + for (const [from, targets] of Object.entries(imports)) { + for (const to of targets) { + const importers = incoming.get(to); + if (importers) importers.push(from); + else incoming.set(to, [from]); + } + } + } + const neighbors = function* (node: string): Generator { + const targets = + options.direction === 'outgoing' ? (imports[node] ?? []) : (incoming.get(node) ?? []); + for (const neighbor of targets) { + const from = options.direction === 'outgoing' ? node : neighbor; + const to = options.direction === 'outgoing' ? neighbor : node; + const detail = graph.importDetails?.[from]?.[to]; + const sources = detail?.sources ?? []; + if (sources.length > 0) { + for (const source of sources) yield { from, to, kind: source.kind, source, verified: true }; + } else { + yield { + from, + to, + kind: 'import', + verified: false, + source: { + line: detail?.line ?? 1, + specifier: '', + kind: 'import', + sourceFingerprint: '', + targetFingerprint: '' + } + }; + } + } + }; + + const maxNodes = Math.max(1, Math.min(500, options.maxNodes ?? 100)); + const maxPaths = Math.max(1, Math.min(500, options.maxPaths ?? 200)); + const maxInspections = Math.max(1, Math.min(10000, options.maxInspections ?? 1000)); + const visitedNodes = new Set([start]); + const queue: Array<{ nodes: string[]; edges: FileGraphEdge[] }> = [{ nodes: [start], edges: [] }]; + const paths: FileGraphTraversal['paths'] = []; + const rejectedEdges: FileGraphEdge[] = []; + let truncated = false; + let inspections = 0; + let inspectionLimited = false; + traversal: for (let cursor = 0; cursor < queue.length; cursor++) { + const current = queue[cursor]!; + if (current.edges.length >= options.maxHops) continue; + for (const edge of neighbors(current.nodes[current.nodes.length - 1]!)) { + if (inspections >= maxInspections) { + truncated = inspectionLimited = true; + break traversal; + } + inspections++; + const next = options.direction === 'outgoing' ? edge.to : edge.from; + if (current.nodes.includes(next)) continue; + if (paths.length >= maxPaths) { + truncated = true; + break traversal; + } + if (!(yield edge)) { + rejectedEdges.push(edge); + continue; + } + if (!visitedNodes.has(next) && visitedNodes.size >= maxNodes) { + truncated = true; + continue; + } + visitedNodes.add(next); + const nextPath = { + nodes: [...current.nodes, next], + edges: [...current.edges, edge] + }; + paths.push(nextPath); + queue.push(nextPath); + } + } + const unresolved: UnresolvedImportEdge[] = []; + for (const edge of graph.unresolvedEdges ?? []) { + if (!visitedNodes.has(edge.from)) continue; + if (inspections >= maxInspections) { + truncated = inspectionLimited = true; + break; + } + inspections++; + unresolved.push(edge); + } + return { paths, unresolved, truncated, rejectedEdges, inspectionLimited }; +} + +export function traverseFileGraph( + graph: TraversableFileGraph, + start: string, + options: FileGraphTraversalOptions +): FileGraphTraversal { + const traversal = fileGraphTraversal(graph, start, options); + let step = traversal.next(); + while (!step.done) step = traversal.next(true); + const { paths, unresolved, truncated } = step.value; + return { paths, unresolved, truncated }; +} + +export async function traverseFileGraphWithVerification( + graph: TraversableFileGraph, + start: string, + options: FileGraphTraversalOptions, + acceptEdge: (edge: FileGraphEdge) => Promise +): Promise { + const traversal = fileGraphTraversal(graph, start, options); + let step = traversal.next(); + while (!step.done) step = traversal.next(await acceptEdge(step.value)); + return step.value; } export class InternalFileGraph { @@ -636,6 +819,7 @@ export class InternalFileGraph { private importedSymbols: Map> = new Map(); // Map: fromFile -> toFile -> edge details (line/symbols) private importDetails: Map> = new Map(); + private unresolvedEdges: UnresolvedImportEdge[] = []; // Root path for relative path conversion private rootPath: string; @@ -674,7 +858,8 @@ export class InternalFileGraph { importingFile: string, importedFile: string, line?: number, - importedSymbols?: string[] + importedSymbols?: string[], + sourceProof?: Omit ): void { const fromFile = this.normalizePath(importingFile); const toFile = this.normalizePath(importedFile); @@ -694,7 +879,7 @@ export class InternalFileGraph { ? importedSymbols.filter((sym) => sym !== '*' && sym !== 'default') : []; - if (normalizedLine || normalizedSymbols.length > 0) { + if (normalizedLine || normalizedSymbols.length > 0 || sourceProof) { if (!this.importDetails.has(fromFile)) { this.importDetails.set(fromFile, new Map()); } @@ -710,9 +895,31 @@ export class InternalFileGraph { for (const sym of normalizedSymbols) mergedSymbolsSet.add(sym); const mergedSymbols = Array.from(mergedSymbolsSet); + const sources = existing.sources ? [...existing.sources] : []; + if (sourceProof && normalizedLine) { + const nextSource: ImportEdgeSource = { ...sourceProof, line: normalizedLine }; + if ( + !sources.some( + (source) => + source.line === nextSource.line && + source.specifier === nextSource.specifier && + source.kind === nextSource.kind + ) + ) { + sources.push(nextSource); + } + } + detailsForFrom.set(toFile, { ...(mergedLine ? { line: mergedLine } : {}), - ...(mergedSymbols.length > 0 ? { importedSymbols: mergedSymbols.sort() } : {}) + ...(mergedSymbols.length > 0 ? { importedSymbols: mergedSymbols.sort() } : {}), + ...(sources.length > 0 + ? { + sources: sources.sort( + (a, b) => a.line - b.line || a.specifier.localeCompare(b.specifier) + ) + } + : {}) }); } @@ -727,6 +934,33 @@ export class InternalFileGraph { } } + trackUnresolved( + importingFile: string, + source: ImportEdgeSource, + diagnostic: UnresolvedSpecifierReason | UnresolvedImportDiagnostic, + kind: GraphRelationKind = source.kind + ): void { + const proof = typeof diagnostic === 'string' ? { reason: diagnostic } : diagnostic; + const edge: UnresolvedImportEdge = { + from: this.normalizePath(importingFile), + kind, + source, + ...('target' in proof + ? { reason: proof.reason, target: proof.target } + : { reason: proof.reason }) + }; + if ( + !this.unresolvedEdges.some( + (existing) => + existing.from === edge.from && + existing.source.line === edge.source.line && + existing.source.specifier === edge.source.specifier + ) + ) { + this.unresolvedEdges.push(edge); + } + } + /** * Track exports from a file */ @@ -865,6 +1099,7 @@ export class InternalFileGraph { imports: Record; exports: Record; importDetails?: Record>; + unresolvedEdges?: UnresolvedImportEdge[]; stats: { files: number; edges: number; avgDependencies: number }; } { const imports: Record = {}; @@ -894,6 +1129,7 @@ export class InternalFileGraph { imports, exports, ...(Object.keys(importDetails).length > 0 ? { importDetails } : {}), + ...(this.unresolvedEdges.length > 0 ? { unresolvedEdges: this.unresolvedEdges } : {}), stats: this.getStats() }; } @@ -906,6 +1142,7 @@ export class InternalFileGraph { imports?: Record; exports?: Record; importDetails?: Record>; + unresolvedEdges?: UnresolvedImportEdge[]; }, rootPath: string ): InternalFileGraph { @@ -944,6 +1181,11 @@ export class InternalFileGraph { } } + graph.unresolvedEdges = (data.unresolvedEdges ?? []).map((edge) => ({ + ...edge, + from: graph.normalizePath(edge.from) + })); + return graph; } } diff --git a/tests/analyzer-registry.test.ts b/tests/analyzer-registry.test.ts index 703ceb7..202007a 100644 --- a/tests/analyzer-registry.test.ts +++ b/tests/analyzer-registry.test.ts @@ -5,6 +5,8 @@ import { NextJsAnalyzer } from '../src/analyzers/nextjs/index'; import { NestJsAnalyzer } from '../src/analyzers/nestjs/index'; import { ReactAnalyzer } from '../src/analyzers/react/index'; import { GenericAnalyzer } from '../src/analyzers/generic/index'; +import angularRoutingFixture from './fixtures/angular-routing-regressions.json'; +import { createHash } from 'node:crypto'; // Register default analyzers analyzerRegistry.register(new AngularAnalyzer()); @@ -13,6 +15,45 @@ analyzerRegistry.register(new NestJsAnalyzer()); analyzerRegistry.register(new ReactAnalyzer()); analyzerRegistry.register(new GenericAnalyzer()); +describe('frozen Angular-local routing source facts', () => { + const registry = new AnalyzerRegistry(); + const angular = new AngularAnalyzer(); + registry.register(angular); + registry.register(new NextJsAnalyzer()); + registry.register(new NestJsAnalyzer()); + registry.register(new ReactAnalyzer()); + registry.register(new GenericAnalyzer()); + + for (const fixture of angularRoutingFixture.cases) { + it(`routes according to actual source evidence: ${fixture.id}`, async () => { + const filePath = `/virtual/${fixture.filePath}`; + expect(angular.canAnalyze(filePath, fixture.source)).toBe(fixture.expectedAngularEligible); + expect(registry.findAnalyzer(filePath, fixture.source)?.name).toBe(fixture.expectedAnalyzer); + if ('origin' in fixture && fixture.origin) { + expect(createHash('sha256').update(fixture.source, 'utf8').digest('hex')).toBe( + fixture.origin.sourceSha256 + ); + } + if ('expectedSourceSymbols' in fixture && fixture.expectedSourceSymbols) { + const result = await registry.analyzeFile(filePath, fixture.source, { + includeScopePrefix: false + }); + expect(result?.metadata.analyzer).toBe('generic'); + expect(result?.metadata.chunkStrategy).toBe('ast-aligned'); + for (const name of fixture.expectedSourceSymbols) { + const symbol = result?.chunks.find((chunk) => chunk.metadata.symbolName === name); + expect(symbol, name).toBeDefined(); + expect(symbol!.metadata.symbolAware).toBe(true); + expect(symbol!.metadata.symbolPath).toContain(name); + expect(fixture.source).toContain(symbol!.content); + expect(symbol!.startLine).toBeGreaterThanOrEqual(1); + expect(symbol!.endLine).toBeLessThanOrEqual(fixture.source.split('\n').length); + } + } + }); + } +}); + describe('AnalyzerRegistry', () => { describe('missing analyzer logging', () => { it('should stay quiet by default when no analyzer matches', async () => { @@ -110,10 +151,56 @@ export class PaymentsService {} expect( analyzerRegistry.findAllAnalyzers('/tmp/payments.service.ts', content).map((a) => a.name) - ).toEqual(['nestjs', 'angular', 'generic']); + ).toEqual(['nestjs', 'generic']); expect(analyzerRegistry.findAnalyzer('/tmp/payments.service.ts', content)?.name).toBe( 'nestjs' ); }); + + it('selects Generic for plain TypeScript generics and retains symbol-aware definitions', async () => { + const source = ` +interface ProjectConfiguration { targets?: Record } +const map = new Map(); + +export function readTargetDefaultsForTarget( + project: ProjectConfiguration, + targetName: string +) { + return map.get(targetName) ?? project.targets?.[targetName]; +} +`; + + expect(analyzerRegistry.findAnalyzer('/tmp/target-defaults.ts', source)?.name).toBe( + 'generic' + ); + const result = await analyzerRegistry.analyzeFile('/tmp/target-defaults.ts', source); + + expect(result?.metadata.analyzer).toBe('generic'); + expect( + result?.chunks.some( + (chunk) => + chunk.metadata.symbolName === 'readTargetDefaultsForTarget' && + chunk.metadata.symbolAware === true + ) + ).toBe(true); + }); + + it('preserves quoted names in TypeScript imports and exports', async () => { + const source = `import { "remote-name" as localName } from 'some-package'; +export { localName as "public-name" };`; + const result = await new GenericAnalyzer().analyze('/tmp/quoted-names.ts', source); + + expect(result.imports).toMatchObject([{ source: 'some-package', imports: ['remote-name'] }]); + expect(result.exports).toContainEqual({ + name: 'public-name', + isDefault: false, + type: 'named' + }); + + const angularResult = await new AngularAnalyzer().analyze('/tmp/quoted-names.ts', source); + expect(angularResult.imports).toMatchObject([ + { source: 'some-package', imports: ['remote-name'] } + ]); + }); }); }); diff --git a/tests/ast-chunker-integration.test.ts b/tests/ast-chunker-integration.test.ts index 16ee73e..6ea2360 100644 --- a/tests/ast-chunker-integration.test.ts +++ b/tests/ast-chunker-integration.test.ts @@ -1,6 +1,9 @@ import { describe, expect, it } from 'vitest'; import { GenericAnalyzer } from '../src/analyzers/generic/index'; import { MAX_AST_CHUNK_FILE_LINES } from '../src/utils/ast-chunker'; +import { createASTAlignedChunks } from '../src/utils/ast-chunker'; +import { extractTreeSitterSymbols } from '../src/utils/tree-sitter'; +import structuralFixture from './fixtures/structural-context-regressions.json'; // --------------------------------------------------------------------------- // Fixtures @@ -69,6 +72,197 @@ export function createUserService(db: Database): UserService { const analyzer = new GenericAnalyzer(); +describe('frozen structural raw chunks', () => { + for (const descriptor of [ + { + name: 'concise-value', + fields: 'value() { closeConnections(); }', + markers: ['closeConnections()'] + }, + { name: 'concise-get', fields: 'get() { return readState(); }', markers: ['readState()'] }, + { + name: 'concise-set', + fields: 'set(next) { writeState(next); }', + markers: ['writeState(next)'] + }, + { + name: 'mixed-accessors', + fields: 'get() { return readState(); }, set: next => writeState(next), enumerable: true', + markers: ['readState()', 'writeState(next)'] + } + ]) { + it(`keeps the nearest concise descriptor owner with its body: ${descriptor.name}`, async () => { + const source = `function createHandlers() { return { shutdown: { ${descriptor.fields} }, sibling: { value() { reloadConnections(); } }, services: { billing: { reconcile() { persistInvoice(); } } } }; }`; + const parsed = await extractTreeSitterSymbols(source, 'javascript'); + expect(parsed).not.toBeNull(); + const owner = parsed!.symbols.find((symbol) => symbol.name === 'shutdown'); + expect(owner).toBeDefined(); + expect(owner!.symbolPath).toEqual(['createHandlers', 'shutdown']); + expect(owner!.content).toBe(`shutdown: { ${descriptor.fields} }`); + const chunks = createASTAlignedChunks(source, parsed!.symbols, { + filePath: '/virtual/concise-descriptor.js', + language: 'javascript', + minChunkLines: 10, + maxChunkLines: 150, + includeScopePrefix: false + }); + const implementation = chunks.find((chunk) => chunk.metadata.symbolName === 'shutdown'); + expect(implementation).toBeDefined(); + expect(implementation!.metadata.symbolPath).toEqual(['createHandlers', 'shutdown']); + expect(implementation!.content).toBe(owner!.content); + for (const marker of descriptor.markers) expect(implementation!.content).toContain(marker); + expect(implementation!.content).not.toContain('reloadConnections()'); + expect(implementation!.content).not.toContain('persistInvoice()'); + expect( + chunks.find((chunk) => chunk.metadata.symbolName === 'reconcile')?.metadata.symbolPath + ).toEqual(['createHandlers', 'services', 'billing', 'reconcile']); + expect( + parsed!.symbols.some( + (symbol) => symbol.symbolPath?.join('.') === 'createHandlers.shutdown.value' + ) + ).toBe(false); + }); + } + + for (const fields of [ + 'value() { closeConnections(); }, unknown: true', + '[operation]() { closeConnections(); }' + ]) { + it(`does not coalesce an unresolved descriptor shape: ${fields}`, async () => { + const source = `function createHandlers() { return { shutdown: { ${fields} } }; }`; + const parsed = await extractTreeSitterSymbols(source, 'javascript'); + expect(parsed).not.toBeNull(); + expect(parsed!.symbols.some((symbol) => symbol.name === 'shutdown')).toBe(false); + }); + } + + it('preserves same-line container statements before and after a nested callback', async () => { + const source = 'function start(){ before(); const hooks={onReady:()=>openQueue()}; after(); }'; + const parsed = await extractTreeSitterSymbols(source, 'javascript'); + const chunks = createASTAlignedChunks(source, parsed!.symbols, { + filePath: '/virtual/container.js', + language: 'javascript', + minChunkLines: 10, + maxChunkLines: 150, + includeScopePrefix: false + }); + expect(chunks.map((chunk) => chunk.content).join('')).toBe(source); + const callback = chunks.find((chunk) => chunk.metadata.symbolName === 'onReady'); + expect(callback).toBeDefined(); + expect(callback!.content).toContain('openQueue()'); + expect(callback!.content).not.toContain('before()'); + expect(callback!.content).not.toContain('after()'); + for (const chunk of chunks) { + expect(chunk.startLine).toBe(1); + expect(chunk.endLine).toBe(1); + expect(source).toContain(chunk.content); + } + }); + + for (const fixture of structuralFixture.cases) { + it(`keeps raw owner bodies isolated: ${fixture.id}`, async () => { + const parsed = await extractTreeSitterSymbols(fixture.source, fixture.language); + expect(parsed).not.toBeNull(); + const chunks = createASTAlignedChunks(fixture.source, parsed!.symbols, { + filePath: '/virtual/structural.ts', + language: fixture.language, + minChunkLines: 10, + maxChunkLines: 150, + includeScopePrefix: false + }); + for (const fact of fixture.sourceSymbols) { + if ('scopeOnly' in fact && fact.scopeOnly) { + expect( + chunks.some((chunk) => + fact.bodyMarkers.every((marker) => chunk.content.includes(marker)) + ) + ).toBe(true); + continue; + } + const matching = chunks.filter( + (chunk) => JSON.stringify(chunk.metadata.symbolPath) === JSON.stringify(fact.ownerChain) + ); + expect(matching.length, `${fixture.id}: ${fact.ownerChain.join(' > ')}`).toBeGreaterThan(0); + for (const marker of fact.bodyMarkers) + expect(matching.some((chunk) => chunk.content.includes(marker))).toBe(true); + for (const chunk of matching) { + expect(fixture.source).toContain(chunk.content); + if ('forbiddenSiblingBodyMarkers' in fact) { + for (const marker of fact.forbiddenSiblingBodyMarkers ?? []) + expect(chunk.content).not.toContain(marker); + } + } + } + }); + } + + it('coalesces only an immediate callable descriptor, without swallowing singleton wrappers', async () => { + const source = [ + 'function buildHandlers() {', + ' return {', + ' shutdown: {', + ' configurable: true,', + ' value: () => { closeConnections(); }', + ' },', + ' services: { billing: { reconcile: () => persistInvoice() } }', + ' };', + '}' + ].join('\n'); + const parsed = await extractTreeSitterSymbols(source, 'javascript'); + const chunks = createASTAlignedChunks(source, parsed!.symbols, { + filePath: '/virtual/descriptor.js', + language: 'javascript', + minChunkLines: 10, + maxChunkLines: 150, + includeScopePrefix: false + }); + const shutdown = chunks.find((chunk) => chunk.metadata.symbolName === 'shutdown'); + expect(shutdown).toBeDefined(); + expect(shutdown!.content).toContain('shutdown: {'); + expect(shutdown!.content).toContain('closeConnections()'); + expect(shutdown!.content).not.toContain('persistInvoice()'); + const reconcile = chunks.find((chunk) => chunk.metadata.symbolName === 'reconcile'); + expect(reconcile).toBeDefined(); + expect(reconcile!.content).not.toContain('services:'); + expect(reconcile!.content).not.toContain('billing:'); + }); + + it('splits raw source before display prefixes without shifting source coordinates', async () => { + const source = [ + 'function consume() {', + '// consume :: ()', + ...Array.from({ length: 175 }, (_, i) => ` step${i}();`), + '}' + ].join('\n'); + const parsed = await extractTreeSitterSymbols(source, 'javascript'); + const options = { + filePath: '/virtual/large.js', + language: 'javascript', + minChunkLines: 10, + maxChunkLines: 35 + }; + const raw = createASTAlignedChunks(source, parsed!.symbols, { + ...options, + includeScopePrefix: false + }); + const displayed = createASTAlignedChunks(source, parsed!.symbols, options); + expect(displayed.map(({ startLine, endLine }) => [startLine, endLine])).toEqual( + raw.map(({ startLine, endLine }) => [startLine, endLine]) + ); + for (const chunk of raw) { + expect(chunk.content).toBe( + source + .split('\n') + .slice(chunk.startLine - 1, chunk.endLine) + .join('\n') + ); + expect(chunk.endLine - chunk.startLine + 1).toBeLessThanOrEqual(35); + } + expect(raw.map((chunk) => chunk.content).join('\n')).toBe(source); + expect(raw[0].content).toContain('// consume :: ()'); + }); +}); + describe('AST Chunker Integration', () => { // Test 1: Supported language, normal file — AST chunks with scope prefixes it('produces AST-aligned chunks with scope prefixes for a normal TypeScript file', async () => { diff --git a/tests/auto-refresh-e2e.test.ts b/tests/auto-refresh-e2e.test.ts index 6e6a7bc..7570447 100644 --- a/tests/auto-refresh-e2e.test.ts +++ b/tests/auto-refresh-e2e.test.ts @@ -1,3 +1,4 @@ +import { resolveIndexArtifact } from '../src/core/index-generation.js'; import { describe, it, expect, beforeEach, afterEach } from 'vitest'; import { promises as fs } from 'fs'; import os from 'os'; @@ -31,12 +32,10 @@ function getKeywordChunks(raw: unknown): Array> { } async function readIndexedContent(rootPath: string): Promise { - const indexPath = path.join(rootPath, CODEBASE_CONTEXT_DIRNAME, KEYWORD_INDEX_FILENAME); + const indexPath = await resolveIndexArtifact(rootPath, KEYWORD_INDEX_FILENAME); const raw = JSON.parse(await fs.readFile(indexPath, 'utf-8')) as unknown; const chunks = getKeywordChunks(raw); - return chunks - .map((chunk) => (typeof chunk.content === 'string' ? chunk.content : '')) - .join('\n'); + return chunks.map((chunk) => (typeof chunk.content === 'string' ? chunk.content : '')).join('\n'); } async function waitFor( @@ -69,7 +68,10 @@ describe('Auto-refresh E2E', () => { tempDir = await fs.mkdtemp(path.join(os.tmpdir(), 'auto-refresh-e2e-')); await fs.mkdir(path.join(tempDir, 'src'), { recursive: true }); await fs.writeFile(path.join(tempDir, 'package.json'), JSON.stringify({ name: 'watch-test' })); - await fs.writeFile(path.join(tempDir, 'src', 'app.ts'), 'export const token = "INITIAL_TOKEN";\n'); + await fs.writeFile( + path.join(tempDir, 'src', 'app.ts'), + 'export const token = "INITIAL_TOKEN";\n' + ); }); afterEach(async () => { @@ -131,7 +133,10 @@ describe('Auto-refresh E2E', () => { try { await watcherReady; - await fs.writeFile(path.join(tempDir, 'src', 'app.ts'), 'export const token = "UPDATED_TOKEN";\n'); + await fs.writeFile( + path.join(tempDir, 'src', 'app.ts'), + 'export const token = "UPDATED_TOKEN";\n' + ); await waitFor( async () => { @@ -146,7 +151,7 @@ describe('Auto-refresh E2E', () => { expect(updatedContent).toContain('UPDATED_TOKEN'); expect(incrementalRuns).toBeGreaterThan(0); } finally { - stopWatcher(); + await stopWatcher(); } }, 20000); }); diff --git a/tests/chunk-source-integrity.test.ts b/tests/chunk-source-integrity.test.ts new file mode 100644 index 0000000..94fe3fc --- /dev/null +++ b/tests/chunk-source-integrity.test.ts @@ -0,0 +1,241 @@ +import { promises as fs } from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { afterEach, expect, it, vi } from 'vitest'; +import { mergeSmallChunks, validateSourceChunks } from '../src/utils/chunking.js'; +import { CodebaseIndexer } from '../src/core/indexer.js'; +import { analyzerRegistry } from '../src/core/analyzer-registry.js'; +import { GenericAnalyzer } from '../src/analyzers/generic/index.js'; +import { AngularAnalyzer } from '../src/analyzers/angular/index.js'; +import { InvalidSourceChunkError } from '../src/errors/index.js'; +import { ReactAnalyzer } from '../src/analyzers/react/index.js'; +import { resolveIndexArtifact } from '../src/core/index-generation.js'; +import type { CodeChunk } from '../src/types/index.js'; +let root = ''; +afterEach(async () => { + vi.restoreAllMocks(); + if (root) await fs.rm(root, { recursive: true, force: true }); + root = ''; +}); +function chunk( + startLine: number, + endLine: number, + content: string, + overrides: Partial = {} +): CodeChunk { + return { + id: `${startLine}-${endLine}`, + filePath: '/project/source.ts', + relativePath: 'source.ts', + startLine, + endLine, + content, + language: 'typescript', + dependencies: [], + imports: [], + exports: [], + tags: [], + metadata: {}, + ...overrides + }; +} +it.each([ + { name: 'reverse', first: chunk(20, 20, 'late'), second: chunk(1, 1, 'early') }, + { name: 'gap', first: chunk(1, 1, 'early'), second: chunk(3, 3, 'late') }, + { name: 'overlap', first: chunk(1, 2, 'a\nb'), second: chunk(2, 3, 'b\nc') }, + { + name: 'foreign file', + first: chunk(1, 1, 'a'), + second: chunk(2, 2, 'b', { filePath: '/project/other.ts' }) + }, + { + name: 'independent owners', + first: chunk(1, 1, 'a', { metadata: { componentName: 'First' } }), + second: chunk(2, 2, 'b', { metadata: { componentName: 'Second' } }) + }, + { + name: 'different metadata', + first: chunk(1, 1, 'a', { metadata: { framework: 'one' } }), + second: chunk(2, 2, 'b', { metadata: { framework: 'two' } }) + } +])('keeps $name chunks as independent source evidence', ({ first, second }) => { + expect(mergeSmallChunks([first, second], 15)).toEqual([first, second]); +}); +it('merges a legal contiguous interval while retaining imports, exports and dependency tags', () => { + const first = chunk(1, 1, 'a', { + dependencies: ['one'], + imports: ['first'], + exports: ['A'], + tags: ['left'] + }); + const second = chunk(2, 2, 'b', { + dependencies: ['two'], + imports: ['second'], + exports: ['B'], + tags: ['right'] + }); + const result = mergeSmallChunks([first, second], 15); + expect(result).toHaveLength(1); + expect(result[0]).toMatchObject({ + startLine: 1, + endLine: 2, + content: 'a\nb', + dependencies: ['one', 'two'], + imports: ['first', 'second'], + exports: ['A', 'B'], + tags: ['left', 'right'] + }); +}); +it('persists actual React TSX chunks whose bodies and intervals match the analyzed file', async () => { + root = await fs.mkdtemp(path.join(os.tmpdir(), 'cbc-react-source-integrity-')); + analyzerRegistry.register(new ReactAnalyzer()); + const source = [ + 'export const First = () =>
render escaped child nodes
', + ...Array(20).fill(''), + 'export const Second = () => render escaped child nodes', + '' + ].join('\n'); + await fs.writeFile(path.join(root, 'components.tsx'), source); + await fs.writeFile( + path.join(root, 'package.json'), + JSON.stringify({ name: 'span-fixture', dependencies: { react: '19.0.0' } }) + ); + await new CodebaseIndexer({ rootPath: root, config: { skipEmbedding: true } }).index(); + const index: { chunks: CodeChunk[] } = JSON.parse( + await fs.readFile(await resolveIndexArtifact(root, 'index.json'), 'utf8') + ); + const spans = index.chunks.filter((item) => item.relativePath === 'components.tsx'); + expect(spans.length).toBeGreaterThanOrEqual(2); + for (const item of spans) { + expect(item.startLine).toBeGreaterThanOrEqual(1); + expect(item.endLine).toBeGreaterThanOrEqual(item.startLine); + expect( + source + .split('\n') + .slice(item.startLine - 1, item.endLine) + .join('\n') + ).toContain(item.content); + } +}); + +it.each([ + { + source: 'const alpha = "😀"; function x() { return alpha }', + start: 1, + end: 1, + body: 'function x() { return alpha }' + }, + { source: 'repeat\nrepeat\n', start: 2, end: 2, body: 'repeat' }, + { source: 'first\nsecond\n', start: 1, end: 3, body: 'first\nsecond\n' }, + { source: ' α\n child', start: 1, end: 2, body: 'α\n child' } +])('accepts exact source evidence $start–$end', ({ source, start, end, body }) => { + expect(() => + validateSourceChunks([chunk(start, end, body)], { + filePath: '/project/source.ts', + relativePath: 'source.ts', + content: source + }) + ).not.toThrow(); +}); +it.each([ + chunk(1, 1, '// Scope: invented\nactual'), + chunk(1, 2, 'only one line'), + chunk(0, 1, 'x'), + chunk(2, 1, 'x'), + chunk(1, 1, 'absent'), + chunk(1, 1, 'actual', { filePath: '/project/foreign.ts' }) +])('rejects fabricated producer evidence $id', (candidate) => { + expect(() => + validateSourceChunks([candidate], { + filePath: '/project/source.ts', + relativePath: 'source.ts', + content: 'actual\n' + }) + ).toThrow(InvalidSourceChunkError); +}); +it.each([ + [ + 'generic', + 'file.ts', + '// α😀\r\nexport const repeat = "x"; export function one() { return repeat }\r\nexport function two() { return repeat }\r\n' + ], + [ + 'angular', + 'component.ts', + 'import { Component } from "@angular/core";\r\n@Component({ selector: "app-test", template: "

α

" })\r\nexport class ExampleComponent { value = "😀"; }\r\n' + ] +])('indexes real %s source with raw padded/partial bodies', async (analyzer, file, source) => { + root = await fs.mkdtemp(path.join(os.tmpdir(), 'cbc-source-boundary-')); + analyzerRegistry.register(new GenericAnalyzer()); + analyzerRegistry.register(new AngularAnalyzer()); + await fs.writeFile(path.join(root, file), source); + await new CodebaseIndexer({ + rootPath: root, + config: { skipEmbedding: true }, + projectOptions: { preferredAnalyzer: analyzer } + }).index(); + const index: { chunks: CodeChunk[] } = JSON.parse( + await fs.readFile(await resolveIndexArtifact(root, 'index.json'), 'utf8') + ); + const selected = index.chunks.filter((item) => item.relativePath === file); + expect(selected.length).toBeGreaterThan(0); + expect(() => + validateSourceChunks(selected, { + filePath: path.join(root, file), + relativePath: file, + content: source.replace(/\r\n/g, '\n') + }) + ).not.toThrow(); +}); +it('aborts malformed producer output instead of publishing a silently skipped file', async () => { + root = await fs.mkdtemp(path.join(os.tmpdir(), 'cbc-source-fatal-')); + analyzerRegistry.register(new GenericAnalyzer()); + const file = path.join(root, 'file.ts'); + await fs.writeFile(file, 'export const value = 1;'); + await new CodebaseIndexer({ + rootPath: root, + config: { skipEmbedding: true }, + projectOptions: { preferredAnalyzer: 'generic' } + }).index(); + const pointer = path.join(root, '.codebase-context', 'index-meta.json'); + const before = await fs.readFile(pointer); + const analyze = analyzerRegistry.analyzeFile.bind(analyzerRegistry); + vi.spyOn(analyzerRegistry, 'analyzeFile').mockImplementation(async (...args) => { + const result = await analyze(...args); + if (result?.chunks[0]) result.chunks[0].endLine = 0; + return result; + }); + await expect( + new CodebaseIndexer({ + rootPath: root, + config: { skipEmbedding: true }, + projectOptions: { preferredAnalyzer: 'generic' } + }).index() + ).rejects.toBeInstanceOf(InvalidSourceChunkError); + expect(await fs.readFile(pointer)).toEqual(before); +}); + +it('keeps ordinary per-file parse failures visible without making them producer-integrity failures', async () => { + root = await fs.mkdtemp(path.join(os.tmpdir(), 'cbc-source-parse-')); + analyzerRegistry.register(new GenericAnalyzer()); + await fs.writeFile(path.join(root, 'good.ts'), 'export const good = 1;'); + await fs.writeFile(path.join(root, 'bad.ts'), 'invalid source'); + const analyze = analyzerRegistry.analyzeFile.bind(analyzerRegistry); + vi.spyOn(analyzerRegistry, 'analyzeFile').mockImplementation(async (...args) => { + if (path.basename(args[0]) === 'bad.ts') + throw new SyntaxError('ordinary analyzer parse failure'); + return analyze(...args); + }); + const stats = await new CodebaseIndexer({ + rootPath: root, + config: { skipEmbedding: true }, + projectOptions: { preferredAnalyzer: 'generic' } + }).index(); + expect(stats.errors).toHaveLength(1); + expect(stats.errors[0].error).toBe('ordinary analyzer parse failure'); + const index: { chunks: CodeChunk[] } = JSON.parse( + await fs.readFile(await resolveIndexArtifact(root, 'index.json'), 'utf8') + ); + expect(index.chunks.some((item) => item.relativePath === 'good.ts')).toBe(true); + expect(index.chunks.some((item) => item.relativePath === 'bad.ts')).toBe(false); +}); diff --git a/tests/cli-entrypoint-runtime.test.ts b/tests/cli-entrypoint-runtime.test.ts index 7bdb291..3413363 100644 --- a/tests/cli-entrypoint-runtime.test.ts +++ b/tests/cli-entrypoint-runtime.test.ts @@ -1,6 +1,13 @@ import { spawnSync } from 'node:child_process'; -import { resolve } from 'node:path'; +import { mkdir, mkdtemp, readFile, readdir, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { mkdtempSync, rmSync, symlinkSync, writeFileSync } from 'node:fs'; +import { pathToFileURL } from 'node:url'; import { describe, expect, it } from 'vitest'; +import { INDEX_FORMAT_VERSION, KEYWORD_INDEX_FILENAME } from '../src/constants/codebase-context.js'; +import { IndexMetaSchema, validateKeywordIndexArtifact } from '../src/core/index-meta.js'; +import { resolveIndexArtifact } from '../src/core/index-generation.js'; const root = resolve(import.meta.dirname, '..'); const entrypoint = resolve(root, 'src', 'index.ts'); @@ -12,29 +19,67 @@ type MapJson = { }; describe('CLI entrypoint runtime', () => { - function runCli(args: string[]) { - return spawnSync(process.execPath, ['--import', 'tsx', entrypoint, ...args], { - cwd: root, - env: { - ...process.env, - CODEBASE_ROOT: root - }, - encoding: 'utf8', - timeout: 120_000 - }); + function runCli(args: string[], projectRoot = root) { + return spawnSync( + process.execPath, + ['--import', import.meta.resolve('tsx'), entrypoint, ...args], + { + cwd: projectRoot, + env: { + ...process.env, + CODEBASE_ROOT: projectRoot + }, + encoding: 'utf8', + timeout: 120_000 + } + ); } - it('dispatches map without loading MCP server runtime on the CLI path', () => { - const result = runCli(['map', '--json']); + it('dispatches map without loading MCP server runtime on the CLI path', async () => { + const projectRoot = await mkdtemp(join(tmpdir(), 'cbc-cli-map-runtime-')); + try { + await writeFile(join(projectRoot, 'example.ts'), 'export const answer = 42;\n'); + const result = runCli(['map', '--json'], projectRoot); - expect(result.status).toBe(0); - expect(result.stderr).not.toContain('ERR_MODULE_NOT_FOUND'); - expect(result.stderr).not.toContain('@modelcontextprotocol/sdk/server/stdio.js'); + expect(result.status, result.stderr).toBe(0); + expect(result.stderr).not.toContain('ERR_MODULE_NOT_FOUND'); + expect(result.stderr).not.toContain('@modelcontextprotocol/sdk/server/stdio.js'); + + const parsed = JSON.parse(result.stdout) as MapJson; + expect(typeof parsed.project).toBe('string'); + expect(parsed.architecture).toBeTruthy(); + expect(Array.isArray(parsed.activePatterns)).toBe(true); + expect(parsed).toMatchObject({ graphStats: { files: 1 } }); + + const metadataPath = join(projectRoot, '.codebase-context', 'index-meta.json'); + const metadataBytes = await readFile(metadataPath, 'utf8'); + const metadata = IndexMetaSchema.parse(JSON.parse(metadataBytes)); + expect(metadata.formatVersion).toBe(INDEX_FORMAT_VERSION); + const keywordPath = await resolveIndexArtifact(projectRoot, KEYWORD_INDEX_FILENAME); + expect(keywordPath).toBe( + join(projectRoot, '.codebase-context', metadata.artifacts.keywordIndex.path) + ); + const keyword = validateKeywordIndexArtifact( + JSON.parse(await readFile(keywordPath, 'utf8')), + metadata + ); + expect(keyword.header.buildId).toBe(metadata.buildId); + expect(keyword.chunks).toContainEqual( + expect.objectContaining({ + relativePath: 'example.ts', + content: expect.stringContaining('export const answer = 42;') + }) + ); - const parsed = JSON.parse(result.stdout) as MapJson; - expect(typeof parsed.project).toBe('string'); - expect(parsed.architecture).toBeTruthy(); - expect(Array.isArray(parsed.activePatterns)).toBe(true); + const repeated = runCli(['map', '--json'], projectRoot); + expect(repeated.status, repeated.stderr).toBe(0); + expect(repeated.stderr).not.toContain('ERR_MODULE_NOT_FOUND'); + expect(repeated.stderr).not.toContain('@modelcontextprotocol/sdk/server/stdio.js'); + expect(JSON.parse(repeated.stdout)).toEqual(parsed); + expect(await readFile(metadataPath, 'utf8')).toBe(metadataBytes); + } finally { + await rm(projectRoot, { recursive: true, force: true }); + } }, 120_000); it('dispatches status without loading MCP server runtime on the CLI path', () => { @@ -54,4 +99,111 @@ describe('CLI entrypoint runtime', () => { expect(parsed.rootPath).toBe(root); expect(parsed.hint).toContain('refresh_index'); }, 120_000); + + it('prints usage for help from a project containing a directory named help', async () => { + const projectRoot = await mkdtemp(join(tmpdir(), 'cbc-cli-help-')); + const helpDirectory = join(projectRoot, 'help'); + + const env = { ...process.env }; + delete env.CODEBASE_ROOT; + + try { + await mkdir(helpDirectory); + + const result = spawnSync( + process.execPath, + ['--import', import.meta.resolve('tsx'), entrypoint, 'help'], + { + cwd: projectRoot, + env, + encoding: 'utf8', + timeout: 10_000 + } + ); + + expect({ + errorCode: result.error?.code, + exitCode: result.status, + helpText: result.stdout.includes('codebase-context [options]'), + commandsSection: result.stdout.includes('Commands:'), + stderr: result.stderr, + projectEntries: await readdir(projectRoot), + helpEntries: await readdir(helpDirectory) + }).toEqual({ + errorCode: undefined, + exitCode: 0, + helpText: true, + commandsSection: true, + stderr: '', + projectEntries: ['help'], + helpEntries: [] + }); + } finally { + await rm(projectRoot, { recursive: true, force: true }); + } + }, 70_000); + it('dispatches CLI help when invoked through a symlink entrypoint', (context) => { + const tempDirectory = mkdtempSync(join(tmpdir(), 'cbc-cli-entrypoint-')); + const linkedEntrypoint = join(tempDirectory, 'codebase-context.ts'); + + try { + symlinkSync(entrypoint, linkedEntrypoint, 'file'); + } catch (error) { + const code = + typeof error === 'object' && + error !== null && + 'code' in error && + typeof error.code === 'string' + ? error.code + : undefined; + rmSync(tempDirectory, { recursive: true, force: true }); + if (process.platform === 'win32' && ['EPERM', 'EACCES', 'ENOSYS'].includes(code ?? '')) { + context.skip(); + return; + } + throw error; + } + + try { + const result = spawnSync(process.execPath, ['--import', 'tsx', linkedEntrypoint, '--help'], { + cwd: root, + env: { ...process.env, CODEBASE_ROOT: root }, + encoding: 'utf8', + timeout: 120_000 + }); + + expect(result.status).toBe(0); + expect(result.stdout).toContain('codebase-context [options]'); + expect(result.stdout).toContain('init [--client ]'); + expect(result.stderr).not.toContain('ERR_MODULE_NOT_FOUND'); + } finally { + rmSync(tempDirectory, { recursive: true, force: true }); + } + }, 120_000); + + it('does not run the server or CLI when imported by an unrelated index.js wrapper', () => { + const tempDirectory = mkdtempSync(join(tmpdir(), 'cbc-cli-import-')); + const wrapper = join(tempDirectory, 'index.js'); + const importUrl = pathToFileURL(entrypoint).href; + writeFileSync( + wrapper, + `process.argv[2] = '--help'; import(${JSON.stringify(importUrl)}).then(() => console.log('IMPORT_COMPLETE'), (error) => { console.error(error); process.exitCode = 1; });`, + 'utf8' + ); + + try { + const result = spawnSync(process.execPath, ['--import', 'tsx', wrapper], { + cwd: root, + env: { ...process.env, CODEBASE_ROOT: root }, + encoding: 'utf8', + timeout: 120_000 + }); + + expect(result.status).toBe(0); + expect(result.stdout.trim()).toBe('IMPORT_COMPLETE'); + expect(result.stderr).toBe(''); + } finally { + rmSync(tempDirectory, { recursive: true, force: true }); + } + }, 120_000); }); diff --git a/tests/cli-init-atomic.test.ts b/tests/cli-init-atomic.test.ts new file mode 100644 index 0000000..b0d5f25 --- /dev/null +++ b/tests/cli-init-atomic.test.ts @@ -0,0 +1,71 @@ +import { promises as fs } from 'node:fs'; +import path from 'node:path'; +import os from 'node:os'; +import { afterEach, describe, expect, it, vi } from 'vitest'; + +const fault = vi.hoisted(() => ({ enabled: false, rename: false })); +vi.mock('node:fs', async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + promises: { + ...actual.promises, + rename: vi.fn(async (...args: Parameters) => { + if (fault.rename) + throw Object.assign(new Error('simulated rename failure'), { code: 'EACCES' }); + return actual.promises.rename(...args); + }), + writeFile: vi.fn(async (...args: Parameters) => { + if (fault.enabled) { + await actual.promises.writeFile(args[0], 'PARTIAL WRITE'); + throw Object.assign(new Error('simulated disk full'), { code: 'ENOSPC' }); + } + return actual.promises.writeFile(...args); + }) + } + }; +}); +import { handleInitCli } from '../src/cli-init.js'; + +describe('init interrupted writes', () => { + let root: string; + const previousExitCode = process.exitCode; + afterEach(async () => { + fault.enabled = false; + fault.rename = false; + process.exitCode = previousExitCode; + if (root) await fs.rm(root, { recursive: true, force: true }); + }); + it('preserves the original config when publication fails after the temporary file is complete', async () => { + root = await fs.mkdtemp(path.join(os.tmpdir(), 'cbc-init-rename-')); + const file = path.join(root, '.mcp.json'); + const original = '{ "customSetting": "keep" }\n'; + await fs.writeFile(file, original); + fault.rename = true; + await handleInitCli(['--client', 'claude-code', '--yes', '--root', root]); + expect(process.exitCode).toBe(1); + expect(await fs.readFile(file, 'utf8')).toBe(original); + expect(await fs.readdir(root)).toEqual(['.mcp.json']); + }); + it.each(['claude-code', 'codex'])( + 'preserves the entire existing %s config on a partial write failure', + async (client) => { + root = await fs.mkdtemp(path.join(os.tmpdir(), 'cbc-init-write-')); + const file = + client === 'codex' + ? path.join(root, '.codex', 'config.toml') + : path.join(root, '.mcp.json'); + const original = + client === 'codex' + ? '# original instructions\nmodel = "keep"\n' + : '{ "customSetting": "keep" }\n'; + await fs.mkdir(path.dirname(file), { recursive: true }); + await fs.writeFile(file, original); + fault.enabled = true; + await handleInitCli(['--client', client, '--yes', '--root', root]); + expect(process.exitCode).toBe(1); + expect(await fs.readFile(file, 'utf8')).toBe(original); + expect(await fs.readdir(path.dirname(file))).toEqual([path.basename(file)]); + } + ); +}); diff --git a/tests/cli-init-bootstrap.test.ts b/tests/cli-init-bootstrap.test.ts new file mode 100644 index 0000000..6134d67 --- /dev/null +++ b/tests/cli-init-bootstrap.test.ts @@ -0,0 +1,213 @@ +import path from 'node:path'; +import * as fs from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { describe, expect, it } from 'vitest'; +import { + detectClients, + generateMcpConfig, + handleInitCli, + _buildMergedMcpContent, + _buildMergedCodexContent +} from '../src/cli-init.js'; +import { resolveRootPath } from '../src/index.js'; +import { parse as parseToml } from 'smol-toml'; + +describe('init bootstrap root', () => { + it('passes the configured project root through the MCP server argv path', () => { + const projectRoot = path.resolve('C:/projects/An App With Spaces'); + const previousArgv = process.argv; + const previousEnvRoot = process.env.CODEBASE_ROOT; + const generated = generateMcpConfig('claude-code', { + rootPath: projectRoot, + nodeExecutable: process.execPath, + serverEntrypoint: path.resolve('C:/codebase-context/dist/index.js') + }); + if (generated.kind !== 'file') throw new Error('expected project MCP config'); + + const parsed = JSON.parse(generated.content) as { + mcpServers: { 'codebase-context': { command: string; args: string[] } }; + }; + const server = parsed.mcpServers['codebase-context']; + expect(server.command).toBe(path.resolve(process.execPath)); + expect(server.args).toEqual([path.resolve('C:/codebase-context/dist/index.js'), projectRoot]); + process.argv = [server.command, ...server.args]; + process.env.CODEBASE_ROOT = path.resolve('C:/projects/Other App'); + + try { + expect(resolveRootPath()).toBe(projectRoot); + } finally { + process.argv = previousArgv; + if (previousEnvRoot === undefined) delete process.env.CODEBASE_ROOT; + else process.env.CODEBASE_ROOT = previousEnvRoot; + } + }); +}); + +describe('init preserves custom launchers', () => { + it('updates the real launch while preserving an earlier apparent launch inside instructions', async () => { + const root = await fs.mkdtemp(path.join(tmpdir(), 'cbc-codex-real-table-')); + const entrypoint = path.join(root, 'package', 'dist', 'index.js'); + const generated = generateMcpConfig('codex', { rootPath: root, serverEntrypoint: entrypoint }); + const instructions = + 'developer_instructions = """\n[mcp_servers.codebase-context]\ncommand = "npx"\nargs = ["--yes", "codebase-context@2.2.0", "C:/fake"]\n"""\n'; + const original = + instructions + + '\n[mcp_servers.codebase-context]\ncommand = "npx" # real\nargs = ["--yes", "codebase-context@2.2.0", "C:/old"]\nstartup_timeout_sec = 75\n'; + await fs.mkdir(path.dirname(generated.path), { recursive: true }); + await fs.writeFile(generated.path, original); + try { + const merged = await _buildMergedCodexContent(generated.path, generated.content); + expect(merged.content.startsWith(instructions)).toBe(true); + const parsed = parseToml(merged.content); + expect(parsed.mcp_servers).toEqual({ + 'codebase-context': { + command: process.execPath, + args: [entrypoint, root], + startup_timeout_sec: 75 + } + }); + expect(parsed.developer_instructions).toBe(parseToml(original).developer_instructions); + expect(merged.content).toContain('# real'); + } finally { + await fs.rm(root, { recursive: true, force: true }); + } + }); + + it.each([ + ['basic multiline', '"""', 'escaped quote: \\" and escaped delimiter: \\"""\n'], + ['literal multiline', "'''", 'literal quotes: " and \\n\n'] + ])( + 'preserves apparent tables inside a %s instruction string', + async (_name, delimiter, prefix) => { + const root = await fs.mkdtemp(path.join(tmpdir(), 'cbc-codex-multiline-')); + const generated = generateMcpConfig('codex', { rootPath: root }); + const original = `developer_instructions = ${delimiter}\n${prefix}[mcp_servers.codebase-context]\ncommand = "npx"\nargs = ["--yes", "codebase-context@2.2.0", "C:/project"]\n${delimiter}\n`; + await fs.mkdir(path.dirname(generated.path), { recursive: true }); + await fs.writeFile(generated.path, original); + try { + const merged = await _buildMergedCodexContent(generated.path, generated.content); + expect(merged.content.startsWith(original)).toBe(true); + expect(merged.content.slice(original.length)).toContain(generated.content.trim()); + const parsed = parseToml(merged.content); + expect(parsed.developer_instructions).toBe(parseToml(original).developer_instructions); + expect(parsed.mcp_servers).toEqual(parseToml(generated.content).mcp_servers); + expect(await fs.readFile(generated.path, 'utf8')).toBe(original); + } finally { + await fs.rm(root, { recursive: true, force: true }); + } + } + ); + + it('distinguishes executable files from directories while detecting tools', async () => { + const root = await fs.mkdtemp(path.join(tmpdir(), 'cbc-detect-files-')); + const bin = path.join(root, 'bin'); + const home = path.join(root, 'home'); + await fs.mkdir(path.join(bin, 'claude'), { recursive: true }); + await fs.writeFile(path.join(root, '.codex'), 'This is a file, not a tool directory.'); + const env = { + PATH: bin, + HOME: home, + USERPROFILE: home, + XDG_CONFIG_HOME: path.join(home, '.config') + }; + try { + expect(await detectClients(root, env)).toEqual([]); + const executable = path.join(bin, process.platform === 'win32' ? 'codex.cmd' : 'codex'); + await fs.writeFile(executable, 'Detection must never execute this file.'); + await fs.chmod(executable, 0o755); + expect(await detectClients(root, env)).toEqual(['codex']); + } finally { + await fs.rm(root, { recursive: true, force: true }); + } + }); + + it('leaves a matching Codex launch and extra settings byte-for-byte unchanged', async () => { + const root = await fs.mkdtemp(path.join(tmpdir(), 'cbc-codex-format-')); + const generated = generateMcpConfig('codex', { rootPath: root }); + const original = ( + 'model = "example"\n\n' + + generated.content + + 'custom_label = "keep"\n' + ).replace(/\n/g, '\r\n'); + await fs.mkdir(path.dirname(generated.path), { recursive: true }); + await fs.writeFile(generated.path, original); + try { + await handleInitCli(['--client', 'codex', '--yes', '--root', root]); + expect(await fs.readFile(generated.path, 'utf8')).toBe(original); + } finally { + await fs.rm(root, { recursive: true, force: true }); + } + }); + + it.each([ + ['claude-code', 'other package'], + ['codex', 'other package'], + ['claude-code', 'custom npx'], + ['codex', 'custom npx'] + ] as const)('preserves %s config with a %s launcher', async (client, kind) => { + const root = await fs.mkdtemp(path.join(tmpdir(), 'cbc-custom-launch-')); + const generated = generateMcpConfig(client, { rootPath: root }); + const launcher = path.join(root, 'node_modules', 'other-package', 'dist', 'index.js'); + const command = kind === 'custom npx' ? path.join(root, 'custom', 'npx.cmd') : 'node'; + const args = + kind === 'custom npx' ? ['--yes', 'codebase-context@2.3.0', root] : [launcher, root]; + const original = + client === 'codex' + ? `[mcp_servers.codebase-context]\ncommand = ${JSON.stringify(command)}\nargs = ${JSON.stringify(args)}\nstartup_timeout_sec = 75\n` + : JSON.stringify({ + mcpServers: { + 'codebase-context': { + command, + args, + env: { KEEP: 'yes' } + } + }, + customSetting: true + }); + await fs.mkdir(path.dirname(generated.path), { recursive: true }); + await fs.writeFile(generated.path, original); + try { + const merge = + client === 'codex' + ? _buildMergedCodexContent(generated.path, generated.content) + : _buildMergedMcpContent(generated.path, generated.content, client); + await expect(merge).rejects.toThrow('different codebase-context'); + expect(await fs.readFile(generated.path, 'utf8')).toBe(original); + } finally { + await fs.rm(root, { recursive: true, force: true }); + } + }); + + it('writes no selected configs when a later client uses a custom launcher', async () => { + const root = await fs.mkdtemp(path.join(tmpdir(), 'cbc-batch-launch-')); + const claude = path.join(root, '.mcp.json'); + const codex = path.join(root, '.codex', 'config.toml'); + const originalClaude = JSON.stringify({ + mcpServers: { other: { command: 'custom' } }, + customSetting: true + }); + const originalCodex = `[mcp_servers.codebase-context]\ncommand = "node"\nargs = [${JSON.stringify(path.join(root, 'other-package', 'dist', 'index.js'))}, ${JSON.stringify(root)}]\n`; + await fs.mkdir(path.dirname(codex), { recursive: true }); + await fs.writeFile(claude, originalClaude); + await fs.writeFile(codex, originalCodex); + const previousExitCode = process.exitCode; + try { + process.exitCode = undefined; + await handleInitCli([ + '--client', + 'claude-code', + '--client', + 'codex', + '--yes', + '--root', + root + ]); + expect(process.exitCode).toBe(1); + expect(await fs.readFile(claude, 'utf8')).toBe(originalClaude); + expect(await fs.readFile(codex, 'utf8')).toBe(originalCodex); + } finally { + process.exitCode = previousExitCode; + await fs.rm(root, { recursive: true, force: true }); + } + }); +}); diff --git a/tests/cli-init-saved-file.test.ts b/tests/cli-init-saved-file.test.ts new file mode 100644 index 0000000..a90beaf --- /dev/null +++ b/tests/cli-init-saved-file.test.ts @@ -0,0 +1,59 @@ +import { mkdtemp, mkdir, readFile, rm, writeFile } from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { describe, expect, it, vi } from 'vitest'; +import { handleInitCli } from '../src/cli-init.js'; + +describe('Codex init with an existing saved config', () => { + it.each([ + { + name: 'bare dotted keys', + toml: 'mcp_servers.codebase-context = { command = "old", args = [] }\n' + }, + { + name: 'a double-quoted root key', + toml: '"mcp_servers".codebase-context = { command = "old", args = [] }\n' + }, + { + name: 'a double-quoted server key', + toml: 'mcp_servers."codebase-context" = { command = "old", args = [] }\n' + }, + { + name: 'double-quoted root and server keys', + toml: '"mcp_servers"."codebase-context" = { command = "old", args = [] }\n' + }, + { + name: 'single-quoted root and server keys', + toml: "'mcp_servers'.'codebase-context' = { command = \"old\", args = [] }\n" + } + ])('refuses $name without changing the saved file', async ({ toml }) => { + const projectRoot = await mkdtemp(path.join(os.tmpdir(), 'cbc-codex-init-')); + const configPath = path.join(projectRoot, '.codex', 'config.toml'); + const initialBytes = Buffer.from(toml, 'utf8'); + const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); + const logSpy = vi.spyOn(console, 'log').mockImplementation(() => {}); + const previousExitCode = process.exitCode; + + try { + await mkdir(path.dirname(configPath), { recursive: true }); + await writeFile(configPath, initialBytes); + process.exitCode = undefined; + + await handleInitCli(['--client', 'codex', '--yes', '--root', projectRoot]); + + const persistedBytes = await readFile(configPath); + expect({ exitCode: process.exitCode, persistedBytes }).toEqual({ + exitCode: 1, + persistedBytes: initialBytes + }); + expect(errorSpy).toHaveBeenCalledWith( + expect.stringContaining('already contains a different codebase-context configuration') + ); + } finally { + process.exitCode = previousExitCode; + errorSpy.mockRestore(); + logSpy.mockRestore(); + await rm(projectRoot, { recursive: true, force: true }); + } + }); +}); diff --git a/tests/cli-init-windows.test.ts b/tests/cli-init-windows.test.ts new file mode 100644 index 0000000..7abf3bd --- /dev/null +++ b/tests/cli-init-windows.test.ts @@ -0,0 +1,67 @@ +import path from 'node:path'; +import { describe, expect, it } from 'vitest'; +import { generateMcpConfig } from '../src/cli-init.js'; +import { mkdtemp, writeFile, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { StdioClientTransport } from '@modelcontextprotocol/sdk/client/stdio.js'; + +describe('saved launch arguments', () => { + it.each(['An App', 'App&MissingCbcLaunchCommand', 'App%cbc_literal%End'])( + 'keeps the repository path %s as one argument beside the npx cache path', + (name) => { + const root = path.resolve('C:/projects', name); + const nodeExecutable = path.resolve('C:/Program Files/nodejs/node.exe'); + const cacheEntrypoint = path.resolve( + 'C:/Users/dev/AppData/Local/npm-cache/_npx/hash/node_modules/codebase-context/dist/index.js' + ); + const generated = generateMcpConfig('claude-code', { + rootPath: root, + nodeExecutable, + serverEntrypoint: cacheEntrypoint + }); + if (generated.kind !== 'file') throw new Error('expected file config'); + const server = JSON.parse(generated.content).mcpServers['codebase-context'] as { + command: string; + args: string[]; + }; + + expect(server.command).toBe(nodeExecutable); + expect(server.args).toEqual([cacheEntrypoint, root]); + } + ); +}); + +describe('direct Node client launch', () => { + it.each(['An App', 'App&MissingCbcLaunchCommand', 'App%cbc_literal%End'])( + 'passes %s literally through the real SDK transport', + async (name) => { + const directory = await mkdtemp(path.join(tmpdir(), 'cbc-direct-launch-')); + const root = path.join(directory, name); + const entrypoint = path.join(directory, 'capture.cjs'); + await writeFile( + entrypoint, + 'console.log(JSON.stringify({jsonrpc:"2.0",method:"captured",params:{args:process.argv.slice(2)}}));' + ); + const generated = generateMcpConfig('claude-code', { + rootPath: root, + nodeExecutable: process.execPath, + serverEntrypoint: entrypoint + }); + const server: { command: string; args: string[] } = JSON.parse(generated.content).mcpServers[ + 'codebase-context' + ]; + const transport = new StdioClientTransport({ ...server, stderr: 'pipe' }); + const result = new Promise((resolve, reject) => { + transport.onmessage = resolve; + transport.onerror = reject; + }); + try { + await transport.start(); + expect(await result).toMatchObject({ params: { args: [root] } }); + } finally { + await transport.close(); + await rm(directory, { recursive: true, force: true }); + } + } + ); +}); diff --git a/tests/cli-init.test.ts b/tests/cli-init.test.ts index 1dc718e..bfff804 100644 --- a/tests/cli-init.test.ts +++ b/tests/cli-init.test.ts @@ -1,155 +1,150 @@ -import { describe, it, expect, vi, beforeEach } from 'vitest'; +import { describe, it, expect, vi, beforeEach, afterAll } from 'vitest'; import path from 'path'; // Mock node:fs/promises at the top level so Vitest can hoist it correctly. -vi.mock('node:fs/promises', async (importOriginal) => { - const actual = await importOriginal(); +vi.mock('node:fs', async (importOriginal) => { + const actual = await importOriginal(); return { ...actual, - access: vi.fn(), - readFile: vi.fn(), - writeFile: vi.fn().mockResolvedValue(undefined), - mkdir: vi.fn().mockResolvedValue(undefined) + promises: { + ...actual.promises, + access: vi.fn(), + stat: vi.fn(), + readFile: vi.fn(), + writeFile: vi.fn().mockResolvedValue(undefined), + mkdir: vi.fn().mockResolvedValue(undefined), + rename: vi.fn().mockResolvedValue(undefined), + rm: vi.fn().mockResolvedValue(undefined) + } }; }); -import * as fsMod from 'node:fs/promises'; +vi.mock('@inquirer/prompts', () => ({ + checkbox: vi.fn() +})); + +import { promises as fsMod } from 'node:fs'; +import { checkbox } from '@inquirer/prompts'; import { + CLIENTS, generateMcpConfig, - generateInstructionBlock, - resolveInstructionFilePath, - _appendInstructionBlock, - _buildMergedMcpContent + detectClients, + resolveInitRoot, + _buildMergedMcpContent, + _buildMergedCodexContent, + handleInitCli } from '../src/cli-init.js'; // --- generateMcpConfig --- +const nodeExecutable = path.resolve('C:/Program Files/nodejs/node.exe'); +const cacheEntrypoint = path.resolve( + 'C:/npm-cache/_npx/hash/node_modules/codebase-context/dist/index.js' +); + +function savedLaunch(rootPath: string) { + return { + rootPath, + nodeExecutable, + serverEntrypoint: cacheEntrypoint + }; +} + describe('generateMcpConfig', () => { - it('claude-code returns a command with mcp add --transport http', () => { - const result = generateMcpConfig('claude-code'); - expect(result.kind).toBe('command'); - if (result.kind !== 'command') return; - expect(result.args[0]).toBe('mcp'); - expect(result.args).toContain('--transport'); - expect(result.args).toContain('http'); - expect(result.args).toContain('codebase-context'); - expect(result.args).toContain('http://127.0.0.1:3100/mcp'); - }); - - it('cursor returns a file at .cursor/mcp.json with http type', () => { - const result = generateMcpConfig('cursor'); + const projectRoot = path.resolve('C:/projects/Example Project'); + + it('saves the npx cache path for Claude Code', () => { + const result = generateMcpConfig('claude-code', savedLaunch(projectRoot)); + expect(result.kind).toBe('file'); if (result.kind !== 'file') return; - expect(result.path).toBe('.cursor/mcp.json'); - const parsed = JSON.parse(result.content) as { - mcpServers: { 'codebase-context': { type: string } }; - }; - expect(parsed.mcpServers['codebase-context'].type).toBe('http'); - }); - - it('codex returns a command with mcp add', () => { - const result = generateMcpConfig('codex'); - expect(result.kind).toBe('command'); - if (result.kind !== 'command') return; - expect(result.args).toContain('mcp'); - expect(result.args).toContain('add'); - expect(result.args).toContain('codebase-context'); - expect(result.args).toContain('http://127.0.0.1:3100/mcp'); + expect(result.path).toBe(path.join(projectRoot, '.mcp.json')); + expect(JSON.parse(result.content)).toEqual({ + mcpServers: { + 'codebase-context': { + command: nodeExecutable, + args: [cacheEntrypoint, projectRoot] + } + } + }); }); - it('opencode returns a file at opencode.json with remote type', () => { - const result = generateMcpConfig('opencode'); + it('Cursor gets a project stdio config that saves the npx cache path', () => { + const result = generateMcpConfig('cursor', savedLaunch(projectRoot)); expect(result.kind).toBe('file'); if (result.kind !== 'file') return; - expect(result.path).toBe('opencode.json'); + expect(result.path).toBe(path.join(projectRoot, '.cursor', 'mcp.json')); const parsed = JSON.parse(result.content) as { - mcp: { 'codebase-context': { type: string } }; + mcpServers: { 'codebase-context': { type: string; command: string; args: string[] } }; }; - expect(parsed.mcp['codebase-context'].type).toBe('remote'); - }); -}); - -// --- generateInstructionBlock --- - -describe('generateInstructionBlock', () => { - it('contains opening and closing delimiters', () => { - const block = generateInstructionBlock(); - expect(block).toContain(''); - expect(block).toContain(''); - }); - - it('contains all five tool call rules', () => { - const block = generateInstructionBlock(); - expect(block).toContain('get_memory'); - expect(block).toContain('search_codebase'); - expect(block).toContain('get_team_patterns'); - expect(block).toContain('remember'); - expect(block).toContain('detect_circular_dependencies'); + expect(parsed.mcpServers['codebase-context']).toEqual({ + type: 'stdio', + command: nodeExecutable, + args: [cacheEntrypoint, projectRoot] + }); }); -}); - -// --- resolveInstructionFilePath --- -describe('resolveInstructionFilePath', () => { - const cwd = '/test-cwd'; - - it('claude-code resolves to CLAUDE.md', () => { - const result = resolveInstructionFilePath('claude-code', cwd); - expect(result).toBe(path.join(cwd, 'CLAUDE.md')); + it('defaults to the current Node binary and the running package file', () => { + const result = generateMcpConfig('claude-code', { rootPath: projectRoot }); + if (result.kind !== 'file') throw new Error('expected file config'); + const parsed = JSON.parse(result.content) as { + mcpServers: { 'codebase-context': { command: string; args: string[] } }; + }; + const server = parsed.mcpServers['codebase-context']; + expect(server.command).toBe(path.resolve(process.execPath)); + expect(server.args?.[0]).toBe(path.resolve(process.argv[1] ?? '')); + expect(server.args?.[1]).toBe(projectRoot); }); - it('cursor resolves to .cursorrules', () => { - const result = resolveInstructionFilePath('cursor', cwd); - expect(result).toBe(path.join(cwd, '.cursorrules')); - }); + it('Codex saves the node binary and the npx cache path', () => { + const result = generateMcpConfig('codex', savedLaunch(projectRoot)); - it('codex resolves to AGENTS.md', () => { - const result = resolveInstructionFilePath('codex', cwd); - expect(result).toBe(path.join(cwd, 'AGENTS.md')); + expect(result.kind).toBe('file'); + if (result.kind !== 'file') return; + expect(result.path).toBe(path.join(projectRoot, '.codex', 'config.toml')); + expect(result.content).toBe( + `[mcp_servers.codebase-context]\ncommand = ${JSON.stringify(nodeExecutable)}\nargs = [${JSON.stringify(cacheEntrypoint)}, ${JSON.stringify(projectRoot)}]\nstartup_timeout_sec = 120\n` + ); }); - it('opencode returns null', () => { - const result = resolveInstructionFilePath('opencode', cwd); - expect(result).toBeNull(); + it('OpenCode uses a local command that saves the npx cache path', () => { + const result = generateMcpConfig('opencode', savedLaunch(projectRoot)); + expect(result.kind).toBe('file'); + if (result.kind !== 'file') return; + expect(result.path).toBe(path.join(projectRoot, 'opencode.json')); + const parsed = JSON.parse(result.content) as { + mcp: { 'codebase-context': { type: string; command: string[] } }; + }; + expect(parsed.mcp['codebase-context']).toEqual({ + type: 'local', + command: [nodeExecutable, cacheEntrypoint, projectRoot] + }); }); }); -// --- _appendInstructionBlock file-write behavior --- - -describe('_appendInstructionBlock', () => { - const accessMock = vi.mocked(fsMod.access); - const readFileMock = vi.mocked(fsMod.readFile); - const writeFileMock = vi.mocked(fsMod.writeFile); - - beforeEach(() => { - vi.resetAllMocks(); - writeFileMock.mockResolvedValue(undefined); +describe('resolveInitRoot', () => { + it('uses explicit root before CODEBASE_ROOT and cwd', () => { + expect( + resolveInitRoot(['--root', 'C:/projects/Explicit Project'], 'C:/projects/Env', 'C:/other') + ).toBe(path.resolve('C:/projects/Explicit Project')); }); - it('appends block when file exists and block not yet present', async () => { - // Simulate file exists (access resolves) and has content without the delimiter - accessMock.mockResolvedValue(undefined); - readFileMock.mockResolvedValue('# existing content' as unknown as Buffer); - - await _appendInstructionBlock('/test/CLAUDE.md'); - - expect(writeFileMock).toHaveBeenCalledOnce(); - const callArgs = writeFileMock.mock.calls[0] as [string, string, ...unknown[]]; - const content = callArgs[1]; - expect(content).toContain('# existing content'); - expect(content).toContain(''); + it('uses CODEBASE_ROOT when no explicit root is supplied', () => { + expect(resolveInitRoot([], 'C:/projects/Env Project', 'C:/other')).toBe( + path.resolve('C:/projects/Env Project') + ); }); - it('skips when block already present', async () => { - accessMock.mockResolvedValue(undefined); - readFileMock.mockResolvedValue( - '# existing content\n\nsome block\n' as unknown as Buffer + it('uses cwd when neither root override is supplied', () => { + expect(resolveInitRoot([], undefined, 'C:/projects/Current Project')).toBe( + path.resolve('C:/projects/Current Project') ); + }); - const result = await _appendInstructionBlock('/test/CLAUDE.md'); - - expect(result).toBe('skipped'); - expect(writeFileMock).not.toHaveBeenCalled(); + it('treats an empty CODEBASE_ROOT as unset', () => { + expect(resolveInitRoot([], '', 'C:/projects/Current Project')).toBe( + path.resolve('C:/projects/Current Project') + ); }); }); @@ -170,12 +165,16 @@ describe('_buildMergedMcpContent', () => { }) as unknown as Buffer ); - const generated = generateMcpConfig('cursor'); + const generated = generateMcpConfig('cursor', { platform: 'linux' }); if (generated.kind !== 'file') throw new Error('expected file config'); - const merged = await _buildMergedMcpContent('/test/.cursor/mcp.json', generated.content, 'cursor'); + const merged = await _buildMergedMcpContent( + '/test/.cursor/mcp.json', + generated.content, + 'cursor' + ); const parsed = JSON.parse(merged.content) as { - mcpServers: Record; + mcpServers: Record; someOtherKey: boolean; }; @@ -185,13 +184,76 @@ describe('_buildMergedMcpContent', () => { type: 'http', url: 'http://127.0.0.1:4000/mcp' }); + expect(parsed.mcpServers['codebase-context']).toMatchObject({ + type: 'stdio', + command: path.resolve(process.execPath) + }); + }); + + it('preserves existing Claude Code mcpServers entries and adds codebase-context', async () => { + readFileMock.mockResolvedValue( + JSON.stringify({ + mcpServers: { + 'existing-server': { type: 'stdio', command: 'node', args: ['server.js'] } + }, + someOtherKey: true + }) as unknown as Buffer + ); + + const generated = generateMcpConfig( + 'claude-code', + savedLaunch(path.resolve('C:/projects/app')) + ); + if (generated.kind !== 'file') throw new Error('expected file config'); + + const merged = await _buildMergedMcpContent( + '/test/.mcp.json', + generated.content, + 'claude-code' + ); + const parsed = JSON.parse(merged.content) as { + mcpServers: Record; + someOtherKey: boolean; + }; + + expect(merged.mergedFromExisting).toBe(true); + expect(parsed.someOtherKey).toBe(true); + expect(parsed.mcpServers['existing-server']).toEqual({ + type: 'stdio', + command: 'node', + args: ['server.js'] + }); expect(parsed.mcpServers['codebase-context']).toEqual({ - type: 'http', - url: 'http://127.0.0.1:3100/mcp' + command: nodeExecutable, + args: [cacheEntrypoint, path.resolve('C:/projects/app')] + }); + }); + + it('refuses to overwrite a disabled Claude Code codebase-context entry', async () => { + readFileMock.mockResolvedValue( + JSON.stringify({ + mcpServers: { + 'codebase-context': { + type: 'stdio', + command: 'node', + args: ['server.js'], + disabled: true + } + } + }) as unknown as Buffer + ); + const generated = generateMcpConfig('claude-code', { + rootPath: path.resolve('C:/projects/app'), + platform: 'win32' }); + if (generated.kind !== 'file') throw new Error('expected file config'); + + await expect( + _buildMergedMcpContent('/test/.mcp.json', generated.content, 'claude-code') + ).rejects.toThrow(/already contains a different codebase-context configuration/i); }); - it('preserves existing opencode mcp entries and updates codebase-context deterministically', async () => { + it('refuses to overwrite an existing conflicting OpenCode codebase-context entry', async () => { readFileMock.mockResolvedValue( JSON.stringify({ $schema: 'https://opencode.ai/config.json', @@ -206,21 +268,574 @@ describe('_buildMergedMcpContent', () => { const generated = generateMcpConfig('opencode'); if (generated.kind !== 'file') throw new Error('expected file config'); - const merged = await _buildMergedMcpContent('/test/opencode.json', generated.content, 'opencode'); + await expect( + _buildMergedMcpContent('/test/opencode.json', generated.content, 'opencode') + ).rejects.toThrow(/already contains a different codebase-context configuration/i); + }); + + it('treats a custom Cursor HTTP entry as a conflict', async () => { + readFileMock.mockResolvedValue( + JSON.stringify({ + mcpServers: { 'codebase-context': { type: 'http', url: 'https://custom/mcp' } } + }) as unknown as Buffer + ); + const generated = generateMcpConfig('cursor', { rootPath: '/test', platform: 'linux' }); + if (generated.kind !== 'file') throw new Error('expected file config'); + + await expect( + _buildMergedMcpContent('/test/.cursor/mcp.json', generated.content, 'cursor') + ).rejects.toThrow(/already contains a different codebase-context configuration/i); + }); + + it('updates a generated JSON launch when the project root moves and preserves entry extras', async () => { + const oldRoot = path.resolve('C:/projects/Old App'); + const newRoot = path.resolve('C:/projects/New App'); + readFileMock.mockResolvedValue( + JSON.stringify({ + customSetting: true, + mcpServers: { + 'codebase-context': { + command: 'C:/node.exe', + args: ['C:/cache/node_modules/codebase-context/dist/index.js', oldRoot], + customTimeout: 8 + } + } + }) as unknown as Buffer + ); + const generated = generateMcpConfig('claude-code', savedLaunch(newRoot)); + if (generated.kind !== 'file') throw new Error('expected file config'); + + const merged = await _buildMergedMcpContent( + '/test/.mcp.json', + generated.content, + 'claude-code' + ); + const parsed = JSON.parse(merged.content) as { + customSetting: boolean; + mcpServers: { 'codebase-context': Record }; + }; + expect(parsed.customSetting).toBe(true); + expect(parsed.mcpServers['codebase-context']).toEqual({ + command: nodeExecutable, + args: [cacheEntrypoint, newRoot], + customTimeout: 8 + }); + }); + + it('updates a previously generated pinned npx command when the project root moves', async () => { + const oldRoot = path.resolve('C:/projects/Old Pinned App'); + const newRoot = path.resolve('C:/projects/New Pinned App'); + readFileMock.mockResolvedValue( + JSON.stringify({ + mcpServers: { + 'codebase-context': { + command: 'npx', + args: ['--yes', 'codebase-context@2.3.0', oldRoot], + customLabel: 'preserve' + } + } + }) as unknown as Buffer + ); + const generated = generateMcpConfig('claude-code', savedLaunch(newRoot)); + if (generated.kind !== 'file') throw new Error('expected file config'); + + const merged = await _buildMergedMcpContent( + '/test/.mcp.json', + generated.content, + 'claude-code' + ); const parsed = JSON.parse(merged.content) as { - mcp: Record; - extra: string; + mcpServers: { 'codebase-context': { command: string; args: string[]; customLabel: string } }; }; + expect(parsed.mcpServers['codebase-context']).toEqual({ + command: nodeExecutable, + args: [cacheEntrypoint, newRoot], + customLabel: 'preserve' + }); + }); +}); + +describe('detectClients', () => { + const accessMock = vi.mocked(fsMod.access); + const missing = () => Object.assign(new Error('missing'), { code: 'ENOENT' }); + + beforeEach(() => { + vi.resetAllMocks(); + accessMock.mockRejectedValue(missing()); + vi.mocked(fsMod.stat).mockImplementation(async (candidate) => { + const executable = String(candidate) + .replace(/\\/g, '/') + .toLowerCase() + .startsWith('c:/tools/'); + return { isFile: () => executable, isDirectory: () => !executable } as Awaited< + ReturnType + >; + }); + }); + + it('uses one four-client table and detects installed tools through PATH candidates', async () => { + accessMock.mockImplementation(async (candidate) => { + const pathValue = String(candidate).toLowerCase(); + if (pathValue.startsWith('c:\\tools\\') && pathValue.endsWith('codex.exe')) return undefined; + throw missing(); + }); + + const clients = await detectClients('/project', { PATH: 'C:/tools' }, 'win32'); + + expect(CLIENTS.map(({ id }) => id)).toEqual(['claude-code', 'codex', 'cursor', 'opencode']); + expect(clients).toContain('codex'); + expect(clients).not.toContain('claude-code'); + expect( + accessMock.mock.calls.some(([candidate]) => String(candidate).includes('C:\\tools')) + ).toBe(true); + }); + + it.each([ + { + name: 'CODEX_HOME', + env: { PATH: '', CODEX_HOME: 'C:/custom-codex-home' }, + marker: 'custom-codex-home', + client: 'codex' + }, + { + name: 'CLAUDE_CONFIG_DIR', + env: { PATH: '', CLAUDE_CONFIG_DIR: 'C:/custom-claude-config' }, + marker: 'custom-claude-config', + client: 'claude-code' + }, + { + name: 'XDG_CONFIG_HOME', + env: { PATH: '', XDG_CONFIG_HOME: 'C:/custom-xdg-config' }, + marker: 'custom-xdg-config', + client: 'opencode' + } + ])('checks $name when detecting installed clients', async ({ env, marker, client }) => { + accessMock.mockImplementation(async (candidate) => { + if (String(candidate).toLowerCase().includes(marker)) return undefined; + throw missing(); + }); + + const clients = await detectClients('/project', env, 'win32'); + + expect(clients).toContain(client); + expect(accessMock.mock.calls.some(([candidate]) => String(candidate).includes(marker))).toBe( + true + ); + }); +}); + +describe('handleInitCli', () => { + const readFileMock = vi.mocked(fsMod.readFile); + const writeFileMock = vi.mocked(fsMod.writeFile); + const mkdirMock = vi.mocked(fsMod.mkdir); + const accessMock = vi.mocked(fsMod.access); + const statMock = vi.mocked(fsMod.stat); + const previousExitCode = process.exitCode; + const previousIsTTY = process.stdin.isTTY; + const previousStdoutIsTTY = process.stdout.isTTY; + + const missing = () => Object.assign(new Error('missing'), { code: 'ENOENT' }); + const detectDirectories = (...names: string[]) => { + accessMock.mockImplementation(async (candidate) => { + const basename = String(candidate).toLowerCase().split(/[\\/]/).at(-1); + if (names.some((name) => basename === (name === 'opencode' ? 'opencode' : `.${name}`))) + return undefined; + throw missing(); + }); + }; + + beforeEach(() => { + vi.resetAllMocks(); + writeFileMock.mockResolvedValue(undefined); + vi.mocked(fsMod.rename).mockResolvedValue(undefined); + vi.mocked(fsMod.rm).mockResolvedValue(undefined); + mkdirMock.mockResolvedValue(undefined); + readFileMock.mockRejectedValue(missing()); + accessMock.mockRejectedValue(missing()); + statMock.mockResolvedValue({ isDirectory: () => true } as Awaited< + ReturnType + >); + process.exitCode = undefined; + Object.defineProperty(process.stdin, 'isTTY', { configurable: true, value: false }); + Object.defineProperty(process.stdout, 'isTTY', { configurable: true, value: false }); + }); + + afterAll(() => { + process.exitCode = previousExitCode; + Object.defineProperty(process.stdin, 'isTTY', { configurable: true, value: previousIsTTY }); + Object.defineProperty(process.stdout, 'isTTY', { + configurable: true, + value: previousStdoutIsTTY + }); + }); + + it('configures all detected clients with --yes', async () => { + const projectRoot = path.resolve('C:/projects/Configured App'); + detectDirectories('claude', 'codex'); + + await handleInitCli(['--yes', '--root', projectRoot]); + + expect(checkbox).not.toHaveBeenCalled(); + expect(vi.mocked(fsMod.rename).mock.calls.map(([, filePath]) => filePath)).toEqual( + expect.arrayContaining([ + path.join(projectRoot, '.mcp.json'), + path.join(projectRoot, '.codex', 'config.toml') + ]) + ); + expect(writeFileMock).toHaveBeenCalledTimes(2); + }); + + it('configures all detected clients without a terminal even without --yes', async () => { + const projectRoot = path.resolve('C:/projects/No Terminal'); + detectDirectories('cursor', 'opencode'); + + await handleInitCli(['--root', projectRoot]); + + expect(checkbox).not.toHaveBeenCalled(); + expect(vi.mocked(fsMod.rename).mock.calls.map(([, filePath]) => filePath)).toEqual( + expect.arrayContaining([ + path.join(projectRoot, '.cursor', 'mcp.json'), + path.join(projectRoot, 'opencode.json') + ]) + ); + expect(writeFileMock).toHaveBeenCalledTimes(2); + }); + + it('lets repeated --client options override detection', async () => { + const projectRoot = path.resolve('C:/projects/Explicit Clients'); + + await handleInitCli([ + '--client', + 'claude-code', + '--client', + 'codex', + '--yes', + '--root', + projectRoot + ]); + + expect(accessMock).not.toHaveBeenCalled(); + expect(vi.mocked(fsMod.rename).mock.calls.map(([, filePath]) => filePath)).toEqual( + expect.arrayContaining([ + path.join(projectRoot, '.mcp.json'), + path.join(projectRoot, '.codex', 'config.toml') + ]) + ); + expect(writeFileMock).toHaveBeenCalledTimes(2); + }); + + it('reports no detected clients without writing when there is no terminal', async () => { + await handleInitCli(['--yes', '--root', path.resolve('C:/projects/Empty')]); + + expect(writeFileMock).not.toHaveBeenCalled(); + expect(mkdirMock).not.toHaveBeenCalled(); + expect(process.exitCode).toBe(1); + }); + + it('shows detected clients checked in the terminal multi-select', async () => { + detectDirectories('claude', 'codex'); + vi.mocked(checkbox).mockResolvedValue(['claude-code']); + Object.defineProperty(process.stdin, 'isTTY', { configurable: true, value: true }); + Object.defineProperty(process.stdout, 'isTTY', { configurable: true, value: true }); + + await handleInitCli(['--root', path.resolve('C:/projects/Interactive')]); + + const choices = vi.mocked(checkbox).mock.calls[0]?.[0].choices; + expect(choices).toEqual( + expect.arrayContaining([ + expect.objectContaining({ value: 'claude-code', checked: true }), + expect.objectContaining({ value: 'codex', checked: true }), + expect.objectContaining({ value: 'cursor', checked: false }), + expect.objectContaining({ value: 'opencode', checked: false }) + ]) + ); + expect(writeFileMock).toHaveBeenCalledOnce(); + }); + + it('leaves all no-detection terminal choices unchecked and writes nothing on empty selection', async () => { + vi.mocked(checkbox).mockResolvedValue([]); + Object.defineProperty(process.stdin, 'isTTY', { configurable: true, value: true }); + Object.defineProperty(process.stdout, 'isTTY', { configurable: true, value: true }); + + await handleInitCli(['--root', path.resolve('C:/projects/Cancelled')]); + + const choices = vi.mocked(checkbox).mock.calls[0]?.[0].choices; + expect(choices).toEqual( + expect.arrayContaining([ + expect.objectContaining({ value: 'claude-code', checked: false }), + expect.objectContaining({ value: 'codex', checked: false }), + expect.objectContaining({ value: 'cursor', checked: false }), + expect.objectContaining({ value: 'opencode', checked: false }) + ]) + ); + expect(writeFileMock).not.toHaveBeenCalled(); + expect(mkdirMock).not.toHaveBeenCalled(); + expect(process.exitCode).toBe(1); + }); + + it('does not write when the terminal prompt throws its cancellation error', async () => { + detectDirectories('claude'); + vi.mocked(checkbox).mockRejectedValue( + Object.assign(new Error('User force closed the prompt'), { name: 'ExitPromptError' }) + ); + Object.defineProperty(process.stdin, 'isTTY', { configurable: true, value: true }); + Object.defineProperty(process.stdout, 'isTTY', { configurable: true, value: true }); + + await handleInitCli(['--root', path.resolve('C:/projects/Prompt Cancelled')]); + + expect(writeFileMock).not.toHaveBeenCalled(); + expect(mkdirMock).not.toHaveBeenCalled(); + }); + + it('requires an existing directory for --root', async () => { + statMock.mockResolvedValue({ isDirectory: () => false } as Awaited< + ReturnType + >); + + await handleInitCli(['--client', 'codex', '--yes', '--root', path.resolve('C:/projects/File')]); + + expect(mkdirMock).not.toHaveBeenCalled(); + expect(writeFileMock).not.toHaveBeenCalled(); + expect(process.exitCode).toBe(1); + }); + + it('does not create configs when --root does not exist', async () => { + statMock.mockRejectedValue(missing()); + + await handleInitCli([ + '--client', + 'codex', + '--yes', + '--root', + path.resolve('C:/projects/Missing Directory') + ]); + + expect(mkdirMock).not.toHaveBeenCalled(); + expect(writeFileMock).not.toHaveBeenCalled(); + expect(process.exitCode).toBe(1); + }); + + it('leaves an identical generated config untouched', async () => { + const projectRoot = path.resolve('C:/projects/Already Configured'); + const generated = generateMcpConfig('claude-code', { rootPath: projectRoot }); + readFileMock.mockResolvedValue(generated.content as unknown as Buffer); + + await handleInitCli(['--client', 'claude-code', '--yes', '--root', projectRoot]); + + expect(writeFileMock).not.toHaveBeenCalled(); + }); + + it('prepares every selected config before writing when a later client conflicts', async () => { + const projectRoot = path.resolve('C:/projects/Batch Conflict'); + const opencodePath = path.join(projectRoot, 'opencode.json'); + readFileMock.mockImplementation(async (filePath) => { + if (String(filePath) === opencodePath) { + return JSON.stringify({ + mcp: { 'codebase-context': { type: 'remote', url: 'https://custom' } } + }) as unknown as Buffer; + } + throw missing(); + }); + + await handleInitCli([ + '--client', + 'claude-code', + '--client', + 'opencode', + '--yes', + '--root', + projectRoot + ]); + + expect(process.exitCode).toBe(1); + expect(mkdirMock).not.toHaveBeenCalled(); + expect(writeFileMock).not.toHaveBeenCalled(); + }); + + it('shows help without prompting or writing files', async () => { + const log = vi.spyOn(console, 'log').mockImplementation(() => undefined); + try { + await handleInitCli(['--help']); + expect(log.mock.calls.flat().join('\n')).toContain( + 'codex mcp add codebase-context -- npx -y codebase-context' + ); + } finally { + log.mockRestore(); + } + + expect(checkbox).not.toHaveBeenCalled(); + expect(writeFileMock).not.toHaveBeenCalled(); + }); +}); + +describe('_buildMergedCodexContent', () => { + const readFileMock = vi.mocked(fsMod.readFile); + const writeFileMock = vi.mocked(fsMod.writeFile); + const mkdirMock = vi.mocked(fsMod.mkdir); + const statMock = vi.mocked(fsMod.stat); + + beforeEach(() => { + vi.resetAllMocks(); + writeFileMock.mockResolvedValue(undefined); + vi.mocked(fsMod.rename).mockResolvedValue(undefined); + vi.mocked(fsMod.rm).mockResolvedValue(undefined); + mkdirMock.mockResolvedValue(undefined); + statMock.mockResolvedValue({ isDirectory: () => true } as Awaited< + ReturnType + >); + readFileMock.mockRejectedValue(Object.assign(new Error('missing'), { code: 'ENOENT' })); + }); + + const runCodexSetupWithExistingConfig = async (existingToml: string) => { + const projectRoot = path.resolve('C:/projects/Codex Existing'); + const initialBytes = Buffer.from(existingToml, 'utf8'); + let persistedBytes = Buffer.from(initialBytes); + readFileMock.mockResolvedValue(existingToml as unknown as Buffer); + writeFileMock.mockImplementation(async (_filePath, content) => { + persistedBytes = Buffer.from(content as string, 'utf8'); + }); + + const previousExitCode = process.exitCode; + process.exitCode = undefined; + try { + await handleInitCli(['--client', 'codex', '--yes', '--root', projectRoot]); + return { initialBytes, persistedBytes, exitCode: process.exitCode }; + } finally { + process.exitCode = previousExitCode; + } + }; + + it('preserves unrelated Codex TOML settings when adding the project server', async () => { + readFileMock.mockResolvedValue( + '[model]\nmodel = "gpt-example"\n\n[mcp_servers.other]\ncommand = "other"\n' as unknown as Buffer + ); + const generated = generateMcpConfig('codex', { + rootPath: 'C:/projects/app', + platform: 'win32' + }); + if (generated.kind !== 'file') throw new Error('expected file config'); + + const merged = await _buildMergedCodexContent(generated.path, generated.content); expect(merged.mergedFromExisting).toBe(true); - expect(parsed.extra).toBe('value'); - expect(parsed.mcp['existing-server']).toEqual({ - type: 'remote', - url: 'http://127.0.0.1:5000/mcp' + expect(merged.content).toContain('[model]\nmodel = "gpt-example"'); + expect(merged.content).toContain('[mcp_servers.other]\ncommand = "other"'); + expect(merged.content).toContain('[mcp_servers.codebase-context]'); + }); + + it('leaves the same Codex codebase-context entry unchanged on a repeated setup', async () => { + const generated = generateMcpConfig('codex', { + rootPath: path.resolve('C:/projects/app'), + platform: 'win32' }); - expect(parsed.mcp['codebase-context']).toEqual({ - type: 'remote', - url: 'http://127.0.0.1:3100/mcp' + if (generated.kind !== 'file') throw new Error('expected file config'); + readFileMock.mockResolvedValue(generated.content as unknown as Buffer); + + const merged = await _buildMergedCodexContent(generated.path, generated.content); + + expect(merged).toEqual({ content: generated.content, mergedFromExisting: true }); + }); + + it('treats quoted table keys as the same idempotent Codex entry', async () => { + const generated = generateMcpConfig('codex', { + rootPath: path.resolve('C:/projects/app'), + platform: 'win32' + }); + if (generated.kind !== 'file') throw new Error('expected file config'); + const existing = generated.content.replace( + '[mcp_servers.codebase-context]', + '["mcp_servers"."codebase-context"]' + ); + readFileMock.mockResolvedValue(existing as unknown as Buffer); + + const merged = await _buildMergedCodexContent(generated.path, generated.content); + + expect(merged).toEqual({ content: existing, mergedFromExisting: true }); + }); + + it('refuses a conflicting or disabled Codex codebase-context entry', async () => { + readFileMock.mockResolvedValue( + '[mcp_servers.codebase-context]\ndisabled = true\ncommand = "old"\n' as unknown as Buffer + ); + const generated = generateMcpConfig('codex', { + rootPath: 'C:/projects/app', + platform: 'win32' }); + if (generated.kind !== 'file') throw new Error('expected file config'); + + await expect(_buildMergedCodexContent(generated.path, generated.content)).rejects.toThrow( + /already contains a different codebase-context configuration/i + ); + }); + + it('updates a generated Codex launch for a moved root while preserving custom settings', async () => { + const oldRoot = path.resolve('C:/projects/Old Codex App'); + const newRoot = path.resolve('C:/projects/New Codex App'); + const existing = + '[mcp_servers.codebase-context]\n' + + 'command = "node"\n' + + `args = ["C:/cache/node_modules/codebase-context/dist/index.js", ${JSON.stringify(oldRoot)}]\n` + + 'startup_timeout_sec = 75\n' + + 'custom_label = "keep me"\n'; + readFileMock.mockResolvedValue(existing as unknown as Buffer); + const generated = generateMcpConfig('codex', savedLaunch(newRoot)); + if (generated.kind !== 'file') throw new Error('expected file config'); + + const merged = await _buildMergedCodexContent(generated.path, generated.content); + + expect(merged.content).toContain(`command = ${JSON.stringify(nodeExecutable)}`); + expect(merged.content).toContain( + `args = [${JSON.stringify(cacheEntrypoint)}, ${JSON.stringify(newRoot)}]` + ); + expect(merged.content).toContain('startup_timeout_sec = 75'); + expect(merged.content).toContain('custom_label = "keep me"'); }); + + it.each([ + { + name: 'double-quoted parent and child keys', + toml: '["mcp_servers"."codebase-context"]\ncommand = "old"\n' + }, + { + name: 'double-quoted parent and bare child key', + toml: '["mcp_servers".codebase-context]\ncommand = "old"\n' + } + ])( + 'refuses to replace a Codex entry under $name without changing its bytes', + async ({ toml }) => { + const result = await runCodexSetupWithExistingConfig(toml); + + expect(result.exitCode).toBe(1); + expect(writeFileMock).not.toHaveBeenCalled(); + expect(result.persistedBytes).toEqual(result.initialBytes); + } + ); + + it.each([ + { + name: 'inline child in the bare parent table', + toml: '[mcp_servers]\ncodebase-context = { command = "old", args = [] }\n' + }, + { + name: 'dotted child in the double-quoted parent table', + toml: '["mcp_servers"]\ncodebase-context.command = "old"\n' + }, + { + name: 'dotted child in the single-quoted parent table', + toml: "['mcp_servers']\n'codebase-context'.command = \"old\"\n" + }, + { + name: 'inline root table under a double-quoted key', + toml: '"mcp_servers" = { "codebase-context" = { command = "old", args = [] } }\n' + } + ])( + 'refuses to replace a Codex entry under $name without changing its bytes', + async ({ toml }) => { + const result = await runCodexSetupWithExistingConfig(toml); + + expect(result.exitCode).toBe(1); + expect(writeFileMock).not.toHaveBeenCalled(); + expect(result.persistedBytes).toEqual(result.initialBytes); + } + ); }); diff --git a/tests/cli-map-export.test.ts b/tests/cli-map-export.test.ts index 78dd9cc..1fa9ee2 100644 --- a/tests/cli-map-export.test.ts +++ b/tests/cli-map-export.test.ts @@ -1,8 +1,9 @@ import { describe, it, expect, afterEach } from 'vitest'; -import { mkdtemp, rm, access, readFile } from 'fs/promises'; +import { mkdtemp, rm, access, readFile, writeFile } from 'fs/promises'; import os from 'os'; import path from 'path'; import { handleMapCli } from '../src/cli-map.js'; +import { resolveIndexArtifact } from '../src/core/index-generation.js'; describe('handleMapCli --export', () => { let tmpDir: string; @@ -60,4 +61,20 @@ describe('handleMapCli --export', () => { else process.env.CODEBASE_ROOT = original; } }); + + it('indexes a project that has no index before printing the map', async () => { + tmpDir = await mkdtemp(path.join(os.tmpdir(), 'cli-map-index-')); + await writeFile(path.join(tmpDir, 'server.ts'), 'export const port = 3000;\n', 'utf-8'); + const original = process.env.CODEBASE_ROOT; + process.env.CODEBASE_ROOT = tmpDir; + try { + await handleMapCli(['--export']); + await access(await resolveIndexArtifact(tmpDir, 'index.json')); + const content = await readFile(path.join(tmpDir, 'CODEBASE_MAP.md'), 'utf-8'); + expect(content).toMatch(/Files: [1-9]/); + } finally { + if (original === undefined) delete process.env.CODEBASE_ROOT; + else process.env.CODEBASE_ROOT = original; + } + }); }); diff --git a/tests/cli-version.test.ts b/tests/cli-version.test.ts new file mode 100644 index 0000000..c4c56f6 --- /dev/null +++ b/tests/cli-version.test.ts @@ -0,0 +1,26 @@ +import { spawnSync } from 'node:child_process'; +import { readFileSync } from 'node:fs'; +import { resolve } from 'node:path'; +import { describe, expect, it } from 'vitest'; + +const root = resolve(import.meta.dirname, '..'); +const packageJson = JSON.parse(readFileSync(resolve(root, 'package.json'), 'utf8')) as { + version: string; +}; + +describe('version flag', () => { + it('prints the package version and exits', () => { + const result = spawnSync( + process.execPath, + ['--import', 'tsx', resolve(root, 'src/index.ts'), '--version'], + { + cwd: root, + encoding: 'utf8', + timeout: 30_000 + } + ); + + expect(result.status).toBe(0); + expect(result.stdout.trim()).toBe(packageJson.version); + }); +}); diff --git a/tests/cli.test.ts b/tests/cli.test.ts index 6ee46d7..2debe67 100644 --- a/tests/cli.test.ts +++ b/tests/cli.test.ts @@ -2,15 +2,32 @@ import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'; import path from 'path'; import os from 'os'; import { promises as fs } from 'fs'; +import { formatRefs } from '../src/cli-formatters.js'; const toolMocks = vi.hoisted(() => ({ dispatchTool: vi.fn() })); +const cliContextMocks = vi.hoisted(() => ({ + loadProjectConfig: vi.fn(), + indexer: vi.fn() +})); + vi.mock('../src/tools/index.js', () => ({ dispatchTool: toolMocks.dispatchTool })); +vi.mock('../src/server/config.js', () => ({ + loadProjectConfig: cliContextMocks.loadProjectConfig +})); + +vi.mock('../src/core/indexer.js', () => ({ + CodebaseIndexer: vi.fn().mockImplementation(() => ({ + index: cliContextMocks.indexer, + getProgress: vi.fn() + })) +})); + import { handleCliCommand, handleMemoryCli } from '../src/cli.js'; describe('CLI', () => { @@ -21,6 +38,12 @@ describe('CLI', () => { beforeEach(() => { toolMocks.dispatchTool.mockReset(); + cliContextMocks.loadProjectConfig.mockReset().mockResolvedValue(undefined); + cliContextMocks.indexer.mockReset().mockResolvedValue({ + indexedFiles: 0, + totalChunks: 0, + duration: 0 + }); originalEnvRoot = process.env.CODEBASE_ROOT; delete process.env.CODEBASE_ROOT; @@ -41,6 +64,23 @@ describe('CLI', () => { else process.env.CODEBASE_ROOT = originalEnvRoot; }); + it('init configures the selected repository without preparing its index', async () => { + const root = await fs.mkdtemp(path.join(os.tmpdir(), 'cbc-init-only-')); + const previousExitCode = process.exitCode; + try { + process.exitCode = undefined; + await handleCliCommand(['init', '--client', 'codex', '--yes', '--root', root]); + expect(cliContextMocks.indexer).not.toHaveBeenCalled(); + expect(cliContextMocks.loadProjectConfig).not.toHaveBeenCalled(); + expect(await fs.readFile(path.join(root, '.codex', 'config.toml'), 'utf8')).toContain( + '[mcp_servers.codebase-context]' + ); + } finally { + process.exitCode = previousExitCode; + await fs.rm(root, { recursive: true, force: true }); + } + }); + it('search errors when --query has no value', async () => { await expect(handleCliCommand(['search', '--query', '--json'])).rejects.toThrow( /process\.exit:1/ @@ -49,6 +89,25 @@ describe('CLI', () => { expect(errorSpy).toHaveBeenCalled(); }); + it('reindex help returns before context creation, indexing, or dispatch', async () => { + await handleCliCommand(['reindex', '--help']); + + const output = logSpy.mock.calls.map((call) => String(call[0] ?? '')).join('\n'); + expect(output).toContain('reindex [--incremental]'); + expect(cliContextMocks.loadProjectConfig).not.toHaveBeenCalled(); + expect(cliContextMocks.indexer).not.toHaveBeenCalled(); + expect(toolMocks.dispatchTool).not.toHaveBeenCalled(); + }); + + it('read-command help returns before context creation or dispatch', async () => { + await handleCliCommand(['status', '--help']); + + const output = logSpy.mock.calls.map((call) => String(call[0] ?? '')).join('\n'); + expect(output).toContain('status'); + expect(cliContextMocks.loadProjectConfig).not.toHaveBeenCalled(); + expect(toolMocks.dispatchTool).not.toHaveBeenCalled(); + }); + it('search dispatches with typed args', async () => { toolMocks.dispatchTool.mockResolvedValue({ content: [{ type: 'text', text: JSON.stringify({ ok: true }) }] @@ -114,7 +173,7 @@ describe('CLI', () => { const out = logSpy.mock.calls.map((c) => String(c[0] ?? '')).join('\n'); expect(out).toMatch(/Index Status/); - expect(out).toMatch(/\+\- Index Status/); + expect(out).toMatch(/\+- Index Status/); expect(out).toMatch(/Progress:/); } finally { if (originalAscii === undefined) delete process.env.CODEBASE_CONTEXT_ASCII; @@ -132,6 +191,45 @@ describe('CLI', () => { expect(logSpy).toHaveBeenCalled(); }); + it('reports unavailable reference sources and labels lexical fallback uncertainty', () => { + const result = { + symbol: 'activeToken', + usageCount: 1, + confidence: 'lexical', + usages: [{ file: 'src/active.ts', line: 1, preview: 'activeToken();' }], + isComplete: false, + coverage: { filesUnavailable: 1 } + }; + + formatRefs(result, 'C:/repo'); + + const output = logSpy.mock.calls + .map((call) => String(call[0] ?? '')) + .join('\n') + .replace(/\s+/g, ' '); + expect(output).toContain('incomplete coverage: 1 indexed source unavailable'); + expect(output).not.toContain('results capped at limit'); + expect(output).toContain('lexical'); + expect(output).toContain('Raw-text fallback may match comments or strings.'); + }); + + it('retains the limit message when complete reference coverage reaches the limit', () => { + const result = { + symbol: 'activeToken', + usageCount: 10, + confidence: 'syntactic', + usages: [{ file: 'src/active.ts', line: 1, preview: 'activeToken();' }], + isComplete: false, + coverage: { filesUnavailable: 0 } + }; + + formatRefs(result, 'C:/repo'); + + const output = logSpy.mock.calls.map((call) => String(call[0] ?? '')).join('\n'); + expect(output).toContain('results capped at limit'); + expect(output).not.toContain('incomplete coverage'); + }); + it('memory list errors on invalid --type', async () => { await expect(handleMemoryCli(['list', '--type', 'nope'])).rejects.toThrow(/process\.exit:1/); expect(errorSpy).toHaveBeenCalled(); diff --git a/tests/codebase-map.test.ts b/tests/codebase-map.test.ts index c90de78..10f5b49 100644 --- a/tests/codebase-map.test.ts +++ b/tests/codebase-map.test.ts @@ -84,15 +84,19 @@ async function removeTempMapProject(rootPath: string): Promise { } function runMapCli(args: string[], rootPath: string) { - return spawnSync(process.execPath, ['--no-deprecation', '--import', 'tsx', ENTRYPOINT, 'map', ...args], { - cwd: CURRENT_REPO_ROOT, - env: { - ...process.env, - CODEBASE_ROOT: rootPath - }, - encoding: 'utf8', - timeout: 120_000 - }); + return spawnSync( + process.execPath, + ['--no-deprecation', '--import', 'tsx', ENTRYPOINT, 'map', ...args], + { + cwd: CURRENT_REPO_ROOT, + env: { + ...process.env, + CODEBASE_ROOT: rootPath + }, + encoding: 'utf8', + timeout: 120_000 + } + ); } // --------------------------------------------------------------------------- @@ -198,6 +202,21 @@ describe('buildCodebaseMap', () => { expect(map.suggestedNextCalls.length).toBeGreaterThan(0); }); + it('counts indexed files when recorded stats say zero', async () => { + const rootPath = await createTempMapProject({ + chunks: [ + { + relativePath: 'src/server.ts', + content: 'export const port = 3000;', + metadata: {} + } as CodeChunk + ], + graph: { stats: { files: 0, edges: 0, avgDependencies: 0 } } + }); + const map = await buildCodebaseMap(createProjectState(rootPath)); + expect(map.graphStats.files).toBe(1); + }); + it('caps suggested next calls at 3', async () => { const project = createProjectState(FIXTURE_ROOT); const map = await buildCodebaseMap(project, { mode: 'full' }); diff --git a/tests/contextbench-baseline-runner.test.ts b/tests/contextbench-baseline-runner.test.ts index 61064a4..881f3ae 100644 --- a/tests/contextbench-baseline-runner.test.ts +++ b/tests/contextbench-baseline-runner.test.ts @@ -1,5 +1,12 @@ import { execFileSync, spawnSync } from 'node:child_process'; -import { appendFileSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { + appendFileSync, + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + writeFileSync +} from 'node:fs'; import { tmpdir } from 'node:os'; import path from 'node:path'; import { describe, expect, it, vi } from 'vitest'; @@ -51,22 +58,17 @@ function childEnv(overrides: NodeJS.ProcessEnv = {}): NodeJS.ProcessEnv { return { ...env, ...overrides }; } -function ignoreWindowsTempCleanupRace(error: unknown): void { - const code = (error as NodeJS.ErrnoException).code; - if (!['EBUSY', 'ENOTEMPTY', 'EPERM'].includes(code ?? '')) throw error; +function cleanupTempRoot(tempRoot: string): void { + rmSync(tempRoot, { + recursive: true, + force: true, + maxRetries: 10, + retryDelay: 200 + }); } function cleanupSessionRoot(sessionRoot: string): void { - try { - rmSync(path.dirname(path.dirname(path.dirname(path.dirname(sessionRoot)))), { - recursive: true, - force: true, - maxRetries: 10, - retryDelay: 200 - }); - } catch (error) { - ignoreWindowsTempCleanupRace(error); - } + cleanupTempRoot(path.dirname(path.dirname(path.dirname(path.dirname(sessionRoot))))); } function tempSessionRoot(phase: 'phase40' | 'phase41' = 'phase40'): string { @@ -94,7 +96,15 @@ function createCleanGitRepo(root: string): string { execFileSync('git', ['add', 'README.md'], { cwd: repoPath, encoding: 'utf8', env: childEnv() }); execFileSync( 'git', - ['-c', 'user.name=ContextBench Test', '-c', 'user.email=contextbench@example.invalid', 'commit', '-m', 'fixture'], + [ + '-c', + 'user.name=ContextBench Test', + '-c', + 'user.email=contextbench@example.invalid', + 'commit', + '-m', + 'fixture' + ], { cwd: repoPath, encoding: 'utf8', env: childEnv() } ); return repoPath; @@ -131,7 +141,11 @@ function writeStubClaude(root: string): string { return stubPath; } -function writeStubEvaluator(root: string, exitCode: 0 | 1, output = JSON.stringify({ score: 1 })): string { +function writeStubEvaluator( + root: string, + exitCode: 0 | 1, + output = JSON.stringify({ score: 1 }) +): string { const stubPath = path.join(root, `stub-evaluator-${exitCode}.cjs`); const serializedOutput = JSON.stringify(output); writeFileSync( @@ -146,14 +160,22 @@ describe('ContextBench Phase 40 baseline runner', () => { it('reserves every required slot and writes terminal missing-evidence rows for blocked lanes', () => { const sessionRoot = tempSessionRoot(); try { - execFileSync('node', ['scripts/contextbench-runner.mjs', '--baseline-snapshot', '--out', sessionRoot], { - encoding: 'utf8' - }); + execFileSync( + 'node', + ['scripts/contextbench-runner.mjs', '--baseline-snapshot', '--out', sessionRoot], + { + encoding: 'utf8' + } + ); const reservations = JSON.parse( readFileSync(path.join(sessionRoot, 'slot-reservations.json'), 'utf8') - ) as { reservations: Array<{ laneId: string; status: string; terminalStatus: string | null }> }; + ) as { + reservations: Array<{ laneId: string; status: string; terminalStatus: string | null }>; + }; expect(reservations.reservations).toHaveLength(20 * 6 * 3); - const blocked = reservations.reservations.filter((slot) => slot.status === 'terminal_missing_evidence'); + const blocked = reservations.reservations.filter( + (slot) => slot.status === 'terminal_missing_evidence' + ); expect(blocked).toHaveLength(20 * 2 * 3); expect([...new Set(blocked.map((slot) => slot.laneId))].sort()).toEqual([ 'codebase-memory-mcp', @@ -176,10 +198,16 @@ describe('ContextBench Phase 40 baseline runner', () => { it('rejects duplicate primary baseline rows during validation', () => { const sessionRoot = tempSessionRoot('phase41'); try { - execFileSync('node', ['scripts/contextbench-runner.mjs', '--baseline-snapshot', '--out', sessionRoot], { - encoding: 'utf8' - }); - const firstRow = readFileSync(path.join(sessionRoot, 'run-manifest.jsonl'), 'utf8').trim().split('\n')[0]; + execFileSync( + 'node', + ['scripts/contextbench-runner.mjs', '--baseline-snapshot', '--out', sessionRoot], + { + encoding: 'utf8' + } + ); + const firstRow = readFileSync(path.join(sessionRoot, 'run-manifest.jsonl'), 'utf8') + .trim() + .split('\n')[0]; appendFileSync(path.join(sessionRoot, 'run-manifest.jsonl'), `${firstRow}\n`, 'utf8'); const result = spawnSync( @@ -191,10 +219,7 @@ describe('ContextBench Phase 40 baseline runner', () => { expect(result.status).not.toBe(0); expect(result.stderr).toContain('duplicate primary baseline row for reservation'); } finally { - rmSync(path.dirname(path.dirname(path.dirname(path.dirname(sessionRoot)))), { - recursive: true, - force: true - }); + cleanupSessionRoot(sessionRoot); } }); @@ -202,9 +227,13 @@ describe('ContextBench Phase 40 baseline runner', () => { const sessionRoot = tempSessionRoot(); const taskId = manifest.tasks[0].instance_id; try { - execFileSync('node', ['scripts/contextbench-runner.mjs', '--baseline-snapshot', '--out', sessionRoot], { - encoding: 'utf8' - }); + execFileSync( + 'node', + ['scripts/contextbench-runner.mjs', '--baseline-snapshot', '--out', sessionRoot], + { + encoding: 'utf8' + } + ); execFileSync( 'node', [ @@ -265,7 +294,13 @@ describe('ContextBench Phase 40 baseline runner', () => { it('records official evaluator invocation metadata for overridden live executor attempts', () => { const tempRoot = mkdtempSync(path.join(tmpdir(), 'contextbench-official-evaluator-')); - const sessionRoot = path.join(tempRoot, 'benchmark-runs', 'contextbench', 'phase41', 'runner-smoke'); + const sessionRoot = path.join( + tempRoot, + 'benchmark-runs', + 'contextbench', + 'phase41', + 'runner-smoke' + ); const taskId = manifest.tasks[0].instance_id; const repoPath = createCleanGitRepo(tempRoot); const payloadPath = writePayloadFile(tempRoot, taskId, repoPath); @@ -275,14 +310,22 @@ describe('ContextBench Phase 40 baseline runner', () => { CONTEXTBENCH_CLAUDE_COMMAND: JSON.stringify([process.execPath, stubClaude]), CONTEXTBENCH_OFFICIAL_EVALUATOR_COMMAND: JSON.stringify([process.execPath, stubEvaluator]), CONTEXTBENCH_LANE_TELEMETRY_JSON: JSON.stringify({ - 'raw-native': { sourceKind: 'proxy', proofSource: 'stubbed_test_proxy', observedTools: ['native-read'] } + 'raw-native': { + sourceKind: 'proxy', + proofSource: 'stubbed_test_proxy', + observedTools: ['native-read'] + } }) }); try { - execFileSync('node', ['scripts/contextbench-runner.mjs', '--baseline-snapshot', '--out', sessionRoot], { - encoding: 'utf8', - env - }); + execFileSync( + 'node', + ['scripts/contextbench-runner.mjs', '--baseline-snapshot', '--out', sessionRoot], + { + encoding: 'utf8', + env + } + ); execFileSync( 'node', [ @@ -329,7 +372,12 @@ describe('ContextBench Phase 40 baseline runner', () => { expect(score.stdoutPath).toBeTruthy(); expect(score.stderrPath).toBeTruthy(); const rawTrace = JSON.parse(readFileSync(attempt?.raw_trace_path ?? '', 'utf8')) as { - laneIsolation?: { proven: boolean; sourceKind: string; proofSource: string; observedTools: string[] }; + laneIsolation?: { + proven: boolean; + sourceKind: string; + proofSource: string; + observedTools: string[]; + }; }; expect(rawTrace.laneIsolation).toMatchObject({ proven: true, @@ -338,13 +386,19 @@ describe('ContextBench Phase 40 baseline runner', () => { observedTools: ['native-read'] }); } finally { - rmSync(tempRoot, { recursive: true, force: true }); + cleanupTempRoot(tempRoot); } }); it('rejects malformed official evaluator output as judge_failed diagnostic evidence', () => { const tempRoot = mkdtempSync(path.join(tmpdir(), 'contextbench-official-evaluator-malformed-')); - const sessionRoot = path.join(tempRoot, 'benchmark-runs', 'contextbench', 'phase41', 'runner-smoke'); + const sessionRoot = path.join( + tempRoot, + 'benchmark-runs', + 'contextbench', + 'phase41', + 'runner-smoke' + ); const taskId = manifest.tasks[0].instance_id; const repoPath = createCleanGitRepo(tempRoot); const payloadPath = writePayloadFile(tempRoot, taskId, repoPath); @@ -354,14 +408,22 @@ describe('ContextBench Phase 40 baseline runner', () => { CONTEXTBENCH_CLAUDE_COMMAND: JSON.stringify([process.execPath, stubClaude]), CONTEXTBENCH_OFFICIAL_EVALUATOR_COMMAND: JSON.stringify([process.execPath, stubEvaluator]), CONTEXTBENCH_LANE_TELEMETRY_JSON: JSON.stringify({ - 'raw-native': { sourceKind: 'proxy', proofSource: 'stubbed_test_proxy', observedTools: ['native-read'] } + 'raw-native': { + sourceKind: 'proxy', + proofSource: 'stubbed_test_proxy', + observedTools: ['native-read'] + } }) }); try { - execFileSync('node', ['scripts/contextbench-runner.mjs', '--baseline-snapshot', '--out', sessionRoot], { - encoding: 'utf8', - env - }); + execFileSync( + 'node', + ['scripts/contextbench-runner.mjs', '--baseline-snapshot', '--out', sessionRoot], + { + encoding: 'utf8', + env + } + ); execFileSync( 'node', [ @@ -401,7 +463,7 @@ describe('ContextBench Phase 40 baseline runner', () => { expect(score.fallbackReason).toBe('official_evaluator_malformed_jsonl'); expect(score.outputHash).toMatch(/^sha256:[a-f0-9]{64}$/); } finally { - rmSync(tempRoot, { recursive: true, force: true }); + cleanupTempRoot(tempRoot); } }); @@ -415,8 +477,16 @@ describe('ContextBench Phase 40 baseline runner', () => { ]; for (const testCase of cases) { - const tempRoot = mkdtempSync(path.join(tmpdir(), 'contextbench-official-evaluator-envelope-')); - const sessionRoot = path.join(tempRoot, 'benchmark-runs', 'contextbench', 'phase41', 'runner-smoke'); + const tempRoot = mkdtempSync( + path.join(tmpdir(), 'contextbench-official-evaluator-envelope-') + ); + const sessionRoot = path.join( + tempRoot, + 'benchmark-runs', + 'contextbench', + 'phase41', + 'runner-smoke' + ); const taskId = manifest.tasks[0].instance_id; const repoPath = createCleanGitRepo(tempRoot); const payloadPath = writePayloadFile(tempRoot, taskId, repoPath); @@ -426,14 +496,22 @@ describe('ContextBench Phase 40 baseline runner', () => { CONTEXTBENCH_CLAUDE_COMMAND: JSON.stringify([process.execPath, stubClaude]), CONTEXTBENCH_OFFICIAL_EVALUATOR_COMMAND: JSON.stringify([process.execPath, stubEvaluator]), CONTEXTBENCH_LANE_TELEMETRY_JSON: JSON.stringify({ - 'raw-native': { sourceKind: 'proxy', proofSource: 'stubbed_test_proxy', observedTools: ['native-read'] } + 'raw-native': { + sourceKind: 'proxy', + proofSource: 'stubbed_test_proxy', + observedTools: ['native-read'] + } }) }); try { - execFileSync('node', ['scripts/contextbench-runner.mjs', '--baseline-snapshot', '--out', sessionRoot], { - encoding: 'utf8', - env - }); + execFileSync( + 'node', + ['scripts/contextbench-runner.mjs', '--baseline-snapshot', '--out', sessionRoot], + { + encoding: 'utf8', + env + } + ); execFileSync( 'node', [ @@ -460,14 +538,20 @@ describe('ContextBench Phase 40 baseline runner', () => { expect(attempt?.status).toBe('judge_failed'); expect(attempt?.scoring.fallbackReason).toBe(testCase.reason); } finally { - rmSync(tempRoot, { recursive: true, force: true }); + cleanupTempRoot(tempRoot); } } }); it('records official evaluator failure as judge_failed without making claims', () => { const tempRoot = mkdtempSync(path.join(tmpdir(), 'contextbench-official-evaluator-fail-')); - const sessionRoot = path.join(tempRoot, 'benchmark-runs', 'contextbench', 'phase41', 'runner-smoke'); + const sessionRoot = path.join( + tempRoot, + 'benchmark-runs', + 'contextbench', + 'phase41', + 'runner-smoke' + ); const taskId = manifest.tasks[0].instance_id; const repoPath = createCleanGitRepo(tempRoot); const payloadPath = writePayloadFile(tempRoot, taskId, repoPath); @@ -478,10 +562,14 @@ describe('ContextBench Phase 40 baseline runner', () => { CONTEXTBENCH_OFFICIAL_EVALUATOR_COMMAND: JSON.stringify([process.execPath, stubEvaluator]) }); try { - execFileSync('node', ['scripts/contextbench-runner.mjs', '--baseline-snapshot', '--out', sessionRoot], { - encoding: 'utf8', - env - }); + execFileSync( + 'node', + ['scripts/contextbench-runner.mjs', '--baseline-snapshot', '--out', sessionRoot], + { + encoding: 'utf8', + env + } + ); execFileSync( 'node', [ @@ -525,16 +613,20 @@ describe('ContextBench Phase 40 baseline runner', () => { expect(score.exitCode).toBe(1); expect(score.exitStatus).toBe(1); } finally { - rmSync(tempRoot, { recursive: true, force: true }); + cleanupTempRoot(tempRoot); } }); it('chunks all-ready-lane execution with max-attempts so live runs are resumable', () => { const sessionRoot = tempSessionRoot('phase41'); try { - execFileSync('node', ['scripts/contextbench-runner.mjs', '--baseline-snapshot', '--out', sessionRoot], { - encoding: 'utf8' - }); + execFileSync( + 'node', + ['scripts/contextbench-runner.mjs', '--baseline-snapshot', '--out', sessionRoot], + { + encoding: 'utf8' + } + ); execFileSync( 'node', [ @@ -591,9 +683,13 @@ describe('ContextBench Phase 40 baseline runner', () => { const sessionRoot = tempSessionRoot('phase41'); const taskId = manifest.tasks[0].instance_id; try { - execFileSync('node', ['scripts/contextbench-runner.mjs', '--baseline-snapshot', '--out', sessionRoot], { - encoding: 'utf8' - }); + execFileSync( + 'node', + ['scripts/contextbench-runner.mjs', '--baseline-snapshot', '--out', sessionRoot], + { + encoding: 'utf8' + } + ); execFileSync( 'node', [ @@ -607,8 +703,16 @@ describe('ContextBench Phase 40 baseline runner', () => { { encoding: 'utf8' } ); const measurement = JSON.parse( - readFileSync(path.join(sessionRoot, 'setup-index', 'raw-native', 'setup-index.json'), 'utf8') - ) as { claimBearing: boolean; setupStatus: string; indexStatus: string; setupLogPath: string }; + readFileSync( + path.join(sessionRoot, 'setup-index', 'raw-native', 'setup-index.json'), + 'utf8' + ) + ) as { + claimBearing: boolean; + setupStatus: string; + indexStatus: string; + setupLogPath: string; + }; expect(measurement).toMatchObject({ claimBearing: false, setupStatus: 'not_required', @@ -652,9 +756,13 @@ describe('ContextBench Phase 40 baseline runner', () => { const sessionRoot = tempSessionRoot('phase41'); const taskId = manifest.tasks[0].instance_id; try { - execFileSync('node', ['scripts/contextbench-runner.mjs', '--baseline-snapshot', '--out', sessionRoot], { - encoding: 'utf8' - }); + execFileSync( + 'node', + ['scripts/contextbench-runner.mjs', '--baseline-snapshot', '--out', sessionRoot], + { + encoding: 'utf8' + } + ); execFileSync( 'node', [ @@ -674,7 +782,8 @@ describe('ContextBench Phase 40 baseline runner', () => { { encoding: 'utf8' } ); const attempt = readRows(sessionRoot).find( - (row) => row.lane_id === 'codebase-context' && row.task_id === taskId && row.repeat_index === 1 + (row) => + row.lane_id === 'codebase-context' && row.task_id === taskId && row.repeat_index === 1 ); expect(attempt?.status).toBe('setup_failed'); expect(attempt?.scoring.fallbackReason).toContain('missing_setup_index_measurement'); @@ -688,16 +797,23 @@ describe('ContextBench Phase 40 baseline runner', () => { const sessionRoot = tempSessionRoot('phase41'); const taskId = manifest.tasks[0].instance_id; try { - execFileSync('node', ['scripts/contextbench-runner.mjs', '--baseline-snapshot', '--out', sessionRoot], { - encoding: 'utf8' - }); + execFileSync( + 'node', + ['scripts/contextbench-runner.mjs', '--baseline-snapshot', '--out', sessionRoot], + { + encoding: 'utf8' + } + ); const logsDir = path.join(sessionRoot, 'manual-setup-index-logs', 'codebase-context'); mkdirSync(logsDir, { recursive: true }); const setupLogPath = path.join(logsDir, 'setup.stdout.log'); const indexLogPath = path.join(logsDir, 'index.stdout.log'); writeFileSync(setupLogPath, 'setup completed\n', 'utf8'); writeFileSync(indexLogPath, 'index completed\n', 'utf8'); - const importPath = path.join(path.dirname(sessionRoot), 'codebase-context-setup-index-import.json'); + const importPath = path.join( + path.dirname(sessionRoot), + 'codebase-context-setup-index-import.json' + ); writeFileSync( importPath, JSON.stringify( @@ -749,7 +865,8 @@ describe('ContextBench Phase 40 baseline runner', () => { { encoding: 'utf8' } ); const attempt = readRows(sessionRoot).find( - (row) => row.lane_id === 'codebase-context' && row.task_id === taskId && row.repeat_index === 1 + (row) => + row.lane_id === 'codebase-context' && row.task_id === taskId && row.repeat_index === 1 ); expect(attempt?.status).toBe('completed'); expect(attempt?.setupIndex).toMatchObject({ @@ -769,9 +886,13 @@ describe('ContextBench Phase 40 baseline runner', () => { const sessionRoot = tempSessionRoot('phase41'); const taskId = manifest.tasks[0].instance_id; try { - execFileSync('node', ['scripts/contextbench-runner.mjs', '--baseline-snapshot', '--out', sessionRoot], { - encoding: 'utf8' - }); + execFileSync( + 'node', + ['scripts/contextbench-runner.mjs', '--baseline-snapshot', '--out', sessionRoot], + { + encoding: 'utf8' + } + ); const logsDir = path.join(sessionRoot, 'manual-setup-index-logs', 'codebase-context'); mkdirSync(logsDir, { recursive: true }); const setupLogPath = path.join(logsDir, 'setup.stdout.log'); @@ -896,7 +1017,8 @@ describe('ContextBench Phase 40 baseline runner', () => { { encoding: 'utf8' } ); const attempt = readRows(sessionRoot).find( - (row) => row.lane_id === 'codebase-context' && row.task_id === taskId && row.repeat_index === 1 + (row) => + row.lane_id === 'codebase-context' && row.task_id === taskId && row.repeat_index === 1 ); expect(attempt?.status).toBe('setup_failed'); expect(attempt?.scoring.fallbackReason).toContain('missing_setup_index_measurement'); @@ -922,9 +1044,13 @@ describe('ContextBench Phase 40 baseline runner', () => { const sessionRoot = tempSessionRoot(); const taskId = manifest.tasks[0].instance_id; try { - execFileSync('node', ['scripts/contextbench-runner.mjs', '--baseline-snapshot', '--out', sessionRoot], { - encoding: 'utf8' - }); + execFileSync( + 'node', + ['scripts/contextbench-runner.mjs', '--baseline-snapshot', '--out', sessionRoot], + { + encoding: 'utf8' + } + ); execFileSync( 'node', [ @@ -942,7 +1068,9 @@ describe('ContextBench Phase 40 baseline runner', () => { { encoding: 'utf8' } ); const rows = readRows(sessionRoot); - const diagnosticRows = rows.filter((row) => row.run_id.startsWith('codebase-context-current')); + const diagnosticRows = rows.filter((row) => + row.run_id.startsWith('codebase-context-current') + ); expect(diagnosticRows.length).toBeGreaterThanOrEqual(3); expect(diagnosticRows.every((row) => row.lane_id === 'codebase-context')).toBe(true); expect(diagnosticRows.every((row) => row.scoring.claimBearing === false)).toBe(true); @@ -957,13 +1085,23 @@ describe('ContextBench Phase 40 baseline runner', () => { it('runs Phase 42 verification as read-only artifact-derived evidence and fails diagnostic sessions closed', () => { const tempRoot = mkdtempSync(path.join(tmpdir(), 'contextbench-phase42-verify-')); - const sessionRoot = path.join(tempRoot, 'benchmark-runs', 'contextbench', 'phase41', 'runner-smoke'); + const sessionRoot = path.join( + tempRoot, + 'benchmark-runs', + 'contextbench', + 'phase41', + 'runner-smoke' + ); const reportPath = path.join(tempRoot, 'phase42-report.json'); const taskId = manifest.tasks[0].instance_id; try { - execFileSync('node', ['scripts/contextbench-runner.mjs', '--baseline-snapshot', '--out', sessionRoot], { - encoding: 'utf8' - }); + execFileSync( + 'node', + ['scripts/contextbench-runner.mjs', '--baseline-snapshot', '--out', sessionRoot], + { + encoding: 'utf8' + } + ); execFileSync( 'node', [ @@ -1022,21 +1160,33 @@ describe('ContextBench Phase 40 baseline runner', () => { expect(report.failureCounts.official_evaluator_missing).toBeGreaterThan(0); expect(report.failureCounts.missing_required_run).toBeGreaterThan(0); expect(report.blockedClaims).toContain('Phase 42 passed'); - expect(readFileSync(path.join(sessionRoot, 'BASELINE-SESSION.json'), 'utf8')).toBe(sessionBefore); + expect(readFileSync(path.join(sessionRoot, 'BASELINE-SESSION.json'), 'utf8')).toBe( + sessionBefore + ); } finally { - rmSync(tempRoot, { recursive: true, force: true }); + cleanupTempRoot(tempRoot); } }); it('classifies diagnostic baseline arms as supplemental during Phase 42 verification', () => { const tempRoot = mkdtempSync(path.join(tmpdir(), 'contextbench-phase42-arms-')); - const sessionRoot = path.join(tempRoot, 'benchmark-runs', 'contextbench', 'phase41', 'runner-smoke'); + const sessionRoot = path.join( + tempRoot, + 'benchmark-runs', + 'contextbench', + 'phase41', + 'runner-smoke' + ); const reportPath = path.join(tempRoot, 'phase42-report.json'); const taskId = manifest.tasks[0].instance_id; try { - execFileSync('node', ['scripts/contextbench-runner.mjs', '--baseline-snapshot', '--out', sessionRoot], { - encoding: 'utf8' - }); + execFileSync( + 'node', + ['scripts/contextbench-runner.mjs', '--baseline-snapshot', '--out', sessionRoot], + { + encoding: 'utf8' + } + ); execFileSync( 'node', [ @@ -1076,16 +1226,20 @@ describe('ContextBench Phase 40 baseline runner', () => { expect(report.failureCounts.unexpected_run_row ?? 0).toBe(0); expect(report.failureCounts.denominator_count_mismatch).toBe(1); } finally { - rmSync(tempRoot, { recursive: true, force: true }); + cleanupTempRoot(tempRoot); } }); it('blocks baseline seal when terminal row completeness lacks Phase 42 claim evidence', () => { const sessionRoot = tempSessionRoot('phase41'); try { - execFileSync('node', ['scripts/contextbench-runner.mjs', '--baseline-snapshot', '--out', sessionRoot], { - encoding: 'utf8' - }); + execFileSync( + 'node', + ['scripts/contextbench-runner.mjs', '--baseline-snapshot', '--out', sessionRoot], + { + encoding: 'utf8' + } + ); execFileSync( 'node', [ diff --git a/tests/contextbench-baseline-schema-gate.test.ts b/tests/contextbench-baseline-schema-gate.test.ts index 69b7a93..0b9b80d 100644 --- a/tests/contextbench-baseline-schema-gate.test.ts +++ b/tests/contextbench-baseline-schema-gate.test.ts @@ -1,12 +1,5 @@ import { execFileSync } from 'node:child_process'; -import { - chmodSync, - mkdtempSync, - readFileSync, - realpathSync, - rmSync, - writeFileSync -} from 'node:fs'; +import { chmodSync, mkdtempSync, readFileSync, realpathSync, rmSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import path from 'node:path'; import { describe, expect, it, vi } from 'vitest'; @@ -44,22 +37,13 @@ function childEnv(overrides: NodeJS.ProcessEnv = {}): NodeJS.ProcessEnv { return { ...env, ...overrides }; } -function ignoreWindowsTempCleanupRace(error: unknown): void { - const code = (error as NodeJS.ErrnoException).code; - if (!['EBUSY', 'ENOTEMPTY', 'EPERM'].includes(code ?? '')) throw error; -} - function cleanupSessionRoot(sessionRoot: string): void { - try { - rmSync(path.dirname(path.dirname(path.dirname(path.dirname(sessionRoot)))), { - recursive: true, - force: true, - maxRetries: 10, - retryDelay: 200 - }); - } catch (error) { - ignoreWindowsTempCleanupRace(error); - } + rmSync(path.dirname(path.dirname(path.dirname(path.dirname(sessionRoot)))), { + recursive: true, + force: true, + maxRetries: 10, + retryDelay: 200 + }); } function normalizeFilesystemPath(filePath: string): string { @@ -454,9 +438,13 @@ describe('ContextBench Phase 40 schema gate', () => { }) ); try { - execFileSync('node', ['scripts/contextbench-runner.mjs', '--baseline-snapshot', '--out', sessionRoot], { - encoding: 'utf8' - }); + execFileSync( + 'node', + ['scripts/contextbench-runner.mjs', '--baseline-snapshot', '--out', sessionRoot], + { + encoding: 'utf8' + } + ); execFileSync( 'node', [ @@ -815,14 +803,8 @@ describe('ContextBench Phase 40 schema gate', () => { expect.arrayContaining(['base_commit_mismatch', 'problem_statement_hash_mismatch']) ); } finally { - rmSync(path.dirname(path.dirname(path.dirname(path.dirname(missingSessionRoot)))), { - recursive: true, - force: true - }); - rmSync(path.dirname(path.dirname(path.dirname(path.dirname(wrongCommitSessionRoot)))), { - recursive: true, - force: true - }); + cleanupSessionRoot(missingSessionRoot); + cleanupSessionRoot(wrongCommitSessionRoot); rmSync(payloadDir, { recursive: true, force: true }); rmSync(wrongCommitRepo, { recursive: true, force: true }); } diff --git a/tests/eval-harness.test.ts b/tests/eval-harness.test.ts index 9d483bb..faf0058 100644 --- a/tests/eval-harness.test.ts +++ b/tests/eval-harness.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it, vi } from 'vitest'; import { CodebaseSearcher } from '../src/core/search.js'; import type { CodeChunk, SearchResult } from '../src/types/index.js'; +import { asSearcherTestHarness, materializeSourceSnapshot } from './helpers/source-snapshot.js'; import type { EvalFixture, EvalQuery } from '../src/eval/types.js'; import { countUtf8Bytes, @@ -24,7 +25,7 @@ function createChunk( filePath, relativePath: filePath.replace(/^.*?[/\\](?:src|libs|apps)[/\\]/, ''), startLine: 1, - endLine: 40, + endLine: content.split('\n').length, language: 'typescript', framework: 'generic', componentType: 'service', @@ -49,10 +50,11 @@ function setupSearcherWithResultsByQuery( byQuery: Record, fallback: SearchResult[] = [] ): CodebaseSearcher { - const searcher = new CodebaseSearcher('C:/repo') as any; - searcher.initialized = true; - searcher.search = vi.fn(async (query: string) => byQuery[query] ?? fallback); - return searcher as CodebaseSearcher; + const searcher = new CodebaseSearcher('C:/repo'); + const harness = asSearcherTestHarness(searcher); + harness.initialized = true; + harness.search = vi.fn(async (query: string) => byQuery[query] ?? fallback); + return searcher; } function fixtureForSingleQuery(query: EvalQuery): EvalFixture { @@ -268,23 +270,30 @@ describe('Eval Harness - integration with CodebaseSearcher (mocked)', () => { { componentType: 'unknown' } ); - const searcher = new CodebaseSearcher('C:/repo') as any; - searcher.initialized = true; - searcher.embeddingProvider = { - embed: vi.fn(async () => new Array(384).fill(0.01)) - }; - searcher.storageProvider = { - search: vi.fn(async () => [ - { chunk: authServiceChunk, score: 0.65 }, - { chunk: e2eChunk, score: 0.55 } - ]), - count: vi.fn(async () => 2) - }; - searcher.fuseIndex = null; - searcher.patternIntelligence = null; + const snapshot = await materializeSourceSnapshot([authServiceChunk, e2eChunk]); + try { + const searcher = new CodebaseSearcher(snapshot.rootPath); + const harness = asSearcherTestHarness(searcher); + harness.initialized = true; + harness.sourceManifest = snapshot.manifest; + harness.embeddingProvider = { + embed: vi.fn(async () => new Array(384).fill(0.01)) + }; + harness.storageProvider = { + search: vi.fn(async () => [ + { chunk: authServiceChunk, score: 0.65 }, + { chunk: e2eChunk, score: 0.55 } + ]), + count: vi.fn(async () => 2) + }; + harness.fuseIndex = null; + harness.patternIntelligence = null; - const results = await (searcher as CodebaseSearcher).search('AuthService', 5); - expect(results.length).toBeGreaterThan(0); - expect(results[0].filePath).toContain('auth.service.ts'); + const results = await searcher.search('AuthService', 5); + expect(results.length).toBeGreaterThan(0); + expect(results[0].filePath).toContain('auth.service.ts'); + } finally { + await snapshot.cleanup(); + } }); }); diff --git a/tests/excluded-import-freshness.test.ts b/tests/excluded-import-freshness.test.ts new file mode 100644 index 0000000..3d469d5 --- /dev/null +++ b/tests/excluded-import-freshness.test.ts @@ -0,0 +1,410 @@ +import { promises as fs } from 'node:fs'; +import path from 'node:path'; +import os from 'node:os'; +import { afterEach, beforeAll, expect, it, vi } from 'vitest'; +import { z } from 'zod'; +import { CodebaseIndexer } from '../src/core/indexer.js'; +import { analyzerRegistry } from '../src/core/analyzer-registry.js'; +import { GenericAnalyzer } from '../src/analyzers/generic/index.js'; +import { resolveIndexArtifact } from '../src/core/index-generation.js'; +import { RELATIONSHIPS_FILENAME } from '../src/constants/codebase-context.js'; +import { makePaths } from '../src/project-state.js'; +import { handle } from '../src/tools/get-code-relationships.js'; +import { InternalSourceResolver } from '../src/utils/internal-source-resolver.js'; +import type { ToolContext } from '../src/tools/types.js'; + +const roots: string[] = []; +beforeAll(() => analyzerRegistry.register(new GenericAnalyzer())); +afterEach(async () => { + vi.restoreAllMocks(); + for (const root of roots.splice(0)) await fs.rm(root, { recursive: true, force: true }); +}); + +async function fixture() { + const root = await fs.mkdtemp(path.join(os.tmpdir(), 'cbc-unindexed-')); + roots.push(root); + return root; +} +async function write(root: string, relative: string, content: string) { + const file = path.join(root, relative); + await fs.mkdir(path.dirname(file), { recursive: true }); + await fs.writeFile(file, content); +} +function index(rootPath: string, exclude?: string[], incrementalOnly = false) { + return new CodebaseIndexer({ + rootPath, + config: { skipEmbedding: true, ...(exclude ? { exclude } : {}) }, + incrementalOnly + }).index(); +} +async function query(rootPath: string) { + const ctx: ToolContext = { + rootPath, + paths: makePaths(rootPath), + indexState: { status: 'ready' }, + performIndexing: () => undefined + }; + const response = await handle({ file: 'main.ts', max_hops: 1 }, ctx); + const text = response.content?.[0]?.text ?? '{}'; + expect(Buffer.byteLength(text)).toBeLessThanOrEqual(4000); + return z + .object({ + status: z.string(), + truncated: z.boolean().optional(), + outputNote: z.string().optional() + }) + .passthrough() + .parse(JSON.parse(text)); +} +const Stored = z + .object({ + header: z.record(z.string(), z.unknown()), + graph: z + .object({ + imports: z.record(z.string(), z.array(z.string())), + unresolvedEdges: z.array(z.record(z.string(), z.unknown())).default([]) + }) + .passthrough() + }) + .passthrough(); +async function sidecar(root: string) { + const file = await resolveIndexArtifact(root, RELATIONSHIPS_FILENAME); + const value = Stored.parse(JSON.parse(await fs.readFile(file, 'utf8'))); + return { file, value, save: () => fs.writeFile(file, JSON.stringify(value)) }; +} + +it.each([ + { statement: "import './dist/target';", exclude: undefined, target: 'dist/target.ts' }, + { statement: "export * from './dist/target';", exclude: undefined, target: 'dist/target.ts' }, + { + statement: "import './private/target';", + exclude: ['**/private/**'], + target: 'private/target.ts' + } +])( + 'keeps an existing excluded target current across full and incremental refresh: $statement', + async ({ statement, exclude, target }) => { + const root = await fixture(); + await write(root, 'main.ts', `${statement}\nexport const main = true;\n`); + await write(root, target, 'export const target = true;\n'); + for (const incremental of [false, false, true]) { + await index(root, exclude, incremental); + expect(await query(root)).toMatchObject({ + status: 'success', + paths: [], + unresolved: [expect.objectContaining({ reason: 'not-indexed', target })] + }); + const stored = await sidecar(root); + expect(stored.value.graph.imports['main.ts'] ?? []).toEqual([]); + expect(stored.value.graph.unresolvedEdges).toContainEqual( + expect.objectContaining({ reason: 'not-indexed', target }) + ); + } + } +); + +it('requires one actual refresh of an old not-found sidecar and then remains current', async () => { + const root = await fixture(); + await write(root, 'main.ts', "import './dist/target';\n"); + await write(root, 'dist/target.ts', 'export const target = true;\n'); + await index(root); + const stored = await sidecar(root); + const edge = stored.value.graph.unresolvedEdges[0]; + if (!edge) throw new Error('Expected stored unresolved import'); + edge.reason = 'not-found'; + delete edge.target; + await stored.save(); + expect(await query(root)).toMatchObject({ status: 'stale_edges', paths: [] }); + for (let refresh = 0; refresh < 2; refresh++) { + await index(root); + expect(await query(root)).toMatchObject({ + status: 'success', + unresolved: [expect.objectContaining({ reason: 'not-indexed' })] + }); + } +}); + +it('does not suppress a newly appeared excluded target using manifest nonmembership', async () => { + const root = await fixture(); + await write(root, 'main.ts', "import './dist/target';\n"); + await index(root); + expect(await query(root)).toMatchObject({ + status: 'success', + unresolved: [expect.objectContaining({ reason: 'not-found' })] + }); + await write(root, 'dist/target.ts', 'export const target = true;\n'); + expect(await query(root)).toMatchObject({ status: 'stale_edges', paths: [] }); + await index(root); + expect(await query(root)).toMatchObject({ + status: 'success', + unresolved: [expect.objectContaining({ reason: 'not-indexed' })] + }); +}); + +it('rejects a new included higher-priority target even when its bytes match the excluded fallback', async () => { + const root = await fixture(); + await write(root, 'main.ts', "import './target';\n"); + await write(root, 'target.js', 'export const target = true;\n'); + await index(root, ['**/*.js']); + expect(await query(root)).toMatchObject({ + status: 'success', + unresolved: [expect.objectContaining({ target: 'target.js' })] + }); + await write(root, 'target.ts', 'export const target = true;\n'); + expect(await query(root)).toMatchObject({ status: 'stale_edges', paths: [], unresolved: [] }); +}); + +it('distinguishes a selected target missing its source fingerprint and recovers after a successful read', async () => { + const root = await fixture(); + await write(root, 'main.ts', "import './target';\n"); + await write(root, 'target.ts', 'export const target = true;\n'); + const target = await fs.realpath(path.join(root, 'target.ts')); + const read = fs.readFile; + const reads = vi.spyOn(fs, 'readFile').mockImplementation((file, options) => { + if (String(file) === target && options === undefined) + return Promise.reject( + Object.assign(new Error('Controlled source read failure'), { code: 'EACCES' }) + ); + return read(file, options); + }); + const stats = await index(root); + reads.mockRestore(); + expect(stats.errors).toContainEqual( + expect.objectContaining({ filePath: path.join(root, 'target.ts'), phase: 'analyzing' }) + ); + const stored = await sidecar(root); + expect(stored.value.graph.unresolvedEdges).toContainEqual( + expect.objectContaining({ reason: 'analysis-unavailable', target: 'target.ts' }) + ); + expect(await query(root)).toMatchObject({ + status: 'success', + paths: [], + unresolved: [expect.objectContaining({ reason: 'analysis-unavailable', target: 'target.ts' })], + message: expect.stringMatching(/indexing errors/i) + }); + await index(root); + expect(await query(root)).toMatchObject({ + status: 'success', + unresolved: [], + paths: [expect.objectContaining({ nodes: ['main.ts', 'target.ts'] })] + }); +}); + +it.each(['throw', 'skip'])( + 'does not confuse post-capture analyzer %s with a missing source fingerprint', + async (behavior) => { + const root = await fixture(); + await write(root, 'main.ts', "import './target';\n"); + await write(root, 'target.ts', 'export const target = true;\n'); + const analyze = analyzerRegistry.analyzeFile.bind(analyzerRegistry); + const analysis = vi + .spyOn(analyzerRegistry, 'analyzeFile') + .mockImplementation((file, content, options) => { + if (file === path.join(root, 'target.ts')) + return behavior === 'throw' + ? Promise.reject(new Error('Controlled analyzer failure after byte capture')) + : Promise.resolve(null); + return analyze(file, content, options); + }); + const stats = await index(root); + analysis.mockRestore(); + if (behavior === 'throw') + expect(stats.errors).toContainEqual(expect.objectContaining({ phase: 'analyzing' })); + else expect(stats.skippedFiles).toBeGreaterThan(0); + expect(await query(root)).toMatchObject({ + status: 'success', + unresolved: [], + paths: [expect.objectContaining({ nodes: ['main.ts', 'target.ts'] })] + }); + } +); + +it('captures physical target identity through root and target junction aliases and detects retargeting', async () => { + const base = await fixture(); + const root = path.join(base, 'physical'); + await write(root, 'main.ts', "import './linked/target';\n"); + await write(root, 'dist/first/target.ts', 'export const target = true;\n'); + await write(root, 'dist/second/target.ts', 'export const target = true;\n'); + const alias = path.join(base, 'alias'); + const link = path.join(root, 'linked'); + await fs.symlink(root, alias, 'junction'); + await fs.symlink(path.join(root, 'dist/first'), link, 'junction'); + await index(root); + expect(await query(alias)).toMatchObject({ + status: 'success', + unresolved: [expect.objectContaining({ target: 'dist/first/target.ts' })] + }); + const stored = await sidecar(alias); + expect(stored.value.graph.unresolvedEdges).toContainEqual( + expect.objectContaining({ target: 'dist/first/target.ts' }) + ); + await fs.unlink(link); + await fs.symlink(path.join(root, 'dist/second'), link, 'junction'); + expect(await query(alias)).toMatchObject({ status: 'stale_edges', paths: [] }); + const outside = path.join(base, 'physical-other'); + await write(outside, 'target.ts', 'export const target = true;\n'); + await fs.unlink(link); + await fs.symlink(outside, link, 'junction'); + expect(await query(alias)).toMatchObject({ status: 'stale_edges', paths: [] }); +}); + +it.each([ + { target: undefined }, + { target: null }, + { target: 4 }, + { target: [] }, + { target: '/absolute.ts' }, + { target: '../escape.ts' }, + { target: 'C:drive-relative.ts' }, + { target: 'a/../escape.ts' } +])('rejects malformed target proof before traversal: $target', async ({ target }) => { + const root = await fixture(); + await write(root, 'main.ts', "import './dist/target';\n"); + await write(root, 'dist/target.ts', 'export const target = true;\n'); + await index(root); + const stored = await sidecar(root); + const edge = stored.value.graph.unresolvedEdges[0]; + if (!edge) throw new Error('Expected unresolved import'); + edge.reason = 'not-indexed'; + edge.target = target; + await stored.save(); + const resolve = vi.spyOn(InternalSourceResolver.prototype, 'resolve'); + expect(await query(root)).toMatchObject({ status: 'graph_unavailable' }); + expect(resolve).not.toHaveBeenCalled(); +}); + +it.each([ + null, + { from: 4 }, + { from: 'main.ts', source: null }, + { from: 'main.ts', reason: 'invented', source: {} } +])('rejects malformed unresolved records before traversal: %j', async (edge) => { + const root = await fixture(); + await write(root, 'main.ts', "import './missing';\n"); + await index(root); + const stored = await sidecar(root); + await fs.writeFile( + stored.file, + JSON.stringify({ ...stored.value, graph: { ...stored.value.graph, unresolvedEdges: [edge] } }) + ); + expect(await query(root)).toMatchObject({ status: 'graph_unavailable' }); +}); + +it('keeps canonical proof captured before a pending selected target link is retargeted', async () => { + const root = await fixture(); + await write(root, 'main.ts', "import './linked/target';\n"); + await write(root, 'first/target.ts', 'export const target = true;\n'); + await write(root, 'second/target.ts', 'export const target = true;\n'); + const link = path.join(root, 'linked'); + await fs.symlink(path.join(root, 'first'), link, 'junction'); + const first = await fs.realpath(path.join(root, 'first/target.ts')); + const read = fs.readFile; + const reads = vi.spyOn(fs, 'readFile').mockImplementation((file, options) => { + if (String(file) === first && options === undefined) + return Promise.reject(new Error('Controlled missing target snapshot')); + return read(file, options); + }); + const resolve = InternalSourceResolver.prototype.resolve; + let retargeted = false; + const resolutions = vi + .spyOn(InternalSourceResolver.prototype, 'resolve') + .mockImplementation(async function ( + this: InternalSourceResolver, + importer, + specifier, + allowUnindexed + ) { + const result = await resolve.call(this, importer, specifier, allowUnindexed); + if (!retargeted && specifier === './linked/target') { + expect(result).toMatchObject({ status: 'resolved', target: 'first/target.ts' }); + retargeted = true; + await fs.unlink(link); + await fs.symlink(path.join(root, 'second'), link, 'junction'); + } + return result; + }); + await index(root); + reads.mockRestore(); + resolutions.mockRestore(); + expect(retargeted).toBe(true); + const stored = await sidecar(root); + expect(stored.value.graph.unresolvedEdges).toContainEqual( + expect.objectContaining({ reason: 'analysis-unavailable', target: 'first/target.ts' }) + ); + expect(await query(root)).toMatchObject({ status: 'stale_edges', paths: [] }); +}); + +it('uses platform case rules for physical target identity without accepting a different POSIX file', async () => { + const root = await fixture(); + await write(root, 'main.ts', "import './dist/Case';\n"); + await write(root, 'dist/Case.ts', 'export const target = true;\n'); + if (process.platform !== 'win32') + await write(root, 'dist/case.ts', 'export const target = true;\n'); + await index(root); + const stored = await sidecar(root); + const edge = stored.value.graph.unresolvedEdges[0]; + if (!edge) throw new Error('Expected unresolved import'); + expect(edge.target).toBe('dist/Case.ts'); + edge.target = 'dist/case.ts'; + await stored.save(); + expect(await query(root)).toMatchObject({ + status: process.platform === 'win32' ? 'success' : 'stale_edges', + paths: [] + }); +}); + +it('rejects changed aliases, missing targets and modified importers without inventing an edge', async () => { + const root = await fixture(); + await write(root, 'main.ts', "import '@generated';\n"); + await write(root, 'dist/first.ts', 'export const target = true;\n'); + await write(root, 'dist/second.ts', 'export const target = true;\n'); + const config = (target: string) => + JSON.stringify({ compilerOptions: { baseUrl: '.', paths: { '@generated': [target] } } }); + await write(root, 'tsconfig.json', config('dist/first.ts')); + await index(root); + expect(await query(root)).toMatchObject({ status: 'success' }); + await write(root, 'tsconfig.json', config('dist/second.ts')); + expect(await query(root)).toMatchObject({ status: 'stale_edges', paths: [] }); + await write(root, 'tsconfig.json', config('dist/first.ts')); + await fs.unlink(path.join(root, 'dist/first.ts')); + expect(await query(root)).toMatchObject({ status: 'stale_edges', paths: [] }); + await write(root, 'dist/first.ts', 'export const target = true;\n'); + await fs.appendFile(path.join(root, 'main.ts'), '// changed importer\n'); + expect(await query(root)).toMatchObject({ status: 'stale_edges', paths: [] }); +}); + +it('keeps target-bearing diagnostic inspections, cached source reads and response bytes bounded', async () => { + const root = await fixture(); + await write(root, 'main.ts', "import './dist/target';\n".repeat(1100)); + await write(root, 'dist/target.ts', 'export const target = true;\n'); + await index(root); + const reads = vi.spyOn(fs, 'readFile'); + const resolutions = vi.spyOn(InternalSourceResolver.prototype, 'resolve'); + const result = await query(root); + expect(result).toMatchObject({ + status: 'success', + truncated: true, + outputNote: expect.stringMatching(/inspection/i) + }); + expect(resolutions).toHaveBeenCalledTimes(1); + expect( + reads.mock.calls.filter(([file]) => String(file) === path.join(root, 'main.ts')) + ).toHaveLength(1); +}); + +it('includes target identity in the unresolved verification cache key', async () => { + const root = await fixture(); + await write(root, 'main.ts', "import './dist/target';\n"); + await write(root, 'dist/target.ts', 'export const target = true;\n'); + await index(root); + const stored = await sidecar(root); + const edge = stored.value.graph.unresolvedEdges[0]; + if (!edge) throw new Error('Expected unresolved import'); + stored.value.graph.unresolvedEdges.push({ ...edge, target: 'dist/another.ts' }); + await stored.save(); + expect(await query(root)).toMatchObject({ + status: 'stale_edges', + unresolved: [expect.objectContaining({ target: 'dist/target.ts' })], + staleUnresolvedEdges: [expect.objectContaining({ target: 'dist/another.ts' })] + }); +}); diff --git a/tests/file-graph-freshness.test.ts b/tests/file-graph-freshness.test.ts new file mode 100644 index 0000000..1884b9e --- /dev/null +++ b/tests/file-graph-freshness.test.ts @@ -0,0 +1,196 @@ +import { createHash } from 'crypto'; +import { promises as fs } from 'fs'; +import os from 'os'; +import path from 'path'; +import { afterEach, expect, it, vi } from 'vitest'; +import { + CODEBASE_CONTEXT_DIRNAME, + INDEX_FORMAT_VERSION, + INDEX_META_FILENAME, + INDEX_META_VERSION, + RELATIONSHIPS_FILENAME +} from '../src/constants/codebase-context.js'; +import { handle as getCodeRelationships } from '../src/tools/get-code-relationships.js'; +import type { ToolContext } from '../src/tools/types.js'; +import { InternalFileGraph } from '../src/utils/usage-tracker.js'; + +const roots: string[] = []; +afterEach(async () => { + vi.restoreAllMocks(); + await Promise.all(roots.splice(0).map((root) => fs.rm(root, { recursive: true, force: true }))); +}); + +async function fixture() { + const root = await fs.mkdtemp(path.join(os.tmpdir(), 'cbc-graph-freshness-')); + roots.push(root); + const graph = new InternalFileGraph(root); + const sources = new Map(); + const baseDir = path.join(root, CODEBASE_CONTEXT_DIRNAME); + await fs.mkdir(baseDir); + const write = async (file: string, source: string) => { + sources.set(file, source); + await fs.writeFile(path.join(root, file), source); + }; + const fingerprint = (file: string) => + createHash('sha256') + .update(sources.get(file) ?? '') + .digest('hex'); + const edge = (from: string, to: string, line = 1, kind: 'import' | 'reexport' = 'import') => { + graph.trackImport(path.join(root, from), path.join(root, to), line, [], { + kind, + specifier: `./${to.slice(0, -3)}`, + sourceFingerprint: fingerprint(from), + targetFingerprint: fingerprint(to) + }); + }; + const save = async () => { + await fs.writeFile( + path.join(baseDir, INDEX_META_FILENAME), + JSON.stringify({ + metaVersion: INDEX_META_VERSION, + formatVersion: INDEX_FORMAT_VERSION, + buildId: 'freshness-fixture', + generatedAt: '2026-10-15T00:00:00.000Z', + toolVersion: 'fixture', + artifacts: { + keywordIndex: { path: 'index.json' }, + vectorDb: { path: 'index', provider: 'none', embeddingEnabled: false } + } + }) + ); + await fs.writeFile( + path.join(baseDir, RELATIONSHIPS_FILENAME), + JSON.stringify({ + header: { buildId: 'freshness-fixture', formatVersion: INDEX_FORMAT_VERSION }, + graph: graph.toJSON() + }) + ); + }; + const ctx: ToolContext = { + rootPath: root, + paths: { + baseDir, + intelligence: path.join(baseDir, 'intelligence.json'), + memory: path.join(baseDir, 'memory.json'), + health: path.join(baseDir, 'health.json'), + keywordIndex: path.join(baseDir, 'index.json'), + vectorDb: path.join(baseDir, 'index') + }, + indexState: { status: 'ready' }, + performIndexing: () => undefined + }; + const query = async ( + file: string, + direction: 'incoming' | 'outgoing', + maxHops = 1, + maxNodes = 100 + ) => { + const response = await getCodeRelationships( + { file, direction, max_hops: maxHops, max_nodes: maxNodes }, + ctx + ); + const text = response.content?.[0]?.text ?? '{}'; + expect(Buffer.byteLength(text, 'utf8')).toBeLessThanOrEqual(4000); + return JSON.parse(text) as { + status: string; + paths: Array<{ nodes: string[]; edges: Array<{ kind: string }> }>; + truncated: boolean; + outputNote?: string; + message: string; + }; + }; + return { root, write, edge, save, query }; +} + +it.each([2, 100])( + 'keeps a current outgoing edge after forty stale targets with accepted node cap %i', + async (maxNodes) => { + const f = await fixture(); + const stale = Array.from({ length: 40 }, (_, n) => `stale-${n}.ts`); + await f.write( + 'entry.ts', + [...stale, 'support.ts'].map((file) => `import './${file.slice(0, -3)}';`).join('\n') + ); + for (const [index, file] of [...stale, 'support.ts'].entries()) { + await f.write(file, 'export const value = true;\n'); + f.edge('entry.ts', file, index + 1); + } + await f.save(); + for (const file of stale) await f.write(file, 'export const changed = true;\n'); + + const result = await f.query('entry.ts', 'outgoing', 1, maxNodes); + expect(result.status).toBe('stale_edges'); + expect(result.paths.map(({ nodes }) => nodes)).toEqual([['entry.ts', 'support.ts']]); + expect(result.paths.some(({ nodes }) => nodes.some((node) => node.startsWith('stale-')))).toBe( + false + ); + } +); + +it.each([2, 100])( + 'keeps a current incoming importer after forty stale importers with accepted node cap %i', + async (maxNodes) => { + const f = await fixture(); + await f.write('target.ts', 'export const value = true;\n'); + const stale = Array.from({ length: 40 }, (_, n) => `stale-${n}.ts`); + for (const file of [...stale, 'current.ts']) { + await f.write(file, "import './target';\n"); + f.edge(file, 'target.ts'); + } + await f.save(); + for (const file of stale) await f.write(file, 'export const changed = true;\n'); + + const result = await f.query('target.ts', 'incoming', 1, maxNodes); + expect(result.paths.map(({ nodes }) => nodes)).toEqual([['target.ts', 'current.ts']]); + } +); + +it('follows a current typed three-hop chain past stale branches and terminates its cycle', async () => { + const f = await fixture(); + const stale = Array.from({ length: 40 }, (_, n) => `stale-${n}.ts`); + await f.write( + 'entry.ts', + [...stale, 'barrel.ts'].map((file) => `import './${file.slice(0, -3)}';`).join('\n') + ); + await f.write('barrel.ts', "export * from './middle';\n"); + await f.write('middle.ts', "import './leaf';\n"); + await f.write('leaf.ts', "import './barrel';\n"); + for (const file of stale) { + await f.write(file, 'export const value = true;\n'); + f.edge('entry.ts', file); + } + f.edge('entry.ts', 'barrel.ts'); + f.edge('barrel.ts', 'middle.ts', 1, 'reexport'); + f.edge('middle.ts', 'leaf.ts'); + f.edge('leaf.ts', 'barrel.ts'); + await f.save(); + for (const file of stale) await f.write(file, 'export const changed = true;\n'); + + const result = await f.query('entry.ts', 'outgoing', 3, 4); + const chain = result.paths.find(({ nodes }) => nodes.length === 4); + expect(chain?.nodes).toEqual(['entry.ts', 'barrel.ts', 'middle.ts', 'leaf.ts']); + expect(chain?.edges.map(({ kind }) => kind)).toEqual(['import', 'reexport', 'import']); + expect(result.paths.every(({ nodes }) => new Set(nodes).size === nodes.length)).toBe(true); +}); + +it('bounds stale edge inspections and explains when a current edge remains beyond that budget', async () => { + const f = await fixture(); + await f.write('entry.ts', `${"import './stale';\n".repeat(1100)}import './current';\n`); + await f.write('stale.ts', 'export const value = true;\n'); + await f.write('current.ts', 'export const current = true;\n'); + for (let line = 1; line <= 1100; line++) f.edge('entry.ts', 'stale.ts', line); + f.edge('entry.ts', 'current.ts', 1101); + await f.save(); + await f.write('stale.ts', 'export const changed = true;\n'); + const reads = vi.spyOn(fs, 'readFile'); + + const result = await f.query('entry.ts', 'outgoing'); + const staleReads = reads.mock.calls.filter( + ([file]) => String(file) === path.join(f.root, 'stale.ts') + ); + expect(staleReads.length).toBeLessThanOrEqual(1000); + expect(result.paths).toEqual([]); + expect(result.truncated).toBe(true); + expect(result.outputNote).toMatch(/inspection/i); + expect(result.message).toMatch(/refresh indexing/i); +}); diff --git a/tests/file-graph-indexing-negatives.test.ts b/tests/file-graph-indexing-negatives.test.ts new file mode 100644 index 0000000..c5ecca8 --- /dev/null +++ b/tests/file-graph-indexing-negatives.test.ts @@ -0,0 +1,42 @@ +import { resolveIndexArtifact } from '../src/core/index-generation.js'; +import { afterAll, expect, it } from 'vitest'; +import { promises as fs } from 'fs'; +import os from 'os'; +import path from 'path'; +import { CodebaseIndexer } from '../src/core/indexer.js'; +import { + CODEBASE_CONTEXT_DIRNAME, + RELATIONSHIPS_FILENAME +} from '../src/constants/codebase-context.js'; + +const root = await fs.mkdtemp(path.join(os.tmpdir(), 'cbc-graph-negative-')); +afterAll(async () => fs.rm(root, { recursive: true, force: true })); + +it('never serializes a guessed .ts target when only a differently named target or no target exists', async () => { + const source = path.join(root, 'src'); + await fs.mkdir(source, { recursive: true }); + await fs.writeFile(path.join(source, 'entry.ts'), "import './missing';\nimport './present';\n"); + await fs.writeFile(path.join(source, 'present.tsx'), 'export const present = true;\n'); + const indexer = new CodebaseIndexer({ rootPath: root, config: { skipEmbedding: true } }); + await indexer.index(); + + const sidecarPath = await resolveIndexArtifact(root, RELATIONSHIPS_FILENAME); + const sidecar = JSON.parse(await fs.readFile(sidecarPath, 'utf8')) as { + graph: { + imports: Record; + importDetails?: Record }>>; + unresolvedEdges?: Array<{ source: { specifier: string }; reason: string }>; + }; + }; + const importer = Object.keys(sidecar.graph.imports).find((file) => file.endsWith('src/entry.ts')); + expect(importer).toBeDefined(); + const targets = sidecar.graph.imports[importer ?? ''] ?? []; + expect(targets).toContain('src/present.tsx'); + expect(targets).not.toContain('src/missing.ts'); + expect(sidecar.graph.unresolvedEdges).toContainEqual( + expect.objectContaining({ + source: expect.objectContaining({ specifier: './missing' }), + reason: 'not-found' + }) + ); +}); diff --git a/tests/file-graph-integration.test.ts b/tests/file-graph-integration.test.ts new file mode 100644 index 0000000..a857b6e --- /dev/null +++ b/tests/file-graph-integration.test.ts @@ -0,0 +1,406 @@ +import { resolveIndexArtifact } from '../src/core/index-generation.js'; +import { afterAll, expect, it } from 'vitest'; +import { promises as fs } from 'fs'; +import os from 'os'; +import path from 'path'; +import { CodebaseIndexer } from '../src/core/indexer.js'; +import { + CODEBASE_CONTEXT_DIRNAME, + RELATIONSHIPS_FILENAME +} from '../src/constants/codebase-context.js'; +import { handle as getCodeRelationships } from '../src/tools/get-code-relationships.js'; +import type { ToolContext } from '../src/tools/types.js'; + +const root = await fs.mkdtemp(path.join(os.tmpdir(), 'cbc-graph-integration-')); +afterAll(async () => fs.rm(root, { recursive: true, force: true })); + +async function write(relative: string, content: string, projectRoot = root): Promise { + const fullPath = path.join(projectRoot, relative); + await fs.mkdir(path.dirname(fullPath), { recursive: true }); + await fs.writeFile(fullPath, content); +} + +async function readGraph(projectRoot = root): Promise<{ + header: { buildId: string }; + graph: { + imports: Record; + importDetails: Record< + string, + Record< + string, + { + sources?: Array<{ + kind: string; + specifier: string; + sourceFingerprint: string; + targetFingerprint: string; + }>; + } + > + >; + unresolvedEdges?: Array<{ source: { specifier: string }; reason: string }>; + }; +}> { + return JSON.parse( + await fs.readFile(await resolveIndexArtifact(projectRoot, RELATIONSHIPS_FILENAME), 'utf8') + ) as { + header: { buildId: string }; + graph: { + imports: Record; + importDetails: Record< + string, + Record< + string, + { + sources?: Array<{ + kind: string; + specifier: string; + sourceFingerprint: string; + targetFingerprint: string; + }>; + } + > + >; + unresolvedEdges?: Array<{ source: { specifier: string }; reason: string }>; + }; + }; +} + +function context(projectRoot = root): ToolContext { + const contextDir = path.join(projectRoot, CODEBASE_CONTEXT_DIRNAME); + return { + rootPath: projectRoot, + paths: { + baseDir: contextDir, + intelligence: path.join(contextDir, 'intelligence.json'), + memory: path.join(contextDir, 'memory.json'), + health: path.join(contextDir, 'health.json'), + keywordIndex: path.join(contextDir, 'keyword-index.json'), + vectorDb: path.join(contextDir, 'vector-db') + }, + indexState: { status: 'ready' }, + performIndexing: () => undefined + }; +} + +it('verifies the incoming three-hop alias chain in a nested root with only tsconfig.base.json', async () => { + const host = path.join(root, 'base-config-host'); + const projectRoot = path.join(host, 'checkout with spaces'); + const utility = 'libs/web/shared/utils/src/lib/time-util.ts'; + const barrel = 'libs/web/shared/utils/src/index.ts'; + const pipe = 'libs/web/shared/pipes/duration-pipe/src/lib/duration.pipe.ts'; + const module = 'libs/web/shared/pipes/duration-pipe/src/lib/duration-pipe.module.ts'; + const alias = '@angular-spotify/web/shared/utils'; + await write( + 'tsconfig.json', + JSON.stringify({ + compilerOptions: { baseUrl: '.', paths: { [alias]: ['wrong-parent-target.ts'] } } + }), + host + ); + await write( + 'tsconfig.base.json', + JSON.stringify({ + compilerOptions: { baseUrl: '.', paths: { [alias]: [barrel] } } + }), + projectRoot + ); + await write( + utility, + 'export class TimeUtil { static formatDuration(value: number) { return String(value); } }\n', + projectRoot + ); + await write(barrel, "export * from './lib/time-util';\n", projectRoot); + await write( + pipe, + `import { TimeUtil } from '${alias}';\nimport '@unmapped/package';\nexport class DurationPipe { transform(value: number) { return TimeUtil.formatDuration(value); } }\n`, + projectRoot + ); + await write( + module, + "import { DurationPipe } from './duration.pipe';\nexport const pipes = [DurationPipe];\n", + projectRoot + ); + + await new CodebaseIndexer({ rootPath: projectRoot, config: { skipEmbedding: true } }).index(); + const graph = await readGraph(projectRoot); + expect(graph.graph.imports[barrel]).toContain(utility); + expect(graph.graph.imports[pipe]).toContain(barrel); + expect(graph.graph.imports[module]).toContain(pipe); + expect(graph.graph.importDetails[pipe]?.[barrel]?.sources?.[0]).toMatchObject({ + kind: 'import', + specifier: alias, + sourceFingerprint: expect.any(String), + targetFingerprint: expect.any(String) + }); + expect(graph.graph.unresolvedEdges).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + source: expect.objectContaining({ specifier: '@unmapped/package' }), + reason: 'external' + }) + ]) + ); + expect(graph.graph.unresolvedEdges?.some(({ source }) => source.specifier === alias)).toBe(false); + + const response = await getCodeRelationships( + { file: utility, direction: 'incoming', max_hops: 3, max_nodes: 10 }, + context(projectRoot) + ); + const result = JSON.parse(response.content?.[0]?.text ?? '{}') as { + status: string; + paths: Array<{ nodes: string[] }>; + staleEdges?: unknown[]; + truncated: boolean; + }; + expect(result.status).toBe('success'); + expect(result.paths.map(({ nodes }) => nodes)).toContainEqual([utility, barrel, pipe, module]); + expect(result.staleEdges ?? []).toHaveLength(0); + expect(result.truncated).toBe(false); +}); + +it('resolves source aliases, directory indexes, typed barrels and three hops, then refreshes after delete/rename', async () => { + await write( + 'tsconfig.json', + JSON.stringify({ + compilerOptions: { baseUrl: '.', paths: { '@core/*': ['packages/core/src/*'] } } + }) + ); + await write( + 'src/main.ts', + "import './barrel';\nimport './present';\nimport './missing';\nimport './dir';\nimport '@core/actual';\nimport 'external-pkg';\n" + ); + await write('src/barrel.ts', "export { actual } from '@core/actual';\n"); + await write('src/present.tsx', 'export const present = true;\n'); + await write('src/dir/index.ts', 'export const indexed = true;\n'); + await write('packages/core/src/actual.ts', "import './impl';\nexport const actual = true;\n"); + await write( + 'packages/core/src/impl.ts', + "import { actual } from '@core/actual';\nexport const impl = actual;\n" + ); + + await new CodebaseIndexer({ rootPath: root, config: { skipEmbedding: true } }).index(); + const graph = await readGraph(); + const main = Object.keys(graph.graph.imports).find((file) => file.endsWith('src/main.ts')); + expect(main).toBeDefined(); + const targets = graph.graph.imports[main ?? ''] ?? []; + expect(targets).toContain('src/present.tsx'); + expect(targets).toContain('src/dir/index.ts'); + expect(targets).not.toContain('src/missing.ts'); + expect(graph.graph.unresolvedEdges).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + source: expect.objectContaining({ specifier: './missing' }), + reason: 'not-found' + }), + expect.objectContaining({ + source: expect.objectContaining({ specifier: 'external-pkg' }), + reason: 'external' + }) + ]) + ); + expect( + graph.graph.importDetails['src/barrel.ts']?.['packages/core/src/actual.ts']?.sources?.[0] + ).toMatchObject({ + kind: 'reexport', + specifier: '@core/actual', + sourceFingerprint: expect.any(String), + targetFingerprint: expect.any(String) + }); + + const response = await getCodeRelationships( + { file: 'src/main.ts', direction: 'outgoing', max_hops: 3, max_nodes: 30 }, + context() + ); + const result = JSON.parse(response.content?.[0]?.text ?? '{}') as { + status: string; + paths: Array<{ nodes: string[] }>; + unresolved: unknown[]; + }; + expect(result.status).toBe('success'); + expect( + result.paths.some( + ({ nodes }) => + nodes.join('|') === + 'src/main.ts|src/barrel.ts|packages/core/src/actual.ts|packages/core/src/impl.ts' + ) + ).toBe(true); + expect(result.unresolved).toHaveLength(2); + + await fs.rename( + path.join(root, 'packages/core/src/actual.ts'), + path.join(root, 'packages/core/src/actual-renamed.ts') + ); + await fs.rm(path.join(root, 'packages/core/src/impl.ts')); + await new CodebaseIndexer({ + rootPath: root, + config: { skipEmbedding: true }, + incrementalOnly: true + }).index(); + const refreshed = await readGraph(); + expect(refreshed.header.buildId).not.toBe(graph.header.buildId); + expect(refreshed.graph.imports['src/barrel.ts'] ?? []).not.toContain( + 'packages/core/src/actual.ts' + ); + expect(refreshed.graph.unresolvedEdges).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + from: 'src/barrel.ts', + source: expect.objectContaining({ specifier: '@core/actual' }) + }) + ]) + ); +}); + +it('rejects graph edges after either current endpoint changes without reindexing', async () => { + await Promise.all([ + fs.rm(path.join(root, 'src'), { recursive: true, force: true }), + fs.rm(path.join(root, 'packages'), { recursive: true, force: true }), + fs.rm(path.join(root, CODEBASE_CONTEXT_DIRNAME), { recursive: true, force: true }), + fs.rm(path.join(root, 'tsconfig.json'), { force: true }) + ]); + await write( + 'tsconfig.json', + JSON.stringify({ + compilerOptions: { baseUrl: '.', paths: { '@core/*': ['packages/core/src/*'] } } + }) + ); + await write('src/main.ts', "import '@core/actual';\n"); + await write('packages/core/src/actual.ts', 'export const actual = true;\n'); + await new CodebaseIndexer({ rootPath: root, config: { skipEmbedding: true } }).index(); + await fs.appendFile( + path.join(root, 'packages/core/src/actual.ts'), + 'export const changed = true;\n' + ); + + const response = await getCodeRelationships({ file: 'src/main.ts', max_hops: 1 }, context()); + const result = JSON.parse(response.content?.[0]?.text ?? '{}') as { + status: string; + paths: unknown[]; + staleEdges?: unknown[]; + }; + expect(result.status).toBe('stale_edges'); + expect(result.paths).toHaveLength(0); + expect(result.staleEdges).toHaveLength(1); + + await new CodebaseIndexer({ rootPath: root, config: { skipEmbedding: true } }).index(); + await fs.appendFile(path.join(root, 'src/main.ts'), '// changed importer\n'); + const changedSourceResponse = await getCodeRelationships( + { file: 'src/main.ts', max_hops: 1 }, + context() + ); + const changedSource = JSON.parse(changedSourceResponse.content?.[0]?.text ?? '{}') as { + status: string; + paths: unknown[]; + staleEdges?: unknown[]; + }; + expect(changedSource.status).toBe('stale_edges'); + expect(changedSource.paths).toHaveLength(0); + expect(changedSource.staleEdges).toHaveLength(1); +}); + +it('marks edges stale when current tsconfig resolution changes or a higher priority target appears', async () => { + await Promise.all([ + fs.rm(path.join(root, 'src'), { recursive: true, force: true }), + fs.rm(path.join(root, 'packages'), { recursive: true, force: true }), + fs.rm(path.join(root, CODEBASE_CONTEXT_DIRNAME), { recursive: true, force: true }), + fs.rm(path.join(root, 'tsconfig.json'), { force: true }) + ]); + await write( + 'tsconfig.json', + JSON.stringify({ + compilerOptions: { baseUrl: '.', paths: { '@core/*': ['packages/core/src/*'] } } + }) + ); + await write('src/main.ts', "import './feature';\nimport '@core/actual';\n"); + await write('src/feature.tsx', 'export const feature = true;\n'); + await write('packages/core/src/actual.ts', 'export const actual = true;\n'); + await write('packages/alternate/src/actual.ts', 'export const actual = false;\n'); + await new CodebaseIndexer({ rootPath: root, config: { skipEmbedding: true } }).index(); + const original = await getCodeRelationships({ file: 'src/main.ts', max_hops: 1 }, context()); + expect(JSON.parse(original.content?.[0]?.text ?? '{}')).toMatchObject({ status: 'success' }); + + await write('src/feature.ts', 'export const feature = false;\n'); + await write( + 'tsconfig.json', + JSON.stringify({ + compilerOptions: { baseUrl: '.', paths: { '@core/*': ['packages/alternate/src/*'] } } + }) + ); + const changedResolution = await getCodeRelationships( + { file: 'src/main.ts', max_hops: 1 }, + context() + ); + const result = JSON.parse(changedResolution.content?.[0]?.text ?? '{}') as { + status: string; + paths: unknown[]; + staleEdges: Array<{ from: string; to: string; specifier: string }>; + }; + expect(result.status).toBe('stale_edges'); + expect(result.paths).toHaveLength(0); + expect(result.staleEdges).toEqual( + expect.arrayContaining([ + expect.objectContaining({ specifier: './feature', to: 'src/feature.tsx' }), + expect.objectContaining({ specifier: '@core/actual', to: 'packages/core/src/actual.ts' }) + ]) + ); +}); + +it('bounds the complete serialized branching response and reports truncation', async () => { + await Promise.all([ + fs.rm(path.join(root, 'src'), { recursive: true, force: true }), + fs.rm(path.join(root, CODEBASE_CONTEXT_DIRNAME), { recursive: true, force: true }), + fs.rm(path.join(root, 'tsconfig.json'), { force: true }) + ]); + const imports: string[] = []; + for (let index = 0; index < 36; index += 1) { + const target = `branch-${String(index).padStart(2, '0')}-long-diagnostic-target-name.ts`; + imports.push(`import './${target.slice(0, -3)}';`); + await write(`src/${target}`, `export const branch${index} = ${index};\n`); + } + await write('src/main.ts', `${imports.join('\n')}\n`); + await new CodebaseIndexer({ rootPath: root, config: { skipEmbedding: true } }).index(); + + const response = await getCodeRelationships( + { file: 'src/main.ts', max_hops: 1, max_nodes: 100 }, + context() + ); + const payload = response.content?.[0]?.text ?? ''; + const result = JSON.parse(payload) as { paths: unknown[]; truncated: boolean }; + expect(Buffer.byteLength(payload, 'utf8')).toBeLessThanOrEqual(4000); + expect(result.paths.length).toBeGreaterThan(0); + expect(result.truncated).toBe(true); +}); + +it('rejects legacy proofless edges and reports a missing sidecar', async () => { + await Promise.all([ + fs.rm(path.join(root, 'src'), { recursive: true, force: true }), + fs.rm(path.join(root, CODEBASE_CONTEXT_DIRNAME), { recursive: true, force: true }), + fs.rm(path.join(root, 'tsconfig.json'), { force: true }) + ]); + await write('src/from.ts', "import './to';\n"); + await write('src/to.ts', 'export const value = true;\n'); + await new CodebaseIndexer({ rootPath: root, config: { skipEmbedding: true } }).index(); + + const sidecarPath = await resolveIndexArtifact(root, RELATIONSHIPS_FILENAME); + const sidecar = JSON.parse(await fs.readFile(sidecarPath, 'utf8')) as { + graph: { + imports: Record; + importDetails: Record>; + }; + }; + sidecar.graph.importDetails = { 'src/from.ts': { 'src/to.ts': {} } }; + await fs.writeFile(sidecarPath, JSON.stringify(sidecar)); + const legacy = await getCodeRelationships({ file: 'src/from.ts', max_hops: 1 }, context()); + expect(JSON.parse(legacy.content?.[0]?.text ?? '{}')).toMatchObject({ + status: 'stale_edges', + paths: [], + truncated: false + }); + + await fs.rm(sidecarPath); + const missing = await getCodeRelationships({ file: 'src/from.ts', max_hops: 1 }, context()); + expect(JSON.parse(missing.content?.[0]?.text ?? '{}')).toMatchObject({ + status: 'graph_unavailable' + }); +}); diff --git a/tests/file-graph-traversal.test.ts b/tests/file-graph-traversal.test.ts new file mode 100644 index 0000000..3d71baa --- /dev/null +++ b/tests/file-graph-traversal.test.ts @@ -0,0 +1,143 @@ +import { describe, expect, it } from 'vitest'; +import { InternalFileGraph, traverseFileGraph } from '../src/utils/usage-tracker.js'; + +describe('verified file graph traversal', () => { + it('round-trips typed source proofs and returns a three-hop path with edge evidence', () => { + const graph = new InternalFileGraph('C:/repo'); + graph.trackImport('C:/repo/src/a.ts', 'C:/repo/src/b.ts', 2, ['B'], { + kind: 'import', + specifier: './b', + sourceFingerprint: 'sha256:a', + targetFingerprint: 'sha256:b' + }); + graph.trackImport('C:/repo/src/b.ts', 'C:/repo/src/c/index.ts', 4, [], { + kind: 'reexport', + specifier: '../c', + sourceFingerprint: 'sha256:b', + targetFingerprint: 'sha256:c' + }); + graph.trackImport('C:/repo/src/c/index.ts', 'C:/repo/src/d.tsx', 1, [], { + kind: 'import', + specifier: '../d', + sourceFingerprint: 'sha256:c', + targetFingerprint: 'sha256:d' + }); + graph.trackUnresolved( + 'C:/repo/src/a.ts', + { + kind: 'import', + line: 5, + specifier: './missing', + sourceFingerprint: 'sha256:a', + targetFingerprint: '' + }, + 'not-found' + ); + + const json = graph.toJSON(); + const restored = InternalFileGraph.fromJSON(json, 'C:/repo').toJSON(); + expect(Object.keys(restored.importDetails ?? {})).toHaveLength(3); + expect(restored.importDetails?.['src/b.ts']?.['src/c/index.ts']?.sources?.[0]).toMatchObject({ + kind: 'reexport', + line: 4, + specifier: '../c' + }); + expect(restored.unresolvedEdges?.[0]?.reason).toBe('not-found'); + + const result = traverseFileGraph(restored, 'src/a.ts', { direction: 'outgoing', maxHops: 3 }); + expect(result.paths.map((entry) => entry.nodes)).toContainEqual([ + 'src/a.ts', + 'src/b.ts', + 'src/c/index.ts', + 'src/d.tsx' + ]); + expect(result.paths.find((entry) => entry.nodes.length === 4)?.edges[1]).toMatchObject({ + kind: 'reexport', + source: { + line: 4, + specifier: '../c', + sourceFingerprint: 'sha256:b', + targetFingerprint: 'sha256:c' + } + }); + expect(result.unresolved).toHaveLength(1); + }); + + it('supports incoming direction, caps hops and terminates cycles', () => { + const graph = new InternalFileGraph('C:/repo'); + graph.trackImport('C:/repo/a.ts', 'C:/repo/b.ts', 1, [], { + kind: 'import', + specifier: './b', + sourceFingerprint: 'a', + targetFingerprint: 'b' + }); + graph.trackImport('C:/repo/b.ts', 'C:/repo/c.ts', 2, [], { + kind: 'import', + specifier: './c', + sourceFingerprint: 'b', + targetFingerprint: 'c' + }); + graph.trackImport('C:/repo/c.ts', 'C:/repo/d.ts', 3, [], { + kind: 'import', + specifier: './d', + sourceFingerprint: 'c', + targetFingerprint: 'd' + }); + graph.trackImport('C:/repo/d.ts', 'C:/repo/a.ts', 4, [], { + kind: 'import', + specifier: './a', + sourceFingerprint: 'd', + targetFingerprint: 'a' + }); + const serialized = graph.toJSON(); + + const outgoing = traverseFileGraph(serialized, 'a.ts', { direction: 'outgoing', maxHops: 2 }); + expect(outgoing.paths.every((entry) => entry.edges.length <= 2)).toBe(true); + expect(outgoing.truncated).toBe(false); + const incoming = traverseFileGraph(serialized, 'd.ts', { direction: 'incoming', maxHops: 1 }); + expect(incoming.paths).toContainEqual({ + nodes: ['d.ts', 'c.ts'], + edges: [expect.objectContaining({ from: 'c.ts', to: 'd.ts' })] + }); + const capped = traverseFileGraph(serialized, 'a.ts', { + direction: 'outgoing', + maxHops: 3, + maxNodes: 2 + }); + expect(capped.truncated).toBe(true); + const outputCapped = traverseFileGraph(serialized, 'a.ts', { + direction: 'outgoing', + maxHops: 3, + maxPaths: 1 + }); + expect(outputCapped.paths).toHaveLength(1); + expect(outputCapped.truncated).toBe(true); + }); + + it.each(['incoming', 'outgoing'] as const)( + 'terminates a cycle reachable inside three hops in %s direction', + (direction) => { + const graph = new InternalFileGraph('C:/repo'); + const pairs = [ + ['a.ts', 'b.ts'], + ['b.ts', 'a.ts'], + ['b.ts', 'c.ts'], + ['c.ts', 'b.ts'] + ] as const; + for (const [from, to] of pairs) { + graph.trackImport(`C:/repo/${from}`, `C:/repo/${to}`, 1, [], { + kind: 'import', + specifier: `./${to.slice(0, -3)}`, + sourceFingerprint: from, + targetFingerprint: to + }); + } + const result = traverseFileGraph(graph.toJSON(), 'a.ts', { direction, maxHops: 3 }); + expect(result.paths.map(({ nodes }) => nodes)).toEqual([ + ['a.ts', 'b.ts'], + ['a.ts', 'b.ts', 'c.ts'] + ]); + expect(result.truncated).toBe(false); + } + ); +}); diff --git a/tests/file-watcher.test.ts b/tests/file-watcher.test.ts index cde1f45..5c11e17 100644 --- a/tests/file-watcher.test.ts +++ b/tests/file-watcher.test.ts @@ -1,7 +1,9 @@ -import { describe, it, expect, beforeEach, afterEach } from 'vitest'; +import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'; import { promises as fs } from 'fs'; import path from 'path'; import os from 'os'; +import { EventEmitter } from 'node:events'; +import chokidar, { type FSWatcher } from 'chokidar'; import { startFileWatcher } from '../src/core/file-watcher.js'; import { rmWithRetries } from './test-helpers.js'; @@ -16,6 +18,55 @@ describe('FileWatcher', () => { await rmWithRetries(tempDir); }); + it('waits for close completion once and ignores events after stop', async () => { + let finishClose!: () => void; + const closeCompletion = new Promise((resolve) => { + finishClose = resolve; + }); + const watcher = new EventEmitter() as unknown as FSWatcher; + watcher.close = vi.fn(() => closeCompletion); + const watchSpy = vi.spyOn(chokidar, 'watch').mockReturnValue(watcher); + const onReady = vi.fn(); + const onChanged = vi.fn(); + let stop: (() => Promise) | undefined; + try { + stop = startFileWatcher({ + rootPath: tempDir, + debounceMs: 20, + onReady, + onChanged + }); + watcher.emit('add', path.join(tempDir, 'before-stop.ts')); + + const firstStop: unknown = stop(); + const secondStop: unknown = stop(); + expect(firstStop).toBeInstanceOf(Promise); + expect(secondStop).toBe(firstStop); + expect(watcher.close).toHaveBeenCalledTimes(1); + + watcher.emit('add', path.join(tempDir, 'after-stop.ts')); + watcher.emit('ready'); + await new Promise((resolve) => setTimeout(resolve, 40)); + expect(onChanged).not.toHaveBeenCalled(); + expect(onReady).not.toHaveBeenCalled(); + + let stopResolved = false; + const completion = Promise.resolve(firstStop).then(() => { + stopResolved = true; + }); + await Promise.resolve(); + expect(stopResolved).toBe(false); + + finishClose(); + await completion; + expect(stopResolved).toBe(true); + } finally { + finishClose(); + watchSpy.mockRestore(); + if (stop) await stop(); + } + }); + it('triggers onChanged after debounce window', async () => { const debounceMs = 400; let callCount = 0; @@ -29,7 +80,9 @@ describe('FileWatcher', () => { rootPath: tempDir, debounceMs, onReady: () => resolveReady(), - onChanged: () => { callCount++; }, + onChanged: () => { + callCount++; + } }); try { @@ -40,7 +93,7 @@ describe('FileWatcher', () => { await new Promise((resolve) => setTimeout(resolve, debounceMs + 1000)); expect(callCount).toBe(1); } finally { - stop(); + await stop(); } }, 8000); @@ -57,7 +110,9 @@ describe('FileWatcher', () => { rootPath: tempDir, debounceMs, onReady: () => resolveReady(), - onChanged: () => { callCount++; }, + onChanged: () => { + callCount++; + } }); try { @@ -72,7 +127,7 @@ describe('FileWatcher', () => { await new Promise((resolve) => setTimeout(resolve, debounceMs + 1200)); expect(callCount).toBe(1); } finally { - stop(); + await stop(); } }, 8000); @@ -89,19 +144,25 @@ describe('FileWatcher', () => { rootPath: tempDir, debounceMs, onReady: () => resolveReady(), - onChanged: () => { callCount++; }, - }); - - // Give chokidar a moment to finish initializing before the first write - await ready; - await fs.writeFile(path.join(tempDir, 'cancel.ts'), 'export const y = 99;'); - // Let chokidar detect the event (including awaitWriteFinish stabilityThreshold) - // but stop before the debounce window expires. - await new Promise((resolve) => setTimeout(resolve, 350)); - stop(); - // Wait past where debounce would have fired - await new Promise((resolve) => setTimeout(resolve, debounceMs + 200)); - expect(callCount).toBe(0); + onChanged: () => { + callCount++; + } + }); + + try { + // Give chokidar a moment to finish initializing before the first write + await ready; + await fs.writeFile(path.join(tempDir, 'cancel.ts'), 'export const y = 99;'); + // Let chokidar detect the event (including awaitWriteFinish stabilityThreshold) + // but stop before the debounce window expires. + await new Promise((resolve) => setTimeout(resolve, 350)); + await stop(); + // Wait past where debounce would have fired + await new Promise((resolve) => setTimeout(resolve, debounceMs + 200)); + expect(callCount).toBe(0); + } finally { + await stop(); + } }, 5000); it('ignores changes to non-tracked file extensions', async () => { @@ -128,7 +189,7 @@ describe('FileWatcher', () => { await new Promise((resolve) => setTimeout(resolve, debounceMs + 700)); expect(callCount).toBe(0); } finally { - stop(); + await stop(); } }, 5000); @@ -156,7 +217,7 @@ describe('FileWatcher', () => { await new Promise((resolve) => setTimeout(resolve, debounceMs + 700)); expect(callCount).toBe(1); } finally { - stop(); + await stop(); } }, 5000); @@ -185,7 +246,84 @@ describe('FileWatcher', () => { await new Promise((resolve) => setTimeout(resolve, debounceMs + 700)); expect(callCount).toBe(1); } finally { - stop(); + await stop(); + } + }, 5000); + + it('prunes excluded directories at any depth without hiding similar names', async () => { + const debounceMs = 150; + let callCount = 0; + + let resolveReady!: () => void; + const ready = new Promise((resolve) => { + resolveReady = resolve; + }); + + const stop = startFileWatcher({ + rootPath: tempDir, + debounceMs, + onReady: () => resolveReady(), + onChanged: () => { + callCount++; + } + }); + + try { + await ready; + for (const excluded of ['nested/node_modules', 'dist', '.codebase-context']) { + const directory = path.join(tempDir, 'parent', excluded); + await fs.mkdir(directory, { recursive: true }); + await fs.writeFile(path.join(directory, 'ignored.ts'), 'export const ignored = true;'); + } + await new Promise((resolve) => setTimeout(resolve, debounceMs + 700)); + expect(callCount).toBe(0); + + const allowed = path.join(tempDir, 'nested', 'node_modules-copy'); + await fs.mkdir(allowed, { recursive: true }); + await fs.writeFile(path.join(allowed, 'included.ts'), 'export const included = true;'); + await new Promise((resolve) => setTimeout(resolve, debounceMs + 700)); + expect(callCount).toBe(1); + } finally { + await stop(); + } + }, 7000); + + it('watches a literal brace-named root and reports add, change, and unlink', async () => { + const braceRoot = path.join(tempDir, '{literal-root}'); + await fs.mkdir(braceRoot); + const debounceMs = 100; + let callCount = 0; + + let resolveReady!: () => void; + const ready = new Promise((resolve) => { + resolveReady = resolve; + }); + + const stop = startFileWatcher({ + rootPath: braceRoot, + debounceMs, + onReady: () => resolveReady(), + onChanged: () => { + callCount++; + } + }); + + const filePath = path.join(braceRoot, 'tracked.ts'); + try { + await ready; + await fs.writeFile(filePath, 'export const value = 1;'); + await new Promise((resolve) => setTimeout(resolve, debounceMs + 450)); + expect(callCount).toBe(1); + + await fs.writeFile(filePath, 'export const value = 2;'); + await new Promise((resolve) => setTimeout(resolve, debounceMs + 450)); + expect(callCount).toBe(2); + + await fs.unlink(filePath); + await new Promise((resolve) => setTimeout(resolve, debounceMs + 450)); + expect(callCount).toBe(3); + } finally { + await stop(); } }, 5000); }); diff --git a/tests/fixtures/angular-routing-regressions.json b/tests/fixtures/angular-routing-regressions.json new file mode 100644 index 0000000..2d71e4b --- /dev/null +++ b/tests/fixtures/angular-routing-regressions.json @@ -0,0 +1,330 @@ +{ + "version": 1, + "purpose": "Frozen source facts for Angular-local analyzer eligibility and registry routing. These controls do not change or score retrieval gold.", + "eligibilityContract": "Ordinary names such as resolve, validate, intercept and canActivate are not framework evidence. Actual Angular imports and Angular-bound syntax establish eligibility; comments, string literals, filenames and other packages do not. Existing extension support is retained. Classification inside confirmed Angular files is a separate responsibility.", + "cases": [ + { + "id": "ordinary-promise-resolve", + "filePath": "src/example.ts", + "source": "export const saveRecord = (data: object) => Promise.resolve(data);", + "expectedAngularEligible": false, + "expectedAnalyzer": "generic", + "expectedSourceSymbols": ["saveRecord"] + }, + { + "id": "node-path-resolve", + "filePath": "src/example.ts", + "source": "import path from 'node:path'; export const resolveFile = (file: string) => path.resolve(file);", + "expectedAngularEligible": false, + "expectedAnalyzer": "generic", + "expectedSourceSymbols": ["resolveFile"] + }, + { + "id": "user-defined-resolve", + "filePath": "src/example.ts", + "source": "export class Resolver { resolve(value: string) { return value; } }", + "expectedAngularEligible": false, + "expectedAnalyzer": "generic" + }, + { + "id": "ordinary-validator", + "filePath": "src/example.ts", + "source": "export class Schema { validate(value: unknown) { return value !== null; } }", + "expectedAngularEligible": false, + "expectedAnalyzer": "generic" + }, + { + "id": "ordinary-interceptor", + "filePath": "src/example.ts", + "source": "export class Middleware { intercept(request: object) { return request; } }", + "expectedAngularEligible": false, + "expectedAnalyzer": "generic" + }, + { + "id": "ordinary-guard-method", + "filePath": "src/example.ts", + "source": "export class Permissions { canActivate() { return true; } canMatch() { return false; } }", + "expectedAngularEligible": false, + "expectedAnalyzer": "generic" + }, + { + "id": "locally-defined-resolve-type", + "filePath": "src/example.ts", + "source": "type ResolveFn = () => string; export const lookup: ResolveFn = () => \"found\";", + "expectedAngularEligible": false, + "expectedAnalyzer": "generic" + }, + { + "id": "locally-defined-validator-interface", + "filePath": "src/example.ts", + "source": "interface Validator { validate(value: unknown): boolean; } export class Schema implements Validator { validate(value: unknown) { return !!value; } }", + "expectedAngularEligible": false, + "expectedAnalyzer": "generic" + }, + { + "id": "decorator-lookalikes-in-comments", + "filePath": "src/example.ts", + "source": "// @Component({selector: \"demo\"})\n/* @Injectable() */\nexport function normalize(value: string) { return value.trim(); }", + "expectedAngularEligible": false, + "expectedAnalyzer": "generic", + "expectedSourceSymbols": ["normalize"] + }, + { + "id": "decorator-lookalikes-in-strings", + "filePath": "src/example.ts", + "source": "export const hints = \"@Component({}) @Injectable() CanActivateFn ResolveFn\"; export const ready = () => true;", + "expectedAngularEligible": false, + "expectedAnalyzer": "generic" + }, + { + "id": "angular-import-lookalike-in-comment", + "filePath": "src/example.ts", + "source": "// import { Component } from '@angular/core';\nexport const value = 1;", + "expectedAngularEligible": false, + "expectedAnalyzer": "generic" + }, + { + "id": "angular-import-lookalike-in-string", + "filePath": "src/example.ts", + "source": "export const example = \"import { Component } from '@angular/core';\";", + "expectedAngularEligible": false, + "expectedAnalyzer": "generic" + }, + { + "id": "similarly-named-nonangular-package", + "filePath": "src/example.ts", + "source": "import { Component } from 'ui-component'; @Component({}) export class Card {}", + "expectedAngularEligible": false, + "expectedAnalyzer": "generic" + }, + { + "id": "nestjs-injectable-is-not-angular", + "filePath": "src/example.ts", + "source": "import { Injectable } from '@nestjs/common'; @Injectable() export class PaymentsService {}", + "expectedAngularEligible": false, + "expectedAnalyzer": "nestjs" + }, + { + "id": "component-filename-is-not-proof", + "filePath": "src/plain.component.ts", + "source": "export function resolve(value: string) { return value; }", + "expectedAngularEligible": false, + "expectedAnalyzer": "generic", + "expectedSourceSymbols": ["resolve"] + }, + { + "id": "angular-component-import", + "filePath": "src/example.ts", + "source": "import { Component } from '@angular/core'; @Component({selector: 'demo', template: ''}) export class DemoComponent {}", + "expectedAngularEligible": true, + "expectedAnalyzer": "angular" + }, + { + "id": "angular-service-import", + "filePath": "src/example.ts", + "source": "import { Injectable } from '@angular/core'; @Injectable({providedIn: 'root'}) export class CacheService {}", + "expectedAngularEligible": true, + "expectedAnalyzer": "angular" + }, + { + "id": "angular-directive-import", + "filePath": "src/example.ts", + "source": "import { Directive } from '@angular/core'; @Directive({selector: '[focus]'}) export class FocusDirective {}", + "expectedAngularEligible": true, + "expectedAnalyzer": "angular" + }, + { + "id": "angular-pipe-import", + "filePath": "src/example.ts", + "source": "import { Pipe } from '@angular/core'; @Pipe({name: 'title'}) export class TitlePipe {}", + "expectedAngularEligible": true, + "expectedAnalyzer": "angular" + }, + { + "id": "angular-module-import", + "filePath": "src/example.ts", + "source": "import { NgModule } from '@angular/core'; @NgModule({}) export class FeatureModule {}", + "expectedAngularEligible": true, + "expectedAnalyzer": "angular" + }, + { + "id": "angular-aliased-component-binding", + "filePath": "src/example.ts", + "source": "import { Component as View } from '@angular/core'; @View({selector: 'demo', template: ''}) export class DemoView {}", + "expectedAngularEligible": true, + "expectedAnalyzer": "angular" + }, + { + "id": "angular-namespace-decorator-binding", + "filePath": "src/example.ts", + "source": "import * as ng from '@angular/core'; @ng.Component({selector: 'demo', template: ''}) export class DemoView {}", + "expectedAngularEligible": true, + "expectedAnalyzer": "angular" + }, + { + "id": "angular-class-resolver-binding", + "filePath": "src/example.ts", + "source": "import { Resolve } from '@angular/router'; export class RecordResolver implements Resolve { resolve() { return 'record'; } }", + "expectedAngularEligible": true, + "expectedAnalyzer": "angular" + }, + { + "id": "angular-functional-resolvefn-binding", + "filePath": "src/example.ts", + "source": "import { ResolveFn } from '@angular/router'; export const recordResolver: ResolveFn = () => 'record';", + "expectedAngularEligible": true, + "expectedAnalyzer": "angular" + }, + { + "id": "angular-aliased-functional-resolver", + "filePath": "src/example.ts", + "source": "import { ResolveFn as Lookup } from '@angular/router'; export const recordResolver: Lookup = () => 'record';", + "expectedAngularEligible": true, + "expectedAnalyzer": "angular" + }, + { + "id": "angular-class-guard-binding", + "filePath": "src/example.ts", + "source": "import { CanActivate } from '@angular/router'; export class AccessGuard implements CanActivate { canActivate() { return true; } }", + "expectedAngularEligible": true, + "expectedAnalyzer": "angular" + }, + { + "id": "angular-functional-guard-binding", + "filePath": "src/example.ts", + "source": "import { CanActivateChildFn } from '@angular/router'; export const childGuard: CanActivateChildFn = () => true;", + "expectedAngularEligible": true, + "expectedAnalyzer": "angular" + }, + { + "id": "angular-functional-interceptor-binding", + "filePath": "src/example.ts", + "source": "import { HttpInterceptorFn as Interceptor } from '@angular/common/http'; export const attachHeaders: Interceptor = (request, next) => next(request);", + "expectedAngularEligible": true, + "expectedAnalyzer": "angular" + }, + { + "id": "angular-functional-validator-binding", + "filePath": "src/example.ts", + "source": "import { AsyncValidatorFn } from '@angular/forms'; export const checkInput: AsyncValidatorFn = control => Promise.resolve(null);", + "expectedAngularEligible": true, + "expectedAnalyzer": "angular" + }, + { + "id": "angular-template-extension-retained", + "filePath": "src/card.html", + "source": "
@if (ready) {

Ready

}
", + "expectedAngularEligible": true, + "expectedAnalyzer": "angular" + }, + { + "id": "angular-style-extension-retained-css", + "filePath": "src/card.css", + "source": ".card { display: block; }", + "expectedAngularEligible": true, + "expectedAnalyzer": "angular" + }, + { + "id": "angular-style-extension-retained-scss", + "filePath": "src/card.scss", + "source": ".card { display: block; }", + "expectedAngularEligible": true, + "expectedAnalyzer": "angular" + }, + { + "id": "angular-style-extension-retained-sass", + "filePath": "src/card.sass", + "source": ".card { display: block; }", + "expectedAngularEligible": true, + "expectedAnalyzer": "angular" + }, + { + "id": "angular-style-extension-retained-less", + "filePath": "src/card.less", + "source": ".card { display: block; }", + "expectedAngularEligible": true, + "expectedAnalyzer": "angular" + }, + { + "id": "javascript-eligibility-unchanged", + "filePath": "src/util.js", + "source": "export const resolveValue = value => Promise.resolve(value);", + "expectedAngularEligible": false, + "expectedAnalyzer": "generic" + }, + { + "id": "other-language-eligibility-unchanged", + "filePath": "src/util.py", + "source": "def resolve(value):\n return value\n", + "expectedAngularEligible": false, + "expectedAnalyzer": "generic" + }, + { + "id": "public-excalidraw-json", + "filePath": "packages/excalidraw/data/json.ts", + "source": "import {\n EXPORT_DATA_TYPES,\n getExportSource,\n MIME_TYPES,\n VERSIONS,\n} from \"@excalidraw/common\";\n\nimport type { ExcalidrawElement, NonDeleted } from \"@excalidraw/element/types\";\n\nimport type { MaybePromise } from \"@excalidraw/common/utility-types\";\n\nimport { cleanAppStateForExport, clearAppStateForDatabase } from \"../appState\";\n\nimport { isImageFileHandle, loadFromBlob } from \"./blob\";\nimport { fileOpen, fileSave } from \"./filesystem\";\n\nimport type { AppState, BinaryFiles, LibraryItems } from \"../types\";\nimport type {\n ExportedDataState,\n ImportedDataState,\n ExportedLibraryData,\n ImportedLibraryData,\n} from \"./types\";\n\nexport type JSONExportData = {\n elements: readonly NonDeleted[];\n appState: AppState;\n files: BinaryFiles;\n};\n\n/**\n * Strips out files which are only referenced by deleted elements\n */\nconst filterOutDeletedFiles = (\n elements: readonly ExcalidrawElement[],\n files: BinaryFiles,\n) => {\n const nextFiles: BinaryFiles = {};\n for (const element of elements) {\n if (\n !element.isDeleted &&\n \"fileId\" in element &&\n element.fileId &&\n files[element.fileId]\n ) {\n nextFiles[element.fileId] = files[element.fileId];\n }\n }\n return nextFiles;\n};\n\nexport const serializeAsJSON = (\n elements: readonly ExcalidrawElement[],\n appState: Partial,\n files: BinaryFiles,\n type: \"local\" | \"database\",\n): string => {\n const data: ExportedDataState = {\n type: EXPORT_DATA_TYPES.excalidraw,\n version: VERSIONS.excalidraw,\n source: getExportSource(),\n elements,\n appState:\n type === \"local\"\n ? cleanAppStateForExport(appState)\n : clearAppStateForDatabase(appState),\n files:\n type === \"local\"\n ? filterOutDeletedFiles(elements, files)\n : // will be stripped from JSON\n undefined,\n };\n\n return JSON.stringify(data, null, 2);\n};\n\nexport const saveAsJSON = async ({\n data,\n filename,\n fileHandle,\n}: {\n data: MaybePromise;\n filename: string;\n fileHandle: AppState[\"fileHandle\"];\n}) => {\n const blob = Promise.resolve(data).then(({ elements, appState, files }) => {\n const serialized = serializeAsJSON(elements, appState, files, \"local\");\n return new Blob([serialized], {\n type: MIME_TYPES.excalidraw,\n });\n });\n\n const savedFileHandle = await fileSave(blob, {\n name: filename,\n extension: \"excalidraw\",\n description: \"Excalidraw file\",\n fileHandle: isImageFileHandle(fileHandle) ? null : fileHandle,\n });\n return { fileHandle: savedFileHandle };\n};\n\nexport const loadFromJSON = async (\n localAppState: AppState,\n localElements: readonly ExcalidrawElement[] | null,\n) => {\n const file = await fileOpen({\n description: \"Excalidraw files\",\n // ToDo: Be over-permissive until https://bugs.webkit.org/show_bug.cgi?id=34442\n // gets resolved. Else, iOS users cannot open `.excalidraw` files.\n // extensions: [\"json\", \"excalidraw\", \"png\", \"svg\"],\n });\n return loadFromBlob(file, localAppState, localElements, file.handle);\n};\n\nexport const isValidExcalidrawData = (data?: {\n type?: any;\n elements?: any;\n appState?: any;\n}): data is ImportedDataState => {\n return (\n data?.type === EXPORT_DATA_TYPES.excalidraw &&\n (!data.elements ||\n (Array.isArray(data.elements) &&\n (!data.appState || typeof data.appState === \"object\")))\n );\n};\n\nexport const isValidLibrary = (json: any): json is ImportedLibraryData => {\n return (\n typeof json === \"object\" &&\n json &&\n json.type === EXPORT_DATA_TYPES.excalidrawLibrary &&\n (json.version === 1 || json.version === 2)\n );\n};\n\nexport const serializeLibraryAsJSON = (libraryItems: LibraryItems) => {\n const data: ExportedLibraryData = {\n type: EXPORT_DATA_TYPES.excalidrawLibrary,\n version: VERSIONS.excalidrawLibrary,\n source: getExportSource(),\n libraryItems,\n };\n return JSON.stringify(data, null, 2);\n};\n\nexport const saveLibraryAsJSON = async (libraryItems: LibraryItems) => {\n const serialized = serializeLibraryAsJSON(libraryItems);\n await fileSave(\n new Blob([serialized], {\n type: MIME_TYPES.excalidrawlib,\n }),\n {\n name: \"library\",\n extension: \"excalidrawlib\",\n description: \"Excalidraw library file\",\n },\n );\n};\n", + "expectedAngularEligible": false, + "expectedAnalyzer": "generic", + "expectedSourceSymbols": ["serializeAsJSON", "saveAsJSON"], + "origin": { + "repositoryUrl": "https://github.com/excalidraw/excalidraw", + "path": "packages/excalidraw/data/json.ts", + "sourceSha256": "8ae8ad064f9a2f2060fe6ca62b8a5cf3cebeaa7dbc7c4ee8b227ed449cf9555b", + "lineStart": 1, + "lineEnd": 159, + "referenceFixture": "tests/fixtures/discovery-excalidraw.json", + "fixtureRepositoryRef": "e18c1dd213000dde0ae94ef7eb00aab537b39708", + "snapshotRefVerification": "not independently established; source bytes and hash are authoritative", + "licenseNotice": "excalidraw-mit" + } + }, + { + "id": "public-angular-spotify-component", + "filePath": "apps/angular-spotify/src/app/app.component.ts", + "source": "import { Component } from '@angular/core';\r\n\r\n@Component({\r\n selector: 'angular-spotify-root',\r\n template: ` `,\r\n styleUrls: ['./app.component.scss']\r\n})\r\nexport class AppComponent {}\r\n", + "expectedAngularEligible": true, + "expectedAnalyzer": "angular", + "origin": { + "repositoryUrl": "https://github.com/trungk18/angular-spotify", + "path": "apps/angular-spotify/src/app/app.component.ts", + "sourceSha256": "187e85f943f37ed8cd87eb7ecfee0b00850128e4e82fadbcf8510170ae962b18", + "lineStart": 1, + "lineEnd": 8, + "repositoryRef": "ff9efa765c53cfde78c9a172c62d515ae8ef9fe0", + "licenseNotice": "angular-spotify-mit" + } + }, + { + "id": "public-angular-spotify-auth-interceptor", + "filePath": "libs/web/auth/util/src/lib/interceptors/auth.interceptor.ts", + "source": "import { Injectable, Provider } from '@angular/core';\r\nimport {\r\n HttpEvent,\r\n HttpInterceptor,\r\n HttpHandler,\r\n HttpRequest,\r\n HTTP_INTERCEPTORS\r\n} from '@angular/common/http';\r\nimport { Observable } from 'rxjs';\r\nimport { AuthStore } from '@angular-spotify/web/auth/data-access';\r\nimport { switchMap, take } from 'rxjs/operators';\r\nimport { shouldSkipInterceptor } from './skip-urls';\r\n\r\n@Injectable()\r\nexport class AuthInterceptor implements HttpInterceptor {\r\n constructor(private authStore: AuthStore) {}\r\n\r\n intercept(req: HttpRequest, next: HttpHandler): Observable> {\r\n if (shouldSkipInterceptor(req.url)) {\r\n return next.handle(req);\r\n }\r\n\r\n return this.authStore.token$.pipe(\r\n take(1),\r\n switchMap((token) => {\r\n if (!token) {\r\n return next.handle(req);\r\n }\r\n const headers = req.headers.set('Authorization', `Bearer ${token}`);\r\n const authReq = req.clone({\r\n headers\r\n });\r\n return next.handle(authReq);\r\n })\r\n );\r\n }\r\n}\r\n\r\nexport const authInterceptorProvider: Provider = {\r\n provide: HTTP_INTERCEPTORS,\r\n useClass: AuthInterceptor,\r\n multi: true\r\n};\r\n", + "expectedAngularEligible": true, + "expectedAnalyzer": "angular", + "origin": { + "repositoryUrl": "https://github.com/trungk18/angular-spotify", + "path": "libs/web/auth/util/src/lib/interceptors/auth.interceptor.ts", + "sourceSha256": "186562e4372376eb2ca1c86d51ac0255d35ef9132f7ee9281cd8eefb1d052e3a", + "lineStart": 1, + "lineEnd": 43, + "repositoryRef": "ff9efa765c53cfde78c9a172c62d515ae8ef9fe0", + "licenseNotice": "angular-spotify-mit" + } + } + ], + "thirdPartyNotices": [ + { + "id": "excalidraw-mit", + "repositoryUrl": "https://github.com/excalidraw/excalidraw", + "licensePath": "LICENSE", + "licenseSourceSha256": "1352d204fdb90d5e482c13139d848bc2c9300a9a7de358a0a558a23e72d0d8be", + "licenseText": "MIT License\n\nCopyright (c) 2020 Excalidraw\n\nPermission is hereby granted, free of charge, to any person obtaining a copy\nof this software and associated documentation files (the \"Software\"), to deal\nin the Software without restriction, including without limitation the rights\nto use, copy, modify, merge, publish, distribute, sublicense, and/or sell\ncopies of the Software, and to permit persons to whom the Software is\nfurnished to do so, subject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in all\ncopies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\nFITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\nAUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\nLIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\nOUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\nSOFTWARE.\n" + }, + { + "id": "angular-spotify-mit", + "repositoryUrl": "https://github.com/trungk18/angular-spotify", + "licensePath": "LICENSE", + "licenseSourceSha256": "002ff3de0a6eebc81bf78cd524b87c5388e334e796c7f8fe29ad829d1e9b64da", + "licenseText": "MIT License\r\n\r\nCopyright (c) 2021 Trung Vo\r\n\r\nPermission is hereby granted, free of charge, to any person obtaining a copy\r\nof this software and associated documentation files (the \"Software\"), to deal\r\nin the Software without restriction, including without limitation the rights\r\nto use, copy, modify, merge, publish, distribute, sublicense, and/or sell\r\ncopies of the Software, and to permit persons to whom the Software is\r\nfurnished to do so, subject to the following conditions:\r\n\r\nThe above copyright notice and this permission notice shall be included in all\r\ncopies or substantial portions of the Software.\r\n\r\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\r\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\r\nFITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\r\nAUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\r\nLIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\r\nOUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\r\nSOFTWARE.\r\n" + } + ] +} diff --git a/tests/fixtures/eval-angular-spotify-supplemental.json b/tests/fixtures/eval-angular-spotify-supplemental.json new file mode 100644 index 0000000..45ec2b7 --- /dev/null +++ b/tests/fixtures/eval-angular-spotify-supplemental.json @@ -0,0 +1,133 @@ +{ + "description": "Supplemental source-grounded retrieval evaluation for angular-spotify.", + "codebase": "angular-spotify", + "repository": "https://github.com/trungk18/angular-spotify", + "repositoryUrl": "https://github.com/trungk18/angular-spotify", + "repositoryRef": "ff9efa765c53cfde78c9a172c62d515ae8ef9fe0", + "repositoryRefStatus": "Task-supplied pinned revision; snapshot has no .git metadata for independent verification.", + "license": "MIT", + "frozenDate": "2026-10-15", + "notes": "Supplemental set authored after candidate-04 source/archive freeze; it is not a pre-implementation evaluation and does not replace the existing evaluation set. Do not infer adoption, release quality, or broad generalization from this set alone.", + "queries": [ + { + "id": 1, + "query": "When a Spotify API call comes back unauthorized, where is that converted into the expired-session UI signal, and what does the interceptor actually do?", + "category": "conceptual_behavior", + "expectedPatterns": ["libs/web/auth/util/src/lib/interceptors/unauthorized.interceptor.ts"], + "expectedNotPatterns": ["libs/web/auth/util/src/lib/interceptors/auth.interceptor.ts"], + "notes": "Distinguishes 401 UI behavior from adding credentials to requests." + }, + { + "id": 2, + "query": "Which URLs bypass the shared HTTP interceptors, including the token endpoint and the external lyrics service, and what is the URL matching rule?", + "category": "exact_identifier_and_configuration", + "expectedPatterns": ["libs/web/auth/util/src/lib/interceptors/skip-urls.ts"], + "expectedNotPatterns": [ + "libs/web/auth/util/src/lib/interceptors/auth.interceptor.ts", + "libs/web/auth/util/src/lib/interceptors/unauthorized.interceptor.ts" + ], + "notes": "The helper holds the endpoint list and substring predicate." + }, + { + "id": 3, + "query": "Trace the Spotify PKCE callback: where is the verifier recovered, the authorization code exchanged, tokens saved, and the pre-login route restored?", + "category": "multi_concept_wiring", + "expectedPatterns": [ + "libs/web/auth/data-access/src/lib/store/auth.store.ts", + "libs/web/auth/data-access/src/lib/models/spotify-authorize.ts" + ], + "expectedNotPatterns": [ + "libs/web/auth/util/src/lib/interceptors/auth.interceptor.ts", + "libs/web/auth/util/src/lib/interceptors/skip-urls.ts" + ], + "notes": "Requires the auth flow store plus its authorize URL helper." + }, + { + "id": 4, + "query": "How does search replace results for a new term but append tracks, artists, albums, and playlists when another page is requested?", + "category": "conceptual_pagination", + "expectedPatterns": ["libs/web/search/data-access/src/lib/store/search.store.ts"], + "expectedNotPatterns": ["libs/web/search/feature/src/lib/search.component.ts"], + "notes": "Targets state/effect behavior, not the search page presentation." + }, + { + "id": 5, + "query": "Where are recent-played pages fetched with Spotify after-cursors and appended into the existing track list?", + "category": "multi_file_data_flow", + "expectedPatterns": [ + "libs/web/home/data-access/src/lib/store/recent-played-tracks/recent-played-tracks.effect.ts", + "libs/web/home/data-access/src/lib/store/recent-played-tracks/recent-played-tracks.reducer.ts" + ], + "expectedNotPatterns": [ + "libs/web/home/ui/recent-played/src/lib/recent-played.component.html" + ], + "notes": "Fetch/cursor and accumulation cross the effect and reducer." + }, + { + "id": 6, + "query": "What shared sentinel directive emits when the bottom enters view, and how does the recent-played component turn that into a load-more action?", + "category": "cross_layer_wiring", + "expectedPatterns": [ + "libs/web/shared/ui/infinite-scroll/src/lib/infinite-scroll.directive.ts", + "libs/web/home/ui/recent-played/src/lib/recent-played.component.ts" + ], + "expectedNotPatterns": [ + "libs/web/home/data-access/src/lib/store/recent-played-tracks/recent-played-tracks.reducer.ts" + ], + "notes": "Separates viewport detection from feature-level dispatch." + }, + { + "id": 7, + "query": "Where are Spotify audio-analysis segments converted into energy, pitch, timbre and beat-phase values before the selected visualizer renderer is updated?", + "category": "conceptual_data_transformation", + "expectedPatterns": ["libs/web/visualizer/ui/src/lib/web-visualizer-ui.component.ts"], + "expectedNotPatterns": ["libs/web/visualizer/data-access/src/lib/store/visualizer.store.ts"], + "notes": "Targets the live analysis-to-audio-data conversion, not selection state." + }, + { + "id": 8, + "query": "How does the visualizer switch into picture-in-picture after leaving its route, and what happens when the route is visited again?", + "category": "navigation_state_behavior", + "expectedPatterns": ["libs/web/visualizer/data-access/src/lib/store/visualizer.store.ts"], + "expectedNotPatterns": ["libs/web/lyrics/ui/lyrics-pip/src/lib/lyrics-pip.component.ts"], + "notes": "Tests route-event state transitions, not the separate lyrics PiP widget." + }, + { + "id": 9, + "query": "Which effect persists the current settings after the volume-persist action, and which service applies the app-specific storage key prefix?", + "category": "exact_identifier_and_wiring", + "expectedPatterns": [ + "libs/web/settings/data-access/src/lib/store/settings.effects.ts", + "libs/web/settings/data-access/src/lib/services/local-storage.service.ts" + ], + "expectedNotPatterns": ["libs/web/auth/data-access/src/lib/store/auth.store.ts"], + "notes": "Connects the settings effect to its storage utility." + }, + { + "id": 10, + "query": "Where does the Browse shell choose between the category index and a category-id route, while registering the categories and category-playlists state/effects?", + "category": "routing_and_registration", + "expectedPatterns": ["libs/web/browse/feature/shell/src/lib/browse-shell.module.ts"], + "expectedNotPatterns": [ + "libs/web/browse/feature/categories/src/lib/categories.component.ts", + "libs/web/browse/feature/category/src/lib/category.component.ts" + ], + "notes": "Looks for the route/state composition module, not either routed view." + } + ], + "idProvenance": { + "mapping": { + "ang-01": 1, + "ang-02": 2, + "ang-03": 3, + "ang-04": 4, + "ang-05": 5, + "ang-06": 6, + "ang-07": 7, + "ang-08": 8, + "ang-09": 9, + "ang-10": 10 + }, + "reason": "declared EvalQuery.id:number; factual schema alignment before first score, with question, expected paths, categories, and order unchanged." + } +} diff --git a/tests/fixtures/eval-excalidraw-supplemental.json b/tests/fixtures/eval-excalidraw-supplemental.json new file mode 100644 index 0000000..15f39b2 --- /dev/null +++ b/tests/fixtures/eval-excalidraw-supplemental.json @@ -0,0 +1,124 @@ +{ + "description": "Supplemental source-grounded retrieval evaluation for excalidraw/excalidraw.", + "codebase": "excalidraw/excalidraw", + "repository": "https://github.com/excalidraw/excalidraw", + "repositoryUrl": "https://github.com/excalidraw/excalidraw", + "repositoryRef": "e18c1dd213000dde0ae94ef7eb00aab537b39708", + "repositoryRefStatus": "Task-supplied pinned revision; snapshot has no .git metadata for independent verification.", + "license": "MIT", + "frozenDate": "2026-10-15", + "notes": "Supplemental set authored after candidate-04 source/archive freeze; it is not a pre-implementation evaluation and does not replace the existing evaluation set. Do not infer adoption, release quality, or broad generalization from this set alone.", + "queries": [ + { + "id": 1, + "query": "Where does the app debounce saving the non-deleted scene and app state to browser storage, then save image binaries separately?", + "category": "multi_concept_storage_flow", + "expectedPatterns": ["excalidraw-app/data/LocalData.ts"], + "expectedNotPatterns": ["excalidraw-app/data/tabSync.ts"], + "notes": "The local persistence entrypoint coordinates both scene storage and file storage." + }, + { + "id": 2, + "query": "How does a local .excalidraw JSON export filter out image files used only by deleted elements, and where is the resulting file saved?", + "category": "edge_case_export", + "expectedPatterns": [ + "packages/excalidraw/data/json.ts", + "packages/excalidraw/data/filesystem.ts" + ], + "expectedNotPatterns": [ + "packages/excalidraw/data/image.ts", + "packages/excalidraw/clipboard.ts" + ], + "notes": "Distinguishes standard JSON-file export from PNG embedding and clipboard serialization." + }, + { + "id": 3, + "query": "When old diagrams are restored, where are legacy text font and line-height values repaired, and how are incomplete arrow point lists normalized?", + "category": "compatibility_repair", + "expectedPatterns": ["packages/excalidraw/data/restore.ts"], + "expectedNotPatterns": ["packages/excalidraw/tests/data/restore.test.ts"], + "notes": "Request asks for implementation; the nearby restore test is an explicit distractor." + }, + { + "id": 4, + "query": "How is scene JSON compressed into a PNG text chunk and decoded back out, including the legacy unencoded scene case?", + "category": "exact_identifier_and_compatibility", + "expectedPatterns": ["packages/excalidraw/data/image.ts"], + "expectedNotPatterns": [ + "packages/excalidraw/data/json.ts", + "packages/excalidraw/tests/data/restore.test.ts" + ], + "notes": "Targets PNG metadata helpers rather than the ordinary .excalidraw JSON serializer." + }, + { + "id": 5, + "query": "How does the app map the browser language to a supported locale, fall back to the default, and cache a user selection?", + "category": "configuration_and_wiring", + "expectedPatterns": [ + "excalidraw-app/app-language/language-detector.ts", + "excalidraw-app/app-language/language-state.ts", + "excalidraw-app/app-language/LanguageList.tsx" + ], + "expectedNotPatterns": ["excalidraw-app/tests/LanguageList.test.tsx"], + "notes": "The detector, state hook and selector are distinct steps; test file is the negative match." + }, + { + "id": 6, + "query": "Where are collaboration scene updates encrypted before socket emission, and where are incoming packets decrypted and reconciled into the scene?", + "category": "cross_module_protocol", + "expectedPatterns": ["excalidraw-app/collab/Portal.tsx", "excalidraw-app/collab/Collab.tsx"], + "expectedNotPatterns": ["excalidraw-app/data/FileManager.ts"], + "notes": "Separates outbound transport framing from inbound room-state handling." + }, + { + "id": 7, + "query": "Which file manager tracks image file versions as fetching, saved or errored so repeated upload and fetch work can be deduplicated?", + "category": "implementation_convention", + "expectedPatterns": ["excalidraw-app/data/FileManager.ts"], + "expectedNotPatterns": ["excalidraw-app/data/fileStatusStore.ts"], + "notes": "Distinguishes per-file manager bookkeeping from the shared UI status snapshot store." + }, + { + "id": 8, + "query": "Where does the legacy localStorage library adapter load and clear saved library items as they migrate to an IndexedDB adapter, and where is that migration wired at startup?", + "category": "migration_and_startup_wiring", + "expectedPatterns": ["excalidraw-app/data/LocalData.ts", "excalidraw-app/App.tsx"], + "expectedNotPatterns": ["packages/excalidraw/data/library.ts"], + "notes": "App startup wires the adapters; LocalData defines both persistence adapters." + }, + { + "id": 9, + "query": "How does this tab decide browser-stored scene or file state is newer than its in-memory version, and update the timestamp marker?", + "category": "state_synchronization", + "expectedPatterns": ["excalidraw-app/data/tabSync.ts"], + "expectedNotPatterns": ["excalidraw-app/data/LocalData.ts"], + "notes": "Targets cross-tab timestamp comparison rather than the main local save path." + }, + { + "id": 10, + "query": "How are clipboard and drag/drop transfer items parsed when the payload mixes text and files, and how are Excalidraw element payloads recognized?", + "category": "mixed_input_edge_case", + "expectedPatterns": ["packages/excalidraw/clipboard.ts"], + "expectedNotPatterns": [ + "packages/excalidraw/tests/clipboard.test.tsx", + "packages/excalidraw/clipboard.test.ts" + ], + "notes": "Implementation source is distinct from both clipboard test suites." + } + ], + "idProvenance": { + "mapping": { + "exc-01": 1, + "exc-02": 2, + "exc-03": 3, + "exc-04": 4, + "exc-05": 5, + "exc-06": 6, + "exc-07": 7, + "exc-08": 8, + "exc-09": 9, + "exc-10": 10 + }, + "reason": "declared EvalQuery.id:number; factual schema alignment before first score, with question, expected paths, categories, and order unchanged." + } +} diff --git a/tests/fixtures/manual-first-use-queries.json b/tests/fixtures/manual-first-use-queries.json new file mode 100644 index 0000000..8a31e56 --- /dev/null +++ b/tests/fixtures/manual-first-use-queries.json @@ -0,0 +1,206 @@ +{ + "scope": "Manual integration check, not a benchmark advantage claim. Queries and expected files frozen before candidate scoring or retrieval repairs.", + "rule": "For retrieval cases, report whether an independently identified relevant file appears in top five; report missing context and failures. The five original Express dogfood questions also need source-grounded answers. The graph question is a capability negative, not a ranked-search success. Do not change expected files after observing candidate output.", + "corpora": { + "express": { + "revision": "0dee2c52c7c873498f8ed7d8ee4ddd0a04d3ada8", + "source": "https://github.com/expressjs/express", + "codeFiles": 141 + }, + "hono": { + "revision": "90d4182aabd328e2ec6af3f25ec62ddc574ad8cb", + "source": "https://github.com/honojs/hono/tree/v4.12.16", + "scope": "src plus root package/docs; 284 TypeScript files, not the entire upstream checkout", + "codeFiles": 284 + } + }, + "cases": [ + { + "id": "e01", + "repo": "express", + "query": "how does routing work", + "expectedFiles": [ + "lib/application.js", + "lib/express.js" + ], + "dogfood": true + }, + { + "id": "e02", + "repo": "express", + "query": "where are errors handled for a request", + "expectedFiles": [ + "lib/application.js", + "examples/error/index.js" + ], + "dogfood": true + }, + { + "id": "e03", + "repo": "express", + "query": "what changes when I add middleware", + "expectedFiles": [ + "lib/application.js", + "test/app.use.js" + ], + "dogfood": true + }, + { + "id": "e04", + "repo": "express", + "query": "where are tests for routing and how do I run them", + "expectedFiles": [ + "test/app.router.js" + ], + "additionalSource": "package.json", + "dogfood": true + }, + { + "id": "e05", + "repo": "express", + "query": "what is the least-connected module", + "expectedFiles": [], + "kind": "capability-negative", + "rule": "Ranked snippets do not answer this graph question. Report an actual connectivity result or an explicit limitation and appropriate next tool.", + "dogfood": true + }, + { + "id": "e06", + "repo": "express", + "query": "how is JSON request body parsing exposed", + "expectedFiles": [ + "lib/express.js" + ] + }, + { + "id": "e07", + "repo": "express", + "query": "where are views rendered and template engines loaded", + "expectedFiles": [ + "lib/view.js", + "lib/application.js" + ] + }, + { + "id": "e08", + "repo": "express", + "query": "how do application settings get stored and inherited", + "expectedFiles": [ + "lib/application.js" + ] + }, + { + "id": "e09", + "repo": "express", + "query": "how does request content type negotiation work", + "expectedFiles": [ + "lib/request.js" + ] + }, + { + "id": "e10", + "repo": "express", + "query": "where is static file middleware exported", + "expectedFiles": [ + "lib/express.js" + ] + }, + { + "id": "e11", + "repo": "express", + "query": "how are objects serialized into JSON responses", + "expectedFiles": [ + "lib/response.js" + ] + }, + { + "id": "h01", + "repo": "hono", + "query": "how are exceptions caught while middleware runs in sequence", + "expectedFiles": [ + "src/compose.ts" + ] + }, + { + "id": "h02", + "repo": "hono", + "query": "where are request headers and query parameters accessed", + "expectedFiles": [ + "src/request.ts" + ] + }, + { + "id": "h03", + "repo": "hono", + "query": "how is a JSON response constructed", + "expectedFiles": [ + "src/context.ts" + ] + }, + { + "id": "h04", + "repo": "hono", + "query": "how do I attach middleware and handlers to routes", + "expectedFiles": [ + "src/hono-base.ts" + ] + }, + { + "id": "h05", + "repo": "hono", + "query": "how is the routing algorithm selected", + "expectedFiles": [ + "src/router/smart-router/router.ts" + ] + }, + { + "id": "h06", + "repo": "hono", + "query": "where are cross origin preflight requests handled", + "expectedFiles": [ + "src/middleware/cors/index.ts" + ] + }, + { + "id": "h07", + "repo": "hono", + "query": "where is incoming request data validated before a handler", + "expectedFiles": [ + "src/validator/validator.ts" + ] + }, + { + "id": "h08", + "repo": "hono", + "query": "how do incoming requests and response times get logged", + "expectedFiles": [ + "src/middleware/logger/index.ts" + ] + }, + { + "id": "h09", + "repo": "hono", + "query": "how are JSX child nodes rendered and escaped", + "expectedFiles": [ + "src/jsx/base.ts" + ] + }, + { + "id": "h10", + "repo": "hono", + "query": "how does the typed HTTP client construct a request", + "expectedFiles": [ + "src/client/client.ts" + ] + }, + { + "id": "h11", + "repo": "hono", + "query": "where are regular expression router tests and how are they run", + "expectedFiles": [ + "src/router/reg-exp-router/router.test.ts" + ], + "additionalSource": "package.json" + } + ] +} diff --git a/tests/fixtures/reference-nomination-parity.json b/tests/fixtures/reference-nomination-parity.json new file mode 100644 index 0000000..3778afd --- /dev/null +++ b/tests/fixtures/reference-nomination-parity.json @@ -0,0 +1,344 @@ +{ + "contract": "Freeze current exact-source reference behavior before any production nomination change. Each expected result is the current parser path (AST when parse succeeds, otherwise lexical fallback).", + "cases": [ + { + "id": "ts-exact-reference", + "language": "typescript", + "symbol": "Scene", + "source": "const Scene = 1; Scene;", + "expected": { "mode": "ast", "count": 2 } + }, + { + "id": "ts-long-identifier", + "language": "typescript", + "symbol": "Scene", + "source": "const SceneData = 1;", + "expected": { "mode": "ast", "count": 0 } + }, + { + "id": "ts-dollar-identifier", + "language": "typescript", + "symbol": "Scene", + "source": "const $Scene = 1;", + "expected": { "mode": "ast", "count": 0 } + }, + { + "id": "ts-underscore-identifier", + "language": "typescript", + "symbol": "Scene", + "source": "const _Scene = 1;", + "expected": { "mode": "ast", "count": 0 } + }, + { + "id": "ts-underscore-suffix", + "language": "typescript", + "symbol": "Scene", + "source": "const Scene_Thing = 1;", + "expected": { "mode": "ast", "count": 0 } + }, + { + "id": "ts-unicode-start-continuation", + "language": "typescript", + "symbol": "Scene", + "source": "const πScene = 1;", + "expected": { "mode": "ast", "count": 0 } + }, + { + "id": "ts-unicode-end-continuation", + "language": "typescript", + "symbol": "Scene", + "source": "const Sceneπ = 1;", + "expected": { "mode": "ast", "count": 0 } + }, + { + "id": "ts-combining-mark-continuation", + "language": "typescript", + "symbol": "Scene", + "source": "const Scené = 1;", + "expected": { "mode": "ast", "count": 0 } + }, + { + "id": "ts-joiner-continuation", + "language": "typescript", + "symbol": "Scene", + "source": "const Scene‌part = 1;", + "expected": { "mode": "ast", "count": 0 } + }, + { + "id": "ts-string-only", + "language": "typescript", + "symbol": "Scene", + "source": "const message = \"Scene\";", + "expected": { "mode": "ast", "count": 0 } + }, + { + "id": "ts-comment-only", + "language": "typescript", + "symbol": "Scene", + "source": "// Scene", + "expected": { "mode": "ast", "count": 0 } + }, + { + "id": "ts-quoted-property", + "language": "typescript", + "symbol": "Scene", + "source": "const value = { \"Scene\": 1 };", + "expected": { "mode": "ast", "count": 0 } + }, + { + "id": "ts-unicode-escape-semantic-name", + "language": "typescript", + "symbol": "Scene", + "source": "const \\u0053cene = 1;", + "expected": { "mode": "ast", "count": 0 } + }, + { + "id": "ts-unicode-escape-raw-query", + "language": "typescript", + "symbol": "\\u0053cene", + "source": "const \\u0053cene = 1;", + "expected": { "mode": "ast", "count": 1 } + }, + { + "id": "ts-incomplete-syntax-fallback", + "language": "typescript", + "symbol": "Scene", + "source": "const Scene =", + "expected": { "mode": "lexical", "count": 1 } + }, + { + "id": "ts-number-error-split", + "language": "typescript", + "symbol": "Scene", + "source": "const value = 0xScene;", + "expected": { "mode": "lexical", "count": 0 } + }, + { + "id": "javascript-number-suffix", + "language": "javascript", + "symbol": "Scene", + "source": "const value = 1Scene;", + "expected": { "mode": "lexical", "count": 0 } + }, + { + "id": "python-number-suffix", + "language": "python", + "symbol": "Scene", + "source": "value = 1Scene", + "expected": { "mode": "lexical", "count": 0 } + }, + { + "id": "go-number-suffix", + "language": "go", + "symbol": "Scene", + "source": "package p\nvar value = 1Scene", + "expected": { "mode": "lexical", "count": 0 } + }, + { + "id": "rust-number-suffix", + "language": "rust", + "symbol": "Scene", + "source": "fn f() { let value = 1Scene; }", + "expected": { "mode": "lexical", "count": 0 } + }, + { + "id": "java-number-suffix", + "language": "java", + "symbol": "Scene", + "source": "class C { int value = 1Scene; }", + "expected": { "mode": "lexical", "count": 0 } + }, + { + "id": "c-number-suffix", + "language": "c", + "symbol": "Scene", + "source": "int value = 1Scene;", + "expected": { "mode": "lexical", "count": 0 } + }, + { + "id": "cpp-number-suffix", + "language": "cpp", + "symbol": "Scene", + "source": "int value = 1Scene;", + "expected": { "mode": "ast", "count": 0 } + }, + { + "id": "c-hex-prefix-split", + "language": "c", + "symbol": "Scene", + "source": "int value = 0xScene;", + "expected": { "mode": "lexical", "count": 0 } + }, + { + "id": "cpp-hex-prefix-split", + "language": "cpp", + "symbol": "Scene", + "source": "int value = 0xScene;", + "expected": { "mode": "ast", "count": 0 } + }, + { + "id": "csharp-number-suffix", + "language": "csharp", + "symbol": "Scene", + "source": "class C { int value = 1Scene; }", + "expected": { "mode": "lexical", "count": 0 } + }, + { + "id": "kotlin-number-suffix", + "language": "kotlin", + "symbol": "Scene", + "source": "val value = 1Scene", + "expected": { "mode": "lexical", "count": 0 } + }, + { + "id": "js-exact-reference", + "language": "javascript", + "symbol": "Scene", + "source": "const Scene = 1; Scene;", + "expected": { "mode": "ast", "count": 2 } + }, + { + "id": "tsx-jsx-identifier", + "language": "typescriptreact", + "symbol": "Scene", + "source": "const view = ;", + "expected": { "mode": "ast", "count": 1 } + }, + { + "id": "python-exact-reference", + "language": "python", + "symbol": "Scene", + "source": "Scene = 1\nprint(Scene)", + "expected": { "mode": "ast", "count": 2 } + }, + { + "id": "python-long-identifier", + "language": "python", + "symbol": "Scene", + "source": "SceneData = 1", + "expected": { "mode": "ast", "count": 0 } + }, + { + "id": "go-exact-reference", + "language": "go", + "symbol": "Scene", + "source": "package p\nvar Scene int\nfunc f() { _ = Scene }", + "expected": { "mode": "ast", "count": 2 } + }, + { + "id": "rust-exact-reference", + "language": "rust", + "symbol": "Scene", + "source": "fn f() { let Scene = 1; println!(\"{}\", Scene); }", + "expected": { "mode": "ast", "count": 2 } + }, + { + "id": "java-type-reference", + "language": "java", + "symbol": "Scene", + "source": "class Scene { Scene value; }", + "expected": { "mode": "ast", "count": 2 } + }, + { + "id": "cpp-exact-reference", + "language": "cpp", + "symbol": "Scene", + "source": "int Scene; int f() { return Scene; }", + "expected": { "mode": "ast", "count": 2 } + }, + { + "id": "c-exact-reference", + "language": "c", + "symbol": "Scene", + "source": "int Scene; int f() { return Scene; }", + "expected": { "mode": "ast", "count": 2 } + }, + { + "id": "csharp-exact-reference", + "language": "csharp", + "symbol": "Scene", + "source": "class Scene { int value; }", + "expected": { "mode": "ast", "count": 1 } + }, + { + "id": "kotlin-current-node-name-contract", + "language": "kotlin", + "symbol": "Scene", + "source": "fun main() { val Scene = 1; println(Scene) }", + "expected": { "mode": "ast", "count": 0 } + }, + { + "id": "unknown-quoted-and-comment-fallback", + "language": "unknown", + "symbol": "Scene", + "source": "val x = \"Scene\" // Scene", + "expected": { "mode": "lexical", "count": 2 } + }, + { + "id": "unknown-long-identifier", + "language": "unknown", + "symbol": "Scene", + "source": "SceneData", + "expected": { "mode": "lexical", "count": 0 } + }, + { + "id": "unknown-dollar-boundary", + "language": "unknown", + "symbol": "Scene", + "source": "$Scene Scene$", + "expected": { "mode": "lexical", "count": 0 } + }, + { + "id": "unknown-non-ascii-boundaries", + "language": "unknown", + "symbol": "Scene", + "source": "πScene Sceneπ", + "expected": { "mode": "lexical", "count": 0 } + }, + { + "id": "unknown-number-error-split", + "language": "unknown", + "symbol": "Scene", + "source": "0xScene", + "expected": { "mode": "lexical", "count": 0 } + }, + { + "id": "unknown-composite-symbol", + "language": "unknown", + "symbol": "foo.bar", + "source": "foo.bar + foobar", + "expected": { "mode": "lexical", "count": 1 } + }, + { + "id": "unknown-raw-escape-spelling", + "language": "unknown", + "symbol": "\\u0053cene", + "source": "\\u0053cene", + "expected": { "mode": "lexical", "count": 1 } + }, + { + "id": "angular-spotify-source-token", + "language": "typescript", + "symbol": "Component", + "source": "Component", + "provenance": { + "repository": "angular-spotify", + "file": "apps/angular-spotify/src/app/app.component.ts", + "sha256": "187e85f943f37ed8cd87eb7ecfee0b00850128e4e82fadbcf8510170ae962b18" + }, + "expected": { "mode": "ast", "count": 1 } + }, + { + "id": "excalidraw-source-token-inside-long-identifier", + "language": "typescript", + "symbol": "Scene", + "source": "initializeScene", + "provenance": { + "repository": "excalidraw-monorepo", + "file": "excalidraw-app/App.tsx", + "sha256": "59789614d257b77bd89a3e304de3247fed4c821401e0e30173ce52d58b71816b" + }, + "expected": { "mode": "ast", "count": 0 } + } + ] +} diff --git a/tests/fixtures/search-rescue-density.json b/tests/fixtures/search-rescue-density.json new file mode 100644 index 0000000..8195bdd --- /dev/null +++ b/tests/fixtures/search-rescue-density.json @@ -0,0 +1,478 @@ +{ + "schemaVersion": 1, + "purpose": "Frozen public controls for a secondary source-local normalized term-frequency density tie-break.", + "formula": "For each existing normalized query group g, tf_g counts normalized body-word occurrences whose existing lexicalAliases(normalizedBodyWord) intersect the aliases in g. Count that body-word occurrence at most once per group, regardless of alias-expansion multiplicity. density = sum_g sqrt(tf_g) / sqrt(max(1, normalizedBodyWordCount)).", + "ordering": "Lexicographic comparison: preserve the existing primary rank, then compare descending density only for equal primary ranks, then preserve original chunk ordinal. Never add density to or otherwise alter the primary score.", + "scope": "Synthetic source snippets and relative file paths. Expected order is a consequence of the frozen lexical formula and existing primary evidence classes; it does not assert semantic correctness.", + "cases": [ + { + "id": "density-breaks-primary-tie", + "query": "invoice approval", + "chunks": [ + { + "id": "long-noise", + "filePath": "src/long-noise.ts", + "content": "invoice approval billing invoice mail report network" + }, + { + "id": "compact", + "filePath": "src/compact.ts", + "content": "invoice approval" + } + ], + "expectedIds": ["compact", "long-noise"], + "expectedPathPatterns": ["src/compact.ts", "src/long-noise.ts"], + "expectedProperties": [ + "primary evidence ties", + "unmatched body vocabulary increases the denominator" + ] + }, + { + "id": "declaration-primary-dominates-density", + "query": "invoice approval", + "chunks": [ + { + "id": "declared-implementation", + "filePath": "src/invoice-approval.ts", + "content": "export function apply(invoice) { const approval = true; return applyPolicy(invoice, approval); } billing mail report network logging validation storage adapter response", + "metadata": { + "functionName": "invoiceApproval" + } + }, + { + "id": "dense-body", + "filePath": "src/repeated.ts", + "content": "invoice approval invoice approval invoice approval" + } + ], + "expectedIds": ["declared-implementation", "dense-body"], + "expectedPathPatterns": ["src/invoice-approval.ts", "src/repeated.ts"], + "expectedProperties": [ + "declaration evidence remains primary", + "density cannot promote a weaker primary candidate" + ] + }, + { + "id": "exact-anchor-primary-dominates-density", + "query": "InvoicePolicy invoice policy", + "chunks": [ + { + "id": "exact-anchor", + "filePath": "src/invoice-policy.ts", + "content": "invoice policy resolves a rule through a versioned registry and validates the result against the selected tenant before returning it", + "metadata": { + "symbolName": "InvoicePolicy" + } + }, + { + "id": "dense-body", + "filePath": "src/repeated-policy.ts", + "content": "invoice policy invoice policy invoice policy" + } + ], + "expectedIds": ["exact-anchor", "dense-body"], + "expectedPathPatterns": ["src/invoice-policy.ts", "src/repeated-policy.ts"], + "expectedProperties": [ + "exact indexed anchor remains primary", + "body density is only a secondary tie-break" + ] + }, + { + "id": "implementation-primary-dominates-test-density", + "query": "invoice approval", + "chunks": [ + { + "id": "implementation", + "filePath": "src/invoice-approval.ts", + "content": "invoice approval validates the request and records the accepted result after checking the account state and policy" + }, + { + "id": "test", + "filePath": "src/__tests__/invoice-approval.test.ts", + "content": "invoice approval invoice approval invoice approval invoice approval" + } + ], + "expectedIds": ["implementation", "test"], + "expectedPathPatterns": ["src/invoice-approval.ts", "src/__tests__/invoice-approval.test.ts"], + "expectedProperties": [ + "test-file primary penalty remains effective", + "repeated test-body terms do not override primary order" + ] + }, + { + "id": "own-implementation-outranks-repeated-type-mentions", + "query": "cache refresh", + "chunks": [ + { + "id": "own-implementation", + "filePath": "src/cache-refresh.ts", + "content": "refreshCache updates the cache after validating the new value and then returns the stored entry", + "metadata": { + "functionName": "refreshCache" + } + }, + { + "id": "repeated-type-mentions", + "filePath": "src/cache-types.ts", + "content": "type CacheRefresh = Cache & Refresh & CacheRefresh & CacheRefresh & CacheRefresh" + } + ], + "expectedIds": ["own-implementation", "repeated-type-mentions"], + "expectedPathPatterns": ["src/cache-refresh.ts", "src/cache-types.ts"], + "expectedProperties": [ + "own declaration evidence remains primary", + "repetition in a type body is lexical evidence only" + ] + }, + { + "id": "tiny-wrapper-does-not-beat-stronger-declaration", + "query": "invoice approval", + "chunks": [ + { + "id": "implementation", + "filePath": "src/invoice-approval.ts", + "content": "invoice approval validates the account, checks the pending state, records the decision, and returns the persisted result", + "metadata": { + "functionName": "invoiceApproval" + } + }, + { + "id": "tiny-wrapper", + "filePath": "src/wrapper.ts", + "content": "invoice approval" + } + ], + "expectedIds": ["implementation", "tiny-wrapper"], + "expectedPathPatterns": ["src/invoice-approval.ts", "src/wrapper.ts"], + "expectedProperties": [ + "declaration primary rank dominates the wrapper's higher density", + "shortness alone does not imply a better result" + ] + }, + { + "id": "metadata-repetition-is-not-body-frequency", + "query": "invoice approval", + "chunks": [ + { + "id": "repeated-metadata-first", + "filePath": "src/a.ts", + "content": "invoice approval validates a request", + "metadata": { + "componentName": "InvoiceApproval InvoiceApproval InvoiceApproval" + } + }, + { + "id": "single-metadata-second", + "filePath": "src/b.ts", + "content": "invoice approval validates a request", + "metadata": { + "componentName": "InvoiceApproval" + } + } + ], + "expectedIds": ["repeated-metadata-first", "single-metadata-second"], + "expectedPathPatterns": ["src/a.ts", "src/b.ts"], + "expectedProperties": [ + "body density is identical", + "metadata repetition does not increase term frequency", + "original ordinal breaks the tie" + ] + }, + { + "id": "body-inflection-alias-matching-preserves-ordinal", + "query": "archive policy", + "chunks": [ + { + "id": "inflected-first", + "filePath": "src/a.ts", + "content": "archiving policy" + }, + { + "id": "canonical-second", + "filePath": "src/b.ts", + "content": "archive policy" + } + ], + "expectedIds": ["inflected-first", "canonical-second"], + "expectedPathPatterns": ["src/a.ts", "src/b.ts"], + "expectedProperties": [ + "existing body lexical aliases are intersected with the query-group aliases", + "archiving and archive each contribute one occurrence to the archive group", + "both bodies have equal density and original ordinal is preserved" + ] + }, + { + "id": "overlapping-aliases-count-one-occurrence-per-group", + "query": "archive archiving policy", + "chunks": [ + { + "id": "archiving-first", + "filePath": "src/a.ts", + "content": "archiving policy" + }, + { + "id": "archive-second", + "filePath": "src/b.ts", + "content": "archive policy" + } + ], + "expectedIds": ["archiving-first", "archive-second"], + "expectedPathPatterns": ["src/a.ts", "src/b.ts"], + "expectedProperties": [ + "archive and archiving belong to one existing alias group", + "one body occurrence contributes once to that group", + "equal density preserves ordinal" + ] + }, + { + "id": "repeat-one-group-does-not-win-on-raw-count", + "query": "archive policy", + "chunks": [ + { + "id": "balanced", + "filePath": "src/balanced.ts", + "content": "archive policy" + }, + { + "id": "repeated-archive", + "filePath": "src/repeated.ts", + "content": "archive archive policy" + } + ], + "expectedIds": ["balanced", "repeated-archive"], + "expectedPathPatterns": ["src/balanced.ts", "src/repeated.ts"], + "expectedProperties": [ + "balanced density is 2/sqrt(2)", + "repeated density is (sqrt(2)+1)/sqrt(3)", + "square-root TF and body length are both applied" + ] + }, + { + "id": "unrelated-billing-mail-report-network-words-only-add-length", + "query": "archive policy", + "chunks": [ + { + "id": "compact", + "filePath": "src/compact.ts", + "content": "archive policy" + }, + { + "id": "unrelated-vocabulary", + "filePath": "src/unrelated.ts", + "content": "archive policy billing invoice mail report network" + } + ], + "expectedIds": ["compact", "unrelated-vocabulary"], + "expectedPathPatterns": ["src/compact.ts", "src/unrelated.ts"], + "expectedProperties": [ + "billing, mail, report, and network words are not query clues", + "unmatched body words increase the denominator only" + ] + }, + { + "id": "overlapping-chunks-are-scored-separately", + "query": "invoice approval", + "chunks": [ + { + "id": "long-overlap", + "filePath": "src/ledger.ts", + "startLine": 1, + "content": "invoice approval\ninvoice approval\nreturn approved\nledger entries retain stable boundaries" + }, + { + "id": "short-overlap", + "filePath": "src/ledger.ts", + "startLine": 2, + "content": "invoice approval\nreturn approved" + } + ], + "expectedIds": ["short-overlap"], + "expectedPathPatterns": ["src/ledger.ts"], + "expectedProperties": [ + "the second chunk exactly matches overlapping source lines", + "rescue returns one representative per source file", + "density ranks body evidence per chunk before file deduplication", + "query term counts are not aggregated across chunks" + ] + }, + { + "id": "identical-full-bodies-preserve-original-ordinal", + "query": "invoice approval", + "chunks": [ + { + "id": "z-first", + "filePath": "src/z.ts", + "content": "invoice approval validates the request" + }, + { + "id": "a-second", + "filePath": "src/a.ts", + "content": "invoice approval validates the request" + } + ], + "expectedIds": ["z-first", "a-second"], + "expectedPathPatterns": ["src/z.ts", "src/a.ts"], + "expectedProperties": [ + "identical normalized bodies have equal density", + "lexicographic path or id does not replace ordinal tie-breaking" + ] + }, + { + "id": "body-evidence-beats-metadata-only-density-zero", + "query": "invoice approval", + "chunks": [ + { + "id": "metadata-only", + "filePath": "src/metadata.ts", + "content": " \n ", + "metadata": { + "componentName": "InvoiceApproval" + } + }, + { + "id": "body-present", + "filePath": "src/body.ts", + "content": "invoice approval", + "metadata": { + "componentName": "InvoiceApproval" + } + } + ], + "expectedIds": ["body-present", "metadata-only"], + "expectedPathPatterns": ["src/body.ts", "src/metadata.ts"], + "expectedDensityById": { + "metadata-only": 0 + }, + "expectedProperties": [ + "primary evidence ties", + "empty body uses max(1, body length)", + "metadata does not enter the density numerator or denominator" + ] + }, + { + "id": "empty-metadata-only-bodies-have-finite-zero-density", + "query": "invoice approval", + "chunks": [ + { + "id": "empty-first", + "filePath": "src/empty-a.ts", + "content": "", + "metadata": { + "componentName": "InvoiceApproval" + } + }, + { + "id": "empty-second", + "filePath": "src/empty-b.ts", + "content": " \n ", + "metadata": { + "componentName": "InvoiceApproval" + } + } + ], + "expectedIds": ["empty-first", "empty-second"], + "expectedPathPatterns": ["src/empty-a.ts", "src/empty-b.ts"], + "expectedDensityById": { + "empty-first": 0, + "empty-second": 0 + }, + "expectedProperties": [ + "density is finite and exactly zero", + "equal primary evidence and density preserve ordinal" + ] + }, + { + "id": "equal-primary-tiny-wrapper-wins-density-with-explicit-limitation", + "query": "archive policy", + "chunks": [ + { + "id": "tiny-wrapper", + "filePath": "src/wrapper.ts", + "content": "archive policy" + }, + { + "id": "repeated-long-body", + "filePath": "src/implementation.ts", + "content": "archive archive policy policy filler filler filler filler" + } + ], + "expectedIds": ["tiny-wrapper", "repeated-long-body"], + "expectedPathPatterns": ["src/wrapper.ts", "src/implementation.ts"], + "limitation": "Primary lexical evidence ties, so the fixed formula ranks the short wrapper first (sqrt(1)+sqrt(1) over sqrt(2), versus sqrt(2)+sqrt(2) over sqrt(8)). This order is not evidence that the wrapper is semantically more useful; preserving this counterexample prevents an unsupported semantic claim.", + "expectedProperties": [ + "equal primary ranks invoke density as a separate secondary comparison", + "the higher-density result may be a wrapper", + "density does not establish semantic usefulness" + ] + }, + { + "id": "ambiguous-primary-density-order-is-not-semantic-proof", + "query": "invoice approval", + "chunks": [ + { + "id": "comment-repetition", + "filePath": "src/comment.ts", + "content": "// invoice approval invoice approval invoice approval" + }, + { + "id": "implementation", + "filePath": "src/implementation.ts", + "content": "export function approve(invoice) { return policyAllowsApproval(invoice); }" + } + ], + "expectedIds": ["comment-repetition", "implementation"], + "expectedPathPatterns": ["src/comment.ts", "src/implementation.ts"], + "limitation": "Primary lexical evidence ties. The fixed formula places the repeated comment first; the available evidence cannot establish whether the comment or implementation is semantically more useful. This expected order is algorithmic, not a semantic winner claim.", + "expectedProperties": [ + "ambiguous exact-primary tie is retained explicitly", + "density does not prove action meaning or semantic relevance" + ] + } + ], + "sourceValidationCases": [ + { + "id": "stale-sources-within-nomination-pool-are-skipped", + "query": "invoice approval", + "staleChunkGenerator": { + "count": 30, + "idPattern": "stale-{index}", + "filePathPattern": "src/stale-{index}.ts", + "content": "invoice approval" + }, + "currentChunks": [ + { + "id": "current", + "filePath": "src/current.ts", + "content": "invoice approval validates current source" + } + ], + "mutateFilePathPattern": "src/stale-{index}.ts", + "replacementSource": "export const unrelated = true;", + "expectedIds": ["current"], + "expectedProperties": [ + "31 candidates fit inside the existing nomination pool", + "stale sources are excluded before results are returned", + "density cannot bypass source validation", + "no beyond-window refill is required" + ] + }, + { + "id": "changed-only-source-cannot-supply-density-evidence", + "query": "invoice approval", + "chunks": [ + { + "id": "changed", + "filePath": "src/changed.ts", + "content": "invoice approval validates the indexed source" + } + ], + "mutateFilePath": "src/changed.ts", + "replacementSource": "export const unrelated = true;", + "expectedIds": [], + "expectedProperties": [ + "changed source is invalidated by the existing harness", + "density cannot bypass source integrity checks" + ] + } + ] +} diff --git a/tests/fixtures/search-rescue-nomination.json b/tests/fixtures/search-rescue-nomination.json new file mode 100644 index 0000000..efd922e --- /dev/null +++ b/tests/fixtures/search-rescue-nomination.json @@ -0,0 +1,382 @@ +{ + "version": 1, + "purpose": "Frozen general rescue-only nomination witnesses and negative controls; synthetic unit contract, not a retrieval accuracy benchmark.", + "contract": { + "scope": "Replace rescue fuzzy OR nomination deliberately; keep strict fuzzy retrieval and exact rescue admission unchanged.", + "nomination": "Conservative literal fragments across content, basename, componentName, symbolName, functionName, className and exports; filters before the dynamic cap; stable original chunk ordinal.", + "window": "min(750,max(256,limit*80)); overflow is disclosed through candidateBudgetExhausted. Witness completeness applies inside the nominated window, not beyond it.", + "ranking": "Existing exact evidence rank and score remain; equal exact evidence uses original ordinal instead of fuzzy nomination score. This is an intentional nomination and tie-order change, not parity.", + "normalization": "Existing ASCII lexical splitting and case folding only; no new Unicode transliteration, synonyms or language-specific rules.", + "fallback": "No fuzzy OR rescan in rescue, including overflow/refill; original strict fuzzy search remains the typo and extended-syntax route." + }, + "lexicalCases": [ + { + "id": "plural-policy-to-singular", + "query": "policy archive", + "content": "/* policies archiving */", + "expected": true, + "expectedScore": 0.5 + }, + { + "id": "singular-to-plural-query", + "query": "policies archives", + "content": "/* policy archive */", + "expected": true, + "expectedScore": 0.5 + }, + { + "id": "plural-library", + "query": "library archive", + "content": "/* libraries archived */", + "expected": true, + "expectedScore": 0.5 + }, + { + "id": "silent-e-ing", + "query": "archive policy", + "content": "/* archiving policies */", + "expected": true, + "expectedScore": 0.5 + }, + { + "id": "ed-to-ing", + "query": "archived policy", + "content": "/* archiving policy */", + "expected": true, + "expectedScore": 0.5 + }, + { + "id": "double-consonant-format", + "query": "formatting invoice", + "content": "/* formatted invoices */", + "expected": true, + "expectedScore": 0.5 + }, + { + "id": "double-consonant-stop", + "query": "stopping invoice", + "content": "/* stopped invoices */", + "expected": true, + "expectedScore": 0.5 + }, + { + "id": "camel-case", + "query": "invoice archive", + "content": "export const archiveInvoices = true;", + "expected": true, + "expectedScore": 0.5 + }, + { + "id": "snake-case", + "query": "credit ledger", + "content": "credit_ledger = True", + "language": "python", + "expected": true, + "expectedScore": 0.5 + }, + { + "id": "acronym-boundary", + "query": "http archive", + "content": "type HTTPArchive = {};", + "expected": true, + "expectedScore": 0.5 + }, + { + "id": "uppercase", + "query": "POLICY ARCHIVE", + "content": "/* POLICIES ARCHIVING */", + "expected": true, + "expectedScore": 0.5 + }, + { + "id": "unicode-surroundings", + "query": "policy archive", + "content": "/* λ archiving 政策 policies café */", + "expected": true, + "expectedScore": 0.5 + }, + { + "id": "unicode-lookalikes-not-folded", + "query": "policy archive", + "content": "/* рolicy аrchive */", + "expected": false + }, + { + "id": "diacritic-not-transliterated", + "query": "policy archive", + "content": "/* policies arçhive */", + "expected": false + }, + { + "id": "repeated-inflections-one-clue", + "query": "policy policies", + "content": "/* policy policies */", + "expected": false + }, + { + "id": "source-inflections-one-clue", + "query": "archive policy", + "content": "/* archived archiving archive */", + "expected": false + }, + { + "id": "substring-collision", + "query": "archive policy", + "content": "/* monarchivel policies */", + "expected": false + }, + { + "id": "single-natural-clue", + "query": "archive", + "content": "/* archive policy */", + "expected": false + }, + { + "id": "stop-word-not-corroboration", + "query": "where archive", + "content": "/* where archive */", + "expected": false + }, + { + "id": "comment-clues-are-lexical", + "query": "archive policy", + "content": "// archive policy", + "expected": true, + "expectedScore": 0.5 + }, + { + "id": "tags-do-not-admit", + "query": "archive policy", + "content": "export const value = 0;", + "tags": ["archive", "policy"], + "expected": false + }, + { + "id": "component-name-only", + "query": "archive policy", + "content": "export const value = 0;", + "metadata": { + "componentName": "ArchivePolicy" + }, + "expected": true, + "expectedScore": 0 + }, + { + "id": "symbol-name-only", + "query": "archive policy", + "content": "export const value = 0;", + "metadata": { + "symbolName": "ArchivePolicy" + }, + "expected": true, + "expectedScore": 0 + }, + { + "id": "basename-only", + "query": "archive policy", + "content": "export const value = 0;", + "filePath": "src/archive-policy.ts", + "expected": true, + "expectedScore": 0.5 + } + ], + "representativeCases": [ + { + "id": "function-metadata-representative", + "query": "archive policy", + "chunks": [ + { + "id": "declaration", + "filePath": "src/a.ts", + "content": "export const value = 0;", + "metadata": { + "functionName": "ArchivePolicy" + } + }, + { + "id": "body", + "filePath": "src/a.ts", + "startLine": 3, + "content": "// archive policy" + } + ], + "expectedIds": ["declaration"] + }, + { + "id": "class-metadata-representative", + "query": "archive policy", + "chunks": [ + { + "id": "declaration", + "filePath": "src/a.ts", + "content": "export const value = 0;", + "metadata": { + "className": "ArchivePolicy" + } + }, + { + "id": "body", + "filePath": "src/a.ts", + "startLine": 3, + "content": "// archive policy" + } + ], + "expectedIds": ["declaration"] + }, + { + "id": "export-metadata-representative", + "query": "archive policy", + "chunks": [ + { + "id": "export", + "filePath": "src/a.ts", + "content": "export const value = 0;", + "exports": ["ArchivePolicy"] + }, + { + "id": "body", + "filePath": "src/a.ts", + "startLine": 3, + "content": "// archive policy" + } + ], + "expectedIds": ["export"] + }, + { + "id": "merged-symbol-representative", + "query": "archive policy", + "chunks": [ + { + "id": "declaration", + "filePath": "src/a.ts", + "content": "export const value = 0;", + "metadata": { + "symbolName": "ArchivePolicy+Other" + } + }, + { + "id": "body", + "filePath": "src/a.ts", + "startLine": 3, + "content": "// archive policy" + } + ], + "expectedIds": ["declaration"] + }, + { + "id": "same-file-disjoint-clues", + "query": "archive policy", + "chunks": [ + { + "id": "first", + "filePath": "src/a.ts", + "content": "// archive" + }, + { + "id": "second", + "filePath": "src/a.ts", + "startLine": 3, + "content": "// policy" + } + ], + "expectedIds": ["first"] + }, + { + "id": "equal-evidence-original-ordinal", + "query": "archive policy", + "chunks": [ + { + "id": "z-first", + "filePath": "src/z.ts", + "content": "// archive policy" + }, + { + "id": "a-second", + "filePath": "src/a.ts", + "content": "// archive policy" + } + ], + "expectedIds": ["z-first", "a-second"] + }, + { + "id": "equal-evidence-reversed-ordinal", + "query": "archive policy", + "chunks": [ + { + "id": "a-first", + "filePath": "src/a.ts", + "content": "// archive policy" + }, + { + "id": "z-second", + "filePath": "src/z.ts", + "content": "// archive policy" + } + ], + "expectedIds": ["a-first", "z-second"] + } + ], + "populationCases": [ + { + "id": "below-minimum-window", + "limit": 1, + "count": 255, + "cap": 256, + "exhausted": false + }, + { + "id": "at-minimum-window", + "limit": 1, + "count": 256, + "cap": 256, + "exhausted": false + }, + { + "id": "over-minimum-window", + "limit": 1, + "count": 257, + "cap": 256, + "exhausted": true + }, + { + "id": "intermediate-window", + "limit": 9, + "count": 721, + "cap": 720, + "exhausted": true + }, + { + "id": "at-maximum-window", + "limit": 10, + "count": 750, + "cap": 750, + "exhausted": false + }, + { + "id": "over-maximum-window", + "limit": 10, + "count": 751, + "cap": 750, + "exhausted": true + } + ], + "lifecycleCases": ["changed", "missing", "directory", "invalid-manifest", "outside-root"], + "graphCases": ["current", "no-edge", "changed-companion", "directory-companion"], + "strictCases": [ + { + "id": "typo-fuzzy-retained", + "query": "archve polcy", + "expectsResult": true + }, + { + "id": "extended-or-retained", + "query": "archive | policy", + "expectsResult": true + }, + { + "id": "exact-identifier-retained", + "query": "ArchivePolicy", + "expectsResult": true + } + ] +} diff --git a/tests/fixtures/structural-context-regressions.json b/tests/fixtures/structural-context-regressions.json new file mode 100644 index 0000000..2d90c33 --- /dev/null +++ b/tests/fixtures/structural-context-regressions.json @@ -0,0 +1,176 @@ +{ + "version": 1, + "purpose": "General JavaScript and TypeScript source facts for structural ownership, bounded source witnesses, and source-versus-synthetic-context regressions. This is a human-checkable regression fixture, not a retrieval-ranking benchmark.", + "ownerChainDefinition": "Each sourceSymbols entry names the exact static lexical route in source from its outer named declaration/property through the executable leaf. Static string keys use their decoded key. For an unresolved dynamic computed key, the route may stop at the nearest established owner; scopeOnly marks that route as context and does not claim that owner declares the callback. Dynamic computed keys are omitted from the route and listed in unresolvedNames. bodyMarkers belong to that entry's implementation body; forbiddenSiblingBodyMarkers are source markers owned by another listed sibling.", + "cases": [ + { + "id": "nested-object-static-owner-chain", + "language": "typescript", + "source": "const services = { billing: { reconcile: async (invoice: Invoice) => { return persistInvoice(invoice); } } };", + "sourceSymbols": [{ "ownerChain": ["services", "billing", "reconcile"], "bodyMarkers": ["persistInvoice(invoice)"] }] + }, + { + "id": "same-leaf-name-under-distinct-owners", + "language": "javascript", + "source": "const billing = {\n refresh: () => billingCache.reload()\n};\nconst catalog = {\n refresh: () => catalogCache.reload()\n};", + "sourceSymbols": [ + { "ownerChain": ["billing", "refresh"], "bodyMarkers": ["billingCache.reload()"], "forbiddenSiblingBodyMarkers": ["catalogCache.reload()"] }, + { "ownerChain": ["catalog", "refresh"], "bodyMarkers": ["catalogCache.reload()"], "forbiddenSiblingBodyMarkers": ["billingCache.reload()"] } + ] + }, + { + "id": "sibling-body-isolation", + "language": "typescript", + "source": "const jobs = {\n export: { run: () => writeArchive() },\n import: { run: () => readArchive() }\n};", + "sourceSymbols": [ + { "ownerChain": ["jobs", "export", "run"], "bodyMarkers": ["writeArchive()"], "forbiddenSiblingBodyMarkers": ["readArchive()"] }, + { "ownerChain": ["jobs", "import", "run"], "bodyMarkers": ["readArchive()"], "forbiddenSiblingBodyMarkers": ["writeArchive()"] } + ] + }, + { + "id": "same-line-property-callback-bodies-stay-distinct", + "language": "javascript", + "source": "const operations = { archive: { run: () => writeArchiveFile() }, restore: { run: () => readArchiveFile() } };", + "sourceSymbols": [ + { "ownerChain": ["operations", "archive", "run"], "bodyMarkers": ["writeArchiveFile()"], "forbiddenSiblingBodyMarkers": ["readArchiveFile()"] }, + { "ownerChain": ["operations", "restore", "run"], "bodyMarkers": ["readArchiveFile()"], "forbiddenSiblingBodyMarkers": ["writeArchiveFile()"] } + ] + }, + { + "id": "static-string-computed-property", + "language": "javascript", + "source": "const actions = { [\"sync-now\"]: () => syncRemote() };", + "sourceSymbols": [{ "ownerChain": ["actions", "sync-now"], "bodyMarkers": ["syncRemote()"] }], + "unresolvedNames": [] + }, + { + "id": "dynamic-computed-property-name-remains-unresolved", + "language": "typescript", + "source": "const actions = { [actionName]: () => runAction() };", + "sourceSymbols": [{ "ownerChain": ["actions"], "scopeOnly": true, "bodyMarkers": ["runAction()"] }], + "unresolvedNames": ["actions[actionName]"] + }, + { + "id": "property-arrow-callback-body", + "language": "typescript", + "source": "const listeners = { onMessage: (message: Message) => { recordMessage(message); acknowledge(message.id); } };", + "sourceSymbols": [{ "ownerChain": ["listeners", "onMessage"], "bodyMarkers": ["recordMessage(message)", "acknowledge(message.id)"] }] + }, + { + "id": "class-method-with-property-callback", + "language": "typescript", + "source": "class QueueWorker { start() { const hooks = { onReady: () => openQueue() }; return hooks; } }", + "sourceSymbols": [{ "ownerChain": ["QueueWorker", "start", "hooks", "onReady"], "bodyMarkers": ["openQueue()"] }] + }, + { + "id": "class-method-sibling-isolation", + "language": "javascript", + "source": "class Cache {\n load() { return readDisk(); }\n save() { return writeDisk(); }\n}", + "sourceSymbols": [ + { "ownerChain": ["Cache", "load"], "bodyMarkers": ["readDisk()"], "forbiddenSiblingBodyMarkers": ["writeDisk()"] }, + { "ownerChain": ["Cache", "save"], "bodyMarkers": ["writeDisk()"], "forbiddenSiblingBodyMarkers": ["readDisk()"] } + ] + }, + { + "id": "standalone-function-declaration", + "language": "typescript", + "source": "export function normalizeRecord(record: RecordInput): RecordOutput { return mapRecord(record); }", + "sourceSymbols": [{ "ownerChain": ["normalizeRecord"], "bodyMarkers": ["mapRecord(record)"] }] + }, + { + "id": "standalone-function-expression", + "language": "javascript", + "source": "const calculateTotal = function (items) { return sumPrices(items); };", + "sourceSymbols": [{ "ownerChain": ["calculateTotal"], "bodyMarkers": ["sumPrices(items)"] }] + }, + { + "id": "class-and-function-compatible-source-facts", + "language": "typescript", + "source": "class Ledger {\n close() { flushEntries(); }\n}\nfunction openLedger() { return createLedger(); }", + "sourceSymbols": [ + { "ownerChain": ["Ledger", "close"], "bodyMarkers": ["flushEntries()"] }, + { "ownerChain": ["openLedger"], "bodyMarkers": ["createLedger()"] } + ] + }, + { + "id": "nested-static-property-chain-three-levels", + "language": "javascript", + "source": "const app = { api: { reports: { build: () => assembleReport() } } };", + "sourceSymbols": [{ "ownerChain": ["app", "api", "reports", "build"], "bodyMarkers": ["assembleReport()"] }] + }, + { + "id": "static-computed-key-inside-nested-owner", + "language": "typescript", + "source": "const routes = { admin: { [\"user-list\"]: () => fetchUsers() } };", + "sourceSymbols": [{ "ownerChain": ["routes", "admin", "user-list"], "bodyMarkers": ["fetchUsers()"] }], + "unresolvedNames": [] + }, + { + "id": "nested-container-retains-distinct-bounded-witness-markers", + "language": "javascript", + "source": "const pipeline = {\n ingest: { run: () => { validateInput(); decodePayload(); persistPayload(); } },\n report: { run: () => renderSummary() }\n};", + "sourceSymbols": [ + { "ownerChain": ["pipeline", "ingest", "run"], "bodyMarkers": ["validateInput()", "decodePayload()", "persistPayload()"], "boundedWitnessMarkers": ["validateInput()", "persistPayload()"], "forbiddenSiblingBodyMarkers": ["renderSummary()"] }, + { "ownerChain": ["pipeline", "report", "run"], "bodyMarkers": ["renderSummary()"], "boundedWitnessMarkers": ["renderSummary()"], "forbiddenSiblingBodyMarkers": ["validateInput()", "decodePayload()", "persistPayload()"] } + ] + }, + { + "id": "method-body-has-interior-witness-and-sibling-control", + "language": "typescript", + "source": "class Importer {\n importRows(rows: Row[]) {\n beginImport();\n const normalized = normalizeRows(rows);\n const checked = checkRows(normalized);\n const saved = saveRows(checked);\n finishImport(saved);\n }\n exportRows() { return writeRows(); }\n}", + "sourceSymbols": [ + { "ownerChain": ["Importer", "importRows"], "bodyMarkers": ["beginImport()", "normalizeRows(rows)", "checkRows(normalized)", "saveRows(checked)", "finishImport(saved)"], "boundedWitnessMarkers": ["beginImport()", "checkRows(normalized)", "finishImport(saved)"], "forbiddenSiblingBodyMarkers": ["writeRows()"] }, + { "ownerChain": ["Importer", "exportRows"], "bodyMarkers": ["writeRows()"], "boundedWitnessMarkers": ["writeRows()"], "forbiddenSiblingBodyMarkers": ["beginImport()", "normalizeRows(rows)", "checkRows(normalized)", "saveRows(checked)", "finishImport(saved)"] } + ] + }, + { + "id": "same-method-name-in-separate-classes", + "language": "typescript", + "source": "class LocalStore {\n clear() { clearLocalFiles(); }\n}\nclass RemoteStore {\n clear() { clearRemoteFiles(); }\n}", + "sourceSymbols": [ + { "ownerChain": ["LocalStore", "clear"], "bodyMarkers": ["clearLocalFiles()"], "forbiddenSiblingBodyMarkers": ["clearRemoteFiles()"] }, + { "ownerChain": ["RemoteStore", "clear"], "bodyMarkers": ["clearRemoteFiles()"], "forbiddenSiblingBodyMarkers": ["clearLocalFiles()"] } + ] + }, + { + "id": "type-declaration-is-not-function-body", + "language": "typescript", + "source": "type RowMapper = (row: Row) => MappedRow;\nfunction mapRow(row: Row): MappedRow { return convertRow(row); }", + "sourceSymbols": [{ "ownerChain": ["mapRow"], "bodyMarkers": ["convertRow(row)"] }], + "supportingOnlyMarkers": ["type RowMapper = (row: Row) => MappedRow"] + }, + { + "id": "test-companion-midchunk-line-is-supporting-context", + "language": "typescript", + "source": "describe(\"invoice parser\", () => {\n it(\"keeps the account id\", () => {\n const parsed = parseInvoice(fixture);\n expect(parsed.accountId).toBe(\"acct-7\");\n });\n});", + "sourceSymbols": [], + "supportingOnlyMarkers": ["expect(parsed.accountId).toBe(\"acct-7\")"] + }, + { + "id": "declaration-companion-midchunk-line-is-supporting-context", + "language": "typescript", + "source": "export interface InvoiceParser {\n parse(source: string): Invoice;\n}\nexport function parseInvoice(source: string): Invoice { return decodeInvoice(source); }", + "sourceSymbols": [{ "ownerChain": ["parseInvoice"], "bodyMarkers": ["decodeInvoice(source)"] }], + "supportingOnlyMarkers": ["parse(source: string): Invoice"] + }, + { + "id": "scope-label-text-is-not-source-body-evidence", + "language": "javascript", + "source": "const workers = {\n email: { send: () => deliverEmail() },\n sms: { send: () => deliverText() }\n};", + "sourceSymbols": [ + { "ownerChain": ["workers", "email", "send"], "bodyMarkers": ["deliverEmail()"], "forbiddenSiblingBodyMarkers": ["deliverText()"] }, + { "ownerChain": ["workers", "sms", "send"], "bodyMarkers": ["deliverText()"], "forbiddenSiblingBodyMarkers": ["deliverEmail()"] } + ], + "syntheticLabelsAbsentFromSource": ["workers > email > send", "workers > sms > send"] + }, + { + "id": "overlapping-name-words-do-not-cross-attribute-sibling-body", + "language": "typescript", + "source": "const tasks = {\n syncUsers: { run: () => uploadUsers() },\n syncTeams: { run: () => uploadTeams() }\n};", + "sourceSymbols": [ + { "ownerChain": ["tasks", "syncUsers", "run"], "bodyMarkers": ["uploadUsers()"], "forbiddenSiblingBodyMarkers": ["uploadTeams()"] }, + { "ownerChain": ["tasks", "syncTeams", "run"], "bodyMarkers": ["uploadTeams()"], "forbiddenSiblingBodyMarkers": ["uploadUsers()"] } + ] + } + ] +} diff --git a/tests/format-five-migration.test.ts b/tests/format-five-migration.test.ts new file mode 100644 index 0000000..8032832 --- /dev/null +++ b/tests/format-five-migration.test.ts @@ -0,0 +1,227 @@ +import { promises as fs } from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { connect } from '@lancedb/lancedb'; +import { afterEach, beforeEach, expect, it, vi } from 'vitest'; +const { provider } = vi.hoisted(() => ({ provider: vi.fn() })); +vi.mock('../src/embeddings/index.js', async (original) => ({ + ...(await original()), + getEmbeddingProvider: provider +})); +import { CodebaseIndexer } from '../src/core/indexer.js'; +import { readIndexMeta, IndexMetaSchema } from '../src/core/index-meta.js'; +import { IndexCapacityError, inspectIndexDiskState } from '../src/core/index-generation.js'; +import { analyzerRegistry } from '../src/core/analyzer-registry.js'; +import { InvalidSourceChunkError } from '../src/errors/index.js'; +import { INDEX_FORMAT_VERSION } from '../src/constants/codebase-context.js'; +let root = ''; +beforeEach(async () => { + root = await fs.mkdtemp(path.join(os.tmpdir(), 'cbc-format-five-')); + await fs.writeFile(path.join(root, 'route.ts'), 'export const route = "original";'); + provider.mockReset().mockResolvedValue({ + name: 'fixture', + modelName: 'migration', + dimensions: 384, + initialize: async () => undefined, + isReady: () => true, + embedBatch: async (texts: string[]) => texts.map(() => Array(384).fill(0.1)) + }); +}); +afterEach(async () => { + vi.restoreAllMocks(); + await fs.rm(root, { recursive: true, force: true }); +}); +const context = () => path.join(root, '.codebase-context'); +const build = (skipEmbedding?: boolean, incrementalOnly = false) => + new CodebaseIndexer({ + rootPath: root, + incrementalOnly, + config: { + ...(skipEmbedding === undefined ? {} : { skipEmbedding }), + embedding: { provider: 'custom', model: 'migration' } + } + }).index(); +async function bytes(directory: string): Promise> { + const result: Record = {}; + for (const item of await fs.readdir(directory, { withFileTypes: true })) { + const full = path.join(directory, item.name); + if (item.isDirectory()) { + for (const [name, value] of Object.entries(await bytes(full))) + result[path.join(item.name, name)] = value; + } else result[item.name] = (await fs.readFile(full)).toString('base64'); + } + return result; +} +async function oldFour() { + const generations = path.join(context(), '.generations'); + for (const id of await fs.readdir(generations)) { + const directory = path.join(generations, id); + for (const name of [ + 'index.json', + 'intelligence.json', + 'relationships.json', + 'health.json', + 'manifest.json', + 'index-meta.json' + ]) { + const file = path.join(directory, name); + const value = JSON.parse(await fs.readFile(file, 'utf8')); + if (value.header) value.header.formatVersion = 4; + if (name === 'index-meta.json') value.formatVersion = 4; + await fs.writeFile(file, JSON.stringify(value)); + } + const marker = path.join(directory, 'index', 'index-build.json'); + const value = JSON.parse(await fs.readFile(marker, 'utf8')); + value.formatVersion = 4; + await fs.writeFile(marker, JSON.stringify(value)); + } + const pointer = path.join(context(), 'index-meta.json'); + const value = JSON.parse(await fs.readFile(pointer, 'utf8')); + value.formatVersion = 4; + await fs.writeFile(pointer, JSON.stringify(value)); + return IndexMetaSchema.parse(value).buildId; +} +async function rows(id: string) { + const db = await connect(path.join(context(), '.generations', id, 'index')); + const table = await db.openTable('code_chunks'); + try { + return (await table.query().toArray()).map((row) => ({ + id: String(row.id), + content: String(row.content), + vector: Array.from(row.vector) + })); + } finally { + table.close(); + db.close(); + } +} +it.each( + [true, false].flatMap((skip) => [true, false].map((incremental) => ({ skip, incremental }))) +)( + 'migrates unchanged old4 mode skip=$skip incremental=$incremental', + async ({ skip, incremental }) => { + await build(skip); + const id = await oldFour(); + const native = skip ? undefined : await rows(id); + const old = await bytes(path.join(context(), '.generations', id)); + await build(undefined, incremental); + const meta = await readIndexMeta(root); + expect(meta.formatVersion).toBe(INDEX_FORMAT_VERSION); + expect(meta.buildId).not.toBe(id); + expect(meta.artifacts.vectorDb.embeddingEnabled).toBe(!skip); + expect(await bytes(path.join(context(), '.generations', id))).toEqual(old); + if (native) expect(await rows(id)).toEqual(native); + } +); +it('preserves actual old4 semantic pointer/vectors when the omitted-mode provider cannot initialize', async () => { + await build(false); + const id = await oldFour(); + const pointer = await fs.readFile(path.join(context(), 'index-meta.json')); + const old = await bytes(path.join(context(), '.generations', id)); + const native = await rows(id); + provider.mockRejectedValue(new Error('semantic runtime unavailable')); + await expect(build(undefined, true)).rejects.toThrow(/semantic runtime unavailable/); + expect(await fs.readFile(path.join(context(), 'index-meta.json'))).toEqual(pointer); + expect(await bytes(path.join(context(), '.generations', id))).toEqual(old); + expect(await rows(id)).toEqual(native); +}); +it.each([true, false])( + 'does not infer fresh mode from missing old4 root metadata skip=%s', + async (skip) => { + await build(skip); + await oldFour(); + await fs.rm(path.join(context(), 'index-meta.json')); + const old = await bytes(path.join(context(), '.generations')); + expect((await inspectIndexDiskState(root)).kind).toBe('recoverable-unpublished'); + await expect(build(undefined, true)).rejects.toThrow(/recover.*mode/i); + expect(await bytes(path.join(context(), '.generations'))).toEqual(old); + } +); +it.each([true, false])( + 'fails bounded migration at three pinned old4 generations incremental=%s', + async (incremental) => { + await build(true); + await build(false); + await build(false); + const active = await oldFour(); + const directory = path.join(context(), '.generations'); + const ids = await fs.readdir(directory); + const pins = path.join(context(), '.pins'); + await fs.mkdir(pins, { recursive: true }); + for (const id of ids.filter((id) => id !== active)) + await fs.writeFile( + path.join(pins, `${id}.json`), + JSON.stringify({ pid: process.pid, buildId: id }) + ); + const pointer = await fs.readFile(path.join(context(), 'index-meta.json')); + const old = await bytes(directory); + const native = await rows(active); + await expect(build(undefined, incremental)).rejects.toBeInstanceOf(IndexCapacityError); + expect(await fs.readFile(path.join(context(), 'index-meta.json'))).toEqual(pointer); + expect(await bytes(directory)).toEqual(old); + expect(await rows(active)).toEqual(native); + expect(await fs.readdir(pins)).toHaveLength(2); + } +); +it('preserves real semantic vectors after fatal producer corruption', async () => { + await build(false); + const id = (await readIndexMeta(root)).buildId; + const pointer = await fs.readFile(path.join(context(), 'index-meta.json')); + const old = await bytes(path.join(context(), '.generations', id)); + const native = await rows(id); + const analyze = analyzerRegistry.analyzeFile.bind(analyzerRegistry); + vi.spyOn(analyzerRegistry, 'analyzeFile').mockImplementation(async (...args) => { + const result = await analyze(...args); + if (result?.chunks[0]) result.chunks[0].filePath = path.join(root, 'foreign.ts'); + return result; + }); + await expect(build(false)).rejects.toBeInstanceOf(InvalidSourceChunkError); + expect(await fs.readFile(path.join(context(), 'index-meta.json'))).toEqual(pointer); + expect(await bytes(path.join(context(), '.generations', id))).toEqual(old); + expect(await rows(id)).toEqual(native); +}); + +it.each([true, false])( + 'rebuilds malformed unchanged old4 rows rather than stamping them as format5 skip=%s', + async (skip) => { + await build(skip); + const id = await oldFour(); + const file = path.join(context(), '.generations', id, 'index.json'); + const index = JSON.parse(await fs.readFile(file, 'utf8')); + index.chunks[0].startLine = 17; + index.chunks[0].endLine = 6; + index.chunks[0].metadata.merged = true; + await fs.writeFile(file, JSON.stringify(index)); + const old = await bytes(path.join(context(), '.generations', id)); + await build(undefined, true); + const meta = await readIndexMeta(root); + expect(meta.formatVersion).toBe(INDEX_FORMAT_VERSION); + expect(meta.buildId).not.toBe(id); + const rebuilt = JSON.parse( + await fs.readFile(path.join(context(), meta.artifacts.keywordIndex.path), 'utf8') + ); + expect(rebuilt.chunks.length).toBeGreaterThan(0); + for (const chunk of rebuilt.chunks) { + expect(chunk.startLine).toBeGreaterThanOrEqual(1); + expect(chunk.endLine).toBeGreaterThanOrEqual(chunk.startLine); + expect(chunk.content).toContain('original'); + } + expect(await bytes(path.join(context(), '.generations', id))).toEqual(old); + } +); +it.each([true, false])( + 'holds all three old4 orphans under explicit capacity recovery skip=%s', + async (skip) => { + await build(true); + await build(false); + await build(false); + await oldFour(); + await fs.rm(path.join(context(), 'index-meta.json')); + const old = await bytes(path.join(context(), '.generations')); + await expect(build(skip)).rejects.toBeInstanceOf(IndexCapacityError); + expect(await bytes(path.join(context(), '.generations'))).toEqual(old); + await expect(fs.access(path.join(context(), 'index-meta.json'))).rejects.toMatchObject({ + code: 'ENOENT' + }); + } +); diff --git a/tests/get-symbol-references.test.ts b/tests/get-symbol-references.test.ts index 6481b46..f3949a4 100644 --- a/tests/get-symbol-references.test.ts +++ b/tests/get-symbol-references.test.ts @@ -7,7 +7,8 @@ import { INDEX_FORMAT_VERSION, INDEX_META_FILENAME, INDEX_META_VERSION, - KEYWORD_INDEX_FILENAME + KEYWORD_INDEX_FILENAME, + MANIFEST_FILENAME } from '../src/constants/codebase-context.js'; type ToolCallRequest = { @@ -22,7 +23,9 @@ type ToolCallResponse = { isError?: boolean; }; -function getToolCallHandler(server: unknown): (request: ToolCallRequest) => Promise { +function getToolCallHandler( + server: unknown +): (request: ToolCallRequest) => Promise { const handlers = (server as { _requestHandlers?: unknown })._requestHandlers; if (!(handlers instanceof Map)) { throw new Error('Expected server._requestHandlers to be a Map'); @@ -34,6 +37,41 @@ function getToolCallHandler(server: unknown): (request: ToolCallRequest) => Prom return handler as (request: ToolCallRequest) => Promise; } +async function writeProjectSource( + rootPath: string, + relativePath: string, + content: string +): Promise { + const absolutePath = path.join(rootPath, relativePath); + await fs.mkdir(path.dirname(absolutePath), { recursive: true }); + await fs.writeFile(absolutePath, content, 'utf-8'); + return absolutePath; +} + +async function writeReferenceManifest( + rootPath: string, + buildId: string, + sourcePaths: string[] +): Promise { + const contextDir = path.join(rootPath, CODEBASE_CONTEXT_DIRNAME); + const metadataPath = path.join(contextDir, INDEX_META_FILENAME); + const meta = JSON.parse(await fs.readFile(metadataPath, 'utf8')) as { + artifacts: Record; + }; + meta.artifacts.manifest = { path: MANIFEST_FILENAME }; + await fs.writeFile(metadataPath, JSON.stringify(meta, null, 2), 'utf8'); + await fs.writeFile( + path.join(contextDir, MANIFEST_FILENAME), + JSON.stringify({ + version: 1, + generatedAt: new Date().toISOString(), + header: { buildId, formatVersion: INDEX_FORMAT_VERSION }, + files: Object.fromEntries(sourcePaths.map((sourcePath) => [sourcePath, 'a'.repeat(16)])) + }), + 'utf8' + ); +} + describe('get_symbol_references MCP tool', () => { let tempRoot: string | null = null; let originalArgv: string[] | null = null; @@ -72,6 +110,36 @@ describe('get_symbol_references MCP tool', () => { expect(TOOLS.some((tool) => tool.name === 'get_symbol_references')).toBe(true); }); + it('marks a manifest-listed source integrity failure as an MCP error', async () => { + if (!tempRoot) throw new Error('tempRoot not initialized'); + + const { server } = await import('../src/index.js'); + const { CodebaseIndexer } = await import('../src/core/indexer.js'); + const sourcePath = await writeProjectSource( + tempRoot, + 'src/a.ts', + 'export const alpha = 1;\nconsole.log(alpha);\n' + ); + await new CodebaseIndexer({ rootPath: tempRoot }).index(); + await fs.unlink(sourcePath); + await fs.mkdir(sourcePath); + + const response = await getToolCallHandler(server)({ + jsonrpc: '2.0', + id: 1, + method: 'tools/call', + params: { name: 'get_symbol_references', arguments: { symbol: 'alpha' } } + }); + + expect(response.isError).toBe(true); + const payload = JSON.parse(response.content[0].text) as { + status: string; + message: string; + }; + expect(payload.status).toBe('error'); + expect(payload.message).toContain('src/a.ts'); + }); + it('returns usageCount and top usages from keyword index', async () => { if (!tempRoot) { throw new Error('tempRoot not initialized'); @@ -90,16 +158,22 @@ describe('get_symbol_references MCP tool', () => { 'utf-8' ); + const sourceA = 'export function alpha() {\n return beta(alpha);\n}\n'; + const sourceB = 'const beta = alpha + 1;\n'; + const filePathA = await writeProjectSource(tempRoot, 'src/a.ts', sourceA); + const filePathB = await writeProjectSource(tempRoot, 'src/b.ts', sourceB); const chunks = [ { - content: 'export function alpha() {\n return beta(alpha);\n}', + content: sourceA, startLine: 10, - relativePath: 'src/a.ts' + relativePath: 'src/a.ts', + filePath: filePathA }, { - content: 'const beta = alpha + 1;\n', + content: sourceB, startLine: 2, - relativePath: 'src/b.ts' + relativePath: 'src/b.ts', + filePath: filePathB } ]; @@ -128,6 +202,7 @@ describe('get_symbol_references MCP tool', () => { ), 'utf-8' ); + await writeReferenceManifest(tempRoot, buildId, ['src/a.ts', 'src/b.ts']); const { server } = await import('../src/index.js'); const handler = getToolCallHandler(server); @@ -151,6 +226,11 @@ describe('get_symbol_references MCP tool', () => { expect(payload.usages.length).toBeLessThanOrEqual(2); expect(payload.confidence).toBe('syntactic'); expect(typeof payload.isComplete).toBe('boolean'); + expect(payload.coverage).toMatchObject({ + candidateFiles: 2, + sourcesRead: 2, + filesUnavailable: 0 + }); for (const usage of payload.usages) { expect(usage.file).toBeTypeOf('string'); @@ -160,6 +240,62 @@ describe('get_symbol_references MCP tool', () => { } }); + it('returns a structured full-reindex error when the reference manifest is missing', async () => { + if (!tempRoot) { + throw new Error('tempRoot not initialized'); + } + + const contextDir = path.join(tempRoot, CODEBASE_CONTEXT_DIRNAME); + await fs.mkdir(contextDir, { recursive: true }); + const buildId = 'test-build-symbol-refs-missing-manifest'; + await fs.mkdir(path.join(contextDir, 'index'), { recursive: true }); + await fs.writeFile( + path.join(contextDir, 'index', 'index-build.json'), + JSON.stringify({ buildId, formatVersion: INDEX_FORMAT_VERSION }), + 'utf8' + ); + await fs.writeFile( + path.join(contextDir, KEYWORD_INDEX_FILENAME), + JSON.stringify({ + header: { buildId, formatVersion: INDEX_FORMAT_VERSION }, + chunks: [] + }), + 'utf8' + ); + await fs.writeFile( + path.join(contextDir, INDEX_META_FILENAME), + JSON.stringify({ + metaVersion: INDEX_META_VERSION, + formatVersion: INDEX_FORMAT_VERSION, + buildId, + generatedAt: new Date().toISOString(), + toolVersion: 'test', + artifacts: { + keywordIndex: { path: KEYWORD_INDEX_FILENAME }, + vectorDb: { path: 'index', provider: 'test' }, + manifest: { path: MANIFEST_FILENAME } + } + }), + 'utf8' + ); + + const { server } = await import('../src/index.js'); + const handler = getToolCallHandler(server); + const response = await handler({ + jsonrpc: '2.0', + id: 2, + method: 'tools/call', + params: { name: 'get_symbol_references', arguments: { symbol: 'alpha' } } + }); + + expect(response.isError).toBe(true); + const payload = JSON.parse(response.content[0].text) as { status: string; message: string }; + expect(payload.status).toBe('error'); + expect(payload.message).toMatch( + /full reindex.*codebase-context reindex without --incremental/i + ); + }); + it('excludes comment and string matches when file exists and Tree-sitter is supported', async () => { if (!tempRoot) { throw new Error('tempRoot not initialized'); @@ -186,7 +322,7 @@ describe('get_symbol_references MCP tool', () => { 'const alpha = 1;', '// alpha only in comment', 'console.log(alpha);', - 'const s = \"alpha\";' + 'const s = "alpha";' ].join('\n'); await fs.writeFile(filePath, fileContent, 'utf-8'); @@ -224,6 +360,7 @@ describe('get_symbol_references MCP tool', () => { ), 'utf-8' ); + await writeReferenceManifest(tempRoot, buildId, ['src/example.ts']); const { server } = await import('../src/index.js'); const handler = getToolCallHandler(server); @@ -272,11 +409,14 @@ describe('get_symbol_references MCP tool', () => { 'utf-8' ); + const source = 'export function alpha() {\n return alpha;\n}\n'; + const filePath = await writeProjectSource(tempRoot, 'src/test.ts', source); const chunks = [ { - content: 'export function alpha() {\n return alpha;\n}', + content: source, startLine: 1, - relativePath: 'src/test.ts' + relativePath: 'src/test.ts', + filePath } ]; @@ -305,6 +445,7 @@ describe('get_symbol_references MCP tool', () => { ), 'utf-8' ); + await writeReferenceManifest(tempRoot, buildId, ['src/test.ts']); const { server } = await import('../src/index.js'); const handler = getToolCallHandler(server); @@ -351,11 +492,13 @@ describe('get_symbol_references MCP tool', () => { // Create a chunk with many matches const content = 'foo foo foo foo foo foo foo foo foo foo foo'; + const filePath = await writeProjectSource(tempRoot, 'src/test.ts', content); const chunks = [ { content, startLine: 1, - relativePath: 'src/test.ts' + relativePath: 'src/test.ts', + filePath } ]; @@ -384,6 +527,7 @@ describe('get_symbol_references MCP tool', () => { ), 'utf-8' ); + await writeReferenceManifest(tempRoot, buildId, ['src/test.ts']); const { server } = await import('../src/index.js'); const handler = getToolCallHandler(server); diff --git a/tests/helpers/index-lease-control-child.ts b/tests/helpers/index-lease-control-child.ts new file mode 100644 index 0000000..7456aa8 --- /dev/null +++ b/tests/helpers/index-lease-control-child.ts @@ -0,0 +1,44 @@ +import { promises as fs } from 'node:fs'; +import { once } from 'node:events'; +import { withIndexWriter } from '../../src/core/index-generation.js'; + +const [mode, target] = process.argv.slice(2); +if (!target) throw new Error('Expected a fixture path'); +const watchdog = setTimeout(() => process.exit(124), 8000); +try { + if (mode === 'reader') { + process.send?.({ type: 'ready' }); + const [request] = await once(process, 'message'); + if ( + !request || + typeof request !== 'object' || + !('target' in request) || + typeof request.target !== 'string' + ) + throw new Error('Expected a ticket read target'); + const handle = await fs.open(request.target, 'r'); + try { + process.send?.({ type: 'held', bytes: (await handle.readFile()).toString('utf8') }); + await once(process, 'message'); + } finally { + await handle.close(); + } + process.send?.({ type: 'closed' }); + } else if (mode === 'writer') { + await withIndexWriter(target, async () => { + process.send?.({ type: 'entered' }); + await once(process, 'message'); + process.send?.({ type: 'leaving', time: Date.now() }); + }); + process.send?.({ type: 'finished' }); + } else throw new Error('Unknown control mode'); +} catch (error) { + process.send?.({ + type: 'failed', + message: error instanceof Error ? error.message : String(error) + }); + process.exitCode = 1; +} finally { + clearTimeout(watchdog); + process.disconnect?.(); +} diff --git a/tests/helpers/index-mode-child.ts b/tests/helpers/index-mode-child.ts new file mode 100644 index 0000000..e5a9918 --- /dev/null +++ b/tests/helpers/index-mode-child.ts @@ -0,0 +1,57 @@ +import { promises as fs } from 'node:fs'; +import path from 'node:path'; +import { CodebaseIndexer } from '../../src/core/indexer.js'; +import { handleMapCli } from '../../src/cli-map.js'; +import { TransformersEmbeddingProvider } from '../../src/embeddings/transformers.js'; + +const [root, action] = process.argv.slice(2); +if (!root) throw new Error('Expected test root'); +// Stub only model computation; real indexer, native storage, leases and publication run. +Object.assign(TransformersEmbeddingProvider.prototype, { + initialize: async () => undefined, + embedBatch: async (texts: string[]) => + texts.map(() => Array.from({ length: 384 }, (_, i) => i / 1000)) +}); +const release = () => new Promise((resolve) => process.once('message', () => resolve())); +const rename = fs.rename.bind(fs); +fs.rename = async (source, target) => { + await rename(source, target); + if ( + String(target).includes(`${path.sep}.writers${path.sep}`) && + String(target).endsWith('.json') + ) { + const ticket: { choosing: boolean; ticket: number } = JSON.parse( + await fs.readFile(target, 'utf8') + ); + if (!ticket.choosing && ticket.ticket > 0) { + process.send?.({ type: action === 'publish' ? 'held' : 'queued' }); + if (action === 'publish') await release(); + } + } +}; +const write = fs.writeFile.bind(fs); +fs.writeFile = async (...args) => { + if (action === 'semantic-stage' && path.basename(String(args[0])) === 'index-build.json') { + process.send?.({ type: 'native' }); + await release(); + } + await write(...args); +}; +try { + if (action === 'map') { + process.env.CODEBASE_ROOT = root; + process.env.CODEBASE_CONTEXT_CONFIG_PATH = path.join(root, 'server-config.json'); + await handleMapCli(['--json']); + } else { + await new CodebaseIndexer({ rootPath: root, config: { skipEmbedding: false } }).index(); + } + process.send?.({ type: 'complete' }); +} catch (error) { + process.send?.({ + type: 'failed', + message: error instanceof Error ? error.message : String(error) + }); + process.exitCode = 1; +} finally { + process.disconnect?.(); +} diff --git a/tests/helpers/index-publication-child.ts b/tests/helpers/index-publication-child.ts new file mode 100644 index 0000000..cd9d36c --- /dev/null +++ b/tests/helpers/index-publication-child.ts @@ -0,0 +1,34 @@ +import { promises as fs } from 'node:fs'; +import path from 'node:path'; +import { CodebaseIndexer } from '../../src/core/indexer.js'; + +const [rootPath, mode, edge] = process.argv.slice(2); +if (!rootPath || !mode || !edge) throw new Error('Expected root, mode and edge'); + +const pause = async () => { + process.send?.({ edge }); + await new Promise(() => undefined); +}; +const rename = fs.rename.bind(fs); +fs.rename = async (source, target) => { + await rename(source, target); + if ( + edge === 'published-meta' && + String(target) === path.join(rootPath, '.codebase-context', 'index-meta.json') + ) { + await pause(); + } +}; +const writeFile = fs.writeFile.bind(fs); +fs.writeFile = async (...args) => { + await writeFile(...args); + if (edge === 'vector-marker' && path.basename(String(args[0])) === 'index-build.json') + await pause(); +}; + +await new CodebaseIndexer({ + rootPath, + config: { skipEmbedding: true }, + incrementalOnly: mode === 'incremental' +}).index(); +process.send?.({ completed: true }); diff --git a/tests/helpers/index-writer-child.ts b/tests/helpers/index-writer-child.ts new file mode 100644 index 0000000..ba09d1f --- /dev/null +++ b/tests/helpers/index-writer-child.ts @@ -0,0 +1,43 @@ +import { promises as fs } from 'node:fs'; +import path from 'node:path'; +import { CodebaseIndexer } from '../../src/core/indexer.js'; + +const [rootPath, hold, mode] = process.argv.slice(2); +if (!rootPath) throw new Error('Expected project root'); +const rename = fs.rename.bind(fs); +fs.rename = async (source, target) => { + await rename(source, target); + if ( + String(target).includes(`${path.sep}.writers${path.sep}`) && + String(target).endsWith('.json') + ) { + const ticket: { choosing: boolean; ticket: number } = JSON.parse( + await fs.readFile(target, 'utf8') + ); + if (!ticket.choosing && ticket.ticket > 0) process.send?.({ type: 'queued' }); + } +}; +const writeFile = fs.writeFile.bind(fs); +fs.writeFile = async (...args) => { + await writeFile(...args); + if (hold === 'hold' && path.basename(String(args[0])) === 'index-build.json') { + process.send?.({ type: 'staged' }); + await new Promise((resolve) => process.once('message', () => resolve())); + } +}; +try { + const stats = await new CodebaseIndexer({ + rootPath, + config: { skipEmbedding: true }, + incrementalOnly: mode !== 'full' + }).index(); + process.send?.({ type: 'complete', incremental: stats.incremental }); +} catch (error) { + process.send?.({ + type: 'failed', + message: error instanceof Error ? error.message : String(error) + }); + process.exitCode = 1; +} finally { + process.disconnect?.(); +} diff --git a/tests/helpers/source-snapshot.ts b/tests/helpers/source-snapshot.ts new file mode 100644 index 0000000..a67293b --- /dev/null +++ b/tests/helpers/source-snapshot.ts @@ -0,0 +1,104 @@ +import { promises as fs } from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import type { CodebaseSearcher } from '../../src/core/search.js'; +import { + hashFileContent, + readManifest, + writeManifest, + type FileManifest +} from '../../src/core/manifest.js'; +import { + CODEBASE_CONTEXT_DIRNAME, + MANIFEST_FILENAME +} from '../../src/constants/codebase-context.js'; +import type { CodeChunk } from '../../src/types/index.js'; + +export interface SourceSnapshot { + rootPath: string; + manifest: FileManifest; + cleanup: () => Promise; +} + +export interface SearcherTestHarness { + initialized: boolean; + sourceManifest: FileManifest | null; + embeddingProvider: unknown; + storageProvider: unknown; + fuseIndex: unknown; + patternIntelligence: unknown; + semanticSearch: unknown; + keywordSearch: unknown; + collectHybridMatches: unknown; + search: CodebaseSearcher['search']; +} + +export function asSearcherTestHarness(searcher: CodebaseSearcher): SearcherTestHarness { + return searcher as unknown as SearcherTestHarness; +} + +function toRelativeSourcePath(filePath: string): string { + const normalized = filePath.replace(/\\/g, '/'); + const relativePath = normalized.replace(/^[A-Za-z]:\/repo\//i, '').replace(/^\/repo\//, ''); + if (path.isAbsolute(relativePath) || relativePath.split('/').includes('..')) { + throw new Error(`Fixture source path must be relative to the synthetic repo: ${filePath}`); + } + return relativePath; +} + +/** Materialize indexed chunks as a confined source tree plus the standard file manifest. */ +export async function materializeSourceSnapshot( + chunks: CodeChunk[], + prefix = 'search-source-fixture-' +): Promise { + const rootPath = await fs.mkdtemp(path.join(os.tmpdir(), prefix)); + try { + const byRelativePath = new Map(); + for (const chunk of chunks) { + const relativePath = toRelativeSourcePath(chunk.filePath); + const sameFile = byRelativePath.get(relativePath) ?? []; + sameFile.push(chunk); + byRelativePath.set(relativePath, sameFile); + } + + const files: Record = {}; + for (const [relativePath, fileChunks] of byRelativePath) { + const finalLine = Math.max(...fileChunks.map((chunk) => chunk.endLine)); + const lines = Array(Math.max(finalLine, 1)).fill('') as string[]; + for (const chunk of fileChunks) { + const contentLines = chunk.content.split('\n'); + contentLines.forEach((line, offset) => { + const lineIndex = chunk.startLine - 1 + offset; + if (lineIndex < lines.length) lines[lineIndex] = line; + }); + } + + const content = `${lines.join('\n')}\n`; + const sourcePath = path.join(rootPath, ...relativePath.split('/')); + await fs.mkdir(path.dirname(sourcePath), { recursive: true }); + await fs.writeFile(sourcePath, content); + files[relativePath] = hashFileContent(content); + for (const chunk of fileChunks) { + chunk.filePath = sourcePath; + chunk.relativePath = relativePath; + } + } + + const manifestPath = path.join(rootPath, CODEBASE_CONTEXT_DIRNAME, MANIFEST_FILENAME); + await writeManifest(manifestPath, { + version: 1, + generatedAt: new Date().toISOString(), + files + }); + const manifest = await readManifest(manifestPath); + if (!manifest) throw new Error('Could not read generated source fixture manifest'); + return { + rootPath, + manifest, + cleanup: () => fs.rm(rootPath, { recursive: true, force: true }) + }; + } catch (error) { + await fs.rm(rootPath, { recursive: true, force: true }); + throw error; + } +} diff --git a/tests/impact-2hop.test.ts b/tests/impact-2hop.test.ts index 010499e..fdd5e00 100644 --- a/tests/impact-2hop.test.ts +++ b/tests/impact-2hop.test.ts @@ -1,3 +1,4 @@ +import { resolveIndexArtifact } from '../src/core/index-generation.js'; import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'; import { promises as fs } from 'fs'; import os from 'os'; @@ -59,75 +60,79 @@ describe('Impact candidates (2-hop)', () => { } }); - it('includes hop 1 and hop 2 candidates in preflight impact.details', async () => { - if (!tempRoot) throw new Error('tempRoot not initialized'); + it( + 'includes hop 1 and hop 2 candidates in preflight impact.details', + async () => { + if (!tempRoot) throw new Error('tempRoot not initialized'); - const rootPath = tempRoot; - const paths = { - baseDir: path.join(rootPath, CODEBASE_CONTEXT_DIRNAME), - memory: path.join(rootPath, CODEBASE_CONTEXT_DIRNAME, MEMORY_FILENAME), - intelligence: path.join(rootPath, CODEBASE_CONTEXT_DIRNAME, INTELLIGENCE_FILENAME), - keywordIndex: path.join(rootPath, CODEBASE_CONTEXT_DIRNAME, KEYWORD_INDEX_FILENAME), - vectorDb: path.join(rootPath, CODEBASE_CONTEXT_DIRNAME, VECTOR_DB_DIRNAME) - }; + const rootPath = tempRoot; + const paths = { + baseDir: path.join(rootPath, CODEBASE_CONTEXT_DIRNAME), + memory: path.join(rootPath, CODEBASE_CONTEXT_DIRNAME, MEMORY_FILENAME), + intelligence: await resolveIndexArtifact(rootPath, INTELLIGENCE_FILENAME), + keywordIndex: await resolveIndexArtifact(rootPath, KEYWORD_INDEX_FILENAME), + vectorDb: await resolveIndexArtifact(rootPath, VECTOR_DB_DIRNAME) + }; - const ctx: ToolContext = { - indexState: { status: 'ready' }, - paths, - rootPath, - performIndexing: () => {} - }; + const ctx: ToolContext = { + indexState: { status: 'ready' }, + paths, + rootPath, + performIndexing: () => {} + }; - const relationshipsPath = path.join(rootPath, CODEBASE_CONTEXT_DIRNAME, RELATIONSHIPS_FILENAME); - const relationshipsRaw = await fs.readFile(relationshipsPath, 'utf-8'); - const relationships = JSON.parse(relationshipsRaw) as { - graph?: { imports?: Record }; - }; - const imports = relationships.graph?.imports ?? {}; - const hasInternalEdge = - (imports['src/b.ts'] ?? []).some((d) => d.endsWith('src/c.ts') || d === 'src/c.ts') && - (imports['src/a.ts'] ?? []).some((d) => d.endsWith('src/b.ts') || d === 'src/b.ts'); - if (!hasInternalEdge) { - throw new Error( - `Expected relationships graph to include src/a.ts -> src/b.ts and src/b.ts -> src/c.ts, got imports keys=${JSON.stringify( - Object.keys(imports) - )}` - ); - } + const relationshipsPath = await resolveIndexArtifact(rootPath, RELATIONSHIPS_FILENAME); + const relationshipsRaw = await fs.readFile(relationshipsPath, 'utf-8'); + const relationships = JSON.parse(relationshipsRaw) as { + graph?: { imports?: Record }; + }; + const imports = relationships.graph?.imports ?? {}; + const hasInternalEdge = + (imports['src/b.ts'] ?? []).some((d) => d.endsWith('src/c.ts') || d === 'src/c.ts') && + (imports['src/a.ts'] ?? []).some((d) => d.endsWith('src/b.ts') || d === 'src/b.ts'); + if (!hasInternalEdge) { + throw new Error( + `Expected relationships graph to include src/a.ts -> src/b.ts and src/b.ts -> src/c.ts, got imports keys=${JSON.stringify( + Object.keys(imports) + )}` + ); + } - const resp = await dispatchTool( - 'search_codebase', - { query: token, intent: 'edit', includeSnippets: false, limit: 1 }, - ctx - ); - - const text = resp.content?.[0]?.text ?? ''; - const parsed = JSON.parse(text) as { - status?: string; - results?: Array<{ file?: string }>; - preflight?: { impact?: { details?: Array<{ file: string; hop: 1 | 2 }> } }; - }; - const results = parsed.results ?? []; - if (!Array.isArray(results) || results.length === 0) { - throw new Error( - `Expected at least one search result for token, got status=${String(parsed.status)} results=${JSON.stringify( - results - )}` + const resp = await dispatchTool( + 'search_codebase', + { query: token, intent: 'edit', includeSnippets: false, limit: 1 }, + ctx ); - } - const details = parsed.preflight?.impact?.details ?? []; - const hasHop1 = details.some((d) => d.file.endsWith('src/b.ts') && d.hop === 1); - if (!hasHop1) { - throw new Error( - `Expected hop 1 candidate src/b.ts, got impact.details=${JSON.stringify(details)}` - ); - } - const hasHop2 = details.some((d) => d.file.endsWith('src/a.ts') && d.hop === 2); - if (!hasHop2) { - throw new Error( - `Expected hop 2 candidate src/a.ts, got impact.details=${JSON.stringify(details)}` - ); - } - }, SLOW_WINDOWS_TEST_TIMEOUT_MS); + const text = resp.content?.[0]?.text ?? ''; + const parsed = JSON.parse(text) as { + status?: string; + results?: Array<{ file?: string }>; + preflight?: { impact?: { details?: Array<{ file: string; hop: 1 | 2 }> } }; + }; + const results = parsed.results ?? []; + if (!Array.isArray(results) || results.length === 0) { + throw new Error( + `Expected at least one search result for token, got status=${String(parsed.status)} results=${JSON.stringify( + results + )}` + ); + } + const details = parsed.preflight?.impact?.details ?? []; + + const hasHop1 = details.some((d) => d.file.endsWith('src/b.ts') && d.hop === 1); + if (!hasHop1) { + throw new Error( + `Expected hop 1 candidate src/b.ts, got impact.details=${JSON.stringify(details)}` + ); + } + const hasHop2 = details.some((d) => d.file.endsWith('src/a.ts') && d.hop === 2); + if (!hasHop2) { + throw new Error( + `Expected hop 2 candidate src/a.ts, got impact.details=${JSON.stringify(details)}` + ); + } + }, + SLOW_WINDOWS_TEST_TIMEOUT_MS + ); }); diff --git a/tests/incremental-indexing.test.ts b/tests/incremental-indexing.test.ts index 09fe132..aee9322 100644 --- a/tests/incremental-indexing.test.ts +++ b/tests/incremental-indexing.test.ts @@ -1,3 +1,4 @@ +import { resolveIndexArtifact } from '../src/core/index-generation.js'; import { describe, it, expect, beforeEach, afterEach } from 'vitest'; import { promises as fs } from 'fs'; import path from 'path'; @@ -32,7 +33,7 @@ describe('Incremental Indexing', () => { await indexer.index(); - const manifestPath = path.join(tempDir, CODEBASE_CONTEXT_DIRNAME, MANIFEST_FILENAME); + const manifestPath = await resolveIndexArtifact(tempDir, MANIFEST_FILENAME); const manifest = await readManifest(manifestPath); expect(manifest).not.toBeNull(); expect(manifest!.version).toBe(1); @@ -136,7 +137,7 @@ describe('Incremental Indexing', () => { const contextDir = path.join(tempDir, CODEBASE_CONTEXT_DIRNAME); await fs.writeFile( - path.join(contextDir, INDEXING_STATS_FILENAME), + await resolveIndexArtifact(path.dirname(contextDir), INDEXING_STATS_FILENAME), JSON.stringify( { indexedFiles: 77, @@ -148,7 +149,10 @@ describe('Incremental Indexing', () => { 2 ) ); - await fs.writeFile(path.join(contextDir, KEYWORD_INDEX_FILENAME), JSON.stringify([])); + await fs.writeFile( + await resolveIndexArtifact(path.dirname(contextDir), KEYWORD_INDEX_FILENAME), + JSON.stringify([]) + ); const incIndexer = new CodebaseIndexer({ rootPath: tempDir, @@ -259,7 +263,7 @@ describe('Incremental Indexing', () => { expect(stats.incremental!.deleted).toBe(0); // Manifest should be created - const manifestPath = path.join(tempDir, CODEBASE_CONTEXT_DIRNAME, MANIFEST_FILENAME); + const manifestPath = await resolveIndexArtifact(tempDir, MANIFEST_FILENAME); const manifest = await readManifest(manifestPath); expect(manifest).not.toBeNull(); }); @@ -274,7 +278,7 @@ describe('Incremental Indexing', () => { }); await indexer1.index(); - const manifestPath = path.join(tempDir, CODEBASE_CONTEXT_DIRNAME, MANIFEST_FILENAME); + const manifestPath = await resolveIndexArtifact(tempDir, MANIFEST_FILENAME); const manifest1 = await readManifest(manifestPath); // Add a new file @@ -288,7 +292,7 @@ describe('Incremental Indexing', () => { }); await indexer2.index(); - const manifest2 = await readManifest(manifestPath); + const manifest2 = await readManifest(await resolveIndexArtifact(tempDir, MANIFEST_FILENAME)); expect(manifest2).not.toBeNull(); // New manifest should have more files @@ -308,7 +312,7 @@ describe('Incremental Indexing', () => { }); await indexer1.index(); - const indexPath = path.join(tempDir, CODEBASE_CONTEXT_DIRNAME, KEYWORD_INDEX_FILENAME); + const indexPath = await resolveIndexArtifact(tempDir, KEYWORD_INDEX_FILENAME); const fullIndexRaw = JSON.parse(await fs.readFile(indexPath, 'utf-8')) as any; const fullIndex = Array.isArray(fullIndexRaw) ? fullIndexRaw @@ -328,7 +332,9 @@ describe('Incremental Indexing', () => { await indexer2.index(); // Keyword index should still contain chunks from ALL files (not just changed ones) - const incrementalRaw = JSON.parse(await fs.readFile(indexPath, 'utf-8')) as any; + const incrementalRaw = JSON.parse( + await fs.readFile(await resolveIndexArtifact(tempDir, KEYWORD_INDEX_FILENAME), 'utf-8') + ) as any; const incrementalIndex = Array.isArray(incrementalRaw) ? incrementalRaw : Array.isArray(incrementalRaw?.chunks) diff --git a/tests/index-generation-vectors.test.ts b/tests/index-generation-vectors.test.ts new file mode 100644 index 0000000..76317ec --- /dev/null +++ b/tests/index-generation-vectors.test.ts @@ -0,0 +1,339 @@ +import { promises as fs } from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { connect } from '@lancedb/lancedb'; +import { afterEach, beforeEach, expect, it, vi } from 'vitest'; +import { resolveIndexArtifact, withIndexSnapshot } from '../src/core/index-generation.js'; + +const { embedBatch, embed } = vi.hoisted(() => ({ + embed: vi.fn(async (text: string) => + Array.from({ length: 384 }, (_, index) => (text.charCodeAt(index % text.length) + index) / 1000) + ), + embedBatch: vi.fn(async (texts: string[]) => + texts.map((text) => + Array.from( + { length: 384 }, + (_, index) => (text.charCodeAt(index % text.length) + index) / 1000 + ) + ) + ) +})); +vi.mock('../src/embeddings/index.js', async (importOriginal) => ({ + ...(await importOriginal()), + getEmbeddingProvider: vi.fn(async () => ({ embedBatch, embed })) +})); +import { CodebaseIndexer } from '../src/core/indexer.js'; +import { CodebaseSearcher } from '../src/core/search.js'; +import * as storage from '../src/storage/index.js'; +import * as embeddings from '../src/embeddings/index.js'; +import { createHash } from 'node:crypto'; +import { handleCliCommand } from '../src/cli.js'; +import { handleMapCli } from '../src/cli-map.js'; +import { readIndexMeta } from '../src/core/index-meta.js'; +import { LanceDBStorageProvider } from '../src/storage/lancedb.js'; +import { IndexCorruptedError } from '../src/errors/index.js'; + +let root = ''; +beforeEach(() => { + vi.mocked(embeddings.getEmbeddingProvider) + .mockReset() + .mockResolvedValue({ + name: 'fixture', + modelName: 'vector-generation-fixture', + dimensions: 384, + initialize: async () => undefined, + isReady: () => true, + embed, + embedBatch + }); +}); +afterEach(async () => { + vi.restoreAllMocks(); + if (root) await fs.rm(root, { recursive: true, force: true }); + root = ''; + embedBatch.mockClear(); + embed.mockClear(); +}); + +async function nativeRows(directory: string) { + const db = await connect(directory); + const table = await db.openTable('code_chunks'); + try { + const result = await table.query().toArray(); + return result.map((row) => ({ + id: String(row.id), + file: String(row.relativePath), + content: String(row.content), + vector: Array.from(row.vector) + })); + } finally { + table.close(); + db.close(); + } +} + +async function rows() { + return withIndexSnapshot(root, async () => nativeRows(await resolveIndexArtifact(root, 'index'))); +} + +async function runMap(skipEmbedding?: boolean, command: 'map' | 'reindex' = 'map') { + const previousRoot = process.env.CODEBASE_ROOT; + const previousConfig = process.env.CODEBASE_CONTEXT_CONFIG_PATH; + const configPath = path.join(root, 'server-config.json'); + await fs.writeFile( + configPath, + JSON.stringify({ + projects: [{ root, ...(skipEmbedding === undefined ? {} : { skipEmbedding }) }] + }) + ); + process.env.CODEBASE_ROOT = root; + process.env.CODEBASE_CONTEXT_CONFIG_PATH = configPath; + try { + if (command === 'map') await handleMapCli(['--json']); + else await handleCliCommand(['reindex']); + } finally { + if (previousRoot === undefined) delete process.env.CODEBASE_ROOT; + else process.env.CODEBASE_ROOT = previousRoot; + if (previousConfig === undefined) delete process.env.CODEBASE_CONTEXT_CONFIG_PATH; + else process.env.CODEBASE_CONTEXT_CONFIG_PATH = previousConfig; + } +} + +async function generationBytes( + directory: string +): Promise> { + const result: Array<{ path: string; bytes: number; sha256: string }> = []; + const visit = async (current: string): Promise => { + for (const entry of (await fs.readdir(current, { withFileTypes: true })).sort((a, b) => + a.name.localeCompare(b.name) + )) { + const filename = path.join(current, entry.name); + if (entry.isDirectory()) await visit(filename); + else { + const bytes = await fs.readFile(filename); + result.push({ + path: path.relative(directory, filename), + bytes: bytes.length, + sha256: createHash('sha256').update(bytes).digest('hex') + }); + } + } + }; + await visit(directory); + return result; +} + +it.each(['map', 'reindex'] as const)( + 'preserves a completed semantic generation and requires explicit recovery when only its root pointer is missing: %s', + async (command) => { + root = await fs.mkdtemp(path.join(os.tmpdir(), 'cbc-orphan-semantic-')); + await fs.writeFile(path.join(root, 'route.ts'), 'export const route = "original";'); + await new CodebaseIndexer({ + rootPath: root, + config: { + skipEmbedding: false, + embedding: { provider: 'custom', model: 'vector-generation-fixture' } + } + }).index(); + const pointer = path.join(root, '.codebase-context', 'index-meta.json'); + const pointerBytes = await fs.readFile(pointer); + const generation = path.dirname(await resolveIndexArtifact(root, 'index.json')); + const vectors = await rows(); + const files = await generationBytes(generation); + expect(vectors.some((row) => row.file === 'route.ts')).toBe(true); + expect(files.some((file) => file.path === 'index-meta.json')).toBe(true); + await fs.rm(pointer); + let failure: unknown; + try { + await runMap(undefined, command); + } catch (error) { + failure = error; + } + const afterPointer = await fs.readFile(pointer).catch((error: unknown) => { + if (error instanceof Error && 'code' in error && error.code === 'ENOENT') return null; + throw error; + }); + expect({ + failed: failure instanceof Error, + pointer: afterPointer, + files: await generationBytes(generation), + vectors: await nativeRows(path.join(generation, 'index')) + }).toEqual({ failed: true, pointer: null, files, vectors }); + // The original complete pointer is retained in test evidence, separate from the deleted authority file. + expect(JSON.parse(pointerBytes.toString()).artifacts.vectorDb.embeddingEnabled).toBe(true); + }, + 30_000 +); + +it('fails explicitly and preserves real semantic metadata and vectors when map finds a missing keyword artifact', async () => { + root = await fs.mkdtemp(path.join(os.tmpdir(), 'cbc-map-corrupt-semantic-')); + await fs.writeFile(path.join(root, 'route.ts'), 'export const route = "original";'); + await new CodebaseIndexer({ + rootPath: root, + config: { + skipEmbedding: false, + embedding: { provider: 'custom', model: 'vector-generation-fixture' } + } + }).index(); + const metaPath = path.join(root, '.codebase-context', 'index-meta.json'); + const beforeMeta = await fs.readFile(metaPath, 'utf8'); + const vectorPath = await resolveIndexArtifact(root, 'index'); + const beforeRows = await nativeRows(vectorPath); + expect(beforeRows.length).toBeGreaterThan(0); + await fs.rm(await resolveIndexArtifact(root, 'index.json')); + let failure: unknown; + try { + await runMap(); + } catch (error) { + failure = error; + } + expect({ + failed: failure instanceof Error, + metadata: await fs.readFile(metaPath, 'utf8'), + vectors: await nativeRows(vectorPath) + }).toEqual({ failed: true, metadata: beforeMeta, vectors: beforeRows }); +}); + +it.each([undefined, true, false])( + 'honors fresh-map project embedding choice %s', + async (skipEmbedding) => { + root = await fs.mkdtemp(path.join(os.tmpdir(), 'cbc-map-mode-')); + await fs.writeFile(path.join(root, 'route.ts'), 'export const route = "original";'); + vi.mocked(embeddings.getEmbeddingProvider).mockClear(); + await runMap(skipEmbedding); + const semantic = skipEmbedding === false; + expect((await readIndexMeta(root)).artifacts.vectorDb.embeddingEnabled).toBe(semantic); + expect(vi.mocked(embeddings.getEmbeddingProvider).mock.calls.length > 0).toBe(semantic); + if (semantic) expect((await rows()).some((row) => row.file === 'route.ts')).toBe(true); + } +); + +it('preserves the committed generation and reports an inaccessible keyword artifact', async () => { + root = await fs.mkdtemp(path.join(os.tmpdir(), 'cbc-map-inaccessible-')); + await fs.writeFile(path.join(root, 'route.ts'), 'export const route = "original";'); + await new CodebaseIndexer({ rootPath: root, config: { skipEmbedding: true } }).index(); + const keyword = await resolveIndexArtifact(root, 'index.json'); + const metaPath = path.join(root, '.codebase-context', 'index-meta.json'); + const before = await fs.readFile(metaPath, 'utf8'); + const original = fs.access; + vi.spyOn(fs, 'access').mockImplementation(async (...args) => { + if (String(args[0]) === keyword) + throw Object.assign(new Error('keyword access denied'), { code: 'EACCES' }); + return original(...args); + }); + await expect(runMap()).rejects.toMatchObject({ code: 'EACCES' }); + expect(await fs.readFile(metaPath, 'utf8')).toBe(before); +}); + +it('copies unchanged real LanceDB vectors while applying incremental add/change/delete', async () => { + root = await fs.mkdtemp(path.join(os.tmpdir(), 'cbc-vector-generation-')); + for (const name of ['unchanged', 'changed', 'deleted']) { + await fs.writeFile(path.join(root, `${name}.ts`), `export const ${name} = "original";`); + } + const options = { + rootPath: root, + config: { + skipEmbedding: false, + embedding: { provider: 'custom' as const, model: 'vector-generation-fixture' } + } + }; + await new CodebaseIndexer(options).index(); + const before = await rows(); + expect(before.map((row) => row.file).sort()).toEqual([ + 'changed.ts', + 'deleted.ts', + 'unchanged.ts' + ]); + embedBatch.mockClear(); + await fs.writeFile(path.join(root, 'changed.ts'), 'export const changed = "replacement";'); + await fs.rm(path.join(root, 'deleted.ts')); + await fs.writeFile(path.join(root, 'added.ts'), 'export const added = "new";'); + const stats = await new CodebaseIndexer({ ...options, incrementalOnly: true }).index(); + expect(stats.incremental).toEqual({ added: 1, changed: 1, deleted: 1, unchanged: 1 }); + const after = await rows(); + expect(after.map((row) => row.file).sort()).toEqual(['added.ts', 'changed.ts', 'unchanged.ts']); + expect(after.find((row) => row.file === 'unchanged.ts')).toEqual( + before.find((row) => row.file === 'unchanged.ts') + ); + expect(after.find((row) => row.file === 'changed.ts')?.content).toContain('replacement'); + expect(after.find((row) => row.file === 'changed.ts')?.vector).not.toEqual( + before.find((row) => row.file === 'changed.ts')?.vector + ); + const embedded = embedBatch.mock.calls.flatMap(([texts]) => texts); + expect(embedded).toHaveLength(2); + expect(embedded.join('\n')).not.toContain('unchanged'); +}, 30_000); + +it('retains keyword fallback when the same semantic searcher cannot reopen its provider', async () => { + root = await fs.mkdtemp(path.join(os.tmpdir(), 'cbc-semantic-fallback-')); + await fs.writeFile(path.join(root, 'route.ts'), 'export function route() { return "route"; }'); + await new CodebaseIndexer({ + rootPath: root, + config: { + skipEmbedding: false, + embedding: { provider: 'custom', model: 'vector-generation-fixture' } + } + }).index(); + vi.spyOn(storage, 'getStorageProvider').mockRejectedValue( + new Error('storage temporarily unavailable') + ); + const searcher = new CodebaseSearcher(root); + expect((await searcher.search('route')).length).toBeGreaterThan(0); + expect((await searcher.search('route')).length).toBeGreaterThan(0); +}); + +it('does not reuse an old embedding provider after mode changes and repeated new-model initialization failures', async () => { + root = await fs.mkdtemp(path.join(os.tmpdir(), 'cbc-semantic-mode-pair-')); + await fs.writeFile(path.join(root, 'route.ts'), 'export function route() { return "route"; }'); + const semanticConfig = { + skipEmbedding: false, + embedding: { provider: 'custom' as const, model: 'vector-generation-fixture' } + }; + await new CodebaseIndexer({ rootPath: root, config: semanticConfig }).index(); + const searcher = new CodebaseSearcher(root); + const query = () => searcher.search('route', 5, undefined, { enableReranker: false }); + expect((await query()).length).toBeGreaterThan(0); + await new CodebaseIndexer({ rootPath: root, config: { skipEmbedding: true } }).index(); + expect((await query()).length).toBeGreaterThan(0); + await new CodebaseIndexer({ + rootPath: root, + config: { + ...semanticConfig, + embedding: { ...semanticConfig.embedding, model: 'vector-generation-fixture-v2' } + } + }).index(); + const initialize = vi + .spyOn(embeddings, 'getEmbeddingProvider') + .mockRejectedValue(new Error('new model temporarily unavailable')); + initialize.mockClear(); + embed.mockClear(); + expect((await query()).length).toBeGreaterThan(0); + expect((await query()).length).toBeGreaterThan(0); + expect(initialize).toHaveBeenCalledTimes(2); + expect(embed).not.toHaveBeenCalled(); +}, 30_000); + +it('preserves successful search and integrity failures when native read-handle cleanup fails', async () => { + root = await fs.mkdtemp(path.join(os.tmpdir(), 'cbc-semantic-close-')); + await fs.writeFile(path.join(root, 'route.ts'), 'export function route() { return "route"; }'); + await new CodebaseIndexer({ + rootPath: root, + config: { + skipEmbedding: false, + embedding: { provider: 'custom', model: 'vector-generation-fixture' } + } + }).index(); + const close = LanceDBStorageProvider.prototype.close; + vi.spyOn(LanceDBStorageProvider.prototype, 'close').mockImplementation(function ( + this: LanceDBStorageProvider + ) { + close.call(this); + throw new Error('native cleanup unavailable'); + }); + const searcher = new CodebaseSearcher(root); + const query = () => searcher.search('route', 5, undefined, { enableReranker: false }); + await expect(query()).resolves.toHaveLength(1); + const original = new IndexCorruptedError('original vector integrity failure'); + vi.spyOn(LanceDBStorageProvider.prototype, 'search').mockRejectedValue(original); + await expect(query()).rejects.toBe(original); +}); diff --git a/tests/index-generation.test.ts b/tests/index-generation.test.ts new file mode 100644 index 0000000..db81d9d --- /dev/null +++ b/tests/index-generation.test.ts @@ -0,0 +1,546 @@ +import { promises as fs } from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { CodebaseIndexer } from '../src/core/indexer.js'; +import { CodebaseSearcher } from '../src/core/search.js'; +import { readIndexMeta, validateIndexArtifacts } from '../src/core/index-meta.js'; +import { + IndexCapacityError, + resolveIndexArtifact, + withIndexSnapshot, + MAX_INDEX_GENERATIONS +} from '../src/core/index-generation.js'; +import { withAvailableIndexSnapshot } from '../src/core/index-generation.js'; +import { IndexMetaSchema } from '../src/core/index-meta.js'; +import { handle as healthTool } from '../src/tools/get-codebase-health.js'; +import { makePaths } from '../src/project-state.js'; + +let root = ''; +afterEach(async () => { + vi.restoreAllMocks(); + if (root) await fs.rm(root, { recursive: true, force: true }); + root = ''; +}); +async function fixture() { + root = await fs.mkdtemp(path.join(os.tmpdir(), 'cbc-generation-')); + await fs.writeFile(path.join(root, 'route.ts'), 'export function route() { return "alpha"; }'); + await new CodebaseIndexer({ rootPath: root, config: { skipEmbedding: true } }).index(); + return readIndexMeta(root); +} +async function update(value: string, incrementalOnly = false) { + await fs.writeFile(path.join(root, 'route.ts'), `export function route() { return "${value}"; }`); + return new CodebaseIndexer({ + rootPath: root, + config: { skipEmbedding: true }, + incrementalOnly + }).index(); +} +async function validate() { + await validateIndexArtifacts(root, await readIndexMeta(root)); +} + +async function legacyFixture() { + const meta = await fixture(); + const context = path.join(root, '.codebase-context'); + const artifacts: Record = {}; + for (const [name, artifact] of Object.entries(meta.artifacts)) { + if ( + typeof artifact !== 'object' || + artifact === null || + !('path' in artifact) || + typeof artifact.path !== 'string' + ) + throw new Error('Invalid fixture artifact'); + const filename = path.basename(artifact.path); + await fs.cp(path.join(context, artifact.path), path.join(context, filename), { + recursive: true + }); + artifacts[name] = { ...artifact, path: filename }; + } + const legacy = IndexMetaSchema.parse({ ...meta, artifacts }); + await fs.writeFile(path.join(context, 'index-meta.json'), JSON.stringify(legacy)); + await fs.rm(path.join(context, '.generations', meta.buildId), { recursive: true }); + return legacy; +} + +describe('index generation fault boundaries', () => { + it('rejects a linked vector marker even when its external build identity matches', async () => { + const meta = await fixture(); + const vector = await resolveIndexArtifact(root, 'index'); + const marker = path.join(vector, 'index-build.json'); + const outside = path.join(root, 'outside-marker.json'); + await fs.copyFile(marker, outside); + await fs.rm(marker); + await fs.symlink(outside, marker, 'file'); + await expect(validateIndexArtifacts(root, meta)).rejects.toThrow(/linked.*artifact/i); + }); + + it('rejects linked artifact files and a linked context directory through a real tool', async () => { + await fixture(); + const keyword = await resolveIndexArtifact(root, 'index.json'); + const outside = path.join(root, 'outside-keyword.json'); + await fs.copyFile(keyword, outside); + await fs.rm(keyword); + await fs.symlink(outside, keyword, 'file'); + await expect(resolveIndexArtifact(root, 'index.json')).rejects.toThrow(/linked.*artifact/i); + await fs.rm(keyword); + await fs.copyFile(outside, keyword); + const context = path.join(root, '.codebase-context'); + const moved = path.join(root, 'outside-context'); + await fs.rename(context, moved); + await fs.symlink(moved, context, process.platform === 'win32' ? 'junction' : 'dir'); + await expect( + healthTool( + {}, + { + rootPath: root, + paths: makePaths(root), + indexState: { status: 'ready' }, + performIndexing: () => undefined + } + ) + ).rejects.toThrow(/linked.*artifact/i); + }); + + it.each([false, true])( + 'preserves the committed build across every staged write failure (incremental=%s)', + async (incrementalOnly) => { + const baseline = await fixture(); + for (const filename of [ + 'index-build.json', + 'index.json', + 'intelligence.json', + 'relationships.json', + 'health.json', + 'manifest.json', + 'indexing-stats.json', + 'index-meta.json' + ]) { + const write = fs.writeFile.bind(fs); + let injected = false; + const spy = vi.spyOn(fs, 'writeFile').mockImplementation(async (...args) => { + await write(...args); + if ( + !injected && + String(args[0]).includes('.generations') && + path.basename(String(args[0])) === filename + ) { + injected = true; + throw new Error(`fault after ${filename}`); + } + }); + try { + await expect(update(`new-${filename}`, incrementalOnly)).rejects.toThrow( + `fault after ${filename}` + ); + expect(injected).toBe(true); + } finally { + spy.mockRestore(); + } + expect((await readIndexMeta(root)).buildId).toBe(baseline.buildId); + await validate(); + } + }, + 60_000 + ); + + it('keeps the previous build when pointer replacement fails', async () => { + const baseline = await fixture(); + const rename = fs.rename.bind(fs); + const pointer = path.join(root, '.codebase-context', 'index-meta.json'); + const spy = vi.spyOn(fs, 'rename').mockImplementation(async (source, target) => { + if (String(target) === pointer) throw new Error('pointer rename failed'); + await rename(source, target); + }); + await expect(update('beta')).rejects.toThrow('pointer rename failed'); + spy.mockRestore(); + expect((await readIndexMeta(root)).buildId).toBe(baseline.buildId); + await validate(); + }); + + it('keeps the previous build when writing temporary publication metadata fails', async () => { + const baseline = await fixture(); + const context = path.join(root, '.codebase-context'); + const write = fs.writeFile.bind(fs); + let injected = false; + const spy = vi.spyOn(fs, 'writeFile').mockImplementation(async (...args) => { + await write(...args); + const filename = String(args[0]); + if ( + path.dirname(filename) === context && + path.basename(filename).startsWith('index-meta.json.') && + filename.endsWith('.tmp') + ) { + injected = true; + throw new Error('temporary publication write failed'); + } + }); + await expect(update('beta')).rejects.toThrow('temporary publication write failed'); + spy.mockRestore(); + expect(injected).toBe(true); + expect((await readIndexMeta(root)).buildId).toBe(baseline.buildId); + await validate(); + }); + + it('does not roll back or delete a committed generation when collection fails', async () => { + const baseline = await fixture(); + const readdir = fs.readdir.bind(fs); + const pointer = path.join(root, '.codebase-context', 'index-meta.json'); + let committed = false; + const rename = fs.rename.bind(fs); + vi.spyOn(fs, 'rename').mockImplementation(async (source, target) => { + await rename(source, target); + if (String(target) === pointer) committed = true; + }); + const spy = vi.spyOn(fs, 'readdir').mockImplementation(async (...args) => { + if (committed && String(args[0]) === path.join(root, '.codebase-context', '.generations')) + throw new Error('cleanup failed'); + return readdir(...args); + }); + await expect(update('beta')).resolves.toHaveProperty('errors', []); + spy.mockRestore(); + expect(committed).toBe(true); + expect((await readIndexMeta(root)).buildId).not.toBe(baseline.buildId); + await validate(); + }); + + it('preserves publication when lease-file cleanup fails after commit', async () => { + const baseline = await fixture(); + const pointer = path.join(root, '.codebase-context', 'index-meta.json'); + let committed = false; + const rename = fs.rename.bind(fs); + vi.spyOn(fs, 'rename').mockImplementation(async (source, target) => { + await rename(source, target); + if (String(target) === pointer) committed = true; + }); + const remove = fs.rm.bind(fs); + const spy = vi.spyOn(fs, 'rm').mockImplementation(async (...args) => { + if ( + committed && + String(args[0]).includes('.publication') && + String(args[0]).endsWith('.json') + ) + throw new Error('lease cleanup failed'); + return remove(...args); + }); + await expect(update('beta')).resolves.toHaveProperty('errors', []); + spy.mockRestore(); + expect((await readIndexMeta(root)).buildId).not.toBe(baseline.buildId); + await validate(); + await expect(update('gamma')).resolves.toHaveProperty('errors', []); + }); +}); + +describe('operation-scoped generation readers', () => { + it('keeps an operation that began without an index on that same unavailable snapshot', async () => { + await fixture(); + await fs.rm(path.join(root, '.codebase-context', 'index-meta.json')); + await withAvailableIndexSnapshot(root, async () => { + const before = await resolveIndexArtifact(root, 'index.json'); + await update('beta'); + expect(await resolveIndexArtifact(root, 'index.json')).toBe(before); + }); + expect((await new CodebaseSearcher(root).search('beta')).length).toBeGreaterThan(0); + }); + + it('protects metadata-free legacy readers while an explicit rebuild migrates their files', async () => { + await legacyFixture(); + await fs.rm(path.join(root, '.codebase-context', 'index-meta.json')); + const entered = Promise.withResolvers(); + const release = Promise.withResolvers(); + const reader = withAvailableIndexSnapshot(root, async () => { + const filename = await resolveIndexArtifact(root, 'index.json'); + entered.resolve(); + await release.promise; + expect(await fs.readFile(filename, 'utf8')).toContain('alpha'); + }); + await entered.promise; + try { + await update('beta'); + } finally { + release.resolve(); + await reader; + } + expect((await new CodebaseSearcher(root).search('beta')).length).toBeGreaterThan(0); + }); + + it('retries a completed pin release after both its release write and unlink temporarily fail', async () => { + await fixture(); + const rename = fs.rename.bind(fs); + const remove = fs.rm.bind(fs); + let releasing = false; + const renameSpy = vi.spyOn(fs, 'rename').mockImplementation(async (source, target) => { + if (releasing && String(target).includes('.pins') && String(target).endsWith('.json')) + throw new Error('pin release write unavailable'); + return rename(source, target); + }); + const removeSpy = vi.spyOn(fs, 'rm').mockImplementation(async (...args) => { + if (releasing && String(args[0]).includes('.pins') && String(args[0]).endsWith('.json')) + throw new Error('pin unlink unavailable'); + return remove(...args); + }); + await withIndexSnapshot(root, async () => { + releasing = true; + }); + releasing = false; + renameSpy.mockRestore(); + removeSpy.mockRestore(); + await update('beta'); + expect(await fs.readdir(path.join(root, '.codebase-context', '.pins'))).toEqual([]); + }); + + it('migrates legacy fixed files while preserving a pinned legacy reader and memory/config bytes', async () => { + const legacy = await legacyFixture(); + const context = path.join(root, '.codebase-context'); + const memory = '[{"memory":"retain this"}]'; + const config = '{"project":"retain this"}'; + await fs.writeFile(path.join(context, 'memory.json'), memory); + await fs.writeFile(path.join(context, 'config.json'), config); + const entered = Promise.withResolvers(); + const release = Promise.withResolvers(); + const reader = withIndexSnapshot(root, async () => { + const filename = await resolveIndexArtifact(root, 'index.json'); + entered.resolve(); + await release.promise; + expect(JSON.parse(await fs.readFile(filename, 'utf8')).header.buildId).toBe(legacy.buildId); + }); + await entered.promise; + const releaseNew = Promise.withResolvers(); + let newReader: Promise | undefined; + try { + await update('beta', true); + const enteredNew = Promise.withResolvers(); + newReader = withIndexSnapshot(root, async () => { + const meta = await readIndexMeta(root); + const filename = await resolveIndexArtifact(root, 'index.json'); + enteredNew.resolve(); + await releaseNew.promise; + expect(JSON.parse(await fs.readFile(filename, 'utf8')).header.buildId).toBe(meta.buildId); + }); + await enteredNew.promise; + await update('gamma', true); + await expect(update('delta', true)).rejects.toBeInstanceOf(IndexCapacityError); + expect(await fs.readdir(path.join(context, '.generations'))).toHaveLength(2); + await fs.access(path.join(context, 'index.json')); + } finally { + release.resolve(); + releaseNew.resolve(); + await Promise.all([reader, newReader]); + } + await update('epsilon', true); + await expect(fs.access(path.join(context, 'index.json'))).rejects.toMatchObject({ + code: 'ENOENT' + }); + expect(await fs.readFile(path.join(context, 'memory.json'), 'utf8')).toBe(memory); + expect(await fs.readFile(path.join(context, 'config.json'), 'utf8')).toBe(config); + await validate(); + }); + + it('indexes and reads one canonical project through a real junction alias', async () => { + await fixture(); + const alias = `${root}-alias`; + await fs.symlink(root, alias, process.platform === 'win32' ? 'junction' : 'dir'); + try { + await expect( + new CodebaseIndexer({ rootPath: alias, config: { skipEmbedding: true } }).index() + ).resolves.toHaveProperty('errors', []); + await fs.writeFile(path.join(root, 'route.ts'), 'export function route() { return "beta"; }'); + await expect( + new CodebaseIndexer({ + rootPath: alias, + config: { skipEmbedding: true }, + incrementalOnly: true + }).index() + ).resolves.toHaveProperty('errors', []); + expect((await new CodebaseSearcher(alias).search('beta')).length).toBeGreaterThan(0); + } finally { + await fs.rm(alias, { recursive: true, force: true }); + } + }); + + it('rejects every mixed legacy/generation artifact path even when headers match', async () => { + const meta = await fixture(); + const context = path.join(root, '.codebase-context'); + for (const artifact of Object.values(meta.artifacts)) { + if ( + typeof artifact !== 'object' || + artifact === null || + !('path' in artifact) || + typeof artifact.path !== 'string' + ) + continue; + await fs.cp( + path.join(context, artifact.path), + path.join(context, path.basename(artifact.path)), + { recursive: true } + ); + } + const legacy = IndexMetaSchema.parse({ + ...meta, + artifacts: Object.fromEntries( + Object.entries(meta.artifacts).map(([name, artifact]) => { + if ( + typeof artifact !== 'object' || + artifact === null || + !('path' in artifact) || + typeof artifact.path !== 'string' + ) + throw new Error('Invalid fixture artifact'); + return [name, { ...artifact, path: path.basename(artifact.path) }]; + }) + ) + }); + for (const [name, filename] of [ + ['vectorDb', 'index'], + ['intelligence', 'intelligence.json'], + ['manifest', 'manifest.json'], + ['indexingStats', 'indexing-stats.json'], + ['relationships', 'relationships.json'], + ['health', 'health.json'] + ]) { + const original = legacy.artifacts[name]; + if (typeof original !== 'object' || original === null) + throw new Error('Missing fixture artifact'); + await fs.writeFile( + path.join(context, 'index-meta.json'), + JSON.stringify({ + ...legacy, + artifacts: { + ...legacy.artifacts, + [name]: { + ...original, + path: meta.artifacts.keywordIndex.path.replace('index.json', filename) + } + } + }) + ); + await expect( + withIndexSnapshot(root, async () => resolveIndexArtifact(root, filename)) + ).rejects.toThrow(/artifact.*path/i); + } + }); + + it('preserves successful and failed read outcomes when pin deletion fails, then reuses capacity', async () => { + await fixture(); + const remove = fs.rm.bind(fs); + const spy = vi.spyOn(fs, 'rm').mockImplementation(async (...args) => { + if (String(args[0]).includes('.pins') && String(args[0]).endsWith('.json')) + throw new Error('pin deletion failed'); + return remove(...args); + }); + const original = new Error('original read failure'); + await expect(withIndexSnapshot(root, async () => 'read complete')).resolves.toBe( + 'read complete' + ); + await expect( + withIndexSnapshot(root, async () => { + throw original; + }) + ).rejects.toBe(original); + for (const value of ['beta', 'gamma', 'delta', 'epsilon']) { + await update(value); + await expect(withIndexSnapshot(root, async () => value)).resolves.toBe(value); + } + spy.mockRestore(); + await expect(update('zeta')).resolves.toHaveProperty('errors', []); + await validate(); + }); + + it('selects the latest incremental baseline inside a writer nested under an old read snapshot', async () => { + await fixture(); + await withIndexSnapshot(root, async () => { + const old = await readIndexMeta(root); + await update('beta'); + await fs.writeFile(path.join(root, 'added.ts'), 'export const added = true;'); + const stats = await new CodebaseIndexer({ + rootPath: root, + config: { skipEmbedding: true }, + incrementalOnly: true + }).index(); + expect(stats.incremental).toEqual({ added: 1, changed: 0, deleted: 0, unchanged: 1 }); + expect((await readIndexMeta(root)).buildId).toBe(old.buildId); + }); + await validate(); + }); + + it('keeps two long readers safe across publications and defers when all three slots are occupied', async () => { + const a = await fixture(); + const enteredA = Promise.withResolvers(); + const releaseA = Promise.withResolvers(); + const readerA = withIndexSnapshot(root, async () => { + const filename = await resolveIndexArtifact(root, 'index.json'); + enteredA.resolve(); + await releaseA.promise; + const artifact = JSON.parse(await fs.readFile(filename, 'utf8')); + expect(artifact.header.buildId).toBe(a.buildId); + }); + await enteredA.promise; + let readerB: Promise | undefined; + const releaseB = Promise.withResolvers(); + try { + await update('beta'); + const b = await readIndexMeta(root); + const enteredB = Promise.withResolvers(); + readerB = withIndexSnapshot(root, async () => { + const filename = await resolveIndexArtifact(root, 'index.json'); + enteredB.resolve(); + await releaseB.promise; + expect(JSON.parse(await fs.readFile(filename, 'utf8')).header.buildId).toBe(b.buildId); + }); + await enteredB.promise; + await update('gamma'); + const c = await readIndexMeta(root); + await expect(update('delta')).rejects.toBeInstanceOf(IndexCapacityError); + expect((await readIndexMeta(root)).buildId).toBe(c.buildId); + expect(await fs.readdir(path.join(root, '.codebase-context', '.generations'))).toHaveLength( + MAX_INDEX_GENERATIONS + ); + } finally { + releaseA.resolve(); + releaseB.resolve(); + await Promise.all([readerA, readerB]); + } + await expect(update('epsilon')).resolves.toHaveProperty('errors', []); + await validate(); + expect( + (await fs.readdir(path.join(root, '.codebase-context', '.generations'))).length + ).toBeLessThanOrEqual(MAX_INDEX_GENERATIONS); + }); + + it('refreshes an existing searcher on every operation and keeps source-drift negatives', async () => { + await fixture(); + const searcher = new CodebaseSearcher(root); + expect((await searcher.search('alpha')).length).toBeGreaterThan(0); + await fs.writeFile(path.join(root, 'route.ts'), 'export function route() { return "beta"; }'); + expect((await searcher.searchWithCoverage('alpha')).sourceCoverage.status).toBe( + 'no_fresh_results' + ); + await update('beta', true); + expect((await searcher.search('beta')).length).toBeGreaterThan(0); + await update('gamma', true); + expect((await searcher.search('gamma')).length).toBeGreaterThan(0); + expect(await fs.readdir(path.join(root, '.codebase-context', '.pins'))).toEqual([]); + }); + + it('keeps validation and loading on the pinned build when publication occurs between them', async () => { + await fixture(); + const filename = await resolveIndexArtifact(root, 'index.json'); + const read = fs.readFile.bind(fs); + let refreshed = false; + const spy = vi.spyOn(fs, 'readFile').mockImplementation(async (...args) => { + const data = await read(...args); + if (!refreshed && String(args[0]) === filename) { + refreshed = true; + await update('beta'); + } + return data; + }); + const outcome = await new CodebaseSearcher(root).searchWithCoverage('alpha'); + spy.mockRestore(); + expect(refreshed).toBe(true); + expect(outcome.results).toEqual([]); + expect(outcome.sourceCoverage.status).toBe('no_fresh_results'); + expect((await new CodebaseSearcher(root).search('beta')).length).toBeGreaterThan(0); + }); +}); diff --git a/tests/index-lease-replacement.test.ts b/tests/index-lease-replacement.test.ts new file mode 100644 index 0000000..fb7af13 --- /dev/null +++ b/tests/index-lease-replacement.test.ts @@ -0,0 +1,430 @@ +import { promises as fs } from 'node:fs'; +import path from 'node:path'; +import os from 'node:os'; +import { fork, type ChildProcess } from 'node:child_process'; +import { fileURLToPath } from 'node:url'; +import { afterEach, expect, it, vi } from 'vitest'; +import { z } from 'zod'; +import { withIndexWriter } from '../src/core/index-generation.js'; +import { atomicWrite } from '../src/utils/atomic-write.js'; + +const roots: string[] = []; +const children = new Map>(); +const Ticket = z.object({ pid: z.number(), choosing: z.boolean(), ticket: z.number() }); +const ChildMessage = z.object({ + type: z.enum(['ready', 'held', 'closed', 'entered', 'leaving', 'finished', 'failed']), + bytes: z.string().optional(), + time: z.number().optional(), + message: z.string().optional() +}); +afterEach(async () => { + vi.restoreAllMocks(); + for (const [child] of children) { + if (child.exitCode === null && child.signalCode === null) child.kill('SIGKILL'); + } + await Promise.all(children.values()); + children.clear(); + for (const root of roots.splice(0)) await fs.rm(root, { recursive: true, force: true }); +}); +async function fixture() { + const root = await fs.mkdtemp(path.join(os.tmpdir(), 'cbc-ticket-replacement-')); + roots.push(root); + return root; +} +function bounded(promise: Promise): Promise { + return new Promise((resolve, reject) => { + const timer = setTimeout(() => reject(new Error('Lease control barrier timed out')), 5000); + promise.then(resolve, reject).finally(() => clearTimeout(timer)); + }); +} +function child(mode: 'reader' | 'writer', target: string) { + const process = fork( + fileURLToPath(new URL('./helpers/index-lease-control-child.ts', import.meta.url)), + [mode, target], + { + execArgv: ['--import', 'tsx'], + stdio: ['ignore', 'ignore', 'pipe', 'ipc'], + windowsHide: true + } + ); + const history: z.infer[] = []; + const listeners = new Set<(message: z.infer) => void>(); + process.on('message', (value) => { + const message = ChildMessage.parse(value); + history.push(message); + for (const listener of listeners) listener(message); + }); + const exited = new Promise((resolve) => process.once('exit', () => resolve())); + children.set(process, exited); + const wait = (type: z.infer['type']) => + bounded( + new Promise>((resolve, reject) => { + const existing = history.find((message) => message.type === type); + if (existing) return resolve(existing); + const receive = (message: z.infer) => { + if (message.type === type) { + listeners.delete(receive); + resolve(message); + } else if (message.type === 'failed') { + listeners.delete(receive); + reject(new Error(message.message)); + } + }; + listeners.add(receive); + }) + ); + return { process, wait, exited, history }; +} +async function hold(target: string, mode: string, warmed?: ReturnType) { + if (mode === 'same') { + const handle = await fs.open(target, 'r'); + return { bytes: (await handle.readFile()).toString('utf8'), close: () => handle.close() }; + } + const reader = warmed ?? child('reader', target); + await reader.wait('ready'); + reader.process.send({ target }); + const held = await reader.wait('held'); + return { + bytes: held.bytes, + close: async () => { + const closed = reader.wait('closed'); + reader.process.send({ close: true }); + await closed; + await bounded(reader.exited); + } + }; +} +function writerTarget(target: unknown) { + return ( + String(target).includes(`${path.sep}.writers${path.sep}`) && String(target).endsWith('.json') + ); +} +async function nextTicket( + source: Parameters[0], + target: Parameters[1] +) { + if (!writerTarget(target)) return null; + return Ticket.parse(JSON.parse(await fs.readFile(source, 'utf8'))); +} + +it.skipIf(process.platform !== 'win32').each([ + { phase: 'numbered', mode: 'same' }, + { phase: 'numbered', mode: 'cross' }, + { phase: 'released', mode: 'same' }, + { phase: 'released', mode: 'cross' } +])( + 'publishes a complete $phase ticket after an ordinary $mode-process reader closes', + async ({ phase, mode }) => { + const root = await fixture(); + const reader = mode === 'cross' ? child('reader', root) : undefined; + if (reader) await reader.wait('ready'); + const rename = fs.rename.bind(fs); + let held: Awaited> | undefined; + let admitted = false; + let failures = 0; + let successes = 0; + let oldBytes: string | undefined; + const publish = vi.spyOn(fs, 'rename').mockImplementation(async (source, target) => { + const next = await nextTicket(source, target); + const selected = + next && !next.choosing && (phase === 'numbered' ? next.ticket > 0 : next.ticket === 0); + if (!selected) return rename(source, target); + const proposed = await fs.readFile(source, 'utf8'); + if (!held) { + oldBytes = await fs.readFile(target, 'utf8'); + held = await hold(String(target), mode, reader); + expect(held.bytes).toBe(oldBytes); + } + try { + await rename(source, target); + successes++; + expect(await fs.readFile(target, 'utf8')).toBe(proposed); + } catch (error) { + failures++; + expect(error).toMatchObject({ code: 'EPERM', syscall: 'rename', dest: String(target) }); + expect(admitted).toBe(phase === 'released'); + expect(await fs.readFile(target, 'utf8')).toBe(oldBytes); + await held.close(); + throw error; + } + }); + await expect( + withIndexWriter(root, async () => { + admitted = true; + return 'successful-operation'; + }) + ).resolves.toBe('successful-operation'); + expect(failures).toBe(1); + expect(successes).toBe(1); + expect(admitted).toBe(true); + expect(await fs.readdir(path.join(root, '.codebase-context/.writers'))).toEqual([]); + publish.mockRestore(); + await expect(withIndexWriter(root, async () => 'next-operation')).resolves.toBe( + 'next-operation' + ); + } +); + +it.skipIf(process.platform !== 'win32').each(['same', 'cross'])( + 'bounds replacement failures under a persistently held %s-process read handle without admission', + async (mode) => { + const root = await fixture(); + const reader = mode === 'cross' ? child('reader', root) : undefined; + if (reader) await reader.wait('ready'); + const rename = fs.rename.bind(fs); + let held: Awaited> | undefined; + let admitted = false; + let acquisitionFailures = 0; + let lastAcquisitionError: unknown; + vi.spyOn(fs, 'rename').mockImplementation(async (source, target) => { + const next = await nextTicket(source, target); + if (next && !next.choosing && next.ticket > 0) { + if (!held) held = await hold(String(target), mode, reader); + try { + await rename(source, target); + } catch (error) { + acquisitionFailures++; + lastAcquisitionError = error; + expect(await fs.readFile(target, 'utf8')).toBe(held.bytes); + throw error; + } + return; + } + return rename(source, target); + }); + try { + const operation = withIndexWriter(root, async () => { + admitted = true; + }); + await expect(operation).rejects.toMatchObject({ code: 'EPERM', syscall: 'rename' }); + await expect(operation).rejects.toBe(lastAcquisitionError); + expect(admitted).toBe(false); + expect(acquisitionFailures).toBeGreaterThan(1); + expect(acquisitionFailures).toBeLessThanOrEqual(8); + expect(await fs.readdir(path.join(root, '.codebase-context/.writers'))).toEqual([]); + } finally { + await held?.close(); + } + } +); + +it.skipIf(process.platform !== 'win32')( + 'starts no further replacement attempt after its retry deadline', + async () => { + const root = await fixture(); + const rename = fs.rename.bind(fs); + const now = Date.now(); + const clock = vi.spyOn(Date, 'now').mockReturnValue(now); + let attempts = 0; + let admitted = false; + let denial: Error | undefined; + vi.spyOn(fs, 'rename').mockImplementation(async (source, target) => { + const next = await nextTicket(source, target); + if (next && !next.choosing && next.ticket > 0) { + attempts++; + clock.mockReturnValue(now + 251); + denial = Object.assign(new Error('Controlled replacement deadline'), { + code: 'EPERM', + syscall: 'rename', + path: String(source), + dest: String(target) + }); + throw denial; + } + return rename(source, target); + }); + await expect( + withIndexWriter(root, async () => { + admitted = true; + }) + ).rejects.toMatchObject({ message: 'Controlled replacement deadline' }); + expect(attempts).toBe(1); + expect(admitted).toBe(false); + } +); + +it.each([ + { code: 'EACCES', syscall: 'rename', mismatch: '' }, + { code: 'EPERM', syscall: 'open', mismatch: '' }, + { code: 'EIO', syscall: 'rename', mismatch: '' }, + { code: 'EPERM', syscall: 'rename', mismatch: 'destination' }, + { code: 'EPERM', syscall: 'rename', mismatch: 'temporary-uuid' }, + { code: 'EPERM', syscall: 'rename', mismatch: 'temporary-directory' }, + { code: 'EPERM', syscall: 'rename', mismatch: 'missing-destination' }, + { code: 'EPERM', syscall: 'rename', mismatch: 'non-string-source' } +])( + 'preserves permanent or unrelated $code/$syscall errors ($mismatch) without retry or admission', + async ({ code, syscall, mismatch }) => { + const root = await fixture(); + const rename = fs.rename.bind(fs); + let attempts = 0; + let admitted = false; + let denial: Error | undefined; + vi.spyOn(fs, 'rename').mockImplementation(async (source, target) => { + const next = await nextTicket(source, target); + if (next && !next.choosing && next.ticket > 0) { + attempts++; + denial = Object.assign(new Error('Controlled permanent or unrelated filesystem error'), { + code, + syscall, + path: + mismatch === 'non-string-source' + ? 4 + : mismatch === 'temporary-uuid' + ? `${target}.not-a-uuid.tmp` + : mismatch === 'temporary-directory' + ? path.join(root, path.basename(String(source))) + : String(source), + ...(mismatch === 'missing-destination' + ? {} + : { + dest: mismatch === 'destination' ? path.join(root, 'foreign.json') : String(target) + }) + }); + throw denial; + } + return rename(source, target); + }); + const failed = withIndexWriter(root, async () => { + admitted = true; + }); + await expect(failed).rejects.toMatchObject({ code, syscall }); + await expect(failed).rejects.toBe(denial); + expect(attempts).toBe(1); + expect(admitted).toBe(false); + expect(await fs.readdir(path.join(root, '.codebase-context/.writers'))).toEqual([]); + } +); + +it('does not retry initial ticket creation', async () => { + const root = await fixture(); + const rename = fs.rename.bind(fs); + let attempts = 0; + let admitted = false; + vi.spyOn(fs, 'rename').mockImplementation(async (source, target) => { + const next = await nextTicket(source, target); + if (next?.choosing) { + attempts++; + throw Object.assign(new Error('Initial ticket publication denied'), { + code: 'EPERM', + syscall: 'rename', + path: String(source), + dest: String(target) + }); + } + return rename(source, target); + }); + await expect( + withIndexWriter(root, async () => { + admitted = true; + }) + ).rejects.toThrow('Initial ticket publication denied'); + expect(attempts).toBe(1); + expect(admitted).toBe(false); +}); + +it('keeps ticket open errors fail closed', async () => { + const root = await fixture(); + const read = fs.readFile.bind(fs); + let attempts = 0; + let admitted = false; + vi.spyOn(fs, 'readFile').mockImplementation((file, options) => { + if (writerTarget(file)) { + attempts++; + return Promise.reject( + Object.assign(new Error('Controlled ticket open denial'), { + code: 'EPERM', + syscall: 'open', + path: String(file) + }) + ); + } + return read(file, options); + }); + await expect( + withIndexWriter(root, async () => { + admitted = true; + }) + ).rejects.toMatchObject({ code: 'EPERM', syscall: 'open' }); + expect(attempts).toBe(1); + expect(admitted).toBe(false); +}); + +it('leaves generic atomic publication errors unchanged', async () => { + const root = await fixture(); + const target = path.join(root, 'ordinary.json'); + await fs.writeFile(target, 'original'); + let attempts = 0; + vi.spyOn(fs, 'rename').mockImplementation(async (source, destination) => { + attempts++; + throw Object.assign(new Error('Generic rename denied'), { + code: 'EPERM', + syscall: 'rename', + path: String(source), + dest: String(destination) + }); + }); + await expect(atomicWrite(target, 'replacement')).rejects.toMatchObject({ code: 'EPERM' }); + expect(attempts).toBe(1); + expect(await fs.readFile(target, 'utf8')).toBe('original'); + expect(await fs.readdir(root)).toEqual(['ordinary.json']); +}); + +it('preserves cross-process writer order and excludes admission until the first operation leaves', async () => { + const root = await fixture(); + const first = child('writer', root); + await first.wait('entered'); + const rename = fs.rename.bind(fs); + const queued = Promise.withResolvers(); + let secondEnteredAt = 0; + vi.spyOn(fs, 'rename').mockImplementation(async (source, target) => { + const next = await nextTicket(source, target); + await rename(source, target); + if (next && !next.choosing && next.ticket > 0) { + expect(next.ticket).toBe(2); + queued.resolve(); + } + }); + const second = withIndexWriter(root, async () => { + secondEnteredAt = Date.now(); + return 'second-operation'; + }); + await bounded(queued.promise); + expect(secondEnteredAt).toBe(0); + const leaving = first.wait('leaving'); + first.process.send({ release: true }); + const exitedSection = await leaving; + await expect(second).resolves.toBe('second-operation'); + expect(secondEnteredAt).toBeGreaterThanOrEqual(exitedSection.time ?? Infinity); + await bounded(first.exited); +}); + +it('recovers the unique ticket of an actually terminated owner', async () => { + const root = await fixture(); + const first = child('writer', root); + await first.wait('entered'); + const directory = path.join(root, '.codebase-context/.writers'); + const abandoned = await fs.readdir(directory); + expect(abandoned).toHaveLength(1); + first.process.kill('SIGKILL'); + await bounded(first.exited); + await expect(withIndexWriter(root, async () => 'recovered')).resolves.toBe('recovered'); + expect(await fs.readdir(directory)).toEqual([]); +}); + +it('rejects malformed foreign ticket bytes without admitting or rewriting them', async () => { + const root = await fixture(); + const directory = path.join(root, '.codebase-context/.writers'); + await fs.mkdir(directory, { recursive: true }); + const foreign = path.join(directory, 'foreign.json'); + const original = '{"pid":"invalid"}'; + await fs.writeFile(foreign, original); + let admitted = false; + await expect( + withIndexWriter(root, async () => { + admitted = true; + }) + ).rejects.toThrow(); + expect(admitted).toBe(false); + expect(await fs.readFile(foreign, 'utf8')).toBe(original); +}); diff --git a/tests/index-mode-matrix.test.ts b/tests/index-mode-matrix.test.ts new file mode 100644 index 0000000..fb0e198 --- /dev/null +++ b/tests/index-mode-matrix.test.ts @@ -0,0 +1,479 @@ +import { promises as fs } from 'node:fs'; +import { randomUUID, createHash } from 'node:crypto'; +import { fork, spawnSync, type ChildProcess } from 'node:child_process'; +import { fileURLToPath } from 'node:url'; +import os from 'node:os'; +import path from 'node:path'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +const { provider } = vi.hoisted(() => ({ provider: vi.fn() })); +vi.mock('../src/embeddings/index.js', async (original) => ({ + ...(await original()), + getEmbeddingProvider: provider +})); +import { CodebaseIndexer } from '../src/core/indexer.js'; +import { handleCliCommand } from '../src/cli.js'; +import { resolveSkipEmbedding, readIndexMeta } from '../src/core/index-meta.js'; +import { + inspectIndexDiskState, + resolveIndexArtifact, + IndexCapacityError +} from '../src/core/index-generation.js'; + +let root = ''; +const children: ChildProcess[] = []; +const previousRoot = process.env.CODEBASE_ROOT; +const previousConfig = process.env.CODEBASE_CONTEXT_CONFIG_PATH; +beforeEach(async () => { + root = await fs.mkdtemp(path.join(os.tmpdir(), 'cbc-mode-matrix-')); + await fs.writeFile(path.join(root, 'route.ts'), 'export const route = "matrix";'); + process.env.CODEBASE_ROOT = root; + process.env.CODEBASE_CONTEXT_CONFIG_PATH = path.join(root, 'server-config.json'); + provider.mockReset().mockResolvedValue({ + name: 'fixture', + modelName: 'matrix', + dimensions: 384, + initialize: async () => undefined, + isReady: () => true, + embedBatch: async (texts: string[]) => + texts.map(() => Array.from({ length: 384 }, (_, i) => i / 1000)), + embed: async () => Array(384).fill(0) + }); +}); +afterEach(async () => { + vi.restoreAllMocks(); + for (const child of children.splice(0)) { + if (child.exitCode !== null || child.signalCode !== null) continue; + const exited = new Promise((resolve) => child.once('exit', () => resolve())); + child.kill('SIGKILL'); + await exited; + } + await fs.rm(root, { recursive: true, force: true }); + if (previousRoot === undefined) delete process.env.CODEBASE_ROOT; + else process.env.CODEBASE_ROOT = previousRoot; + if (previousConfig === undefined) delete process.env.CODEBASE_CONTEXT_CONFIG_PATH; + else process.env.CODEBASE_CONTEXT_CONFIG_PATH = previousConfig; +}); +async function config(skipEmbedding?: boolean) { + await fs.writeFile( + path.join(root, 'server-config.json'), + JSON.stringify({ + projects: [{ root, ...(skipEmbedding === undefined ? {} : { skipEmbedding }) }] + }) + ); +} +async function build(skipEmbedding: boolean) { + return new CodebaseIndexer({ rootPath: root, config: { skipEmbedding } }).index(); +} +async function digest( + directory: string +): Promise> { + const result: Array<{ path: string; bytes: number; sha256: string }> = []; + const visit = async (current: string): Promise => { + for (const entry of (await fs.readdir(current, { withFileTypes: true })).sort((a, b) => + a.name.localeCompare(b.name) + )) { + const filename = path.join(current, entry.name); + if (entry.isDirectory()) await visit(filename); + else { + const data = await fs.readFile(filename); + result.push({ + path: path.relative(directory, filename), + bytes: data.length, + sha256: createHash('sha256').update(data).digest('hex') + }); + } + } + }; + await visit(directory); + return result; +} +async function legacy(skipEmbedding: boolean) { + await build(skipEmbedding); + const pointer = path.join(root, '.codebase-context', 'index-meta.json'); + const meta = await readIndexMeta(root); + const directory = path.dirname(await resolveIndexArtifact(root, 'index.json')); + for (const entry of await fs.readdir(directory)) { + if (entry !== 'index-meta.json') + await fs.cp(path.join(directory, entry), path.join(root, '.codebase-context', entry), { + recursive: true + }); + } + const fixed = JSON.parse(await fs.readFile(path.join(directory, 'index-meta.json'), 'utf8')); + expect(fixed.buildId).toBe(meta.buildId); + await fs.writeFile(pointer, JSON.stringify(fixed)); + await fs.rm(path.join(root, '.codebase-context', '.generations'), { recursive: true }); +} +const scenarios = [ + { name: 'fresh default', setup: async () => config(), mode: true }, + { name: 'fresh explicit keyword', setup: async () => config(true), mode: true }, + { name: 'fresh explicit semantic', setup: async () => config(false), mode: false }, + { + name: 'recorded keyword', + setup: async () => { + await build(true); + await config(); + }, + mode: true + }, + { + name: 'recorded semantic', + setup: async () => { + await build(false); + await config(); + }, + mode: false + }, + { + name: 'legacy keyword', + setup: async () => { + await legacy(true); + await config(); + }, + mode: true + }, + { + name: 'legacy semantic', + setup: async () => { + await legacy(false); + await config(); + }, + mode: false + }, + { + name: 'memory only', + setup: async () => { + await fs.mkdir(path.join(root, '.codebase-context')); + await fs.writeFile(path.join(root, '.codebase-context', 'memory.json'), '{}'); + await config(); + }, + mode: true + } +]; +describe.each(['map', 'reindex'] as const)('mode matrix through %s', (command) => { + it.each(scenarios)('$name', async ({ setup, mode }) => { + await setup(); + provider.mockClear(); + await handleCliCommand([command, '--json']); + expect(await resolveSkipEmbedding(root)).toBe(mode); + expect((await readIndexMeta(root)).artifacts.vectorDb.embeddingEnabled).toBe(!mode); + if (mode) expect(provider).not.toHaveBeenCalled(); + }); + it('preserves an out-of-order stage instead of treating known filenames as a first build', async () => { + await build(true); + await config(); + const generation = path.dirname(await resolveIndexArtifact(root, 'index.json')); + await fs.rm(path.join(root, '.codebase-context', 'index-meta.json')); + for (const name of await fs.readdir(generation)) { + if (name !== 'health.json') await fs.rm(path.join(generation, name), { recursive: true }); + } + const before = await digest(generation); + await expect(handleCliCommand([command, '--json'])).rejects.toThrow(/recover|explicit/i); + expect(await digest(generation)).toEqual(before); + }); + it.each(['keyword', 'semantic'] as const)( + 'requires explicit recovery of a completed %s orphan', + async (mode) => { + await build(mode === 'keyword'); + await config(); + const directory = path.join(root, '.codebase-context', '.generations'); + const before = await digest(directory); + await fs.rm(path.join(root, '.codebase-context', 'index-meta.json')); + await expect(handleCliCommand([command, '--json'])).rejects.toThrow(/explicit|recover/i); + expect(await digest(directory)).toEqual(before); + await expect( + fs.access(path.join(root, '.codebase-context', 'index-meta.json')) + ).rejects.toMatchObject({ code: 'ENOENT' }); + } + ); + it.each(['malformed', 'future', 'truncated-local', 'mismatched-local', 'unknown-stage'] as const)( + 'preserves %s evidence', + async (variant) => { + await build(true); + await config(); + const pointer = path.join(root, '.codebase-context', 'index-meta.json'); + const generation = path.dirname(await resolveIndexArtifact(root, 'index.json')); + if (variant === 'malformed') await fs.writeFile(pointer, '{'); + else if (variant === 'future') { + const meta = await readIndexMeta(root); + await fs.writeFile(pointer, JSON.stringify({ ...meta, formatVersion: 999 })); + } else { + await fs.rm(pointer); + if (variant === 'unknown-stage') + await fs.writeFile(path.join(generation, 'unknown.bin'), 'hold'); + else if (variant === 'truncated-local') + await fs.writeFile(path.join(generation, 'index-meta.json'), '{'); + else { + const local = JSON.parse( + await fs.readFile(path.join(generation, 'index-meta.json'), 'utf8') + ); + local.buildId = randomUUID(); + await fs.writeFile(path.join(generation, 'index-meta.json'), JSON.stringify(local)); + } + } + const before = await digest(path.join(root, '.codebase-context', '.generations')); + await expect(handleCliCommand([command, '--json'])).rejects.toThrow(); + expect(await digest(path.join(root, '.codebase-context', '.generations'))).toEqual(before); + } + ); +}); + +it.each([true, false])( + 'preserves three mixed completed orphans and reports capacity even with explicit skipEmbedding=%s', + async (skipEmbedding) => { + await build(false); + await build(true); + await build(false); + const directory = path.join(root, '.codebase-context', '.generations'); + const before = await digest(directory); + await fs.rm(path.join(root, '.codebase-context', 'index-meta.json')); + const attempt = build(skipEmbedding); + await expect(attempt).rejects.toBeInstanceOf(IndexCapacityError); + await expect(attempt).rejects.toThrow(/operator-selected.*outside.*explicit/i); + expect(await digest(directory)).toEqual(before); + expect(await fs.readdir(directory)).toHaveLength(3); + } +); +it.each( + [1, 2].flatMap((count) => [true, false].map((skipEmbedding) => ({ count, skipEmbedding }))) +)( + 'makes explicit progress with $count completed orphans and skipEmbedding=$skipEmbedding without deleting bytes', + async ({ count, skipEmbedding }) => { + for (let i = 0; i < count; i++) await build(i % 2 === 0); + const directory = path.join(root, '.codebase-context', '.generations'); + const before = await digest(directory); + await fs.rm(path.join(root, '.codebase-context', 'index-meta.json')); + await build(skipEmbedding); + const after = await digest(directory); + for (const file of before) expect(after).toContainEqual(file); + expect((await readIndexMeta(root)).artifacts.vectorDb.embeddingEnabled).toBe(!skipEmbedding); + } +); +it('holds an unknown UUID generation during explicit recovery', async () => { + const directory = path.join(root, '.codebase-context', '.generations', randomUUID()); + await fs.mkdir(directory, { recursive: true }); + await fs.writeFile(path.join(directory, 'unknown.bin'), 'preserve'); + const before = await digest(directory); + await build(true); + expect(await digest(directory)).toEqual(before); +}); + +it.each([true, false])( + 'holds unknown and completed data at capacity with explicit skipEmbedding=%s', + async (skipEmbedding) => { + await build(true); + await build(false); + await fs.rm(path.join(root, '.codebase-context', 'index-meta.json')); + const directory = path.join(root, '.codebase-context', '.generations'); + const unknown = path.join(directory, randomUUID()); + await fs.mkdir(unknown); + await fs.writeFile(path.join(unknown, 'unknown.bin'), 'preserve'); + const before = await digest(directory); + await expect(build(skipEmbedding)).rejects.toBeInstanceOf(IndexCapacityError); + expect(await digest(directory)).toEqual(before); + } +); +for (const command of ['map', 'reindex'] as const) { + it(`preserves unversioned legacy bytes through ${command} without inventing a mode`, async () => { + await legacy(true); + await config(); + const pointer = path.join(root, '.codebase-context', 'index-meta.json'); + await fs.rm(pointer); + const before = await digest(path.join(root, '.codebase-context')); + expect(await inspectIndexDiskState(root)).toEqual({ kind: 'legacy', mode: 'unknown' }); + if (command === 'map') await handleCliCommand([command, '--json']); + else await expect(handleCliCommand([command, '--json'])).rejects.toThrow(/recover|explicit/i); + expect(await digest(path.join(root, '.codebase-context'))).toEqual(before); + await expect(fs.access(pointer)).rejects.toMatchObject({ code: 'ENOENT' }); + }); + it(`rejects unsupported future fixed-layout metadata through ${command}`, async () => { + await legacy(true); + await config(); + const pointer = path.join(root, '.codebase-context', 'index-meta.json'); + const meta = await readIndexMeta(root); + await fs.writeFile(pointer, JSON.stringify({ ...meta, metaVersion: 999 })); + const before = await digest(path.join(root, '.codebase-context')); + await expect(handleCliCommand([command, '--json'])).rejects.toThrow(); + expect(await digest(path.join(root, '.codebase-context'))).toEqual(before); + }); + it(`reports pointer permission failure through ${command} without publication`, async () => { + await build(true); + await config(); + const pointer = path.join(root, '.codebase-context', 'index-meta.json'); + const before = await digest(path.join(root, '.codebase-context', '.generations')); + const read = fs.readFile.bind(fs); + vi.spyOn(fs, 'readFile').mockImplementation(async (...args) => { + if (String(args[0]) === pointer) + throw Object.assign(new Error('pointer access denied'), { code: 'EACCES' }); + return read(...args); + }); + await expect(inspectIndexDiskState(root)).rejects.toMatchObject({ code: 'EACCES' }); + await expect(handleCliCommand([command, '--json'])).rejects.toThrow(/denied|unreadable/i); + expect(await digest(path.join(root, '.codebase-context', '.generations'))).toEqual(before); + }); + it(`preserves linked orphan contents through ${command}`, async () => { + await build(true); + await config(); + const generation = path.dirname(await resolveIndexArtifact(root, 'index.json')); + await fs.rm(path.join(root, '.codebase-context', 'index-meta.json')); + await fs.rm(path.join(generation, 'index'), { recursive: true }); + const outside = path.join(root, 'outside'); + await fs.mkdir(outside); + await fs.writeFile(path.join(outside, 'held.bin'), 'held'); + await fs.symlink( + outside, + path.join(generation, 'index'), + process.platform === 'win32' ? 'junction' : 'dir' + ); + const before = await digest(outside); + await expect(handleCliCommand([command, '--json'])).rejects.toThrow(/recover|explicit|linked/i); + expect(await digest(outside)).toEqual(before); + expect((await fs.lstat(path.join(generation, 'index'))).isSymbolicLink()).toBe(true); + }); +} + +for (const command of ['map', 'reindex'] as const) { + it.each([1, 2, 3])( + `preserves format-%i fixed-layout recorded mode through ${command}`, + async (formatVersion) => { + const skipEmbedding = formatVersion !== 2; + await legacy(skipEmbedding); + await config(); + const pointer = path.join(root, '.codebase-context', 'index-meta.json'); + const meta = await readIndexMeta(root); + await fs.writeFile(pointer, JSON.stringify({ ...meta, formatVersion })); + const before = await digest(path.join(root, '.codebase-context')); + expect(await resolveSkipEmbedding(root)).toBe(skipEmbedding); + if (command === 'map') { + await expect(handleCliCommand([command, '--json'])).rejects.toThrow(/version|rebuild/i); + expect(await digest(path.join(root, '.codebase-context'))).toEqual(before); + } else { + await handleCliCommand([command, '--json']); + expect((await readIndexMeta(root)).artifacts.vectorDb.embeddingEnabled).toBe( + !skipEmbedding + ); + } + } + ); +} +it.each(['index-generation', 'index-meta'])( + 'cold-imports %s first and resolves a fresh mode without a provider', + (first) => { + const entry = new URL(`../src/core/${first}.ts`, import.meta.url).href; + const metadata = new URL('../src/core/index-meta.ts', import.meta.url).href; + const script = `await import(${JSON.stringify(entry)}); const {resolveSkipEmbedding}=await import(${JSON.stringify(metadata)}); console.log((await resolveSkipEmbedding(${JSON.stringify(root)})) ? "keyword" : "semantic");`; + const result = spawnSync( + process.execPath, + ['--import', 'tsx', '--input-type=module', '-e', script], + { encoding: 'utf8', timeout: 15_000, windowsHide: true } + ); + expect(result.status).toBe(0); + expect(result.stdout.trim()).toBe('keyword'); + expect(result.stderr).toBe(''); + } +); + +function child(action: string, helper = 'index-mode-child.ts') { + const process = fork( + fileURLToPath(new URL(`./helpers/${helper}`, import.meta.url)), + helper === 'index-publication-child.ts' ? [root, 'full', 'vector-marker'] : [root, action], + { execArgv: ['--import', 'tsx'], stdio: ['ignore', 'ignore', 'pipe', 'ipc'] } + ); + children.push(process); + const seen: string[] = []; + const listeners = new Set<() => void>(); + let stderr = ''; + process.stderr?.on('data', (data: Buffer) => { + stderr += data.toString(); + }); + process.on('message', (message) => { + if (typeof message === 'object' && message !== null) { + if ('type' in message && typeof message.type === 'string') seen.push(message.type); + else if ('edge' in message) seen.push('marker'); + } + for (const check of listeners) check(); + }); + process.on('exit', () => { + for (const check of listeners) check(); + }); + const wait = (type: string) => + new Promise((resolve, reject) => { + const finish = (error?: Error) => { + clearTimeout(timer); + listeners.delete(check); + if (error) reject(error); + else resolve(); + }; + const check = () => { + if (seen.includes(type)) finish(); + else if ( + seen.includes('failed') || + process.exitCode !== null || + process.signalCode !== null + ) + finish(new Error(`Child stopped: ${seen} ${stderr}`)); + }; + // Stops on this event, child exit, or20seconds; afterEach kills only these children. + const timer = setTimeout( + () => finish(new Error(`Child did not report ${type}: ${stderr}`)), + 20_000 + ); + listeners.add(check); + check(); + }); + const kill = async () => { + const exited = new Promise((resolve) => process.once('exit', () => resolve())); + process.kill('SIGKILL'); + await exited; + }; + return { process, wait, kill }; +} +it('rechecks first-build eligibility under the writer after another process publishes semantic data', async () => { + await config(true); + const publisher = child('publish'); + await publisher.wait('held'); + const map = child('map'); + await map.wait('queued'); + publisher.process.send({ release: true }); + await publisher.wait('complete'); + const before = await fs.readFile(path.join(root, '.codebase-context', 'index-meta.json')); + expect((await readIndexMeta(root)).artifacts.vectorDb.embeddingEnabled).toBe(true); + await map.wait('complete'); + expect(await fs.readFile(path.join(root, '.codebase-context', 'index-meta.json'))).toEqual( + before + ); + expect(await fs.readdir(path.join(root, '.codebase-context', '.generations'))).toHaveLength(1); +}, 30_000); +it.each(['map', 'reindex'] as const)( + 'resumes a real killed model-free marker-only first build through %s', + async (command) => { + await config(); + const first = child('marker', 'index-publication-child.ts'); + await first.wait('marker'); + await first.kill(); + expect((await inspectIndexDiskState(root)).kind).toBe('incomplete-first-build'); + await handleCliCommand([command, '--json']); + expect((await readIndexMeta(root)).artifacts.vectorDb.embeddingEnabled).toBe(false); + expect(await fs.readdir(path.join(root, '.codebase-context', '.generations'))).toHaveLength(2); + }, + 30_000 +); +it.each(['map', 'reindex'] as const)( + 'preserves a real killed semantic native stage before its marker through %s', + async (command) => { + await config(); + const first = child('semantic-stage'); + await first.wait('native'); + await first.kill(); + const directory = path.join(root, '.codebase-context', '.generations'); + const before = await digest(directory); + expect(before.some((file) => file.path.includes('.lance'))).toBe(true); + expect( + before.some( + (file) => file.path.endsWith('index-meta.json') || file.path.endsWith('index-build.json') + ) + ).toBe(false); + await expect(handleCliCommand([command, '--json'])).rejects.toThrow(/recover|explicit/i); + expect(await digest(directory)).toEqual(before); + }, + 30_000 +); diff --git a/tests/index-publication.test.ts b/tests/index-publication.test.ts new file mode 100644 index 0000000..6ee5d40 --- /dev/null +++ b/tests/index-publication.test.ts @@ -0,0 +1,125 @@ +import { fork } from 'node:child_process'; +import { promises as fs } from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { afterEach, describe, expect, it } from 'vitest'; +import { CodebaseIndexer } from '../src/core/indexer.js'; +import { readIndexMeta, validateIndexArtifacts } from '../src/core/index-meta.js'; +import { handle as searchTool } from '../src/tools/search-codebase.js'; +import { makePaths, getOrCreateProject, clearProjects } from '../src/project-state.js'; + +const roots: string[] = []; +afterEach(async () => { + await clearProjects(); + for (const root of roots.splice(0)) await fs.rm(root, { recursive: true, force: true }); +}); + +async function fixture() { + const root = await fs.mkdtemp(path.join(os.tmpdir(), 'cbc-publication-')); + roots.push(root); + await fs.writeFile(path.join(root, 'route.ts'), 'export function route() { return "old"; }'); + await new CodebaseIndexer({ rootPath: root, config: { skipEmbedding: true } }).index(); + return root; +} + +async function interrupt(root: string, mode: string, edge: string) { + const child = fork( + fileURLToPath(new URL('./helpers/index-publication-child.ts', import.meta.url)), + [root, mode, edge], + { + execArgv: ['--import', 'tsx'], + stdio: ['ignore', 'pipe', 'pipe', 'ipc'] + } + ); + let stderr = ''; + child.stderr?.on('data', (data: Buffer) => { + stderr += data.toString(); + }); + child.stdout?.resume(); + let watchdog: ReturnType | undefined; + try { + await new Promise((resolve, reject) => { + // End on an observed edge/exit or after twenty seconds; finally kills this child. + watchdog = setTimeout( + () => reject(new Error(`Child did not reach ${edge}: ${stderr}`)), + 20_000 + ); + child.once('error', reject); + child.once('exit', (code) => + reject(new Error(`Child exited before edge (${code}): ${stderr}`)) + ); + child.on('message', (message) => { + if (typeof message === 'object' && message !== null && 'edge' in message) resolve(); + else reject(new Error(`Child did not reach edge: ${JSON.stringify(message)}`)); + }); + }); + } finally { + clearTimeout(watchdog); + const exited = new Promise((resolve) => child.once('exit', () => resolve())); + child.kill('SIGKILL'); + if (child.exitCode === null && child.signalCode === null) await exited; + } +} + +describe('interrupted index publication', () => { + it('serves the committed index through the direct MCP tool during refresh', async () => { + const root = await fixture(); + const response = await searchTool( + { query: 'route' }, + { + rootPath: root, + paths: makePaths(root), + indexState: { status: 'indexing' }, + performIndexing: () => undefined + } + ); + const payload: { status: string; results?: unknown[] } = JSON.parse( + response.content?.[0]?.text ?? '{}' + ); + expect(payload.status).toBe('success'); + expect(payload.results?.length).toBeGreaterThan(0); + }); + + it('serves the committed index through the native MCP gate during refresh', async () => { + const root = await fixture(); + const oldRoot = process.env.CODEBASE_ROOT; + const oldArgv = process.argv; + process.env.CODEBASE_ROOT = root; + process.argv = [process.execPath, 'publication-test']; + try { + const { server } = await import('../src/index.js'); + const project = getOrCreateProject(root); + project.initPromise = Promise.resolve(); + project.indexState.status = 'indexing'; + // Invoke the real SDK-registered MCP handler, with the real search/index code. + const handler = Reflect.get(server, '_requestHandlers').get('tools/call'); + const response = await handler({ + method: 'tools/call', + params: { name: 'search_codebase', arguments: { query: 'route', project: root } } + }); + const payload: { status: string; results?: unknown[] } = JSON.parse(response.content[0].text); + expect(payload.status).toBe('success'); + expect(payload.results?.length).toBeGreaterThan(0); + } finally { + process.argv = oldArgv; + if (oldRoot === undefined) delete process.env.CODEBASE_ROOT; + else process.env.CODEBASE_ROOT = oldRoot; + } + }); + + it.each([ + ['full', 'published-meta'], + ['incremental', 'vector-marker'] + ])( + 'keeps a complete committed index after killed %s at %s', + async (mode, edge) => { + const root = await fixture(); + await fs.writeFile(path.join(root, 'route.ts'), 'export function route() { return "new"; }'); + await interrupt(root, mode, edge); + const meta = await readIndexMeta(root); + await expect(validateIndexArtifacts(root, meta)).resolves.toBeUndefined(); + }, + 30_000 + ); +}); diff --git a/tests/index-versioning-migration.test.ts b/tests/index-versioning-migration.test.ts index a541f17..57e03d5 100644 --- a/tests/index-versioning-migration.test.ts +++ b/tests/index-versioning-migration.test.ts @@ -27,6 +27,12 @@ vi.mock('../src/core/search.js', async () => { async search(query: string, limit: number, filters?: unknown, options?: unknown) { return searchMocks.search(query, limit, filters, options); } + async searchWithCoverage(query: string, limit: number, filters?: unknown, options?: unknown) { + return { + results: await this.search(query, limit, filters, options), + sourceCoverage: { status: 'current', checkedFiles: 0, skippedFiles: 0 } + }; + } } return { CodebaseSearcher }; @@ -207,7 +213,7 @@ describe('index versioning migration (MIGR-01)', () => { afterEach(async () => { const { clearProjects } = await import('../src/project-state.js'); - clearProjects(); + await clearProjects(); if (originalArgv) process.argv = originalArgv; if (originalEnvRoot === undefined) delete process.env.CODEBASE_ROOT; @@ -425,7 +431,7 @@ describe('index-consuming allowlist enforcement', () => { afterEach(async () => { const { clearProjects } = await import('../src/project-state.js'); - clearProjects(); + await clearProjects(); if (originalArgv) process.argv = originalArgv; if (originalEnvRoot === undefined) delete process.env.CODEBASE_ROOT; @@ -481,7 +487,8 @@ describe('index-consuming allowlist enforcement', () => { } })); - vi.doMock('../src/core/index-meta.js', () => ({ + vi.doMock('../src/core/index-meta.js', async (importOriginal) => ({ + ...(await importOriginal()), readIndexMeta, validateIndexArtifacts })); diff --git a/tests/index-writer-process.test.ts b/tests/index-writer-process.test.ts new file mode 100644 index 0000000..dfe7303 --- /dev/null +++ b/tests/index-writer-process.test.ts @@ -0,0 +1,122 @@ +import { fork, type ChildProcess } from 'node:child_process'; +import { promises as fs } from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { afterEach, expect, it } from 'vitest'; +import { CodebaseIndexer } from '../src/core/indexer.js'; +import { readIndexMeta, validateIndexArtifacts } from '../src/core/index-meta.js'; +import { CodebaseSearcher } from '../src/core/search.js'; + +let root = ''; +const children: ChildProcess[] = []; +afterEach(async () => { + for (const child of children.splice(0)) { + if (child.exitCode !== null || child.signalCode !== null) continue; + const exited = new Promise((resolve) => child.once('exit', () => resolve())); + child.kill('SIGKILL'); + await exited; + } + if (root) await fs.rm(root, { recursive: true, force: true }); + root = ''; +}); + +function writer(hold: boolean) { + const child = fork( + fileURLToPath(new URL('./helpers/index-writer-child.ts', import.meta.url)), + [root, hold ? 'hold' : 'run'], + { + execArgv: ['--import', 'tsx'], + stdio: ['ignore', 'ignore', 'pipe', 'ipc'] + } + ); + children.push(child); + const messages: Array<{ type: string; incremental?: unknown; message?: string }> = []; + const listeners = new Set<() => void>(); + let stderr = ''; + child.stderr?.on('data', (data: Buffer) => { + stderr += data.toString(); + }); + child.on('message', (message) => { + if ( + typeof message !== 'object' || + message === null || + !('type' in message) || + typeof message.type !== 'string' + ) + return; + messages.push({ ...message, type: message.type }); + for (const listener of listeners) listener(); + }); + child.on('exit', () => { + for (const listener of listeners) listener(); + }); + const wait = (type: string) => + new Promise<{ type: string; incremental?: unknown }>((resolve, reject) => { + // Test-child watchdog: stop after twenty seconds or as soon as this event arrives. + const timer = setTimeout( + () => finish(new Error(`Child did not report ${type}: ${stderr}`)), + 20_000 + ); + const finish = (error?: Error) => { + clearTimeout(timer); + listeners.delete(check); + if (error) reject(error); + }; + const check = () => { + const message = messages.find((candidate) => candidate.type === type); + if (message) { + finish(); + resolve(message); + } else if ( + messages.some((candidate) => candidate.type === 'failed') || + child.exitCode !== null || + child.signalCode !== null + ) + finish(new Error(`Writer stopped: ${JSON.stringify(messages)} ${stderr}`)); + }; + listeners.add(check); + check(); + }); + return { child, wait }; +} + +async function fixture() { + root = await fs.mkdtemp(path.join(os.tmpdir(), 'cbc-writers-')); + await fs.writeFile(path.join(root, 'a.ts'), 'export const first = "alpha";'); + await fs.writeFile(path.join(root, 'b.ts'), 'export const second = "alpha";'); + await new CodebaseIndexer({ rootPath: root, config: { skipEmbedding: true } }).index(); +} + +it('serializes real processes before baseline reads while searches use the committed generation', async () => { + await fixture(); + await fs.writeFile(path.join(root, 'a.ts'), 'export const first = "beta";'); + const first = writer(true); + await first.wait('staged'); + // A live writer never blocks admission to the previous complete generation. + expect((await new CodebaseSearcher(root).search('second')).length).toBeGreaterThan(0); + await fs.writeFile(path.join(root, 'b.ts'), 'export const second = "beta";'); + const second = writer(false); + await second.wait('queued'); + first.child.send({ release: true }); + await first.wait('complete'); + const completed = await second.wait('complete'); + expect(completed.incremental).toEqual({ added: 0, changed: 1, deleted: 0, unchanged: 1 }); + await validateIndexArtifacts(root, await readIndexMeta(root)); + expect((await new CodebaseSearcher(root).search('beta')).length).toBeGreaterThan(0); +}, 30_000); + +it('recovers a killed writer without stealing a live writer lease', async () => { + await fixture(); + await fs.writeFile(path.join(root, 'a.ts'), 'export const first = "beta";'); + const first = writer(true); + await first.wait('staged'); + const second = writer(false); + await second.wait('queued'); + const exited = new Promise((resolve) => first.child.once('exit', () => resolve())); + first.child.kill('SIGKILL'); + await exited; + const completed = await second.wait('complete'); + expect(completed.incremental).toEqual({ added: 0, changed: 1, deleted: 0, unchanged: 1 }); + await validateIndexArtifacts(root, await readIndexMeta(root)); +}, 30_000); diff --git a/tests/indexer-analyzer-hints.test.ts b/tests/indexer-analyzer-hints.test.ts index c7a1adf..ad75865 100644 --- a/tests/indexer-analyzer-hints.test.ts +++ b/tests/indexer-analyzer-hints.test.ts @@ -1,3 +1,4 @@ +import { resolveIndexArtifact } from '../src/core/index-generation.js'; import { afterEach, beforeAll, describe, expect, it, vi } from 'vitest'; import { promises as fs } from 'fs'; import os from 'os'; @@ -17,7 +18,7 @@ type IndexChunk = { }; async function readIndexedChunks(rootPath: string): Promise { - const indexPath = path.join(rootPath, CODEBASE_CONTEXT_DIRNAME, KEYWORD_INDEX_FILENAME); + const indexPath = await resolveIndexArtifact(rootPath, KEYWORD_INDEX_FILENAME); const indexRaw = JSON.parse(await fs.readFile(indexPath, 'utf-8')) as Record; if (Array.isArray(indexRaw)) { diff --git a/tests/indexer-exclude-patterns.test.ts b/tests/indexer-exclude-patterns.test.ts index a147b71..fd2fff4 100644 --- a/tests/indexer-exclude-patterns.test.ts +++ b/tests/indexer-exclude-patterns.test.ts @@ -1,3 +1,4 @@ +import { resolveIndexArtifact } from '../src/core/index-generation.js'; import { afterEach, describe, expect, it } from 'vitest'; import { promises as fs } from 'fs'; import os from 'os'; @@ -58,7 +59,7 @@ describe('Indexer exclude patterns — nested directories', () => { await indexer.index(); - const indexPath = path.join(tempDir, CODEBASE_CONTEXT_DIRNAME, KEYWORD_INDEX_FILENAME); + const indexPath = await resolveIndexArtifact(tempDir, KEYWORD_INDEX_FILENAME); const indexRaw = JSON.parse(await fs.readFile(indexPath, 'utf-8')) as Record; let chunks: Array<{ filePath: string }>; diff --git a/tests/indexer-large-file-skip.test.ts b/tests/indexer-large-file-skip.test.ts index e54e513..4fc03c3 100644 --- a/tests/indexer-large-file-skip.test.ts +++ b/tests/indexer-large-file-skip.test.ts @@ -1,3 +1,4 @@ +import { resolveIndexArtifact } from '../src/core/index-generation.js'; import { afterEach, beforeEach, describe, expect, it } from 'vitest'; import { promises as fs } from 'fs'; import os from 'os'; @@ -46,7 +47,7 @@ describe('Indexer large file skip regression', () => { await indexer.index(); - const indexPath = path.join(tempDir, CODEBASE_CONTEXT_DIRNAME, KEYWORD_INDEX_FILENAME); + const indexPath = await resolveIndexArtifact(tempDir, KEYWORD_INDEX_FILENAME); const indexRaw = JSON.parse(await fs.readFile(indexPath, 'utf-8')) as any; const chunks = ( Array.isArray(indexRaw) ? indexRaw : Array.isArray(indexRaw?.chunks) ? indexRaw.chunks : [] @@ -75,7 +76,7 @@ describe('Indexer large file skip regression', () => { await indexer.index(); - const indexPath = path.join(tempDir, CODEBASE_CONTEXT_DIRNAME, KEYWORD_INDEX_FILENAME); + const indexPath = await resolveIndexArtifact(tempDir, KEYWORD_INDEX_FILENAME); const indexRaw = JSON.parse(await fs.readFile(indexPath, 'utf-8')) as { chunks?: unknown[]; }; diff --git a/tests/indexer-model-mode.test.ts b/tests/indexer-model-mode.test.ts new file mode 100644 index 0000000..de0c9b9 --- /dev/null +++ b/tests/indexer-model-mode.test.ts @@ -0,0 +1,444 @@ +import { resolveIndexArtifact } from '../src/core/index-generation.js'; +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { promises as fs } from 'node:fs'; +import { writeFileSync } from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import structuralFixture from './fixtures/structural-context-regressions.json'; +import type { CodeChunk } from '../src/types/index.js'; +import { + resolveSkipEmbedding, + checkEmbeddingMismatch, + type IndexMeta +} from '../src/core/index-meta.js'; +import { + CODEBASE_CONTEXT_DIRNAME, + INDEX_FORMAT_VERSION, + INDEX_META_FILENAME, + INDEX_META_VERSION, + HEALTH_FILENAME, + INTELLIGENCE_FILENAME, + KEYWORD_INDEX_FILENAME, + MANIFEST_FILENAME, + RELATIONSHIPS_FILENAME, + VECTOR_DB_DIRNAME +} from '../src/constants/codebase-context.js'; + +const { embeddingInit, storageInit, embeddingRuntimeAssert, storageRuntimeAssert, rerankInit } = + vi.hoisted(() => ({ + embeddingInit: vi.fn(), + storageInit: vi.fn(), + embeddingRuntimeAssert: vi.fn(), + storageRuntimeAssert: vi.fn(), + rerankInit: vi.fn(async (_query: string, results: unknown) => results) + })); +vi.mock('../src/embeddings/index.js', async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + assertEmbeddingRuntimeAvailable: embeddingRuntimeAssert, + getEmbeddingProvider: embeddingInit + }; +}); +vi.mock('../src/storage/index.js', async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + assertStorageRuntimeAvailable: storageRuntimeAssert, + getStorageProvider: storageInit + }; +}); +vi.mock('../src/core/reranker.js', () => ({ + rerank: rerankInit, + getRerankerStatus: vi.fn(() => 'fallback') +})); + +import { CodebaseSearcher } from '../src/core/search.js'; +import { CodebaseIndexer } from '../src/core/indexer.js'; + +describe('persisted embedding mode', () => { + let rootPath: string | undefined; + + async function retainLegacyFormat(legacyRoot: string, formatVersion: number): Promise { + const contextDir = path.join(legacyRoot, CODEBASE_CONTEXT_DIRNAME); + const metaPath = path.join(contextDir, INDEX_META_FILENAME); + const meta = JSON.parse(await fs.readFile(metaPath, 'utf8')) as IndexMeta; + meta.formatVersion = formatVersion; + await fs.writeFile(metaPath, JSON.stringify(meta)); + for (const filename of [ + KEYWORD_INDEX_FILENAME, + INTELLIGENCE_FILENAME, + RELATIONSHIPS_FILENAME, + HEALTH_FILENAME, + MANIFEST_FILENAME + ]) { + const artifactPath = await resolveIndexArtifact(legacyRoot, filename, meta); + const artifact = JSON.parse(await fs.readFile(artifactPath, 'utf8')) as { + header: { formatVersion: number }; + }; + artifact.header.formatVersion = formatVersion; + await fs.writeFile(artifactPath, JSON.stringify(artifact)); + } + await fs.writeFile( + path.join( + await resolveIndexArtifact(legacyRoot, VECTOR_DB_DIRNAME, meta), + 'index-build.json' + ), + JSON.stringify({ buildId: meta.buildId, formatVersion }) + ); + return meta; + } + + afterEach(async () => { + if (rootPath) await fs.rm(rootPath, { recursive: true, force: true }); + rootPath = undefined; + }); + + it('defaults a genuinely fresh folder to model-free and honors explicit semantic opt-in', async () => { + rootPath = await fs.mkdtemp(path.join(os.tmpdir(), 'cbc-mode-fresh-')); + await expect(resolveSkipEmbedding(rootPath)).resolves.toBe(true); + await expect(resolveSkipEmbedding(rootPath, false)).resolves.toBe(false); + }); + + it.each([1, 2, 3])( + 'preserves a format-%i recorded semantic or model-free mode when no choice is supplied', + async (formatVersion) => { + rootPath = await fs.mkdtemp(path.join(os.tmpdir(), 'cbc-mode-existing-')); + await fs.mkdir(path.join(rootPath, '.codebase-context'), { recursive: true }); + const writeMeta = async (embeddingEnabled: boolean): Promise => { + const meta: IndexMeta = { + metaVersion: 1, + formatVersion, + buildId: 'build-1', + generatedAt: new Date().toISOString(), + toolVersion: '1.0.0', + artifacts: { + keywordIndex: { path: 'index.json' }, + vectorDb: { + path: 'vector-db', + provider: 'lancedb', + embeddingProvider: 'transformers', + embeddingModel: 'Xenova/bge-small-en-v1.5', + embeddingEnabled + } + } + }; + await fs.writeFile( + path.join(rootPath!, '.codebase-context', 'index-meta.json'), + JSON.stringify(meta) + ); + }; + + await writeMeta(false); + await expect(resolveSkipEmbedding(rootPath)).resolves.toBe(true); + await expect(resolveSkipEmbedding(rootPath, false)).resolves.toBe(false); + await writeMeta(true); + await expect(resolveSkipEmbedding(rootPath)).resolves.toBe(false); + } + ); + + it('fails closed when index artifacts exist but mode metadata cannot be read', async () => { + rootPath = await fs.mkdtemp(path.join(os.tmpdir(), 'cbc-mode-corrupt-')); + const contextDir = path.join(rootPath, '.codebase-context'); + await fs.mkdir(contextDir); + await fs.writeFile(await resolveIndexArtifact(path.dirname(contextDir), 'index.json'), '{}'); + await expect(resolveSkipEmbedding(rootPath)).rejects.toThrow(/Cannot recover embedding mode/); + await expect(resolveSkipEmbedding(rootPath, true)).resolves.toBe(true); + }); + + it('fails closed on unknown state under the index directory but accepts memory-only state', async () => { + rootPath = await fs.mkdtemp(path.join(os.tmpdir(), 'cbc-mode-unknown-')); + const contextDir = path.join(rootPath, CODEBASE_CONTEXT_DIRNAME); + await fs.mkdir(contextDir); + await fs.writeFile(path.join(contextDir, 'unrecognized-state.json'), '{}'); + await expect(resolveSkipEmbedding(rootPath)).rejects.toThrow(/Cannot recover embedding mode/); + + await fs.rm(contextDir, { recursive: true, force: true }); + await fs.mkdir(contextDir); + await fs.writeFile(path.join(contextDir, 'memory.json'), '{}'); + await expect(resolveSkipEmbedding(rootPath)).resolves.toBe(true); + }); + + it('forces rebuilds when enabled state changes, while mode-free provider changes do not matter', () => { + const meta = { + metaVersion: 1, + formatVersion: 1, + buildId: 'build-1', + generatedAt: new Date().toISOString(), + toolVersion: '1.0.0', + artifacts: { + keywordIndex: { path: 'index.json' }, + vectorDb: { + path: 'vector-db', + provider: 'lancedb', + embeddingEnabled: false + } + } + } satisfies IndexMeta; + expect(checkEmbeddingMismatch(meta, 'openai', 'some-model', false)).toBe(false); + expect(checkEmbeddingMismatch(meta, 'transformers', 'some-model', true)).toBe(true); + }); + + it('initializes keyword search without embedding or vector storage in model-free mode', async () => { + rootPath = await fs.mkdtemp(path.join(os.tmpdir(), 'cbc-search-model-free-')); + await fs.writeFile( + path.join(rootPath, 'source.ts'), + 'export const source = "model-free source";\n' + ); + // A positive reader case needs a real build-bound source manifest, not an + // empty hand-written cache with no manifest path/header. + const stats = await new CodebaseIndexer({ rootPath }).index(); + expect(stats.indexedFiles).toBe(1); + + const searcher = new CodebaseSearcher(rootPath); + await searcher.initialize(); + const results = await searcher.search('source', 5, undefined, { + useSemanticSearch: true, + enableReranker: true + }); + expect(results[0]?.snippet).toContain('model-free source'); + expect(embeddingInit).not.toHaveBeenCalled(); + expect(storageInit).not.toHaveBeenCalled(); + expect(rerankInit).not.toHaveBeenCalled(); + }); + + it('indexes real source by default and preserves the model-free mode on rebuild', async () => { + rootPath = await fs.mkdtemp(path.join(os.tmpdir(), 'cbc-index-model-free-')); + const sourcePath = path.join(rootPath, 'source.ts'); + await fs.writeFile(sourcePath, 'export const sourcePreserved = "original source";\n'); + + const first = new CodebaseIndexer({ rootPath }); + const firstStats = await first.index(); + expect(firstStats.errors).toEqual([]); + + const contextDir = path.join(rootPath, CODEBASE_CONTEXT_DIRNAME); + const firstMeta = JSON.parse( + await fs.readFile(path.join(contextDir, INDEX_META_FILENAME), 'utf8') + ) as { artifacts: { vectorDb: { embeddingEnabled: boolean } } }; + expect(firstMeta.artifacts.vectorDb.embeddingEnabled).toBe(false); + expect(embeddingRuntimeAssert).not.toHaveBeenCalled(); + expect(storageRuntimeAssert).not.toHaveBeenCalled(); + expect(embeddingInit).not.toHaveBeenCalled(); + const indexPath = await resolveIndexArtifact(path.dirname(contextDir), KEYWORD_INDEX_FILENAME); + const initialIndex = await fs.readFile(indexPath, 'utf8'); + expect(initialIndex).toContain('original source'); + + await fs.writeFile(sourcePath, 'export const sourcePreserved = "updated source";\n'); + const second = new CodebaseIndexer({ rootPath }); + const secondStats = await second.index(); + expect(secondStats.errors).toEqual([]); + const secondMeta = JSON.parse( + await fs.readFile(path.join(contextDir, INDEX_META_FILENAME), 'utf8') + ) as { artifacts: { vectorDb: { embeddingEnabled: boolean } } }; + expect(secondMeta.artifacts.vectorDb.embeddingEnabled).toBe(false); + const rebuiltIndex = await fs.readFile( + await resolveIndexArtifact(rootPath, KEYWORD_INDEX_FILENAME), + 'utf8' + ); + expect(rebuiltIndex).toContain('updated source'); + expect(rebuiltIndex).not.toContain('original source'); + expect(embeddingInit).not.toHaveBeenCalled(); + }, 60000); + + it('persists exact raw AST bodies, keeps same-line siblings separate, and preserves prefix-like source comments', async () => { + rootPath = await fs.mkdtemp(path.join(os.tmpdir(), 'cbc-raw-ast-')); + const fixture = structuralFixture.cases.find( + (item) => item.id === 'same-line-property-callback-bodies-stay-distinct' + ); + expect(fixture).toBeDefined(); + await fs.writeFile(path.join(rootPath, 'handlers.js'), fixture!.source); + const commented = 'function consume() {\n// consume :: ()\n consumeMessage();\n}\n'; + await fs.writeFile(path.join(rootPath, 'consume.js'), commented); + await new CodebaseIndexer({ rootPath }).index(); + const persisted = JSON.parse( + await fs.readFile(await resolveIndexArtifact(rootPath, KEYWORD_INDEX_FILENAME), 'utf8') + ) as { chunks: CodeChunk[] }; + for (const fact of fixture!.sourceSymbols) { + const own = persisted.chunks.filter( + (chunk) => JSON.stringify(chunk.metadata.symbolPath) === JSON.stringify(fact.ownerChain) + ); + expect(own.length).toBeGreaterThan(0); + for (const marker of fact.bodyMarkers) + expect(own.some((chunk) => chunk.content.includes(marker))).toBe(true); + for (const chunk of own) { + expect(fixture!.source).toContain(chunk.content); + if ('forbiddenSiblingBodyMarkers' in fact) { + for (const marker of fact.forbiddenSiblingBodyMarkers ?? []) + expect(chunk.content).not.toContain(marker); + } + } + } + const consume = persisted.chunks.find((chunk) => chunk.metadata.symbolName === 'consume'); + expect(consume).toBeDefined(); + expect(consume!.content).toBe(commented.trimEnd()); + expect(consume!.content.match(/\/\/ consume :: \(\)/g)).toHaveLength(1); + }, 60000); + + it('persists concise descriptor implementations under their nearest static owner', async () => { + rootPath = await fs.mkdtemp(path.join(os.tmpdir(), 'cbc-concise-descriptor-')); + const source = + 'function createHandlers() { return { shutdown: { value() { closeConnections(); } }, state: { get() { return readState(); }, set: next => writeState(next) }, sibling: { value() { reloadConnections(); } } }; }'; + await fs.writeFile(path.join(rootPath, 'handlers.js'), source); + await new CodebaseIndexer({ rootPath }).index(); + const persisted = JSON.parse( + await fs.readFile(await resolveIndexArtifact(rootPath, KEYWORD_INDEX_FILENAME), 'utf8') + ) as { chunks: CodeChunk[] }; + for (const fact of [ + { + name: 'shutdown', + body: 'shutdown: { value() { closeConnections(); } }', + forbidden: ['readState()', 'reloadConnections()'] + }, + { + name: 'state', + body: 'state: { get() { return readState(); }, set: next => writeState(next) }', + forbidden: ['closeConnections()', 'reloadConnections()'] + } + ]) { + const own = persisted.chunks.filter((chunk) => chunk.metadata.symbolName === fact.name); + expect(own).toHaveLength(1); + expect(own[0].content).toBe(fact.body); + expect(own[0].metadata.symbolPath).toEqual(['createHandlers', fact.name]); + for (const marker of fact.forbidden) expect(own[0].content).not.toContain(marker); + } + }, 60000); + + it.each([1, 2, 3])( + 'rebuilds an otherwise consistent format-%i index on an unchanged incremental pass', + async (formatVersion) => { + rootPath = await fs.mkdtemp(path.join(os.tmpdir(), 'cbc-format-recovery-')); + await fs.writeFile( + path.join(rootPath, 'source.ts'), + 'export function handle() { return dispatch(); }\n' + ); + await new CodebaseIndexer({ rootPath }).index(); + const contextDir = path.join(rootPath, CODEBASE_CONTEXT_DIRNAME); + const metaPath = path.join(contextDir, INDEX_META_FILENAME); + const oldMeta = await retainLegacyFormat(rootPath, formatVersion); + await expect(resolveSkipEmbedding(rootPath)).resolves.toBe(true); + const stats = await new CodebaseIndexer({ rootPath, incrementalOnly: true }).index(); + expect(stats.errors).toEqual([]); + const rebuilt = JSON.parse(await fs.readFile(metaPath, 'utf8')) as IndexMeta; + expect(rebuilt.formatVersion).toBe(INDEX_FORMAT_VERSION); + expect(rebuilt.buildId).not.toBe(oldMeta.buildId); + expect(rebuilt.artifacts.vectorDb.embeddingEnabled).toBe(false); + }, + 60000 + ); + + it.each([1, 2, 3])( + 'preserves a format-%i successful index when semantic migration opt-in cannot initialize', + async (formatVersion) => { + rootPath = await fs.mkdtemp(path.join(os.tmpdir(), 'cbc-format-atomic-')); + await fs.writeFile( + path.join(rootPath, 'source.ts'), + 'export function handle() { return dispatch(); }\n' + ); + await new CodebaseIndexer({ rootPath }).index(); + const contextDir = path.join(rootPath, CODEBASE_CONTEXT_DIRNAME); + const metaPath = path.join(contextDir, INDEX_META_FILENAME); + const legacyMeta = await retainLegacyFormat(rootPath, formatVersion); + const originalMeta = await fs.readFile(metaPath, 'utf8'); + const originalIndex = await fs.readFile( + await resolveIndexArtifact(rootPath, KEYWORD_INDEX_FILENAME, legacyMeta), + 'utf8' + ); + await expect(resolveSkipEmbedding(rootPath)).resolves.toBe(true); + const unavailable = new Error('migration semantic runtime unavailable'); + embeddingRuntimeAssert.mockImplementationOnce(() => { + throw unavailable; + }); + await expect( + new CodebaseIndexer({ + rootPath, + incrementalOnly: true, + config: { skipEmbedding: false } + }).index() + ).rejects.toBe(unavailable); + expect(await fs.readFile(metaPath, 'utf8')).toBe(originalMeta); + expect( + await fs.readFile( + await resolveIndexArtifact(rootPath, KEYWORD_INDEX_FILENAME, legacyMeta), + 'utf8' + ) + ).toBe(originalIndex); + }, + 60000 + ); + + it('checks semantic dependencies before creating or replacing index artifacts', async () => { + rootPath = await fs.mkdtemp(path.join(os.tmpdir(), 'cbc-semantic-runtime-')); + await fs.writeFile(path.join(rootPath, 'source.ts'), 'export const source = 1;\n'); + const missingRuntime = new Error('semantic dependency unavailable'); + embeddingRuntimeAssert.mockImplementationOnce(() => { + throw missingRuntime; + }); + + const indexer = new CodebaseIndexer({ rootPath, config: { skipEmbedding: false } }); + await expect(indexer.index()).rejects.toBe(missingRuntime); + // Writer coordination exists before baseline reads; no index artifact is created. + expect(await fs.readdir(path.join(rootPath, CODEBASE_CONTEXT_DIRNAME))).toEqual(['.writers']); + expect(await fs.readdir(path.join(rootPath, CODEBASE_CONTEXT_DIRNAME, '.writers'))).toEqual([]); + expect(storageRuntimeAssert).not.toHaveBeenCalled(); + }); + + it('binds the manifest digest to the exact bytes analyzed, even if the file changes during analysis', async () => { + // Frozen S2 regression: analyzer receives snapshot A, source changes to B before publish. + // The manifest must not certify stale chunks as if they came from B. + const { analyzerRegistry } = await import('../src/core/analyzer-registry.js'); + rootPath = await fs.mkdtemp(path.join(os.tmpdir(), 'cbc-manifest-snapshot-')); + const sourcePath = path.join(rootPath, 'snapshot.ts'); + const analyzed = 'export const snapshotTarget = "A";\n'; + const changed = 'export const replacementTarget = "B";\n'; + await fs.writeFile(sourcePath, analyzed); + const original = analyzerRegistry.analyzeFile.bind(analyzerRegistry); + const spy = vi.spyOn(analyzerRegistry, 'analyzeFile').mockImplementation(async (...args) => { + const result = await original(...args); + if (args[0] === sourcePath) await fs.writeFile(sourcePath, changed); + return result; + }); + try { + await new CodebaseIndexer({ rootPath }).index(); + } finally { + spy.mockRestore(); + } + const manifest = JSON.parse( + await fs.readFile(await resolveIndexArtifact(rootPath, MANIFEST_FILENAME), 'utf8') + ) as { files: Record }; + const expected = (await import('node:crypto')) + .createHash('sha256') + .update(analyzed) + .digest('hex') + .slice(0, 16); + expect(manifest.files['snapshot.ts']).toBe(expected); + }); + + it('fails an incremental build when a pre-hashed unchanged file drifts before analysis', async () => { + rootPath = await fs.mkdtemp(path.join(os.tmpdir(), 'cbc-incremental-snapshot-')); + const triggerPath = path.join(rootPath, 'a-trigger.ts'); + const unchangedPath = path.join(rootPath, 'z-unchanged.ts'); + await fs.writeFile(triggerPath, 'export const trigger = 1;\n'); + await fs.writeFile(unchangedPath, 'export const stableTarget = "old";\n'); + await new CodebaseIndexer({ rootPath }).index(); + const metaPath = path.join(rootPath, CODEBASE_CONTEXT_DIRNAME, INDEX_META_FILENAME); + const priorMeta = JSON.parse(await fs.readFile(metaPath, 'utf8')) as { buildId: string }; + + await fs.writeFile(triggerPath, 'export const trigger = 2;\n'); + let mutated = false; + await expect( + new CodebaseIndexer({ + rootPath, + incrementalOnly: true, + onProgress: (progress) => { + if (progress.phase === 'analyzing' && !mutated) { + mutated = true; + writeFileSync(unchangedPath, 'export const replacementTarget = "new";\n'); + } + } + }).index() + ).rejects.toThrow(/Source changed during incremental scan/); + expect(mutated).toBe(true); + const currentMeta = JSON.parse(await fs.readFile(metaPath, 'utf8')) as { buildId: string }; + expect(currentMeta.buildId).toBe(priorMeta.buildId); + }, 60000); +}); diff --git a/tests/indexer-preview.test.ts b/tests/indexer-preview.test.ts new file mode 100644 index 0000000..140eb72 --- /dev/null +++ b/tests/indexer-preview.test.ts @@ -0,0 +1,327 @@ +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { promises as fs } from 'node:fs'; +import { renameSync, unlinkSync } from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { CodebaseIndexer, selectFilesForIndex } from '../src/core/indexer.js'; +import { handleCliCommand } from '../src/cli.js'; +import { resolveIndexArtifact } from '../src/core/index-generation.js'; + +describe('index preview enumeration', () => { + let root: string; + + afterEach(async () => { + if (root) await fs.rm(root, { recursive: true, force: true }); + }); + + it.each(['reindex', 'map'])( + 'applies configured exclusions and analyzer hints to the actual CLI %s index', + async (command) => { + root = await fs.mkdtemp(path.join(os.tmpdir(), 'cbc-cli-project-options-')); + await fs.mkdir(path.join(root, 'src')); + await fs.mkdir(path.join(root, 'private')); + await fs.writeFile(path.join(root, 'src', 'keep.customcode'), 'export const keep = 1;'); + await fs.writeFile(path.join(root, 'private', 'secret.ts'), 'export const secret = 1;'); + const configPath = path.join(root, 'server-config.json'); + await fs.writeFile( + configPath, + JSON.stringify({ + projects: [ + { + root, + excludePatterns: ['**/private/**'], + analyzerHints: { analyzer: 'generic', extensions: ['.customcode'] } + } + ] + }) + ); + const previousConfig = process.env.CODEBASE_CONTEXT_CONFIG_PATH; + const previousRoot = process.env.CODEBASE_ROOT; + const output = vi.spyOn(console, 'log').mockImplementation(() => undefined); + process.env.CODEBASE_CONTEXT_CONFIG_PATH = configPath; + process.env.CODEBASE_ROOT = root; + try { + await handleCliCommand( + command === 'reindex' ? ['reindex', '--no-embeddings'] : ['map', '--json'] + ); + const artifact: { chunks: Array<{ relativePath: string }> } = JSON.parse( + await fs.readFile(await resolveIndexArtifact(root, 'index.json'), 'utf8') + ); + const paths = artifact.chunks.map((chunk) => chunk.relativePath); + expect({ + privateIncluded: paths.includes('private/secret.ts'), + customIncluded: paths.includes('src/keep.customcode') + }).toEqual({ privateIncluded: false, customIncluded: true }); + } finally { + output.mockRestore(); + if (previousConfig === undefined) delete process.env.CODEBASE_CONTEXT_CONFIG_PATH; + else process.env.CODEBASE_CONTEXT_CONFIG_PATH = previousConfig; + if (previousRoot === undefined) delete process.env.CODEBASE_ROOT; + else process.env.CODEBASE_ROOT = previousRoot; + } + } + ); + + it('reports only files the indexer will select and explains observed skips', async () => { + root = await fs.mkdtemp(path.join(os.tmpdir(), 'indexer-preview-')); + await fs.mkdir(path.join(root, 'packages', 'api'), { recursive: true }); + await fs.mkdir(path.join(root, 'packages', 'web', 'dist'), { recursive: true }); + await fs.writeFile(path.join(root, '.gitignore'), 'ignored.ts\n'); + await fs.writeFile(path.join(root, 'packages', 'api', 'keep.ts'), 'export const keep = 1;\n'); + await fs.writeFile(path.join(root, 'ignored.ts'), 'export const ignored = 1;\n'); + await fs.writeFile(path.join(root, 'large.ts'), `export const large = '${'x'.repeat(80)}';\n`); + await fs.writeFile( + path.join(root, 'packages', 'web', 'dist', 'bundle.ts'), + 'export const bundle = 1;\n' + ); + await fs.writeFile(path.join(root, 'notes.txt'), 'not a supported source file'); + + const preview = await new CodebaseIndexer({ + rootPath: root, + config: { skipEmbedding: true, parsing: { maxFileSize: 48 } } + }).previewFiles(); + + expect(preview.rootPath).toBe(root); + expect(preview.eligibleFiles).toEqual(['packages/api/keep.ts']); + expect(preview.skippedFiles).toEqual([ + { path: 'ignored.ts', reason: 'ignored by .gitignore' }, + { path: 'large.ts', reason: 'exceeds configured maximum file size' } + ]); + expect(preview.excludedPatterns).toContain('**/dist/**'); + expect(preview.excludedPatterns.some((pattern) => pattern.includes('dist/bundle.ts'))).toBe( + false + ); + await expect(fs.access(path.join(root, '.codebase-context'))).rejects.toMatchObject({ + code: 'ENOENT' + }); + }); + + it('honors project config exclusions in the Codex init preview', async () => { + root = await fs.mkdtemp(path.join(os.tmpdir(), 'indexer-preview-config-')); + await fs.mkdir(path.join(root, 'src'), { recursive: true }); + await fs.mkdir(path.join(root, 'private'), { recursive: true }); + await fs.writeFile(path.join(root, 'src', 'keep.ts'), 'export const keep = 1;\n'); + await fs.writeFile(path.join(root, 'private', 'secret.ts'), 'export const secret = 1;\n'); + const configPath = path.join(root, 'server-config.json'); + await fs.writeFile( + configPath, + JSON.stringify({ projects: [{ root, excludePatterns: ['**/private/**'] }] }) + ); + const previousConfigPath = process.env.CODEBASE_CONTEXT_CONFIG_PATH; + process.env.CODEBASE_CONTEXT_CONFIG_PATH = configPath; + const output = vi.spyOn(console, 'log').mockImplementation(() => undefined); + try { + await handleCliCommand(['init', '--client', 'codex', '--root', root, '--preview']); + const rendered = output.mock.calls.flat().join('\n'); + expect(rendered).toContain('src/keep.ts'); + expect(rendered).not.toContain('private/secret.ts'); + expect(rendered).toContain('**/private/**'); + } finally { + output.mockRestore(); + if (previousConfigPath === undefined) delete process.env.CODEBASE_CONTEXT_CONFIG_PATH; + else process.env.CODEBASE_CONTEXT_CONFIG_PATH = previousConfigPath; + } + }); + + it('reports overflow with a count and bounded examples, without materializing an overflow list', () => { + const files = Array.from({ length: 10025 }, (_, index) => `file-${index}.ts`); + + const selection = selectFilesForIndex(files); + + expect(selection.selected).toHaveLength(10000); + expect(selection.overLimitCount).toBe(25); + expect(selection.overLimitSamples).toHaveLength(20); + expect(selection.overLimitSamples[0]).toBe('file-10000.ts'); + }); + + it('rechecks gitignore after preview instead of indexing a stale selection', async () => { + root = await fs.mkdtemp(path.join(os.tmpdir(), 'indexer-preview-stale-')); + await fs.writeFile(path.join(root, 'keep.ts'), 'export const keep = 1;\n'); + await fs.writeFile(path.join(root, 'stale.ts'), 'export const stale = 1;\n'); + const indexer = new CodebaseIndexer({ + rootPath: root, + config: { skipEmbedding: true } + }); + + const preview = await indexer.previewFiles(); + expect(preview.eligibleFiles).toContain('stale.ts'); + await fs.writeFile(path.join(root, '.gitignore'), 'stale.ts\n'); + + const stats = await indexer.index(); + + expect(stats.totalFiles).toBe(1); + }); + + it('does not enumerate source symlinks whose canonical target is outside the root', async () => { + root = await fs.mkdtemp(path.join(os.tmpdir(), 'indexer-preview-symlink-')); + const outsideRoot = await fs.mkdtemp(path.join(os.tmpdir(), 'indexer-preview-outside-')); + try { + const outsideFile = path.join(outsideRoot, 'outside.ts'); + const linkPath = path.join(root, 'linked.ts'); + await fs.writeFile(outsideFile, 'export const outside = true;\n'); + try { + await fs.symlink(outsideFile, linkPath, 'file'); + } catch (error) { + const code = (error as NodeJS.ErrnoException).code; + if (process.platform === 'win32' && (code === 'EPERM' || code === 'EACCES')) { + throw new Error( + `Symlink containment oracle could not run: Windows denied symlink creation (${code}).` + ); + } + throw error; + } + + const preview = await new CodebaseIndexer({ rootPath: root }).previewFiles(); + + expect(preview.eligibleFiles).not.toContain('linked.ts'); + expect(preview.skippedFiles).toContainEqual({ + path: 'linked.ts', + reason: 'source resolves outside selected root' + }); + } finally { + await fs.rm(outsideRoot, { recursive: true, force: true }); + } + }); + + it('does not let an in-root alias bypass a configured exclusion', async () => { + root = await fs.mkdtemp(path.join(os.tmpdir(), 'indexer-preview-inroot-alias-')); + const privateDir = path.join(root, 'private'); + await fs.mkdir(privateDir, { recursive: true }); + const canonicalFile = path.join(privateDir, 'secret.ts'); + const aliasPath = path.join(root, 'alias.ts'); + await fs.writeFile(canonicalFile, 'export const secret = true;\n'); + try { + await fs.symlink(canonicalFile, aliasPath, 'file'); + } catch (error) { + const code = (error as NodeJS.ErrnoException).code; + if (process.platform === 'win32' && (code === 'EPERM' || code === 'EACCES')) { + throw new Error( + `In-root alias oracle could not run: Windows denied symlink creation (${code}).` + ); + } + throw error; + } + + const preview = await new CodebaseIndexer({ + rootPath: root, + config: { exclude: ['**/private/**'] } + }).previewFiles(); + + expect(preview.eligibleFiles).not.toContain('alias.ts'); + expect(preview.eligibleFiles).not.toContain('private/secret.ts'); + expect(preview.skippedFiles).toContainEqual({ + path: 'alias.ts', + reason: 'source alias is not indexed; use its canonical path' + }); + }); + + it('counts one canonical source when multiple in-root aliases point to it', async () => { + root = await fs.mkdtemp(path.join(os.tmpdir(), 'indexer-preview-aliases-')); + const canonicalFile = path.join(root, 'real.ts'); + await fs.writeFile(canonicalFile, 'export const real = true;\n'); + for (const name of ['first.ts', 'second.ts']) { + try { + await fs.symlink(canonicalFile, path.join(root, name), 'file'); + } catch (error) { + const code = (error as NodeJS.ErrnoException).code; + if (process.platform === 'win32' && (code === 'EPERM' || code === 'EACCES')) { + throw new Error( + `Alias deduplication oracle could not run: Windows denied symlink creation (${code}).` + ); + } + throw error; + } + } + + const preview = await new CodebaseIndexer({ rootPath: root }).previewFiles(); + + expect(preview.eligibleFiles).toEqual(['real.ts']); + expect(preview.skippedFileCount).toBe(2); + expect(preview.skippedFiles.map((file) => file.path).sort()).toEqual(['first.ts', 'second.ts']); + }); + + it('rechecks containment immediately before reading a source replaced by an outside symlink', async () => { + root = await fs.mkdtemp(path.join(os.tmpdir(), 'indexer-read-symlink-')); + const outsideRoot = await fs.mkdtemp(path.join(os.tmpdir(), 'indexer-read-outside-')); + const sourcePath = path.join(root, 'source.ts'); + const replacementPath = path.join(root, 'replacement-link'); + try { + const outsideFile = path.join(outsideRoot, 'outside.ts'); + await fs.writeFile(sourcePath, 'export const inside = true;\n'); + await fs.writeFile(outsideFile, 'export const outside = true;\n'); + try { + await fs.symlink(outsideFile, replacementPath, 'file'); + } catch (error) { + const code = (error as NodeJS.ErrnoException).code; + if (process.platform === 'win32' && (code === 'EPERM' || code === 'EACCES')) { + throw new Error( + `Pre-read containment oracle could not run: Windows denied symlink creation (${code}).` + ); + } + throw error; + } + + let replaced = false; + const indexer = new CodebaseIndexer({ + rootPath: root, + config: { skipEmbedding: true }, + onProgress: ({ phase }) => { + if (phase !== 'analyzing' || replaced) return; + unlinkSync(sourcePath); + renameSync(replacementPath, sourcePath); + replaced = true; + } + }); + + const stats = await indexer.index(); + + expect(replaced).toBe(true); + expect(stats.indexedFiles).toBe(0); + expect( + stats.errors.some((error) => error.error.includes('Source resolves outside selected root')) + ).toBe(true); + } finally { + await fs.rm(outsideRoot, { recursive: true, force: true }); + } + }); + + it('rejects a source replaced before read by an alias to another in-root file', async () => { + root = await fs.mkdtemp(path.join(os.tmpdir(), 'indexer-read-inroot-alias-')); + const sourcePath = path.join(root, 'source.ts'); + const canonicalPath = path.join(root, 'canonical.ts'); + const replacementPath = path.join(root, 'replacement-link'); + await fs.writeFile(sourcePath, 'export const original = true;\n'); + await fs.writeFile(canonicalPath, 'export const canonical = true;\n'); + try { + await fs.symlink(canonicalPath, replacementPath, 'file'); + } catch (error) { + const code = (error as NodeJS.ErrnoException).code; + if (process.platform === 'win32' && (code === 'EPERM' || code === 'EACCES')) { + throw new Error( + `In-root read-alias oracle could not run: Windows denied symlink creation (${code}).` + ); + } + throw error; + } + + let replaced = false; + const indexer = new CodebaseIndexer({ + rootPath: root, + config: { skipEmbedding: true }, + onProgress: ({ phase }) => { + if (phase !== 'analyzing' || replaced) return; + unlinkSync(sourcePath); + renameSync(replacementPath, sourcePath); + replaced = true; + } + }); + + const stats = await indexer.index(); + + expect(replaced).toBe(true); + expect(stats.indexedFiles).toBe(1); + expect(stats.errors.some((error) => error.error.includes('Source alias is not indexed'))).toBe( + true + ); + }); +}); diff --git a/tests/internal-file-graph-serialization.test.ts b/tests/internal-file-graph-serialization.test.ts index 1f773a6..8432ba0 100644 --- a/tests/internal-file-graph-serialization.test.ts +++ b/tests/internal-file-graph-serialization.test.ts @@ -4,6 +4,28 @@ import os from 'os'; import { InternalFileGraph } from '../src/utils/usage-tracker.js'; describe('InternalFileGraph serialization', () => { + it.each(['not-indexed', 'analysis-unavailable'] as const)( + 'round-trips the captured canonical target of a %s diagnostic unchanged', + (reason) => { + const rootPath = path.join(os.tmpdir(), 'canonical-graph'); + const graph = new InternalFileGraph(rootPath); + graph.trackUnresolved( + path.join(rootPath, 'main.ts'), + { + line: 1, + specifier: './linked/target', + kind: 'import', + sourceFingerprint: 'source-sha256', + targetFingerprint: '' + }, + { reason, target: 'physical/target.ts' } + ); + const restored = InternalFileGraph.fromJSON(graph.toJSON(), rootPath).toJSON(); + expect(restored.unresolvedEdges).toEqual(graph.toJSON().unresolvedEdges); + expect(restored.unresolvedEdges?.[0]).toMatchObject({ reason, target: 'physical/target.ts' }); + expect(restored.imports).toEqual({}); + } + ); it('round-trips importDetails and importedSymbols behavior', () => { const rootPath = path.join(os.tmpdir(), `ifg-${Date.now()}`); const graph = new InternalFileGraph(rootPath); @@ -12,7 +34,23 @@ describe('InternalFileGraph serialization', () => { const importingFile = path.join(rootPath, 'src', 'importer.ts'); graph.trackExports(exportedFile, [{ name: 'Foo', type: 'function' }]); - graph.trackImport(importingFile, exportedFile, 12, ['Foo']); + graph.trackImport(importingFile, exportedFile, 12, ['Foo'], { + kind: 'reexport', + specifier: './exported', + sourceFingerprint: 'source-sha256', + targetFingerprint: 'target-sha256' + }); + graph.trackUnresolved( + importingFile, + { + line: 18, + specifier: './missing', + kind: 'import', + sourceFingerprint: 'source-sha256', + targetFingerprint: '' + }, + 'not-found' + ); const json = graph.toJSON(); expect(json.importDetails).toBeDefined(); @@ -20,9 +58,17 @@ describe('InternalFileGraph serialization', () => { const restored = InternalFileGraph.fromJSON(json, rootPath); const restoredJson = restored.toJSON(); expect(restoredJson.importDetails).toEqual(json.importDetails); + expect(restoredJson.unresolvedEdges).toEqual(json.unresolvedEdges); + expect( + restoredJson.importDetails?.['src/importer.ts']?.['src/exported.ts']?.sources?.[0] + ).toMatchObject({ + specifier: './exported', + kind: 'reexport', + sourceFingerprint: 'source-sha256', + targetFingerprint: 'target-sha256' + }); const unused = restored.findUnusedExports(); expect(unused.length).toBe(0); }); }); - diff --git a/tests/internal-source-resolver.test.ts b/tests/internal-source-resolver.test.ts new file mode 100644 index 0000000..612422b --- /dev/null +++ b/tests/internal-source-resolver.test.ts @@ -0,0 +1,242 @@ +import { afterAll, expect, it } from 'vitest'; +import { promises as fs } from 'fs'; +import os from 'os'; +import path from 'path'; +import ts from 'typescript'; +import { + extractSourceReexports, + InternalSourceResolver, + loadTsconfigCompilerOptions +} from '../src/utils/internal-source-resolver.js'; + +const root = await fs.mkdtemp(path.join(os.tmpdir(), 'cbc-source-resolver-')); +afterAll(async () => fs.rm(root, { recursive: true, force: true })); + +it('captures the same physical target from selected and unindexed resolution through root aliases', async () => { + const project = path.join(root, 'physical-root'); + await fs.mkdir(path.join(project, 'src'), { recursive: true }); + await fs.mkdir(path.join(project, 'physical'), { recursive: true }); + await fs.writeFile(path.join(project, 'src/main.ts'), "import '../linked/target';\n"); + await fs.writeFile(path.join(project, 'physical/target.ts'), 'export const target = true;\n'); + await fs.symlink(path.join(project, 'physical'), path.join(project, 'linked'), 'junction'); + const alias = path.join(root, 'root-alias'); + await fs.symlink(project, alias, 'junction'); + const importer = path.join(alias, 'src/main.ts'); + const target = path.join(alias, 'physical/target.ts'); + const selected = await InternalSourceResolver.create(alias, [importer, target]); + expect(await selected.resolve(importer, '../linked/target')).toEqual({ + status: 'resolved', + path: target, + target: 'physical/target.ts' + }); + const unindexed = await InternalSourceResolver.create(alias, [importer]); + expect(await unindexed.resolve(importer, '../linked/target')).toEqual({ + status: 'unresolved', + reason: 'not-indexed', + target: 'physical/target.ts' + }); +}); + +async function configProject(name: string): Promise { + const project = path.join(root, name, 'project'); + await fs.mkdir(project, { recursive: true }); + await fs.writeFile( + path.join(root, name, 'tsconfig.json'), + JSON.stringify({ + compilerOptions: { baseUrl: '.', paths: { '@ancestor/*': ['ancestor/*'] } } + }) + ); + return project; +} + +it('loads aliases from the selected root tsconfig.base.json instead of an ancestor config', async () => { + const project = await configProject('base-only'); + await fs.writeFile( + path.join(project, 'tsconfig.base.json'), + JSON.stringify({ + compilerOptions: { baseUrl: '.', paths: { '@shared/*': ['src/*'] } } + }) + ); + await fs.mkdir(path.join(project, 'src')); + const importer = path.join(project, 'src/entry.ts'); + const target = path.join(project, 'src/util.ts'); + await Promise.all([ + fs.writeFile(importer, ''), + fs.writeFile(target, 'export const value = true;') + ]); + + const compilerOptions = loadTsconfigCompilerOptions(project); + expect(path.resolve(compilerOptions.baseUrl ?? '')).toBe(project); + expect(compilerOptions.paths).toEqual({ '@shared/*': ['src/*'] }); + const resolver = await InternalSourceResolver.create(project, [importer, target], { + compilerOptions + }); + expect(await resolver.resolve(importer, '@shared/util')).toEqual({ + status: 'resolved', + path: target, + target: 'src/util.ts' + }); + expect(await resolver.resolve(importer, '@unmapped/util')).toEqual({ + status: 'unresolved', + reason: 'external' + }); +}); + +it('prefers the selected root tsconfig.json without merging its base-config sibling', async () => { + const project = await configProject('primary'); + await fs.writeFile( + path.join(project, 'tsconfig.json'), + JSON.stringify({ + compilerOptions: { baseUrl: '.', paths: { '@primary/*': ['primary/*'] } } + }) + ); + await fs.writeFile( + path.join(project, 'tsconfig.base.json'), + JSON.stringify({ + compilerOptions: { baseUrl: '.', paths: { '@base/*': ['base/*'] } } + }) + ); + + const compilerOptions = loadTsconfigCompilerOptions(project); + expect(path.resolve(compilerOptions.baseUrl ?? '')).toBe(project); + expect(compilerOptions.paths).toEqual({ '@primary/*': ['primary/*'] }); +}); + +it('uses resolver defaults when the selected root has no config, without borrowing an ancestor', async () => { + const project = await configProject('no-config'); + expect(loadTsconfigCompilerOptions(project)).toEqual({ + module: ts.ModuleKind.Preserve, + moduleResolution: ts.ModuleResolutionKind.Bundler + }); +}); + +it.each(['tsconfig.json', 'tsconfig.base.json'])( + 'preserves explicit extends from the selected %s', + async (configName) => { + const project = await configProject(`extends-${configName}`); + await fs.writeFile( + path.join(path.dirname(project), 'shared.json'), + JSON.stringify({ + compilerOptions: { + baseUrl: './project', + paths: { '@inherited/*': ['src/*'] }, + strict: true + } + }) + ); + await fs.writeFile( + path.join(project, configName), + JSON.stringify({ extends: '../shared.json' }) + ); + + const compilerOptions = loadTsconfigCompilerOptions(project); + expect(path.resolve(compilerOptions.baseUrl ?? '')).toBe(project); + expect(compilerOptions.paths).toEqual({ '@inherited/*': ['src/*'] }); + expect(compilerOptions.strict).toBe(true); + } +); + +it('keeps defaults for a malformed primary config instead of silently loading the base sibling', async () => { + const project = await configProject('malformed-primary'); + await fs.writeFile(path.join(project, 'tsconfig.json'), '{'); + await fs.writeFile( + path.join(project, 'tsconfig.base.json'), + JSON.stringify({ + compilerOptions: { baseUrl: '.', paths: { '@base/*': ['src/*'] } } + }) + ); + expect(loadTsconfigCompilerOptions(project)).toEqual({ + module: ts.ModuleKind.Preserve, + moduleResolution: ts.ModuleResolutionKind.Bundler + }); +}); + +it('resolves only indexed TS/TSX files, directory indexes, aliases, and JS specifiers to source', async () => { + const importer = path.join(root, 'apps/web/src/entry.ts'); + const jsAliasTarget = path.join(root, 'apps/web/src/actual.ts'); + const targets = [ + path.join(root, 'apps/web/src/view.tsx'), + path.join(root, 'packages/core/src/index.ts'), + path.join(root, 'packages/core/src/actual.ts') + ]; + await fs.mkdir(path.dirname(importer), { recursive: true }); + await fs.mkdir(path.dirname(targets[0]!), { recursive: true }); + await fs.mkdir(path.dirname(targets[1]!), { recursive: true }); + await Promise.all([ + fs.writeFile(importer, ''), + fs.writeFile(jsAliasTarget, ''), + ...targets.map((file) => fs.writeFile(file, '')) + ]); + const resolver = await InternalSourceResolver.create( + root, + [importer, jsAliasTarget, ...targets], + { + baseUrl: '.', + paths: { '@core/*': ['packages/core/src/*'] } + } + ); + + expect(await resolver.resolve(importer, './view')).toMatchObject({ + status: 'resolved', + path: targets[0] + }); + expect(await resolver.resolve(importer, '../../../packages/core/src')).toMatchObject({ + status: 'resolved', + path: targets[1] + }); + expect(await resolver.resolve(importer, '@core/actual')).toMatchObject({ + status: 'resolved', + path: targets[2] + }); + expect(await resolver.resolve(importer, '@core/missing')).toMatchObject({ + status: 'unresolved', + reason: 'not-found' + }); + expect(await resolver.resolve(importer, 'external-package')).toMatchObject({ + status: 'unresolved', + reason: 'external' + }); + expect(await resolver.resolve(importer, './ghost')).toMatchObject({ + status: 'unresolved', + reason: 'not-found' + }); + expect(await resolver.resolve(importer, './actual.js')).toMatchObject({ + status: 'resolved', + path: jsAliasTarget + }); +}); + +it('extracts only explicit AST re-exports and rejects paths resolving outside the root', async () => { + expect( + extractSourceReexports("export { thing } from './impl';\nexport * from './other';", 'index.ts') + ).toEqual([ + { specifier: './impl', line: 1 }, + { specifier: './other', line: 2 } + ]); + expect(extractSourceReexports("import './impl';", 'index.ts')).toEqual([]); + + const outside = await fs.mkdtemp(path.join(os.tmpdir(), 'cbc-outside-')); + const importer = path.join(root, 'src/in-root.ts'); + const escape = path.join(root, 'src/escape'); + await fs.mkdir(path.dirname(importer), { recursive: true }); + await fs.writeFile(importer, ''); + let symlinkCreated = false; + try { + await fs.writeFile(path.join(outside, 'file.ts'), 'export const outside = true;'); + await fs.symlink(outside, escape, 'junction'); + symlinkCreated = true; + } catch { + // Junction creation can be disabled by the host; direct traversal tests still cover outside-root paths. + } + if (symlinkCreated) { + const resolver = await InternalSourceResolver.create(root, [ + importer, + path.join(escape, 'file.ts') + ]); + expect(await resolver.resolve(importer, './escape/file')).toMatchObject({ + status: 'unresolved', + reason: 'outside-root' + }); + } + await fs.rm(outside, { recursive: true, force: true }); +}); diff --git a/tests/lancedb-corruption.test.ts b/tests/lancedb-corruption.test.ts index 7770a90..e728014 100644 --- a/tests/lancedb-corruption.test.ts +++ b/tests/lancedb-corruption.test.ts @@ -28,11 +28,15 @@ describe('LanceDBStorageProvider corruption detection', () => { }); it('throws IndexCorruptedError when vector column missing during initialize()', async () => { + const closeTable = vi.fn(); + const closeConnection = vi.fn(); const db = { tableNames: vi.fn(async () => ['code_chunks']), openTable: vi.fn(async () => ({ - schema: vi.fn(async () => ({ fields: [{ name: 'id' }] })) - })) + schema: vi.fn(async () => ({ fields: [{ name: 'id' }] })), + close: closeTable + })), + close: closeConnection }; lancedb.connect.mockResolvedValue(db); @@ -41,6 +45,8 @@ describe('LanceDBStorageProvider corruption detection', () => { const provider = new LanceDBStorageProvider(); await expect(provider.initialize(tempDir)).rejects.toBeInstanceOf(IndexCorruptedError); + expect(closeTable).toHaveBeenCalledOnce(); + expect(closeConnection).toHaveBeenCalledOnce(); // dropTable is no longer called within initialize (senior mindset: separation of concerns) }); @@ -121,7 +127,7 @@ describe('LanceDBStorageProvider corruption detection', () => { expect(deleted).toBe(3); expect(deleteSpy).toHaveBeenCalledWith( - "\"filePath\" IN ('C:/repo/src/a.ts', 'C:/repo/src/b''test.ts')" + "`filePath` IN ('C:/repo/src/a.ts', 'C:/repo/src/b''test.ts')" ); }); @@ -148,6 +154,6 @@ describe('LanceDBStorageProvider corruption detection', () => { }); expect(results).toEqual([]); - expect(whereSpy).toHaveBeenCalledWith("\"componentType\" = 'service' AND layer = 'business'"); + expect(whereSpy).toHaveBeenCalledWith("`componentType` = 'service' AND layer = 'business'"); }); }); diff --git a/tests/multi-project-routing.test.ts b/tests/multi-project-routing.test.ts index fa8f181..31f4a03 100644 --- a/tests/multi-project-routing.test.ts +++ b/tests/multi-project-routing.test.ts @@ -64,6 +64,12 @@ vi.mock('../src/core/search.js', async () => { async search(query: string, limit: number, filters?: unknown, options?: unknown) { return searchMocks.search(this.rootPath, query, limit, filters, options); } + async searchWithCoverage(query: string, limit: number, filters?: unknown, options?: unknown) { + return { + results: await this.search(query, limit, filters, options), + sourceCoverage: { status: 'current', checkedFiles: 0, skippedFiles: 0 } + }; + } } return { CodebaseSearcher }; @@ -221,7 +227,7 @@ describe('multi-project routing', () => { afterEach(async () => { const { clearProjects } = await import('../src/project-state.js'); - clearProjects(); + await clearProjects(); if (originalArgv) process.argv = originalArgv; if (originalEnvRoot === undefined) { @@ -367,6 +373,31 @@ describe('multi-project routing', () => { } }); + it('reports status-check guidance during an initial uncommitted build', async () => { + const { server } = await import('../src/index.js'); + const { getOrCreateProject } = await import('../src/project-state.js'); + const handler = (server as unknown as TestServer)._requestHandlers.get('tools/call'); + if (!handler) throw new Error('tools/call handler not registered'); + + const project = getOrCreateProject(primaryRoot); + const originalStatus = project.indexState.status; + project.indexState.status = 'indexing'; + await fs.rm(path.join(primaryRoot, CODEBASE_CONTEXT_DIRNAME, INDEX_META_FILENAME)); + try { + const response = await callTool(handler, 20, 'search_codebase', { + query: 'feature', + project: primaryRoot + }); + const payload = parsePayload(response) as { status: string; message: string }; + + expect(payload.status).toBe('indexing'); + expect(payload.message).toBe('Index build in progress. Check indexing status, then retry.'); + expect(searchMocks.search).not.toHaveBeenCalled(); + } finally { + project.indexState.status = originalStatus; + } + }); + it('triggers a background rebuild for a corrupted explicit project without falling back to cwd', async () => { delete process.env.CODEBASE_ROOT; delete process.argv[2]; @@ -398,7 +429,7 @@ describe('multi-project routing', () => { }; expect(payload.status).toBe('indexing'); - expect(payload.message).toContain('retry shortly'); + expect(payload.message).toBe('Index rebuild in progress. Check indexing status, then retry.'); expect(payload.index?.action).toBe('rebuild-started'); expect(String(payload.index?.reason || '')).toContain('Index meta'); } finally { @@ -480,6 +511,76 @@ describe('multi-project routing', () => { ); }); + it('reserves watcher slots while concurrent opens evict and reopen a closing project', async () => { + const roots = [primaryRoot, secondaryRoot, nestedProjectRoot]; + for (let index = 0; index < 4; index += 1) { + const root = await fs.mkdtemp(path.join(os.tmpdir(), `cc-watcher-limit-${index}-`)); + roots.push(root); + await seedValidIndex(root); + } + + const oldestRoot = roots[0]!; + const [openingRootA, openingRootB] = roots.slice(5); + if (!openingRootA || !openingRootB) throw new Error('expected two new watcher roots'); + + let finishOldestClose!: () => void; + const oldestClose = new Promise((resolve) => { + finishOldestClose = resolve; + }); + const stopCalls = new Map(); + watcherMocks.start.mockImplementation(({ rootPath }: { rootPath: string }) => () => { + stopCalls.set(rootPath, (stopCalls.get(rootPath) ?? 0) + 1); + return rootPath === oldestRoot ? oldestClose : Promise.resolve(); + }); + + const { server } = await import('../src/index.js'); + const handler = (server as unknown as TestServer)._requestHandlers.get('tools/call'); + if (!handler) throw new Error('tools/call handler not registered'); + + try { + for (const [id, root] of roots.slice(0, 5).entries()) { + await callTool(handler, 200 + id, 'search_codebase', { query: 'feature', project: root }); + } + + const opening = [ + callTool(handler, 210, 'search_codebase', { query: 'feature', project: openingRootA }), + callTool(handler, 211, 'search_codebase', { query: 'feature', project: openingRootB }) + ]; + await vi.waitFor(() => { + expect(stopCalls.get(oldestRoot)).toBeGreaterThan(0); + }); + + const reopening = [ + callTool(handler, 212, 'search_codebase', { query: 'feature', project: oldestRoot }), + callTool(handler, 213, 'search_codebase', { query: 'feature', project: oldestRoot }) + ]; + await vi.waitFor(() => { + expect(watcherMocks.start).toHaveBeenCalledWith( + expect.objectContaining({ rootPath: oldestRoot }) + ); + }); + expect( + watcherMocks.start.mock.calls.filter(([options]) => options.rootPath === oldestRoot) + ).toHaveLength(1); + + finishOldestClose(); + await Promise.all([...opening, ...reopening]); + + const { getAllProjects } = await import('../src/project-state.js'); + expect(getAllProjects().filter((project) => project.stopWatcher)).toHaveLength(5); + expect( + watcherMocks.start.mock.calls.filter(([options]) => options.rootPath === oldestRoot) + ).toHaveLength(2); + expect(stopCalls.get(oldestRoot)).toBe(1); + } finally { + finishOldestClose(); + await fs.rm(roots[3]!, { recursive: true, force: true }); + await fs.rm(roots[4]!, { recursive: true, force: true }); + await fs.rm(roots[5]!, { recursive: true, force: true }); + await fs.rm(roots[6]!, { recursive: true, force: true }); + } + }); + it('uses the active project for later tool calls and returns project metadata', async () => { const { server } = await import('../src/index.js'); const handler = (server as unknown as TestServer)._requestHandlers.get('tools/call'); @@ -591,7 +692,9 @@ describe('multi-project routing', () => { it('lists bounded and full context resources for active and project-scoped flows', async () => { const { server } = await import('../src/index.js'); const toolHandler = (server as unknown as TestServer)._requestHandlers.get('tools/call'); - const resourcesHandler = (server as unknown as TestServer)._requestHandlers.get('resources/list'); + const resourcesHandler = (server as unknown as TestServer)._requestHandlers.get( + 'resources/list' + ); if (!toolHandler || !resourcesHandler) { throw new Error('required handlers not registered'); diff --git a/tests/project-state.test.ts b/tests/project-state.test.ts index 92820e5..49314a7 100644 --- a/tests/project-state.test.ts +++ b/tests/project-state.test.ts @@ -11,8 +11,8 @@ import { import { CODEBASE_CONTEXT_DIRNAME } from '../src/constants/codebase-context.js'; describe('project-state', () => { - afterEach(() => { - clearProjects(); + afterEach(async () => { + await clearProjects(); vi.restoreAllMocks(); }); @@ -85,33 +85,54 @@ describe('project-state', () => { }); describe('clearProjects', () => { - it('empties the map', () => { + it('empties the map', async () => { getOrCreateProject('/projects/app-a'); getOrCreateProject('/projects/app-b'); - clearProjects(); + await clearProjects(); // After clearing, getOrCreateProject should return a fresh instance const p = getOrCreateProject('/projects/app-a'); expect(p.indexState.status).toBe('idle'); }); - it('calls stopWatcher on projects with watchers', () => { + it('calls stopWatcher on projects with watchers', async () => { const p = getOrCreateProject('/projects/my-app'); - const stopSpy = vi.fn(); + const stopSpy = vi.fn(async () => undefined); p.stopWatcher = stopSpy; - clearProjects(); + await clearProjects(); expect(stopSpy).toHaveBeenCalledTimes(1); }); + + it('waits for watcher close after synchronously clearing project state', async () => { + let finishClose!: () => void; + const closeCompletion = new Promise((resolve) => { + finishClose = resolve; + }); + const project = getOrCreateProject('/projects/closing-app'); + project.stopWatcher = vi.fn(() => closeCompletion); + + let clearResolved = false; + const clearing = clearProjects().then(() => { + clearResolved = true; + }); + expect(getProject('/projects/closing-app')).toBeUndefined(); + await Promise.resolve(); + expect(clearResolved).toBe(false); + + finishClose(); + await clearing; + expect(clearResolved).toBe(true); + }); }); describe('removeProject', () => { - it('removes a project and stops its watcher', () => { + it('removes a project and stops its watcher', async () => { const project = getOrCreateProject('/projects/my-app'); - const stopSpy = vi.fn(); + const stopSpy = vi.fn(async () => undefined); project.stopWatcher = stopSpy; - removeProject('/projects/my-app'); + await removeProject('/projects/my-app'); expect(stopSpy).toHaveBeenCalledTimes(1); expect(getProject('/projects/my-app')).toBeUndefined(); diff --git a/tests/react-analyzer.test.ts b/tests/react-analyzer.test.ts index 2cb7939..eaed855 100644 --- a/tests/react-analyzer.test.ts +++ b/tests/react-analyzer.test.ts @@ -14,6 +14,48 @@ describe('ReactAnalyzer', () => { expect(analyzer.canAnalyze('/tmp/file.js', 'React.createElement("div");')).toBe(true); }); + it.each(['.ts', '.mts', '.cts'])( + 'does not claim %s files for TypeScript generic or angle-assertion syntax alone', + (extension) => { + const analyzer = new ReactAnalyzer(); + + expect( + analyzer.canAnalyze( + `/tmp/file${extension}`, + 'const map = new Map();' + ) + ).toBe(false); + expect( + analyzer.canAnalyze(`/tmp/file${extension}`, 'const value = input;') + ).toBe(false); + } + ); + + it.each(['.ts', '.mts', '.cts'])( + 'continues to recognize explicit React markers in %s files', + (extension) => { + const analyzer = new ReactAnalyzer(); + + expect( + analyzer.canAnalyze(`/tmp/file${extension}`, 'import { useState } from "react";') + ).toBe(true); + expect(analyzer.canAnalyze(`/tmp/file${extension}`, 'const React = require("react");')).toBe( + true + ); + expect(analyzer.canAnalyze(`/tmp/file${extension}`, 'React.createElement("div");')).toBe( + true + ); + } + ); + + it('continues to claim JSX extensions and JavaScript JSX syntax', () => { + const analyzer = new ReactAnalyzer(); + + expect(analyzer.canAnalyze('/tmp/file.tsx')).toBe(true); + expect(analyzer.canAnalyze('/tmp/file.jsx')).toBe(true); + expect(analyzer.canAnalyze('/tmp/file.js', 'const element =
;')).toBe(true); + }); + it('detects components, hooks, context, and ecosystem signals', async () => { const analyzer = new ReactAnalyzer(); const filePath = path.join(process.cwd(), 'src', 'components', 'MyWidget.tsx'); @@ -54,7 +96,9 @@ export class LegacyWidget extends Component { expect(result.framework).toBe('react'); expect( - result.components.some((component) => component.componentType === 'hook' && component.name === 'useTheme') + result.components.some( + (component) => component.componentType === 'hook' && component.name === 'useTheme' + ) ).toBe(true); expect( result.components.some( @@ -67,7 +111,10 @@ export class LegacyWidget extends Component { ) ).toBe(true); - const patterns = (result.metadata.detectedPatterns || []) as Array<{ category: string; name: string }>; + const patterns = (result.metadata.detectedPatterns || []) as Array<{ + category: string; + name: string; + }>; expect(patterns).toContainEqual({ category: 'stateManagement', name: 'React Context' }); expect(patterns).toContainEqual({ category: 'reactivity', name: 'Suspense' }); expect(patterns).toContainEqual({ category: 'reactivity', name: 'Memoization' }); @@ -99,7 +146,10 @@ export function CheckoutForm({ cartPromise }: { cartPromise: Promise } `; const result = await analyzer.analyze(filePath, code); - const patterns = (result.metadata.detectedPatterns || []) as Array<{ category: string; name: string }>; + const patterns = (result.metadata.detectedPatterns || []) as Array<{ + category: string; + name: string; + }>; expect(patterns).toContainEqual({ category: 'reactHooks', name: 'Built-in hooks' }); expect(patterns).toContainEqual({ category: 'reactivity', name: 'Actions' }); diff --git a/tests/reference-nomination-parity.test.ts b/tests/reference-nomination-parity.test.ts new file mode 100644 index 0000000..0ee36dc --- /dev/null +++ b/tests/reference-nomination-parity.test.ts @@ -0,0 +1,88 @@ +import { readFileSync } from 'node:fs'; +import path from 'node:path'; +import { describe, expect, it } from 'vitest'; +import { + hasIdentifierCandidate, + findIdentifierOccurrences, + findLexicalIdentifierOccurrences +} from '../src/utils/tree-sitter'; + +interface ParityCase { + id: string; + language: string; + symbol: string; + source: string; + expected: { mode: 'ast' | 'lexical'; count: number }; +} + +interface ParityFixture { + contract: string; + cases: ParityCase[]; +} + +const fixturePath = path.join( + process.cwd(), + 'tests', + 'fixtures', + 'reference-nomination-parity.json' +); +const fixture = JSON.parse(readFileSync(fixturePath, 'utf8')) as ParityFixture; + +describe('reference nomination parity fixture', () => { + it('freezes current parser and lexical behavior across grammar and uncertainty cases', async () => { + expect(fixture.cases.length).toBeGreaterThanOrEqual(20); + + for (const example of fixture.cases) { + const ast = await findIdentifierOccurrences(example.source, example.language, example.symbol); + const mode = ast === null ? 'lexical' : 'ast'; + const occurrences = + ast === null ? findLexicalIdentifierOccurrences(example.source, example.symbol) : ast; + + expect({ mode, count: occurrences.length }, example.id).toEqual(example.expected); + } + }); + + it('preserves the frozen result with the production ASCII-conservative nominator', async () => { + const omittedCandidates: string[] = []; + for (const example of fixture.cases) { + const nominated = hasIdentifierCandidate(example.source, example.symbol); + if (example.source.includes(example.symbol) && !nominated) { + omittedCandidates.push(example.id); + } + const ast = nominated + ? await findIdentifierOccurrences(example.source, example.language, example.symbol) + : []; + const occurrences = !nominated + ? [] + : ast === null + ? findLexicalIdentifierOccurrences(example.source, example.symbol) + : ast; + + expect(occurrences.length, example.id).toBe(example.expected.count); + } + + expect(omittedCandidates).toEqual([ + 'ts-long-identifier', + 'ts-dollar-identifier', + 'ts-underscore-identifier', + 'ts-underscore-suffix', + 'ts-number-error-split', + 'javascript-number-suffix', + 'python-number-suffix', + 'go-number-suffix', + 'rust-number-suffix', + 'java-number-suffix', + 'c-number-suffix', + 'cpp-number-suffix', + 'c-hex-prefix-split', + 'cpp-hex-prefix-split', + 'csharp-number-suffix', + 'kotlin-number-suffix', + 'python-long-identifier', + 'unknown-long-identifier', + 'unknown-dollar-boundary', + 'unknown-number-error-split', + 'excalidraw-source-token-inside-long-identifier' + ]); + }); +}); diff --git a/tests/refresh-index-mode.test.ts b/tests/refresh-index-mode.test.ts new file mode 100644 index 0000000..c6412fc --- /dev/null +++ b/tests/refresh-index-mode.test.ts @@ -0,0 +1,110 @@ +import { describe, expect, it, vi } from 'vitest'; +import * as embeddings from '../src/embeddings/index.js'; +import { OptionalDependencyError } from '../src/errors/index.js'; +import * as storage from '../src/storage/index.js'; +import { handle } from '../src/tools/refresh-index.js'; +import type { ToolContext } from '../src/tools/types.js'; + +function makeContext(status: ToolContext['indexState']['status'] = 'idle') { + const indexer = {} as NonNullable; + const stats = {} as NonNullable; + return { + indexState: { status, lastIndexed: new Date(1), indexer, stats }, + paths: {} as ToolContext['paths'], + rootPath: '/project', + performIndexing: vi.fn() + } satisfies ToolContext; +} + +describe('refresh_index embedding mode', () => { + it('forwards explicit semantic opt-in and preserves omission for recorded mode resolution', async () => { + const context = makeContext(); + const embeddingAssert = vi + .spyOn(embeddings, 'assertEmbeddingRuntimeAvailable') + .mockImplementation(() => {}); + const storageAssert = vi + .spyOn(storage, 'assertStorageRuntimeAvailable') + .mockImplementation(() => {}); + try { + await handle( + { skipEmbedding: false, incrementalOnly: true, reason: 'owner choice' }, + context + ); + await handle({ incrementalOnly: true }, context); + + expect(embeddingAssert).toHaveBeenCalledTimes(1); + expect(storageAssert).toHaveBeenCalledTimes(1); + expect(context.performIndexing).toHaveBeenNthCalledWith(1, true, 'owner choice', false); + expect(context.performIndexing).toHaveBeenNthCalledWith(2, true, undefined, undefined); + } finally { + embeddingAssert.mockRestore(); + storageAssert.mockRestore(); + } + }); + + it('rejects invalid or busy mode changes before starting indexing', async () => { + const invalidContext = makeContext(); + const invalid = await handle({ skipEmbedding: 'false' }, invalidContext); + expect(invalid.isError).toBe(true); + expect(invalidContext.performIndexing).not.toHaveBeenCalled(); + + const busyContext = makeContext('indexing'); + const busy = await handle({ skipEmbedding: false }, busyContext); + expect(busy.isError).toBe(true); + expect(busyContext.performIndexing).not.toHaveBeenCalled(); + }); + + it('rejects missing embedding runtime before dispatch and preserves prior ready state', async () => { + const context = makeContext('ready'); + const originalState = { ...context.indexState }; + const missingRuntime = new OptionalDependencyError('@huggingface/transformers', '^4.2.0'); + const embeddingAssert = vi + .spyOn(embeddings, 'assertEmbeddingRuntimeAvailable') + .mockImplementationOnce(() => { + throw missingRuntime; + }); + const storageAssert = vi.spyOn(storage, 'assertStorageRuntimeAvailable'); + + try { + const response = await handle({ skipEmbedding: false }, context); + expect(response.isError).toBe(true); + expect(response.content?.[0].text).toContain('@huggingface/transformers'); + expect(response.content?.[0].text).toContain('npm install'); + expect(context.performIndexing).not.toHaveBeenCalled(); + expect(context.indexState).toEqual(originalState); + expect(embeddingAssert).toHaveBeenCalledTimes(1); + expect(storageAssert).not.toHaveBeenCalled(); + } finally { + embeddingAssert.mockRestore(); + storageAssert.mockRestore(); + } + }); + + it('rejects missing vector storage after embedding preflight without changing project state', async () => { + const context = makeContext('ready'); + const originalState = { ...context.indexState }; + const embeddingAssert = vi + .spyOn(embeddings, 'assertEmbeddingRuntimeAvailable') + .mockImplementation(() => {}); + const missingRuntime = new OptionalDependencyError('@lancedb/lancedb', '^0.4.0'); + const storageAssert = vi + .spyOn(storage, 'assertStorageRuntimeAvailable') + .mockImplementationOnce(() => { + throw missingRuntime; + }); + + try { + const response = await handle({ skipEmbedding: false }, context); + expect(response.isError).toBe(true); + expect(response.content?.[0].text).toContain('@lancedb/lancedb'); + expect(response.content?.[0].text).toContain('npm install'); + expect(context.performIndexing).not.toHaveBeenCalled(); + expect(context.indexState).toEqual(originalState); + expect(embeddingAssert).toHaveBeenCalledTimes(1); + expect(storageAssert).toHaveBeenCalledTimes(1); + } finally { + embeddingAssert.mockRestore(); + storageAssert.mockRestore(); + } + }); +}); diff --git a/tests/relationship-response-budget.test.ts b/tests/relationship-response-budget.test.ts new file mode 100644 index 0000000..f7eb50b --- /dev/null +++ b/tests/relationship-response-budget.test.ts @@ -0,0 +1,52 @@ +import { expect, it } from 'vitest'; +import { finalizeJsonTextPayload } from '../src/index.js'; + +it('keeps the final relationship text within budget after routing metadata is added', () => { + const graphPayload = { + status: 'success', + file: 'packages/example/src/main.ts', + direction: 'outgoing', + maxHops: 3, + paths: Array.from({ length: 16 }, (_, index) => ({ + nodes: [ + 'packages/example/src/main.ts', + `packages/example/src/branch-${index}-with-a-long-source-path.ts` + ], + edges: [ + { + from: 'packages/example/src/main.ts', + to: `packages/example/src/branch-${index}-with-a-long-source-path.ts`, + kind: 'import', + source: { line: index + 1, specifier: `./branch-${index}-with-a-long-source-path` } + } + ] + })), + unresolved: [], + truncated: false, + message: + 'Relationships are limited to verified imports and re-exports; dynamic calls are not inferred.' + }; + const routedPayload = { + ...graphPayload, + index: { status: 'ready', lastIndexed: '2026-10-11T04:30:00.000Z', buildId: 'b'.repeat(64) }, + project: { + id: 'p'.repeat(64), + rootPath: 'C:/projects/consumer', + source: 'root', + label: 'consumer project' + } + }; + + const finalText = finalizeJsonTextPayload(routedPayload, true); + expect(Buffer.byteLength(finalText, 'utf8')).toBeLessThanOrEqual(4000); + const finalized = JSON.parse(finalText) as { + paths: unknown[]; + truncated: boolean; + index: unknown; + project: unknown; + }; + expect(finalized.truncated).toBe(true); + expect(finalized.paths.length).toBeGreaterThan(0); + expect(finalized.index).toEqual(routedPayload.index); + expect(finalized.project).toEqual(routedPayload.project); +}); diff --git a/tests/relationship-sidecar.test.ts b/tests/relationship-sidecar.test.ts index d4ac6da..3415d39 100644 --- a/tests/relationship-sidecar.test.ts +++ b/tests/relationship-sidecar.test.ts @@ -1,3 +1,4 @@ +import { resolveIndexArtifact } from '../src/core/index-generation.js'; import { describe, it, expect, beforeAll, afterAll } from 'vitest'; import { promises as fs } from 'fs'; import path from 'path'; @@ -6,7 +7,8 @@ import { readIndexMeta, validateIndexArtifacts } from '../src/core/index-meta.js import { CODEBASE_CONTEXT_DIRNAME, RELATIONSHIPS_FILENAME, - INDEX_META_FILENAME + INDEX_META_FILENAME, + INDEX_FORMAT_VERSION } from '../src/constants/codebase-context.js'; async function createTempDir(): Promise { @@ -60,7 +62,10 @@ describe('Relationship Sidecar', () => { // Check relationships.json exists const contextDir = path.join(tmpDir, CODEBASE_CONTEXT_DIRNAME); - const relationshipsPath = path.join(contextDir, RELATIONSHIPS_FILENAME); + const relationshipsPath = await resolveIndexArtifact( + path.dirname(contextDir), + RELATIONSHIPS_FILENAME + ); const relationshipsExists = await fs .access(relationshipsPath) .then(() => true) @@ -74,7 +79,7 @@ describe('Relationship Sidecar', () => { expect(relationships.header).toBeDefined(); expect(relationships.header.buildId).toBeDefined(); expect(typeof relationships.header.buildId).toBe('string'); - expect(relationships.header.formatVersion).toBe(1); + expect(relationships.header.formatVersion).toBe(INDEX_FORMAT_VERSION); expect(relationships.generatedAt).toBeDefined(); expect(relationships.graph).toBeDefined(); expect(typeof relationships.graph.imports).toBe('object'); @@ -115,7 +120,10 @@ describe('Relationship Sidecar', () => { const metaRaw = await fs.readFile(path.join(contextDir, INDEX_META_FILENAME), 'utf-8'); const meta = JSON.parse(metaRaw); - const relRaw = await fs.readFile(path.join(contextDir, RELATIONSHIPS_FILENAME), 'utf-8'); + const relRaw = await fs.readFile( + await resolveIndexArtifact(path.dirname(contextDir), RELATIONSHIPS_FILENAME), + 'utf-8' + ); const relationships = JSON.parse(relRaw); expect(relationships.header.buildId).toBe(meta.buildId); @@ -131,7 +139,10 @@ describe('Relationship Sidecar', () => { it('updates relationships.json on incremental index', async () => { // Record the initial buildId const contextDir = path.join(tmpDir, CODEBASE_CONTEXT_DIRNAME); - const relRaw1 = await fs.readFile(path.join(contextDir, RELATIONSHIPS_FILENAME), 'utf-8'); + const relRaw1 = await fs.readFile( + await resolveIndexArtifact(path.dirname(contextDir), RELATIONSHIPS_FILENAME), + 'utf-8' + ); const relationships1 = JSON.parse(relRaw1); // Add a new file @@ -151,7 +162,10 @@ describe('Relationship Sidecar', () => { await indexer.index(); // Read updated relationships - const relRaw2 = await fs.readFile(path.join(contextDir, RELATIONSHIPS_FILENAME), 'utf-8'); + const relRaw2 = await fs.readFile( + await resolveIndexArtifact(path.dirname(contextDir), RELATIONSHIPS_FILENAME), + 'utf-8' + ); const relationships2 = JSON.parse(relRaw2); // The file should still have valid structure after incremental update @@ -177,7 +191,10 @@ describe('Relationship Sidecar', () => { await indexer.index(); // Check relationships.json exists again - const relationshipsPath = path.join(contextDir, RELATIONSHIPS_FILENAME); + const relationshipsPath = await resolveIndexArtifact( + path.dirname(contextDir), + RELATIONSHIPS_FILENAME + ); const relationshipsExists = await fs .access(relationshipsPath) .then(() => true) @@ -189,7 +206,7 @@ describe('Relationship Sidecar', () => { const relationships = JSON.parse(raw); expect(relationships.header).toBeDefined(); - expect(relationships.header.formatVersion).toBe(1); + expect(relationships.header.formatVersion).toBe(INDEX_FORMAT_VERSION); expect(relationships.graph).toBeDefined(); expect(relationships.symbols).toBeDefined(); expect(relationships.stats).toBeDefined(); diff --git a/tests/release-truth-surfaces.test.ts b/tests/release-truth-surfaces.test.ts index f362ba4..7cd1707 100644 --- a/tests/release-truth-surfaces.test.ts +++ b/tests/release-truth-surfaces.test.ts @@ -114,8 +114,83 @@ describe('release truth surfaces', () => { expect(visualsDoc).toContain('Historical snapshot'); }); - it('keeps the manual publish fallback aligned to the package version', () => { - expect(workflow).toContain(`description: 'Tag to publish (e.g. v${expectedVersion})'`); - expect(workflow).toContain(`default: 'v${expectedVersion}'`); + it('requires an explicit release tag for the manual publish fallback', () => { + const manualTagInput = workflow.match(/^ {6}tag:\r?\n((?: {8}[^\r\n]*\r?\n)+)/m)?.[1] ?? ''; + + expect(manualTagInput).toMatch(/^\s*required:\s*true\s*$/m); + expect(manualTagInput).toMatch(/^\s*type:\s*string\s*$/m); + expect(manualTagInput).not.toMatch(/^\s*default:/m); + expect(workflow).toMatch(/^ release:\r?\n {4}types: \[published\]/m); + expect(workflow).toContain( + 'ref: ${{ github.event.inputs.tag || github.event.release.tag_name }}' + ); + }); + + it('keeps pinned install commands on the published npm version', () => { + const publishedVersion = '2.2.0'; + const userFacingDocs = [ + 'README.md', + 'docs/capabilities.md', + 'docs/cli.md', + 'docs/client-setup.md', + 'docs/demo.md', + 'docs/review-context.md', + 'docs/templates.md' + ]; + const pinnedCommands = userFacingDocs.flatMap((path) => { + const page = readOptionalText(path); + if (!page) return []; + return [...page.matchAll(/codebase-context@(\d+\.\d+\.\d+)/g)] + .filter((match) => { + const version = match[1]; + if (version === publishedVersion) { + return false; + } + if ( + version === '2.4.0' && + (path === 'docs/review-context.md' || path === 'docs/client-setup.md') + ) { + const index = match.index ?? 0; + const surroundingText = page.slice( + Math.max(0, index - 300), + Math.min(page.length, index + 300) + ); + if (/requires\s+(?:the\s+unpublished\s+)?2\.4\.0/i.test(surroundingText)) { + return false; + } + } + return true; + }) + .map((match) => `${path}: ${match[0]}`); + }); + + expect(pinnedCommands).toEqual([]); + expect(readText('README.md')).toContain('npx -y codebase-context'); + expect(readText('docs/client-setup.md')).toContain('npx -y codebase-context'); + expect(readText('docs/client-setup.md')).toContain( + 'node /path/to/codebase-context/dist/index.js init' + ); + expect(readText('docs/client-setup.md')).not.toContain('codebase-context@2.4.0 init'); + expect(readText('docs/review-context.md')).toContain('--package=codebase-context@2.4.0'); + }); + + it('does not describe the unpublished 2.4.0 preview as current or published', () => { + const capabilities = readText('docs/capabilities.md'); + + expect(capabilities).not.toContain('what `codebase-context` ships today'); + expect(capabilities).not.toContain('Published 2.4.0 has'); + }); + + it('keeps release wording, analyzer claims and setup checks accurate', () => { + const capabilities = readText('docs/capabilities.md'); + const setup = readText('docs/client-setup.md'); + const beforeFirstClient = setup.slice(0, setup.indexOf('## Claude Code')); + + expect(capabilities).not.toMatch(/\b2\.4\.0 (?:setup )?preview\b/i); + expect(`${readme}\n${capabilities}`).not.toMatch(/opt-in NestJS|NestJS \(opt-in\)/i); + expect(capabilities).not.toContain('analyzers: { nestjs: { enabled: true } }'); + expect(capabilities).not.toContain('generates HTTP setup'); + expect(setup).not.toContain('package still declares Node 18+'); + expect(beforeFirstClient).toContain('`2.2.0`'); }); }); diff --git a/tests/reranker.test.ts b/tests/reranker.test.ts index 7bee16c..aaa035a 100644 --- a/tests/reranker.test.ts +++ b/tests/reranker.test.ts @@ -1,6 +1,7 @@ -import { describe, it, expect } from 'vitest'; +import { describe, it, expect, vi } from 'vitest'; import { isAmbiguous } from '../src/core/reranker.js'; -import type { SearchResult } from '../src/types/index.js'; +import type { CodeChunk, SearchResult } from '../src/types/index.js'; +import { asSearcherTestHarness, materializeSourceSnapshot } from './helpers/source-snapshot.js'; function makeResult(score: number, filePath: string): SearchResult { return { @@ -29,23 +30,20 @@ describe('Reranker ambiguity detection', () => { const results = [ makeResult(0.95, '/a.ts'), makeResult(0.75, '/b.ts'), - makeResult(0.60, '/c.ts') + makeResult(0.6, '/c.ts') ]; expect(isAmbiguous(results)).toBe(false); }); it('returns false for fewer than 3 results', () => { - const results = [ - makeResult(0.85, '/a.ts'), - makeResult(0.84, '/b.ts') - ]; + const results = [makeResult(0.85, '/a.ts'), makeResult(0.84, '/b.ts')]; expect(isAmbiguous(results)).toBe(false); }); it('correctly handles edge case at threshold boundary', () => { // Gap of exactly 0.08 is NOT below the < 0.08 threshold const results = [ - makeResult(0.90, '/a.ts'), + makeResult(0.9, '/a.ts'), makeResult(0.85, '/b.ts'), makeResult(0.82, '/c.ts') ]; @@ -53,7 +51,7 @@ describe('Reranker ambiguity detection', () => { // Gap of 0.07 IS below the threshold const ambiguous = [ - makeResult(0.90, '/a.ts'), + makeResult(0.9, '/a.ts'), makeResult(0.86, '/b.ts'), makeResult(0.83, '/c.ts') ]; @@ -67,93 +65,94 @@ describe('File-level dedupe in search results', () => { // The dedupe logic is in scoreAndSortResults — tested indirectly by // ensuring results contain unique file paths const { CodebaseSearcher } = await import('../src/core/search.js'); - const { vi } = await import('vitest'); - - const searcher = new CodebaseSearcher('C:/repo') as any; - searcher.initialized = true; - searcher.embeddingProvider = { - embed: vi.fn(async () => [0.1, 0.2]) - }; - searcher.storageProvider = { - search: vi.fn(async () => [ - { - chunk: { - id: 'chunk1', - content: 'class Foo {}', - filePath: 'C:/repo/src/foo.ts', - relativePath: 'src/foo.ts', - startLine: 1, - endLine: 10, - language: 'typescript', - framework: 'generic', - componentType: 'service', - layer: 'core', - dependencies: [], - imports: [], - exports: [], - tags: [], - metadata: {} - }, - score: 0.9 - }, - { - chunk: { - id: 'chunk2', - content: 'class Bar extends Foo {}', - filePath: 'C:/repo/src/foo.ts', // same file, different chunk - relativePath: 'src/foo.ts', - startLine: 11, - endLine: 20, - language: 'typescript', - framework: 'generic', - componentType: 'service', - layer: 'core', - dependencies: [], - imports: [], - exports: [], - tags: [], - metadata: {} - }, - score: 0.85 - }, - { - chunk: { - id: 'chunk3', - content: 'class Baz {}', - filePath: 'C:/repo/src/baz.ts', // different file - relativePath: 'src/baz.ts', - startLine: 1, - endLine: 10, - language: 'typescript', - framework: 'generic', - componentType: 'service', - layer: 'core', - dependencies: [], - imports: [], - exports: [], - tags: [], - metadata: {} - }, - score: 0.80 - } - ]), - count: vi.fn(async () => 3) - }; - searcher.fuseIndex = null; - searcher.patternIntelligence = null; + const chunks: CodeChunk[] = [ + { + id: 'chunk1', + content: 'class Foo {}', + filePath: 'C:/repo/src/foo.ts', + relativePath: 'src/foo.ts', + startLine: 1, + endLine: 1, + language: 'typescript', + framework: 'generic', + componentType: 'service', + layer: 'core', + dependencies: [], + imports: [], + exports: [], + tags: [], + metadata: {} + }, + { + id: 'chunk2', + content: 'class Bar extends Foo {}', + filePath: 'C:/repo/src/foo.ts', + relativePath: 'src/foo.ts', + startLine: 11, + endLine: 11, + language: 'typescript', + framework: 'generic', + componentType: 'service', + layer: 'core', + dependencies: [], + imports: [], + exports: [], + tags: [], + metadata: {} + }, + { + id: 'chunk3', + content: 'class Baz {}', + filePath: 'C:/repo/src/baz.ts', + relativePath: 'src/baz.ts', + startLine: 1, + endLine: 1, + language: 'typescript', + framework: 'generic', + componentType: 'service', + layer: 'core', + dependencies: [], + imports: [], + exports: [], + tags: [], + metadata: {} + } + ]; + const snapshot = await materializeSourceSnapshot(chunks, 'reranker-source-'); + try { + const searcher = new CodebaseSearcher(snapshot.rootPath); + const harness = asSearcherTestHarness(searcher); + harness.initialized = true; + harness.sourceManifest = snapshot.manifest; + harness.embeddingProvider = { + embed: vi.fn(async () => [0.1, 0.2]) + }; + harness.storageProvider = { + search: vi.fn(async () => [ + { chunk: chunks[0], score: 0.9 }, + { chunk: chunks[1], score: 0.85 }, + { chunk: chunks[2], score: 0.8 } + ]), + count: vi.fn(async () => 3) + }; + harness.fuseIndex = null; + harness.patternIntelligence = null; - const results = await (searcher as any).search('Foo class', 5, undefined, { - useSemanticSearch: true, - useKeywordSearch: false, - enableReranker: false, - enableLowConfidenceRescue: false, - enableQueryExpansion: false - }); + const results = await searcher.search('Foo class', 5, undefined, { + useSemanticSearch: true, + useKeywordSearch: false, + enableReranker: false, + enableLowConfidenceRescue: false, + enableQueryExpansion: false + }); - // Should have 2 unique files, not 3 results (two from foo.ts) - const filePaths = results.map((r: any) => r.filePath); - expect(filePaths.length).toBe(2); - expect(filePaths[0]).toContain('foo.ts'); - expect(filePaths[1]).toContain('baz.ts'); + // Should have 2 unique files, not 3 results (two from foo.ts) + const filePaths = results.map((result) => result.filePath); + expect(filePaths.length).toBe(2); + expect(filePaths[0]).toContain('foo.ts'); + expect(filePaths[1]).toContain('baz.ts'); + } finally { + await snapshot.cleanup(); + } }); }); diff --git a/tests/review-context.test.ts b/tests/review-context.test.ts index 52dca41..c097455 100644 --- a/tests/review-context.test.ts +++ b/tests/review-context.test.ts @@ -13,7 +13,9 @@ import type { SearchResponse } from '../src/tools/types.js'; describe('review context', () => { it('parses modified, added, deleted and renamed files', () => { expect( - parseNameStatus(['M\tsrc/a.ts', 'A\tsrc/b.ts', 'D\tsrc/c.ts', 'R100\tsrc/old.ts\tsrc/new.ts'].join('\n')) + parseNameStatus( + ['M\tsrc/a.ts', 'A\tsrc/b.ts', 'D\tsrc/c.ts', 'R100\tsrc/old.ts\tsrc/new.ts'].join('\n') + ) ).toEqual([ { path: 'src/a.ts', status: 'M', rawStatus: 'M' }, { path: 'src/b.ts', status: 'A', rawStatus: 'A' }, @@ -54,7 +56,9 @@ describe('review context', () => { }); it('detects binary patches', () => { - expect(inspectPatch('diff --git a/a.png b/a.png\nBinary files a/a.png and b/a.png differ')).toEqual({ + expect( + inspectPatch('diff --git a/a.png b/a.png\nBinary files a/a.png and b/a.png differ') + ).toEqual({ additions: 0, deletions: 0, binary: true @@ -119,6 +123,45 @@ describe('review context', () => { expect(fingerprintDiff('diff body\n')).not.toBe(fingerprintDiff('diff body')); }); + it.each(['partial', 'no_fresh_results'] as const)( + 'keeps %s source coverage visible to review-context consumers', + async (status) => { + const rawDiff = '@@ -1 +1 @@\n-oldSession();\n+rotateSessionToken();'; + const sourceCoverage = { + status, + checkedFiles: status === 'partial' ? 2 : 1, + skippedFiles: 1, + refreshAction: 'refresh_index' + }; + const packet = await buildReviewContextPacket({ + base: 'main', + head: 'HEAD', + baseCommit: 'a'.repeat(40), + headCommit: 'b'.repeat(40), + rawDiff, + changedFiles: [{ path: 'src/auth.ts', status: 'M', rawStatus: 'M' }], + patchesByPath: new Map([['src/auth.ts', rawDiff]]), + search: async () => ({ + status: 'ok', + searchQuality: { status: 'ok', confidence: 0.9 }, + sourceCoverage, + preflight: { ready: false, reason: 'Indexed candidates are no longer current.' }, + results: + status === 'partial' + ? [{ file: 'src/current.ts:1-2', summary: 'Current source', score: 0.9 }] + : [] + }) + }); + + expect(packet.searches[0]?.sourceCoverage).toEqual(sourceCoverage); + expect(packet.searches[0]?.preflight?.ready).toBe(false); + expect(packet.warnings).toContain( + 'Search source coverage is incomplete for query: rotateSessionToken oldSession' + ); + expect(packet.searches[0]?.results).toHaveLength(status === 'partial' ? 1 : 0); + } + ); + it('builds a bounded packet and keeps failed searches explicit', async () => { const rawDiff = [ 'diff --git a/src/auth.ts b/src/auth.ts', diff --git a/tests/review-repair-regressions.test.ts b/tests/review-repair-regressions.test.ts new file mode 100644 index 0000000..549db68 --- /dev/null +++ b/tests/review-repair-regressions.test.ts @@ -0,0 +1,489 @@ +import { promises as fs } from 'node:fs'; +import path from 'node:path'; +import os from 'node:os'; +import { afterEach, beforeAll, expect, it, vi } from 'vitest'; +import chokidar from 'chokidar'; +import { z } from 'zod'; +import { CodebaseIndexer } from '../src/core/indexer.js'; +import { CodebaseSearcher, type SearchOptions } from '../src/core/search.js'; +import { analyzerRegistry } from '../src/core/analyzer-registry.js'; +import { GenericAnalyzer } from '../src/analyzers/generic/index.js'; +import { startFileWatcher } from '../src/core/file-watcher.js'; +import { readReferenceManifest } from '../src/core/manifest.js'; +import { readIndexMeta } from '../src/core/index-meta.js'; +import { resolveIndexArtifact, withIndexSnapshot } from '../src/core/index-generation.js'; +import { IndexCorruptedError } from '../src/errors/index.js'; +import { makePaths } from '../src/project-state.js'; +import { handle as relationships } from '../src/tools/get-code-relationships.js'; +import type { ToolContext } from '../src/tools/types.js'; +import * as embeddingProviders from '../src/embeddings/index.js'; +import { InternalSourceResolver } from '../src/utils/internal-source-resolver.js'; +import * as reranker from '../src/core/reranker.js'; +import * as searchQuality from '../src/core/search-quality.js'; +import Fuse from 'fuse.js'; + +const roots: string[] = []; +const RelationshipReply = z + .object({ + status: z.string(), + paths: z.array(z.unknown()), + unresolved: z.array(z.unknown()), + truncated: z.boolean(), + outputNote: z.string().optional() + }) + .passthrough(); +const StoredKeyword = z + .object({ + chunks: z.array( + z.object({ content: z.string(), metadata: z.record(z.string(), z.unknown()) }).passthrough() + ) + }) + .passthrough(); +beforeAll(() => analyzerRegistry.register(new GenericAnalyzer())); +afterEach(async () => { + vi.restoreAllMocks(); + for (const root of roots.splice(0)) await fs.rm(root, { recursive: true, force: true }); +}); + +it('confines watcher exclusions to descendants, including Windows case and different-drive paths', async () => { + const root = await fixture(); + const watch = vi.spyOn(chokidar, 'watch'); + const stop = startFileWatcher({ rootPath: root, onChanged: () => undefined }); + try { + const ignored = watch.mock.calls[0]?.[1]?.ignored; + if (typeof ignored !== 'function') + throw new Error('Watcher must supply its root-scoped predicate'); + expect(ignored(root)).toBe(false); + expect(ignored(path.join(root, 'src', 'main.ts'))).toBe(false); + expect(ignored(path.join(root, 'node_modules', 'nested.ts'))).toBe(true); + expect(ignored(path.join(root, '..', 'sibling', 'main.ts'))).toBe(true); + if (process.platform === 'win32') { + expect(ignored(root.toUpperCase())).toBe(false); + expect(ignored(path.join(root.toUpperCase(), 'src', 'main.ts'))).toBe(false); + const drive = path.parse(root).root.toUpperCase().startsWith('C:') ? 'D:' : 'C:'; + expect(ignored(`${drive}\\elsewhere\\main.ts`)).toBe(true); + } + } finally { + await stop(); + } +}); + +it('rejects a mismatched source manifest before attempting a recorded semantic provider', async () => { + const root = await fixture(); + await write(root, 'main.ts', 'export const quartzBeacon = true;\n'); + await index(root); + const meta = await readIndexMeta(root); + const manifestPath = await resolveIndexArtifact(root, 'manifest.json'); + const original = await fs.readFile(manifestPath); + const manifest = await readReferenceManifest(root, meta); + if (manifest.header) manifest.header.buildId = 'foreign'; + meta.artifacts.vectorDb.embeddingEnabled = true; + await fs.writeFile(path.join(root, '.codebase-context', 'index-meta.json'), JSON.stringify(meta)); + await fs.writeFile(manifestPath, JSON.stringify(manifest)); + const provider = vi + .spyOn(embeddingProviders, 'getEmbeddingProvider') + .mockRejectedValue(new Error('Controlled unavailable model; no network')); + await expect(new CodebaseSearcher(root).initialize()).rejects.toBeInstanceOf(IndexCorruptedError); + expect(provider).not.toHaveBeenCalled(); + await fs.writeFile(manifestPath, original); + await new CodebaseSearcher(root).initialize(); + expect(provider).toHaveBeenCalledTimes(1); +}); + +it('keeps exact candidate bodies through real keyword rescue and test fallback, then a reranker permutation', async () => { + const root = await fixture(); + await write( + root, + 'main.ts', + 'function start(){ before(); const hooks={onReady:()=>routing()}; after(); }\n' + ); + await write( + root, + 'main.test.ts', + 'function check(){ beforeTest(); const hooks={onReady:()=>routing()}; afterTest(); }\n' + ); + await index(root); + const meta = await readIndexMeta(root); + // Admit the reranking path while making model initialization fail without any + // network/storage request. Actual file indexing and keyword retrieval are real. + meta.artifacts.vectorDb.embeddingEnabled = true; + await fs.writeFile(path.join(root, '.codebase-context', 'index-meta.json'), JSON.stringify(meta)); + vi.spyOn(embeddingProviders, 'getEmbeddingProvider').mockRejectedValue( + new Error('Controlled unavailable model') + ); + vi.spyOn(searchQuality, 'assessSearchQuality').mockReturnValue({ + status: 'low_confidence', + confidence: 0, + signals: ['Controlled branch selection'] + }); + const queries = vi.spyOn(Fuse.prototype, 'search'); + const rerank = vi + .spyOn(reranker, 'rerank') + .mockImplementation(async (_query, candidates) => candidates.slice().reverse()); + const result = await new CodebaseSearcher(root).searchWithCoverage('routing', 5, undefined, { + ...options, + enableLowConfidenceRescue: true, + enableReranker: true + }); + expect(rerank).toHaveBeenCalledTimes(1); + expect( + queries.mock.calls.some( + ([query]) => typeof query === 'string' && query !== 'routing' && query.includes('routing') + ) + ).toBe(true); + expect(result.results.map((hit) => path.basename(hit.filePath)).sort()).toEqual([ + 'main.test.ts', + 'main.ts' + ]); + for (const hit of result.results) { + expect(hit.metadata.symbolName).toBe('onReady'); + expect(hit.snippet).toBe('onReady:()=>routing()'); + } +}); + +it('serves a complete legacy fixed layout with a matching header and rejects a later mixed manifest', async () => { + const root = await fixture(); + await write(root, 'main.ts', 'export const quartzBeacon = true;\n'); + await index(root); + const meta = await readIndexMeta(root); + const context = path.join(root, '.codebase-context'); + for (const artifact of Object.values(meta.artifacts)) { + if (!artifact) continue; + const original = path.join(context, artifact.path); + artifact.path = path.basename(artifact.path); + await fs.cp(original, path.join(context, artifact.path), { recursive: true }); + } + await fs.writeFile(path.join(context, 'index-meta.json'), JSON.stringify(meta)); + const result = await new CodebaseSearcher(root).search('quartzBeacon', 5, undefined, options); + expect(result.length).toBeGreaterThan(0); + const manifest = await readReferenceManifest(root, meta); + if (manifest.header) manifest.header.buildId = 'another-legacy-build'; + await fs.writeFile(path.join(context, 'manifest.json'), JSON.stringify(manifest)); + await expect(new CodebaseSearcher(root).initialize()).rejects.toBeInstanceOf(IndexCorruptedError); +}); + +it.each(['escaping-path', 'malformed-date', 'missing-file'])( + 'rejects a %s source manifest with current build identity', + async (variant) => { + const root = await fixture(); + await write(root, 'main.ts', 'export const quartzBeacon = true;\n'); + await index(root); + const meta = await readIndexMeta(root); + const manifest = await readReferenceManifest(root, meta); + const manifestPath = await resolveIndexArtifact(root, 'manifest.json'); + if (variant === 'missing-file') await fs.unlink(manifestPath); + else { + if (variant === 'escaping-path') manifest.files['../outside.ts'] = '0123456789abcdef'; + else manifest.generatedAt = 'not a timestamp'; + await fs.writeFile(manifestPath, JSON.stringify(manifest)); + } + await expect(new CodebaseSearcher(root).initialize()).rejects.toBeInstanceOf( + IndexCorruptedError + ); + } +); + +async function graphQuery(rootPath: string) { + const ctx: ToolContext = { + rootPath, + paths: makePaths(rootPath), + indexState: { status: 'ready' }, + performIndexing: () => undefined + }; + const response = await relationships({ file: 'main.ts', direction: 'outgoing' }, ctx); + return RelationshipReply.parse(JSON.parse(response.content[0]?.text ?? '{}')); +} + +it('rechecks aliases and extension priority without changes to importer bytes', async () => { + const root = await fixture(); + await write(root, 'main.ts', "import '@late';\nimport './choice';\n"); + await write( + root, + 'tsconfig.json', + JSON.stringify({ compilerOptions: { baseUrl: '.', paths: { '@late': ['absent.ts'] } } }) + ); + await write(root, 'target.ts', 'export const target = true;\n'); + await write(root, 'choice.tsx', 'export const choice = true;\n'); + await index(root); + expect(await graphQuery(root)).toMatchObject({ status: 'success' }); + await write( + root, + 'tsconfig.json', + JSON.stringify({ compilerOptions: { baseUrl: '.', paths: { '@late': ['target.ts'] } } }) + ); + await write(root, 'choice.ts', 'export const alternate = true;\n'); + expect(await graphQuery(root)).toMatchObject({ + status: 'stale_edges', + paths: [], + staleUnresolvedEdges: [expect.objectContaining({ source: { line: 1, specifier: '@late' } })] + }); + await index(root); + expect(await graphQuery(root)).toMatchObject({ status: 'success', unresolved: [] }); +}); + +it('does not call an outside-root target current when it appears through a directory link', async () => { + const base = await fixture(); + const root = path.join(base, 'project'); + const outside = path.join(base, 'outside'); + await write(root, 'main.ts', "import './linked/target';\n"); + await write(outside, 'target.ts', 'export const outside = true;\n'); + await index(root); + expect(await graphQuery(root)).toMatchObject({ + status: 'success', + unresolved: [expect.objectContaining({ reason: 'not-found' })] + }); + await fs.symlink(outside, path.join(root, 'linked'), 'junction'); + expect(await graphQuery(root)).toMatchObject({ + status: 'stale_edges', + paths: [], + unresolved: [] + }); +}); + +it('resolves repeated unresolved proofs once per operation and observes later appearance', async () => { + const root = await fixture(); + await write(root, 'main.ts', "import './missing';\nimport './missing';\n"); + await index(root); + const resolve = vi.spyOn(InternalSourceResolver.prototype, 'resolve'); + expect(await graphQuery(root)).toMatchObject({ status: 'success' }); + expect(resolve).toHaveBeenCalledTimes(1); + resolve.mockClear(); + await write(root, 'missing.ts', 'export const available = true;\n'); + expect(await graphQuery(root)).toMatchObject({ status: 'stale_edges', paths: [] }); + expect(resolve).toHaveBeenCalledTimes(1); +}); + +it('checks only the unresolved candidates already admitted by the shared inspection allowance', async () => { + const root = await fixture(); + await write( + root, + 'main.ts', + Array.from({ length: 1005 }, (_, n) => `import './missing${n}';`).join('\n') + ); + await index(root); + const resolve = vi.spyOn(InternalSourceResolver.prototype, 'resolve'); + const result = await graphQuery(root); + expect(resolve).toHaveBeenCalledTimes(1000); + expect(result).toMatchObject({ status: 'success', truncated: true }); + expect(result.outputNote).toContain('inspection budget'); + expect(result.unresolved.length).toBeLessThanOrEqual(20); +}); +async function fixture() { + const root = await fs.mkdtemp(path.join(os.tmpdir(), 'cbc-review-repair-')); + roots.push(root); + return root; +} +async function write(root: string, file: string, content: string) { + await fs.mkdir(path.dirname(path.join(root, file)), { recursive: true }); + await fs.writeFile(path.join(root, file), content); +} +function index(rootPath: string) { + return new CodebaseIndexer({ + rootPath, + config: { skipEmbedding: true }, + projectOptions: { preferredAnalyzer: 'generic' } + }).index(); +} +const options: SearchOptions = { + useSemanticSearch: false, + useKeywordSearch: true, + enableQueryExpansion: false, + enableLowConfidenceRescue: false, + enableReranker: false, + profile: 'explore' +}; + +it.each(['clean-first', 'forged-first'])( + 'rejects a forged sibling after source caching (%s)', + async (order) => { + const root = await fixture(); + await write( + root, + 'main.ts', + 'function first(){const hooks={quartzBeacon:()=>readFirst()};}\nfunction second(){const hooks={quartzBeacon:()=>readSecond()};}\n' + ); + await index(root); + const file = await resolveIndexArtifact(root, 'index.json'); + const stored = StoredKeyword.parse(JSON.parse(await fs.readFile(file, 'utf8'))); + const callbacks = stored.chunks.filter((chunk) => chunk.metadata.symbolName === 'quartzBeacon'); + expect(callbacks).toHaveLength(2); + const forged = callbacks.find((chunk) => chunk.content.includes('readSecond')); + if (!forged) throw Error('Actual analyzer must produce the second callback'); + forged.content = '// forged context\n' + forged.content; + if (order === 'forged-first') stored.chunks.reverse(); + const bytes = JSON.stringify(stored); + await fs.writeFile(file, bytes); + await expect( + new CodebaseSearcher(root).search('quartzBeacon', 5, undefined, options) + ).rejects.toBeInstanceOf(IndexCorruptedError); + expect(await fs.readFile(file, 'utf8')).toBe(bytes); + } +); + +it('keeps valid presentation context in metadata separate from exact partial source bodies', async () => { + const root = await fixture(); + await write( + root, + 'main.ts', + 'function start(){ before(); const hooks={quartzBeacon:()=>openQueue("雪")}; after(); }\r\n' + ); + await index(root); + const file = await resolveIndexArtifact(root, 'index.json'); + const stored = StoredKeyword.parse(JSON.parse(await fs.readFile(file, 'utf8'))); + for (const chunk of stored.chunks) + chunk.metadata.scopeContext = '// indexing-only signature header'; + await fs.writeFile(file, JSON.stringify(stored)); + const result = await new CodebaseSearcher(root).search('quartzBeacon', 5, undefined, options); + expect(result[0]?.metadata.scopeContext).toBe('// indexing-only signature header'); + expect(result[0]?.snippet).toBe('quartzBeacon:()=>openQueue("雪")'); +}); + +it.each(['worktrees', '.worktrees', 'build', 'selected-root-is-build'])( + 'watches the selected project despite excluded ancestor %s, but excludes internal build', + async (name) => { + const base = await fixture(); + const root = + name === 'selected-root-is-build' + ? path.join(base, 'build') + : path.join(base, name, 'project'); + await write(root, 'main.ts', 'export const before = 1;\n'); + await write(root, 'build/internal.ts', 'export const excluded = 1;\n'); + const ready = Promise.withResolvers(); + const changed = Promise.withResolvers(); + let changes = 0; + const stop = startFileWatcher({ + rootPath: root, + debounceMs: 20, + onReady: ready.resolve, + onChanged: () => { + changes++; + changed.resolve(); + } + }); + try { + await ready.promise; + await write(root, 'main.ts', 'export const after = 2;\n'); + const observed = await Promise.race([ + changed.promise.then(() => true), + new Promise((resolve) => setTimeout(() => resolve(false), 1500)) + ]); + expect(observed).toBe(true); + await write(root, 'build/internal.ts', 'export const excluded = 2;\n'); + await new Promise((resolve) => setTimeout(resolve, 600)); + expect(changes).toBe(1); + } finally { + await stop(); + } + } +); + +it('retains the exact same-line callback owner body and abstains after drift/deletion', async () => { + const root = await fixture(); + const source = + 'function start(){ before(); const hooks={onReady:()=>openQueue()}; after(); }\r\nfunction again(){ beforeAgain(); const hooks={onReady:()=>openAgain("雪")}; afterAgain(); }\r\n'; + await write(root, 'main.ts', source); + await index(root); + const searcher = new CodebaseSearcher(root); + for (const [query, owner, expected] of [ + ['openQueue', 'start', 'onReady:()=>openQueue()'], + ['openAgain', 'again', 'onReady:()=>openAgain("雪")'] + ]) { + const result = await searcher.searchWithCoverage(query, 5, undefined, options); + expect(result.sourceCoverage.status).toBe('current'); + expect(result.results[0]?.metadata.symbolPath).toContain(owner); + expect(result.results[0]?.snippet).toBe(expected); + } + await write(root, 'main.ts', source.replace('openQueue()', 'changedQueue()')); + expect( + (await searcher.searchWithCoverage('openQueue', 5, undefined, options)).sourceCoverage.status + ).toBe('no_fresh_results'); + await fs.unlink(path.join(root, 'main.ts')); + expect((await searcher.searchWithCoverage('openAgain', 5, undefined, options)).results).toEqual( + [] + ); +}); + +it.each(['foreign', 'missing', 'wrong-format'])( + 'rejects a %s manifest header even when every current source hash matches', + async (variant) => { + const root = await fixture(); + await write(root, 'main.ts', 'export function quartzBeacon() { return "alive"; }\n'); + await index(root); + const meta = await readIndexMeta(root); + const manifest = await readReferenceManifest(root, meta); + if (variant === 'missing') delete manifest.header; + else if (manifest.header) { + if (variant === 'foreign') manifest.header.buildId = 'another-generation'; + else manifest.header.formatVersion--; + } + await fs.writeFile(await resolveIndexArtifact(root, 'manifest.json'), JSON.stringify(manifest)); + await expect( + new CodebaseSearcher(root).search('quartzBeacon', 5, undefined, options) + ).rejects.toBeInstanceOf(IndexCorruptedError); + } +); + +it('keeps manifest reads pinned during publication and refreshes an existing searcher on its next operation', async () => { + const root = await fixture(); + await write(root, 'main.ts', 'export const quartzBeacon = 1;\n'); + await index(root); + const searcher = new CodebaseSearcher(root); + expect((await searcher.search('quartzBeacon', 5, undefined, options)).length).toBeGreaterThan(0); + const entered = Promise.withResolvers(); + const release = Promise.withResolvers(); + const reader = withIndexSnapshot(root, async () => { + const old = await readIndexMeta(root); + entered.resolve(); + await release.promise; + expect((await readReferenceManifest(root, old)).header?.buildId).toBe(old.buildId); + }); + try { + await entered.promise; + await write(root, 'main.ts', 'export const newBeacon = 2;\n'); + await index(root); + const current = await readIndexMeta(root); + const manifest = await readReferenceManifest(root, current); + if (manifest.header) manifest.header.buildId = 'mixed-new-manifest'; + await fs.writeFile(await resolveIndexArtifact(root, 'manifest.json'), JSON.stringify(manifest)); + await expect(searcher.search('newBeacon', 5, undefined, options)).rejects.toBeInstanceOf( + IndexCorruptedError + ); + } finally { + release.resolve(); + await reader; + } +}); + +it.each([ + { specifier: './missing', target: 'missing.ts' }, + { specifier: './missing', target: 'missing.tsx' }, + { specifier: './missing.js', target: 'missing.ts' } +])( + 'marks an unresolved $specifier stale when $target appears, then follows it only after refresh', + async ({ specifier, target }) => { + const root = await fixture(); + await write(root, 'main.ts', `import '${specifier}';\nexport const value = true;\n`); + await index(root); + const query = () => graphQuery(root); + expect(await query()).toMatchObject({ + status: 'success', + paths: [], + unresolved: [expect.objectContaining({ reason: 'not-found' })] + }); + await write(root, target, 'export const target = true;\n'); + expect(await query()).toMatchObject({ + status: 'stale_edges', + paths: [], + unresolved: [], + staleUnresolvedEdges: [expect.objectContaining({ reason: 'not-found' })] + }); + await index(root); + expect(await query()).toMatchObject({ + status: 'success', + paths: [expect.objectContaining({ nodes: ['main.ts', target] })], + unresolved: [] + }); + await fs.unlink(path.join(root, target)); + expect(await query()).toMatchObject({ status: 'stale_edges', paths: [] }); + } +); diff --git a/tests/search-codebase-auto-heal.test.ts b/tests/search-codebase-auto-heal.test.ts index 13cd1a0..67a4731 100644 --- a/tests/search-codebase-auto-heal.test.ts +++ b/tests/search-codebase-auto-heal.test.ts @@ -1,3 +1,4 @@ +import { resolveIndexArtifact } from '../src/core/index-generation.js'; import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'; import { promises as fs } from 'fs'; import os from 'os'; @@ -19,6 +20,21 @@ const indexerMocks = vi.hoisted(() => ({ index: vi.fn() })); +function getToolsCallHandler(server: unknown) { + const handlers = ( + server as { + _requestHandlers: { + get( + name: string + ): ((request: unknown) => Promise<{ content: Array<{ text: string }> }>) | undefined; + }; + } + )._requestHandlers; + const handler = handlers.get('tools/call'); + if (!handler) throw new Error('tools/call handler not registered'); + return handler; +} + vi.mock('../src/core/search.js', async () => { class CodebaseSearcher { constructor(_rootPath: string) {} @@ -26,6 +42,13 @@ vi.mock('../src/core/search.js', async () => { async search(query: string, limit: number, filters?: unknown) { return searchMocks.search(query, limit, filters); } + + async searchWithCoverage(query: string, limit: number, filters?: unknown) { + return { + results: await this.search(query, limit, filters), + sourceCoverage: { status: 'current', checkedFiles: 0, skippedFiles: 0 } + }; + } } return { CodebaseSearcher }; @@ -40,7 +63,7 @@ vi.mock('../src/core/indexer.js', () => { } async index() { - indexerMocks.index(); + await indexerMocks.index(); return { totalFiles: 0, indexedFiles: 0, @@ -143,37 +166,112 @@ describe('search_codebase auto-heal', () => { } }); - it('fires background re-index and returns retry message when IndexCorruptedError is thrown', async () => { - const { IndexCorruptedError } = await import('../src/errors/index.js'); + it.each([ + [ + 'missing initial index metadata', + 'Index meta missing or unreadable (rebuild required): ENOENT' + ], + ['genuine index corruption', 'LanceDB index corrupted: missing vector column'] + ])( + 'starts background re-index and returns retry guidance for %s', + async (_scenario, errorMessage) => { + const { IndexCorruptedError } = await import('../src/errors/index.js'); + + searchMocks.search.mockRejectedValueOnce(new IndexCorruptedError(errorMessage)); + const releaseIndexing = Promise.withResolvers(); + indexerMocks.index.mockImplementationOnce(() => releaseIndexing.promise); + + const { server } = await import('../src/index.js'); + const handler = (server as any)._requestHandlers.get('tools/call'); + + let response; + try { + response = await handler({ + jsonrpc: '2.0', + id: 1, + method: 'tools/call', + params: { + name: 'search_codebase', + arguments: { + query: 'test' + } + } + }); + + const payload = JSON.parse(response.content[0].text); + expect(payload.status).toBe('indexing'); + expect(payload.message).toBe( + 'Index is unavailable or not ready. Indexing has started. Check indexing status, then retry.' + ); + expect(searchMocks.search).toHaveBeenCalledTimes(1); + expect(payload.index).toMatchObject({ + status: 'indexing', + confidence: 'low', + action: 'rebuild-started' + }); + expect(payload.project.indexStatus).toBe('indexing'); + } finally { + releaseIndexing.resolve(); + } + const { getOrCreateProject } = await import('../src/project-state.js'); + const deadline = Date.now() + 1000; + while (getOrCreateProject(tempRoot!).indexState.status === 'indexing') { + if (Date.now() >= deadline) throw new Error('Controlled index fixture did not finish'); + await new Promise((resolve) => setTimeout(resolve, 5)); + } + }, + 15000 + ); + + it('returns status-check guidance without searching during an initial uncommitted build', async () => { + if (!tempRoot) throw new Error('tempRoot not initialized'); - searchMocks.search.mockRejectedValueOnce( - new IndexCorruptedError('LanceDB index corrupted: missing vector column') + await fs.rm(path.join(tempRoot, CODEBASE_CONTEXT_DIRNAME, INDEX_META_FILENAME)); + const { handle } = await import('../src/tools/search-codebase.js'); + const baseDir = path.join(tempRoot, CODEBASE_CONTEXT_DIRNAME); + const response = await handle( + { query: 'search something' }, + { + indexState: { status: 'indexing' }, + paths: { + baseDir, + memory: path.join(baseDir, 'memory.json'), + intelligence: await resolveIndexArtifact(path.dirname(baseDir), 'intelligence.json'), + health: await resolveIndexArtifact(path.dirname(baseDir), 'health.json'), + keywordIndex: await resolveIndexArtifact(path.dirname(baseDir), KEYWORD_INDEX_FILENAME), + vectorDb: await resolveIndexArtifact(path.dirname(baseDir), VECTOR_DB_DIRNAME) + }, + rootPath: tempRoot, + performIndexing: vi.fn() + } ); - const { server } = await import('../src/index.js'); - const handler = (server as any)._requestHandlers.get('tools/call'); + const payload = JSON.parse(response.content?.[0]?.text ?? '{}'); + expect(payload.status).toBe('indexing'); + expect(payload.message).toBe('Index build in progress. Check indexing status, then retry.'); + expect(searchMocks.search).not.toHaveBeenCalled(); + }); + it('rebuilds an existing semantic index before search when startup explicitly requests model-free mode', async () => { + process.argv.push('--no-embeddings'); + const { server } = await import('../src/index.js'); + const handler = getToolsCallHandler(server); const response = await handler({ jsonrpc: '2.0', - id: 1, + id: 2, method: 'tools/call', - params: { - name: 'search_codebase', - arguments: { - query: 'test' - } - } + params: { name: 'search_codebase', arguments: { query: 'test' } } }); - const payload = JSON.parse(response.content[0].text); expect(payload.status).toBe('indexing'); expect(payload.message).toContain('retry'); - expect(searchMocks.search).toHaveBeenCalledTimes(1); + expect(indexerMocks.index).toHaveBeenCalledTimes(1); + expect(searchMocks.search).not.toHaveBeenCalled(); }, 15000); it('returns invalid_params when search_codebase query is missing', async () => { const { server } = await import('../src/index.js'); - const handler = (server as any)._requestHandlers.get('tools/call'); + const handler = getToolsCallHandler(server); const response = await handler({ jsonrpc: '2.0', @@ -213,7 +311,7 @@ describe('search_codebase auto-heal', () => { ); const { server } = await import('../src/index.js'); - const handler = (server as any)._requestHandlers.get('tools/call'); + const handler = getToolsCallHandler(server); const response = await handler({ jsonrpc: '2.0', diff --git a/tests/search-compact-mode.test.ts b/tests/search-compact-mode.test.ts index c4d573c..267db28 100644 --- a/tests/search-compact-mode.test.ts +++ b/tests/search-compact-mode.test.ts @@ -2,7 +2,10 @@ import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'; import { promises as fs } from 'fs'; import os from 'os'; import path from 'path'; +import { hashFileContent, writeManifest } from '../src/core/manifest.js'; +import { IndexMetaSchema } from '../src/core/index-meta.js'; import type { SearchResponse } from '../src/tools/types.js'; +import structuralContextFixture from './fixtures/structural-context-regressions.json'; import { CODEBASE_CONTEXT_DIRNAME, INDEX_FORMAT_VERSION, @@ -10,6 +13,7 @@ import { INDEX_META_VERSION, INTELLIGENCE_FILENAME, KEYWORD_INDEX_FILENAME, + MANIFEST_FILENAME, RELATIONSHIPS_FILENAME, VECTOR_DB_DIRNAME } from '../src/constants/codebase-context.js'; @@ -25,6 +29,13 @@ vi.mock('../src/core/search.js', async () => { async search(query: string, limit: number, filters?: unknown) { return searchMocks.search(query, limit, filters); } + + async searchWithCoverage(query: string, limit: number, filters?: unknown) { + return { + results: await this.search(query, limit, filters), + sourceCoverage: { status: 'current', checkedFiles: 0, skippedFiles: 0 } + }; + } } return { CodebaseSearcher }; @@ -98,7 +109,8 @@ describe('search_codebase compact/full mode', () => { toolVersion: 'test', artifacts: { keywordIndex: { path: KEYWORD_INDEX_FILENAME }, - vectorDb: { path: VECTOR_DB_DIRNAME, provider: 'lancedb' }, + // This keyword-only fixture has no semantic table. + vectorDb: { path: VECTOR_DB_DIRNAME, provider: 'lancedb', embeddingEnabled: false }, intelligence: { path: INTELLIGENCE_FILENAME } } }, @@ -125,7 +137,11 @@ describe('search_codebase compact/full mode', () => { primary: { name: 'Signals', frequency: '78%', - trend: 'Rising' + trend: 'Rising', + canonicalExample: { + file: 'src/auth/auth.service.ts', + snippet: 'export class AuthService { getToken() { return token; } }' + } } }, componentArchitecture: { @@ -287,7 +303,9 @@ describe('search_codebase compact/full mode', () => { // Test 4: Compact response includes budget, patternSummary, bestExample, nextHops it('compact response includes budget, patternSummary, bestExample, and nextHops', async () => { - searchMocks.search.mockResolvedValueOnce([makeResult()]); + searchMocks.search.mockResolvedValueOnce([ + makeResult({ filePath: path.join(tempRoot!, 'src/auth/auth.service.ts') }) + ]); const { server } = await import('../src/index.js'); const handler = ( @@ -310,14 +328,168 @@ describe('search_codebase compact/full mode', () => { const payload = parseSearchResponse(response.content[0].text); expect(payload.budget).toBeDefined(); expect(payload.budget?.mode).toBe('compact'); - // patternSummary should include pattern info from intelligence.json + // Only patterns backed by source in a returned result belong in this summary. expect(typeof payload.patternSummary).toBe('string'); expect(payload.patternSummary).toContain('Signals'); + expect(payload.patternSummary).not.toContain('Standalone Components'); // bestExample should be the top golden file expect(payload.bestExample).toBe('src/auth/auth.service.ts'); // nextHops should have at least 1 entry expect(Array.isArray(payload.nextHops)).toBe(true); expect(payload.nextHops?.length ?? 0).toBeGreaterThan(0); + expect(payload.nextHops?.some((hop) => hop.tool === 'read_file')).toBe(false); + expect(payload.nextHops?.[0].tool).toBe('get_code_relationships'); + }); + + it('does not recommend an unrelated golden example for a different source result', async () => { + searchMocks.search.mockResolvedValueOnce([ + makeResult({ filePath: path.join(tempRoot!, 'src/cli-init.ts') }) + ]); + const { server } = await import('../src/index.js'); + const handler = ( + server as unknown as { + _requestHandlers?: Map< + string, + (r: unknown) => Promise<{ content: Array<{ type: string; text: string }> }> + >; + } + )._requestHandlers?.get('tools/call'); + if (!handler) throw new Error('Expected tools/call handler'); + const response = await handler({ + jsonrpc: '2.0', + id: 1, + method: 'tools/call', + params: { name: 'search_codebase', arguments: { query: 'setup wizard' } } + }); + const payload = parseSearchResponse(response.content[0].text); + expect(payload.bestExample).toBeUndefined(); + expect(payload.nextHops?.some((hop) => hop.tool === 'read_file')).toBe(false); + }); + + it('omits stale pattern examples from summaries, advice, and evidence', async () => { + const intelligencePath = path.join(tempRoot!, CODEBASE_CONTEXT_DIRNAME, INTELLIGENCE_FILENAME); + const intelligence = JSON.parse(await fs.readFile(intelligencePath, 'utf-8')) as { + patterns: Record< + string, + { primary?: Record; alsoDetected?: Array> } + >; + }; + intelligence.patterns.stateManagement.primary = { + name: 'Signals', + frequency: '78%', + trend: 'Rising', + canonicalExample: { + file: 'src/auth/auth.service.ts', + snippet: 'export function obsoleteSignalStore() {}' + } + }; + intelligence.patterns.stateManagement.alsoDetected = [ + { + name: 'Legacy Store', + frequency: '22%', + trend: 'Declining', + canonicalExample: { + file: 'src/auth/auth.service.ts', + snippet: 'export function obsoleteLegacyStore() {}' + } + } + ]; + await fs.writeFile(intelligencePath, JSON.stringify(intelligence), 'utf-8'); + searchMocks.search.mockResolvedValueOnce([ + makeResult(), + makeResult({ filePath: 'src/auth/token.service.ts' }), + makeResult({ filePath: 'src/auth/session.service.ts' }) + ]); + + const { server } = await import('../src/index.js'); + const handler = ( + server as { + _requestHandlers?: Map< + string, + (r: unknown) => Promise<{ content: Array<{ type: string; text: string }> }> + >; + } + )._requestHandlers?.get('tools/call'); + if (!handler) throw new Error('Expected tools/call handler'); + + const response = await handler({ + jsonrpc: '2.0', + id: 1, + method: 'tools/call', + params: { + name: 'search_codebase', + arguments: { query: 'auth service', intent: 'edit' } + } + }); + const payload = parseSearchResponse(response.content[0].text); + expect(payload.patternSummary).toBeUndefined(); + expect(payload.preflight?.ready).toBe(false); + expect(payload.preflight?.patterns).toBeUndefined(); + }); + + it('keeps advice for a current canonical example and excludes unrelated patterns', async () => { + searchMocks.search.mockResolvedValueOnce([makeResult()]); + + const { server } = await import('../src/index.js'); + const handler = ( + server as { + _requestHandlers?: Map< + string, + (r: unknown) => Promise<{ content: Array<{ type: string; text: string }> }> + >; + } + )._requestHandlers?.get('tools/call'); + if (!handler) throw new Error('Expected tools/call handler'); + + const response = await handler({ + jsonrpc: '2.0', + id: 1, + method: 'tools/call', + params: { + name: 'search_codebase', + arguments: { query: 'auth service', intent: 'edit' } + } + }); + const payload = parseSearchResponse(response.content[0].text); + expect(payload.searchQuality?.status).toBe('ok'); + expect(payload.preflight?.patterns?.do).toContain('Signals'); + expect(payload.preflight?.patterns?.do).not.toContain('Standalone Components'); + expect(payload.preflight?.patterns?.do?.some((pattern) => pattern.includes('adoption'))).toBe( + false + ); + }); + + it('suppresses grounded pattern advice when search quality requires abstention', async () => { + searchMocks.search.mockResolvedValueOnce([ + makeResult({ score: 0.2 }), + makeResult({ filePath: 'src/auth/auth.service2.ts', score: 0.19 }), + makeResult({ filePath: 'src/auth/auth.service3.ts', score: 0.18 }) + ]); + + const { server } = await import('../src/index.js'); + const handler = ( + server as { + _requestHandlers?: Map< + string, + (r: unknown) => Promise<{ content: Array<{ type: string; text: string }> }> + >; + } + )._requestHandlers?.get('tools/call'); + if (!handler) throw new Error('Expected tools/call handler'); + + const response = await handler({ + jsonrpc: '2.0', + id: 1, + method: 'tools/call', + params: { + name: 'search_codebase', + arguments: { query: 'auth service', intent: 'edit' } + } + }); + const payload = parseSearchResponse(response.content[0].text); + expect(payload.searchQuality?.status).toBe('low_confidence'); + expect(payload.preflight?.abstain).toBe(true); + expect(payload.preflight?.patterns).toBeUndefined(); }); it('compact mode serializes symbol intelligence fields from chunk metadata', async () => { @@ -361,6 +533,107 @@ describe('search_codebase compact/full mode', () => { ); }); + it.each([ + { + id: 'test-companion-midchunk-line-is-supporting-context', + query: 'accountId', + context: [ + "import { parseInvoice } from './invoice-parser';", + "import { fixture } from './invoice-fixture';", + 'const options = {};' + ] + }, + { + id: 'declaration-companion-midchunk-line-is-supporting-context', + query: 'InvoiceParser parse', + context: [ + "import type { Invoice } from './invoice';", + "import type { ParserOptions } from './parser-options';", + 'const source = readSource();' + ] + } + ])('compact previews retain query-relevant source lines for $id', async (testCase) => { + const fixtureCase = structuralContextFixture.cases.find((entry) => entry.id === testCase.id); + if (!fixtureCase) throw new Error(`Missing structural fixture case: ${testCase.id}`); + const marker = fixtureCase.supportingOnlyMarkers?.[0]; + if (!marker) throw new Error(`Missing supporting source marker: ${testCase.id}`); + + searchMocks.search.mockResolvedValueOnce([ + makeResult({ snippet: [...testCase.context, fixtureCase.source].join('\n') }) + ]); + + const { server } = await import('../src/index.js'); + const handler = ( + server as { + _requestHandlers?: Map< + string, + (request: unknown) => Promise<{ content: Array<{ type: string; text: string }> }> + >; + } + )._requestHandlers?.get('tools/call'); + if (!handler) throw new Error('Expected tools/call handler'); + + const response = await handler({ + jsonrpc: '2.0', + id: 1, + method: 'tools/call', + params: { + name: 'search_codebase', + arguments: { query: testCase.query } + } + }); + const payload = parseSearchResponse(response.content[0].text); + const preview = payload.results[0]?.signaturePreview; + + expect(preview).toContain(marker); + expect((preview ?? '').split('\n').length).toBeLessThanOrEqual(3); + }); + + it('keeps compact preview lines contiguous around the strongest source match', async () => { + const fixtureCase = structuralContextFixture.cases.find( + (entry) => entry.id === 'method-body-has-interior-witness-and-sibling-control' + ); + if (!fixtureCase) throw new Error('Missing bounded method-body fixture case'); + + searchMocks.search.mockResolvedValueOnce([ + makeResult({ snippet: fixtureCase.source, startLine: 1, endLine: 12 }) + ]); + + const { server } = await import('../src/index.js'); + const handler = ( + server as { + _requestHandlers?: Map< + string, + (request: unknown) => Promise<{ content: Array<{ type: string; text: string }> }> + >; + } + )._requestHandlers?.get('tools/call'); + if (!handler) throw new Error('Expected tools/call handler'); + + const response = await handler({ + jsonrpc: '2.0', + id: 1, + method: 'tools/call', + params: { + name: 'search_codebase', + arguments: { query: 'beginImport finishImport saved' } + } + }); + const payload = parseSearchResponse(response.content[0].text); + const previewLines = (payload.results[0]?.signaturePreview ?? '') + .split('\n') + .map((line) => line.trim()); + const sourceLines = fixtureCase.source.split('\n').map((line) => line.trim()); + const previewStart = sourceLines.indexOf(previewLines[0] ?? ''); + + expect(previewLines).toContain('finishImport(saved);'); + expect(previewLines.length).toBeGreaterThan(0); + expect(previewLines.length).toBeLessThanOrEqual(3); + expect(sourceLines.slice(previewStart, previewStart + previewLines.length)).toEqual( + previewLines + ); + }); + it('adds an exact tokenEstimate advisory to compact responses', async () => { searchMocks.search.mockResolvedValueOnce([makeResult()]); @@ -517,64 +790,83 @@ describe('search_codebase compact/full mode', () => { expect(payload.results[0].complexity).toBe(7); }); - it('real CodebaseSearcher preserves chunk imports and exports', async () => { - if (!tempRoot) throw new Error('tempRoot not initialized'); - - const ctxDir = path.join(tempRoot, CODEBASE_CONTEXT_DIRNAME); - const actualChunk = { - id: 'auth-chunk', - content: - 'import { tokenStore } from "./token-store";\nexport class AuthService {\n getToken() { return tokenStore.read(); }\n}\nexport const AUTH_TOKEN = "auth";', - filePath: path.join(tempRoot, 'src', 'auth', 'auth.service.ts'), - relativePath: 'src/auth/auth.service.ts', - startLine: 1, - endLine: 5, - language: 'ts', - dependencies: [], - imports: [ - 'src/auth/token-store.ts', - 'src/auth/session.ts', - 'src/shared/logger.ts', - 'src/config/env.ts', - 'src/http/client.ts' - ], - exports: ['AuthService', 'AUTH_TOKEN'], - tags: ['service'], - metadata: { - className: 'AuthService', - symbolAware: true, - symbolName: 'AuthService', - symbolKind: 'class' - } - }; - - await fs.writeFile( - path.join(ctxDir, KEYWORD_INDEX_FILENAME), - JSON.stringify( - { - header: { buildId: 'test-build-compact', formatVersion: INDEX_FORMAT_VERSION }, - chunks: [actualChunk] - }, - null, - 2 - ), - 'utf-8' - ); - - const actualSearchModule = - await vi.importActual('../src/core/search.js'); - const searcher = new actualSearchModule.CodebaseSearcher(tempRoot); - const results = await searcher.search('AuthService token', 5, undefined, { - useSemanticSearch: false, - useKeywordSearch: true, - enableReranker: false - }); + it( + 'real CodebaseSearcher preserves chunk imports and exports', + async () => { + if (!tempRoot) throw new Error('tempRoot not initialized'); + + const ctxDir = path.join(tempRoot, CODEBASE_CONTEXT_DIRNAME); + const actualChunk = { + id: 'auth-chunk', + content: + 'import { tokenStore } from "./token-store";\nexport class AuthService {\n getToken() { return tokenStore.read(); }\n}\nexport const AUTH_TOKEN = "auth";', + filePath: path.join(tempRoot, 'src', 'auth', 'auth.service.ts'), + relativePath: 'src/auth/auth.service.ts', + startLine: 1, + endLine: 5, + language: 'ts', + dependencies: [], + imports: [ + 'src/auth/token-store.ts', + 'src/auth/session.ts', + 'src/shared/logger.ts', + 'src/config/env.ts', + 'src/http/client.ts' + ], + exports: ['AuthService', 'AUTH_TOKEN'], + tags: ['service'], + metadata: { + className: 'AuthService', + symbolAware: true, + symbolName: 'AuthService', + symbolKind: 'class' + } + }; + + await fs.writeFile( + path.join(ctxDir, KEYWORD_INDEX_FILENAME), + JSON.stringify( + { + header: { buildId: 'test-build-compact', formatVersion: INDEX_FORMAT_VERSION }, + chunks: [actualChunk] + }, + null, + 2 + ), + 'utf-8' + ); - expect(results).toHaveLength(1); - expect(results[0].filePath).toBe(actualChunk.filePath); - expect(results[0].imports).toEqual(actualChunk.imports); - expect(results[0].exports).toEqual(actualChunk.exports); - }, SLOW_WINDOWS_TEST_TIMEOUT_MS); + // This real-search fixture must represent actual indexed source. Older setup + // only wrote cached chunks, which cannot establish current-source validity. + await fs.mkdir(path.dirname(actualChunk.filePath), { recursive: true }); + await fs.writeFile(actualChunk.filePath, actualChunk.content, 'utf-8'); + await writeManifest(path.join(ctxDir, MANIFEST_FILENAME), { + version: 1, + generatedAt: new Date().toISOString(), + header: { buildId: 'test-build-compact', formatVersion: INDEX_FORMAT_VERSION }, + files: { [actualChunk.relativePath]: hashFileContent(actualChunk.content) } + }); + const metaPath = path.join(ctxDir, INDEX_META_FILENAME); + const meta = IndexMetaSchema.parse(JSON.parse(await fs.readFile(metaPath, 'utf8'))); + meta.artifacts.manifest = { path: MANIFEST_FILENAME }; + await fs.writeFile(metaPath, JSON.stringify(meta)); + + const actualSearchModule = + await vi.importActual('../src/core/search.js'); + const searcher = new actualSearchModule.CodebaseSearcher(tempRoot); + const results = await searcher.search('AuthService token', 5, undefined, { + useSemanticSearch: false, + useKeywordSearch: true, + enableReranker: false + }); + + expect(results).toHaveLength(1); + expect(results[0].filePath).toBe(actualChunk.filePath); + expect(results[0].imports).toEqual(actualChunk.imports); + expect(results[0].exports).toEqual(actualChunk.exports); + }, + SLOW_WINDOWS_TEST_TIMEOUT_MS + ); it('adds a warning only when the final full payload exceeds the compact budget threshold', async () => { const oversizedSummary = 'Token-heavy summary '.repeat(1200); diff --git a/tests/search-decision-card.test.ts b/tests/search-decision-card.test.ts index c6d77ae..23e4ece 100644 --- a/tests/search-decision-card.test.ts +++ b/tests/search-decision-card.test.ts @@ -174,214 +174,231 @@ export class ProfileService { } }, SLOW_WINDOWS_TEST_TIMEOUT_MS); - it('intent="edit" with multiple results returns full decision card with ready field', async () => { - if (!tempRoot) throw new Error('tempRoot not initialized'); - - const { server } = await import('../src/index.js'); - const handler = getToolCallHandler(server); - - const response = await handler({ - jsonrpc: '2.0', - id: 1, - method: 'tools/call', - params: { - name: 'search_codebase', - arguments: { - query: 'getToken', - intent: 'edit' + it( + 'intent="edit" with multiple results returns full decision card with ready field', + async () => { + if (!tempRoot) throw new Error('tempRoot not initialized'); + + const { server } = await import('../src/index.js'); + const handler = getToolCallHandler(server); + + const response = await handler({ + jsonrpc: '2.0', + id: 1, + method: 'tools/call', + params: { + name: 'search_codebase', + arguments: { + query: 'getToken', + intent: 'edit' + } } - } - }); + }); - expect(response.content).toBeDefined(); - expect(response.content.length).toBeGreaterThan(0); - const content = response.content[0]; - expect(content.type).toBe('text'); + expect(response.content).toBeDefined(); + expect(response.content.length).toBeGreaterThan(0); + const content = response.content[0]; + expect(content.type).toBe('text'); - const parsed = JSON.parse(content.text) as SearchToolPayload; - expect(parsed.results).toBeDefined(); - expect(parsed.results.length).toBeGreaterThan(0); + const parsed = JSON.parse(content.text) as SearchToolPayload; + expect(parsed.results).toBeDefined(); + expect(parsed.results.length).toBeGreaterThan(0); - const preflight = parsed.preflight; - expect(preflight).toBeDefined(); - if (!preflight) { - throw new Error('Expected preflight payload for edit intent'); - } - expect(preflight.ready).toBeDefined(); - expect(typeof preflight.ready).toBe('boolean'); - }, SLOW_WINDOWS_TEST_TIMEOUT_MS); - - it('decision card has all expected fields when returned', async () => { - if (!tempRoot) throw new Error('tempRoot not initialized'); - - const { server } = await import('../src/index.js'); - const handler = getToolCallHandler(server); - - const response = await handler({ - jsonrpc: '2.0', - id: 1, - method: 'tools/call', - params: { - name: 'search_codebase', - arguments: { - query: 'AuthService', - intent: 'edit' - } + const preflight = parsed.preflight; + expect(preflight).toBeDefined(); + if (!preflight) { + throw new Error('Expected preflight payload for edit intent'); } - }); - - const content = response.content[0]; - const parsed = JSON.parse(content.text) as SearchToolPayload; - const preflight = parsed.preflight; - expect(preflight).toBeDefined(); - if (!preflight) { - throw new Error('Expected preflight payload for edit intent'); - } - - // preflight should have ready as minimum - expect(preflight.ready).toBeDefined(); - expect(typeof preflight.ready).toBe('boolean'); - - // Optional fields can be present - if (preflight.nextAction) { - expect(typeof preflight.nextAction).toBe('string'); - } - if (preflight.patterns) { - expect(typeof preflight.patterns).toBe('object'); - } - if (preflight.warnings) { - expect(Array.isArray(preflight.warnings)).toBe(true); - } - if (preflight.bestExample) { - expect(typeof preflight.bestExample).toBe('string'); - } - if (preflight.impact) { - expect(typeof preflight.impact).toBe('object'); - } - if (preflight.whatWouldHelp) { - expect(Array.isArray(preflight.whatWouldHelp)).toBe(true); - } - }, SLOW_WINDOWS_TEST_TIMEOUT_MS); - - it('intent="explore" returns lightweight preflight', async () => { - if (!tempRoot) throw new Error('tempRoot not initialized'); - - const { server } = await import('../src/index.js'); - const handler = getToolCallHandler(server); - - const response = await handler({ - jsonrpc: '2.0', - id: 1, - method: 'tools/call', - params: { - name: 'search_codebase', - arguments: { - query: 'AuthService', - intent: 'explore' + expect(preflight.ready).toBeDefined(); + expect(typeof preflight.ready).toBe('boolean'); + }, + SLOW_WINDOWS_TEST_TIMEOUT_MS + ); + + it( + 'decision card has all expected fields when returned', + async () => { + if (!tempRoot) throw new Error('tempRoot not initialized'); + + const { server } = await import('../src/index.js'); + const handler = getToolCallHandler(server); + + const response = await handler({ + jsonrpc: '2.0', + id: 1, + method: 'tools/call', + params: { + name: 'search_codebase', + arguments: { + query: 'AuthService', + intent: 'edit' + } } + }); + + const content = response.content[0]; + const parsed = JSON.parse(content.text) as SearchToolPayload; + const preflight = parsed.preflight; + expect(preflight).toBeDefined(); + if (!preflight) { + throw new Error('Expected preflight payload for edit intent'); } - }); - - const content = response.content[0]; - const parsed = JSON.parse(content.text) as SearchToolPayload; - const preflight = parsed.preflight; - // For explore intent, preflight should be lite: { ready, reason? } - if (preflight) { + // preflight should have ready as minimum expect(preflight.ready).toBeDefined(); expect(typeof preflight.ready).toBe('boolean'); - // Should NOT have full decision card fields for explore - } - }, SLOW_WINDOWS_TEST_TIMEOUT_MS); - it('includes snippet field when includeSnippets=true', async () => { - if (!tempRoot) throw new Error('tempRoot not initialized'); - - const { server } = await import('../src/index.js'); - const handler = getToolCallHandler(server); - - const response = await handler({ - jsonrpc: '2.0', - id: 1, - method: 'tools/call', - params: { - name: 'search_codebase', - arguments: { - query: 'getToken', - mode: 'full', - includeSnippets: true - } + // Optional fields can be present + if (preflight.nextAction) { + expect(typeof preflight.nextAction).toBe('string'); } - }); - - const content = response.content[0]; - const parsed = JSON.parse(content.text) as SearchToolPayload; - - expect(parsed.results).toBeDefined(); - expect(parsed.results.length).toBeGreaterThan(0); - - // At least some results should have a snippet - const withSnippets = parsed.results.filter((result) => result.snippet); - expect(withSnippets.length).toBeGreaterThan(0); - }, SLOW_WINDOWS_TEST_TIMEOUT_MS); - - it('does not include snippet field when includeSnippets=false', async () => { - if (!tempRoot) throw new Error('tempRoot not initialized'); - - const { server } = await import('../src/index.js'); - const handler = getToolCallHandler(server); - - const response = await handler({ - jsonrpc: '2.0', - id: 1, - method: 'tools/call', - params: { - name: 'search_codebase', - arguments: { - query: 'getToken', - includeSnippets: false - } + if (preflight.patterns) { + expect(typeof preflight.patterns).toBe('object'); } - }); - - const content = response.content[0]; - const parsed = JSON.parse(content.text) as SearchToolPayload; - - expect(parsed.results).toBeDefined(); - // All results should not have snippet field - parsed.results.forEach((result) => { - expect(result.snippet).toBeUndefined(); - }); - }, SLOW_WINDOWS_TEST_TIMEOUT_MS); - - it('scope header starts snippet when includeSnippets=true', async () => { - if (!tempRoot) throw new Error('tempRoot not initialized'); - - const { server } = await import('../src/index.js'); - const handler = getToolCallHandler(server); - - const response = await handler({ - jsonrpc: '2.0', - id: 1, - method: 'tools/call', - params: { - name: 'search_codebase', - arguments: { - query: 'getToken', - mode: 'full', - includeSnippets: true - } + if (preflight.warnings) { + expect(Array.isArray(preflight.warnings)).toBe(true); } - }); + if (preflight.bestExample) { + expect(typeof preflight.bestExample).toBe('string'); + } + if (preflight.impact) { + expect(typeof preflight.impact).toBe('object'); + } + if (preflight.whatWouldHelp) { + expect(Array.isArray(preflight.whatWouldHelp)).toBe(true); + } + }, + SLOW_WINDOWS_TEST_TIMEOUT_MS + ); + + it( + 'intent="explore" omits edit readiness', + async () => { + if (!tempRoot) throw new Error('tempRoot not initialized'); + + const { server } = await import('../src/index.js'); + const handler = getToolCallHandler(server); + + const response = await handler({ + jsonrpc: '2.0', + id: 1, + method: 'tools/call', + params: { + name: 'search_codebase', + arguments: { + query: 'AuthService', + intent: 'explore' + } + } + }); + + const content = response.content[0]; + const parsed = JSON.parse(content.text) as SearchToolPayload; + expect(parsed.preflight).toBeUndefined(); + }, + SLOW_WINDOWS_TEST_TIMEOUT_MS + ); + + it( + 'includes snippet field when includeSnippets=true', + async () => { + if (!tempRoot) throw new Error('tempRoot not initialized'); + + const { server } = await import('../src/index.js'); + const handler = getToolCallHandler(server); + + const response = await handler({ + jsonrpc: '2.0', + id: 1, + method: 'tools/call', + params: { + name: 'search_codebase', + arguments: { + query: 'getToken', + mode: 'full', + includeSnippets: true + } + } + }); + + const content = response.content[0]; + const parsed = JSON.parse(content.text) as SearchToolPayload; + + expect(parsed.results).toBeDefined(); + expect(parsed.results.length).toBeGreaterThan(0); + + // At least some results should have a snippet + const withSnippets = parsed.results.filter((result) => result.snippet); + expect(withSnippets.length).toBeGreaterThan(0); + }, + SLOW_WINDOWS_TEST_TIMEOUT_MS + ); + + it( + 'does not include snippet field when includeSnippets=false', + async () => { + if (!tempRoot) throw new Error('tempRoot not initialized'); + + const { server } = await import('../src/index.js'); + const handler = getToolCallHandler(server); + + const response = await handler({ + jsonrpc: '2.0', + id: 1, + method: 'tools/call', + params: { + name: 'search_codebase', + arguments: { + query: 'getToken', + includeSnippets: false + } + } + }); + + const content = response.content[0]; + const parsed = JSON.parse(content.text) as SearchToolPayload; + + expect(parsed.results).toBeDefined(); + // All results should not have snippet field + parsed.results.forEach((result) => { + expect(result.snippet).toBeUndefined(); + }); + }, + SLOW_WINDOWS_TEST_TIMEOUT_MS + ); + + it( + 'scope header starts snippet when includeSnippets=true', + async () => { + if (!tempRoot) throw new Error('tempRoot not initialized'); + + const { server } = await import('../src/index.js'); + const handler = getToolCallHandler(server); + + const response = await handler({ + jsonrpc: '2.0', + id: 1, + method: 'tools/call', + params: { + name: 'search_codebase', + arguments: { + query: 'getToken', + mode: 'full', + includeSnippets: true + } + } + }); - const content = response.content[0]; - const parsed = JSON.parse(content.text) as SearchToolPayload; + const content = response.content[0]; + const parsed = JSON.parse(content.text) as SearchToolPayload; - const withSnippet = parsed.results.find((result) => result.snippet); - if (withSnippet && withSnippet.snippet) { - // Scope header should be a comment line - const firstLine = withSnippet.snippet.split('\n')[0].trim(); - expect(firstLine).toMatch(/^\/\//); - } - }, SLOW_WINDOWS_TEST_TIMEOUT_MS); + const withSnippet = parsed.results.find((result) => result.snippet); + if (withSnippet && withSnippet.snippet) { + // Scope header should be a comment line + const firstLine = withSnippet.snippet.split('\n')[0].trim(); + expect(firstLine).toMatch(/^\/\//); + } + }, + SLOW_WINDOWS_TEST_TIMEOUT_MS + ); }); diff --git a/tests/search-edit-preflight-lite.test.ts b/tests/search-edit-preflight-lite.test.ts index 5ee335e..45fd6c1 100644 --- a/tests/search-edit-preflight-lite.test.ts +++ b/tests/search-edit-preflight-lite.test.ts @@ -12,10 +12,50 @@ import { VECTOR_DB_DIRNAME } from '../src/constants/codebase-context.js'; +type SourceCoverageMock = { + status: 'current' | 'partial' | 'no_fresh_results'; + checkedFiles: number; + skippedFiles: number; + refreshAction?: string; +}; + +type ToolCallRequest = { + jsonrpc: '2.0'; + id: number; + method: 'tools/call'; + params: { name: string; arguments: Record }; +}; + +type ToolCallResponse = { content: Array<{ type: string; text: string }> }; + +type SearchPayload = { + status: string; + searchQuality?: { status: string; confidence: number }; + sourceCoverage?: SourceCoverageMock; + preflight?: { ready: boolean; abstain?: boolean; reason?: string }; +}; + const searchMocks = vi.hoisted(() => ({ - search: vi.fn() + search: vi.fn(), + sourceCoverage: vi.fn( + (): SourceCoverageMock => ({ + status: 'current', + checkedFiles: 0, + skippedFiles: 0 + }) + ) })); +function getToolCallHandler( + server: unknown +): (request: ToolCallRequest) => Promise { + const handlers = (server as { _requestHandlers?: unknown })._requestHandlers; + if (!(handlers instanceof Map)) throw new Error('Expected server._requestHandlers to be a Map'); + const handler = handlers.get('tools/call'); + if (typeof handler !== 'function') throw new Error('Expected tools/call handler'); + return handler as (request: ToolCallRequest) => Promise; +} + vi.mock('../src/core/search.js', async () => { class CodebaseSearcher { constructor(_rootPath: string) {} @@ -23,6 +63,13 @@ vi.mock('../src/core/search.js', async () => { async search(query: string, limit: number, filters?: unknown) { return searchMocks.search(query, limit, filters); } + + async searchWithCoverage(query: string, limit: number, filters?: unknown) { + return { + results: await this.search(query, limit, filters), + sourceCoverage: searchMocks.sourceCoverage() + }; + } } return { CodebaseSearcher }; @@ -35,6 +82,11 @@ describe('search_codebase preflight', () => { beforeEach(async () => { searchMocks.search.mockReset(); + searchMocks.sourceCoverage.mockReset().mockReturnValue({ + status: 'current', + checkedFiles: 0, + skippedFiles: 0 + }); vi.resetModules(); originalArgv = [...process.argv]; @@ -122,7 +174,7 @@ describe('search_codebase preflight', () => { } }); - it('returns preflight with ready flag when intent is omitted (intelligence present)', async () => { + it('omits edit readiness for discovery when intent is omitted', async () => { searchMocks.search.mockResolvedValueOnce([ { summary: 'Auth interceptor implementation', @@ -137,7 +189,7 @@ describe('search_codebase preflight', () => { ]); const { server } = await import('../src/index.js'); - const handler = (server as any)._requestHandlers.get('tools/call'); + const handler = getToolCallHandler(server); const response = await handler({ jsonrpc: '2.0', @@ -151,11 +203,51 @@ describe('search_codebase preflight', () => { } }); - const payload = JSON.parse(response.content[0].text); + const payload = JSON.parse(response.content[0].text) as SearchPayload; expect(payload.status).toBe('success'); - // Preflight is flattened to { ready, reason } - expect(payload.preflight).toBeTruthy(); - expect(typeof payload.preflight.ready).toBe('boolean'); + expect(payload.searchQuality).toBeDefined(); + expect(payload.preflight).toBeUndefined(); + }); + + it('keeps currentness and retrieval uncertainty visible for explicit exploration', async () => { + searchMocks.search.mockResolvedValueOnce([ + { + summary: 'Local cache recovery path', + snippet: 'reloadCache() { return readSnapshot(); }', + filePath: 'src/cache/recovery.ts', + startLine: 14, + endLine: 19, + score: 0.05, + language: 'ts', + metadata: {} + } + ]); + searchMocks.sourceCoverage.mockReturnValueOnce({ + status: 'partial', + checkedFiles: 2, + skippedFiles: 1, + refreshAction: 'Refresh the index to inspect remaining candidates.' + }); + + const { server } = await import('../src/index.js'); + const handler = getToolCallHandler(server); + const response = await handler({ + jsonrpc: '2.0', + id: 2, + method: 'tools/call', + params: { + name: 'search_codebase', + arguments: { query: 'cache recovery', intent: 'explore' } + } + }); + + const payload = JSON.parse(response.content[0].text) as SearchPayload; + expect(payload.sourceCoverage?.status).toBe('partial'); + expect(payload.sourceCoverage?.refreshAction).toBe( + 'Refresh the index to inspect remaining candidates.' + ); + expect(payload.searchQuality?.status).toBeDefined(); + expect(payload.preflight).toBeUndefined(); }); it('returns preflight with ready flag when intent="edit"', async () => { @@ -173,7 +265,7 @@ describe('search_codebase preflight', () => { ]); const { server } = await import('../src/index.js'); - const handler = (server as any)._requestHandlers.get('tools/call'); + const handler = getToolCallHandler(server); const response = await handler({ jsonrpc: '2.0', @@ -188,10 +280,47 @@ describe('search_codebase preflight', () => { } }); - const payload = JSON.parse(response.content[0].text); + const payload = JSON.parse(response.content[0].text) as SearchPayload; expect(payload.status).toBe('success'); - // Preflight is flattened to { ready, reason } expect(payload.preflight).toBeTruthy(); - expect(typeof payload.preflight.ready).toBe('boolean'); + expect(typeof payload.preflight?.ready).toBe('boolean'); + }); + + it('keeps the explicit edit gate closed when source coverage is partial', async () => { + searchMocks.search.mockResolvedValueOnce([ + { + summary: 'Local cache recovery path', + snippet: 'reloadCache() { return readSnapshot(); }', + filePath: 'src/cache/recovery.ts', + startLine: 14, + endLine: 19, + score: 0.05, + language: 'ts', + metadata: {} + } + ]); + searchMocks.sourceCoverage.mockReturnValueOnce({ + status: 'partial', + checkedFiles: 2, + skippedFiles: 1, + refreshAction: 'Refresh the index to inspect remaining candidates.' + }); + + const { server } = await import('../src/index.js'); + const handler = getToolCallHandler(server); + const response = await handler({ + jsonrpc: '2.0', + id: 3, + method: 'tools/call', + params: { + name: 'search_codebase', + arguments: { query: 'cache recovery', intent: 'edit' } + } + }); + + const payload = JSON.parse(response.content[0].text) as SearchPayload; + expect(payload.preflight?.ready).toBe(false); + expect(payload.preflight?.abstain).toBe(true); + expect(payload.preflight?.reason).toBe('Refresh the index to inspect remaining candidates.'); }); }); diff --git a/tests/search-health-scope.test.ts b/tests/search-health-scope.test.ts index 0e175bf..4fd1acb 100644 --- a/tests/search-health-scope.test.ts +++ b/tests/search-health-scope.test.ts @@ -16,6 +16,13 @@ vi.mock('../src/core/search.js', async () => { async search(query: string, limit: number, filters?: unknown) { return searchMocks.search(query, limit, filters); } + + async searchWithCoverage(query: string, limit: number, filters?: unknown) { + return { + results: await this.search(query, limit, filters), + sourceCoverage: { status: 'current', checkedFiles: 0, skippedFiles: 0 } + }; + } } return { CodebaseSearcher }; diff --git a/tests/search-memory-scope.test.ts b/tests/search-memory-scope.test.ts new file mode 100644 index 0000000..9b544c5 --- /dev/null +++ b/tests/search-memory-scope.test.ts @@ -0,0 +1,124 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { promises as fs } from 'fs'; +import os from 'os'; +import path from 'path'; +import { handle } from '../src/tools/search-codebase.js'; +import type { ToolContext } from '../src/tools/types.js'; + +const searchMocks = vi.hoisted(() => ({ search: vi.fn() })); + +vi.mock('../src/core/search.js', async () => { + class CodebaseSearcher { + constructor(_rootPath: string) {} + + async search(query: string, limit: number, filters?: unknown) { + return searchMocks.search(query, limit, filters); + } + + async searchWithCoverage(query: string, limit: number, filters?: unknown) { + return { + results: await this.search(query, limit, filters), + sourceCoverage: { status: 'current', checkedFiles: 1, skippedFiles: 0 } + }; + } + } + + return { CodebaseSearcher }; +}); + +describe('search_codebase memory relevance at the tool seam', () => { + let tempRoot: string; + let ctx: ToolContext; + + beforeEach(async () => { + searchMocks.search.mockReset(); + tempRoot = await fs.mkdtemp(path.join(os.tmpdir(), 'search-memory-scope-')); + const contextDir = path.join(tempRoot, '.codebase-context'); + await fs.mkdir(contextDir, { recursive: true }); + + ctx = { + indexState: { status: 'ready' }, + paths: { + baseDir: contextDir, + memory: path.join(contextDir, 'memory.json'), + intelligence: path.join(contextDir, 'intelligence.json'), + keywordIndex: path.join(contextDir, 'index.json'), + vectorDb: path.join(contextDir, 'index') + }, + rootPath: tempRoot, + performIndexing: () => undefined + }; + + const date = new Date().toISOString(); + await fs.writeFile( + ctx.paths.memory, + JSON.stringify([ + { + id: 'auth-path-only', + type: 'decision', + category: 'architecture', + memory: 'Prefer React Signals for local view state', + reason: 'Keep the widget state independent of network retry behavior.', + date, + scope: { kind: 'file', file: 'src/auth/auth.service.ts' } + }, + { + id: 'cross-file-auth-decision', + type: 'decision', + category: 'architecture', + memory: 'Use AuthService for CLI token bootstrap', + reason: 'Keep CLI setup on the shared token refresh path.', + date, + scope: { + kind: 'symbol', + file: 'src/auth/auth.service.ts', + symbol: 'AuthService' + } + } + ]), + 'utf-8' + ); + await fs.writeFile( + ctx.paths.intelligence, + JSON.stringify({ + header: { buildId: 'scope-test-build', formatVersion: 1 }, + generatedAt: date, + patterns: {}, + goldenFiles: [] + }), + 'utf-8' + ); + }); + + afterEach(async () => { + await fs.rm(tempRoot, { recursive: true, force: true }); + }); + + it('excludes path-only auth memories while retaining a relevant cross-file decision', async () => { + searchMocks.search.mockResolvedValueOnce([ + { + summary: 'Auth CLI bootstrap', + snippet: 'export function bootstrapAuthCli() { return authService.refreshToken(); }', + filePath: 'src/cli/bootstrap.ts', + startLine: 1, + endLine: 4, + score: 0.92, + language: 'ts', + metadata: { symbolName: 'bootstrapAuthCli', symbolKind: 'function' }, + relevanceReason: 'Matches auth CLI setup' + } + ]); + + const response = await handle({ query: 'auth CLI', intent: 'edit', mode: 'full' }, ctx); + const payload = JSON.parse(response.content?.[0]?.text ?? '{}') as { + relatedMemories?: string[]; + }; + + expect(payload.relatedMemories).toContainEqual( + expect.stringContaining('Use AuthService for CLI token bootstrap') + ); + expect(payload.relatedMemories).not.toContainEqual( + expect.stringContaining('Prefer React Signals for local view state') + ); + }); +}); diff --git a/tests/search-model-free-fallback.test.ts b/tests/search-model-free-fallback.test.ts new file mode 100644 index 0000000..88814fc --- /dev/null +++ b/tests/search-model-free-fallback.test.ts @@ -0,0 +1,819 @@ +import { resolveIndexArtifact } from '../src/core/index-generation.js'; +import { describe, expect, it } from 'vitest'; +import { createHash } from 'node:crypto'; +import { promises as fs } from 'node:fs'; +import path from 'node:path'; +import Fuse from 'fuse.js'; +import type { CodeChunk, SearchFilters } from '../src/types/index.js'; +import { CodebaseSearcher } from '../src/core/search.js'; +import { assessSearchQuality } from '../src/core/search-quality.js'; +import { GenericAnalyzer } from '../src/analyzers/generic/index.js'; +import { CodebaseIndexer } from '../src/core/indexer.js'; +import { analyzerRegistry } from '../src/core/analyzer-registry.js'; +import { InternalFileGraph } from '../src/utils/usage-tracker.js'; +import { + InternalSourceResolver, + loadTsconfigCompilerOptions +} from '../src/utils/internal-source-resolver.js'; +import { + CODEBASE_CONTEXT_DIRNAME, + INTELLIGENCE_FILENAME, + MANIFEST_FILENAME +} from '../src/constants/codebase-context.js'; +import { asSearcherTestHarness, materializeSourceSnapshot } from './helpers/source-snapshot.js'; + +function chunk(filePath: string, content: string, symbolName?: string): CodeChunk { + return { + id: filePath, + content, + filePath, + relativePath: filePath, + startLine: 1, + endLine: content.split('\n').length, + language: 'typescript', + framework: 'generic', + componentType: 'module', + dependencies: [], + imports: [], + exports: [], + tags: [], + metadata: symbolName ? { symbolName, componentName: symbolName } : {} + }; +} + +async function searchFixture(chunks: CodeChunk[], withImportGraph = false) { + const snapshot = await materializeSourceSnapshot(chunks, 'cbc-model-free-query-'); + const searcher = new CodebaseSearcher(snapshot.rootPath); + const harness = asSearcherTestHarness(searcher) as ReturnType & { + chunks: CodeChunk[]; + loadPatternIntelligence: () => Promise; + semanticSearch: ( + query: string, + limit: number, + filters?: SearchFilters + ) => Promise>; + }; + harness.initialized = true; + harness.sourceManifest = snapshot.manifest; + harness.chunks = chunks; + harness.embeddingProvider = null; + harness.storageProvider = null; + harness.fuseIndex = new Fuse(chunks, { + keys: [ + { name: 'content', weight: 0.4 }, + { name: 'metadata.componentName', weight: 0.25 }, + { name: 'filePath', weight: 0.15 }, + { name: 'relativePath', weight: 0.15 }, + { name: 'componentType', weight: 0.15 }, + { name: 'layer', weight: 0.1 }, + { name: 'tags', weight: 0.15 } + ], + includeScore: true, + threshold: 0.4, + useExtendedSearch: true, + ignoreLocation: true + }); + // Use the same analyzer, resolver and graph serializer as indexing. Text in + // comments/strings and unproven chunk.imports must not become graph edges. + const graph = new InternalFileGraph(snapshot.rootPath); + const reloadGraph = async (data: ReturnType) => { + await fs.writeFile( + await resolveIndexArtifact(snapshot.rootPath, INTELLIGENCE_FILENAME), + JSON.stringify({ internalFileGraph: data }) + ); + await harness.loadPatternIntelligence(); + }; + if (withImportGraph) { + const files = [...new Set(chunks.map((item) => item.filePath))]; + const resolver = await InternalSourceResolver.create(snapshot.rootPath, files, { + compilerOptions: { + ...loadTsconfigCompilerOptions(snapshot.rootPath), + resolveJsonModule: true + } + }); + const analyzer = new GenericAnalyzer(); + for (const file of files.filter((file) => file.endsWith('.ts'))) { + const content = await fs.readFile(file, 'utf8'); + const analysis = await analyzer.analyze(file, content); + for (const imported of analysis.imports) { + const target = await resolver.resolve(file, imported.source); + if (target.status !== 'resolved' || !target.path) continue; + graph.trackImport(file, target.path, imported.line, imported.imports, { + kind: 'import', + specifier: imported.source, + sourceFingerprint: createHash('sha256').update(content).digest('hex'), + targetFingerprint: createHash('sha256') + .update(await fs.readFile(target.path)) + .digest('hex') + }); + } + } + await reloadGraph(graph.toJSON()); + } + const search = (query: string, filters?: SearchFilters) => + searcher.search(query, 5, filters, { + useSemanticSearch: false, + enableQueryExpansion: false, + enableLowConfidenceRescue: false, + enableReranker: false + }); + const searchWithSemantic = (query: string) => + searcher.search(query, 5, undefined, { + useSemanticSearch: true, + useKeywordSearch: true, + enableQueryExpansion: false, + enableLowConfidenceRescue: false, + enableReranker: false + }); + return { + rootPath: snapshot.rootPath, + search, + searchWithSemantic, + searcher: harness, + graph: graph.toJSON(), + reloadGraph, + cleanup: snapshot.cleanup + }; +} + +describe('model-free query fallback', () => { + it.each(['ParcelApiClient', 'parcelapiclient', 'Where is ParcelApiClient retry policy?'])( + 'keeps an exact declared class ahead of a same-suffix consumer for %s', + async (query) => { + const declaration = chunk( + 'src/parcels.ts', + 'export class ParcelApiClient { retryPolicy = "bounded"; }' + ); + declaration.componentType = 'unknown'; + declaration.metadata = { componentName: 'ParcelApiClient' }; + const consumer = chunk( + 'src/ParcelApiClient-consumer.ts', + 'export class CachedParcelApiClient { client: ParcelApiClient; retryPolicy = "cached"; }' + ); + consumer.metadata = { componentName: 'CachedParcelApiClient' }; + consumer.componentType = 'service'; + consumer.layer = 'business'; + const fixture = await searchFixture([consumer, declaration]); + try { + const results = await fixture.search(query); + expect(results[0]?.filePath).toContain('parcels.ts'); + const exact = await fixture.search('ParcelApiClient'); + expect(assessSearchQuality('ParcelApiClient', exact).status).toBe('ok'); + expect(await fixture.search('QuantumPineappleService')).toEqual([]); + } finally { + await fixture.cleanup(); + } + } + ); + + it('keeps a re-export mention below its indexed declaration for an exact class query', async () => { + const declaration = chunk( + 'src/parcels.ts', + 'export class ParcelApiClient { retryPolicy = "bounded"; }' + ); + declaration.metadata = { componentName: 'ParcelApiClient' }; + declaration.componentType = 'unknown'; + const barrel = chunk( + 'src/ParcelApiClient.ts', + 'export { ParcelApiClient } from "./parcels.js";' + ); + barrel.exports = ['ParcelApiClient']; + barrel.componentType = 'service'; + barrel.layer = 'business'; + const fixture = await searchFixture([barrel, declaration]); + try { + expect((await fixture.search('ParcelApiClient'))[0]?.filePath).toContain('parcels.ts'); + } finally { + await fixture.cleanup(); + } + }); + + it('retrieves an inflected action in a source declaration ahead of incidental configuration words', async () => { + const fixture = await searchFixture([ + chunk( + 'src/order-batch.ts', + 'export function batchOrders(orders: string[]) { return orders.map(order => [order]); }', + 'batchOrders' + ), + chunk( + 'jest.config.ts', + 'export default { displayName: "orders", coverageDirectory: "coverage/order-processing" };' + ) + ]); + try { + const results = await fixture.search('How are orders batched for processing?'); + expect(results[0]?.filePath).toContain('order-batch.ts'); + } finally { + await fixture.cleanup(); + } + }); + + it('uses the same identifier boundaries for natural questions and Python snake_case source', async () => { + const implementation = chunk( + 'billing/archive.py', + 'def archive_invoices(records):\n return [record for record in records if record.paid]', + 'archive_invoices' + ); + implementation.language = 'python'; + const fixture = await searchFixture([ + implementation, + chunk('src/invoice.ts', 'export const invoices = [];') + ]); + try { + const results = await fixture.search('Where are invoices archived before deletion?', { + language: 'python' + }); + expect(results[0]?.filePath).toContain('archive.py'); + expect(await fixture.search('Where are photons archived before teleportation?')).toEqual([]); + } finally { + await fixture.cleanup(); + } + }); + + it('does not let fuzzy full-question configuration hits hide grounded declaration evidence', async () => { + const fixture = await searchFixture([ + chunk( + 'src/channel.ts', + 'export function setAudioVolume(volume: number) { return volume; }', + 'setAudioVolume' + ), + chunk( + 'tsconfig.json', + '{ "compilerOptions": { "allowJs": true }, "adjust": "auditor", "volume": "logs" }' + ) + ]); + try { + const results = await fixture.search('adjust audio volume'); + expect(results[0]?.filePath).toContain('channel.ts'); + const exact = await fixture.search('setAudioVolume'); + expect(exact[0]?.filePath).toContain('channel.ts'); + } finally { + await fixture.cleanup(); + } + }); + + it('does not count inflections of one clue as independent corroboration', async () => { + const fixture = await searchFixture([ + chunk('src/archive.ts', 'export function archive() { return true; }', 'archive'), + chunk('src/organization.ts', 'export const organization = "archive";') + ]); + try { + expect(await fixture.search('Where are archives archived holographically?')).toEqual([]); + expect(await fixture.search('Where are organic organs archived?')).toEqual([]); + } finally { + await fixture.cleanup(); + } + }); + + it('keeps concrete declaration relevance above unrelated component and layer labels', async () => { + const declaration = chunk( + 'src/budget.ts', + 'export function calculateRetryBudget() { return 3; }', + 'calculateRetryBudget' + ); + declaration.componentType = 'unknown'; + const mention = chunk( + 'src/controller.ts', + 'export class Controller { label = "retry budget"; }', + 'Controller' + ); + mention.componentType = 'service'; + mention.layer = 'business'; + const fixture = await searchFixture([mention, declaration]); + try { + expect((await fixture.search('How is the retry budget calculated?'))[0]?.filePath).toContain( + 'budget.ts' + ); + } finally { + await fixture.cleanup(); + } + }); + + it.each([ + { + file: 'jobs/batch.ts', + source: + 'export function batchOrders(orders: string[]) { return orders.map(order => [order]); }', + query: 'How are orders batched for processing?', + exact: 'batchOrders' + }, + { + file: 'billing/archive.py', + source: + 'def archive_invoices(records):\n return [record for record in records if record.paid]', + query: 'Where are invoices archived before deletion?', + exact: 'archive_invoices' + } + ])('retrieves real GenericAnalyzer chunks for $file', async ({ file, source, query, exact }) => { + const analyzer = new GenericAnalyzer(); + const analysis = await analyzer.analyze(path.resolve(file), `${source}\n`, { + includeScopePrefix: false + }); + for (const item of analysis.chunks) { + item.relativePath = file; + item.filePath = file; + } + expect(analysis.chunks.length).toBeGreaterThan(0); + const fixture = await searchFixture(analysis.chunks); + try { + expect((await fixture.search(query))[0]?.filePath).toContain(path.basename(file)); + expect((await fixture.search(exact))[0]?.filePath).toContain(path.basename(file)); + } finally { + await fixture.cleanup(); + } + }); + + it('uses default model-free indexing and search for inflected TypeScript and Python questions', async () => { + const sources = [ + chunk( + 'jobs/batch.ts', + 'export function batchOrders(orders: string[]) { return orders.map(order => [order]); }' + ), + chunk( + 'billing/archive.py', + 'def archive_invoices(records):\n return [record for record in records if record.paid]' + ) + ]; + const snapshot = await materializeSourceSnapshot(sources, 'cbc-model-free-indexed-'); + analyzerRegistry.register(new GenericAnalyzer()); + try { + // The helper's manifest supports in-memory tests; this case starts from + // ordinary unindexed source and lets the real indexer own all artifacts. + await fs.unlink(await resolveIndexArtifact(snapshot.rootPath, MANIFEST_FILENAME)); + await fs.rmdir(path.join(snapshot.rootPath, CODEBASE_CONTEXT_DIRNAME)); + const indexer = new CodebaseIndexer({ rootPath: snapshot.rootPath }); + await indexer.index(); + const searcher = new CodebaseSearcher(snapshot.rootPath); + const typescript = await searcher.search('How are orders batched for processing?'); + const python = await searcher.search('Where are invoices archived before deletion?'); + expect(typescript[0]?.filePath).toContain('batch.ts'); + expect(python[0]?.filePath).toContain('archive.py'); + expect((await searcher.search('batchOrders'))[0]?.filePath).toContain('batch.ts'); + expect((await searcher.search('archive_invoices'))[0]?.filePath).toContain('archive.py'); + } finally { + await snapshot.cleanup(); + } + }); + + it('replenishes a saturated rescue evidence window after one source with many chunks changes', async () => { + const stale = Array.from({ length: 800 }, (_, index) => ({ + ...chunk('src/billing.ts', 'export const billing = { invoice: true, refund: true };'), + id: `billing-${index}` + })); + const support = chunk( + 'src/support.ts', + 'export const support = { invoice: true, refund: true };' + ); + const fixture = await searchFixture([...stale, support]); + try { + await fs.writeFile(stale[0].filePath, 'export const changed = true;\n'); + const matches = await fixture.search('billing invoice refund scheduling'); + expect(matches.map((result) => path.basename(result.filePath))).toEqual(['support.ts']); + expect(matches[0].snippet).toContain('support'); + } finally { + await fixture.cleanup(); + } + }); + + it('recovers separate source and policy files without requiring a manually written OR query', async () => { + const fixture = await searchFixture( + [ + chunk( + 'src/target.ts', + 'import { REQUIRED_POLICY } from "./policy.js";\nexport function resolvePilotPolicy(kind: string) { return kind + REQUIRED_POLICY; }', + 'resolvePilotPolicy' + ), + chunk('src/policy.ts', 'export const REQUIRED_POLICY = "review-before-deploy";') + ], + true + ); + try { + const results = await fixture.search('resolvePilotPolicy review-before-deploy'); + expect(results.map((result) => result.filePath.split(/[\\/]/).pop())).toEqual( + expect.arrayContaining(['target.ts', 'policy.ts']) + ); + const explicitOr = await fixture.search('resolvePilotPolicy | review-before-deploy'); + expect(explicitOr.map((result) => result.filePath.split(/[\\/]/).pop())).toEqual( + expect.arrayContaining(['target.ts', 'policy.ts']) + ); + } finally { + await fixture.cleanup(); + } + }); + + it('finds an implementation from an ordinary negative-limit suggestion question', async () => { + const fixture = await searchFixture([ + chunk( + 'packages/nx/src/utils/string-similarity.ts', + 'Returns up to limit candidates. Candidates not similar enough to input are excluded from suggestions.\nexport function findClosestMatches(input: string, candidates: string[], limit = 3) { return candidates.slice(0, limit); }', + 'findClosestMatches' + ), + chunk('packages/nx/src/utils/worker-limit.ts', 'export const workerLimit = 100;'), + chunk( + 'packages/nx/src/utils/catalog-helper.ts', + 'export function formatSuggestions() { return "suggestions"; }' + ) + ]); + try { + const results = await fixture.search( + 'Why do negative limits still yield multiple Nx typo suggestions?' + ); + expect(results[0]?.filePath).toContain('string-similarity.ts'); + expect( + assessSearchQuality( + 'Why do negative limits still yield multiple Nx typo suggestions?', + results + ).status + ).toBe('low_confidence'); + } finally { + await fixture.cleanup(); + } + }); + + it('does not manufacture matches from irrelevant text or weaken exact symbol search', async () => { + const fixture = await searchFixture([ + chunk( + 'src/session.ts', + 'export function authenticateUser() { return "token"; }', + 'authenticateUser' + ), + chunk('src/policy.ts', 'export const SESSION_POLICY = "login-required";') + ]); + try { + expect(await fixture.search('holographic pineapple zebra quantum')).toEqual([]); + const exact = await fixture.search('authenticateUser'); + expect(exact[0]?.filePath).toContain('session.ts'); + } finally { + await fixture.cleanup(); + } + }); + + it('does not promote unrelated single-term hits into a cross-file answer', async () => { + const fixture = await searchFixture([ + chunk('src/green.ts', 'export const hydroponics = 1;'), + chunk('src/red.ts', 'export const transcendence = 1;') + ]); + try { + expect(await fixture.search('hydroponics transcendence')).toEqual([]); + expect(await fixture.search('hydroponics transcendence', { language: 'python' })).toEqual([]); + } finally { + await fixture.cleanup(); + } + }); + + it('respects filters when the fallback has enough supporting words', async () => { + const fixture = await searchFixture([ + chunk('src/policy.ts', 'export const deploymentPolicy = "review before deploy";') + ]); + try { + expect(await fixture.search('review deploy', { language: 'python' })).toEqual([]); + expect((await fixture.search('review deploy')).at(0)?.filePath).toContain('policy.ts'); + } finally { + await fixture.cleanup(); + } + }); + + it('combines evidence from separate chunks in one source file without repeating that file', async () => { + const first = chunk('src/combined.ts', 'export const hydroponics = true;'); + first.id = 'combined-first'; + const second = chunk('src/combined.ts', 'export const transcendence = true;'); + second.id = 'combined-second'; + second.startLine = 2; + second.endLine = 2; + const fixture = await searchFixture([first, second]); + try { + const results = await fixture.search('hydroponics transcendence'); + expect(results).toHaveLength(1); + expect(results[0].filePath).toContain('combined.ts'); + } finally { + await fixture.cleanup(); + } + }); + + it('rejects an exact symbol when all the other substantive query terms are unsupported', async () => { + const fixture = await searchFixture([ + chunk( + 'src/auth.ts', + 'export function authenticateUser() { return true; }', + 'authenticateUser' + ) + ]); + try { + expect(await fixture.search('authenticateUser holographic pineapple zebra quantum')).toEqual( + [] + ); + expect((await fixture.search('authenticateUser'))[0]?.filePath).toContain('auth.ts'); + } finally { + await fixture.cleanup(); + } + }); + + it('does not corroborate an exact symbol using evidence from an unrelated file', async () => { + const fixture = await searchFixture([ + chunk( + 'src/auth.ts', + 'export function authenticateUser() { return true; }', + 'authenticateUser' + ), + chunk('src/billing.ts', 'export const billing = { invoice: "paid", refund: "pending" };') + ]); + try { + const matches = await fixture.search('authenticateUser billing invoice refund'); + expect(matches.map((result) => result.filePath.split(/[\\/]/).pop())).toEqual(['billing.ts']); + } finally { + await fixture.cleanup(); + } + }); + + it('does not treat a longer indexed symbol as an exact query symbol anchor', async () => { + const fixture = await searchFixture( + [ + chunk( + 'src/auth.ts', + 'import "./billing.js";\nexport function authenticateUserAdmin() { return true; }', + 'authenticateUserAdmin' + ), + chunk('src/billing.ts', 'export const billing = { invoice: "paid", refund: "pending" };') + ], + true + ); + try { + const matches = await fixture.search('authenticateUser billing invoice refund'); + expect(matches.map((result) => path.basename(result.filePath))).toEqual(['billing.ts']); + } finally { + await fixture.cleanup(); + } + }); + + it('does not use a comment-only symbol mention as an exact import anchor', async () => { + const fixture = await searchFixture( + [ + chunk('src/auth.ts', 'import "./billing.js";\n// authenticateUser was renamed last year.'), + chunk('src/billing.ts', 'export const billing = { invoice: "paid", refund: "pending" };') + ], + true + ); + try { + const matches = await fixture.search('authenticateUser billing invoice refund'); + expect(matches.map((result) => path.basename(result.filePath))).toEqual(['billing.ts']); + } finally { + await fixture.cleanup(); + } + }); + + it.each([ + '// import "./billing.js";', + '/* import "./billing.js"; */', + 'const example = \'import "./billing.js";\';', + 'const example = `import "./billing.js";`;', + 'const example = \'import("./billing.js")\';', + 'const example = \'require("./billing.js")\';', + 'const example = \'from "./billing.js"\';' + ])('does not use fake import text as cross-file evidence: %s', async (fakeImport) => { + const auth = chunk( + 'src/auth.ts', + `${fakeImport}\nexport function authenticateUser() { return true; }`, + 'authenticateUser' + ); + auth.imports = ['./billing.js']; // Raw chunk metadata has no source proof either. + const fixture = await searchFixture( + [ + auth, + chunk('src/billing.ts', 'export const billing = { invoice: "paid", refund: "pending" };') + ], + true + ); + try { + expect(fixture.graph.imports['src/auth.ts'] ?? []).toEqual([]); + const matches = await fixture.search('authenticateUser billing invoice refund'); + expect(matches.map((result) => path.basename(result.filePath))).toEqual(['billing.ts']); + } finally { + await fixture.cleanup(); + } + }); + + it('fails closed when a genuine import has no indexed graph proof', async () => { + const fixture = await searchFixture([ + chunk( + 'src/auth.ts', + 'import "./billing.js";\nexport function authenticateUser() { return true; }', + 'authenticateUser' + ), + chunk('src/billing.ts', 'export const billing = { invoice: "paid", refund: "pending" };') + ]); + try { + const matches = await fixture.search('authenticateUser billing invoice refund'); + expect(matches.map((result) => path.basename(result.filePath))).toEqual(['billing.ts']); + } finally { + await fixture.cleanup(); + } + }); + + it.each(['legacy', 'sourceFingerprint', 'targetFingerprint'] as const)( + 'fails closed for %s graph proof', + async (invalidProof) => { + const fixture = await searchFixture( + [ + chunk( + 'src/auth.ts', + 'import "./billing.js";\nexport function authenticateUser() { return true; }', + 'authenticateUser' + ), + chunk('src/billing.ts', 'export const billing = { invoice: "paid", refund: "pending" };') + ], + true + ); + try { + const detail = fixture.graph.importDetails!['src/auth.ts']['src/billing.ts']; + expect(detail.sources).toHaveLength(1); + if (invalidProof === 'legacy') delete detail.sources; + else detail.sources![0][invalidProof] = '0'.repeat(64); + await fixture.reloadGraph(fixture.graph); + const matches = await fixture.search('authenticateUser billing invoice refund'); + expect(matches.map((result) => path.basename(result.filePath))).toEqual(['billing.ts']); + } finally { + await fixture.cleanup(); + } + } + ); + + it('does not conflate a resolved JSON import with a same-stem TypeScript file', async () => { + const json = chunk('src/billing.json', '{}'); + json.language = 'json'; + const fixture = await searchFixture( + [ + chunk( + 'src/auth.ts', + 'import "./billing.json";\nexport function authenticateUser() { return true; }', + 'authenticateUser' + ), + json, + chunk('src/billing.ts', 'export const billing = { invoice: "paid", refund: "pending" };') + ], + true + ); + try { + expect(fixture.graph.imports['src/auth.ts']).toEqual(['src/billing.json']); + const matches = await fixture.search('authenticateUser billing invoice refund', { + language: 'typescript' + }); + expect(matches.map((result) => path.basename(result.filePath))).toEqual(['billing.ts']); + } finally { + await fixture.cleanup(); + } + }); + + it('excludes an anchor when its only graph companion no longer matches the snapshot', async () => { + const auth = chunk( + 'src/auth.ts', + 'import "./billing.js";\nexport function authenticateUser() { return true; }', + 'authenticateUser' + ); + const billing = chunk( + 'src/billing.ts', + 'export const billing = { invoice: "paid", refund: "pending" };' + ); + const fixture = await searchFixture([auth, billing], true); + try { + await fs.writeFile(billing.filePath, 'export const unrelated = true;'); + // Contract migration: drift must not become corruption or lend stale graph proof. + expect(await fixture.search('authenticateUser billing invoice refund')).toEqual([]); + } finally { + await fixture.cleanup(); + } + }); + + it('preserves a fresh semantic nominee while excluding its stale graph companion', async () => { + const auth = chunk( + 'src/auth.ts', + 'import "./billing.js";\nexport function authenticateUser() { return true; }', + 'authenticateUser' + ); + const billing = chunk( + 'src/billing.ts', + 'export const billing = { invoice: "paid", refund: "pending" };' + ); + const fixture = await searchFixture([auth, billing], true); + try { + await fs.writeFile(billing.filePath, 'export const unrelated = true;'); + fixture.searcher.embeddingProvider = {}; + fixture.searcher.storageProvider = {}; + fixture.searcher.semanticSearch = async () => [{ chunk: auth, score: 0.9 }]; + + const results = await fixture.searchWithSemantic('authenticateUser billing invoice refund'); + expect(results.map((result) => path.basename(result.filePath))).toEqual(['auth.ts']); + expect(results[0].snippet).toContain('authenticateUser'); + } finally { + await fixture.cleanup(); + } + }); + + it('does not corroborate an anchor after alias configuration redirects its unchanged import', async () => { + const fixture = await searchFixture([ + chunk( + 'src/auth.ts', + 'import "@billing";\nexport function authenticateUser() { return true; }', + 'authenticateUser' + ), + chunk('src/billing.ts', 'export const billing = { invoice: "paid", refund: "pending" };'), + chunk('src/other.ts', 'export const unrelated = true;') + ]); + const configPath = path.join(fixture.rootPath, 'tsconfig.json'); + try { + await fs.writeFile( + configPath, + JSON.stringify({ + compilerOptions: { baseUrl: '.', paths: { '@billing': ['src/billing.ts'] } } + }) + ); + const authPath = path.join(fixture.rootPath, 'src/auth.ts'); + const billingPath = path.join(fixture.rootPath, 'src/billing.ts'); + const graph = new InternalFileGraph(fixture.rootPath); + graph.trackImport(authPath, billingPath, 1, [], { + kind: 'import', + specifier: '@billing', + sourceFingerprint: createHash('sha256') + .update(await fs.readFile(authPath)) + .digest('hex'), + targetFingerprint: createHash('sha256') + .update(await fs.readFile(billingPath)) + .digest('hex') + }); + await fixture.reloadGraph(graph.toJSON()); + expect( + (await fixture.search('authenticateUser billing invoice refund')) + .map((result) => path.basename(result.filePath)) + .sort() + ).toEqual(['auth.ts', 'billing.ts']); + await fs.writeFile( + configPath, + JSON.stringify({ + compilerOptions: { baseUrl: '.', paths: { '@billing': ['src/other.ts'] } } + }) + ); + // Same endpoint bytes and same initialized searcher: resolution is a per-query witness. + expect( + (await fixture.search('authenticateUser billing invoice refund')).map((result) => + path.basename(result.filePath) + ) + ).toEqual(['billing.ts']); + } finally { + await fixture.cleanup(); + } + }); + + it.skipIf(process.platform !== 'linux')( + 'keeps case-distinct indexed import targets separate', + async () => { + const fixture = await searchFixture( + [ + chunk( + 'src/auth.ts', + 'import "./Billing.js";\nexport function authenticateUser() { return true; }', + 'authenticateUser' + ), + chunk('src/Billing.ts', 'export const unrelated = true;'), + chunk('src/billing.ts', 'export const billing = { invoice: "paid", refund: "pending" };') + ], + true + ); + try { + expect(fixture.graph.imports['src/auth.ts']).toEqual(['src/Billing.ts']); + const matches = await fixture.search('authenticateUser billing invoice refund'); + expect(matches.map((result) => path.basename(result.filePath))).toEqual(['billing.ts']); + } finally { + await fixture.cleanup(); + } + } + ); + + it('does not turn generic TypeScript syntax words into a confident match', async () => { + const fixture = await searchFixture([ + chunk('src/unconnected.ts', 'export function unconnected() { return 1; }') + ]); + try { + expect(await fixture.search('configure function return on thing')).toEqual([]); + } finally { + await fixture.cleanup(); + } + }); + + it('applies a language filter before the bounded rescue-candidate cutoff', async () => { + const decoys = Array.from({ length: 1051 }, (_, index) => { + const item = chunk(`src/noise-${index}.py`, 'system error algorithm'); + item.language = 'python'; + return item; + }); + const target = chunk('src/algorithm.ts', 'export const algorithm = "cohesion";'); + const fixture = await searchFixture([...decoys, target]); + try { + const matches = await fixture.search('system error algorithm cohesion', { + language: 'typescript' + }); + expect(matches[0]?.filePath).toContain('algorithm.ts'); + expect(matches).toHaveLength(1); + } finally { + await fixture.cleanup(); + } + }); +}); diff --git a/tests/search-quality.test.ts b/tests/search-quality.test.ts index 52adffe..f3703b0 100644 --- a/tests/search-quality.test.ts +++ b/tests/search-quality.test.ts @@ -26,7 +26,7 @@ describe('assessSearchQuality', () => { expect(quality.status).toBe('low_confidence'); expect(quality.confidence).toBe(0); expect(quality.signals).toContain('no results returned'); - expect(quality.nextSteps?.length).toBeGreaterThan(0); + expect(quality.nextSteps?.[0]).toMatch(/not evidence that the code is missing/i); }); it('flags test-artifact dominance for non-testing queries', () => { diff --git a/tests/search-ranking.test.ts b/tests/search-ranking.test.ts index 24449e4..d64883d 100644 --- a/tests/search-ranking.test.ts +++ b/tests/search-ranking.test.ts @@ -1,15 +1,25 @@ import { describe, it, expect, vi } from 'vitest'; import type { CodeChunk } from '../src/types/index.js'; import { CodebaseSearcher } from '../src/core/search.js'; +import { + asSearcherTestHarness, + materializeSourceSnapshot, + type SourceSnapshot +} from './helpers/source-snapshot.js'; -function createChunk(id: string, filePath: string, content: string): CodeChunk { +function createChunk( + id: string, + filePath: string, + content: string, + metadata: CodeChunk['metadata'] = {} +): CodeChunk { return { id, content, filePath, relativePath: filePath.replace(/^.*?Repos\//, ''), startLine: 1, - endLine: 40, + endLine: content.split('\n').length, language: 'typescript', framework: 'generic', componentType: 'service', @@ -18,28 +28,84 @@ function createChunk(id: string, filePath: string, content: string): CodeChunk { imports: [], exports: [], tags: [], - metadata: {} + metadata }; } function setupSemanticOnlySearcher( - results: { chunk: CodeChunk; score: number }[] + results: { chunk: CodeChunk; score: number }[], + snapshot: SourceSnapshot ): CodebaseSearcher { - const searcher = new CodebaseSearcher('C:/repo') as any; - searcher.initialized = true; - searcher.embeddingProvider = { + const searcher = new CodebaseSearcher(snapshot.rootPath); + const harness = asSearcherTestHarness(searcher); + harness.initialized = true; + harness.sourceManifest = snapshot.manifest; + harness.embeddingProvider = { embed: vi.fn(async () => [0.1, 0.2]) }; - searcher.storageProvider = { + harness.storageProvider = { search: vi.fn(async () => results), count: vi.fn(async () => results.length) }; - searcher.fuseIndex = null; - searcher.patternIntelligence = null; - return searcher as CodebaseSearcher; + harness.fuseIndex = null; + harness.patternIntelligence = null; + return searcher; } describe('CodebaseSearcher query-aware ranking', () => { + it('keeps same-named symbols from different files and deduplicates chunks within each file', async () => { + const symbolPath = ['readTargetDefaultsForTarget']; + const metadata = { + symbolName: 'readTargetDefaultsForTarget', + symbolPath, + symbolAware: true + }; + const firstFileBest = createChunk( + 'target-best', + 'C:/repo/packages/nx/target-defaults.ts', + 'export function readTargetDefaultsForTarget() { return "nx"; }', + metadata + ); + const firstFileDuplicate = createChunk( + 'target-duplicate', + 'C:/repo/packages/nx/target-defaults.ts', + 'export function readTargetDefaultsForTarget() { return "nx"; }', + metadata + ); + const secondFile = createChunk( + 'wrapper', + 'C:/repo/packages/devkit/target-defaults.ts', + 'export function readTargetDefaultsForTarget() { return "devkit"; }', + metadata + ); + + const snapshot = await materializeSourceSnapshot( + [firstFileBest, firstFileDuplicate, secondFile], + 'search-symbol-file-identity-' + ); + try { + const searcher = setupSemanticOnlySearcher( + [ + { chunk: firstFileBest, score: 0.9 }, + { chunk: firstFileDuplicate, score: 0.85 }, + { chunk: secondFile, score: 0.8 } + ], + snapshot + ); + + const results = await searcher.search('readTargetDefaultsForTarget', 2); + + expect(results).toHaveLength(2); + expect(results.map((result) => result.filePath)).toEqual([ + firstFileBest.filePath, + secondFile.filePath + ]); + expect(results[0].score).toBeGreaterThan(results[1].score); + } finally { + await snapshot.cleanup(); + } + }); + it('de-prioritizes spec files for non-testing queries', async () => { const specChunk = createChunk( 'spec', @@ -52,14 +118,24 @@ describe('CodebaseSearcher query-aware ranking', () => { 'export class SessionManager {}' ); - const searcher = setupSemanticOnlySearcher([ - { chunk: specChunk, score: 0.75 }, - { chunk: implChunk, score: 0.68 } - ]); - - const results = await searcher.search('Refactor session management flow', 2); - expect(results[0].filePath).toContain('session-manager.ts'); - expect(results[0].filePath).not.toContain('.spec.ts'); + const snapshot = await materializeSourceSnapshot( + [specChunk, implChunk], + 'search-ranking-source-' + ); + try { + const searcher = setupSemanticOnlySearcher( + [ + { chunk: specChunk, score: 0.75 }, + { chunk: implChunk, score: 0.68 } + ], + snapshot + ); + const results = await searcher.search('Refactor session management flow', 2); + expect(results[0].filePath).toContain('session-manager.ts'); + expect(results[0].filePath).not.toContain('.spec.ts'); + } finally { + await snapshot.cleanup(); + } }); it('keeps spec files prioritized for testing queries', async () => { @@ -74,13 +150,23 @@ describe('CodebaseSearcher query-aware ranking', () => { 'export class SessionManager {}' ); - const searcher = setupSemanticOnlySearcher([ - { chunk: specChunk, score: 0.75 }, - { chunk: implChunk, score: 0.68 } - ]); - - const results = await searcher.search('Update unit tests for session manager with mocks', 2); - expect(results[0].filePath).toContain('.spec.ts'); + const snapshot = await materializeSourceSnapshot( + [specChunk, implChunk], + 'search-ranking-source-' + ); + try { + const searcher = setupSemanticOnlySearcher( + [ + { chunk: specChunk, score: 0.75 }, + { chunk: implChunk, score: 0.68 } + ], + snapshot + ); + const results = await searcher.search('Update unit tests for session manager with mocks', 2); + expect(results[0].filePath).toContain('.spec.ts'); + } finally { + await snapshot.cleanup(); + } }); it('de-prioritizes Windows e2e paths for non-testing queries', async () => { @@ -95,12 +181,22 @@ describe('CodebaseSearcher query-aware ranking', () => { 'export class SessionManager {}' ); - const searcher = setupSemanticOnlySearcher([ - { chunk: e2eChunk, score: 0.75 }, - { chunk: implChunk, score: 0.72 } - ]); - - const results = await searcher.search('session login flow', 2); - expect(results[0].filePath.toLowerCase()).toContain('session-manager.ts'); + const snapshot = await materializeSourceSnapshot( + [e2eChunk, implChunk], + 'search-ranking-source-' + ); + try { + const searcher = setupSemanticOnlySearcher( + [ + { chunk: e2eChunk, score: 0.75 }, + { chunk: implChunk, score: 0.72 } + ], + snapshot + ); + const results = await searcher.search('session login flow', 2); + expect(results[0].filePath.toLowerCase()).toContain('session-manager.ts'); + } finally { + await snapshot.cleanup(); + } }); }); diff --git a/tests/search-recovery-sdk.test.ts b/tests/search-recovery-sdk.test.ts new file mode 100644 index 0000000..0d513c4 --- /dev/null +++ b/tests/search-recovery-sdk.test.ts @@ -0,0 +1,319 @@ +import { promises as fs } from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { fork, type ChildProcess } from 'node:child_process'; +import { fileURLToPath } from 'node:url'; +import { Client } from '@modelcontextprotocol/sdk/client/index.js'; +import { StdioClientTransport } from '@modelcontextprotocol/sdk/client/stdio.js'; +import { z } from 'zod'; +import { IndexSignalSchema } from '../src/core/index-signal.js'; +import { afterEach, expect, it } from 'vitest'; +import { CodebaseIndexer } from '../src/core/indexer.js'; +import { resolveIndexArtifact } from '../src/core/index-generation.js'; +import { makePaths } from '../src/project-state.js'; +import { handle as searchTool } from '../src/tools/search-codebase.js'; +import type { ToolContext } from '../src/tools/types.js'; + +let root = ''; +const clients: Array<{ client: Client; transport: StdioClientTransport }> = []; +const children: ChildProcess[] = []; +const nativeSessions = new Map string }>(); +afterEach(async () => { + for (const item of clients.splice(0)) { + await item.client.close().catch(() => {}); + await item.transport.close().catch(() => {}); + } + nativeSessions.clear(); + for (const child of children.splice(0)) { + if (child.exitCode !== null || child.signalCode !== null) continue; + const exited = new Promise((resolve) => child.once('exit', () => resolve())); + child.kill('SIGKILL'); + await exited; + } + if (root) await fs.rm(root, { recursive: true, force: true }); + root = ''; +}); +async function fixture() { + root = await fs.mkdtemp(path.join(os.tmpdir(), 'cbc-late-recovery-')); + await fs.writeFile(path.join(root, 'route.ts'), 'export function route() { return "original"; }'); + await new CodebaseIndexer({ rootPath: root, config: { skipEmbedding: true } }).index(); +} +async function corruptLocation() { + const indexPath = await resolveIndexArtifact(root, 'index.json'); + const value = JSON.parse(await fs.readFile(indexPath, 'utf8')); + value.chunks[0].filePath = path.join(root, '..', 'foreign.ts'); + await fs.writeFile(indexPath, JSON.stringify(value)); +} +const NativePayloadSchema = z + .object({ + status: z.string(), + index: IndexSignalSchema.optional(), + project: z.object({ indexStatus: z.string() }).passthrough(), + error: z.string().optional(), + message: z.string().optional() + }) + .passthrough(); +function decode(response: unknown): z.infer { + const envelope = z + .object({ + content: z.array(z.object({ type: z.string(), text: z.string().optional() }).passthrough()) + }) + .parse(response); + const item = envelope.content.find((item) => item.type === 'text'); + if (!item?.text) throw Error('Expected JSON text'); + return NativePayloadSchema.parse(JSON.parse(item.text)); +} + +async function server() { + const home = path.join(root, 'profile'); + await fs.mkdir(home); + const config = path.join(home, 'server.json'); + await fs.writeFile(config, '{}'); + const transport = new StdioClientTransport({ + command: process.execPath, + args: [ + '--import', + import.meta.resolve('tsx'), + fileURLToPath(new URL('../src/index.ts', import.meta.url)), + root + ], + cwd: root, + env: { + ...process.env, + CODEBASE_CONTEXT_CONFIG_PATH: config, + HOME: home, + USERPROFILE: home, + HF_HOME: path.join(home, 'models') + }, + stderr: 'pipe' + }); + let stderr = ''; + transport.stderr?.on('data', (data) => { + stderr += data.toString(); + }); + const client = new Client({ name: 'late-index-state-regression', version: '1.0.0' }); + nativeSessions.set(client, { root, stderr: () => stderr }); + clients.push({ client, transport }); + try { + await client.connect(transport); + } catch (error) { + throw new Error(`SDK connection failed: ${stderr}`, { cause: error }); + } + return { client, stderr: () => stderr }; +} +async function captureEnvelope(client: Client, phase: string, response: T): Promise { + const session = nativeSessions.get(client); + const envelope = { + phase, + time: new Date().toISOString(), + root: session?.root, + response, + serverStderr: session?.stderr() + }; + const diagnosticPath = process.env.CBC_TEST_SDK_DIAGNOSTICS; + if (diagnosticPath) await fs.appendFile(diagnosticPath, JSON.stringify(envelope) + '\n'); + return response; +} +async function status(client: Client, expected: string, phase = 'status') { + const deadline = Date.now() + 20000; + for (;;) { + const response = await captureEnvelope( + client, + phase, + await client.callTool({ name: 'get_indexing_status', arguments: {} }) + ); + let result: ReturnType; + try { + result = decode(response); + } catch (error) { + const session = nativeSessions.get(client); + throw new Error( + `Native status decoding failed before ${expected}: ${JSON.stringify({ phase, response, serverStderr: session?.stderr() })}`, + { cause: error } + ); + } + if (result.status === expected) return result; + if (Date.now() >= deadline) throw Error(`Did not reach ${expected}: ${JSON.stringify(result)}`); + await new Promise((resolve) => setTimeout(resolve, 25)); + } +} +async function holdWriter() { + const child = fork( + fileURLToPath(new URL('./helpers/index-writer-child.ts', import.meta.url)), + [root, 'hold', 'full'], + { execArgv: ['--import', 'tsx'], stdio: ['ignore', 'ignore', 'pipe', 'ipc'] } + ); + children.push(child); + let stderr = ''; + child.stderr?.on('data', (data) => { + stderr += data.toString(); + }); + await new Promise((resolve, reject) => { + const timer = setTimeout(() => reject(Error(`Writer did not stage: ${stderr}`)), 20000); + child.on('message', (message) => { + if (typeof message === 'object' && message !== null && 'type' in message) { + if (message.type === 'staged') { + clearTimeout(timer); + resolve(); + } else if (message.type === 'failed') { + clearTimeout(timer); + reject(Error(JSON.stringify(message))); + } + } + }); + child.once('error', reject); + }); + return child; +} +it('uses actual late-corruption state through the SDK and does not duplicate an already-running rebuild', async () => { + await fixture(); + const s = await server(); + await status(s.client, 'ready', 'initial ready'); + await corruptLocation(); + const held = await holdWriter(); + const query = () => + s.client.callTool( + { name: 'search_codebase', arguments: { query: 'route', limit: 5 } }, + undefined, + { timeout: 20000 } + ); + const first = decode(await captureEnvelope(s.client, 'first corrupt search', await query())); + expect(first.status).toBe('indexing'); + expect(first.index).toMatchObject({ + status: 'indexing', + confidence: 'low', + action: 'rebuild-started' + }); + expect(first.project.indexStatus).toBe('indexing'); + const starts = s.stderr().match(/Indexing \(full\)/g)?.length ?? 0; + const second = decode( + await captureEnvelope(s.client, 'second search while held writer', await query()) + ); + expect(second.status).toBe('indexing'); + expect(second.index?.status).toBe('indexing'); + expect(second.project.indexStatus).toBe('indexing'); + expect(s.stderr().match(/Indexing \(full\)/g)?.length ?? 0).toBe(starts); + const exited = new Promise((resolve) => held.once('exit', () => resolve())); + held.kill('SIGKILL'); + await exited; + await status(s.client, 'ready', 'after killed held writer'); + const healthy = decode( + await captureEnvelope(s.client, 'healthy search after recovery', await query()) + ); + expect(healthy.status).toBe('success'); + expect(healthy.index).toMatchObject({ status: 'ready', action: 'served' }); + expect(healthy.project.indexStatus).toBe('ready'); +}, 30000); +it('auto-heals a forged stored body through the real SDK without exposing its header', async () => { + await fixture(); + const s = await server(); + await status(s.client, 'ready'); + const file = await resolveIndexArtifact(root, 'index.json'); + const value = z + .object({ chunks: z.array(z.object({ content: z.string() }).passthrough()) }) + .passthrough() + .parse(JSON.parse(await fs.readFile(file, 'utf8'))); + for (const chunk of value.chunks) + chunk.content = '// FORGED_HEADER_MUST_NOT_ESCAPE\n' + chunk.content; + await fs.writeFile(file, JSON.stringify(value)); + const query = () => + s.client.callTool( + { name: 'search_codebase', arguments: { query: 'route', limit: 5 } }, + undefined, + { timeout: 20000 } + ); + const first = await query(); + expect(JSON.stringify(first)).not.toContain('FORGED_HEADER_MUST_NOT_ESCAPE'); + expect(decode(first).index?.action).not.toBe('served'); + await status(s.client, 'ready'); + const healthy = await query(); + expect(decode(healthy).status).toBe('success'); + expect(decode(healthy).index).toMatchObject({ status: 'ready', action: 'served' }); + expect(JSON.stringify(healthy)).not.toContain('FORGED_HEADER_MUST_NOT_ESCAPE'); +}, 30000); +it('reports failed capacity recovery through the real SDK without changing the old pointer/vector marker', async () => { + await fixture(); + await new CodebaseIndexer({ rootPath: root, config: { skipEmbedding: true } }).index(); + await new CodebaseIndexer({ rootPath: root, config: { skipEmbedding: true } }).index(); + const directory = path.join(root, '.codebase-context', '.generations'); + const pointer = path.join(root, '.codebase-context', 'index-meta.json'); + const meta = JSON.parse(await fs.readFile(pointer, 'utf8')); + const pins = path.join(root, '.codebase-context', '.pins'); + await fs.mkdir(pins, { recursive: true }); + for (const id of (await fs.readdir(directory)).filter((id) => id !== meta.buildId)) + await fs.writeFile( + path.join(pins, `${id}.json`), + JSON.stringify({ pid: process.pid, buildId: id }) + ); + const s = await server(); + await status(s.client, 'ready'); + await corruptLocation(); + const before = await fs.readFile(pointer); + const marker = path.join(directory, meta.buildId, 'index', 'index-build.json'); + const markerBytes = await fs.readFile(marker); + const response = decode( + await s.client.callTool( + { name: 'search_codebase', arguments: { query: 'route', limit: 5 } }, + undefined, + { timeout: 20000 } + ) + ); + expect(response.index?.status).toBe(response.status); + expect(response.index?.confidence).toBe('low'); + expect(response.index?.action).not.toBe('served'); + const failed = await status(s.client, 'error'); + expect(failed.project.indexStatus).toBe('error'); + expect(failed.error ?? failed.message).toMatch(/capacity|deferred/i); + expect(await fs.readFile(pointer)).toEqual(before); + expect(await fs.readFile(marker)).toEqual(markerBytes); +}, 30000); +it('keeps healthy artifact status and stale-source abstention distinct through a cold SDK session', async () => { + await fixture(); + await fs.writeFile( + path.join(root, 'route.ts'), + 'export function renamed() { return "changed before startup"; }' + ); + const pointer = path.join(root, '.codebase-context', 'index-meta.json'); + const before = await fs.readFile(pointer); + const s = await server(); + await status(s.client, 'ready'); + const result = decode( + await s.client.callTool( + { name: 'search_codebase', arguments: { query: 'route', limit: 5 } }, + undefined, + { timeout: 20000 } + ) + ); + expect(result.status).toBe('no_fresh_results'); + expect(result.index).toMatchObject({ status: 'ready', action: 'served' }); + expect(result.project.indexStatus).toBe('ready'); + expect(await fs.readFile(pointer)).toEqual(before); +}, 30000); +it.each(['throw', 'reject'] as const)( + 'reports an immediately %s launch without claiming indexing or ready', + async (kind) => { + await fixture(); + await corruptLocation(); + const ctx: ToolContext = { + rootPath: root, + paths: makePaths(root), + indexState: { status: 'ready' }, + performIndexing: () => { + if (kind === 'throw') throw Error('launch denied'); + return Promise.reject(Error('launch denied')); + } + }; + const result = await searchTool({ query: 'route' }, ctx); + const text = result.content?.[0]?.text; + if (!text) throw Error('Expected response'); + const value = JSON.parse(text); + expect(value.status).toBe('error'); + expect(value.index).toMatchObject({ + status: 'error', + confidence: 'low', + action: 'rebuild-failed' + }); + expect(ctx.indexState.status).toBe('error'); + expect(result.isError).toBe(true); + } +); diff --git a/tests/search-reranker-health.test.ts b/tests/search-reranker-health.test.ts index aea0cfd..6ff8ae8 100644 --- a/tests/search-reranker-health.test.ts +++ b/tests/search-reranker-health.test.ts @@ -29,6 +29,13 @@ vi.mock('../src/core/search.js', async () => { async search(query: string, limit: number, filters?: unknown) { return searchMocks.search(query, limit, filters); } + + async searchWithCoverage(query: string, limit: number, filters?: unknown) { + return { + results: await this.search(query, limit, filters), + sourceCoverage: { status: 'current', checkedFiles: 0, skippedFiles: 0 } + }; + } } return { CodebaseSearcher }; diff --git a/tests/search-rescue-density.test.ts b/tests/search-rescue-density.test.ts new file mode 100644 index 0000000..b88cb8a --- /dev/null +++ b/tests/search-rescue-density.test.ts @@ -0,0 +1,151 @@ +import { describe, expect, it } from 'vitest'; +import { promises as fs } from 'node:fs'; +import Fuse, { type IFuseOptions } from 'fuse.js'; +import { CodebaseSearcher } from '../src/core/search.js'; +import type { CodeChunk } from '../src/types/index.js'; +import { asSearcherTestHarness, materializeSourceSnapshot } from './helpers/source-snapshot.js'; + +interface DensityChunkSpec { + id: string; + filePath: string; + content: string; + startLine?: number; + metadata?: CodeChunk['metadata']; +} + +interface DensityCase { + id: string; + query: string; + chunks: DensityChunkSpec[]; + expectedIds: string[]; + expectedPathPatterns: string[]; + limitation?: string; +} + +interface GeneratedChunkCase { + count: number; + idPattern: string; + filePathPattern: string; + content: string; +} + +interface SourceValidationCase { + id: string; + query: string; + chunks?: DensityChunkSpec[]; + staleChunkGenerator?: GeneratedChunkCase; + currentChunks?: DensityChunkSpec[]; + mutateFilePath?: string; + mutateFilePathPattern?: string; + replacementSource: string; + expectedIds: string[]; +} + +interface DensityFixture { + cases: DensityCase[]; + sourceValidationCases: SourceValidationCase[]; +} + +interface DensityHarness { + chunks: CodeChunk[]; + fuseIndex: Fuse; + rescueKeywordMatches: ( + query: string, + limit: number + ) => Promise>; +} + +const fixture = JSON.parse( + await fs.readFile(new URL('./fixtures/search-rescue-density.json', import.meta.url), 'utf8') +) as DensityFixture; + +const fuseOptions: IFuseOptions = { + keys: [ + { name: 'content', weight: 0.4 }, + { name: 'metadata.componentName', weight: 0.25 }, + { name: 'filePath', weight: 0.15 }, + { name: 'relativePath', weight: 0.15 }, + { name: 'componentType', weight: 0.15 }, + { name: 'layer', weight: 0.1 }, + { name: 'tags', weight: 0.15 } + ], + includeScore: true, + threshold: 0.4, + useExtendedSearch: true, + ignoreLocation: true +}; + +function makeChunk(spec: DensityChunkSpec, index = 0): CodeChunk { + const startLine = spec.startLine ?? 1; + return { + id: spec.id || `density-${index}`, + filePath: spec.filePath, + relativePath: spec.filePath, + content: spec.content, + startLine, + endLine: startLine + spec.content.split('\n').length - 1, + language: 'typescript', + framework: 'generic', + componentType: 'module', + dependencies: [], + imports: [], + exports: [], + tags: [], + metadata: spec.metadata ?? {} + }; +} + +async function setup(chunks: CodeChunk[]) { + const snapshot = await materializeSourceSnapshot(chunks, 'cbc-rescue-density-'); + const searcher = new CodebaseSearcher(snapshot.rootPath); + const base = asSearcherTestHarness(searcher); + base.initialized = true; + base.sourceManifest = snapshot.manifest; + base.embeddingProvider = null; + base.storageProvider = null; + const fuse = new Fuse(chunks, fuseOptions); + base.fuseIndex = fuse; + const harness = searcher as unknown as DensityHarness; + harness.chunks = chunks; + return { ...snapshot, harness, fuse }; +} + +describe('frozen rescue density behavior controls', () => { + it.each(fixture.cases)('$id', async (control) => { + const subject = await setup(control.chunks.map(makeChunk)); + try { + const results = await subject.harness.rescueKeywordMatches(control.query, 10); + expect(results.map(({ item }) => item.id)).toEqual(control.expectedIds); + expect(results.map(({ item }) => item.relativePath)).toEqual(control.expectedPathPatterns); + } finally { + await subject.cleanup(); + } + }); + + it.each(fixture.sourceValidationCases)('$id', async (control) => { + const chunks = control.staleChunkGenerator + ? Array.from({ length: control.staleChunkGenerator.count }, (_, index) => { + const generated = control.staleChunkGenerator!; + return makeChunk({ + id: generated.idPattern.replace('{index}', String(index)), + filePath: generated.filePathPattern.replace('{index}', String(index)), + content: generated.content + }); + }).concat((control.currentChunks ?? []).map(makeChunk)) + : (control.chunks ?? []).map(makeChunk); + const subject = await setup(chunks); + try { + const changedChunks = control.mutateFilePathPattern + ? chunks.filter(({ id }) => id.startsWith('stale-')) + : chunks.filter(({ relativePath }) => relativePath === control.mutateFilePath); + for (const stale of changedChunks) { + await fs.writeFile(stale.filePath, control.replacementSource); + } + + const results = await subject.harness.rescueKeywordMatches(control.query, 5); + expect(results.map(({ item }) => item.id)).toEqual(control.expectedIds); + } finally { + await subject.cleanup(); + } + }); +}); diff --git a/tests/search-rescue-nomination.test.ts b/tests/search-rescue-nomination.test.ts new file mode 100644 index 0000000..1585be9 --- /dev/null +++ b/tests/search-rescue-nomination.test.ts @@ -0,0 +1,378 @@ +import { describe, expect, it, vi } from 'vitest'; +import { promises as fs } from 'node:fs'; +import path from 'node:path'; +import Fuse, { type FuseResult, type IFuseOptions } from 'fuse.js'; +import { CodebaseSearcher } from '../src/core/search.js'; +import { IndexCorruptedError } from '../src/errors/index.js'; +import type { CodeChunk, SearchFilters } from '../src/types/index.js'; +import { asSearcherTestHarness, materializeSourceSnapshot } from './helpers/source-snapshot.js'; + +interface ChunkSpec { + id?: string; + filePath?: string; + content: string; + startLine?: number; + language?: string; + metadata?: CodeChunk['metadata']; + exports?: string[]; + tags?: string[]; +} + +interface NominationFixture { + lexicalCases: Array< + ChunkSpec & { id: string; query: string; expected: boolean; expectedScore?: number } + >; + representativeCases: Array<{ + id: string; + query: string; + chunks: Array; + expectedIds: string[]; + }>; + populationCases: Array<{ + id: string; + limit: number; + count: number; + cap: number; + exhausted: boolean; + }>; + lifecycleCases: Array<'changed' | 'missing' | 'directory' | 'invalid-manifest' | 'outside-root'>; + graphCases: Array<'current' | 'no-edge' | 'changed-companion' | 'directory-companion'>; + strictCases: Array<{ id: string; query: string; expectsResult: boolean }>; +} + +interface Validation { + sources: Map; + candidateBudgetExhausted?: true; +} + +interface RescueHarness { + chunks: CodeChunk[]; + indexedImports: Map>; + indexedFileKey: (file: string) => string; + lexicalEvidence: (text: string) => Set; + rescueKeywordMatches: ( + query: string, + limit: number, + filters?: SearchFilters, + validation?: Validation + ) => Promise[]>; + keywordSearch: ( + query: string, + limit: number, + filters?: SearchFilters + ) => Promise>; +} + +const fixture = JSON.parse( + await fs.readFile(new URL('./fixtures/search-rescue-nomination.json', import.meta.url), 'utf8') +) as NominationFixture; + +const options: IFuseOptions = { + keys: [ + { name: 'content', weight: 0.4 }, + { name: 'metadata.componentName', weight: 0.25 }, + { name: 'filePath', weight: 0.15 }, + { name: 'relativePath', weight: 0.15 }, + { name: 'componentType', weight: 0.15 }, + { name: 'layer', weight: 0.1 }, + { name: 'tags', weight: 0.15 } + ], + includeScore: true, + threshold: 0.4, + useExtendedSearch: true, + ignoreLocation: true +}; + +function chunk(spec: ChunkSpec, index = 0): CodeChunk { + const startLine = spec.startLine ?? 1; + return { + id: spec.id ?? `chunk-${index}`, + filePath: spec.filePath ?? 'src/a.ts', + relativePath: spec.filePath ?? 'src/a.ts', + content: spec.content, + startLine, + endLine: startLine + spec.content.split('\n').length - 1, + language: spec.language ?? 'typescript', + framework: 'generic', + componentType: 'module', + dependencies: [], + imports: [], + exports: spec.exports ?? [], + tags: spec.tags ?? [], + metadata: spec.metadata ?? {} + }; +} + +async function setup(chunks: CodeChunk[]) { + const snapshot = await materializeSourceSnapshot(chunks, 'cbc-nomination-contract-'); + const searcher = new CodebaseSearcher(snapshot.rootPath); + const base = asSearcherTestHarness(searcher); + base.initialized = true; + base.sourceManifest = snapshot.manifest; + base.embeddingProvider = null; + base.storageProvider = null; + const fuse = new Fuse(chunks, options); + base.fuseIndex = fuse; + const harness = searcher as unknown as RescueHarness; + harness.chunks = chunks; + return { ...snapshot, searcher, harness, fuse }; +} + +describe('frozen rescue-only literal nomination contract', () => { + // These are lexical unit controls, not an accuracy score. In particular, a + // comment can provide a lexical clue without proving a program relationship. + it.each(fixture.lexicalCases)('$id', async (control) => { + const source = chunk(control); + const subject = await setup([source]); + try { + const results = await subject.harness.rescueKeywordMatches(control.query, 5); + expect(results.map(({ item }) => item.id)).toEqual(control.expected ? [source.id] : []); + if (control.expected) expect(results[0]?.score).toBe(control.expectedScore); + } finally { + await subject.cleanup(); + } + }); + + it.each(fixture.representativeCases)('$id', async (control) => { + const subject = await setup(control.chunks.map(chunk)); + try { + const results = await subject.harness.rescueKeywordMatches(control.query, 5); + expect(results.map(({ item }) => item.id)).toEqual(control.expectedIds); + expect(results.every(({ score }) => score !== undefined && score >= 0 && score <= 1)).toBe( + true + ); + } finally { + await subject.cleanup(); + } + }); + + it('does not invoke corpus fuzzy matching to nominate or rank a rescue witness', async () => { + const subject = await setup([chunk({ content: '// archiving policies' })]); + const fuzzy = vi.spyOn(Fuse.prototype, 'search'); + try { + expect(await subject.harness.rescueKeywordMatches('archive policy', 5)).toHaveLength(1); + expect(fuzzy).not.toHaveBeenCalled(); + } finally { + fuzzy.mockRestore(); + await subject.cleanup(); + } + }); + + it.each(fixture.populationCases)('$id', async (control) => { + const chunks = Array.from({ length: control.count }, (_, index) => + chunk({ + id: `nominee-${index}`, + filePath: index === control.cap ? 'src/b.ts' : 'src/a.ts', + content: '// archive policy' + }) + ); + const subject = await setup(chunks); + const evidence = vi.spyOn(subject.harness, 'lexicalEvidence'); + const validation: Validation = { sources: new Map() }; + try { + const results = await subject.harness.rescueKeywordMatches( + 'archive policy', + control.limit, + undefined, + validation + ); + // Overflow is a declared bounded miss, not an invented complete search. + expect(results.map(({ item }) => item.id)).toEqual(['nominee-0']); + expect(Boolean(validation.candidateBudgetExhausted)).toBe(control.exhausted); + expect(evidence.mock.calls.length).toBeLessThanOrEqual(control.cap * 2); + } finally { + await subject.cleanup(); + } + }); + + it.each(['language', 'framework', 'componentType', 'layer', 'tags'] as const)( + 'applies %s before the candidate window', + async (field) => { + const rejected = Array.from({ length: 760 }, (_, index) => + chunk({ + id: `excluded-${index}`, + filePath: 'src/a.ts', + content: '// archive policy' + }) + ); + const target = chunk({ id: 'included', filePath: 'src/b.ts', content: '// archive policy' }); + const filters: SearchFilters = {}; + if (field === 'language') { + filters.language = 'python'; + target.language = 'python'; + } + if (field === 'framework') { + filters.framework = 'test-framework'; + target.framework = 'test-framework'; + } + if (field === 'componentType') { + filters.componentType = 'function'; + target.componentType = 'function'; + } + if (field === 'layer') { + filters.layer = 'business'; + target.layer = 'business'; + } + if (field === 'tags') { + filters.tags = ['included']; + target.tags = ['included']; + } + const subject = await setup([...rejected, target]); + const validation: Validation = { sources: new Map() }; + try { + const results = await subject.harness.rescueKeywordMatches( + 'archive policy', + 1, + filters, + validation + ); + expect(results.map(({ item }) => item.id)).toEqual(['included']); + expect(validation.candidateBudgetExhausted).toBeUndefined(); + } finally { + await subject.cleanup(); + } + } + ); + + it.each(fixture.lifecycleCases)('preserves %s source handling', async (state) => { + const source = chunk({ content: '// archive policy' }); + const subject = await setup([source]); + const validation: Validation = { sources: new Map() }; + try { + if (state === 'changed') await fs.writeFile(source.filePath, '// replacement\n'); + if (state === 'missing' || state === 'directory') await fs.unlink(source.filePath); + if (state === 'directory') await fs.mkdir(source.filePath); + if (state === 'invalid-manifest') subject.manifest.files['src/a.ts'] = 'invalid'; + if (state === 'outside-root') + source.filePath = path.resolve(subject.rootPath, '../escape.ts'); + const operation = subject.harness.rescueKeywordMatches( + 'archive policy', + 5, + undefined, + validation + ); + if (state === 'changed' || state === 'missing') { + expect(await operation).toEqual([]); + expect(validation.sources.get(source.filePath)).toBeNull(); + } else { + await expect(operation).rejects.toBeInstanceOf(IndexCorruptedError); + } + } finally { + await subject.cleanup(); + } + }); + + it('refills past all chunks of a stale source without rerunning Fuse', async () => { + const stale = Array.from({ length: 270 }, (_, index) => + chunk({ + id: `stale-${index}`, + content: '// archive policy' + }) + ); + const current = chunk({ id: 'current', filePath: 'src/b.ts', content: '// archive policy' }); + const subject = await setup([...stale, current]); + const fuzzy = vi.spyOn(Fuse.prototype, 'search'); + try { + await fs.writeFile(stale[0].filePath, '// replacement\n'); + const results = await subject.harness.rescueKeywordMatches('archive policy', 1); + expect(results.map(({ item }) => item.id)).toEqual(['current']); + expect(fuzzy).not.toHaveBeenCalled(); + } finally { + fuzzy.mockRestore(); + await subject.cleanup(); + } + }); + + it('exposes nomination overflow through the public source coverage result', async () => { + const chunks = Array.from({ length: 752 }, (_, index) => + chunk({ + id: `split-${index}`, + content: index % 2 === 0 ? '// archive' : '// policy', + startLine: index % 2 === 0 ? 1 : 3 + }) + ); + const subject = await setup(chunks); + try { + const outcome = await subject.searcher.searchWithCoverage('archive policy', 1, undefined, { + useSemanticSearch: false, + enableQueryExpansion: false, + enableLowConfidenceRescue: false, + enableReranker: false + }); + expect(outcome.results).toHaveLength(1); + expect(outcome.sourceCoverage.status).toBe('partial'); + expect(outcome.sourceCoverage.candidateBudgetExhausted).toBe(true); + } finally { + await subject.cleanup(); + } + }); + + it.each(fixture.graphCases)('requires current resolved graph proof: %s', async (state) => { + const anchor = chunk({ + id: 'anchor', + content: 'import { rules } from "./b.js";\nexport class LedgerGateway {}', + metadata: { className: 'LedgerGateway' } + }); + const companion = chunk({ + id: 'companion', + filePath: 'src/b.ts', + content: 'export const rules = { archive: true, policy: true };' + }); + const subject = await setup([anchor, companion]); + if (state !== 'no-edge') { + subject.harness.indexedImports = new Map([ + [ + subject.harness.indexedFileKey(anchor.filePath), + new Map([[subject.harness.indexedFileKey(companion.filePath), ['./b.js']]]) + ] + ]); + } + try { + if (state === 'changed-companion') + await fs.writeFile(companion.filePath, 'export const rules = {};\n'); + if (state === 'directory-companion') { + await fs.unlink(companion.filePath); + await fs.mkdir(companion.filePath); + } + const operation = subject.harness.rescueKeywordMatches('LedgerGateway archive policy', 5); + if (state === 'directory-companion') { + await expect(operation).rejects.toBeInstanceOf(IndexCorruptedError); + } else { + const ids = (await operation).map(({ item }) => item.id); + expect(ids.includes('anchor')).toBe(state === 'current'); + } + } finally { + await subject.cleanup(); + } + }); + + it.each(fixture.strictCases)('retains original strict Fuse route: $id', async (control) => { + const source = chunk({ content: 'export class ArchivePolicy {}\n// archive policy' }); + const subject = await setup([source]); + const fuzzy = vi.spyOn(subject.fuse, 'search'); + try { + const results = await subject.harness.keywordSearch(control.query, 5); + expect(fuzzy).toHaveBeenCalledWith(control.query); + expect(results.length > 0).toBe(control.expectsResult); + } finally { + await subject.cleanup(); + } + }); + + it.each([ + 'archive | policy', + '^archive policy', + 'archive policy$', + '!archive policy', + '=archive policy' + ])('keeps explicit syntax out of lexical rescue: %s', async (query) => { + const subject = await setup([chunk({ content: '// archive policy' })]); + const fuzzy = vi.spyOn(subject.fuse, 'search'); + try { + expect(await subject.harness.rescueKeywordMatches(query, 5)).toEqual([]); + expect(fuzzy).not.toHaveBeenCalled(); + } finally { + await subject.cleanup(); + } + }); +}); diff --git a/tests/search-retrieval-strategy.test.ts b/tests/search-retrieval-strategy.test.ts index 0cb92a2..166d5db 100644 --- a/tests/search-retrieval-strategy.test.ts +++ b/tests/search-retrieval-strategy.test.ts @@ -1,6 +1,10 @@ import { describe, it, expect, vi } from 'vitest'; import type { CodeChunk } from '../src/types/index.js'; import { CodebaseSearcher } from '../src/core/search.js'; +import { asSearcherTestHarness, materializeSourceSnapshot } from './helpers/source-snapshot.js'; +import { promises as fs } from 'node:fs'; +import path from 'node:path'; +import { IndexCorruptedError } from '../src/errors/index.js'; function createChunk(id: string, filePath: string, content: string): CodeChunk { return { @@ -9,7 +13,7 @@ function createChunk(id: string, filePath: string, content: string): CodeChunk { filePath, relativePath: filePath, startLine: 1, - endLine: 20, + endLine: content.split('\n').length, language: 'typescript', framework: 'generic', componentType: 'service', @@ -22,31 +26,226 @@ function createChunk(id: string, filePath: string, content: string): CodeChunk { }; } +async function createSourceBackedSearcher(chunks: CodeChunk[]) { + const snapshot = await materializeSourceSnapshot(chunks, 'retrieval-source-'); + const searcher = new CodebaseSearcher(snapshot.rootPath); + const harness = asSearcherTestHarness(searcher); + harness.initialized = true; + harness.sourceManifest = snapshot.manifest; + harness.embeddingProvider = {}; + harness.storageProvider = {}; + harness.fuseIndex = null; + harness.patternIntelligence = null; + harness.semanticSearch = vi.fn(async () => []); + harness.keywordSearch = vi.fn(async () => []); + return { + searcher, + harness, + cleanup: snapshot.cleanup + }; +} + describe('CodebaseSearcher retrieval strategy', () => { + it.each(['ordinary', 'integrity'] as const)( + 'preserves validated candidates on ordinary refill failure and propagates integrity failure (%s)', + async (failure) => { + const stale = Array.from({ length: 100 }, (_, n) => + createChunk( + `stale-${n}`, + 'src/quartzBeacon.ts', + 'export function quartzBeacon() { return true; }' + ) + ); + const support = createChunk( + 'support', + 'src/support.ts', + 'export function support() { return quartzBeacon(); }' + ); + const chunks = [...stale, support]; + const fixture = await createSourceBackedSearcher(chunks); + const harness = fixture.harness as typeof fixture.harness & { chunks: CodeChunk[] }; + harness.chunks = chunks; + Reflect.deleteProperty(harness, 'semanticSearch'); + const embed = vi.fn(async () => [0.1, 0.2]); + const vectorSearch = vi + .fn() + .mockResolvedValueOnce( + [...stale.slice(0, 29), support].map((chunk) => ({ chunk, score: 0.9, distance: 0.1 })) + ) + .mockRejectedValueOnce( + failure === 'integrity' + ? new IndexCorruptedError('Invalid vector index build identity') + : new Error('Semantic refill temporarily unavailable') + ); + harness.embeddingProvider = { embed }; + harness.storageProvider = { search: vectorSearch }; + try { + await fs.writeFile(stale[0].filePath, 'export const changed = true;\n'); + const outcome = fixture.searcher.searchWithCoverage( + 'ordinary semantic request', + 1, + undefined, + { + useKeywordSearch: false, + useSemanticSearch: true, + enableQueryExpansion: false, + enableLowConfidenceRescue: false, + enableReranker: false + } + ); + if (failure === 'integrity') { + await expect(outcome).rejects.toBeInstanceOf(IndexCorruptedError); + } else { + const result = await outcome; + expect(result.results.map((match) => path.basename(match.filePath))).toEqual([ + 'support.ts' + ]); + expect(result.results[0].snippet).toContain('support'); + expect(result.sourceCoverage).toMatchObject({ + status: 'partial', + checkedFiles: 2, + skippedFiles: 1, + candidateRetrievalFailed: true + }); + expect(result.sourceCoverage.candidateBudgetExhausted).toBeUndefined(); + } + expect(embed).toHaveBeenCalledTimes(1); + expect(vectorSearch).toHaveBeenCalledTimes(2); + } finally { + await fixture.cleanup(); + } + } + ); + + it('discloses semantic exhaustion when widening reaches the bounded cumulative retrieval work', async () => { + const stale = Array.from({ length: 1000 }, (_, n) => + createChunk( + `stale-${n}`, + 'src/quartzBeacon.ts', + 'export function quartzBeacon() { return true; }' + ) + ); + const support = createChunk( + 'support', + 'src/support.ts', + 'export function support() { return quartzBeacon(); }' + ); + const chunks = [...stale, support]; + const fixture = await createSourceBackedSearcher(chunks); + const harness = fixture.harness as typeof fixture.harness & { chunks: CodeChunk[] }; + harness.chunks = chunks; + Reflect.deleteProperty(harness, 'semanticSearch'); + const embed = vi.fn(async () => [0.1, 0.2]); + const vectorSearch = vi.fn(async (_vector: number[], limit: number) => + chunks.slice(0, limit).map((chunk) => ({ chunk, score: 0.9, distance: 0.1 })) + ); + harness.embeddingProvider = { embed }; + harness.storageProvider = { search: vectorSearch }; + try { + await fs.writeFile(stale[0].filePath, 'export const changed = true;\n'); + const outcome = await fixture.searcher.searchWithCoverage('quartzBeacon', 1, undefined, { + useKeywordSearch: false, + useSemanticSearch: true, + enableQueryExpansion: false, + enableLowConfidenceRescue: false, + enableReranker: false + }); + expect(outcome.results).toEqual([]); + expect(outcome.sourceCoverage).toMatchObject({ + status: 'no_fresh_results', + checkedFiles: 1, + skippedFiles: 1, + candidateBudgetExhausted: true + }); + expect(embed).toHaveBeenCalledTimes(1); + expect(vectorSearch).toHaveBeenCalledTimes(2); + expect( + vectorSearch.mock.calls.reduce((sum, [, limit]) => sum + limit, 0) + ).toBeLessThanOrEqual(750); + expect(Math.max(...vectorSearch.mock.calls.map(([, limit]) => limit))).toBeLessThanOrEqual( + chunks.length + ); + } finally { + await fixture.cleanup(); + } + }); + + it.each([false, true])( + 'replenishes semantic stale prefixes only when needed (changed=%s) with one embedding', + async (changed) => { + const staleChunks = Array.from({ length: 100 }, (_, n) => + createChunk( + `stale-${n}`, + 'src/quartzBeacon.ts', + 'export function quartzBeacon() { return true; }' + ) + ); + const support = createChunk( + 'support', + 'src/support.ts', + 'export function support() { return quartzBeacon(); }' + ); + const chunks = [...staleChunks, support]; + const fixture = await createSourceBackedSearcher(chunks); + const harness = fixture.harness as typeof fixture.harness & { chunks: CodeChunk[] }; + harness.chunks = chunks; + Reflect.deleteProperty(harness, 'semanticSearch'); + const embed = vi.fn(async () => [0.1, 0.2]); + const vectorSearch = vi.fn(async (_vector: number[], limit: number) => + chunks.slice(0, limit).map((chunk) => ({ chunk, score: 0.9, distance: 0.1 })) + ); + harness.embeddingProvider = { embed }; + harness.storageProvider = { search: vectorSearch }; + try { + if (changed) await fs.writeFile(staleChunks[0].filePath, 'export const changed = true;\n'); + const outcome = await fixture.searcher.searchWithCoverage('quartzBeacon', 1, undefined, { + useKeywordSearch: false, + useSemanticSearch: true, + enableQueryExpansion: false, + enableLowConfidenceRescue: false, + enableReranker: false + }); + expect(outcome.results).toHaveLength(1); + expect(path.basename(outcome.results[0].filePath)).toBe( + changed ? 'support.ts' : 'quartzBeacon.ts' + ); + expect(embed).toHaveBeenCalledTimes(1); + expect(vectorSearch).toHaveBeenCalledTimes(changed ? 2 : 1); + expect(vectorSearch.mock.calls[0][1]).toBe(30); + if (changed) { + expect(vectorSearch.mock.calls[1][1]).toBe(101); + expect(outcome.sourceCoverage).toMatchObject({ + status: 'partial', + checkedFiles: 2, + skippedFiles: 1 + }); + } + } finally { + await fixture.cleanup(); + } + } + ); + it('enforces candidate floor independently from user limit', async () => { const implChunk = createChunk( 'impl', 'src/core/auth-service.ts', 'export class AuthService {}' ); - const searcher = new CodebaseSearcher('C:/repo') as any; - - searcher.initialized = true; - searcher.embeddingProvider = {}; - searcher.storageProvider = {}; - searcher.fuseIndex = null; - searcher.patternIntelligence = null; - searcher.semanticSearch = vi.fn(async () => [{ chunk: implChunk, score: 0.7 }]); - searcher.keywordSearch = vi.fn(async () => []); - - await searcher.search('authentication login', 1, undefined, { - enableQueryExpansion: false, - enableLowConfidenceRescue: false, - candidateFloor: 30 - }); - - expect(searcher.semanticSearch).toHaveBeenCalledTimes(1); - expect(searcher.semanticSearch.mock.calls[0][1]).toBe(30); + const { searcher, harness, cleanup } = await createSourceBackedSearcher([implChunk]); + const semanticSearch = vi.fn(async () => [{ chunk: implChunk, score: 0.7 }]); + harness.semanticSearch = semanticSearch; + try { + await searcher.search('authentication login', 1, undefined, { + enableQueryExpansion: false, + enableLowConfidenceRescue: false, + candidateFloor: 30 + }); + expect(semanticSearch).toHaveBeenCalledTimes(1); + expect(semanticSearch.mock.calls[0][1]).toBe(30); + } finally { + await cleanup(); + } }); it('uses bounded query expansion for intent-heavy queries', async () => { @@ -55,24 +254,20 @@ describe('CodebaseSearcher retrieval strategy', () => { 'src/core/router-service.ts', 'export class RouterService {}' ); - const searcher = new CodebaseSearcher('C:/repo') as any; - - searcher.initialized = true; - searcher.embeddingProvider = {}; - searcher.storageProvider = {}; - searcher.fuseIndex = null; - searcher.patternIntelligence = null; - searcher.semanticSearch = vi.fn(async () => [{ chunk: implChunk, score: 0.65 }]); - searcher.keywordSearch = vi.fn(async () => []); - - await searcher.search('authentication login', 3, undefined, { - enableQueryExpansion: true, - enableLowConfidenceRescue: false - }); - - const semanticQueries = searcher.semanticSearch.mock.calls.map((call: any[]) => call[0]); - expect(semanticQueries[0]).toBe('authentication login'); - expect(semanticQueries.length).toBeLessThanOrEqual(2); + const { searcher, harness, cleanup } = await createSourceBackedSearcher([implChunk]); + const semanticSearch = vi.fn(async (_query: string) => [{ chunk: implChunk, score: 0.65 }]); + harness.semanticSearch = semanticSearch; + try { + await searcher.search('authentication login', 3, undefined, { + enableQueryExpansion: true, + enableLowConfidenceRescue: false + }); + const semanticQueries = semanticSearch.mock.calls.map(([query]) => query); + expect(semanticQueries[0]).toBe('authentication login'); + expect(semanticQueries.length).toBeLessThanOrEqual(2); + } finally { + await cleanup(); + } }); it('runs low-confidence rescue and can replace poor primary ranking', async () => { @@ -87,14 +282,9 @@ describe('CodebaseSearcher retrieval strategy', () => { 'export class AuthCallbackComponent {}' ); - const searcher = new CodebaseSearcher('C:/repo') as any; - searcher.initialized = true; - searcher.embeddingProvider = {}; - searcher.storageProvider = {}; - searcher.fuseIndex = null; - searcher.patternIntelligence = null; + const { searcher, harness, cleanup } = await createSourceBackedSearcher([specChunk, implChunk]); - searcher.semanticSearch = vi.fn(async (query: string) => { + const semanticSearch = vi.fn(async (query: string) => { if (query.includes('router') || query.includes('navigation')) { return [ { chunk: implChunk, score: 0.8 }, @@ -107,16 +297,18 @@ describe('CodebaseSearcher retrieval strategy', () => { { chunk: implChunk, score: 0.35 } ]; }); - searcher.keywordSearch = vi.fn(async () => []); - - const results = await searcher.search('navigate to page after login redirect', 2, undefined, { - enableQueryExpansion: false, - enableLowConfidenceRescue: true, - profile: 'edit' - }); - - expect(results[0].filePath).toContain('auth-callback.component.ts'); - expect(results[0].filePath).not.toContain('.spec.ts'); + harness.semanticSearch = semanticSearch; + try { + const results = await searcher.search('navigate to page after login redirect', 2, undefined, { + enableQueryExpansion: false, + enableLowConfidenceRescue: true, + profile: 'edit' + }); + expect(results[0].filePath).toContain('auth-callback.component.ts'); + expect(results[0].filePath).not.toContain('.spec.ts'); + } finally { + await cleanup(); + } }); it('applies intent-based keyword-heavy weights for exact-name queries', async () => { @@ -126,24 +318,19 @@ describe('CodebaseSearcher retrieval strategy', () => { 'export class AuthService { login() {} }' ); - const searcher = new CodebaseSearcher('C:/repo') as any; - searcher.initialized = true; - searcher.embeddingProvider = {}; - searcher.storageProvider = {}; - searcher.fuseIndex = null; - searcher.patternIntelligence = null; + const { searcher, harness, cleanup } = await createSourceBackedSearcher([implChunk]); let capturedSemanticWeight: number | undefined; let capturedKeywordWeight: number | undefined; // Intercept collectHybridMatches to capture the weights used - searcher.collectHybridMatches = vi.fn( + harness.collectHybridMatches = vi.fn( async ( - _variants: any, - _limit: any, - _filters: any, - _useSemantic: any, - _useKeyword: any, + _variants: unknown, + _limit: unknown, + _filters: unknown, + _useSemantic: unknown, + _useKeyword: unknown, semWeight: number, kwWeight: number ) => { @@ -153,22 +340,23 @@ describe('CodebaseSearcher retrieval strategy', () => { semantic: new Map([ ['impl', { chunk: implChunk, ranks: [{ rank: 0, weight: semWeight }] }] ]), - keyword: new Map([ - ['impl', { chunk: implChunk, ranks: [{ rank: 0, weight: kwWeight }] }] - ]) + keyword: new Map([['impl', { chunk: implChunk, ranks: [{ rank: 0, weight: kwWeight }] }]]) }; } ); // AuthService is PascalCase → EXACT_NAME intent → keyword: 0.6, semantic: 0.4 - await searcher.search('AuthService', 3, undefined, { - enableQueryExpansion: false, - enableLowConfidenceRescue: false - }); - - expect(capturedKeywordWeight).toBeGreaterThan(capturedSemanticWeight!); - expect(capturedKeywordWeight).toBeCloseTo(0.6, 1); - expect(capturedSemanticWeight).toBeCloseTo(0.4, 1); + try { + await searcher.search('AuthService', 3, undefined, { + enableQueryExpansion: false, + enableLowConfidenceRescue: false + }); + expect(capturedKeywordWeight).toBeGreaterThan(capturedSemanticWeight!); + expect(capturedKeywordWeight).toBeCloseTo(0.6, 1); + expect(capturedSemanticWeight).toBeCloseTo(0.4, 1); + } finally { + await cleanup(); + } }); it('produces scores below 1.0 for weak single-list retrievals', async () => { @@ -179,22 +367,20 @@ describe('CodebaseSearcher retrieval strategy', () => { ); const weakChunk = createChunk('weak', 'src/utils/helpers.ts', 'export function helper() {}'); - const searcher = new CodebaseSearcher('C:/repo') as any; - searcher.initialized = true; - searcher.embeddingProvider = {}; - searcher.storageProvider = {}; - searcher.fuseIndex = null; - searcher.patternIntelligence = null; + const { searcher, harness, cleanup } = await createSourceBackedSearcher([ + strongChunk, + weakChunk + ]); // Mock collectHybridMatches: strong chunk rank 0 in both channels, // weak chunk rank 5 in keyword only - searcher.collectHybridMatches = vi.fn( + harness.collectHybridMatches = vi.fn( async ( - _variants: any, - _limit: any, - _filters: any, - _useSemantic: any, - _useKeyword: any, + _variants: unknown, + _limit: unknown, + _filters: unknown, + _useSemantic: unknown, + _useKeyword: unknown, semWeight: number, kwWeight: number ) => ({ @@ -208,17 +394,20 @@ describe('CodebaseSearcher retrieval strategy', () => { }) ); - const results = await searcher.search('authentication login', 10, undefined, { - enableQueryExpansion: false, - enableLowConfidenceRescue: false - }); - - expect(results.length).toBeGreaterThanOrEqual(2); - expect(results[0].score).toBeGreaterThan(0); - // Key invariant: strong result (rank 0, both channels) must score - // meaningfully higher than weak result (rank 5, keyword only). - // With normalization-by-actual-max, the ratio collapses. - const ratio = results[0].score / results[1].score; - expect(ratio).toBeGreaterThan(1.5); + try { + const results = await searcher.search('authentication login', 10, undefined, { + enableQueryExpansion: false, + enableLowConfidenceRescue: false + }); + expect(results.length).toBeGreaterThanOrEqual(2); + expect(results[0].score).toBeGreaterThan(0); + // Key invariant: strong result (rank 0, both channels) must score + // meaningfully higher than weak result (rank 5, keyword only). + // With normalization-by-actual-max, the ratio collapses. + const ratio = results[0].score / results[1].score; + expect(ratio).toBeGreaterThan(1.5); + } finally { + await cleanup(); + } }); }); diff --git a/tests/search-safe-01.test.ts b/tests/search-safe-01.test.ts index bee7c75..e2b7cc0 100644 --- a/tests/search-safe-01.test.ts +++ b/tests/search-safe-01.test.ts @@ -24,6 +24,13 @@ vi.mock('../src/core/search.js', async () => { async search(query: string, limit: number, filters?: unknown) { return searchMocks.search(query, limit, filters); } + + async searchWithCoverage(query: string, limit: number, filters?: unknown) { + return { + results: await this.search(query, limit, filters), + sourceCoverage: { status: 'current', checkedFiles: 0, skippedFiles: 0 } + }; + } } return { CodebaseSearcher }; diff --git a/tests/search-scoped-memory.test.ts b/tests/search-scoped-memory.test.ts index 0f6df64..ae21450 100644 --- a/tests/search-scoped-memory.test.ts +++ b/tests/search-scoped-memory.test.ts @@ -16,6 +16,13 @@ vi.mock('../src/core/search.js', async () => { async search(query: string, limit: number, filters?: unknown) { return searchMocks.search(query, limit, filters); } + + async searchWithCoverage(query: string, limit: number, filters?: unknown) { + return { + results: await this.search(query, limit, filters), + sourceCoverage: { status: 'current', checkedFiles: 0, skippedFiles: 0 } + }; + } } return { CodebaseSearcher }; diff --git a/tests/search-source-validity.test.ts b/tests/search-source-validity.test.ts new file mode 100644 index 0000000..a02e399 --- /dev/null +++ b/tests/search-source-validity.test.ts @@ -0,0 +1,961 @@ +import { resolveIndexArtifact } from '../src/core/index-generation.js'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { promises as fs } from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { CodebaseIndexer } from '../src/core/indexer.js'; +import { CodebaseSearcher, type SearchOptions } from '../src/core/search.js'; +import { analyzerRegistry } from '../src/core/analyzer-registry.js'; +import { GenericAnalyzer } from '../src/analyzers/generic/index.js'; +import { IndexCorruptedError } from '../src/errors/index.js'; +import { + CODEBASE_CONTEXT_DIRNAME, + HEALTH_FILENAME, + INTELLIGENCE_FILENAME, + KEYWORD_INDEX_FILENAME, + MANIFEST_FILENAME, + MEMORY_FILENAME, + VECTOR_DB_DIRNAME +} from '../src/constants/codebase-context.js'; +import { findSymbolReferences } from '../src/core/symbol-references.js'; +import type { CodeChunk, SearchResult } from '../src/types/index.js'; +import { handle as searchCodebase } from '../src/tools/search-codebase.js'; +import type { SearchResponse, ToolContext } from '../src/tools/types.js'; + +const KEYWORD_ONLY_OPTIONS: SearchOptions = { + useSemanticSearch: false, + useKeywordSearch: true, + enableQueryExpansion: false, + enableLowConfidenceRescue: false, + enableReranker: false, + profile: 'explore' +}; + +function isContained(rootPath: string, candidatePath: string): boolean { + const relativePath = path.relative(rootPath, candidatePath); + return ( + relativePath === '' || + (relativePath !== '..' && + !relativePath.startsWith(`..${path.sep}`) && + !path.isAbsolute(relativePath)) + ); +} + +async function indexProject(rootPath: string, extraFileExtensions: string[] = []): Promise { + const indexer = new CodebaseIndexer({ + rootPath, + config: { + skipEmbedding: true, + embedding: { provider: 'openai', model: 'text-embedding-3-small' }, + parsing: { maxFileSize: 1024 * 1024 } + }, + projectOptions: { + preferredAnalyzer: 'generic', + extraFileExtensions + } + }); + await indexer.index(); +} + +async function createSourceFile( + rootPath: string, + relativePath: string, + content: string +): Promise { + const filePath = path.join(rootPath, relativePath); + await fs.mkdir(path.dirname(filePath), { recursive: true }); + await fs.writeFile(filePath, content, 'utf8'); +} + +function searchContext(rootPath: string): ToolContext { + const baseDir = path.join(rootPath, CODEBASE_CONTEXT_DIRNAME); + return { + rootPath, + indexState: { status: 'ready' }, + paths: { + baseDir, + keywordIndex: path.join(baseDir, KEYWORD_INDEX_FILENAME), + intelligence: path.join(baseDir, INTELLIGENCE_FILENAME), + memory: path.join(baseDir, MEMORY_FILENAME), + health: path.join(baseDir, HEALTH_FILENAME), + vectorDb: path.join(baseDir, VECTOR_DB_DIRNAME) + }, + performIndexing: vi.fn() + }; +} + +async function expectSearchResultsOpenInsideRoot( + rootPath: string, + results: SearchResult[], + expectedRelativePath: string +): Promise { + expect(results.length).toBeGreaterThan(0); + for (const result of results) { + expect(path.isAbsolute(result.filePath)).toBe(true); + expect(isContained(rootPath, result.filePath)).toBe(true); + expect(path.relative(rootPath, result.filePath).replace(/\\/g, '/')).toBe(expectedRelativePath); + const source = await fs.readFile(result.filePath, 'utf8'); + const sourceLines = source.replace(/\r\n/g, '\n').split('\n'); + const indexedRange = sourceLines.slice(result.startLine - 1, result.endLine).join('\n'); + expect(indexedRange).toContain('quartzBeacon'); + expect(indexedRange).toContain(result.snippet.replace(/\n\n\.\.\. \[\d+ more lines\]$/, '')); + expect(result.startLine).toBeGreaterThan(0); + expect(result.endLine).toBeGreaterThanOrEqual(result.startLine); + } +} + +describe('search source-path validity', () => { + let tempRoot: string; + let originalEmbeddingProvider: string | undefined; + let originalOpenAiKey: string | undefined; + let consoleWarnSpy: ReturnType; + + beforeEach(async () => { + tempRoot = await fs.mkdtemp(path.join(os.tmpdir(), 'cbc-search-source-validity-')); + originalEmbeddingProvider = process.env.EMBEDDING_PROVIDER; + originalOpenAiKey = process.env.OPENAI_API_KEY; + // CodebaseSearcher uses its real provider-selection path, which fails closed before any + // network request when OpenAI is selected without a key; keyword search remains available. + process.env.EMBEDDING_PROVIDER = 'openai'; + delete process.env.OPENAI_API_KEY; + consoleWarnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {}); + analyzerRegistry.register(new GenericAnalyzer()); + }); + + afterEach(async () => { + consoleWarnSpy.mockRestore(); + if (originalEmbeddingProvider === undefined) delete process.env.EMBEDDING_PROVIDER; + else process.env.EMBEDDING_PROVIDER = originalEmbeddingProvider; + if (originalOpenAiKey === undefined) delete process.env.OPENAI_API_KEY; + else process.env.OPENAI_API_KEY = originalOpenAiKey; + await fs.rm(tempRoot, { recursive: true, force: true }); + }); + + it.each(['changed', 'deleted', 'unreadable'] as const)( + 'keeps a fresh lower nominee when the top source is %s without rebuilding', + async (drift) => { + const rootPath = path.join(tempRoot, drift); + const topPath = path.join(rootPath, 'src/quartzBeacon.ts'); + await createSourceFile( + rootPath, + 'src/quartzBeacon.ts', + 'export function quartzBeacon() { return "old top"; }\n' + ); + await createSourceFile( + rootPath, + 'src/support.ts', + 'export function support() { return quartzBeacon(); }\n' + ); + await indexProject(rootPath); + const indexPath = await resolveIndexArtifact(rootPath, KEYWORD_INDEX_FILENAME); + const indexBefore = await fs.readFile(indexPath, 'utf8'); + if (drift === 'changed') await fs.writeFile(topPath, 'export const changed = true;\n'); + if (drift === 'deleted') await fs.rm(topPath); + const readFile = fs.readFile.bind(fs); + const readSpy = + drift === 'unreadable' + ? vi + .spyOn(fs, 'readFile') + .mockImplementation((...args: Parameters) => { + if (path.resolve(String(args[0])) === path.resolve(topPath)) { + return Promise.reject( + Object.assign(new Error('Permission denied'), { code: 'EACCES' }) + ); + } + return readFile(...args); + }) + : undefined; + try { + const ctx = searchContext(rootPath); + const response = await searchCodebase( + { query: 'quartzBeacon', limit: 1, intent: 'edit' }, + ctx + ); + const payload = JSON.parse(response.content![0].text) as SearchResponse; + expect(payload.status).toBe('success'); + expect(payload.results).toHaveLength(1); + expect(payload.results[0].file).toContain('support.ts:'); + expect(payload.sourceCoverage).toMatchObject({ + status: 'partial', + checkedFiles: 2, + skippedFiles: 1 + }); + expect(payload.preflight?.ready).toBe(false); + expect(payload.preflight?.nextAction).toMatch(/refresh/i); + expect(JSON.stringify(payload)).not.toContain('old top'); + expect(ctx.performIndexing).not.toHaveBeenCalled(); + expect(await fs.readFile(indexPath, 'utf8')).toBe(indexBefore); + } finally { + readSpy?.mockRestore(); + } + } + ); + + it.each(['compact', 'full'])( + 'distinguishes stale-only evidence from no nominations in %s mode', + async (mode) => { + const rootPath = path.join(tempRoot, mode); + await createSourceFile( + rootPath, + 'src/quartzBeacon.ts', + 'export function quartzBeacon() { return "old top"; }\n' + ); + await indexProject(rootPath); + await fs.writeFile( + path.join(rootPath, 'src/quartzBeacon.ts'), + 'export const different = true;\n' + ); + const ctx = searchContext(rootPath); + const stale = await searchCodebase({ query: 'quartzBeacon', mode }, ctx); + const payload = JSON.parse(stale.content![0].text) as SearchResponse & { message: string }; + expect(payload.status).toBe('no_fresh_results'); + expect(payload.message).toMatch(/no fresh/i); + expect(payload.sourceCoverage).toMatchObject({ status: 'no_fresh_results', skippedFiles: 1 }); + expect(payload.sourceCoverage?.refreshAction).toMatch(/refresh_index/); + expect(payload.results).toEqual([]); + expect(payload.preflight).toBeUndefined(); + expect(ctx.performIndexing).not.toHaveBeenCalled(); + expect(JSON.stringify(payload)).not.toContain('old top'); + for (const intent of ['edit', 'refactor', 'migrate'] as const) { + const guarded = await searchCodebase({ query: 'quartzBeacon', mode, intent }, ctx); + const guardedPayload = JSON.parse(guarded.content![0].text) as SearchResponse; + expect(guardedPayload.status).toBe('no_fresh_results'); + expect(guardedPayload.sourceCoverage?.status).toBe('no_fresh_results'); + expect(guardedPayload.preflight?.ready).toBe(false); + } + const empty = await searchCodebase({ query: 'nonexistentUnrelatedNomination', mode }, ctx); + const emptyPayload = JSON.parse(empty.content![0].text) as SearchResponse; + expect(emptyPayload.status).toBe('success'); + expect(emptyPayload.results).toEqual([]); + expect(emptyPayload.sourceCoverage).toBeUndefined(); + } + ); + + it('rejects an injected nominee absent from the manifest even with a fresh lower result', async () => { + const rootPath = path.join(tempRoot, 'injected source'); + await createSourceFile(rootPath, 'src/quartzBeacon.ts', 'export const quartzBeacon = true;\n'); + await indexProject(rootPath); + const indexPath = await resolveIndexArtifact(rootPath, KEYWORD_INDEX_FILENAME); + const index = JSON.parse(await fs.readFile(indexPath, 'utf8')) as { chunks: CodeChunk[] }; + await createSourceFile( + rootPath, + 'src/injected.ts', + 'export const quartzBeacon = "injected";\n' + ); + index.chunks.push({ + ...index.chunks[0], + id: 'injected', + filePath: path.join(rootPath, 'src/injected.ts'), + relativePath: 'src/injected.ts' + }); + await fs.writeFile(indexPath, JSON.stringify(index)); + await expect( + new CodebaseSearcher(rootPath).search('quartzBeacon', 1, undefined, KEYWORD_ONLY_OPTIONS) + ).rejects.toThrow(/absent from its manifest/); + }); + + it('keeps concurrent query coverage independent on one initialized searcher', async () => { + const rootPath = path.join(tempRoot, 'concurrent coverage'); + await createSourceFile(rootPath, 'src/quartzBeacon.ts', 'export const quartzBeacon = true;\n'); + await createSourceFile(rootPath, 'src/stableBeacon.ts', 'export const stableBeacon = true;\n'); + await indexProject(rootPath); + await fs.writeFile( + path.join(rootPath, 'src/quartzBeacon.ts'), + 'export const changed = true;\n' + ); + const searcher = new CodebaseSearcher(rootPath); + await searcher.initialize(); + const [stale, current] = await Promise.all([ + searcher.searchWithCoverage('quartzBeacon', 1, undefined, KEYWORD_ONLY_OPTIONS), + searcher.searchWithCoverage('stableBeacon', 1, undefined, KEYWORD_ONLY_OPTIONS) + ]); + expect(stale.results).toEqual([]); + expect(stale.sourceCoverage).toMatchObject({ + status: 'no_fresh_results', + checkedFiles: 1, + skippedFiles: 1 + }); + expect(current.results).toHaveLength(1); + expect(current.sourceCoverage).toEqual({ status: 'current', checkedFiles: 1, skippedFiles: 0 }); + }); + + it.each(['compact', 'full'] as const)( + 'replenishes current candidates while retaining mutation gates in %s mode', + async (mode) => { + const rootPath = path.join(tempRoot, 'one stale source many chunks'); + const sourcePath = path.join(rootPath, 'src/quartzBeacon.ts'); + const supportPath = path.join(rootPath, 'src/support.ts'); + const source = + Array.from( + { length: 100 }, + (_, n) => + `export function quartzBeacon${n}() {\n${Array.from( + { length: 30 }, + (_, m) => ` const helper${m} = "quartzBeacon";` + ).join('\n')}\n return "quartzBeacon";\n}` + ).join('\n') + '\n'; + await createSourceFile(rootPath, 'src/quartzBeacon.ts', source); + await createSourceFile( + rootPath, + 'src/support.ts', + 'export function support() { return quartzBeacon0(); }\n' + ); + await indexProject(rootPath); + const indexPath = await resolveIndexArtifact(rootPath, KEYWORD_INDEX_FILENAME); + const indexBefore = await fs.readFile(indexPath, 'utf8'); + const index = JSON.parse(indexBefore) as { chunks: CodeChunk[] }; + expect( + index.chunks.filter((chunk) => chunk.relativePath === 'src/quartzBeacon.ts').length + ).toBeGreaterThan(30); + const readSpy = vi.spyOn(fs, 'readFile'); + const realPathSpy = vi.spyOn(fs, 'realpath'); + try { + const fresh = await new CodebaseSearcher(rootPath).searchWithCoverage( + 'quartzBeacon', + 1, + undefined, + KEYWORD_ONLY_OPTIONS + ); + expect(fresh.results).toHaveLength(1); + expect(fresh.sourceCoverage).toEqual({ + status: 'current', + checkedFiles: 1, + skippedFiles: 0 + }); + await fs.writeFile(sourcePath, 'export const changed = true;\n'); + for (const limit of [1, 5]) { + readSpy.mockClear(); + realPathSpy.mockClear(); + const result = await new CodebaseSearcher(rootPath).searchWithCoverage( + 'quartzBeacon', + limit, + undefined, + KEYWORD_ONLY_OPTIONS + ); + await expectSearchResultsOpenInsideRoot(rootPath, result.results, 'src/support.ts'); + expect(result.results.length).toBeLessThanOrEqual(limit); + expect(result.sourceCoverage).toMatchObject({ + status: 'partial', + checkedFiles: 2, + skippedFiles: 1 + }); + expect(result.sourceCoverage.candidateBudgetExhausted).toBeUndefined(); + const reads = readSpy.mock.calls.map(([file]) => path.resolve(String(file))); + expect(reads.filter((file) => file === sourcePath)).toHaveLength(1); + // The assertion helper opens the returned file once as a host would. + expect(reads.filter((file) => file === supportPath)).toHaveLength(2); + expect( + realPathSpy.mock.calls.filter(([file]) => path.resolve(String(file)) === rootPath) + ).toHaveLength(1); + } + const ctx = searchContext(rootPath); + const response = await searchCodebase({ query: 'quartzBeacon', limit: 1, mode }, ctx); + const payload = JSON.parse(response.content![0].text) as SearchResponse; + expect(payload.status).toBe('success'); + expect(payload.results).toHaveLength(1); + expect(payload.results[0].file).toContain('support.ts:'); + expect(payload.sourceCoverage?.status).toBe('partial'); + expect(payload.preflight).toBeUndefined(); + for (const intent of ['edit', 'refactor', 'migrate'] as const) { + const guarded = await searchCodebase( + { query: 'quartzBeacon', limit: 1, mode, intent }, + ctx + ); + const guardedPayload = JSON.parse(guarded.content![0].text) as SearchResponse; + expect(guardedPayload.sourceCoverage?.status).toBe('partial'); + expect(guardedPayload.preflight?.ready).toBe(false); + expect(guardedPayload.preflight?.nextAction).toMatch(/refresh/i); + } + expect(ctx.performIndexing).not.toHaveBeenCalled(); + expect(await fs.readFile(indexPath, 'utf8')).toBe(indexBefore); + } finally { + readSpy.mockRestore(); + realPathSpy.mockRestore(); + } + } + ); + + it('discloses exhausted source inspection budget instead of claiming all nominated evidence is stale', async () => { + const rootPath = path.join(tempRoot, 'bounded stale sources'); + for (let n = 0; n < 35; n++) { + await createSourceFile( + rootPath, + `src/quartzBeacon${n}.ts`, + `export function quartzBeacon${n}() { return "quartzBeacon"; }\n` + ); + } + await createSourceFile( + rootPath, + 'src/support.ts', + 'export function support() { return quartzBeacon0(); }\n' + ); + await indexProject(rootPath); + for (let n = 0; n < 35; n++) { + await fs.writeFile( + path.join(rootPath, `src/quartzBeacon${n}.ts`), + 'export const changed = true;\n' + ); + } + const outcome = await new CodebaseSearcher(rootPath).searchWithCoverage( + 'quartzBeacon', + 1, + undefined, + KEYWORD_ONLY_OPTIONS + ); + expect(outcome.results).toEqual([]); + expect(outcome.sourceCoverage).toMatchObject({ + status: 'no_fresh_results', + candidateBudgetExhausted: true + }); + expect(outcome.sourceCoverage.checkedFiles).toBeLessThanOrEqual(30); + const ctx = searchContext(rootPath); + const response = await searchCodebase({ query: 'quartzBeacon', limit: 1 }, ctx); + const payload = JSON.parse(response.content![0].text) as SearchResponse & { message: string }; + expect(payload.status).toBe('no_fresh_results'); + expect(payload.message).toMatch(/bounded candidate window.*uninspected/i); + expect(payload.sourceCoverage?.candidateBudgetExhausted).toBe(true); + expect(ctx.performIndexing).not.toHaveBeenCalled(); + }); + + it('rejects malformed ranges in lower nominated evidence before limiting to a fresh top file', async () => { + const rootPath = path.join(tempRoot, 'lower invalid range'); + await createSourceFile( + rootPath, + 'src/quartzBeacon.ts', + 'export function quartzBeacon() { return true; }\n' + ); + await createSourceFile(rootPath, 'src/support.ts', 'export const support = quartzBeacon;\n'); + await indexProject(rootPath); + const before = await new CodebaseSearcher(rootPath).search( + 'quartzBeacon', + 1, + undefined, + KEYWORD_ONLY_OPTIONS + ); + expect(path.basename(before[0].filePath)).toBe('quartzBeacon.ts'); + const indexPath = await resolveIndexArtifact(rootPath, KEYWORD_INDEX_FILENAME); + const index = JSON.parse(await fs.readFile(indexPath, 'utf8')) as { chunks: CodeChunk[] }; + for (const chunk of index.chunks.filter((chunk) => chunk.relativePath === 'src/support.ts')) { + chunk.endLine = 99999; + } + await fs.writeFile(indexPath, JSON.stringify(index)); + await expect( + new CodebaseSearcher(rootPath).search('quartzBeacon', 1, undefined, KEYWORD_ONLY_OPTIONS) + ).rejects.toThrow(/source line range is invalid/i); + }); + + it('stores root-relative chunk paths when indexing nested roots from an ancestor cwd', async () => { + const ancestorPath = path.join(tempRoot, 'consumer'); + const rootPath = path.join(ancestorPath, 'project-a'); + const independentRootPath = path.join(ancestorPath, 'project-b'); + await createSourceFile( + rootPath, + 'src/quartzBeacon.ts', + 'export function quartzBeacon() { return "first"; }\nquartzBeacon();\n' + ); + await createSourceFile( + rootPath, + 'src/stableBeacon.ts', + 'export function stableBeacon() { return "stable"; }\nstableBeacon();\n' + ); + await createSourceFile( + independentRootPath, + 'src/quartzBeacon.ts', + 'export function quartzBeacon() { return "independent"; }\nquartzBeacon();\n' + ); + + const cwdSpy = vi.spyOn(process, 'cwd').mockReturnValue(ancestorPath); + try { + await indexProject(rootPath); + + const indexPath = await resolveIndexArtifact(rootPath, KEYWORD_INDEX_FILENAME); + const initialIndex = JSON.parse(await fs.readFile(indexPath, 'utf8')) as { + chunks: Array<{ + content: string; + filePath: string; + id: string; + relativePath: string; + startLine: number; + endLine: number; + }>; + }; + expect(initialIndex.chunks.length).toBeGreaterThan(0); + expect( + initialIndex.chunks.every((chunk) => !chunk.relativePath.startsWith('project-a/')) + ).toBe(true); + expect(initialIndex.chunks.every((chunk) => chunk.relativePath.startsWith('src/'))).toBe( + true + ); + + const firstResults = await new CodebaseSearcher(rootPath).search( + 'quartzBeacon', + 5, + undefined, + KEYWORD_ONLY_OPTIONS + ); + await expectSearchResultsOpenInsideRoot(rootPath, firstResults, 'src/quartzBeacon.ts'); + + const stableBefore = initialIndex.chunks.filter((chunk) => + chunk.relativePath.endsWith('stableBeacon.ts') + ); + expect(stableBefore.length).toBeGreaterThan(0); + await createSourceFile( + rootPath, + 'src/quartzBeacon.ts', + 'export function quartzBeacon() { return "edited"; }\nquartzBeacon();\n' + ); + cwdSpy.mockReturnValue(tempRoot); + await new CodebaseIndexer({ + rootPath, + config: { + skipEmbedding: true, + embedding: { provider: 'openai', model: 'text-embedding-3-small' }, + parsing: { maxFileSize: 1024 * 1024 } + }, + projectOptions: { preferredAnalyzer: 'generic' }, + incrementalOnly: true + }).index(); + + const incrementallyIndexed = JSON.parse(await fs.readFile(indexPath, 'utf8')) as { + chunks: Array<{ + content: string; + filePath: string; + id: string; + relativePath: string; + startLine: number; + endLine: number; + }>; + }; + expect(incrementallyIndexed.chunks.length).toBeGreaterThan(0); + expect( + incrementallyIndexed.chunks.every( + (chunk) => + chunk.relativePath === 'src/quartzBeacon.ts' || + chunk.relativePath === 'src/stableBeacon.ts' + ) + ).toBe(true); + const stableAfter = incrementallyIndexed.chunks.filter((chunk) => + chunk.relativePath.endsWith('stableBeacon.ts') + ); + expect(stableAfter).toHaveLength(stableBefore.length); + expect( + stableAfter.map(({ relativePath, filePath, content, startLine, endLine }) => ({ + relativePath, + filePath, + content, + startLine, + endLine + })) + ).toEqual( + stableBefore.map(({ relativePath, filePath, content, startLine, endLine }) => ({ + relativePath, + filePath, + content, + startLine, + endLine + })) + ); + + const editedResults = await new CodebaseSearcher(rootPath).search( + 'quartzBeacon', + 5, + undefined, + KEYWORD_ONLY_OPTIONS + ); + await expectSearchResultsOpenInsideRoot(rootPath, editedResults, 'src/quartzBeacon.ts'); + expect(editedResults.some((result) => result.snippet.includes('edited'))).toBe(true); + + cwdSpy.mockReturnValue(ancestorPath); + await indexProject(independentRootPath); + const independentIndex = JSON.parse( + await fs.readFile( + await resolveIndexArtifact(independentRootPath, KEYWORD_INDEX_FILENAME), + 'utf8' + ) + ) as { chunks: Array<{ relativePath: string }> }; + expect(independentIndex.chunks.length).toBeGreaterThan(0); + expect(independentIndex.chunks.every((chunk) => chunk.relativePath.startsWith('src/'))).toBe( + true + ); + const independentResults = await new CodebaseSearcher(independentRootPath).search( + 'quartzBeacon', + 5, + undefined, + KEYWORD_ONLY_OPTIONS + ); + await expectSearchResultsOpenInsideRoot( + independentRootPath, + independentResults, + 'src/quartzBeacon.ts' + ); + } finally { + cwdSpy.mockRestore(); + } + }); + + it('rejects a copied index that points outside the selected root, then searches rebuilt current ranges', async () => { + const originalRoot = path.join(tempRoot, 'source repo with spaces'); + const copiedRoot = path.join(tempRoot, 'copied repo with spaces'); + const initialSource = [ + 'export function quartzBeacon() {', + ' return "original";', + '}', + 'quartzBeacon();', + '' + ].join('\n'); + await createSourceFile(originalRoot, 'src/quartzBeacon.ts', initialSource); + await indexProject(originalRoot); + await fs.cp(originalRoot, copiedRoot, { recursive: true }); + await createSourceFile( + copiedRoot, + 'src/quartzBeacon.ts', + `// copied header one\n// copied header two\n${initialSource}` + ); + + const staleSearcher = new CodebaseSearcher(copiedRoot); + await expect( + staleSearcher.search('quartzBeacon', 5, undefined, KEYWORD_ONLY_OPTIONS) + ).rejects.toBeInstanceOf(IndexCorruptedError); + + // The existing recovery action is a complete index rebuild, which must replace absolute + // chunk paths and stale line ranges with evidence from the selected copied repository. + await indexProject(copiedRoot); + const rebuiltResults = await new CodebaseSearcher(copiedRoot).search( + 'quartzBeacon', + 5, + undefined, + KEYWORD_ONLY_OPTIONS + ); + await expectSearchResultsOpenInsideRoot(copiedRoot, rebuiltResults, 'src/quartzBeacon.ts'); + expect(rebuiltResults.some((result) => result.startLine >= 3)).toBe(true); + }); + + it('rejects a forged stored context header without rewriting the corrupted index', async () => { + const rootPath = path.join(tempRoot, 'annotated chunks'); + await createSourceFile( + rootPath, + 'src/quartzBeacon.ts', + 'export function quartzBeacon(): string {\r\n return "current source";\r\n}\r\nquartzBeacon();\r\n' + ); + await indexProject(rootPath); + const indexPath = await resolveIndexArtifact(rootPath, KEYWORD_INDEX_FILENAME); + const index = JSON.parse(await fs.readFile(indexPath, 'utf8')) as { + chunks: Array<{ content: string }>; + }; + for (const chunk of index.chunks) + chunk.content = '// indexing-only signature header\n' + chunk.content; + const indexedBytes = JSON.stringify(index); + await fs.writeFile(indexPath, indexedBytes, 'utf8'); + + // Format 5 persists raw bodies. A stored presentation prefix is corruption, + // not authority to expand the excerpt to surrounding lines or sanitize it. + await expect( + new CodebaseSearcher(rootPath).search('quartzBeacon', 5, undefined, KEYWORD_ONLY_OPTIONS) + ).rejects.toBeInstanceOf(IndexCorruptedError); + expect(await fs.readFile(indexPath, 'utf8')).toBe(indexedBytes); + }); + + it('preserves source projection when Windows opens the same root with different casing', async () => { + const rootPath = path.join(tempRoot, 'mixed-case-project'); + await createSourceFile( + rootPath, + 'src/quartzBeacon.ts', + 'export function quartzBeacon() { return 1; }\n' + ); + await indexProject(rootPath); + const differentlyCasedRoot = path.join(tempRoot, 'MIXED-CASE-PROJECT'); + const caseVariantExists = await fs.access(differentlyCasedRoot).then( + () => true, + () => false + ); + const search = new CodebaseSearcher(differentlyCasedRoot).search( + 'quartzBeacon', + 5, + undefined, + KEYWORD_ONLY_OPTIONS + ); + if (caseVariantExists) { + await expectSearchResultsOpenInsideRoot( + differentlyCasedRoot, + await search, + 'src/quartzBeacon.ts' + ); + } else { + // Filesystem behavior, not the OS name, decides whether this is the same project. + await expect(search).rejects.toBeInstanceOf(IndexCorruptedError); + } + }); + + it('rejects an indexed directory redirected outside the root even when source bytes match', async () => { + const rootPath = path.join(tempRoot, 'junction-project'); + const outsidePath = path.join(tempRoot, 'external-source'); + await createSourceFile( + rootPath, + 'src/quartzBeacon.ts', + 'export function quartzBeacon() { return 1; }\n' + ); + await indexProject(rootPath); + // Only this test's owned directory is moved; the index retains the original path/hash. + await fs.rename(path.join(rootPath, 'src'), outsidePath); + await fs.symlink( + outsidePath, + path.join(rootPath, 'src'), + process.platform === 'win32' ? 'junction' : 'dir' + ); + await expect( + new CodebaseSearcher(rootPath).search('quartzBeacon', 5, undefined, KEYWORD_ONLY_OPTIONS) + ).rejects.toBeInstanceOf(IndexCorruptedError); + expect(await fs.readFile(path.join(outsidePath, 'quartzBeacon.ts'), 'utf8')).toContain( + 'quartzBeacon' + ); + }); + + it('permits an indexed directory linked to a source location still inside its own root', async () => { + const rootPath = path.join(tempRoot, 'real-project'); + await createSourceFile( + rootPath, + 'src/quartzBeacon.ts', + 'export function quartzBeacon() { return 1; }\n' + ); + await indexProject(rootPath); + const internalSourcePath = path.join(rootPath, 'internal-source'); + await fs.rename(path.join(rootPath, 'src'), internalSourcePath); + await fs.symlink( + internalSourcePath, + path.join(rootPath, 'src'), + process.platform === 'win32' ? 'junction' : 'dir' + ); + const results = await new CodebaseSearcher(rootPath).search( + 'quartzBeacon', + 5, + undefined, + KEYWORD_ONLY_OPTIONS + ); + await expectSearchResultsOpenInsideRoot(rootPath, results, 'src/quartzBeacon.ts'); + }); + + it.each([-1, 1.5, 99999])( + 'rejects invalid returned source bounds (%s) without fabricating a snippet', + async (endLine) => { + const rootPath = path.join(tempRoot, 'invalid range'); + await createSourceFile( + rootPath, + 'src/quartzBeacon.ts', + 'export function quartzBeacon() { return 1; }\n' + ); + await indexProject(rootPath); + const indexPath = await resolveIndexArtifact(rootPath, KEYWORD_INDEX_FILENAME); + const index = JSON.parse(await fs.readFile(indexPath, 'utf8')) as { + chunks: Array<{ startLine: number; endLine: number }>; + }; + for (const chunk of index.chunks) { + chunk.startLine = 1; + chunk.endLine = endLine; + } + await fs.writeFile(indexPath, JSON.stringify(index), 'utf8'); + await expect( + new CodebaseSearcher(rootPath).search('quartzBeacon', 5, undefined, KEYWORD_ONLY_OPTIONS) + ).rejects.toThrow(/source line range is invalid.*rebuild required/i); + } + ); + + it('excludes stale ranges after a same-root source shift until an explicit refresh', async () => { + const rootPath = path.join(tempRoot, 'shifted source repo with spaces'); + const originalSource = [ + 'export function quartzBeacon() {', + ' return "original";', + '}', + 'quartzBeacon();', + '' + ].join('\n'); + await createSourceFile(rootPath, 'src/quartzBeacon.ts', originalSource); + await indexProject(rootPath); + await createSourceFile( + rootPath, + 'src/quartzBeacon.ts', + `// shifted header one\n// shifted header two\n${originalSource}` + ); + + // Owner-selected contract migration: ordinary drift is excluded with explicit + // coverage, while missing membership, malformed ranges and confinement remain errors. + const stale = await new CodebaseSearcher(rootPath).searchWithCoverage( + 'quartzBeacon', + 5, + undefined, + KEYWORD_ONLY_OPTIONS + ); + expect(stale.results).toEqual([]); + expect(stale.sourceCoverage).toMatchObject({ status: 'no_fresh_results', skippedFiles: 1 }); + + await indexProject(rootPath); + const rebuiltResults = await new CodebaseSearcher(rootPath).search( + 'quartzBeacon', + 5, + undefined, + KEYWORD_ONLY_OPTIONS + ); + await expectSearchResultsOpenInsideRoot(rootPath, rebuiltResults, 'src/quartzBeacon.ts'); + expect(rebuiltResults.some((result) => result.startLine >= 3)).toBe(true); + }); + it('excludes a deleted top result while returning the valid lower file before and after refresh', async () => { + const rootPath = path.join(tempRoot, 'deleted source repo with spaces'); + await createSourceFile( + rootPath, + 'src/quartzBeacon.ts', + 'export function quartzBeacon() {\n return "top";\n}\nquartzBeacon();\n' + ); + await createSourceFile( + rootPath, + 'src/support.ts', + 'export function useBeacon() {\n return quartzBeacon();\n}\n' + ); + await indexProject(rootPath); + await fs.rm(path.join(rootPath, 'src', 'quartzBeacon.ts')); + + const staleSearcher = new CodebaseSearcher(rootPath); + const current = await staleSearcher.searchWithCoverage( + 'quartzBeacon', + 5, + undefined, + KEYWORD_ONLY_OPTIONS + ); + await expectSearchResultsOpenInsideRoot(rootPath, current.results, 'src/support.ts'); + expect(current.sourceCoverage).toMatchObject({ status: 'partial', skippedFiles: 1 }); + + await indexProject(rootPath); + const rebuiltResults = await new CodebaseSearcher(rootPath).search( + 'quartzBeacon', + 5, + undefined, + KEYWORD_ONLY_OPTIONS + ); + await expectSearchResultsOpenInsideRoot(rootPath, rebuiltResults, 'src/support.ts'); + }); + + it('fails closed when the source manifest is missing, then recovers after a full rebuild', async () => { + const rootPath = path.join(tempRoot, 'missing manifest repo with spaces'); + await createSourceFile( + rootPath, + 'src/quartzBeacon.ts', + 'export function quartzBeacon() {\n return "source";\n}\nquartzBeacon();\n' + ); + await indexProject(rootPath); + await fs.rm(await resolveIndexArtifact(rootPath, MANIFEST_FILENAME)); + + await expect( + new CodebaseSearcher(rootPath).search('quartzBeacon', 5, undefined, KEYWORD_ONLY_OPTIONS) + ).rejects.toBeInstanceOf(IndexCorruptedError); + + await indexProject(rootPath); + const rebuiltResults = await new CodebaseSearcher(rootPath).search( + 'quartzBeacon', + 5, + undefined, + KEYWORD_ONLY_OPTIONS + ); + await expectSearchResultsOpenInsideRoot(rootPath, rebuiltResults, 'src/quartzBeacon.ts'); + }); + + it('rejects traversal paths even when the outside target exists', async () => { + const rootPath = path.join(tempRoot, 'path traversal repo with spaces'); + const outsidePath = path.join(tempRoot, 'outside.ts'); + await createSourceFile( + rootPath, + 'src/quartzBeacon.ts', + 'export function quartzBeacon() {\n return "inside";\n}\nquartzBeacon();\n' + ); + await createSourceFile( + tempRoot, + 'outside.ts', + 'export function quartzBeacon() {\n return "outside";\n}\nquartzBeacon();\n' + ); + await indexProject(rootPath); + + const indexPath = await resolveIndexArtifact(rootPath, KEYWORD_INDEX_FILENAME); + const parsedIndex = JSON.parse(await fs.readFile(indexPath, 'utf8')) as { + header: { buildId: string; formatVersion: number }; + chunks: Array<{ content: string; filePath: string; relativePath: string }>; + }; + const candidateChunk = parsedIndex.chunks.find( + (chunk) => + path.resolve(rootPath, chunk.filePath) === path.join(rootPath, 'src', 'quartzBeacon.ts') + ); + if (!candidateChunk) throw new Error('Indexed source chunk missing from test fixture'); + candidateChunk.filePath = outsidePath; + candidateChunk.relativePath = '../outside.ts'; + await fs.writeFile(indexPath, JSON.stringify(parsedIndex), 'utf8'); + + await expect( + new CodebaseSearcher(rootPath).search('quartzBeacon', 5, undefined, KEYWORD_ONLY_OPTIONS) + ).rejects.toBeInstanceOf(IndexCorruptedError); + + await indexProject(rootPath); + const rebuiltResults = await new CodebaseSearcher(rootPath).search( + 'quartzBeacon', + 5, + undefined, + KEYWORD_ONLY_OPTIONS + ); + await expectSearchResultsOpenInsideRoot(rootPath, rebuiltResults, 'src/quartzBeacon.ts'); + expect( + rebuiltResults.every( + (result) => !path.resolve(result.filePath).startsWith(path.resolve(outsidePath)) + ) + ).toBe(true); + }); + it('finds new references in current parser-unavailable source even when stale chunks lack the symbol', async () => { + const rootPath = path.join(tempRoot, 'changed source repo with spaces'); + const originalSource = [ + 'export function staleToken() {', + ' return "old";', + '}', + 'staleToken();', + '' + ].join('\n'); + await createSourceFile(rootPath, 'src/widget.sfc', originalSource); + await indexProject(rootPath, ['.sfc']); + + const currentSource = [ + 'export function currentOnly() {', + ' return "new";', + '}', + 'currentOnly();', + '' + ].join('\n'); + await createSourceFile(rootPath, 'src/widget.sfc', currentSource); + + const staleSymbol = await findSymbolReferences(rootPath, 'staleToken'); + const newSymbol = await findSymbolReferences(rootPath, 'currentOnly'); + expect(staleSymbol.status).toBe('success'); + if (staleSymbol.status !== 'success') throw new Error('Expected a successful reference query'); + expect(staleSymbol.usageCount).toBe(0); + expect(staleSymbol.usages).toEqual([]); + expect(newSymbol.status).toBe('success'); + if (newSymbol.status !== 'success') throw new Error('Expected a successful reference query'); + expect(newSymbol.usageCount).toBe(2); + expect(newSymbol.usages.map((usage) => usage.line)).toEqual([1, 4]); + expect(newSymbol.usages.every((usage) => usage.file === 'src/widget.sfc')).toBe(true); + }); + + it('does not return symbol references through a directory junction outside the project', async () => { + const rootPath = path.join(tempRoot, 'symbol-junction-project'); + const outsidePath = path.join(tempRoot, 'symbol-external-source'); + await createSourceFile( + rootPath, + 'src/quartzBeacon.ts', + 'export function quartzBeacon() { return 1; }\nquartzBeacon();\n' + ); + await indexProject(rootPath); + await fs.rename(path.join(rootPath, 'src'), outsidePath); + await fs.symlink( + outsidePath, + path.join(rootPath, 'src'), + process.platform === 'win32' ? 'junction' : 'dir' + ); + + const result = await findSymbolReferences(rootPath, 'quartzBeacon'); + expect(result.status).toBe('error'); + if (result.status !== 'error') throw new Error('Expected source-confinement error'); + expect(result.message).toMatch(/outside the selected project/i); + }); +}); diff --git a/tests/server-config.test.ts b/tests/server-config.test.ts index eb7f6c6..e2ba8a2 100644 --- a/tests/server-config.test.ts +++ b/tests/server-config.test.ts @@ -113,10 +113,7 @@ describe('loadServerConfig', () => { const projA = path.join(os.tmpdir(), 'ccc-test-proj-a'); const projB = path.join(os.tmpdir(), 'ccc-test-proj-b'); const config = JSON.stringify({ - projects: [ - { root: projA, excludePatterns: ['**/dist/**'] }, - { root: projB } - ], + projects: [{ root: projA, excludePatterns: ['**/dist/**'] }, { root: projB }], server: { port: 5199, host: '0.0.0.0' } }); await withTempConfig(config, async (filePath) => { @@ -171,6 +168,26 @@ describe('loadServerConfig', () => { }); }); + it('preserves explicit per-project embedding mode, including false opt-in', async () => { + const config = JSON.stringify({ + projects: [ + { root: '~/model-free', skipEmbedding: true }, + { root: '~/semantic', skipEmbedding: false }, + { root: '~/invalid', skipEmbedding: 'false' } + ] + }); + + await withTempConfig(config, async (filePath) => { + process.env.CODEBASE_CONTEXT_CONFIG_PATH = filePath; + const result = await loadServerConfig(); + expect(result?.projects?.map((project) => project.skipEmbedding)).toEqual([ + true, + false, + undefined + ]); + }); + }); + it('loads parsing configuration for the requested CLI project root', async () => { const projectRoot = path.join(os.tmpdir(), 'ccc-large-cli-project'); const config = JSON.stringify({ diff --git a/tests/source-interval.test.ts b/tests/source-interval.test.ts new file mode 100644 index 0000000..7a6a272 --- /dev/null +++ b/tests/source-interval.test.ts @@ -0,0 +1,89 @@ +import { expect, it } from 'vitest'; +import { isSourceBodyWithinInterval } from '../src/utils/source-interval.js'; + +it.each([ + { name: 'empty source body', source: '', content: '', startLine: 1, endLine: 1, valid: true }, + { name: 'absent source line', source: '', content: '', startLine: 2, endLine: 2, valid: false }, + { + name: 'trailing newline', + source: 'value\n', + content: 'value\n', + startLine: 1, + endLine: 2, + valid: true + }, + { + name: 'unclaimed trailing newline', + source: 'value\n', + content: 'value\n', + startLine: 1, + endLine: 1, + valid: false + }, + { + name: 'partial Unicode AST body', + source: 'before(); onReady:()=>open("雪"); after();', + content: 'onReady:()=>open("雪")', + startLine: 1, + endLine: 1, + valid: true + }, + { + name: 'normalized CRLF', + source: 'before();\r\nonReady:()=>open("雪")\r\n', + content: 'onReady:()=>open("雪")', + startLine: 2, + endLine: 2, + valid: true + }, + { + name: 'duplicate text inside interval', + source: 'same();\nsame();\n', + content: 'same();', + startLine: 2, + endLine: 2, + valid: true + }, + { + name: 'text exists only outside interval', + source: 'before();\nafter();', + content: 'before();', + startLine: 2, + endLine: 2, + valid: false + }, + { + name: 'forged prefix', + source: 'onReady:()=>open()', + content: '// prefix\nonReady:()=>open()', + startLine: 1, + endLine: 1, + valid: false + }, + { + name: 'fabricated body in valid range', + source: 'onReady:()=>open()', + content: 'onReady:()=>erase()', + startLine: 1, + endLine: 1, + valid: false + }, + { + name: 'reversed interval', + source: 'a\nb', + content: 'b', + startLine: 2, + endLine: 1, + valid: false + }, + { + name: 'nonintegral interval', + source: 'a', + content: 'a', + startLine: 1.5, + endLine: 1.5, + valid: false + } +])('checks $name without salvaging a different body', ({ source, valid, ...chunk }) => { + expect(isSourceBodyWithinInterval(chunk, source.replace(/\r\n/g, '\n').split('\n'))).toBe(valid); +}); diff --git a/tests/symbol-chunk-merge.test.ts b/tests/symbol-chunk-merge.test.ts index b87ed75..de20086 100644 --- a/tests/symbol-chunk-merge.test.ts +++ b/tests/symbol-chunk-merge.test.ts @@ -1,3 +1,4 @@ +import { resolveIndexArtifact } from '../src/core/index-generation.js'; import { afterEach, beforeEach, describe, expect, it } from 'vitest'; import { promises as fs } from 'fs'; import os from 'os'; @@ -47,7 +48,7 @@ describe('Symbol-aware chunk merge guard', () => { await indexer.index(); - const indexPath = path.join(tempDir, CODEBASE_CONTEXT_DIRNAME, KEYWORD_INDEX_FILENAME); + const indexPath = await resolveIndexArtifact(tempDir, KEYWORD_INDEX_FILENAME); const indexRaw = JSON.parse(await fs.readFile(indexPath, 'utf-8')) as any; const allChunks = ( Array.isArray(indexRaw) ? indexRaw : Array.isArray(indexRaw?.chunks) ? indexRaw.chunks : [] diff --git a/tests/symbol-references.test.ts b/tests/symbol-references.test.ts new file mode 100644 index 0000000..33384a0 --- /dev/null +++ b/tests/symbol-references.test.ts @@ -0,0 +1,852 @@ +import { resolveIndexArtifact } from '../src/core/index-generation.js'; +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { promises as fs } from 'fs'; +import os from 'os'; +import path from 'path'; +import { + CODEBASE_CONTEXT_DIRNAME, + INDEX_FORMAT_VERSION, + INDEX_META_FILENAME, + INDEX_META_VERSION, + KEYWORD_INDEX_FILENAME, + MANIFEST_FILENAME, + MEMORY_FILENAME +} from '../src/constants/codebase-context.js'; +import { CodebaseIndexer } from '../src/core/indexer.js'; +import { analyzerRegistry } from '../src/core/analyzer-registry.js'; +import { GenericAnalyzer } from '../src/analyzers/generic/index.js'; + +const { findIdentifierOccurrences } = vi.hoisted(() => ({ + findIdentifierOccurrences: vi.fn() +})); + +vi.mock('../src/utils/tree-sitter.js', async (importOriginal) => ({ + ...(await importOriginal()), + findIdentifierOccurrences +})); + +import { findSymbolReferences } from '../src/core/symbol-references.js'; + +if (!analyzerRegistry.get('generic')) analyzerRegistry.register(new GenericAnalyzer()); + +function createGate() { + let release!: () => void; + const promise = new Promise((resolve) => { + release = resolve; + }); + return { promise, release }; +} + +describe('findSymbolReferences', () => { + let rootPath: string | null = null; + + afterEach(async () => { + vi.restoreAllMocks(); + vi.clearAllMocks(); + if (rootPath) { + await fs.rm(rootPath, { recursive: true, force: true }); + rootPath = null; + } + }); + + async function createIndex(files: Array<{ path: string; exists?: boolean; source?: string }>) { + rootPath = await fs.mkdtemp(path.join(os.tmpdir(), 'cbc-symbol-references-')); + const chunks = []; + for (const entry of files) { + const absolutePath = path.join(rootPath, entry.path); + if (entry.exists !== false) { + await fs.mkdir(path.dirname(absolutePath), { recursive: true }); + await fs.writeFile(absolutePath, entry.source ?? '', 'utf8'); + } + chunks.push({ relativePath: entry.path, filePath: absolutePath }); + } + const contextPath = path.join(rootPath, CODEBASE_CONTEXT_DIRNAME); + await fs.mkdir(contextPath, { recursive: true }); + await fs.writeFile( + await resolveIndexArtifact(path.dirname(contextPath), KEYWORD_INDEX_FILENAME), + JSON.stringify({ + header: { buildId: 'test', formatVersion: INDEX_FORMAT_VERSION }, + chunks + }), + 'utf8' + ); + await fs.writeFile( + path.join(contextPath, INDEX_META_FILENAME), + JSON.stringify({ + metaVersion: INDEX_META_VERSION, + formatVersion: INDEX_FORMAT_VERSION, + buildId: 'test', + generatedAt: new Date().toISOString(), + toolVersion: 'test', + artifacts: { + keywordIndex: { path: KEYWORD_INDEX_FILENAME }, + vectorDb: { path: 'index', provider: 'test' }, + manifest: { path: MANIFEST_FILENAME } + } + }), + 'utf8' + ); + await fs.writeFile( + await resolveIndexArtifact(path.dirname(contextPath), MANIFEST_FILENAME), + JSON.stringify({ + version: 1, + generatedAt: new Date().toISOString(), + header: { buildId: 'test', formatVersion: INDEX_FORMAT_VERSION }, + // Deliberately stale hashes prove source text, not stored hashes, decides matches. + files: Object.fromEntries(files.map((file) => [file.path, '0'.repeat(16)])) + }), + 'utf8' + ); + return rootPath; + } + + async function rewriteManifest(root: string, value: unknown): Promise { + const contextPath = path.join(root, CODEBASE_CONTEXT_DIRNAME); + await fs.writeFile( + await resolveIndexArtifact(path.dirname(contextPath), MANIFEST_FILENAME), + JSON.stringify(value), + 'utf8' + ); + } + + async function rewriteManifestArtifactPath(root: string, manifestPath: string): Promise { + const metadataPath = path.join(root, CODEBASE_CONTEXT_DIRNAME, INDEX_META_FILENAME); + const meta = JSON.parse(await fs.readFile(metadataPath, 'utf8')) as { + artifacts: Record; + }; + meta.artifacts.manifest = { path: manifestPath }; + await fs.writeFile(metadataPath, JSON.stringify(meta), 'utf8'); + } + + it.each([ + ['missing keyword index header', (index: Record) => delete index.header], + [ + 'mismatched keyword index build identity', + (index: Record) => { + const header = index.header as Record; + index.header = { ...header, buildId: 'different-build' }; + } + ] + ])('rejects a real-built index with %s', async (_description, mutate) => { + findIdentifierOccurrences.mockResolvedValue([]); + rootPath = await fs.mkdtemp(path.join(os.tmpdir(), 'cbc-symbol-index-identity-')); + const sourcePath = path.join(rootPath, 'src', 'identity.ts'); + await fs.mkdir(path.dirname(sourcePath), { recursive: true }); + await fs.writeFile(sourcePath, 'export function identity() { return identity; }\n', 'utf8'); + await new CodebaseIndexer({ + rootPath, + config: { skipEmbedding: true }, + projectOptions: { preferredAnalyzer: 'generic' } + }).index(); + + const keywordPath = await resolveIndexArtifact(rootPath, KEYWORD_INDEX_FILENAME); + const parsed = JSON.parse(await fs.readFile(keywordPath, 'utf8')) as Record; + mutate(parsed); + await fs.writeFile(keywordPath, JSON.stringify(parsed), 'utf8'); + + await expect(findSymbolReferences(rootPath, 'identity')).rejects.toThrow(/keyword index/i); + }); + + it('finds current references in files omitted by the keyword chunk cap', async () => { + findIdentifierOccurrences.mockResolvedValue([ + { line: 1, startIndex: 0, endIndex: 1, nodeType: 'identifier' } + ]); + rootPath = await fs.mkdtemp(path.join(os.tmpdir(), 'cbc-symbol-index-cap-')); + const sources = new Map([ + ['src/a-first.ts', 'export function firstOnlyToken() { return 1; }\n'], + ['src/z-second.ts', 'export function secondOnlyToken() { return 2; }\n'] + ]); + for (const [relativePath, source] of sources) { + const absolutePath = path.join(rootPath, relativePath); + await fs.mkdir(path.dirname(absolutePath), { recursive: true }); + await fs.writeFile(absolutePath, source, 'utf8'); + } + + const stats = await new CodebaseIndexer({ + rootPath, + config: { skipEmbedding: true, parsing: { maxChunks: 1, maxFileSize: 1024 * 1024 } }, + projectOptions: { preferredAnalyzer: 'generic' } + }).index(); + + const contextPath = path.join(rootPath, CODEBASE_CONTEXT_DIRNAME); + const keywordIndex = JSON.parse( + await fs.readFile( + await resolveIndexArtifact(path.dirname(contextPath), KEYWORD_INDEX_FILENAME), + 'utf8' + ) + ) as { + header: { buildId: string; formatVersion: number }; + chunks: Array<{ relativePath?: string }>; + }; + const manifest = JSON.parse( + await fs.readFile( + await resolveIndexArtifact(path.dirname(contextPath), MANIFEST_FILENAME), + 'utf8' + ) + ) as { + header?: { buildId: string; formatVersion: number }; + files: Record; + }; + const indexedPaths = new Set( + keywordIndex.chunks.flatMap((chunk) => (chunk.relativePath ? [chunk.relativePath] : [])) + ); + const omittedPath = Object.keys(manifest.files).find((file) => !indexedPaths.has(file)); + expect(stats.indexedFiles).toBe(2); + expect(stats.totalChunks).toBeGreaterThan(keywordIndex.chunks.length); + expect(keywordIndex.chunks).toHaveLength(1); + expect(manifest.header).toEqual(keywordIndex.header); + expect(omittedPath).toBeDefined(); + if (!omittedPath) throw new Error('Expected one manifest-listed file omitted by the chunk cap'); + + const omittedSymbol = new Map([ + ['src/a-first.ts', 'firstOnlyToken'], + ['src/z-second.ts', 'secondOnlyToken'] + ]).get(omittedPath); + expect(omittedSymbol).toBeDefined(); + if (!omittedSymbol) throw new Error(`Unexpected omitted source path: ${omittedPath}`); + + const result = await findSymbolReferences(rootPath, omittedSymbol); + + expect(result.status).toBe('success'); + if (result.status !== 'success') throw new Error('Expected successful current-source scan'); + expect(result.usageCount).toBe(1); + expect(result.usages).toMatchObject([{ file: omittedPath, line: 1 }]); + expect(result.isComplete).toBe(true); + expect(result.coverage).toMatchObject({ + candidateFiles: 2, + sourcesRead: 2, + filesUnavailable: 0 + }); + }); + + it('writes a build-bound manifest after an incremental add/change/delete pass', async () => { + rootPath = await fs.mkdtemp(path.join(os.tmpdir(), 'cbc-symbol-manifest-incremental-')); + const sourceFiles = new Map([ + ['src/keep.ts', 'export const beforeChange = 1;\n'], + ['src/remove.ts', 'export const removedSource = 1;\n'] + ]); + for (const [relativePath, source] of sourceFiles) { + const absolutePath = path.join(rootPath, relativePath); + await fs.mkdir(path.dirname(absolutePath), { recursive: true }); + await fs.writeFile(absolutePath, source, 'utf8'); + } + const config = { skipEmbedding: true, parsing: { maxChunks: 1, maxFileSize: 1024 * 1024 } }; + await new CodebaseIndexer({ + rootPath, + config, + projectOptions: { preferredAnalyzer: 'generic' } + }).index(); + + const contextPath = path.join(rootPath, CODEBASE_CONTEXT_DIRNAME); + const firstMeta = JSON.parse( + await fs.readFile(path.join(contextPath, INDEX_META_FILENAME), 'utf8') + ) as { buildId: string; formatVersion: number }; + const firstManifest = JSON.parse( + await fs.readFile( + await resolveIndexArtifact(path.dirname(contextPath), MANIFEST_FILENAME), + 'utf8' + ) + ) as { header?: { buildId: string; formatVersion: number } }; + expect(firstManifest.header).toEqual({ + buildId: firstMeta.buildId, + formatVersion: firstMeta.formatVersion + }); + + await fs.rm(path.join(rootPath, 'src', 'remove.ts')); + await fs.writeFile(path.join(rootPath, 'src', 'keep.ts'), 'export const changedToken = 2;\n'); + await fs.writeFile(path.join(rootPath, 'src', 'added.ts'), 'export const addedToken = 3;\n'); + const incremental = await new CodebaseIndexer({ + rootPath, + config, + projectOptions: { preferredAnalyzer: 'generic' }, + incrementalOnly: true + }).index(); + + const nextMeta = JSON.parse( + await fs.readFile(path.join(contextPath, INDEX_META_FILENAME), 'utf8') + ) as { buildId: string; formatVersion: number }; + const nextManifest = JSON.parse( + await fs.readFile( + await resolveIndexArtifact(path.dirname(contextPath), MANIFEST_FILENAME), + 'utf8' + ) + ) as { header?: { buildId: string; formatVersion: number }; files: Record }; + expect(incremental.incremental).toMatchObject({ added: 1, changed: 1, deleted: 1 }); + expect(nextManifest.header).toEqual({ + buildId: nextMeta.buildId, + formatVersion: nextMeta.formatVersion + }); + expect(Object.keys(nextManifest.files).sort()).toEqual(['src/added.ts', 'src/keep.ts']); + }); + + it('does not stamp a headerless legacy manifest on a no-op incremental run', async () => { + rootPath = await fs.mkdtemp(path.join(os.tmpdir(), 'cbc-symbol-manifest-legacy-')); + const sourcePath = path.join(rootPath, 'src', 'legacy.ts'); + await fs.mkdir(path.dirname(sourcePath), { recursive: true }); + await fs.writeFile(sourcePath, 'export const legacyToken = 1;\n', 'utf8'); + await new CodebaseIndexer({ + rootPath, + config: { skipEmbedding: true }, + projectOptions: { preferredAnalyzer: 'generic' } + }).index(); + + const contextPath = path.join(rootPath, CODEBASE_CONTEXT_DIRNAME); + const manifestPath = await resolveIndexArtifact(path.dirname(contextPath), MANIFEST_FILENAME); + const initialManifest = JSON.parse(await fs.readFile(manifestPath, 'utf8')) as Record< + string, + unknown + >; + expect(initialManifest.header).toBeDefined(); + delete initialManifest.header; + await fs.writeFile(manifestPath, JSON.stringify(initialManifest), 'utf8'); + + await new CodebaseIndexer({ + rootPath, + config: { skipEmbedding: true }, + projectOptions: { preferredAnalyzer: 'generic' }, + incrementalOnly: true + }).index(); + + const afterIncremental = JSON.parse(await fs.readFile(manifestPath, 'utf8')) as Record< + string, + unknown + >; + const meta = JSON.parse( + await fs.readFile(path.join(contextPath, INDEX_META_FILENAME), 'utf8') + ) as { artifacts: { vectorDb: { embeddingEnabled?: boolean } } }; + expect(afterIncremental.header).toBeUndefined(); + expect(meta.artifacts.vectorDb.embeddingEnabled).toBe(false); + await expect(findSymbolReferences(rootPath, 'legacyToken')).rejects.toThrow(/full reindex/i); + + const memoryPath = path.join(contextPath, MEMORY_FILENAME); + const memory = JSON.stringify([{ type: 'note', memory: 'preserve this note' }]); + await fs.writeFile(memoryPath, memory, 'utf8'); + await new CodebaseIndexer({ + rootPath, + config: { skipEmbedding: true }, + projectOptions: { preferredAnalyzer: 'generic' } + }).index(); + + const repairedManifest = JSON.parse( + await fs.readFile(await resolveIndexArtifact(rootPath, MANIFEST_FILENAME), 'utf8') + ) as { + header?: { buildId: string; formatVersion: number }; + }; + const repairedMeta = JSON.parse( + await fs.readFile(path.join(contextPath, INDEX_META_FILENAME), 'utf8') + ) as { + buildId: string; + formatVersion: number; + artifacts: { vectorDb: { embeddingEnabled?: boolean } }; + }; + expect(repairedManifest.header).toEqual({ + buildId: repairedMeta.buildId, + formatVersion: repairedMeta.formatVersion + }); + expect(repairedMeta.artifacts.vectorDb.embeddingEnabled).toBe(false); + expect(await fs.readFile(memoryPath, 'utf8')).toBe(memory); + findIdentifierOccurrences.mockResolvedValue([ + { line: 1, startIndex: 0, endIndex: 1, nodeType: 'identifier' } + ]); + const repairedResult = await findSymbolReferences(rootPath, 'legacyToken'); + expect(repairedResult.status).toBe('success'); + if (repairedResult.status !== 'success') + throw new Error('Expected reference lookup after full reindex'); + expect(repairedResult.usageCount).toBe(1); + }); + + it('requires a readable versioned manifest for reference coverage', async () => { + const root = await createIndex([{ path: 'src/a.ts', source: 'alpha();' }]); + await fs.rm(await resolveIndexArtifact(root, MANIFEST_FILENAME)); + + await expect(findSymbolReferences(root, 'alpha')).rejects.toThrow(/manifest.*full reindex/i); + }); + + it('requires a build-bound manifest instead of accepting the legacy headerless format', async () => { + const root = await createIndex([{ path: 'src/a.ts', source: 'alpha();' }]); + const manifestPath = await resolveIndexArtifact(root, MANIFEST_FILENAME); + const manifest = JSON.parse(await fs.readFile(manifestPath, 'utf8')) as Record; + delete manifest.header; + await rewriteManifest(root, manifest); + + await expect(findSymbolReferences(root, 'alpha')).rejects.toThrow(/manifest.*full reindex/i); + }); + + it.each([ + ['buildId', { buildId: 'different-build', formatVersion: INDEX_FORMAT_VERSION }], + ['formatVersion', { buildId: 'test', formatVersion: INDEX_FORMAT_VERSION + 1 }] + ])('rejects manifest header with mismatched %s', async (_field, header) => { + const root = await createIndex([{ path: 'src/a.ts', source: 'alpha();' }]); + const manifestPath = await resolveIndexArtifact(root, MANIFEST_FILENAME); + const manifest = JSON.parse(await fs.readFile(manifestPath, 'utf8')) as { + header: unknown; + [key: string]: unknown; + }; + manifest.header = header; + await rewriteManifest(root, manifest); + + await expect(findSymbolReferences(root, 'alpha')).rejects.toThrow( + /manifest.*(buildId|formatVersion)/i + ); + }); + + it.each([ + ['null', null], + ['an array', []], + ['a malformed hash', { 'src/a.ts': 'not-a-hash' }] + ])('rejects manifest file entries supplied as %s', async (_description, files) => { + const root = await createIndex([{ path: 'src/a.ts', source: 'alpha();' }]); + const manifestPath = await resolveIndexArtifact(root, MANIFEST_FILENAME); + const manifest = JSON.parse(await fs.readFile(manifestPath, 'utf8')) as { + [key: string]: unknown; + }; + manifest.files = files; + await rewriteManifest(root, manifest); + + await expect(findSymbolReferences(root, 'alpha')).rejects.toThrow(/manifest/i); + }); + + it('rejects manifest artifact metadata that traverses outside the context directory', async () => { + const root = await createIndex([{ path: 'src/a.ts', source: 'alpha();' }]); + await fs.writeFile( + path.join(root, MANIFEST_FILENAME), + await fs.readFile(await resolveIndexArtifact(root, MANIFEST_FILENAME), 'utf8'), + 'utf8' + ); + await rewriteManifestArtifactPath(root, `../${MANIFEST_FILENAME}`); + + await expect(findSymbolReferences(root, 'alpha')).rejects.toThrow(/manifest.*path/i); + }); + + it('rejects traversal in manifest source paths before scanning sources', async () => { + const root = await createIndex([{ path: 'src/a.ts', source: 'alpha();' }]); + const manifestPath = await resolveIndexArtifact(root, MANIFEST_FILENAME); + const manifest = JSON.parse(await fs.readFile(manifestPath, 'utf8')) as { + [key: string]: unknown; + }; + manifest.files = { '../outside.ts': '0'.repeat(16) }; + await rewriteManifest(root, manifest); + + await expect(findSymbolReferences(root, 'alpha')).rejects.toThrow(/manifest.*path/i); + }); + + it('skips parser work when current source text cannot contain the symbol', async () => { + const root = await createIndex([{ path: 'src/a.ts', source: 'const beta = 1;' }]); + findIdentifierOccurrences.mockResolvedValue([]); + + const result = await findSymbolReferences(root, 'alpha'); + + expect(result.status).toBe('success'); + expect(findIdentifierOccurrences).not.toHaveBeenCalled(); + if (result.status !== 'success') throw new Error('Expected complete current-source scan'); + expect(result.coverage).toMatchObject({ + candidateFiles: 1, + sourcesRead: 1, + filesWithoutText: 1, + parserCalls: 0 + }); + }); + + it('skips longer-identifier candidates without changing exact references or text coverage', async () => { + const root = await createIndex([ + { path: 'src/long.ts', source: 'const SceneData = 1;' }, + { path: 'src/exact.ts', source: 'const Scene = 1; Scene();' } + ]); + findIdentifierOccurrences.mockResolvedValue([ + { line: 1, startIndex: 6, endIndex: 11, nodeType: 'identifier' } + ]); + + const result = await findSymbolReferences(root, 'Scene'); + + expect(result.status).toBe('success'); + if (result.status !== 'success') throw new Error('Expected successful current-source scan'); + expect(findIdentifierOccurrences).toHaveBeenCalledTimes(1); + expect(findIdentifierOccurrences).toHaveBeenCalledWith( + 'const Scene = 1; Scene();', + 'typescript', + 'Scene' + ); + expect(findIdentifierOccurrences).not.toHaveBeenCalledWith( + 'const SceneData = 1;', + 'typescript', + 'Scene' + ); + expect(result.usageCount).toBe(1); + expect(result.usages).toMatchObject([{ file: 'src/exact.ts', line: 1 }]); + expect(result.coverage).toMatchObject({ + candidateFiles: 2, + sourcesRead: 2, + filesWithoutText: 0, + parserCalls: 1 + }); + }); + + it('finds references added after indexing by checking current source', async () => { + const root = await createIndex([{ path: 'src/a.ts', source: 'const previousOnly = 1;' }]); + await fs.writeFile( + path.join(root, 'src/a.ts'), + 'const currentOnly = 1; currentOnly();', + 'utf8' + ); + findIdentifierOccurrences.mockResolvedValue(null); + + const result = await findSymbolReferences(root, 'currentOnly'); + + expect(result.status).toBe('success'); + if (result.status !== 'success') throw new Error('Expected successful current-source scan'); + expect(result.usageCount).toBe(2); + expect(result.coverage).toMatchObject({ sourcesRead: 1, parserCalls: 1 }); + }); + + it.each(['$alpha', 'προβολή'])( + 'matches the full identifier spelling %s in lexical fallback', + async (symbol) => { + const source = `x${symbol} suffix_${symbol} $${symbol} ${symbol} `; + const root = await createIndex([{ path: 'src/a.unknown', source }]); + findIdentifierOccurrences.mockResolvedValue(null); + + const result = await findSymbolReferences(root, symbol); + + expect(result.status).toBe('success'); + if (result.status !== 'success') throw new Error('Expected successful lexical scan'); + expect(result.usageCount).toBe(1); + expect(result.confidence).toBe('lexical'); + expect(result.coverage.lexicalFallbackFiles).toBe(1); + } + ); + + it('counts observed references beyond the usage limit and reports truncation', async () => { + const root = await createIndex([ + { path: 'src/a.unknown', source: 'activeToken(); activeToken(); activeToken();' } + ]); + findIdentifierOccurrences.mockResolvedValue(null); + + const result = await findSymbolReferences(root, 'activeToken', 2); + + expect(result.status).toBe('success'); + if (result.status !== 'success') throw new Error('Expected successful bounded scan'); + expect(result.usages).toHaveLength(2); + expect(result.usageCount).toBe(3); + expect(result.isComplete).toBe(false); + }); + + it('keeps observed references when an indexed source is unavailable', async () => { + const root = await createIndex([ + { path: 'src/missing.ts', exists: false }, + { path: 'src/readable.unknown', source: 'activeToken();' } + ]); + findIdentifierOccurrences.mockResolvedValue(null); + + const result = await findSymbolReferences(root, 'activeToken'); + + expect(result.status).toBe('success'); + if (result.status !== 'success') throw new Error('Expected partial successful scan'); + expect(result.usageCount).toBe(1); + expect(result.isComplete).toBe(false); + expect(result.coverage.filesUnavailable).toBe(1); + }); + + it('bounds source reads to four and preserves manifest order with sequential parsing', async () => { + const files = Array.from({ length: 9 }, (_, index) => ({ + path: `src/${index}.unknown`, + source: index < 2 ? 'previousOnly();' : `// file ${index}\nπροβολή();\n` + })); + const root = await createIndex(files); + await fs.writeFile(path.join(root, files[0].path), '// current source\r\nπροβολή();\r\n'); + // Every other source is manifest-only; capped keyword chunks are not the inventory. + await fs.writeFile( + await resolveIndexArtifact(root, KEYWORD_INDEX_FILENAME), + JSON.stringify({ + header: { buildId: 'test', formatVersion: INDEX_FORMAT_VERSION }, + chunks: [{ relativePath: files[0].path }] + }) + ); + const sourcePaths = new Map(files.map((file, index) => [path.join(root, file.path), index])); + const gates = files.map(() => createGate()); + const parserGate = createGate(); + const parserStarted = createGate(); + const readsStarted: number[] = []; + const readsCompleted: number[] = []; + let activeReads = 0; + let maxActiveReads = 0; + let activeParsers = 0; + let maxActiveParsers = 0; + const originalReadFile = fs.readFile; + vi.spyOn(fs, 'readFile').mockImplementation(async (...args) => { + const index = typeof args[0] === 'string' ? sourcePaths.get(args[0]) : undefined; + if (index === undefined) return originalReadFile(...args); + readsStarted.push(index); + activeReads += 1; + maxActiveReads = Math.max(maxActiveReads, activeReads); + try { + await gates[index].promise; + const content = await originalReadFile(...args); + readsCompleted.push(index); + return content; + } finally { + activeReads -= 1; + } + }); + findIdentifierOccurrences.mockImplementation(async () => { + activeParsers += 1; + maxActiveParsers = Math.max(maxActiveParsers, activeParsers); + parserStarted.release(); + await parserGate.promise; + activeParsers -= 1; + return null; + }); + const scan = findSymbolReferences(root, 'προβολή', 3); + try { + await vi.waitFor(() => expect(readsStarted).toHaveLength(4)); + expect([...readsStarted].sort()).toEqual([0, 1, 2, 3]); + for (const index of [3, 2, 1, 0]) { + gates[index].release(); + await vi.waitFor(() => expect(readsCompleted).toContain(index)); + } + await parserStarted.promise; + // Completed buffers remain bounded while the shared parser is busy. + expect(readsStarted).toHaveLength(4); + expect(findIdentifierOccurrences).toHaveBeenCalledTimes(1); + parserGate.release(); + for (const gate of gates) gate.release(); + const result = await scan; + + expect(maxActiveReads).toBe(4); + expect(maxActiveParsers).toBe(1); + expect(readsCompleted.slice(0, 4)).toEqual([3, 2, 1, 0]); + expect(new Set(readsStarted).size).toBe(9); + expect(result.status).toBe('success'); + if (result.status !== 'success') throw new Error('Expected bounded current-source scan'); + expect(result.usageCount).toBe(8); + expect(result.usages.map((usage) => usage.file)).toEqual([ + 'src/0.unknown', + 'src/2.unknown', + 'src/3.unknown' + ]); + expect(result.usages[0]).toEqual({ + file: 'src/0.unknown', + line: 2, + preview: '// current source\nπροβολή();' + }); + expect(result.coverage).toEqual({ + candidateFiles: 9, + sourcesRead: 9, + filesWithoutText: 1, + filesUnavailable: 0, + parserCalls: 8, + lexicalFallbackFiles: 8 + }); + expect(result.confidence).toBe('lexical'); + expect(result.isComplete).toBe(false); + } finally { + for (const gate of gates) gate.release(); + parserGate.release(); + await scan; + findIdentifierOccurrences.mockReset(); + } + }); + + it.each(['ENOENT', 'ENOTDIR', 'EACCES', 'EPERM'])( + 'keeps full observed counts across batches when a source read reports %s', + async (code) => { + const files = Array.from({ length: 6 }, (_, index) => ({ + path: `src/${index}.unknown`, + source: 'alpha();' + })); + const root = await createIndex(files); + const unavailable = path.join(root, files[1].path); + const originalReadFile = fs.readFile; + vi.spyOn(fs, 'readFile').mockImplementation(async (...args) => { + if (args[0] === unavailable) throw Object.assign(new Error('unavailable'), { code }); + return originalReadFile(...args); + }); + findIdentifierOccurrences.mockResolvedValue(null); + + const result = await findSymbolReferences(root, 'alpha', 1); + + expect(result.status).toBe('success'); + if (result.status !== 'success') throw new Error('Expected partial current-source scan'); + expect(result.usageCount).toBe(5); + expect(result.usages.map((usage) => usage.file)).toEqual(['src/0.unknown']); + expect(result.coverage).toMatchObject({ + candidateFiles: 6, + sourcesRead: 5, + filesUnavailable: 1, + parserCalls: 5 + }); + expect(result.isComplete).toBe(false); + } + ); + + it.each(['realpath', 'stat', 'readFile'] as const)( + 'fails the entire batch closed on an unexpected %s error without parsing it', + async (operation) => { + const files = Array.from({ length: 5 }, (_, index) => ({ + path: `src/${index}.unknown`, + source: 'alpha();' + })); + const root = await createIndex(files); + const fatal = path.join(root, files[1].path); + const later = path.join(root, files[4].path); + const originalReadFile = fs.readFile; + const laterReads: unknown[] = []; + // The operation-specific spies retain the real fs signatures. + if (operation === 'realpath') { + const realpath = fs.realpath; + vi.spyOn(fs, 'realpath').mockImplementation(async (...args) => { + if (args[0] === fatal) throw Object.assign(new Error('fatal'), { code: 'EIO' }); + return realpath(...args); + }); + } else if (operation === 'stat') { + const stat = fs.stat; + vi.spyOn(fs, 'stat').mockImplementation(async (...args) => { + if (args[0] === fatal) throw Object.assign(new Error('fatal'), { code: 'EIO' }); + return stat(...args); + }); + } + vi.spyOn(fs, 'readFile').mockImplementation(async (...args) => { + if (args[0] === later) laterReads.push(args[0]); + if (operation === 'readFile' && args[0] === fatal) { + throw Object.assign(new Error('fatal'), { code: 'EIO' }); + } + return originalReadFile(...args); + }); + findIdentifierOccurrences.mockResolvedValue(null); + + const result = await findSymbolReferences(root, 'alpha'); + + expect(result.status).toBe('error'); + if (result.status !== 'error') throw new Error('Expected fail-closed scan'); + expect(result.message).toContain(files[1].path); + expect(findIdentifierOccurrences).not.toHaveBeenCalled(); + expect(laterReads).toEqual([]); + } + ); + + it('settles pending source read failures before returning a fatal batch', async () => { + const files = Array.from({ length: 5 }, (_, index) => ({ + path: `src/${index}.unknown`, + source: 'alpha();' + })); + const root = await createIndex(files); + const first = path.join(root, files[0].path); + const pending = path.join(root, files[1].path); + const pendingRelease = createGate(); + const originalReadFile = fs.readFile; + let pendingStarted = false; + let pendingSettled = false; + let scanSettled = false; + vi.spyOn(fs, 'readFile').mockImplementation(async (...args) => { + if (args[0] === first) throw Object.assign(new Error('first failure'), { code: 'EIO' }); + if (args[0] === pending) { + pendingStarted = true; + await pendingRelease.promise; + pendingSettled = true; + throw Object.assign(new Error('later failure'), { code: 'EMFILE' }); + } + return originalReadFile(...args); + }); + findIdentifierOccurrences.mockResolvedValue(null); + const scan = findSymbolReferences(root, 'alpha').finally(() => { + scanSettled = true; + }); + try { + await vi.waitFor(() => expect(pendingStarted).toBe(true)); + expect(scanSettled).toBe(false); + expect(pendingSettled).toBe(false); + pendingRelease.release(); + const result = await scan; + expect(result.status).toBe('error'); + if (result.status !== 'error') throw new Error('Expected fail-closed scan'); + expect(result.message).toContain(files[0].path); + expect(pendingSettled).toBe(true); + expect(findIdentifierOccurrences).not.toHaveBeenCalled(); + } finally { + pendingRelease.release(); + await scan; + } + }); + + it('fails a batch closed when a manifest-listed source becomes a directory', async () => { + const files = Array.from({ length: 5 }, (_, index) => ({ + path: `src/${index}.unknown`, + source: 'alpha();' + })); + const root = await createIndex(files); + const invalidSource = path.join(root, files[1].path); + await fs.unlink(invalidSource); + await fs.mkdir(invalidSource); + findIdentifierOccurrences.mockResolvedValue(null); + + const result = await findSymbolReferences(root, 'alpha'); + + expect(result.status).toBe('error'); + if (result.status !== 'error') throw new Error('Expected file-integrity error'); + expect(result.message).toContain(files[1].path); + expect(findIdentifierOccurrences).not.toHaveBeenCalled(); + }); + + it('fails closed when an indexed source resolves outside the project', async () => { + const root = await createIndex([{ path: 'src/a.ts', source: 'alpha();' }]); + const outsidePath = await fs.mkdtemp(path.join(os.tmpdir(), 'cbc-symbol-outside-')); + try { + const externalFile = path.join(outsidePath, 'external.ts'); + await fs.writeFile(externalFile, 'alpha();', 'utf8'); + const indexPath = await resolveIndexArtifact(root, KEYWORD_INDEX_FILENAME); + await fs.writeFile( + indexPath, + JSON.stringify({ + header: { buildId: 'test', formatVersion: INDEX_FORMAT_VERSION }, + chunks: [{ filePath: externalFile }] + }), + 'utf8' + ); + + const result = await findSymbolReferences(root, 'alpha'); + + expect(result.status).toBe('error'); + if (result.status !== 'error') throw new Error('Expected path-confinement error'); + expect(result.message).toMatch(/invalid|outside/i); + } finally { + await fs.rm(outsidePath, { recursive: true, force: true }); + } + }); + + it('fails closed when a manifest-listed source symlink resolves outside the project', async ({ + skip + }) => { + const root = await createIndex([{ path: 'src/anchor.ts', source: 'safeOnly();' }]); + const outsidePath = await fs.mkdtemp(path.join(os.tmpdir(), 'cbc-symbol-manifest-link-')); + try { + const externalFile = path.join(outsidePath, 'external.ts'); + await fs.writeFile(externalFile, 'export const outsideOnly = 1;\n', 'utf8'); + const linkPath = path.join(root, 'src', 'link.ts'); + await fs.mkdir(path.dirname(linkPath), { recursive: true }); + try { + await fs.symlink(externalFile, linkPath, 'file'); + } catch (error) { + const code = + error !== null && typeof error === 'object' && 'code' in error ? error.code : undefined; + if (code === 'EPERM' || code === 'EACCES' || code === 'ENOSYS') { + skip(`Symlink creation is unavailable in this environment (${String(code)})`); + } + throw error; + } + const manifestPath = await resolveIndexArtifact(root, MANIFEST_FILENAME); + const manifest = JSON.parse(await fs.readFile(manifestPath, 'utf8')) as { + files: Record; + [key: string]: unknown; + }; + manifest.files['src/link.ts'] = '0'.repeat(16); + await rewriteManifest(root, manifest); + + const result = await findSymbolReferences(root, 'outsideOnly'); + + expect(result.status).toBe('error'); + if (result.status !== 'error') throw new Error('Expected symlink confinement error'); + expect(result.message).toMatch(/outside/i); + } finally { + await fs.rm(outsidePath, { recursive: true, force: true }); + } + }); +}); diff --git a/tests/tools/dispatch.test.ts b/tests/tools/dispatch.test.ts index d51f313..366d642 100644 --- a/tests/tools/dispatch.test.ts +++ b/tests/tools/dispatch.test.ts @@ -1,10 +1,11 @@ import { describe, it, expect } from 'vitest'; +import { fileURLToPath } from 'node:url'; import { TOOLS, dispatchTool } from '../../src/tools/index.js'; import type { ToolContext } from '../../src/tools/types.js'; describe('Tool Dispatch', () => { - it('exports all 11 tools', () => { - expect(TOOLS.length).toBe(11); + it('exports all 12 tools including source relationship navigation', () => { + expect(TOOLS.length).toBe(12); expect(TOOLS.map((t) => t.name)).toEqual([ 'search_codebase', 'get_codebase_metadata', @@ -16,7 +17,8 @@ describe('Tool Dispatch', () => { 'detect_circular_dependencies', 'remember', 'get_memory', - 'get_codebase_health' + 'get_codebase_health', + 'get_code_relationships' ]); }); @@ -96,5 +98,11 @@ describe('Tool Dispatch', () => { expect(result.content![0].type).toBe('text'); const text = result.content![0].text; expect(text).toContain('idle'); + const status = JSON.parse(text) as { rootPath: string; loadedStatusModulePath: string }; + expect(status.rootPath).toBe('/tmp'); + expect(status.loadedStatusModulePath).toBe( + fileURLToPath(new URL('../../src/tools/get-indexing-status.ts', import.meta.url)) + ); + expect(status.loadedStatusModulePath).not.toBe(status.rootPath); }); }); diff --git a/tests/tree-sitter-symbols.test.ts b/tests/tree-sitter-symbols.test.ts index b8ce9dd..9511df2 100644 --- a/tests/tree-sitter-symbols.test.ts +++ b/tests/tree-sitter-symbols.test.ts @@ -1,6 +1,11 @@ import { describe, expect, it } from 'vitest'; import { GenericAnalyzer } from '../src/analyzers/generic/index'; -import { extractTreeSitterSymbols, supportsTreeSitter } from '../src/utils/tree-sitter'; +import structuralFixture from './fixtures/structural-context-regressions.json'; +import { + extractTreeSitterSymbols, + findIdentifierOccurrences, + supportsTreeSitter +} from '../src/utils/tree-sitter'; describe('Tree-sitter symbol extraction', () => { it('extracts TypeScript symbols including function variables', async () => { @@ -80,4 +85,70 @@ describe('Tree-sitter symbol extraction', () => { it('reports unsupported language grammars', () => { expect(supportsTreeSitter('markdown')).toBe(false); }); + + it('matches complete dollar, underscore, and Unicode identifiers in parsed code', async () => { + const source = [ + 'const alpha = $alpha;', + 'const alphaSuffix = 1;', + 'const _alpha = 2;', + 'const προβολή = alpha;', + '// alpha and $alpha are comments', + 'const message = "alpha $alpha";' + ].join('\n'); + + const alpha = await findIdentifierOccurrences(source, 'typescript', 'alpha'); + const dollar = await findIdentifierOccurrences(source, 'typescript', '$alpha'); + const unicode = await findIdentifierOccurrences(source, 'typescript', 'προβολή'); + + expect(alpha?.map((occurrence) => occurrence.line)).toEqual([1, 4]); + expect(dollar?.map((occurrence) => occurrence.line)).toEqual([1]); + expect(unicode?.map((occurrence) => occurrence.line)).toEqual([4]); + }); +}); + +describe('frozen structural source facts', () => { + it('does not connect static ancestors across a dynamic method or field boundary', async () => { + const cases = [ + 'class C { [name](){ const inner=()=>run(); } }', + 'class C { [name]=()=>{ const inner=()=>run(); }; }' + ]; + for (const source of cases) { + const parsed = await extractTreeSitterSymbols(source, 'javascript'); + expect(parsed).not.toBeNull(); + const inner = parsed!.symbols.find((symbol) => symbol.name === 'inner') as + | (NonNullable['symbols'][number] & { symbolPath?: string[] }) + | undefined; + expect(inner).toBeDefined(); + expect(inner!.symbolPath).toEqual(['inner']); + } + }); + + for (const fixture of structuralFixture.cases) { + it(`preserves exact static ownership and implementation: ${fixture.id}`, async () => { + const extracted = await extractTreeSitterSymbols(fixture.source, fixture.language); + expect(extracted).not.toBeNull(); + for (const fact of fixture.sourceSymbols) { + if ('scopeOnly' in fact && fact.scopeOnly) { + expect(extracted!.symbols.some((symbol) => symbol.name.includes('[actionName]'))).toBe( + false + ); + continue; + } + const symbol = extracted!.symbols.find((candidate) => { + const sourceSymbol = candidate as typeof candidate & { symbolPath?: string[] }; + return JSON.stringify(sourceSymbol.symbolPath) === JSON.stringify(fact.ownerChain); + }); + expect(symbol, `${fixture.id}: ${fact.ownerChain.join(' > ')}`).toBeDefined(); + for (const marker of fact.bodyMarkers) expect(symbol!.content).toContain(marker); + if ('forbiddenSiblingBodyMarkers' in fact) { + for (const marker of fact.forbiddenSiblingBodyMarkers ?? []) { + expect(symbol!.content).not.toContain(marker); + } + } + expect(fixture.source).toContain(symbol!.content); + expect(symbol!.startLine).toBeGreaterThanOrEqual(1); + expect(symbol!.endLine).toBeLessThanOrEqual(fixture.source.split('\n').length); + } + }); + } }); diff --git a/vitest.config.mjs b/vitest.config.mjs index 6b6958c..fd318e9 100644 --- a/vitest.config.mjs +++ b/vitest.config.mjs @@ -2,6 +2,8 @@ import { defineConfig } from 'vitest/config'; export default defineConfig({ test: { + // File-heavy subprocess fixtures compete for native resources on Windows. + ...(process.platform === 'win32' ? { maxWorkers: 1 } : {}), include: ['tests/**/*.test.ts'], environment: 'node', testTimeout: 15000