diff --git a/Cargo.lock b/Cargo.lock index 860f7e0e1d..4238a98de7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6430,6 +6430,7 @@ dependencies = [ "base64 0.23.1", "criterion", "flate2", + "gix", "hex", "ignore", "miniz_oxide", diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler.rs index a325e59813..d3ea24dc19 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler.rs @@ -201,7 +201,7 @@ mod activation; pub mod branch_generations; pub mod branch_publication; mod cadence; -mod classification; +use tracedecay_code_index::source_walk::classification; mod demand_admission; mod freshness_witness; pub use freshness_witness::sealed_source_bytes; diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/classification.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/classification.rs deleted file mode 100644 index 8f057e94a2..0000000000 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/classification.rs +++ /dev/null @@ -1,347 +0,0 @@ -//! Truthful gix-based change classification for incremental indexing. -//! -//! Filesystem events and hook payloads only *hint* which paths may have moved. -//! The authoritative changed/added/deleted/renamed set is always recomputed -//! here from gix's HEAD-tree/index/worktree status, so a duplicate hook, a -//! save-without-change, or a dropped watcher event can never fabricate work or -//! hide a real change. -//! -//! Committed, staged, unstaged, untracked, and deleted paths are kept -//! *distinct*. Deletions become tombstone candidates. Renames are deliberately -//! not tracked as a distinct class: for indexing a rename is just a deletion of -//! the source plus an addition of the destination, so rename detection is -//! disabled and gix reports the two halves independently. The scheduler -//! consumes the derived candidate and changed sets to build an incremental -//! batch. - -use std::collections::BTreeSet; -use std::path::Path; - -use gix::bstr::ByteSlice; -use tracedecay_runtime_core::git_repository::GIT_STATUS_MODIFICATION_CHECK_THREADS; - -/// Failure to classify worktree changes through gix. -#[derive(Debug, thiserror::Error)] -pub enum ClassificationErrorV1 { - #[error("code-index classification: {0}")] - Git(String), -} - -/// The truthful disposition of one repository-relative path. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub enum WorktreeChangeClassV1 { - /// Added to the index versus HEAD (staged new file). - StagedAdded, - /// Content differs between HEAD tree and index (staged modification). - StagedModified, - /// Present in HEAD tree, removed from the index (staged deletion). - StagedDeleted, - /// Worktree content differs from the index (unstaged modification). - UnstagedModified, - /// Tracked file missing from the worktree (unstaged deletion). - UnstagedDeleted, - /// Present on disk with no index relation (untracked, incl. intent-to-add). - Untracked, - /// Merge conflict; content is not a single truthful revision. - Conflicted, -} - -impl WorktreeChangeClassV1 { - /// Whether this class removes a path (so its prior chunks become tombstones) - /// rather than presenting current content to index. - pub fn is_deletion(self) -> bool { - matches!(self, Self::StagedDeleted | Self::UnstagedDeleted) - } - - /// Whether this class contributes a present file whose bytes should be - /// hashed and considered for (re)indexing. - pub fn presents_content(self) -> bool { - !self.is_deletion() - } -} - -/// One classified path. -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct ClassifiedChangeV1 { - pub path: String, - pub class: WorktreeChangeClassV1, -} - -/// One classified change whose Git path bytes are not UTF-8. No `String` -/// names it, so it lives outside `changes`: a lossy decode could equal a -/// real UTF-8 path and make the string-keyed candidate and changed sets -/// treat this change as that file's. -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct NonUtf8ClassifiedChangeV1 { - /// The exact Git path bytes. - pub git_path: Vec, - pub class: WorktreeChangeClassV1, -} - -/// A complete, truthful classification of one worktree snapshot. -#[derive(Clone, Debug, Default)] -pub struct WorktreeChangeClassificationV1 { - /// Repository-relative paths tracked in the index (the committed/staged - /// baseline present in the checkout). - committed_baseline: BTreeSet, - changes: Vec, - /// Changes whose Git path bytes are not UTF-8, kept byte-exact so the - /// worktree is still dirty on a non-UTF-8-only transition. - non_utf8_changes: Vec, - /// Present paths whose Git bytes are not UTF-8, so no `String` above - /// names them. - non_utf8_paths: BTreeSet>, -} - -impl WorktreeChangeClassificationV1 { - /// Classify the current status of `repository` truthfully. - #[tracing::instrument(name = "daemon.code_index.capture.classify", level = "trace", skip_all)] - pub fn classify(repository: &gix::Repository) -> Result { - let index = repository - .index_or_empty() - .map_err(|error| ClassificationErrorV1::Git(error.to_string()))?; - let skip_validation = gix::index::entry::Flags::UPTODATE - | gix::index::entry::Flags::SKIP_WORKTREE - | gix::index::entry::Flags::ASSUME_VALID - | gix::index::entry::Flags::FSMONITOR_VALID; - let mut committed_baseline = BTreeSet::new(); - let mut non_utf8_paths = BTreeSet::new(); - let mut has_skipped_entries = false; - for entry in index.entries() { - has_skipped_entries |= entry.flags.intersects(skip_validation); - let path: &[u8] = entry.path(&index).as_ref(); - match std::str::from_utf8(path) { - Ok(path) => { - committed_baseline.insert(path.to_owned()); - } - Err(_) => { - non_utf8_paths.insert(path.to_vec()); - } - } - } - let mut status_index = gix::worktree::IndexPersistedOrInMemory::from(index); - if has_skipped_entries { - // These flags are Git status shortcuts, not evidence that source - // bytes still match the index. Recheck only those entries through - // gix's ordinary filtered blob comparison. An mtime at the index - // timestamp is racy even when all stat fields match, so neither - // the flags nor preserved file metadata can suppress that check. - // This private index is never written back to the user's checkout. - let mut checked = status_index.into_owned(); - let racy_mtime = checked.timestamp().into(); - for (entry, path) in checked.entries_mut_with_paths() { - if entry.flags.intersects(skip_validation) { - // Sparse entries deliberately omit worktree content. - // Preserve that omission; only materialized skipped - // paths need to prove their bytes against the index. - if entry - .flags - .contains(gix::index::entry::Flags::SKIP_WORKTREE) - && (entry.mode.is_sparse() - || repository.workdir().is_some_and(|root| { - gix::path::from_bstr(path).is_ok_and(|relative| { - std::fs::symlink_metadata(root.join(relative)).is_err_and( - |error| error.kind() == std::io::ErrorKind::NotFound, - ) - }) - })) - { - continue; - } - entry.flags.remove(skip_validation); - entry.stat.mtime = racy_mtime; - } - } - status_index = checked.into(); - } - - let mut changes = Vec::new(); - let mut non_utf8_changes = Vec::new(); - let status = repository - .status(gix::progress::Discard) - .map_err(|error| ClassificationErrorV1::Git(error.to_string()))? - .index(status_index) - // Emit untracked files, but not ignored ones: ignored/generated - // content is out of indexing scope. - .untracked_files(gix::status::UntrackedFiles::Files) - // Rename detection is deliberately disabled on both the tree↔index - // and index↔worktree axes: for indexing a rename is just a deletion - // plus an addition, and rewrite tracking is expensive. Content - // reuse still happens via the content-addressed byte pool. - .tree_index_track_renames(gix::status::tree_index::TrackRenames::Disabled) - .index_worktree_rewrites(None) - // Submodule content belongs to its own repository identity and is - // never indexed as part of this worktree. - .index_worktree_submodules(None) - .index_worktree_options_mut(|options| { - options.thread_limit = Some(GIT_STATUS_MODIFICATION_CHECK_THREADS); - }) - .into_iter(Vec::::new()) - .map_err(|error| ClassificationErrorV1::Git(error.to_string()))?; - for item in status { - let item = item.map_err(|error| ClassificationErrorV1::Git(error.to_string()))?; - let location: &[u8] = item.location().as_ref(); - if let Some(class) = classify_item(&item) { - match location.to_str() { - Ok(path) => changes.push(ClassifiedChangeV1 { - path: path.to_owned(), - class, - }), - Err(_) => { - if class.presents_content() { - non_utf8_paths.insert(location.to_vec()); - } else { - non_utf8_paths.remove(location); - } - non_utf8_changes.push(NonUtf8ClassifiedChangeV1 { - git_path: location.to_vec(), - class, - }); - } - } - } - } - - Ok(Self { - committed_baseline, - changes, - non_utf8_changes, - non_utf8_paths, - }) - } - - /// Paths whose Git bytes are not UTF-8, present or not. - pub fn non_utf8_paths(&self) -> &BTreeSet> { - &self.non_utf8_paths - } - - /// Paths no logical path can name because their Git bytes are not UTF-8, - /// limited to those naming a regular file (through links) under - /// `project_root`, the files capture would read. - pub fn non_utf8_files(&self, project_root: &Path) -> Vec> { - self.non_utf8_paths - .iter() - .filter(|path| { - gix::path::from_byte_slice(path) - .is_ok_and(|relative| project_root.join(relative).is_file()) - }) - .cloned() - .collect() - } - - /// Present files worth hashing and considering for (re)indexing: the - /// committed baseline, plus untracked/added/rename-destination paths, minus - /// any path removed by a deletion. - pub fn candidate_paths(&self) -> BTreeSet { - let mut candidates = self.committed_baseline.clone(); - for change in &self.changes { - if change.class.presents_content() { - candidates.insert(change.path.clone()); - } else { - candidates.remove(&change.path); - } - } - candidates - } - - /// Whether the worktree carries any change relative to the committed - /// baseline, UTF-8-named or not. - pub fn has_changes(&self) -> bool { - !(self.changes().is_empty() && self.non_utf8_changes().is_empty()) - } - - /// Whether any classified change is a merge conflict. - pub fn has_conflicted(&self) -> bool { - self.changes() - .iter() - .map(|change| change.class) - .chain(self.non_utf8_changes().iter().map(|change| change.class)) - .any(|class| class == WorktreeChangeClassV1::Conflicted) - } - - /// Paths whose indexing evidence changed relative to the last generation: - /// every staged, unstaged, or untracked path (deletions included so - /// tombstones flow through). This is a hint to narrow work, not the identity - /// authority; the generation planner still compares content digests. - pub fn changed_paths(&self) -> BTreeSet { - self.changes - .iter() - .map(|change| change.path.clone()) - .collect() - } - - /// Paths removed from the present snapshot (staged or unstaged deletions). - #[cfg(test)] - pub fn deleted_paths(&self) -> BTreeSet { - self.changes - .iter() - .filter(|change| change.class.is_deletion()) - .map(|change| change.path.clone()) - .collect() - } - - /// All classified changes a `String` can name (for reporting and tests). - /// Non-UTF-8 changes live under [`Self::non_utf8_changes`]. - pub fn changes(&self) -> &[ClassifiedChangeV1] { - &self.changes - } - - /// All classified changes whose Git path bytes are not UTF-8, kept - /// byte-exact (for reporting and tests). - pub fn non_utf8_changes(&self) -> &[NonUtf8ClassifiedChangeV1] { - &self.non_utf8_changes - } - - /// The class recorded for `path`, if any change touched it. - #[cfg(test)] - pub fn class_of(&self, path: &str) -> Option { - self.changes - .iter() - .find(|change| change.path == path) - .map(|change| change.class) - } -} - -/// Map one gix status item to a truthful class, or `None` for items that carry -/// no indexing signal on their own. -/// -/// Rename detection is disabled on both status axes (see [`classify`]), so gix -/// reports a move as an independent deletion of the source plus an addition of -/// the destination. The `Rewrite` variants therefore cannot occur here; they -/// are mapped to `None` defensively rather than fabricating a rename class. -fn classify_item(item: &gix::status::Item) -> Option { - use gix::diff::index::ChangeRef; - use gix::status::Item; - use gix::status::index_worktree::Item as IndexWorktreeItem; - use gix::status::plumbing::index_as_worktree::{Change as WorktreeChange, EntryStatus}; - - Some(match item { - // HEAD tree ↔ index: staged changes. - Item::TreeIndex(change) => match change { - ChangeRef::Addition { .. } => WorktreeChangeClassV1::StagedAdded, - ChangeRef::Deletion { .. } => WorktreeChangeClassV1::StagedDeleted, - ChangeRef::Modification { .. } => WorktreeChangeClassV1::StagedModified, - // Unreachable with rename detection disabled; skip rather than - // classify a move as anything other than delete + add. - ChangeRef::Rewrite { .. } => return None, - }, - // Index ↔ worktree: unstaged / untracked changes. - Item::IndexWorktree(worktree) => match worktree { - IndexWorktreeItem::Modification { status, .. } => match status { - EntryStatus::Change(WorktreeChange::Removed) => { - WorktreeChangeClassV1::UnstagedDeleted - } - EntryStatus::Conflict { .. } => WorktreeChangeClassV1::Conflicted, - EntryStatus::IntentToAdd => WorktreeChangeClassV1::Untracked, - // Any other worktree change (content, type, submodule) is an - // unstaged modification of present content. - EntryStatus::Change(_) | EntryStatus::NeedsUpdate(_) => { - WorktreeChangeClassV1::UnstagedModified - } - }, - IndexWorktreeItem::DirectoryContents { .. } => WorktreeChangeClassV1::Untracked, - // Unreachable with rewrite tracking disabled; skip defensively. - IndexWorktreeItem::Rewrite { .. } => return None, - }, - }) -} diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/freshness_witness.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/freshness_witness.rs index 8392b54955..fe936ae032 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/freshness_witness.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/freshness_witness.rs @@ -76,10 +76,12 @@ pub struct WorktreeStatSweepV1 { pub fn worktree_stat_sweep( project_root: &Path, ignored_source_admissions: &[CodeIndexIgnoredSourceAdmissionV1], + include_submodules: bool, ) -> Result { let repository = tracedecay_runtime_core::git_open::open(project_root) .map_err(|error| CodeIndexSchedulerErrorV1::Git(error.to_string()))?; - let candidate_roster = source_candidates(&repository, ignored_source_admissions)?; + let candidate_roster = + source_candidates(&repository, ignored_source_admissions, include_submodules)?; // One sweep span covers the O(candidates) stat walk. Never emit one // trace event per file. let mut buf = Vec::new(); @@ -120,16 +122,36 @@ struct SourceCandidatesV1 { fn source_candidates( repository: &gix::Repository, ignored_source_admissions: &[CodeIndexIgnoredSourceAdmissionV1], + include_submodules: bool, ) -> Result { - let classification = classification::WorktreeChangeClassificationV1::classify(repository) - .map_err(|error| CodeIndexSchedulerErrorV1::Git(error.to_string()))?; + let classification = + classification::WorktreeChangeClassificationV1::for_policy(repository, include_submodules) + .map_err(|error| CodeIndexSchedulerErrorV1::Git(error.to_string()))?; let registry = StaticLanguageRegistry::new(); let admitted_paths = ignored_source_admissions .iter() .map(|admission| admission.logical_path.as_str()) .collect::>(); let mut candidate_paths = classification.candidate_paths(); - candidate_paths.extend(admitted_paths.iter().map(|path| (*path).to_owned())); + let project_root = repository.workdir().ok_or_else(|| { + CodeIndexSchedulerErrorV1::Git("source freshness requires a worktree".to_owned()) + })?; + // Explicit ignored-file enrollment lifts Git ignore rules, never the + // parent's index/HEAD gitlink boundary. A staged gitlink removal and + // missing child .git marker must not re-admit its files through freshness. + for path in &admitted_paths { + let relative = Path::new(path); + if classification.beneath_gitlink(relative) + || tracedecay_code_index::source_walk::has_nested_repository_ancestor( + project_root, + relative, + )? + || tracedecay_code_index::source_walk::is_source_symlink(project_root, relative)? + { + continue; + } + candidate_paths.insert((*path).to_owned()); + } let mut unrepresentable = classification .non_utf8_paths() .iter() @@ -233,13 +255,22 @@ fn read_candidate( None, ) } else { - ignored_dependencies::read_bounded_snapshot_source( - &project_root.join(&candidate.logical_path), - None, - ) + let path = resolved_candidate_path(project_root, &candidate.logical_path)?; + ignored_dependencies::read_bounded_snapshot_source(&path, None) } } +fn resolved_candidate_path(root: &Path, logical_path: &str) -> std::io::Result { + tracedecay_code_index::source_walk::resolve_parent_source(root, Path::new(logical_path))? + .map(|relative| root.join(relative)) + .ok_or_else(|| { + std::io::Error::new( + std::io::ErrorKind::InvalidData, + "source alias is outside the parent Git inventory", + ) + }) +} + /// The canonical content digest capture would seal for `raw`: the digest of /// the sanitized bytes. fn sanitized_digest( @@ -478,6 +509,7 @@ fn roster_evidence( repository: &gix::Repository, project_root: &Path, sampled_at: ChangeClockReading, + ignored_source_admissions: &[CodeIndexIgnoredSourceAdmissionV1], ) -> Option)>> { let mut recorder = DirectoryEvidenceV1 { root: project_root.to_path_buf(), @@ -501,6 +533,37 @@ fn roster_evidence( } recorder.record_directory(project_root.to_path_buf()); let index = repository.index_or_empty().ok()?; + // Git's directory walk may not descend into tracked-only directories. + // Their nested .git markers still change source admission without a + // parent index or HEAD update, so key each tracked and explicitly + // admitted source ancestor independently of dirwalk traversal. + let mut source_ancestors = BTreeSet::new(); + let mut collect_ancestors = |relative: &Path| { + let mut current = relative.parent(); + while let Some(parent) = current { + if parent.as_os_str().is_empty() { + break; + } + source_ancestors.insert(parent.to_path_buf()); + current = parent.parent(); + } + }; + for entry in index.entries() { + collect_ancestors(gix::path::from_bstr(entry.path(&index)).ok()?.as_ref()); + } + for admission in ignored_source_admissions { + if validate_code_logical_path(&admission.logical_path).is_err() { + return None; + } + collect_ancestors(Path::new(&admission.logical_path)); + } + for relative in source_ancestors { + let directory = project_root.join(relative); + recorder.record_directory(directory.clone()); + // A missing marker is evidence too: a newly created gitdir file or + // directory must invalidate the roster even if Git ignores it. + recorder.record(directory.join(".git"), false); + } let options = repository .dirwalk_options() .ok()? @@ -585,12 +648,27 @@ impl SourceSweepCacheV1 { let Ok(repository) = tracedecay_runtime_core::git_open::open(project_root) else { return (false, stats); }; - let evidence = roster_evidence(&repository, project_root, sampled_at); + // Parent-only directory evidence cannot vouch for child Git indexes, + // refs or untracked entries. Enumerate both Git rosters on every child proof. + let evidence = (!self.path_policy.includes_submodules()) + .then(|| { + roster_evidence( + &repository, + project_root, + sampled_at, + ignored_source_admissions, + ) + }) + .flatten(); let Ok(SourceCandidatesV1 { candidates, unrepresentable, .. - }) = source_candidates(&repository, ignored_source_admissions) + }) = source_candidates( + &repository, + ignored_source_admissions, + self.path_policy.includes_submodules(), + ) else { return (false, stats); }; @@ -668,12 +746,14 @@ impl SourceSweepCacheV1 { &mut self, project_root: &Path, manifest: &SourceContentManifestV1, + candidate_paths: &BTreeSet, shutting_down: &AtomicBool, ) -> Result, CodeIndexSchedulerErrorV1> { let registry = StaticLanguageRegistry::new(); let candidates = manifest .files .keys() + .filter(|path| candidate_paths.contains(*path)) .filter_map(|logical_path| source_candidate(®istry, logical_path.clone(), false)) .collect::>(); let present = present_candidates( @@ -771,7 +851,12 @@ impl SourceSweepCacheV1 { if disputed.is_empty() { return Ok(disputed); } - Ok(clean_filter_mismatches(project_root, disputed, manifest)) + Ok(clean_filter_mismatches( + project_root, + disputed, + manifest, + self.path_policy.includes_submodules(), + )) } } @@ -815,28 +900,80 @@ fn clean_filter_mismatches<'a>( project_root: &Path, disputed: Vec<&'a StatCandidateV1>, manifest: &SourceContentManifestV1, + include_submodules: bool, ) -> Vec<&'a StatCandidateV1> { let Ok(repository) = tracedecay_runtime_core::git_open::open(project_root) else { return disputed; }; + let mut remaining = disputed; + let mut mismatches = Vec::new(); + if include_submodules { + let Ok(children) = + tracedecay_code_index::source_walk::submodules::live_submodules(&repository) + else { + return remaining; + }; + for (prefix, binding) in children { + let (local, other): (Vec<_>, Vec<_>) = remaining + .into_iter() + .partition(|candidate| Path::new(&candidate.logical_path).starts_with(&prefix)); + remaining = other; + if local.is_empty() { + continue; + } + match tracedecay_runtime_core::git_open::open(&binding.git_dir) { + Ok(child) => mismatches.extend(filter_mismatches_in_repository( + project_root, + &child, + Path::new(&prefix), + local, + manifest, + )), + Err(_) => mismatches.extend(local), + } + } + } + mismatches.extend(filter_mismatches_in_repository( + project_root, + &repository, + Path::new(""), + remaining, + manifest, + )); + mismatches +} + +fn filter_mismatches_in_repository<'a>( + project_root: &Path, + repository: &gix::Repository, + prefix: &Path, + disputed: Vec<&'a StatCandidateV1>, + manifest: &SourceContentManifestV1, +) -> Vec<&'a StatCandidateV1> { let Ok((mut pipeline, index)) = repository.filter_pipeline(None) else { return disputed; }; + let repository_root = project_root.join(prefix); disputed .into_iter() .filter(|candidate| { let Some(expected) = manifest.files.get(&candidate.logical_path) else { return true; }; + let Ok(path) = resolved_candidate_path(project_root, &candidate.logical_path) else { + return true; + }; + let Ok(rela_path) = path.strip_prefix(&repository_root) else { + return true; + }; if candidate.explicitly_admitted || index - .entry_by_path(BStr::new(candidate.logical_path.as_str())) + .entry_by_path(BStr::new(rela_path.as_os_str().as_encoded_bytes())) .is_none() { return true; } - let rela_path = Path::new(&candidate.logical_path); - let Ok(file) = std::fs::File::open(project_root.join(rela_path)) else { + let Ok(file) = std::fs::File::open(&path) else { return true; }; let Ok(mut converted) = pipeline.convert_to_git(file, rela_path, &index) else { @@ -860,30 +997,33 @@ fn clean_filter_mismatches<'a>( /// disk first, then through the repository's own filter pipeline, the same /// two readings [`clean_filter_mismatches`] accepts as current. pub fn sealed_source_bytes( - project_root: &Path, + boundaries: &tracedecay_code_index::source_walk::submodules::LiveSourceBoundaries, logical_path: &str, language: &LanguageId, sealed: &ContentDigest, ) -> Result>, CodeIndexSchedulerErrorV1> { - let raw = - ignored_dependencies::read_bounded_snapshot_source(&project_root.join(logical_path), None)?; + let Some((root, relative)) = boundaries.resolve(Path::new(logical_path))? else { + return Ok(None); + }; + let raw = ignored_dependencies::read_bounded_snapshot_source(&root.join(&relative), None)?; let (sanitized, _, _) = privacy::sanitize_code_file(language, &raw)?; if content_digest(&sanitized) == *sealed { return Ok(Some(sanitized)); } - // ponytail: one repository open and filter pipeline per filtered file; - // hoist them into a caller-held reader if a caller ever needs hundreds. - let repository = tracedecay_runtime_core::git_open::open(project_root) + let repository = tracedecay_runtime_core::git_open::open(root) .map_err(|error| CodeIndexSchedulerErrorV1::Git(error.to_string()))?; let (mut pipeline, index) = repository .filter_pipeline(None) .map_err(|error| CodeIndexSchedulerErrorV1::Git(error.to_string()))?; - if index.entry_by_path(BStr::new(logical_path)).is_none() { + if index + .entry_by_path(BStr::new(relative.as_os_str().as_encoded_bytes())) + .is_none() + { return Ok(None); } let mut filtered = Vec::new(); pipeline - .convert_to_git(raw.as_slice(), Path::new(logical_path), &index) + .convert_to_git(raw.as_slice(), relative.as_ref(), &index) .map_err(|error| CodeIndexSchedulerErrorV1::Git(error.to_string()))? .read_to_end(&mut filtered)?; let (sanitized, _, _) = privacy::sanitize_code_file(language, &filtered)?; @@ -1017,3 +1157,162 @@ mod tests { assert_eq!(first, sample(&path, sampled_at).unwrap().unwrap()); } } + +#[cfg(all(test, unix))] +mod tracked_parent_boundary_tests { + use std::path::Path; + use std::sync::Arc; + use std::time::{Duration, Instant}; + + use super::{ + CachedCandidateRosterV1, ChangeClockReading, CodeIndexIgnoredSourceAdmissionV1, + roster_evidence, source_candidates, + }; + + fn git(root: &Path, args: &[&str]) { + let program = tracedecay_runtime_core::git::try_git_program().expect("fixture Git"); + let mut command = std::process::Command::new(program); + command.current_dir(root); + for (key, _) in std::env::vars_os() { + if key.to_string_lossy().starts_with("GIT_") { + command.env_remove(key); + } + } + let output = command + .env("GIT_CONFIG_NOSYSTEM", "1") + .env("GIT_CONFIG_GLOBAL", root.join(".fixture-gitconfig")) + .args([ + "-c", + "user.name=Fixture", + "-c", + "user.email=fixture@example.invalid", + "-c", + "commit.gpgsign=false", + ]) + .args(args) + .output() + .expect("run fixture Git"); + assert!( + output.status.success(), + "Git {args:?}: {}", + String::from_utf8_lossy(&output.stderr) + ); + } + + #[test] + fn ignored_source_cannot_reenter_markerless_parent_gitlink_freshness() { + let fixture = tempfile::tempdir().expect("Git fixture"); + let parent = fixture.path().join("parent"); + let child = fixture.path().join("child"); + std::fs::create_dir_all(&parent).unwrap(); + std::fs::create_dir_all(&child).unwrap(); + + git(&child, &["init", "-q"]); + std::fs::write(child.join("child.rs"), "pub fn child() {}\n").unwrap(); + git(&child, &["add", "child.rs"]); + git(&child, &["commit", "-qm", "child"]); + + git(&parent, &["init", "-q"]); + std::fs::create_dir_all(parent.join("ignored")).unwrap(); + std::fs::write(parent.join(".gitignore"), "ignored/\n").unwrap(); + std::fs::write(parent.join("parent.rs"), "pub fn parent() {}\n").unwrap(); + std::fs::write(parent.join("ignored/ordinary.rs"), "pub fn ordinary() {}\n").unwrap(); + git(&parent, &["add", ".gitignore", "parent.rs"]); + git(&parent, &["commit", "-qm", "parent"]); + git( + &parent, + &[ + "-c", + "protocol.file.allow=always", + "submodule", + "add", + "-q", + "../child", + "app", + ], + ); + git(&parent, &["add", ".gitmodules", "app"]); + git(&parent, &["commit", "-qm", "registered submodule"]); + + // Git still records the child gitlink in HEAD after staging its + // removal. Its missing marker makes child.rs appear untracked, but + // an old explicit ignored admission must not revive it in freshness. + std::fs::remove_file(parent.join("app/.git")).unwrap(); + git(&parent, &["rm", "-q", "--cached", "app"]); + let repository = tracedecay_runtime_core::git_open::open(&parent).unwrap(); + let admitted = [ + CodeIndexIgnoredSourceAdmissionV1 { + logical_path: "app/child.rs".to_owned(), + }, + CodeIndexIgnoredSourceAdmissionV1 { + logical_path: "ignored/ordinary.rs".to_owned(), + }, + ]; + let candidates = source_candidates(&repository, &admitted, false).unwrap(); + let paths = candidates + .candidates + .iter() + .map(|candidate| candidate.logical_path.as_str()) + .collect::>(); + assert!(paths.contains("parent.rs")); + assert!(paths.contains("ignored/ordinary.rs")); + assert!( + !paths.contains("app/child.rs"), + "ignored admission must not cross a HEAD gitlink boundary" + ); + } + + #[test] + fn cached_tracked_directory_roster_invalidates_on_nested_git_marker() { + let root = tempfile::tempdir().expect("parent checkout"); + let tracked = root.path().join("src/nested/tracked.rs"); + std::fs::create_dir_all(tracked.parent().unwrap()).unwrap(); + std::fs::write(&tracked, "pub fn tracked() {}\n").unwrap(); + git(root.path(), &["init", "-q"]); + git(root.path(), &["add", "src/nested/tracked.rs"]); + git(root.path(), &["commit", "-qm", "parent source"]); + + let repository = + tracedecay_runtime_core::git_open::open(root.path()).expect("open parent Git"); + let deadline = Instant::now() + Duration::from_secs(5); + let evidence = loop { + if let Some(evidence) = + roster_evidence(&repository, root.path(), ChangeClockReading::now(), &[]) + { + break evidence; + } + assert!( + Instant::now() < deadline, + "tracked parent directories never became settleable" + ); + std::thread::sleep(Duration::from_millis(20)); + }; + let marker = root.path().join("src/nested/.git"); + assert!( + evidence + .iter() + .any(|(path, key)| path == &marker && key.is_none()), + "missing nested marker must be sampled independently of gix dirwalk" + ); + + let cached = CachedCandidateRosterV1 { + git_metadata_signature: "unchanged-parent-git".to_owned(), + admitted_paths: Vec::new(), + evidence, + candidates: Arc::new(Vec::new()), + unrepresentable: Arc::new(Vec::new()), + }; + assert!(cached.holds("unchanged-parent-git", &[])); + std::fs::write(&marker, "gitdir: ../../.git/modules/nested\n") + .expect("create nested gitdir marker"); + assert!( + !cached.holds("unchanged-parent-git", &[]), + "a new nested repository marker must invalidate the cached roster even when parent Git metadata is unchanged" + ); + std::fs::remove_file(&marker).unwrap(); + assert!( + !cached.holds("unchanged-parent-git", &[]), + "removing the marker also changes the tracked ancestor directory evidence" + ); + } +} diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/git_tree_capture.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/git_tree_capture.rs index 7db65b9c88..ef4a82f133 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/git_tree_capture.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/git_tree_capture.rs @@ -16,7 +16,9 @@ use tracedecay_domain::{ }; use tracedecay_privacy::CODE_SOURCE_SANITIZER_VERSION_V1; use tracedecay_query::code_search::CodeIndexSearchUnavailableReasonV1; -use tracedecay_runtime_core::git_repository::resolve_tree_link; +use tracedecay_runtime_core::git_repository::{ + GitNativeCandidateTreeVisitError, GitTreeSubmodulePins, resolve_tree_link, +}; use tracedecay_runtime_core::resident_memory::ResidentMemoryReservationV1; use super::{ @@ -33,6 +35,9 @@ use tracedecay_code_index::production::{ CodeIndexRepositoryParseIdentityV1, }; +type TreeBlobVisitor<'a> = + dyn FnMut(&[u8], &[u8]) -> Result<(), CodeIndexSearchUnavailableReasonV1> + 'a; + #[derive(Clone, Debug, PartialEq, Eq)] pub struct ExactGitTreeSourceV1 { pub reference: tracedecay_domain::RefId, @@ -161,7 +166,7 @@ pub(super) fn representable_logical_path(git_path: &[u8]) -> Option<&str> { /// Whether capture would parse `git_path` as source. A committed symlink at /// such a path is read through to its target, as a checkout reads it, so its /// link text never becomes source. -fn is_source_path(registry: &StaticLanguageRegistry, git_path: &[u8]) -> bool { +pub(super) fn is_source_path(registry: &StaticLanguageRegistry, git_path: &[u8]) -> bool { representable_logical_path(git_path) .and_then(|logical_path| Path::new(logical_path).extension()) .and_then(|extension| extension.to_str()) @@ -396,9 +401,8 @@ impl CodeIndexWorktreeSchedulerV1 { } /// HEAD's tree standing in for a clean checkout. Git stores a symlink as - /// its target path, while worktree capture and every source sweep read - /// through the link, so a tree holding a source symlink would seal bytes - /// the next sweep disproves. That tree is `GenerationUnavailable` here and + /// its target path, while the live roster excludes the link. A tree holding + /// a source symlink would seal bytes the next sweep disproves. That tree is `GenerationUnavailable` here and /// the checkout is captured from disk instead. pub(super) fn capture_clean_head_tree_snapshot( &self, @@ -458,8 +462,15 @@ impl CodeIndexWorktreeSchedulerV1 { source.tree.clone(), control, |visitor| { + let mut gitlinks = Vec::new(); for entry in entries { control.termination().map_or(Ok(()), Err)?; + if self.path_policy.includes_submodules() && entry.mode.is_commit() { + let path = entry.filepath.to_str().map_err(|_| { + CodeIndexSearchUnavailableReasonV1::GenerationUnavailable + })?; + gitlinks.push((path.to_owned(), entry.oid)); + } if entry.mode.is_tree() || entry.mode.is_commit() { continue; } @@ -480,11 +491,61 @@ impl CodeIndexWorktreeSchedulerV1 { } visitor(entry.filepath.as_ref(), &blob.data)?; } - Ok(()) + let pins = GitTreeSubmodulePins::from_tree(&tree, gitlinks) + .map_err(|_| CodeIndexSearchUnavailableReasonV1::GenerationUnavailable)?; + self.visit_pinned_submodule_blobs(&repository, &pins, control, visitor) }, ) } + fn visit_pinned_submodule_blobs( + &self, + parent: &gix::Repository, + pins: &GitTreeSubmodulePins, + control: &branch_generations::BranchGenerationReadControlV1, + visitor: &mut TreeBlobVisitor<'_>, + ) -> Result<(), CodeIndexSearchUnavailableReasonV1> { + let unavailable = CodeIndexSearchUnavailableReasonV1::GenerationUnavailable; + let children = + tracedecay_code_index::source_walk::submodules::pinned_submodules(parent, pins) + .map_err(|_| unavailable)?; + let registry = StaticLanguageRegistry::new(); + for (prefix, (repo, pin)) in children { + control.termination().map_or(Ok(()), Err)?; + let tree = repo + .find_commit(pin) + .and_then(|commit| commit.tree()) + .map_err(|_| unavailable)?; + for entry in tree + .traverse() + .breadthfirst + .files() + .map_err(|_| unavailable)? + { + control.termination().map_or(Ok(()), Err)?; + if !entry.mode.is_blob() { + continue; + } + let path = [prefix.as_bytes(), b"/", entry.filepath.as_slice()].concat(); + let Some(logical_path) = representable_logical_path(&path) else { + // Capture records the unrepresentable name without consulting its blob. + visitor(&path, &[])?; + continue; + }; + if !is_source_path(®istry, &path) + || !tracedecay_code_index::source_walk::submodules::is_source_location( + Path::new(logical_path), + ) + { + continue; + } + let blob = repo.find_blob(entry.oid).map_err(|_| unavailable)?; + visitor(&path, &blob.data)?; + } + } + Ok(()) + } + #[tracing::instrument( name = "daemon.code_index.capture.native_candidate_tree", level = "trace", @@ -510,14 +571,24 @@ impl CodeIndexWorktreeSchedulerV1 { expected_tree.clone(), control, |visitor| { - candidate.visit_blobs(|path| is_source_path(®istry, path), visitor).map_err(|error| match error { - tracedecay_runtime_core::git_repository::GitNativeCandidateTreeVisitError::Repository(_) => { - control + let pins = candidate + .visit_blobs( + self.path_policy.includes_submodules(), + |path| is_source_path(®istry, path), + &mut *visitor, + ) + .map_err(|error| match error { + GitNativeCandidateTreeVisitError::Repository(_) => control .termination() - .unwrap_or(CodeIndexSearchUnavailableReasonV1::GenerationUnavailable) - } - tracedecay_runtime_core::git_repository::GitNativeCandidateTreeVisitError::Visitor(reason) => reason, - }) + .unwrap_or(CodeIndexSearchUnavailableReasonV1::GenerationUnavailable), + GitNativeCandidateTreeVisitError::Visitor(reason) => reason, + })?; + if pins.gitlinks.is_empty() { + return Ok(()); + } + let parent = tracedecay_runtime_core::git_open::open(&self.project_root) + .map_err(|_| CodeIndexSearchUnavailableReasonV1::GenerationUnavailable)?; + self.visit_pinned_submodule_blobs(&parent, &pins, control, visitor) }, ) } @@ -528,9 +599,7 @@ impl CodeIndexWorktreeSchedulerV1 { source_revision: Option, tree: tracedecay_domain::TreeId, control: &branch_generations::BranchGenerationReadControlV1, - visit: impl FnOnce( - &mut dyn FnMut(&[u8], &[u8]) -> Result<(), CodeIndexSearchUnavailableReasonV1>, - ) -> Result<(), CodeIndexSearchUnavailableReasonV1>, + visit: impl FnOnce(&mut TreeBlobVisitor<'_>) -> Result<(), CodeIndexSearchUnavailableReasonV1>, ) -> Result { let registry = StaticLanguageRegistry::new(); let mut roster = CapturedFileRosterV1::default(); @@ -601,6 +670,7 @@ impl CodeIndexWorktreeSchedulerV1 { snapshot_content_identity(&files, &sanitization_receipts, &omitted_sources); Ok(CapturedSnapshotV1 { repository_parse_identity: CodeIndexRepositoryParseIdentityV1 { + include_submodules: self.path_policy.includes_submodules(), tree: Some(tree), dirty: tracedecay_domain::RepositoryDirtyStateV1::Clean, }, @@ -1326,16 +1396,41 @@ mod tests { ], "only source links that name a file in the tree are captured" ); + let live = present( + &scheduler + .capture_authoritative_snapshot_without_active_generation_reuse(None) + .expect("capture the checkout from disk") + .snapshot, + ); assert_eq!( - branch, - present( - &scheduler - .capture_authoritative_snapshot_without_active_generation_reuse(None) - .expect("capture the checkout from disk") - .snapshot - ), - "a branch capture must index the bytes a checkout reads through each link" + branch, live, + "same-parent source aliases resolve to the live checkout's bytes" ); + let changed = b"pub fn linked_target() -> u32 { 19 }\n"; + std::fs::write(root.join("src/real.rs"), changed).expect("edit only the alias target"); + let edited = present( + &scheduler + .capture_authoritative_snapshot_without_active_generation_reuse(None) + .expect("capture unstaged target through each tracked alias") + .snapshot, + ); + let expected = tracedecay_code_index::chunks::content_digest(changed); + for alias in [ + "src/alias.rs", + "src/nested/chained.rs", + "src/through_directory.rs", + ] { + assert!( + edited + .iter() + .any(|(path, digest)| path == alias && *digest == expected) + ); + assert!( + branch + .iter() + .any(|(path, digest)| path == alias && *digest != expected) + ); + } let notes = captured .snapshot .files diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/identity.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/identity.rs index 88ca93c0a4..ed1378ee97 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/identity.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/identity.rs @@ -154,6 +154,11 @@ pub struct GitMetadataFingerprintV1 { /// branch switch changes this even when mtime resolution is too coarse to /// observe, so content is compared alongside mtimes. head_contents: Option, + // Include the registered child pin, bound checkout and resolved HEAD in the same + // read-admission proof as the parent. A failed probe cannot renew that proof. + submodule_refs: Option, + submodule_indexes: Option, + submodule_probe_failed: bool, } impl GitMetadataFingerprintV1 { @@ -176,37 +181,84 @@ impl GitMetadataFingerprintV1 { packed_refs: mtime(&common_dir.join("packed-refs")), refs_heads: refs_heads_signature(&common_dir.join("refs").join("heads")), head_contents: std::fs::read_to_string(&head_path).ok(), + ..Self::default() } } + pub(super) fn for_policy(project_root: &Path, include_submodules: bool) -> Self { + let mut fingerprint = Self::capture(project_root); + if include_submodules { + let probes = (|| { + let parent = tracedecay_runtime_core::git_open::open(project_root).ok()?; + let children = + tracedecay_code_index::source_walk::submodules::live_submodules(&parent) + .ok()?; + let mut refs = Vec::new(); + let mut indexes = Vec::new(); + for (path, binding) in children { + let child = tracedecay_runtime_core::git_open::open(&binding.git_dir).ok()?; + let head = child.head_id().ok()?.detach(); + refs.push(( + path.clone(), + binding.git_dir, + binding.pin.to_string(), + head.to_string(), + )); + indexes.push((path, mtime(&child.index_path()))); + } + Some(( + tracedecay_domain::canonical_sha256(&refs).ok()?, + tracedecay_domain::canonical_sha256(&indexes).ok()?, + )) + })(); + match probes { + Some((refs, indexes)) => { + fingerprint.submodule_refs = Some(refs.as_str().to_owned()); + fingerprint.submodule_indexes = Some(indexes.as_str().to_owned()); + } + None => fingerprint.submodule_probe_failed = true, + } + } + fingerprint + } + /// Whether the sampled metadata differs from a previously captured value. pub fn differs_from(&self, other: &GitMetadataFingerprintV1) -> bool { - self != other + self.submodule_probe_failed || other.submodule_probe_failed || self != other } /// Whether HEAD and every branch ref are unchanged. The index is excluded: /// opening the repository may refresh `.git/index` on its own, while a /// commit, checkout, or ref update always moves HEAD or a ref. pub fn same_refs_as(&self, other: &GitMetadataFingerprintV1) -> bool { - self.head == other.head + !self.submodule_probe_failed + && !other.submodule_probe_failed + && self.submodule_refs == other.submodule_refs + && self.head == other.head && self.packed_refs == other.packed_refs && self.refs_heads == other.refs_heads && self.head_contents == other.head_contents } + /// Whether an authenticated submodule probe can establish a freshness witness. + pub(super) fn can_prove_freshness(&self) -> bool { + !self.submodule_probe_failed + } + /// A stable, order-independent textual signature of this fingerprint. /// /// Used to persist a restore-time freshness witness: two fingerprints share - /// a signature iff they are structurally equal (`PartialEq`), so comparing - /// the recomputed signature against a witness recorded at seal time is a - /// faithful stand-in for `differs_from`. `SystemTime` mtimes are rendered as + /// a signature iff they are structurally equal (`PartialEq`). A failed + /// submodule probe is explicitly encoded, but even matching failed probes + /// cannot prove freshness: callers must reject them, as `differs_from` does. + /// `SystemTime` mtimes are rendered as /// nanos-since-epoch so the encoding is deterministic across processes. pub fn stable_signature(&self) -> String { fn render_time(time: Option) -> String { time.and_then(|time| time.duration_since(UNIX_EPOCH).ok()) .map_or_else(|| "-".to_owned(), |elapsed| elapsed.as_nanos().to_string()) } - format!( + let mut signature = format!( "head={}|index={}|packed_refs={}|refs_heads={}|head_contents={}", render_time(self.head), render_time(self.index), @@ -216,7 +268,19 @@ impl GitMetadataFingerprintV1 { || "-".to_owned(), |value| super::sha256_hex(value.as_bytes()) ), - ) + ); + if let Some(refs) = &self.submodule_refs { + signature.push_str("|submodule_refs="); + signature.push_str(refs); + } + if let Some(indexes) = &self.submodule_indexes { + signature.push_str("|submodule_indexes="); + signature.push_str(indexes); + } + if self.submodule_probe_failed { + signature.push_str("|submodule_probe_failed=true"); + } + signature } } @@ -444,6 +508,49 @@ mod tests { ); } + #[test] + fn failed_submodule_probe_cannot_reuse_a_persisted_parent_only_witness() { + let root = TempDir::new().expect("non-repository root"); + let parent_only = GitMetadataFingerprintV1::capture(root.path()); + let failed = GitMetadataFingerprintV1::for_policy(root.path(), true); + + assert!( + failed.submodule_probe_failed, + "opt-in probe must fail closed" + ); + assert!(!failed.can_prove_freshness()); + assert!(failed.differs_from(&parent_only)); + assert_ne!( + failed.stable_signature(), + parent_only.stable_signature(), + "restore cannot mistake failed authentication for parent-only freshness" + ); + assert_eq!( + GitMetadataFingerprintV1::for_policy(root.path(), false).stable_signature(), + parent_only.stable_signature(), + "the default-off persisted witness remains stable" + ); + let failed_again = GitMetadataFingerprintV1::for_policy(root.path(), true); + assert_eq!(failed.stable_signature(), failed_again.stable_signature()); + assert!( + !failed_again.can_prove_freshness(), + "repeated matching failures cannot authorize retained reuse" + ); + } + + #[test] + fn successful_submodule_probe_has_a_distinct_stable_witness() { + let repo = init_repo(&[("src/lib.rs", "pub fn example() {}\n")]); + let disabled = GitMetadataFingerprintV1::for_policy(repo.path(), false); + let enabled = GitMetadataFingerprintV1::for_policy(repo.path(), true); + + assert!(enabled.can_prove_freshness()); + assert!(!disabled.submodule_probe_failed); + assert!(enabled.submodule_refs.is_some()); + assert!(enabled.submodule_indexes.is_some()); + assert_ne!(enabled.stable_signature(), disabled.stable_signature()); + } + #[test] fn git_metadata_fingerprint_detects_a_commit() { let repo = init_repo(&[("src/lib.rs", "pub fn a() {}\n")]); diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/ignored_dependencies.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/ignored_dependencies.rs index a5475fc590..f0d6c56fa0 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/ignored_dependencies.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/ignored_dependencies.rs @@ -14,6 +14,7 @@ use tracedecay_code_index::production::{ CodeIndexInterruptionV1, CodeIndexProductionErrorV1, MAX_IGNORED_DEPENDENCY_ENTRYPOINT_BYTES_V1, }; +use tracedecay_code_index::source_walk::parent_gitlink_paths; use tracedecay_contracts::ResolvedScope; use tracedecay_contracts::code_index_freshness::CodeIndexConvergenceParkedV1; use tracedecay_domain::{CodeGenerationId, SanitizerDispositionV1}; @@ -259,7 +260,7 @@ impl CodeIndexWorktreeSchedulerV1 { ) -> Result { let source_control = self.reconcile_control(); let began_under = source_control.expected_epoch(); - let captured_under = super::identity::GitMetadataFingerprintV1::capture(&self.project_root); + let captured_under = self.freshness_fence().git_metadata(&self.project_root); let _workers = self.ensure_worker_plan()?; let _worker_memory = match self.reserve_worker_memory() { Ok(reservation) => reservation, @@ -861,6 +862,20 @@ fn prove_gix_ignored(root: &Path, logical_path: &str) -> Result<(), CodeIndexSch { return Err(CodeIndexIgnoredDependencyRefusalV1::NotIgnored.into()); } + // Admission and live source walks share the union of staged and HEAD + // gitlinks: staging a removal cannot grant trust over the child checkout. + let parent_gitlinks = parent_gitlink_paths(root) + .map_err(|error| CodeIndexSchedulerErrorV1::Git(error.message))?; + let mut ancestor = Path::new(logical_path).parent(); + while let Some(relative) = ancestor { + if relative + .to_str() + .is_some_and(|path| parent_gitlinks.contains(path)) + { + return Err(CodeIndexIgnoredDependencyRefusalV1::NotIgnored.into()); + } + ancestor = relative.parent(); + } let mut excludes = repository .excludes( &index, @@ -933,3 +948,146 @@ fn map_reconcile_interruption_result( ) -> Result { result.map_err(map_reconcile_interruption) } + +#[cfg(test)] +mod tests { + use std::fs; + use std::path::Path; + use std::process::{Command, Output}; + + use tracedecay_runtime_core::path_safety::canonical_existing_identity; + + use super::{ + CodeIndexIgnoredDependencyRefusalV1, CodeIndexSchedulerErrorV1, prove_gix_ignored, + read_bounded_admitted_source, + }; + + fn fixture_git(root: &Path, args: &[&str]) -> Output { + let program = tracedecay_runtime_core::git::try_git_program().expect("Git program"); + let mut command = Command::new(program); + command.current_dir(root); + for (key, _) in std::env::vars_os() { + if key.to_string_lossy().starts_with("GIT_") { + command.env_remove(key); + } + } + let output = command + .env("GIT_CONFIG_NOSYSTEM", "1") + .env("GIT_CONFIG_GLOBAL", root.join(".fixture-gitconfig")) + .args([ + "-c", + "user.name=Fixture", + "-c", + "user.email=fixture@example.invalid", + "-c", + "commit.gpgsign=false", + ]) + .args(args) + .output() + .expect("run fixture Git"); + assert!( + output.status.success(), + "Git {args:?}: {}", + String::from_utf8_lossy(&output.stderr) + ); + output + } + + #[test] + fn parent_gitlink_cannot_be_admitted_as_ignored_dependency_without_gitdir_marker() { + let fixture = tempfile::tempdir().expect("repository fixture"); + let parent = fixture.path().join("parent"); + let child = fixture.path().join("child"); + fs::create_dir_all(&parent).unwrap(); + fs::create_dir_all(&child).unwrap(); + let parent = canonical_existing_identity(&parent).expect("canonical project root"); + + fixture_git(&child, &["init", "-q"]); + fs::write(child.join("index.ts"), "export type Child = number;\n").unwrap(); + fixture_git(&child, &["add", "."]); + fixture_git(&child, &["commit", "-qm", "child"]); + + fixture_git(&parent, &["init", "-q"]); + fs::write(parent.join(".gitignore"), "node_modules/\n").unwrap(); + fixture_git(&parent, &["add", ".gitignore"]); + fixture_git(&parent, &["commit", "-qm", "parent"]); + fixture_git( + &parent, + &[ + "-c", + "protocol.file.allow=always", + "submodule", + "add", + "-f", + "-q", + "../child", + "node_modules/pkg", + ], + ); + + let staged = fixture_git(&parent, &["ls-files", "--stage", "--", "node_modules/pkg"]); + assert!(staged.stdout.starts_with(b"160000 ")); + assert!(staged.stdout.ends_with(b" 0\tnode_modules/pkg\n")); + assert!(parent.join(".gitmodules").is_file()); + + fs::remove_file(parent.join("node_modules/pkg/.git")).unwrap(); + assert!(parent.join("node_modules/pkg/index.ts").is_file()); + let denied = prove_gix_ignored(&parent, "node_modules/pkg/index.ts"); + assert!(matches!( + denied, + Err(CodeIndexSchedulerErrorV1::IgnoredDependency( + CodeIndexIgnoredDependencyRefusalV1::NotIgnored + )) + )); + let admitted = read_bounded_admitted_source(&parent, "node_modules/pkg/index.ts", None); + assert!( + matches!( + admitted, + Err(CodeIndexSchedulerErrorV1::IgnoredDependency( + CodeIndexIgnoredDependencyRefusalV1::NotIgnored + )) + ), + "unexpected source admission: {admitted:?}" + ); + + // Removing the staged gitlink does not authorize the child while + // the parent HEAD still owns the repository boundary. + fixture_git(&parent, &["add", ".gitmodules", ".gitignore"]); + fixture_git(&parent, &["commit", "-qm", "parent with submodule"]); + fixture_git(&parent, &["rm", "--cached", "-f", "node_modules/pkg"]); + assert_eq!( + fixture_git(&parent, &["ls-files", "--stage", "--", "node_modules/pkg"]).stdout, + b"" + ); + assert!( + fixture_git(&parent, &["ls-tree", "HEAD", "--", "node_modules/pkg"]) + .stdout + .starts_with(b"160000 commit ") + ); + let denied = prove_gix_ignored(&parent, "node_modules/pkg/index.ts"); + assert!(matches!( + denied, + Err(CodeIndexSchedulerErrorV1::IgnoredDependency( + CodeIndexIgnoredDependencyRefusalV1::NotIgnored + )) + )); + let denied_read = read_bounded_admitted_source(&parent, "node_modules/pkg/index.ts", None); + assert!(matches!( + denied_read, + Err(CodeIndexSchedulerErrorV1::IgnoredDependency( + CodeIndexIgnoredDependencyRefusalV1::NotIgnored + )) + )); + + // Ordinary ignored sources are still eligible; only gitlink ancestry + // withdraws the source trust that the ignored-dependency proof needs. + fs::create_dir_all(parent.join("node_modules/loose")).unwrap(); + fs::write( + parent.join("node_modules/loose/index.ts"), + "export type Loose = string;\n", + ) + .unwrap(); + prove_gix_ignored(&parent, "node_modules/loose/index.ts") + .expect("non-repository ignored source remains admissible"); + } +} diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/reconcile.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/reconcile.rs index 9ca91320c7..56e5ea5a84 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/reconcile.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/reconcile.rs @@ -567,6 +567,7 @@ pub(crate) struct SourceFreshnessFenceV1 { source_epoch: Arc, /// Held only by source sweeps, never across a scheduler pass. sweep_cache: Arc>, + include_submodules: bool, pending_source: Option>, } @@ -590,6 +591,10 @@ pub(super) struct SourceFreshnessFenceStateV1 { } impl SourceFreshnessFenceV1 { + pub(super) fn git_metadata(&self, root: &Path) -> identity::GitMetadataFingerprintV1 { + identity::GitMetadataFingerprintV1::for_policy(root, self.include_submodules) + } + fn unverified(source_epoch: Arc, path_policy: IndexPathPolicyV1) -> Self { Self { state: Arc::new(Mutex::new(SourceFreshnessFenceStateV1 { @@ -604,6 +609,7 @@ impl SourceFreshnessFenceV1 { })), last_reconciled_at_micros: Arc::new(AtomicI64::new(0)), source_epoch, + include_submodules: path_policy.includes_submodules(), sweep_cache: Arc::new(Mutex::new(SourceSweepCacheV1::new(path_policy))), pending_source: None, } @@ -657,7 +663,7 @@ impl SourceFreshnessFenceV1 { state.source_witness = source_witness; state.source_roster = source_roster.to_vec(); state.last_reconciled_at = Instant::now(); - state.verified_against_source = true; + state.verified_against_source = state.git_metadata.can_prove_freshness(); state.reconciled_without_generation = reconciled_without_generation; state.reconciled_source_epoch = source_epoch; state.successor_build_in_flight = false; @@ -723,7 +729,8 @@ impl SourceFreshnessFenceV1 { pub(super) fn verified_git_metadata_moved(&self, project_root: &Path) -> bool { let state = self.snapshot(); state.verified_against_source - && identity::GitMetadataFingerprintV1::capture(project_root) + && self + .git_metadata(project_root) .differs_from(&state.git_metadata) } @@ -760,7 +767,8 @@ impl SourceFreshnessFenceV1 { } state.verified_against_source && self.source_epoch.load(Ordering::Acquire) == state.reconciled_source_epoch - && !identity::GitMetadataFingerprintV1::capture(project_root) + && !self + .git_metadata(project_root) .differs_from(&state.git_metadata) } @@ -837,7 +845,7 @@ impl SourceFreshnessFenceV1 { let mut state = self.state.lock().unwrap_or_else(PoisonError::into_inner); state.git_metadata = git_metadata; state.last_reconciled_at = Instant::now(); - state.verified_against_source = true; + state.verified_against_source = state.git_metadata.can_prove_freshness(); self.last_reconciled_at_micros .store(micros, Ordering::Release); } @@ -875,6 +883,10 @@ impl SourceFreshnessFenceV1 { manifest: &SourceContentManifestV1, shutting_down: &AtomicBool, ) -> (bool, freshness_witness::SourceSweepStatsV1) { + // Equal source bytes cannot authenticate a previously selected child checkout. + if !git_metadata.can_prove_freshness() { + return (false, freshness_witness::SourceSweepStatsV1::default()); + } self.sweep_cache .lock() .unwrap_or_else(PoisonError::into_inner) @@ -893,12 +905,13 @@ impl SourceFreshnessFenceV1 { &self, project_root: &Path, manifest: &SourceContentManifestV1, + candidate_paths: &BTreeSet, shutting_down: &AtomicBool, ) -> Result, CodeIndexSchedulerErrorV1> { self.sweep_cache .lock() .unwrap_or_else(PoisonError::into_inner) - .disputed_sources(project_root, manifest, shutting_down) + .disputed_sources(project_root, manifest, candidate_paths, shutting_down) } /// Sweep the current witness and report what the sweep had to read. @@ -916,7 +929,7 @@ impl SourceFreshnessFenceV1 { self.sweep_sources( project_root, &freshness.source_roster, - &identity::GitMetadataFingerprintV1::capture(project_root), + &self.git_metadata(project_root), &witness.content_manifest, shutting_down, ) @@ -937,7 +950,7 @@ impl SourceFreshnessFenceV1 { if !freshness.verified_against_source { return FreshnessProbeVerdictV1::Unverified; } - let git_metadata = identity::GitMetadataFingerprintV1::capture(project_root); + let git_metadata = self.git_metadata(project_root); if git_metadata.differs_from(&freshness.git_metadata) { return FreshnessProbeVerdictV1::Moved; } @@ -1336,6 +1349,7 @@ impl CodeIndexWorktreeSchedulerV1 { .with_shutdown_signal(Arc::clone(&shutting_down)) .with_decoded_content(byte_pool.decoded_content.clone()); let production_config = CodeIndexProductionConfigV1 { + include_submodules: path_policy.includes_submodules(), project_id: project_id.clone(), repository: repository_id.clone(), sanitizer_revision, @@ -1996,8 +2010,10 @@ impl CodeIndexWorktreeSchedulerV1 { { return Ok(None); } - let metadata = identity::GitMetadataFingerprintV1::capture(&self.project_root); - if witness.git_metadata_signature != metadata.stable_signature() { + let metadata = self.freshness_fence.git_metadata(&self.project_root); + if !metadata.can_prove_freshness() + || witness.git_metadata_signature != metadata.stable_signature() + { return Ok(None); } // Re-swept under the adopted ignored-source roster. Equal metadata is @@ -2052,8 +2068,10 @@ impl CodeIndexWorktreeSchedulerV1 { if witness.generation_id != pointer.generation_id { return false; } - let metadata = identity::GitMetadataFingerprintV1::capture(&self.project_root); - if witness.git_metadata_signature != metadata.stable_signature() { + let metadata = self.freshness_fence.git_metadata(&self.project_root); + if !metadata.can_prove_freshness() + || witness.git_metadata_signature != metadata.stable_signature() + { return false; } self.worktree_stat_sweep() @@ -2234,7 +2252,7 @@ impl CodeIndexWorktreeSchedulerV1 { )); } self.identity = resolved; - let sampled_metadata = identity::GitMetadataFingerprintV1::capture(&self.project_root); + let sampled_metadata = self.freshness_fence.git_metadata(&self.project_root); let Some(sampled_sweep) = self.worktree_stat_sweep().ok() else { return Ok(None); }; @@ -2249,8 +2267,10 @@ impl CodeIndexWorktreeSchedulerV1 { // generation's sealed file digests; its matching stat signature is // the negative cache that lets a moved tree skip the byte comparison. let source_manifest = SourceContentManifestV1::for_snapshot(metadata.snapshot()); - let sealed_bytes_match = - retained_is_reusable && !has_hints && self.sources_match_manifest(&source_manifest); + let sealed_bytes_match = retained_is_reusable + && !has_hints + && sampled_metadata.can_prove_freshness() + && self.sources_match_manifest(&source_manifest); #[cfg(test)] if let Some(hook) = self.after_retained_source_check.clone() { hook(); @@ -2343,7 +2363,7 @@ impl CodeIndexWorktreeSchedulerV1 { let captured = self .capture_authoritative_snapshot_without_active_generation_reuse(Some(&control)) .map_err(|error| prefer_resident_memory_refusal(&control, error))?; - let git_metadata = identity::GitMetadataFingerprintV1::capture(&self.project_root); + let git_metadata = self.freshness_fence.git_metadata(&self.project_root); let stat_signature = self.worktree_stat_signature().ok(); let drained_hints = { let mut hints = self @@ -2927,7 +2947,7 @@ impl CodeIndexWorktreeSchedulerV1 { // bound to; a HEAD move under the same worktree is allowed and simply // records a new source revision, so the served generation is never // mis-attributed across identities. - let captured_under = identity::GitMetadataFingerprintV1::capture(&self.project_root); + let captured_under = self.freshness_fence.git_metadata(&self.project_root); let resolved = identity::IndexingIdentityV1::resolve(&self.project_root) .map_err(|error| CodeIndexSchedulerErrorV1::Identity(error.to_string()))?; if !resolved.authorizes_reuse_of(&self.identity) { @@ -3199,7 +3219,7 @@ impl CodeIndexWorktreeSchedulerV1 { source_epoch: u64, captured_under: &identity::GitMetadataFingerprintV1, ) { - let sampled = identity::GitMetadataFingerprintV1::capture(&self.project_root); + let sampled = self.freshness_fence.git_metadata(&self.project_root); let metadata = if sampled.same_refs_as(captured_under) { sampled } else { @@ -3318,7 +3338,9 @@ impl CodeIndexWorktreeSchedulerV1 { } let freshness = self.freshness_fence.snapshot(); if !freshness.verified_against_source - || identity::GitMetadataFingerprintV1::capture(&self.project_root) + || self + .freshness_fence + .git_metadata(&self.project_root) .differs_from(&freshness.git_metadata) || freshness.last_reconciled_at.elapsed() >= self.policy.staleness_threshold { @@ -3338,7 +3360,9 @@ impl CodeIndexWorktreeSchedulerV1 { } let freshness = self.freshness_fence.snapshot(); if !freshness.verified_against_source - || identity::GitMetadataFingerprintV1::capture(&self.project_root) + || self + .freshness_fence + .git_metadata(&self.project_root) .differs_from(&freshness.git_metadata) { self.request_background_reconcile(); @@ -3364,7 +3388,7 @@ impl CodeIndexWorktreeSchedulerV1 { .sweep_sources( &self.project_root, &self.ignored_source_admissions, - &identity::GitMetadataFingerprintV1::capture(&self.project_root), + &self.freshness_fence.git_metadata(&self.project_root), manifest, &self.shutting_down, ) @@ -3376,7 +3400,7 @@ impl CodeIndexWorktreeSchedulerV1 { fn source_witness_matches_worktree(&self, freshness: &SourceFreshnessFenceStateV1) -> bool { self.freshness_fence.source_witness_matches( freshness, - &identity::GitMetadataFingerprintV1::capture(&self.project_root), + &self.freshness_fence.git_metadata(&self.project_root), &self.project_root, &self.shutting_down, ) @@ -3432,7 +3456,7 @@ impl CodeIndexWorktreeSchedulerV1 { // Sample after the walk. `git_open::open` inside the digest comparison can // move index metadata; storing the post-walk sample is what keeps the // next probe from calling that side effect a new generation. - let git_metadata = identity::GitMetadataFingerprintV1::capture(&self.project_root); + let git_metadata = self.freshness_fence.git_metadata(&self.project_root); self.freshness_fence.rebind_admission_clock(git_metadata); self.freshness_fence .source_currency_witness_for(generation_id, snapshot_content_identity) @@ -3450,7 +3474,11 @@ impl CodeIndexWorktreeSchedulerV1 { fn worktree_stat_sweep( &self, ) -> Result { - freshness_witness::worktree_stat_sweep(&self.project_root, &self.ignored_source_admissions) + freshness_witness::worktree_stat_sweep( + &self.project_root, + &self.ignored_source_admissions, + self.path_policy.includes_submodules(), + ) } /// Deliver debounced hook hints (exact touched paths) into the incremental @@ -3495,7 +3523,9 @@ impl CodeIndexWorktreeSchedulerV1 { // suppressing cadence on open-time clocks is not. return Ok(Some(self.reconcile_now()?)); } - let git_changed = identity::GitMetadataFingerprintV1::capture(&self.project_root) + let git_changed = self + .freshness_fence + .git_metadata(&self.project_root) .differs_from(&freshness.git_metadata); if git_changed { // Tier 1: a git-mediated mutation is authoritative evidence; reconcile. @@ -3957,7 +3987,28 @@ impl CodeIndexWorktreeSchedulerV1 { .ignored_source_admissions .iter() .any(|admission| admission.logical_path == logical_path); - self.capture_admitted_candidate(registry, logical_path, control, explicitly_admitted) + let child_root = if self.path_policy.includes_submodules() && !explicitly_admitted { + let repository = tracedecay_runtime_core::git_open::open(&self.project_root) + .map_err(|error| CodeIndexSchedulerErrorV1::Git(error.to_string()))?; + let classification = + classification::WorktreeChangeClassificationV1::for_policy(&repository, true) + .map_err(|error| CodeIndexSchedulerErrorV1::Git(error.to_string()))?; + if classification.beneath_gitlink(Path::new(logical_path)) + && classification.child_root(logical_path).is_none() + { + return Ok(CapturedFileOutcomeV1::Absent); + } + classification.child_root(logical_path).map(str::to_owned) + } else { + None + }; + self.capture_admitted_candidate( + registry, + logical_path, + control, + explicitly_admitted, + child_root.as_deref(), + ) } fn ignored_admission_paths(&self) -> BTreeSet<&str> { @@ -3973,8 +4024,55 @@ impl CodeIndexWorktreeSchedulerV1 { logical_path: &str, control: Option<&dyn CodeIndexExecutionControlV1>, explicitly_admitted: bool, + child_root: Option<&str>, ) -> Result { - let absolute = self.project_root.join(logical_path); + // Explicit ignored-source admission cannot cross a nested checkout + // any more than an ordinary Git candidate or interactive source walk. + let relative = Path::new(logical_path); + let absolute = self.project_root.join(relative); + let (source_root, source_relative) = match child_root { + Some(prefix) if !explicitly_admitted => { + if !git_tree_capture::is_source_path(registry, logical_path.as_bytes()) { + return Ok(CapturedFileOutcomeV1::Absent); + } + let root = self.project_root.join(prefix); + if root.canonicalize().ok().as_ref() != Some(&root) + || !absolute + .symlink_metadata() + .is_ok_and(|metadata| metadata.is_file()) + { + return Ok(CapturedFileOutcomeV1::Absent); + } + let relative = relative.strip_prefix(prefix).map_err(|_| { + CodeIndexSchedulerErrorV1::Git("invalid bound submodule path".into()) + })?; + (root, relative) + } + _ => (self.project_root.clone(), relative), + }; + let resolved = if tracedecay_code_index::source_walk::has_nested_repository_ancestor( + &source_root, + source_relative, + )? { + Err(CodeIndexIgnoredDependencyRefusalV1::NotIgnored) + } else if explicitly_admitted + && tracedecay_code_index::source_walk::is_source_symlink(&source_root, source_relative)? + { + Err(CodeIndexIgnoredDependencyRefusalV1::SymlinkEscape) + } else { + tracedecay_code_index::source_walk::resolve_parent_source( + &source_root, + source_relative, + )? + .ok_or(CodeIndexIgnoredDependencyRefusalV1::SymlinkEscape) + }; + let resolved = match resolved { + Ok(path) => path, + // Retained admissions name required bytes. A refusal must remain visible. + Err(reason) if explicitly_admitted => return Err(reason.into()), + Err(_) => return Ok(CapturedFileOutcomeV1::Absent), + }; + let absolute = source_root.join(resolved); match absolute.metadata() { Ok(metadata) if !metadata.is_file() => return Ok(CapturedFileOutcomeV1::Absent), Ok(_) => {} @@ -4036,8 +4134,11 @@ impl CodeIndexWorktreeSchedulerV1 { // Classify committed/staged/unstaged/untracked/deleted/renamed paths // truthfully from gix. Deletions drop out of the present candidate set; // their tombstones flow through `changed_paths`. - let classification = classification::WorktreeChangeClassificationV1::classify(&repository) - .map_err(|error| CodeIndexSchedulerErrorV1::Git(error.to_string()))?; + let classification = classification::WorktreeChangeClassificationV1::for_policy( + &repository, + self.path_policy.includes_submodules(), + ) + .map_err(|error| CodeIndexSchedulerErrorV1::Git(error.to_string()))?; if self.shutting_down.load(Ordering::Acquire) { return Err(cancelled_code_index_reconcile()); } @@ -4114,6 +4215,7 @@ impl CodeIndexWorktreeSchedulerV1 { // assume-unchanged. A sealed row is reused only while its bytes still // carry its digest; every row they disprove beyond the visible commit // and status deltas is a worktree change stat could not see. + let mut candidate_paths = classification.candidate_paths(); let stat_blind_paths = match reusable_active.as_ref() { Some(active) => { let visible = classification.changed_paths(); @@ -4121,6 +4223,7 @@ impl CodeIndexWorktreeSchedulerV1 { .disputed_sources( &self.project_root, &SourceContentManifestV1::for_snapshot(active.snapshot()), + &candidate_paths, &self.shutting_down, )? .into_iter() @@ -4181,7 +4284,6 @@ impl CodeIndexWorktreeSchedulerV1 { let source_revision = (self.ignored_source_admissions.is_empty() && !has_changes) .then(|| self.identity.head_commit().cloned()) .flatten(); - let mut candidate_paths = classification.candidate_paths(); let mut changed_paths = classification.changed_paths(); changed_paths.extend(tree_delta); changed_paths.extend(stat_blind_paths); @@ -4262,6 +4364,7 @@ impl CodeIndexWorktreeSchedulerV1 { logical_path, control, admitted_paths.contains(logical_path.as_str()), + classification.child_root(logical_path), ) }) }) @@ -4352,6 +4455,7 @@ impl CodeIndexWorktreeSchedulerV1 { snapshot_content_identity(&files, &sanitization_receipts, &omitted_sources); let captured = CapturedSnapshotV1 { repository_parse_identity: CodeIndexRepositoryParseIdentityV1 { + include_submodules: self.path_policy.includes_submodules(), tree: self.identity.head_tree().cloned(), dirty, }, diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/index_path_policy_tests.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/index_path_policy_tests.rs index 06cf0bd3f3..9ec5037f42 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/index_path_policy_tests.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/index_path_policy_tests.rs @@ -191,3 +191,300 @@ async fn hook_hints_for_excluded_paths_queue_no_work() { ); registry.shutdown().await; } + +#[test] +fn submodule_policy_binds_live_graph_and_historical_child_pins_across_restart() { + use super::{git_stdout, run_git_in}; + use crate::code_index_scheduler::{ + branch_generations::BranchGenerationReadControlV1, git_tree_capture::ExactGitTreeSourceV1, + }; + use tracedecay_code_index::grep_search::{GrepSearchQuery, search_tree_with_cancel}; + use tracedecay_domain::{CommitId, RefId, RelationEdgeKindV1, RepositoryDirtyStateV1, TreeId}; + + let parent = GitFixture::new(&[( + "dev/launch.ts", + "import { feature } from '../app/lib';\nexport function launch() { return feature(); }\n", + )]); + let child = GitFixture::new(&[ + ( + "lib.ts", + "export function feature() { return 'pinned_symbol'; }\n", + ), + ("gone.ts", "export function gone() {}\n"), + ("media.bin", "local binary fixture"), + (".gitignore", "*.private.ts\n"), + ]); + run_git_in( + parent.path(), + &[ + "-c", + "protocol.file.allow=always", + "submodule", + "add", + "-q", + child.path().to_str().unwrap(), + "app", + ], + ); + run_git_in(parent.path(), &["commit", "-qm", "bind application"]); + run_git_in(parent.path(), &["branch", "pinned"]); + let pinned = ExactGitTreeSourceV1 { + reference: RefId::new("refs/heads/pinned").unwrap(), + revision: CommitId::new(git_stdout(parent.path(), &["rev-parse", "HEAD"])).unwrap(), + tree: TreeId::new(git_stdout(parent.path(), &["rev-parse", "HEAD^{tree}"])).unwrap(), + }; + let store = TempDir::new().unwrap(); + let enabled = policy(&[], &[]).with_submodules(true); + let grep = |needle: &str, policy: &IndexPathPolicyV1| { + search_tree_with_cancel( + parent.path(), + &GrepSearchQuery { + pattern: needle.to_owned(), + fixed_strings: true, + case_sensitive: true, + path_glob: Some("app/**".into()), + scope_prefix: None, + context_lines: 0, + max_results: 8, + }, + policy, + || false, + ) + .unwrap() + .hits + .into_iter() + .map(|hit| hit.file.to_string()) + .collect::>() + }; + let child_path = std::path::Path::new("app/lib.ts"); + let direct_reads = |enabled| { + tracedecay_code_index::source_walk::submodules::LiveSourceBoundaries::open( + parent.path(), + enabled, + ) + .unwrap() + }; + assert!(direct_reads(false).resolve(child_path).unwrap().is_none()); + let active_reads = direct_reads(true); + assert!(active_reads.resolve(child_path).unwrap().is_some()); + let mut off = start(&parent, store.path(), policy(&[], &[])); + off.reconcile_now().unwrap(); + assert_eq!(indexed_paths(&off), ["dev/launch.ts"]); + assert_eq!( + grep("pinned_symbol", &policy(&[], &[])), + Vec::::new() + ); + drop(off); + + let mut on = start(&parent, store.path(), enabled.clone()); + on.reconcile_now().unwrap(); + assert_eq!( + indexed_paths(&on), + ["app/gone.ts", "app/lib.ts", "dev/launch.ts"] + ); + assert_eq!(grep("pinned_symbol", &enabled), ["app/lib.ts"]); + let assert_edge = + |generation: &tracedecay_code_index::production::CodeIndexPublishedGenerationV1| { + let symbol = |name: &str| { + generation + .symbols() + .symbols + .iter() + .find(|item| item.simple_name == name) + .unwrap() + .occurrence + .clone() + }; + assert!( + generation + .edges() + .iter() + .any(|edge| edge.kind == RelationEdgeKindV1::Calls + && edge.from_occurrence == symbol("launch") + && edge.to_occurrence == symbol("feature")) + ); + }; + assert_edge(on.latest_complete().unwrap().generation()); + let checkout = parent.path().join("app"); + std::fs::write( + checkout.join("ignored.private.ts"), + "const private_marker = 1;", + ) + .unwrap(); + assert_eq!(grep("private_marker", &enabled), Vec::::new()); + std::fs::write( + checkout.join("lib.ts"), + "export function feature() { return 'successor_symbol'; }\n", + ) + .unwrap(); + std::fs::remove_file(checkout.join("gone.ts")).unwrap(); + run_git_in(&checkout, &["add", "-A"]); + run_git_in( + &checkout, + &["commit", "-qm", "move child HEAD independently"], + ); + std::fs::write( + checkout.join("pending.ts"), + "export function pending() {}\n", + ) + .unwrap(); + assert!(on.ensure_fresh_for_query().unwrap().is_some()); + assert_eq!( + indexed_paths(&on), + ["app/lib.ts", "app/pending.ts", "dev/launch.ts"] + ); + assert_eq!( + on.latest_complete() + .unwrap() + .generation() + .repository_parse_identity() + .dirty, + RepositoryDirtyStateV1::Dirty + ); + assert_eq!(grep("successor_symbol", &enabled), ["app/lib.ts"]); + run_git_in(parent.path(), &["submodule", "deinit", "-f", "app"]); + assert!(!checkout.join(".git").exists()); + assert!(active_reads.resolve(child_path).unwrap().is_none()); + let historical = on + .publish_exact_git_tree_generation( + &pinned, + &BranchGenerationReadControlV1 { + deadline: None, + cancellation: None, + }, + ) + .unwrap(); + let historical = historical.generation(); + assert_edge(historical); + assert!( + !historical + .snapshot() + .files + .iter() + .any(|file| file.logical_path == "app/media.bin") + ); + assert!( + historical + .snapshot() + .files + .iter() + .any(|file| file.logical_path == "app/gone.ts") + ); + assert!( + !historical + .snapshot() + .files + .iter() + .any(|file| file.logical_path == "app/pending.ts") + ); + let pinned_file = historical + .snapshot() + .files + .iter() + .find(|file| file.logical_path == "app/lib.ts") + .unwrap(); + assert_eq!( + pinned_file.content_digest, + tracedecay_code_index::chunks::content_digest( + b"export function feature() { return 'pinned_symbol'; }\n" + ) + ); + drop(on); + let mut off = start(&parent, store.path(), policy(&[], &[])); + off.reconcile_now().unwrap(); + assert_eq!(indexed_paths(&off), ["dev/launch.ts"]); +} + +#[test] +fn parent_filtered_source_does_not_require_submodule_registration_when_disabled() { + use super::run_git_in; + use tracedecay_code_index::chunks::content_digest; + use tracedecay_domain::LanguageId; + + let source = "pub fn parent() {}\n"; + let parent = GitFixture::new(&[ + ("lib.rs", source), + (".gitmodules", "[submodule \"child\"]\npath = child\n"), + ]); + run_git_in(parent.path(), &["config", "core.autocrlf", "true"]); + std::fs::write(parent.path().join("lib.rs"), source.replace('\n', "\r\n")).unwrap(); + std::fs::write(parent.path().join(".gitmodules"), "[unfinished").unwrap(); + let boundaries = tracedecay_code_index::source_walk::submodules::LiveSourceBoundaries::open( + parent.path(), + false, + ) + .unwrap(); + let sealed = super::super::sealed_source_bytes( + &boundaries, + "lib.rs", + &LanguageId::new("rust").unwrap(), + &content_digest(source.as_bytes()), + ) + .unwrap(); + assert_eq!(sealed.as_deref(), Some(source.as_bytes())); +} + +#[test] +fn retained_graph_checks_require_a_successful_submodule_probe_each_time() { + let registration = "[submodule \"child\"]\npath = child\n"; + let parent = GitFixture::new(&[ + ("lib.rs", "pub fn first() {}\n"), + (".gitmodules", registration), + ]); + let store = TempDir::new().unwrap(); + // Keep the sealed attribution worktree-relative across the configuration edit. + std::fs::write(parent.path().join("lib.rs"), "pub fn latest() {}\n").unwrap(); + let mut scheduler = start( + &parent, + store.path(), + policy(&[], &[]).with_submodules(true), + ); + published(scheduler.reconcile_now().unwrap()); + let metadata = scheduler + .publication + .load_active_manifest() + .unwrap() + .unwrap() + .metadata; + // A partially staged edit leaves the worktree and its source bytes readable, + // while the index registration still cannot establish submodule identity. + let registration_path = parent.path().join(".gitmodules"); + std::fs::write(®istration_path, "[unfinished").unwrap(); + super::run_git_in(parent.path(), &["add", ".gitmodules"]); + std::fs::write(®istration_path, registration).unwrap(); + for _ in 0..2 { + assert!( + !scheduler + .freshness_fence() + .git_metadata(parent.path()) + .can_prove_freshness() + ); + let serving_witness = scheduler.currency_witness_for_sealed_snapshot( + &metadata.manifest().generation_id, + &metadata.snapshot().content_identity, + ); + let retained = scheduler + .reconcile_retained_text_generation_with(&metadata, false) + .unwrap(); + assert_eq!( + (serving_witness.is_some(), retained.is_some()), + (false, false), + "unchanged source bytes cannot replace a failed submodule probe" + ); + } + super::run_git_in(parent.path(), &["add", ".gitmodules"]); + assert!(matches!( + scheduler + .reconcile_retained_text_generation_with(&metadata, false) + .unwrap(), + Some(CodeIndexReconcileOutcomeV1::Noop(_)) + )); + assert!( + scheduler + .currency_witness_for_sealed_snapshot( + &metadata.manifest().generation_id, + &metadata.snapshot().content_identity, + ) + .is_some() + ); +} diff --git a/crates/tracedecay-code-index-runtime/tests/code_index_scheduler/reconcile.rs b/crates/tracedecay-code-index-runtime/tests/code_index_scheduler/reconcile.rs index 35ca13c8c1..318dc166f9 100644 --- a/crates/tracedecay-code-index-runtime/tests/code_index_scheduler/reconcile.rs +++ b/crates/tracedecay-code-index-runtime/tests/code_index_scheduler/reconcile.rs @@ -825,10 +825,8 @@ fn unfollowable_committed_source_links_are_absent_from_disk_capture() { ); } -/// Git stores a committed source symlink as its target path, while the -/// worktree reads through it. Sealing the link text from HEAD's tree left the -/// first post-publication sweep disproving an unchanged checkout, so every -/// mount rebuilt once and served the rebuild as dirty. +/// Publishing the link target bytes under both tracked paths must preserve +/// the clean HEAD revision and avoid a redundant post-publication rebuild. #[cfg(unix)] #[test] fn committed_source_symlink_publishes_a_current_clean_generation() { @@ -851,16 +849,19 @@ fn committed_source_symlink_publishes_a_current_clean_generation() { !scheduler.request_fresh_now_background(), "the sweep after publication must find the unchanged checkout current" ); + let latest = scheduler.latest_complete().expect("published generation"); + let snapshot = latest.generation.snapshot(); assert_eq!( - scheduler - .latest_complete() - .expect("published generation") - .generation - .snapshot() - .source_revision - .as_ref(), - scheduler.identity.head_commit(), - "a clean checkout's generation must carry HEAD's revision" + snapshot.source_revision.as_ref(), + scheduler.identity.head_commit() + ); + assert_eq!( + snapshot + .files + .iter() + .map(|file| file.logical_path.as_str()) + .collect::>(), + ["src/lib.rs", "src/pricing.rs", "src/pricing_alias.rs"] ); } diff --git a/crates/tracedecay-code-index/Cargo.toml b/crates/tracedecay-code-index/Cargo.toml index 5404891c52..cba8706cb2 100644 --- a/crates/tracedecay-code-index/Cargo.toml +++ b/crates/tracedecay-code-index/Cargo.toml @@ -66,6 +66,7 @@ lang-lean = ["tracedecay-code-extraction/lang-lean"] [dependencies] ast-grep-core = "0.50" flate2.workspace = true +gix.workspace = true miniz_oxide.workspace = true base64.workspace = true hex.workspace = true diff --git a/crates/tracedecay-code-index/benches/partitioned_codec.rs b/crates/tracedecay-code-index/benches/partitioned_codec.rs index 12a962fddf..c9330ca623 100644 --- a/crates/tracedecay-code-index/benches/partitioned_codec.rs +++ b/crates/tracedecay-code-index/benches/partitioned_codec.rs @@ -286,6 +286,7 @@ fn build_generation( invalidations: BTreeSet::new(), ignored_source_admissions: Vec::new(), repository_parse_identity: CodeIndexRepositoryParseIdentityV1 { + include_submodules: false, tree: None, dirty: RepositoryDirtyStateV1::Dirty, }, @@ -315,6 +316,7 @@ fn build_generation( fn production_config() -> Result> { Ok(CodeIndexProductionConfigV1 { + include_submodules: false, project_id: id::("project.partitioned-codec")?, repository: id::("repository.partitioned-codec")?, sanitizer_revision: id::("sanitizer.partitioned-codec.v1")?, diff --git a/crates/tracedecay-code-index/benches/sealed_storage.rs b/crates/tracedecay-code-index/benches/sealed_storage.rs index f3cc2da231..7e927b88c2 100644 --- a/crates/tracedecay-code-index/benches/sealed_storage.rs +++ b/crates/tracedecay-code-index/benches/sealed_storage.rs @@ -468,6 +468,7 @@ fn request( invalidations: BTreeSet::new(), ignored_source_admissions: Vec::new(), repository_parse_identity: CodeIndexRepositoryParseIdentityV1 { + include_submodules: false, tree: Some(id::(tree)?), dirty: RepositoryDirtyStateV1::Clean, }, @@ -482,6 +483,7 @@ fn request( fn config() -> Result> { Ok(CodeIndexProductionConfigV1 { + include_submodules: false, project_id: id::("project.sealed-storage")?, repository: id::("repository.sealed-storage")?, sanitizer_revision: id::("sanitizer.sealed-storage.v1")?, diff --git a/crates/tracedecay-code-index/src/production/ignored_sources.rs b/crates/tracedecay-code-index/src/production/ignored_sources.rs index 2a103a5303..beafc93a38 100644 --- a/crates/tracedecay-code-index/src/production/ignored_sources.rs +++ b/crates/tracedecay-code-index/src/production/ignored_sources.rs @@ -53,6 +53,9 @@ pub struct CodeIndexBuildRequestV1 { #[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] #[serde(deny_unknown_fields)] pub struct CodeIndexRepositoryParseIdentityV1 { + /// Older sealed generations used the default parent-only roster. + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + pub include_submodules: bool, pub tree: Option, pub dirty: RepositoryDirtyStateV1, } diff --git a/crates/tracedecay-code-index/src/production/lexical_page_source.rs b/crates/tracedecay-code-index/src/production/lexical_page_source.rs index 76497c639a..df469bda13 100644 --- a/crates/tracedecay-code-index/src/production/lexical_page_source.rs +++ b/crates/tracedecay-code-index/src/production/lexical_page_source.rs @@ -1332,6 +1332,7 @@ impl VerifiedSealedTextGenerationMetadataV1 { &self.snapshot, config, ); + compatibility.observe_submodule_policy(self.repository_parse_identity(), config); self.chunk_policy.observe(config, &mut compatibility); compatibility } diff --git a/crates/tracedecay-code-index/src/production/lexical_page_source_tests.rs b/crates/tracedecay-code-index/src/production/lexical_page_source_tests.rs index 6ce3129396..6b2276f9b9 100644 --- a/crates/tracedecay-code-index/src/production/lexical_page_source_tests.rs +++ b/crates/tracedecay-code-index/src/production/lexical_page_source_tests.rs @@ -611,6 +611,7 @@ fn fixture_for_source_files( invalidations: BTreeSet::new(), ignored_source_admissions: Vec::new(), repository_parse_identity: CodeIndexRepositoryParseIdentityV1 { + include_submodules: false, tree: None, dirty: RepositoryDirtyStateV1::Dirty, }, @@ -624,6 +625,7 @@ fn fixture_for_source_files( }; let mut owner = CodeIndexProductionOwnerV1::new( CodeIndexProductionConfigV1 { + include_submodules: false, project_id: ProjectId::new("project.lexical-page-batch").expect("fixture project ID"), repository: RepositoryId::new("repository.lexical-page-batch") .expect("fixture repository ID"), diff --git a/crates/tracedecay-code-index/src/production/mod.rs b/crates/tracedecay-code-index/src/production/mod.rs index b4b48db5ac..af485652be 100644 --- a/crates/tracedecay-code-index/src/production/mod.rs +++ b/crates/tracedecay-code-index/src/production/mod.rs @@ -151,6 +151,7 @@ pub const DAEMON_CODE_INDEX_CHUNKER_REVISION: &str = "chunker.daemon.v5"; /// Immutable configuration retained by one production index owner. #[derive(Clone, Debug)] pub struct CodeIndexProductionConfigV1 { + pub include_submodules: bool, pub project_id: ProjectId, pub repository: RepositoryId, pub sanitizer_revision: SanitizerRevision, @@ -192,6 +193,7 @@ pub enum CodeIndexGenerationIncompatibilityV1 { SanitizerRevision, PolicyRevision, MixedPolicyRevisions, + SubmodulePolicy, ChunkerRevision, PrivacyDomain, PrivacyKeyEpoch, @@ -203,6 +205,7 @@ impl CodeIndexGenerationIncompatibilityV1 { Self::Project => "project", Self::LanguageRevisions => "language_revisions", Self::SanitizerRevision => "sanitizer_revision", + Self::SubmodulePolicy => "submodule_policy", Self::PolicyRevision => "policy_revision", Self::MixedPolicyRevisions => "mixed_policy_revisions", Self::ChunkerRevision => "chunker_revision", @@ -226,6 +229,17 @@ pub struct CodeIndexGenerationCompatibilityV1 { } impl CodeIndexGenerationCompatibilityV1 { + fn observe_submodule_policy( + &mut self, + identity: &CodeIndexRepositoryParseIdentityV1, + config: &CodeIndexProductionConfigV1, + ) { + if identity.include_submodules != config.include_submodules { + self.incompatibilities + .insert(CodeIndexGenerationIncompatibilityV1::SubmodulePolicy); + } + } + fn for_metadata( manifest: &CodeGenerationManifestV1, snapshot: &SanitizedCodeSnapshotV1, @@ -1456,6 +1470,7 @@ impl CodeIndexPublishedGenerationV1 { &self.snapshot, config, ); + compatibility.observe_submodule_policy(self.repository_parse_identity(), config); self.chunk_policy_summary() .observe(config, &mut compatibility); compatibility @@ -2364,6 +2379,11 @@ where crate::observe::record_generation_state("building"); crate::observe::record_rebuild_state("unknown"); lexical_page_source::checkpoint(control)?; + if request.repository_parse_identity.include_submodules != self.config.include_submodules { + return Err(CodeIndexProductionErrorV1::Contract( + "snapshot submodule policy differs from owner".to_owned(), + )); + } let ignored_source_roster = IgnoredSourceRosterV1::admit( &request.snapshot, &request.repository_parse_identity, diff --git a/crates/tracedecay-code-index/src/production/sparse_increment_tests.rs b/crates/tracedecay-code-index/src/production/sparse_increment_tests.rs index 01eb1dcb76..66d8af7915 100644 --- a/crates/tracedecay-code-index/src/production/sparse_increment_tests.rs +++ b/crates/tracedecay-code-index/src/production/sparse_increment_tests.rs @@ -100,6 +100,7 @@ pub(super) fn request(files: &[(&str, &str, &str)], sealed_at: i64) -> CodeIndex invalidations: BTreeSet::new(), ignored_source_admissions: Vec::new(), repository_parse_identity: CodeIndexRepositoryParseIdentityV1 { + include_submodules: false, tree: None, dirty: tracedecay_domain::RepositoryDirtyStateV1::Dirty, }, diff --git a/crates/tracedecay-code-index/src/production/worker_tests.rs b/crates/tracedecay-code-index/src/production/worker_tests.rs index 7f210388b6..71b9eedfcc 100644 --- a/crates/tracedecay-code-index/src/production/worker_tests.rs +++ b/crates/tracedecay-code-index/src/production/worker_tests.rs @@ -39,6 +39,7 @@ where pub(super) fn worker_config() -> CodeIndexProductionConfigV1 { CodeIndexProductionConfigV1 { + include_submodules: false, project_id: worker_id("project.worker"), repository: worker_id("repository.worker"), sanitizer_revision: worker_id::("sanitizer.v1"), @@ -87,6 +88,7 @@ pub(super) fn worker_request_with_source( invalidations: BTreeSet::new(), ignored_source_admissions: Vec::new(), repository_parse_identity: CodeIndexRepositoryParseIdentityV1 { + include_submodules: false, tree: None, dirty: tracedecay_domain::RepositoryDirtyStateV1::Dirty, }, diff --git a/crates/tracedecay-code-index/src/source_walk.rs b/crates/tracedecay-code-index/src/source_walk.rs index 63d4be6e40..93b6b3847e 100644 --- a/crates/tracedecay-code-index/src/source_walk.rs +++ b/crates/tracedecay-code-index/src/source_walk.rs @@ -9,13 +9,21 @@ //! walk is therefore public rather than crate-private, the audit in the root //! crate reuses this policy instead of restating it. -use std::path::{Path, PathBuf}; -use std::sync::Arc; +pub mod classification; +pub mod submodules; +mod tracked_links; +pub use tracked_links::resolve_parent_source; + +use std::collections::BTreeSet; +use std::path::{Component, Path, PathBuf}; +use std::sync::{Arc, Mutex, PoisonError}; use ignore::overrides::{Override, OverrideBuilder}; use ignore::{Walk, WalkBuilder}; use tracedecay_domain::{IndexPathPolicyV1, forward_slash_path}; +pub type SourceWalk = std::iter::Flatten>; + #[derive(Debug)] pub struct SourceWalkError { pub glob: String, @@ -24,6 +32,7 @@ pub struct SourceWalkError { /// Conservative directory reachability for a positive override. The override /// matcher remains authoritative for files, including directory inheritance. +#[derive(Clone)] struct PathGlobScope { literal_prefix: PathBuf, may_match_descendants: bool, @@ -103,12 +112,268 @@ impl PathGlobScope { } } +const MAX_REPOSITORY_NAMES: usize = 8; +const MAX_REPOSITORY_PATH_CHARS: usize = 160; + +fn nested_repository_marker(directory: &Path) -> std::io::Result { + match std::fs::symlink_metadata(directory.join(".git")) { + Ok(_) => Ok(true), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(false), + Err(error) => Err(error), + } +} + +/// Default-deny all gitlinks in the parent's Git index even when a child +/// checkout's .git marker has been removed. A gitlink is a repository boundary, +/// not an ordinary directory the live source walker can authenticate. +/// This is deny-only; it never enables submodule admission. +pub fn parent_gitlink_paths(project_root: &Path) -> Result, SourceWalkError> { + let repository = match tracedecay_runtime_core::git_open::open(project_root) { + Ok(repository) => repository, + Err(error) => { + // Non-Git projects still support source walks. An initialized + // checkout whose Git authority is unreadable must fail closed. + return match nested_repository_marker(project_root) { + Ok(false) => Ok(BTreeSet::new()), + Ok(true) => Err(SourceWalkError { + glob: String::new(), + message: format!("cannot inspect parent Git boundaries: {error}"), + }), + Err(marker_error) => Err(SourceWalkError { + glob: String::new(), + message: format!("cannot inspect parent Git marker: {marker_error}"), + }), + }; + } + }; + if repository + .workdir() + .and_then(|workdir| workdir.canonicalize().ok()) + != project_root.canonicalize().ok() + { + return Err(SourceWalkError { + glob: String::new(), + message: "source walk root is not the opened Git worktree".to_owned(), + }); + } + let index = repository + .index_or_empty() + .map_err(|error| SourceWalkError { + glob: String::new(), + message: format!("cannot inspect parent Git index: {error}"), + })?; + let mut paths = BTreeSet::new(); + for entry in index.entries() { + if entry.mode.is_submodule() { + let path: &[u8] = entry.path(&index).as_ref(); + let relative = std::str::from_utf8(path).map_err(|_| SourceWalkError { + glob: String::new(), + message: "non-UTF-8 parent gitlink cannot be safely scoped".to_owned(), + })?; + paths.insert(relative.to_owned()); + } + } + // A staged gitlink removal does not erase the checkout boundary from + // HEAD. Without this second authority, a child whose .git marker was + // removed becomes ordinary live source until the parent commits. + let head = repository.head().map_err(|error| SourceWalkError { + glob: String::new(), + message: format!("cannot inspect parent Git HEAD: {error}"), + })?; + if head.id().is_some() { + let commit = repository.head_commit().map_err(|error| SourceWalkError { + glob: String::new(), + message: format!("cannot resolve parent Git HEAD: {error}"), + })?; + let tree = commit.tree().map_err(|error| SourceWalkError { + glob: String::new(), + message: format!("cannot read parent Git HEAD tree: {error}"), + })?; + let entries = tree + .traverse() + .breadthfirst + .files() + .map_err(|error| SourceWalkError { + glob: String::new(), + message: format!("cannot walk parent Git HEAD tree: {error}"), + })?; + for entry in entries { + if !entry.mode.is_commit() { + continue; + } + let path: &[u8] = entry.filepath.as_ref(); + let relative = std::str::from_utf8(path).map_err(|_| SourceWalkError { + glob: String::new(), + message: "non-UTF-8 HEAD gitlink cannot be safely scoped".to_owned(), + })?; + paths.insert(relative.to_owned()); + } + } else if head.referent_name().is_none() { + return Err(SourceWalkError { + glob: String::new(), + message: "parent Git HEAD has neither a commit nor an unborn branch".to_owned(), + }); + } + Ok(paths) +} + +/// Reject a source beneath another checkout or a symlinked directory. +/// The root's own Git metadata is exempt. This boundary is shared by +/// interactive source walks and the indexed worktree candidate roster, +/// so neither may admit bytes the other refuses. +pub fn has_nested_repository_ancestor( + project_root: &Path, + relative_path: &Path, +) -> std::io::Result { + if relative_path + .components() + .any(|component| !matches!(component, Component::Normal(_))) + { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidInput, + "source path must be repository-relative", + )); + } + + let mut parent = relative_path.parent(); + while let Some(relative_dir) = parent { + if relative_dir.as_os_str().is_empty() { + break; + } + let directory = project_root.join(relative_dir); + match std::fs::symlink_metadata(&directory) { + Ok(metadata) if !metadata.is_dir() => return Ok(true), + Ok(_) => {} + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => return Err(error), + } + if nested_repository_marker(&directory)? { + return Ok(true); + } + parent = relative_dir.parent(); + } + Ok(false) +} + +/// Distinguish leaf links before selecting their source authority. +/// Missing paths remain possible tombstones and are not symlink exclusions. +pub fn is_source_symlink(project_root: &Path, relative_path: &Path) -> std::io::Result { + if relative_path.as_os_str().is_empty() + || relative_path + .components() + .any(|component| !matches!(component, Component::Normal(_))) + { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidInput, + "source path must be repository-relative", + )); + } + match std::fs::symlink_metadata(project_root.join(relative_path)) { + Ok(metadata) => Ok(metadata.file_type().is_symlink()), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(false), + Err(error) => Err(error), + } +} + +/// Diagnostics emitted by the same source walker that enforces the nested +/// repository boundary. Only directories within the walk's policy are named. +#[derive(Clone, Default)] +pub(crate) struct SourceWalkDiagnostics { + skipped_nested_repositories: Arc>, +} + +#[derive(Default)] +struct NestedRepositorySummary { + total: usize, + names: BTreeSet, +} + +impl SourceWalkDiagnostics { + fn record_skipped_nested_repository(&self, relative: String) { + let mut summary = self + .skipped_nested_repositories + .lock() + .unwrap_or_else(PoisonError::into_inner); + // A pruned directory is visited once. Keep only the smallest names, + // independent of filesystem iteration order, rather than collecting + // every path and truncating only after the walk has finished. + summary.total += 1; + summary.names.insert(relative); + if summary.names.len() > MAX_REPOSITORY_NAMES { + let _ = summary.names.pop_last(); + } + } + + pub(crate) fn skipped_nested_repositories_note(&self) -> Option { + let summary = self + .skipped_nested_repositories + .lock() + .unwrap_or_else(PoisonError::into_inner); + if summary.total == 0 { + return None; + } + let names = summary + .names + .iter() + .map(|path| { + // Bound by characters before escaping, so truncation cannot + // split UTF-8 or an escape. Quoting prevents a filename from + // injecting new lines or delimiters into an agent's report. + let mut chars = path.chars(); + let prefix = chars + .by_ref() + .take(MAX_REPOSITORY_PATH_CHARS) + .collect::(); + let suffix = if chars.next().is_some() { "…" } else { "" }; + format!("{prefix:?}{suffix}") + }) + .collect::>(); + let remaining = summary.total - summary.names.len(); + let suffix = if remaining == 0 { + String::new() + } else { + format!(" (and {remaining} more)") + }; + Some(format!( + "Skipped nested repositories outside parent index: {}{suffix}", + names.join(", ") + )) + } +} + #[tracing::instrument(name = "code_index.capture.source_walk", level = "trace", skip_all)] pub fn source_walk( project_root: &Path, path_glob: Option<&str>, path_policy: &IndexPathPolicyV1, -) -> Result { +) -> Result { + source_walk_inner(project_root, path_glob, path_policy, None) +} + +/// Expose excluded nested repository roots without changing source admission. +/// The unmounted-file audit uses this to avoid falsely implying completeness. +/// It uses the same traversal rather than an independent repository walker. +pub(crate) fn source_walk_with_diagnostics( + project_root: &Path, + path_glob: Option<&str>, + path_policy: &IndexPathPolicyV1, +) -> Result<(SourceWalk, SourceWalkDiagnostics), SourceWalkError> { + let diagnostics = SourceWalkDiagnostics::default(); + let walk = source_walk_inner( + project_root, + path_glob, + path_policy, + Some(diagnostics.clone()), + )?; + Ok((walk, diagnostics)) +} + +fn source_walk_inner( + project_root: &Path, + path_glob: Option<&str>, + path_policy: &IndexPathPolicyV1, + diagnostics: Option, +) -> Result { let overrides = build_overrides(project_root, path_glob)?; let has_positive_override = overrides .as_ref() @@ -117,68 +382,145 @@ pub fn source_walk( let path_glob_scope = path_glob.and_then(PathGlobScope::from_path_glob); let filter_root = project_root.to_path_buf(); let path_policy = path_policy.clone(); + let scoped_policy = { + let root = filter_root.clone(); + let scope = path_glob_scope.clone(); + move |path: &Path, is_dir: bool| { + has_positive_override + && (generated_dir_overrides + .as_ref() + .is_some_and(|overrides| overrides.matched(path, is_dir).is_whitelist()) + || (is_dir + && scope + .as_ref() + .is_some_and(|scope| scope.allows(&root, path)))) + } + }; + let boundaries = + submodules::CheckoutBoundaries::open(project_root, path_policy.includes_submodules())?; + let mut parent_gitlinks = boundaries.gitlinks; + let mut authenticated = BTreeSet::new(); + let mut child_files = BTreeSet::new(); + for (path, child) in boundaries.children { + let absolute = project_root.join(&path); + let policy = if scoped_policy(&absolute, true) { + path_policy.without_generated_dir_defaults() + } else { + &path_policy + }; + if policy.excludes_directory(&path) + || path_glob_scope + .as_ref() + .is_some_and(|scope| !scope.may_contain_match(project_root, &absolute)) + { + continue; + } + let inventory = child.sources()?; + child_files.extend( + inventory + .files + .into_iter() + .map(|file| format!("{path}/{file}")), + ); + // Only direct registrations belong to this parent. + for nested in inventory.gitlinks { + parent_gitlinks.insert(format!("{path}/{nested}")); + } + authenticated.insert(path); + } - let mut builder = WalkBuilder::new(project_root); - builder - .follow_links(false) - .hidden(false) - .git_ignore(true) - .git_global(true) - .git_exclude(true) - .add_custom_ignore_filename(".gitignore") - .filter_entry(move |entry| { - if entry.depth() == 0 { - return true; - } - let segment = entry.file_name().to_string_lossy(); - if segment == ".git" || segment == ".tracedecay" { - return false; + let roots = authenticated + .iter() + .map(|path| project_root.join(path)) + .collect::>(); + let child_dirs = child_files + .iter() + .flat_map(|path| Path::new(path).ancestors().skip(1)) + .map(Path::to_owned) + .collect::>(); + + let accept = Arc::new(move |entry: &ignore::DirEntry, in_child: bool| { + let segment = entry.file_name().to_string_lossy(); + if segment == ".git" || segment == ".tracedecay" { + return false; + } + let Ok(relative) = entry.path().strip_prefix(&filter_root) else { + return false; + }; + let relative = forward_slash_path(relative); + let is_dir = entry.file_type().is_some_and(|kind| kind.is_dir()); + if is_dir + && path_glob_scope + .as_ref() + .is_some_and(|scope| !scope.may_contain_match(&filter_root, entry.path())) + { + return false; + } + // An entry the glob names, or a directory it must pass through, + // is judged without the generated-directory defaults: the scope + // is the operator asking for that noise. Every other exclusion + // still applies; only `index.include.v1` lifts those. + let path_policy = if scoped_policy(entry.path(), is_dir) { + path_policy.without_generated_dir_defaults() + } else { + &path_policy + }; + if !is_dir { + return !path_policy.excludes(&relative) + && (!in_child || child_files.contains(&relative)); + } + // Excluded directories are outside this audit, not missing source + // caused by the repository boundary. Do not probe or name them. + if path_policy.excludes_directory(&relative) { + return false; + } + // Only an independently bound direct gitlink may cross a checkout boundary. + if authenticated.contains(&relative) { + return in_child; + } + if parent_gitlinks.contains(&relative) { + if let Some(diagnostics) = &diagnostics { + diagnostics.record_skipped_nested_repository(relative); } - let Ok(relative) = entry.path().strip_prefix(&filter_root) else { - return false; - }; - let relative = forward_slash_path(relative); - let is_dir = entry.file_type().is_some_and(|kind| kind.is_dir()); - if is_dir - && path_glob_scope - .as_ref() - .is_some_and(|scope| !scope.may_contain_match(&filter_root, entry.path())) - { - return false; + return false; + } + match nested_repository_marker(entry.path()) { + Ok(true) => { + if let Some(diagnostics) = &diagnostics { + diagnostics.record_skipped_nested_repository(relative); + } + false } - // An entry the glob names, or a directory it must pass through, - // is judged without the generated-directory defaults: the scope - // is the operator asking for that noise. Every other exclusion - // still applies; only `index.include.v1` lifts those. - let explicitly_requested = has_positive_override - && (generated_dir_overrides.as_ref().is_some_and(|overrides| { - overrides.matched(entry.path(), is_dir).is_whitelist() - }) || (is_dir - && path_glob_scope - .as_ref() - .is_some_and(|scope| scope.allows(&filter_root, entry.path())))); - let path_policy = if explicitly_requested { - path_policy.without_generated_dir_defaults() + Ok(false) => !in_child || child_dirs.contains(Path::new(&relative)), + // The walk cannot prove that an unreadable Git marker is + // absent. Never admit files across that boundary. + Err(_) => false, + } + }); + let mut builders = vec![(WalkBuilder::new(project_root), false)]; + builders.extend(roots.into_iter().map(|root| (WalkBuilder::new(root), true))); + let walks = builders + .into_iter() + .map(|(mut builder, in_child)| { + // The child Git roster already applied that checkout's ignores, including + // tracked exceptions. Running parent ignore rules over it would hide those files. + if in_child { + builder.standard_filters(false); } else { - &path_policy - }; - if !is_dir { - return !path_policy.excludes(&relative); + builder.add_custom_ignore_filename(".gitignore"); } - // Includes select paths, not repository authority. A parent - // gitlink does not capture the child's files or prove their - // freshness, so even registered submodules remain separate - // sources. Linked worktrees and nested clones share this boundary. - // The project root is deliberately exempt at depth zero above. - if std::fs::symlink_metadata(entry.path().join(".git")).is_ok() { - return false; + let accept = Arc::clone(&accept); + builder + .follow_links(false) + .hidden(false) + .filter_entry(move |entry| accept(entry, in_child)); + if let Some(overrides) = &overrides { + builder.overrides(overrides.clone()); } - !path_policy.excludes_directory(&relative) - }); - if let Some(overrides) = overrides { - builder.overrides(overrides); - } - Ok(builder.build()) + builder.build() + }) + .collect::>(); + Ok(walks.into_iter().flatten()) } /// Project-relative path rendered with forward slashes. @@ -212,6 +554,9 @@ fn build_overrides( } } +#[cfg(test)] +mod nested_repository_tests; + #[cfg(test)] mod tests { use std::fs; @@ -220,7 +565,68 @@ mod tests { use tempfile::TempDir; use tracedecay_domain::IndexPathPolicyV1; - use super::source_walk; + use super::{ + has_nested_repository_ancestor, is_source_symlink, source_walk, + source_walk_with_diagnostics, + }; + + #[test] + fn nested_repository_boundary_excludes_parent_markers_and_path_escape() { + let root = TempDir::new().unwrap(); + fs::create_dir_all(root.path().join("parent/nested/.git")).unwrap(); + fs::write( + root.path().join("parent/nested/.git/HEAD"), + "ref: refs/heads/main\n", + ) + .unwrap(); + assert!( + !has_nested_repository_ancestor(root.path(), std::path::Path::new("parent/source.rs")) + .unwrap() + ); + assert!( + has_nested_repository_ancestor( + root.path(), + std::path::Path::new("parent/nested/source.rs") + ) + .unwrap() + ); + assert!( + has_nested_repository_ancestor(root.path(), std::path::Path::new("../escape.rs")) + .is_err() + ); + assert!( + has_nested_repository_ancestor(root.path(), std::path::Path::new("/absolute.rs")) + .is_err() + ); + } + + #[cfg(unix)] + #[test] + fn nested_repository_boundary_excludes_symlinked_ancestors() { + use std::os::unix::fs::symlink; + + let root = TempDir::new().unwrap(); + let external = TempDir::new().unwrap(); + symlink(external.path(), root.path().join("foreign")).unwrap(); + assert!( + has_nested_repository_ancestor(root.path(), std::path::Path::new("foreign/source.rs")) + .unwrap() + ); + } + + #[cfg(unix)] + #[test] + fn leaf_link_detection_preserves_missing_paths() { + use std::os::unix::fs::symlink; + + let root = TempDir::new().unwrap(); + fs::write(root.path().join("real.rs"), "pub fn real() {}\n").unwrap(); + symlink("real.rs", root.path().join("alias.rs")).unwrap(); + assert!(is_source_symlink(root.path(), std::path::Path::new("alias.rs")).unwrap()); + assert!(!is_source_symlink(root.path(), std::path::Path::new("real.rs")).unwrap()); + assert!(!is_source_symlink(root.path(), std::path::Path::new("missing.rs")).unwrap()); + assert!(is_source_symlink(root.path(), std::path::Path::new("../escape.rs")).is_err()); + } fn walked_files(root: &std::path::Path, path_policy: &IndexPathPolicyV1) -> Vec { let mut files = source_walk(root, None, path_policy) @@ -341,7 +747,7 @@ mod tests { #[test] fn nested_repository_is_not_part_of_the_parent_source_tree() { let root = TempDir::new().expect("project root"); - fs::create_dir(root.path().join(".git")).expect("parent git directory"); + fixture_git(root.path(), &["init", "-q"]); fs::create_dir_all(root.path().join("src")).expect("parent source directory"); fs::write(root.path().join("src/lib.rs"), "pub fn parent() {}\n").expect("parent source"); fs::create_dir_all(root.path().join("plain")).expect("plain directory"); @@ -435,6 +841,20 @@ mod tests { assert!(index.stdout.ends_with(b" 0\tapp\0")); assert!(root.join("app/src/feature.rs").is_file()); + let (walk, diagnostics) = source_walk_with_diagnostics( + &root, + None, + &IndexPathPolicyV1::new(vec![], vec![]).unwrap(), + ) + .expect("source walk with nested repository diagnostics"); + for entry in walk { + entry.expect("source entry"); + } + assert_eq!( + diagnostics.skipped_nested_repositories_note().as_deref(), + Some("Skipped nested repositories outside parent index: \"app\"") + ); + for rule in ["app/**", "app/src/feature.rs", "app/src/fe*.rs"] { let policy = IndexPathPolicyV1::new(vec![], vec![rule.into()]).unwrap(); let files = walked_files(&root, &policy); @@ -453,12 +873,103 @@ mod tests { "{rule}: an explicit scope cannot cross repositories" ); } + + let enabled = IndexPathPolicyV1::new(vec![], vec![]) + .unwrap() + .with_submodules(true); + fixture_git( + &child, + &[ + "worktree", + "add", + "-q", + "--detach", + root.join("other-worktree").to_str().unwrap(), + "HEAD", + ], + ); + fixture_git( + &root, + &["clone", "-q", child.to_str().unwrap(), "other-clone"], + ); + let files = walked_files(&root, &enabled); + assert!(files.contains(&PathBuf::from("app/src/feature.rs"))); + assert!(files.iter().all(|path|!path.starts_with("other-worktree") && !path.starts_with("other-clone"))); + fixture_git( + &child, + &[ + "worktree", + "remove", + root.join("other-worktree").to_str().unwrap(), + ], + ); + fs::remove_dir_all(root.join("other-clone")).unwrap(); + fs::remove_file(root.join(".gitmodules")).unwrap(); + assert!(!walked_files(&root, &enabled).contains(&PathBuf::from("app/src/feature.rs"))); + fixture_git(&root, &["checkout", "--", ".gitmodules"]); + // A missing child .git marker must not turn the parent's stage-0 + // gitlink into ordinary live source under an explicit include. + fs::remove_file(root.join("app/.git")).expect("remove child gitdir marker"); + assert!(root.join("app/src/feature.rs").is_file()); + let policy = IndexPathPolicyV1::new(vec![], vec!["app/**".into()]).unwrap(); + assert!(!walked_files(&root, &enabled).contains(&PathBuf::from("app/src/feature.rs"))); + let (walk, diagnostics) = + source_walk_with_diagnostics(&root, None, &policy).expect("source walk"); + let files = walk + .map(|entry| entry.expect("source entry")) + .filter(|entry| entry.file_type().is_some_and(|kind| kind.is_file())) + .map(|entry| entry.into_path()) + .collect::>(); + assert!( + files.iter().all(|path| !path.starts_with(root.join("app"))), + "a registered gitlink stays excluded without a child marker: {files:?}" + ); + assert_eq!( + diagnostics.skipped_nested_repositories_note().as_deref(), + Some("Skipped nested repositories outside parent index: \"app\"") + ); + let explicitly_scoped = source_walk(&root, Some("app/**"), &policy) + .expect("explicit gitlink scope") + .map(|entry| entry.expect("source entry")) + .filter(|entry| entry.file_type().is_some_and(|kind| kind.is_file())) + .count(); + assert_eq!(explicitly_scoped, 0); + + // Staging removal of the gitlink also removes it from the parent + // index, but HEAD still owns that repository boundary. Neither an + // explicit include nor a path glob may adopt its live child bytes. + fixture_git(&root, &["add", ".gitmodules", "src/parent.rs"]); + fixture_git(&root, &["commit", "-qm", "parent with submodule"]); + fixture_git(&root, &["rm", "--cached", "-f", "app"]); + assert!( + fixture_git(&root, &["ls-files", "--stage", "--", "app"]) + .stdout + .is_empty() + ); + assert!( + fixture_git(&root, &["ls-tree", "HEAD", "--", "app"]) + .stdout + .starts_with(b"160000 commit ") + ); + assert!(root.join("app/src/feature.rs").is_file()); + + let files = walked_files(&root, &policy); + assert!( + files.iter().all(|path| !path.starts_with("app")), + "HEAD gitlink is still an excluded source boundary: {files:?}" + ); + let scoped = source_walk(&root, Some("app/**"), &policy) + .expect("scope cannot cross HEAD gitlink") + .map(|entry| entry.expect("source entry")) + .filter(|entry| entry.file_type().is_some_and(|kind| kind.is_file())) + .count(); + assert_eq!(scoped, 0); } #[test] fn submodule_metadata_cannot_break_an_unrelated_source_include() { let root = TempDir::new().expect("project root"); - fs::create_dir_all(root.path().join(".git")).unwrap(); + fixture_git(root.path(), &["init", "-q"]); fs::create_dir_all(root.path().join("src")).unwrap(); fs::create_dir_all(root.path().join("app/.git")).unwrap(); fs::write(root.path().join("src/keep.rs"), "pub fn keep() {}\n").unwrap(); diff --git a/crates/tracedecay-code-index/src/source_walk/classification.rs b/crates/tracedecay-code-index/src/source_walk/classification.rs new file mode 100644 index 0000000000..2042361ca3 --- /dev/null +++ b/crates/tracedecay-code-index/src/source_walk/classification.rs @@ -0,0 +1,748 @@ +//! Truthful gix-based change classification for incremental indexing. +//! +//! Filesystem events and hook payloads only *hint* which paths may have moved. +//! The authoritative changed/added/deleted/renamed set is always recomputed +//! here from gix's HEAD-tree/index/worktree status, so a duplicate hook, a +//! save-without-change, or a dropped watcher event can never fabricate work or +//! hide a real change. +//! +//! Committed, staged, unstaged, untracked, and deleted paths are kept +//! *distinct*. Deletions become tombstone candidates. Renames are deliberately +//! not tracked as a distinct class: for indexing a rename is just a deletion of +//! the source plus an addition of the destination, so rename detection is +//! disabled and gix reports the two halves independently. The scheduler +//! consumes the derived candidate and changed sets to build an incremental +//! batch. + +use std::collections::{BTreeMap, BTreeSet}; +use std::path::Path; + +use super::{ + has_nested_repository_ancestor, parent_gitlink_paths, resolve_parent_source, + submodules::live_submodules, +}; +use gix::bstr::ByteSlice; +use tracedecay_runtime_core::git_repository::GIT_STATUS_MODIFICATION_CHECK_THREADS; + +/// Failure to classify worktree changes through gix. +#[derive(Debug, thiserror::Error)] +pub enum ClassificationErrorV1 { + #[error("code-index classification: {0}")] + Git(String), + #[error("code-index source boundary: {0}")] + SourceBoundary(#[from] std::io::Error), +} + +/// The truthful disposition of one repository-relative path. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum WorktreeChangeClassV1 { + /// Added to the index versus HEAD (staged new file). + StagedAdded, + /// Content differs between HEAD tree and index (staged modification). + StagedModified, + /// Present in HEAD tree, removed from the index (staged deletion). + StagedDeleted, + /// Worktree content differs from the index (unstaged modification). + UnstagedModified, + /// Tracked file missing from the worktree (unstaged deletion). + UnstagedDeleted, + /// Present on disk with no index relation (untracked, incl. intent-to-add). + Untracked, + /// Merge conflict; content is not a single truthful revision. + Conflicted, +} + +impl WorktreeChangeClassV1 { + /// Whether this class removes a path (so its prior chunks become tombstones) + /// rather than presenting current content to index. + pub fn is_deletion(self) -> bool { + matches!(self, Self::StagedDeleted | Self::UnstagedDeleted) + } + + /// Whether this class contributes a present file whose bytes should be + /// hashed and considered for (re)indexing. + pub fn presents_content(self) -> bool { + !self.is_deletion() + } +} + +/// One classified path. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ClassifiedChangeV1 { + pub path: String, + pub class: WorktreeChangeClassV1, +} + +/// One classified change whose Git path bytes are not UTF-8. No `String` +/// names it, so it lives outside `changes`: a lossy decode could equal a +/// real UTF-8 path and make the string-keyed candidate and changed sets +/// treat this change as that file's. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct NonUtf8ClassifiedChangeV1 { + /// The exact Git path bytes. + pub git_path: Vec, + pub class: WorktreeChangeClassV1, +} + +/// A complete, truthful classification of one worktree snapshot. +#[derive(Clone, Debug, Default)] +pub struct WorktreeChangeClassificationV1 { + /// Repository-relative paths tracked in the index (the committed/staged + /// baseline present in the checkout). + committed_baseline: BTreeSet, + changes: Vec, + /// Changes whose Git path bytes are not UTF-8, kept byte-exact so the + /// worktree is still dirty on a non-UTF-8-only transition. + non_utf8_changes: Vec, + /// Present paths whose Git bytes are not UTF-8, so no `String` above + /// names them. + non_utf8_paths: BTreeSet>, + /// Files still tracked by the parent but hidden behind a nested checkout + /// or symlinked ancestor. These are tombstones, not indexable candidates. + nested_repository_exclusions: BTreeSet, + parent_gitlinks: BTreeSet, + /// Direct parent-relative child paths admitted from that checkout's own Git roster. + child_sources: BTreeMap, + child_revision_changed: bool, +} + +impl WorktreeChangeClassificationV1 { + /// Classify the current status of `repository` truthfully. + #[tracing::instrument(name = "daemon.code_index.capture.classify", level = "trace", skip_all)] + pub fn classify(repository: &gix::Repository) -> Result { + let index = repository + .index_or_empty() + .map_err(|error| ClassificationErrorV1::Git(error.to_string()))?; + let skip_validation = gix::index::entry::Flags::UPTODATE + | gix::index::entry::Flags::SKIP_WORKTREE + | gix::index::entry::Flags::ASSUME_VALID + | gix::index::entry::Flags::FSMONITOR_VALID; + let mut committed_baseline = BTreeSet::new(); + let mut non_utf8_paths = BTreeSet::new(); + let mut has_skipped_entries = false; + for entry in index.entries() { + has_skipped_entries |= entry.flags.intersects(skip_validation); + let path: &[u8] = entry.path(&index).as_ref(); + match std::str::from_utf8(path) { + Ok(path) => { + committed_baseline.insert(path.to_owned()); + } + Err(_) => { + non_utf8_paths.insert(path.to_vec()); + } + } + } + let mut status_index = gix::worktree::IndexPersistedOrInMemory::from(index); + if has_skipped_entries { + // These flags are Git status shortcuts, not evidence that source + // bytes still match the index. Recheck only those entries through + // gix's ordinary filtered blob comparison. An mtime at the index + // timestamp is racy even when all stat fields match, so neither + // the flags nor preserved file metadata can suppress that check. + // This private index is never written back to the user's checkout. + let mut checked = status_index.into_owned(); + let racy_mtime = checked.timestamp().into(); + for (entry, path) in checked.entries_mut_with_paths() { + if entry.flags.intersects(skip_validation) { + // Sparse entries deliberately omit worktree content. + // Preserve that omission; only materialized skipped + // paths need to prove their bytes against the index. + if entry + .flags + .contains(gix::index::entry::Flags::SKIP_WORKTREE) + && (entry.mode.is_sparse() + || repository.workdir().is_some_and(|root| { + gix::path::from_bstr(path).is_ok_and(|relative| { + std::fs::symlink_metadata(root.join(relative)).is_err_and( + |error| error.kind() == std::io::ErrorKind::NotFound, + ) + }) + })) + { + continue; + } + entry.flags.remove(skip_validation); + entry.stat.mtime = racy_mtime; + } + } + status_index = checked.into(); + } + + let mut changes = Vec::new(); + let mut non_utf8_changes = Vec::new(); + let status = repository + .status(gix::progress::Discard) + .map_err(|error| ClassificationErrorV1::Git(error.to_string()))? + .index(status_index) + // Emit untracked files, but not ignored ones: ignored/generated + // content is out of indexing scope. + .untracked_files(gix::status::UntrackedFiles::Files) + // Rename detection is deliberately disabled on both the tree↔index + // and index↔worktree axes: for indexing a rename is just a deletion + // plus an addition, and rewrite tracking is expensive. Content + // reuse still happens via the content-addressed byte pool. + .tree_index_track_renames(gix::status::tree_index::TrackRenames::Disabled) + .index_worktree_rewrites(None) + // Submodule content belongs to its own repository identity and is + // never indexed as part of this worktree. + .index_worktree_submodules(None) + .index_worktree_options_mut(|options| { + options.thread_limit = Some(GIT_STATUS_MODIFICATION_CHECK_THREADS); + }) + .into_iter(Vec::::new()) + .map_err(|error| ClassificationErrorV1::Git(error.to_string()))?; + for item in status { + let item = item.map_err(|error| ClassificationErrorV1::Git(error.to_string()))?; + let location: &[u8] = item.location().as_ref(); + if let Some(class) = classify_item(&item) { + match location.to_str() { + Ok(path) => changes.push(ClassifiedChangeV1 { + path: path.to_owned(), + class, + }), + Err(_) => { + if class.presents_content() { + non_utf8_paths.insert(location.to_vec()); + } else { + non_utf8_paths.remove(location); + } + non_utf8_changes.push(NonUtf8ClassifiedChangeV1 { + git_path: location.to_vec(), + class, + }); + } + } + } + } + + // A nested .git marker can appear under paths that are still in + // the parent's index without changing Git status. The live walker + // refuses those files; classification must also exclude them and + // invalidate clean-tree and active-generation reuse. Resolve each + // parent directory once rather than probing per source file. + let mut classification = Self { + committed_baseline, + changes, + non_utf8_changes, + non_utf8_paths, + ..Self::default() + }; + if let Some(project_root) = repository.workdir() { + // The live walker also denies HEAD gitlinks staged for removal. + // Without this check, Git reports their markerless child sources + // as untracked, and worktree capture can re-admit those bytes. + classification.parent_gitlinks = parent_gitlink_paths(project_root) + .map_err(|error| ClassificationErrorV1::Git(error.message))?; + let mut checked_parents = BTreeMap::new(); + for path in classification + .committed_baseline + .iter() + .chain(classification.changes.iter().map(|change| &change.path)) + { + let relative = Path::new(path); + let parent = relative.parent().unwrap_or_else(|| Path::new("")); + let excluded = if classification.beneath_gitlink(relative) + || resolve_parent_source(project_root, relative)?.is_none() + { + true + } else { + match checked_parents.get(parent) { + Some(excluded) => *excluded, + None => { + let excluded = has_nested_repository_ancestor(project_root, relative)?; + checked_parents.insert(parent.to_path_buf(), excluded); + excluded + } + } + }; + if excluded { + classification + .nested_repository_exclusions + .insert(path.clone()); + } + } + } + + Ok(classification) + } + + pub fn for_policy( + repository: &gix::Repository, + include_submodules: bool, + ) -> Result { + let mut parent = Self::classify(repository)?; + if !include_submodules || parent.parent_gitlinks.is_empty() { + return Ok(parent); + } + let children = live_submodules(repository) + .map_err(|error| ClassificationErrorV1::Git(error.message))?; + // A missing checkout must not let the clean parent-tree shortcut import children. + parent.child_revision_changed = parent + .parent_gitlinks + .iter() + .any(|path| !children.contains_key(path)); + for (prefix, binding) in children { + let child = tracedecay_runtime_core::git_open::open(&binding.git_dir) + .map_err(|error| ClassificationErrorV1::Git(error.to_string()))?; + let child_head = child + .head_id() + .map_err(|error| ClassificationErrorV1::Git(error.to_string()))?; + parent.child_revision_changed |= child_head.detach() != binding.pin; + let local = Self::classify(&child)?; + for path in local.regular_child_files(&binding.worktree)? { + let logical = format!("{prefix}/{path}"); + parent.committed_baseline.insert(logical.clone()); + parent.child_sources.insert(logical, prefix.clone()); + } + for change in local.changes { + let path = format!("{prefix}/{}", change.path); + if !change.class.presents_content() || parent.child_sources.contains_key(&path) { + parent.changes.push(ClassifiedChangeV1 { + path, + class: change.class, + }); + } + } + parent.nested_repository_exclusions.extend( + local + .nested_repository_exclusions + .into_iter() + .map(|path| format!("{prefix}/{path}")), + ); + let qualify = |path: &[u8]| [prefix.as_bytes(), b"/", path].concat(); + parent + .non_utf8_paths + .extend(local.non_utf8_paths.into_iter().map(|path| qualify(&path))); + parent + .non_utf8_changes + .extend(local.non_utf8_changes.into_iter().map(|change| { + NonUtf8ClassifiedChangeV1 { + git_path: qualify(&change.git_path), + class: change.class, + } + })); + } + Ok(parent) + } + + pub(super) fn regular_child_files(&self, root: &Path) -> std::io::Result> { + let mut files = BTreeSet::new(); + for path in self.candidate_paths() { + if !super::submodules::is_source_location(Path::new(&path)) { + continue; + } + match root.join(&path).symlink_metadata() { + Ok(metadata) if metadata.is_file() => { + files.insert(path); + } + Ok(_) => {} + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => return Err(error), + } + } + Ok(files) + } + + pub fn child_root(&self, logical_path: &str) -> Option<&str> { + self.child_sources.get(logical_path).map(String::as_str) + } + + pub(super) fn gitlink_paths(&self) -> &BTreeSet { + &self.parent_gitlinks + } + + /// Explicit enrollment cannot cross a parent gitlink staged for removal. + pub fn beneath_gitlink(&self, path: &Path) -> bool { + path.ancestors() + .skip(1) + .filter_map(Path::to_str) + .any(|ancestor| self.parent_gitlinks.contains(ancestor)) + } + + /// Paths whose Git bytes are not UTF-8, present or not. + pub fn non_utf8_paths(&self) -> &BTreeSet> { + &self.non_utf8_paths + } + + /// Paths no logical path can name because their Git bytes are not UTF-8, + /// limited to those naming a regular file (through links) under + /// `project_root`, the files capture would read. + pub fn non_utf8_files(&self, project_root: &Path) -> Vec> { + self.non_utf8_paths + .iter() + .filter(|path| { + gix::path::from_byte_slice(path) + .is_ok_and(|relative| project_root.join(relative).is_file()) + }) + .cloned() + .collect() + } + + /// Present files worth hashing and considering for (re)indexing: the + /// committed baseline, plus untracked/added/rename-destination paths, minus + /// any path removed by a deletion. + pub fn candidate_paths(&self) -> BTreeSet { + let mut candidates = self.committed_baseline.clone(); + for change in &self.changes { + if change.class.presents_content() { + candidates.insert(change.path.clone()); + } else { + candidates.remove(&change.path); + } + } + candidates.retain(|path| !self.nested_repository_exclusions.contains(path)); + candidates + } + + /// Whether the worktree carries any change relative to the committed + /// baseline, UTF-8-named or not. + pub fn has_changes(&self) -> bool { + !(self.changes().is_empty() + && self.non_utf8_changes().is_empty() + && self.nested_repository_exclusions.is_empty() + && !self.child_revision_changed) + } + + /// Whether any classified change is a merge conflict. + pub fn has_conflicted(&self) -> bool { + self.changes() + .iter() + .map(|change| change.class) + .chain(self.non_utf8_changes().iter().map(|change| change.class)) + .any(|class| class == WorktreeChangeClassV1::Conflicted) + } + + /// Paths whose indexing evidence changed relative to the last generation: + /// every staged, unstaged, or untracked path (deletions included so + /// tombstones flow through). This is a hint to narrow work, not the identity + /// authority; the generation planner still compares content digests. + pub fn changed_paths(&self) -> BTreeSet { + let mut changed = self + .changes + .iter() + .map(|change| change.path.clone()) + .collect::>(); + changed.extend(self.nested_repository_exclusions.iter().cloned()); + changed + } + + /// Paths removed from the present snapshot (staged or unstaged deletions). + #[cfg(any(test, feature = "test-helpers"))] + pub fn deleted_paths(&self) -> BTreeSet { + self.changes + .iter() + .filter(|change| change.class.is_deletion()) + .map(|change| change.path.clone()) + .collect() + } + + /// All classified changes a `String` can name (for reporting and tests). + /// Non-UTF-8 changes live under [`Self::non_utf8_changes`]. + pub fn changes(&self) -> &[ClassifiedChangeV1] { + &self.changes + } + + /// All classified changes whose Git path bytes are not UTF-8, kept + /// byte-exact (for reporting and tests). + pub fn non_utf8_changes(&self) -> &[NonUtf8ClassifiedChangeV1] { + &self.non_utf8_changes + } + + /// The class recorded for `path`, if any change touched it. + #[cfg(any(test, feature = "test-helpers"))] + pub fn class_of(&self, path: &str) -> Option { + self.changes + .iter() + .find(|change| change.path == path) + .map(|change| change.class) + } +} + +/// Map one gix status item to a truthful class, or `None` for items that carry +/// no indexing signal on their own. +/// +/// Rename detection is disabled on both status axes (see [`classify`]), so gix +/// reports a move as an independent deletion of the source plus an addition of +/// the destination. The `Rewrite` variants therefore cannot occur here; they +/// are mapped to `None` defensively rather than fabricating a rename class. +fn classify_item(item: &gix::status::Item) -> Option { + use gix::diff::index::ChangeRef; + use gix::status::Item; + use gix::status::index_worktree::Item as IndexWorktreeItem; + use gix::status::plumbing::index_as_worktree::{Change as WorktreeChange, EntryStatus}; + + Some(match item { + // HEAD tree ↔ index: staged changes. + Item::TreeIndex(change) => match change { + ChangeRef::Addition { .. } => WorktreeChangeClassV1::StagedAdded, + ChangeRef::Deletion { .. } => WorktreeChangeClassV1::StagedDeleted, + ChangeRef::Modification { .. } => WorktreeChangeClassV1::StagedModified, + // Unreachable with rename detection disabled; skip rather than + // classify a move as anything other than delete + add. + ChangeRef::Rewrite { .. } => return None, + }, + // Index ↔ worktree: unstaged / untracked changes. + Item::IndexWorktree(worktree) => match worktree { + IndexWorktreeItem::Modification { status, .. } => match status { + EntryStatus::Change(WorktreeChange::Removed) => { + WorktreeChangeClassV1::UnstagedDeleted + } + EntryStatus::Conflict { .. } => WorktreeChangeClassV1::Conflicted, + EntryStatus::IntentToAdd => WorktreeChangeClassV1::Untracked, + // Any other worktree change (content, type, submodule) is an + // unstaged modification of present content. + EntryStatus::Change(_) | EntryStatus::NeedsUpdate(_) => { + WorktreeChangeClassV1::UnstagedModified + } + }, + IndexWorktreeItem::DirectoryContents { .. } => WorktreeChangeClassV1::Untracked, + // Unreachable with rewrite tracking disabled; skip defensively. + IndexWorktreeItem::Rewrite { .. } => return None, + }, + }) +} + +#[cfg(test)] +mod nested_repository_tests { + use std::fs; + use std::path::Path; + use std::process::Command; + + use super::WorktreeChangeClassificationV1; + + fn fixture_git(root: &Path, args: &[&str]) { + let program = tracedecay_runtime_core::git::try_git_program().expect("fixture Git program"); + let mut command = Command::new(program); + command.current_dir(root); + for (key, _) in std::env::vars_os() { + if key.to_string_lossy().starts_with("GIT_") { + command.env_remove(key); + } + } + let output = command + .env("GIT_CONFIG_NOSYSTEM", "1") + .env("GIT_CONFIG_GLOBAL", root.join(".fixture-gitconfig")) + .args([ + "-c", + "user.name=Fixture", + "-c", + "user.email=fixture@example.invalid", + "-c", + "commit.gpgsign=false", + ]) + .args(args) + .output() + .expect("fixture Git command"); + assert!( + output.status.success(), + "Git {args:?}: {}", + String::from_utf8_lossy(&output.stderr) + ); + } + + #[test] + fn nested_checkout_marker_tombstones_parent_tracked_sources() { + let root = tempfile::tempdir().unwrap(); + fixture_git(root.path(), &["init", "-q"]); + for path in ["src/keep.rs", "nested/foreign.rs", "linked/foreign.rs"] { + let full = root.path().join(path); + fs::create_dir_all(full.parent().unwrap()).unwrap(); + fs::write(full, "pub fn source() {}\n").unwrap(); + } + fixture_git(root.path(), &["add", "."]); + fixture_git(root.path(), &["commit", "-qm", "parent tracked files"]); + + let repository = tracedecay_runtime_core::git_open::open(root.path()).unwrap(); + let before = WorktreeChangeClassificationV1::classify(&repository).unwrap(); + assert!(!before.has_changes()); + assert!(before.candidate_paths().contains("nested/foreign.rs")); + + fs::create_dir(root.path().join("nested/.git")).unwrap(); + fs::write( + root.path().join("nested/.git/HEAD"), + "ref: refs/heads/main\n", + ) + .unwrap(); + fs::write( + root.path().join("linked/.git"), + "gitdir: /untrusted/worktrees/linked\n", + ) + .unwrap(); + let hidden = WorktreeChangeClassificationV1::classify(&repository).unwrap(); + let candidates = hidden.candidate_paths(); + assert!(candidates.contains("src/keep.rs")); + assert!(!candidates.contains("nested/foreign.rs")); + assert!(!candidates.contains("linked/foreign.rs")); + assert!( + hidden.has_changes(), + "a clean HEAD-tree capture is now unsafe" + ); + assert!(hidden.changed_paths().contains("nested/foreign.rs")); + assert!(hidden.changed_paths().contains("linked/foreign.rs")); + + fs::remove_dir_all(root.path().join("nested/.git")).unwrap(); + fs::remove_file(root.path().join("linked/.git")).unwrap(); + let restored = WorktreeChangeClassificationV1::classify(&repository).unwrap(); + assert!(!restored.has_changes()); + assert!(restored.candidate_paths().contains("nested/foreign.rs")); + assert!(restored.candidate_paths().contains("linked/foreign.rs")); + } + + #[cfg(unix)] + #[test] + fn tracked_symlink_into_submodule_is_not_parent_source() { + use std::os::unix::fs::symlink; + + let fixture = tempfile::tempdir().unwrap(); + let parent = fixture.path().join("parent"); + let child = fixture.path().join("child"); + fs::create_dir_all(&parent).unwrap(); + fs::create_dir_all(&child).unwrap(); + fixture_git(&child, &["init", "-q"]); + fs::create_dir_all(child.join("src")).unwrap(); + fs::write(child.join("src/lib.rs"), "pub fn child() {}\n").unwrap(); + fixture_git(&child, &["add", "."]); + fixture_git(&child, &["commit", "-qm", "child"]); + fixture_git(&parent, &["init", "-q"]); + fixture_git( + &parent, + &[ + "-c", + "protocol.file.allow=always", + "submodule", + "add", + "-q", + "../child", + "app", + ], + ); + symlink("app/src/lib.rs", parent.join("bridge.rs")).unwrap(); + fs::write(parent.join("parent.rs"), "pub fn parent() {}\n").unwrap(); + symlink("parent.rs", parent.join("local.rs")).unwrap(); + fixture_git( + &parent, + &[ + "add", + ".gitmodules", + "app", + "bridge.rs", + "parent.rs", + "local.rs", + ], + ); + fixture_git( + &parent, + &["commit", "-qm", "registered child and source alias"], + ); + assert_eq!( + fs::read_to_string(parent.join("bridge.rs")).unwrap(), + "pub fn child() {}\n", + "fixture must prove the parent symlink actually reaches child bytes" + ); + assert!( + fs::symlink_metadata(parent.join("bridge.rs")) + .unwrap() + .file_type() + .is_symlink() + ); + + let repository = tracedecay_runtime_core::git_open::open(&parent).unwrap(); + for include_submodules in [false, true] { + let classified = + WorktreeChangeClassificationV1::for_policy(&repository, include_submodules) + .unwrap(); + let candidates = classified.candidate_paths(); + let boundaries = crate::source_walk::submodules::LiveSourceBoundaries::open( + &parent, + include_submodules, + ) + .unwrap(); + let (_, relative) = boundaries.resolve(Path::new("local.rs")).unwrap().unwrap(); + assert_eq!(relative.as_ref(), Path::new("parent.rs")); + assert!( + boundaries + .resolve(Path::new("bridge.rs")) + .unwrap() + .is_none() + ); + assert!(candidates.contains("local.rs")); + assert!(!candidates.contains("bridge.rs")); + assert!(classified.changed_paths().contains("bridge.rs")); + assert!( + classified.has_changes(), + "do not reuse HEAD tree source bytes" + ); + assert_eq!(candidates.contains("app/src/lib.rs"), include_submodules); + } + symlink("parent.rs", parent.join("untracked_alias.rs")).unwrap(); + let classification = WorktreeChangeClassificationV1::classify(&repository).unwrap(); + assert!( + !classification + .candidate_paths() + .contains("untracked_alias.rs") + ); + } + + #[test] + fn staged_gitlink_removal_without_child_marker_does_not_admit_child_sources() { + let fixture = tempfile::tempdir().unwrap(); + let parent = fixture.path().join("parent"); + let child = fixture.path().join("child"); + fs::create_dir_all(&parent).unwrap(); + fs::create_dir_all(&child).unwrap(); + + fixture_git(&child, &["init", "-q"]); + fs::create_dir_all(child.join("src")).unwrap(); + fs::write(child.join("src/lib.rs"), "pub fn child() {}\n").unwrap(); + fixture_git(&child, &["add", "."]); + fixture_git(&child, &["commit", "-qm", "child"]); + + fixture_git(&parent, &["init", "-q"]); + fs::create_dir_all(parent.join("src")).unwrap(); + fs::write(parent.join("src/lib.rs"), "pub fn parent() {}\n").unwrap(); + fixture_git(&parent, &["add", "."]); + fixture_git(&parent, &["commit", "-qm", "parent"]); + fixture_git( + &parent, + &[ + "-c", + "protocol.file.allow=always", + "submodule", + "add", + "-q", + "../child", + "app", + ], + ); + fixture_git(&parent, &["add", ".gitmodules", "app"]); + fixture_git(&parent, &["commit", "-qm", "register submodule"]); + + // A registered, clean gitlink is not itself a source file or a + // dirty nested descendant. It must not defeat clean HEAD reuse. + let clean_repository = tracedecay_runtime_core::git_open::open(&parent).unwrap(); + let clean = WorktreeChangeClassificationV1::classify(&clean_repository).unwrap(); + assert!(!clean.has_changes()); + assert!(clean.candidate_paths().contains("src/lib.rs")); + + // Git still records app as a mode-160000 gitlink in HEAD, but + // removing the staged gitlink and child marker makes child files + // appear untracked to the parent. They must never enter its index. + fs::remove_file(parent.join("app/.git")).unwrap(); + fixture_git(&parent, &["rm", "-q", "--cached", "app"]); + assert!(parent.join("app/src/lib.rs").is_file()); + + let repository = tracedecay_runtime_core::git_open::open(&parent).unwrap(); + let classified = WorktreeChangeClassificationV1::classify(&repository).unwrap(); + assert!( + classified + .changes() + .iter() + .any(|change| change.path == "app/src/lib.rs"), + "fixture must exercise a child file reported by parent Git status" + ); + assert!(classified.has_changes(), "clean HEAD reuse must be refused"); + assert!(!classified.candidate_paths().contains("app/src/lib.rs")); + assert!(classified.changed_paths().contains("app/src/lib.rs")); + assert!(classified.candidate_paths().contains("src/lib.rs")); + } +} diff --git a/crates/tracedecay-code-index/src/source_walk/nested_repository_tests.rs b/crates/tracedecay-code-index/src/source_walk/nested_repository_tests.rs new file mode 100644 index 0000000000..406671e045 --- /dev/null +++ b/crates/tracedecay-code-index/src/source_walk/nested_repository_tests.rs @@ -0,0 +1,197 @@ +use std::collections::BTreeSet; +use std::path::{Path, PathBuf}; + +use tracedecay_domain::IndexPathPolicyV1; +use tracedecay_domain::errors::TraceDecayError; + +use super::{SourceWalkDiagnostics, source_walk_with_diagnostics}; +use crate::unmounted_files::audit_project; + +fn write(root: &Path, relative: &str, contents: &str) { + let path = root.join(relative); + std::fs::create_dir_all(path.parent().expect("fixture parent")).unwrap(); + std::fs::write(path, contents).unwrap(); +} + +#[test] +fn repository_diagnostic_sample_is_bounded_and_order_independent() { + let expected = (0..8) + .map(|index| format!("repo-{index:02}")) + .collect::>(); + let orders = [ + (0..24).collect::>(), + (0..24).rev().collect::>(), + ]; + for order in orders { + let diagnostics = SourceWalkDiagnostics::default(); + assert!(diagnostics.skipped_nested_repositories_note().is_none()); + for index in order { + diagnostics.record_skipped_nested_repository(format!("repo-{index:02}")); + let summary = diagnostics.skipped_nested_repositories.lock().unwrap(); + assert!(summary.names.len() <= 8, "collection must be bounded too"); + } + let summary = diagnostics.skipped_nested_repositories.lock().unwrap(); + assert_eq!(summary.total, 24); + assert_eq!(summary.names, expected); + drop(summary); + let note = diagnostics.skipped_nested_repositories_note().unwrap(); + assert_eq!( + note, + "Skipped nested repositories outside parent index: \ + \"repo-00\", \"repo-01\", \"repo-02\", \"repo-03\", \ + \"repo-04\", \"repo-05\", \"repo-06\", \"repo-07\" (and 16 more)" + ); + } +} + +#[test] +fn repository_diagnostics_follow_policy_without_crossing_boundaries() { + let root = tempfile::tempdir().unwrap(); + write(root.path(), "keep.rs", "pub fn parent() {}\n"); + for directory in ["app", "excluded", "node_modules"] { + write( + root.path(), + &format!("{directory}/.git"), + "gitdir: foreign\n", + ); + write( + root.path(), + &format!("{directory}/child.rs"), + "child source\n", + ); + } + for (include, expected_note) in [ + ( + vec![], + "Skipped nested repositories outside parent index: \"app\"", + ), + ( + vec!["excluded/child.rs".to_owned()], + "Skipped nested repositories outside parent index: \"app\", \"excluded\"", + ), + ] { + let policy = IndexPathPolicyV1::new( + vec!["excluded/**".to_owned(), "node_modules/**".to_owned()], + include, + ) + .unwrap(); + let (walk, diagnostics) = + source_walk_with_diagnostics(root.path(), None, &policy).expect("source walk"); + let files = walk + .map(|entry| entry.expect("source entry")) + .filter(|entry| entry.file_type().is_some_and(|kind| kind.is_file())) + .map(|entry| { + entry + .path() + .strip_prefix(root.path()) + .unwrap() + .to_path_buf() + }) + .collect::>(); + assert_eq!(files, vec![PathBuf::from("keep.rs")]); + assert_eq!( + diagnostics.skipped_nested_repositories_note().as_deref(), + Some(expected_note) + ); + } +} + +#[test] +fn audit_bounds_repository_summary_and_preserves_unsupported_caveat() { + let root = tempfile::tempdir().unwrap(); + write(root.path(), "main.go", "package main\n"); + for index in (0..12).rev() { + write( + root.path(), + &format!("repo-{index:02}/.git/HEAD"), + "ref: refs/heads/main\n", + ); + write( + root.path(), + &format!("repo-{index:02}/child.go"), + "package child\n", + ); + } + let policy = IndexPathPolicyV1::new(vec![], vec![]).unwrap(); + let audit = audit_project(root.path(), &policy).expect("audit"); + let go = audit + .ecosystems + .iter() + .find(|ecosystem| ecosystem.ecosystem == "go") + .expect("parent Go source was counted"); + assert_eq!(go.scanned_file_count, 1); + assert_eq!(go.status.as_str(), "unsupported"); + assert_eq!( + go.note.as_deref(), + Some( + "1 go source file(s) are present and were not audited, \ + this report cannot say whether any of them is unreachable \ + Skipped nested repositories outside parent index: \ + \"repo-00\", \"repo-01\", \"repo-02\", \"repo-03\", \ + \"repo-04\", \"repo-05\", \"repo-06\", \"repo-07\" (and 4 more)" + ) + ); +} + +#[test] +fn repository_diagnostic_paths_are_clipped_on_character_boundaries() { + let diagnostics = SourceWalkDiagnostics::default(); + diagnostics.record_skipped_nested_repository("é".repeat(161)); + assert_eq!( + diagnostics.skipped_nested_repositories_note().unwrap(), + format!( + "Skipped nested repositories outside parent index: \"{}\"…", + "é".repeat(160) + ) + ); +} + +#[cfg(unix)] +#[test] +fn audit_quotes_control_characters_in_repository_names() { + let root = tempfile::tempdir().unwrap(); + write(root.path(), "keep.py", "print('parent')\n"); + write(root.path(), "line\nbreak/.git", "gitdir: foreign\n"); + write(root.path(), "line\nbreak/child.py", "print('child')\n"); + let policy = IndexPathPolicyV1::new(vec![], vec![]).unwrap(); + let audit = audit_project(root.path(), &policy).expect("audit"); + let python = audit + .ecosystems + .iter() + .find(|ecosystem| ecosystem.ecosystem == "python") + .expect("parent Python source was counted"); + assert_eq!(python.scanned_file_count, 1); + let note = python.note.as_deref().expect("scope diagnostic"); + assert!(note.contains("\"line\\nbreak\""), "{note}"); + assert!(!note.contains('\n'), "a path must not inject report lines"); + assert!(root.path().join("line\nbreak/child.py").is_file()); +} + +#[test] +fn missing_source_root_is_an_error_not_a_clean_audit() { + let root = tempfile::tempdir().unwrap(); + let missing = root.path().join("missing"); + let policy = IndexPathPolicyV1::new(vec![], vec![]).unwrap(); + match audit_project(&missing, &policy) { + Err(TraceDecayError::File { path, message }) => { + assert_eq!(path, missing.display().to_string()); + assert!(message.contains("source walk did not complete")); + } + Err(error) => panic!("unexpected error: {error}"), + Ok(_) => panic!("a failed walk must not report an empty successful audit"), + } +} + +#[test] +fn incomplete_git_root_is_refused_before_walking_sources() { + let root = tempfile::tempdir().unwrap(); + write(root.path(), "src/lib.rs", "pub fn source() {}"); + let policy = IndexPathPolicyV1::new(vec![], vec![]).unwrap(); + assert!(audit_project(root.path(), &policy).is_ok()); + + std::fs::create_dir(root.path().join(".git")).unwrap(); + assert!(matches!( + audit_project(root.path(), &policy), + Err(TraceDecayError::Config { message }) if message.contains("parent Git boundaries") + )); +} diff --git a/crates/tracedecay-code-index/src/source_walk/submodules.rs b/crates/tracedecay-code-index/src/source_walk/submodules.rs new file mode 100644 index 0000000000..0c068e8d32 --- /dev/null +++ b/crates/tracedecay-code-index/src/source_walk/submodules.rs @@ -0,0 +1,302 @@ +//! A parent gitlink names a commit, not the worktree that may supply its bytes. +//! Bind the registration, object store, and checkout before crossing that boundary. + +use std::borrow::Cow; +use std::collections::{BTreeMap, BTreeSet}; +use std::path::{Component, Path, PathBuf}; + +use gix::bstr::ByteSlice; + +use super::{SourceWalkError, has_nested_repository_ancestor}; + +fn failure(error: impl std::fmt::Display) -> SourceWalkError { + SourceWalkError { + glob: String::new(), + message: format!("cannot inspect registered submodule: {error}"), + } +} + +#[derive(Clone, Debug)] +pub struct AuthenticatedSubmodule { + pub worktree: PathBuf, + pub git_dir: PathBuf, + pub pin: gix::ObjectId, +} + +pub(super) struct ChildSources { + pub files: BTreeSet, + pub gitlinks: BTreeSet, +} + +impl AuthenticatedSubmodule { + pub(super) fn sources(&self) -> Result { + let repository = tracedecay_runtime_core::git_open::open(&self.git_dir).map_err(failure)?; + let inventory = + super::classification::WorktreeChangeClassificationV1::classify(&repository) + .map_err(failure)?; + Ok(ChildSources { + files: inventory + .regular_child_files(&self.worktree) + .map_err(failure)?, + gitlinks: inventory.gitlink_paths().clone(), + }) + } +} + +pub(super) struct CheckoutBoundaries { + pub gitlinks: BTreeSet, + pub children: BTreeMap, +} + +impl CheckoutBoundaries { + pub(super) fn open(root: &Path, include_submodules: bool) -> Result { + let gitlinks = super::parent_gitlink_paths(root)?; + let children = if include_submodules && !gitlinks.is_empty() { + let parent = tracedecay_runtime_core::git_open::open(root).map_err(failure)?; + live_submodules(&parent)? + } else { + BTreeMap::new() + }; + Ok(Self { gitlinks, children }) + } +} + +fn normal_relative(path: &Path) -> bool { + !path.as_os_str().is_empty() + && path + .components() + .all(|component| matches!(component, Component::Normal(_))) +} + +/// Administrative data is outside every parent source surface, even with an explicit glob. +pub fn is_source_location(relative: &Path) -> bool { + normal_relative(relative) + && !relative.components().any(|component| { + component.as_os_str() == ".git" || component.as_os_str() == ".tracedecay" + }) +} + +fn registrations(bytes: &[u8]) -> Result, SourceWalkError> { + let modules = gix::submodule::File::from_bytes(bytes, None, &gix::config::File::default()) + .map_err(failure)?; + let mut registrations = BTreeMap::new(); + for name in modules.names() { + let name = gix::validate::submodule::name(name).map_err(failure)?; + let path = modules.path(name).map_err(failure)?; + let path = path.to_str().map_err(failure)?; + if !is_source_location(Path::new(path)) { + continue; + } + let name = PathBuf::from(name.to_str().map_err(failure)?); + if !normal_relative(&name) + || !normal_relative(Path::new(path)) + || registrations.insert(path.to_owned(), name).is_some() + { + return Err(failure("ambiguous or invalid .gitmodules path")); + } + } + Ok(registrations) +} + +fn module_stores<'a>( + parent: &'a gix::Repository, + name: &'a Path, +) -> impl Iterator + 'a { + [parent.git_dir(), parent.common_dir()] + .into_iter() + .filter_map(move |directory| { + let expected = directory.canonicalize().ok()?.join("modules").join(name); + (expected.canonicalize().ok()? == expected).then_some(expected) + }) +} + +fn open_pinned_store(expected: &Path, pin: gix::ObjectId) -> Option { + let child = tracedecay_runtime_core::git_open::open(expected).ok()?; + if child.git_dir().canonicalize().ok()? != expected + || child.common_dir().canonicalize().ok()? != expected + { + return None; + } + child.find_commit(pin).ok()?; + Some(child) +} + +fn bound_checkout( + parent: &gix::Repository, + path: &str, + name: &Path, + pin: gix::ObjectId, +) -> Option { + let root = parent.workdir()?.canonicalize().ok()?; + let path = Path::new(path); + if has_nested_repository_ancestor(&root, path).ok()? { + return None; + } + let worktree = root.join(path); + let marker = worktree.join(".git"); + if worktree.canonicalize().ok()? != worktree || !marker.symlink_metadata().ok()?.is_file() { + return None; + } + let git_dir = gix::discover::path::from_gitdir_file(&marker) + .ok()? + .canonicalize() + .ok()?; + // Git initializes children of a linked parent under that worktree's + // administrative directory. Check the name-derived, canonical location + // before opening any repository named by the checkout's marker. + let expected = module_stores(parent, name).find(|candidate| *candidate == git_dir)?; + let child = open_pinned_store(&expected, pin)?; + if child.workdir()?.canonicalize().ok()? != worktree { + return None; + } + Some(AuthenticatedSubmodule { + worktree, + git_dir: expected, + pin, + }) +} + +pub fn live_submodules( + parent: &gix::Repository, +) -> Result, SourceWalkError> { + let mut children = BTreeMap::new(); + let index = parent.index_or_empty().map_err(failure)?; + let stage = gix::index::entry::Stage::Unconflicted; + let Some(modules) = index.entry_by_path_and_stage(".gitmodules".into(), stage) else { + return Ok(children); + }; + if !modules + .mode + .to_tree_entry_mode() + .is_some_and(|mode| mode.is_blob()) + { + return Ok(children); + } + let blob = parent.find_blob(modules.id).map_err(failure)?; + let registered = registrations(&blob.data)?; + // An unstaged deletion or replacement of the registration cannot authorize a checkout. + let Some(live_registration) = parent.open_modules_file().map_err(failure)? else { + return Ok(children); + }; + for (path, name) in registered { + let Some(entry) = index.entry_by_path_and_stage(path.as_bytes().as_bstr(), stage) else { + continue; + }; + if !entry.mode.is_submodule() { + continue; + } + if !live_registration + .path(name.as_os_str().as_encoded_bytes().as_bstr()) + .is_ok_and(|current| current.as_slice() == path.as_bytes()) + { + continue; + } + if let Some(child) = bound_checkout(parent, &path, &name, entry.id) { + children.insert(path, child); + } + } + Ok(children) +} + +/// Batch the same Git inventory for foreground graph readers. A graph path can +/// outlive its checkout binding, even when a replacement file has equal bytes. +pub struct LiveSourceBoundaries { + project_root: PathBuf, + gitlinks: BTreeSet, + children: BTreeMap)>, +} + +impl LiveSourceBoundaries { + pub fn open(root: &Path, include_submodules: bool) -> Result { + let boundaries = CheckoutBoundaries::open(root, include_submodules)?; + let mut children = BTreeMap::new(); + for (prefix, child) in boundaries.children { + let files = child.sources()?.files; + children.insert(prefix, (child, files)); + } + Ok(Self { + project_root: root.to_owned(), + gitlinks: boundaries.gitlinks, + children, + }) + } + + /// The owning checkout and its relative file path, after checking the + /// current filesystem boundary before reading any bytes. + pub fn resolve<'a, 'p>( + &'a self, + path: &'p Path, + ) -> std::io::Result)>> { + if !is_source_location(path) { + return Ok(None); + } + let mut root = self.project_root.as_path(); + let mut relative = path; + for ancestor in path.ancestors().skip(1).filter_map(Path::to_str) { + if !self.gitlinks.contains(ancestor) { + continue; + } + let Some((child, files)) = self.children.get(ancestor) else { + return Ok(None); + }; + relative = path.strip_prefix(ancestor).map_err(std::io::Error::other)?; + if !relative.to_str().is_some_and(|file| files.contains(file)) + || child.worktree.canonicalize().ok().as_ref() != Some(&child.worktree) + { + return Ok(None); + } + let marker = child.worktree.join(".git"); + if !marker + .symlink_metadata() + .is_ok_and(|metadata| metadata.is_file()) + || gix::discover::path::from_gitdir_file(&marker) + .ok() + .and_then(|target| target.canonicalize().ok()) + .as_ref() + != Some(&child.git_dir) + { + return Ok(None); + } + root = &child.worktree; + break; + } + if has_nested_repository_ancestor(root, relative)? { + return Ok(None); + } + if root == self.project_root { + Ok(super::resolve_parent_source(root, relative)?.map(|path| (root, path))) + } else if super::is_source_symlink(root, relative)? { + Ok(None) + } else { + Ok(Some((root, Cow::Borrowed(relative)))) + } + } +} + +/// Only the selected parent tree may supply historical registrations and pins. +pub fn pinned_submodules( + parent: &gix::Repository, + pins: &tracedecay_runtime_core::git_repository::GitTreeSubmodulePins, +) -> Result, SourceWalkError> { + let mut children = BTreeMap::new(); + if pins.gitlinks.is_empty() { + return Ok(children); + } + let registrations = registrations(&pins.registration)?; + for (path, pin) in &pins.gitlinks { + let Some(name) = registrations.get(path) else { + continue; + }; + // The checkout can move or be deinitialized after the parent revision. + // Its current bytes and .git marker are never historical authorities. + let repo = module_stores(parent, name) + .find_map(|directory| open_pinned_store(&directory, *pin)) + .ok_or_else(|| { + failure(format!( + "registered object store or pinned commit unavailable for {path}" + )) + })?; + children.insert(path.clone(), (repo, *pin)); + } + Ok(children) +} diff --git a/crates/tracedecay-code-index/src/source_walk/tracked_links.rs b/crates/tracedecay-code-index/src/source_walk/tracked_links.rs new file mode 100644 index 0000000000..35d47fa914 --- /dev/null +++ b/crates/tracedecay-code-index/src/source_walk/tracked_links.rs @@ -0,0 +1,94 @@ +//! Resolve a tracked source alias using the parent Git index and live worktree. + +use std::borrow::Cow; +use std::io; +use std::path::{Component, Path}; + +use gix::bstr::ByteSlice; +use gix::index::entry::{Mode, Stage}; +use tracedecay_runtime_core::git_repository::{RepositoryLinkEntry, resolve_repository_link}; + +use super::{is_source_symlink, nested_repository_marker, submodules::is_source_location}; + +/// Ordinary files keep their original path. A live parent alias may only read +/// a tracked regular file through tracked links and parent-owned directories. +pub fn resolve_parent_source<'a>( + root: &Path, + relative: &'a Path, +) -> io::Result>> { + if !is_source_symlink(root, relative)? { + return Ok(Some(Cow::Borrowed(relative))); + } + if !nested_repository_marker(root)? { + return Ok(None); + } + let repo = tracedecay_runtime_core::git_open::open(root).map_err(io::Error::other)?; + let index = repo.index_or_empty().map_err(io::Error::other)?; + let head = repo.head().map_err(io::Error::other)?; + let tree = if head.id().is_some() { + Some(repo.head_tree().map_err(io::Error::other)?) + } else { + None + }; + let alias = gix::path::into_bstr(relative).map_err(io::Error::other)?; + let alias = gix::path::to_unix_separators_on_windows(alias); + if !index + .entry_by_path_and_stage(&alias, Stage::Unconflicted) + .is_some_and(|entry| entry.mode == Mode::SYMLINK) + { + return Ok(None); + } + resolve_repository_link(b"", &alias, |components| { + let bytes = components.join(&b'/'); + let path = gix::path::from_byte_slice(&bytes).map_err(io::Error::other)?; + if !is_source_location(path) { + return Ok(None); + } + let absolute = root.join(path); + let metadata = match absolute.symlink_metadata() { + Ok(metadata) => metadata, + Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(None), + Err(error) => return Err(error), + }; + let indexed = index.entry_by_path_and_stage(bytes.as_bstr(), Stage::Unconflicted); + if metadata.is_dir() { + // A gitlink staged for removal still bounds the old checkout even + // after its marker disappears. Check HEAD before descending. + let head_entry = tree + .as_ref() + .map(|tree| tree.lookup_entry(components.iter().map(Vec::as_slice))) + .transpose() + .map_err(io::Error::other)? + .flatten(); + if index.entry_by_path(bytes.as_bstr()).is_some() + || head_entry.is_some_and(|entry| entry.mode().is_commit()) + || nested_repository_marker(&absolute)? + { + return Ok(None); + } + return Ok(Some(RepositoryLinkEntry::Directory)); + } + let Some(entry) = indexed else { + return Ok(None); + }; + if metadata.file_type().is_symlink() && entry.mode == Mode::SYMLINK { + let target = std::fs::read_link(&absolute)?; + if target + .components() + .any(|component| matches!(component, Component::Prefix(_) | Component::RootDir)) + { + return Ok(None); + } + let target = gix::path::into_bstr(target).map_err(io::Error::other)?; + return Ok(Some(RepositoryLinkEntry::Link( + gix::path::to_unix_separators_on_windows(target).to_vec(), + ))); + } + Ok((metadata.is_file() + && entry + .mode + .to_tree_entry_mode() + .is_some_and(|mode| mode.is_blob())) + .then(|| RepositoryLinkEntry::File(Cow::Owned(path.to_owned())))) + }) +} diff --git a/crates/tracedecay-code-index/src/unmounted_files.rs b/crates/tracedecay-code-index/src/unmounted_files.rs index 1d1cb19e49..31cf2d6f59 100644 --- a/crates/tracedecay-code-index/src/unmounted_files.rs +++ b/crates/tracedecay-code-index/src/unmounted_files.rs @@ -152,19 +152,22 @@ pub struct ProjectAudit { pub(super) struct ProjectFiles { root: PathBuf, files: Vec, + nested_repository_note: Option, } impl ProjectFiles { fn collect(project_root: &Path, path_policy: &IndexPathPolicyV1) -> Result { - let walk = - crate::source_walk::source_walk(project_root, None, path_policy).map_err(|error| { - TraceDecayError::Config { + let (walk, diagnostics) = + crate::source_walk::source_walk_with_diagnostics(project_root, None, path_policy) + .map_err(|error| TraceDecayError::Config { message: format!("source walk rejected its own scope: {}", error.message), - } - })?; + })?; let mut files = Vec::new(); for entry in walk { - let Ok(entry) = entry else { continue }; + let entry = entry.map_err(|error| TraceDecayError::File { + message: format!("source walk did not complete: {error}"), + path: project_root.display().to_string(), + })?; if !entry.file_type().is_some_and(|kind| kind.is_file()) { continue; } @@ -174,6 +177,7 @@ impl ProjectFiles { Ok(Self { root: project_root.to_path_buf(), files, + nested_repository_note: diagnostics.skipped_nested_repositories_note(), }) } @@ -258,6 +262,14 @@ pub fn audit_project(project_root: &Path, path_policy: &IndexPathPolicyV1) -> Re }; let mut ecosystems = vec![rust, typescript]; ecosystems.extend(unmodelled_ecosystems(&files)); + if let Some(summary) = &files.nested_repository_note { + for ecosystem in &mut ecosystems { + ecosystem.note = Some(match ecosystem.note.take() { + Some(existing) => format!("{existing} {summary}"), + None => summary.clone(), + }); + } + } Ok(ProjectAudit { ecosystems }) } @@ -395,6 +407,44 @@ mod tests { ); } + #[test] + fn audit_reports_nested_repository_exclusion_without_claiming_child_code() { + let root = tempfile::tempdir().expect("project fixture"); + write( + root.path(), + "Cargo.toml", + "[package]\nname = \"parent\"\nversion = \"0.0.0\"\n", + ); + write(root.path(), "src/lib.rs", "pub fn parent() {}\n"); + write(root.path(), "app/.git", "gitdir: ../.git/modules/app\n"); + write(root.path(), "app/src/child.rs", "pub fn child() {}\n"); + write(root.path(), "unrelated/.git/HEAD", "ref: refs/heads/main\n"); + write( + root.path(), + "unrelated/src/foreign.rs", + "pub fn foreign() {}\n", + ); + + let audit = audit_project(root.path(), &no_exclusions()).expect("audit"); + let rust = ecosystem(&audit, "rust"); + assert_eq!( + rust.scanned_file_count, 1, + "child repositories are not parent source" + ); + assert!(rust.unmounted.iter().all(|file| { + !file.file.starts_with("app/") && !file.file.starts_with("unrelated/") + })); + let note = rust.note.as_deref().expect("explicit scope diagnostic"); + assert!(note.contains("Skipped nested repositories"), "{note}"); + assert!(note.contains("app") && note.contains("unrelated"), "{note}"); + assert!(audit.ecosystems.iter().all(|ecosystem| { + ecosystem + .note + .as_deref() + .is_some_and(|note| note.contains("Skipped nested repositories")) + })); + } + /// Both modelled ecosystems report in one pass over a polyglot repository. #[test] fn a_polyglot_project_reports_one_section_per_ecosystem() { diff --git a/crates/tracedecay-code-index/tests/code_index_suite/production_orchestration.rs b/crates/tracedecay-code-index/tests/code_index_suite/production_orchestration.rs index 1e1558def4..5ef6dd9e3d 100644 --- a/crates/tracedecay-code-index/tests/code_index_suite/production_orchestration.rs +++ b/crates/tracedecay-code-index/tests/code_index_suite/production_orchestration.rs @@ -277,6 +277,7 @@ impl CodeIndexExecutionControlV1 for MutableCancellationControl { pub(super) fn config() -> CodeIndexProductionConfigV1 { CodeIndexProductionConfigV1 { + include_submodules: false, project_id: id::("project.production"), repository: id::("repository.production"), sanitizer_revision: id::("sanitizer.v1"), @@ -319,6 +320,7 @@ fn request_in_scope( request.snapshot.worktree = worktree.map(id::); request.snapshot.source_revision = Some(id::(source_revision)); request.repository_parse_identity = CodeIndexRepositoryParseIdentityV1 { + include_submodules: false, tree: Some(id::(&format!("tree.{source_revision}"))), dirty: RepositoryDirtyStateV1::Dirty, }; @@ -365,6 +367,7 @@ fn request_at_path( invalidations: BTreeSet::new(), ignored_source_admissions: Vec::new(), repository_parse_identity: CodeIndexRepositoryParseIdentityV1 { + include_submodules: false, tree: None, dirty: RepositoryDirtyStateV1::Dirty, }, @@ -1033,6 +1036,7 @@ fn resumed_parse_quanta_publish_the_same_complete_generation() { invalidations: BTreeSet::new(), ignored_source_admissions: Vec::new(), repository_parse_identity: CodeIndexRepositoryParseIdentityV1 { + include_submodules: false, tree: None, dirty: RepositoryDirtyStateV1::Dirty, }, @@ -3003,6 +3007,7 @@ fn partitioned_request( invalidations: BTreeSet::new(), ignored_source_admissions: Vec::new(), repository_parse_identity: CodeIndexRepositoryParseIdentityV1 { + include_submodules: false, tree: None, dirty: RepositoryDirtyStateV1::Dirty, }, diff --git a/crates/tracedecay-code-index/tests/code_index_suite/production_orchestration/parallel_equivalence.rs b/crates/tracedecay-code-index/tests/code_index_suite/production_orchestration/parallel_equivalence.rs index a23f502b50..7f70e4b9f8 100644 --- a/crates/tracedecay-code-index/tests/code_index_suite/production_orchestration/parallel_equivalence.rs +++ b/crates/tracedecay-code-index/tests/code_index_suite/production_orchestration/parallel_equivalence.rs @@ -92,6 +92,7 @@ fn multi_file_request(file_count: usize, sealed_at: i64) -> CodeIndexBuildReques invalidations: BTreeSet::new(), ignored_source_admissions: Vec::new(), repository_parse_identity: CodeIndexRepositoryParseIdentityV1 { + include_submodules: false, tree: None, dirty: RepositoryDirtyStateV1::Dirty, }, diff --git a/crates/tracedecay-code-index/tests/resident_accounting.rs b/crates/tracedecay-code-index/tests/resident_accounting.rs index 18a6afcf49..05fd76256c 100644 --- a/crates/tracedecay-code-index/tests/resident_accounting.rs +++ b/crates/tracedecay-code-index/tests/resident_accounting.rs @@ -154,6 +154,7 @@ fn request(files: usize) -> CodeIndexBuildRequestV1 { invalidations: BTreeSet::new(), ignored_source_admissions: Vec::new(), repository_parse_identity: CodeIndexRepositoryParseIdentityV1 { + include_submodules: false, tree: None, dirty: RepositoryDirtyStateV1::Dirty, }, @@ -168,6 +169,7 @@ fn request(files: usize) -> CodeIndexBuildRequestV1 { fn config() -> CodeIndexProductionConfigV1 { CodeIndexProductionConfigV1 { + include_submodules: false, project_id: id::("project.accounting"), repository: id::("repository.accounting"), sanitizer_revision: id::("sanitizer.v1"), diff --git a/crates/tracedecay-configuration/src/config/mod.rs b/crates/tracedecay-configuration/src/config/mod.rs index 12628536d7..52d0ca5725 100644 --- a/crates/tracedecay-configuration/src/config/mod.rs +++ b/crates/tracedecay-configuration/src/config/mod.rs @@ -22,9 +22,9 @@ use tracedecay_domain::configuration::{ AUTOMATION_SETTINGS_SETTING_KEY, AutomationSettingsV1, ConfigurationRevisionId, ConfigurationSnapshotV1, ConfigurationValueV1, INDEX_EXCLUDE_SETTING_KEY, INDEX_EXTRACT_DOCSTRINGS_SETTING_KEY, INDEX_INCLUDE_SETTING_KEY, - INDEX_MAX_FILE_SIZE_SETTING_KEY, INDEX_NATIVE_GRAPH_ACTIVATION_SETTING_KEY, - INDEX_TRACK_CALL_SITES_SETTING_KEY, LCM_SUMMARIZER_EXECUTABLES_SETTING_KEY, - LcmSummarizerExecutablesV1, SYNC_AUTO_INIT_SETTING_KEY, + INDEX_INCLUDE_SUBMODULES_SETTING_KEY, INDEX_MAX_FILE_SIZE_SETTING_KEY, + INDEX_NATIVE_GRAPH_ACTIVATION_SETTING_KEY, INDEX_TRACK_CALL_SITES_SETTING_KEY, + LCM_SUMMARIZER_EXECUTABLES_SETTING_KEY, LcmSummarizerExecutablesV1, SYNC_AUTO_INIT_SETTING_KEY, SYNC_AUTO_TRACK_PR_BRANCHES_SETTING_KEY, SYNC_AUTO_TRACK_PR_POLL_SECS_SETTING_KEY, SYNC_AUTO_WATCH_SETTING_KEY, SYNC_BACKSTOP_INTERVAL_MINS_SETTING_KEY, SYNC_BRANCH_GC_DAYS_SETTING_KEY, SYNC_READ_COOLDOWN_SECS_SETTING_KEY, @@ -277,7 +277,11 @@ fn runtime_config_from_snapshot( required_string_list(snapshot, INDEX_EXCLUDE_SETTING_KEY)?, required_string_list(snapshot, INDEX_INCLUDE_SETTING_KEY)?, ) - .map_err(|error| config_error(format!("resolved index path policy is invalid: {error}")))?, + .map_err(|error| config_error(format!("resolved index path policy is invalid: {error}")))? + .with_submodules(required_bool( + snapshot, + INDEX_INCLUDE_SUBMODULES_SETTING_KEY, + )?), max_file_size: required_unsigned(snapshot, INDEX_MAX_FILE_SIZE_SETTING_KEY)?, extract_docstrings: required_bool(snapshot, INDEX_EXTRACT_DOCSTRINGS_SETTING_KEY)?, track_call_sites: required_bool(snapshot, INDEX_TRACK_CALL_SITES_SETTING_KEY)?, @@ -460,6 +464,28 @@ mod tests { } } + #[test] + fn submodule_policy_is_default_off_and_materializes_the_project_override() { + let default = + PinnedRuntimeConfiguration::new(target(), revision(), resolved(BTreeMap::new())) + .unwrap(); + assert!(!default.config().index_paths.includes_submodules()); + let entries = BTreeMap::from([( + SettingKey::new(super::INDEX_INCLUDE_SUBMODULES_SETTING_KEY).unwrap(), + ConfigurationValueV1::Boolean(true), + )]); + let configured = + PinnedRuntimeConfiguration::new(target(), revision(), resolved(entries)).unwrap(); + assert!(configured.config().index_paths.includes_submodules()); + assert!( + configured + .config() + .index_paths + .without_generated_dir_defaults() + .includes_submodules() + ); + } + #[test] fn pin_rejects_a_snapshot_missing_a_required_setting() { let complete = resolved(BTreeMap::new()); diff --git a/crates/tracedecay-domain/src/configuration.rs b/crates/tracedecay-domain/src/configuration.rs index 601a3eb328..35d41cb26c 100644 --- a/crates/tracedecay-domain/src/configuration.rs +++ b/crates/tracedecay-domain/src/configuration.rs @@ -68,6 +68,7 @@ pub fn is_profile_setting_key(key: &SettingKey) -> bool { /// Canonical project-scoped runtime settings. pub const INDEX_EXCLUDE_SETTING_KEY: &str = "index.exclude.v1"; +pub const INDEX_INCLUDE_SUBMODULES_SETTING_KEY: &str = "index.include_submodules.v1"; pub const INDEX_INCLUDE_SETTING_KEY: &str = "index.include.v1"; pub const INDEX_MAX_FILE_SIZE_SETTING_KEY: &str = "index.max_file_size.v1"; pub const INDEX_EXTRACT_DOCSTRINGS_SETTING_KEY: &str = "index.extract_docstrings.v1"; @@ -110,6 +111,7 @@ pub const CONFIGURATION_SETTING_KEYS_V1: &[&str] = &[ USER_WORK_EXPERTISE_CONSENT_SETTING_KEY, INDEX_EXCLUDE_SETTING_KEY, INDEX_INCLUDE_SETTING_KEY, + INDEX_INCLUDE_SUBMODULES_SETTING_KEY, INDEX_MAX_FILE_SIZE_SETTING_KEY, INDEX_EXTRACT_DOCSTRINGS_SETTING_KEY, INDEX_TRACK_CALL_SITES_SETTING_KEY, diff --git a/crates/tracedecay-domain/src/lib.rs b/crates/tracedecay-domain/src/lib.rs index ffbef3e2d1..764d8cd231 100644 --- a/crates/tracedecay-domain/src/lib.rs +++ b/crates/tracedecay-domain/src/lib.rs @@ -104,9 +104,10 @@ pub use configuration::{ ContextScoutConfigurationStateV1, ContextScoutConfiguredModelPathV1, ContextScoutSettingsV1, CredentialReferenceId, CrossMergeModeV1, CrossMergePolicyV1, GitHubStackedPullRequestPolicyV1, HistoryRewritePolicyV1, INDEX_EXCLUDE_SETTING_KEY, INDEX_EXTRACT_DOCSTRINGS_SETTING_KEY, - INDEX_INCLUDE_SETTING_KEY, INDEX_MAX_FILE_SIZE_SETTING_KEY, - INDEX_NATIVE_GRAPH_ACTIVATION_SETTING_KEY, INDEX_TRACK_CALL_SITES_SETTING_KEY, - LCM_SUMMARIZER_EXECUTABLES_SETTING_KEY, LcmSummarizerExecutableV1, LcmSummarizerExecutablesV1, + INDEX_INCLUDE_SETTING_KEY, INDEX_INCLUDE_SUBMODULES_SETTING_KEY, + INDEX_MAX_FILE_SIZE_SETTING_KEY, INDEX_NATIVE_GRAPH_ACTIVATION_SETTING_KEY, + INDEX_TRACK_CALL_SITES_SETTING_KEY, LCM_SUMMARIZER_EXECUTABLES_SETTING_KEY, + LcmSummarizerExecutableV1, LcmSummarizerExecutablesV1, MAX_WORK_EXPERTISE_CONSENT_LIFETIME_MICROS_V1, PROJECT_WORK_EXPERTISE_CONSENT_SETTING_KEY, ProtectedApplyRequest, ProtectedChange, ProtectedChangePlan, ProtectedChangeSnapshotError, ProtectedRefDispositionV1, ProtectedRefRuleV1, ProtectedRefSelectorV1, QueryCollectionId, diff --git a/crates/tracedecay-domain/src/source_path_policy.rs b/crates/tracedecay-domain/src/source_path_policy.rs index 9871687d7e..9a921d6fb8 100644 --- a/crates/tracedecay-domain/src/source_path_policy.rs +++ b/crates/tracedecay-domain/src/source_path_policy.rs @@ -60,6 +60,7 @@ pub struct IndexPathPatternError { /// files. #[derive(Clone, Debug)] pub struct IndexPathPolicyV1 { + include_submodules: bool, exclude: Vec, include: Vec, exclude_set: GlobSet, @@ -78,7 +79,9 @@ pub struct IndexPathPolicyV1 { impl PartialEq for IndexPathPolicyV1 { fn eq(&self, other: &Self) -> bool { - self.exclude == other.exclude && self.include == other.include + self.include_submodules == other.include_submodules + && self.exclude == other.exclude + && self.include == other.include } } @@ -109,6 +112,7 @@ impl IndexPathPolicyV1 { .then(|| Self::new(configured, include.clone()).map(Box::new)) .transpose()?; Ok(Self { + include_submodules: false, exclude, include, exclude_set, @@ -132,6 +136,18 @@ impl IndexPathPolicyV1 { .unwrap_or(self) } + pub fn with_submodules(mut self, enabled: bool) -> Self { + self.include_submodules = enabled; + if let Some(scoped) = self.without_generated_dir_defaults.as_mut() { + scoped.include_submodules = enabled; + } + self + } + + pub fn includes_submodules(&self) -> bool { + self.include_submodules + } + pub fn exclude_patterns(&self) -> &[String] { &self.exclude } diff --git a/crates/tracedecay-global-db/src/configuration/registry.rs b/crates/tracedecay-global-db/src/configuration/registry.rs index 97d964bf4a..02c3b283cb 100644 --- a/crates/tracedecay-global-db/src/configuration/registry.rs +++ b/crates/tracedecay-global-db/src/configuration/registry.rs @@ -8,9 +8,10 @@ use tracedecay_domain::configuration::{ AnalyzerSettingsV1, CONFIGURATION_SETTING_KEYS_V1, CONTEXT_SCOUT_SETTINGS_SETTING_KEY, CodeIndexWorkerSelectionV1, ConfigurationValueKindV1, ConfigurationValueV1, ContextScoutSettingsV1, INDEX_EXCLUDE_SETTING_KEY, INDEX_EXTRACT_DOCSTRINGS_SETTING_KEY, - INDEX_INCLUDE_SETTING_KEY, INDEX_MAX_FILE_SIZE_SETTING_KEY, - INDEX_NATIVE_GRAPH_ACTIVATION_SETTING_KEY, INDEX_TRACK_CALL_SITES_SETTING_KEY, - LCM_SUMMARIZER_EXECUTABLES_SETTING_KEY, LcmSummarizerExecutablesV1, PROFILE_SETTING_KEYS_V1, + INDEX_INCLUDE_SETTING_KEY, INDEX_INCLUDE_SUBMODULES_SETTING_KEY, + INDEX_MAX_FILE_SIZE_SETTING_KEY, INDEX_NATIVE_GRAPH_ACTIVATION_SETTING_KEY, + INDEX_TRACK_CALL_SITES_SETTING_KEY, LCM_SUMMARIZER_EXECUTABLES_SETTING_KEY, + LcmSummarizerExecutablesV1, PROFILE_SETTING_KEYS_V1, PROJECT_WORK_EXPERTISE_CONSENT_SETTING_KEY, RestartRequirementV1, SOURCE_BINDINGS_SETTING_KEY, SYNC_AUTO_INIT_SETTING_KEY, SYNC_AUTO_TRACK_PR_BRANCHES_SETTING_KEY, SYNC_AUTO_TRACK_PR_POLL_SECS_SETTING_KEY, SYNC_AUTO_WATCH_SETTING_KEY, @@ -35,7 +36,7 @@ pub const MAX_PROXIMITY_RISK_THRESHOLD_BASIS_POINTS_V1: u64 = 10_000; /// Registry schema revision. Increment only when setting-definition semantics /// change, not when a setting value changes. -pub const CONFIGURATION_REGISTRY_SCHEMA_REVISION: u16 = 8; +pub const CONFIGURATION_REGISTRY_SCHEMA_REVISION: u16 = 9; #[derive(Debug, Error)] pub enum ConfigurationRegistryError { @@ -523,6 +524,12 @@ fn register_project_settings( SettingSensitivityV1::Sensitive, RestartRequirementV1::DaemonRestart, ), + ( + INDEX_INCLUDE_SUBMODULES_SETTING_KEY, + ConfigurationValueV1::Boolean(false), + SettingSensitivityV1::Public, + RestartRequirementV1::DaemonRestart, + ), ( INDEX_MAX_FILE_SIZE_SETTING_KEY, ConfigurationValueV1::Unsigned(defaults.max_file_size), @@ -779,7 +786,10 @@ mod user_profile_settings_tests { ); let definition = registry.definition(&key).expect("definition"); - assert_eq!(definition.schema_revision, 8); + assert_eq!( + definition.schema_revision, + CONFIGURATION_REGISTRY_SCHEMA_REVISION + ); assert_eq!(definition.scope, SettingScopeV1::UserProfile); assert_eq!( definition.value_kind, diff --git a/crates/tracedecay-global-db/src/configuration/store.rs b/crates/tracedecay-global-db/src/configuration/store.rs index f20a0c2bf6..ff751c349a 100644 --- a/crates/tracedecay-global-db/src/configuration/store.rs +++ b/crates/tracedecay-global-db/src/configuration/store.rs @@ -16,11 +16,11 @@ use tracedecay_domain::configuration::{ ConfigurationAuditEvent, ConfigurationAuditEventId, ConfigurationAuditEventKindV1, ConfigurationCandidateV1, ConfigurationIdempotencyKey, ConfigurationLayerIdV1, ConfigurationReceiptId, ConfigurationRevisionId, ConfigurationSnapshotV1, ConfigurationValueV1, - INDEX_NATIVE_GRAPH_ACTIVATION_SETTING_KEY, LCM_SUMMARIZER_EXECUTABLES_SETTING_KEY, - ProtectedChange, ProtectedChangePlan, ProtectedChangeSnapshotError, - RedactedConfigurationChangeV1, RollbackModeV1, SOURCE_BINDINGS_SETTING_KEY, - SYNC_WATCH_LINKED_WORKTREES_SETTING_KEY, ScopeControlOperationV1, SettingKey, - USER_CODE_INDEX_WORKERS_SETTING_KEY, USER_EXTRACTION_TIMEOUT_SECS_SETTING_KEY, + INDEX_INCLUDE_SUBMODULES_SETTING_KEY, INDEX_NATIVE_GRAPH_ACTIVATION_SETTING_KEY, + LCM_SUMMARIZER_EXECUTABLES_SETTING_KEY, ProtectedChange, ProtectedChangePlan, + ProtectedChangeSnapshotError, RedactedConfigurationChangeV1, RollbackModeV1, + SOURCE_BINDINGS_SETTING_KEY, SYNC_WATCH_LINKED_WORKTREES_SETTING_KEY, ScopeControlOperationV1, + SettingKey, USER_CODE_INDEX_WORKERS_SETTING_KEY, USER_EXTRACTION_TIMEOUT_SECS_SETTING_KEY, USER_REPEATED_QUERY_HINTS_SETTING_KEY, USER_WATCHER_DEBOUNCE_MS_SETTING_KEY, UserProfileId, WORK_TOPOLOGY_POLICY_SETTING_KEY, }; @@ -349,6 +349,7 @@ impl<'db> GlobalDbConfigurationControlStore<'db> { occurred_at, ®istry, &[ + INDEX_INCLUDE_SUBMODULES_SETTING_KEY, INDEX_NATIVE_GRAPH_ACTIVATION_SETTING_KEY, LCM_SUMMARIZER_EXECUTABLES_SETTING_KEY, SYNC_WATCH_LINKED_WORKTREES_SETTING_KEY, diff --git a/crates/tracedecay-global-db/src/configuration/store/tests/mod.rs b/crates/tracedecay-global-db/src/configuration/store/tests/mod.rs index 5e1b7f7d9c..8c6b4ab128 100644 --- a/crates/tracedecay-global-db/src/configuration/store/tests/mod.rs +++ b/crates/tracedecay-global-db/src/configuration/store/tests/mod.rs @@ -50,13 +50,19 @@ fn incomplete_snapshot_requires_reset_instead_of_default_repair() { } #[asupersync::test] -async fn linked_worktree_default_converges_into_existing_snapshot() { +async fn linked_worktree_and_submodule_defaults_converge_into_existing_snapshot() { let (_directory, runtime, root) = Box::pin(global_setup()).await; - let key = SettingKey::new(SYNC_WATCH_LINKED_WORKTREES_SETTING_KEY).unwrap(); + let keys = [ + SYNC_WATCH_LINKED_WORKTREES_SETTING_KEY, + tracedecay_domain::configuration::INDEX_INCLUDE_SUBMODULES_SETTING_KEY, + ] + .map(|name| SettingKey::new(name).unwrap()); let mut effective_values = root.snapshot.effective_values.clone(); let mut provenance = root.snapshot.provenance.clone(); - effective_values.remove(&key); - provenance.remove(&key); + for key in &keys { + effective_values.remove(key); + provenance.remove(key); + } let historical_snapshot = ConfigurationSnapshotV1::new(effective_values, provenance).unwrap(); let db = runtime .registered_database(HostAdmissionScope::Project) @@ -69,13 +75,18 @@ async fn linked_worktree_default_converges_into_existing_snapshot() { ) .await .unwrap(); - transaction - .execute( - "DELETE FROM configuration_entries WHERE revision_id = ?1 AND key = ?2", - tracedecay_runtime_core::db::engine::params![root.revision_id.as_str(), key.as_str()], - ) - .await - .unwrap(); + for key in &keys { + transaction + .execute( + "DELETE FROM configuration_entries WHERE revision_id = ?1 AND key = ?2", + tracedecay_runtime_core::db::engine::params![ + root.revision_id.as_str(), + key.as_str() + ], + ) + .await + .unwrap(); + } point_revision_at_snapshot(&transaction, &root.revision_id, &historical_snapshot).await; transaction.commit().await.unwrap(); let store = GlobalDbConfigurationControlStore::new_registered(db); @@ -86,11 +97,13 @@ async fn linked_worktree_default_converges_into_existing_snapshot() { .unwrap(); assert_ne!(converged.revision_id, root.revision_id); - assert_eq!( - converged.snapshot.effective_values.get(&key), - Some(&ConfigurationValueV1::Boolean(false)) - ); - assert!(converged.snapshot.provenance.contains_key(&key)); + for key in &keys { + assert_eq!( + converged.snapshot.effective_values.get(key), + Some(&ConfigurationValueV1::Boolean(false)) + ); + assert!(converged.snapshot.provenance.contains_key(key)); + } validate_snapshot_registry_completeness(&converged.snapshot).unwrap(); } diff --git a/crates/tracedecay-mcp/src/handlers/analysis/field_sites.rs b/crates/tracedecay-mcp/src/handlers/analysis/field_sites.rs index 75f161b692..d3df5f2f09 100644 --- a/crates/tracedecay-mcp/src/handlers/analysis/field_sites.rs +++ b/crates/tracedecay-mcp/src/handlers/analysis/field_sites.rs @@ -1,5 +1,6 @@ //! `tracedecay_field_sites`, read and write references to a named field. +use tracedecay_code_index::source_walk::submodules::LiveSourceBoundaries; use tracedecay_code_index_runtime::code_index_scheduler::sealed_source_bytes; use tracedecay_contracts::retrieval::{ FieldSiteV1, FieldSitesResultV1, FieldSitesSurfaceRequestV1, @@ -13,6 +14,7 @@ pub(super) async fn compute_field_sites( graph: &tracedecay_graph_query::VerifiedGraphQuery, args: Value, scope_prefix: Option<&str>, + include_submodules: bool, ) -> Result { let request: FieldSitesSurfaceRequestV1 = decode_primitive_request(&args, "tracedecay_field_sites")?; @@ -61,9 +63,16 @@ pub(super) async fn compute_field_sites( let mut reads: Vec = Vec::new(); let mut touched: Vec = Vec::new(); + let boundaries = LiveSourceBoundaries::open(&project_root, include_submodules) + .map_err(|error| TraceDecayError::project_route( + "verified-field-source-unavailable", true, error.message, + ))?; let needle = format!(".{field_name}"); 'outer: for file in &files { - let abs = project_root.join(file); + let Some((root, relative)) = boundaries.resolve(Path::new(file))? else { + continue; + }; + let abs = root.join(relative); let Ok(disk_source) = tracedecay_runtime_core::sync::read_source_file(&abs) else { continue; }; @@ -76,7 +85,7 @@ pub(super) async fn compute_field_sites( if !disk_source.contains(&needle) { continue; } - let source = sealed_field_source(&project_root, file, sealed_files.get(file))?; + let source = sealed_field_source(&boundaries, file, sealed_files.get(file))?; let masked = path_is_rust(file).then(|| { tracedecay_code_extraction::source_mask::masked_rust_source_with( &source, @@ -195,7 +204,7 @@ pub(super) async fn compute_field_sites( /// the generation sealed, which differ from the checkout when git's clean /// filters rewrite it (a CRLF checkout under `core.autocrlf=true`). fn sealed_field_source( - project_root: &Path, + boundaries: &LiveSourceBoundaries, file: &str, sealed: Option<&(LanguageId, ContentDigest)>, ) -> Result { @@ -208,7 +217,7 @@ fn sealed_field_source( ), )); }; - let bytes = sealed_source_bytes(project_root, file, language, digest) + let bytes = sealed_source_bytes(boundaries, file, language, digest) .map_err(|error| { TraceDecayError::project_route( "verified-field-source-unavailable", diff --git a/crates/tracedecay-mcp/src/handlers/analysis/mod.rs b/crates/tracedecay-mcp/src/handlers/analysis/mod.rs index a6008e9cfd..33a5babdad 100644 --- a/crates/tracedecay-mcp/src/handlers/analysis/mod.rs +++ b/crates/tracedecay-mcp/src/handlers/analysis/mod.rs @@ -100,7 +100,13 @@ pub async fn compute_analysis_report( compute_constructors(&graph, args, scope_prefix).await } ApplicationSurfaceOperation::FieldSites => { - compute_field_sites(&graph, args, scope_prefix).await + compute_field_sites( + &graph, + args, + scope_prefix, + ctx.index_path_policy().includes_submodules(), + ) + .await } operation => Err(unknown_tool_error(operation.mcp_tool_name())), } diff --git a/crates/tracedecay-mcp/src/handlers/graph/rename_preview.rs b/crates/tracedecay-mcp/src/handlers/graph/rename_preview.rs index 35f13c1928..f7ab389acf 100644 --- a/crates/tracedecay-mcp/src/handlers/graph/rename_preview.rs +++ b/crates/tracedecay-mcp/src/handlers/graph/rename_preview.rs @@ -1,12 +1,13 @@ //! `tracedecay_rename_preview`: what a rename of one symbol would touch. use std::collections::HashMap; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; use std::sync::Arc; use std::sync::atomic::{AtomicBool, Ordering}; use std::time::Duration; use serde_json::Value; +use tracedecay_code_index::source_walk::submodules::LiveSourceBoundaries; use tracedecay_code_index_runtime::code_index_scheduler::sealed_source_bytes; use tracedecay_contracts::graph_tool::{GraphToolCompletionV1, GraphToolResultV1}; use tracedecay_contracts::retrieval::{ @@ -175,6 +176,7 @@ struct RenameReferenceSiteInput { } struct PreviewFileWalk { + include_submodules: bool, project_root: PathBuf, declaration_line: u32, result: RenamePreviewPrimitiveResultV1, @@ -197,14 +199,30 @@ impl PreviewFileWalk { let mut text_only_matches = HashMap::new(); // Only the current file's decoded text and line index are resident. // Ordinals retain the graph's reference order across the grouped walk. + let boundaries = LiveSourceBoundaries::open(&self.project_root, self.include_submodules) + .map_err(|error| { + TraceDecayError::project_route( + "verified-reference-source-unavailable", + true, + error.message, + ) + })?; for file in files { self.control.check()?; let has_references = reference_inputs .peek() .is_some_and(|input| input.file == file); - let text = match tracedecay_runtime_core::sync::read_source_file( - &self.project_root.join(&file), - ) { + let Some((root, relative)) = boundaries.resolve(Path::new(&file))? else { + if has_references { + return Err(TraceDecayError::project_route( + "verified-reference-source-unavailable", + true, + format!("'{file}' is outside the current checkout source boundary"), + )); + } + continue; + }; + let text = match tracedecay_runtime_core::sync::read_source_file(&root.join(relative)) { Ok(text) => text, Err(error) if has_references => return Err(error.into()), // Declaration snippets are optional; reference evidence is not. @@ -217,7 +235,7 @@ impl PreviewFileWalk { // after the seal only its current text remains to map against. let sealed = match self.sealed_files.get(&file) { Some((language, digest)) => sealed_source_bytes( - &self.project_root, + &boundaries, &file, language, digest, @@ -427,6 +445,7 @@ pub async fn compute_rename_preview( freshness: None, }; let walk = PreviewFileWalk { + include_submodules: ctx.index_path_policy().includes_submodules(), project_root: ctx.project_root().to_path_buf(), declaration_line, result, diff --git a/crates/tracedecay-mcp/src/handlers/grep.rs b/crates/tracedecay-mcp/src/handlers/grep.rs index 1cb0965588..1d09dc452e 100644 --- a/crates/tracedecay-mcp/src/handlers/grep.rs +++ b/crates/tracedecay-mcp/src/handlers/grep.rs @@ -477,11 +477,17 @@ mod tests { "// UNIQUE_GENERATED_DIR_TOKEN\n", ) .expect("source fixture"); - std::fs::write( - project.path().join(".git"), - "gitdir: UNIQUE_GENERATED_DIR_TOKEN\n", - ) - .expect("linked-worktree git file fixture"); + let administration = tempfile::tempdir().expect("isolated Git administration"); + let git = tracedecay_runtime_core::git::try_git_program().expect("Git program"); + let initialized = std::process::Command::new(git) + .args(["init", "--quiet", "--template=", "--separate-git-dir"]) + .arg(administration.path().join("UNIQUE_GENERATED_DIR_TOKEN")) + .arg(project.path()) + .env("HOME", administration.path()) + .env("GIT_CONFIG_NOSYSTEM", "1") + .status() + .expect("initialize checkout with a Git pointer file"); + assert!(initialized.success()); std::fs::write( project.path().join(".tracedecay/internal.txt"), "UNIQUE_GENERATED_DIR_TOKEN\n", diff --git a/crates/tracedecay-query/benches/code_lexical_catchup.rs b/crates/tracedecay-query/benches/code_lexical_catchup.rs index 434a75740a..6b45e53cd6 100644 --- a/crates/tracedecay-query/benches/code_lexical_catchup.rs +++ b/crates/tracedecay-query/benches/code_lexical_catchup.rs @@ -184,6 +184,7 @@ fn build_fixture() -> Fixture { invalidations: BTreeSet::new(), ignored_source_admissions: Vec::new(), repository_parse_identity: CodeIndexRepositoryParseIdentityV1 { + include_submodules: false, tree: None, dirty: RepositoryDirtyStateV1::Dirty, }, @@ -195,6 +196,7 @@ fn build_fixture() -> Fixture { }, }; let config = CodeIndexProductionConfigV1 { + include_submodules: false, project_id: id::("project.catchup-benchmark"), repository: repository.clone(), sanitizer_revision, diff --git a/crates/tracedecay-query/src/bin/tracedecay_index_bench.rs b/crates/tracedecay-query/src/bin/tracedecay_index_bench.rs index 64681d7855..7841942625 100644 --- a/crates/tracedecay-query/src/bin/tracedecay_index_bench.rs +++ b/crates/tracedecay-query/src/bin/tracedecay_index_bench.rs @@ -561,6 +561,7 @@ fn run(options: &Options) -> Result { let repository = identity::("repository.index-bench"); let sanitizer_revision = identity::("sanitizer.index-bench.v1"); let config = CodeIndexProductionConfigV1 { + include_submodules: false, project_id: identity::("project.index-bench"), repository: repository.clone(), sanitizer_revision: sanitizer_revision.clone(), @@ -736,6 +737,7 @@ fn build_request( invalidations: BTreeSet::new(), ignored_source_admissions: Vec::new(), repository_parse_identity: CodeIndexRepositoryParseIdentityV1 { + include_submodules: false, tree: Some(identity::(tree)), dirty: RepositoryDirtyStateV1::Clean, }, diff --git a/crates/tracedecay-query/src/bin/tracedecay_search_bench.rs b/crates/tracedecay-query/src/bin/tracedecay_search_bench.rs index d1d6a515c8..9b9cd38fde 100644 --- a/crates/tracedecay-query/src/bin/tracedecay_search_bench.rs +++ b/crates/tracedecay-query/src/bin/tracedecay_search_bench.rs @@ -329,6 +329,7 @@ fn run(options: &Options) -> Result { let repository = identity::("repository.search-bench"); let sanitizer_revision = identity::("sanitizer.search-bench.v1"); let config = CodeIndexProductionConfigV1 { + include_submodules: false, project_id: identity::("project.search-bench"), repository: repository.clone(), sanitizer_revision: sanitizer_revision.clone(), @@ -726,6 +727,7 @@ fn build_request( invalidations: BTreeSet::new(), ignored_source_admissions: Vec::new(), repository_parse_identity: CodeIndexRepositoryParseIdentityV1 { + include_submodules: false, tree: Some(identity::("tree.search-bench.clean")), dirty: RepositoryDirtyStateV1::Clean, }, diff --git a/crates/tracedecay-query/tests/search_quality_suite/candidate_producers.rs b/crates/tracedecay-query/tests/search_quality_suite/candidate_producers.rs index f186edc00e..aa1def7ed4 100644 --- a/crates/tracedecay-query/tests/search_quality_suite/candidate_producers.rs +++ b/crates/tracedecay-query/tests/search_quality_suite/candidate_producers.rs @@ -425,6 +425,7 @@ pub(crate) fn real_lexical_source_fixture_from_sources( invalidations: BTreeSet::new(), ignored_source_admissions: Vec::new(), repository_parse_identity: CodeIndexRepositoryParseIdentityV1 { + include_submodules: false, tree: None, dirty: RepositoryDirtyStateV1::Dirty, }, @@ -436,6 +437,7 @@ pub(crate) fn real_lexical_source_fixture_from_sources( }, }; let config = CodeIndexProductionConfigV1 { + include_submodules: false, project_id: id::("project.artifact"), repository: repository.clone(), sanitizer_revision, diff --git a/crates/tracedecay-runtime-core/src/git_repository.rs b/crates/tracedecay-runtime-core/src/git_repository.rs index 13c8dadc6d..e978a2cce0 100644 --- a/crates/tracedecay-runtime-core/src/git_repository.rs +++ b/crates/tracedecay-runtime-core/src/git_repository.rs @@ -33,9 +33,9 @@ pub use history::{ pub use native_integration::{ GitNativeApplyOutcome, GitNativeCandidateTreeV1, GitNativeCandidateTreeVisitError, GitNativeIntegrationMode, GitNativePreflight, GitNativePreflightCaptureError, - GitNativePreflightDisposition, GitNativeUnsupportedReason, + GitNativePreflightDisposition, GitNativeUnsupportedReason, GitTreeSubmodulePins, }; -pub use tree_links::resolve_tree_link; +pub use tree_links::{RepositoryLinkEntry, resolve_repository_link, resolve_tree_link}; /// A typed failure from the in-process Git repository authority. #[derive(Clone, Debug, thiserror::Error)] diff --git a/crates/tracedecay-runtime-core/src/git_repository/native_integration.rs b/crates/tracedecay-runtime-core/src/git_repository/native_integration.rs index e09324c149..ada2a0215a 100644 --- a/crates/tracedecay-runtime-core/src/git_repository/native_integration.rs +++ b/crates/tracedecay-runtime-core/src/git_repository/native_integration.rs @@ -67,6 +67,40 @@ pub struct GitNativeCandidateTreeV1<'repository> { cancellation: &'repository CancellationToken, } +/// Registration and child commits taken from one immutable parent tree. +#[derive(Debug, Default)] +pub struct GitTreeSubmodulePins { + pub registration: Vec, + pub gitlinks: Vec<(String, gix::ObjectId)>, +} + +impl GitTreeSubmodulePins { + pub fn from_tree( + tree: &gix::Tree<'_>, + gitlinks: Vec<(String, gix::ObjectId)>, + ) -> Result { + if gitlinks.is_empty() { + return Ok(Self::default()); + } + let registration = match tree + .find_entry(".gitmodules") + .filter(|entry| entry.mode().is_blob()) + { + Some(entry) => tree + .repo + .find_blob(entry.object_id()) + .map_err(|error| operation("read pinned .gitmodules", error))? + .data + .clone(), + None => Vec::new(), + }; + Ok(Self { + registration, + gitlinks, + }) + } +} + impl GitNativeCandidateTreeV1<'_> { pub fn tree(&self) -> Result { oid(self.tree) @@ -78,9 +112,10 @@ impl GitNativeCandidateTreeV1<'_> { /// when it names none; every other symlink is visited with its link text. pub fn visit_blobs( &self, + include_submodules: bool, read_through: impl Fn(&[u8]) -> bool, mut visitor: impl FnMut(&[u8], &[u8]) -> Result<(), E>, - ) -> Result<(), GitNativeCandidateTreeVisitError> { + ) -> Result> { let tree = self .repository .find_tree(self.tree) @@ -90,6 +125,7 @@ impl GitNativeCandidateTreeV1<'_> { repository_visit_error("native candidate tree traversal", error) })?; entries.sort_by(|left, right| left.filepath.cmp(&right.filepath)); + let mut gitlinks = Vec::new(); for entry in entries { if self.cancellation.is_cancelled() { return Err(GitNativeCandidateTreeVisitError::Repository( @@ -99,6 +135,12 @@ impl GitNativeCandidateTreeV1<'_> { }, )); } + if include_submodules && entry.mode.is_commit() { + let path = entry.filepath.to_str().map_err(|error| { + repository_visit_error("native candidate gitlink path", error) + })?; + gitlinks.push((path.to_owned(), entry.oid)); + } if entry.mode.is_tree() || entry.mode.is_commit() { continue; } @@ -120,7 +162,8 @@ impl GitNativeCandidateTreeV1<'_> { visitor(entry.filepath.as_ref(), &blob.data) .map_err(GitNativeCandidateTreeVisitError::Visitor)?; } - Ok(()) + GitTreeSubmodulePins::from_tree(&tree, gitlinks) + .map_err(GitNativeCandidateTreeVisitError::Repository) } } diff --git a/crates/tracedecay-runtime-core/src/git_repository/tree_links.rs b/crates/tracedecay-runtime-core/src/git_repository/tree_links.rs index ccee3e049f..31e81fe652 100644 --- a/crates/tracedecay-runtime-core/src/git_repository/tree_links.rs +++ b/crates/tracedecay-runtime-core/src/git_repository/tree_links.rs @@ -15,6 +15,42 @@ pub fn resolve_tree_link( link_path: &[u8], target: &[u8], ) -> Result, GitRepositoryError> { + resolve_repository_link(link_path, target, |path| { + let Some(entry) = tree + .lookup_entry(path.iter().map(Vec::as_slice)) + .map_err(|error| operation("tree link lookup", error))? + else { + return Ok(None); + }; + let mode = entry.mode(); + Ok(if mode.is_tree() { + Some(RepositoryLinkEntry::Directory) + } else if mode.is_link() { + let link = entry + .object() + .map_err(|error| operation("tree link target", error))?; + Some(RepositoryLinkEntry::Link(link.data.clone())) + } else if mode.is_blob() { + Some(RepositoryLinkEntry::File(entry.object_id())) + } else { + None + }) + }) +} + +pub enum RepositoryLinkEntry { + Directory, + Link(Vec), + File(T), +} + +/// Walk relative link components with the caller's repository authority. Each +/// directory is checked before descending, including one followed by `..`. +pub fn resolve_repository_link( + link_path: &[u8], + target: &[u8], + mut lookup: impl FnMut(&[Vec]) -> Result>, E>, +) -> Result, E> { let mut resolved = components(link_path).collect::>(); resolved.pop(); let mut pending = Vec::new(); @@ -33,26 +69,17 @@ pub fn resolve_tree_link( } _ => resolved.push(component), } - let Some(entry) = tree - .lookup_entry(resolved.iter().map(Vec::as_slice)) - .map_err(|error| operation("tree link lookup", error))? - else { - return Ok(None); - }; - let mode = entry.mode(); - if mode.is_tree() { - continue; - } - if !mode.is_link() { - return Ok((pending.is_empty() && mode.is_blob()).then(|| entry.object_id())); - } - hops += 1; - resolved.pop(); - let link = entry - .object() - .map_err(|error| operation("tree link target", error))?; - if hops > MAX_LINK_HOPS || !push_target(&mut pending, &link.data) { - return Ok(None); + match lookup(&resolved)? { + Some(RepositoryLinkEntry::Directory) => {} + Some(RepositoryLinkEntry::Link(target)) => { + hops += 1; + resolved.pop(); + if hops > MAX_LINK_HOPS || !push_target(&mut pending, &target) { + return Ok(None); + } + } + Some(RepositoryLinkEntry::File(file)) if pending.is_empty() => return Ok(Some(file)), + _ => return Ok(None), } } Ok(None) diff --git a/crates/tracedecay-runtime-core/tests/runtime_core_suite/native_integration_authority.rs b/crates/tracedecay-runtime-core/tests/runtime_core_suite/native_integration_authority.rs index a1dc0be714..2f8d6c85f0 100644 --- a/crates/tracedecay-runtime-core/tests/runtime_core_suite/native_integration_authority.rs +++ b/crates/tracedecay-runtime-core/tests/runtime_core_suite/native_integration_authority.rs @@ -268,6 +268,7 @@ fn synthetic_candidate_tree_is_streamed_in_order_before_object_memory_drops() { let mut paths = Vec::new(); candidate .visit_blobs( + false, |_| false, |path, _bytes| { paths.push(String::from_utf8(path.to_vec()).expect("UTF-8 test path")); @@ -306,6 +307,7 @@ fn synthetic_candidate_tree_is_streamed_in_order_before_object_memory_drops() { |_, candidate| { candidate .visit_blobs( + false, |_| false, |_, _| { cancelled.cancel(); @@ -358,6 +360,7 @@ fn candidate_tree_reads_accepted_symlinks_through_to_their_blobs() { let mut visited = Vec::new(); candidate .visit_blobs( + false, |path| path.ends_with(b".rs"), |path, bytes| { visited.push(( diff --git a/crates/tracedecay-search-eval/src/candidate_output.rs b/crates/tracedecay-search-eval/src/candidate_output.rs index dcb3c2bb70..4bb11bb8d5 100644 --- a/crates/tracedecay-search-eval/src/candidate_output.rs +++ b/crates/tracedecay-search-eval/src/candidate_output.rs @@ -1324,6 +1324,7 @@ fn publish_scope_generation( invalidations: BTreeSet::new(), ignored_source_admissions: Vec::new(), repository_parse_identity: CodeIndexRepositoryParseIdentityV1 { + include_submodules: false, tree: None, dirty: RepositoryDirtyStateV1::Dirty, }, @@ -1331,6 +1332,7 @@ fn publish_scope_generation( target_projection_key, }; let config = CodeIndexProductionConfigV1 { + include_submodules: false, project_id: id::("project.candidate.fixture")?, repository: id::("repository.candidate.fixture")?, sanitizer_revision: id::("sanitizer.candidate.v1")?, diff --git a/crates/tracedecay-search-eval/src/candidate_output/cancellation.rs b/crates/tracedecay-search-eval/src/candidate_output/cancellation.rs index eff54a990f..b258849ab2 100644 --- a/crates/tracedecay-search-eval/src/candidate_output/cancellation.rs +++ b/crates/tracedecay-search-eval/src/candidate_output/cancellation.rs @@ -72,6 +72,7 @@ pub(super) fn prove_cancellation( invalidations: BTreeSet::new(), ignored_source_admissions: Vec::new(), repository_parse_identity: CodeIndexRepositoryParseIdentityV1 { + include_submodules: false, tree: None, dirty: RepositoryDirtyStateV1::Dirty, }, @@ -84,6 +85,7 @@ pub(super) fn prove_cancellation( }; let generation_scope = CodeIndexGenerationScopeV1::for_snapshot(&request.snapshot); let config = CodeIndexProductionConfigV1 { + include_submodules: false, project_id: id::("project.candidate.cancel")?, repository: id::("repository.candidate.cancel")?, sanitizer_revision: id::("sanitizer.candidate.v1")?, diff --git a/docs/USER-GUIDE.md b/docs/USER-GUIDE.md index d7eb98fc29..5ef9797bc9 100644 --- a/docs/USER-GUIDE.md +++ b/docs/USER-GUIDE.md @@ -209,6 +209,24 @@ The value replaces the whole list, so start from the current effective value malformed pattern is refused before anything is written. There is no per-run folder exclusion; `init` and `sync` take no folder flags. +Direct Git submodules are excluded by default. To index their checked-out +files under parent-relative paths, set the project’s Boolean +`index.include_submodules.v1` setting and restart the daemon. + +```bash +tracedecay tool configuration_set --args '{"layer":{"kind":"project","project_id":""},"key":"index.include_submodules.v1","value":{"kind":"boolean","value":true},"expected_revision":"","idempotency_key":""}' +``` + +The parent index and `.gitmodules` must identify the child, and its checkout +must use the matching repository under the parent's Git modules directory. +Live indexing includes child worktree changes. The parent graph retains tracked +source aliases that resolve through tracked links to a tracked parent file. A +parent alias cannot cross into a child checkout. Historical indexing reads the +child commit recorded by the selected parent revision from the local object +store. Initialize the submodule locally before requesting that history. +Linked worktrees, unrelated clones, and deeper nested repositories stay +outside the parent index. + ### Seeing what changed Use the daemon's status/coverage result to see the selected generation, exact