From 38bc131598d5d993295f97b7001811209a1b16a5 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sat, 7 Nov 2026 17:10:12 -0800 Subject: [PATCH 01/33] fix(code-index): report skipped nested repositories in unmounted audit Keep parent repository source admission unchanged; use one shared walker for exclusion evidence instead of a second filesystem scan. Refs #4593 --- .../tracedecay-code-index/src/source_walk.rs | 64 +++++++++++++++- .../src/unmounted_files.rs | 76 +++++++++++++++++-- 2 files changed, 132 insertions(+), 8 deletions(-) diff --git a/crates/tracedecay-code-index/src/source_walk.rs b/crates/tracedecay-code-index/src/source_walk.rs index 63d4be6e40..a02ecb2e20 100644 --- a/crates/tracedecay-code-index/src/source_walk.rs +++ b/crates/tracedecay-code-index/src/source_walk.rs @@ -9,8 +9,9 @@ //! walk is therefore public rather than crate-private, the audit in the root //! crate reuses this policy instead of restating it. +use std::collections::BTreeSet; use std::path::{Path, PathBuf}; -use std::sync::Arc; +use std::sync::{Arc, Mutex, PoisonError}; use ignore::overrides::{Override, OverrideBuilder}; use ignore::{Walk, WalkBuilder}; @@ -103,11 +104,51 @@ impl PathGlobScope { } } +/// Diagnostics emitted by the same source walker that enforces the nested +/// repository boundary. Only directories the walk actually reaches are named. +#[derive(Clone, Default)] +pub(crate) struct SourceWalkDiagnostics { + skipped_nested_repositories: Arc>>, +} + +impl SourceWalkDiagnostics { + pub(crate) fn skipped_nested_repositories(&self) -> Vec { + self.skipped_nested_repositories + .lock() + .unwrap_or_else(PoisonError::into_inner) + .iter() + .cloned() + .collect() + } +} + #[tracing::instrument(name = "code_index.capture.source_walk", level = "trace", skip_all)] pub fn source_walk( project_root: &Path, path_glob: Option<&str>, path_policy: &IndexPathPolicyV1, +) -> Result { + source_walk_inner(project_root, path_glob, path_policy, None) +} + +/// Expose excluded nested repository roots without changing source admission. +/// The unmounted-file audit uses this to avoid falsely implying completeness. +/// It uses the same traversal rather than an independent repository walker. +pub(crate) fn source_walk_with_diagnostics( + project_root: &Path, + path_glob: Option<&str>, + path_policy: &IndexPathPolicyV1, +) -> Result<(Walk, SourceWalkDiagnostics), SourceWalkError> { + let diagnostics = SourceWalkDiagnostics::default(); + let walk = source_walk_inner(project_root, path_glob, path_policy, Some(diagnostics.clone()))?; + Ok((walk, diagnostics)) +} + +fn source_walk_inner( + project_root: &Path, + path_glob: Option<&str>, + path_policy: &IndexPathPolicyV1, + diagnostics: Option, ) -> Result { let overrides = build_overrides(project_root, path_glob)?; let has_positive_override = overrides @@ -171,6 +212,13 @@ pub fn source_walk( // sources. Linked worktrees and nested clones share this boundary. // The project root is deliberately exempt at depth zero above. if std::fs::symlink_metadata(entry.path().join(".git")).is_ok() { + if let Some(diagnostics) = &diagnostics { + diagnostics + .skipped_nested_repositories + .lock() + .unwrap_or_else(PoisonError::into_inner) + .insert(relative); + } return false; } !path_policy.excludes_directory(&relative) @@ -220,7 +268,7 @@ mod tests { use tempfile::TempDir; use tracedecay_domain::IndexPathPolicyV1; - use super::source_walk; + use super::{source_walk, source_walk_with_diagnostics}; fn walked_files(root: &std::path::Path, path_policy: &IndexPathPolicyV1) -> Vec { let mut files = source_walk(root, None, path_policy) @@ -435,6 +483,18 @@ mod tests { assert!(index.stdout.ends_with(b" 0\tapp\0")); assert!(root.join("app/src/feature.rs").is_file()); + let (walk, diagnostics) = source_walk_with_diagnostics( + &root, + None, + &IndexPathPolicyV1::new(vec![], vec![]).unwrap(), + ) + .expect("source walk with nested repository diagnostics"); + let _visited = walk.map(|entry| entry.expect("source entry")).count(); + assert_eq!( + diagnostics.skipped_nested_repositories(), + vec!["app".to_owned()] + ); + for rule in ["app/**", "app/src/feature.rs", "app/src/fe*.rs"] { let policy = IndexPathPolicyV1::new(vec![], vec![rule.into()]).unwrap(); let files = walked_files(&root, &policy); diff --git a/crates/tracedecay-code-index/src/unmounted_files.rs b/crates/tracedecay-code-index/src/unmounted_files.rs index 1d1cb19e49..3f140bdebf 100644 --- a/crates/tracedecay-code-index/src/unmounted_files.rs +++ b/crates/tracedecay-code-index/src/unmounted_files.rs @@ -152,16 +152,19 @@ pub struct ProjectAudit { pub(super) struct ProjectFiles { root: PathBuf, files: Vec, + skipped_nested_repositories: Vec, } impl ProjectFiles { fn collect(project_root: &Path, path_policy: &IndexPathPolicyV1) -> Result { - let walk = - crate::source_walk::source_walk(project_root, None, path_policy).map_err(|error| { - TraceDecayError::Config { - message: format!("source walk rejected its own scope: {}", error.message), - } - })?; + let (walk, diagnostics) = crate::source_walk::source_walk_with_diagnostics( + project_root, + None, + path_policy, + ) + .map_err(|error| TraceDecayError::Config { + message: format!("source walk rejected its own scope: {}", error.message), + })?; let mut files = Vec::new(); for entry in walk { let Ok(entry) = entry else { continue }; @@ -174,6 +177,7 @@ impl ProjectFiles { Ok(Self { root: project_root.to_path_buf(), files, + skipped_nested_repositories: diagnostics.skipped_nested_repositories(), }) } @@ -258,6 +262,32 @@ pub fn audit_project(project_root: &Path, path_policy: &IndexPathPolicyV1) -> Re }; let mut ecosystems = vec![rust, typescript]; ecosystems.extend(unmodelled_ecosystems(&files)); + if !files.skipped_nested_repositories.is_empty() { + // Preserve existing ecosystem caveats and bound the diagnostic text. + // Skipped child repositories are not indexed as parent source. + let named = files + .skipped_nested_repositories + .iter() + .take(8) + .cloned() + .collect::>(); + let remaining = files.skipped_nested_repositories.len() - named.len(); + let summary = format!( + "Skipped nested repositories outside parent index: {}{}", + named.join(", "), + if remaining == 0 { + String::new() + } else { + format!(" (and {remaining} more)") + } + ); + for ecosystem in &mut ecosystems { + ecosystem.note = Some(match ecosystem.note.take() { + Some(existing) => format!("{existing} {summary}"), + None => summary.clone(), + }); + } + } Ok(ProjectAudit { ecosystems }) } @@ -395,6 +425,40 @@ mod tests { ); } + #[test] + fn audit_reports_nested_repository_exclusion_without_claiming_child_code() { + let root = tempfile::tempdir().expect("project fixture"); + write( + root.path(), + "Cargo.toml", + "[package]\nname = \"parent\"\nversion = \"0.0.0\"\n", + ); + write(root.path(), "src/lib.rs", "pub fn parent() {}\n"); + write(root.path(), "app/.git", "gitdir: ../.git/modules/app\n"); + write(root.path(), "app/src/child.rs", "pub fn child() {}\n"); + write(root.path(), "unrelated/.git/HEAD", "ref: refs/heads/main\n"); + write(root.path(), "unrelated/src/foreign.rs", "pub fn foreign() {}\n"); + + let audit = audit_project(root.path(), &no_exclusions()).expect("audit"); + let rust = ecosystem(&audit, "rust"); + assert_eq!( + rust.scanned_file_count, 1, + "child repositories are not parent source" + ); + assert!(rust.unmounted.iter().all(|file| { + !file.file.starts_with("app/") && !file.file.starts_with("unrelated/") + })); + let note = rust.note.as_deref().expect("explicit scope diagnostic"); + assert!(note.contains("Skipped nested repositories"), "{note}"); + assert!(note.contains("app") && note.contains("unrelated"), "{note}"); + assert!(audit.ecosystems.iter().all(|ecosystem| { + ecosystem + .note + .as_deref() + .is_some_and(|note| note.contains("Skipped nested repositories")) + })); + } + /// Both modelled ecosystems report in one pass over a polyglot repository. #[test] fn a_polyglot_project_reports_one_section_per_ecosystem() { From 1f8c1997090e4631ee38f50a32af4186e8153c12 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sat, 7 Nov 2026 18:14:21 -0800 Subject: [PATCH 02/33] style(code-index): format nested-repository diagnostics --- .../tracedecay-code-index/src/source_walk.rs | 7 ++++++- .../src/unmounted_files.rs | 19 ++++++++++--------- 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/crates/tracedecay-code-index/src/source_walk.rs b/crates/tracedecay-code-index/src/source_walk.rs index a02ecb2e20..c0eebaeda3 100644 --- a/crates/tracedecay-code-index/src/source_walk.rs +++ b/crates/tracedecay-code-index/src/source_walk.rs @@ -140,7 +140,12 @@ pub(crate) fn source_walk_with_diagnostics( path_policy: &IndexPathPolicyV1, ) -> Result<(Walk, SourceWalkDiagnostics), SourceWalkError> { let diagnostics = SourceWalkDiagnostics::default(); - let walk = source_walk_inner(project_root, path_glob, path_policy, Some(diagnostics.clone()))?; + let walk = source_walk_inner( + project_root, + path_glob, + path_policy, + Some(diagnostics.clone()), + )?; Ok((walk, diagnostics)) } diff --git a/crates/tracedecay-code-index/src/unmounted_files.rs b/crates/tracedecay-code-index/src/unmounted_files.rs index 3f140bdebf..5e5ee2e690 100644 --- a/crates/tracedecay-code-index/src/unmounted_files.rs +++ b/crates/tracedecay-code-index/src/unmounted_files.rs @@ -157,14 +157,11 @@ pub(super) struct ProjectFiles { impl ProjectFiles { fn collect(project_root: &Path, path_policy: &IndexPathPolicyV1) -> Result { - let (walk, diagnostics) = crate::source_walk::source_walk_with_diagnostics( - project_root, - None, - path_policy, - ) - .map_err(|error| TraceDecayError::Config { - message: format!("source walk rejected its own scope: {}", error.message), - })?; + let (walk, diagnostics) = + crate::source_walk::source_walk_with_diagnostics(project_root, None, path_policy) + .map_err(|error| TraceDecayError::Config { + message: format!("source walk rejected its own scope: {}", error.message), + })?; let mut files = Vec::new(); for entry in walk { let Ok(entry) = entry else { continue }; @@ -437,7 +434,11 @@ mod tests { write(root.path(), "app/.git", "gitdir: ../.git/modules/app\n"); write(root.path(), "app/src/child.rs", "pub fn child() {}\n"); write(root.path(), "unrelated/.git/HEAD", "ref: refs/heads/main\n"); - write(root.path(), "unrelated/src/foreign.rs", "pub fn foreign() {}\n"); + write( + root.path(), + "unrelated/src/foreign.rs", + "pub fn foreign() {}\n", + ); let audit = audit_project(root.path(), &no_exclusions()).expect("audit"); let rust = ecosystem(&audit, "rust"); From 8d6f07e1bb326630a313e1911d8bd271f337dd6c Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sat, 7 Nov 2026 19:16:18 -0800 Subject: [PATCH 03/33] fix(code-index): bound repository audits and report walk failures Limit skipped-repository samples during traversal, honor directory policy before collecting diagnostics, and quote bounded path labels. Propagate source-walk failures instead of returning an empty successful audit. Add behavioral coverage for bounded sampling, policy scope, preserved ecosystem caveats, Unicode/control-character paths, and missing roots. Refs #4593 --- .../tracedecay-code-index/src/source_walk.rs | 85 +++++++-- .../source_walk/nested_repository_tests.rs | 169 ++++++++++++++++++ .../src/unmounted_files.rs | 29 +-- 3 files changed, 246 insertions(+), 37 deletions(-) create mode 100644 crates/tracedecay-code-index/src/source_walk/nested_repository_tests.rs diff --git a/crates/tracedecay-code-index/src/source_walk.rs b/crates/tracedecay-code-index/src/source_walk.rs index c0eebaeda3..e360e4e680 100644 --- a/crates/tracedecay-code-index/src/source_walk.rs +++ b/crates/tracedecay-code-index/src/source_walk.rs @@ -104,21 +104,72 @@ impl PathGlobScope { } } +const MAX_REPOSITORY_NAMES: usize = 8; +const MAX_REPOSITORY_PATH_CHARS: usize = 160; + /// Diagnostics emitted by the same source walker that enforces the nested -/// repository boundary. Only directories the walk actually reaches are named. +/// repository boundary. Only directories within the walk's policy are named. #[derive(Clone, Default)] pub(crate) struct SourceWalkDiagnostics { - skipped_nested_repositories: Arc>>, + skipped_nested_repositories: Arc>, +} + +#[derive(Default)] +struct NestedRepositorySummary { + total: usize, + names: BTreeSet, } impl SourceWalkDiagnostics { - pub(crate) fn skipped_nested_repositories(&self) -> Vec { - self.skipped_nested_repositories + fn record_skipped_nested_repository(&self, relative: String) { + let mut summary = self + .skipped_nested_repositories .lock() - .unwrap_or_else(PoisonError::into_inner) + .unwrap_or_else(PoisonError::into_inner); + // A pruned directory is visited once. Keep only the smallest names, + // independent of filesystem iteration order, rather than collecting + // every path and truncating only after the walk has finished. + summary.total += 1; + summary.names.insert(relative); + if summary.names.len() > MAX_REPOSITORY_NAMES { + let _ = summary.names.pop_last(); + } + } + + pub(crate) fn skipped_nested_repositories_note(&self) -> Option { + let summary = self + .skipped_nested_repositories + .lock() + .unwrap_or_else(PoisonError::into_inner); + if summary.total == 0 { + return None; + } + let names = summary + .names .iter() - .cloned() - .collect() + .map(|path| { + // Bound by characters before escaping, so truncation cannot + // split UTF-8 or an escape. Quoting prevents a filename from + // injecting new lines or delimiters into an agent's report. + let mut chars = path.chars(); + let prefix = chars + .by_ref() + .take(MAX_REPOSITORY_PATH_CHARS) + .collect::(); + let suffix = if chars.next().is_some() { "…" } else { "" }; + format!("{prefix:?}{suffix}") + }) + .collect::>(); + let remaining = summary.total - summary.names.len(); + let suffix = if remaining == 0 { + String::new() + } else { + format!(" (and {remaining} more)") + }; + Some(format!( + "Skipped nested repositories outside parent index: {}{suffix}", + names.join(", ") + )) } } @@ -211,6 +262,11 @@ fn source_walk_inner( if !is_dir { return !path_policy.excludes(&relative); } + // Excluded directories are outside this audit, not missing source + // caused by the repository boundary. Do not probe or name them. + if path_policy.excludes_directory(&relative) { + return false; + } // Includes select paths, not repository authority. A parent // gitlink does not capture the child's files or prove their // freshness, so even registered submodules remain separate @@ -218,15 +274,11 @@ fn source_walk_inner( // The project root is deliberately exempt at depth zero above. if std::fs::symlink_metadata(entry.path().join(".git")).is_ok() { if let Some(diagnostics) = &diagnostics { - diagnostics - .skipped_nested_repositories - .lock() - .unwrap_or_else(PoisonError::into_inner) - .insert(relative); + diagnostics.record_skipped_nested_repository(relative); } return false; } - !path_policy.excludes_directory(&relative) + true }); if let Some(overrides) = overrides { builder.overrides(overrides); @@ -265,6 +317,9 @@ fn build_overrides( } } +#[cfg(test)] +mod nested_repository_tests; + #[cfg(test)] mod tests { use std::fs; @@ -496,8 +551,8 @@ mod tests { .expect("source walk with nested repository diagnostics"); let _visited = walk.map(|entry| entry.expect("source entry")).count(); assert_eq!( - diagnostics.skipped_nested_repositories(), - vec!["app".to_owned()] + diagnostics.skipped_nested_repositories_note().as_deref(), + Some("Skipped nested repositories outside parent index: \"app\"") ); for rule in ["app/**", "app/src/feature.rs", "app/src/fe*.rs"] { diff --git a/crates/tracedecay-code-index/src/source_walk/nested_repository_tests.rs b/crates/tracedecay-code-index/src/source_walk/nested_repository_tests.rs new file mode 100644 index 0000000000..5242c8305f --- /dev/null +++ b/crates/tracedecay-code-index/src/source_walk/nested_repository_tests.rs @@ -0,0 +1,169 @@ +use std::collections::BTreeSet; +use std::path::{Path, PathBuf}; + +use tracedecay_domain::IndexPathPolicyV1; +use tracedecay_domain::errors::TraceDecayError; + +use super::{SourceWalkDiagnostics, source_walk_with_diagnostics}; +use crate::unmounted_files::audit_project; + +fn write(root: &Path, relative: &str, contents: &str) { + let path = root.join(relative); + std::fs::create_dir_all(path.parent().expect("fixture parent")).unwrap(); + std::fs::write(path, contents).unwrap(); +} + +#[test] +fn repository_diagnostic_sample_is_bounded_and_order_independent() { + let expected = (0..8) + .map(|index| format!("repo-{index:02}")) + .collect::>(); + let orders = [ + (0..24).collect::>(), + (0..24).rev().collect::>(), + ]; + for order in orders { + let diagnostics = SourceWalkDiagnostics::default(); + assert!(diagnostics.skipped_nested_repositories_note().is_none()); + for index in order { + diagnostics.record_skipped_nested_repository(format!("repo-{index:02}")); + let summary = diagnostics.skipped_nested_repositories.lock().unwrap(); + assert!(summary.names.len() <= 8, "collection must be bounded too"); + } + let summary = diagnostics.skipped_nested_repositories.lock().unwrap(); + assert_eq!(summary.total, 24); + assert_eq!(summary.names, expected); + drop(summary); + let note = diagnostics.skipped_nested_repositories_note().unwrap(); + assert_eq!( + note, + "Skipped nested repositories outside parent index: \ + \"repo-00\", \"repo-01\", \"repo-02\", \"repo-03\", \ + \"repo-04\", \"repo-05\", \"repo-06\", \"repo-07\" (and 16 more)" + ); + } +} + +#[test] +fn repository_diagnostics_follow_policy_without_crossing_boundaries() { + let root = tempfile::tempdir().unwrap(); + write(root.path(), "keep.rs", "pub fn parent() {}\n"); + for directory in ["app", "excluded", "node_modules"] { + write(root.path(), &format!("{directory}/.git"), "gitdir: foreign\n"); + write(root.path(), &format!("{directory}/child.rs"), "child source\n"); + } + for (include, expected_note) in [ + ( + vec![], + "Skipped nested repositories outside parent index: \"app\"", + ), + ( + vec!["excluded/child.rs".to_owned()], + "Skipped nested repositories outside parent index: \"app\", \"excluded\"", + ), + ] { + let policy = IndexPathPolicyV1::new( + vec!["excluded/**".to_owned(), "node_modules/**".to_owned()], + include, + ) + .unwrap(); + let (walk, diagnostics) = source_walk_with_diagnostics(root.path(), None, &policy) + .expect("source walk"); + let files = walk + .map(|entry| entry.expect("source entry")) + .filter(|entry| entry.file_type().is_some_and(|kind| kind.is_file())) + .map(|entry| entry.path().strip_prefix(root.path()).unwrap().to_path_buf()) + .collect::>(); + assert_eq!(files, vec![PathBuf::from("keep.rs")]); + assert_eq!( + diagnostics.skipped_nested_repositories_note().as_deref(), + Some(expected_note) + ); + } +} + +#[test] +fn audit_bounds_repository_summary_and_preserves_unsupported_caveat() { + let root = tempfile::tempdir().unwrap(); + write(root.path(), "main.go", "package main\n"); + for index in (0..12).rev() { + write( + root.path(), + &format!("repo-{index:02}/.git/HEAD"), + "ref: refs/heads/main\n", + ); + write( + root.path(), + &format!("repo-{index:02}/child.go"), + "package child\n", + ); + } + let policy = IndexPathPolicyV1::new(vec![], vec![]).unwrap(); + let audit = audit_project(root.path(), &policy).expect("audit"); + let go = audit + .ecosystems + .iter() + .find(|ecosystem| ecosystem.ecosystem == "go") + .expect("parent Go source was counted"); + assert_eq!(go.scanned_file_count, 1); + assert_eq!(go.status.as_str(), "unsupported"); + assert_eq!( + go.note.as_deref(), + Some( + "1 go source file(s) are present and were not audited, \ + this report cannot say whether any of them is unreachable \ + Skipped nested repositories outside parent index: \ + \"repo-00\", \"repo-01\", \"repo-02\", \"repo-03\", \ + \"repo-04\", \"repo-05\", \"repo-06\", \"repo-07\" (and 4 more)" + ) + ); +} + +#[test] +fn repository_diagnostic_paths_are_clipped_on_character_boundaries() { + let diagnostics = SourceWalkDiagnostics::default(); + diagnostics.record_skipped_nested_repository("é".repeat(161)); + assert_eq!( + diagnostics.skipped_nested_repositories_note().unwrap(), + format!( + "Skipped nested repositories outside parent index: \"{}\"…", + "é".repeat(160) + ) + ); +} + +#[cfg(unix)] +#[test] +fn audit_quotes_control_characters_in_repository_names() { + let root = tempfile::tempdir().unwrap(); + write(root.path(), "keep.py", "print('parent')\n"); + write(root.path(), "line\nbreak/.git", "gitdir: foreign\n"); + write(root.path(), "line\nbreak/child.py", "print('child')\n"); + let policy = IndexPathPolicyV1::new(vec![], vec![]).unwrap(); + let audit = audit_project(root.path(), &policy).expect("audit"); + let python = audit + .ecosystems + .iter() + .find(|ecosystem| ecosystem.ecosystem == "python") + .expect("parent Python source was counted"); + assert_eq!(python.scanned_file_count, 1); + let note = python.note.as_deref().expect("scope diagnostic"); + assert!(note.contains("\"line\\nbreak\""), "{note}"); + assert!(!note.contains('\n'), "a path must not inject report lines"); + assert!(root.path().join("line\nbreak/child.py").is_file()); +} + +#[test] +fn missing_source_root_is_an_error_not_a_clean_audit() { + let root = tempfile::tempdir().unwrap(); + let missing = root.path().join("missing"); + let policy = IndexPathPolicyV1::new(vec![], vec![]).unwrap(); + match audit_project(&missing, &policy) { + Err(TraceDecayError::File { path, message }) => { + assert_eq!(path, missing.display().to_string()); + assert!(message.contains("source walk did not complete")); + } + Err(error) => panic!("unexpected error: {error}"), + Ok(_) => panic!("a failed walk must not report an empty successful audit"), + } +} diff --git a/crates/tracedecay-code-index/src/unmounted_files.rs b/crates/tracedecay-code-index/src/unmounted_files.rs index 5e5ee2e690..31cf2d6f59 100644 --- a/crates/tracedecay-code-index/src/unmounted_files.rs +++ b/crates/tracedecay-code-index/src/unmounted_files.rs @@ -152,7 +152,7 @@ pub struct ProjectAudit { pub(super) struct ProjectFiles { root: PathBuf, files: Vec, - skipped_nested_repositories: Vec, + nested_repository_note: Option, } impl ProjectFiles { @@ -164,7 +164,10 @@ impl ProjectFiles { })?; let mut files = Vec::new(); for entry in walk { - let Ok(entry) = entry else { continue }; + let entry = entry.map_err(|error| TraceDecayError::File { + message: format!("source walk did not complete: {error}"), + path: project_root.display().to_string(), + })?; if !entry.file_type().is_some_and(|kind| kind.is_file()) { continue; } @@ -174,7 +177,7 @@ impl ProjectFiles { Ok(Self { root: project_root.to_path_buf(), files, - skipped_nested_repositories: diagnostics.skipped_nested_repositories(), + nested_repository_note: diagnostics.skipped_nested_repositories_note(), }) } @@ -259,25 +262,7 @@ pub fn audit_project(project_root: &Path, path_policy: &IndexPathPolicyV1) -> Re }; let mut ecosystems = vec![rust, typescript]; ecosystems.extend(unmodelled_ecosystems(&files)); - if !files.skipped_nested_repositories.is_empty() { - // Preserve existing ecosystem caveats and bound the diagnostic text. - // Skipped child repositories are not indexed as parent source. - let named = files - .skipped_nested_repositories - .iter() - .take(8) - .cloned() - .collect::>(); - let remaining = files.skipped_nested_repositories.len() - named.len(); - let summary = format!( - "Skipped nested repositories outside parent index: {}{}", - named.join(", "), - if remaining == 0 { - String::new() - } else { - format!(" (and {remaining} more)") - } - ); + if let Some(summary) = &files.nested_repository_note { for ecosystem in &mut ecosystems { ecosystem.note = Some(match ecosystem.note.take() { Some(existing) => format!("{existing} {summary}"), From 1b49877815c81451007e66a4f2d54e4a92a06f6d Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sat, 7 Nov 2026 20:39:59 -0800 Subject: [PATCH 04/33] style(code-index): format nested repository regressions Apply the exact formatter changes reported by the hosted Bazel rustfmt_check_2024 failure on PR #4801. Refs #4593 --- .../source_walk/nested_repository_tests.rs | 24 +++++++++++++++---- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/crates/tracedecay-code-index/src/source_walk/nested_repository_tests.rs b/crates/tracedecay-code-index/src/source_walk/nested_repository_tests.rs index 5242c8305f..0b77991f58 100644 --- a/crates/tracedecay-code-index/src/source_walk/nested_repository_tests.rs +++ b/crates/tracedecay-code-index/src/source_walk/nested_repository_tests.rs @@ -49,8 +49,16 @@ fn repository_diagnostics_follow_policy_without_crossing_boundaries() { let root = tempfile::tempdir().unwrap(); write(root.path(), "keep.rs", "pub fn parent() {}\n"); for directory in ["app", "excluded", "node_modules"] { - write(root.path(), &format!("{directory}/.git"), "gitdir: foreign\n"); - write(root.path(), &format!("{directory}/child.rs"), "child source\n"); + write( + root.path(), + &format!("{directory}/.git"), + "gitdir: foreign\n", + ); + write( + root.path(), + &format!("{directory}/child.rs"), + "child source\n", + ); } for (include, expected_note) in [ ( @@ -67,12 +75,18 @@ fn repository_diagnostics_follow_policy_without_crossing_boundaries() { include, ) .unwrap(); - let (walk, diagnostics) = source_walk_with_diagnostics(root.path(), None, &policy) - .expect("source walk"); + let (walk, diagnostics) = + source_walk_with_diagnostics(root.path(), None, &policy).expect("source walk"); let files = walk .map(|entry| entry.expect("source entry")) .filter(|entry| entry.file_type().is_some_and(|kind| kind.is_file())) - .map(|entry| entry.path().strip_prefix(root.path()).unwrap().to_path_buf()) + .map(|entry| { + entry + .path() + .strip_prefix(root.path()) + .unwrap() + .to_path_buf() + }) .collect::>(); assert_eq!(files, vec![PathBuf::from("keep.rs")]); assert_eq!( From 624cc8338a1cfe8486b0638ba88fbbae0df9fa33 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sat, 7 Nov 2026 21:05:56 -0800 Subject: [PATCH 05/33] fix(code-index): align nested checkout capture with source walks Use one source boundary for interactive walks, Git candidate classification, worktree capture (including ignored-file admissions), and freshness. Parent tracked files hidden by a new nested .git marker become tombstones, invalidate clean HEAD-tree reuse, and return when the marker disappears. Reject source paths passing through symlinked directories or lexical escapes. Add real Git classification and filesystem regressions for nested directory, gitdir-file, symlink, path traversal and removal behavior. Refs #4593 --- .../code_index_scheduler/classification.rs | 134 +++++++++++++++++- .../code_index_scheduler/freshness_witness.rs | 14 +- .../src/code_index_scheduler/reconcile.rs | 8 ++ .../tracedecay-code-index/src/source_walk.rs | 107 +++++++++++++- 4 files changed, 251 insertions(+), 12 deletions(-) diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/classification.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/classification.rs index 8f057e94a2..ab168be996 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/classification.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/classification.rs @@ -14,10 +14,11 @@ //! consumes the derived candidate and changed sets to build an incremental //! batch. -use std::collections::BTreeSet; +use std::collections::{BTreeMap, BTreeSet}; use std::path::Path; use gix::bstr::ByteSlice; +use tracedecay_code_index::source_walk::has_nested_repository_ancestor; use tracedecay_runtime_core::git_repository::GIT_STATUS_MODIFICATION_CHECK_THREADS; /// Failure to classify worktree changes through gix. @@ -25,6 +26,8 @@ use tracedecay_runtime_core::git_repository::GIT_STATUS_MODIFICATION_CHECK_THREA pub enum ClassificationErrorV1 { #[error("code-index classification: {0}")] Git(String), + #[error("code-index source boundary: {0}")] + SourceBoundary(#[from] std::io::Error), } /// The truthful disposition of one repository-relative path. @@ -91,6 +94,9 @@ pub struct WorktreeChangeClassificationV1 { /// Present paths whose Git bytes are not UTF-8, so no `String` above /// names them. non_utf8_paths: BTreeSet>, + /// Files still tracked by the parent but hidden behind a nested checkout + /// or symlinked ancestor. These are tombstones, not indexable candidates. + nested_repository_exclusions: BTreeSet, } impl WorktreeChangeClassificationV1 { @@ -202,11 +208,40 @@ impl WorktreeChangeClassificationV1 { } } + // A nested .git marker can appear under paths that are still in + // the parent's index without changing Git status. The live walker + // refuses those files; classification must also exclude them and + // invalidate clean-tree and active-generation reuse. Resolve each + // parent directory once rather than probing per source file. + let mut nested_repository_exclusions = BTreeSet::new(); + if let Some(project_root) = repository.workdir() { + let mut checked_parents = BTreeMap::new(); + for path in committed_baseline + .iter() + .chain(changes.iter().map(|change| &change.path)) + { + let relative = Path::new(path); + let parent = relative.parent().unwrap_or_else(|| Path::new("")); + let excluded = match checked_parents.get(parent) { + Some(excluded) => *excluded, + None => { + let excluded = has_nested_repository_ancestor(project_root, relative)?; + checked_parents.insert(parent.to_path_buf(), excluded); + excluded + } + }; + if excluded { + nested_repository_exclusions.insert(path.clone()); + } + } + } + Ok(Self { committed_baseline, changes, non_utf8_changes, non_utf8_paths, + nested_repository_exclusions, }) } @@ -241,13 +276,16 @@ impl WorktreeChangeClassificationV1 { candidates.remove(&change.path); } } + candidates.retain(|path| !self.nested_repository_exclusions.contains(path)); candidates } /// Whether the worktree carries any change relative to the committed /// baseline, UTF-8-named or not. pub fn has_changes(&self) -> bool { - !(self.changes().is_empty() && self.non_utf8_changes().is_empty()) + !(self.changes().is_empty() + && self.non_utf8_changes().is_empty() + && self.nested_repository_exclusions.is_empty()) } /// Whether any classified change is a merge conflict. @@ -264,10 +302,13 @@ impl WorktreeChangeClassificationV1 { /// tombstones flow through). This is a hint to narrow work, not the identity /// authority; the generation planner still compares content digests. pub fn changed_paths(&self) -> BTreeSet { - self.changes + let mut changed = self + .changes .iter() .map(|change| change.path.clone()) - .collect() + .collect::>(); + changed.extend(self.nested_repository_exclusions.iter().cloned()); + changed } /// Paths removed from the present snapshot (staged or unstaged deletions). @@ -345,3 +386,88 @@ fn classify_item(item: &gix::status::Item) -> Option { }, }) } + +#[cfg(test)] +mod nested_repository_tests { + use std::fs; + use std::path::Path; + use std::process::Command; + + use super::WorktreeChangeClassificationV1; + + fn fixture_git(root: &Path, args: &[&str]) { + let program = + tracedecay_runtime_core::git::try_git_program().expect("fixture Git program"); + let mut command = Command::new(program); + command.current_dir(root); + for (key, _) in std::env::vars_os() { + if key.to_string_lossy().starts_with("GIT_") { + command.env_remove(key); + } + } + let output = command + .env("GIT_CONFIG_NOSYSTEM", "1") + .env("GIT_CONFIG_GLOBAL", root.join(".fixture-gitconfig")) + .args([ + "-c", + "user.name=Fixture", + "-c", + "user.email=fixture@example.invalid", + "-c", + "commit.gpgsign=false", + ]) + .args(args) + .output() + .expect("fixture Git command"); + assert!( + output.status.success(), + "Git {args:?}: {}", + String::from_utf8_lossy(&output.stderr) + ); + } + + #[test] + fn nested_checkout_marker_tombstones_parent_tracked_sources() { + let root = tempfile::tempdir().unwrap(); + fixture_git(root.path(), &["init", "-q"]); + for path in ["src/keep.rs", "nested/foreign.rs", "linked/foreign.rs"] { + let full = root.path().join(path); + fs::create_dir_all(full.parent().unwrap()).unwrap(); + fs::write(full, "pub fn source() {}\n").unwrap(); + } + fixture_git(root.path(), &["add", "."]); + fixture_git(root.path(), &["commit", "-qm", "parent tracked files"]); + + let repository = tracedecay_runtime_core::git_open::open(root.path()).unwrap(); + let before = WorktreeChangeClassificationV1::classify(&repository).unwrap(); + assert!(!before.has_changes()); + assert!(before.candidate_paths().contains("nested/foreign.rs")); + + fs::create_dir(root.path().join("nested/.git")).unwrap(); + fs::write( + root.path().join("nested/.git/HEAD"), + "ref: refs/heads/main\n", + ) + .unwrap(); + fs::write( + root.path().join("linked/.git"), + "gitdir: /untrusted/worktrees/linked\n", + ) + .unwrap(); + let hidden = WorktreeChangeClassificationV1::classify(&repository).unwrap(); + let candidates = hidden.candidate_paths(); + assert!(candidates.contains("src/keep.rs")); + assert!(!candidates.contains("nested/foreign.rs")); + assert!(!candidates.contains("linked/foreign.rs")); + assert!(hidden.has_changes(), "a clean HEAD-tree capture is now unsafe"); + assert!(hidden.changed_paths().contains("nested/foreign.rs")); + assert!(hidden.changed_paths().contains("linked/foreign.rs")); + + fs::remove_dir_all(root.path().join("nested/.git")).unwrap(); + fs::remove_file(root.path().join("linked/.git")).unwrap(); + let restored = WorktreeChangeClassificationV1::classify(&repository).unwrap(); + assert!(!restored.has_changes()); + assert!(restored.candidate_paths().contains("nested/foreign.rs")); + assert!(restored.candidate_paths().contains("linked/foreign.rs")); + } +} diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/freshness_witness.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/freshness_witness.rs index 8392b54955..48639d5beb 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/freshness_witness.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/freshness_witness.rs @@ -129,7 +129,19 @@ fn source_candidates( .map(|admission| admission.logical_path.as_str()) .collect::>(); let mut candidate_paths = classification.candidate_paths(); - candidate_paths.extend(admitted_paths.iter().map(|path| (*path).to_owned())); + let project_root = repository.workdir().ok_or_else(|| { + CodeIndexSchedulerErrorV1::Git("source freshness requires a worktree".to_owned()) + })?; + for path in &admitted_paths { + // Explicit ignored-file enrollment lifts Git ignore rules, not the + // nested repository boundary shared with capture and live search. + if !tracedecay_code_index::source_walk::has_nested_repository_ancestor( + project_root, + Path::new(path), + )? { + candidate_paths.insert((*path).to_owned()); + } + } let mut unrepresentable = classification .non_utf8_paths() .iter() diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/reconcile.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/reconcile.rs index 9ca91320c7..e2d51bfeb6 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/reconcile.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/reconcile.rs @@ -3974,6 +3974,14 @@ impl CodeIndexWorktreeSchedulerV1 { control: Option<&dyn CodeIndexExecutionControlV1>, explicitly_admitted: bool, ) -> Result { + // Explicit ignored-source admission cannot cross a nested checkout + // any more than an ordinary Git candidate or interactive source walk. + if tracedecay_code_index::source_walk::has_nested_repository_ancestor( + &self.project_root, + std::path::Path::new(logical_path), + )? { + return Ok(CapturedFileOutcomeV1::Absent); + } let absolute = self.project_root.join(logical_path); match absolute.metadata() { Ok(metadata) if !metadata.is_file() => return Ok(CapturedFileOutcomeV1::Absent), diff --git a/crates/tracedecay-code-index/src/source_walk.rs b/crates/tracedecay-code-index/src/source_walk.rs index e360e4e680..6a363aed0d 100644 --- a/crates/tracedecay-code-index/src/source_walk.rs +++ b/crates/tracedecay-code-index/src/source_walk.rs @@ -10,7 +10,7 @@ //! crate reuses this policy instead of restating it. use std::collections::BTreeSet; -use std::path::{Path, PathBuf}; +use std::path::{Component, Path, PathBuf}; use std::sync::{Arc, Mutex, PoisonError}; use ignore::overrides::{Override, OverrideBuilder}; @@ -107,6 +107,52 @@ impl PathGlobScope { const MAX_REPOSITORY_NAMES: usize = 8; const MAX_REPOSITORY_PATH_CHARS: usize = 160; +fn nested_repository_marker(directory: &Path) -> std::io::Result { + match std::fs::symlink_metadata(directory.join(".git")) { + Ok(_) => Ok(true), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(false), + Err(error) => Err(error), + } +} + +/// Reject a source beneath another checkout or a symlinked directory. +/// The root's own Git metadata is exempt. This boundary is shared by +/// interactive source walks and the indexed worktree candidate roster, +/// so neither may admit bytes the other refuses. +pub fn has_nested_repository_ancestor( + project_root: &Path, + relative_path: &Path, +) -> std::io::Result { + if relative_path + .components() + .any(|component| !matches!(component, Component::Normal(_))) + { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidInput, + "source path must be repository-relative", + )); + } + + let mut parent = relative_path.parent(); + while let Some(relative_dir) = parent { + if relative_dir.as_os_str().is_empty() { + break; + } + let directory = project_root.join(relative_dir); + match std::fs::symlink_metadata(&directory) { + Ok(metadata) if !metadata.is_dir() => return Ok(true), + Ok(_) => {} + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => return Err(error), + } + if nested_repository_marker(&directory)? { + return Ok(true); + } + parent = relative_dir.parent(); + } + Ok(false) +} + /// Diagnostics emitted by the same source walker that enforces the nested /// repository boundary. Only directories within the walk's policy are named. #[derive(Clone, Default)] @@ -272,13 +318,18 @@ fn source_walk_inner( // freshness, so even registered submodules remain separate // sources. Linked worktrees and nested clones share this boundary. // The project root is deliberately exempt at depth zero above. - if std::fs::symlink_metadata(entry.path().join(".git")).is_ok() { - if let Some(diagnostics) = &diagnostics { - diagnostics.record_skipped_nested_repository(relative); + match nested_repository_marker(entry.path()) { + Ok(true) => { + if let Some(diagnostics) = &diagnostics { + diagnostics.record_skipped_nested_repository(relative); + } + false } - return false; + Ok(false) => true, + // The walk cannot prove that an unreadable Git marker is + // absent. Never admit files across that boundary. + Err(_) => false, } - true }); if let Some(overrides) = overrides { builder.overrides(overrides); @@ -328,7 +379,49 @@ mod tests { use tempfile::TempDir; use tracedecay_domain::IndexPathPolicyV1; - use super::{source_walk, source_walk_with_diagnostics}; + use super::{ + has_nested_repository_ancestor, source_walk, source_walk_with_diagnostics, + }; + + #[test] + fn nested_repository_boundary_excludes_parent_markers_and_path_escape() { + let root = TempDir::new().unwrap(); + fs::create_dir_all(root.path().join("parent/nested/.git")).unwrap(); + fs::write(root.path().join("parent/nested/.git/HEAD"), "ref: refs/heads/main\n").unwrap(); + assert!( + !has_nested_repository_ancestor(root.path(), std::path::Path::new("parent/source.rs")) + .unwrap() + ); + assert!( + has_nested_repository_ancestor( + root.path(), + std::path::Path::new("parent/nested/source.rs") + ) + .unwrap() + ); + assert!( + has_nested_repository_ancestor(root.path(), std::path::Path::new("../escape.rs")) + .is_err() + ); + assert!( + has_nested_repository_ancestor(root.path(), std::path::Path::new("/absolute.rs")) + .is_err() + ); + } + + #[cfg(unix)] + #[test] + fn nested_repository_boundary_excludes_symlinked_ancestors() { + use std::os::unix::fs::symlink; + + let root = TempDir::new().unwrap(); + let external = TempDir::new().unwrap(); + symlink(external.path(), root.path().join("foreign")).unwrap(); + assert!( + has_nested_repository_ancestor(root.path(), std::path::Path::new("foreign/source.rs")) + .unwrap() + ); + } fn walked_files(root: &std::path::Path, path_policy: &IndexPathPolicyV1) -> Vec { let mut files = source_walk(root, None, path_policy) From 9beb7b89f6ae6ca0c03b0713e561d5d792745334 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sat, 7 Nov 2026 22:39:23 -0800 Subject: [PATCH 06/33] style(code-index): format nested repository boundary tests Apply four exact replacements from hosted Bazel rustfmt_check_2024 at PR #4801 head 624cc833; no behavioral source change. Refs #4593 --- .../src/code_index_scheduler/classification.rs | 8 +++++--- crates/tracedecay-code-index/src/source_walk.rs | 10 ++++++---- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/classification.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/classification.rs index ab168be996..f8ef32be30 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/classification.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/classification.rs @@ -396,8 +396,7 @@ mod nested_repository_tests { use super::WorktreeChangeClassificationV1; fn fixture_git(root: &Path, args: &[&str]) { - let program = - tracedecay_runtime_core::git::try_git_program().expect("fixture Git program"); + let program = tracedecay_runtime_core::git::try_git_program().expect("fixture Git program"); let mut command = Command::new(program); command.current_dir(root); for (key, _) in std::env::vars_os() { @@ -459,7 +458,10 @@ mod nested_repository_tests { assert!(candidates.contains("src/keep.rs")); assert!(!candidates.contains("nested/foreign.rs")); assert!(!candidates.contains("linked/foreign.rs")); - assert!(hidden.has_changes(), "a clean HEAD-tree capture is now unsafe"); + assert!( + hidden.has_changes(), + "a clean HEAD-tree capture is now unsafe" + ); assert!(hidden.changed_paths().contains("nested/foreign.rs")); assert!(hidden.changed_paths().contains("linked/foreign.rs")); diff --git a/crates/tracedecay-code-index/src/source_walk.rs b/crates/tracedecay-code-index/src/source_walk.rs index 6a363aed0d..515a679fce 100644 --- a/crates/tracedecay-code-index/src/source_walk.rs +++ b/crates/tracedecay-code-index/src/source_walk.rs @@ -379,15 +379,17 @@ mod tests { use tempfile::TempDir; use tracedecay_domain::IndexPathPolicyV1; - use super::{ - has_nested_repository_ancestor, source_walk, source_walk_with_diagnostics, - }; + use super::{has_nested_repository_ancestor, source_walk, source_walk_with_diagnostics}; #[test] fn nested_repository_boundary_excludes_parent_markers_and_path_escape() { let root = TempDir::new().unwrap(); fs::create_dir_all(root.path().join("parent/nested/.git")).unwrap(); - fs::write(root.path().join("parent/nested/.git/HEAD"), "ref: refs/heads/main\n").unwrap(); + fs::write( + root.path().join("parent/nested/.git/HEAD"), + "ref: refs/heads/main\n", + ) + .unwrap(); assert!( !has_nested_repository_ancestor(root.path(), std::path::Path::new("parent/source.rs")) .unwrap() From 2386440dfa50f47393d1f380f0d4f43c707e95c9 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 00:43:28 -0800 Subject: [PATCH 07/33] fix(code-index): invalidate cached rosters on nested Git markers Sample .git marker evidence for tracked and explicitly admitted source ancestors even when Git dirwalk never recurses into tracked-only folders. Test marker creation and removal against a real Git parent index. Refs #4593 --- .../code_index_scheduler/freshness_witness.rs | 132 +++++++++++++++++- 1 file changed, 131 insertions(+), 1 deletion(-) diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/freshness_witness.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/freshness_witness.rs index 48639d5beb..1e5e03e860 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/freshness_witness.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/freshness_witness.rs @@ -490,6 +490,7 @@ fn roster_evidence( repository: &gix::Repository, project_root: &Path, sampled_at: ChangeClockReading, + ignored_source_admissions: &[CodeIndexIgnoredSourceAdmissionV1], ) -> Option)>> { let mut recorder = DirectoryEvidenceV1 { root: project_root.to_path_buf(), @@ -513,6 +514,37 @@ fn roster_evidence( } recorder.record_directory(project_root.to_path_buf()); let index = repository.index_or_empty().ok()?; + // Git's directory walk may not descend into tracked-only directories. + // Their nested .git markers still change source admission without a + // parent index or HEAD update, so key each tracked and explicitly + // admitted source ancestor independently of dirwalk traversal. + let mut source_ancestors = BTreeSet::new(); + let mut collect_ancestors = |relative: &Path| { + let mut current = relative.parent(); + while let Some(parent) = current { + if parent.as_os_str().is_empty() { + break; + } + source_ancestors.insert(parent.to_path_buf()); + current = parent.parent(); + } + }; + for entry in index.entries() { + collect_ancestors(gix::path::from_bstr(entry.path(&index)).ok()?); + } + for admission in ignored_source_admissions { + if validate_code_logical_path(&admission.logical_path).is_err() { + return None; + } + collect_ancestors(Path::new(&admission.logical_path)); + } + for relative in source_ancestors { + let directory = project_root.join(relative); + recorder.record_directory(directory.clone()); + // A missing marker is evidence too: a newly created gitdir file or + // directory must invalidate the roster even if Git ignores it. + recorder.record(directory.join(".git"), false); + } let options = repository .dirwalk_options() .ok()? @@ -597,7 +629,12 @@ impl SourceSweepCacheV1 { let Ok(repository) = tracedecay_runtime_core::git_open::open(project_root) else { return (false, stats); }; - let evidence = roster_evidence(&repository, project_root, sampled_at); + let evidence = roster_evidence( + &repository, + project_root, + sampled_at, + ignored_source_admissions, + ); let Ok(SourceCandidatesV1 { candidates, unrepresentable, @@ -1029,3 +1066,96 @@ mod tests { assert_eq!(first, sample(&path, sampled_at).unwrap().unwrap()); } } + +#[cfg(all(test, unix))] +mod tracked_parent_boundary_tests { + use std::path::Path; + use std::sync::Arc; + use std::time::{Duration, Instant}; + + use super::{CachedCandidateRosterV1, ChangeClockReading, roster_evidence}; + + fn git(root: &Path, args: &[&str]) { + let program = tracedecay_runtime_core::git::try_git_program().expect("fixture Git"); + let mut command = std::process::Command::new(program); + command.current_dir(root); + for (key, _) in std::env::vars_os() { + if key.to_string_lossy().starts_with("GIT_") { + command.env_remove(key); + } + } + let output = command + .env("GIT_CONFIG_NOSYSTEM", "1") + .env("GIT_CONFIG_GLOBAL", root.join(".fixture-gitconfig")) + .args([ + "-c", + "user.name=Fixture", + "-c", + "user.email=fixture@example.invalid", + "-c", + "commit.gpgsign=false", + ]) + .args(args) + .output() + .expect("run fixture Git"); + assert!( + output.status.success(), + "Git {args:?}: {}", + String::from_utf8_lossy(&output.stderr) + ); + } + + #[test] + fn cached_tracked_directory_roster_invalidates_on_nested_git_marker() { + let root = tempfile::tempdir().expect("parent checkout"); + let tracked = root.path().join("src/nested/tracked.rs"); + std::fs::create_dir_all(tracked.parent().unwrap()).unwrap(); + std::fs::write(&tracked, "pub fn tracked() {}\n").unwrap(); + git(root.path(), &["init", "-q"]); + git(root.path(), &["add", "src/nested/tracked.rs"]); + git(root.path(), &["commit", "-qm", "parent source"]); + + let repository = + tracedecay_runtime_core::git_open::open(root.path()).expect("open parent Git"); + let deadline = Instant::now() + Duration::from_secs(5); + let evidence = loop { + if let Some(evidence) = + roster_evidence(&repository, root.path(), ChangeClockReading::now(), &[]) + { + break evidence; + } + assert!( + Instant::now() < deadline, + "tracked parent directories never became settleable" + ); + std::thread::sleep(Duration::from_millis(20)); + }; + let marker = root.path().join("src/nested/.git"); + assert!( + evidence + .iter() + .any(|(path, key)| path == &marker && key.is_none()), + "missing nested marker must be sampled independently of gix dirwalk" + ); + + let cached = CachedCandidateRosterV1 { + git_metadata_signature: "unchanged-parent-git".to_owned(), + admitted_paths: Vec::new(), + evidence, + candidates: Arc::new(Vec::new()), + unrepresentable: Arc::new(Vec::new()), + }; + assert!(cached.holds("unchanged-parent-git", &[])); + std::fs::write(&marker, "gitdir: ../../.git/modules/nested\n") + .expect("create nested gitdir marker"); + assert!( + !cached.holds("unchanged-parent-git", &[]), + "a new nested repository marker must invalidate the cached roster even when parent Git metadata is unchanged" + ); + std::fs::remove_file(&marker).unwrap(); + assert!( + cached.holds("unchanged-parent-git", &[]), + "the pre-marker evidence should be valid again after removing it" + ); + } +} From 0119cf9ff865c41a92c4a9ecb1cd09f15ea0322d Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 00:44:27 -0800 Subject: [PATCH 08/33] test(code-index): assert roster invalidation across marker removal Directory metadata changes when the marker is removed, so the original cached roster must not be treated as restored merely by marker absence. Refs #4593 --- .../src/code_index_scheduler/freshness_witness.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/freshness_witness.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/freshness_witness.rs index 1e5e03e860..8eb1ba4fe0 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/freshness_witness.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/freshness_witness.rs @@ -1154,8 +1154,8 @@ mod tracked_parent_boundary_tests { ); std::fs::remove_file(&marker).unwrap(); assert!( - cached.holds("unchanged-parent-git", &[]), - "the pre-marker evidence should be valid again after removing it" + !cached.holds("unchanged-parent-git", &[]), + "removing the marker also changes the tracked ancestor directory evidence" ); } } From 704a9aac135046fe1d720242a482561a61eb5274 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 01:16:52 -0800 Subject: [PATCH 09/33] fix(code-index): refuse markerless parent gitlink directories A registered gitlink remains an untrusted nested source boundary even if its child .git marker is absent. Keep live grep and audit in default-deny parity with the parent index; include/glob cannot lift repository trust. Exercise a real registered Git submodule with its marker removed. Refs #4593 --- .../tracedecay-code-index/src/source_walk.rs | 80 +++++++++++++++++++ 1 file changed, 80 insertions(+) diff --git a/crates/tracedecay-code-index/src/source_walk.rs b/crates/tracedecay-code-index/src/source_walk.rs index 515a679fce..9e3f4e973b 100644 --- a/crates/tracedecay-code-index/src/source_walk.rs +++ b/crates/tracedecay-code-index/src/source_walk.rs @@ -115,6 +115,52 @@ fn nested_repository_marker(directory: &Path) -> std::io::Result { } } +/// Default-deny all gitlinks in the parent's Git index even when a child +/// checkout's .git marker has been removed. A gitlink is a repository boundary, +/// not an ordinary directory the live source walker can authenticate. +/// This is deny-only; it never enables submodule admission. +fn parent_gitlink_paths(project_root: &Path) -> Result, SourceWalkError> { + let marker = project_root.join(".git"); + let repository = match tracedecay_runtime_core::git_open::open(project_root) { + Ok(repository) => repository, + Err(error) => { + // Non-Git projects still support source walks. An initialized + // checkout whose Git authority is unreadable must fail closed. + if marker.is_file() || marker.join("HEAD").is_file() { + return Err(SourceWalkError { + glob: String::new(), + message: format!("cannot inspect parent Git boundaries: {error}"), + }); + } + return Ok(BTreeSet::new()); + } + }; + if repository.workdir().and_then(|workdir| workdir.canonicalize().ok()) + != project_root.canonicalize().ok() + { + return Err(SourceWalkError { + glob: String::new(), + message: "source walk root is not the opened Git worktree".to_owned(), + }); + } + let index = repository.index_or_empty().map_err(|error| SourceWalkError { + glob: String::new(), + message: format!("cannot inspect parent Git index: {error}"), + })?; + let mut paths = BTreeSet::new(); + for entry in index.entries() { + if entry.mode.is_commit() { + let path: &[u8] = entry.path(&index).as_ref(); + let relative = std::str::from_utf8(path).map_err(|_| SourceWalkError { + glob: String::new(), + message: "non-UTF-8 parent gitlink cannot be safely scoped".to_owned(), + })?; + paths.insert(relative.to_owned()); + } + } + Ok(paths) +} + /// Reject a source beneath another checkout or a symlinked directory. /// The root's own Git metadata is exempt. This boundary is shared by /// interactive source walks and the indexed worktree candidate roster, @@ -260,6 +306,7 @@ fn source_walk_inner( let path_glob_scope = path_glob.and_then(PathGlobScope::from_path_glob); let filter_root = project_root.to_path_buf(); let path_policy = path_policy.clone(); + let parent_gitlinks = parent_gitlink_paths(project_root)?; let mut builder = WalkBuilder::new(project_root); builder @@ -318,6 +365,12 @@ fn source_walk_inner( // freshness, so even registered submodules remain separate // sources. Linked worktrees and nested clones share this boundary. // The project root is deliberately exempt at depth zero above. + if parent_gitlinks.contains(&relative) { + if let Some(diagnostics) = &diagnostics { + diagnostics.record_skipped_nested_repository(relative); + } + return false; + } match nested_repository_marker(entry.path()) { Ok(true) => { if let Some(diagnostics) = &diagnostics { @@ -668,6 +721,33 @@ mod tests { "{rule}: an explicit scope cannot cross repositories" ); } + + // A missing child .git marker must not turn the parent's stage-0 + // gitlink into ordinary live source under an explicit include. + fs::remove_file(root.join("app/.git")).expect("remove child gitdir marker"); + assert!(root.join("app/src/feature.rs").is_file()); + let policy = IndexPathPolicyV1::new(vec![], vec!["app/**".into()]).unwrap(); + let (walk, diagnostics) = + source_walk_with_diagnostics(&root, None, &policy).expect("source walk"); + let files = walk + .map(|entry| entry.expect("source entry")) + .filter(|entry| entry.file_type().is_some_and(|kind| kind.is_file())) + .map(|entry| entry.into_path()) + .collect::>(); + assert!( + files.iter().all(|path| !path.starts_with(root.join("app"))), + "a registered gitlink stays excluded without a child marker: {files:?}" + ); + assert_eq!( + diagnostics.skipped_nested_repositories_note().as_deref(), + Some("Skipped nested repositories outside parent index: \"app\"") + ); + let explicitly_scoped = source_walk(&root, Some("app/**"), &policy) + .expect("explicit gitlink scope") + .map(|entry| entry.expect("source entry")) + .filter(|entry| entry.file_type().is_some_and(|kind| kind.is_file())) + .count(); + assert_eq!(explicitly_scoped, 0); } #[test] From 1899d261c9c6486f572a0cbdf619d07315b65378 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 01:50:36 -0800 Subject: [PATCH 10/33] style(code-index): satisfy Bazel formatting for gitlink guard Apply the two exact rustfmt_check_2024 hunks from hosted run 42466514013. No source behavior changes; authenticated submodule indexing remains pending. Refs #4593 --- crates/tracedecay-code-index/src/source_walk.rs | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/crates/tracedecay-code-index/src/source_walk.rs b/crates/tracedecay-code-index/src/source_walk.rs index 9e3f4e973b..8605696607 100644 --- a/crates/tracedecay-code-index/src/source_walk.rs +++ b/crates/tracedecay-code-index/src/source_walk.rs @@ -135,7 +135,9 @@ fn parent_gitlink_paths(project_root: &Path) -> Result, SourceW return Ok(BTreeSet::new()); } }; - if repository.workdir().and_then(|workdir| workdir.canonicalize().ok()) + if repository + .workdir() + .and_then(|workdir| workdir.canonicalize().ok()) != project_root.canonicalize().ok() { return Err(SourceWalkError { @@ -143,10 +145,12 @@ fn parent_gitlink_paths(project_root: &Path) -> Result, SourceW message: "source walk root is not the opened Git worktree".to_owned(), }); } - let index = repository.index_or_empty().map_err(|error| SourceWalkError { - glob: String::new(), - message: format!("cannot inspect parent Git index: {error}"), - })?; + let index = repository + .index_or_empty() + .map_err(|error| SourceWalkError { + glob: String::new(), + message: format!("cannot inspect parent Git index: {error}"), + })?; let mut paths = BTreeSet::new(); for entry in index.entries() { if entry.mode.is_commit() { From 77332ede0f9b9cb557bd41185203eba9500c3f0a Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 04:54:15 -0800 Subject: [PATCH 11/33] fix(code-index): deny ignored admission beneath Git gitlinks A parent stage-0 gitlink remains tracked repository source even when its child checkout loses the .git marker. Check every indexed ancestor before an ignored dependency is admitted or revalidated, preserving the existing proof for ordinary ignored sources. Add a real local Git submodule regression that removes the child marker and checks both the admission proof and production source-read refusal. Refs #4593 --- .../ignored_dependencies.rs | 122 ++++++++++++++++++ 1 file changed, 122 insertions(+) diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/ignored_dependencies.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/ignored_dependencies.rs index a5475fc590..089b3f80d1 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/ignored_dependencies.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/ignored_dependencies.rs @@ -861,6 +861,20 @@ fn prove_gix_ignored(root: &Path, logical_path: &str) -> Result<(), CodeIndexSch { return Err(CodeIndexIgnoredDependencyRefusalV1::NotIgnored.into()); } + // A registered parent gitlink is still tracked source when the child + // checkout loses its .git marker. Ignore rules may match its descendants, + // but that cannot authorize indexing those descendants as dependencies. + let mut ancestor = Path::new(logical_path).parent(); + while let Some(relative) = ancestor { + if relative + .to_str() + .and_then(|path| index.entry_by_path(path.as_bytes().as_bstr())) + .is_some_and(|entry| entry.mode.is_commit()) + { + return Err(CodeIndexIgnoredDependencyRefusalV1::NotIgnored.into()); + } + ancestor = relative.parent(); + } let mut excludes = repository .excludes( &index, @@ -933,3 +947,111 @@ fn map_reconcile_interruption_result( ) -> Result { result.map_err(map_reconcile_interruption) } + +#[cfg(test)] +mod tests { + use std::fs; + use std::path::Path; + use std::process::{Command, Output}; + + use super::{ + CodeIndexIgnoredDependencyRefusalV1, CodeIndexSchedulerErrorV1, prove_gix_ignored, + read_bounded_admitted_source, + }; + + fn fixture_git(root: &Path, args: &[&str]) -> Output { + let program = tracedecay_runtime_core::git::try_git_program().expect("Git program"); + let mut command = Command::new(program); + command.current_dir(root); + for (key, _) in std::env::vars_os() { + if key.to_string_lossy().starts_with("GIT_") { + command.env_remove(key); + } + } + let output = command + .env("GIT_CONFIG_NOSYSTEM", "1") + .env("GIT_CONFIG_GLOBAL", root.join(".fixture-gitconfig")) + .args([ + "-c", + "user.name=Fixture", + "-c", + "user.email=fixture@example.invalid", + "-c", + "commit.gpgsign=false", + ]) + .args(args) + .output() + .expect("run fixture Git"); + assert!( + output.status.success(), + "Git {args:?}: {}", + String::from_utf8_lossy(&output.stderr) + ); + output + } + + #[test] + fn parent_gitlink_cannot_be_admitted_as_ignored_dependency_without_gitdir_marker() { + let fixture = tempfile::tempdir().expect("repository fixture"); + let parent = fixture.path().join("parent"); + let child = fixture.path().join("child"); + fs::create_dir_all(&parent).unwrap(); + fs::create_dir_all(&child).unwrap(); + + fixture_git(&child, &["init", "-q"]); + fs::write(child.join("index.ts"), "export type Child = number;\n").unwrap(); + fixture_git(&child, &["add", "."]); + fixture_git(&child, &["commit", "-qm", "child"]); + + fixture_git(&parent, &["init", "-q"]); + fs::write(parent.join(".gitignore"), "node_modules/\n").unwrap(); + fixture_git(&parent, &["add", ".gitignore"]); + fixture_git(&parent, &["commit", "-qm", "parent"]); + fixture_git( + &parent, + &[ + "-c", + "protocol.file.allow=always", + "submodule", + "add", + "-f", + "-q", + "../child", + "node_modules/pkg", + ], + ); + + let staged = fixture_git(&parent, &["ls-files", "--stage", "--", "node_modules/pkg"]); + assert!(staged.stdout.starts_with(b"160000 ")); + assert!(staged.stdout.ends_with(b" 0\tnode_modules/pkg\n")); + assert!(parent.join(".gitmodules").is_file()); + + fs::remove_file(parent.join("node_modules/pkg/.git")).unwrap(); + assert!(parent.join("node_modules/pkg/index.ts").is_file()); + let denied = prove_gix_ignored(&parent, "node_modules/pkg/index.ts"); + assert!(matches!( + denied, + Err(CodeIndexSchedulerErrorV1::IgnoredDependency( + CodeIndexIgnoredDependencyRefusalV1::NotIgnored + )) + )); + let admitted = read_bounded_admitted_source(&parent, "node_modules/pkg/index.ts", None); + assert!(matches!( + admitted, + Err(CodeIndexSchedulerErrorV1::IgnoredDependency( + CodeIndexIgnoredDependencyRefusalV1::NotIgnored + )) + )); + + // Ordinary ignored sources are still eligible; only gitlink ancestry + // withdraws the source trust that the ignored-dependency proof needs. + fs::create_dir_all(parent.join("node_modules/loose")).unwrap(); + fs::write( + parent.join("node_modules/loose/index.ts"), + "export type Loose = string;\n", + ) + .unwrap(); + prove_gix_ignored(&parent, "node_modules/loose/index.ts") + .expect("non-repository ignored source remains admissible"); + } +} From 2b230e8232f6ffd23675223c261fcfbd0a26e80a Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 08:25:50 -0800 Subject: [PATCH 12/33] fix(code-index): preserve HEAD gitlink boundary on staged removal Refs #4593. Full authenticated child indexing still incomplete. --- .../tracedecay-code-index/src/source_walk.rs | 71 +++++++++++++++++++ 1 file changed, 71 insertions(+) diff --git a/crates/tracedecay-code-index/src/source_walk.rs b/crates/tracedecay-code-index/src/source_walk.rs index 8605696607..d0d3c932ab 100644 --- a/crates/tracedecay-code-index/src/source_walk.rs +++ b/crates/tracedecay-code-index/src/source_walk.rs @@ -162,6 +162,47 @@ fn parent_gitlink_paths(project_root: &Path) -> Result, SourceW paths.insert(relative.to_owned()); } } + // A staged gitlink removal does not erase the checkout boundary from + // HEAD. Without this second authority, a child whose .git marker was + // removed becomes ordinary live source until the parent commits. + let head = repository.head().map_err(|error| SourceWalkError { + glob: String::new(), + message: format!("cannot inspect parent Git HEAD: {error}"), + })?; + if head.id().is_some() { + let commit = repository.head_commit().map_err(|error| SourceWalkError { + glob: String::new(), + message: format!("cannot resolve parent Git HEAD: {error}"), + })?; + let tree = commit.tree().map_err(|error| SourceWalkError { + glob: String::new(), + message: format!("cannot read parent Git HEAD tree: {error}"), + })?; + let entries = tree + .traverse() + .breadthfirst + .files() + .map_err(|error| SourceWalkError { + glob: String::new(), + message: format!("cannot walk parent Git HEAD tree: {error}"), + })?; + for entry in entries { + if !entry.mode.is_commit() { + continue; + } + let path: &[u8] = entry.filepath.as_ref(); + let relative = std::str::from_utf8(path).map_err(|_| SourceWalkError { + glob: String::new(), + message: "non-UTF-8 HEAD gitlink cannot be safely scoped".to_owned(), + })?; + paths.insert(relative.to_owned()); + } + } else if head.referent_name().is_none() { + return Err(SourceWalkError { + glob: String::new(), + message: "parent Git HEAD has neither a commit nor an unborn branch".to_owned(), + }); + } Ok(paths) } @@ -752,6 +793,36 @@ mod tests { .filter(|entry| entry.file_type().is_some_and(|kind| kind.is_file())) .count(); assert_eq!(explicitly_scoped, 0); + + // Staging removal of the gitlink also removes it from the parent + // index, but HEAD still owns that repository boundary. Neither an + // explicit include nor a path glob may adopt its live child bytes. + fixture_git(&root, &["add", ".gitmodules", "src/parent.rs"]); + fixture_git(&root, &["commit", "-qm", "parent with submodule"]); + fixture_git(&root, &["rm", "--cached", "-f", "app"]); + assert!( + fixture_git(&root, &["ls-files", "--stage", "--", "app"]) + .stdout + .is_empty() + ); + assert!( + fixture_git(&root, &["ls-tree", "HEAD", "--", "app"]) + .stdout + .starts_with(b"160000 commit ") + ); + assert!(root.join("app/src/feature.rs").is_file()); + + let files = walked_files(&root, &policy); + assert!( + files.iter().all(|path| !path.starts_with("app")), + "HEAD gitlink is still an excluded source boundary: {files:?}" + ); + let scoped = source_walk(&root, Some("app/**"), &policy) + .expect("scope cannot cross HEAD gitlink") + .map(|entry| entry.expect("source entry")) + .filter(|entry| entry.file_type().is_some_and(|kind| kind.is_file())) + .count(); + assert_eq!(scoped, 0); } #[test] From f925fe659de81b368a6bd6c2cb40835cb472f179 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 09:07:45 -0800 Subject: [PATCH 13/33] fix(code-index): deny staged-removal gitlink in source admission Refs #4593. Reuse the parent's index-plus-HEAD boundary for ignored sources and cover the real staged-removal checkout scenario. The authenticated opt-in indexing feature remains incomplete. --- .../ignored_dependencies.rs | 42 ++++++++++++++++--- .../tracedecay-code-index/src/source_walk.rs | 2 +- 2 files changed, 38 insertions(+), 6 deletions(-) diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/ignored_dependencies.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/ignored_dependencies.rs index 089b3f80d1..709e72145e 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/ignored_dependencies.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/ignored_dependencies.rs @@ -14,6 +14,7 @@ use tracedecay_code_index::production::{ CodeIndexInterruptionV1, CodeIndexProductionErrorV1, MAX_IGNORED_DEPENDENCY_ENTRYPOINT_BYTES_V1, }; +use tracedecay_code_index::source_walk::parent_gitlink_paths; use tracedecay_contracts::ResolvedScope; use tracedecay_contracts::code_index_freshness::CodeIndexConvergenceParkedV1; use tracedecay_domain::{CodeGenerationId, SanitizerDispositionV1}; @@ -861,15 +862,15 @@ fn prove_gix_ignored(root: &Path, logical_path: &str) -> Result<(), CodeIndexSch { return Err(CodeIndexIgnoredDependencyRefusalV1::NotIgnored.into()); } - // A registered parent gitlink is still tracked source when the child - // checkout loses its .git marker. Ignore rules may match its descendants, - // but that cannot authorize indexing those descendants as dependencies. + // Admission and live source walks share the union of staged and HEAD + // gitlinks: staging a removal cannot grant trust over the child checkout. + let parent_gitlinks = parent_gitlink_paths(root) + .map_err(|error| CodeIndexSchedulerErrorV1::Git(error.message))?; let mut ancestor = Path::new(logical_path).parent(); while let Some(relative) = ancestor { if relative .to_str() - .and_then(|path| index.entry_by_path(path.as_bytes().as_bstr())) - .is_some_and(|entry| entry.mode.is_commit()) + .is_some_and(|path| parent_gitlinks.contains(path)) { return Err(CodeIndexIgnoredDependencyRefusalV1::NotIgnored.into()); } @@ -1043,6 +1044,37 @@ mod tests { )) )); + // Removing the staged gitlink does not authorize the child while + // the parent HEAD still owns the repository boundary. + fixture_git(&parent, &["add", ".gitmodules", ".gitignore"]); + fixture_git(&parent, &["commit", "-qm", "parent with submodule"]); + fixture_git(&parent, &["rm", "--cached", "-f", "node_modules/pkg"]); + assert!( + fixture_git(&parent, &["ls-files", "--stage", "--", "node_modules/pkg"]) + .stdout + .is_empty() + ); + assert!( + fixture_git(&parent, &["ls-tree", "HEAD", "--", "node_modules/pkg"]) + .stdout + .starts_with(b"160000 commit ") + ); + let denied = prove_gix_ignored(&parent, "node_modules/pkg/index.ts"); + assert!(matches!( + denied, + Err(CodeIndexSchedulerErrorV1::IgnoredDependency( + CodeIndexIgnoredDependencyRefusalV1::NotIgnored + )) + )); + let denied_read = + read_bounded_admitted_source(&parent, "node_modules/pkg/index.ts", None); + assert!(matches!( + denied_read, + Err(CodeIndexSchedulerErrorV1::IgnoredDependency( + CodeIndexIgnoredDependencyRefusalV1::NotIgnored + )) + )); + // Ordinary ignored sources are still eligible; only gitlink ancestry // withdraws the source trust that the ignored-dependency proof needs. fs::create_dir_all(parent.join("node_modules/loose")).unwrap(); diff --git a/crates/tracedecay-code-index/src/source_walk.rs b/crates/tracedecay-code-index/src/source_walk.rs index d0d3c932ab..8d49b8dc92 100644 --- a/crates/tracedecay-code-index/src/source_walk.rs +++ b/crates/tracedecay-code-index/src/source_walk.rs @@ -119,7 +119,7 @@ fn nested_repository_marker(directory: &Path) -> std::io::Result { /// checkout's .git marker has been removed. A gitlink is a repository boundary, /// not an ordinary directory the live source walker can authenticate. /// This is deny-only; it never enables submodule admission. -fn parent_gitlink_paths(project_root: &Path) -> Result, SourceWalkError> { +pub fn parent_gitlink_paths(project_root: &Path) -> Result, SourceWalkError> { let marker = project_root.join(".git"); let repository = match tracedecay_runtime_core::git_open::open(project_root) { Ok(repository) => repository, From bcd2b46a77824ebeb7dd0fbe6806336c1f21fa13 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 09:52:59 -0800 Subject: [PATCH 14/33] style(code-index): fix ignored gitlink regression formatting Refs #4593 --- .../src/code_index_scheduler/ignored_dependencies.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/ignored_dependencies.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/ignored_dependencies.rs index 709e72145e..cfae3260dc 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/ignored_dependencies.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/ignored_dependencies.rs @@ -1066,8 +1066,7 @@ mod tests { CodeIndexIgnoredDependencyRefusalV1::NotIgnored )) )); - let denied_read = - read_bounded_admitted_source(&parent, "node_modules/pkg/index.ts", None); + let denied_read = read_bounded_admitted_source(&parent, "node_modules/pkg/index.ts", None); assert!(matches!( denied_read, Err(CodeIndexSchedulerErrorV1::IgnoredDependency( From 5edd6c6cb4915da74f5f7a392e340b364064710f Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 09:56:52 -0800 Subject: [PATCH 15/33] fix(code-index): preserve HEAD gitlink boundaries in classification Refs #4593 --- .../code_index_scheduler/classification.rs | 84 +++++++++++++++++-- 1 file changed, 77 insertions(+), 7 deletions(-) diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/classification.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/classification.rs index f8ef32be30..c6c110eba0 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/classification.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/classification.rs @@ -18,7 +18,7 @@ use std::collections::{BTreeMap, BTreeSet}; use std::path::Path; use gix::bstr::ByteSlice; -use tracedecay_code_index::source_walk::has_nested_repository_ancestor; +use tracedecay_code_index::source_walk::{has_nested_repository_ancestor, parent_gitlink_paths}; use tracedecay_runtime_core::git_repository::GIT_STATUS_MODIFICATION_CHECK_THREADS; /// Failure to classify worktree changes through gix. @@ -215,6 +215,11 @@ impl WorktreeChangeClassificationV1 { // parent directory once rather than probing per source file. let mut nested_repository_exclusions = BTreeSet::new(); if let Some(project_root) = repository.workdir() { + // The live walker also denies HEAD gitlinks staged for removal. + // Without this check, Git reports their markerless child sources + // as untracked, and worktree capture can re-admit those bytes. + let parent_gitlinks = parent_gitlink_paths(project_root) + .map_err(|error| ClassificationErrorV1::Git(error.message))?; let mut checked_parents = BTreeMap::new(); for path in committed_baseline .iter() @@ -222,12 +227,20 @@ impl WorktreeChangeClassificationV1 { { let relative = Path::new(path); let parent = relative.parent().unwrap_or_else(|| Path::new("")); - let excluded = match checked_parents.get(parent) { - Some(excluded) => *excluded, - None => { - let excluded = has_nested_repository_ancestor(project_root, relative)?; - checked_parents.insert(parent.to_path_buf(), excluded); - excluded + let excluded = if relative + .ancestors() + .filter_map(Path::to_str) + .any(|ancestor| parent_gitlinks.contains(ancestor)) + { + true + } else { + match checked_parents.get(parent) { + Some(excluded) => *excluded, + None => { + let excluded = has_nested_repository_ancestor(project_root, relative)?; + checked_parents.insert(parent.to_path_buf(), excluded); + excluded + } } }; if excluded { @@ -472,4 +485,61 @@ mod nested_repository_tests { assert!(restored.candidate_paths().contains("nested/foreign.rs")); assert!(restored.candidate_paths().contains("linked/foreign.rs")); } + + #[test] + fn staged_gitlink_removal_without_child_marker_does_not_admit_child_sources() { + let fixture = tempfile::tempdir().unwrap(); + let parent = fixture.path().join("parent"); + let child = fixture.path().join("child"); + fs::create_dir_all(&parent).unwrap(); + fs::create_dir_all(&child).unwrap(); + + fixture_git(&child, &["init", "-q"]); + fs::create_dir_all(child.join("src")).unwrap(); + fs::write(child.join("src/lib.rs"), "pub fn child() {}\n").unwrap(); + fixture_git(&child, &["add", "."]); + fixture_git(&child, &["commit", "-qm", "child"]); + + fixture_git(&parent, &["init", "-q"]); + fs::create_dir_all(parent.join("src")).unwrap(); + fs::write(parent.join("src/lib.rs"), "pub fn parent() {}\n").unwrap(); + fixture_git(&parent, &["add", "."]); + fixture_git(&parent, &["commit", "-qm", "parent"]); + fixture_git( + &parent, + &[ + "-c", + "protocol.file.allow=always", + "submodule", + "add", + "-q", + "../child", + "app", + ], + ); + fixture_git(&parent, &["add", ".gitmodules", "app"]); + fixture_git(&parent, &["commit", "-qm", "register submodule"]); + + // Git still records app as a mode-160000 gitlink in HEAD, but + // removing the staged gitlink and child marker makes child files + // appear untracked to the parent. They must never enter its index. + fs::remove_file(parent.join("app/.git")).unwrap(); + fixture_git(&parent, &["rm", "-q", "--cached", "app"]); + assert!(parent.join("app/src/lib.rs").is_file()); + + let repository = tracedecay_runtime_core::git_open::open(&parent).unwrap(); + let classified = WorktreeChangeClassificationV1::classify(&repository).unwrap(); + assert!( + classified + .changes() + .iter() + .any(|change| change.path == "app/src/lib.rs"), + "fixture must exercise a child file reported by parent Git status" + ); + assert!(classified.has_changes(), "clean HEAD reuse must be refused"); + assert!(!classified.candidate_paths().contains("app/src/lib.rs")); + assert!(classified.changed_paths().contains("app/src/lib.rs")); + assert!(classified.candidate_paths().contains("src/lib.rs")); + } + } From d31966d29f5630b479168dafad49ae162ba96c27 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 09:58:03 -0800 Subject: [PATCH 16/33] fix(code-index): preserve clean registered gitlink state Refs #4593 --- .../src/code_index_scheduler/classification.rs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/classification.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/classification.rs index c6c110eba0..f5dee93d95 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/classification.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/classification.rs @@ -229,6 +229,7 @@ impl WorktreeChangeClassificationV1 { let parent = relative.parent().unwrap_or_else(|| Path::new("")); let excluded = if relative .ancestors() + .skip(1) .filter_map(Path::to_str) .any(|ancestor| parent_gitlinks.contains(ancestor)) { @@ -520,6 +521,13 @@ mod nested_repository_tests { fixture_git(&parent, &["add", ".gitmodules", "app"]); fixture_git(&parent, &["commit", "-qm", "register submodule"]); + // A registered, clean gitlink is not itself a source file or a + // dirty nested descendant. It must not defeat clean HEAD reuse. + let clean_repository = tracedecay_runtime_core::git_open::open(&parent).unwrap(); + let clean = WorktreeChangeClassificationV1::classify(&clean_repository).unwrap(); + assert!(!clean.has_changes()); + assert!(clean.candidate_paths().contains("src/lib.rs")); + // Git still records app as a mode-160000 gitlink in HEAD, but // removing the staged gitlink and child marker makes child files // appear untracked to the parent. They must never enter its index. @@ -541,5 +549,4 @@ mod nested_repository_tests { assert!(classified.changed_paths().contains("app/src/lib.rs")); assert!(classified.candidate_paths().contains("src/lib.rs")); } - } From a04503a06d1d98b458e86d8499ba9f7d557ca60d Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 10:53:08 -0800 Subject: [PATCH 17/33] fix(code-index): deny gitlink children in freshness roster Refs #4593. Do not merge until integrated submodule admission and Bazel regression acceptance. --- .../code_index_scheduler/freshness_witness.rs | 98 +++++++++++++++++-- 1 file changed, 90 insertions(+), 8 deletions(-) diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/freshness_witness.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/freshness_witness.rs index 8eb1ba4fe0..28f60ca543 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/freshness_witness.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/freshness_witness.rs @@ -132,15 +132,27 @@ fn source_candidates( let project_root = repository.workdir().ok_or_else(|| { CodeIndexSchedulerErrorV1::Git("source freshness requires a worktree".to_owned()) })?; + // Explicit ignored-file enrollment lifts Git ignore rules, never the + // parent's index/HEAD gitlink boundary. A staged gitlink removal and + // missing child .git marker must not re-admit its files through freshness. + let parent_gitlinks = tracedecay_code_index::source_walk::parent_gitlink_paths(project_root) + .map_err(|error| CodeIndexSchedulerErrorV1::Git(error.message))?; for path in &admitted_paths { - // Explicit ignored-file enrollment lifts Git ignore rules, not the - // nested repository boundary shared with capture and live search. - if !tracedecay_code_index::source_walk::has_nested_repository_ancestor( - project_root, - Path::new(path), - )? { - candidate_paths.insert((*path).to_owned()); + let relative = Path::new(path); + let beneath_gitlink = relative + .ancestors() + .skip(1) + .filter_map(Path::to_str) + .any(|ancestor| parent_gitlinks.contains(ancestor)); + if beneath_gitlink + || tracedecay_code_index::source_walk::has_nested_repository_ancestor( + project_root, + relative, + )? + { + continue; } + candidate_paths.insert((*path).to_owned()); } let mut unrepresentable = classification .non_utf8_paths() @@ -1073,7 +1085,10 @@ mod tracked_parent_boundary_tests { use std::sync::Arc; use std::time::{Duration, Instant}; - use super::{CachedCandidateRosterV1, ChangeClockReading, roster_evidence}; + use super::{ + CachedCandidateRosterV1, ChangeClockReading, CodeIndexIgnoredSourceAdmissionV1, + roster_evidence, source_candidates, + }; fn git(root: &Path, args: &[&str]) { let program = tracedecay_runtime_core::git::try_git_program().expect("fixture Git"); @@ -1105,6 +1120,73 @@ mod tracked_parent_boundary_tests { ); } + #[test] + fn ignored_source_cannot_reenter_markerless_parent_gitlink_freshness() { + let fixture = tempfile::tempdir().expect("Git fixture"); + let parent = fixture.path().join("parent"); + let child = fixture.path().join("child"); + std::fs::create_dir_all(&parent).unwrap(); + std::fs::create_dir_all(&child).unwrap(); + + git(&child, &["init", "-q"]); + std::fs::write(child.join("child.rs"), "pub fn child() {}\n").unwrap(); + git(&child, &["add", "child.rs"]); + git(&child, &["commit", "-qm", "child"]); + + git(&parent, &["init", "-q"]); + std::fs::create_dir_all(parent.join("ignored")).unwrap(); + std::fs::write(parent.join(".gitignore"), "ignored/\n").unwrap(); + std::fs::write(parent.join("parent.rs"), "pub fn parent() {}\n").unwrap(); + std::fs::write( + parent.join("ignored/ordinary.rs"), + "pub fn ordinary() {}\n", + ) + .unwrap(); + git(&parent, &["add", ".gitignore", "parent.rs"]); + git(&parent, &["commit", "-qm", "parent"]); + git( + &parent, + &[ + "-c", + "protocol.file.allow=always", + "submodule", + "add", + "-q", + "../child", + "app", + ], + ); + git(&parent, &["add", ".gitmodules", "app"]); + git(&parent, &["commit", "-qm", "registered submodule"]); + + // Git still records the child gitlink in HEAD after staging its + // removal. Its missing marker makes child.rs appear untracked, but + // an old explicit ignored admission must not revive it in freshness. + std::fs::remove_file(parent.join("app/.git")).unwrap(); + git(&parent, &["rm", "-q", "--cached", "app"]); + let repository = tracedecay_runtime_core::git_open::open(&parent).unwrap(); + let admitted = [ + CodeIndexIgnoredSourceAdmissionV1 { + logical_path: "app/child.rs".to_owned(), + }, + CodeIndexIgnoredSourceAdmissionV1 { + logical_path: "ignored/ordinary.rs".to_owned(), + }, + ]; + let candidates = source_candidates(&repository, &admitted).unwrap(); + let paths = candidates + .candidates + .iter() + .map(|candidate| candidate.logical_path.as_str()) + .collect::>(); + assert!(paths.contains("parent.rs")); + assert!(paths.contains("ignored/ordinary.rs")); + assert!( + !paths.contains("app/child.rs"), + "ignored admission must not cross a HEAD gitlink boundary" + ); + } + #[test] fn cached_tracked_directory_roster_invalidates_on_nested_git_marker() { let root = tempfile::tempdir().expect("parent checkout"); From d6e0edfe128b74abab74a255185729ca827129d6 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 11:20:49 -0800 Subject: [PATCH 18/33] style(code-index): format nested repository fixture --- .../src/code_index_scheduler/freshness_witness.rs | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/freshness_witness.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/freshness_witness.rs index 28f60ca543..6160643331 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/freshness_witness.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/freshness_witness.rs @@ -1137,11 +1137,7 @@ mod tracked_parent_boundary_tests { std::fs::create_dir_all(parent.join("ignored")).unwrap(); std::fs::write(parent.join(".gitignore"), "ignored/\n").unwrap(); std::fs::write(parent.join("parent.rs"), "pub fn parent() {}\n").unwrap(); - std::fs::write( - parent.join("ignored/ordinary.rs"), - "pub fn ordinary() {}\n", - ) - .unwrap(); + std::fs::write(parent.join("ignored/ordinary.rs"), "pub fn ordinary() {}\n").unwrap(); git(&parent, &["add", ".gitignore", "parent.rs"]); git(&parent, &["commit", "-qm", "parent"]); git( From 1bbf847d0340238b91df20188278caec6d8fa6ea Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 12:26:06 -0800 Subject: [PATCH 19/33] fix(code-index): reuse verified Git boundaries during freshness --- .../code_index_scheduler/classification.rs | 44 +++++++++++-------- .../code_index_scheduler/freshness_witness.rs | 9 +--- .../tracedecay-code-index/src/source_walk.rs | 19 ++++---- .../source_walk/nested_repository_tests.rs | 14 ++++++ 4 files changed, 52 insertions(+), 34 deletions(-) diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/classification.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/classification.rs index f5dee93d95..b69b8c16d4 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/classification.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/classification.rs @@ -97,6 +97,7 @@ pub struct WorktreeChangeClassificationV1 { /// Files still tracked by the parent but hidden behind a nested checkout /// or symlinked ancestor. These are tombstones, not indexable candidates. nested_repository_exclusions: BTreeSet, + parent_gitlinks: BTreeSet, } impl WorktreeChangeClassificationV1 { @@ -213,26 +214,28 @@ impl WorktreeChangeClassificationV1 { // refuses those files; classification must also exclude them and // invalidate clean-tree and active-generation reuse. Resolve each // parent directory once rather than probing per source file. - let mut nested_repository_exclusions = BTreeSet::new(); + let mut classification = Self { + committed_baseline, + changes, + non_utf8_changes, + non_utf8_paths, + ..Self::default() + }; if let Some(project_root) = repository.workdir() { // The live walker also denies HEAD gitlinks staged for removal. // Without this check, Git reports their markerless child sources // as untracked, and worktree capture can re-admit those bytes. - let parent_gitlinks = parent_gitlink_paths(project_root) + classification.parent_gitlinks = parent_gitlink_paths(project_root) .map_err(|error| ClassificationErrorV1::Git(error.message))?; let mut checked_parents = BTreeMap::new(); - for path in committed_baseline + for path in classification + .committed_baseline .iter() - .chain(changes.iter().map(|change| &change.path)) + .chain(classification.changes.iter().map(|change| &change.path)) { let relative = Path::new(path); let parent = relative.parent().unwrap_or_else(|| Path::new("")); - let excluded = if relative - .ancestors() - .skip(1) - .filter_map(Path::to_str) - .any(|ancestor| parent_gitlinks.contains(ancestor)) - { + let excluded = if classification.beneath_gitlink(relative) { true } else { match checked_parents.get(parent) { @@ -245,18 +248,23 @@ impl WorktreeChangeClassificationV1 { } }; if excluded { - nested_repository_exclusions.insert(path.clone()); + classification + .nested_repository_exclusions + .insert(path.clone()); } } } - Ok(Self { - committed_baseline, - changes, - non_utf8_changes, - non_utf8_paths, - nested_repository_exclusions, - }) + Ok(classification) + } + + /// Explicitly enrolled files must respect the same Git boundary snapshot + /// as ordinary candidates, including a gitlink staged for removal. + pub(super) fn beneath_gitlink(&self, path: &Path) -> bool { + path.ancestors() + .skip(1) + .filter_map(Path::to_str) + .any(|ancestor| self.parent_gitlinks.contains(ancestor)) } /// Paths whose Git bytes are not UTF-8, present or not. diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/freshness_witness.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/freshness_witness.rs index 6160643331..c0edd6eb30 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/freshness_witness.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/freshness_witness.rs @@ -135,16 +135,9 @@ fn source_candidates( // Explicit ignored-file enrollment lifts Git ignore rules, never the // parent's index/HEAD gitlink boundary. A staged gitlink removal and // missing child .git marker must not re-admit its files through freshness. - let parent_gitlinks = tracedecay_code_index::source_walk::parent_gitlink_paths(project_root) - .map_err(|error| CodeIndexSchedulerErrorV1::Git(error.message))?; for path in &admitted_paths { let relative = Path::new(path); - let beneath_gitlink = relative - .ancestors() - .skip(1) - .filter_map(Path::to_str) - .any(|ancestor| parent_gitlinks.contains(ancestor)); - if beneath_gitlink + if classification.beneath_gitlink(relative) || tracedecay_code_index::source_walk::has_nested_repository_ancestor( project_root, relative, diff --git a/crates/tracedecay-code-index/src/source_walk.rs b/crates/tracedecay-code-index/src/source_walk.rs index 8d49b8dc92..bc41a8beb7 100644 --- a/crates/tracedecay-code-index/src/source_walk.rs +++ b/crates/tracedecay-code-index/src/source_walk.rs @@ -120,19 +120,22 @@ fn nested_repository_marker(directory: &Path) -> std::io::Result { /// not an ordinary directory the live source walker can authenticate. /// This is deny-only; it never enables submodule admission. pub fn parent_gitlink_paths(project_root: &Path) -> Result, SourceWalkError> { - let marker = project_root.join(".git"); let repository = match tracedecay_runtime_core::git_open::open(project_root) { Ok(repository) => repository, Err(error) => { // Non-Git projects still support source walks. An initialized // checkout whose Git authority is unreadable must fail closed. - if marker.is_file() || marker.join("HEAD").is_file() { - return Err(SourceWalkError { + return match nested_repository_marker(project_root) { + Ok(false) => Ok(BTreeSet::new()), + Ok(true) => Err(SourceWalkError { glob: String::new(), message: format!("cannot inspect parent Git boundaries: {error}"), - }); - } - return Ok(BTreeSet::new()); + }), + Err(marker_error) => Err(SourceWalkError { + glob: String::new(), + message: format!("cannot inspect parent Git marker: {marker_error}"), + }), + }; } }; if repository @@ -642,7 +645,7 @@ mod tests { #[test] fn nested_repository_is_not_part_of_the_parent_source_tree() { let root = TempDir::new().expect("project root"); - fs::create_dir(root.path().join(".git")).expect("parent git directory"); + fixture_git(root.path(), &["init", "-q"]); fs::create_dir_all(root.path().join("src")).expect("parent source directory"); fs::write(root.path().join("src/lib.rs"), "pub fn parent() {}\n").expect("parent source"); fs::create_dir_all(root.path().join("plain")).expect("plain directory"); @@ -828,7 +831,7 @@ mod tests { #[test] fn submodule_metadata_cannot_break_an_unrelated_source_include() { let root = TempDir::new().expect("project root"); - fs::create_dir_all(root.path().join(".git")).unwrap(); + fixture_git(root.path(), &["init", "-q"]); fs::create_dir_all(root.path().join("src")).unwrap(); fs::create_dir_all(root.path().join("app/.git")).unwrap(); fs::write(root.path().join("src/keep.rs"), "pub fn keep() {}\n").unwrap(); diff --git a/crates/tracedecay-code-index/src/source_walk/nested_repository_tests.rs b/crates/tracedecay-code-index/src/source_walk/nested_repository_tests.rs index 0b77991f58..406671e045 100644 --- a/crates/tracedecay-code-index/src/source_walk/nested_repository_tests.rs +++ b/crates/tracedecay-code-index/src/source_walk/nested_repository_tests.rs @@ -181,3 +181,17 @@ fn missing_source_root_is_an_error_not_a_clean_audit() { Ok(_) => panic!("a failed walk must not report an empty successful audit"), } } + +#[test] +fn incomplete_git_root_is_refused_before_walking_sources() { + let root = tempfile::tempdir().unwrap(); + write(root.path(), "src/lib.rs", "pub fn source() {}"); + let policy = IndexPathPolicyV1::new(vec![], vec![]).unwrap(); + assert!(audit_project(root.path(), &policy).is_ok()); + + std::fs::create_dir(root.path().join(".git")).unwrap(); + assert!(matches!( + audit_project(root.path(), &policy), + Err(TraceDecayError::Config { message }) if message.contains("parent Git boundaries") + )); +} From dd1ddca5b0fdcb5220e81ac7fa56187410a95b4e Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 12:49:52 -0800 Subject: [PATCH 20/33] fix(code-index): recognize submodule modes in the Git index --- crates/tracedecay-code-index/src/source_walk.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/tracedecay-code-index/src/source_walk.rs b/crates/tracedecay-code-index/src/source_walk.rs index bc41a8beb7..d8e5ca885f 100644 --- a/crates/tracedecay-code-index/src/source_walk.rs +++ b/crates/tracedecay-code-index/src/source_walk.rs @@ -156,7 +156,7 @@ pub fn parent_gitlink_paths(project_root: &Path) -> Result, Sou })?; let mut paths = BTreeSet::new(); for entry in index.entries() { - if entry.mode.is_commit() { + if entry.mode.is_submodule() { let path: &[u8] = entry.path(&index).as_ref(); let relative = std::str::from_utf8(path).map_err(|_| SourceWalkError { glob: String::new(), From 1f7b37aa41da122818ed3ea67b75657bc9266f00 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 12:55:55 -0800 Subject: [PATCH 21/33] fix(code-index): borrow Git paths for roster freshness checks --- .../src/code_index_scheduler/freshness_witness.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/freshness_witness.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/freshness_witness.rs index c0edd6eb30..721491ddc4 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/freshness_witness.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/freshness_witness.rs @@ -535,7 +535,7 @@ fn roster_evidence( } }; for entry in index.entries() { - collect_ancestors(gix::path::from_bstr(entry.path(&index)).ok()?); + collect_ancestors(gix::path::from_bstr(entry.path(&index)).ok()?.as_ref()); } for admission in ignored_source_admissions { if validate_code_logical_path(&admission.logical_path).is_err() { From 5eaa4993902f1f9dd545eb92b4aed304737f2a36 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 12:59:40 -0800 Subject: [PATCH 22/33] test(code-index): bind ignored reads to the canonical fixture root --- .../ignored_dependencies.rs | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/ignored_dependencies.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/ignored_dependencies.rs index cfae3260dc..2819eb7815 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/ignored_dependencies.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/ignored_dependencies.rs @@ -955,6 +955,8 @@ mod tests { use std::path::Path; use std::process::{Command, Output}; + use tracedecay_runtime_core::path_safety::canonical_existing_identity; + use super::{ CodeIndexIgnoredDependencyRefusalV1, CodeIndexSchedulerErrorV1, prove_gix_ignored, read_bounded_admitted_source, @@ -998,6 +1000,7 @@ mod tests { let child = fixture.path().join("child"); fs::create_dir_all(&parent).unwrap(); fs::create_dir_all(&child).unwrap(); + let parent = canonical_existing_identity(&parent).expect("canonical project root"); fixture_git(&child, &["init", "-q"]); fs::write(child.join("index.ts"), "export type Child = number;\n").unwrap(); @@ -1037,12 +1040,15 @@ mod tests { )) )); let admitted = read_bounded_admitted_source(&parent, "node_modules/pkg/index.ts", None); - assert!(matches!( - admitted, - Err(CodeIndexSchedulerErrorV1::IgnoredDependency( - CodeIndexIgnoredDependencyRefusalV1::NotIgnored - )) - )); + assert!( + matches!( + admitted, + Err(CodeIndexSchedulerErrorV1::IgnoredDependency( + CodeIndexIgnoredDependencyRefusalV1::NotIgnored + )) + ), + "unexpected source admission: {admitted:?}" + ); // Removing the staged gitlink does not authorize the child while // the parent HEAD still owns the repository boundary. From bc7c1fc9edbee03318afd4d408374b0c3cb5adca Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 13:55:19 -0800 Subject: [PATCH 23/33] fix(code-index): refuse symlinked source leaf capture Refs #4593. Preserve live-walk exclusion through classification, worktree capture, and freshness even when a tracked symlink resolves into a registered child repository. Adds real Git submodule and symlink regressions. Partial boundary repair; keep PR draft. --- .../code_index_scheduler/classification.rs | 63 ++++++++++++++++++- .../code_index_scheduler/freshness_witness.rs | 13 ++++ .../src/code_index_scheduler/reconcile.rs | 6 +- .../tracedecay-code-index/src/source_walk.rs | 40 +++++++++++- 4 files changed, 118 insertions(+), 4 deletions(-) diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/classification.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/classification.rs index b69b8c16d4..0550c4a843 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/classification.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/classification.rs @@ -18,7 +18,9 @@ use std::collections::{BTreeMap, BTreeSet}; use std::path::Path; use gix::bstr::ByteSlice; -use tracedecay_code_index::source_walk::{has_nested_repository_ancestor, parent_gitlink_paths}; +use tracedecay_code_index::source_walk::{ + has_nested_repository_ancestor, is_source_symlink, parent_gitlink_paths, +}; use tracedecay_runtime_core::git_repository::GIT_STATUS_MODIFICATION_CHECK_THREADS; /// Failure to classify worktree changes through gix. @@ -235,7 +237,13 @@ impl WorktreeChangeClassificationV1 { { let relative = Path::new(path); let parent = relative.parent().unwrap_or_else(|| Path::new("")); - let excluded = if classification.beneath_gitlink(relative) { + // The live walker never follows a symlink file, even when + // the lexical parent is ordinary and the target is inside a + // registered submodule. Do not capture its target through a + // clean-tree or worktree path. + let excluded = if classification.beneath_gitlink(relative) + || is_source_symlink(project_root, relative)? + { true } else { match checked_parents.get(parent) { @@ -495,6 +503,57 @@ mod nested_repository_tests { assert!(restored.candidate_paths().contains("linked/foreign.rs")); } + #[cfg(unix)] + #[test] + fn tracked_symlink_into_submodule_is_not_parent_source() { + use std::os::unix::fs::symlink; + + let fixture = tempfile::tempdir().unwrap(); + let parent = fixture.path().join("parent"); + let child = fixture.path().join("child"); + fs::create_dir_all(&parent).unwrap(); + fs::create_dir_all(&child).unwrap(); + fixture_git(&child, &["init", "-q"]); + fs::create_dir_all(child.join("src")).unwrap(); + fs::write(child.join("src/lib.rs"), "pub fn child() {}\n").unwrap(); + fixture_git(&child, &["add", "."]); + fixture_git(&child, &["commit", "-qm", "child"]); + fixture_git(&parent, &["init", "-q"]); + fixture_git( + &parent, + &[ + "-c", + "protocol.file.allow=always", + "submodule", + "add", + "-q", + "../child", + "app", + ], + ); + symlink("app/src/lib.rs", parent.join("bridge.rs")).unwrap(); + fixture_git(&parent, &["add", ".gitmodules", "app", "bridge.rs"]); + fixture_git(&parent, &["commit", "-qm", "registered child and source alias"]); + assert_eq!( + fs::read_to_string(parent.join("bridge.rs")).unwrap(), + "pub fn child() {}\n", + "fixture must prove the parent symlink actually reaches child bytes" + ); + assert!( + fs::symlink_metadata(parent.join("bridge.rs")) + .unwrap() + .file_type() + .is_symlink() + ); + + let repository = tracedecay_runtime_core::git_open::open(&parent).unwrap(); + let classified = WorktreeChangeClassificationV1::classify(&repository).unwrap(); + assert!(!classified.candidate_paths().contains("bridge.rs")); + assert!(classified.changed_paths().contains("bridge.rs")); + assert!(classified.has_changes(), "do not reuse HEAD tree source bytes"); + assert!(!classified.candidate_paths().contains("app/src/lib.rs")); + } + #[test] fn staged_gitlink_removal_without_child_marker_does_not_admit_child_sources() { let fixture = tempfile::tempdir().unwrap(); diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/freshness_witness.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/freshness_witness.rs index 721491ddc4..d81c425e9e 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/freshness_witness.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/freshness_witness.rs @@ -142,6 +142,7 @@ fn source_candidates( project_root, relative, )? + || tracedecay_code_index::source_walk::is_source_symlink(project_root, relative)? { continue; } @@ -243,6 +244,18 @@ fn read_candidate( project_root: &Path, candidate: &StatCandidateV1, ) -> Result, CodeIndexSchedulerErrorV1> { + // A link installed after candidate classification must not turn a + // freshness proof into a read of a different repository's source bytes. + if tracedecay_code_index::source_walk::is_source_symlink( + project_root, + Path::new(&candidate.logical_path), + )? { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidData, + "source symlink is outside the parent source inventory", + ) + .into()); + } if candidate.explicitly_admitted { ignored_dependencies::read_explicitly_admitted_source( project_root, diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/reconcile.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/reconcile.rs index e2d51bfeb6..5724543eff 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/reconcile.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/reconcile.rs @@ -3976,9 +3976,13 @@ impl CodeIndexWorktreeSchedulerV1 { ) -> Result { // Explicit ignored-source admission cannot cross a nested checkout // any more than an ordinary Git candidate or interactive source walk. + let relative = std::path::Path::new(logical_path); if tracedecay_code_index::source_walk::has_nested_repository_ancestor( &self.project_root, - std::path::Path::new(logical_path), + relative, + )? || tracedecay_code_index::source_walk::is_source_symlink( + &self.project_root, + relative, )? { return Ok(CapturedFileOutcomeV1::Absent); } diff --git a/crates/tracedecay-code-index/src/source_walk.rs b/crates/tracedecay-code-index/src/source_walk.rs index d8e5ca885f..a72a87b77e 100644 --- a/crates/tracedecay-code-index/src/source_walk.rs +++ b/crates/tracedecay-code-index/src/source_walk.rs @@ -247,6 +247,28 @@ pub fn has_nested_repository_ancestor( Ok(false) } +/// A worktree source entry must be a real file, not a link to bytes outside +/// the parent's Git authority. The shared live walker does not follow leaf +/// symlinks; classification, capture, and freshness must make the same choice. +/// Missing paths remain possible tombstones and are not symlink exclusions. +pub fn is_source_symlink(project_root: &Path, relative_path: &Path) -> std::io::Result { + if relative_path.as_os_str().is_empty() + || relative_path + .components() + .any(|component| !matches!(component, Component::Normal(_))) + { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidInput, + "source path must be repository-relative", + )); + } + match std::fs::symlink_metadata(project_root.join(relative_path)) { + Ok(metadata) => Ok(metadata.file_type().is_symlink()), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(false), + Err(error) => Err(error), + } +} + /// Diagnostics emitted by the same source walker that enforces the nested /// repository boundary. Only directories within the walk's policy are named. #[derive(Clone, Default)] @@ -480,7 +502,9 @@ mod tests { use tempfile::TempDir; use tracedecay_domain::IndexPathPolicyV1; - use super::{has_nested_repository_ancestor, source_walk, source_walk_with_diagnostics}; + use super::{ + has_nested_repository_ancestor, is_source_symlink, source_walk, source_walk_with_diagnostics, + }; #[test] fn nested_repository_boundary_excludes_parent_markers_and_path_escape() { @@ -526,6 +550,20 @@ mod tests { ); } + #[cfg(unix)] + #[test] + fn readable_symlink_leaf_is_not_an_admissible_source() { + use std::os::unix::fs::symlink; + + let root = TempDir::new().unwrap(); + fs::write(root.path().join("real.rs"), "pub fn real() {}\n").unwrap(); + symlink("real.rs", root.path().join("alias.rs")).unwrap(); + assert!(is_source_symlink(root.path(), std::path::Path::new("alias.rs")).unwrap()); + assert!(!is_source_symlink(root.path(), std::path::Path::new("real.rs")).unwrap()); + assert!(!is_source_symlink(root.path(), std::path::Path::new("missing.rs")).unwrap()); + assert!(is_source_symlink(root.path(), std::path::Path::new("../escape.rs")).is_err()); + } + fn walked_files(root: &std::path::Path, path_policy: &IndexPathPolicyV1) -> Vec { let mut files = source_walk(root, None, path_policy) .expect("source walk") From 632729906cbf0fe46b858bd556e716550db062b2 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 01:52:34 -0800 Subject: [PATCH 24/33] test(automation): bind scheduler fixtures to backend identities Repair three hosted automation_runner_test failures by supplying the configured executable identity required by the production prefilter. Use the same identity for the prefilter and retained runner, preserving manual-schedule, exact-ledger, live-lock and lock-race assertions. Add a production-prefilter regression for unconfigured refusal, deduplication and same-occurrence recovery after binding, with zero backend calls and no effectful terminal. Do not relax backend admission, change timeouts, or alter the graph publication changes. Refs #4532 --- .../tests/automation_runner_test/jobs.rs | 124 ++++++++++++++++-- 1 file changed, 110 insertions(+), 14 deletions(-) diff --git a/crates/tracedecay/tests/automation_runner_test/jobs.rs b/crates/tracedecay/tests/automation_runner_test/jobs.rs index 740283d56e..7342786381 100644 --- a/crates/tracedecay/tests/automation_runner_test/jobs.rs +++ b/crates/tracedecay/tests/automation_runner_test/jobs.rs @@ -86,6 +86,7 @@ fn enabled_job_config() -> AutomationSettingsV1 { struct ContentBackend { calls: AtomicUsize, content: &'static str, + executable: Option, } impl ContentBackend { @@ -93,9 +94,15 @@ impl ContentBackend { Self { calls: AtomicUsize::new(0), content, + executable: None, } } + fn with_executable(mut self, executable: &Path) -> Self { + self.executable = Some(executable.to_path_buf()); + self + } + fn calls(&self) -> usize { self.calls.load(Ordering::SeqCst) } @@ -126,7 +133,7 @@ impl AgentTaskBackend for ContentBackend { } fn executable(&self) -> Option<&std::path::Path> { - None + self.executable.as_deref() } } @@ -988,6 +995,10 @@ async fn scheduler_trigger_skips_repeat_and_respects_lock_discipline() { let mut job = sample_job("sched-job"); job.schedule = Some("manual".to_string()); let config = enabled_job_config(); + // Reach the schedule gate with a bound identity, not the earlier + // backend-unconfigured refusal. This gate never invokes the backend. + let executable = tempfile::NamedTempFile::new().unwrap(); + let backend = ContentBackend::new("unused").with_executable(executable.path()); // Manual schedule: every scheduler tick of the same occurrence reaches // the prefilter, which records the skip once. @@ -996,7 +1007,7 @@ async fn scheduler_trigger_skips_repeat_and_respects_lock_discipline() { let run = evaluate_and_record_scheduler_skip( &dashboard_root, &config, - None, + backend.executable(), &job, "sched-occurrence", None, @@ -1019,7 +1030,7 @@ async fn scheduler_trigger_skips_repeat_and_respects_lock_discipline() { let repeated = evaluate_and_record_scheduler_skip( &dashboard_root, &config, - None, + backend.executable(), &job, "sched-occurrence", None, @@ -1031,6 +1042,76 @@ async fn scheduler_trigger_skips_repeat_and_respects_lock_discipline() { repeated.ledger_record, records[0], "an immediate repeat skip must return the exact durable prior row" ); + assert_eq!(backend.calls(), 0); +} + +#[tokio::test] +async fn scheduler_prefilter_rejects_unconfigured_backend_and_recovers_without_effect() { + let temp = tempdir().unwrap(); + let dashboard_root = temp.path().join("dashboard"); + let job = sample_job("backend-recovery"); + let config = enabled_job_config(); + let occurrence = "backend-recovery-occurrence"; + let backend = ContentBackend::new("unused"); + + let first = evaluate_and_record_scheduler_skip( + &dashboard_root, + &config, + backend.executable(), + &job, + occurrence, + None, + ) + .await + .unwrap() + .expect("a due job still requires a configured backend"); + assert_eq!(first.report["reason"], json!("backend_unconfigured")); + assert_eq!(first.ledger_record.status, AutomationRunStatus::Skipped); + assert_ne!(first.run_id, occurrence); + let repeated = evaluate_and_record_scheduler_skip( + &dashboard_root, + &config, + backend.executable(), + &job, + occurrence, + None, + ) + .await + .unwrap() + .unwrap(); + assert_eq!(repeated.ledger_record, first.ledger_record); + + let executable = tempfile::NamedTempFile::new().unwrap(); + let backend = backend.with_executable(executable.path()); + assert!( + evaluate_and_record_scheduler_skip( + &dashboard_root, + &config, + backend.executable(), + &job, + occurrence, + None, + ) + .await + .unwrap() + .is_none(), + "binding the backend must admit the same due occurrence without cooldown" + ); + assert_eq!(backend.calls(), 0); + assert_eq!( + load_run_records(&dashboard_root, 10).await.unwrap(), + vec![first.ledger_record], + "advisory admission must not append an effectful terminal" + ); + assert!( + tracedecay_automation_runtime::automation::run_ledger::find_run_record_exact_bounded( + &dashboard_root, + occurrence, + ) + .await + .unwrap() + .is_none() + ); } #[tokio::test] @@ -1206,15 +1287,22 @@ async fn scheduler_prefilter_live_lock_wins_over_not_due_summary() { let config = enabled_job_config(); let occurrence = "locked-prefilter-occurrence"; - let prefilter = - evaluate_and_record_scheduler_skip(&dashboard_root, &config, None, &job, occurrence, None) - .await - .unwrap() - .expect("the live job lock must produce a diagnostic"); + let executable = tempfile::NamedTempFile::new().unwrap(); + let backend = ContentBackend::new("unused").with_executable(executable.path()); + let prefilter = evaluate_and_record_scheduler_skip( + &dashboard_root, + &config, + backend.executable(), + &job, + occurrence, + None, + ) + .await + .unwrap() + .expect("the live job lock must produce a diagnostic"); assert_eq!(prefilter.report["reason"], json!("scheduler_lock_active")); assert_ne!(prefilter.run_id, occurrence); - let backend = ContentBackend::new("unused"); let admitted_job = tracedecay_automation_runtime::automation::jobs::admit_job(&job).unwrap(); let retained = run_user_job_with_backend_for_retained_settlement( &dashboard_root, @@ -1262,12 +1350,21 @@ async fn retained_scheduler_runner_reacquires_after_due_prefilter() { job.schedule = Some("every 1h".to_owned()); let config = enabled_job_config(); let occurrence = "prefilter-gap-occurrence"; + let executable = tempfile::NamedTempFile::new().unwrap(); + let backend = ContentBackend::new("unused").with_executable(executable.path()); assert!( - evaluate_and_record_scheduler_skip(&dashboard_root, &config, None, &job, occurrence, None) - .await - .unwrap() - .is_none(), + evaluate_and_record_scheduler_skip( + &dashboard_root, + &config, + backend.executable(), + &job, + occurrence, + None, + ) + .await + .unwrap() + .is_none(), "a new interval job is advisory-due before effect admission" ); let _raced_in = @@ -1276,7 +1373,6 @@ async fn retained_scheduler_runner_reacquires_after_due_prefilter() { .unwrap() .expect("the advisory prefilter must release its reservation lock"); - let backend = ContentBackend::new("unused"); let admitted_job = tracedecay_automation_runtime::automation::jobs::admit_job(&job).unwrap(); let retained = run_user_job_with_backend_for_retained_settlement( &dashboard_root, From 3fdb50ddbe2b9fe19f80f3bb94b96d2c738c66b0 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 14:03:19 -0800 Subject: [PATCH 25/33] feat(code-index): index authenticated direct submodules --- Cargo.lock | 1 + .../src/code_index_scheduler.rs | 2 +- .../code_index_scheduler/freshness_witness.rs | 128 +++++++-- .../code_index_scheduler/git_tree_capture.rs | 119 ++++++-- .../src/code_index_scheduler/identity.rs | 60 +++- .../ignored_dependencies.rs | 2 +- .../src/code_index_scheduler/reconcile.rs | 124 +++++++-- .../tests/index_path_policy_tests.rs | 213 ++++++++++++++ .../tests/code_index_scheduler/reconcile.rs | 26 +- crates/tracedecay-code-index/Cargo.toml | 1 + .../benches/partitioned_codec.rs | 2 + .../benches/sealed_storage.rs | 2 + .../src/production/ignored_sources.rs | 3 + .../src/production/lexical_page_source.rs | 1 + .../production/lexical_page_source_tests.rs | 2 + .../src/production/mod.rs | 20 ++ .../src/production/sparse_increment_tests.rs | 1 + .../src/production/worker_tests.rs | 2 + .../tracedecay-code-index/src/source_walk.rs | 262 +++++++++++++----- .../src/source_walk}/classification.rs | 115 +++++++- .../src/source_walk/submodules.rs | 188 +++++++++++++ .../production_orchestration.rs | 5 + .../parallel_equivalence.rs | 1 + .../tests/resident_accounting.rs | 2 + .../src/config/mod.rs | 34 ++- crates/tracedecay-domain/src/configuration.rs | 2 + crates/tracedecay-domain/src/lib.rs | 7 +- .../src/source_path_policy.rs | 18 +- .../src/configuration/registry.rs | 20 +- .../src/configuration/store.rs | 11 +- .../src/configuration/store/tests/mod.rs | 45 +-- .../src/handlers/analysis/field_sites.rs | 7 +- .../src/handlers/analysis/mod.rs | 8 +- .../src/handlers/graph/rename_preview.rs | 16 ++ .../benches/code_lexical_catchup.rs | 2 + .../src/bin/tracedecay_index_bench.rs | 2 + .../src/bin/tracedecay_search_bench.rs | 2 + .../candidate_producers.rs | 2 + .../src/git_repository.rs | 2 +- .../src/git_repository/native_integration.rs | 47 +++- .../native_integration_authority.rs | 3 + .../src/candidate_output.rs | 2 + .../src/candidate_output/cancellation.rs | 2 + docs/USER-GUIDE.md | 16 ++ 44 files changed, 1321 insertions(+), 209 deletions(-) rename crates/{tracedecay-code-index-runtime/src/code_index_scheduler => tracedecay-code-index/src/source_walk}/classification.rs (85%) create mode 100644 crates/tracedecay-code-index/src/source_walk/submodules.rs diff --git a/Cargo.lock b/Cargo.lock index 860f7e0e1d..4238a98de7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6430,6 +6430,7 @@ dependencies = [ "base64 0.23.1", "criterion", "flate2", + "gix", "hex", "ignore", "miniz_oxide", diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler.rs index a325e59813..d3ea24dc19 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler.rs @@ -201,7 +201,7 @@ mod activation; pub mod branch_generations; pub mod branch_publication; mod cadence; -mod classification; +use tracedecay_code_index::source_walk::classification; mod demand_admission; mod freshness_witness; pub use freshness_witness::sealed_source_bytes; diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/freshness_witness.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/freshness_witness.rs index d81c425e9e..b98634731b 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/freshness_witness.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/freshness_witness.rs @@ -76,10 +76,12 @@ pub struct WorktreeStatSweepV1 { pub fn worktree_stat_sweep( project_root: &Path, ignored_source_admissions: &[CodeIndexIgnoredSourceAdmissionV1], + include_submodules: bool, ) -> Result { let repository = tracedecay_runtime_core::git_open::open(project_root) .map_err(|error| CodeIndexSchedulerErrorV1::Git(error.to_string()))?; - let candidate_roster = source_candidates(&repository, ignored_source_admissions)?; + let candidate_roster = + source_candidates(&repository, ignored_source_admissions, include_submodules)?; // One sweep span covers the O(candidates) stat walk. Never emit one // trace event per file. let mut buf = Vec::new(); @@ -120,9 +122,11 @@ struct SourceCandidatesV1 { fn source_candidates( repository: &gix::Repository, ignored_source_admissions: &[CodeIndexIgnoredSourceAdmissionV1], + include_submodules: bool, ) -> Result { - let classification = classification::WorktreeChangeClassificationV1::classify(repository) - .map_err(|error| CodeIndexSchedulerErrorV1::Git(error.to_string()))?; + let classification = + classification::WorktreeChangeClassificationV1::for_policy(repository, include_submodules) + .map_err(|error| CodeIndexSchedulerErrorV1::Git(error.to_string()))?; let registry = StaticLanguageRegistry::new(); let admitted_paths = ignored_source_admissions .iter() @@ -647,17 +651,27 @@ impl SourceSweepCacheV1 { let Ok(repository) = tracedecay_runtime_core::git_open::open(project_root) else { return (false, stats); }; - let evidence = roster_evidence( - &repository, - project_root, - sampled_at, - ignored_source_admissions, - ); + // Parent-only directory evidence cannot vouch for child Git indexes, + // refs or untracked entries. Enumerate both Git rosters on every child proof. + let evidence = (!self.path_policy.includes_submodules()) + .then(|| { + roster_evidence( + &repository, + project_root, + sampled_at, + ignored_source_admissions, + ) + }) + .flatten(); let Ok(SourceCandidatesV1 { candidates, unrepresentable, .. - }) = source_candidates(&repository, ignored_source_admissions) + }) = source_candidates( + &repository, + ignored_source_admissions, + self.path_policy.includes_submodules(), + ) else { return (false, stats); }; @@ -735,12 +749,14 @@ impl SourceSweepCacheV1 { &mut self, project_root: &Path, manifest: &SourceContentManifestV1, + candidate_paths: &BTreeSet, shutting_down: &AtomicBool, ) -> Result, CodeIndexSchedulerErrorV1> { let registry = StaticLanguageRegistry::new(); let candidates = manifest .files .keys() + .filter(|path| candidate_paths.contains(*path)) .filter_map(|logical_path| source_candidate(®istry, logical_path.clone(), false)) .collect::>(); let present = present_candidates( @@ -838,7 +854,12 @@ impl SourceSweepCacheV1 { if disputed.is_empty() { return Ok(disputed); } - Ok(clean_filter_mismatches(project_root, disputed, manifest)) + Ok(clean_filter_mismatches( + project_root, + disputed, + manifest, + self.path_policy.includes_submodules(), + )) } } @@ -882,10 +903,56 @@ fn clean_filter_mismatches<'a>( project_root: &Path, disputed: Vec<&'a StatCandidateV1>, manifest: &SourceContentManifestV1, + include_submodules: bool, ) -> Vec<&'a StatCandidateV1> { let Ok(repository) = tracedecay_runtime_core::git_open::open(project_root) else { return disputed; }; + let mut remaining = disputed; + let mut mismatches = Vec::new(); + if include_submodules { + let Ok(children) = + tracedecay_code_index::source_walk::submodules::live_submodules(&repository) + else { + return remaining; + }; + for (prefix, binding) in children { + let (local, other): (Vec<_>, Vec<_>) = remaining + .into_iter() + .partition(|candidate| Path::new(&candidate.logical_path).starts_with(&prefix)); + remaining = other; + if local.is_empty() { + continue; + } + match tracedecay_runtime_core::git_open::open(&binding.git_dir) { + Ok(child) => mismatches.extend(filter_mismatches_in_repository( + project_root, + &child, + Path::new(&prefix), + local, + manifest, + )), + Err(_) => mismatches.extend(local), + } + } + } + mismatches.extend(filter_mismatches_in_repository( + project_root, + &repository, + Path::new(""), + remaining, + manifest, + )); + mismatches +} + +fn filter_mismatches_in_repository<'a>( + project_root: &Path, + repository: &gix::Repository, + prefix: &Path, + disputed: Vec<&'a StatCandidateV1>, + manifest: &SourceContentManifestV1, +) -> Vec<&'a StatCandidateV1> { let Ok((mut pipeline, index)) = repository.filter_pipeline(None) else { return disputed; }; @@ -895,15 +962,17 @@ fn clean_filter_mismatches<'a>( let Some(expected) = manifest.files.get(&candidate.logical_path) else { return true; }; + let Ok(rela_path) = Path::new(&candidate.logical_path).strip_prefix(prefix) else { + return true; + }; if candidate.explicitly_admitted || index - .entry_by_path(BStr::new(candidate.logical_path.as_str())) + .entry_by_path(BStr::new(rela_path.as_os_str().as_encoded_bytes())) .is_none() { return true; } - let rela_path = Path::new(&candidate.logical_path); - let Ok(file) = std::fs::File::open(project_root.join(rela_path)) else { + let Ok(file) = std::fs::File::open(project_root.join(&candidate.logical_path)) else { return true; }; let Ok(mut converted) = pipeline.convert_to_git(file, rela_path, &index) else { @@ -930,8 +999,13 @@ pub fn sealed_source_bytes( project_root: &Path, logical_path: &str, language: &LanguageId, + include_submodules: bool, sealed: &ContentDigest, ) -> Result>, CodeIndexSchedulerErrorV1> { + if tracedecay_code_index::source_walk::is_source_symlink(project_root, Path::new(logical_path))? + { + return Ok(None); + } let raw = ignored_dependencies::read_bounded_snapshot_source(&project_root.join(logical_path), None)?; let (sanitized, _, _) = privacy::sanitize_code_file(language, &raw)?; @@ -942,15 +1016,35 @@ pub fn sealed_source_bytes( // hoist them into a caller-held reader if a caller ever needs hundreds. let repository = tracedecay_runtime_core::git_open::open(project_root) .map_err(|error| CodeIndexSchedulerErrorV1::Git(error.to_string()))?; + let mut repository = repository; + let mut relative = Path::new(logical_path); + if include_submodules { + let children = tracedecay_code_index::source_walk::submodules::live_submodules(&repository) + .map_err(|error| CodeIndexSchedulerErrorV1::Git(error.message))?; + if let Some((prefix, binding)) = children + .iter() + .find(|(prefix, _)| relative.starts_with(prefix)) + { + let child_path = relative + .strip_prefix(prefix) + .map_err(|_| CodeIndexSchedulerErrorV1::Git("invalid child filter path".into()))?; + repository = tracedecay_runtime_core::git_open::open(&binding.git_dir) + .map_err(|error| CodeIndexSchedulerErrorV1::Git(error.to_string()))?; + relative = child_path; + } + } let (mut pipeline, index) = repository .filter_pipeline(None) .map_err(|error| CodeIndexSchedulerErrorV1::Git(error.to_string()))?; - if index.entry_by_path(BStr::new(logical_path)).is_none() { + if index + .entry_by_path(BStr::new(relative.as_os_str().as_encoded_bytes())) + .is_none() + { return Ok(None); } let mut filtered = Vec::new(); pipeline - .convert_to_git(raw.as_slice(), Path::new(logical_path), &index) + .convert_to_git(raw.as_slice(), relative, &index) .map_err(|error| CodeIndexSchedulerErrorV1::Git(error.to_string()))? .read_to_end(&mut filtered)?; let (sanitized, _, _) = privacy::sanitize_code_file(language, &filtered)?; @@ -1175,7 +1269,7 @@ mod tracked_parent_boundary_tests { logical_path: "ignored/ordinary.rs".to_owned(), }, ]; - let candidates = source_candidates(&repository, &admitted).unwrap(); + let candidates = source_candidates(&repository, &admitted, false).unwrap(); let paths = candidates .candidates .iter() diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/git_tree_capture.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/git_tree_capture.rs index 7db65b9c88..922d0c889e 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/git_tree_capture.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/git_tree_capture.rs @@ -16,7 +16,9 @@ use tracedecay_domain::{ }; use tracedecay_privacy::CODE_SOURCE_SANITIZER_VERSION_V1; use tracedecay_query::code_search::CodeIndexSearchUnavailableReasonV1; -use tracedecay_runtime_core::git_repository::resolve_tree_link; +use tracedecay_runtime_core::git_repository::{ + GitNativeCandidateTreeVisitError, GitTreeSubmodulePins, resolve_tree_link, +}; use tracedecay_runtime_core::resident_memory::ResidentMemoryReservationV1; use super::{ @@ -161,7 +163,7 @@ pub(super) fn representable_logical_path(git_path: &[u8]) -> Option<&str> { /// Whether capture would parse `git_path` as source. A committed symlink at /// such a path is read through to its target, as a checkout reads it, so its /// link text never becomes source. -fn is_source_path(registry: &StaticLanguageRegistry, git_path: &[u8]) -> bool { +pub(super) fn is_source_path(registry: &StaticLanguageRegistry, git_path: &[u8]) -> bool { representable_logical_path(git_path) .and_then(|logical_path| Path::new(logical_path).extension()) .and_then(|extension| extension.to_str()) @@ -396,9 +398,8 @@ impl CodeIndexWorktreeSchedulerV1 { } /// HEAD's tree standing in for a clean checkout. Git stores a symlink as - /// its target path, while worktree capture and every source sweep read - /// through the link, so a tree holding a source symlink would seal bytes - /// the next sweep disproves. That tree is `GenerationUnavailable` here and + /// its target path, while the live roster excludes the link. A tree holding + /// a source symlink would seal bytes the next sweep disproves. That tree is `GenerationUnavailable` here and /// the checkout is captured from disk instead. pub(super) fn capture_clean_head_tree_snapshot( &self, @@ -458,8 +459,15 @@ impl CodeIndexWorktreeSchedulerV1 { source.tree.clone(), control, |visitor| { + let mut gitlinks = Vec::new(); for entry in entries { control.termination().map_or(Ok(()), Err)?; + if self.path_policy.includes_submodules() && entry.mode.is_commit() { + let path = entry.filepath.to_str().map_err(|_| { + CodeIndexSearchUnavailableReasonV1::GenerationUnavailable + })?; + gitlinks.push((path.to_owned(), entry.oid)); + } if entry.mode.is_tree() || entry.mode.is_commit() { continue; } @@ -480,11 +488,61 @@ impl CodeIndexWorktreeSchedulerV1 { } visitor(entry.filepath.as_ref(), &blob.data)?; } - Ok(()) + let pins = GitTreeSubmodulePins::from_tree(&tree, gitlinks) + .map_err(|_| CodeIndexSearchUnavailableReasonV1::GenerationUnavailable)?; + self.visit_pinned_submodule_blobs(&repository, &pins, control, visitor) }, ) } + fn visit_pinned_submodule_blobs( + &self, + parent: &gix::Repository, + pins: &GitTreeSubmodulePins, + control: &branch_generations::BranchGenerationReadControlV1, + visitor: &mut dyn FnMut(&[u8], &[u8]) -> Result<(), CodeIndexSearchUnavailableReasonV1>, + ) -> Result<(), CodeIndexSearchUnavailableReasonV1> { + let unavailable = CodeIndexSearchUnavailableReasonV1::GenerationUnavailable; + let children = + tracedecay_code_index::source_walk::submodules::pinned_submodules(parent, pins) + .map_err(|_| unavailable)?; + let registry = StaticLanguageRegistry::new(); + for (prefix, (repo, pin)) in children { + control.termination().map_or(Ok(()), Err)?; + let tree = repo + .find_commit(pin) + .and_then(|commit| commit.tree()) + .map_err(|_| unavailable)?; + for entry in tree + .traverse() + .breadthfirst + .files() + .map_err(|_| unavailable)? + { + control.termination().map_or(Ok(()), Err)?; + if !entry.mode.is_blob() { + continue; + } + let path = [prefix.as_bytes(), b"/", entry.filepath.as_slice()].concat(); + let Some(logical_path) = representable_logical_path(&path) else { + // Capture records the unrepresentable name without consulting its blob. + visitor(&path, &[])?; + continue; + }; + if !is_source_path(®istry, &path) + || !tracedecay_code_index::source_walk::submodules::is_source_location( + Path::new(logical_path), + ) + { + continue; + } + let blob = repo.find_blob(entry.oid).map_err(|_| unavailable)?; + visitor(&path, &blob.data)?; + } + } + Ok(()) + } + #[tracing::instrument( name = "daemon.code_index.capture.native_candidate_tree", level = "trace", @@ -510,14 +568,24 @@ impl CodeIndexWorktreeSchedulerV1 { expected_tree.clone(), control, |visitor| { - candidate.visit_blobs(|path| is_source_path(®istry, path), visitor).map_err(|error| match error { - tracedecay_runtime_core::git_repository::GitNativeCandidateTreeVisitError::Repository(_) => { - control + let pins = candidate + .visit_blobs( + self.path_policy.includes_submodules(), + |path| is_source_path(®istry, path), + &mut *visitor, + ) + .map_err(|error| match error { + GitNativeCandidateTreeVisitError::Repository(_) => control .termination() - .unwrap_or(CodeIndexSearchUnavailableReasonV1::GenerationUnavailable) - } - tracedecay_runtime_core::git_repository::GitNativeCandidateTreeVisitError::Visitor(reason) => reason, - }) + .unwrap_or(CodeIndexSearchUnavailableReasonV1::GenerationUnavailable), + GitNativeCandidateTreeVisitError::Visitor(reason) => reason, + })?; + if pins.gitlinks.is_empty() { + return Ok(()); + } + let parent = tracedecay_runtime_core::git_open::open(&self.project_root) + .map_err(|_| CodeIndexSearchUnavailableReasonV1::GenerationUnavailable)?; + self.visit_pinned_submodule_blobs(&parent, &pins, control, visitor) }, ) } @@ -601,6 +669,7 @@ impl CodeIndexWorktreeSchedulerV1 { snapshot_content_identity(&files, &sanitization_receipts, &omitted_sources); Ok(CapturedSnapshotV1 { repository_parse_identity: CodeIndexRepositoryParseIdentityV1 { + include_submodules: self.path_policy.includes_submodules(), tree: Some(tree), dirty: tracedecay_domain::RepositoryDirtyStateV1::Clean, }, @@ -1326,16 +1395,24 @@ mod tests { ], "only source links that name a file in the tree are captured" ); + let live = present( + &scheduler + .capture_authoritative_snapshot_without_active_generation_reuse(None) + .expect("capture the checkout from disk") + .snapshot, + ); assert_eq!( - branch, - present( - &scheduler - .capture_authoritative_snapshot_without_active_generation_reuse(None) - .expect("capture the checkout from disk") - .snapshot - ), - "a branch capture must index the bytes a checkout reads through each link" + live.iter() + .map(|(path, _)| path.as_str()) + .collect::>(), + ["AGENTS.md", "src/real.rs"] ); + for regular in live { + assert!( + branch.contains(®ular), + "historical and live regular files must carry the same source digest" + ); + } let notes = captured .snapshot .files diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/identity.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/identity.rs index 88ca93c0a4..728e42d7b4 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/identity.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/identity.rs @@ -154,6 +154,11 @@ pub struct GitMetadataFingerprintV1 { /// branch switch changes this even when mtime resolution is too coarse to /// observe, so content is compared alongside mtimes. head_contents: Option, + // Include the registered child pin, bound checkout and resolved HEAD in the same + // read-admission proof as the parent. A failed probe cannot renew that proof. + submodule_refs: Option, + submodule_indexes: Option, + submodule_probe_failed: bool, } impl GitMetadataFingerprintV1 { @@ -176,19 +181,60 @@ impl GitMetadataFingerprintV1 { packed_refs: mtime(&common_dir.join("packed-refs")), refs_heads: refs_heads_signature(&common_dir.join("refs").join("heads")), head_contents: std::fs::read_to_string(&head_path).ok(), + ..Self::default() } } + pub(super) fn for_policy(project_root: &Path, include_submodules: bool) -> Self { + let mut fingerprint = Self::capture(project_root); + if include_submodules { + let probes = (|| { + let parent = tracedecay_runtime_core::git_open::open(project_root).ok()?; + let children = + tracedecay_code_index::source_walk::submodules::live_submodules(&parent) + .ok()?; + let mut refs = Vec::new(); + let mut indexes = Vec::new(); + for (path, binding) in children { + let child = tracedecay_runtime_core::git_open::open(&binding.git_dir).ok()?; + let head = child.head_id().ok()?.detach(); + refs.push(( + path.clone(), + binding.git_dir, + binding.pin.to_string(), + head.to_string(), + )); + indexes.push((path, mtime(&child.index_path()))); + } + Some(( + tracedecay_domain::canonical_sha256(&refs).ok()?, + tracedecay_domain::canonical_sha256(&indexes).ok()?, + )) + })(); + match probes { + Some((refs, indexes)) => { + fingerprint.submodule_refs = Some(refs.as_str().to_owned()); + fingerprint.submodule_indexes = Some(indexes.as_str().to_owned()); + } + None => fingerprint.submodule_probe_failed = true, + } + } + fingerprint + } + /// Whether the sampled metadata differs from a previously captured value. pub fn differs_from(&self, other: &GitMetadataFingerprintV1) -> bool { - self != other + self.submodule_probe_failed || other.submodule_probe_failed || self != other } /// Whether HEAD and every branch ref are unchanged. The index is excluded: /// opening the repository may refresh `.git/index` on its own, while a /// commit, checkout, or ref update always moves HEAD or a ref. pub fn same_refs_as(&self, other: &GitMetadataFingerprintV1) -> bool { - self.head == other.head + !self.submodule_probe_failed + && !other.submodule_probe_failed + && self.submodule_refs == other.submodule_refs + && self.head == other.head && self.packed_refs == other.packed_refs && self.refs_heads == other.refs_heads && self.head_contents == other.head_contents @@ -206,7 +252,7 @@ impl GitMetadataFingerprintV1 { time.and_then(|time| time.duration_since(UNIX_EPOCH).ok()) .map_or_else(|| "-".to_owned(), |elapsed| elapsed.as_nanos().to_string()) } - format!( + let mut signature = format!( "head={}|index={}|packed_refs={}|refs_heads={}|head_contents={}", render_time(self.head), render_time(self.index), @@ -216,7 +262,13 @@ impl GitMetadataFingerprintV1 { || "-".to_owned(), |value| super::sha256_hex(value.as_bytes()) ), - ) + ); + if let (Some(refs), Some(indexes)) = (&self.submodule_refs, &self.submodule_indexes) { + signature.push_str(&format!( + "|submodule_refs={refs}|submodule_indexes={indexes}" + )); + } + signature } } diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/ignored_dependencies.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/ignored_dependencies.rs index 2819eb7815..23dacd76f5 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/ignored_dependencies.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/ignored_dependencies.rs @@ -260,7 +260,7 @@ impl CodeIndexWorktreeSchedulerV1 { ) -> Result { let source_control = self.reconcile_control(); let began_under = source_control.expected_epoch(); - let captured_under = super::identity::GitMetadataFingerprintV1::capture(&self.project_root); + let captured_under = self.freshness_fence().git_metadata(&self.project_root); let _workers = self.ensure_worker_plan()?; let _worker_memory = match self.reserve_worker_memory() { Ok(reservation) => reservation, diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/reconcile.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/reconcile.rs index 5724543eff..4113a4ff27 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/reconcile.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/reconcile.rs @@ -567,6 +567,7 @@ pub(crate) struct SourceFreshnessFenceV1 { source_epoch: Arc, /// Held only by source sweeps, never across a scheduler pass. sweep_cache: Arc>, + include_submodules: bool, pending_source: Option>, } @@ -590,6 +591,10 @@ pub(super) struct SourceFreshnessFenceStateV1 { } impl SourceFreshnessFenceV1 { + pub(super) fn git_metadata(&self, root: &Path) -> identity::GitMetadataFingerprintV1 { + identity::GitMetadataFingerprintV1::for_policy(root, self.include_submodules) + } + fn unverified(source_epoch: Arc, path_policy: IndexPathPolicyV1) -> Self { Self { state: Arc::new(Mutex::new(SourceFreshnessFenceStateV1 { @@ -604,6 +609,7 @@ impl SourceFreshnessFenceV1 { })), last_reconciled_at_micros: Arc::new(AtomicI64::new(0)), source_epoch, + include_submodules: path_policy.includes_submodules(), sweep_cache: Arc::new(Mutex::new(SourceSweepCacheV1::new(path_policy))), pending_source: None, } @@ -723,7 +729,8 @@ impl SourceFreshnessFenceV1 { pub(super) fn verified_git_metadata_moved(&self, project_root: &Path) -> bool { let state = self.snapshot(); state.verified_against_source - && identity::GitMetadataFingerprintV1::capture(project_root) + && self + .git_metadata(project_root) .differs_from(&state.git_metadata) } @@ -760,7 +767,8 @@ impl SourceFreshnessFenceV1 { } state.verified_against_source && self.source_epoch.load(Ordering::Acquire) == state.reconciled_source_epoch - && !identity::GitMetadataFingerprintV1::capture(project_root) + && !self + .git_metadata(project_root) .differs_from(&state.git_metadata) } @@ -893,12 +901,13 @@ impl SourceFreshnessFenceV1 { &self, project_root: &Path, manifest: &SourceContentManifestV1, + candidate_paths: &BTreeSet, shutting_down: &AtomicBool, ) -> Result, CodeIndexSchedulerErrorV1> { self.sweep_cache .lock() .unwrap_or_else(PoisonError::into_inner) - .disputed_sources(project_root, manifest, shutting_down) + .disputed_sources(project_root, manifest, candidate_paths, shutting_down) } /// Sweep the current witness and report what the sweep had to read. @@ -916,7 +925,7 @@ impl SourceFreshnessFenceV1 { self.sweep_sources( project_root, &freshness.source_roster, - &identity::GitMetadataFingerprintV1::capture(project_root), + &self.git_metadata(project_root), &witness.content_manifest, shutting_down, ) @@ -937,7 +946,7 @@ impl SourceFreshnessFenceV1 { if !freshness.verified_against_source { return FreshnessProbeVerdictV1::Unverified; } - let git_metadata = identity::GitMetadataFingerprintV1::capture(project_root); + let git_metadata = self.git_metadata(project_root); if git_metadata.differs_from(&freshness.git_metadata) { return FreshnessProbeVerdictV1::Moved; } @@ -1336,6 +1345,7 @@ impl CodeIndexWorktreeSchedulerV1 { .with_shutdown_signal(Arc::clone(&shutting_down)) .with_decoded_content(byte_pool.decoded_content.clone()); let production_config = CodeIndexProductionConfigV1 { + include_submodules: path_policy.includes_submodules(), project_id: project_id.clone(), repository: repository_id.clone(), sanitizer_revision, @@ -1996,7 +2006,7 @@ impl CodeIndexWorktreeSchedulerV1 { { return Ok(None); } - let metadata = identity::GitMetadataFingerprintV1::capture(&self.project_root); + let metadata = self.freshness_fence.git_metadata(&self.project_root); if witness.git_metadata_signature != metadata.stable_signature() { return Ok(None); } @@ -2052,7 +2062,7 @@ impl CodeIndexWorktreeSchedulerV1 { if witness.generation_id != pointer.generation_id { return false; } - let metadata = identity::GitMetadataFingerprintV1::capture(&self.project_root); + let metadata = self.freshness_fence.git_metadata(&self.project_root); if witness.git_metadata_signature != metadata.stable_signature() { return false; } @@ -2234,7 +2244,7 @@ impl CodeIndexWorktreeSchedulerV1 { )); } self.identity = resolved; - let sampled_metadata = identity::GitMetadataFingerprintV1::capture(&self.project_root); + let sampled_metadata = self.freshness_fence.git_metadata(&self.project_root); let Some(sampled_sweep) = self.worktree_stat_sweep().ok() else { return Ok(None); }; @@ -2343,7 +2353,7 @@ impl CodeIndexWorktreeSchedulerV1 { let captured = self .capture_authoritative_snapshot_without_active_generation_reuse(Some(&control)) .map_err(|error| prefer_resident_memory_refusal(&control, error))?; - let git_metadata = identity::GitMetadataFingerprintV1::capture(&self.project_root); + let git_metadata = self.freshness_fence.git_metadata(&self.project_root); let stat_signature = self.worktree_stat_signature().ok(); let drained_hints = { let mut hints = self @@ -2927,7 +2937,7 @@ impl CodeIndexWorktreeSchedulerV1 { // bound to; a HEAD move under the same worktree is allowed and simply // records a new source revision, so the served generation is never // mis-attributed across identities. - let captured_under = identity::GitMetadataFingerprintV1::capture(&self.project_root); + let captured_under = self.freshness_fence.git_metadata(&self.project_root); let resolved = identity::IndexingIdentityV1::resolve(&self.project_root) .map_err(|error| CodeIndexSchedulerErrorV1::Identity(error.to_string()))?; if !resolved.authorizes_reuse_of(&self.identity) { @@ -3199,7 +3209,7 @@ impl CodeIndexWorktreeSchedulerV1 { source_epoch: u64, captured_under: &identity::GitMetadataFingerprintV1, ) { - let sampled = identity::GitMetadataFingerprintV1::capture(&self.project_root); + let sampled = self.freshness_fence.git_metadata(&self.project_root); let metadata = if sampled.same_refs_as(captured_under) { sampled } else { @@ -3318,7 +3328,9 @@ impl CodeIndexWorktreeSchedulerV1 { } let freshness = self.freshness_fence.snapshot(); if !freshness.verified_against_source - || identity::GitMetadataFingerprintV1::capture(&self.project_root) + || self + .freshness_fence + .git_metadata(&self.project_root) .differs_from(&freshness.git_metadata) || freshness.last_reconciled_at.elapsed() >= self.policy.staleness_threshold { @@ -3338,7 +3350,9 @@ impl CodeIndexWorktreeSchedulerV1 { } let freshness = self.freshness_fence.snapshot(); if !freshness.verified_against_source - || identity::GitMetadataFingerprintV1::capture(&self.project_root) + || self + .freshness_fence + .git_metadata(&self.project_root) .differs_from(&freshness.git_metadata) { self.request_background_reconcile(); @@ -3364,7 +3378,7 @@ impl CodeIndexWorktreeSchedulerV1 { .sweep_sources( &self.project_root, &self.ignored_source_admissions, - &identity::GitMetadataFingerprintV1::capture(&self.project_root), + &self.freshness_fence.git_metadata(&self.project_root), manifest, &self.shutting_down, ) @@ -3376,7 +3390,7 @@ impl CodeIndexWorktreeSchedulerV1 { fn source_witness_matches_worktree(&self, freshness: &SourceFreshnessFenceStateV1) -> bool { self.freshness_fence.source_witness_matches( freshness, - &identity::GitMetadataFingerprintV1::capture(&self.project_root), + &self.freshness_fence.git_metadata(&self.project_root), &self.project_root, &self.shutting_down, ) @@ -3432,7 +3446,7 @@ impl CodeIndexWorktreeSchedulerV1 { // Sample after the walk. `git_open::open` inside the digest comparison can // move index metadata; storing the post-walk sample is what keeps the // next probe from calling that side effect a new generation. - let git_metadata = identity::GitMetadataFingerprintV1::capture(&self.project_root); + let git_metadata = self.freshness_fence.git_metadata(&self.project_root); self.freshness_fence.rebind_admission_clock(git_metadata); self.freshness_fence .source_currency_witness_for(generation_id, snapshot_content_identity) @@ -3450,7 +3464,11 @@ impl CodeIndexWorktreeSchedulerV1 { fn worktree_stat_sweep( &self, ) -> Result { - freshness_witness::worktree_stat_sweep(&self.project_root, &self.ignored_source_admissions) + freshness_witness::worktree_stat_sweep( + &self.project_root, + &self.ignored_source_admissions, + self.path_policy.includes_submodules(), + ) } /// Deliver debounced hook hints (exact touched paths) into the incremental @@ -3495,7 +3513,9 @@ impl CodeIndexWorktreeSchedulerV1 { // suppressing cadence on open-time clocks is not. return Ok(Some(self.reconcile_now()?)); } - let git_changed = identity::GitMetadataFingerprintV1::capture(&self.project_root) + let git_changed = self + .freshness_fence + .git_metadata(&self.project_root) .differs_from(&freshness.git_metadata); if git_changed { // Tier 1: a git-mediated mutation is authoritative evidence; reconcile. @@ -3957,7 +3977,28 @@ impl CodeIndexWorktreeSchedulerV1 { .ignored_source_admissions .iter() .any(|admission| admission.logical_path == logical_path); - self.capture_admitted_candidate(registry, logical_path, control, explicitly_admitted) + let child_root = if self.path_policy.includes_submodules() && !explicitly_admitted { + let repository = tracedecay_runtime_core::git_open::open(&self.project_root) + .map_err(|error| CodeIndexSchedulerErrorV1::Git(error.to_string()))?; + let classification = + classification::WorktreeChangeClassificationV1::for_policy(&repository, true) + .map_err(|error| CodeIndexSchedulerErrorV1::Git(error.to_string()))?; + if classification.beneath_gitlink(Path::new(logical_path)) + && classification.child_root(logical_path).is_none() + { + return Ok(CapturedFileOutcomeV1::Absent); + } + classification.child_root(logical_path).map(str::to_owned) + } else { + None + }; + self.capture_admitted_candidate( + registry, + logical_path, + control, + explicitly_admitted, + child_root.as_deref(), + ) } fn ignored_admission_paths(&self) -> BTreeSet<&str> { @@ -3973,20 +4014,41 @@ impl CodeIndexWorktreeSchedulerV1 { logical_path: &str, control: Option<&dyn CodeIndexExecutionControlV1>, explicitly_admitted: bool, + child_root: Option<&str>, ) -> Result { // Explicit ignored-source admission cannot cross a nested checkout // any more than an ordinary Git candidate or interactive source walk. - let relative = std::path::Path::new(logical_path); + let relative = Path::new(logical_path); + let absolute = self.project_root.join(relative); + let (source_root, source_relative) = match child_root { + Some(prefix) if !explicitly_admitted => { + if !git_tree_capture::is_source_path(registry, logical_path.as_bytes()) { + return Ok(CapturedFileOutcomeV1::Absent); + } + let root = self.project_root.join(prefix); + if root.canonicalize().ok().as_ref() != Some(&root) + || !absolute + .symlink_metadata() + .is_ok_and(|metadata| metadata.is_file()) + { + return Ok(CapturedFileOutcomeV1::Absent); + } + let relative = relative.strip_prefix(prefix).map_err(|_| { + CodeIndexSchedulerErrorV1::Git("invalid bound submodule path".into()) + })?; + (root, relative) + } + _ => (self.project_root.clone(), relative), + }; if tracedecay_code_index::source_walk::has_nested_repository_ancestor( - &self.project_root, - relative, + &source_root, + source_relative, )? || tracedecay_code_index::source_walk::is_source_symlink( - &self.project_root, - relative, + &source_root, + source_relative, )? { return Ok(CapturedFileOutcomeV1::Absent); } - let absolute = self.project_root.join(logical_path); match absolute.metadata() { Ok(metadata) if !metadata.is_file() => return Ok(CapturedFileOutcomeV1::Absent), Ok(_) => {} @@ -4048,8 +4110,11 @@ impl CodeIndexWorktreeSchedulerV1 { // Classify committed/staged/unstaged/untracked/deleted/renamed paths // truthfully from gix. Deletions drop out of the present candidate set; // their tombstones flow through `changed_paths`. - let classification = classification::WorktreeChangeClassificationV1::classify(&repository) - .map_err(|error| CodeIndexSchedulerErrorV1::Git(error.to_string()))?; + let classification = classification::WorktreeChangeClassificationV1::for_policy( + &repository, + self.path_policy.includes_submodules(), + ) + .map_err(|error| CodeIndexSchedulerErrorV1::Git(error.to_string()))?; if self.shutting_down.load(Ordering::Acquire) { return Err(cancelled_code_index_reconcile()); } @@ -4126,6 +4191,7 @@ impl CodeIndexWorktreeSchedulerV1 { // assume-unchanged. A sealed row is reused only while its bytes still // carry its digest; every row they disprove beyond the visible commit // and status deltas is a worktree change stat could not see. + let mut candidate_paths = classification.candidate_paths(); let stat_blind_paths = match reusable_active.as_ref() { Some(active) => { let visible = classification.changed_paths(); @@ -4133,6 +4199,7 @@ impl CodeIndexWorktreeSchedulerV1 { .disputed_sources( &self.project_root, &SourceContentManifestV1::for_snapshot(active.snapshot()), + &candidate_paths, &self.shutting_down, )? .into_iter() @@ -4193,7 +4260,6 @@ impl CodeIndexWorktreeSchedulerV1 { let source_revision = (self.ignored_source_admissions.is_empty() && !has_changes) .then(|| self.identity.head_commit().cloned()) .flatten(); - let mut candidate_paths = classification.candidate_paths(); let mut changed_paths = classification.changed_paths(); changed_paths.extend(tree_delta); changed_paths.extend(stat_blind_paths); @@ -4274,6 +4340,7 @@ impl CodeIndexWorktreeSchedulerV1 { logical_path, control, admitted_paths.contains(logical_path.as_str()), + classification.child_root(logical_path), ) }) }) @@ -4364,6 +4431,7 @@ impl CodeIndexWorktreeSchedulerV1 { snapshot_content_identity(&files, &sanitization_receipts, &omitted_sources); let captured = CapturedSnapshotV1 { repository_parse_identity: CodeIndexRepositoryParseIdentityV1 { + include_submodules: self.path_policy.includes_submodules(), tree: self.identity.head_tree().cloned(), dirty, }, diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/index_path_policy_tests.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/index_path_policy_tests.rs index 06cf0bd3f3..f90c046bd0 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/index_path_policy_tests.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/index_path_policy_tests.rs @@ -191,3 +191,216 @@ async fn hook_hints_for_excluded_paths_queue_no_work() { ); registry.shutdown().await; } + +#[test] +fn submodule_policy_binds_live_graph_and_historical_child_pins_across_restart() { + use super::{git_stdout, run_git_in}; + use crate::code_index_scheduler::{ + branch_generations::BranchGenerationReadControlV1, git_tree_capture::ExactGitTreeSourceV1, + }; + use tracedecay_code_index::grep_search::{GrepSearchQuery, search_tree_with_cancel}; + use tracedecay_domain::{CommitId, RefId, RelationEdgeKindV1, RepositoryDirtyStateV1, TreeId}; + + let parent = GitFixture::new(&[( + "dev/launch.ts", + "import { feature } from '../app/lib';\nexport function launch() { return feature(); }\n", + )]); + let child = GitFixture::new(&[ + ( + "lib.ts", + "export function feature() { return 'pinned_symbol'; }\n", + ), + ("gone.ts", "export function gone() {}\n"), + ("media.bin", "local binary fixture"), + (".gitignore", "*.private.ts\n"), + ]); + run_git_in( + parent.path(), + &[ + "-c", + "protocol.file.allow=always", + "submodule", + "add", + "-q", + child.path().to_str().unwrap(), + "app", + ], + ); + run_git_in(parent.path(), &["commit", "-qm", "bind application"]); + run_git_in(parent.path(), &["branch", "pinned"]); + let pinned = ExactGitTreeSourceV1 { + reference: RefId::new("refs/heads/pinned").unwrap(), + revision: CommitId::new(git_stdout(parent.path(), &["rev-parse", "HEAD"])).unwrap(), + tree: TreeId::new(git_stdout(parent.path(), &["rev-parse", "HEAD^{tree}"])).unwrap(), + }; + let store = TempDir::new().unwrap(); + let enabled = policy(&[], &[]).with_submodules(true); + let grep = |needle: &str, policy: &IndexPathPolicyV1| { + search_tree_with_cancel( + parent.path(), + &GrepSearchQuery { + pattern: needle.to_owned(), + fixed_strings: true, + case_sensitive: true, + path_glob: Some("app/**".into()), + scope_prefix: None, + context_lines: 0, + max_results: 8, + }, + policy, + || false, + ) + .unwrap() + .hits + .into_iter() + .map(|hit| hit.file.to_string()) + .collect::>() + }; + let mut off = start(&parent, store.path(), policy(&[], &[])); + off.reconcile_now().unwrap(); + assert_eq!(indexed_paths(&off), ["dev/launch.ts"]); + assert!(grep("pinned_symbol", &policy(&[], &[])).is_empty()); + drop(off); + + let mut on = start(&parent, store.path(), enabled.clone()); + on.reconcile_now().unwrap(); + assert_eq!( + indexed_paths(&on), + ["app/gone.ts", "app/lib.ts", "dev/launch.ts"] + ); + assert_eq!(grep("pinned_symbol", &enabled), ["app/lib.ts"]); + let assert_edge = + |generation: &tracedecay_code_index::production::CodeIndexPublishedGenerationV1| { + let symbol = |name: &str| { + generation + .symbols() + .symbols + .iter() + .find(|item| item.simple_name == name) + .unwrap() + .occurrence + .clone() + }; + assert!( + generation + .edges() + .iter() + .any(|edge| edge.kind == RelationEdgeKindV1::Calls + && edge.from_occurrence == symbol("launch") + && edge.to_occurrence == symbol("feature")) + ); + }; + assert_edge(on.latest_complete().unwrap().generation()); + let checkout = parent.path().join("app"); + std::fs::write( + checkout.join("ignored.private.ts"), + "const private_marker = 1;", + ) + .unwrap(); + assert!(grep("private_marker", &enabled).is_empty()); + std::fs::write( + checkout.join("lib.ts"), + "export function feature() { return 'successor_symbol'; }\n", + ) + .unwrap(); + std::fs::remove_file(checkout.join("gone.ts")).unwrap(); + run_git_in(&checkout, &["add", "-A"]); + run_git_in( + &checkout, + &["commit", "-qm", "move child HEAD independently"], + ); + std::fs::write( + checkout.join("pending.ts"), + "export function pending() {}\n", + ) + .unwrap(); + assert!(on.ensure_fresh_for_query().unwrap().is_some()); + assert_eq!( + indexed_paths(&on), + ["app/lib.ts", "app/pending.ts", "dev/launch.ts"] + ); + assert_eq!( + on.latest_complete() + .unwrap() + .generation() + .repository_parse_identity() + .dirty, + RepositoryDirtyStateV1::Dirty + ); + assert_eq!(grep("successor_symbol", &enabled), ["app/lib.ts"]); + run_git_in(parent.path(), &["submodule", "deinit", "-f", "app"]); + assert!(!checkout.join(".git").exists()); + let historical = on + .publish_exact_git_tree_generation( + &pinned, + &BranchGenerationReadControlV1 { + deadline: None, + cancellation: None, + }, + ) + .unwrap(); + let historical = historical.generation(); + assert_edge(historical); + assert!( + !historical + .snapshot() + .files + .iter() + .any(|file| file.logical_path == "app/media.bin") + ); + assert!( + historical + .snapshot() + .files + .iter() + .any(|file| file.logical_path == "app/gone.ts") + ); + assert!( + !historical + .snapshot() + .files + .iter() + .any(|file| file.logical_path == "app/pending.ts") + ); + let pinned_file = historical + .snapshot() + .files + .iter() + .find(|file| file.logical_path == "app/lib.ts") + .unwrap(); + assert_eq!( + pinned_file.content_digest, + tracedecay_code_index::chunks::content_digest( + b"export function feature() { return 'pinned_symbol'; }\n" + ) + ); + drop(on); + let mut off = start(&parent, store.path(), policy(&[], &[])); + off.reconcile_now().unwrap(); + assert_eq!(indexed_paths(&off), ["dev/launch.ts"]); +} + +#[test] +fn parent_filtered_source_does_not_require_submodule_registration_when_disabled() { + use super::run_git_in; + use tracedecay_code_index::chunks::content_digest; + use tracedecay_domain::LanguageId; + + let source = "pub fn parent() {}\n"; + let parent = GitFixture::new(&[ + ("lib.rs", source), + (".gitmodules", "[submodule \"child\"]\npath = child\n"), + ]); + run_git_in(parent.path(), &["config", "core.autocrlf", "true"]); + std::fs::write(parent.path().join("lib.rs"), source.replace('\n', "\r\n")).unwrap(); + std::fs::write(parent.path().join(".gitmodules"), "[unfinished").unwrap(); + let sealed = super::super::sealed_source_bytes( + parent.path(), + "lib.rs", + &LanguageId::new("rust").unwrap(), + false, + &content_digest(source.as_bytes()), + ) + .unwrap(); + assert_eq!(sealed.as_deref(), Some(source.as_bytes())); +} diff --git a/crates/tracedecay-code-index-runtime/tests/code_index_scheduler/reconcile.rs b/crates/tracedecay-code-index-runtime/tests/code_index_scheduler/reconcile.rs index 35ca13c8c1..5aa18b0df2 100644 --- a/crates/tracedecay-code-index-runtime/tests/code_index_scheduler/reconcile.rs +++ b/crates/tracedecay-code-index-runtime/tests/code_index_scheduler/reconcile.rs @@ -825,13 +825,11 @@ fn unfollowable_committed_source_links_are_absent_from_disk_capture() { ); } -/// Git stores a committed source symlink as its target path, while the -/// worktree reads through it. Sealing the link text from HEAD's tree left the -/// first post-publication sweep disproving an unchanged checkout, so every -/// mount rebuilt once and served the rebuild as dirty. +/// A readable live symlink stays outside the worktree roster. The remaining +/// files can still establish a current generation without naming the full HEAD tree. #[cfg(unix)] #[test] -fn committed_source_symlink_publishes_a_current_clean_generation() { +fn committed_source_symlink_publishes_current_regular_files_only() { let fixture = GitFixture::new(&[ ("src/lib.rs", "pub mod pricing;\n"), ("src/pricing.rs", "pub fn total() -> u64 { 1 }\n"), @@ -851,16 +849,16 @@ fn committed_source_symlink_publishes_a_current_clean_generation() { !scheduler.request_fresh_now_background(), "the sweep after publication must find the unchanged checkout current" ); + let latest = scheduler.latest_complete().expect("published generation"); + let snapshot = latest.generation.snapshot(); + assert!(snapshot.source_revision.is_none()); assert_eq!( - scheduler - .latest_complete() - .expect("published generation") - .generation - .snapshot() - .source_revision - .as_ref(), - scheduler.identity.head_commit(), - "a clean checkout's generation must carry HEAD's revision" + snapshot + .files + .iter() + .map(|file| file.logical_path.as_str()) + .collect::>(), + ["src/lib.rs", "src/pricing.rs"] ); } diff --git a/crates/tracedecay-code-index/Cargo.toml b/crates/tracedecay-code-index/Cargo.toml index 5404891c52..cba8706cb2 100644 --- a/crates/tracedecay-code-index/Cargo.toml +++ b/crates/tracedecay-code-index/Cargo.toml @@ -66,6 +66,7 @@ lang-lean = ["tracedecay-code-extraction/lang-lean"] [dependencies] ast-grep-core = "0.50" flate2.workspace = true +gix.workspace = true miniz_oxide.workspace = true base64.workspace = true hex.workspace = true diff --git a/crates/tracedecay-code-index/benches/partitioned_codec.rs b/crates/tracedecay-code-index/benches/partitioned_codec.rs index 12a962fddf..c9330ca623 100644 --- a/crates/tracedecay-code-index/benches/partitioned_codec.rs +++ b/crates/tracedecay-code-index/benches/partitioned_codec.rs @@ -286,6 +286,7 @@ fn build_generation( invalidations: BTreeSet::new(), ignored_source_admissions: Vec::new(), repository_parse_identity: CodeIndexRepositoryParseIdentityV1 { + include_submodules: false, tree: None, dirty: RepositoryDirtyStateV1::Dirty, }, @@ -315,6 +316,7 @@ fn build_generation( fn production_config() -> Result> { Ok(CodeIndexProductionConfigV1 { + include_submodules: false, project_id: id::("project.partitioned-codec")?, repository: id::("repository.partitioned-codec")?, sanitizer_revision: id::("sanitizer.partitioned-codec.v1")?, diff --git a/crates/tracedecay-code-index/benches/sealed_storage.rs b/crates/tracedecay-code-index/benches/sealed_storage.rs index f3cc2da231..7e927b88c2 100644 --- a/crates/tracedecay-code-index/benches/sealed_storage.rs +++ b/crates/tracedecay-code-index/benches/sealed_storage.rs @@ -468,6 +468,7 @@ fn request( invalidations: BTreeSet::new(), ignored_source_admissions: Vec::new(), repository_parse_identity: CodeIndexRepositoryParseIdentityV1 { + include_submodules: false, tree: Some(id::(tree)?), dirty: RepositoryDirtyStateV1::Clean, }, @@ -482,6 +483,7 @@ fn request( fn config() -> Result> { Ok(CodeIndexProductionConfigV1 { + include_submodules: false, project_id: id::("project.sealed-storage")?, repository: id::("repository.sealed-storage")?, sanitizer_revision: id::("sanitizer.sealed-storage.v1")?, diff --git a/crates/tracedecay-code-index/src/production/ignored_sources.rs b/crates/tracedecay-code-index/src/production/ignored_sources.rs index 2a103a5303..beafc93a38 100644 --- a/crates/tracedecay-code-index/src/production/ignored_sources.rs +++ b/crates/tracedecay-code-index/src/production/ignored_sources.rs @@ -53,6 +53,9 @@ pub struct CodeIndexBuildRequestV1 { #[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] #[serde(deny_unknown_fields)] pub struct CodeIndexRepositoryParseIdentityV1 { + /// Older sealed generations used the default parent-only roster. + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + pub include_submodules: bool, pub tree: Option, pub dirty: RepositoryDirtyStateV1, } diff --git a/crates/tracedecay-code-index/src/production/lexical_page_source.rs b/crates/tracedecay-code-index/src/production/lexical_page_source.rs index 76497c639a..df469bda13 100644 --- a/crates/tracedecay-code-index/src/production/lexical_page_source.rs +++ b/crates/tracedecay-code-index/src/production/lexical_page_source.rs @@ -1332,6 +1332,7 @@ impl VerifiedSealedTextGenerationMetadataV1 { &self.snapshot, config, ); + compatibility.observe_submodule_policy(self.repository_parse_identity(), config); self.chunk_policy.observe(config, &mut compatibility); compatibility } diff --git a/crates/tracedecay-code-index/src/production/lexical_page_source_tests.rs b/crates/tracedecay-code-index/src/production/lexical_page_source_tests.rs index 6ce3129396..6b2276f9b9 100644 --- a/crates/tracedecay-code-index/src/production/lexical_page_source_tests.rs +++ b/crates/tracedecay-code-index/src/production/lexical_page_source_tests.rs @@ -611,6 +611,7 @@ fn fixture_for_source_files( invalidations: BTreeSet::new(), ignored_source_admissions: Vec::new(), repository_parse_identity: CodeIndexRepositoryParseIdentityV1 { + include_submodules: false, tree: None, dirty: RepositoryDirtyStateV1::Dirty, }, @@ -624,6 +625,7 @@ fn fixture_for_source_files( }; let mut owner = CodeIndexProductionOwnerV1::new( CodeIndexProductionConfigV1 { + include_submodules: false, project_id: ProjectId::new("project.lexical-page-batch").expect("fixture project ID"), repository: RepositoryId::new("repository.lexical-page-batch") .expect("fixture repository ID"), diff --git a/crates/tracedecay-code-index/src/production/mod.rs b/crates/tracedecay-code-index/src/production/mod.rs index b4b48db5ac..af485652be 100644 --- a/crates/tracedecay-code-index/src/production/mod.rs +++ b/crates/tracedecay-code-index/src/production/mod.rs @@ -151,6 +151,7 @@ pub const DAEMON_CODE_INDEX_CHUNKER_REVISION: &str = "chunker.daemon.v5"; /// Immutable configuration retained by one production index owner. #[derive(Clone, Debug)] pub struct CodeIndexProductionConfigV1 { + pub include_submodules: bool, pub project_id: ProjectId, pub repository: RepositoryId, pub sanitizer_revision: SanitizerRevision, @@ -192,6 +193,7 @@ pub enum CodeIndexGenerationIncompatibilityV1 { SanitizerRevision, PolicyRevision, MixedPolicyRevisions, + SubmodulePolicy, ChunkerRevision, PrivacyDomain, PrivacyKeyEpoch, @@ -203,6 +205,7 @@ impl CodeIndexGenerationIncompatibilityV1 { Self::Project => "project", Self::LanguageRevisions => "language_revisions", Self::SanitizerRevision => "sanitizer_revision", + Self::SubmodulePolicy => "submodule_policy", Self::PolicyRevision => "policy_revision", Self::MixedPolicyRevisions => "mixed_policy_revisions", Self::ChunkerRevision => "chunker_revision", @@ -226,6 +229,17 @@ pub struct CodeIndexGenerationCompatibilityV1 { } impl CodeIndexGenerationCompatibilityV1 { + fn observe_submodule_policy( + &mut self, + identity: &CodeIndexRepositoryParseIdentityV1, + config: &CodeIndexProductionConfigV1, + ) { + if identity.include_submodules != config.include_submodules { + self.incompatibilities + .insert(CodeIndexGenerationIncompatibilityV1::SubmodulePolicy); + } + } + fn for_metadata( manifest: &CodeGenerationManifestV1, snapshot: &SanitizedCodeSnapshotV1, @@ -1456,6 +1470,7 @@ impl CodeIndexPublishedGenerationV1 { &self.snapshot, config, ); + compatibility.observe_submodule_policy(self.repository_parse_identity(), config); self.chunk_policy_summary() .observe(config, &mut compatibility); compatibility @@ -2364,6 +2379,11 @@ where crate::observe::record_generation_state("building"); crate::observe::record_rebuild_state("unknown"); lexical_page_source::checkpoint(control)?; + if request.repository_parse_identity.include_submodules != self.config.include_submodules { + return Err(CodeIndexProductionErrorV1::Contract( + "snapshot submodule policy differs from owner".to_owned(), + )); + } let ignored_source_roster = IgnoredSourceRosterV1::admit( &request.snapshot, &request.repository_parse_identity, diff --git a/crates/tracedecay-code-index/src/production/sparse_increment_tests.rs b/crates/tracedecay-code-index/src/production/sparse_increment_tests.rs index 01eb1dcb76..66d8af7915 100644 --- a/crates/tracedecay-code-index/src/production/sparse_increment_tests.rs +++ b/crates/tracedecay-code-index/src/production/sparse_increment_tests.rs @@ -100,6 +100,7 @@ pub(super) fn request(files: &[(&str, &str, &str)], sealed_at: i64) -> CodeIndex invalidations: BTreeSet::new(), ignored_source_admissions: Vec::new(), repository_parse_identity: CodeIndexRepositoryParseIdentityV1 { + include_submodules: false, tree: None, dirty: tracedecay_domain::RepositoryDirtyStateV1::Dirty, }, diff --git a/crates/tracedecay-code-index/src/production/worker_tests.rs b/crates/tracedecay-code-index/src/production/worker_tests.rs index 7f210388b6..71b9eedfcc 100644 --- a/crates/tracedecay-code-index/src/production/worker_tests.rs +++ b/crates/tracedecay-code-index/src/production/worker_tests.rs @@ -39,6 +39,7 @@ where pub(super) fn worker_config() -> CodeIndexProductionConfigV1 { CodeIndexProductionConfigV1 { + include_submodules: false, project_id: worker_id("project.worker"), repository: worker_id("repository.worker"), sanitizer_revision: worker_id::("sanitizer.v1"), @@ -87,6 +88,7 @@ pub(super) fn worker_request_with_source( invalidations: BTreeSet::new(), ignored_source_admissions: Vec::new(), repository_parse_identity: CodeIndexRepositoryParseIdentityV1 { + include_submodules: false, tree: None, dirty: tracedecay_domain::RepositoryDirtyStateV1::Dirty, }, diff --git a/crates/tracedecay-code-index/src/source_walk.rs b/crates/tracedecay-code-index/src/source_walk.rs index a72a87b77e..78c682e297 100644 --- a/crates/tracedecay-code-index/src/source_walk.rs +++ b/crates/tracedecay-code-index/src/source_walk.rs @@ -9,6 +9,9 @@ //! walk is therefore public rather than crate-private, the audit in the root //! crate reuses this policy instead of restating it. +pub mod classification; +pub mod submodules; + use std::collections::BTreeSet; use std::path::{Component, Path, PathBuf}; use std::sync::{Arc, Mutex, PoisonError}; @@ -17,6 +20,8 @@ use ignore::overrides::{Override, OverrideBuilder}; use ignore::{Walk, WalkBuilder}; use tracedecay_domain::{IndexPathPolicyV1, forward_slash_path}; +pub type SourceWalk = std::iter::Flatten>; + #[derive(Debug)] pub struct SourceWalkError { pub glob: String, @@ -25,6 +30,7 @@ pub struct SourceWalkError { /// Conservative directory reachability for a positive override. The override /// matcher remains authoritative for files, including directory inheritance. +#[derive(Clone)] struct PathGlobScope { literal_prefix: PathBuf, may_match_descendants: bool, @@ -340,7 +346,7 @@ pub fn source_walk( project_root: &Path, path_glob: Option<&str>, path_policy: &IndexPathPolicyV1, -) -> Result { +) -> Result { source_walk_inner(project_root, path_glob, path_policy, None) } @@ -351,7 +357,7 @@ pub(crate) fn source_walk_with_diagnostics( project_root: &Path, path_glob: Option<&str>, path_policy: &IndexPathPolicyV1, -) -> Result<(Walk, SourceWalkDiagnostics), SourceWalkError> { +) -> Result<(SourceWalk, SourceWalkDiagnostics), SourceWalkError> { let diagnostics = SourceWalkDiagnostics::default(); let walk = source_walk_inner( project_root, @@ -367,7 +373,7 @@ fn source_walk_inner( path_glob: Option<&str>, path_policy: &IndexPathPolicyV1, diagnostics: Option, -) -> Result { +) -> Result { let overrides = build_overrides(project_root, path_glob)?; let has_positive_override = overrides .as_ref() @@ -376,88 +382,163 @@ fn source_walk_inner( let path_glob_scope = path_glob.and_then(PathGlobScope::from_path_glob); let filter_root = project_root.to_path_buf(); let path_policy = path_policy.clone(); - let parent_gitlinks = parent_gitlink_paths(project_root)?; - - let mut builder = WalkBuilder::new(project_root); - builder - .follow_links(false) - .hidden(false) - .git_ignore(true) - .git_global(true) - .git_exclude(true) - .add_custom_ignore_filename(".gitignore") - .filter_entry(move |entry| { - if entry.depth() == 0 { - return true; - } - let segment = entry.file_name().to_string_lossy(); - if segment == ".git" || segment == ".tracedecay" { - return false; - } - let Ok(relative) = entry.path().strip_prefix(&filter_root) else { - return false; - }; - let relative = forward_slash_path(relative); - let is_dir = entry.file_type().is_some_and(|kind| kind.is_dir()); - if is_dir - && path_glob_scope + let scoped_policy = { + let root = filter_root.clone(); + let scope = path_glob_scope.clone(); + move |path: &Path, is_dir: bool| { + has_positive_override + && (generated_dir_overrides .as_ref() - .is_some_and(|scope| !scope.may_contain_match(&filter_root, entry.path())) - { - return false; + .is_some_and(|overrides| overrides.matched(path, is_dir).is_whitelist()) + || (is_dir + && scope + .as_ref() + .is_some_and(|scope| scope.allows(&root, path)))) + } + }; + let mut parent_gitlinks = parent_gitlink_paths(project_root)?; + let mut authenticated = BTreeSet::new(); + let mut child_files = BTreeSet::new(); + if path_policy.includes_submodules() && !parent_gitlinks.is_empty() { + let parent = tracedecay_runtime_core::git_open::open(project_root).map_err(|error| { + SourceWalkError { + glob: String::new(), + message: error.to_string(), } - // An entry the glob names, or a directory it must pass through, - // is judged without the generated-directory defaults: the scope - // is the operator asking for that noise. Every other exclusion - // still applies; only `index.include.v1` lifts those. - let explicitly_requested = has_positive_override - && (generated_dir_overrides.as_ref().is_some_and(|overrides| { - overrides.matched(entry.path(), is_dir).is_whitelist() - }) || (is_dir - && path_glob_scope - .as_ref() - .is_some_and(|scope| scope.allows(&filter_root, entry.path())))); - let path_policy = if explicitly_requested { + })?; + for (path, child) in submodules::live_submodules(&parent)? { + let absolute = project_root.join(&path); + let policy = if scoped_policy(&absolute, true) { path_policy.without_generated_dir_defaults() } else { &path_policy }; - if !is_dir { - return !path_policy.excludes(&relative); + if policy.excludes_directory(&path) + || path_glob_scope + .as_ref() + .is_some_and(|scope| !scope.may_contain_match(project_root, &absolute)) + { + continue; } - // Excluded directories are outside this audit, not missing source - // caused by the repository boundary. Do not probe or name them. - if path_policy.excludes_directory(&relative) { - return false; + let repo = + tracedecay_runtime_core::git_open::open(&child.git_dir).map_err(|error| { + SourceWalkError { + glob: String::new(), + message: error.to_string(), + } + })?; + let inventory = classification::WorktreeChangeClassificationV1::classify(&repo) + .map_err(|error| SourceWalkError { + glob: String::new(), + message: error.to_string(), + })?; + let files = inventory + .regular_child_files(&child.worktree) + .map_err(|error| SourceWalkError { + glob: String::new(), + message: error.to_string(), + })?; + child_files.extend(files.into_iter().map(|file| format!("{path}/{file}"))); + // Only direct registrations belong to this parent. + for nested in inventory.gitlink_paths() { + parent_gitlinks.insert(format!("{path}/{nested}")); + } + authenticated.insert(path); + } + } + + let roots = authenticated + .iter() + .map(|path| project_root.join(path)) + .collect::>(); + let child_dirs = child_files + .iter() + .flat_map(|path| Path::new(path).ancestors().skip(1)) + .map(Path::to_owned) + .collect::>(); + + let accept = Arc::new(move |entry: &ignore::DirEntry, in_child: bool| { + let segment = entry.file_name().to_string_lossy(); + if segment == ".git" || segment == ".tracedecay" { + return false; + } + let Ok(relative) = entry.path().strip_prefix(&filter_root) else { + return false; + }; + let relative = forward_slash_path(relative); + let is_dir = entry.file_type().is_some_and(|kind| kind.is_dir()); + if is_dir + && path_glob_scope + .as_ref() + .is_some_and(|scope| !scope.may_contain_match(&filter_root, entry.path())) + { + return false; + } + // An entry the glob names, or a directory it must pass through, + // is judged without the generated-directory defaults: the scope + // is the operator asking for that noise. Every other exclusion + // still applies; only `index.include.v1` lifts those. + let path_policy = if scoped_policy(entry.path(), is_dir) { + path_policy.without_generated_dir_defaults() + } else { + &path_policy + }; + if !is_dir { + return !path_policy.excludes(&relative) + && (!in_child || child_files.contains(&relative)); + } + // Excluded directories are outside this audit, not missing source + // caused by the repository boundary. Do not probe or name them. + if path_policy.excludes_directory(&relative) { + return false; + } + // Only an independently bound direct gitlink may cross a checkout boundary. + if authenticated.contains(&relative) { + return in_child; + } + if parent_gitlinks.contains(&relative) { + if let Some(diagnostics) = &diagnostics { + diagnostics.record_skipped_nested_repository(relative); } - // Includes select paths, not repository authority. A parent - // gitlink does not capture the child's files or prove their - // freshness, so even registered submodules remain separate - // sources. Linked worktrees and nested clones share this boundary. - // The project root is deliberately exempt at depth zero above. - if parent_gitlinks.contains(&relative) { + return false; + } + match nested_repository_marker(entry.path()) { + Ok(true) => { if let Some(diagnostics) = &diagnostics { diagnostics.record_skipped_nested_repository(relative); } - return false; + false } - match nested_repository_marker(entry.path()) { - Ok(true) => { - if let Some(diagnostics) = &diagnostics { - diagnostics.record_skipped_nested_repository(relative); - } - false - } - Ok(false) => true, - // The walk cannot prove that an unreadable Git marker is - // absent. Never admit files across that boundary. - Err(_) => false, + Ok(false) => !in_child || child_dirs.contains(Path::new(&relative)), + // The walk cannot prove that an unreadable Git marker is + // absent. Never admit files across that boundary. + Err(_) => false, + } + }); + let mut builders = vec![(WalkBuilder::new(project_root), false)]; + builders.extend(roots.into_iter().map(|root| (WalkBuilder::new(root), true))); + let walks = builders + .into_iter() + .map(|(mut builder, in_child)| { + // The child Git roster already applied that checkout's ignores, including + // tracked exceptions. Running parent ignore rules over it would hide those files. + if in_child { + builder.standard_filters(false); + } else { + builder.add_custom_ignore_filename(".gitignore"); } - }); - if let Some(overrides) = overrides { - builder.overrides(overrides); - } - Ok(builder.build()) + let accept = Arc::clone(&accept); + builder + .follow_links(false) + .hidden(false) + .filter_entry(move |entry| accept(entry, in_child)); + if let Some(overrides) = &overrides { + builder.overrides(overrides.clone()); + } + builder.build() + }) + .collect::>(); + Ok(walks.into_iter().flatten()) } /// Project-relative path rendered with forward slashes. @@ -503,7 +584,8 @@ mod tests { use tracedecay_domain::IndexPathPolicyV1; use super::{ - has_nested_repository_ancestor, is_source_symlink, source_walk, source_walk_with_diagnostics, + has_nested_repository_ancestor, is_source_symlink, source_walk, + source_walk_with_diagnostics, }; #[test] @@ -808,11 +890,45 @@ mod tests { ); } + let enabled = IndexPathPolicyV1::new(vec![], vec![]) + .unwrap() + .with_submodules(true); + fixture_git( + &child, + &[ + "worktree", + "add", + "-q", + "--detach", + root.join("other-worktree").to_str().unwrap(), + "HEAD", + ], + ); + fixture_git( + &root, + &["clone", "-q", child.to_str().unwrap(), "other-clone"], + ); + let files = walked_files(&root, &enabled); + assert!(files.contains(&PathBuf::from("app/src/feature.rs"))); + assert!(files.iter().all(|path|!path.starts_with("other-worktree") && !path.starts_with("other-clone"))); + fixture_git( + &child, + &[ + "worktree", + "remove", + root.join("other-worktree").to_str().unwrap(), + ], + ); + fs::remove_dir_all(root.join("other-clone")).unwrap(); + fs::remove_file(root.join(".gitmodules")).unwrap(); + assert!(!walked_files(&root, &enabled).contains(&PathBuf::from("app/src/feature.rs"))); + fixture_git(&root, &["checkout", "--", ".gitmodules"]); // A missing child .git marker must not turn the parent's stage-0 // gitlink into ordinary live source under an explicit include. fs::remove_file(root.join("app/.git")).expect("remove child gitdir marker"); assert!(root.join("app/src/feature.rs").is_file()); let policy = IndexPathPolicyV1::new(vec![], vec!["app/**".into()]).unwrap(); + assert!(!walked_files(&root, &enabled).contains(&PathBuf::from("app/src/feature.rs"))); let (walk, diagnostics) = source_walk_with_diagnostics(&root, None, &policy).expect("source walk"); let files = walk diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/classification.rs b/crates/tracedecay-code-index/src/source_walk/classification.rs similarity index 85% rename from crates/tracedecay-code-index-runtime/src/code_index_scheduler/classification.rs rename to crates/tracedecay-code-index/src/source_walk/classification.rs index 0550c4a843..931bdf5329 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/classification.rs +++ b/crates/tracedecay-code-index/src/source_walk/classification.rs @@ -17,10 +17,11 @@ use std::collections::{BTreeMap, BTreeSet}; use std::path::Path; -use gix::bstr::ByteSlice; -use tracedecay_code_index::source_walk::{ +use super::{ has_nested_repository_ancestor, is_source_symlink, parent_gitlink_paths, + submodules::live_submodules, }; +use gix::bstr::ByteSlice; use tracedecay_runtime_core::git_repository::GIT_STATUS_MODIFICATION_CHECK_THREADS; /// Failure to classify worktree changes through gix. @@ -100,6 +101,9 @@ pub struct WorktreeChangeClassificationV1 { /// or symlinked ancestor. These are tombstones, not indexable candidates. nested_repository_exclusions: BTreeSet, parent_gitlinks: BTreeSet, + /// Direct parent-relative child paths admitted from that checkout's own Git roster. + child_sources: BTreeMap, + child_revision_changed: bool, } impl WorktreeChangeClassificationV1 { @@ -266,9 +270,93 @@ impl WorktreeChangeClassificationV1 { Ok(classification) } - /// Explicitly enrolled files must respect the same Git boundary snapshot - /// as ordinary candidates, including a gitlink staged for removal. - pub(super) fn beneath_gitlink(&self, path: &Path) -> bool { + pub fn for_policy( + repository: &gix::Repository, + include_submodules: bool, + ) -> Result { + let mut parent = Self::classify(repository)?; + if !include_submodules || parent.parent_gitlinks.is_empty() { + return Ok(parent); + } + let children = live_submodules(repository) + .map_err(|error| ClassificationErrorV1::Git(error.message))?; + // A missing checkout must not let the clean parent-tree shortcut import children. + parent.child_revision_changed = parent + .parent_gitlinks + .iter() + .any(|path| !children.contains_key(path)); + for (prefix, binding) in children { + let child = tracedecay_runtime_core::git_open::open(&binding.git_dir) + .map_err(|error| ClassificationErrorV1::Git(error.to_string()))?; + let child_head = child + .head_id() + .map_err(|error| ClassificationErrorV1::Git(error.to_string()))?; + parent.child_revision_changed |= child_head.detach() != binding.pin; + let local = Self::classify(&child)?; + for path in local.regular_child_files(&binding.worktree)? { + let logical = format!("{prefix}/{path}"); + parent.committed_baseline.insert(logical.clone()); + parent.child_sources.insert(logical, prefix.clone()); + } + for change in local.changes { + let path = format!("{prefix}/{}", change.path); + if !change.class.presents_content() || parent.child_sources.contains_key(&path) { + parent.changes.push(ClassifiedChangeV1 { + path, + class: change.class, + }); + } + } + parent.nested_repository_exclusions.extend( + local + .nested_repository_exclusions + .into_iter() + .map(|path| format!("{prefix}/{path}")), + ); + let qualify = |path: &[u8]| [prefix.as_bytes(), b"/", path].concat(); + parent + .non_utf8_paths + .extend(local.non_utf8_paths.into_iter().map(|path| qualify(&path))); + parent + .non_utf8_changes + .extend(local.non_utf8_changes.into_iter().map(|change| { + NonUtf8ClassifiedChangeV1 { + git_path: qualify(&change.git_path), + class: change.class, + } + })); + } + Ok(parent) + } + + pub(super) fn regular_child_files(&self, root: &Path) -> std::io::Result> { + let mut files = BTreeSet::new(); + for path in self.candidate_paths() { + if !super::submodules::is_source_location(Path::new(&path)) { + continue; + } + match root.join(&path).symlink_metadata() { + Ok(metadata) if metadata.is_file() => { + files.insert(path); + } + Ok(_) => {} + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => return Err(error), + } + } + Ok(files) + } + + pub fn child_root(&self, logical_path: &str) -> Option<&str> { + self.child_sources.get(logical_path).map(String::as_str) + } + + pub(super) fn gitlink_paths(&self) -> &BTreeSet { + &self.parent_gitlinks + } + + /// Explicit enrollment cannot cross a parent gitlink staged for removal. + pub fn beneath_gitlink(&self, path: &Path) -> bool { path.ancestors() .skip(1) .filter_map(Path::to_str) @@ -315,7 +403,8 @@ impl WorktreeChangeClassificationV1 { pub fn has_changes(&self) -> bool { !(self.changes().is_empty() && self.non_utf8_changes().is_empty() - && self.nested_repository_exclusions.is_empty()) + && self.nested_repository_exclusions.is_empty() + && !self.child_revision_changed) } /// Whether any classified change is a merge conflict. @@ -342,7 +431,7 @@ impl WorktreeChangeClassificationV1 { } /// Paths removed from the present snapshot (staged or unstaged deletions). - #[cfg(test)] + #[cfg(any(test, feature = "test-helpers"))] pub fn deleted_paths(&self) -> BTreeSet { self.changes .iter() @@ -364,7 +453,7 @@ impl WorktreeChangeClassificationV1 { } /// The class recorded for `path`, if any change touched it. - #[cfg(test)] + #[cfg(any(test, feature = "test-helpers"))] pub fn class_of(&self, path: &str) -> Option { self.changes .iter() @@ -533,7 +622,10 @@ mod nested_repository_tests { ); symlink("app/src/lib.rs", parent.join("bridge.rs")).unwrap(); fixture_git(&parent, &["add", ".gitmodules", "app", "bridge.rs"]); - fixture_git(&parent, &["commit", "-qm", "registered child and source alias"]); + fixture_git( + &parent, + &["commit", "-qm", "registered child and source alias"], + ); assert_eq!( fs::read_to_string(parent.join("bridge.rs")).unwrap(), "pub fn child() {}\n", @@ -550,7 +642,10 @@ mod nested_repository_tests { let classified = WorktreeChangeClassificationV1::classify(&repository).unwrap(); assert!(!classified.candidate_paths().contains("bridge.rs")); assert!(classified.changed_paths().contains("bridge.rs")); - assert!(classified.has_changes(), "do not reuse HEAD tree source bytes"); + assert!( + classified.has_changes(), + "do not reuse HEAD tree source bytes" + ); assert!(!classified.candidate_paths().contains("app/src/lib.rs")); } diff --git a/crates/tracedecay-code-index/src/source_walk/submodules.rs b/crates/tracedecay-code-index/src/source_walk/submodules.rs new file mode 100644 index 0000000000..5c08ebbcd1 --- /dev/null +++ b/crates/tracedecay-code-index/src/source_walk/submodules.rs @@ -0,0 +1,188 @@ +//! A parent gitlink names a commit, not the worktree that may supply its bytes. +//! Bind the registration, object store, and checkout before crossing that boundary. + +use std::collections::BTreeMap; +use std::path::{Component, Path, PathBuf}; + +use gix::bstr::ByteSlice; + +use super::{SourceWalkError, has_nested_repository_ancestor}; + +fn failure(error: impl std::fmt::Display) -> SourceWalkError { + SourceWalkError { + glob: String::new(), + message: format!("cannot inspect registered submodule: {error}"), + } +} + +#[derive(Clone, Debug)] +pub struct AuthenticatedSubmodule { + pub worktree: PathBuf, + pub git_dir: PathBuf, + pub pin: gix::ObjectId, +} + +fn normal_relative(path: &Path) -> bool { + !path.as_os_str().is_empty() + && path + .components() + .all(|component| matches!(component, Component::Normal(_))) +} + +/// Administrative data is outside every parent source surface, even with an explicit glob. +pub fn is_source_location(relative: &Path) -> bool { + normal_relative(relative) + && !relative.components().any(|component| { + component.as_os_str() == ".git" || component.as_os_str() == ".tracedecay" + }) +} + +fn registrations(bytes: &[u8]) -> Result, SourceWalkError> { + let modules = gix::submodule::File::from_bytes(bytes, None, &gix::config::File::default()) + .map_err(failure)?; + let mut registrations = BTreeMap::new(); + for name in modules.names() { + let name = gix::validate::submodule::name(name).map_err(failure)?; + let path = modules.path(name).map_err(failure)?; + let path = path.to_str().map_err(failure)?; + if !is_source_location(Path::new(path)) { + continue; + } + let name = PathBuf::from(name.to_str().map_err(failure)?); + if !normal_relative(&name) + || !normal_relative(Path::new(path)) + || registrations.insert(path.to_owned(), name).is_some() + { + return Err(failure("ambiguous or invalid .gitmodules path")); + } + } + Ok(registrations) +} + +fn module_stores<'a>( + parent: &'a gix::Repository, + name: &'a Path, +) -> impl Iterator + 'a { + [parent.git_dir(), parent.common_dir()] + .into_iter() + .filter_map(move |directory| { + let expected = directory.canonicalize().ok()?.join("modules").join(name); + (expected.canonicalize().ok()? == expected).then_some(expected) + }) +} + +fn open_pinned_store(expected: &Path, pin: gix::ObjectId) -> Option { + let child = tracedecay_runtime_core::git_open::open(expected).ok()?; + if child.git_dir().canonicalize().ok()? != expected + || child.common_dir().canonicalize().ok()? != expected + { + return None; + } + child.find_commit(pin).ok()?; + Some(child) +} + +fn bound_checkout( + parent: &gix::Repository, + path: &str, + name: &Path, + pin: gix::ObjectId, +) -> Option { + let root = parent.workdir()?.canonicalize().ok()?; + let path = Path::new(path); + if has_nested_repository_ancestor(&root, path).ok()? { + return None; + } + let worktree = root.join(path); + let marker = worktree.join(".git"); + if worktree.canonicalize().ok()? != worktree || !marker.symlink_metadata().ok()?.is_file() { + return None; + } + let git_dir = gix::discover::path::from_gitdir_file(&marker) + .ok()? + .canonicalize() + .ok()?; + // Git initializes children of a linked parent under that worktree's + // administrative directory. Check the name-derived, canonical location + // before opening any repository named by the checkout's marker. + let expected = module_stores(parent, name).find(|candidate| *candidate == git_dir)?; + let child = open_pinned_store(&expected, pin)?; + if child.workdir()?.canonicalize().ok()? != worktree { + return None; + } + Some(AuthenticatedSubmodule { + worktree, + git_dir: expected, + pin, + }) +} + +pub fn live_submodules( + parent: &gix::Repository, +) -> Result, SourceWalkError> { + let mut children = BTreeMap::new(); + let index = parent.index_or_empty().map_err(failure)?; + let stage = gix::index::entry::Stage::Unconflicted; + let Some(modules) = index.entry_by_path_and_stage(".gitmodules".into(), stage) else { + return Ok(children); + }; + if !modules + .mode + .to_tree_entry_mode() + .is_some_and(|mode| mode.is_blob()) + { + return Ok(children); + } + let blob = parent.find_blob(modules.id).map_err(failure)?; + let registered = registrations(&blob.data)?; + // An unstaged deletion or replacement of the registration cannot authorize a checkout. + let Some(live_registration) = parent.open_modules_file().map_err(failure)? else { + return Ok(children); + }; + for (path, name) in registered { + let Some(entry) = index.entry_by_path_and_stage(path.as_bytes().as_bstr(), stage) else { + continue; + }; + if !entry.mode.is_submodule() { + continue; + } + if !live_registration + .path(name.as_os_str().as_encoded_bytes().as_bstr()) + .is_ok_and(|current| current.as_slice() == path.as_bytes()) + { + continue; + } + if let Some(child) = bound_checkout(parent, &path, &name, entry.id) { + children.insert(path, child); + } + } + Ok(children) +} + +/// Only the selected parent tree may supply historical registrations and pins. +pub fn pinned_submodules( + parent: &gix::Repository, + pins: &tracedecay_runtime_core::git_repository::GitTreeSubmodulePins, +) -> Result, SourceWalkError> { + let mut children = BTreeMap::new(); + if pins.gitlinks.is_empty() { + return Ok(children); + } + let registrations = registrations(&pins.registration)?; + for (path, pin) in &pins.gitlinks { + let Some(name) = registrations.get(path) else { + continue; + }; + // The checkout can move or be deinitialized after the parent revision. + // Its current bytes and .git marker are never historical authorities. + let repo = module_stores(parent, name) + .find_map(|directory| open_pinned_store(&directory, *pin)) + .ok_or_else(|| { + failure(format!( + "registered object store or pinned commit unavailable for {path}" + )) + })?; + children.insert(path.clone(), (repo, *pin)); + } + Ok(children) +} diff --git a/crates/tracedecay-code-index/tests/code_index_suite/production_orchestration.rs b/crates/tracedecay-code-index/tests/code_index_suite/production_orchestration.rs index 1e1558def4..5ef6dd9e3d 100644 --- a/crates/tracedecay-code-index/tests/code_index_suite/production_orchestration.rs +++ b/crates/tracedecay-code-index/tests/code_index_suite/production_orchestration.rs @@ -277,6 +277,7 @@ impl CodeIndexExecutionControlV1 for MutableCancellationControl { pub(super) fn config() -> CodeIndexProductionConfigV1 { CodeIndexProductionConfigV1 { + include_submodules: false, project_id: id::("project.production"), repository: id::("repository.production"), sanitizer_revision: id::("sanitizer.v1"), @@ -319,6 +320,7 @@ fn request_in_scope( request.snapshot.worktree = worktree.map(id::); request.snapshot.source_revision = Some(id::(source_revision)); request.repository_parse_identity = CodeIndexRepositoryParseIdentityV1 { + include_submodules: false, tree: Some(id::(&format!("tree.{source_revision}"))), dirty: RepositoryDirtyStateV1::Dirty, }; @@ -365,6 +367,7 @@ fn request_at_path( invalidations: BTreeSet::new(), ignored_source_admissions: Vec::new(), repository_parse_identity: CodeIndexRepositoryParseIdentityV1 { + include_submodules: false, tree: None, dirty: RepositoryDirtyStateV1::Dirty, }, @@ -1033,6 +1036,7 @@ fn resumed_parse_quanta_publish_the_same_complete_generation() { invalidations: BTreeSet::new(), ignored_source_admissions: Vec::new(), repository_parse_identity: CodeIndexRepositoryParseIdentityV1 { + include_submodules: false, tree: None, dirty: RepositoryDirtyStateV1::Dirty, }, @@ -3003,6 +3007,7 @@ fn partitioned_request( invalidations: BTreeSet::new(), ignored_source_admissions: Vec::new(), repository_parse_identity: CodeIndexRepositoryParseIdentityV1 { + include_submodules: false, tree: None, dirty: RepositoryDirtyStateV1::Dirty, }, diff --git a/crates/tracedecay-code-index/tests/code_index_suite/production_orchestration/parallel_equivalence.rs b/crates/tracedecay-code-index/tests/code_index_suite/production_orchestration/parallel_equivalence.rs index a23f502b50..7f70e4b9f8 100644 --- a/crates/tracedecay-code-index/tests/code_index_suite/production_orchestration/parallel_equivalence.rs +++ b/crates/tracedecay-code-index/tests/code_index_suite/production_orchestration/parallel_equivalence.rs @@ -92,6 +92,7 @@ fn multi_file_request(file_count: usize, sealed_at: i64) -> CodeIndexBuildReques invalidations: BTreeSet::new(), ignored_source_admissions: Vec::new(), repository_parse_identity: CodeIndexRepositoryParseIdentityV1 { + include_submodules: false, tree: None, dirty: RepositoryDirtyStateV1::Dirty, }, diff --git a/crates/tracedecay-code-index/tests/resident_accounting.rs b/crates/tracedecay-code-index/tests/resident_accounting.rs index 18a6afcf49..05fd76256c 100644 --- a/crates/tracedecay-code-index/tests/resident_accounting.rs +++ b/crates/tracedecay-code-index/tests/resident_accounting.rs @@ -154,6 +154,7 @@ fn request(files: usize) -> CodeIndexBuildRequestV1 { invalidations: BTreeSet::new(), ignored_source_admissions: Vec::new(), repository_parse_identity: CodeIndexRepositoryParseIdentityV1 { + include_submodules: false, tree: None, dirty: RepositoryDirtyStateV1::Dirty, }, @@ -168,6 +169,7 @@ fn request(files: usize) -> CodeIndexBuildRequestV1 { fn config() -> CodeIndexProductionConfigV1 { CodeIndexProductionConfigV1 { + include_submodules: false, project_id: id::("project.accounting"), repository: id::("repository.accounting"), sanitizer_revision: id::("sanitizer.v1"), diff --git a/crates/tracedecay-configuration/src/config/mod.rs b/crates/tracedecay-configuration/src/config/mod.rs index 12628536d7..52d0ca5725 100644 --- a/crates/tracedecay-configuration/src/config/mod.rs +++ b/crates/tracedecay-configuration/src/config/mod.rs @@ -22,9 +22,9 @@ use tracedecay_domain::configuration::{ AUTOMATION_SETTINGS_SETTING_KEY, AutomationSettingsV1, ConfigurationRevisionId, ConfigurationSnapshotV1, ConfigurationValueV1, INDEX_EXCLUDE_SETTING_KEY, INDEX_EXTRACT_DOCSTRINGS_SETTING_KEY, INDEX_INCLUDE_SETTING_KEY, - INDEX_MAX_FILE_SIZE_SETTING_KEY, INDEX_NATIVE_GRAPH_ACTIVATION_SETTING_KEY, - INDEX_TRACK_CALL_SITES_SETTING_KEY, LCM_SUMMARIZER_EXECUTABLES_SETTING_KEY, - LcmSummarizerExecutablesV1, SYNC_AUTO_INIT_SETTING_KEY, + INDEX_INCLUDE_SUBMODULES_SETTING_KEY, INDEX_MAX_FILE_SIZE_SETTING_KEY, + INDEX_NATIVE_GRAPH_ACTIVATION_SETTING_KEY, INDEX_TRACK_CALL_SITES_SETTING_KEY, + LCM_SUMMARIZER_EXECUTABLES_SETTING_KEY, LcmSummarizerExecutablesV1, SYNC_AUTO_INIT_SETTING_KEY, SYNC_AUTO_TRACK_PR_BRANCHES_SETTING_KEY, SYNC_AUTO_TRACK_PR_POLL_SECS_SETTING_KEY, SYNC_AUTO_WATCH_SETTING_KEY, SYNC_BACKSTOP_INTERVAL_MINS_SETTING_KEY, SYNC_BRANCH_GC_DAYS_SETTING_KEY, SYNC_READ_COOLDOWN_SECS_SETTING_KEY, @@ -277,7 +277,11 @@ fn runtime_config_from_snapshot( required_string_list(snapshot, INDEX_EXCLUDE_SETTING_KEY)?, required_string_list(snapshot, INDEX_INCLUDE_SETTING_KEY)?, ) - .map_err(|error| config_error(format!("resolved index path policy is invalid: {error}")))?, + .map_err(|error| config_error(format!("resolved index path policy is invalid: {error}")))? + .with_submodules(required_bool( + snapshot, + INDEX_INCLUDE_SUBMODULES_SETTING_KEY, + )?), max_file_size: required_unsigned(snapshot, INDEX_MAX_FILE_SIZE_SETTING_KEY)?, extract_docstrings: required_bool(snapshot, INDEX_EXTRACT_DOCSTRINGS_SETTING_KEY)?, track_call_sites: required_bool(snapshot, INDEX_TRACK_CALL_SITES_SETTING_KEY)?, @@ -460,6 +464,28 @@ mod tests { } } + #[test] + fn submodule_policy_is_default_off_and_materializes_the_project_override() { + let default = + PinnedRuntimeConfiguration::new(target(), revision(), resolved(BTreeMap::new())) + .unwrap(); + assert!(!default.config().index_paths.includes_submodules()); + let entries = BTreeMap::from([( + SettingKey::new(super::INDEX_INCLUDE_SUBMODULES_SETTING_KEY).unwrap(), + ConfigurationValueV1::Boolean(true), + )]); + let configured = + PinnedRuntimeConfiguration::new(target(), revision(), resolved(entries)).unwrap(); + assert!(configured.config().index_paths.includes_submodules()); + assert!( + configured + .config() + .index_paths + .without_generated_dir_defaults() + .includes_submodules() + ); + } + #[test] fn pin_rejects_a_snapshot_missing_a_required_setting() { let complete = resolved(BTreeMap::new()); diff --git a/crates/tracedecay-domain/src/configuration.rs b/crates/tracedecay-domain/src/configuration.rs index 601a3eb328..35d41cb26c 100644 --- a/crates/tracedecay-domain/src/configuration.rs +++ b/crates/tracedecay-domain/src/configuration.rs @@ -68,6 +68,7 @@ pub fn is_profile_setting_key(key: &SettingKey) -> bool { /// Canonical project-scoped runtime settings. pub const INDEX_EXCLUDE_SETTING_KEY: &str = "index.exclude.v1"; +pub const INDEX_INCLUDE_SUBMODULES_SETTING_KEY: &str = "index.include_submodules.v1"; pub const INDEX_INCLUDE_SETTING_KEY: &str = "index.include.v1"; pub const INDEX_MAX_FILE_SIZE_SETTING_KEY: &str = "index.max_file_size.v1"; pub const INDEX_EXTRACT_DOCSTRINGS_SETTING_KEY: &str = "index.extract_docstrings.v1"; @@ -110,6 +111,7 @@ pub const CONFIGURATION_SETTING_KEYS_V1: &[&str] = &[ USER_WORK_EXPERTISE_CONSENT_SETTING_KEY, INDEX_EXCLUDE_SETTING_KEY, INDEX_INCLUDE_SETTING_KEY, + INDEX_INCLUDE_SUBMODULES_SETTING_KEY, INDEX_MAX_FILE_SIZE_SETTING_KEY, INDEX_EXTRACT_DOCSTRINGS_SETTING_KEY, INDEX_TRACK_CALL_SITES_SETTING_KEY, diff --git a/crates/tracedecay-domain/src/lib.rs b/crates/tracedecay-domain/src/lib.rs index ffbef3e2d1..764d8cd231 100644 --- a/crates/tracedecay-domain/src/lib.rs +++ b/crates/tracedecay-domain/src/lib.rs @@ -104,9 +104,10 @@ pub use configuration::{ ContextScoutConfigurationStateV1, ContextScoutConfiguredModelPathV1, ContextScoutSettingsV1, CredentialReferenceId, CrossMergeModeV1, CrossMergePolicyV1, GitHubStackedPullRequestPolicyV1, HistoryRewritePolicyV1, INDEX_EXCLUDE_SETTING_KEY, INDEX_EXTRACT_DOCSTRINGS_SETTING_KEY, - INDEX_INCLUDE_SETTING_KEY, INDEX_MAX_FILE_SIZE_SETTING_KEY, - INDEX_NATIVE_GRAPH_ACTIVATION_SETTING_KEY, INDEX_TRACK_CALL_SITES_SETTING_KEY, - LCM_SUMMARIZER_EXECUTABLES_SETTING_KEY, LcmSummarizerExecutableV1, LcmSummarizerExecutablesV1, + INDEX_INCLUDE_SETTING_KEY, INDEX_INCLUDE_SUBMODULES_SETTING_KEY, + INDEX_MAX_FILE_SIZE_SETTING_KEY, INDEX_NATIVE_GRAPH_ACTIVATION_SETTING_KEY, + INDEX_TRACK_CALL_SITES_SETTING_KEY, LCM_SUMMARIZER_EXECUTABLES_SETTING_KEY, + LcmSummarizerExecutableV1, LcmSummarizerExecutablesV1, MAX_WORK_EXPERTISE_CONSENT_LIFETIME_MICROS_V1, PROJECT_WORK_EXPERTISE_CONSENT_SETTING_KEY, ProtectedApplyRequest, ProtectedChange, ProtectedChangePlan, ProtectedChangeSnapshotError, ProtectedRefDispositionV1, ProtectedRefRuleV1, ProtectedRefSelectorV1, QueryCollectionId, diff --git a/crates/tracedecay-domain/src/source_path_policy.rs b/crates/tracedecay-domain/src/source_path_policy.rs index 9871687d7e..9a921d6fb8 100644 --- a/crates/tracedecay-domain/src/source_path_policy.rs +++ b/crates/tracedecay-domain/src/source_path_policy.rs @@ -60,6 +60,7 @@ pub struct IndexPathPatternError { /// files. #[derive(Clone, Debug)] pub struct IndexPathPolicyV1 { + include_submodules: bool, exclude: Vec, include: Vec, exclude_set: GlobSet, @@ -78,7 +79,9 @@ pub struct IndexPathPolicyV1 { impl PartialEq for IndexPathPolicyV1 { fn eq(&self, other: &Self) -> bool { - self.exclude == other.exclude && self.include == other.include + self.include_submodules == other.include_submodules + && self.exclude == other.exclude + && self.include == other.include } } @@ -109,6 +112,7 @@ impl IndexPathPolicyV1 { .then(|| Self::new(configured, include.clone()).map(Box::new)) .transpose()?; Ok(Self { + include_submodules: false, exclude, include, exclude_set, @@ -132,6 +136,18 @@ impl IndexPathPolicyV1 { .unwrap_or(self) } + pub fn with_submodules(mut self, enabled: bool) -> Self { + self.include_submodules = enabled; + if let Some(scoped) = self.without_generated_dir_defaults.as_mut() { + scoped.include_submodules = enabled; + } + self + } + + pub fn includes_submodules(&self) -> bool { + self.include_submodules + } + pub fn exclude_patterns(&self) -> &[String] { &self.exclude } diff --git a/crates/tracedecay-global-db/src/configuration/registry.rs b/crates/tracedecay-global-db/src/configuration/registry.rs index 97d964bf4a..02c3b283cb 100644 --- a/crates/tracedecay-global-db/src/configuration/registry.rs +++ b/crates/tracedecay-global-db/src/configuration/registry.rs @@ -8,9 +8,10 @@ use tracedecay_domain::configuration::{ AnalyzerSettingsV1, CONFIGURATION_SETTING_KEYS_V1, CONTEXT_SCOUT_SETTINGS_SETTING_KEY, CodeIndexWorkerSelectionV1, ConfigurationValueKindV1, ConfigurationValueV1, ContextScoutSettingsV1, INDEX_EXCLUDE_SETTING_KEY, INDEX_EXTRACT_DOCSTRINGS_SETTING_KEY, - INDEX_INCLUDE_SETTING_KEY, INDEX_MAX_FILE_SIZE_SETTING_KEY, - INDEX_NATIVE_GRAPH_ACTIVATION_SETTING_KEY, INDEX_TRACK_CALL_SITES_SETTING_KEY, - LCM_SUMMARIZER_EXECUTABLES_SETTING_KEY, LcmSummarizerExecutablesV1, PROFILE_SETTING_KEYS_V1, + INDEX_INCLUDE_SETTING_KEY, INDEX_INCLUDE_SUBMODULES_SETTING_KEY, + INDEX_MAX_FILE_SIZE_SETTING_KEY, INDEX_NATIVE_GRAPH_ACTIVATION_SETTING_KEY, + INDEX_TRACK_CALL_SITES_SETTING_KEY, LCM_SUMMARIZER_EXECUTABLES_SETTING_KEY, + LcmSummarizerExecutablesV1, PROFILE_SETTING_KEYS_V1, PROJECT_WORK_EXPERTISE_CONSENT_SETTING_KEY, RestartRequirementV1, SOURCE_BINDINGS_SETTING_KEY, SYNC_AUTO_INIT_SETTING_KEY, SYNC_AUTO_TRACK_PR_BRANCHES_SETTING_KEY, SYNC_AUTO_TRACK_PR_POLL_SECS_SETTING_KEY, SYNC_AUTO_WATCH_SETTING_KEY, @@ -35,7 +36,7 @@ pub const MAX_PROXIMITY_RISK_THRESHOLD_BASIS_POINTS_V1: u64 = 10_000; /// Registry schema revision. Increment only when setting-definition semantics /// change, not when a setting value changes. -pub const CONFIGURATION_REGISTRY_SCHEMA_REVISION: u16 = 8; +pub const CONFIGURATION_REGISTRY_SCHEMA_REVISION: u16 = 9; #[derive(Debug, Error)] pub enum ConfigurationRegistryError { @@ -523,6 +524,12 @@ fn register_project_settings( SettingSensitivityV1::Sensitive, RestartRequirementV1::DaemonRestart, ), + ( + INDEX_INCLUDE_SUBMODULES_SETTING_KEY, + ConfigurationValueV1::Boolean(false), + SettingSensitivityV1::Public, + RestartRequirementV1::DaemonRestart, + ), ( INDEX_MAX_FILE_SIZE_SETTING_KEY, ConfigurationValueV1::Unsigned(defaults.max_file_size), @@ -779,7 +786,10 @@ mod user_profile_settings_tests { ); let definition = registry.definition(&key).expect("definition"); - assert_eq!(definition.schema_revision, 8); + assert_eq!( + definition.schema_revision, + CONFIGURATION_REGISTRY_SCHEMA_REVISION + ); assert_eq!(definition.scope, SettingScopeV1::UserProfile); assert_eq!( definition.value_kind, diff --git a/crates/tracedecay-global-db/src/configuration/store.rs b/crates/tracedecay-global-db/src/configuration/store.rs index f20a0c2bf6..ff751c349a 100644 --- a/crates/tracedecay-global-db/src/configuration/store.rs +++ b/crates/tracedecay-global-db/src/configuration/store.rs @@ -16,11 +16,11 @@ use tracedecay_domain::configuration::{ ConfigurationAuditEvent, ConfigurationAuditEventId, ConfigurationAuditEventKindV1, ConfigurationCandidateV1, ConfigurationIdempotencyKey, ConfigurationLayerIdV1, ConfigurationReceiptId, ConfigurationRevisionId, ConfigurationSnapshotV1, ConfigurationValueV1, - INDEX_NATIVE_GRAPH_ACTIVATION_SETTING_KEY, LCM_SUMMARIZER_EXECUTABLES_SETTING_KEY, - ProtectedChange, ProtectedChangePlan, ProtectedChangeSnapshotError, - RedactedConfigurationChangeV1, RollbackModeV1, SOURCE_BINDINGS_SETTING_KEY, - SYNC_WATCH_LINKED_WORKTREES_SETTING_KEY, ScopeControlOperationV1, SettingKey, - USER_CODE_INDEX_WORKERS_SETTING_KEY, USER_EXTRACTION_TIMEOUT_SECS_SETTING_KEY, + INDEX_INCLUDE_SUBMODULES_SETTING_KEY, INDEX_NATIVE_GRAPH_ACTIVATION_SETTING_KEY, + LCM_SUMMARIZER_EXECUTABLES_SETTING_KEY, ProtectedChange, ProtectedChangePlan, + ProtectedChangeSnapshotError, RedactedConfigurationChangeV1, RollbackModeV1, + SOURCE_BINDINGS_SETTING_KEY, SYNC_WATCH_LINKED_WORKTREES_SETTING_KEY, ScopeControlOperationV1, + SettingKey, USER_CODE_INDEX_WORKERS_SETTING_KEY, USER_EXTRACTION_TIMEOUT_SECS_SETTING_KEY, USER_REPEATED_QUERY_HINTS_SETTING_KEY, USER_WATCHER_DEBOUNCE_MS_SETTING_KEY, UserProfileId, WORK_TOPOLOGY_POLICY_SETTING_KEY, }; @@ -349,6 +349,7 @@ impl<'db> GlobalDbConfigurationControlStore<'db> { occurred_at, ®istry, &[ + INDEX_INCLUDE_SUBMODULES_SETTING_KEY, INDEX_NATIVE_GRAPH_ACTIVATION_SETTING_KEY, LCM_SUMMARIZER_EXECUTABLES_SETTING_KEY, SYNC_WATCH_LINKED_WORKTREES_SETTING_KEY, diff --git a/crates/tracedecay-global-db/src/configuration/store/tests/mod.rs b/crates/tracedecay-global-db/src/configuration/store/tests/mod.rs index 5e1b7f7d9c..8c6b4ab128 100644 --- a/crates/tracedecay-global-db/src/configuration/store/tests/mod.rs +++ b/crates/tracedecay-global-db/src/configuration/store/tests/mod.rs @@ -50,13 +50,19 @@ fn incomplete_snapshot_requires_reset_instead_of_default_repair() { } #[asupersync::test] -async fn linked_worktree_default_converges_into_existing_snapshot() { +async fn linked_worktree_and_submodule_defaults_converge_into_existing_snapshot() { let (_directory, runtime, root) = Box::pin(global_setup()).await; - let key = SettingKey::new(SYNC_WATCH_LINKED_WORKTREES_SETTING_KEY).unwrap(); + let keys = [ + SYNC_WATCH_LINKED_WORKTREES_SETTING_KEY, + tracedecay_domain::configuration::INDEX_INCLUDE_SUBMODULES_SETTING_KEY, + ] + .map(|name| SettingKey::new(name).unwrap()); let mut effective_values = root.snapshot.effective_values.clone(); let mut provenance = root.snapshot.provenance.clone(); - effective_values.remove(&key); - provenance.remove(&key); + for key in &keys { + effective_values.remove(key); + provenance.remove(key); + } let historical_snapshot = ConfigurationSnapshotV1::new(effective_values, provenance).unwrap(); let db = runtime .registered_database(HostAdmissionScope::Project) @@ -69,13 +75,18 @@ async fn linked_worktree_default_converges_into_existing_snapshot() { ) .await .unwrap(); - transaction - .execute( - "DELETE FROM configuration_entries WHERE revision_id = ?1 AND key = ?2", - tracedecay_runtime_core::db::engine::params![root.revision_id.as_str(), key.as_str()], - ) - .await - .unwrap(); + for key in &keys { + transaction + .execute( + "DELETE FROM configuration_entries WHERE revision_id = ?1 AND key = ?2", + tracedecay_runtime_core::db::engine::params![ + root.revision_id.as_str(), + key.as_str() + ], + ) + .await + .unwrap(); + } point_revision_at_snapshot(&transaction, &root.revision_id, &historical_snapshot).await; transaction.commit().await.unwrap(); let store = GlobalDbConfigurationControlStore::new_registered(db); @@ -86,11 +97,13 @@ async fn linked_worktree_default_converges_into_existing_snapshot() { .unwrap(); assert_ne!(converged.revision_id, root.revision_id); - assert_eq!( - converged.snapshot.effective_values.get(&key), - Some(&ConfigurationValueV1::Boolean(false)) - ); - assert!(converged.snapshot.provenance.contains_key(&key)); + for key in &keys { + assert_eq!( + converged.snapshot.effective_values.get(key), + Some(&ConfigurationValueV1::Boolean(false)) + ); + assert!(converged.snapshot.provenance.contains_key(key)); + } validate_snapshot_registry_completeness(&converged.snapshot).unwrap(); } diff --git a/crates/tracedecay-mcp/src/handlers/analysis/field_sites.rs b/crates/tracedecay-mcp/src/handlers/analysis/field_sites.rs index 75f161b692..d8e2a8c5c1 100644 --- a/crates/tracedecay-mcp/src/handlers/analysis/field_sites.rs +++ b/crates/tracedecay-mcp/src/handlers/analysis/field_sites.rs @@ -13,6 +13,7 @@ pub(super) async fn compute_field_sites( graph: &tracedecay_graph_query::VerifiedGraphQuery, args: Value, scope_prefix: Option<&str>, + include_submodules: bool, ) -> Result { let request: FieldSitesSurfaceRequestV1 = decode_primitive_request(&args, "tracedecay_field_sites")?; @@ -63,6 +64,7 @@ pub(super) async fn compute_field_sites( let needle = format!(".{field_name}"); 'outer: for file in &files { + if tracedecay_code_index::source_walk::is_source_symlink(&project_root,Path::new(file))? { continue; } let abs = project_root.join(file); let Ok(disk_source) = tracedecay_runtime_core::sync::read_source_file(&abs) else { continue; @@ -76,7 +78,7 @@ pub(super) async fn compute_field_sites( if !disk_source.contains(&needle) { continue; } - let source = sealed_field_source(&project_root, file, sealed_files.get(file))?; + let source = sealed_field_source(&project_root, file, sealed_files.get(file), include_submodules)?; let masked = path_is_rust(file).then(|| { tracedecay_code_extraction::source_mask::masked_rust_source_with( &source, @@ -198,6 +200,7 @@ fn sealed_field_source( project_root: &Path, file: &str, sealed: Option<&(LanguageId, ContentDigest)>, + include_submodules: bool, ) -> Result { let Some((language, digest)) = sealed else { return Err(TraceDecayError::project_route( @@ -208,7 +211,7 @@ fn sealed_field_source( ), )); }; - let bytes = sealed_source_bytes(project_root, file, language, digest) + let bytes = sealed_source_bytes(project_root, file, language, include_submodules, digest) .map_err(|error| { TraceDecayError::project_route( "verified-field-source-unavailable", diff --git a/crates/tracedecay-mcp/src/handlers/analysis/mod.rs b/crates/tracedecay-mcp/src/handlers/analysis/mod.rs index a6008e9cfd..33a5babdad 100644 --- a/crates/tracedecay-mcp/src/handlers/analysis/mod.rs +++ b/crates/tracedecay-mcp/src/handlers/analysis/mod.rs @@ -100,7 +100,13 @@ pub async fn compute_analysis_report( compute_constructors(&graph, args, scope_prefix).await } ApplicationSurfaceOperation::FieldSites => { - compute_field_sites(&graph, args, scope_prefix).await + compute_field_sites( + &graph, + args, + scope_prefix, + ctx.index_path_policy().includes_submodules(), + ) + .await } operation => Err(unknown_tool_error(operation.mcp_tool_name())), } diff --git a/crates/tracedecay-mcp/src/handlers/graph/rename_preview.rs b/crates/tracedecay-mcp/src/handlers/graph/rename_preview.rs index 35f13c1928..8822df7de2 100644 --- a/crates/tracedecay-mcp/src/handlers/graph/rename_preview.rs +++ b/crates/tracedecay-mcp/src/handlers/graph/rename_preview.rs @@ -175,6 +175,7 @@ struct RenameReferenceSiteInput { } struct PreviewFileWalk { + include_submodules: bool, project_root: PathBuf, declaration_line: u32, result: RenamePreviewPrimitiveResultV1, @@ -202,6 +203,19 @@ impl PreviewFileWalk { let has_references = reference_inputs .peek() .is_some_and(|input| input.file == file); + if tracedecay_code_index::source_walk::is_source_symlink( + &self.project_root, + std::path::Path::new(&file), + )? { + if has_references { + return Err(TraceDecayError::project_route( + "verified-reference-source-unavailable", + true, + format!("a source symlink replaced '{file}'"), + )); + } + continue; + } let text = match tracedecay_runtime_core::sync::read_source_file( &self.project_root.join(&file), ) { @@ -220,6 +234,7 @@ impl PreviewFileWalk { &self.project_root, &file, language, + self.include_submodules, digest, ) .map_err(|error| { @@ -427,6 +442,7 @@ pub async fn compute_rename_preview( freshness: None, }; let walk = PreviewFileWalk { + include_submodules: ctx.index_path_policy().includes_submodules(), project_root: ctx.project_root().to_path_buf(), declaration_line, result, diff --git a/crates/tracedecay-query/benches/code_lexical_catchup.rs b/crates/tracedecay-query/benches/code_lexical_catchup.rs index 434a75740a..6b45e53cd6 100644 --- a/crates/tracedecay-query/benches/code_lexical_catchup.rs +++ b/crates/tracedecay-query/benches/code_lexical_catchup.rs @@ -184,6 +184,7 @@ fn build_fixture() -> Fixture { invalidations: BTreeSet::new(), ignored_source_admissions: Vec::new(), repository_parse_identity: CodeIndexRepositoryParseIdentityV1 { + include_submodules: false, tree: None, dirty: RepositoryDirtyStateV1::Dirty, }, @@ -195,6 +196,7 @@ fn build_fixture() -> Fixture { }, }; let config = CodeIndexProductionConfigV1 { + include_submodules: false, project_id: id::("project.catchup-benchmark"), repository: repository.clone(), sanitizer_revision, diff --git a/crates/tracedecay-query/src/bin/tracedecay_index_bench.rs b/crates/tracedecay-query/src/bin/tracedecay_index_bench.rs index 64681d7855..7841942625 100644 --- a/crates/tracedecay-query/src/bin/tracedecay_index_bench.rs +++ b/crates/tracedecay-query/src/bin/tracedecay_index_bench.rs @@ -561,6 +561,7 @@ fn run(options: &Options) -> Result { let repository = identity::("repository.index-bench"); let sanitizer_revision = identity::("sanitizer.index-bench.v1"); let config = CodeIndexProductionConfigV1 { + include_submodules: false, project_id: identity::("project.index-bench"), repository: repository.clone(), sanitizer_revision: sanitizer_revision.clone(), @@ -736,6 +737,7 @@ fn build_request( invalidations: BTreeSet::new(), ignored_source_admissions: Vec::new(), repository_parse_identity: CodeIndexRepositoryParseIdentityV1 { + include_submodules: false, tree: Some(identity::(tree)), dirty: RepositoryDirtyStateV1::Clean, }, diff --git a/crates/tracedecay-query/src/bin/tracedecay_search_bench.rs b/crates/tracedecay-query/src/bin/tracedecay_search_bench.rs index d1d6a515c8..9b9cd38fde 100644 --- a/crates/tracedecay-query/src/bin/tracedecay_search_bench.rs +++ b/crates/tracedecay-query/src/bin/tracedecay_search_bench.rs @@ -329,6 +329,7 @@ fn run(options: &Options) -> Result { let repository = identity::("repository.search-bench"); let sanitizer_revision = identity::("sanitizer.search-bench.v1"); let config = CodeIndexProductionConfigV1 { + include_submodules: false, project_id: identity::("project.search-bench"), repository: repository.clone(), sanitizer_revision: sanitizer_revision.clone(), @@ -726,6 +727,7 @@ fn build_request( invalidations: BTreeSet::new(), ignored_source_admissions: Vec::new(), repository_parse_identity: CodeIndexRepositoryParseIdentityV1 { + include_submodules: false, tree: Some(identity::("tree.search-bench.clean")), dirty: RepositoryDirtyStateV1::Clean, }, diff --git a/crates/tracedecay-query/tests/search_quality_suite/candidate_producers.rs b/crates/tracedecay-query/tests/search_quality_suite/candidate_producers.rs index f186edc00e..aa1def7ed4 100644 --- a/crates/tracedecay-query/tests/search_quality_suite/candidate_producers.rs +++ b/crates/tracedecay-query/tests/search_quality_suite/candidate_producers.rs @@ -425,6 +425,7 @@ pub(crate) fn real_lexical_source_fixture_from_sources( invalidations: BTreeSet::new(), ignored_source_admissions: Vec::new(), repository_parse_identity: CodeIndexRepositoryParseIdentityV1 { + include_submodules: false, tree: None, dirty: RepositoryDirtyStateV1::Dirty, }, @@ -436,6 +437,7 @@ pub(crate) fn real_lexical_source_fixture_from_sources( }, }; let config = CodeIndexProductionConfigV1 { + include_submodules: false, project_id: id::("project.artifact"), repository: repository.clone(), sanitizer_revision, diff --git a/crates/tracedecay-runtime-core/src/git_repository.rs b/crates/tracedecay-runtime-core/src/git_repository.rs index 13c8dadc6d..bfb65d1212 100644 --- a/crates/tracedecay-runtime-core/src/git_repository.rs +++ b/crates/tracedecay-runtime-core/src/git_repository.rs @@ -33,7 +33,7 @@ pub use history::{ pub use native_integration::{ GitNativeApplyOutcome, GitNativeCandidateTreeV1, GitNativeCandidateTreeVisitError, GitNativeIntegrationMode, GitNativePreflight, GitNativePreflightCaptureError, - GitNativePreflightDisposition, GitNativeUnsupportedReason, + GitNativePreflightDisposition, GitNativeUnsupportedReason, GitTreeSubmodulePins, }; pub use tree_links::resolve_tree_link; diff --git a/crates/tracedecay-runtime-core/src/git_repository/native_integration.rs b/crates/tracedecay-runtime-core/src/git_repository/native_integration.rs index e09324c149..ada2a0215a 100644 --- a/crates/tracedecay-runtime-core/src/git_repository/native_integration.rs +++ b/crates/tracedecay-runtime-core/src/git_repository/native_integration.rs @@ -67,6 +67,40 @@ pub struct GitNativeCandidateTreeV1<'repository> { cancellation: &'repository CancellationToken, } +/// Registration and child commits taken from one immutable parent tree. +#[derive(Debug, Default)] +pub struct GitTreeSubmodulePins { + pub registration: Vec, + pub gitlinks: Vec<(String, gix::ObjectId)>, +} + +impl GitTreeSubmodulePins { + pub fn from_tree( + tree: &gix::Tree<'_>, + gitlinks: Vec<(String, gix::ObjectId)>, + ) -> Result { + if gitlinks.is_empty() { + return Ok(Self::default()); + } + let registration = match tree + .find_entry(".gitmodules") + .filter(|entry| entry.mode().is_blob()) + { + Some(entry) => tree + .repo + .find_blob(entry.object_id()) + .map_err(|error| operation("read pinned .gitmodules", error))? + .data + .clone(), + None => Vec::new(), + }; + Ok(Self { + registration, + gitlinks, + }) + } +} + impl GitNativeCandidateTreeV1<'_> { pub fn tree(&self) -> Result { oid(self.tree) @@ -78,9 +112,10 @@ impl GitNativeCandidateTreeV1<'_> { /// when it names none; every other symlink is visited with its link text. pub fn visit_blobs( &self, + include_submodules: bool, read_through: impl Fn(&[u8]) -> bool, mut visitor: impl FnMut(&[u8], &[u8]) -> Result<(), E>, - ) -> Result<(), GitNativeCandidateTreeVisitError> { + ) -> Result> { let tree = self .repository .find_tree(self.tree) @@ -90,6 +125,7 @@ impl GitNativeCandidateTreeV1<'_> { repository_visit_error("native candidate tree traversal", error) })?; entries.sort_by(|left, right| left.filepath.cmp(&right.filepath)); + let mut gitlinks = Vec::new(); for entry in entries { if self.cancellation.is_cancelled() { return Err(GitNativeCandidateTreeVisitError::Repository( @@ -99,6 +135,12 @@ impl GitNativeCandidateTreeV1<'_> { }, )); } + if include_submodules && entry.mode.is_commit() { + let path = entry.filepath.to_str().map_err(|error| { + repository_visit_error("native candidate gitlink path", error) + })?; + gitlinks.push((path.to_owned(), entry.oid)); + } if entry.mode.is_tree() || entry.mode.is_commit() { continue; } @@ -120,7 +162,8 @@ impl GitNativeCandidateTreeV1<'_> { visitor(entry.filepath.as_ref(), &blob.data) .map_err(GitNativeCandidateTreeVisitError::Visitor)?; } - Ok(()) + GitTreeSubmodulePins::from_tree(&tree, gitlinks) + .map_err(GitNativeCandidateTreeVisitError::Repository) } } diff --git a/crates/tracedecay-runtime-core/tests/runtime_core_suite/native_integration_authority.rs b/crates/tracedecay-runtime-core/tests/runtime_core_suite/native_integration_authority.rs index a1dc0be714..2f8d6c85f0 100644 --- a/crates/tracedecay-runtime-core/tests/runtime_core_suite/native_integration_authority.rs +++ b/crates/tracedecay-runtime-core/tests/runtime_core_suite/native_integration_authority.rs @@ -268,6 +268,7 @@ fn synthetic_candidate_tree_is_streamed_in_order_before_object_memory_drops() { let mut paths = Vec::new(); candidate .visit_blobs( + false, |_| false, |path, _bytes| { paths.push(String::from_utf8(path.to_vec()).expect("UTF-8 test path")); @@ -306,6 +307,7 @@ fn synthetic_candidate_tree_is_streamed_in_order_before_object_memory_drops() { |_, candidate| { candidate .visit_blobs( + false, |_| false, |_, _| { cancelled.cancel(); @@ -358,6 +360,7 @@ fn candidate_tree_reads_accepted_symlinks_through_to_their_blobs() { let mut visited = Vec::new(); candidate .visit_blobs( + false, |path| path.ends_with(b".rs"), |path, bytes| { visited.push(( diff --git a/crates/tracedecay-search-eval/src/candidate_output.rs b/crates/tracedecay-search-eval/src/candidate_output.rs index dcb3c2bb70..4bb11bb8d5 100644 --- a/crates/tracedecay-search-eval/src/candidate_output.rs +++ b/crates/tracedecay-search-eval/src/candidate_output.rs @@ -1324,6 +1324,7 @@ fn publish_scope_generation( invalidations: BTreeSet::new(), ignored_source_admissions: Vec::new(), repository_parse_identity: CodeIndexRepositoryParseIdentityV1 { + include_submodules: false, tree: None, dirty: RepositoryDirtyStateV1::Dirty, }, @@ -1331,6 +1332,7 @@ fn publish_scope_generation( target_projection_key, }; let config = CodeIndexProductionConfigV1 { + include_submodules: false, project_id: id::("project.candidate.fixture")?, repository: id::("repository.candidate.fixture")?, sanitizer_revision: id::("sanitizer.candidate.v1")?, diff --git a/crates/tracedecay-search-eval/src/candidate_output/cancellation.rs b/crates/tracedecay-search-eval/src/candidate_output/cancellation.rs index eff54a990f..b258849ab2 100644 --- a/crates/tracedecay-search-eval/src/candidate_output/cancellation.rs +++ b/crates/tracedecay-search-eval/src/candidate_output/cancellation.rs @@ -72,6 +72,7 @@ pub(super) fn prove_cancellation( invalidations: BTreeSet::new(), ignored_source_admissions: Vec::new(), repository_parse_identity: CodeIndexRepositoryParseIdentityV1 { + include_submodules: false, tree: None, dirty: RepositoryDirtyStateV1::Dirty, }, @@ -84,6 +85,7 @@ pub(super) fn prove_cancellation( }; let generation_scope = CodeIndexGenerationScopeV1::for_snapshot(&request.snapshot); let config = CodeIndexProductionConfigV1 { + include_submodules: false, project_id: id::("project.candidate.cancel")?, repository: id::("repository.candidate.cancel")?, sanitizer_revision: id::("sanitizer.candidate.v1")?, diff --git a/docs/USER-GUIDE.md b/docs/USER-GUIDE.md index d7eb98fc29..15a3ee9014 100644 --- a/docs/USER-GUIDE.md +++ b/docs/USER-GUIDE.md @@ -209,6 +209,22 @@ The value replaces the whole list, so start from the current effective value malformed pattern is refused before anything is written. There is no per-run folder exclusion; `init` and `sync` take no folder flags. +Direct Git submodules are excluded by default. To index their checked-out +files under parent-relative paths, set the project’s Boolean +`index.include_submodules.v1` setting and restart the daemon. + +```bash +tracedecay tool configuration_set --args '{"layer":{"kind":"project","project_id":""},"key":"index.include_submodules.v1","value":{"kind":"boolean","value":true},"expected_revision":"","idempotency_key":""}' +``` + +The parent index and `.gitmodules` must identify the child, and its checkout +must use the matching repository under the parent's Git modules directory. +Live indexing includes child worktree changes. Historical indexing reads the +child commit recorded by the selected parent revision from the local object +store. Initialize the submodule locally before requesting that history. +Linked worktrees, unrelated clones, and deeper nested repositories stay +outside the parent index. + ### Seeing what changed Use the daemon's status/coverage result to see the selected generation, exact From b96724957d79e242838c22b14b48c9d7c17e64b1 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 14:42:02 -0800 Subject: [PATCH 26/33] fix(graph): bind foreground reads to current checkout sources --- .../code_index_scheduler/freshness_witness.rs | 32 +---- .../tests/index_path_policy_tests.rs | 20 +++- .../tracedecay-code-index/src/source_walk.rs | 74 +++++------- .../src/source_walk/submodules.rs | 111 +++++++++++++++++- .../src/handlers/analysis/field_sites.rs | 18 ++- .../src/handlers/graph/rename_preview.rs | 27 +++-- 6 files changed, 188 insertions(+), 94 deletions(-) diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/freshness_witness.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/freshness_witness.rs index b98634731b..00228326aa 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/freshness_witness.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/freshness_witness.rs @@ -996,43 +996,21 @@ fn filter_mismatches_in_repository<'a>( /// disk first, then through the repository's own filter pipeline, the same /// two readings [`clean_filter_mismatches`] accepts as current. pub fn sealed_source_bytes( - project_root: &Path, + boundaries: &tracedecay_code_index::source_walk::submodules::LiveSourceBoundaries, logical_path: &str, language: &LanguageId, - include_submodules: bool, sealed: &ContentDigest, ) -> Result>, CodeIndexSchedulerErrorV1> { - if tracedecay_code_index::source_walk::is_source_symlink(project_root, Path::new(logical_path))? - { + let Some((root, relative)) = boundaries.resolve(Path::new(logical_path))? else { return Ok(None); - } - let raw = - ignored_dependencies::read_bounded_snapshot_source(&project_root.join(logical_path), None)?; + }; + let raw = ignored_dependencies::read_bounded_snapshot_source(&root.join(relative), None)?; let (sanitized, _, _) = privacy::sanitize_code_file(language, &raw)?; if content_digest(&sanitized) == *sealed { return Ok(Some(sanitized)); } - // ponytail: one repository open and filter pipeline per filtered file; - // hoist them into a caller-held reader if a caller ever needs hundreds. - let repository = tracedecay_runtime_core::git_open::open(project_root) + let repository = tracedecay_runtime_core::git_open::open(root) .map_err(|error| CodeIndexSchedulerErrorV1::Git(error.to_string()))?; - let mut repository = repository; - let mut relative = Path::new(logical_path); - if include_submodules { - let children = tracedecay_code_index::source_walk::submodules::live_submodules(&repository) - .map_err(|error| CodeIndexSchedulerErrorV1::Git(error.message))?; - if let Some((prefix, binding)) = children - .iter() - .find(|(prefix, _)| relative.starts_with(prefix)) - { - let child_path = relative - .strip_prefix(prefix) - .map_err(|_| CodeIndexSchedulerErrorV1::Git("invalid child filter path".into()))?; - repository = tracedecay_runtime_core::git_open::open(&binding.git_dir) - .map_err(|error| CodeIndexSchedulerErrorV1::Git(error.to_string()))?; - relative = child_path; - } - } let (mut pipeline, index) = repository .filter_pipeline(None) .map_err(|error| CodeIndexSchedulerErrorV1::Git(error.to_string()))?; diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/index_path_policy_tests.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/index_path_policy_tests.rs index f90c046bd0..c0e00751f8 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/index_path_policy_tests.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/index_path_policy_tests.rs @@ -256,6 +256,17 @@ fn submodule_policy_binds_live_graph_and_historical_child_pins_across_restart() .map(|hit| hit.file.to_string()) .collect::>() }; + let child_path = std::path::Path::new("app/lib.ts"); + let direct_reads = |enabled| { + tracedecay_code_index::source_walk::submodules::LiveSourceBoundaries::open( + parent.path(), + enabled, + ) + .unwrap() + }; + assert!(direct_reads(false).resolve(child_path).unwrap().is_none()); + let active_reads = direct_reads(true); + assert!(active_reads.resolve(child_path).unwrap().is_some()); let mut off = start(&parent, store.path(), policy(&[], &[])); off.reconcile_now().unwrap(); assert_eq!(indexed_paths(&off), ["dev/launch.ts"]); @@ -330,6 +341,7 @@ fn submodule_policy_binds_live_graph_and_historical_child_pins_across_restart() assert_eq!(grep("successor_symbol", &enabled), ["app/lib.ts"]); run_git_in(parent.path(), &["submodule", "deinit", "-f", "app"]); assert!(!checkout.join(".git").exists()); + assert!(active_reads.resolve(child_path).unwrap().is_none()); let historical = on .publish_exact_git_tree_generation( &pinned, @@ -394,11 +406,15 @@ fn parent_filtered_source_does_not_require_submodule_registration_when_disabled( run_git_in(parent.path(), &["config", "core.autocrlf", "true"]); std::fs::write(parent.path().join("lib.rs"), source.replace('\n', "\r\n")).unwrap(); std::fs::write(parent.path().join(".gitmodules"), "[unfinished").unwrap(); - let sealed = super::super::sealed_source_bytes( + let boundaries = tracedecay_code_index::source_walk::submodules::LiveSourceBoundaries::open( parent.path(), + false, + ) + .unwrap(); + let sealed = super::super::sealed_source_bytes( + &boundaries, "lib.rs", &LanguageId::new("rust").unwrap(), - false, &content_digest(source.as_bytes()), ) .unwrap(); diff --git a/crates/tracedecay-code-index/src/source_walk.rs b/crates/tracedecay-code-index/src/source_walk.rs index 78c682e297..3701db9d5f 100644 --- a/crates/tracedecay-code-index/src/source_walk.rs +++ b/crates/tracedecay-code-index/src/source_walk.rs @@ -396,55 +396,37 @@ fn source_walk_inner( .is_some_and(|scope| scope.allows(&root, path)))) } }; - let mut parent_gitlinks = parent_gitlink_paths(project_root)?; + let boundaries = + submodules::CheckoutBoundaries::open(project_root, path_policy.includes_submodules())?; + let mut parent_gitlinks = boundaries.gitlinks; let mut authenticated = BTreeSet::new(); let mut child_files = BTreeSet::new(); - if path_policy.includes_submodules() && !parent_gitlinks.is_empty() { - let parent = tracedecay_runtime_core::git_open::open(project_root).map_err(|error| { - SourceWalkError { - glob: String::new(), - message: error.to_string(), - } - })?; - for (path, child) in submodules::live_submodules(&parent)? { - let absolute = project_root.join(&path); - let policy = if scoped_policy(&absolute, true) { - path_policy.without_generated_dir_defaults() - } else { - &path_policy - }; - if policy.excludes_directory(&path) - || path_glob_scope - .as_ref() - .is_some_and(|scope| !scope.may_contain_match(project_root, &absolute)) - { - continue; - } - let repo = - tracedecay_runtime_core::git_open::open(&child.git_dir).map_err(|error| { - SourceWalkError { - glob: String::new(), - message: error.to_string(), - } - })?; - let inventory = classification::WorktreeChangeClassificationV1::classify(&repo) - .map_err(|error| SourceWalkError { - glob: String::new(), - message: error.to_string(), - })?; - let files = inventory - .regular_child_files(&child.worktree) - .map_err(|error| SourceWalkError { - glob: String::new(), - message: error.to_string(), - })?; - child_files.extend(files.into_iter().map(|file| format!("{path}/{file}"))); - // Only direct registrations belong to this parent. - for nested in inventory.gitlink_paths() { - parent_gitlinks.insert(format!("{path}/{nested}")); - } - authenticated.insert(path); + for (path, child) in boundaries.children { + let absolute = project_root.join(&path); + let policy = if scoped_policy(&absolute, true) { + path_policy.without_generated_dir_defaults() + } else { + &path_policy + }; + if policy.excludes_directory(&path) + || path_glob_scope + .as_ref() + .is_some_and(|scope| !scope.may_contain_match(project_root, &absolute)) + { + continue; + } + let inventory = child.sources()?; + child_files.extend( + inventory + .files + .into_iter() + .map(|file| format!("{path}/{file}")), + ); + // Only direct registrations belong to this parent. + for nested in inventory.gitlinks { + parent_gitlinks.insert(format!("{path}/{nested}")); } + authenticated.insert(path); } let roots = authenticated diff --git a/crates/tracedecay-code-index/src/source_walk/submodules.rs b/crates/tracedecay-code-index/src/source_walk/submodules.rs index 5c08ebbcd1..461f79aff4 100644 --- a/crates/tracedecay-code-index/src/source_walk/submodules.rs +++ b/crates/tracedecay-code-index/src/source_walk/submodules.rs @@ -1,7 +1,7 @@ //! A parent gitlink names a commit, not the worktree that may supply its bytes. //! Bind the registration, object store, and checkout before crossing that boundary. -use std::collections::BTreeMap; +use std::collections::{BTreeMap, BTreeSet}; use std::path::{Component, Path, PathBuf}; use gix::bstr::ByteSlice; @@ -22,6 +22,44 @@ pub struct AuthenticatedSubmodule { pub pin: gix::ObjectId, } +pub(super) struct ChildSources { + pub files: BTreeSet, + pub gitlinks: BTreeSet, +} + +impl AuthenticatedSubmodule { + pub(super) fn sources(&self) -> Result { + let repository = tracedecay_runtime_core::git_open::open(&self.git_dir).map_err(failure)?; + let inventory = + super::classification::WorktreeChangeClassificationV1::classify(&repository) + .map_err(failure)?; + Ok(ChildSources { + files: inventory + .regular_child_files(&self.worktree) + .map_err(failure)?, + gitlinks: inventory.gitlink_paths().clone(), + }) + } +} + +pub(super) struct CheckoutBoundaries { + pub gitlinks: BTreeSet, + pub children: BTreeMap, +} + +impl CheckoutBoundaries { + pub(super) fn open(root: &Path, include_submodules: bool) -> Result { + let gitlinks = super::parent_gitlink_paths(root)?; + let children = if include_submodules && !gitlinks.is_empty() { + let parent = tracedecay_runtime_core::git_open::open(root).map_err(failure)?; + live_submodules(&parent)? + } else { + BTreeMap::new() + }; + Ok(Self { gitlinks, children }) + } +} + fn normal_relative(path: &Path) -> bool { !path.as_os_str().is_empty() && path @@ -159,6 +197,77 @@ pub fn live_submodules( Ok(children) } +/// Batch the same Git inventory for foreground graph readers. A graph path can +/// outlive its checkout binding, even when a replacement file has equal bytes. +pub struct LiveSourceBoundaries { + project_root: PathBuf, + gitlinks: BTreeSet, + children: BTreeMap)>, +} + +impl LiveSourceBoundaries { + pub fn open(root: &Path, include_submodules: bool) -> Result { + let boundaries = CheckoutBoundaries::open(root, include_submodules)?; + let mut children = BTreeMap::new(); + for (prefix, child) in boundaries.children { + let files = child.sources()?.files; + children.insert(prefix, (child, files)); + } + Ok(Self { + project_root: root.to_owned(), + gitlinks: boundaries.gitlinks, + children, + }) + } + + /// The owning checkout and its relative file path, after checking the + /// current filesystem boundary before reading any bytes. + pub fn resolve<'a, 'p>( + &'a self, + path: &'p Path, + ) -> std::io::Result> { + if !is_source_location(path) { + return Ok(None); + } + let mut root = self.project_root.as_path(); + let mut relative = path; + for ancestor in path.ancestors().skip(1).filter_map(Path::to_str) { + if !self.gitlinks.contains(ancestor) { + continue; + } + let Some((child, files)) = self.children.get(ancestor) else { + return Ok(None); + }; + relative = path.strip_prefix(ancestor).map_err(std::io::Error::other)?; + if !relative.to_str().is_some_and(|file| files.contains(file)) + || child.worktree.canonicalize().ok().as_ref() != Some(&child.worktree) + { + return Ok(None); + } + let marker = child.worktree.join(".git"); + if !marker + .symlink_metadata() + .is_ok_and(|metadata| metadata.is_file()) + || gix::discover::path::from_gitdir_file(&marker) + .ok() + .and_then(|target| target.canonicalize().ok()) + .as_ref() + != Some(&child.git_dir) + { + return Ok(None); + } + root = &child.worktree; + break; + } + if has_nested_repository_ancestor(root, relative)? + || super::is_source_symlink(root, relative)? + { + return Ok(None); + } + Ok(Some((root, relative))) + } +} + /// Only the selected parent tree may supply historical registrations and pins. pub fn pinned_submodules( parent: &gix::Repository, diff --git a/crates/tracedecay-mcp/src/handlers/analysis/field_sites.rs b/crates/tracedecay-mcp/src/handlers/analysis/field_sites.rs index d8e2a8c5c1..d3df5f2f09 100644 --- a/crates/tracedecay-mcp/src/handlers/analysis/field_sites.rs +++ b/crates/tracedecay-mcp/src/handlers/analysis/field_sites.rs @@ -1,5 +1,6 @@ //! `tracedecay_field_sites`, read and write references to a named field. +use tracedecay_code_index::source_walk::submodules::LiveSourceBoundaries; use tracedecay_code_index_runtime::code_index_scheduler::sealed_source_bytes; use tracedecay_contracts::retrieval::{ FieldSiteV1, FieldSitesResultV1, FieldSitesSurfaceRequestV1, @@ -62,10 +63,16 @@ pub(super) async fn compute_field_sites( let mut reads: Vec = Vec::new(); let mut touched: Vec = Vec::new(); + let boundaries = LiveSourceBoundaries::open(&project_root, include_submodules) + .map_err(|error| TraceDecayError::project_route( + "verified-field-source-unavailable", true, error.message, + ))?; let needle = format!(".{field_name}"); 'outer: for file in &files { - if tracedecay_code_index::source_walk::is_source_symlink(&project_root,Path::new(file))? { continue; } - let abs = project_root.join(file); + let Some((root, relative)) = boundaries.resolve(Path::new(file))? else { + continue; + }; + let abs = root.join(relative); let Ok(disk_source) = tracedecay_runtime_core::sync::read_source_file(&abs) else { continue; }; @@ -78,7 +85,7 @@ pub(super) async fn compute_field_sites( if !disk_source.contains(&needle) { continue; } - let source = sealed_field_source(&project_root, file, sealed_files.get(file), include_submodules)?; + let source = sealed_field_source(&boundaries, file, sealed_files.get(file))?; let masked = path_is_rust(file).then(|| { tracedecay_code_extraction::source_mask::masked_rust_source_with( &source, @@ -197,10 +204,9 @@ pub(super) async fn compute_field_sites( /// the generation sealed, which differ from the checkout when git's clean /// filters rewrite it (a CRLF checkout under `core.autocrlf=true`). fn sealed_field_source( - project_root: &Path, + boundaries: &LiveSourceBoundaries, file: &str, sealed: Option<&(LanguageId, ContentDigest)>, - include_submodules: bool, ) -> Result { let Some((language, digest)) = sealed else { return Err(TraceDecayError::project_route( @@ -211,7 +217,7 @@ fn sealed_field_source( ), )); }; - let bytes = sealed_source_bytes(project_root, file, language, include_submodules, digest) + let bytes = sealed_source_bytes(boundaries, file, language, digest) .map_err(|error| { TraceDecayError::project_route( "verified-field-source-unavailable", diff --git a/crates/tracedecay-mcp/src/handlers/graph/rename_preview.rs b/crates/tracedecay-mcp/src/handlers/graph/rename_preview.rs index 8822df7de2..f7ab389acf 100644 --- a/crates/tracedecay-mcp/src/handlers/graph/rename_preview.rs +++ b/crates/tracedecay-mcp/src/handlers/graph/rename_preview.rs @@ -1,12 +1,13 @@ //! `tracedecay_rename_preview`: what a rename of one symbol would touch. use std::collections::HashMap; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; use std::sync::Arc; use std::sync::atomic::{AtomicBool, Ordering}; use std::time::Duration; use serde_json::Value; +use tracedecay_code_index::source_walk::submodules::LiveSourceBoundaries; use tracedecay_code_index_runtime::code_index_scheduler::sealed_source_bytes; use tracedecay_contracts::graph_tool::{GraphToolCompletionV1, GraphToolResultV1}; use tracedecay_contracts::retrieval::{ @@ -198,27 +199,30 @@ impl PreviewFileWalk { let mut text_only_matches = HashMap::new(); // Only the current file's decoded text and line index are resident. // Ordinals retain the graph's reference order across the grouped walk. + let boundaries = LiveSourceBoundaries::open(&self.project_root, self.include_submodules) + .map_err(|error| { + TraceDecayError::project_route( + "verified-reference-source-unavailable", + true, + error.message, + ) + })?; for file in files { self.control.check()?; let has_references = reference_inputs .peek() .is_some_and(|input| input.file == file); - if tracedecay_code_index::source_walk::is_source_symlink( - &self.project_root, - std::path::Path::new(&file), - )? { + let Some((root, relative)) = boundaries.resolve(Path::new(&file))? else { if has_references { return Err(TraceDecayError::project_route( "verified-reference-source-unavailable", true, - format!("a source symlink replaced '{file}'"), + format!("'{file}' is outside the current checkout source boundary"), )); } continue; - } - let text = match tracedecay_runtime_core::sync::read_source_file( - &self.project_root.join(&file), - ) { + }; + let text = match tracedecay_runtime_core::sync::read_source_file(&root.join(relative)) { Ok(text) => text, Err(error) if has_references => return Err(error.into()), // Declaration snippets are optional; reference evidence is not. @@ -231,10 +235,9 @@ impl PreviewFileWalk { // after the seal only its current text remains to map against. let sealed = match self.sealed_files.get(&file) { Some((language, digest)) => sealed_source_bytes( - &self.project_root, + &boundaries, &file, language, - self.include_submodules, digest, ) .map_err(|error| { From b25c3f54da4482046eb9e173757234b14d590044 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 16:24:38 -0800 Subject: [PATCH 27/33] fix(code-index): reject failed submodule freshness witnesses Refs #4593. Encode a failed authenticated probe in the persisted metadata signature, and never treat repeated failed probes as retained-generation freshness evidence. Add positive and negative behavioral regressions. --- .../src/code_index_scheduler/identity.rs | 64 +++++++++++++++++-- .../src/code_index_scheduler/reconcile.rs | 9 ++- 2 files changed, 64 insertions(+), 9 deletions(-) diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/identity.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/identity.rs index 728e42d7b4..39f0813488 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/identity.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/identity.rs @@ -240,12 +240,18 @@ impl GitMetadataFingerprintV1 { && self.head_contents == other.head_contents } + /// Whether an authenticated submodule probe can establish a freshness witness. + pub(super) fn can_prove_freshness(&self) -> bool { + !self.submodule_probe_failed + } + /// A stable, order-independent textual signature of this fingerprint. /// /// Used to persist a restore-time freshness witness: two fingerprints share - /// a signature iff they are structurally equal (`PartialEq`), so comparing - /// the recomputed signature against a witness recorded at seal time is a - /// faithful stand-in for `differs_from`. `SystemTime` mtimes are rendered as + /// a signature iff they are structurally equal (`PartialEq`). A failed + /// submodule probe is explicitly encoded, but even matching failed probes + /// cannot prove freshness: callers must reject them, as `differs_from` does. + /// `SystemTime` mtimes are rendered as /// nanos-since-epoch so the encoding is deterministic across processes. pub fn stable_signature(&self) -> String { fn render_time(time: Option) -> String { @@ -263,10 +269,14 @@ impl GitMetadataFingerprintV1 { |value| super::sha256_hex(value.as_bytes()) ), ); - if let (Some(refs), Some(indexes)) = (&self.submodule_refs, &self.submodule_indexes) { - signature.push_str(&format!( - "|submodule_refs={refs}|submodule_indexes={indexes}" - )); + if let Some(refs) = &self.submodule_refs { + signature.push_str(&format!("|submodule_refs={refs}")); + } + if let Some(indexes) = &self.submodule_indexes { + signature.push_str(&format!("|submodule_indexes={indexes}")); + } + if self.submodule_probe_failed { + signature.push_str("|submodule_probe_failed=true"); } signature } @@ -496,6 +506,46 @@ mod tests { ); } + #[test] + fn failed_submodule_probe_cannot_reuse_a_persisted_parent_only_witness() { + let root = TempDir::new().expect("non-repository root"); + let parent_only = GitMetadataFingerprintV1::capture(root.path()); + let failed = GitMetadataFingerprintV1::for_policy(root.path(), true); + + assert!(failed.submodule_probe_failed, "opt-in probe must fail closed"); + assert!(!failed.can_prove_freshness()); + assert!(failed.differs_from(&parent_only)); + assert_ne!( + failed.stable_signature(), + parent_only.stable_signature(), + "restore cannot mistake failed authentication for parent-only freshness" + ); + assert_eq!( + GitMetadataFingerprintV1::for_policy(root.path(), false).stable_signature(), + parent_only.stable_signature(), + "the default-off persisted witness remains stable" + ); + let failed_again = GitMetadataFingerprintV1::for_policy(root.path(), true); + assert_eq!(failed.stable_signature(), failed_again.stable_signature()); + assert!( + !failed_again.can_prove_freshness(), + "repeated matching failures cannot authorize retained reuse" + ); + } + + #[test] + fn successful_submodule_probe_has_a_distinct_stable_witness() { + let repo = init_repo(&[("src/lib.rs", "pub fn example() {}\n")]); + let disabled = GitMetadataFingerprintV1::for_policy(repo.path(), false); + let enabled = GitMetadataFingerprintV1::for_policy(repo.path(), true); + + assert!(enabled.can_prove_freshness()); + assert!(!disabled.submodule_probe_failed); + assert!(enabled.submodule_refs.is_some()); + assert!(enabled.submodule_indexes.is_some()); + assert_ne!(enabled.stable_signature(), disabled.stable_signature()); + } + #[test] fn git_metadata_fingerprint_detects_a_commit() { let repo = init_repo(&[("src/lib.rs", "pub fn a() {}\n")]); diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/reconcile.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/reconcile.rs index 4113a4ff27..c9c21b6666 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/reconcile.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/reconcile.rs @@ -2007,7 +2007,9 @@ impl CodeIndexWorktreeSchedulerV1 { return Ok(None); } let metadata = self.freshness_fence.git_metadata(&self.project_root); - if witness.git_metadata_signature != metadata.stable_signature() { + if !metadata.can_prove_freshness() + || witness.git_metadata_signature != metadata.stable_signature() + { return Ok(None); } // Re-swept under the adopted ignored-source roster. Equal metadata is @@ -2063,7 +2065,9 @@ impl CodeIndexWorktreeSchedulerV1 { return false; } let metadata = self.freshness_fence.git_metadata(&self.project_root); - if witness.git_metadata_signature != metadata.stable_signature() { + if !metadata.can_prove_freshness() + || witness.git_metadata_signature != metadata.stable_signature() + { return false; } self.worktree_stat_sweep() @@ -2266,6 +2270,7 @@ impl CodeIndexWorktreeSchedulerV1 { hook(); } let quiet_witness = sealed_bytes_match + && sampled_metadata.can_prove_freshness() && witness.as_ref().is_some_and(|witness| { witness.git_metadata_signature == sampled_metadata.stable_signature() && witness.stat_signature == sampled_sweep.signature From ebd52fe3e494b0cfb03a1bd488c1db280a407dcf Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 16:36:57 -0800 Subject: [PATCH 28/33] fix(code-index): require valid probes for retained graph reads --- .../src/code_index_scheduler/reconcile.rs | 15 +++-- .../tests/index_path_policy_tests.rs | 65 +++++++++++++++++++ 2 files changed, 75 insertions(+), 5 deletions(-) diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/reconcile.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/reconcile.rs index c9c21b6666..a820832fb4 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/reconcile.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/reconcile.rs @@ -663,7 +663,7 @@ impl SourceFreshnessFenceV1 { state.source_witness = source_witness; state.source_roster = source_roster.to_vec(); state.last_reconciled_at = Instant::now(); - state.verified_against_source = true; + state.verified_against_source = state.git_metadata.can_prove_freshness(); state.reconciled_without_generation = reconciled_without_generation; state.reconciled_source_epoch = source_epoch; state.successor_build_in_flight = false; @@ -845,7 +845,7 @@ impl SourceFreshnessFenceV1 { let mut state = self.state.lock().unwrap_or_else(PoisonError::into_inner); state.git_metadata = git_metadata; state.last_reconciled_at = Instant::now(); - state.verified_against_source = true; + state.verified_against_source = state.git_metadata.can_prove_freshness(); self.last_reconciled_at_micros .store(micros, Ordering::Release); } @@ -883,6 +883,10 @@ impl SourceFreshnessFenceV1 { manifest: &SourceContentManifestV1, shutting_down: &AtomicBool, ) -> (bool, freshness_witness::SourceSweepStatsV1) { + // Equal source bytes cannot authenticate a previously selected child checkout. + if !git_metadata.can_prove_freshness() { + return (false, freshness_witness::SourceSweepStatsV1::default()); + } self.sweep_cache .lock() .unwrap_or_else(PoisonError::into_inner) @@ -2263,14 +2267,15 @@ impl CodeIndexWorktreeSchedulerV1 { // generation's sealed file digests; its matching stat signature is // the negative cache that lets a moved tree skip the byte comparison. let source_manifest = SourceContentManifestV1::for_snapshot(metadata.snapshot()); - let sealed_bytes_match = - retained_is_reusable && !has_hints && self.sources_match_manifest(&source_manifest); + let sealed_bytes_match = retained_is_reusable + && !has_hints + && sampled_metadata.can_prove_freshness() + && self.sources_match_manifest(&source_manifest); #[cfg(test)] if let Some(hook) = self.after_retained_source_check.clone() { hook(); } let quiet_witness = sealed_bytes_match - && sampled_metadata.can_prove_freshness() && witness.as_ref().is_some_and(|witness| { witness.git_metadata_signature == sampled_metadata.stable_signature() && witness.stat_signature == sampled_sweep.signature diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/index_path_policy_tests.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/index_path_policy_tests.rs index c0e00751f8..57e30b6b5e 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/index_path_policy_tests.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/index_path_policy_tests.rs @@ -420,3 +420,68 @@ fn parent_filtered_source_does_not_require_submodule_registration_when_disabled( .unwrap(); assert_eq!(sealed.as_deref(), Some(source.as_bytes())); } + +#[test] +fn retained_graph_checks_require_a_successful_submodule_probe_each_time() { + let registration = "[submodule \"child\"]\npath = child\n"; + let parent = GitFixture::new(&[ + ("lib.rs", "pub fn first() {}\n"), + (".gitmodules", registration), + ]); + let store = TempDir::new().unwrap(); + // Keep the sealed attribution worktree-relative across the configuration edit. + std::fs::write(parent.path().join("lib.rs"), "pub fn latest() {}\n").unwrap(); + let mut scheduler = start( + &parent, + store.path(), + policy(&[], &[]).with_submodules(true), + ); + published(scheduler.reconcile_now().unwrap()); + let metadata = scheduler + .publication + .load_active_manifest() + .unwrap() + .unwrap() + .metadata; + // A partially staged edit leaves the worktree and its source bytes readable, + // while the index registration still cannot establish submodule identity. + let registration_path = parent.path().join(".gitmodules"); + std::fs::write(®istration_path, "[unfinished").unwrap(); + super::run_git_in(parent.path(), &["add", ".gitmodules"]); + std::fs::write(®istration_path, registration).unwrap(); + for _ in 0..2 { + assert!( + !scheduler + .freshness_fence() + .git_metadata(parent.path()) + .can_prove_freshness() + ); + let serving_witness = scheduler.currency_witness_for_sealed_snapshot( + &metadata.manifest().generation_id, + &metadata.snapshot().content_identity, + ); + let retained = scheduler + .reconcile_retained_text_generation_with(&metadata, false) + .unwrap(); + assert_eq!( + (serving_witness.is_some(), retained.is_some()), + (false, false), + "unchanged source bytes cannot replace a failed submodule probe" + ); + } + super::run_git_in(parent.path(), &["add", ".gitmodules"]); + assert!(matches!( + scheduler + .reconcile_retained_text_generation_with(&metadata, false) + .unwrap(), + Some(CodeIndexReconcileOutcomeV1::Noop(_)) + )); + assert!( + scheduler + .currency_witness_for_sealed_snapshot( + &metadata.manifest().generation_id, + &metadata.snapshot().content_identity, + ) + .is_some() + ); +} From 478da430ede75cb22cb1f7ffc97a5f14364a0f46 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 16:37:44 -0800 Subject: [PATCH 29/33] style(code-index): format submodule witness assertion --- .../src/code_index_scheduler/identity.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/identity.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/identity.rs index 39f0813488..71d215b6b8 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/identity.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/identity.rs @@ -512,7 +512,10 @@ mod tests { let parent_only = GitMetadataFingerprintV1::capture(root.path()); let failed = GitMetadataFingerprintV1::for_policy(root.path(), true); - assert!(failed.submodule_probe_failed, "opt-in probe must fail closed"); + assert!( + failed.submodule_probe_failed, + "opt-in probe must fail closed" + ); assert!(!failed.can_prove_freshness()); assert!(failed.differs_from(&parent_only)); assert_ne!( From 2b752c786481b9d4e4fa3d6d837459325b7f526f Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 17:31:12 -0800 Subject: [PATCH 30/33] fix(code-index): clear submodule indexing Clippy diagnostics --- .../src/code_index_scheduler/git_tree_capture.rs | 9 +++++---- .../src/code_index_scheduler/identity.rs | 6 ++++-- .../src/code_index_scheduler/ignored_dependencies.rs | 7 +++---- .../tests/index_path_policy_tests.rs | 7 +++++-- crates/tracedecay-code-index/src/source_walk.rs | 4 +++- 5 files changed, 20 insertions(+), 13 deletions(-) diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/git_tree_capture.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/git_tree_capture.rs index 922d0c889e..26b798f880 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/git_tree_capture.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/git_tree_capture.rs @@ -35,6 +35,9 @@ use tracedecay_code_index::production::{ CodeIndexRepositoryParseIdentityV1, }; +type TreeBlobVisitor<'a> = + dyn FnMut(&[u8], &[u8]) -> Result<(), CodeIndexSearchUnavailableReasonV1> + 'a; + #[derive(Clone, Debug, PartialEq, Eq)] pub struct ExactGitTreeSourceV1 { pub reference: tracedecay_domain::RefId, @@ -500,7 +503,7 @@ impl CodeIndexWorktreeSchedulerV1 { parent: &gix::Repository, pins: &GitTreeSubmodulePins, control: &branch_generations::BranchGenerationReadControlV1, - visitor: &mut dyn FnMut(&[u8], &[u8]) -> Result<(), CodeIndexSearchUnavailableReasonV1>, + visitor: &mut TreeBlobVisitor<'_>, ) -> Result<(), CodeIndexSearchUnavailableReasonV1> { let unavailable = CodeIndexSearchUnavailableReasonV1::GenerationUnavailable; let children = @@ -596,9 +599,7 @@ impl CodeIndexWorktreeSchedulerV1 { source_revision: Option, tree: tracedecay_domain::TreeId, control: &branch_generations::BranchGenerationReadControlV1, - visit: impl FnOnce( - &mut dyn FnMut(&[u8], &[u8]) -> Result<(), CodeIndexSearchUnavailableReasonV1>, - ) -> Result<(), CodeIndexSearchUnavailableReasonV1>, + visit: impl FnOnce(&mut TreeBlobVisitor<'_>) -> Result<(), CodeIndexSearchUnavailableReasonV1>, ) -> Result { let registry = StaticLanguageRegistry::new(); let mut roster = CapturedFileRosterV1::default(); diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/identity.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/identity.rs index 71d215b6b8..ed1378ee97 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/identity.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/identity.rs @@ -270,10 +270,12 @@ impl GitMetadataFingerprintV1 { ), ); if let Some(refs) = &self.submodule_refs { - signature.push_str(&format!("|submodule_refs={refs}")); + signature.push_str("|submodule_refs="); + signature.push_str(refs); } if let Some(indexes) = &self.submodule_indexes { - signature.push_str(&format!("|submodule_indexes={indexes}")); + signature.push_str("|submodule_indexes="); + signature.push_str(indexes); } if self.submodule_probe_failed { signature.push_str("|submodule_probe_failed=true"); diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/ignored_dependencies.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/ignored_dependencies.rs index 23dacd76f5..f0d6c56fa0 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/ignored_dependencies.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/ignored_dependencies.rs @@ -1055,10 +1055,9 @@ mod tests { fixture_git(&parent, &["add", ".gitmodules", ".gitignore"]); fixture_git(&parent, &["commit", "-qm", "parent with submodule"]); fixture_git(&parent, &["rm", "--cached", "-f", "node_modules/pkg"]); - assert!( - fixture_git(&parent, &["ls-files", "--stage", "--", "node_modules/pkg"]) - .stdout - .is_empty() + assert_eq!( + fixture_git(&parent, &["ls-files", "--stage", "--", "node_modules/pkg"]).stdout, + b"" ); assert!( fixture_git(&parent, &["ls-tree", "HEAD", "--", "node_modules/pkg"]) diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/index_path_policy_tests.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/index_path_policy_tests.rs index 57e30b6b5e..9ec5037f42 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/index_path_policy_tests.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/index_path_policy_tests.rs @@ -270,7 +270,10 @@ fn submodule_policy_binds_live_graph_and_historical_child_pins_across_restart() let mut off = start(&parent, store.path(), policy(&[], &[])); off.reconcile_now().unwrap(); assert_eq!(indexed_paths(&off), ["dev/launch.ts"]); - assert!(grep("pinned_symbol", &policy(&[], &[])).is_empty()); + assert_eq!( + grep("pinned_symbol", &policy(&[], &[])), + Vec::::new() + ); drop(off); let mut on = start(&parent, store.path(), enabled.clone()); @@ -308,7 +311,7 @@ fn submodule_policy_binds_live_graph_and_historical_child_pins_across_restart() "const private_marker = 1;", ) .unwrap(); - assert!(grep("private_marker", &enabled).is_empty()); + assert_eq!(grep("private_marker", &enabled), Vec::::new()); std::fs::write( checkout.join("lib.ts"), "export function feature() { return 'successor_symbol'; }\n", diff --git a/crates/tracedecay-code-index/src/source_walk.rs b/crates/tracedecay-code-index/src/source_walk.rs index 3701db9d5f..e227f3032d 100644 --- a/crates/tracedecay-code-index/src/source_walk.rs +++ b/crates/tracedecay-code-index/src/source_walk.rs @@ -847,7 +847,9 @@ mod tests { &IndexPathPolicyV1::new(vec![], vec![]).unwrap(), ) .expect("source walk with nested repository diagnostics"); - let _visited = walk.map(|entry| entry.expect("source entry")).count(); + for entry in walk { + entry.expect("source entry"); + } assert_eq!( diagnostics.skipped_nested_repositories_note().as_deref(), Some("Skipped nested repositories outside parent index: \"app\"") From 26d7171bf87eebd8cd9a75abc24d9e3107670f44 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 17:51:17 -0800 Subject: [PATCH 31/33] fix(code-index): preserve retained source boundary refusals --- .../src/code_index_scheduler/reconcile.rs | 19 ++++++++++++++++--- crates/tracedecay-mcp/src/handlers/grep.rs | 16 +++++++++++----- 2 files changed, 27 insertions(+), 8 deletions(-) diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/reconcile.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/reconcile.rs index a820832fb4..6e0a7feb15 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/reconcile.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/reconcile.rs @@ -4050,14 +4050,27 @@ impl CodeIndexWorktreeSchedulerV1 { } _ => (self.project_root.clone(), relative), }; - if tracedecay_code_index::source_walk::has_nested_repository_ancestor( + let refusal = if tracedecay_code_index::source_walk::has_nested_repository_ancestor( &source_root, source_relative, - )? || tracedecay_code_index::source_walk::is_source_symlink( + )? { + Some(CodeIndexIgnoredDependencyRefusalV1::NotIgnored) + } else if tracedecay_code_index::source_walk::is_source_symlink( &source_root, source_relative, )? { - return Ok(CapturedFileOutcomeV1::Absent); + Some(CodeIndexIgnoredDependencyRefusalV1::SymlinkEscape) + } else { + None + }; + if let Some(reason) = refusal { + // A retained admission names required bytes. Omitting them silently + // would turn a source refusal into a broken snapshot contract. + return if explicitly_admitted { + Err(reason.into()) + } else { + Ok(CapturedFileOutcomeV1::Absent) + }; } match absolute.metadata() { Ok(metadata) if !metadata.is_file() => return Ok(CapturedFileOutcomeV1::Absent), diff --git a/crates/tracedecay-mcp/src/handlers/grep.rs b/crates/tracedecay-mcp/src/handlers/grep.rs index 1cb0965588..1d09dc452e 100644 --- a/crates/tracedecay-mcp/src/handlers/grep.rs +++ b/crates/tracedecay-mcp/src/handlers/grep.rs @@ -477,11 +477,17 @@ mod tests { "// UNIQUE_GENERATED_DIR_TOKEN\n", ) .expect("source fixture"); - std::fs::write( - project.path().join(".git"), - "gitdir: UNIQUE_GENERATED_DIR_TOKEN\n", - ) - .expect("linked-worktree git file fixture"); + let administration = tempfile::tempdir().expect("isolated Git administration"); + let git = tracedecay_runtime_core::git::try_git_program().expect("Git program"); + let initialized = std::process::Command::new(git) + .args(["init", "--quiet", "--template=", "--separate-git-dir"]) + .arg(administration.path().join("UNIQUE_GENERATED_DIR_TOKEN")) + .arg(project.path()) + .env("HOME", administration.path()) + .env("GIT_CONFIG_NOSYSTEM", "1") + .status() + .expect("initialize checkout with a Git pointer file"); + assert!(initialized.success()); std::fs::write( project.path().join(".tracedecay/internal.txt"), "UNIQUE_GENERATED_DIR_TOKEN\n", From da69c12c2a3b511cdf33d02c2377e8ff5dc0b294 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Mon, 9 Nov 2026 00:18:56 -0800 Subject: [PATCH 32/33] fix(code-index): retain source aliases within the parent checkout --- .../code_index_scheduler/freshness_witness.rs | 41 ++++---- .../code_index_scheduler/git_tree_capture.rs | 31 ++++-- .../src/code_index_scheduler/reconcile.rs | 35 +++---- .../tracedecay-code-index/src/source_walk.rs | 8 +- .../src/source_walk/classification.rs | 60 +++++++++--- .../src/source_walk/submodules.rs | 15 ++- .../src/source_walk/tracked_links.rs | 94 +++++++++++++++++++ .../src/git_repository.rs | 2 +- .../src/git_repository/tree_links.rs | 67 +++++++++---- docs/USER-GUIDE.md | 4 +- 10 files changed, 269 insertions(+), 88 deletions(-) create mode 100644 crates/tracedecay-code-index/src/source_walk/tracked_links.rs diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/freshness_witness.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/freshness_witness.rs index 00228326aa..fe936ae032 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/freshness_witness.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/freshness_witness.rs @@ -248,18 +248,6 @@ fn read_candidate( project_root: &Path, candidate: &StatCandidateV1, ) -> Result, CodeIndexSchedulerErrorV1> { - // A link installed after candidate classification must not turn a - // freshness proof into a read of a different repository's source bytes. - if tracedecay_code_index::source_walk::is_source_symlink( - project_root, - Path::new(&candidate.logical_path), - )? { - return Err(std::io::Error::new( - std::io::ErrorKind::InvalidData, - "source symlink is outside the parent source inventory", - ) - .into()); - } if candidate.explicitly_admitted { ignored_dependencies::read_explicitly_admitted_source( project_root, @@ -267,13 +255,22 @@ fn read_candidate( None, ) } else { - ignored_dependencies::read_bounded_snapshot_source( - &project_root.join(&candidate.logical_path), - None, - ) + let path = resolved_candidate_path(project_root, &candidate.logical_path)?; + ignored_dependencies::read_bounded_snapshot_source(&path, None) } } +fn resolved_candidate_path(root: &Path, logical_path: &str) -> std::io::Result { + tracedecay_code_index::source_walk::resolve_parent_source(root, Path::new(logical_path))? + .map(|relative| root.join(relative)) + .ok_or_else(|| { + std::io::Error::new( + std::io::ErrorKind::InvalidData, + "source alias is outside the parent Git inventory", + ) + }) +} + /// The canonical content digest capture would seal for `raw`: the digest of /// the sanitized bytes. fn sanitized_digest( @@ -956,13 +953,17 @@ fn filter_mismatches_in_repository<'a>( let Ok((mut pipeline, index)) = repository.filter_pipeline(None) else { return disputed; }; + let repository_root = project_root.join(prefix); disputed .into_iter() .filter(|candidate| { let Some(expected) = manifest.files.get(&candidate.logical_path) else { return true; }; - let Ok(rela_path) = Path::new(&candidate.logical_path).strip_prefix(prefix) else { + let Ok(path) = resolved_candidate_path(project_root, &candidate.logical_path) else { + return true; + }; + let Ok(rela_path) = path.strip_prefix(&repository_root) else { return true; }; if candidate.explicitly_admitted @@ -972,7 +973,7 @@ fn filter_mismatches_in_repository<'a>( { return true; } - let Ok(file) = std::fs::File::open(project_root.join(&candidate.logical_path)) else { + let Ok(file) = std::fs::File::open(&path) else { return true; }; let Ok(mut converted) = pipeline.convert_to_git(file, rela_path, &index) else { @@ -1004,7 +1005,7 @@ pub fn sealed_source_bytes( let Some((root, relative)) = boundaries.resolve(Path::new(logical_path))? else { return Ok(None); }; - let raw = ignored_dependencies::read_bounded_snapshot_source(&root.join(relative), None)?; + let raw = ignored_dependencies::read_bounded_snapshot_source(&root.join(&relative), None)?; let (sanitized, _, _) = privacy::sanitize_code_file(language, &raw)?; if content_digest(&sanitized) == *sealed { return Ok(Some(sanitized)); @@ -1022,7 +1023,7 @@ pub fn sealed_source_bytes( } let mut filtered = Vec::new(); pipeline - .convert_to_git(raw.as_slice(), relative, &index) + .convert_to_git(raw.as_slice(), relative.as_ref(), &index) .map_err(|error| CodeIndexSchedulerErrorV1::Git(error.to_string()))? .read_to_end(&mut filtered)?; let (sanitized, _, _) = privacy::sanitize_code_file(language, &filtered)?; diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/git_tree_capture.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/git_tree_capture.rs index 26b798f880..ef4a82f133 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/git_tree_capture.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/git_tree_capture.rs @@ -1403,15 +1403,32 @@ mod tests { .snapshot, ); assert_eq!( - live.iter() - .map(|(path, _)| path.as_str()) - .collect::>(), - ["AGENTS.md", "src/real.rs"] + branch, live, + "same-parent source aliases resolve to the live checkout's bytes" ); - for regular in live { + let changed = b"pub fn linked_target() -> u32 { 19 }\n"; + std::fs::write(root.join("src/real.rs"), changed).expect("edit only the alias target"); + let edited = present( + &scheduler + .capture_authoritative_snapshot_without_active_generation_reuse(None) + .expect("capture unstaged target through each tracked alias") + .snapshot, + ); + let expected = tracedecay_code_index::chunks::content_digest(changed); + for alias in [ + "src/alias.rs", + "src/nested/chained.rs", + "src/through_directory.rs", + ] { + assert!( + edited + .iter() + .any(|(path, digest)| path == alias && *digest == expected) + ); assert!( - branch.contains(®ular), - "historical and live regular files must carry the same source digest" + branch + .iter() + .any(|(path, digest)| path == alias && *digest != expected) ); } let notes = captured diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/reconcile.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/reconcile.rs index 6e0a7feb15..56e5ea5a84 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/reconcile.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/reconcile.rs @@ -4050,28 +4050,29 @@ impl CodeIndexWorktreeSchedulerV1 { } _ => (self.project_root.clone(), relative), }; - let refusal = if tracedecay_code_index::source_walk::has_nested_repository_ancestor( + let resolved = if tracedecay_code_index::source_walk::has_nested_repository_ancestor( &source_root, source_relative, )? { - Some(CodeIndexIgnoredDependencyRefusalV1::NotIgnored) - } else if tracedecay_code_index::source_walk::is_source_symlink( - &source_root, - source_relative, - )? { - Some(CodeIndexIgnoredDependencyRefusalV1::SymlinkEscape) + Err(CodeIndexIgnoredDependencyRefusalV1::NotIgnored) + } else if explicitly_admitted + && tracedecay_code_index::source_walk::is_source_symlink(&source_root, source_relative)? + { + Err(CodeIndexIgnoredDependencyRefusalV1::SymlinkEscape) } else { - None + tracedecay_code_index::source_walk::resolve_parent_source( + &source_root, + source_relative, + )? + .ok_or(CodeIndexIgnoredDependencyRefusalV1::SymlinkEscape) }; - if let Some(reason) = refusal { - // A retained admission names required bytes. Omitting them silently - // would turn a source refusal into a broken snapshot contract. - return if explicitly_admitted { - Err(reason.into()) - } else { - Ok(CapturedFileOutcomeV1::Absent) - }; - } + let resolved = match resolved { + Ok(path) => path, + // Retained admissions name required bytes. A refusal must remain visible. + Err(reason) if explicitly_admitted => return Err(reason.into()), + Err(_) => return Ok(CapturedFileOutcomeV1::Absent), + }; + let absolute = source_root.join(resolved); match absolute.metadata() { Ok(metadata) if !metadata.is_file() => return Ok(CapturedFileOutcomeV1::Absent), Ok(_) => {} diff --git a/crates/tracedecay-code-index/src/source_walk.rs b/crates/tracedecay-code-index/src/source_walk.rs index e227f3032d..93b6b3847e 100644 --- a/crates/tracedecay-code-index/src/source_walk.rs +++ b/crates/tracedecay-code-index/src/source_walk.rs @@ -11,6 +11,8 @@ pub mod classification; pub mod submodules; +mod tracked_links; +pub use tracked_links::resolve_parent_source; use std::collections::BTreeSet; use std::path::{Component, Path, PathBuf}; @@ -253,9 +255,7 @@ pub fn has_nested_repository_ancestor( Ok(false) } -/// A worktree source entry must be a real file, not a link to bytes outside -/// the parent's Git authority. The shared live walker does not follow leaf -/// symlinks; classification, capture, and freshness must make the same choice. +/// Distinguish leaf links before selecting their source authority. /// Missing paths remain possible tombstones and are not symlink exclusions. pub fn is_source_symlink(project_root: &Path, relative_path: &Path) -> std::io::Result { if relative_path.as_os_str().is_empty() @@ -616,7 +616,7 @@ mod tests { #[cfg(unix)] #[test] - fn readable_symlink_leaf_is_not_an_admissible_source() { + fn leaf_link_detection_preserves_missing_paths() { use std::os::unix::fs::symlink; let root = TempDir::new().unwrap(); diff --git a/crates/tracedecay-code-index/src/source_walk/classification.rs b/crates/tracedecay-code-index/src/source_walk/classification.rs index 931bdf5329..2042361ca3 100644 --- a/crates/tracedecay-code-index/src/source_walk/classification.rs +++ b/crates/tracedecay-code-index/src/source_walk/classification.rs @@ -18,7 +18,7 @@ use std::collections::{BTreeMap, BTreeSet}; use std::path::Path; use super::{ - has_nested_repository_ancestor, is_source_symlink, parent_gitlink_paths, + has_nested_repository_ancestor, parent_gitlink_paths, resolve_parent_source, submodules::live_submodules, }; use gix::bstr::ByteSlice; @@ -241,12 +241,8 @@ impl WorktreeChangeClassificationV1 { { let relative = Path::new(path); let parent = relative.parent().unwrap_or_else(|| Path::new("")); - // The live walker never follows a symlink file, even when - // the lexical parent is ordinary and the target is inside a - // registered submodule. Do not capture its target through a - // clean-tree or worktree path. let excluded = if classification.beneath_gitlink(relative) - || is_source_symlink(project_root, relative)? + || resolve_parent_source(project_root, relative)?.is_none() { true } else { @@ -621,7 +617,19 @@ mod nested_repository_tests { ], ); symlink("app/src/lib.rs", parent.join("bridge.rs")).unwrap(); - fixture_git(&parent, &["add", ".gitmodules", "app", "bridge.rs"]); + fs::write(parent.join("parent.rs"), "pub fn parent() {}\n").unwrap(); + symlink("parent.rs", parent.join("local.rs")).unwrap(); + fixture_git( + &parent, + &[ + "add", + ".gitmodules", + "app", + "bridge.rs", + "parent.rs", + "local.rs", + ], + ); fixture_git( &parent, &["commit", "-qm", "registered child and source alias"], @@ -639,14 +647,40 @@ mod nested_repository_tests { ); let repository = tracedecay_runtime_core::git_open::open(&parent).unwrap(); - let classified = WorktreeChangeClassificationV1::classify(&repository).unwrap(); - assert!(!classified.candidate_paths().contains("bridge.rs")); - assert!(classified.changed_paths().contains("bridge.rs")); + for include_submodules in [false, true] { + let classified = + WorktreeChangeClassificationV1::for_policy(&repository, include_submodules) + .unwrap(); + let candidates = classified.candidate_paths(); + let boundaries = crate::source_walk::submodules::LiveSourceBoundaries::open( + &parent, + include_submodules, + ) + .unwrap(); + let (_, relative) = boundaries.resolve(Path::new("local.rs")).unwrap().unwrap(); + assert_eq!(relative.as_ref(), Path::new("parent.rs")); + assert!( + boundaries + .resolve(Path::new("bridge.rs")) + .unwrap() + .is_none() + ); + assert!(candidates.contains("local.rs")); + assert!(!candidates.contains("bridge.rs")); + assert!(classified.changed_paths().contains("bridge.rs")); + assert!( + classified.has_changes(), + "do not reuse HEAD tree source bytes" + ); + assert_eq!(candidates.contains("app/src/lib.rs"), include_submodules); + } + symlink("parent.rs", parent.join("untracked_alias.rs")).unwrap(); + let classification = WorktreeChangeClassificationV1::classify(&repository).unwrap(); assert!( - classified.has_changes(), - "do not reuse HEAD tree source bytes" + !classification + .candidate_paths() + .contains("untracked_alias.rs") ); - assert!(!classified.candidate_paths().contains("app/src/lib.rs")); } #[test] diff --git a/crates/tracedecay-code-index/src/source_walk/submodules.rs b/crates/tracedecay-code-index/src/source_walk/submodules.rs index 461f79aff4..0c068e8d32 100644 --- a/crates/tracedecay-code-index/src/source_walk/submodules.rs +++ b/crates/tracedecay-code-index/src/source_walk/submodules.rs @@ -1,6 +1,7 @@ //! A parent gitlink names a commit, not the worktree that may supply its bytes. //! Bind the registration, object store, and checkout before crossing that boundary. +use std::borrow::Cow; use std::collections::{BTreeMap, BTreeSet}; use std::path::{Component, Path, PathBuf}; @@ -225,7 +226,7 @@ impl LiveSourceBoundaries { pub fn resolve<'a, 'p>( &'a self, path: &'p Path, - ) -> std::io::Result> { + ) -> std::io::Result)>> { if !is_source_location(path) { return Ok(None); } @@ -259,12 +260,16 @@ impl LiveSourceBoundaries { root = &child.worktree; break; } - if has_nested_repository_ancestor(root, relative)? - || super::is_source_symlink(root, relative)? - { + if has_nested_repository_ancestor(root, relative)? { return Ok(None); } - Ok(Some((root, relative))) + if root == self.project_root { + Ok(super::resolve_parent_source(root, relative)?.map(|path| (root, path))) + } else if super::is_source_symlink(root, relative)? { + Ok(None) + } else { + Ok(Some((root, Cow::Borrowed(relative)))) + } } } diff --git a/crates/tracedecay-code-index/src/source_walk/tracked_links.rs b/crates/tracedecay-code-index/src/source_walk/tracked_links.rs new file mode 100644 index 0000000000..35d47fa914 --- /dev/null +++ b/crates/tracedecay-code-index/src/source_walk/tracked_links.rs @@ -0,0 +1,94 @@ +//! Resolve a tracked source alias using the parent Git index and live worktree. + +use std::borrow::Cow; +use std::io; +use std::path::{Component, Path}; + +use gix::bstr::ByteSlice; +use gix::index::entry::{Mode, Stage}; +use tracedecay_runtime_core::git_repository::{RepositoryLinkEntry, resolve_repository_link}; + +use super::{is_source_symlink, nested_repository_marker, submodules::is_source_location}; + +/// Ordinary files keep their original path. A live parent alias may only read +/// a tracked regular file through tracked links and parent-owned directories. +pub fn resolve_parent_source<'a>( + root: &Path, + relative: &'a Path, +) -> io::Result>> { + if !is_source_symlink(root, relative)? { + return Ok(Some(Cow::Borrowed(relative))); + } + if !nested_repository_marker(root)? { + return Ok(None); + } + let repo = tracedecay_runtime_core::git_open::open(root).map_err(io::Error::other)?; + let index = repo.index_or_empty().map_err(io::Error::other)?; + let head = repo.head().map_err(io::Error::other)?; + let tree = if head.id().is_some() { + Some(repo.head_tree().map_err(io::Error::other)?) + } else { + None + }; + let alias = gix::path::into_bstr(relative).map_err(io::Error::other)?; + let alias = gix::path::to_unix_separators_on_windows(alias); + if !index + .entry_by_path_and_stage(&alias, Stage::Unconflicted) + .is_some_and(|entry| entry.mode == Mode::SYMLINK) + { + return Ok(None); + } + resolve_repository_link(b"", &alias, |components| { + let bytes = components.join(&b'/'); + let path = gix::path::from_byte_slice(&bytes).map_err(io::Error::other)?; + if !is_source_location(path) { + return Ok(None); + } + let absolute = root.join(path); + let metadata = match absolute.symlink_metadata() { + Ok(metadata) => metadata, + Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(None), + Err(error) => return Err(error), + }; + let indexed = index.entry_by_path_and_stage(bytes.as_bstr(), Stage::Unconflicted); + if metadata.is_dir() { + // A gitlink staged for removal still bounds the old checkout even + // after its marker disappears. Check HEAD before descending. + let head_entry = tree + .as_ref() + .map(|tree| tree.lookup_entry(components.iter().map(Vec::as_slice))) + .transpose() + .map_err(io::Error::other)? + .flatten(); + if index.entry_by_path(bytes.as_bstr()).is_some() + || head_entry.is_some_and(|entry| entry.mode().is_commit()) + || nested_repository_marker(&absolute)? + { + return Ok(None); + } + return Ok(Some(RepositoryLinkEntry::Directory)); + } + let Some(entry) = indexed else { + return Ok(None); + }; + if metadata.file_type().is_symlink() && entry.mode == Mode::SYMLINK { + let target = std::fs::read_link(&absolute)?; + if target + .components() + .any(|component| matches!(component, Component::Prefix(_) | Component::RootDir)) + { + return Ok(None); + } + let target = gix::path::into_bstr(target).map_err(io::Error::other)?; + return Ok(Some(RepositoryLinkEntry::Link( + gix::path::to_unix_separators_on_windows(target).to_vec(), + ))); + } + Ok((metadata.is_file() + && entry + .mode + .to_tree_entry_mode() + .is_some_and(|mode| mode.is_blob())) + .then(|| RepositoryLinkEntry::File(Cow::Owned(path.to_owned())))) + }) +} diff --git a/crates/tracedecay-runtime-core/src/git_repository.rs b/crates/tracedecay-runtime-core/src/git_repository.rs index bfb65d1212..e978a2cce0 100644 --- a/crates/tracedecay-runtime-core/src/git_repository.rs +++ b/crates/tracedecay-runtime-core/src/git_repository.rs @@ -35,7 +35,7 @@ pub use native_integration::{ GitNativeIntegrationMode, GitNativePreflight, GitNativePreflightCaptureError, GitNativePreflightDisposition, GitNativeUnsupportedReason, GitTreeSubmodulePins, }; -pub use tree_links::resolve_tree_link; +pub use tree_links::{RepositoryLinkEntry, resolve_repository_link, resolve_tree_link}; /// A typed failure from the in-process Git repository authority. #[derive(Clone, Debug, thiserror::Error)] diff --git a/crates/tracedecay-runtime-core/src/git_repository/tree_links.rs b/crates/tracedecay-runtime-core/src/git_repository/tree_links.rs index ccee3e049f..31e81fe652 100644 --- a/crates/tracedecay-runtime-core/src/git_repository/tree_links.rs +++ b/crates/tracedecay-runtime-core/src/git_repository/tree_links.rs @@ -15,6 +15,42 @@ pub fn resolve_tree_link( link_path: &[u8], target: &[u8], ) -> Result, GitRepositoryError> { + resolve_repository_link(link_path, target, |path| { + let Some(entry) = tree + .lookup_entry(path.iter().map(Vec::as_slice)) + .map_err(|error| operation("tree link lookup", error))? + else { + return Ok(None); + }; + let mode = entry.mode(); + Ok(if mode.is_tree() { + Some(RepositoryLinkEntry::Directory) + } else if mode.is_link() { + let link = entry + .object() + .map_err(|error| operation("tree link target", error))?; + Some(RepositoryLinkEntry::Link(link.data.clone())) + } else if mode.is_blob() { + Some(RepositoryLinkEntry::File(entry.object_id())) + } else { + None + }) + }) +} + +pub enum RepositoryLinkEntry { + Directory, + Link(Vec), + File(T), +} + +/// Walk relative link components with the caller's repository authority. Each +/// directory is checked before descending, including one followed by `..`. +pub fn resolve_repository_link( + link_path: &[u8], + target: &[u8], + mut lookup: impl FnMut(&[Vec]) -> Result>, E>, +) -> Result, E> { let mut resolved = components(link_path).collect::>(); resolved.pop(); let mut pending = Vec::new(); @@ -33,26 +69,17 @@ pub fn resolve_tree_link( } _ => resolved.push(component), } - let Some(entry) = tree - .lookup_entry(resolved.iter().map(Vec::as_slice)) - .map_err(|error| operation("tree link lookup", error))? - else { - return Ok(None); - }; - let mode = entry.mode(); - if mode.is_tree() { - continue; - } - if !mode.is_link() { - return Ok((pending.is_empty() && mode.is_blob()).then(|| entry.object_id())); - } - hops += 1; - resolved.pop(); - let link = entry - .object() - .map_err(|error| operation("tree link target", error))?; - if hops > MAX_LINK_HOPS || !push_target(&mut pending, &link.data) { - return Ok(None); + match lookup(&resolved)? { + Some(RepositoryLinkEntry::Directory) => {} + Some(RepositoryLinkEntry::Link(target)) => { + hops += 1; + resolved.pop(); + if hops > MAX_LINK_HOPS || !push_target(&mut pending, &target) { + return Ok(None); + } + } + Some(RepositoryLinkEntry::File(file)) if pending.is_empty() => return Ok(Some(file)), + _ => return Ok(None), } } Ok(None) diff --git a/docs/USER-GUIDE.md b/docs/USER-GUIDE.md index 15a3ee9014..5ef9797bc9 100644 --- a/docs/USER-GUIDE.md +++ b/docs/USER-GUIDE.md @@ -219,7 +219,9 @@ tracedecay tool configuration_set --args '{"layer":{"kind":"project","project_id The parent index and `.gitmodules` must identify the child, and its checkout must use the matching repository under the parent's Git modules directory. -Live indexing includes child worktree changes. Historical indexing reads the +Live indexing includes child worktree changes. The parent graph retains tracked +source aliases that resolve through tracked links to a tracked parent file. A +parent alias cannot cross into a child checkout. Historical indexing reads the child commit recorded by the selected parent revision from the local object store. Initialize the submodule locally before requesting that history. Linked worktrees, unrelated clones, and deeper nested repositories stay From f36fbb3841056efbef51057d6e6fdc9573bc81b2 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Mon, 9 Nov 2026 00:38:44 -0800 Subject: [PATCH 33/33] test(code-index): require clean revisions for tracked aliases --- .../tests/code_index_scheduler/reconcile.rs | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/crates/tracedecay-code-index-runtime/tests/code_index_scheduler/reconcile.rs b/crates/tracedecay-code-index-runtime/tests/code_index_scheduler/reconcile.rs index 5aa18b0df2..318dc166f9 100644 --- a/crates/tracedecay-code-index-runtime/tests/code_index_scheduler/reconcile.rs +++ b/crates/tracedecay-code-index-runtime/tests/code_index_scheduler/reconcile.rs @@ -825,11 +825,11 @@ fn unfollowable_committed_source_links_are_absent_from_disk_capture() { ); } -/// A readable live symlink stays outside the worktree roster. The remaining -/// files can still establish a current generation without naming the full HEAD tree. +/// Publishing the link target bytes under both tracked paths must preserve +/// the clean HEAD revision and avoid a redundant post-publication rebuild. #[cfg(unix)] #[test] -fn committed_source_symlink_publishes_current_regular_files_only() { +fn committed_source_symlink_publishes_a_current_clean_generation() { let fixture = GitFixture::new(&[ ("src/lib.rs", "pub mod pricing;\n"), ("src/pricing.rs", "pub fn total() -> u64 { 1 }\n"), @@ -851,14 +851,17 @@ fn committed_source_symlink_publishes_current_regular_files_only() { ); let latest = scheduler.latest_complete().expect("published generation"); let snapshot = latest.generation.snapshot(); - assert!(snapshot.source_revision.is_none()); + assert_eq!( + snapshot.source_revision.as_ref(), + scheduler.identity.head_commit() + ); assert_eq!( snapshot .files .iter() .map(|file| file.logical_path.as_str()) .collect::>(), - ["src/lib.rs", "src/pricing.rs"] + ["src/lib.rs", "src/pricing.rs", "src/pricing_alias.rs"] ); }