use std::collections::{BTreeMap, HashMap, HashSet};
use std::ops::{Deref, DerefMut};
use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering};
use std::sync::{Arc, OnceLock};
use arc_swap::{ArcSwap, ArcSwapOption};
use parking_lot::{Mutex, MutexGuard};
use std::time::{Duration, Instant, SystemTime};
use rayon::prelude::*;
use tracing::{debug, error, info, warn};
use super::knowledge_authority::{
AuthorityLimits, AuthorityTemporalIndex, KnowledgeAuthorityView, build_knowledge_authority,
};
use super::knowledge_bridge::{BridgeLimits, KnowledgeBridge, build_knowledge_bridge};
use super::query::RepoOutlineView;
use crate::domain::ParseDiagnostic;
use crate::domain::index::{AdmissionDecision, AdmissionTier, SkipReason, SkippedFile};
use crate::domain::{
CatalogEntry, CoverageStatus, FileClassification, FileDisposition, FileOutcome,
FileProcessingResult, FreshnessReason, FreshnessStatus, HardSkipReason, HistoryCoverage,
HistoryLimit, LanguageId, ManifestResourceUsage, MetadataOnlyReason, ProjectStateDir,
ReferenceRecord, RepositoryManifest, ScoutDecision, SourceId, SourceIdentity,
SourceResponseEnvelope, SourceVersion, StatePlacement, SymbolRecord, WorkingTreeState,
find_enclosing_symbol,
};
use crate::{discovery, parsing};
pub(crate) fn normalize_root(root: &Path) -> PathBuf {
dunce::canonicalize(root).unwrap_or_else(|_| root.to_path_buf())
}
#[derive(Debug)]
pub(crate) struct ReloadCancelled;
impl std::fmt::Display for ReloadCancelled {
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
formatter.write_str("embedded reload cancelled during close")
}
}
impl std::error::Error for ReloadCancelled {}
pub(crate) fn reload_cancelled() -> anyhow::Error {
anyhow::Error::new(ReloadCancelled)
}
pub(crate) fn reload_was_cancelled(error: &anyhow::Error) -> bool {
error.downcast_ref::<ReloadCancelled>().is_some()
}
fn check_reload_cancelled(cancel: Option<&AtomicBool>) -> anyhow::Result<()> {
if cancel.is_some_and(|flag| flag.load(Ordering::Acquire)) {
Err(reload_cancelled())
} else {
Ok(())
}
}
#[cfg(feature = "__test-internals")]
struct ReloadGateState {
pause_after_parses: usize,
parsed: AtomicUsize,
blocked: std::sync::Mutex<bool>,
reached: std::sync::Condvar,
released: AtomicBool,
release_on_cancel: bool,
}
#[cfg(feature = "__test-internals")]
static RELOAD_GATES: OnceLock<std::sync::Mutex<HashMap<PathBuf, Arc<ReloadGateState>>>> =
OnceLock::new();
#[cfg(feature = "__test-internals")]
pub struct ReloadGateForTest {
root: PathBuf,
state: Arc<ReloadGateState>,
}
#[cfg(feature = "__test-internals")]
pub fn hold_reload_after_parses_for_test(
root: &Path,
pause_after_parses: usize,
) -> ReloadGateForTest {
assert!(
pause_after_parses > 0,
"reload gate needs a positive parse count"
);
install_reload_gate(root, pause_after_parses, true)
}
#[cfg(feature = "__test-internals")]
pub fn hold_reload_after_discovery_for_test(root: &Path) -> ReloadGateForTest {
install_reload_gate(root, 0, true)
}
#[cfg(feature = "__test-internals")]
pub fn hold_reload_through_cancel_for_test(
root: &Path,
pause_after_parses: usize,
) -> ReloadGateForTest {
assert!(
pause_after_parses > 0,
"reload gate needs a positive parse count"
);
install_reload_gate(root, pause_after_parses, false)
}
#[cfg(feature = "__test-internals")]
fn install_reload_gate(
root: &Path,
pause_after_parses: usize,
release_on_cancel: bool,
) -> ReloadGateForTest {
let root = normalize_root(root);
let state = Arc::new(ReloadGateState {
pause_after_parses,
parsed: AtomicUsize::new(0),
blocked: std::sync::Mutex::new(false),
reached: std::sync::Condvar::new(),
released: AtomicBool::new(false),
release_on_cancel,
});
let gates = RELOAD_GATES.get_or_init(|| std::sync::Mutex::new(HashMap::new()));
let previous = gates
.lock()
.expect("reload gate registry")
.insert(root.clone(), Arc::clone(&state));
assert!(previous.is_none(), "a root can hold only one reload gate");
ReloadGateForTest { root, state }
}
#[cfg(feature = "__test-internals")]
impl ReloadGateForTest {
pub fn wait_until_blocked(&self, timeout: Duration) -> bool {
let blocked = self.state.blocked.lock().expect("reload gate state");
let (blocked, _) = self
.state
.reached
.wait_timeout_while(blocked, timeout, |blocked| !*blocked)
.expect("reload gate state");
*blocked
}
pub fn release(&self) {
self.state.released.store(true, Ordering::Release);
self.state.reached.notify_all();
}
}
#[cfg(feature = "__test-internals")]
impl Drop for ReloadGateForTest {
fn drop(&mut self) {
self.release();
let gates = RELOAD_GATES.get_or_init(|| std::sync::Mutex::new(HashMap::new()));
let mut gates = gates.lock().expect("reload gate registry");
if gates
.get(&self.root)
.is_some_and(|current| Arc::ptr_eq(current, &self.state))
{
gates.remove(&self.root);
}
}
}
#[cfg(feature = "__test-internals")]
fn pause_after_parse_for_test(source_scope: &Path, cancel: Option<&AtomicBool>) {
let Some(state) = RELOAD_GATES
.get()
.and_then(|gates| gates.lock().ok()?.get(source_scope).cloned())
else {
return;
};
if state.parsed.fetch_add(1, Ordering::AcqRel) + 1 != state.pause_after_parses {
return;
}
park_reload_gate(&state, cancel);
}
#[cfg(feature = "__test-internals")]
fn pause_after_discovery_for_test(source_scope: &Path, cancel: Option<&AtomicBool>) {
let source_scope = normalize_root(source_scope);
let Some(state) = RELOAD_GATES
.get()
.and_then(|gates| gates.lock().ok()?.get(&source_scope).cloned())
else {
return;
};
if state.pause_after_parses == 0 {
park_reload_gate(&state, cancel);
}
}
#[cfg(feature = "__test-internals")]
fn park_reload_gate(state: &ReloadGateState, cancel: Option<&AtomicBool>) {
let mut blocked = state.blocked.lock().expect("reload gate state");
*blocked = true;
state.reached.notify_all();
while !(state.released.load(Ordering::Acquire)
|| (state.release_on_cancel && cancel.is_some_and(|flag| flag.load(Ordering::Acquire))))
{
let (next, _) = state
.reached
.wait_timeout(blocked, Duration::from_millis(5))
.expect("reload gate state");
blocked = next;
}
}
#[cfg(not(feature = "__test-internals"))]
fn pause_after_parse_for_test(_source_scope: &Path, _cancel: Option<&AtomicBool>) {}
#[cfg(not(feature = "__test-internals"))]
fn pause_after_discovery_for_test(_source_scope: &Path, _cancel: Option<&AtomicBool>) {}
#[cfg(feature = "__test-internals")]
static DERIVED_STAGE_HOLD: AtomicBool = AtomicBool::new(false);
#[cfg(feature = "__test-internals")]
static DERIVED_STAGE_REACHED: AtomicBool = AtomicBool::new(false);
#[cfg(feature = "__test-internals")]
pub struct DerivedStageHoldForTest;
#[cfg(feature = "__test-internals")]
pub fn hold_derived_stage_for_test() -> DerivedStageHoldForTest {
DERIVED_STAGE_REACHED.store(false, Ordering::Release);
DERIVED_STAGE_HOLD.store(true, Ordering::Release);
DerivedStageHoldForTest
}
#[cfg(feature = "__test-internals")]
impl DerivedStageHoldForTest {
pub fn wait_until_blocked(&self, timeout: Duration) -> bool {
let deadline = Instant::now() + timeout;
while Instant::now() < deadline {
if DERIVED_STAGE_REACHED.load(Ordering::Acquire) {
return true;
}
std::thread::sleep(Duration::from_millis(1));
}
DERIVED_STAGE_REACHED.load(Ordering::Acquire)
}
}
#[cfg(feature = "__test-internals")]
impl Drop for DerivedStageHoldForTest {
fn drop(&mut self) {
DERIVED_STAGE_HOLD.store(false, Ordering::Release);
}
}
fn pause_between_derived_stages(cancel: Option<&AtomicBool>) -> anyhow::Result<()> {
#[cfg(feature = "__test-internals")]
if DERIVED_STAGE_HOLD.load(Ordering::Acquire) {
DERIVED_STAGE_REACHED.store(true, Ordering::Release);
loop {
check_reload_cancelled(cancel)?;
std::thread::sleep(Duration::from_millis(1));
}
}
let _ = cancel;
Ok(())
}
pub(crate) struct ReloadProgressSink {
files_discovered: AtomicU64,
files_parsed: AtomicU64,
symbols_found: AtomicU64,
}
impl ReloadProgressSink {
pub(crate) fn new() -> Arc<Self> {
Arc::new(Self {
files_discovered: AtomicU64::new(0),
files_parsed: AtomicU64::new(0),
symbols_found: AtomicU64::new(0),
})
}
pub(crate) fn reset(&self) {
self.files_discovered.store(0, Ordering::Release);
self.files_parsed.store(0, Ordering::Release);
self.symbols_found.store(0, Ordering::Release);
}
pub(crate) fn snapshot(&self) -> (u64, u64, u64) {
(
self.files_discovered.load(Ordering::Acquire),
self.files_parsed.load(Ordering::Acquire),
self.symbols_found.load(Ordering::Acquire),
)
}
}
static RELOAD_PROGRESS: OnceLock<std::sync::Mutex<HashMap<PathBuf, Arc<ReloadProgressSink>>>> =
OnceLock::new();
pub(crate) struct ReloadProgressGuard {
root: PathBuf,
}
impl Drop for ReloadProgressGuard {
fn drop(&mut self) {
if let Some(map) = RELOAD_PROGRESS.get() {
map.lock().expect("reload progress").remove(&self.root);
}
}
}
pub(crate) fn register_reload_progress(
root: &Path,
sink: Arc<ReloadProgressSink>,
) -> ReloadProgressGuard {
let root = normalize_root(root);
RELOAD_PROGRESS
.get_or_init(|| std::sync::Mutex::new(HashMap::new()))
.lock()
.expect("reload progress")
.insert(root.clone(), sink);
ReloadProgressGuard { root }
}
fn reload_progress_for(root: &Path) -> Option<Arc<ReloadProgressSink>> {
let root = normalize_root(root);
RELOAD_PROGRESS
.get()
.and_then(|map| map.lock().ok()?.get(&root).cloned())
}
fn note_reload_discovered(root: &Path, count: u64) {
if let Some(sink) = reload_progress_for(root) {
sink.files_discovered.store(count, Ordering::Release);
}
}
fn note_reload_parsed(root: &Path) {
if let Some(sink) = reload_progress_for(root) {
sink.files_parsed.fetch_add(1, Ordering::AcqRel);
}
}
fn note_reload_symbols(root: &Path, count: u64) {
if let Some(sink) = reload_progress_for(root) {
sink.symbols_found.fetch_add(count, Ordering::AcqRel);
}
}
#[cfg(feature = "__test-internals")]
struct CensusHoldState {
blocked: std::sync::Mutex<bool>,
reached: std::sync::Condvar,
released: AtomicBool,
}
#[cfg(feature = "__test-internals")]
static CENSUS_HOLDS: OnceLock<std::sync::Mutex<HashMap<PathBuf, Arc<CensusHoldState>>>> =
OnceLock::new();
#[cfg(feature = "__test-internals")]
pub struct CensusHoldForTest {
root: PathBuf,
state: Arc<CensusHoldState>,
}
#[cfg(feature = "__test-internals")]
pub fn hold_census_after_first_read_for_test(root: &Path) -> CensusHoldForTest {
let root = normalize_root(root);
let state = Arc::new(CensusHoldState {
blocked: std::sync::Mutex::new(false),
reached: std::sync::Condvar::new(),
released: AtomicBool::new(false),
});
let previous = CENSUS_HOLDS
.get_or_init(|| std::sync::Mutex::new(HashMap::new()))
.lock()
.expect("census hold registry")
.insert(root.clone(), Arc::clone(&state));
assert!(previous.is_none(), "a root can hold only one census gate");
CensusHoldForTest { root, state }
}
#[cfg(feature = "__test-internals")]
impl CensusHoldForTest {
pub fn wait_until_blocked(&self, timeout: Duration) -> bool {
let blocked = self.state.blocked.lock().expect("census hold state");
let (blocked, _) = self
.state
.reached
.wait_timeout_while(blocked, timeout, |blocked| !*blocked)
.expect("census hold state");
*blocked
}
pub fn release(&self) {
self.state.released.store(true, Ordering::Release);
self.state.reached.notify_all();
}
}
#[cfg(feature = "__test-internals")]
impl Drop for CensusHoldForTest {
fn drop(&mut self) {
self.release();
let holds = CENSUS_HOLDS.get_or_init(|| std::sync::Mutex::new(HashMap::new()));
let mut holds = holds.lock().expect("census hold registry");
if holds
.get(&self.root)
.is_some_and(|current| Arc::ptr_eq(current, &self.state))
{
holds.remove(&self.root);
}
}
}
pub(crate) fn pause_census_after_first_read_for_test(root: &Path) {
#[cfg(feature = "__test-internals")]
{
let root = normalize_root(root);
let Some(state) = CENSUS_HOLDS
.get()
.and_then(|holds| holds.lock().ok()?.get(&root).cloned())
else {
return;
};
let mut blocked = state.blocked.lock().expect("census hold state");
*blocked = true;
state.reached.notify_all();
while !state.released.load(Ordering::Acquire) {
let (next, _) = state
.reached
.wait_timeout(blocked, Duration::from_millis(5))
.expect("census hold state");
blocked = next;
}
}
#[cfg(not(feature = "__test-internals"))]
{
let _ = root;
}
}
#[cfg(windows)]
const INDEXING_THREAD_STACK_SIZE_ENV: &str = "SYMFORGE_INDEXING_THREAD_STACK_BYTES";
#[cfg(windows)]
const DEFAULT_INDEXING_THREAD_STACK_BYTES: usize = 4 * 1024 * 1024;
#[cfg(windows)]
const MIN_INDEXING_THREAD_STACK_BYTES: usize = 3 * 1024 * 1024;
static INDEXING_THREAD_POOL: OnceLock<rayon::ThreadPool> = OnceLock::new();
#[cfg(windows)]
fn indexing_thread_stack_size() -> usize {
match std::env::var(INDEXING_THREAD_STACK_SIZE_ENV) {
Ok(raw) => match raw.parse::<usize>() {
Ok(bytes) if bytes >= MIN_INDEXING_THREAD_STACK_BYTES => bytes,
Ok(bytes) => {
warn!(
env = INDEXING_THREAD_STACK_SIZE_ENV,
requested = bytes,
minimum = MIN_INDEXING_THREAD_STACK_BYTES,
"indexing thread stack size too small; using Windows minimum"
);
MIN_INDEXING_THREAD_STACK_BYTES
}
Err(error) => {
warn!(
env = INDEXING_THREAD_STACK_SIZE_ENV,
value = %raw,
%error,
default = DEFAULT_INDEXING_THREAD_STACK_BYTES,
"invalid indexing thread stack size; using default"
);
DEFAULT_INDEXING_THREAD_STACK_BYTES
}
},
Err(_) => DEFAULT_INDEXING_THREAD_STACK_BYTES,
}
}
const INDEXING_THREADS_ENV: &str = "SYMFORGE_INDEXING_THREADS";
fn indexing_thread_count() -> Option<usize> {
let raw = std::env::var(INDEXING_THREADS_ENV).ok()?;
match raw.trim().parse::<usize>() {
Ok(count) if count >= 1 => Some(count),
_ => {
warn!(
env = INDEXING_THREADS_ENV,
value = %raw,
"ignoring invalid indexing thread count; using rayon default"
);
None
}
}
}
fn indexing_thread_pool() -> &'static rayon::ThreadPool {
INDEXING_THREAD_POOL.get_or_init(|| {
let builder = rayon::ThreadPoolBuilder::new()
.thread_name(|index| format!("symforge-index-{}", index));
let builder = match indexing_thread_count() {
Some(count) => {
info!(
count,
env = INDEXING_THREADS_ENV,
"initializing indexing thread pool with explicit worker count"
);
builder.num_threads(count)
}
None => builder,
};
#[cfg(windows)]
let builder = {
let stack_size = indexing_thread_stack_size();
info!(
stack_size,
env = INDEXING_THREAD_STACK_SIZE_ENV,
"initializing indexing thread pool with explicit worker stack size"
);
builder.stack_size(stack_size)
};
builder
.build()
.expect("indexing thread pool should initialize")
})
}
const MAX_INFLIGHT_BYTES_ENV: &str = "SYMFORGE_MAX_INFLIGHT_BYTES";
const DEFAULT_MAX_INFLIGHT_BYTES: u64 = 512 * 1024 * 1024;
struct InflightByteBudget {
total: u64,
state: std::sync::Mutex<u64>,
available: std::sync::Condvar,
}
impl InflightByteBudget {
fn new(total: u64) -> Self {
let total = total.max(1);
Self {
total,
state: std::sync::Mutex::new(total),
available: std::sync::Condvar::new(),
}
}
fn from_env() -> Self {
let total = std::env::var(MAX_INFLIGHT_BYTES_ENV)
.ok()
.and_then(|v| v.trim().parse::<u64>().ok())
.filter(|&v| v > 0)
.unwrap_or(DEFAULT_MAX_INFLIGHT_BYTES);
Self::new(total)
}
fn acquire(self: &Arc<Self>, bytes: u64) -> InflightPermit {
let want = bytes.min(self.total).max(1);
let mut available = self.state.lock().expect("inflight budget mutex poisoned");
while *available < want {
available = self
.available
.wait(available)
.expect("inflight budget condvar poisoned");
}
*available -= want;
InflightPermit {
budget: Arc::clone(self),
held: want,
}
}
#[cfg(test)]
fn available_bytes(&self) -> u64 {
*self.state.lock().expect("inflight budget mutex poisoned")
}
}
pub fn configured_inflight_byte_budget() -> u64 {
InflightByteBudget::from_env().total
}
struct InflightPermit {
budget: Arc<InflightByteBudget>,
held: u64,
}
impl Drop for InflightPermit {
fn drop(&mut self) {
let mut available = self
.budget
.state
.lock()
.expect("inflight budget mutex poisoned");
*available += self.held;
self.budget.available.notify_all();
}
}
struct StagedContentAccounting {
ceiling: u64,
used: AtomicU64,
}
impl StagedContentAccounting {
fn new(ceiling: u64) -> Self {
Self {
ceiling,
used: AtomicU64::new(0),
}
}
fn handoff(&self, bytes: u64, permit: Option<InflightPermit>) -> bool {
let reserved = self
.used
.fetch_update(Ordering::AcqRel, Ordering::Acquire, |used| {
used.checked_add(bytes).filter(|next| *next <= self.ceiling)
})
.is_ok();
drop(permit);
reserved
}
#[cfg(test)]
fn used_bytes(&self) -> u64 {
self.used.load(Ordering::Acquire)
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
struct StableReadLimits {
per_file_bytes: u64,
inflight_bytes: u64,
}
#[derive(Clone, Debug, PartialEq, Eq)]
struct StableReadPass {
bytes: Option<Vec<u8>>,
length: u64,
hash: [u8; 32],
handle_before: crate::domain::FileStamp,
handle_after: crate::domain::FileStamp,
path_after: crate::domain::FileStamp,
}
trait StableReadAccess {
fn first_pass(&self, path: &Path, max_bytes: usize) -> std::io::Result<StableReadPass>;
fn second_pass(&self, path: &Path, max_bytes: usize) -> std::io::Result<StableReadPass>;
}
struct FilesystemStableReadAccess;
fn file_stamp_from_metadata(metadata: &std::fs::Metadata) -> crate::domain::FileStamp {
crate::domain::FileStamp {
size: metadata.len(),
created_hint: metadata.created().ok(),
modified_hint: metadata.modified().ok(),
platform_id: None,
}
}
impl FilesystemStableReadAccess {
fn read_pass(
path: &Path,
max_bytes: usize,
retain_bytes: bool,
) -> std::io::Result<StableReadPass> {
use sha2::{Digest, Sha256};
use std::io::Read;
let mut file = std::fs::File::open(path)?;
let handle_before = file_stamp_from_metadata(&file.metadata()?);
if usize::try_from(handle_before.size).map_or(true, |size| size > max_bytes) {
return Err(std::io::Error::new(
std::io::ErrorKind::InvalidData,
"stable read exceeded its pre-authorized bound",
));
}
let mut bytes = retain_bytes.then(Vec::new);
if let Some(bytes) = bytes.as_mut() {
bytes
.try_reserve_exact(handle_before.size as usize)
.map_err(|_| std::io::Error::other("stable read reservation failed"))?;
}
let mut hasher = Sha256::new();
let mut length = 0_u64;
let mut buffer = [0_u8; 64 * 1024];
loop {
let read = file.read(&mut buffer)?;
if read == 0 {
break;
}
length = length
.checked_add(read as u64)
.ok_or_else(|| std::io::Error::other("stable read length overflow"))?;
if length > max_bytes as u64 {
return Err(std::io::Error::new(
std::io::ErrorKind::InvalidData,
"stable read exceeded its pre-authorized bound",
));
}
hasher.update(&buffer[..read]);
if let Some(bytes) = bytes.as_mut() {
bytes.extend_from_slice(&buffer[..read]);
}
}
let handle_after = file_stamp_from_metadata(&file.metadata()?);
let path_after = file_stamp_from_metadata(&std::fs::metadata(path)?);
Ok(StableReadPass {
bytes,
length,
hash: hasher.finalize().into(),
handle_before,
handle_after,
path_after,
})
}
}
impl StableReadAccess for FilesystemStableReadAccess {
fn first_pass(&self, path: &Path, max_bytes: usize) -> std::io::Result<StableReadPass> {
Self::read_pass(path, max_bytes, true)
}
fn second_pass(&self, path: &Path, max_bytes: usize) -> std::io::Result<StableReadPass> {
Self::read_pass(path, max_bytes, false)
}
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub(crate) enum StableReadOutcome {
Accepted {
bytes: Vec<u8>,
hash: [u8; 32],
},
HardSkip {
reason: HardSkipReason,
},
Unreadable {
stage: crate::domain::AccessStage,
kind: crate::domain::AccessErrorKind,
},
UnstableDuringRead,
}
fn stable_read_with_access(
path: &Path,
scout_stamp: &crate::domain::FileStamp,
limits: StableReadLimits,
access: &impl StableReadAccess,
) -> StableReadOutcome {
let Ok(max_bytes) = usize::try_from(scout_stamp.size) else {
return StableReadOutcome::HardSkip {
reason: HardSkipReason::PerFileCeiling,
};
};
if scout_stamp.size > limits.per_file_bytes || scout_stamp.size > limits.inflight_bytes {
return StableReadOutcome::HardSkip {
reason: HardSkipReason::PerFileCeiling,
};
}
let first = match access.first_pass(path, max_bytes) {
Ok(first) => first,
Err(error) if error.kind() == std::io::ErrorKind::InvalidData => {
return StableReadOutcome::UnstableDuringRead;
}
Err(error) => {
return StableReadOutcome::Unreadable {
stage: crate::domain::AccessStage::FullRead,
kind: discovery::access_error_kind(error.kind()),
};
}
};
if &first.handle_before != scout_stamp
|| &first.handle_after != scout_stamp
|| &first.path_after != scout_stamp
{
return StableReadOutcome::UnstableDuringRead;
}
let Some(bytes) = first.bytes else {
return StableReadOutcome::UnstableDuringRead;
};
if first.length != scout_stamp.size
|| usize::try_from(first.length).ok() != Some(bytes.len())
|| first.hash != crate::hash::digest(&bytes)
{
return StableReadOutcome::UnstableDuringRead;
}
let second = match access.second_pass(path, max_bytes) {
Ok(second) => second,
Err(error) if error.kind() == std::io::ErrorKind::InvalidData => {
return StableReadOutcome::UnstableDuringRead;
}
Err(error) => {
return StableReadOutcome::Unreadable {
stage: crate::domain::AccessStage::FullRead,
kind: discovery::access_error_kind(error.kind()),
};
}
};
if &second.handle_before != scout_stamp
|| &second.handle_after != scout_stamp
|| &second.path_after != scout_stamp
|| second.length != first.length
|| second.hash != first.hash
{
return StableReadOutcome::UnstableDuringRead;
}
StableReadOutcome::Accepted {
bytes,
hash: first.hash,
}
}
fn stable_read_with_retries(
path: &Path,
scout_stamp: &crate::domain::FileStamp,
limits: StableReadLimits,
access: &impl StableReadAccess,
) -> StableReadOutcome {
const MAX_ATTEMPTS: usize = 3;
for _ in 0..MAX_ATTEMPTS {
let outcome = stable_read_with_access(path, scout_stamp, limits, access);
if !matches!(outcome, StableReadOutcome::UnstableDuringRead) {
return outcome;
}
}
StableReadOutcome::UnstableDuringRead
}
pub(crate) fn stable_read_file(
path: &Path,
scout_stamp: &crate::domain::FileStamp,
) -> StableReadOutcome {
stable_read_with_retries(
path,
scout_stamp,
StableReadLimits {
per_file_bytes: crate::domain::index::HARD_SKIP_BYTES,
inflight_bytes: InflightByteBudget::from_env().total,
},
&FilesystemStableReadAccess,
)
}
#[derive(Clone, Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
pub enum ParseStatus {
Parsed,
PartialParse { warning: String },
Failed { error: String },
}
#[derive(Clone, Debug)]
pub struct IndexedFile {
pub relative_path: String,
pub language: LanguageId,
pub classification: FileClassification,
pub content: Vec<u8>,
pub symbols: Vec<SymbolRecord>,
pub parse_status: ParseStatus,
pub parse_diagnostic: Option<ParseDiagnostic>,
pub byte_len: u64,
pub content_hash: String,
pub references: Vec<ReferenceRecord>,
pub alias_map: HashMap<String, String>,
pub mtime_secs: u64,
}
#[derive(Clone, Debug)]
pub struct ReferenceLocation {
pub file_path: String,
pub reference_idx: u32,
}
impl IndexedFile {
pub fn from_parse_result(result: FileProcessingResult, content: Vec<u8>) -> Self {
let parse_status = match &result.outcome {
FileOutcome::Processed => ParseStatus::Parsed,
FileOutcome::PartialParse { warning } => ParseStatus::PartialParse {
warning: warning.clone(),
},
FileOutcome::Failed { error } => ParseStatus::Failed {
error: error.clone(),
},
};
let FileProcessingResult {
relative_path,
language,
classification,
outcome: _,
parse_diagnostic,
symbols,
byte_len,
content_hash,
references: raw_references,
alias_map,
} = result;
let symbol_byte_ranges: std::collections::HashSet<(u32, u32)> =
symbols.iter().map(|s| s.byte_range).collect();
let references: Vec<ReferenceRecord> = raw_references
.into_iter()
.filter(|r| !symbol_byte_ranges.contains(&r.byte_range))
.map(|mut r| {
if r.enclosing_symbol_index.is_none() {
r.enclosing_symbol_index = find_enclosing_symbol(&symbols, r.line_range.0);
}
r
})
.collect();
IndexedFile {
relative_path,
language,
classification,
content,
symbols,
parse_status,
parse_diagnostic,
byte_len,
content_hash,
references,
alias_map,
mtime_secs: 0,
}
}
pub fn with_mtime(mut self, mtime_secs: u64) -> Self {
self.mtime_secs = mtime_secs;
self
}
}
impl AsRef<IndexedFile> for IndexedFile {
fn as_ref(&self) -> &IndexedFile {
self
}
}
pub struct CircuitBreakerState {
total: AtomicUsize,
failed: AtomicUsize,
tripped: AtomicBool,
threshold: f64,
failure_details: Mutex<Vec<(String, String)>>,
}
impl Clone for CircuitBreakerState {
fn clone(&self) -> Self {
Self {
total: AtomicUsize::new(self.total.load(Ordering::Relaxed)),
failed: AtomicUsize::new(self.failed.load(Ordering::Relaxed)),
tripped: AtomicBool::new(self.tripped.load(Ordering::Relaxed)),
threshold: self.threshold,
failure_details: Mutex::new(self.failure_details.lock().clone()),
}
}
}
impl CircuitBreakerState {
pub fn new(threshold: f64) -> Self {
Self {
total: AtomicUsize::new(0),
failed: AtomicUsize::new(0),
tripped: AtomicBool::new(false),
threshold,
failure_details: Mutex::new(Vec::new()),
}
}
pub fn from_env() -> Self {
let threshold = std::env::var("SYMFORGE_CB_THRESHOLD")
.ok()
.and_then(|v| v.parse::<f64>().ok())
.unwrap_or(0.20);
Self::new(threshold)
}
pub fn record_success(&self) {
self.total.fetch_add(1, Ordering::Relaxed);
}
pub fn record_failure(&self, path: &str, reason: &str) {
self.total.fetch_add(1, Ordering::Relaxed);
self.failed.fetch_add(1, Ordering::Relaxed);
let mut details = self.failure_details.lock();
if details.len() < 5 {
details.push((path.to_string(), reason.to_string()));
}
}
pub fn should_abort(&self) -> bool {
let total = self.total.load(Ordering::Relaxed);
if total < 5 {
return false;
}
let failed = self.failed.load(Ordering::Relaxed);
let rate = failed as f64 / total as f64;
if rate > self.threshold {
self.tripped.store(true, Ordering::Relaxed);
true
} else {
false
}
}
pub fn is_tripped(&self) -> bool {
self.tripped.load(Ordering::Relaxed)
}
pub fn summary(&self) -> String {
let total = self.total.load(Ordering::Relaxed);
let failed = self.failed.load(Ordering::Relaxed);
let rate = if total > 0 {
(failed as f64 / total as f64 * 100.0) as u32
} else {
0
};
let details = self.failure_details.lock();
let top_failures: Vec<String> = details
.iter()
.take(3)
.map(|(p, r)| format!(" - {p}: {r}"))
.collect();
let mut msg = format!(
"circuit breaker tripped: {failed}/{total} files failed ({rate}% > {}%)",
(self.threshold * 100.0) as u32
);
if !top_failures.is_empty() {
msg.push_str("\nTop failures:\n");
msg.push_str(&top_failures.join("\n"));
}
msg
}
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub enum IndexState {
Empty,
Loading,
Ready,
CircuitBreakerTripped {
summary: String,
},
}
#[derive(Copy, Clone, Debug, PartialEq, Eq)]
pub enum IndexLoadSource {
EmptyBootstrap,
FreshLoad,
SnapshotRestore,
}
impl IndexLoadSource {
pub fn label(self) -> &'static str {
match self {
Self::EmptyBootstrap => "empty_bootstrap",
Self::FreshLoad => "fresh_load",
Self::SnapshotRestore => "snapshot_restore",
}
}
}
const SNAPSHOT_VERIFY_MISMATCH_PATH_LIMIT: usize = 10;
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct SnapshotVerifyReport {
pub mismatch_count: usize,
pub mismatched_paths: Vec<String>,
pub reason: Option<String>,
pub unverified: Arc<BTreeMap<String, String>>,
pub discovery_error: Option<String>,
}
impl SnapshotVerifyReport {
pub fn from_mismatched_paths(mut paths: Vec<String>) -> Self {
paths.sort();
paths.dedup();
let mismatch_count = paths.len();
paths.truncate(SNAPSHOT_VERIFY_MISMATCH_PATH_LIMIT);
Self {
mismatch_count,
mismatched_paths: paths,
reason: None,
unverified: Arc::default(),
discovery_error: None,
}
}
pub fn with_reason(mut self, reason: String) -> Self {
if !reason.is_empty() {
self.reason = Some(reason);
}
self
}
pub fn with_unverified(mut self, unverified: BTreeMap<String, String>) -> Self {
self.unverified = Arc::new(unverified);
self
}
pub fn with_discovery_error(mut self, discovery_error: Option<String>) -> Self {
self.discovery_error = discovery_error;
self
}
pub fn empty() -> Self {
Self {
mismatch_count: 0,
mismatched_paths: Vec::new(),
reason: None,
unverified: Arc::default(),
discovery_error: None,
}
}
pub fn has_mismatches(&self) -> bool {
self.mismatch_count > 0 || self.discovery_error.is_some()
}
fn drop_unverified(&mut self, path: &str) -> bool {
if !self.unverified.contains_key(path) {
return false;
}
Arc::make_mut(&mut self.unverified).remove(path);
self.mismatch_count = self.mismatch_count.saturating_sub(1);
self.mismatched_paths
.retain(|mismatched| mismatched != path);
true
}
pub fn omitted_path_count(&self) -> usize {
self.mismatch_count
.saturating_sub(self.mismatched_paths.len())
}
}
pub(crate) struct SnapshotVerifyCompletion {
pub report: SnapshotVerifyReport,
pub restored_rows: Vec<(String, Option<Arc<IndexedFile>>)>,
}
impl From<SnapshotVerifyReport> for SnapshotVerifyCompletion {
fn from(report: SnapshotVerifyReport) -> Self {
Self {
report,
restored_rows: Vec::new(),
}
}
}
const SNAPSHOT_VERIFY_PHASE_STAT_PASS: u8 = 0;
const SNAPSHOT_VERIFY_PHASE_REVERIFY: u8 = 1;
const SNAPSHOT_VERIFY_PHASE_PUBLISH: u8 = 2;
const SNAPSHOT_VERIFY_PHASE_SPOT_CHECK: u8 = 3;
#[derive(Clone, Debug)]
pub struct SnapshotVerifyProgress(Arc<SnapshotVerifyCounters>);
#[derive(Debug)]
struct SnapshotVerifyCounters {
started_at: SystemTime,
restored_files: usize,
phase: std::sync::atomic::AtomicU8,
total: AtomicUsize,
processed: AtomicUsize,
}
impl PartialEq for SnapshotVerifyProgress {
fn eq(&self, other: &Self) -> bool {
Arc::ptr_eq(&self.0, &other.0)
}
}
impl Eq for SnapshotVerifyProgress {}
impl SnapshotVerifyProgress {
pub fn started(started_at: SystemTime, restored_files: usize) -> Self {
Self(Arc::new(SnapshotVerifyCounters {
started_at,
restored_files,
phase: std::sync::atomic::AtomicU8::new(SNAPSHOT_VERIFY_PHASE_STAT_PASS),
total: AtomicUsize::new(0),
processed: AtomicUsize::new(0),
}))
}
pub fn started_at(&self) -> SystemTime {
self.0.started_at
}
pub fn start_spot_check(&self, total: usize) {
self.0.total.store(total, Ordering::Release);
self.0.processed.store(0, Ordering::Release);
self.0
.phase
.store(SNAPSHOT_VERIFY_PHASE_SPOT_CHECK, Ordering::Release);
}
pub fn start_reverify(&self, total: usize) {
self.0.processed.store(0, Ordering::Release);
self.0.total.store(total, Ordering::Release);
self.0
.phase
.store(SNAPSHOT_VERIFY_PHASE_REVERIFY, Ordering::Release);
}
pub fn add_processed(&self, files: usize) {
self.0.processed.fetch_add(files, Ordering::AcqRel);
}
pub fn enter_publish(&self) {
self.0
.phase
.store(SNAPSHOT_VERIFY_PHASE_PUBLISH, Ordering::Release);
}
pub fn describe(&self) -> String {
let elapsed = self
.0
.started_at
.elapsed()
.map(|elapsed| elapsed.as_secs())
.unwrap_or(0);
let restored = self.0.restored_files;
let processed = self.0.processed.load(Ordering::Acquire);
let total = self.0.total.load(Ordering::Acquire);
match self.0.phase.load(Ordering::Acquire) {
SNAPSHOT_VERIFY_PHASE_STAT_PASS => {
format!("phase=stat_pass restored_files={restored} elapsed={elapsed}s")
}
SNAPSHOT_VERIFY_PHASE_SPOT_CHECK => format!(
"phase=spot_check processed={processed}/{total} restored_files={restored} \
elapsed={elapsed}s"
),
SNAPSHOT_VERIFY_PHASE_REVERIFY => format!(
"phase=reverify processed={processed}/{total} restored_files={restored} \
elapsed={elapsed}s"
),
_ => format!(
"phase=publish processed={processed}/{total} restored_files={restored} \
elapsed={elapsed}s"
),
}
}
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub enum SnapshotVerifyState {
NotNeeded,
Pending,
Running(SnapshotVerifyProgress),
Completed(SnapshotVerifyReport),
Failed(SnapshotVerifyReport),
}
impl SnapshotVerifyState {
pub fn completed_without_mismatches() -> Self {
Self::Completed(SnapshotVerifyReport::empty())
}
pub fn completed_with_mismatches(paths: Vec<String>) -> Self {
Self::Completed(SnapshotVerifyReport::from_mismatched_paths(paths))
}
}
#[derive(Copy, Clone, Debug, PartialEq, Eq)]
pub enum PublishedIndexStatus {
Empty,
Loading,
Ready,
Degraded,
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct PublishedIndexState {
pub generation: u64,
pub status: PublishedIndexStatus,
pub degraded_summary: Option<String>,
pub file_count: usize,
pub parsed_count: usize,
pub partial_parse_count: usize,
pub unexpected_partial_parse_count: usize,
pub expected_vendor_partial_parse_count: usize,
pub expected_generated_partial_parse_count: usize,
pub expected_test_fixture_partial_parse_count: usize,
pub expected_template_dsl_partial_parse_count: usize,
pub expected_framework_partial_parse_count: usize,
pub expected_language_partial_parse_count: usize,
pub failed_count: usize,
pub partial_parse_files: Vec<String>,
pub unexpected_partial_parse_files: Vec<String>,
pub expected_vendor_partial_parse_files: Vec<String>,
pub expected_generated_partial_parse_files: Vec<String>,
pub expected_test_fixture_partial_parse_files: Vec<String>,
pub expected_template_dsl_partial_parse_files: Vec<String>,
pub expected_framework_partial_parse_files: Vec<String>,
pub expected_language_partial_parse_files: Vec<String>,
pub failed_files: Vec<(String, String)>,
pub symbol_count: usize,
pub loaded_at_system: SystemTime,
pub load_duration: Duration,
pub load_source: IndexLoadSource,
pub snapshot_verify_state: SnapshotVerifyState,
pub is_empty: bool,
pub tier_counts: (usize, usize, usize),
pub local_empty_reason: Option<String>,
pub untracked_indexed: usize,
pub indexed_root: Option<PathBuf>,
}
pub struct CodeSignalsSnapshot {
pub state: super::git_temporal::GitTemporalState,
pub temporal: Arc<super::git_temporal::GitTemporalIndex>,
pub computed_for_content_generation: u64,
pub computed_for_source_version: SourceVersion,
pub coverage: Arc<HistoryCoverage>,
}
pub struct PublishedGeneration {
pub publication_generation: u64,
pub content_generation: u64,
pub project_generation: u64,
pub source: Option<Arc<SourceIdentity>>,
pub source_version: Option<Arc<SourceVersion>>,
pub freshness: Arc<FreshnessStatus>,
pub manifest: Option<Arc<RepositoryManifest>>,
pub code_signals: Arc<CodeSignalsSnapshot>,
pub bridge: Arc<KnowledgeBridge>,
pub authority: Arc<KnowledgeAuthorityView>,
pub live: Arc<LiveIndex>,
pub health: Arc<PublishedIndexState>,
pub outline: Arc<RepoOutlineView>,
}
impl PublishedGeneration {
pub fn source_response_envelope(&self) -> Option<SourceResponseEnvelope> {
let manifest = self.manifest.as_ref()?;
Some(SourceResponseEnvelope {
source: self.source.as_ref()?.as_ref().clone(),
source_version: self.source_version.as_ref()?.as_ref().clone(),
publication_generation: self.publication_generation,
content_generation: self.content_generation,
freshness: self.freshness.as_ref().clone(),
manifest_digest: manifest.digest.clone(),
coverage: manifest.coverage,
})
}
}
pub struct PublishedSourceSet {
pub registry_generation: u64,
pub current_source_id: SourceId,
pub sources: BTreeMap<SourceId, Arc<PublishedGeneration>>,
}
impl PublishedSourceSet {
pub fn current_generation(&self) -> Arc<PublishedGeneration> {
Arc::clone(
self.sources
.get(&self.current_source_id)
.expect("published source set must contain its current source"),
)
}
fn next_after_current_publish(
&self,
current_source_id: SourceId,
current_generation: Arc<PublishedGeneration>,
) -> PublishedSourceSet {
let mut sources = self.sources.clone();
if self.current_source_id != current_source_id {
sources.remove(&self.current_source_id);
}
sources.insert(current_source_id.clone(), current_generation);
PublishedSourceSet {
registry_generation: self.registry_generation.saturating_add(1),
current_source_id,
sources,
}
}
}
fn unbound_source_id() -> SourceId {
SourceId::new("symforge:unbound-source")
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct PublicationFence {
pub publication_generation: u64,
pub content_generation: u64,
pub project_generation: u64,
}
impl PublicationFence {
pub(crate) fn from_published(published: &PublishedGeneration) -> Self {
Self {
publication_generation: published.publication_generation,
content_generation: published.content_generation,
project_generation: published.project_generation,
}
}
}
pub(crate) struct IndexedFilePublicationReceipt {
pub published: Arc<PublishedGeneration>,
pub snapshot_created: bool,
}
pub(crate) enum FencedRemoval {
Removed(Arc<PublishedGeneration>),
NothingHeld,
Rejected,
}
pub(crate) struct SnapshotVerifiedFile {
pub path: String,
pub scouted: crate::domain::ScoutedEntry,
pub admission: SnapshotVerifiedAdmission,
pub base_hash: Option<String>,
}
impl SnapshotVerifiedFile {
pub(crate) fn resident_bytes(&self) -> usize {
match &self.admission {
SnapshotVerifiedAdmission::Indexed { file, .. } => file.content.len(),
SnapshotVerifiedAdmission::Terminal(_) => 0,
}
}
}
pub(crate) enum SnapshotVerifiedAdmission {
Indexed {
file: Box<IndexedFile>,
targets: crate::domain::IndexTargets,
},
Terminal(FileDisposition),
}
#[derive(Debug, Default)]
pub(crate) struct SnapshotVerifyBatchReceipt {
pub removed: Vec<String>,
pub indexed: Vec<String>,
pub terminal: Vec<String>,
pub unchanged: Vec<String>,
pub superseded: Vec<String>,
pub refused: Vec<String>,
pub completed: bool,
}
#[cfg(test)]
pub(crate) fn snapshot_verify_test_panic_path() -> &'static parking_lot::Mutex<Option<String>> {
static PATH: parking_lot::Mutex<Option<String>> = parking_lot::Mutex::new(None);
&PATH
}
fn verify_in_flight(state: &SnapshotVerifyState) -> bool {
matches!(
state,
SnapshotVerifyState::Pending | SnapshotVerifyState::Running(_)
)
}
fn snapshot_verified_stamp_matches(scouted: &crate::domain::ScoutedEntry) -> bool {
let Some(path) = scouted.absolute_path.as_deref() else {
return false;
};
std::fs::metadata(path).is_ok_and(|meta| {
meta.len() == scouted.stamp.size && meta.modified().ok() == scouted.stamp.modified_hint
})
}
pub struct PreparedKnowledgeBridge {
fence: PublicationFence,
bridge: Arc<KnowledgeBridge>,
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct AuthorityPublicationFence {
pub publication: PublicationFence,
pub source: Option<SourceIdentity>,
pub source_version: Option<SourceVersion>,
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct GitTemporalPublicationFence {
pub project_generation: u64,
pub content_generation: u64,
pub source: Option<SourceIdentity>,
pub source_version: Option<SourceVersion>,
}
pub struct PreparedKnowledgeAuthority {
fence: AuthorityPublicationFence,
authority: Arc<KnowledgeAuthorityView>,
}
fn capture_published_manifest(
live: &LiveIndex,
scout_plan: Option<&discovery::ScoutPlan>,
) -> Option<Arc<RepositoryManifest>> {
let Some(root) = live.indexed_root.as_deref() else {
if !live.is_empty {
if live.load_source() == IndexLoadSource::EmptyBootstrap {
debug!(
files = live.files.len(),
local_empty_reason = live.local_empty_reason().is_some(),
"bootstrap placeholder published with no indexed_root; guarded as not-Ready"
);
} else {
warn!(
files = live.files.len(),
load_source = live.load_source().label(),
"publishing a non-empty index with no indexed_root; source identity unbound"
);
}
}
return None;
};
let canonical_root = match dunce::canonicalize(root) {
Ok(canonical_root) => canonical_root,
Err(error) => {
warn!(
root = %root.display(),
%error,
"failed to canonicalize the indexed root; publishing without source identity"
);
return None;
}
};
let project_id = crate::discovery::project_id_for_canonical_root(&canonical_root);
let captured = match super::persist::capture_repository_source(&canonical_root, &project_id) {
Ok(captured) => captured,
Err(error) => {
warn!(%error, "failed to capture published source identity");
return None;
}
};
let coverage = scout_plan.map_or_else(
|| {
if manifest_requires_degraded_coverage(live) {
CoverageStatus::Degraded
} else {
CoverageStatus::Complete
}
},
|plan| plan.coverage,
);
let issues = scout_plan
.map(|plan| plan.issues.clone())
.unwrap_or_default();
let usage = scout_plan
.map(|plan| plan.usage)
.unwrap_or_else(|| ManifestResourceUsage {
catalog_entries: live.manifest_entries.len() as u64,
catalog_metadata_bytes: serde_json::to_vec(&live.manifest_entries)
.map(|bytes| bytes.len() as u64)
.unwrap_or_default(),
admitted_content_bytes: live.files.values().map(|file| file.byte_len).sum(),
});
match RepositoryManifest::new(
1,
1,
crate::knowledge::SECRET_POLICY_VERSION,
captured.source,
captured.source_version,
coverage,
live.manifest_entries.clone(),
issues,
usage,
) {
Ok(manifest) => Some(Arc::new(manifest)),
Err(error) => {
warn!(%error, "failed to build canonical published manifest");
None
}
}
}
fn capture_history_coverage(
live: &LiveIndex,
temporal_state: &super::git_temporal::GitTemporalState,
) -> Arc<HistoryCoverage> {
let Some(root) = live.indexed_root.as_deref() else {
return Arc::new(HistoryCoverage {
complete_to_root: false,
limitations: vec![HistoryLimit::Unavailable],
});
};
let mut coverage = super::persist::capture_history_coverage(root);
let mut add_limit = |limit| {
if !coverage.limitations.contains(&limit) {
coverage.limitations.push(limit);
}
};
match temporal_state {
super::git_temporal::GitTemporalState::Ready => {
add_limit(HistoryLimit::WindowLimited);
add_limit(HistoryLimit::RenameFollowLimited);
}
super::git_temporal::GitTemporalState::Pending
| super::git_temporal::GitTemporalState::Computing
| super::git_temporal::GitTemporalState::Unavailable(_) => {
add_limit(HistoryLimit::Unavailable);
}
}
coverage.complete_to_root = false;
Arc::new(coverage)
}
fn build_published_authority(
live: &LiveIndex,
source: Option<&SourceIdentity>,
source_version: Option<&SourceVersion>,
content_generation: u64,
bridge: &KnowledgeBridge,
code_signals: &CodeSignalsSnapshot,
manifest: Option<&RepositoryManifest>,
) -> Arc<KnowledgeAuthorityView> {
let (Some(source), Some(source_version)) = (source, source_version) else {
return Arc::new(KnowledgeAuthorityView::default());
};
let temporal =
AuthorityTemporalIndex::from_components(live, Some(source_version), code_signals);
Arc::new(build_knowledge_authority(
live,
source,
source_version,
content_generation,
bridge,
&temporal,
manifest.map_or(crate::knowledge::SECRET_POLICY_VERSION, |manifest| {
manifest.secret_policy_version
}),
&AuthorityLimits::default(),
))
}
#[derive(Clone)]
pub struct LiveIndex {
pub(crate) files: HashMap<String, Arc<IndexedFile>>,
pub(crate) loaded_at: Instant,
pub(crate) loaded_at_system: SystemTime,
pub(crate) load_duration: Duration,
pub(crate) cb_state: CircuitBreakerState,
pub(crate) is_empty: bool,
pub(crate) load_source: IndexLoadSource,
pub(crate) snapshot_verify_state: SnapshotVerifyState,
pub(crate) reverse_index: HashMap<String, Vec<ReferenceLocation>>,
pub(crate) files_by_basename: HashMap<String, Vec<String>>,
pub(crate) files_by_dir_component: HashMap<String, Vec<String>>,
pub(crate) trigram_index: super::trigram::TrigramIndex,
pub(crate) gitignore: Option<ignore::gitignore::Gitignore>,
pub(crate) manifest_entries: Vec<CatalogEntry>,
pub(crate) coupling_store: Option<Arc<super::coupling::CouplingStore>>,
pub(crate) local_empty_reason: Arc<parking_lot::RwLock<Option<String>>>,
pub(crate) indexed_root: Option<PathBuf>,
}
#[derive(Clone, Debug)]
pub struct PreUpdateSnapshot {
pub content: Vec<u8>,
pub symbols: Vec<PreUpdateSymbol>,
}
#[derive(Clone, Debug)]
struct StoredPreUpdateSnapshot {
snapshot: PreUpdateSnapshot,
replacement: PublicationFence,
}
fn pre_update_snapshot(existing: &IndexedFile) -> PreUpdateSnapshot {
PreUpdateSnapshot {
content: existing.content.clone(),
symbols: existing
.symbols
.iter()
.map(|symbol| PreUpdateSymbol {
name: symbol.name.clone(),
kind: symbol.kind.to_string(),
line_range: symbol.line_range,
byte_range: symbol.byte_range,
})
.collect(),
}
}
#[derive(Clone, Debug)]
pub struct PreUpdateSymbol {
pub name: String,
pub kind: String,
pub line_range: (u32, u32),
pub byte_range: (u32, u32),
}
pub struct SharedIndexHandle {
live: ArcSwap<LiveIndex>,
published_source_set: ArcSwap<PublishedSourceSet>,
project_state_dir: ArcSwapOption<ProjectStateDir>,
source_exclusions: ArcSwap<discovery::SourceExclusions>,
scout_plan: ArcSwapOption<discovery::ScoutPlan>,
freshness_status: ArcSwap<FreshnessStatus>,
write_mutex: Mutex<()>,
impact_mutex: tokio::sync::Mutex<()>,
published_state: ArcSwap<PublishedIndexState>,
next_generation: AtomicU64,
project_generation: AtomicU64,
last_reset_project_generation: AtomicU64,
admitted_root: AtomicU64,
admitted_physical_anchor: parking_lot::Mutex<
Option<crate::live_index::index_lifecycle::physical_root::PhysicalRootAnchor>,
>,
physical_replacement_latched: std::sync::atomic::AtomicBool,
rejected_stale_mutations: AtomicU64,
git_temporal: ArcSwap<super::git_temporal::GitTemporalIndex>,
pub(super) git_temporal_jobs: Mutex<super::git_temporal::GitTemporalJobQueue>,
pre_update_snapshots: Mutex<HashMap<String, StoredPreUpdateSnapshot>>,
ref_reconcile_lock: Mutex<()>,
}
pub struct SharedIndexWriteGuard<'a> {
handle: &'a SharedIndexHandle,
_mutex: MutexGuard<'a, ()>,
index: Option<LiveIndex>,
dirty: bool,
}
fn live_path_changed_since_reload_watermark(
watermark: &LiveIndex,
file: &IndexedFile,
path: &str,
) -> bool {
watermark
.files
.get(path)
.is_none_or(|before| before.content_hash != file.content_hash)
}
fn merge_post_watermark_live_admissions(
watermark: &LiveIndex,
current: &LiveIndex,
candidate: &mut LiveIndex,
) -> anyhow::Result<()> {
for (path, file) in ¤t.files {
if !live_path_changed_since_reload_watermark(watermark, file, path) {
continue;
}
if candidate.files.contains_key(path) {
if candidate
.files
.get(path)
.is_some_and(|existing| existing.content_hash != file.content_hash)
{
anyhow::bail!(
"reload refused: live mutation for '{path}' during candidate build \
conflicts with the disk-built candidate"
);
}
continue;
}
let path_owned = path.clone();
let file_clone = (**file).clone();
let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
candidate.update_file(path_owned, file_clone);
}));
if result.is_err() {
anyhow::bail!(
"reload refused: failed to carry live mutation for '{path}' into candidate"
);
}
}
Ok(())
}
fn indexed_tree_absent_from_disk(watermark: &LiveIndex, root: &Path) -> bool {
!watermark.files.is_empty()
&& watermark
.files
.keys()
.all(|relative| !root.join(relative).exists())
}
impl SharedIndexHandle {
pub fn new(index: LiveIndex) -> Self {
Self::new_with_scout_plan(index, None)
}
fn new_with_scout_plan(
index: LiveIndex,
scout_plan: Option<Arc<discovery::ScoutPlan>>,
) -> Self {
Self::new_with_scout_plan_and_code_signals(index, scout_plan, None)
}
fn new_with_scout_plan_and_code_signals(
index: LiveIndex,
scout_plan: Option<Arc<discovery::ScoutPlan>>,
code_signals: Option<CodeSignalsSnapshot>,
) -> Self {
let publication_started = Instant::now();
let manifest = capture_published_manifest(&index, scout_plan.as_deref());
info!(
"publication/manifest in {:?}",
publication_started.elapsed()
);
let source = manifest
.as_ref()
.map(|manifest| Arc::new(manifest.source.clone()));
let source_version = manifest
.as_ref()
.map(|manifest| Arc::new(manifest.source_version.clone()));
let code_signals = Arc::new(code_signals.unwrap_or_else(|| {
let temporal = Arc::new(super::git_temporal::GitTemporalIndex::pending());
let coverage = capture_history_coverage(&index, &temporal.state);
CodeSignalsSnapshot {
state: temporal.state.clone(),
temporal,
computed_for_content_generation: 0,
computed_for_source_version: source_version.as_deref().cloned().unwrap_or(
SourceVersion {
branch: None,
commit: None,
working_tree: WorkingTreeState::Unknown,
},
),
coverage,
}
}));
let temporal = Arc::clone(&code_signals.temporal);
let admitted_root = index
.indexed_root
.as_deref()
.map(crate::live_index::index_lifecycle::activation::project_source_authority)
.map(|authority| authority.admission_root().as_u64())
.unwrap_or(0);
let admitted_physical_anchor = index.indexed_root.as_deref().and_then(
crate::live_index::index_lifecycle::physical_root::PhysicalRootAnchor::observe,
);
let freshness_status = if scout_plan
.as_deref()
.is_some_and(|plan| plan.coverage == crate::domain::CoverageStatus::Degraded)
{
FreshnessStatus::Degraded {
last_valid_content_generation: 0,
reason_codes: vec![FreshnessReason::ReconciliationPending],
}
} else if index.is_empty
|| !matches!(index.snapshot_verify_state, SnapshotVerifyState::NotNeeded)
{
FreshnessStatus::Verifying
} else {
FreshnessStatus::Current
};
let phase = Instant::now();
let published_state = Arc::new(PublishedIndexState::capture(0, &index));
info!("publication/state capture in {:?}", phase.elapsed());
let phase = Instant::now();
let published_repo_outline = Arc::new(index.capture_repo_outline_view());
info!("publication/repo outline in {:?}", phase.elapsed());
let phase = Instant::now();
let bridge = source
.as_deref()
.map(|source| {
Arc::new(build_knowledge_bridge(
&index,
source,
0,
&BridgeLimits::default(),
))
})
.unwrap_or_else(|| Arc::new(KnowledgeBridge::default()));
info!(
"publication/knowledge bridge ({} cards) in {:?}",
bridge.cards.len(),
phase.elapsed()
);
let phase = Instant::now();
let authority = build_published_authority(
&index,
source.as_deref(),
source_version.as_deref(),
0,
&bridge,
&code_signals,
manifest.as_deref(),
);
info!("publication/authority in {:?}", phase.elapsed());
let live = Arc::new(index);
let published_generation = Arc::new(PublishedGeneration {
publication_generation: 0,
content_generation: 0,
project_generation: 0,
source: source.clone(),
source_version,
freshness: Arc::new(freshness_status.clone()),
manifest,
code_signals,
bridge,
authority,
live: Arc::clone(&live),
health: Arc::clone(&published_state),
outline: Arc::clone(&published_repo_outline),
});
let current_source_id = source
.as_ref()
.map(|source| source.source_id.clone())
.unwrap_or_else(unbound_source_id);
let mut sources = BTreeMap::new();
sources.insert(current_source_id.clone(), published_generation);
let published_source_set = Arc::new(PublishedSourceSet {
registry_generation: 0,
current_source_id,
sources,
});
info!(
"index publication (manifest + bridge + authority) in {:?}",
publication_started.elapsed()
);
Self {
live: ArcSwap::new(live),
published_source_set: ArcSwap::new(published_source_set),
project_state_dir: ArcSwapOption::empty(),
source_exclusions: ArcSwap::new(Arc::new(discovery::SourceExclusions::default())),
scout_plan: ArcSwapOption::new(scout_plan),
freshness_status: ArcSwap::new(Arc::new(freshness_status)),
write_mutex: Mutex::new(()),
impact_mutex: tokio::sync::Mutex::new(()),
published_state: ArcSwap::new(published_state),
next_generation: AtomicU64::new(1),
project_generation: AtomicU64::new(0),
last_reset_project_generation: AtomicU64::new(0),
admitted_root: AtomicU64::new(admitted_root),
admitted_physical_anchor: Mutex::new(admitted_physical_anchor),
physical_replacement_latched: AtomicBool::new(false),
rejected_stale_mutations: AtomicU64::new(0),
git_temporal: ArcSwap::new(temporal),
git_temporal_jobs: Mutex::new(super::git_temporal::GitTemporalJobQueue::default()),
pre_update_snapshots: Mutex::new(HashMap::new()),
ref_reconcile_lock: Mutex::new(()),
}
}
pub(crate) fn try_lock_ref_reconcile(&self) -> Option<MutexGuard<'_, ()>> {
self.ref_reconcile_lock.try_lock()
}
pub(crate) fn build_ref_source_generation(
&self,
index: LiveIndex,
repository_id: crate::domain::index::RepositoryId,
ref_name: &str,
tip_commit: &str,
scout_coverage: crate::domain::CoverageStatus,
) -> Arc<PublishedGeneration> {
let source_id = crate::domain::index::SourceId::new(format!(
"symforge:git-ref:{}:{ref_name}",
repository_id.as_str()
));
let publication_generation = self.next_generation.fetch_add(1, Ordering::Relaxed);
let content_generation = {
let published = self.published_source_set.load_full();
match published.sources.get(&source_id) {
Some(previous) => {
let previous_tip = previous
.source_version
.as_ref()
.and_then(|version| version.commit.as_deref());
if previous_tip == Some(tip_commit) {
previous.content_generation
} else {
previous.content_generation.saturating_add(1)
}
}
None => 1,
}
};
let source = Arc::new(SourceIdentity {
repository_id,
source_id,
location: crate::domain::index::SourceLocation::GitRef {
name: ref_name.to_string(),
},
});
let source_version = Arc::new(SourceVersion {
branch: Some(ref_name.to_string()),
commit: Some(tip_commit.to_string()),
working_tree: WorkingTreeState::NotApplicable,
});
let temporal = Arc::new(super::git_temporal::GitTemporalIndex::pending());
let coverage = capture_history_coverage(&index, &temporal.state);
let code_signals = Arc::new(CodeSignalsSnapshot {
state: temporal.state.clone(),
temporal,
computed_for_content_generation: content_generation,
computed_for_source_version: (*source_version).clone(),
coverage,
});
let usage = ManifestResourceUsage {
catalog_entries: index.files.len() as u64,
catalog_metadata_bytes: 0,
admitted_content_bytes: index.files.values().map(|file| file.byte_len).sum(),
};
let manifest = RepositoryManifest::new(
1,
1,
crate::knowledge::SECRET_POLICY_VERSION,
(*source).clone(),
(*source_version).clone(),
scout_coverage,
Vec::new(),
Vec::new(),
usage,
)
.ok()
.map(Arc::new);
let freshness = if index.is_empty {
FreshnessStatus::Verifying
} else {
FreshnessStatus::Current
};
let published_state =
Arc::new(PublishedIndexState::capture(publication_generation, &index));
let outline = Arc::new(index.capture_repo_outline_view());
let bridge = Arc::new(build_knowledge_bridge(
&index,
&source,
content_generation,
&BridgeLimits::default(),
));
let authority = build_published_authority(
&index,
Some(&source),
Some(&source_version),
content_generation,
&bridge,
&code_signals,
manifest.as_deref(),
);
let live = Arc::new(index);
Arc::new(PublishedGeneration {
publication_generation,
content_generation,
project_generation: 0,
source: Some(source),
source_version: Some(source_version),
freshness: Arc::new(freshness),
manifest,
code_signals,
bridge,
authority,
live,
health: published_state,
outline,
})
}
pub(crate) fn publish_ref_source(&self, generation: Arc<PublishedGeneration>) {
let source_id = generation
.source
.as_ref()
.expect("ref-source generation carries a source identity")
.source_id
.clone();
let _guard = self.write_mutex.lock();
let current = self.published_source_set.load_full();
let mut sources = current.sources.clone();
sources.insert(source_id, generation);
self.published_source_set
.store(Arc::new(PublishedSourceSet {
registry_generation: current.registry_generation.saturating_add(1),
current_source_id: current.current_source_id.clone(),
sources,
}));
}
pub(crate) fn remove_ref_source(&self, source_id: &crate::domain::index::SourceId) -> bool {
let _guard = self.write_mutex.lock();
let current = self.published_source_set.load_full();
if source_id == ¤t.current_source_id || !current.sources.contains_key(source_id) {
return false;
}
let mut sources = current.sources.clone();
sources.remove(source_id);
self.published_source_set
.store(Arc::new(PublishedSourceSet {
registry_generation: current.registry_generation.saturating_add(1),
current_source_id: current.current_source_id.clone(),
sources,
}));
true
}
pub fn shared(index: LiveIndex) -> Arc<Self> {
Arc::new(Self::new(index))
}
#[cfg(test)]
pub(crate) fn shared_with_code_signals(
index: LiveIndex,
code_signals: CodeSignalsSnapshot,
) -> Arc<Self> {
Arc::new(Self::new_with_scout_plan_and_code_signals(
index,
None,
Some(code_signals),
))
}
pub(crate) fn shared_with_source_exclusions_and_code_signals(
index: LiveIndex,
source_exclusions: discovery::SourceExclusions,
code_signals: CodeSignalsSnapshot,
) -> Arc<Self> {
let handle = Self::new_with_scout_plan_and_code_signals(index, None, Some(code_signals));
handle.source_exclusions.store(Arc::new(source_exclusions));
Arc::new(handle)
}
fn shared_with_scout_plan(
index: LiveIndex,
scout_plan: discovery::ScoutPlan,
source_exclusions: discovery::SourceExclusions,
) -> Arc<Self> {
let handle = Self::new_with_scout_plan(index, Some(Arc::new(scout_plan)));
handle.source_exclusions.store(Arc::new(source_exclusions));
Arc::new(handle)
}
pub fn shared_for_state_placement(
index: LiveIndex,
root: &Path,
state_placement: &StatePlacement,
) -> Arc<Self> {
let handle = Self::new(index);
handle
.source_exclusions
.store(Arc::new(discovery::SourceExclusions::for_state_placement(
root,
state_placement,
)));
handle
.project_state_dir
.store(state_placement.directory().cloned().map(Arc::new));
Arc::new(handle)
}
pub fn shared_for_state_placement_with_code_signals(
index: LiveIndex,
root: &Path,
state_placement: &StatePlacement,
code_signals: CodeSignalsSnapshot,
) -> Arc<Self> {
let handle = Self::new_with_scout_plan_and_code_signals(index, None, Some(code_signals));
handle
.source_exclusions
.store(Arc::new(discovery::SourceExclusions::for_state_placement(
root,
state_placement,
)));
handle
.project_state_dir
.store(state_placement.directory().cloned().map(Arc::new));
Arc::new(handle)
}
#[must_use]
pub fn project_state_dir(&self) -> Option<Arc<ProjectStateDir>> {
self.project_state_dir.load_full()
}
#[must_use]
pub fn scout_plan(&self) -> Option<Arc<discovery::ScoutPlan>> {
self.scout_plan.load_full()
}
#[must_use]
pub(crate) fn source_exclusions(&self) -> Arc<discovery::SourceExclusions> {
self.source_exclusions.load_full()
}
fn recompute_freshness_locked(
&self,
live: &LiveIndex,
scout_plan: Option<&discovery::ScoutPlan>,
) -> bool {
let current = self.freshness_status.load_full();
let (last_valid_content_generation, reason_codes) = match current.as_ref() {
FreshnessStatus::Degraded {
last_valid_content_generation,
reason_codes,
} => (*last_valid_content_generation, reason_codes.as_slice()),
FreshnessStatus::Current | FreshnessStatus::Verifying => (
self.published_generation().content_generation,
&[] as &[FreshnessReason],
),
};
let observation_failed = live.manifest_entries.iter().any(|entry| {
matches!(
entry.disposition,
FileDisposition::Unreadable { .. } | FileDisposition::UnstableDuringRead
)
});
let reconciliation_pending =
scout_plan.is_some_and(|plan| plan.coverage == crate::domain::CoverageStatus::Degraded);
let snapshot_verification_failed =
matches!(
&live.snapshot_verify_state,
SnapshotVerifyState::Completed(report) if report.has_mismatches()
) || matches!(live.snapshot_verify_state, SnapshotVerifyState::Failed(_));
let mut next_reasons = Vec::new();
for reason in reason_codes.iter().copied().filter(|reason| {
!matches!(
reason,
FreshnessReason::ObservationFailed
| FreshnessReason::ReconciliationPending
| FreshnessReason::SnapshotVerificationFailed
)
}) {
if !next_reasons.contains(&reason) {
next_reasons.push(reason);
}
}
if observation_failed && !next_reasons.contains(&FreshnessReason::ObservationFailed) {
next_reasons.push(FreshnessReason::ObservationFailed);
}
if reconciliation_pending && !next_reasons.contains(&FreshnessReason::ReconciliationPending)
{
next_reasons.push(FreshnessReason::ReconciliationPending);
}
if snapshot_verification_failed
&& !next_reasons.contains(&FreshnessReason::SnapshotVerificationFailed)
{
next_reasons.push(FreshnessReason::SnapshotVerificationFailed);
}
let next = if next_reasons.is_empty() {
if matches!(
live.snapshot_verify_state,
SnapshotVerifyState::Pending | SnapshotVerifyState::Running(_)
) {
FreshnessStatus::Verifying
} else {
FreshnessStatus::Current
}
} else {
FreshnessStatus::Degraded {
last_valid_content_generation,
reason_codes: next_reasons,
}
};
if &next != current.as_ref() {
self.freshness_status.store(Arc::new(next));
true
} else {
false
}
}
fn scout_plan_with_entry_locked(
&self,
entry: crate::domain::ScoutedEntry,
live: &LiveIndex,
) -> anyhow::Result<Option<Arc<discovery::ScoutPlan>>> {
let Some(current) = self.scout_plan.load_full() else {
return Ok(None);
};
let mut plan = (*current).clone();
let entry_changed = match plan
.entries
.iter()
.position(|existing| existing.path == entry.path)
{
Some(position) if plan.entries[position] == entry => false,
Some(position) => {
plan.entries[position] = entry;
true
}
None => {
plan.entries.push(entry);
true
}
};
discovery::refresh_scout_plan(&mut plan)?;
if manifest_requires_degraded_coverage(live) {
plan.coverage = crate::domain::CoverageStatus::Degraded;
}
if !entry_changed && plan.coverage == current.coverage {
return Ok(None);
}
Ok(Some(Arc::new(plan)))
}
fn scout_plan_without_path_locked(
&self,
path: &str,
) -> anyhow::Result<Option<Arc<discovery::ScoutPlan>>> {
let Some(current) = self.scout_plan.load_full() else {
return Ok(None);
};
let mut plan = (*current).clone();
let entry_count = plan.entries.len();
plan.entries
.retain(|entry| entry.path.normalized_utf8.as_deref() != Some(path));
if plan.entries.len() == entry_count {
return Ok(None);
}
discovery::refresh_scout_plan(&mut plan)?;
Ok(Some(Arc::new(plan)))
}
pub(crate) fn publish_reconciled_scout_plan_at_generation(
&self,
baseline: Option<&discovery::ScoutPlan>,
mut scout_plan: discovery::ScoutPlan,
expected_gen: u64,
) -> bool {
let _wg = self.write_mutex.lock();
let current_gen = self.project_generation.load(Ordering::Acquire);
if current_gen != expected_gen {
self.rejected_stale_mutations
.fetch_add(1, Ordering::Relaxed);
tracing::trace!(
expected_gen,
current_gen,
"rejecting stale scout-plan publication"
);
return false;
}
if let Some(current) = self.scout_plan.load_full() {
let baseline_entries: HashMap<crate::domain::CatalogPath, crate::domain::ScoutedEntry> =
baseline
.map(|plan| {
plan.entries
.iter()
.cloned()
.map(|entry| (entry.path.clone(), entry))
.collect()
})
.unwrap_or_default();
let current_entries: HashMap<crate::domain::CatalogPath, crate::domain::ScoutedEntry> =
current
.entries
.iter()
.cloned()
.map(|entry| (entry.path.clone(), entry))
.collect();
let mut fresh_entries: HashMap<
crate::domain::CatalogPath,
crate::domain::ScoutedEntry,
> = scout_plan
.entries
.drain(..)
.map(|entry| (entry.path.clone(), entry))
.collect();
let changed_paths: HashSet<crate::domain::CatalogPath> = baseline_entries
.keys()
.chain(current_entries.keys())
.cloned()
.collect();
for path in changed_paths {
if current_entries.get(&path) == baseline_entries.get(&path) {
continue;
}
match current_entries.get(&path) {
Some(entry) => {
fresh_entries.insert(path, entry.clone());
}
None => {
fresh_entries.remove(&path);
}
}
}
scout_plan.entries = fresh_entries.into_values().collect();
}
if let Err(error) = discovery::refresh_scout_plan(&mut scout_plan) {
tracing::error!(%error, "failed to refresh reconciled scout-plan accounting");
return false;
}
if manifest_requires_degraded_coverage(&self.live.load_full()) {
scout_plan.coverage = crate::domain::CoverageStatus::Degraded;
}
let live = self.live.load_full();
let scout_plan = Arc::new(scout_plan);
let current_scout_plan = self.scout_plan.load_full();
let scout_plan_changed = match current_scout_plan.as_deref() {
Some(current) => {
current.coverage != scout_plan.coverage
|| current.entries != scout_plan.entries
|| current.issues != scout_plan.issues
|| current.usage != scout_plan.usage
}
None => true,
};
self.scout_plan.store(Some(Arc::clone(&scout_plan)));
let freshness_changed = self.recompute_freshness_locked(&live, Some(&scout_plan));
if scout_plan_changed || freshness_changed {
self.swap_and_publish_retaining_content((*live).clone());
}
true
}
pub(crate) fn terminal_dispositions(
&self,
) -> Arc<Vec<(String, crate::domain::FileDisposition)>> {
Arc::new(
self.published_generation()
.live
.manifest_entries
.iter()
.map(|entry| {
(
scouted_catalog_path(&entry.path).to_string(),
entry.disposition.clone(),
)
})
.collect(),
)
}
#[must_use]
pub fn freshness_status(&self) -> Arc<FreshnessStatus> {
Arc::clone(&self.published_generation().freshness)
}
pub(crate) fn latch_observer_gap(&self) {
let _wg = self.write_mutex.lock();
self.latch_observer_gap_locked();
}
pub(crate) fn latch_observer_gap_at_generation(
&self,
expected_project_generation: u64,
) -> bool {
let _wg = self.write_mutex.lock();
if self.published_generation().project_generation != expected_project_generation {
return false;
}
self.latch_observer_gap_locked();
true
}
fn latch_observer_gap_locked(&self) {
let current = self.freshness_status.load_full();
let (last_valid, mut reasons) = match current.as_ref() {
FreshnessStatus::Degraded {
last_valid_content_generation,
reason_codes,
} => (*last_valid_content_generation, reason_codes.clone()),
FreshnessStatus::Current | FreshnessStatus::Verifying => {
(self.published_generation().content_generation, Vec::new())
}
};
if reasons.contains(&FreshnessReason::WatcherUnavailable) {
return;
}
reasons.push(FreshnessReason::WatcherUnavailable);
self.freshness_status
.store(Arc::new(FreshnessStatus::Degraded {
last_valid_content_generation: last_valid,
reason_codes: reasons,
}));
let live = (*self.live.load_full()).clone();
self.swap_and_publish_retaining_content(live);
}
pub(crate) fn set_freshness_status(&self, status: FreshnessStatus) {
let _wg = self.write_mutex.lock();
self.freshness_status.store(Arc::new(status));
let live = (*self.live.load_full()).clone();
self.swap_and_publish_retaining_content(live);
}
pub fn read(&self) -> Arc<LiveIndex> {
Arc::clone(&self.published_generation().live)
}
pub(crate) async fn lock_impact_analysis(&self) -> tokio::sync::MutexGuard<'_, ()> {
self.impact_mutex.lock().await
}
pub fn published_generation(&self) -> Arc<PublishedGeneration> {
self.published_source_set.load_full().current_generation()
}
pub fn published_source_set(&self) -> Arc<PublishedSourceSet> {
self.published_source_set.load_full()
}
pub fn publication_fence(&self) -> PublicationFence {
let published = self.published_generation();
PublicationFence {
publication_generation: published.publication_generation,
content_generation: published.content_generation,
project_generation: published.project_generation,
}
}
pub fn matches_publication_fence(&self, expected: PublicationFence) -> bool {
self.publication_fence() == expected
}
pub fn prepare_bridge_rebuild(&self) -> PreparedKnowledgeBridge {
let published = self.published_generation();
let fence = PublicationFence {
publication_generation: published.publication_generation,
content_generation: published.content_generation,
project_generation: published.project_generation,
};
let bridge = published
.source
.as_deref()
.map(|source| {
Arc::new(build_knowledge_bridge(
&published.live,
source,
published.content_generation,
&BridgeLimits::default(),
))
})
.unwrap_or_else(|| Arc::new(KnowledgeBridge::default()));
PreparedKnowledgeBridge { fence, bridge }
}
pub fn publish_prepared_bridge(&self, prepared: PreparedKnowledgeBridge) -> bool {
let _write_guard = self.write_mutex.lock();
let previous_source_set = self.published_source_set.load_full();
let previous = previous_source_set.current_generation();
let current_fence = PublicationFence {
publication_generation: previous.publication_generation,
content_generation: previous.content_generation,
project_generation: previous.project_generation,
};
if current_fence != prepared.fence {
self.rejected_stale_mutations.fetch_add(1, Ordering::AcqRel);
return false;
}
let generation = self.next_generation.fetch_add(1, Ordering::Relaxed);
let authority = build_published_authority(
&previous.live,
previous.source.as_deref(),
previous.source_version.as_deref(),
previous.content_generation,
&prepared.bridge,
&previous.code_signals,
previous.manifest.as_deref(),
);
let published_generation = Arc::new(PublishedGeneration {
publication_generation: generation,
content_generation: previous.content_generation,
project_generation: previous.project_generation,
source: previous.source.clone(),
source_version: previous.source_version.clone(),
freshness: Arc::clone(&previous.freshness),
manifest: previous.manifest.clone(),
code_signals: Arc::clone(&previous.code_signals),
bridge: prepared.bridge,
authority,
live: Arc::clone(&previous.live),
health: Arc::clone(&previous.health),
outline: Arc::clone(&previous.outline),
});
self.published_source_set
.store(Arc::new(previous_source_set.next_after_current_publish(
previous_source_set.current_source_id.clone(),
published_generation,
)));
true
}
pub fn prepare_authority_rebuild(&self) -> PreparedKnowledgeAuthority {
let published = self.published_generation();
let fence = AuthorityPublicationFence {
publication: PublicationFence {
publication_generation: published.publication_generation,
content_generation: published.content_generation,
project_generation: published.project_generation,
},
source: published.source.as_deref().cloned(),
source_version: published.source_version.as_deref().cloned(),
};
let authority = build_published_authority(
&published.live,
published.source.as_deref(),
published.source_version.as_deref(),
published.content_generation,
&published.bridge,
&published.code_signals,
published.manifest.as_deref(),
);
PreparedKnowledgeAuthority { fence, authority }
}
pub fn publish_prepared_authority(&self, prepared: PreparedKnowledgeAuthority) -> bool {
let _write_guard = self.write_mutex.lock();
let previous_source_set = self.published_source_set.load_full();
let previous = previous_source_set.current_generation();
let current_fence = AuthorityPublicationFence {
publication: PublicationFence {
publication_generation: previous.publication_generation,
content_generation: previous.content_generation,
project_generation: previous.project_generation,
},
source: previous.source.as_deref().cloned(),
source_version: previous.source_version.as_deref().cloned(),
};
if current_fence != prepared.fence {
self.rejected_stale_mutations.fetch_add(1, Ordering::AcqRel);
return false;
}
let generation = self.next_generation.fetch_add(1, Ordering::Relaxed);
let published_generation = Arc::new(PublishedGeneration {
publication_generation: generation,
content_generation: previous.content_generation,
project_generation: previous.project_generation,
source: previous.source.clone(),
source_version: previous.source_version.clone(),
freshness: Arc::clone(&previous.freshness),
manifest: previous.manifest.clone(),
code_signals: Arc::clone(&previous.code_signals),
bridge: Arc::clone(&previous.bridge),
authority: prepared.authority,
live: Arc::clone(&previous.live),
health: Arc::clone(&previous.health),
outline: Arc::clone(&previous.outline),
});
self.published_source_set
.store(Arc::new(previous_source_set.next_after_current_publish(
previous_source_set.current_source_id.clone(),
published_generation,
)));
true
}
pub fn refresh_source_metadata(&self) -> bool {
let _write_guard = self.write_mutex.lock();
let previous = self.published_generation();
let live = self.live.load_full();
let scout_plan = self.scout_plan.load_full();
let candidate = capture_published_manifest(&live, scout_plan.as_deref());
let candidate_source = candidate.as_ref().map(|manifest| &manifest.source);
let candidate_version = candidate.as_ref().map(|manifest| &manifest.source_version);
if candidate_source == previous.source.as_deref()
&& candidate_version == previous.source_version.as_deref()
{
return false;
}
self.swap_and_publish_retaining_content((*live).clone());
true
}
pub fn git_temporal_publication_fence(&self) -> GitTemporalPublicationFence {
self.refresh_source_metadata();
let published = self.published_generation();
GitTemporalPublicationFence {
project_generation: published.project_generation,
content_generation: published.content_generation,
source: published.source.as_deref().cloned(),
source_version: published.source_version.as_deref().cloned(),
}
}
pub fn write(&self) -> SharedIndexWriteGuard<'_> {
let mutex = self.write_mutex.lock();
let snapshot = (*self.live.load_full()).clone();
SharedIndexWriteGuard {
handle: self,
_mutex: mutex,
index: Some(snapshot),
dirty: false,
}
}
pub fn published_state(&self) -> Arc<PublishedIndexState> {
Arc::clone(&self.published_generation().health)
}
pub fn published_repo_outline(&self) -> Arc<RepoOutlineView> {
Arc::clone(&self.published_generation().outline)
}
pub fn current_project_generation(&self) -> u64 {
self.project_generation.load(Ordering::Acquire)
}
pub fn current_reset_project_generation(&self) -> Option<u64> {
match self.last_reset_project_generation.load(Ordering::Acquire) {
0 => None,
generation => Some(generation),
}
}
pub fn mark_index_folder_reset(&self) -> u64 {
let generation = self.current_project_generation();
self.last_reset_project_generation
.store(generation, Ordering::Release);
generation
}
#[allow(dead_code)]
pub fn current_rejected_stale_mutations(&self) -> u64 {
self.rejected_stale_mutations.load(Ordering::Relaxed)
}
pub(crate) fn note_rejected_stale_mutation(&self) {
self.rejected_stale_mutations
.fetch_add(1, Ordering::Relaxed);
}
pub fn reload(&self, root: &Path) -> anyhow::Result<()> {
self.reload_for_binding(root, None)
}
pub(crate) fn reload_for_binding(
&self,
root: &Path,
project_state_dir: Option<ProjectStateDir>,
) -> anyhow::Result<()> {
self.reload_for_binding_with_exclusions(
root,
project_state_dir,
discovery::SourceExclusions::default(),
)
}
pub fn reload_for_state_placement(
&self,
root: &Path,
state_placement: &StatePlacement,
) -> anyhow::Result<()> {
self.reload_for_binding_with_exclusions(
root,
state_placement.directory().cloned(),
discovery::SourceExclusions::for_state_placement(root, state_placement),
)
}
pub(crate) fn reload_for_binding_with_exclusions(
&self,
root: &Path,
project_state_dir: Option<ProjectStateDir>,
source_exclusions: discovery::SourceExclusions,
) -> anyhow::Result<()> {
self.reload_for_binding_with_exclusions_cancellation(
root,
project_state_dir,
source_exclusions,
None,
)
}
pub(crate) fn reload_for_binding_with_exclusions_cancellable(
&self,
root: &Path,
project_state_dir: Option<ProjectStateDir>,
source_exclusions: discovery::SourceExclusions,
cancel: &AtomicBool,
) -> anyhow::Result<()> {
self.reload_for_binding_with_exclusions_cancellation(
root,
project_state_dir,
source_exclusions,
Some(cancel),
)
}
fn reload_for_binding_with_exclusions_cancellation(
&self,
root: &Path,
project_state_dir: Option<ProjectStateDir>,
source_exclusions: discovery::SourceExclusions,
cancel: Option<&AtomicBool>,
) -> anyhow::Result<()> {
check_reload_cancelled(cancel)?;
let watermark_live = self.live.load_full();
let data = LiveIndex::build_reload_data_for_binding_with_exclusions_cancellable(
root,
project_state_dir.as_ref(),
&source_exclusions,
cancel,
)?;
check_reload_cancelled(cancel)?;
let admitted_anchor = *self.admitted_physical_anchor.lock();
let path_vanish_replacement =
crate::live_index::index_lifecycle::activation::project_source_authority_path_vanish_pending(
root,
) || indexed_tree_absent_from_disk(&watermark_live, root);
crate::live_index::index_lifecycle::activation::evict_project_source_authority_if_anchor_changed(
root,
admitted_anchor,
);
let authority =
crate::live_index::index_lifecycle::activation::project_source_authority(root);
let anchor_replacement = admitted_anchor.is_some()
&& admitted_anchor
!= crate::live_index::index_lifecycle::physical_root::PhysicalRootAnchor::observe(
root,
);
let admission_identity_mismatch =
crate::live_index::index_lifecycle::physical_root::PhysicalRootIdentity::from_stored(
self.admitted_root.load(Ordering::Acquire),
)
.is_some_and(|admitted| admitted != authority.admission_root());
let observed_replacement =
anchor_replacement || path_vanish_replacement || admission_identity_mismatch;
if observed_replacement {
self.physical_replacement_latched
.store(true, Ordering::Release);
}
let physical_replacement =
observed_replacement || self.physical_replacement_latched.load(Ordering::Acquire);
let scout_plan = Arc::clone(&data.scout_plan);
let is_degraded = matches!(scout_plan.coverage, crate::domain::CoverageStatus::Degraded);
let mut live = LiveIndex::from_reload_data(data);
#[cfg(test)]
reload_outside_lock::fire();
let _wg = self.write_mutex.lock();
check_reload_cancelled(cancel)?;
let current_live = self.live.load_full();
merge_post_watermark_live_admissions(&watermark_live, ¤t_live, &mut live)?;
self.source_exclusions.store(Arc::new(source_exclusions));
self.scout_plan.store(Some(scout_plan));
let latched_gap = matches!(
self.freshness_status.load_full().as_ref(),
FreshnessStatus::Degraded { reason_codes, .. }
if reason_codes.contains(&FreshnessReason::WatcherUnavailable)
);
let mut reason_codes = Vec::new();
if (is_degraded || physical_replacement)
&& !reason_codes.contains(&FreshnessReason::ReconciliationPending)
{
reason_codes.push(FreshnessReason::ReconciliationPending);
}
if latched_gap {
reason_codes.push(FreshnessReason::WatcherUnavailable);
}
self.freshness_status
.store(Arc::new(if reason_codes.is_empty() {
FreshnessStatus::Current
} else {
FreshnessStatus::Degraded {
last_valid_content_generation: self.published_generation().content_generation,
reason_codes,
}
}));
self.project_state_dir
.store(project_state_dir.map(Arc::new));
self.project_generation.fetch_add(1, Ordering::AcqRel);
#[cfg(test)]
reload_mid_commit::fire();
self.pre_update_snapshots.lock().clear();
self.swap_and_publish(live);
self.last_reset_project_generation
.store(0, Ordering::Release);
Ok(())
}
pub(crate) fn is_source_excluded(&self, relative_path: &Path) -> bool {
self.source_exclusions
.load()
.excludes_relative(relative_path)
}
fn record_pre_update_snapshot_for_publication(
&self,
path: &str,
snapshot: Option<PreUpdateSnapshot>,
replacement: PublicationFence,
) -> bool {
let snapshot_created = snapshot.is_some();
let mut snapshots = self.pre_update_snapshots.lock();
if let Some(snapshot) = snapshot {
snapshots.insert(
path.to_string(),
StoredPreUpdateSnapshot {
snapshot,
replacement,
},
);
} else {
snapshots.remove(path);
}
snapshot_created
}
pub fn reset_to_empty(&self) {
let _wg = self.write_mutex.lock();
self.source_exclusions
.store(Arc::new(discovery::SourceExclusions::default()));
self.scout_plan.store(None);
self.freshness_status
.store(Arc::new(FreshnessStatus::Verifying));
self.project_generation.fetch_add(1, Ordering::AcqRel);
self.swap_and_publish(LiveIndex::empty_live_index());
self.last_reset_project_generation
.store(0, Ordering::Release);
self.pre_update_snapshots.lock().clear();
}
pub fn update_file(&self, path: String, file: IndexedFile) {
let _wg = self.write_mutex.lock();
let current = self.live.load_full();
let path_clone = path.clone();
let pre_update = current.get_file(&path).map(pre_update_snapshot);
let mut live = (*current).clone();
let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
live.update_file(path, file);
}));
match result {
Ok(()) => {
self.swap_and_publish(live);
let replacement = self.publication_fence();
self.record_pre_update_snapshot_for_publication(
&path_clone,
pre_update,
replacement,
);
}
Err(panic_info) => {
let msg = panic_info
.downcast_ref::<String>()
.map(|s| s.as_str())
.or_else(|| panic_info.downcast_ref::<&str>().copied())
.unwrap_or("unknown");
tracing::error!(
"index mutation panicked for '{}': {} — original index preserved",
path_clone,
msg
);
}
}
}
pub fn update_file_at_generation(
&self,
path: &str,
file: IndexedFile,
expected_gen: u64,
) -> bool {
let _wg = self.write_mutex.lock();
let current_gen = self.project_generation.load(Ordering::Acquire);
if current_gen != expected_gen {
self.rejected_stale_mutations
.fetch_add(1, Ordering::Relaxed);
tracing::trace!(
path,
expected_gen,
current_gen,
"rejecting stale indexed-file update"
);
return false;
}
let current = self.live.load_full();
let pre_update = current.get_file(path).map(pre_update_snapshot);
let mut live = (*current).clone();
let path_owned = path.to_string();
let path_clone = path_owned.clone();
let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
live.update_file(path_owned, file);
}));
match result {
Ok(()) => {
self.swap_and_publish(live);
let replacement = self.publication_fence();
self.record_pre_update_snapshot_for_publication(
&path_clone,
pre_update,
replacement,
);
}
Err(panic_info) => {
let msg = panic_info
.downcast_ref::<String>()
.map(|s| s.as_str())
.or_else(|| panic_info.downcast_ref::<&str>().copied())
.unwrap_or("unknown");
tracing::error!(
"index mutation panicked for '{}': {} — original index preserved",
path_clone,
msg
);
}
}
true
}
pub(crate) fn publish_indexed_file_at_generation(
&self,
path: &str,
file: IndexedFile,
scouted: crate::domain::ScoutedEntry,
targets: crate::domain::IndexTargets,
expected_gen: u64,
expected_index_state_generation: u64,
) -> Option<IndexedFilePublicationReceipt> {
let _wg = self.write_mutex.lock();
let current_gen = self.project_generation.load(Ordering::Acquire);
if current_gen != expected_gen {
self.rejected_stale_mutations
.fetch_add(1, Ordering::Relaxed);
tracing::trace!(
path,
expected_gen,
current_gen,
"rejecting stale indexed-file publication"
);
return None;
}
let current_index_state_generation = self.published_state.load().generation;
if current_index_state_generation != expected_index_state_generation {
tracing::trace!(
path,
expected_index_state_generation,
current_index_state_generation,
"rejecting indexed-file publication prepared from a stale source bundle"
);
return None;
}
let current = self.live.load_full();
let pre_update = current.get_file(path).map(pre_update_snapshot);
let parse_status = match &file.parse_status {
ParseStatus::Parsed => crate::domain::index::ParseStatus::Parsed,
ParseStatus::PartialParse { .. } => crate::domain::index::ParseStatus::PartialParse,
ParseStatus::Failed { .. } => crate::domain::index::ParseStatus::Failed,
};
let disposition = FileDisposition::Indexed {
targets,
parse_status,
};
let manifest_entry =
catalog_entry_from_scout(&scouted, disposition, Some(file.content_hash.clone()));
let mut live = (*current).clone();
let path_owned = path.to_string();
let path_for_log = path_owned.clone();
let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
live.update_file(path_owned, file);
live.upsert_manifest_entry(manifest_entry);
}));
if let Err(panic_info) = result {
let msg = panic_info
.downcast_ref::<String>()
.map(|message| message.as_str())
.or_else(|| panic_info.downcast_ref::<&str>().copied())
.unwrap_or("unknown");
tracing::error!(
"indexed-file publication panicked for '{}': {} — original index preserved",
path_for_log,
msg
);
return None;
}
let updated_scout_plan = match self.scout_plan_with_entry_locked(scouted, &live) {
Ok(plan) => plan,
Err(error) => {
tracing::error!(path, %error, "failed to refresh indexed-file scout plan");
return None;
}
};
if let Some(plan) = updated_scout_plan {
self.scout_plan.store(Some(plan));
}
let scout_plan = self.scout_plan.load_full();
self.recompute_freshness_locked(&live, scout_plan.as_deref());
self.swap_and_publish(live);
let published = self.published_generation();
let replacement = PublicationFence::from_published(&published);
let snapshot_created =
self.record_pre_update_snapshot_for_publication(path, pre_update, replacement);
Some(IndexedFilePublicationReceipt {
published,
snapshot_created,
})
}
pub fn touch_mtime(&self, path: &str, new_mtime: u64) {
let _wg = self.write_mutex.lock();
let current = self.live.load_full();
if let Some(file) = current.files.get(path)
&& file.mtime_secs != new_mtime
{
let mut live = (*current).clone();
let mut updated = (**live.files.get(path).unwrap()).clone();
updated.mtime_secs = new_mtime;
live.files.insert(path.to_string(), Arc::new(updated));
self.swap_and_publish_retaining_content(live);
}
}
pub fn touch_mtime_at_generation(&self, path: &str, new_mtime: u64, expected_gen: u64) -> bool {
let _wg = self.write_mutex.lock();
let current_gen = self.project_generation.load(Ordering::Acquire);
if current_gen != expected_gen {
self.rejected_stale_mutations
.fetch_add(1, Ordering::Relaxed);
tracing::trace!(
path,
expected_gen,
current_gen,
"rejecting stale mtime touch"
);
return false;
}
let current = self.live.load_full();
if let Some(file) = current.files.get(path)
&& file.mtime_secs != new_mtime
{
let mut live = (*current).clone();
let mut updated = (**live.files.get(path).unwrap()).clone();
updated.mtime_secs = new_mtime;
live.files.insert(path.to_string(), Arc::new(updated));
self.swap_and_publish_retaining_content(live);
}
true
}
pub(crate) fn publish_hash_skip_at_generation(
&self,
path: &str,
new_mtime: u64,
scouted: crate::domain::ScoutedEntry,
targets: crate::domain::IndexTargets,
expected_gen: u64,
expected_index_state_generation: u64,
) -> Option<Arc<PublishedGeneration>> {
let _wg = self.write_mutex.lock();
let current_gen = self.project_generation.load(Ordering::Acquire);
if current_gen != expected_gen {
self.rejected_stale_mutations
.fetch_add(1, Ordering::Relaxed);
tracing::trace!(
path,
expected_gen,
current_gen,
"rejecting stale hash-skip publication"
);
return None;
}
let current_index_state_generation = self.published_state.load().generation;
if current_index_state_generation != expected_index_state_generation {
tracing::trace!(
path,
expected_index_state_generation,
current_index_state_generation,
"rejecting hash-skip prepared from a stale source bundle"
);
return None;
}
let current = self.live.load_full();
let mut live = (*current).clone();
let mut live_changed = false;
let file = current.files.get(path)?;
if file.mtime_secs != new_mtime && new_mtime != 0 {
let mut updated = (**file).clone();
updated.mtime_secs = new_mtime;
live.files.insert(path.to_string(), Arc::new(updated));
live_changed = true;
}
let parse_status = match &file.parse_status {
ParseStatus::Parsed => crate::domain::index::ParseStatus::Parsed,
ParseStatus::PartialParse { .. } => crate::domain::index::ParseStatus::PartialParse,
ParseStatus::Failed { .. } => crate::domain::index::ParseStatus::Failed,
};
let indexed_disposition = FileDisposition::Indexed {
targets,
parse_status,
};
let manifest_entry = catalog_entry_from_scout(
&scouted,
indexed_disposition,
Some(file.content_hash.clone()),
);
let manifest_changed = !current
.manifest_entries
.iter()
.any(|entry| entry == &manifest_entry);
if manifest_changed {
live.upsert_manifest_entry(manifest_entry);
}
let updated_scout_plan = match self.scout_plan_with_entry_locked(scouted, &live) {
Ok(plan) => plan,
Err(error) => {
tracing::error!(path, %error, "failed to refresh hash-skip scout plan");
return None;
}
};
if !live_changed && updated_scout_plan.is_none() && !manifest_changed {
return Some(self.published_generation());
}
if let Some(plan) = updated_scout_plan {
self.scout_plan.store(Some(plan));
}
let scout_plan = self.scout_plan.load_full();
self.recompute_freshness_locked(&live, scout_plan.as_deref());
self.swap_and_publish(live);
Some(self.published_generation())
}
pub fn add_file(&self, path: String, file: IndexedFile) {
let _wg = self.write_mutex.lock();
let current = self.live.load_full();
let pre_update = current.get_file(&path).map(pre_update_snapshot);
let mut live = (*current).clone();
let path_clone = path.clone();
let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
live.add_file(path, file);
}));
match result {
Ok(()) => {
self.swap_and_publish(live);
let replacement = self.publication_fence();
self.record_pre_update_snapshot_for_publication(
&path_clone,
pre_update,
replacement,
);
}
Err(panic_info) => {
let msg = panic_info
.downcast_ref::<String>()
.map(|s| s.as_str())
.or_else(|| panic_info.downcast_ref::<&str>().copied())
.unwrap_or("unknown");
tracing::error!(
"index add panicked for '{}': {} — original index preserved",
path_clone,
msg
);
}
}
}
pub fn remove_file(&self, path: &str) {
let _wg = self.write_mutex.lock();
let mut live = (*self.live.load_full()).clone();
let path_owned = path.to_string();
let mut removed = false;
let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
removed = live.remove_file(path);
}));
match result {
Ok(()) => {
if !removed {
return;
}
self.swap_and_publish(live);
self.pre_update_snapshots.lock().remove(path);
}
Err(panic_info) => {
let msg = panic_info
.downcast_ref::<String>()
.map(|s| s.as_str())
.or_else(|| panic_info.downcast_ref::<&str>().copied())
.unwrap_or("unknown");
tracing::error!(
"index remove panicked for '{}': {} — original index preserved",
path_owned,
msg
);
}
}
}
pub fn remove_file_at_generation(&self, path: &str, expected_gen: u64) -> bool {
matches!(
self.remove_file_outcome_at_generation(path, expected_gen),
FencedRemoval::Removed(_)
)
}
pub(crate) fn remove_file_outcome_at_generation(
&self,
path: &str,
expected_gen: u64,
) -> FencedRemoval {
self.remove_file_with_fences(path, expected_gen, None, None, None)
}
pub fn remove_file_at_publication_fence(&self, path: &str, expected: PublicationFence) -> bool {
matches!(
self.remove_file_at_publication_fence_with_receipt(path, expected),
FencedRemoval::Removed(_)
)
}
pub(crate) fn remove_file_at_publication_fence_with_receipt(
&self,
path: &str,
expected: PublicationFence,
) -> FencedRemoval {
self.remove_file_with_fences(
path,
expected.project_generation,
None,
Some(expected),
None,
)
}
pub(crate) fn remove_file_if_absent_at_publication_fence_with_receipt(
&self,
path: &str,
absolute_path: &Path,
expected: PublicationFence,
) -> FencedRemoval {
self.remove_file_with_fences(
path,
expected.project_generation,
None,
None,
Some(absolute_path),
)
}
pub(crate) fn remove_file_if_scout_entry_at_generation(
&self,
path: &str,
expected_entry: &crate::domain::ScoutedEntry,
expected_gen: u64,
) -> bool {
matches!(
self.remove_file_with_fences(
path,
expected_gen,
Some(expected_entry),
None,
expected_entry.absolute_path.as_deref(),
),
FencedRemoval::Removed(_)
)
}
fn remove_file_with_fences(
&self,
path: &str,
expected_gen: u64,
expected_scout_entry: Option<&crate::domain::ScoutedEntry>,
expected_publication: Option<PublicationFence>,
expected_absent_path: Option<&Path>,
) -> FencedRemoval {
let _wg = self.write_mutex.lock();
let current_gen = self.project_generation.load(Ordering::Acquire);
if current_gen != expected_gen {
self.rejected_stale_mutations
.fetch_add(1, Ordering::Relaxed);
tracing::trace!(
path,
expected_gen,
current_gen,
"rejecting stale file removal"
);
return FencedRemoval::Rejected;
}
if let Some(expected_publication) = expected_publication {
let current_publication = self.publication_fence();
if current_publication != expected_publication {
tracing::trace!(
path,
?expected_publication,
?current_publication,
"rejecting stale file removal publication"
);
return FencedRemoval::Rejected;
}
}
if let Some(expected_scout_entry) = expected_scout_entry {
let current_plan = self.scout_plan.load_full();
let current_entry = current_plan.as_ref().and_then(|plan| {
plan.entries
.iter()
.find(|entry| entry.path == expected_scout_entry.path)
});
if current_entry != Some(expected_scout_entry) {
tracing::trace!(
path,
"rejecting file removal because its scouted base changed"
);
return FencedRemoval::Rejected;
}
}
if let Some(absolute_path) = expected_absent_path {
match std::fs::symlink_metadata(absolute_path) {
Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
Ok(_) => {
tracing::trace!(
path,
"rejecting file removal because the path was recreated"
);
return FencedRemoval::Rejected;
}
Err(error) => {
tracing::trace!(
path,
%error,
"rejecting file removal because absence could not be confirmed"
);
return FencedRemoval::Rejected;
}
}
}
let mut live = (*self.live.load_full()).clone();
let path_owned = path.to_string();
let mut removed_from_live = false;
let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
removed_from_live = live.remove_file(path);
}));
if let Err(panic_info) = result {
let msg = panic_info
.downcast_ref::<String>()
.map(|s| s.as_str())
.or_else(|| panic_info.downcast_ref::<&str>().copied())
.unwrap_or("unknown");
tracing::error!(
"index remove panicked for '{}': {} — original index preserved",
path_owned,
msg
);
return FencedRemoval::Rejected;
}
let updated_scout_plan = match self.scout_plan_without_path_locked(path) {
Ok(plan) => plan,
Err(error) => {
tracing::error!(path, %error, "failed to refresh scout plan after removal");
return FencedRemoval::Rejected;
}
};
if !removed_from_live && updated_scout_plan.is_none() {
return FencedRemoval::NothingHeld;
}
if let Some(plan) = updated_scout_plan {
self.scout_plan.store(Some(plan));
}
let scout_plan = self.scout_plan.load_full();
self.recompute_freshness_locked(&live, scout_plan.as_deref());
self.swap_and_publish(live);
self.pre_update_snapshots.lock().remove(path);
FencedRemoval::Removed(self.published_generation())
}
pub(crate) fn publish_terminal_disposition_at_generation(
&self,
path: &str,
scouted: crate::domain::ScoutedEntry,
disposition: crate::domain::FileDisposition,
expected_gen: u64,
expected_index_state_generation: u64,
) -> Option<Arc<PublishedGeneration>> {
let _wg = self.write_mutex.lock();
let current_gen = self.project_generation.load(Ordering::Acquire);
if current_gen != expected_gen {
self.rejected_stale_mutations
.fetch_add(1, Ordering::Relaxed);
tracing::trace!(
path,
expected_gen,
current_gen,
"rejecting stale terminal disposition"
);
return None;
}
let current_index_state_generation = self.published_state.load().generation;
if current_index_state_generation != expected_index_state_generation {
tracing::trace!(
path,
expected_index_state_generation,
current_index_state_generation,
"rejecting terminal disposition prepared from a stale source bundle"
);
return None;
}
let manifest_entry = catalog_entry_from_scout(&scouted, disposition.clone(), None);
let mut live = (*self.live.load_full()).clone();
let retains_last_valid_content = matches!(
&disposition,
crate::domain::FileDisposition::Unreadable { .. }
| crate::domain::FileDisposition::UnstableDuringRead
);
let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
if !retains_last_valid_content {
live.remove_file(path);
}
live.upsert_manifest_entry(manifest_entry);
}));
if let Err(panic_info) = result {
let msg = panic_info
.downcast_ref::<String>()
.map(|s| s.as_str())
.or_else(|| panic_info.downcast_ref::<&str>().copied())
.unwrap_or("unknown");
tracing::error!(
"terminal disposition mutation panicked for '{}': {} — original index preserved",
path,
msg
);
return None;
}
let updated_scout_plan = match self.scout_plan_with_entry_locked(scouted, &live) {
Ok(plan) => plan,
Err(error) => {
tracing::error!(path, %error, "failed to refresh terminal scout plan");
return None;
}
};
if let Some(plan) = updated_scout_plan {
self.scout_plan.store(Some(plan));
}
if retains_last_valid_content {
let scout_plan = self.scout_plan.load_full();
self.recompute_freshness_locked(&live, scout_plan.as_deref());
self.swap_and_publish_retaining_content(live);
} else {
let scout_plan = self.scout_plan.load_full();
self.recompute_freshness_locked(&live, scout_plan.as_deref());
self.swap_and_publish(live);
self.pre_update_snapshots.lock().remove(path);
}
Some(self.published_generation())
}
pub fn mark_snapshot_verify_running(&self) {
let expected = self.publication_fence();
let _ = self.mark_snapshot_verify_running_at_fence(expected);
}
pub fn mark_snapshot_verify_started_at_generation(
&self,
expected_gen: u64,
progress: SnapshotVerifyProgress,
) -> bool {
let _wg = self.write_mutex.lock();
if self.project_generation.load(Ordering::Acquire) != expected_gen {
self.note_rejected_stale_mutation();
return false;
}
let current = self.live.load_full();
if current.load_source != IndexLoadSource::SnapshotRestore
|| !verify_in_flight(¤t.snapshot_verify_state)
{
return true;
}
let mut live = (*current).clone();
live.mark_snapshot_verify_running(progress);
let scout_plan = self.scout_plan.load_full();
self.recompute_freshness_locked(&live, scout_plan.as_deref());
self.swap_and_publish_retaining_content(live);
true
}
pub fn mark_snapshot_verify_running_at_fence(
&self,
expected: PublicationFence,
) -> Option<PublicationFence> {
let _wg = self.write_mutex.lock();
if self.publication_fence() != expected {
self.note_rejected_stale_mutation();
return None;
}
let mut live = (*self.live.load_full()).clone();
let restored_files = live.files.len();
live.mark_snapshot_verify_running(SnapshotVerifyProgress::started(
SystemTime::now(),
restored_files,
));
let scout_plan = self.scout_plan.load_full();
self.recompute_freshness_locked(&live, scout_plan.as_deref());
self.swap_and_publish_retaining_content(live);
Some(self.publication_fence())
}
pub fn mark_snapshot_verify_completed(&self, mismatched_paths: Vec<String>) {
let expected = self.publication_fence();
let _ = self.mark_snapshot_verify_completed_at_fence(expected, mismatched_paths);
}
pub(crate) fn mark_snapshot_verify_completed_at_generation(
&self,
expected_gen: u64,
completion: impl Into<SnapshotVerifyCompletion>,
) -> bool {
let _wg = self.write_mutex.lock();
if self.project_generation.load(Ordering::Acquire) != expected_gen {
self.note_rejected_stale_mutation();
return false;
}
let current = self.live.load_full();
if current.load_source != IndexLoadSource::SnapshotRestore {
return true;
}
if !verify_in_flight(¤t.snapshot_verify_state) {
return true;
}
let mut live = (*current).clone();
let withheld = live.resolve_snapshot_verify(completion.into());
let scout_plan = self.scout_plan.load_full();
self.recompute_freshness_locked(&live, scout_plan.as_deref());
if withheld {
self.swap_and_publish(live);
} else {
self.swap_and_publish_retaining_content(live);
}
true
}
pub fn mark_snapshot_verify_failed_at_generation(
&self,
expected_gen: u64,
report: SnapshotVerifyReport,
) -> bool {
let _wg = self.write_mutex.lock();
if self.project_generation.load(Ordering::Acquire) != expected_gen {
self.note_rejected_stale_mutation();
return false;
}
let current = self.live.load_full();
if current.load_source != IndexLoadSource::SnapshotRestore
|| !verify_in_flight(¤t.snapshot_verify_state)
{
return true;
}
let mut live = (*current).clone();
live.snapshot_verify_state = SnapshotVerifyState::Failed(report);
let scout_plan = self.scout_plan.load_full();
self.recompute_freshness_locked(&live, scout_plan.as_deref());
self.swap_and_publish_retaining_content(live);
true
}
pub(crate) fn publish_snapshot_verify_at_generation(
&self,
expected_gen: u64,
removals: &[(String, PathBuf)],
files: Vec<SnapshotVerifiedFile>,
completion: Option<SnapshotVerifyCompletion>,
) -> Option<SnapshotVerifyBatchReceipt> {
let _wg = self.write_mutex.lock();
if self.project_generation.load(Ordering::Acquire) != expected_gen {
self.note_rejected_stale_mutation();
return None;
}
let current = self.live.load_full();
let mut live = (*current).clone();
let mut receipt = SnapshotVerifyBatchReceipt::default();
let mut removed_paths = Vec::new();
let mut evicted_rows = Vec::new();
let mut scouted_entries = Vec::new();
let catalog: HashMap<String, usize> = live
.manifest_entries
.iter()
.enumerate()
.map(|(position, entry)| (scouted_catalog_path(&entry.path).to_string(), position))
.collect();
let mut catalog_drop: HashSet<String> = HashSet::new();
let mut catalog_set: Vec<CatalogEntry> = Vec::new();
let mut panicked = false;
for (path, absolute_path) in removals {
match std::fs::symlink_metadata(absolute_path) {
Err(error) if error.kind() == std::io::ErrorKind::NotFound => {
removed_paths.push(path.clone());
let row_removed = live.remove_row(path);
let entry_held = catalog.contains_key(path);
if entry_held {
catalog_drop.insert(path.clone());
}
if row_removed || entry_held {
receipt.removed.push(path.clone());
}
}
_ => receipt.refused.push(path.clone()),
}
}
for verified in files {
let current_hash = live
.files
.get(&verified.path)
.map(|file| file.content_hash.as_str());
if current_hash != verified.base_hash.as_deref() {
receipt.superseded.push(verified.path);
continue;
}
if !snapshot_verified_stamp_matches(&verified.scouted) {
receipt.refused.push(verified.path);
continue;
}
match verified.admission {
SnapshotVerifiedAdmission::Indexed { file, targets } => {
let parse_status = match &file.parse_status {
ParseStatus::Parsed => crate::domain::index::ParseStatus::Parsed,
ParseStatus::PartialParse { .. } => {
crate::domain::index::ParseStatus::PartialParse
}
ParseStatus::Failed { .. } => crate::domain::index::ParseStatus::Failed,
};
let manifest_entry = catalog_entry_from_scout(
&verified.scouted,
FileDisposition::Indexed {
targets,
parse_status,
},
Some(file.content_hash.clone()),
);
let path = verified.path.clone();
let applied = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
live.update_file_with_manifest(path.clone(), *file, false);
}));
if applied.is_err() {
tracing::error!(
path = %verified.path,
"snapshot-verify admission panicked; path removed and retried alone"
);
live.remove_row(&verified.path);
panicked = true;
receipt.refused.push(verified.path);
continue;
}
catalog_set.push(manifest_entry);
receipt.indexed.push(verified.path);
}
SnapshotVerifiedAdmission::Terminal(disposition) => {
let manifest_entry =
catalog_entry_from_scout(&verified.scouted, disposition, None);
let row_removed = live.remove_row(&verified.path);
if row_removed {
evicted_rows.push(verified.path.clone());
}
let entry_changed = catalog
.get(&verified.path)
.is_none_or(|&position| live.manifest_entries[position] != manifest_entry);
if entry_changed {
catalog_set.push(manifest_entry);
}
if row_removed || entry_changed {
receipt.terminal.push(verified.path);
} else {
receipt.unchanged.push(verified.path);
}
}
}
scouted_entries.push(verified.scouted);
}
if panicked {
live.rebuild_reverse_index();
}
let mut content_changed = panicked
|| !receipt.removed.is_empty()
|| !receipt.indexed.is_empty()
|| !receipt.terminal.is_empty();
if !catalog_drop.is_empty() || !catalog_set.is_empty() {
catalog_drop.extend(
catalog_set
.iter()
.map(|entry| scouted_catalog_path(&entry.path).to_string()),
);
live.manifest_entries
.retain(|entry| !catalog_drop.contains(scouted_catalog_path(&entry.path)));
live.manifest_entries.append(&mut catalog_set);
}
if content_changed {
live.sort_manifest_entries();
}
if let Some(completion) = completion
&& receipt.refused.is_empty()
&& verify_in_flight(&live.snapshot_verify_state)
&& live.load_source == IndexLoadSource::SnapshotRestore
{
content_changed |= live.resolve_snapshot_verify(completion);
receipt.completed = true;
}
if !content_changed && !receipt.completed {
return Some(receipt);
}
match self.scout_plan_with_batch_locked(scouted_entries, &removed_paths, &live) {
Ok(Some(plan)) => self.scout_plan.store(Some(plan)),
Ok(None) => {}
Err(error) => {
tracing::error!(%error, "failed to refresh scout plan for a snapshot-verify publication");
let mut refused = std::mem::take(&mut receipt.refused);
refused.append(&mut receipt.removed);
refused.append(&mut receipt.indexed);
refused.append(&mut receipt.terminal);
refused.append(&mut receipt.unchanged);
return Some(SnapshotVerifyBatchReceipt {
refused,
superseded: receipt.superseded,
..SnapshotVerifyBatchReceipt::default()
});
}
}
let scout_plan = self.scout_plan.load_full();
self.recompute_freshness_locked(&live, scout_plan.as_deref());
if content_changed {
self.swap_and_publish(live);
let mut snapshots = self.pre_update_snapshots.lock();
for path in removed_paths.iter().chain(&evicted_rows) {
snapshots.remove(path);
}
} else {
self.swap_and_publish_retaining_content(live);
}
Some(receipt)
}
fn scout_plan_with_batch_locked(
&self,
entries: Vec<crate::domain::ScoutedEntry>,
removed_paths: &[String],
live: &LiveIndex,
) -> anyhow::Result<Option<Arc<discovery::ScoutPlan>>> {
let Some(current) = self.scout_plan.load_full() else {
return Ok(None);
};
let mut plan = (*current).clone();
let before = plan.entries.len();
let removed: std::collections::HashSet<&str> =
removed_paths.iter().map(String::as_str).collect();
plan.entries.retain(|entry| {
entry
.path
.normalized_utf8
.as_deref()
.is_none_or(|path| !removed.contains(path))
});
let mut changed = plan.entries.len() != before;
let mut positions: HashMap<crate::domain::CatalogPath, usize> = plan
.entries
.iter()
.enumerate()
.map(|(position, entry)| (entry.path.clone(), position))
.collect();
for entry in entries {
match positions.get(&entry.path) {
Some(&position) if plan.entries[position] == entry => {}
Some(&position) => {
plan.entries[position] = entry;
changed = true;
}
None => {
positions.insert(entry.path.clone(), plan.entries.len());
plan.entries.push(entry);
changed = true;
}
}
}
discovery::refresh_scout_plan(&mut plan)?;
if manifest_requires_degraded_coverage(live) {
plan.coverage = crate::domain::CoverageStatus::Degraded;
}
if !changed && plan.coverage == current.coverage {
return Ok(None);
}
Ok(Some(Arc::new(plan)))
}
pub fn mark_snapshot_verify_completed_at_fence(
&self,
expected: PublicationFence,
mismatched_paths: Vec<String>,
) -> bool {
let _wg = self.write_mutex.lock();
if self.publication_fence() != expected {
self.note_rejected_stale_mutation();
return false;
}
let mut live = (*self.live.load_full()).clone();
live.mark_snapshot_verify_completed(mismatched_paths);
let scout_plan = self.scout_plan.load_full();
self.recompute_freshness_locked(&live, scout_plan.as_deref());
self.swap_and_publish_retaining_content(live);
true
}
fn swap_and_publish(&self, live: LiveIndex) {
self.swap_and_publish_with_content_change_and_hook(live, true, || {});
}
fn swap_and_publish_retaining_content(&self, live: LiveIndex) {
self.swap_and_publish_with_content_change_and_hook(live, false, || {});
}
#[cfg(test)]
fn swap_and_publish_with_hook<F>(&self, live: LiveIndex, after_live_swap: F)
where
F: FnOnce(),
{
self.swap_and_publish_with_content_change_and_hook(live, true, after_live_swap);
}
fn swap_and_publish_with_content_change_and_hook<F>(
&self,
live: LiveIndex,
content_changed: bool,
after_live_swap: F,
) where
F: FnOnce(),
{
let generation = self.next_generation.fetch_add(1, Ordering::Relaxed);
let published_state = Arc::new(PublishedIndexState::capture(generation, &live));
let published_repo_outline = Arc::new(live.capture_repo_outline_view());
let previous_source_set = self.published_source_set.load_full();
let previous = previous_source_set.current_generation();
let content_generation = if content_changed {
previous.content_generation.saturating_add(1)
} else {
previous.content_generation
};
let freshness = self.freshness_status.load_full();
let scout_plan = self.scout_plan.load_full();
let manifest = capture_published_manifest(&live, scout_plan.as_deref());
let temporal = self.git_temporal.load_full();
let history_coverage = capture_history_coverage(&live, &temporal.state);
let source = manifest
.as_ref()
.map(|manifest| Arc::new(manifest.source.clone()));
let source_version = manifest
.as_ref()
.map(|manifest| Arc::new(manifest.source_version.clone()));
let code_signals = if Arc::ptr_eq(&temporal, &previous.code_signals.temporal) {
Arc::clone(&previous.code_signals)
} else {
Arc::new(CodeSignalsSnapshot {
state: temporal.state.clone(),
temporal,
computed_for_content_generation: content_generation,
computed_for_source_version: source_version.as_deref().cloned().unwrap_or(
SourceVersion {
branch: None,
commit: None,
working_tree: WorkingTreeState::Unknown,
},
),
coverage: history_coverage,
})
};
let bridge = match source.as_deref() {
Some(source) if !content_changed && previous.source.as_deref() == Some(source) => {
Arc::clone(&previous.bridge)
}
Some(source) => Arc::new(build_knowledge_bridge(
&live,
source,
content_generation,
&BridgeLimits::default(),
)),
None => Arc::new(KnowledgeBridge::default()),
};
let authority = build_published_authority(
&live,
source.as_deref(),
source_version.as_deref(),
content_generation,
&bridge,
&code_signals,
manifest.as_deref(),
);
let live = Arc::new(live);
let published_generation = Arc::new(PublishedGeneration {
publication_generation: generation,
content_generation,
project_generation: self.project_generation.load(Ordering::Acquire),
source: source.clone(),
source_version,
freshness,
manifest,
code_signals,
bridge,
authority,
live: Arc::clone(&live),
health: Arc::clone(&published_state),
outline: Arc::clone(&published_repo_outline),
});
let current_source_id = source
.as_ref()
.map(|source| source.source_id.clone())
.unwrap_or_else(unbound_source_id);
let published_source_set = Arc::new(
previous_source_set.next_after_current_publish(current_source_id, published_generation),
);
self.live.store(live);
after_live_swap();
self.published_state.store(published_state);
self.published_source_set.store(published_source_set);
}
pub fn git_temporal(&self) -> Arc<super::git_temporal::GitTemporalIndex> {
Arc::clone(&self.published_generation().code_signals.temporal)
}
pub fn take_pre_update_snapshot(&self, path: &str) -> Option<PreUpdateSnapshot> {
let _wg = self.write_mutex.lock();
self.pre_update_snapshots
.lock()
.remove(path)
.map(|stored| stored.snapshot)
}
pub fn take_pre_update_snapshot_at_generation(
&self,
path: &str,
expected_gen: u64,
) -> Option<PreUpdateSnapshot> {
let _wg = self.write_mutex.lock();
let current_gen = self.project_generation.load(Ordering::Acquire);
if current_gen != expected_gen {
self.rejected_stale_mutations
.fetch_add(1, Ordering::Relaxed);
tracing::trace!(
path,
expected_gen,
current_gen,
"rejecting stale pre-update snapshot take"
);
return None;
}
self.pre_update_snapshots
.lock()
.remove(path)
.map(|stored| stored.snapshot)
}
pub(crate) fn peek_pre_update_snapshot_at_generation(
&self,
path: &str,
expected_gen: u64,
) -> Option<(PreUpdateSnapshot, PublicationFence)> {
let _wg = self.write_mutex.lock();
let current_gen = self.project_generation.load(Ordering::Acquire);
if current_gen != expected_gen {
self.rejected_stale_mutations
.fetch_add(1, Ordering::Relaxed);
tracing::trace!(
path,
expected_gen,
current_gen,
"rejecting stale pre-update snapshot peek"
);
return None;
}
self.pre_update_snapshots
.lock()
.get(path)
.map(|stored| (stored.snapshot.clone(), stored.replacement))
}
pub(crate) fn take_pre_update_snapshot_for_publication_at_generation(
&self,
path: &str,
expected_gen: u64,
replacement: PublicationFence,
) -> Option<PreUpdateSnapshot> {
let _wg = self.write_mutex.lock();
let current_gen = self.project_generation.load(Ordering::Acquire);
if current_gen != expected_gen {
self.rejected_stale_mutations
.fetch_add(1, Ordering::Relaxed);
tracing::trace!(
path,
expected_gen,
current_gen,
"rejecting stale publication-bound pre-update snapshot take"
);
return None;
}
let mut snapshots = self.pre_update_snapshots.lock();
if snapshots
.get(path)
.is_none_or(|stored| stored.replacement != replacement)
{
return None;
}
snapshots.remove(path).map(|stored| stored.snapshot)
}
pub fn take_pre_update_symbols(&self, path: &str) -> Option<Vec<PreUpdateSymbol>> {
self.take_pre_update_snapshot(path)
.map(|snapshot| snapshot.symbols)
}
pub fn update_git_temporal(&self, index: super::git_temporal::GitTemporalIndex) {
let _wg = self.write_mutex.lock();
self.git_temporal.store(Arc::new(index));
let live = (*self.live.load_full()).clone();
self.swap_and_publish_retaining_content(live);
}
pub fn update_git_temporal_at_fence(
&self,
index: super::git_temporal::GitTemporalIndex,
fence: &GitTemporalPublicationFence,
) -> bool {
let _wg = self.write_mutex.lock();
let live = self.live.load_full();
let scout_plan = self.scout_plan.load_full();
let candidate = capture_published_manifest(&live, scout_plan.as_deref());
let candidate_source = candidate.as_ref().map(|manifest| &manifest.source);
let candidate_version = candidate.as_ref().map(|manifest| &manifest.source_version);
let published = self.published_generation();
let current_gen = self.project_generation.load(Ordering::Acquire);
let matches = current_gen == fence.project_generation
&& published.content_generation == fence.content_generation
&& candidate_source == fence.source.as_ref()
&& candidate_version == fence.source_version.as_ref();
if !matches {
self.rejected_stale_mutations
.fetch_add(1, Ordering::Relaxed);
if candidate.is_some()
&& (candidate_source != published.source.as_deref()
|| candidate_version != published.source_version.as_deref())
{
self.swap_and_publish_retaining_content((*live).clone());
}
tracing::trace!(
expected_project_generation = fence.project_generation,
current_gen,
expected_content_generation = fence.content_generation,
current_content_generation = published.content_generation,
"rejecting stale git temporal publication"
);
return false;
}
self.git_temporal.store(Arc::new(index));
self.swap_and_publish_retaining_content((*live).clone());
true
}
pub fn set_local_empty_reason(&self, reason: Option<String>) {
let _wg = self.write_mutex.lock();
let mut live = (*self.live.load_full()).clone();
live.local_empty_reason = Arc::new(parking_lot::RwLock::new(reason));
self.swap_and_publish_retaining_content(live);
}
pub(crate) fn mark_bootstrap_loading(&self) {
let _wg = self.write_mutex.lock();
let mut live = (*self.live.load_full()).clone();
live.is_empty = false;
self.swap_and_publish_retaining_content(live);
}
}
impl<'a> Deref for SharedIndexWriteGuard<'a> {
type Target = LiveIndex;
fn deref(&self) -> &Self::Target {
self.index
.as_ref()
.expect("SharedIndexWriteGuard used after drop")
}
}
impl DerefMut for SharedIndexWriteGuard<'_> {
fn deref_mut(&mut self) -> &mut Self::Target {
self.dirty = true;
self.index
.as_mut()
.expect("SharedIndexWriteGuard used after drop")
}
}
impl Drop for SharedIndexWriteGuard<'_> {
fn drop(&mut self) {
if self.dirty
&& let Some(live) = self.index.take()
{
self.handle.swap_and_publish(live);
self.handle.pre_update_snapshots.lock().clear();
}
}
}
#[cfg(test)]
mod reload_mid_commit {
use std::cell::RefCell;
type Hook = Box<dyn Fn()>;
thread_local! {
static HOOK: RefCell<Option<Hook>> = const { RefCell::new(None) };
}
pub(crate) struct MidCommitGuard;
impl Drop for MidCommitGuard {
fn drop(&mut self) {
HOOK.with(|h| *h.borrow_mut() = None);
}
}
pub(crate) fn install(hook: impl Fn() + 'static) -> MidCommitGuard {
HOOK.with(|h| *h.borrow_mut() = Some(Box::new(hook)));
MidCommitGuard
}
pub(crate) fn fire() {
let hook = HOOK.with(|h| h.borrow_mut().take());
if let Some(hook) = hook {
hook();
}
}
}
#[cfg(all(test, feature = "server"))]
pub(crate) use reload_mid_commit::install as install_reload_mid_commit_hook;
#[cfg(test)]
mod reload_outside_lock {
use std::cell::RefCell;
type Hook = Box<dyn Fn()>;
thread_local! {
static HOOK: RefCell<Option<Hook>> = const { RefCell::new(None) };
}
pub(crate) struct OutsideLockGuard;
impl Drop for OutsideLockGuard {
fn drop(&mut self) {
HOOK.with(|h| *h.borrow_mut() = None);
}
}
pub(crate) fn install(hook: impl Fn() + 'static) -> OutsideLockGuard {
HOOK.with(|h| *h.borrow_mut() = Some(Box::new(hook)));
OutsideLockGuard
}
pub(crate) fn fire() {
let hook = HOOK.with(|h| h.borrow_mut().take());
if let Some(hook) = hook {
hook();
}
}
}
pub type SharedIndex = Arc<SharedIndexHandle>;
const PUBLISHED_QUARANTINE_LIST_CAP: usize = 1000;
impl PublishedIndexState {
fn capture(generation: u64, index: &LiveIndex) -> Self {
let (status, degraded_summary) = match index.index_state() {
IndexState::Empty => (PublishedIndexStatus::Empty, None),
IndexState::Loading => (PublishedIndexStatus::Loading, None),
IndexState::Ready => (PublishedIndexStatus::Ready, None),
IndexState::CircuitBreakerTripped { summary } => {
(PublishedIndexStatus::Degraded, Some(summary))
}
};
let stats = index.health_stats();
Self {
generation,
status,
degraded_summary,
file_count: stats.file_count,
parsed_count: stats.parsed_count,
partial_parse_count: stats.partial_parse_count,
unexpected_partial_parse_count: stats.unexpected_partial_parse_count,
expected_vendor_partial_parse_count: stats.expected_vendor_partial_parse_count,
expected_generated_partial_parse_count: stats.expected_generated_partial_parse_count,
expected_test_fixture_partial_parse_count: stats
.expected_test_fixture_partial_parse_count,
expected_template_dsl_partial_parse_count: stats
.expected_template_dsl_partial_parse_count,
expected_framework_partial_parse_count: stats.expected_framework_partial_parse_count,
expected_language_partial_parse_count: stats.expected_language_partial_parse_count,
failed_count: stats.failed_count,
partial_parse_files: stats
.partial_parse_files
.into_iter()
.take(PUBLISHED_QUARANTINE_LIST_CAP)
.collect(),
unexpected_partial_parse_files: stats
.unexpected_partial_parse_files
.into_iter()
.take(PUBLISHED_QUARANTINE_LIST_CAP)
.collect(),
expected_vendor_partial_parse_files: stats
.expected_vendor_partial_parse_files
.into_iter()
.take(PUBLISHED_QUARANTINE_LIST_CAP)
.collect(),
expected_generated_partial_parse_files: stats
.expected_generated_partial_parse_files
.into_iter()
.take(PUBLISHED_QUARANTINE_LIST_CAP)
.collect(),
expected_test_fixture_partial_parse_files: stats
.expected_test_fixture_partial_parse_files
.into_iter()
.take(PUBLISHED_QUARANTINE_LIST_CAP)
.collect(),
expected_template_dsl_partial_parse_files: stats
.expected_template_dsl_partial_parse_files
.into_iter()
.take(PUBLISHED_QUARANTINE_LIST_CAP)
.collect(),
expected_framework_partial_parse_files: stats
.expected_framework_partial_parse_files
.into_iter()
.take(PUBLISHED_QUARANTINE_LIST_CAP)
.collect(),
expected_language_partial_parse_files: stats
.expected_language_partial_parse_files
.into_iter()
.take(PUBLISHED_QUARANTINE_LIST_CAP)
.collect(),
failed_files: stats
.failed_files
.into_iter()
.take(PUBLISHED_QUARANTINE_LIST_CAP)
.collect(),
symbol_count: stats.symbol_count,
loaded_at_system: index.loaded_at_system,
load_duration: stats.load_duration,
load_source: index.load_source,
snapshot_verify_state: index.snapshot_verify_state.clone(),
is_empty: index.is_empty,
tier_counts: stats.tier_counts,
local_empty_reason: stats.local_empty_reason,
untracked_indexed: stats.untracked_indexed,
indexed_root: index.indexed_root.clone(),
}
}
pub fn status_label(&self) -> &'static str {
match self.status {
PublishedIndexStatus::Empty => "Empty",
PublishedIndexStatus::Loading => "Loading",
PublishedIndexStatus::Ready => "Ready",
PublishedIndexStatus::Degraded => "Degraded",
}
}
}
pub(crate) struct DerivedIndices {
pub trigram_index: super::trigram::TrigramIndex,
pub reverse_index: HashMap<String, Vec<ReferenceLocation>>,
pub files_by_basename: HashMap<String, Vec<String>>,
pub files_by_dir_component: HashMap<String, Vec<String>>,
}
impl DerivedIndices {
fn build_from_files_cancellable(
files: &HashMap<String, Arc<IndexedFile>>,
cancel: Option<&AtomicBool>,
) -> anyhow::Result<Self> {
check_reload_cancelled(cancel)?;
let trigram_index = super::trigram::TrigramIndex::build_from_files(files);
check_reload_cancelled(cancel)?;
pause_between_derived_stages(cancel)?;
let reverse_index = build_reverse_index_from_files(files);
check_reload_cancelled(cancel)?;
let (files_by_basename, files_by_dir_component) = build_path_indices_from_files(files);
check_reload_cancelled(cancel)?;
Ok(Self {
trigram_index,
reverse_index,
files_by_basename,
files_by_dir_component,
})
}
}
pub(crate) struct ReloadData {
pub files: HashMap<String, Arc<IndexedFile>>,
pub scout_plan: Arc<discovery::ScoutPlan>,
pub manifest_entries: Vec<CatalogEntry>,
pub cb_state: CircuitBreakerState,
pub load_duration: Duration,
pub gitignore: Option<ignore::gitignore::Gitignore>,
pub derived: DerivedIndices,
pub coupling_store: Option<Arc<super::coupling::CouplingStore>>,
pub indexed_root: PathBuf,
}
pub(crate) fn build_reverse_index_from_files(
files: &HashMap<String, Arc<IndexedFile>>,
) -> HashMap<String, Vec<ReferenceLocation>> {
let mut idx: HashMap<String, Vec<ReferenceLocation>> = HashMap::new();
for (file_path, indexed_file) in files {
for (reference_idx, reference) in indexed_file.references.iter().enumerate() {
idx.entry(reference.name.clone())
.or_default()
.push(ReferenceLocation {
file_path: file_path.clone(),
reference_idx: reference_idx as u32,
});
}
}
idx
}
pub(crate) fn build_path_indices_from_files(
files: &HashMap<String, Arc<IndexedFile>>,
) -> (HashMap<String, Vec<String>>, HashMap<String, Vec<String>>) {
let mut by_basename: HashMap<String, Vec<String>> = HashMap::new();
let mut by_dir_component: HashMap<String, Vec<String>> = HashMap::new();
for path in files.keys() {
if let Some(basename) = basename_key(path) {
insert_sorted_unique(by_basename.entry(basename).or_default(), path);
}
for component in dir_component_keys(path) {
insert_sorted_unique(by_dir_component.entry(component).or_default(), path);
}
}
(by_basename, by_dir_component)
}
#[allow(clippy::large_enum_variant)]
enum AdmissionOutcome {
Parsed(String, IndexedFile, crate::domain::IndexTargets),
Terminal {
path: String,
disposition: crate::domain::FileDisposition,
},
}
pub(super) fn scouted_catalog_path(path: &crate::domain::CatalogPath) -> &str {
path.normalized_utf8
.as_deref()
.unwrap_or(path.public_id.as_str())
}
fn scouted_entry_path(entry: &crate::domain::ScoutedEntry) -> &str {
scouted_catalog_path(&entry.path)
}
fn manifest_requires_degraded_coverage(live: &LiveIndex) -> bool {
live.manifest_entries.iter().any(|entry| {
matches!(
entry.disposition,
FileDisposition::Unreadable { .. }
| FileDisposition::UnstableDuringRead
| FileDisposition::AbortedCircuitBreaker
)
})
}
fn catalog_entry_from_scout(
scouted: &crate::domain::ScoutedEntry,
disposition: FileDisposition,
content_hash: Option<String>,
) -> CatalogEntry {
CatalogEntry {
path: scouted.path.clone(),
size: scouted.stamp.size,
language: scouted.language,
classification: scouted.classification,
disposition,
content_hash,
}
}
fn manifest_entries_from_scout(
scout_plan: &discovery::ScoutPlan,
terminal_dispositions: Vec<(String, FileDisposition)>,
files: &HashMap<String, Arc<IndexedFile>>,
) -> anyhow::Result<Vec<CatalogEntry>> {
let mut dispositions: HashMap<String, FileDisposition> =
terminal_dispositions.into_iter().collect();
let mut entries = Vec::with_capacity(scout_plan.entries.len());
for scouted in &scout_plan.entries {
let path = scouted_entry_path(scouted);
let disposition = dispositions.remove(path).ok_or_else(|| {
anyhow::anyhow!("manifest disposition missing for scouted path {path}")
})?;
let content_hash = match &disposition {
FileDisposition::Indexed { .. } => Some(
files
.get(path)
.ok_or_else(|| anyhow::anyhow!("indexed manifest path missing content {path}"))?
.content_hash
.clone(),
),
_ => None,
};
entries.push(catalog_entry_from_scout(scouted, disposition, content_hash));
}
if !dispositions.is_empty() {
let mut unexpected: Vec<_> = dispositions.into_keys().collect();
unexpected.sort();
anyhow::bail!(
"manifest dispositions without scout entries: {}",
unexpected.join(", ")
);
}
entries.sort_by_cached_key(|entry| {
let path = entry
.path
.normalized_utf8
.as_deref()
.unwrap_or(entry.path.public_id.as_str());
(path.to_lowercase(), path.to_string())
});
Ok(entries)
}
pub(super) fn compatibility_admission_decision(entry: &CatalogEntry) -> Option<AdmissionDecision> {
let (tier, reason) = match &entry.disposition {
FileDisposition::Indexed { .. } => return None,
FileDisposition::MetadataOnly { reason } => (
AdmissionTier::MetadataOnly,
match reason {
MetadataOnlyReason::Lockfile => SkipReason::DependencyLockfile,
MetadataOnlyReason::Binary => SkipReason::BinaryContent,
MetadataOnlyReason::OversizedData => SkipReason::SizeThreshold,
MetadataOnlyReason::GeneratedOrVendor => {
let path = scouted_catalog_path(&entry.path);
let admission =
discovery::classify_admission(Path::new(path), entry.size, None);
if admission.reason == Some(SkipReason::DenylistedExtension) {
SkipReason::DenylistedExtension
} else if entry.classification.is_generated
|| entry.classification.is_vendor
|| Path::new(path).parent().is_some_and(|parent| {
parent.components().any(|component| {
discovery::is_generated_output_dir_name(
&component.as_os_str().to_string_lossy(),
)
})
})
{
SkipReason::GeneratedOutput
} else {
SkipReason::Untracked
}
}
MetadataOnlyReason::SensitivePath { .. }
| MetadataOnlyReason::SensitiveContent { .. } => SkipReason::PolicyWithheld,
MetadataOnlyReason::LfsPointer { .. } => SkipReason::LfsPointer,
MetadataOnlyReason::UnsupportedPathEncoding
| MetadataOnlyReason::PathMetadataTooLarge => SkipReason::UnsupportedPath,
MetadataOnlyReason::UnsupportedTextEncoding => SkipReason::UnsupportedTextEncoding,
},
),
FileDisposition::HardSkip { reason } => (
AdmissionTier::HardSkip,
match reason {
HardSkipReason::PerFileCeiling => SkipReason::SizeCeiling,
HardSkipReason::ArtifactType => SkipReason::DenylistedExtension,
},
),
FileDisposition::Unreadable { .. }
| FileDisposition::UnstableDuringRead
| FileDisposition::AbortedCircuitBreaker => {
(AdmissionTier::MetadataOnly, SkipReason::Unreadable)
}
};
Some(AdmissionDecision::skip(tier, reason))
}
fn disposition_from_admission(decision: AdmissionDecision) -> crate::domain::FileDisposition {
match decision.tier {
AdmissionTier::HardSkip => crate::domain::FileDisposition::HardSkip {
reason: if matches!(decision.reason, Some(SkipReason::SizeCeiling)) {
HardSkipReason::PerFileCeiling
} else {
HardSkipReason::ArtifactType
},
},
AdmissionTier::MetadataOnly => crate::domain::FileDisposition::MetadataOnly {
reason: match decision.reason {
Some(SkipReason::DependencyLockfile) => MetadataOnlyReason::Lockfile,
Some(SkipReason::BinaryContent) => MetadataOnlyReason::Binary,
Some(SkipReason::SizeThreshold) => MetadataOnlyReason::OversizedData,
Some(SkipReason::DenylistedExtension)
| Some(SkipReason::GeneratedOutput)
| Some(SkipReason::Untracked) => MetadataOnlyReason::GeneratedOrVendor,
Some(SkipReason::SizeCeiling) => MetadataOnlyReason::OversizedData,
Some(SkipReason::UnsupportedTextEncoding)
| Some(SkipReason::PolicyWithheld)
| Some(SkipReason::LfsPointer)
| Some(SkipReason::UnsupportedPath)
| Some(SkipReason::Unreadable) => {
debug_assert!(
!matches!(
decision.reason,
Some(SkipReason::PolicyWithheld)
| Some(SkipReason::LfsPointer)
| Some(SkipReason::UnsupportedPath)
| Some(SkipReason::Unreadable)
),
"display-only SkipReason {:?} round-tripped into a \
MetadataOnlyReason; it would resurface as a false \
encoding verdict",
decision.reason
);
MetadataOnlyReason::UnsupportedTextEncoding
}
Some(SkipReason::UnsupportedLanguage) | None => {
MetadataOnlyReason::UnsupportedTextEncoding
}
},
},
AdmissionTier::Normal => crate::domain::FileDisposition::MetadataOnly {
reason: MetadataOnlyReason::UnsupportedTextEncoding,
},
}
}
fn terminal_admission_outcome(
entry: &discovery::DiscoveredEntry,
_decision: AdmissionDecision,
disposition: crate::domain::FileDisposition,
) -> AdmissionOutcome {
AdmissionOutcome::Terminal {
path: entry.relative_path.clone(),
disposition,
}
}
pub(crate) struct AdmitParseResult {
pub files: HashMap<String, Arc<IndexedFile>>,
pub terminal_dispositions: Vec<(String, crate::domain::FileDisposition)>,
pub coverage: crate::domain::CoverageStatus,
pub cb_state: CircuitBreakerState,
}
#[derive(Clone, Debug, PartialEq, Eq, Hash)]
pub(crate) struct CircuitBreakerScope {
source: PathBuf,
lane: crate::domain::IndexTargets,
stage: String,
}
impl CircuitBreakerScope {
fn new(source: PathBuf, lane: crate::domain::IndexTargets, stage: impl Into<String>) -> Self {
Self {
source,
lane,
stage: stage.into(),
}
}
}
struct ParseFoldResult {
files: HashMap<String, Arc<IndexedFile>>,
dispositions: Vec<(String, crate::domain::FileDisposition)>,
coverage: crate::domain::CoverageStatus,
cb_state: CircuitBreakerState,
}
fn fold_parse_results_for_scope(
mut parse_results: Vec<(String, IndexedFile)>,
cb_state: CircuitBreakerState,
scope: CircuitBreakerScope,
) -> ParseFoldResult {
parse_results.sort_by(|left, right| left.0.cmp(&right.0));
let mut files = HashMap::with_capacity(parse_results.len());
let mut dispositions = Vec::with_capacity(parse_results.len());
let mut remaining = parse_results.into_iter();
let mut cb_tripped = false;
while let Some((path, indexed_file)) = remaining.next() {
let parse_status = match &indexed_file.parse_status {
ParseStatus::Parsed => crate::domain::index::ParseStatus::Parsed,
ParseStatus::PartialParse { .. } => crate::domain::index::ParseStatus::PartialParse,
ParseStatus::Failed { error } => {
cb_state.record_failure(&path, error);
crate::domain::index::ParseStatus::Failed
}
};
if !matches!(indexed_file.parse_status, ParseStatus::Failed { .. }) {
cb_state.record_success();
}
dispositions.push((
path.clone(),
crate::domain::FileDisposition::Indexed {
targets: scope.lane,
parse_status,
},
));
files.insert(path, Arc::new(indexed_file));
if cb_state.should_abort() {
error!("{}", cb_state.summary());
cb_tripped = true;
dispositions
.extend(remaining.map(|(path, _)| {
(path, crate::domain::FileDisposition::AbortedCircuitBreaker)
}));
break;
}
}
if cb_tripped {
cb_state.tripped.store(true, Ordering::Relaxed);
}
ParseFoldResult {
files,
dispositions,
coverage: if cb_tripped {
crate::domain::CoverageStatus::Degraded
} else {
crate::domain::CoverageStatus::Complete
},
cb_state,
}
}
#[derive(Clone)]
struct PlannedIngest {
stamp: crate::domain::FileStamp,
targets: crate::domain::IndexTargets,
}
struct LegacyExecutionProjection {
entries: Vec<discovery::DiscoveredEntry>,
ingest_plans: HashMap<String, PlannedIngest>,
terminal_dispositions: Vec<(String, crate::domain::FileDisposition)>,
}
fn project_scout_for_legacy_execution(plan: &discovery::ScoutPlan) -> LegacyExecutionProjection {
let mut executable = Vec::new();
let mut ingest_plans = HashMap::new();
let mut terminal_dispositions = Vec::new();
for entry in &plan.entries {
let path = entry
.path
.normalized_utf8
.clone()
.unwrap_or_else(|| entry.path.public_id.clone());
match &entry.decision {
ScoutDecision::Ingest { targets } => {
let Some(relative_path) = entry.path.normalized_utf8.clone() else {
terminal_dispositions.push((
path,
crate::domain::FileDisposition::MetadataOnly {
reason: MetadataOnlyReason::UnsupportedPathEncoding,
},
));
continue;
};
let Some(absolute_path) = entry.absolute_path.clone() else {
terminal_dispositions.push((
path,
crate::domain::FileDisposition::Unreadable {
stage: crate::domain::AccessStage::Metadata,
kind: crate::domain::AccessErrorKind::Other,
},
));
continue;
};
ingest_plans.insert(
relative_path.clone(),
PlannedIngest {
stamp: entry.stamp.clone(),
targets: *targets,
},
);
executable.push(discovery::DiscoveredEntry {
relative_os_path: PathBuf::from(&relative_path),
relative_path,
absolute_path,
file_size: entry.stamp.size,
language: entry.language,
classification: entry.classification,
});
}
ScoutDecision::MetadataOnly { reason } => {
terminal_dispositions.push((
path,
crate::domain::FileDisposition::MetadataOnly {
reason: reason.clone(),
},
));
}
ScoutDecision::HardSkip { reason } => {
terminal_dispositions.push((
path,
crate::domain::FileDisposition::HardSkip { reason: *reason },
));
}
ScoutDecision::Unavailable { stage, kind } => {
terminal_dispositions.push((
path,
crate::domain::FileDisposition::Unreadable {
stage: *stage,
kind: *kind,
},
));
}
}
}
terminal_dispositions.sort_by(|left, right| left.0.cmp(&right.0));
LegacyExecutionProjection {
entries: executable,
ingest_plans,
terminal_dispositions,
}
}
fn admit_and_parse_entries(
entries: &[crate::discovery::DiscoveredEntry],
ingest_plans: &HashMap<String, PlannedIngest>,
exclude_untracked_set: &Option<std::collections::HashSet<String>>,
generated_output_demotions: &std::collections::HashSet<String>,
source_scope: PathBuf,
cancel: Option<&AtomicBool>,
) -> anyhow::Result<AdmitParseResult> {
use crate::discovery::classify_admission;
check_reload_cancelled(cancel)?;
let cancelled_during_parse = AtomicBool::new(false);
let inflight_budget = Arc::new(InflightByteBudget::from_env());
let staged_ceiling = ingest_plans.values().fold(0_u64, |total, planned| {
total.saturating_add(planned.stamp.size)
});
let staged_accounting = Arc::new(StagedContentAccounting::new(staged_ceiling));
let outcomes: Vec<AdmissionOutcome> = indexing_thread_pool().install(|| {
entries
.par_iter()
.map(|entry| {
if cancel.is_some_and(|flag| flag.load(Ordering::Acquire)) {
cancelled_during_parse.store(true, Ordering::Release);
return terminal_admission_outcome(
entry,
AdmissionDecision::skip(
AdmissionTier::MetadataOnly,
SkipReason::UnsupportedLanguage,
),
crate::domain::FileDisposition::Unreadable {
stage: crate::domain::AccessStage::Metadata,
kind: crate::domain::AccessErrorKind::Other,
},
);
}
let Some(planned) = ingest_plans.get(&entry.relative_path) else {
return terminal_admission_outcome(
entry,
AdmissionDecision::skip(
AdmissionTier::MetadataOnly,
SkipReason::UnsupportedLanguage,
),
crate::domain::FileDisposition::Unreadable {
stage: crate::domain::AccessStage::Metadata,
kind: crate::domain::AccessErrorKind::Other,
},
);
};
let decision_pre = classify_admission(&entry.absolute_path, entry.file_size, None);
if !matches!(decision_pre.tier, AdmissionTier::Normal) {
let disposition = disposition_from_admission(decision_pre);
return terminal_admission_outcome(entry, decision_pre, disposition);
}
if generated_output_demotions.contains(&entry.relative_path) {
let decision = AdmissionDecision::skip(
AdmissionTier::MetadataOnly,
SkipReason::GeneratedOutput,
);
let disposition = disposition_from_admission(decision);
return terminal_admission_outcome(entry, decision, disposition);
}
if let Some(tracked) = exclude_untracked_set.as_ref()
&& !tracked.contains(&entry.relative_path)
{
let decision =
AdmissionDecision::skip(AdmissionTier::MetadataOnly, SkipReason::Untracked);
let disposition = disposition_from_admission(decision);
return terminal_admission_outcome(entry, decision, disposition);
}
if planned.stamp.size > inflight_budget.total {
let decision =
AdmissionDecision::skip(AdmissionTier::HardSkip, SkipReason::SizeCeiling);
return terminal_admission_outcome(
entry,
decision,
crate::domain::FileDisposition::HardSkip {
reason: HardSkipReason::PerFileCeiling,
},
);
}
let permit = Some(inflight_budget.acquire(planned.stamp.size));
if cancel.is_some_and(|flag| flag.load(Ordering::Acquire)) {
cancelled_during_parse.store(true, Ordering::Release);
return terminal_admission_outcome(
entry,
AdmissionDecision::skip(
AdmissionTier::MetadataOnly,
SkipReason::UnsupportedLanguage,
),
crate::domain::FileDisposition::Unreadable {
stage: crate::domain::AccessStage::Metadata,
kind: crate::domain::AccessErrorKind::Other,
},
);
}
let stable = stable_read_with_retries(
&entry.absolute_path,
&planned.stamp,
StableReadLimits {
per_file_bytes: crate::domain::index::HARD_SKIP_BYTES,
inflight_bytes: inflight_budget.total,
},
&FilesystemStableReadAccess,
);
let (bytes, accepted_hash) = match stable {
StableReadOutcome::Accepted { bytes, hash } => (bytes, hash),
StableReadOutcome::HardSkip { reason } => {
let decision = AdmissionDecision::skip(
AdmissionTier::HardSkip,
SkipReason::SizeCeiling,
);
return terminal_admission_outcome(
entry,
decision,
crate::domain::FileDisposition::HardSkip { reason },
);
}
StableReadOutcome::Unreadable { stage, kind } => {
return terminal_admission_outcome(
entry,
AdmissionDecision::skip(
AdmissionTier::MetadataOnly,
SkipReason::UnsupportedLanguage,
),
crate::domain::FileDisposition::Unreadable { stage, kind },
);
}
StableReadOutcome::UnstableDuringRead => {
return terminal_admission_outcome(
entry,
AdmissionDecision::skip(
AdmissionTier::MetadataOnly,
SkipReason::UnsupportedLanguage,
),
crate::domain::FileDisposition::UnstableDuringRead,
);
}
};
let decision_post =
classify_admission(&entry.absolute_path, entry.file_size, Some(&bytes));
if !matches!(decision_post.tier, AdmissionTier::Normal) {
let disposition = disposition_from_admission(decision_post);
return terminal_admission_outcome(entry, decision_post, disposition);
}
if let crate::knowledge::StableContentAdmission::MetadataOnly(reason) =
crate::knowledge::classify_stable_content(
&entry.relative_path,
planned.targets,
&bytes,
)
{
return terminal_admission_outcome(
entry,
AdmissionDecision::skip(
AdmissionTier::MetadataOnly,
SkipReason::UnsupportedLanguage,
),
crate::domain::FileDisposition::MetadataOnly { reason },
);
}
let language = entry.language.unwrap_or(LanguageId::Text);
let mtime_secs = planned
.stamp
.modified_hint
.and_then(|modified| modified.duration_since(std::time::UNIX_EPOCH).ok())
.map(|duration| duration.as_secs())
.unwrap_or(0);
let relative_path = entry.relative_path.clone();
let classification =
FileClassification::for_indexed_path(&relative_path, planned.targets);
let result = parsing::process_file_with_classification(
&relative_path,
&bytes,
language,
classification,
);
note_reload_parsed(&source_scope);
pause_after_parse_for_test(&source_scope, cancel);
if cancel.is_some_and(|flag| flag.load(Ordering::Acquire)) {
cancelled_during_parse.store(true, Ordering::Release);
return terminal_admission_outcome(
entry,
AdmissionDecision::skip(
AdmissionTier::MetadataOnly,
SkipReason::UnsupportedLanguage,
),
crate::domain::FileDisposition::Unreadable {
stage: crate::domain::AccessStage::Metadata,
kind: crate::domain::AccessErrorKind::Other,
},
);
}
let indexed = IndexedFile::from_parse_result(result, bytes).with_mtime(mtime_secs);
note_reload_symbols(&source_scope, indexed.symbols.len() as u64);
debug_assert_eq!(crate::hash::digest(&indexed.content), accepted_hash);
let resident_bytes = u64::try_from(indexed.content.len()).unwrap_or(u64::MAX);
if !staged_accounting.handoff(resident_bytes, permit) {
let decision =
AdmissionDecision::skip(AdmissionTier::HardSkip, SkipReason::SizeCeiling);
return terminal_admission_outcome(
entry,
decision,
crate::domain::FileDisposition::HardSkip {
reason: HardSkipReason::PerFileCeiling,
},
);
}
AdmissionOutcome::Parsed(relative_path, indexed, planned.targets)
})
.collect()
});
if cancelled_during_parse.load(Ordering::Acquire)
|| cancel.is_some_and(|flag| flag.load(Ordering::Acquire))
{
return Err(reload_cancelled());
}
let mut code_results: Vec<(String, IndexedFile)> = Vec::new();
let mut knowledge_results: Vec<(String, IndexedFile)> = Vec::new();
let mut combined_results: Vec<(String, IndexedFile)> = Vec::new();
let mut terminal_dispositions = Vec::new();
let mut execution_degraded = false;
for outcome in outcomes {
match outcome {
AdmissionOutcome::Parsed(path, indexed, targets) => match targets {
crate::domain::IndexTargets::Code => code_results.push((path, indexed)),
crate::domain::IndexTargets::Knowledge => {
knowledge_results.push((path, indexed));
}
crate::domain::IndexTargets::CodeAndKnowledge => {
combined_results.push((path, indexed));
}
},
AdmissionOutcome::Terminal { path, disposition } => {
execution_degraded |= matches!(
disposition,
crate::domain::FileDisposition::Unreadable { .. }
| crate::domain::FileDisposition::UnstableDuringRead
);
terminal_dispositions.push((path, disposition));
}
}
}
let staged_count = code_results.len() + knowledge_results.len() + combined_results.len();
info!(
"admission + parse: {} staged, {} terminal",
staged_count,
terminal_dispositions.len()
);
let mut files = HashMap::with_capacity(staged_count);
let mut breaker_coverage = crate::domain::CoverageStatus::Complete;
let mut cb_state = None;
for (lane, parse_results) in [
(crate::domain::IndexTargets::Code, code_results),
(crate::domain::IndexTargets::Knowledge, knowledge_results),
(
crate::domain::IndexTargets::CodeAndKnowledge,
combined_results,
),
] {
if parse_results.is_empty() {
continue;
}
let ParseFoldResult {
files: lane_files,
dispositions,
coverage,
cb_state: lane_cb_state,
} = fold_parse_results_for_scope(
parse_results,
CircuitBreakerState::from_env(),
CircuitBreakerScope::new(source_scope.clone(), lane, "parse"),
);
files.extend(lane_files);
terminal_dispositions.extend(dispositions);
if matches!(coverage, crate::domain::CoverageStatus::Degraded) {
breaker_coverage = crate::domain::CoverageStatus::Degraded;
}
if cb_state
.as_ref()
.is_none_or(|current: &CircuitBreakerState| {
!current.is_tripped() && lane_cb_state.is_tripped()
})
{
cb_state = Some(lane_cb_state);
}
}
terminal_dispositions.sort_by(|left, right| left.0.cmp(&right.0));
let coverage = if execution_degraded
|| matches!(breaker_coverage, crate::domain::CoverageStatus::Degraded)
{
crate::domain::CoverageStatus::Degraded
} else {
crate::domain::CoverageStatus::Complete
};
Ok(AdmitParseResult {
files,
terminal_dispositions,
coverage,
cb_state: cb_state.unwrap_or_else(CircuitBreakerState::from_env),
})
}
impl LiveIndex {
pub fn load(root: &Path) -> anyhow::Result<SharedIndex> {
Self::load_with_project_state(root, None, discovery::SourceExclusions::default())
}
pub fn load_for_state_placement(
root: &Path,
state_placement: &StatePlacement,
) -> anyhow::Result<SharedIndex> {
let shared = Self::load_with_project_state(
root,
state_placement.directory(),
discovery::SourceExclusions::for_state_placement(root, state_placement),
)?;
shared
.project_state_dir
.store(state_placement.directory().cloned().map(Arc::new));
Ok(shared)
}
fn load_with_project_state(
root: &Path,
project_state_dir: Option<&ProjectStateDir>,
source_exclusions: discovery::SourceExclusions,
) -> anyhow::Result<SharedIndex> {
let start = Instant::now();
info!("LiveIndex::load starting at {:?}", root);
let mut scout_plan = discovery::scout_repository_with_exclusions(root, &source_exclusions)?;
let projection = project_scout_for_legacy_execution(&scout_plan);
info!(
"scouted {} catalog entries ({} executable by the legacy index)",
scout_plan.entries.len(),
projection.entries.len(),
);
let exclude_untracked_set = discovery::tracked_path_set_for_exclusion(root);
let generated_output_demotions =
discovery::untracked_generated_output_demotions(root, &projection.entries);
let LegacyExecutionProjection {
entries: all_entries,
ingest_plans,
terminal_dispositions: mut scout_terminal_dispositions,
} = projection;
let AdmitParseResult {
files,
mut terminal_dispositions,
coverage,
cb_state,
} = admit_and_parse_entries(
&all_entries,
&ingest_plans,
&exclude_untracked_set,
&generated_output_demotions,
normalize_root(root),
None,
)?;
if matches!(coverage, crate::domain::CoverageStatus::Degraded) {
scout_plan.coverage = crate::domain::CoverageStatus::Degraded;
}
terminal_dispositions.append(&mut scout_terminal_dispositions);
terminal_dispositions.sort_by(|left, right| left.0.cmp(&right.0));
debug_assert_eq!(terminal_dispositions.len(), scout_plan.entries.len());
let manifest_entries =
manifest_entries_from_scout(&scout_plan, terminal_dispositions, &files)?;
let admit_parse_elapsed = start.elapsed();
info!(
"admission + parse complete: {} files, {} symbols, {} manifest entries, {:?}",
files.len(),
files.values().map(|f| f.symbols.len()).sum::<usize>(),
manifest_entries.len(),
admit_parse_elapsed
);
let phase = Instant::now();
let trigram_index = super::trigram::TrigramIndex::build_from_files(&files);
let trigram_elapsed = phase.elapsed();
info!("trigram index built in {:?}", trigram_elapsed);
let phase = Instant::now();
let gitignore = discovery::load_gitignore(root);
let coupling_store = project_state_dir
.and_then(|state_dir| super::coupling::init_coupling_store(root, state_dir));
let stores_elapsed = phase.elapsed();
info!("gitignore + coupling store ready in {:?}", stores_elapsed);
let mut index = LiveIndex {
files,
loaded_at: Instant::now(),
loaded_at_system: SystemTime::now(),
load_duration: Duration::ZERO,
cb_state,
is_empty: false,
load_source: IndexLoadSource::FreshLoad,
snapshot_verify_state: SnapshotVerifyState::NotNeeded,
reverse_index: HashMap::new(),
files_by_basename: HashMap::new(),
files_by_dir_component: HashMap::new(),
trigram_index,
gitignore,
manifest_entries,
coupling_store,
local_empty_reason: Arc::new(parking_lot::RwLock::new(None)),
indexed_root: Some(normalize_root(root)),
};
let phase = Instant::now();
index.rebuild_reverse_index();
index.rebuild_path_indices();
let derived_elapsed = phase.elapsed();
index.load_duration = start.elapsed();
info!(
"LiveIndex loaded: {} files, {} manifest entries, {:?} total \
(admission+parse {:?}, trigram {:?}, stores {:?}, derived indices {:?})",
index.files.len(),
index.manifest_entries.len(),
index.load_duration,
admit_parse_elapsed,
trigram_elapsed,
stores_elapsed,
derived_elapsed
);
crate::live_index::frecency::ensure_bump_hook_registered();
Ok(SharedIndexHandle::shared_with_scout_plan(
index,
scout_plan,
source_exclusions,
))
}
pub(crate) fn empty_live_index() -> LiveIndex {
LiveIndex {
files: HashMap::new(),
loaded_at: Instant::now(),
loaded_at_system: SystemTime::now(),
load_duration: Duration::ZERO,
cb_state: CircuitBreakerState::new(0.20),
is_empty: true,
load_source: IndexLoadSource::EmptyBootstrap,
snapshot_verify_state: SnapshotVerifyState::NotNeeded,
reverse_index: HashMap::new(),
files_by_basename: HashMap::new(),
files_by_dir_component: HashMap::new(),
trigram_index: super::trigram::TrigramIndex::new(),
gitignore: None,
manifest_entries: Vec::new(),
coupling_store: None,
local_empty_reason: Arc::new(parking_lot::RwLock::new(None)),
indexed_root: None,
}
}
pub(crate) fn from_source_files(files: HashMap<String, Arc<IndexedFile>>) -> LiveIndex {
let is_empty = files.is_empty();
let trigram_index = super::trigram::TrigramIndex::build_from_files(&files);
let mut index = LiveIndex {
files,
loaded_at: Instant::now(),
loaded_at_system: SystemTime::now(),
load_duration: Duration::ZERO,
cb_state: CircuitBreakerState::new(0.20),
is_empty,
load_source: IndexLoadSource::FreshLoad,
snapshot_verify_state: SnapshotVerifyState::NotNeeded,
reverse_index: HashMap::new(),
files_by_basename: HashMap::new(),
files_by_dir_component: HashMap::new(),
trigram_index,
gitignore: None,
manifest_entries: Vec::new(),
coupling_store: None,
local_empty_reason: Arc::new(parking_lot::RwLock::new(None)),
indexed_root: None,
};
index.rebuild_reverse_index();
index.rebuild_path_indices();
index
}
pub fn from_indexed_files(
root: &Path,
files: Vec<(String, IndexedFile)>,
) -> anyhow::Result<SharedIndex> {
let canonical = dunce::canonicalize(root).map_err(|error| {
anyhow::anyhow!("index root {} does not resolve: {error}", root.display())
})?;
let mut index = Self::from_source_files(
files
.into_iter()
.map(|(path, file)| (path, Arc::new(file)))
.collect(),
);
index.indexed_root = Some(canonical);
Ok(SharedIndexHandle::shared(index))
}
pub fn empty() -> SharedIndex {
SharedIndexHandle::shared(Self::empty_live_index())
}
pub fn set_local_empty_reason(&self, reason: Option<String>) {
*self.local_empty_reason.write() = reason;
}
pub fn local_empty_reason(&self) -> Option<String> {
self.local_empty_reason.read().clone()
}
pub(crate) fn refuses_watcher_observed_admission(&self) -> bool {
self.load_source == IndexLoadSource::EmptyBootstrap
&& self.indexed_root.is_none()
&& self.local_empty_reason().is_none()
}
pub fn coupling_store(&self) -> Option<&super::coupling::CouplingStore> {
self.coupling_store.as_deref()
}
fn upsert_manifest_entry(&mut self, entry: CatalogEntry) {
self.upsert_manifest_entry_with_sort(entry, true);
}
fn upsert_manifest_entry_with_sort(&mut self, entry: CatalogEntry, sort: bool) {
let path = scouted_catalog_path(&entry.path).to_string();
self.manifest_entries
.retain(|existing| scouted_catalog_path(&existing.path) != path);
self.manifest_entries.push(entry);
if sort {
self.sort_manifest_entries();
}
}
fn sort_manifest_entries(&mut self) {
self.manifest_entries.sort_by_cached_key(|entry| {
let path = entry
.path
.normalized_utf8
.as_deref()
.unwrap_or(entry.path.public_id.as_str());
(path.to_lowercase(), path.to_string())
});
}
fn remove_manifest_entry(&mut self, path: &str) -> bool {
let before = self.manifest_entries.len();
self.manifest_entries
.retain(|entry| scouted_catalog_path(&entry.path) != path);
self.manifest_entries.len() != before
}
pub fn compatibility_skipped_files(&self) -> Vec<SkippedFile> {
self.manifest_entries
.iter()
.filter_map(|entry| {
let decision = compatibility_admission_decision(entry)?;
let path = scouted_catalog_path(&entry.path).to_string();
let extension = Path::new(&path)
.extension()
.and_then(|extension| extension.to_str())
.map(ToOwned::to_owned);
Some(SkippedFile {
path,
size: entry.size,
extension,
decision,
})
})
.collect()
}
pub fn tier_counts(&self) -> (usize, usize, usize) {
let mut tier1 = 0;
let mut tier2 = 0;
let mut tier3 = 0;
for entry in &self.manifest_entries {
match entry.disposition {
FileDisposition::Indexed { .. } => tier1 += 1,
FileDisposition::HardSkip { .. } => tier3 += 1,
FileDisposition::MetadataOnly { .. }
| FileDisposition::Unreadable { .. }
| FileDisposition::UnstableDuringRead
| FileDisposition::AbortedCircuitBreaker => tier2 += 1,
}
}
if self.manifest_entries.is_empty() {
tier1 = self.files.len();
}
(tier1, tier2, tier3)
}
pub(crate) fn build_reload_data(root: &Path) -> anyhow::Result<ReloadData> {
Self::build_reload_data_for_binding(root, None)
}
pub(crate) fn build_reload_data_for_binding(
root: &Path,
project_state_dir: Option<&ProjectStateDir>,
) -> anyhow::Result<ReloadData> {
Self::build_reload_data_for_binding_with_exclusions(
root,
project_state_dir,
&discovery::SourceExclusions::default(),
)
}
pub(crate) fn build_reload_data_for_binding_with_exclusions(
root: &Path,
project_state_dir: Option<&ProjectStateDir>,
source_exclusions: &discovery::SourceExclusions,
) -> anyhow::Result<ReloadData> {
Self::build_reload_data_for_binding_with_exclusions_cancellable(
root,
project_state_dir,
source_exclusions,
None,
)
}
fn build_reload_data_for_binding_with_exclusions_cancellable(
root: &Path,
project_state_dir: Option<&ProjectStateDir>,
source_exclusions: &discovery::SourceExclusions,
cancel: Option<&AtomicBool>,
) -> anyhow::Result<ReloadData> {
let start = Instant::now();
check_reload_cancelled(cancel)?;
info!("LiveIndex::build_reload_data starting at {:?}", root);
if !root.exists() {
anyhow::bail!(
"discovery error: root path does not exist: {}",
root.display()
);
}
let mut scout_plan = match cancel {
Some(cancel) => discovery::scout_repository_with_exclusions_cancellable(
root,
source_exclusions,
cancel,
)?,
None => discovery::scout_repository_with_exclusions(root, source_exclusions)?,
};
check_reload_cancelled(cancel)?;
let projection = project_scout_for_legacy_execution(&scout_plan);
note_reload_discovered(root, projection.entries.len() as u64);
pause_after_discovery_for_test(root, cancel);
info!(
"scouted {} catalog entries ({} executable by the legacy index)",
scout_plan.entries.len(),
projection.entries.len(),
);
let exclude_untracked_set = discovery::tracked_path_set_for_exclusion(root);
check_reload_cancelled(cancel)?;
let generated_output_demotions =
discovery::untracked_generated_output_demotions(root, &projection.entries);
check_reload_cancelled(cancel)?;
let LegacyExecutionProjection {
entries: all_entries,
ingest_plans,
terminal_dispositions: mut scout_terminal_dispositions,
} = projection;
let AdmitParseResult {
files: new_files,
mut terminal_dispositions,
coverage,
cb_state: new_cb,
} = admit_and_parse_entries(
&all_entries,
&ingest_plans,
&exclude_untracked_set,
&generated_output_demotions,
normalize_root(root),
cancel,
)?;
check_reload_cancelled(cancel)?;
if matches!(coverage, crate::domain::CoverageStatus::Degraded) {
scout_plan.coverage = crate::domain::CoverageStatus::Degraded;
}
terminal_dispositions.append(&mut scout_terminal_dispositions);
terminal_dispositions.sort_by(|left, right| left.0.cmp(&right.0));
debug_assert_eq!(terminal_dispositions.len(), scout_plan.entries.len());
let manifest_entries =
manifest_entries_from_scout(&scout_plan, terminal_dispositions, &new_files)?;
let phase = Instant::now();
let derived = DerivedIndices::build_from_files_cancellable(&new_files, cancel)?;
let derived_elapsed = phase.elapsed();
check_reload_cancelled(cancel)?;
let gitignore = discovery::load_gitignore(root);
let coupling_store = project_state_dir
.and_then(|state_dir| super::coupling::init_coupling_store(root, state_dir));
check_reload_cancelled(cancel)?;
let load_duration = start.elapsed();
info!(
"LiveIndex::build_reload_data done: {} files, {} symbols, {} manifest entries, \
{:?} total (derived indices {:?})",
new_files.len(),
new_files.values().map(|f| f.symbols.len()).sum::<usize>(),
manifest_entries.len(),
load_duration,
derived_elapsed
);
Ok(ReloadData {
files: new_files,
scout_plan: Arc::new(scout_plan),
manifest_entries,
cb_state: new_cb,
load_duration,
gitignore,
derived,
coupling_store,
indexed_root: normalize_root(root),
})
}
pub(crate) fn apply_reload_data(&mut self, data: ReloadData) {
self.files = data.files;
self.loaded_at = Instant::now();
self.loaded_at_system = SystemTime::now();
self.load_duration = data.load_duration;
self.cb_state = data.cb_state;
self.is_empty = false;
self.local_empty_reason.write().take();
self.load_source = IndexLoadSource::FreshLoad;
self.snapshot_verify_state = SnapshotVerifyState::NotNeeded;
self.trigram_index = data.derived.trigram_index;
self.reverse_index = data.derived.reverse_index;
self.files_by_basename = data.derived.files_by_basename;
self.files_by_dir_component = data.derived.files_by_dir_component;
self.gitignore = data.gitignore;
self.manifest_entries = data.manifest_entries;
self.coupling_store = data.coupling_store;
self.indexed_root = Some(data.indexed_root);
}
fn from_reload_data(data: ReloadData) -> Self {
let mut live = Self::empty_live_index();
live.apply_reload_data(data);
live
}
pub fn reload(&mut self, root: &Path) -> anyhow::Result<()> {
let data = Self::build_reload_data(root)?;
self.apply_reload_data(data);
Ok(())
}
pub fn update_file(&mut self, path: String, file: IndexedFile) {
self.update_file_with_manifest(path, file, true);
}
fn update_file_with_manifest(&mut self, path: String, file: IndexedFile, manifest: bool) {
self.release_unverified(&path);
let old_ref_names: Vec<String> = self
.files
.get(&path)
.map(|f| f.references.iter().map(|r| r.name.clone()).collect())
.unwrap_or_default();
let had_existing = !old_ref_names.is_empty() || self.files.contains_key(&path);
let manifest_entry = manifest.then(|| self.derived_manifest_entry(&path, &file));
self.files.insert(path.clone(), Arc::new(file));
#[cfg(test)]
if snapshot_verify_test_panic_path()
.lock()
.take_if(|injected| *injected == path)
.is_some()
{
panic!("injected panic between a row insert and its index updates");
}
if let Some(manifest_entry) = manifest_entry {
self.upsert_manifest_entry_with_sort(manifest_entry, true);
}
if had_existing {
self.remove_path_indices_for_path(&path);
}
for name in &old_ref_names {
if let Some(locs) = self.reverse_index.get_mut(name) {
locs.retain(|loc| loc.file_path != path);
if locs.is_empty() {
self.reverse_index.remove(name);
}
}
}
self.trigram_index
.update_file(&path, &self.files[&path].content);
self.insert_reverse_index_for_path(&path);
self.insert_path_indices_for_path(&path);
self.is_empty = false;
self.loaded_at_system = SystemTime::now();
}
fn derived_manifest_entry(&self, path: &str, file: &IndexedFile) -> CatalogEntry {
let (catalog_path, targets) = self
.manifest_entries
.iter()
.find(|entry| scouted_catalog_path(&entry.path) == path)
.map(|entry| {
let targets = match entry.disposition {
FileDisposition::Indexed { targets, .. } => targets,
_ if file.language.is_code_language() => crate::domain::IndexTargets::Code,
_ => crate::domain::IndexTargets::Knowledge,
};
(entry.path.clone(), targets)
})
.unwrap_or_else(|| {
let targets = if file.language.is_code_language() {
crate::domain::IndexTargets::Code
} else {
crate::domain::IndexTargets::Knowledge
};
(
crate::domain::CatalogPath {
public_id: path.to_string(),
normalized_utf8: Some(path.to_string()),
},
targets,
)
});
let parse_status = match &file.parse_status {
ParseStatus::Parsed => crate::domain::index::ParseStatus::Parsed,
ParseStatus::PartialParse { .. } => crate::domain::index::ParseStatus::PartialParse,
ParseStatus::Failed { .. } => crate::domain::index::ParseStatus::Failed,
};
CatalogEntry {
path: catalog_path,
size: file.byte_len,
language: Some(file.language),
classification: file.classification,
disposition: FileDisposition::Indexed {
targets,
parse_status,
},
content_hash: Some(file.content_hash.clone()),
}
}
pub(crate) fn is_path_gitignored(&self, relative_path: &str) -> bool {
let Some(gitignore) = self.gitignore.as_ref() else {
return false;
};
if std::path::Path::new(relative_path).has_root() {
return false;
}
gitignore
.matched_path_or_any_parents(relative_path, false)
.is_ignore()
}
pub fn add_file(&mut self, path: String, file: IndexedFile) {
self.update_file(path, file);
}
pub fn remove_file(&mut self, path: &str) -> bool {
let released = self.release_unverified(path);
let removed_file = self.remove_row(path);
let removed_manifest = self.remove_manifest_entry(path);
if removed_file || removed_manifest {
self.loaded_at_system = SystemTime::now();
}
released || removed_file || removed_manifest
}
fn remove_row(&mut self, path: &str) -> bool {
self.remove_reverse_index_for_path(path);
let removed = self.files.remove(path).is_some();
if removed {
self.trigram_index.remove_file(path);
self.remove_path_indices_for_path(path);
}
removed
}
pub fn unverified_since_restore(&self, path: &str) -> Option<&str> {
match &self.snapshot_verify_state {
SnapshotVerifyState::Completed(report) => {
report.unverified.get(path).map(String::as_str)
}
_ => None,
}
}
pub fn withheld_since_restore(&self) -> &BTreeMap<String, String> {
static NONE: BTreeMap<String, String> = BTreeMap::new();
match &self.snapshot_verify_state {
SnapshotVerifyState::Completed(report) => &report.unverified,
_ => &NONE,
}
}
fn release_unverified(&mut self, path: &str) -> bool {
match &mut self.snapshot_verify_state {
SnapshotVerifyState::Completed(report) => report.drop_unverified(path),
_ => false,
}
}
fn resolve_snapshot_verify(&mut self, completion: SnapshotVerifyCompletion) -> bool {
let SnapshotVerifyCompletion {
mut report,
restored_rows,
} = completion;
let mut withheld = false;
for (path, restored) in restored_rows {
let current = self.files.get(&path);
match (current, restored) {
(None, _) => {}
(Some(current), Some(restored)) if Arc::ptr_eq(current, &restored) => {
withheld |= self.remove_row(&path);
}
_ => {
report.drop_unverified(&path);
}
}
}
self.snapshot_verify_state = SnapshotVerifyState::Completed(report);
withheld
}
fn remove_reverse_index_for_path(&mut self, path: &str) {
if let Some(file) = self.files.get(path) {
let names: Vec<String> = file.references.iter().map(|r| r.name.clone()).collect();
for name in names {
if let Some(locs) = self.reverse_index.get_mut(&name) {
locs.retain(|loc| loc.file_path != path);
if locs.is_empty() {
self.reverse_index.remove(&name);
}
}
}
}
}
fn insert_reverse_index_for_path(&mut self, path: &str) {
if let Some(file) = self.files.get(path) {
for (reference_idx, reference) in file.references.iter().enumerate() {
self.reverse_index
.entry(reference.name.clone())
.or_default()
.push(ReferenceLocation {
file_path: path.to_string(),
reference_idx: reference_idx as u32,
});
}
}
}
pub(crate) fn rebuild_reverse_index(&mut self) {
self.reverse_index = build_reverse_index_from_files(&self.files);
}
pub(crate) fn rebuild_path_indices(&mut self) {
let (by_basename, by_dir_component) = build_path_indices_from_files(&self.files);
self.files_by_basename = by_basename;
self.files_by_dir_component = by_dir_component;
}
fn insert_path_indices_for_path(&mut self, path: &str) {
if let Some(basename) = basename_key(path) {
insert_sorted_unique(self.files_by_basename.entry(basename).or_default(), path);
}
for component in dir_component_keys(path) {
insert_sorted_unique(
self.files_by_dir_component.entry(component).or_default(),
path,
);
}
}
fn remove_path_indices_for_path(&mut self, path: &str) {
if let Some(basename) = basename_key(path)
&& let Some(paths) = self.files_by_basename.get_mut(&basename)
{
remove_sorted_path(paths, path);
if paths.is_empty() {
self.files_by_basename.remove(&basename);
}
}
for component in dir_component_keys(path) {
if let Some(paths) = self.files_by_dir_component.get_mut(&component) {
remove_sorted_path(paths, path);
if paths.is_empty() {
self.files_by_dir_component.remove(&component);
}
}
}
}
pub fn load_source(&self) -> IndexLoadSource {
self.load_source
}
pub fn snapshot_verify_state(&self) -> SnapshotVerifyState {
self.snapshot_verify_state.clone()
}
pub(crate) fn reflects_scouted_entry(&self, entry: &crate::domain::ScoutedEntry) -> bool {
let Some(path) = entry.path.normalized_utf8.as_deref() else {
return false;
};
match crate::live_index::single_file::catalog_terminal_disposition(&entry.decision) {
Some(disposition) => {
!self.files.contains_key(path)
&& self.manifest_entries.contains(&catalog_entry_from_scout(
entry,
disposition,
None,
))
}
None if matches!(entry.decision, crate::domain::ScoutDecision::Ingest { .. }) => {
let mtime_secs = entry
.stamp
.modified_hint
.and_then(|time| time.duration_since(std::time::UNIX_EPOCH).ok())
.map(|duration| duration.as_secs());
self.files.get(path).is_some_and(|row| {
row.byte_len == entry.stamp.size && Some(row.mtime_secs) == mtime_secs
})
}
None => false,
}
}
pub(crate) fn mark_snapshot_verify_running(&mut self, progress: SnapshotVerifyProgress) {
if self.load_source == IndexLoadSource::SnapshotRestore {
self.snapshot_verify_state = SnapshotVerifyState::Running(progress);
}
}
pub(crate) fn mark_snapshot_verify_completed(&mut self, mismatched_paths: Vec<String>) {
if self.load_source == IndexLoadSource::SnapshotRestore {
self.snapshot_verify_state =
SnapshotVerifyState::completed_with_mismatches(mismatched_paths);
}
}
}
fn basename_key(path: &str) -> Option<String> {
Path::new(path)
.file_name()
.and_then(|name| name.to_str())
.map(|name| name.to_ascii_lowercase())
}
fn dir_component_keys(path: &str) -> Vec<String> {
let components: Vec<&str> = path
.split(['/', '\\'])
.filter(|component| !component.is_empty())
.collect();
if components.len() <= 1 {
return Vec::new();
}
let mut seen = HashSet::new();
let mut keys = Vec::new();
for component in &components[..components.len() - 1] {
let key = component.to_ascii_lowercase();
if seen.insert(key.clone()) {
keys.push(key);
}
}
keys.sort();
keys
}
fn insert_sorted_unique(paths: &mut Vec<String>, path: &str) {
match paths.binary_search_by(|existing| existing.as_str().cmp(path)) {
Ok(_) => {}
Err(pos) => paths.insert(pos, path.to_string()),
}
}
fn remove_sorted_path(paths: &mut Vec<String>, path: &str) {
if let Ok(pos) = paths.binary_search_by(|existing| existing.as_str().cmp(path)) {
paths.remove(pos);
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::domain::{
FileOutcome, LanguageId, ReferenceKind, ReferenceRecord, SymbolKind, SymbolRecord,
};
use std::fs;
use std::sync::Mutex as StdMutex;
use tempfile::TempDir;
static COUPLING_ENV_LOCK: StdMutex<()> = StdMutex::new(());
struct CouplingEnvGuard {
previous: Option<String>,
}
#[allow(unsafe_code)] impl CouplingEnvGuard {
fn set(value: Option<&str>) -> Self {
let previous =
std::env::var(crate::live_index::coupling::lifecycle::COUPLING_FLAG_ENV).ok();
unsafe {
match value {
Some(value) => std::env::set_var(
crate::live_index::coupling::lifecycle::COUPLING_FLAG_ENV,
value,
),
None => std::env::remove_var(
crate::live_index::coupling::lifecycle::COUPLING_FLAG_ENV,
),
}
}
Self { previous }
}
}
#[allow(unsafe_code)] impl Drop for CouplingEnvGuard {
fn drop(&mut self) {
unsafe {
match self.previous.as_deref() {
Some(value) => std::env::set_var(
crate::live_index::coupling::lifecycle::COUPLING_FLAG_ENV,
value,
),
None => std::env::remove_var(
crate::live_index::coupling::lifecycle::COUPLING_FLAG_ENV,
),
}
}
}
}
fn runtime_canary() -> String {
["runtime", "-", "canary", "-", "segment"].concat()
}
fn dummy_symbol() -> SymbolRecord {
let byte_range = (0, 10);
SymbolRecord {
name: "foo".to_string(),
kind: SymbolKind::Function,
depth: 0,
sort_order: 0,
byte_range,
item_byte_range: Some(byte_range),
line_range: (0, 1),
doc_byte_range: None,
}
}
fn make_result(outcome: FileOutcome, symbols: Vec<SymbolRecord>) -> FileProcessingResult {
FileProcessingResult {
relative_path: "test.rs".to_string(),
language: LanguageId::Rust,
classification: crate::domain::FileClassification::for_code_path("test.rs"),
outcome,
parse_diagnostic: None,
symbols,
byte_len: 42,
content_hash: "abc123".to_string(),
references: vec![],
alias_map: std::collections::HashMap::new(),
}
}
#[test]
fn test_indexed_file_maps_processed_status() {
let result = make_result(FileOutcome::Processed, vec![dummy_symbol()]);
let indexed = IndexedFile::from_parse_result(result, b"fn foo() {}".to_vec());
assert_eq!(indexed.parse_status, ParseStatus::Parsed);
assert_eq!(indexed.symbols.len(), 1);
}
#[test]
fn test_indexed_file_maps_partial_parse_keeps_symbols() {
let result = make_result(
FileOutcome::PartialParse {
warning: "syntax error".to_string(),
},
vec![dummy_symbol()],
);
let indexed = IndexedFile::from_parse_result(result, b"fn bad(".to_vec());
assert!(matches!(
indexed.parse_status,
ParseStatus::PartialParse { .. }
));
assert_eq!(
indexed.symbols.len(),
1,
"symbols kept even on partial parse"
);
}
#[test]
fn test_indexed_file_maps_failed_status_empty_symbols_content_preserved() {
let result = make_result(
FileOutcome::Failed {
error: "parse failed".to_string(),
},
vec![],
);
let content = b"some content bytes".to_vec();
let indexed = IndexedFile::from_parse_result(result, content.clone());
assert!(matches!(indexed.parse_status, ParseStatus::Failed { .. }));
assert!(indexed.symbols.is_empty(), "failed parse has no symbols");
assert_eq!(
indexed.content, content,
"content bytes stored even on failure"
);
}
#[test]
fn test_circuit_breaker_does_not_trip_at_20pct_of_10_files() {
let cb = CircuitBreakerState::new(0.20);
for _ in 0..8 {
cb.record_success();
}
for i in 0..2 {
cb.record_failure(&format!("file{i}.rs"), "error");
}
assert!(
!cb.should_abort(),
"2/10 = 20% should NOT trip (threshold not exceeded)"
);
}
#[test]
fn test_circuit_breaker_trips_at_30pct_of_10_files() {
let cb = CircuitBreakerState::new(0.20);
for _ in 0..7 {
cb.record_success();
}
for i in 0..3 {
cb.record_failure(&format!("file{i}.rs"), "error");
}
assert!(cb.should_abort(), "3/10 = 30% should trip");
}
#[test]
fn test_circuit_breaker_does_not_trip_on_tiny_repos() {
let cb = CircuitBreakerState::new(0.20);
cb.record_failure("a.rs", "err");
cb.record_failure("b.rs", "err");
cb.record_failure("c.rs", "err");
assert!(
!cb.should_abort(),
"< 5 files processed: circuit breaker must not trip"
);
}
#[test]
fn test_circuit_breaker_threshold_configurable() {
let cb = CircuitBreakerState::new(0.10);
for _ in 0..9 {
cb.record_success();
}
cb.record_failure("file.rs", "error");
assert!(!cb.should_abort(), "10% == threshold, not exceeded");
let cb2 = CircuitBreakerState::new(0.10);
for _ in 0..8 {
cb2.record_success();
}
for i in 0..2 {
cb2.record_failure(&format!("file{i}.rs"), "error");
}
assert!(cb2.should_abort(), "20% > 10% threshold should trip");
}
fn write_file(dir: &Path, name: &str, content: &str) {
let path = dir.join(name);
if let Some(p) = path.parent() {
fs::create_dir_all(p).unwrap();
}
fs::write(path, content).unwrap();
}
mod reload_commit_carry {
use super::*;
fn indexed_file_with_hash(path: &str, body: &[u8], content_hash: &str) -> IndexedFile {
let mut file = make_indexed_file_for_mutation(path);
file.content = body.to_vec();
file.byte_len = body.len() as u64;
file.content_hash = content_hash.to_string();
file
}
#[test]
fn carries_post_watermark_admission_absent_from_candidate() {
let tmp = TempDir::new().unwrap();
write_file(tmp.path(), "src/base.rs", "fn base() {}\n");
let shared = LiveIndex::load(tmp.path()).unwrap();
let watermark = shared.read().as_ref().clone();
let mut candidate = {
let data =
LiveIndex::build_reload_data(tmp.path()).expect("reload data should build");
LiveIndex::from_reload_data(data)
};
assert!(
!candidate.files.contains_key("src/watcher_admitted.rs"),
"precondition: candidate built from disk must not yet include the watcher file"
);
let mut current = watermark.clone();
current.update_file(
"src/watcher_admitted.rs".to_string(),
indexed_file_with_hash(
"src/watcher_admitted.rs",
b"fn watcher_admitted() {}\n",
"watcher_hash",
),
);
merge_post_watermark_live_admissions(&watermark, ¤t, &mut candidate)
.expect("carry should succeed for an absent post-watermark admission");
assert!(
candidate.get_file("src/watcher_admitted.rs").is_some(),
"post-watermark live admission must survive reload commit"
);
}
#[test]
fn fails_closed_when_candidate_would_overwrite_post_watermark_live_mutation() {
let tmp = TempDir::new().unwrap();
write_file(tmp.path(), "src/stale.rs", "fn stale_v1() {}\n");
let shared = LiveIndex::load(tmp.path()).unwrap();
let watermark = shared.read().as_ref().clone();
let mut candidate = {
let data =
LiveIndex::build_reload_data(tmp.path()).expect("reload data should build");
LiveIndex::from_reload_data(data)
};
let mut current = watermark.clone();
current.update_file(
"src/stale.rs".to_string(),
indexed_file_with_hash("src/stale.rs", b"fn stale_v2() {}\n", "hash_v2"),
);
let candidate_hash = candidate
.get_file("src/stale.rs")
.expect("precondition: candidate contains stale.rs")
.content_hash
.clone();
let error = merge_post_watermark_live_admissions(&watermark, ¤t, &mut candidate)
.expect_err("conflicting post-watermark mutation must fail closed");
assert!(
error
.to_string()
.contains("conflicts with the disk-built candidate"),
"unexpected error: {error:#}"
);
assert_eq!(
candidate
.get_file("src/stale.rs")
.expect("candidate file must remain present")
.content_hash,
candidate_hash,
"candidate must remain unchanged on refusal"
);
}
}
mod reload_admission_tiering {
use super::*;
use crate::domain::index::SkipReason;
fn reload_index(root: &Path) -> LiveIndex {
let data = LiveIndex::build_reload_data(root).expect("reload should succeed");
let mut index = LiveIndex::empty_live_index();
index.apply_reload_data(data);
index
}
#[test]
fn reload_demotes_lockfile_to_tier2_with_dependency_lockfile_reason() {
let tmp = TempDir::new().unwrap();
write_file(tmp.path(), "src/main.rs", "fn main() {}\n");
write_file(
tmp.path(),
"package-lock.json",
"{ \"name\": \"x\", \"lockfileVersion\": 3, \"packages\": {} }\n",
);
let index = reload_index(tmp.path());
assert!(
!index.files.contains_key("package-lock.json"),
"lockfile must not be parsed/indexed on reload; files = {:?}",
index.files.keys().collect::<Vec<_>>()
);
assert!(
index.files.contains_key("src/main.rs"),
"normal source must still be indexed on reload"
);
let lockfile_skip = index
.compatibility_skipped_files()
.into_iter()
.find(|sf| sf.path.replace('\\', "/") == "package-lock.json")
.expect("lockfile must be recorded in skipped_files on reload");
assert_eq!(
lockfile_skip.tier(),
AdmissionTier::MetadataOnly,
"lockfile must be Tier-2 metadata-only"
);
assert_eq!(
lockfile_skip.reason(),
Some(SkipReason::DependencyLockfile),
"lockfile skip reason must be DependencyLockfile"
);
assert_eq!(
index.tier_counts(),
(1, 1, 0),
"reload tier counts must be (tier1=1, tier2=1, tier3=0)"
);
}
#[test]
fn reload_demotes_oversized_text_file_to_tier2_size_threshold() {
let tmp = TempDir::new().unwrap();
write_file(tmp.path(), "src/lib.rs", "pub fn helper() -> i32 { 42 }\n");
let big = "x".repeat(1_500_000);
write_file(tmp.path(), "data/big.json", &big);
let index = reload_index(tmp.path());
assert!(
!index.files.contains_key("data/big.json"),
"oversized file must not be parsed/indexed on reload"
);
assert!(
index.files.contains_key("src/lib.rs"),
"normal source must still be indexed on reload"
);
let big_skip = index
.compatibility_skipped_files()
.into_iter()
.find(|sf| sf.path.replace('\\', "/") == "data/big.json")
.expect("oversized file must be recorded in skipped_files on reload");
assert_eq!(big_skip.tier(), AdmissionTier::MetadataOnly);
assert_eq!(
big_skip.reason(),
Some(SkipReason::SizeThreshold),
"oversized file skip reason must be SizeThreshold"
);
assert_eq!(
index.tier_counts(),
(1, 1, 0),
"reload tier counts must be (tier1=1, tier2=1, tier3=0)"
);
}
}
#[test]
fn test_live_index_load_valid_files_produces_ready_state() {
let tmp = TempDir::new().unwrap();
write_file(tmp.path(), "a.rs", "fn alpha() {}");
write_file(tmp.path(), "b.py", "def beta(): pass");
write_file(tmp.path(), "c.js", "function gamma() {}");
write_file(tmp.path(), "d.ts", "function delta(): void {}");
write_file(tmp.path(), "e.go", "package main\nfunc epsilon() {}");
let shared = LiveIndex::load(tmp.path()).unwrap();
let index = shared.read();
assert!(
!index.cb_state.is_tripped(),
"valid files should not trip circuit breaker"
);
assert_eq!(index.file_count(), 5);
assert_eq!(index.load_source(), IndexLoadSource::FreshLoad);
assert_eq!(
index.snapshot_verify_state(),
SnapshotVerifyState::NotNeeded
);
}
#[test]
fn coupling_store_accessor_is_none_when_flag_unset() {
let _lock = COUPLING_ENV_LOCK.lock().unwrap();
let _env = CouplingEnvGuard::set(None);
let tmp = TempDir::new().unwrap();
git2::Repository::init(tmp.path()).unwrap();
write_file(tmp.path(), "src/lib.rs", "pub fn alpha() {}");
let project_state =
crate::domain::ProjectStateDir::new(tmp.path().join(crate::paths::SYMFORGE_DIR_NAME));
let placement = crate::domain::StatePlacement::ProjectLocal {
directory: project_state.clone(),
};
let shared = LiveIndex::load_for_state_placement(tmp.path(), &placement).unwrap();
let db_path = crate::live_index::coupling::lifecycle::coupling_db_path(&project_state);
assert!(shared.read().coupling_store().is_none());
assert!(
!db_path.exists(),
"flag-off load must not create the coupling database"
);
}
#[test]
fn coupling_store_accessor_is_some_when_flag_enabled_for_git_workspace() {
let _lock = COUPLING_ENV_LOCK.lock().unwrap();
let _env = CouplingEnvGuard::set(Some("1"));
let tmp = TempDir::new().unwrap();
git2::Repository::init(tmp.path()).unwrap();
write_file(tmp.path(), "src/lib.rs", "pub fn alpha() {}");
let project_state =
crate::domain::ProjectStateDir::new(tmp.path().join(crate::paths::SYMFORGE_DIR_NAME));
let placement = crate::domain::StatePlacement::ProjectLocal {
directory: project_state,
};
let shared = LiveIndex::load_for_state_placement(tmp.path(), &placement).unwrap();
let index = shared.read();
let store = index
.coupling_store()
.expect("flag-on git workspace should expose coupling store");
assert_eq!(
store.schema_version().unwrap(),
crate::live_index::coupling::schema::CURRENT_SCHEMA_VERSION
);
}
#[test]
fn test_live_index_load_circuit_breaker_not_tripped_with_all_languages() {
let tmp = TempDir::new().unwrap();
write_file(tmp.path(), "a.rs", "fn alpha() {}");
write_file(tmp.path(), "b.py", "def beta(): pass");
write_file(tmp.path(), "c.js", "function gamma() {}");
write_file(tmp.path(), "x.swift", "class A {}");
write_file(tmp.path(), "y.php", "<?php class B {}");
write_file(tmp.path(), "z.pl", "sub greet { print \"hi\"; }");
let shared = LiveIndex::load(tmp.path()).unwrap();
let index = shared.read();
assert!(
!index.cb_state.is_tripped(),
"all-parseable files should not trip circuit breaker"
);
}
#[test]
fn test_live_index_file_count() {
let tmp = TempDir::new().unwrap();
write_file(tmp.path(), "a.rs", "fn a() {}");
write_file(tmp.path(), "b.rs", "fn b() {}");
write_file(tmp.path(), "c.rs", "fn c() {}");
let shared = LiveIndex::load(tmp.path()).unwrap();
let index = shared.read();
assert_eq!(index.file_count(), 3);
}
static INFLIGHT_ENV_LOCK: StdMutex<()> = StdMutex::new(());
struct InflightEnvGuard {
previous: Option<String>,
}
#[allow(unsafe_code)] impl InflightEnvGuard {
fn set(value: Option<&str>) -> Self {
let previous = std::env::var(MAX_INFLIGHT_BYTES_ENV).ok();
unsafe {
match value {
Some(value) => std::env::set_var(MAX_INFLIGHT_BYTES_ENV, value),
None => std::env::remove_var(MAX_INFLIGHT_BYTES_ENV),
}
}
Self { previous }
}
}
#[allow(unsafe_code)] impl Drop for InflightEnvGuard {
fn drop(&mut self) {
unsafe {
match self.previous.as_deref() {
Some(value) => std::env::set_var(MAX_INFLIGHT_BYTES_ENV, value),
None => std::env::remove_var(MAX_INFLIGHT_BYTES_ENV),
}
}
}
}
#[test]
fn stable_read_refuses_over_ceiling_before_allocation() {
struct PanicStableReadAccess;
impl StableReadAccess for PanicStableReadAccess {
fn first_pass(
&self,
_path: &Path,
_max_bytes: usize,
) -> std::io::Result<StableReadPass> {
panic!("over-ceiling input must be rejected before allocation/read")
}
fn second_pass(
&self,
_path: &Path,
_max_bytes: usize,
) -> std::io::Result<StableReadPass> {
panic!("over-ceiling input must be rejected before allocation/read")
}
}
let scout_stamp = crate::domain::FileStamp {
size: 2_048,
created_hint: None,
modified_hint: None,
platform_id: None,
};
let outcome = stable_read_with_access(
Path::new("oversized.rs"),
&scout_stamp,
StableReadLimits {
per_file_bytes: 1_024,
inflight_bytes: 4_096,
},
&PanicStableReadAccess,
);
assert!(matches!(
outcome,
StableReadOutcome::HardSkip {
reason: HardSkipReason::PerFileCeiling
}
));
}
#[test]
fn read_larger_than_inflight_budget_is_terminal_hard_skip() {
struct PanicOverInflightAccess;
impl StableReadAccess for PanicOverInflightAccess {
fn first_pass(
&self,
_path: &Path,
_max_bytes: usize,
) -> std::io::Result<StableReadPass> {
panic!("over-inflight input must be rejected before allocation/read")
}
fn second_pass(
&self,
_path: &Path,
_max_bytes: usize,
) -> std::io::Result<StableReadPass> {
panic!("over-inflight input must be rejected before allocation/read")
}
}
let stamp = crate::domain::FileStamp {
size: 2_048,
created_hint: None,
modified_hint: None,
platform_id: None,
};
let outcome = stable_read_with_access(
Path::new("over-inflight.rs"),
&stamp,
StableReadLimits {
per_file_bytes: 4_096,
inflight_bytes: 1_024,
},
&PanicOverInflightAccess,
);
assert!(matches!(
outcome,
StableReadOutcome::HardSkip {
reason: HardSkipReason::PerFileCeiling
}
));
}
#[test]
fn stable_read_rejects_changed_manifest_stamp() {
use std::cell::Cell;
struct ChangedStampAccess<'a> {
first_calls: &'a Cell<usize>,
second_calls: &'a Cell<usize>,
first: StableReadPass,
}
impl StableReadAccess for ChangedStampAccess<'_> {
fn first_pass(
&self,
_path: &Path,
_max_bytes: usize,
) -> std::io::Result<StableReadPass> {
self.first_calls.set(self.first_calls.get() + 1);
Ok(self.first.clone())
}
fn second_pass(
&self,
_path: &Path,
_max_bytes: usize,
) -> std::io::Result<StableReadPass> {
self.second_calls.set(self.second_calls.get() + 1);
panic!("changed first-pass stamp must reject before the second read")
}
}
let scout_stamp = crate::domain::FileStamp {
size: 4,
created_hint: None,
modified_hint: Some(std::time::UNIX_EPOCH),
platform_id: None,
};
let changed_stamp = crate::domain::FileStamp {
modified_hint: Some(std::time::UNIX_EPOCH + Duration::from_secs(1)),
..scout_stamp.clone()
};
let first_calls = Cell::new(0);
let second_calls = Cell::new(0);
let access = ChangedStampAccess {
first_calls: &first_calls,
second_calls: &second_calls,
first: StableReadPass {
bytes: Some(b"same".to_vec()),
length: 4,
hash: crate::hash::digest(b"same"),
handle_before: changed_stamp.clone(),
handle_after: changed_stamp.clone(),
path_after: changed_stamp,
},
};
let outcome = stable_read_with_access(
Path::new("changed.rs"),
&scout_stamp,
StableReadLimits {
per_file_bytes: 1_024,
inflight_bytes: 1_024,
},
&access,
);
assert!(matches!(outcome, StableReadOutcome::UnstableDuringRead));
assert_eq!(first_calls.get(), 1);
assert_eq!(
second_calls.get(),
0,
"changed scout/open-handle state must short-circuit the second pass"
);
}
#[test]
fn read_failure_retains_unreadable_disposition() {
use std::cell::Cell;
struct FailingReadAccess<'a> {
first_calls: &'a Cell<usize>,
second_calls: &'a Cell<usize>,
}
impl StableReadAccess for FailingReadAccess<'_> {
fn first_pass(
&self,
_path: &Path,
_max_bytes: usize,
) -> std::io::Result<StableReadPass> {
self.first_calls.set(self.first_calls.get() + 1);
Err(std::io::Error::new(
std::io::ErrorKind::PermissionDenied,
"injected read refusal",
))
}
fn second_pass(
&self,
_path: &Path,
_max_bytes: usize,
) -> std::io::Result<StableReadPass> {
self.second_calls.set(self.second_calls.get() + 1);
panic!("failed first read must not attempt a second pass")
}
}
let stamp = crate::domain::FileStamp {
size: 4,
created_hint: None,
modified_hint: None,
platform_id: None,
};
let first_calls = Cell::new(0);
let second_calls = Cell::new(0);
let access = FailingReadAccess {
first_calls: &first_calls,
second_calls: &second_calls,
};
let outcome = stable_read_with_access(
Path::new("locked.rs"),
&stamp,
StableReadLimits {
per_file_bytes: 1_024,
inflight_bytes: 1_024,
},
&access,
);
assert!(matches!(
outcome,
StableReadOutcome::Unreadable {
stage: crate::domain::AccessStage::FullRead,
kind: crate::domain::AccessErrorKind::PermissionDenied,
}
));
assert_eq!(first_calls.get(), 1);
assert_eq!(second_calls.get(), 0);
}
#[test]
fn stable_read_double_pass_rejects_same_stamp_torn_write() {
struct ScriptedStableReadAccess {
first: StableReadPass,
second: StableReadPass,
}
impl StableReadAccess for ScriptedStableReadAccess {
fn first_pass(
&self,
_path: &Path,
_max_bytes: usize,
) -> std::io::Result<StableReadPass> {
Ok(self.first.clone())
}
fn second_pass(
&self,
_path: &Path,
_max_bytes: usize,
) -> std::io::Result<StableReadPass> {
Ok(self.second.clone())
}
}
let stamp = crate::domain::FileStamp {
size: 4,
created_hint: None,
modified_hint: Some(std::time::UNIX_EPOCH),
platform_id: None,
};
let pass = |bytes: Option<Vec<u8>>, payload: &[u8]| StableReadPass {
bytes,
length: 4,
hash: crate::hash::digest(payload),
handle_before: stamp.clone(),
handle_after: stamp.clone(),
path_after: stamp.clone(),
};
let limits = StableReadLimits {
per_file_bytes: 1_024,
inflight_bytes: 1_024,
};
let accepted = stable_read_with_access(
Path::new("stable.rs"),
&stamp,
limits,
&ScriptedStableReadAccess {
first: pass(Some(b"same".to_vec()), b"same"),
second: pass(None, b"same"),
},
);
assert!(matches!(
accepted,
StableReadOutcome::Accepted { ref bytes, hash }
if bytes == b"same" && hash == crate::hash::digest(b"same")
));
let torn = stable_read_with_access(
Path::new("torn.rs"),
&stamp,
limits,
&ScriptedStableReadAccess {
first: pass(Some(b"same".to_vec()), b"same"),
second: pass(None, b"torn"),
},
);
assert!(matches!(torn, StableReadOutcome::UnstableDuringRead));
}
#[test]
fn filesystem_stable_read_accepts_unchanged_bytes_exactly() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("exact.rs");
let expected = b"fn exact() {\r\n println!(\"byte exact\");\r\n}\r\n";
std::fs::write(&path, expected).unwrap();
let stamp = file_stamp_from_metadata(&std::fs::metadata(&path).unwrap());
let outcome = stable_read_with_retries(
&path,
&stamp,
StableReadLimits {
per_file_bytes: crate::domain::index::HARD_SKIP_BYTES,
inflight_bytes: crate::domain::index::HARD_SKIP_BYTES,
},
&FilesystemStableReadAccess,
);
assert!(matches!(
outcome,
StableReadOutcome::Accepted { bytes, hash }
if bytes == expected && hash == crate::hash::digest(expected)
));
}
#[test]
fn inflight_budget_releases_on_permit_drop() {
let budget = Arc::new(InflightByteBudget::new(1000));
assert_eq!(budget.available_bytes(), 1000);
let permit_a = budget.acquire(400);
assert_eq!(budget.available_bytes(), 600);
let permit_b = budget.acquire(600);
assert_eq!(budget.available_bytes(), 0);
drop(permit_a);
assert_eq!(budget.available_bytes(), 400);
drop(permit_b);
assert_eq!(budget.available_bytes(), 1000);
}
#[test]
fn inflight_permit_releases_at_staged_handoff_without_deadlock() {
let inflight = Arc::new(InflightByteBudget::new(4));
let staged = StagedContentAccounting::new(8);
let first = inflight.acquire(4);
assert_eq!(inflight.available_bytes(), 0);
assert!(staged.handoff(4, Some(first)));
assert_eq!(staged.used_bytes(), 4);
assert_eq!(inflight.available_bytes(), 4);
let second = inflight.acquire(4);
assert_eq!(inflight.available_bytes(), 0);
assert!(staged.handoff(4, Some(second)));
assert_eq!(staged.used_bytes(), 8);
assert_eq!(inflight.available_bytes(), 4);
}
#[test]
fn inflight_budget_clamps_oversized_request_to_total() {
let budget = Arc::new(InflightByteBudget::new(256));
let permit = budget.acquire(10_000_000);
assert_eq!(budget.available_bytes(), 0, "clamped to the full budget");
drop(permit);
assert_eq!(budget.available_bytes(), 256);
}
#[test]
fn inflight_budget_zero_total_does_not_deadlock() {
let budget = Arc::new(InflightByteBudget::new(0));
assert!(budget.available_bytes() >= 1);
let permit = budget.acquire(123);
drop(permit);
assert!(budget.available_bytes() >= 1);
}
#[test]
fn inflight_budget_blocks_until_capacity_frees() {
use std::sync::mpsc;
use std::thread;
let budget = Arc::new(InflightByteBudget::new(512 * 1024));
let first = budget.acquire(512 * 1024);
assert_eq!(budget.available_bytes(), 0);
let (tx, rx) = mpsc::channel();
let budget_clone = Arc::clone(&budget);
let waiter = thread::spawn(move || {
let _permit = budget_clone.acquire(512 * 1024);
tx.send(()).expect("send acquisition signal");
thread::sleep(Duration::from_millis(20));
});
assert!(
rx.recv_timeout(Duration::from_millis(100)).is_err(),
"second acquirer should block while the budget is exhausted"
);
drop(first);
rx.recv_timeout(Duration::from_secs(2))
.expect("waiter should acquire once budget frees");
waiter.join().expect("waiter thread joins");
assert_eq!(budget.available_bytes(), 512 * 1024);
}
#[test]
fn load_under_tight_inflight_budget_still_indexes_all_large_files() {
let _lock = INFLIGHT_ENV_LOCK.lock().unwrap();
let _env = InflightEnvGuard::set(Some(&(512 * 1024).to_string()));
let tmp = TempDir::new().unwrap();
const FNS_PER_FILE: u32 = 24_000;
let mut body = String::with_capacity(400 * 1024);
for i in 0..FNS_PER_FILE {
use std::fmt::Write;
writeln!(body, "fn f_{i:07}() {{}}").unwrap();
}
assert!(
(body.len() as u64) < crate::domain::index::METADATA_ONLY_BYTES,
"fixture ({} bytes) must stay Normal-tier (under the {} metadata ceiling)",
body.len(),
crate::domain::index::METADATA_ONLY_BYTES
);
const FILE_COUNT: usize = 6;
for n in 0..FILE_COUNT {
write_file(tmp.path(), &format!("big_{n}.rs"), &body);
}
write_file(tmp.path(), "tiny.rs", "fn tiny() {}");
let shared = LiveIndex::load(tmp.path()).unwrap();
let index = shared.read();
assert!(
!index.cb_state.is_tripped(),
"valid large files must not trip the circuit breaker"
);
assert_eq!(
index.file_count(),
FILE_COUNT + 1,
"all large files plus the tiny file must remain indexed under the cap"
);
assert!(
index.symbol_count() as u64 >= FILE_COUNT as u64 * FNS_PER_FILE as u64,
"large files must be fully parsed ({} symbols), not metadata-skipped",
index.symbol_count()
);
}
#[test]
fn test_live_index_symbol_count() {
let tmp = TempDir::new().unwrap();
write_file(tmp.path(), "a.rs", "fn foo() {}\nfn bar() {}");
write_file(tmp.path(), "b.rs", "fn baz() {}");
let shared = LiveIndex::load(tmp.path()).unwrap();
let index = shared.read();
assert_eq!(index.symbol_count(), 3);
}
#[test]
fn test_live_index_empty_has_zero_files() {
let shared = LiveIndex::empty();
let index = shared.read();
assert_eq!(index.file_count(), 0);
assert_eq!(index.load_source(), IndexLoadSource::EmptyBootstrap);
assert_eq!(
index.snapshot_verify_state(),
SnapshotVerifyState::NotNeeded
);
}
#[test]
fn test_shared_index_handle_preserves_read_write_access() {
let shared = LiveIndex::empty();
{
let mut live = shared.write();
live.add_file(
"src/new.rs".to_string(),
make_indexed_file_for_mutation("src/new.rs"),
);
}
let index = shared.read();
assert!(index.get_file("src/new.rs").is_some());
}
#[test]
fn test_shared_index_handle_published_state_tracks_generation_and_counts() {
let shared = LiveIndex::empty();
let initial = shared.published_state();
assert_eq!(initial.generation, 0);
assert_eq!(initial.status, PublishedIndexStatus::Empty);
assert_eq!(initial.degraded_summary, None);
assert_eq!(initial.file_count, 0);
assert_eq!(initial.parsed_count, 0);
assert_eq!(initial.partial_parse_count, 0);
assert_eq!(initial.failed_count, 0);
assert_eq!(initial.load_source, IndexLoadSource::EmptyBootstrap);
shared.add_file(
"src/new.rs".to_string(),
make_indexed_file_for_mutation("src/new.rs"),
);
let after_add = shared.published_state();
assert_eq!(after_add.generation, 1);
assert_eq!(after_add.status, PublishedIndexStatus::Loading);
assert_eq!(after_add.degraded_summary, None);
assert_eq!(after_add.file_count, 1);
assert_eq!(after_add.parsed_count, 1);
assert_eq!(after_add.partial_parse_count, 0);
assert_eq!(after_add.failed_count, 0);
assert_eq!(after_add.symbol_count, 1);
shared.remove_file("src/new.rs");
let after_remove = shared.published_state();
assert_eq!(after_remove.generation, 2);
assert_eq!(after_remove.status, PublishedIndexStatus::Loading);
assert_eq!(after_remove.degraded_summary, None);
assert_eq!(after_remove.file_count, 0);
assert_eq!(after_remove.symbol_count, 0);
}
#[test]
fn published_generation_is_atomic_under_concurrent_reloads() {
let shared = LiveIndex::empty();
let mut replacement = (*shared.live.load_full()).clone();
replacement.add_file(
"src/new_generation.rs".to_string(),
make_indexed_file_for_mutation("src/new_generation.rs"),
);
let _writer = shared.write_mutex.lock();
shared.swap_and_publish_with_hook(replacement, || {
let live_file_count = shared.read().file_count();
let health_file_count = shared.published_state().file_count;
let outline_file_count = shared.published_repo_outline().total_files;
assert_eq!(
live_file_count, health_file_count,
"live content and health must come from one captured generation"
);
assert_eq!(
health_file_count, outline_file_count,
"health and outline must come from one captured generation"
);
});
}
#[test]
fn test_reset_to_empty_invalidates_populated_index_and_bumps_generation() {
let shared = LiveIndex::empty();
shared.add_file(
"src/old_project.rs".to_string(),
make_indexed_file_for_mutation("src/old_project.rs"),
);
let before = shared.published_state();
assert_eq!(before.file_count, 1, "precondition: index has stale file");
let project_gen_before = shared.current_project_generation();
shared.write().manifest_entries = vec![make_manifest_entry(
"src/old_project.rs",
1,
FileDisposition::Indexed {
targets: crate::domain::IndexTargets::Code,
parse_status: crate::domain::index::ParseStatus::Parsed,
},
)];
shared.reset_to_empty();
let after = shared.published_state();
assert_eq!(
after.file_count, 0,
"reset_to_empty must drop all indexed files so ensure_local_index reloads the new root"
);
assert_eq!(
after.symbol_count, 0,
"reset_to_empty must drop all symbols"
);
assert_eq!(
after.status,
PublishedIndexStatus::Empty,
"reset_to_empty must publish Empty status"
);
assert_eq!(
after.load_source,
IndexLoadSource::EmptyBootstrap,
"reset_to_empty must mark the index as a fresh empty bootstrap"
);
assert!(
shared.read().get_file("src/old_project.rs").is_none(),
"stale file must be unreachable after reset"
);
assert!(
shared.current_project_generation() > project_gen_before,
"reset_to_empty must bump project generation to fence stale watcher mutations"
);
assert_eq!(
shared.published_generation().project_generation,
shared.current_project_generation(),
"the replacement project's generation must be captured inside the same publication root"
);
assert!(
shared.terminal_dispositions().is_empty(),
"reset_to_empty must drop terminal dispositions from the previous project"
);
}
#[test]
fn rejected_stale_mutations_counter_increments_on_fence_rejection() {
let dir_a = TempDir::new().unwrap();
write_file(dir_a.path(), "src/a.rs", "pub fn from_a() {}\n");
let shared = LiveIndex::load(dir_a.path()).unwrap();
let gen_a = shared.current_project_generation();
assert_eq!(shared.current_rejected_stale_mutations(), 0);
let dir_b = TempDir::new().unwrap();
write_file(dir_b.path(), "src/b.rs", "pub fn from_b() {}\n");
shared.reload(dir_b.path()).unwrap();
assert!(
shared.current_project_generation() > gen_a,
"reload must advance project generation before stale mutations are checked"
);
assert_eq!(
shared.published_generation().project_generation,
shared.current_project_generation(),
"reload must publish the new project generation inside the replacement root"
);
assert!(!shared.remove_file_at_generation("src/a.rs", gen_a));
assert_eq!(shared.current_rejected_stale_mutations(), 1);
let indexed = make_indexed_file_for_mutation("src/stale.rs");
assert!(!shared.update_file_at_generation("src/stale.rs", indexed, gen_a));
assert_eq!(shared.current_rejected_stale_mutations(), 2);
}
#[test]
fn reload_builds_a_direct_replacement_without_mutating_previous_generation() {
let shared = LiveIndex::empty();
shared.set_local_empty_reason(Some("prior-generation-reason".to_string()));
let previous = shared.published_generation();
assert_eq!(
previous.live.local_empty_reason().as_deref(),
Some("prior-generation-reason")
);
let replacement = TempDir::new().unwrap();
write_file(
replacement.path(),
"src/replacement.rs",
"pub fn replacement() {}\n",
);
shared.reload(replacement.path()).unwrap();
let current = shared.published_generation();
assert_eq!(current.live.local_empty_reason(), None);
assert_eq!(
previous.live.local_empty_reason().as_deref(),
Some("prior-generation-reason"),
"reload must not clear an Arc-owned field inside the previously published root"
);
assert!(!Arc::ptr_eq(&previous.live, ¤t.live));
}
#[test]
fn test_shared_index_handle_write_guard_publishes_on_drop() {
let shared = LiveIndex::empty();
{
let mut live = shared.write();
live.add_file(
"src/new.rs".to_string(),
make_indexed_file_for_mutation("src/new.rs"),
);
}
let after_add = shared.published_state();
assert_eq!(after_add.generation, 1);
assert_eq!(after_add.status, PublishedIndexStatus::Loading);
assert_eq!(after_add.degraded_summary, None);
assert_eq!(after_add.file_count, 1);
{
let mut live = shared.write();
live.remove_file("src/new.rs");
}
let after_remove = shared.published_state();
assert_eq!(after_remove.generation, 2);
assert_eq!(after_remove.status, PublishedIndexStatus::Loading);
assert_eq!(after_remove.degraded_summary, None);
assert_eq!(after_remove.file_count, 0);
}
#[test]
fn generic_write_guard_invalidates_pre_update_snapshot_tokens() {
let shared = LiveIndex::empty();
shared.add_file(
"src/tracked.rs".to_string(),
make_indexed_file_for_mutation("src/tracked.rs"),
);
shared.update_file(
"src/tracked.rs".to_string(),
make_indexed_file_for_mutation("src/tracked.rs"),
);
let generation = shared.current_project_generation();
assert!(
shared
.peek_pre_update_snapshot_at_generation("src/tracked.rs", generation)
.is_some(),
"the exact update must create a baseline token"
);
{
let mut live = shared.write();
live.add_file(
"src/other.rs".to_string(),
make_indexed_file_for_mutation("src/other.rs"),
);
}
assert!(
shared
.peek_pre_update_snapshot_at_generation("src/tracked.rs", generation)
.is_none(),
"an arbitrary write publication must invalidate old path tokens"
);
}
#[test]
fn scout_reconciliation_removal_refuses_a_recreated_disk_path() {
let dir = TempDir::new().expect("root");
let path = dir.path().join("lib.rs");
fs::write(&path, "pub fn before() {}\n").expect("seed");
let shared = LiveIndex::load(dir.path()).expect("load");
let expected_entry = shared
.scout_plan
.load_full()
.expect("scout plan")
.entries
.iter()
.find(|entry| entry.path.normalized_utf8.as_deref() == Some("lib.rs"))
.cloned()
.expect("lib.rs scout entry");
fs::remove_file(&path).expect("scan observes absence");
fs::write(&path, "pub fn recreated() {}\n").expect("recreate before removal");
assert!(
!shared.remove_file_if_scout_entry_at_generation(
"lib.rs",
&expected_entry,
shared.current_project_generation(),
),
"a complete-scout deletion must re-confirm absence at publication"
);
assert!(shared.read().get_file("lib.rs").is_some());
}
#[test]
fn mtime_only_update_is_visible_in_published_root_without_advancing_content_generation() {
let shared = LiveIndex::empty();
shared.add_file(
"src/touched.rs".to_string(),
make_indexed_file_for_mutation("src/touched.rs"),
);
let before = shared.published_generation();
let before_mtime = before
.live
.get_file("src/touched.rs")
.expect("fixture file")
.mtime_secs;
let next_mtime = before_mtime.saturating_add(42);
shared.touch_mtime("src/touched.rs", next_mtime);
let after = shared.published_generation();
assert_eq!(
after
.live
.get_file("src/touched.rs")
.expect("published fixture file")
.mtime_secs,
next_mtime,
"mtime-only updates must be visible through the immutable publication root"
);
assert!(after.publication_generation > before.publication_generation);
assert_eq!(after.content_generation, before.content_generation);
}
#[test]
fn local_empty_reason_is_published_in_the_same_immutable_root() {
let shared = LiveIndex::empty();
let before = shared.published_generation();
shared.set_local_empty_reason(Some("workspace is not bound".to_string()));
let after = shared.published_generation();
assert_eq!(
after.health.local_empty_reason.as_deref(),
Some("workspace is not bound")
);
assert_eq!(
after.live.local_empty_reason().as_deref(),
Some("workspace is not bound")
);
assert!(after.publication_generation > before.publication_generation);
assert_eq!(after.content_generation, before.content_generation);
assert_eq!(before.live.local_empty_reason(), None);
}
#[test]
fn published_source_set_is_the_single_atomic_root_for_current_source() {
let tmp = TempDir::new().unwrap();
write_file(tmp.path(), "src/lib.rs", "pub fn source_set() {}\n");
let shared = LiveIndex::load(tmp.path()).unwrap();
let before = shared.published_source_set();
let before_current = before.current_generation();
assert_eq!(before.sources.len(), 1);
assert_eq!(
before_current
.manifest
.as_ref()
.expect("bound manifest")
.source
.source_id,
before.current_source_id
);
shared.update_file(
"src/next.rs".to_string(),
make_indexed_file_for_mutation("src/next.rs"),
);
let after = shared.published_source_set();
assert!(after.registry_generation > before.registry_generation);
assert!(Arc::ptr_eq(&before_current, &before.current_generation()));
assert!(!Arc::ptr_eq(&before_current, &after.current_generation()));
}
#[test]
fn test_shared_index_handle_published_state_tracks_verify_transitions() {
let mut live = make_empty_live_index();
live.is_empty = false;
live.load_source = IndexLoadSource::SnapshotRestore;
live.snapshot_verify_state = SnapshotVerifyState::Pending;
let shared = SharedIndexHandle::shared(live);
let initial = shared.published_state();
assert_eq!(initial.file_count, 0);
assert_eq!(initial.partial_parse_count, 0);
assert_eq!(initial.failed_count, 0);
shared.mark_snapshot_verify_running();
let running = shared.published_state();
assert_eq!(running.generation, 1);
assert_eq!(running.status, PublishedIndexStatus::Loading);
assert_eq!(running.degraded_summary, None);
assert!(matches!(
running.snapshot_verify_state,
SnapshotVerifyState::Running(_)
));
assert_eq!(running.file_count, initial.file_count);
assert_eq!(running.partial_parse_count, initial.partial_parse_count);
assert_eq!(running.failed_count, initial.failed_count);
shared.mark_snapshot_verify_completed(Vec::new());
let completed = shared.published_state();
assert_eq!(completed.generation, 2);
assert_eq!(
completed.snapshot_verify_state,
SnapshotVerifyState::completed_without_mismatches()
);
assert_eq!(completed.file_count, initial.file_count);
assert_eq!(completed.partial_parse_count, initial.partial_parse_count);
assert_eq!(completed.failed_count, initial.failed_count);
}
#[test]
fn snapshot_verify_transitions_preserve_other_degraded_reasons() {
let project = TempDir::new().unwrap();
write_file(project.path(), "main.rs", "fn main() {}\n");
let loaded = LiveIndex::load(project.path()).unwrap();
let mut live = (*loaded.published_generation().live).clone();
live.load_source = IndexLoadSource::SnapshotRestore;
live.snapshot_verify_state = SnapshotVerifyState::Pending;
let mut degraded_plan = (*loaded.scout_plan().expect("cold scout plan")).clone();
degraded_plan.issues.push(crate::domain::ScoutIssue {
path_id: Some("transient-walk-entry".to_string()),
safe_path: Some("temporarily-locked".to_string()),
kind: crate::domain::ScoutIssueKind::DirectoryEntryUnreadable {
kind: crate::domain::AccessErrorKind::PermissionDenied,
},
safe_message: "directory entry unavailable".to_string(),
});
crate::discovery::refresh_scout_plan(&mut degraded_plan).unwrap();
let shared = SharedIndexHandle::shared_with_scout_plan(
live,
degraded_plan,
crate::discovery::SourceExclusions::default(),
);
assert!(matches!(
shared.published_generation().freshness.as_ref(),
FreshnessStatus::Degraded { reason_codes, .. }
if reason_codes.contains(&FreshnessReason::ReconciliationPending)
));
let running_fence = shared
.mark_snapshot_verify_running_at_fence(shared.publication_fence())
.expect("verification starts at the captured fence");
assert!(matches!(
shared.published_generation().freshness.as_ref(),
FreshnessStatus::Degraded { reason_codes, .. }
if reason_codes.contains(&FreshnessReason::ReconciliationPending)
));
assert!(shared.mark_snapshot_verify_completed_at_fence(running_fence, Vec::new()));
let completed = shared.published_generation();
assert!(matches!(
completed.live.snapshot_verify_state,
SnapshotVerifyState::Completed(_)
));
assert!(matches!(
completed.freshness.as_ref(),
FreshnessStatus::Degraded { reason_codes, .. }
if reason_codes.contains(&FreshnessReason::ReconciliationPending)
&& !reason_codes.contains(&FreshnessReason::SnapshotVerificationFailed)
));
}
#[test]
fn resolve_settles_a_republished_path_in_count_and_set_together() {
let mut live = make_empty_live_index();
live.is_empty = false;
live.load_source = IndexLoadSource::SnapshotRestore;
live.files.insert(
"src/a.rs".to_string(),
Arc::new(make_indexed_file_for_mutation("src/a.rs")),
);
let seed = Arc::new(make_indexed_file_for_mutation("src/a.rs"));
let report = SnapshotVerifyReport::from_mismatched_paths(vec!["src/a.rs".to_string()])
.with_unverified(BTreeMap::from([(
"src/a.rs".to_string(),
"it could not be read".to_string(),
)]));
let withheld = live.resolve_snapshot_verify(SnapshotVerifyCompletion {
report,
restored_rows: vec![("src/a.rs".to_string(), Some(seed))],
});
assert!(!withheld, "another writer's row is current, not withheld");
assert!(live.files.contains_key("src/a.rs"));
match &live.snapshot_verify_state {
SnapshotVerifyState::Completed(report) => {
assert!(report.unverified.is_empty());
assert_eq!(report.mismatch_count, 0, "{report:?}");
assert!(report.mismatched_paths.is_empty(), "{report:?}");
}
other => panic!("expected a completed verify, got {other:?}"),
}
}
#[test]
fn test_shared_index_handle_published_state_bounds_snapshot_verify_mismatch_paths() {
let mut live = make_empty_live_index();
live.is_empty = false;
live.load_source = IndexLoadSource::SnapshotRestore;
live.snapshot_verify_state = SnapshotVerifyState::Pending;
let shared = SharedIndexHandle::shared(live);
let mismatch_paths = (0..12)
.rev()
.map(|i| format!("src/mismatch_{i:02}.rs"))
.collect::<Vec<_>>();
shared.mark_snapshot_verify_completed(mismatch_paths);
let completed = shared.published_state();
match &completed.snapshot_verify_state {
SnapshotVerifyState::Completed(report) => {
assert_eq!(report.mismatch_count, 12);
assert_eq!(report.mismatched_paths.len(), 10);
assert_eq!(report.mismatched_paths[0], "src/mismatch_00.rs");
assert_eq!(report.mismatched_paths[9], "src/mismatch_09.rs");
assert_eq!(report.omitted_path_count(), 2);
}
other => panic!("expected completed snapshot verify report, got {other:?}"),
}
}
#[test]
fn test_shared_index_handle_published_state_captures_degraded_summary() {
let mut live = make_empty_live_index();
live.is_empty = false;
for _ in 0..3 {
live.cb_state.record_failure("src/bad.rs", "parse failure");
}
for _ in 0..7 {
live.cb_state.record_success();
}
assert!(live.cb_state.should_abort(), "circuit breaker should trip");
let shared = SharedIndexHandle::shared(live);
let published = shared.published_state();
assert_eq!(published.status, PublishedIndexStatus::Degraded);
assert!(
published
.degraded_summary
.as_deref()
.is_some_and(|summary| summary.contains("circuit breaker tripped")),
"expected degraded summary, got {:?}",
published.degraded_summary
);
}
#[test]
fn test_shared_index_handle_published_repo_outline_tracks_mutations() {
let shared = LiveIndex::empty();
let initial = shared.published_repo_outline();
assert_eq!(initial.total_files, 0);
assert_eq!(initial.total_symbols, 0);
assert!(initial.files.is_empty());
shared.add_file(
"src/main.rs".to_string(),
make_indexed_file_for_mutation("src/main.rs"),
);
let after_add = shared.published_repo_outline();
assert_eq!(after_add.total_files, 1);
assert_eq!(after_add.total_symbols, 1);
assert_eq!(after_add.files[0].relative_path, "src/main.rs");
{
let mut live = shared.write();
live.remove_file("src/main.rs");
}
let after_remove = shared.published_repo_outline();
assert_eq!(after_remove.total_files, 0);
assert_eq!(after_remove.total_symbols, 0);
assert!(after_remove.files.is_empty());
}
#[test]
fn test_live_index_empty_returns_empty_state() {
let shared = LiveIndex::empty();
let index = shared.read();
assert_eq!(index.index_state(), IndexState::Empty);
}
#[test]
fn test_live_index_empty_is_not_ready() {
let shared = LiveIndex::empty();
let index = shared.read();
assert!(!index.is_ready(), "empty index should not be ready");
}
#[test]
fn bootstrap_placeholder_that_admitted_a_file_is_loading_not_ready() {
let shared = LiveIndex::empty();
shared.update_file(
"src/admitted_before_load.rs".to_string(),
make_indexed_file_for_mutation("src/admitted_before_load.rs"),
);
assert_eq!(shared.read().index_state(), IndexState::Loading);
}
#[test]
fn from_indexed_files_is_ready_and_source_bound() {
let tmp = TempDir::new().unwrap();
let shared = LiveIndex::from_indexed_files(
tmp.path(),
vec![(
"src/lib.rs".to_string(),
make_indexed_file_for_mutation("src/lib.rs"),
)],
)
.expect("existing root");
assert_eq!(shared.read().index_state(), IndexState::Ready);
assert_eq!(shared.read().load_source(), IndexLoadSource::FreshLoad);
assert!(
shared.published_state().indexed_root.is_some(),
"an in-memory index built at a root records that root"
);
assert!(
shared.published_generation().source.is_some(),
"a rooted in-memory index publishes a bound source identity"
);
let placeholder = LiveIndex::empty();
placeholder.update_file(
"src/lib.rs".to_string(),
make_indexed_file_for_mutation("src/lib.rs"),
);
assert_eq!(placeholder.read().index_state(), IndexState::Loading);
assert!(placeholder.published_generation().source.is_none());
assert!(LiveIndex::from_indexed_files(&tmp.path().join("nope"), Vec::new()).is_err());
}
#[test]
fn unbound_no_root_placeholder_that_admitted_a_file_stays_empty_not_loading() {
let shared = LiveIndex::empty();
shared.set_local_empty_reason(Some("workspace is not bound".to_string()));
shared.update_file(
"src/admitted_via_sidecar.rs".to_string(),
make_indexed_file_for_mutation("src/admitted_via_sidecar.rs"),
);
assert_eq!(shared.read().index_state(), IndexState::Empty);
}
#[test]
fn mutation_on_a_rooted_loaded_index_stays_ready() {
let tmp = TempDir::new().unwrap();
write_file(tmp.path(), "a.rs", "fn alpha() {}");
let shared = LiveIndex::empty();
{
let mut index = shared.write();
index.reload(tmp.path()).expect("reload should succeed");
}
assert_eq!(shared.published_state().status, PublishedIndexStatus::Ready);
shared.update_file(
"src/added_after_load.rs".to_string(),
make_indexed_file_for_mutation("src/added_after_load.rs"),
);
assert_eq!(shared.published_state().status, PublishedIndexStatus::Ready);
assert_eq!(shared.read().index_state(), IndexState::Ready);
}
#[test]
fn unbound_bootstrap_rebinds_writable_project_without_restart() {
let tmp = TempDir::new().unwrap();
write_file(tmp.path(), "a.rs", "fn alpha() {}");
write_file(tmp.path(), "b.rs", "fn beta() {}");
let shared = LiveIndex::empty();
{
let index = shared.read();
index.set_local_empty_reason(Some("workspace is not bound".to_owned()));
assert!(index.local_empty_reason().is_some());
}
{
let mut index = shared.write();
index.reload(tmp.path()).expect("reload should succeed");
}
let index = shared.read();
assert_eq!(index.file_count(), 2);
assert!(index.is_ready(), "after reload should be ready");
assert_eq!(index.index_state(), IndexState::Ready);
assert_eq!(index.load_source(), IndexLoadSource::FreshLoad);
assert_eq!(index.local_empty_reason(), None);
assert_eq!(
index.snapshot_verify_state(),
SnapshotVerifyState::NotNeeded
);
}
#[test]
fn test_live_index_reload_invalid_root_returns_error() {
let shared = LiveIndex::empty();
let mut index = shared.write();
let result = index.reload(Path::new("/nonexistent/path/that/does/not/exist"));
assert!(
result.is_err(),
"reload on invalid root should return error"
);
}
#[test]
fn failed_reload_preserves_previous_generation() {
let tmp = TempDir::new().unwrap();
let shared = LiveIndex::empty();
shared.add_file(
"src/retained.rs".to_string(),
make_indexed_file_for_mutation("src/retained.rs"),
);
let before = shared.published_generation();
let result = shared.reload(&tmp.path().join("missing-repository"));
assert!(result.is_err(), "invalid reload input must fail");
let after = shared.published_generation();
assert!(
Arc::ptr_eq(&before, &after),
"failed replacement construction must retain the exact published generation"
);
}
#[test]
fn failed_observation_publishes_degraded_last_valid_wrapper() {
let path = "src/retained.rs";
let shared = LiveIndex::empty();
shared.add_file(path.to_string(), make_indexed_file_for_mutation(path));
let before = shared.published_generation();
let before_file = Arc::clone(before.live.files.get(path).unwrap());
let scouted = crate::domain::ScoutedEntry {
path: crate::domain::CatalogPath {
public_id: path.to_string(),
normalized_utf8: Some(path.to_string()),
},
absolute_path: None,
stamp: crate::domain::FileStamp {
size: before_file.byte_len,
created_hint: None,
modified_hint: None,
platform_id: None,
},
language: Some(LanguageId::Rust),
classification: before_file.classification,
decision: crate::domain::ScoutDecision::Unavailable {
stage: crate::domain::AccessStage::FullRead,
kind: crate::domain::AccessErrorKind::PermissionDenied,
},
};
assert!(
shared
.publish_terminal_disposition_at_generation(
path,
scouted,
FileDisposition::Unreadable {
stage: crate::domain::AccessStage::FullRead,
kind: crate::domain::AccessErrorKind::PermissionDenied,
},
shared.current_project_generation(),
before.health.generation,
)
.is_some()
);
let after = shared.published_generation();
assert!(after.publication_generation > before.publication_generation);
assert_eq!(
after.content_generation, before.content_generation,
"failed observation must not mint a new content generation"
);
assert!(Arc::ptr_eq(
&before_file,
after.live.files.get(path).unwrap()
));
assert!(matches!(
&*shared.freshness_status(),
FreshnessStatus::Degraded {
last_valid_content_generation: _,
reason_codes,
} if reason_codes.contains(&FreshnessReason::ObservationFailed)
));
}
#[test]
fn reload_after_tree_mutation_reaches_current_on_same_physical_root() {
let project = TempDir::new().unwrap();
write_file(project.path(), "src/lib.rs", "pub fn first() {}\n");
let shared = LiveIndex::load(project.path()).unwrap();
assert_eq!(
shared.freshness_status().as_ref(),
&FreshnessStatus::Current,
"precondition: cold load is Current"
);
write_file(project.path(), "src/extra.rs", "pub fn added() {}\n");
shared.reload(project.path()).expect("reload after create");
assert_eq!(
shared.freshness_status().as_ref(),
&FreshnessStatus::Current,
"same physical root after tree mutation must not stay ReconciliationPending"
);
assert!(
shared.read().get_file("src/extra.rs").is_some(),
"reload must index the new file"
);
shared.reload(project.path()).expect("second clean reload");
assert_eq!(
shared.freshness_status().as_ref(),
&FreshnessStatus::Current,
"subsequent clean reload on unchanged physical root must stay Current"
);
}
#[test]
fn healed_observation_restores_current_freshness() {
let project = TempDir::new().unwrap();
let path = "src/retained.rs";
write_file(project.path(), path, "fn retained() {}\n");
let shared = LiveIndex::load(project.path()).unwrap();
let before = shared.published_generation();
let mut scouted = shared
.scout_plan()
.expect("cold scout plan")
.entries
.iter()
.find(|entry| entry.path.normalized_utf8.as_deref() == Some(path))
.expect("retained file scout entry")
.clone();
scouted.decision = crate::domain::ScoutDecision::Unavailable {
stage: crate::domain::AccessStage::FullRead,
kind: crate::domain::AccessErrorKind::PermissionDenied,
};
assert!(
shared
.publish_terminal_disposition_at_generation(
path,
scouted,
FileDisposition::Unreadable {
stage: crate::domain::AccessStage::FullRead,
kind: crate::domain::AccessErrorKind::PermissionDenied,
},
shared.current_project_generation(),
before.health.generation,
)
.is_some()
);
assert!(matches!(
shared.freshness_status().as_ref(),
FreshnessStatus::Degraded { reason_codes, .. }
if reason_codes.contains(&FreshnessReason::ObservationFailed)
));
write_file(project.path(), path, "fn retained_and_healed() {}\n");
assert_eq!(
crate::live_index::single_file::update_file_from_disk(&shared, project.path(), path),
crate::live_index::single_file::ReindexResult::Reindexed
);
let healed = shared.published_generation();
assert_eq!(healed.freshness.as_ref(), &FreshnessStatus::Current);
assert_eq!(healed.live.index_state(), IndexState::Ready);
}
#[test]
fn healed_observation_does_not_bypass_pending_snapshot_verification() {
let project = TempDir::new().unwrap();
let path = "src/retained.rs";
write_file(project.path(), path, "fn retained() {}\n");
let mut live = LiveIndex::from_source_files(HashMap::from([(
path.to_string(),
Arc::new(make_indexed_file_for_mutation(path)),
)]));
live.indexed_root = Some(dunce::canonicalize(project.path()).unwrap());
live.snapshot_verify_state = SnapshotVerifyState::Pending;
let shared = SharedIndexHandle::shared(live);
let before = shared.published_generation();
assert_eq!(before.freshness.as_ref(), &FreshnessStatus::Verifying);
let scouted = crate::domain::ScoutedEntry {
path: crate::domain::CatalogPath {
public_id: path.to_string(),
normalized_utf8: Some(path.to_string()),
},
absolute_path: Some(project.path().join(path)),
stamp: crate::domain::FileStamp {
size: 0,
created_hint: None,
modified_hint: None,
platform_id: None,
},
language: Some(LanguageId::Rust),
classification: crate::domain::FileClassification::for_code_path(path),
decision: crate::domain::ScoutDecision::Unavailable {
stage: crate::domain::AccessStage::FullRead,
kind: crate::domain::AccessErrorKind::PermissionDenied,
},
};
assert!(
shared
.publish_terminal_disposition_at_generation(
path,
scouted,
FileDisposition::Unreadable {
stage: crate::domain::AccessStage::FullRead,
kind: crate::domain::AccessErrorKind::PermissionDenied,
},
shared.current_project_generation(),
before.health.generation,
)
.is_some()
);
assert!(matches!(
shared.freshness_status().as_ref(),
FreshnessStatus::Degraded { reason_codes, .. }
if reason_codes.contains(&FreshnessReason::ObservationFailed)
));
write_file(project.path(), path, "fn retained_and_healed() {}\n");
assert_eq!(
crate::live_index::single_file::update_file_from_disk(&shared, project.path(), path),
crate::live_index::single_file::ReindexResult::Reindexed
);
let healed = shared.published_generation();
assert_eq!(healed.freshness.as_ref(), &FreshnessStatus::Verifying);
assert_eq!(
healed.live.snapshot_verify_state,
SnapshotVerifyState::Pending
);
}
#[test]
fn reconciled_coverage_publishes_freshness_once_per_transition() {
let project = TempDir::new().unwrap();
write_file(project.path(), "main.rs", "fn main() {}\n");
let shared = LiveIndex::load(project.path()).unwrap();
let complete_plan = (*shared.scout_plan().expect("cold scout plan")).clone();
let mut degraded_plan = complete_plan.clone();
degraded_plan.issues.push(crate::domain::ScoutIssue {
path_id: Some("transient-walk-entry".to_string()),
safe_path: Some("temporarily-locked".to_string()),
kind: crate::domain::ScoutIssueKind::DirectoryEntryUnreadable {
kind: crate::domain::AccessErrorKind::PermissionDenied,
},
safe_message: "directory entry unavailable".to_string(),
});
crate::discovery::refresh_scout_plan(&mut degraded_plan).unwrap();
let project_generation = shared.current_project_generation();
let initial = shared.published_generation();
assert!(shared.publish_reconciled_scout_plan_at_generation(
Some(&complete_plan),
degraded_plan.clone(),
project_generation,
));
let degraded = shared.published_generation();
assert!(degraded.publication_generation > initial.publication_generation);
assert_eq!(degraded.content_generation, initial.content_generation);
assert!(matches!(
degraded.freshness.as_ref(),
FreshnessStatus::Degraded { reason_codes, .. }
if reason_codes.contains(&FreshnessReason::ReconciliationPending)
));
assert!(shared.publish_reconciled_scout_plan_at_generation(
Some(°raded_plan),
complete_plan.clone(),
project_generation,
));
let healed = shared.published_generation();
assert!(healed.publication_generation > degraded.publication_generation);
assert_eq!(healed.content_generation, initial.content_generation);
assert_eq!(healed.freshness.as_ref(), &FreshnessStatus::Current);
assert!(shared.publish_reconciled_scout_plan_at_generation(
Some(&complete_plan),
complete_plan.clone(),
project_generation,
));
let unchanged = shared.published_generation();
assert_eq!(
unchanged.publication_generation, healed.publication_generation,
"an unchanged Complete reconciliation must not mint a publication"
);
assert_eq!(unchanged.content_generation, healed.content_generation);
}
#[test]
fn a_latched_gap_survives_an_ordinary_complete_reconcile() {
let project = TempDir::new().unwrap();
write_file(
project.path(),
"main.rs",
"fn main() {}
",
);
let shared = LiveIndex::load(project.path()).unwrap();
let complete_plan = (*shared.scout_plan().expect("cold scout plan")).clone();
let project_generation = shared.current_project_generation();
shared.latch_observer_gap();
assert!(
matches!(
shared.published_generation().freshness.as_ref(),
FreshnessStatus::Degraded { reason_codes, .. }
if reason_codes.contains(&FreshnessReason::WatcherUnavailable)
),
"the latch must reach the published bundle"
);
assert!(shared.publish_reconciled_scout_plan_at_generation(
Some(&complete_plan),
complete_plan.clone(),
project_generation,
));
assert!(
matches!(
shared.published_generation().freshness.as_ref(),
FreshnessStatus::Degraded { reason_codes, .. }
if reason_codes.contains(&FreshnessReason::WatcherUnavailable)
),
"an ordinary Complete reconcile must not retire an observer gap: it \
rebuilds present state and proves nothing about the missed window"
);
}
#[test]
fn degraded_initial_scout_is_published_in_the_trust_bundle() {
let project = TempDir::new().unwrap();
write_file(project.path(), "main.rs", "fn main() {}\n");
let loaded = LiveIndex::load(project.path()).unwrap();
let mut degraded_plan = (*loaded.scout_plan().expect("cold scout plan")).clone();
degraded_plan.issues.push(crate::domain::ScoutIssue {
path_id: Some("transient-walk-entry".to_string()),
safe_path: Some("temporarily-locked".to_string()),
kind: crate::domain::ScoutIssueKind::DirectoryEntryUnreadable {
kind: crate::domain::AccessErrorKind::PermissionDenied,
},
safe_message: "directory entry unavailable".to_string(),
});
crate::discovery::refresh_scout_plan(&mut degraded_plan).unwrap();
let shared = SharedIndexHandle::shared_with_scout_plan(
(*loaded.published_generation().live).clone(),
degraded_plan,
crate::discovery::SourceExclusions::default(),
);
assert!(matches!(
shared.freshness_status().as_ref(),
FreshnessStatus::Degraded { reason_codes, .. }
if reason_codes.contains(&FreshnessReason::ReconciliationPending)
));
assert!(matches!(
shared.published_generation().freshness.as_ref(),
FreshnessStatus::Degraded { reason_codes, .. }
if reason_codes.contains(&FreshnessReason::ReconciliationPending)
));
}
#[test]
fn test_live_index_loaded_at_system_is_recent() {
use std::time::SystemTime;
let before = SystemTime::now();
let shared = LiveIndex::empty();
let index = shared.read();
let after = SystemTime::now();
let ts = index.loaded_at_system();
assert!(
ts >= before,
"loaded_at_system should be >= before creation"
);
assert!(ts <= after, "loaded_at_system should be <= after creation");
}
#[test]
fn test_concurrent_readers_no_deadlock() {
use std::thread;
let tmp = TempDir::new().unwrap();
write_file(tmp.path(), "a.rs", "fn foo() {}");
write_file(tmp.path(), "b.rs", "fn bar() {}");
write_file(tmp.path(), "c.rs", "fn baz() {}");
let shared = LiveIndex::load(tmp.path()).unwrap();
let handles: Vec<_> = (0..8)
.map(|_| {
let shared_clone = Arc::clone(&shared);
thread::spawn(move || {
let index = shared_clone.read();
let _ = index.file_count();
let _ = index.symbol_count();
})
})
.collect();
for h in handles {
h.join().expect("reader thread should not panic");
}
}
fn make_indexed_file_for_mutation(path: &str) -> IndexedFile {
IndexedFile {
relative_path: path.to_string(),
language: LanguageId::Rust,
classification: crate::domain::FileClassification::for_code_path(path),
content: b"fn test() {}".to_vec(),
symbols: vec![dummy_symbol()],
parse_status: ParseStatus::Parsed,
parse_diagnostic: None,
byte_len: 12,
content_hash: "abc123".to_string(),
references: vec![],
alias_map: std::collections::HashMap::new(),
mtime_secs: 0,
}
}
fn make_empty_live_index() -> LiveIndex {
LiveIndex {
files: HashMap::new(),
loaded_at: Instant::now(),
loaded_at_system: SystemTime::now(),
load_duration: Duration::ZERO,
cb_state: CircuitBreakerState::new(0.20),
is_empty: false,
load_source: IndexLoadSource::FreshLoad,
snapshot_verify_state: SnapshotVerifyState::NotNeeded,
reverse_index: HashMap::new(),
files_by_basename: HashMap::new(),
files_by_dir_component: HashMap::new(),
trigram_index: crate::live_index::trigram::TrigramIndex::new(),
gitignore: None,
manifest_entries: Vec::new(),
coupling_store: None,
local_empty_reason: Arc::new(parking_lot::RwLock::new(None)),
indexed_root: None,
}
}
fn make_manifest_entry(path: &str, size: u64, disposition: FileDisposition) -> CatalogEntry {
CatalogEntry {
path: crate::domain::CatalogPath {
public_id: path.to_string(),
normalized_utf8: Some(path.to_string()),
},
size,
language: None,
classification: FileClassification {
class: crate::domain::FileClass::Text,
is_generated: false,
is_test: false,
is_vendor: false,
is_config: false,
},
disposition,
content_hash: None,
}
}
#[test]
fn test_live_index_load_builds_path_indices() {
let dir = TempDir::new().expect("failed to create tempdir");
fs::create_dir_all(dir.path().join("src")).expect("failed to create src dir");
fs::create_dir_all(dir.path().join("tests")).expect("failed to create tests dir");
write_file(dir.path(), "src/lib.rs", "pub fn lib_fn() {}");
write_file(dir.path(), "tests/lib.rs", "fn test_lib() {}");
let shared = LiveIndex::load(dir.path()).expect("LiveIndex::load failed");
let index = shared.read();
assert_eq!(
index.files_by_basename.get("lib.rs"),
Some(&vec!["src/lib.rs".to_string(), "tests/lib.rs".to_string()])
);
assert_eq!(
index.files_by_dir_component.get("src"),
Some(&vec!["src/lib.rs".to_string()])
);
assert_eq!(
index.files_by_dir_component.get("tests"),
Some(&vec!["tests/lib.rs".to_string()])
);
}
#[test]
fn test_live_index_reload_rebuilds_path_indices() {
let dir = TempDir::new().expect("failed to create tempdir");
fs::create_dir_all(dir.path().join("src")).expect("failed to create src dir");
write_file(dir.path(), "src/alpha.rs", "fn alpha() {}");
let shared = LiveIndex::load(dir.path()).expect("LiveIndex::load failed");
fs::remove_file(dir.path().join("src/alpha.rs")).expect("failed to remove alpha");
fs::create_dir_all(dir.path().join("tests")).expect("failed to create tests dir");
write_file(dir.path(), "tests/beta.rs", "fn beta() {}");
{
let mut index = shared.write();
index.reload(dir.path()).expect("reload should succeed");
}
let index = shared.read();
assert!(!index.files_by_basename.contains_key("alpha.rs"));
assert_eq!(
index.files_by_basename.get("beta.rs"),
Some(&vec!["tests/beta.rs".to_string()])
);
assert!(!index.files_by_dir_component.contains_key("src"));
assert_eq!(
index.files_by_dir_component.get("tests"),
Some(&vec!["tests/beta.rs".to_string()])
);
}
#[test]
fn reload_outside_lock_admitted_mutation_survives_swap_or_publish_fails_closed() {
let dir = TempDir::new().expect("failed to create tempdir");
fs::create_dir_all(dir.path().join("src")).expect("failed to create src dir");
write_file(dir.path(), "src/initial.rs", "fn initial() {}");
let shared = LiveIndex::load(dir.path()).expect("LiveIndex::load failed");
let mutation_path = "src/admitted_during_outside_lock_build.rs";
let shared_for_hook = Arc::clone(&shared);
let _guard = reload_outside_lock::install(move || {
let file = make_indexed_file_for_mutation(mutation_path);
shared_for_hook.update_file(mutation_path.to_string(), file);
assert!(
shared_for_hook.read().get_file(mutation_path).is_some(),
"precondition: mutation must be present on live index at outside-lock seam"
);
});
let reload_result = shared.reload(dir.path());
assert!(
reload_result.is_ok(),
"precondition: reload must succeed (false-success on broken tip)"
);
assert!(
shared.read().get_file(mutation_path).is_some(),
"P1-04 GREEN contract: a mutation admitted during the outside-lock build \
window must survive the swap, or publish must fail closed. On broken tip \
the wholesale swap discards it while reload still returns Ok"
);
}
#[test]
fn test_dir_component_keys_deduplicate_and_accept_backslashes() {
assert_eq!(
dir_component_keys("src\\live_index\\src\\store.rs"),
vec!["live_index".to_string(), "src".to_string()]
);
}
#[test]
fn test_update_file_inserts_and_updates_timestamp() {
let mut index = make_empty_live_index();
let before = SystemTime::now();
let file = make_indexed_file_for_mutation("src/new.rs");
index.update_file("src/new.rs".to_string(), file);
let after = SystemTime::now();
assert!(
index.get_file("src/new.rs").is_some(),
"file should be inserted"
);
assert_eq!(
index.files_by_basename.get("new.rs"),
Some(&vec!["src/new.rs".to_string()])
);
assert_eq!(
index.files_by_dir_component.get("src"),
Some(&vec!["src/new.rs".to_string()])
);
let ts = index.loaded_at_system;
assert!(ts >= before, "loaded_at_system should be >= before update");
assert!(ts <= after, "loaded_at_system should be <= after update");
}
#[test]
fn test_update_file_replaces_existing() {
let mut index = make_empty_live_index();
let file1 = IndexedFile {
relative_path: "src/foo.rs".to_string(),
language: LanguageId::Rust,
classification: crate::domain::FileClassification::for_code_path("src/foo.rs"),
content: b"fn old() {}".to_vec(),
symbols: vec![],
parse_status: ParseStatus::Parsed,
parse_diagnostic: None,
byte_len: 11,
content_hash: "old_hash".to_string(),
references: vec![],
alias_map: std::collections::HashMap::new(),
mtime_secs: 0,
};
index.update_file("src/foo.rs".to_string(), file1);
let file2 = IndexedFile {
relative_path: "src/foo.rs".to_string(),
language: LanguageId::Rust,
classification: crate::domain::FileClassification::for_code_path("src/foo.rs"),
content: b"fn new() {}".to_vec(),
symbols: vec![dummy_symbol()],
parse_status: ParseStatus::Parsed,
parse_diagnostic: None,
byte_len: 11,
content_hash: "new_hash".to_string(),
references: vec![],
alias_map: std::collections::HashMap::new(),
mtime_secs: 0,
};
index.update_file("src/foo.rs".to_string(), file2);
let retrieved = index.get_file("src/foo.rs").unwrap();
assert_eq!(
retrieved.content_hash, "new_hash",
"should have replaced the file"
);
assert_eq!(index.file_count(), 1, "should still have exactly 1 file");
assert_eq!(
index.files_by_basename.get("foo.rs"),
Some(&vec!["src/foo.rs".to_string()])
);
assert_eq!(
index.files_by_dir_component.get("src"),
Some(&vec!["src/foo.rs".to_string()])
);
}
#[test]
fn test_add_file_inserts_new() {
let mut index = make_empty_live_index();
assert_eq!(index.file_count(), 0);
let file = make_indexed_file_for_mutation("src/new.rs");
index.add_file("src/new.rs".to_string(), file);
assert_eq!(
index.file_count(),
1,
"file count should increase by 1 after add_file"
);
assert!(index.get_file("src/new.rs").is_some());
}
#[test]
fn test_remove_file_removes_existing() {
let mut index = make_empty_live_index();
let file = make_indexed_file_for_mutation("src/to_delete.rs");
index.update_file("src/to_delete.rs".to_string(), file);
assert_eq!(index.file_count(), 1);
assert_eq!(index.tier_counts(), (1, 0, 0));
index.remove_file("src/to_delete.rs");
assert!(
index.get_file("src/to_delete.rs").is_none(),
"file should be removed"
);
assert_eq!(index.file_count(), 0);
assert_eq!(
index.tier_counts(),
(0, 0, 0),
"removal must clear the canonical manifest entry with the indexed bytes"
);
assert!(!index.files_by_basename.contains_key("to_delete.rs"));
assert!(!index.files_by_dir_component.contains_key("src"));
}
#[test]
fn test_remove_file_nonexistent_is_noop() {
let mut index = make_empty_live_index();
let known_ts = index.loaded_at_system;
std::thread::sleep(Duration::from_millis(5));
index.remove_file("nonexistent.rs");
assert_eq!(
index.loaded_at_system, known_ts,
"loaded_at_system must NOT change when removing non-existent file"
);
}
#[test]
fn test_file_count_after_mutations() {
let mut index = make_empty_live_index();
assert_eq!(index.file_count(), 0);
index.add_file("a.rs".to_string(), make_indexed_file_for_mutation("a.rs"));
assert_eq!(index.file_count(), 1);
index.add_file("b.rs".to_string(), make_indexed_file_for_mutation("b.rs"));
assert_eq!(index.file_count(), 2);
index.update_file("a.rs".to_string(), make_indexed_file_for_mutation("a.rs"));
assert_eq!(index.file_count(), 2, "update does not add a new entry");
index.remove_file("a.rs");
assert_eq!(index.file_count(), 1);
index.remove_file("nonexistent.rs");
assert_eq!(
index.file_count(),
1,
"removing nonexistent does not change count"
);
}
fn make_ref(name: &str, kind: ReferenceKind, line: u32) -> ReferenceRecord {
ReferenceRecord {
name: name.to_string(),
qualified_name: None,
kind,
byte_range: (0, 1),
line_range: (line, line),
enclosing_symbol_index: None,
}
}
fn make_indexed_file_with_refs(path: &str, refs: Vec<ReferenceRecord>) -> IndexedFile {
IndexedFile {
relative_path: path.to_string(),
language: LanguageId::Rust,
classification: crate::domain::FileClassification::for_code_path(path),
content: b"fn test() {}".to_vec(),
symbols: vec![],
parse_status: ParseStatus::Parsed,
parse_diagnostic: None,
byte_len: 12,
content_hash: "abc".to_string(),
references: refs,
alias_map: std::collections::HashMap::new(),
mtime_secs: 0,
}
}
#[test]
fn test_indexed_file_from_parse_result_transfers_refs_and_alias_map() {
use std::collections::HashMap;
let mut alias_map = HashMap::new();
alias_map.insert("Map".to_string(), "HashMap".to_string());
let refs = vec![make_ref("foo", ReferenceKind::Call, 1)];
let result = FileProcessingResult {
relative_path: "test.rs".to_string(),
language: LanguageId::Rust,
classification: crate::domain::FileClassification::for_code_path("test.rs"),
outcome: FileOutcome::Processed,
parse_diagnostic: None,
symbols: vec![],
byte_len: 0,
content_hash: "abc".to_string(),
references: refs.clone(),
alias_map: alias_map.clone(),
};
let indexed = IndexedFile::from_parse_result(result, vec![]);
assert_eq!(indexed.references.len(), 1);
assert_eq!(indexed.references[0].name, "foo");
assert_eq!(
indexed.alias_map.get("Map").map(|s| s.as_str()),
Some("HashMap")
);
}
#[test]
fn publishing_and_removing_a_ref_source_bumps_registry_without_touching_current_lane() {
let handle = SharedIndexHandle::new(LiveIndex::from_source_files(HashMap::new()));
let before = handle.published_source_set();
let before_registry = before.registry_generation;
let current_id = before.current_source_id.clone();
let current_lane = before.current_generation();
let (before_pub, before_content, before_project) = (
current_lane.publication_generation,
current_lane.content_generation,
current_lane.project_generation,
);
let ref_gen = handle.build_ref_source_generation(
LiveIndex::from_source_files(HashMap::new()),
crate::domain::index::RepositoryId::new("repo-under-test"),
"refs/heads/feature",
"0123456789abcdef0123456789abcdef01234567",
crate::domain::CoverageStatus::Complete,
);
let ref_source_id = ref_gen
.source
.as_ref()
.expect("ref source identity")
.source_id
.clone();
assert_ne!(ref_source_id, current_id);
handle.publish_ref_source(ref_gen);
let after = handle.published_source_set();
assert_eq!(
after.registry_generation,
before_registry + 1,
"registry bumps"
);
assert_eq!(after.sources.len(), 2);
assert!(after.sources.contains_key(&ref_source_id));
assert_eq!(after.current_source_id, current_id);
let after_current = after.current_generation();
assert_eq!(after_current.publication_generation, before_pub);
assert_eq!(after_current.content_generation, before_content);
assert_eq!(
after_current.project_generation, before_project,
"a P1 ref add must not advance the current lane's generations"
);
assert!(handle.remove_ref_source(&ref_source_id));
let removed = handle.published_source_set();
assert_eq!(removed.registry_generation, before_registry + 2);
assert_eq!(removed.sources.len(), 1);
assert!(!removed.sources.contains_key(&ref_source_id));
assert!(
!handle.remove_ref_source(¤t_id),
"the current lane can never be removed as a ref source"
);
}
#[test]
fn p0_publishes_preserve_published_ref_lanes() {
let handle = SharedIndexHandle::new(LiveIndex::from_source_files(HashMap::new()));
let current_id = handle.published_source_set().current_source_id.clone();
let ref_gen = handle.build_ref_source_generation(
LiveIndex::from_source_files(HashMap::new()),
crate::domain::index::RepositoryId::new("repo-under-test"),
"refs/heads/feature",
"0123456789abcdef0123456789abcdef01234567",
crate::domain::CoverageStatus::Complete,
);
let ref_source_id = ref_gen
.source
.as_ref()
.expect("ref source identity")
.source_id
.clone();
handle.publish_ref_source(ref_gen);
assert_eq!(handle.published_source_set().sources.len(), 2);
let before = handle.published_source_set();
handle.swap_and_publish(LiveIndex::from_source_files(HashMap::new()));
let after = handle.published_source_set();
assert!(
after.sources.contains_key(&ref_source_id),
"a P0 content publish must not drop a published ref lane"
);
assert_eq!(after.current_source_id, current_id);
assert!(
after.registry_generation > before.registry_generation,
"a source-map change advances registry_generation"
);
assert!(
after.current_generation().publication_generation
> before.current_generation().publication_generation,
"the current lane advances on a P0 publish"
);
let prepared = handle.prepare_authority_rebuild();
assert!(handle.publish_prepared_authority(prepared));
let after_auth = handle.published_source_set();
assert!(
after_auth.sources.contains_key(&ref_source_id),
"a P0 authority publish must not drop a published ref lane"
);
assert_eq!(after_auth.sources.len(), 2);
}
#[test]
fn test_rebuild_reverse_index_builds_name_to_locations() {
let mut index = make_empty_live_index();
let refs_a = vec![
make_ref("process", ReferenceKind::Call, 5),
make_ref("load", ReferenceKind::Call, 10),
];
let refs_b = vec![make_ref("process", ReferenceKind::Call, 3)];
index.add_file(
"a.rs".to_string(),
make_indexed_file_with_refs("a.rs", refs_a),
);
index.add_file(
"b.rs".to_string(),
make_indexed_file_with_refs("b.rs", refs_b),
);
let locs = index
.reverse_index
.get("process")
.expect("process should be in reverse index");
assert_eq!(locs.len(), 2, "process referenced in 2 files");
let locs_load = index
.reverse_index
.get("load")
.expect("load should be in reverse index");
assert_eq!(locs_load.len(), 1);
assert_eq!(locs_load[0].file_path, "a.rs");
assert_eq!(locs_load[0].reference_idx, 1);
}
#[test]
fn test_rebuild_reverse_index_consistent_after_update_file() {
let mut index = make_empty_live_index();
let refs_old = vec![make_ref("old_func", ReferenceKind::Call, 1)];
index.add_file(
"src.rs".to_string(),
make_indexed_file_with_refs("src.rs", refs_old),
);
assert!(index.reverse_index.contains_key("old_func"));
let refs_new = vec![make_ref("new_func", ReferenceKind::Call, 1)];
index.update_file(
"src.rs".to_string(),
make_indexed_file_with_refs("src.rs", refs_new),
);
assert!(
!index.reverse_index.contains_key("old_func"),
"stale entry should be gone"
);
assert!(
index.reverse_index.contains_key("new_func"),
"new entry should be present"
);
}
#[test]
fn test_rebuild_reverse_index_excludes_removed_file() {
let mut index = make_empty_live_index();
let refs = vec![make_ref("target_fn", ReferenceKind::Call, 2)];
index.add_file(
"will_delete.rs".to_string(),
make_indexed_file_with_refs("will_delete.rs", refs),
);
assert!(index.reverse_index.contains_key("target_fn"));
index.remove_file("will_delete.rs");
assert!(
!index.reverse_index.contains_key("target_fn"),
"removed file's refs should be gone"
);
}
#[test]
fn test_reference_location_fields() {
let loc = ReferenceLocation {
file_path: "src/main.rs".to_string(),
reference_idx: 3,
};
assert_eq!(loc.file_path, "src/main.rs");
assert_eq!(loc.reference_idx, 3);
}
#[test]
fn test_empty_live_index_has_empty_reverse_index() {
let index = make_empty_live_index();
assert!(
index.reverse_index.is_empty(),
"fresh index should have empty reverse index"
);
}
#[test]
fn test_incremental_reverse_index_matches_full_rebuild() {
let mut index = make_empty_live_index();
let refs_a = vec![
make_ref("shared_fn", ReferenceKind::Call, 1),
make_ref("only_a", ReferenceKind::Call, 5),
];
let refs_b = vec![
make_ref("shared_fn", ReferenceKind::Call, 2),
make_ref("only_b", ReferenceKind::Call, 8),
];
index.add_file(
"a.rs".to_string(),
make_indexed_file_with_refs("a.rs", refs_a),
);
index.add_file(
"b.rs".to_string(),
make_indexed_file_with_refs("b.rs", refs_b),
);
let refs_a_new = vec![
make_ref("shared_fn", ReferenceKind::Call, 1),
make_ref("replaced_a", ReferenceKind::Call, 10),
];
index.update_file(
"a.rs".to_string(),
make_indexed_file_with_refs("a.rs", refs_a_new),
);
let incremental: HashMap<String, Vec<(String, u32)>> = index
.reverse_index
.iter()
.map(|(k, v)| {
let mut locs: Vec<(String, u32)> = v
.iter()
.map(|l| (l.file_path.clone(), l.reference_idx))
.collect();
locs.sort();
(k.clone(), locs)
})
.collect();
index.rebuild_reverse_index();
let full_rebuild: HashMap<String, Vec<(String, u32)>> = index
.reverse_index
.iter()
.map(|(k, v)| {
let mut locs: Vec<(String, u32)> = v
.iter()
.map(|l| (l.file_path.clone(), l.reference_idx))
.collect();
locs.sort();
(k.clone(), locs)
})
.collect();
assert_eq!(
incremental, full_rebuild,
"incremental update should produce same result as full rebuild"
);
assert!(
!index.reverse_index.contains_key("only_a"),
"only_a should be gone after update"
);
assert!(
index.reverse_index.contains_key("replaced_a"),
"replaced_a should be present"
);
assert!(
index.reverse_index.contains_key("only_b"),
"only_b should still be present from b.rs"
);
let shared = index.reverse_index.get("shared_fn").unwrap();
assert_eq!(shared.len(), 2, "shared_fn still referenced in both files");
}
#[test]
fn test_incremental_reverse_index_remove() {
let mut index = make_empty_live_index();
let refs_a = vec![
make_ref("common", ReferenceKind::Call, 1),
make_ref("unique_a", ReferenceKind::Call, 3),
];
let refs_b = vec![
make_ref("common", ReferenceKind::Call, 2),
make_ref("unique_b", ReferenceKind::Call, 4),
];
index.add_file(
"a.rs".to_string(),
make_indexed_file_with_refs("a.rs", refs_a),
);
index.add_file(
"b.rs".to_string(),
make_indexed_file_with_refs("b.rs", refs_b),
);
index.remove_file("a.rs");
assert!(
!index.reverse_index.contains_key("unique_a"),
"unique_a should be removed with a.rs"
);
assert!(
index.reverse_index.contains_key("unique_b"),
"unique_b should survive"
);
let common_locs = index
.reverse_index
.get("common")
.expect("common should still exist from b.rs");
assert_eq!(common_locs.len(), 1);
assert_eq!(common_locs[0].file_path, "b.rs");
let incremental: HashMap<String, Vec<(String, u32)>> = index
.reverse_index
.iter()
.map(|(k, v)| {
let mut locs: Vec<(String, u32)> = v
.iter()
.map(|l| (l.file_path.clone(), l.reference_idx))
.collect();
locs.sort();
(k.clone(), locs)
})
.collect();
index.rebuild_reverse_index();
let full_rebuild: HashMap<String, Vec<(String, u32)>> = index
.reverse_index
.iter()
.map(|(k, v)| {
let mut locs: Vec<(String, u32)> = v
.iter()
.map(|l| (l.file_path.clone(), l.reference_idx))
.collect();
locs.sort();
(k.clone(), locs)
})
.collect();
assert_eq!(
incremental, full_rebuild,
"incremental remove should match full rebuild"
);
}
#[test]
fn test_circuit_breaker_deterministic_after_sort() {
let mut results: Vec<(String, bool)> = vec![
("a/f00.rs".to_string(), true),
("a/f01.rs".to_string(), true),
("a/f02.rs".to_string(), true),
("a/f03.rs".to_string(), true),
("a/f04.rs".to_string(), true),
("a/f05.rs".to_string(), false),
("a/f06.rs".to_string(), false),
("a/f07.rs".to_string(), false),
("a/f08.rs".to_string(), false),
("a/f09.rs".to_string(), false),
];
let run_cb = |items: &[(String, bool)]| -> Option<String> {
let cb = CircuitBreakerState::new(0.20);
for (path, ok) in items {
if *ok {
cb.record_success();
} else {
cb.record_failure(path, "parse error");
}
if cb.should_abort() {
return Some(path.clone());
}
}
None
};
results.sort_by(|a, b| a.0.cmp(&b.0));
let trip_sorted = run_cb(&results);
results.reverse();
results.sort_by(|a, b| a.0.cmp(&b.0)); let trip_sorted2 = run_cb(&results);
assert_eq!(
trip_sorted, trip_sorted2,
"sorted runs must trip at the same path"
);
assert!(trip_sorted.is_some(), "circuit breaker should have tripped");
let mut reversed: Vec<(String, bool)> = results.clone();
reversed.reverse(); let trip_unsorted = run_cb(&reversed);
assert_ne!(
trip_sorted, trip_unsorted,
"unsorted order should trip at a different (earlier) path, proving sort is needed"
);
}
#[test]
fn circuit_breaker_tail_retains_aborted_dispositions() {
let parse_results = (0..10)
.map(|index| {
let path = format!("a/f{index:02}.rs");
let outcome = if index < 5 {
FileOutcome::Processed
} else {
FileOutcome::Failed {
error: "parse failed".to_string(),
}
};
let mut result = make_result(outcome, vec![]);
result.relative_path = path.clone();
let indexed = IndexedFile::from_parse_result(result, b"fixture".to_vec());
(path, indexed)
})
.collect();
let folded = fold_parse_results_for_scope(
parse_results,
CircuitBreakerState::new(0.20),
CircuitBreakerScope::new(
PathBuf::from("source-a"),
crate::domain::IndexTargets::Code,
"parse",
),
);
let aborted_paths: Vec<_> = folded
.dispositions
.iter()
.filter_map(|(path, disposition)| {
matches!(
disposition,
crate::domain::FileDisposition::AbortedCircuitBreaker
)
.then_some(path.as_str())
})
.collect();
assert_eq!(folded.dispositions.len(), 10);
assert_eq!(aborted_paths, ["a/f07.rs", "a/f08.rs", "a/f09.rs"]);
assert!(folded.files.contains_key("a/f06.rs"));
assert!(!folded.files.contains_key("a/f07.rs"));
}
#[test]
fn circuit_breaker_trip_is_scoped_and_degraded() {
let results = |fail_after: Option<usize>| {
(0..10)
.map(|index| {
let path = format!("a/f{index:02}.rs");
let outcome = if fail_after.is_some_and(|start| index >= start) {
FileOutcome::Failed {
error: "parse failed".to_string(),
}
} else {
FileOutcome::Processed
};
let mut result = make_result(outcome, vec![]);
result.relative_path = path.clone();
(
path,
IndexedFile::from_parse_result(result, b"fixture".to_vec()),
)
})
.collect()
};
let tripped_scope = CircuitBreakerScope::new(
PathBuf::from("source-a"),
crate::domain::IndexTargets::Code,
"parse",
);
let unaffected_scopes = [
CircuitBreakerScope::new(
PathBuf::from("source-a"),
crate::domain::IndexTargets::Knowledge,
"parse",
),
CircuitBreakerScope::new(
PathBuf::from("source-a"),
crate::domain::IndexTargets::Code,
"stable-read",
),
CircuitBreakerScope::new(
PathBuf::from("source-b"),
crate::domain::IndexTargets::Code,
"parse",
),
];
let tripped = fold_parse_results_for_scope(
results(Some(5)),
CircuitBreakerState::new(0.20),
tripped_scope.clone(),
);
assert_eq!(tripped.coverage, crate::domain::CoverageStatus::Degraded);
assert_eq!(
tripped
.dispositions
.iter()
.filter(|(_, disposition)| matches!(
disposition,
crate::domain::FileDisposition::AbortedCircuitBreaker
))
.count(),
3
);
for scope in unaffected_scopes {
let unaffected =
fold_parse_results_for_scope(results(None), CircuitBreakerState::new(0.20), scope);
assert_eq!(unaffected.coverage, crate::domain::CoverageStatus::Complete);
assert_eq!(unaffected.files.len(), 10);
assert!(
unaffected
.dispositions
.iter()
.all(|(_, disposition)| !matches!(
disposition,
crate::domain::FileDisposition::AbortedCircuitBreaker
))
);
}
}
mod sf009_untracked_surfacing {
use super::*;
use crate::discovery::{self, EXCLUDE_UNTRACKED_ENV};
static EXCLUDE_UNTRACKED_ENV_LOCK: StdMutex<()> = StdMutex::new(());
struct ExcludeUntrackedEnvGuard {
previous: Option<String>,
}
#[allow(unsafe_code)] impl ExcludeUntrackedEnvGuard {
fn set(value: Option<&str>) -> Self {
let previous = std::env::var(EXCLUDE_UNTRACKED_ENV).ok();
unsafe {
match value {
Some(v) => std::env::set_var(EXCLUDE_UNTRACKED_ENV, v),
None => std::env::remove_var(EXCLUDE_UNTRACKED_ENV),
}
}
Self { previous }
}
fn apply(&self, value: Option<&str>) {
unsafe {
match value {
Some(v) => std::env::set_var(EXCLUDE_UNTRACKED_ENV, v),
None => std::env::remove_var(EXCLUDE_UNTRACKED_ENV),
}
}
}
}
#[allow(unsafe_code)] impl Drop for ExcludeUntrackedEnvGuard {
fn drop(&mut self) {
unsafe {
match self.previous.as_deref() {
Some(v) => std::env::set_var(EXCLUDE_UNTRACKED_ENV, v),
None => std::env::remove_var(EXCLUDE_UNTRACKED_ENV),
}
}
}
}
fn git(root: &Path, args: &[&str]) {
let status = crate::process_util::hidden_command("git")
.args(args)
.current_dir(root)
.output()
.expect("git command should run");
assert!(
status.status.success(),
"git {:?} failed: {}",
args,
String::from_utf8_lossy(&status.stderr)
);
}
fn init_repo(root: &Path) {
git(root, &["init"]);
git(root, &["config", "user.email", "test@test.com"]);
git(root, &["config", "user.name", "Test"]);
}
#[test]
fn report_bug_does_not_reproduce_generic_text_has_zero_code_symbols() {
let tmp = TempDir::new().unwrap();
init_repo(tmp.path());
write_file(tmp.path(), "src/main.rs", "fn main() {}");
write_file(tmp.path(), "notes.txt", "scratch notes, unknown ext");
write_file(tmp.path(), ".probe.txt", "dotfile scratch");
git(tmp.path(), &["add", "src/main.rs", "notes.txt"]);
git(tmp.path(), &["commit", "-m", "init"]);
let entries = discovery::discover_all_files(tmp.path()).unwrap();
let paths: Vec<&str> = entries.iter().map(|e| e.relative_path.as_str()).collect();
assert!(
paths.contains(&".probe.txt"),
"repository-owned dotfile should reach admission: {paths:?}"
);
let notes = entries
.iter()
.find(|e| e.relative_path == "notes.txt")
.expect("notes.txt should be discovered");
assert_eq!(notes.language, Some(LanguageId::Text));
let probe = entries
.iter()
.find(|e| e.relative_path == ".probe.txt")
.expect(".probe.txt should be discovered");
assert_eq!(probe.language, Some(LanguageId::Text));
let shared = LiveIndex::load(tmp.path()).unwrap();
let index = shared.read();
for path in ["notes.txt", ".probe.txt"] {
let file = index
.get_file(path)
.unwrap_or_else(|| panic!("generic knowledge file missing: {path}"));
assert_eq!(file.language, LanguageId::Text);
assert!(file.symbols.is_empty());
assert!(matches!(
index
.manifest_entries
.iter()
.find(|entry| entry.path.normalized_utf8.as_deref() == Some(path))
.map(|entry| &entry.disposition),
Some(FileDisposition::Indexed {
targets: crate::domain::IndexTargets::Knowledge,
..
})
));
}
}
#[test]
fn untracked_recognized_ext_file_is_surfaced() {
let tmp = TempDir::new().unwrap();
init_repo(tmp.path());
write_file(tmp.path(), "src/main.rs", "fn main() {}");
git(tmp.path(), &["add", "src/main.rs"]);
git(tmp.path(), &["commit", "-m", "init"]);
write_file(tmp.path(), "scratch.rs", "fn scratch() {}");
let shared = LiveIndex::load(tmp.path()).unwrap();
let index = shared.read();
let stats = index.health_stats();
assert_eq!(
stats.untracked_indexed, 1,
"exactly one untracked recognized-ext indexed file expected"
);
#[cfg(feature = "server")]
{
let report = crate::protocol::format::health_report_from_stats("Ready", &stats, 0);
assert!(
report.contains("indexed untracked files: 1"),
"health line should surface the untracked count: {report}"
);
}
}
#[test]
fn fail_open_no_git_repo_reports_zero() {
let tmp = TempDir::new().unwrap();
write_file(tmp.path(), "src/main.rs", "fn main() {}");
write_file(tmp.path(), "src/lib.rs", "pub fn lib() {}");
write_file(tmp.path(), "scratch.rs", "fn scratch() {}");
let shared = LiveIndex::load(tmp.path()).unwrap();
let index = shared.read();
let stats = index.health_stats();
assert_eq!(
stats.untracked_indexed, 0,
"with no git repo the feature must fail open to 0, not count every file"
);
#[cfg(feature = "server")]
{
let report = crate::protocol::format::health_report_from_stats("Ready", &stats, 0);
assert!(
!report.contains("indexed untracked files:"),
"no untracked line should appear when the count is 0: {report}"
);
}
}
#[test]
fn fully_tracked_repo_reports_zero() {
let tmp = TempDir::new().unwrap();
init_repo(tmp.path());
write_file(tmp.path(), "src/main.rs", "fn main() {}");
write_file(tmp.path(), "src/lib.rs", "pub fn lib() {}");
git(tmp.path(), &["add", "."]);
git(tmp.path(), &["commit", "-m", "init"]);
let shared = LiveIndex::load(tmp.path()).unwrap();
let index = shared.read();
let stats = index.health_stats();
assert_eq!(
stats.untracked_indexed, 0,
"a fully-tracked repo must report zero untracked indexed files"
);
}
#[test]
fn exclude_untracked_env_gate_demotes_only_when_enabled() {
let _lock = EXCLUDE_UNTRACKED_ENV_LOCK.lock().unwrap();
let env = ExcludeUntrackedEnvGuard::set(None);
let tmp = TempDir::new().unwrap();
init_repo(tmp.path());
write_file(tmp.path(), "src/main.rs", "fn main() {}");
git(tmp.path(), &["add", "src/main.rs"]);
git(tmp.path(), &["commit", "-m", "init"]);
write_file(tmp.path(), "scratch.rs", "fn scratch() {}");
{
assert!(!discovery::exclude_untracked_enabled());
let shared = LiveIndex::load(tmp.path()).unwrap();
let index = shared.read();
assert!(
index.files.contains_key("scratch.rs"),
"default policy must still index the untracked file (admission unchanged)"
);
assert_eq!(index.health_stats().untracked_indexed, 1);
}
env.apply(Some("1"));
{
assert!(discovery::exclude_untracked_enabled());
let shared = LiveIndex::load(tmp.path()).unwrap();
let index = shared.read();
assert!(
!index.files.contains_key("scratch.rs"),
"with the opt-in policy ON the untracked file is demoted out of Tier-1"
);
assert!(
index.files.contains_key("src/main.rs"),
"tracked files remain Tier-1 under the opt-in policy"
);
assert!(
index
.compatibility_skipped_files()
.iter()
.any(|sf| sf.path == "scratch.rs"
&& sf.reason() == Some(crate::domain::index::SkipReason::Untracked)),
"demoted file should be a Tier-2 skip with the Untracked reason"
);
assert_eq!(index.health_stats().untracked_indexed, 0);
}
}
}
#[test]
fn test_tier_counts() {
let mut index = make_empty_live_index();
assert_eq!(index.tier_counts(), (0, 0, 0));
index.manifest_entries.extend([
make_manifest_entry(
"model.bin",
1000,
FileDisposition::MetadataOnly {
reason: MetadataOnlyReason::GeneratedOrVendor,
},
),
make_manifest_entry(
"huge.dat",
200_000_000,
FileDisposition::HardSkip {
reason: HardSkipReason::PerFileCeiling,
},
),
]);
assert_eq!(index.tier_counts(), (0, 1, 1));
}
#[test]
fn load_routes_unknown_utf8_files_as_generic_knowledge() {
let tmp = TempDir::new().unwrap();
write_file(tmp.path(), "src/main.rs", "fn main() {}");
write_file(
tmp.path(),
"tests/unit/foo.tcl",
"proc foo {} { return 1 }\n",
);
write_file(tmp.path(), "scripts/setup.sh", "#!/bin/sh\necho hi\n");
write_file(tmp.path(), "LICENSE", "MIT License\n");
let shared = LiveIndex::load(tmp.path()).unwrap();
let index = shared.read();
let (tier1, tier2, tier3) = index.tier_counts();
assert_eq!(
tier1, 4,
"safe UTF-8 files must be resident Tier-1 evidence"
);
assert_eq!(tier2, 0);
assert_eq!(tier3, 0);
for path in ["tests/unit/foo.tcl", "scripts/setup.sh", "LICENSE"] {
let file = index
.get_file(path)
.unwrap_or_else(|| panic!("generic knowledge file missing: {path}"));
assert_eq!(file.language, LanguageId::Text);
assert!(file.classification.is_text());
assert!(file.symbols.is_empty());
assert!(matches!(
index
.manifest_entries
.iter()
.find(|entry| entry.path.normalized_utf8.as_deref() == Some(path))
.map(|entry| &entry.disposition),
Some(crate::domain::FileDisposition::Indexed {
targets: crate::domain::IndexTargets::Knowledge,
..
})
));
}
}
#[test]
fn unknown_utf8_file_becomes_generic_knowledge() {
let tmp = TempDir::new().unwrap();
let exact = b"first fact\r\nsecond fact without final newline";
let path = tmp.path().join("notes").join("facts.unknown-format");
std::fs::create_dir_all(path.parent().unwrap()).unwrap();
std::fs::write(&path, exact).unwrap();
let shared = LiveIndex::load(tmp.path()).expect("generic UTF-8 load must succeed");
let index = shared.read();
let file = index
.get_file("notes/facts.unknown-format")
.expect("safe unknown UTF-8 must become resident knowledge");
assert_eq!(file.language, LanguageId::Text);
assert!(file.classification.is_text());
assert_eq!(file.content.as_slice(), exact);
assert!(file.symbols.is_empty());
let disposition = index
.manifest_entries
.iter()
.find(|entry| {
entry.path.normalized_utf8.as_deref() == Some("notes/facts.unknown-format")
})
.map(|entry| &entry.disposition)
.expect("generic knowledge must retain a manifest disposition");
assert!(matches!(
disposition,
FileDisposition::Indexed {
targets: crate::domain::IndexTargets::Knowledge,
parse_status: crate::domain::index::ParseStatus::Parsed
}
));
}
#[test]
fn sensitive_files_remain_catalog_only_with_zero_value_leakage() {
let tmp = TempDir::new().unwrap();
let canary = runtime_canary();
let password_kw = ["pass", "word"].concat();
let token_kw = ["to", "ken"].concat();
write_file(tmp.path(), ".env", &format!("{password_kw}={canary}\n"));
write_file(
tmp.path(),
"notes/guide.txt",
&format!("{token_kw}={canary}\n"),
);
let shared = LiveIndex::load(tmp.path()).expect("sensitive fixture must load safely");
let index = shared.read();
assert!(index.get_file(".env").is_none());
assert!(index.get_file("notes/guide.txt").is_none());
let env = index
.manifest_entries
.iter()
.find(|entry| entry.path.normalized_utf8.as_deref() == Some(".env"))
.expect("sensitive path must remain cataloged");
assert!(matches!(
env.disposition,
FileDisposition::MetadataOnly {
reason: MetadataOnlyReason::SensitivePath { .. }
}
));
assert!(env.content_hash.is_none());
let content = index
.manifest_entries
.iter()
.find(|entry| entry.path.normalized_utf8.as_deref() == Some("notes/guide.txt"))
.expect("detector-positive content must remain cataloged");
assert!(matches!(
content.disposition,
FileDisposition::MetadataOnly {
reason: MetadataOnlyReason::SensitiveContent { .. }
}
));
assert!(content.content_hash.is_none());
let serialized = serde_json::to_vec(&index.manifest_entries).unwrap();
assert!(
!serialized
.windows(canary.len())
.any(|window| window == canary.as_bytes())
);
}
#[test]
fn security_demotions_project_to_policy_withheld_not_a_language_verdict() {
let tmp = TempDir::new().unwrap();
let canary = runtime_canary();
let password_kw = ["pass", "word"].concat();
let token_kw = ["to", "ken"].concat();
write_file(tmp.path(), ".env", &format!("{password_kw}={canary}\n"));
write_file(
tmp.path(),
"src/config.rs",
&format!("let {token_kw} = \"{canary}\";\n"),
);
let shared = LiveIndex::load(tmp.path()).expect("sensitive fixture must load safely");
let index = shared.read();
for path in [".env", "src/config.rs"] {
let entry = index
.manifest_entries
.iter()
.find(|entry| entry.path.normalized_utf8.as_deref() == Some(path))
.unwrap_or_else(|| panic!("{path} must remain cataloged"));
assert!(
matches!(
entry.disposition,
FileDisposition::MetadataOnly {
reason: MetadataOnlyReason::SensitivePath { .. }
| MetadataOnlyReason::SensitiveContent { .. }
}
),
"{path} must be a security demotion, got {:?}",
entry.disposition
);
let decision = compatibility_admission_decision(entry)
.unwrap_or_else(|| panic!("{path} is Tier-2, so it must project a decision"));
assert_eq!(
decision.reason,
Some(SkipReason::PolicyWithheld),
"{path} must project to PolicyWithheld"
);
assert_ne!(
decision.reason,
Some(SkipReason::UnsupportedLanguage),
"{path} must never be reported as a language problem"
);
}
}
#[test]
fn safe_template_path_still_runs_content_detector() {
let tmp = TempDir::new().unwrap();
let canary = runtime_canary();
let password_kw = ["pass", "word"].concat();
write_file(
tmp.path(),
".env.example",
&format!("{password_kw}={canary}\n"),
);
let shared = LiveIndex::load(tmp.path()).expect("safe template fixture must load");
let index = shared.read();
let entry = index
.manifest_entries
.iter()
.find(|entry| entry.path.normalized_utf8.as_deref() == Some(".env.example"))
.expect("template must remain cataloged");
assert!(matches!(
entry.disposition,
FileDisposition::MetadataOnly {
reason: MetadataOnlyReason::SensitiveContent { .. }
}
));
assert!(entry.content_hash.is_none());
assert!(index.get_file(".env.example").is_none());
}
#[test]
fn lfs_pointer_is_catalog_only_and_never_knowledge_searchable() {
let tmp = TempDir::new().unwrap();
write_file(
tmp.path(),
"docs/asset.txt",
"version https://git-lfs.github.com/spec/v1\noid sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef\nsize 42\n",
);
let shared = LiveIndex::load(tmp.path()).expect("LFS pointer fixture must load");
let index = shared.read();
let entry = index
.manifest_entries
.iter()
.find(|entry| entry.path.normalized_utf8.as_deref() == Some("docs/asset.txt"))
.expect("pointer must remain cataloged");
assert!(matches!(
entry.disposition,
FileDisposition::MetadataOnly {
reason: MetadataOnlyReason::LfsPointer { .. }
}
));
assert!(entry.content_hash.is_none());
assert!(index.get_file("docs/asset.txt").is_none());
}
#[test]
fn non_utf8_text_is_catalog_only_without_lossy_evidence() {
let tmp = TempDir::new().unwrap();
let mut bytes = vec![b'a'; crate::domain::index::BINARY_SNIFF_BYTES];
bytes.extend_from_slice(&[0xff, 0xfe, b'!']);
let path = tmp.path().join("notes.txt");
std::fs::write(&path, &bytes).unwrap();
let shared = LiveIndex::load(tmp.path()).expect("invalid text must fail closed");
let index = shared.read();
let entry = index
.manifest_entries
.iter()
.find(|entry| entry.path.normalized_utf8.as_deref() == Some("notes.txt"))
.expect("invalid text must remain cataloged");
assert!(matches!(
entry.disposition,
FileDisposition::MetadataOnly {
reason: MetadataOnlyReason::UnsupportedTextEncoding
}
));
assert!(entry.content_hash.is_none());
assert!(index.get_file("notes.txt").is_none());
}
#[test]
fn load_and_reload_publish_authoritative_scout_plan() {
let tmp = TempDir::new().unwrap();
write_file(tmp.path(), "src/main.rs", "fn main() {}\n");
write_file(tmp.path(), "notes.txt", "repository knowledge\n");
let shared = LiveIndex::load(tmp.path()).unwrap();
let initial = shared
.scout_plan()
.expect("fresh load must retain its authoritative scout plan");
assert_eq!(initial.entries.len(), 2);
assert_eq!(
shared
.terminal_dispositions()
.iter()
.map(|(path, _)| path.as_str())
.collect::<Vec<_>>(),
vec!["notes.txt", "src/main.rs"]
);
assert!(initial.entries.iter().any(|entry| {
entry.path.normalized_utf8.as_deref() == Some("src/main.rs")
&& matches!(
entry.decision,
crate::domain::ScoutDecision::Ingest {
targets: crate::domain::IndexTargets::Code
}
)
}));
assert!(initial.entries.iter().any(|entry| {
entry.path.normalized_utf8.as_deref() == Some("notes.txt")
&& matches!(
entry.decision,
crate::domain::ScoutDecision::Ingest {
targets: crate::domain::IndexTargets::Knowledge
}
)
}));
write_file(tmp.path(), "README.md", "# Added later\n");
shared.reload(tmp.path()).unwrap();
let reloaded = shared
.scout_plan()
.expect("reload must atomically replace the authoritative scout plan");
assert_eq!(reloaded.entries.len(), 3);
assert_eq!(
shared
.terminal_dispositions()
.iter()
.map(|(path, _)| path.as_str())
.collect::<Vec<_>>(),
vec!["notes.txt", "README.md", "src/main.rs"]
);
assert!(
reloaded
.entries
.iter()
.any(|entry| entry.path.normalized_utf8.as_deref() == Some("README.md"))
);
}
#[test]
fn legacy_execution_projection_retains_one_outcome_per_scout_entry() {
let tmp = TempDir::new().unwrap();
write_file(tmp.path(), "src/main.rs", "fn main() {}\n");
write_file(tmp.path(), "notes.txt", "repository knowledge\n");
write_file(tmp.path(), "Cargo.lock", "version = 4\n");
let scout = discovery::scout_repository(tmp.path()).unwrap();
let projection = project_scout_for_legacy_execution(&scout);
let accounted = projection.entries.len() + projection.terminal_dispositions.len();
let mut paths: Vec<_> = projection
.entries
.iter()
.map(|entry| entry.relative_path.as_str())
.chain(
projection
.terminal_dispositions
.iter()
.map(|(path, _)| path.as_str()),
)
.collect();
paths.sort_unstable();
paths.dedup();
assert_eq!(accounted, scout.entries.len());
assert_eq!(paths.len(), scout.entries.len());
}
#[test]
fn compatibility_skips_are_projected_from_manifest_entries() {
let mut index = make_empty_live_index();
index.manifest_entries = vec![crate::domain::CatalogEntry {
path: crate::domain::CatalogPath {
public_id: "Cargo.lock".to_string(),
normalized_utf8: Some("Cargo.lock".to_string()),
},
size: 128,
language: Some(crate::domain::LanguageId::Toml),
classification: crate::domain::FileClassification {
class: crate::domain::FileClass::Text,
is_generated: false,
is_test: false,
is_vendor: false,
is_config: true,
},
disposition: crate::domain::FileDisposition::MetadataOnly {
reason: crate::domain::MetadataOnlyReason::Lockfile,
},
content_hash: None,
}];
let projected = index.compatibility_skipped_files();
assert_eq!(projected.len(), 1);
assert_eq!(projected[0].path, "Cargo.lock");
assert_eq!(projected[0].size, 128);
assert_eq!(
projected[0].decision.reason,
Some(SkipReason::DependencyLockfile)
);
assert_eq!(index.tier_counts(), (0, 1, 0));
}
#[test]
fn tier_and_parse_counters_reconcile_on_synthetic_index() {
let mut index = make_empty_live_index();
let mut parsed = make_indexed_file_for_mutation("src/parsed.rs");
parsed.parse_status = ParseStatus::Parsed;
index.update_file("src/parsed.rs".into(), parsed);
let mut partial = make_indexed_file_for_mutation("src/partial.rs");
partial.parse_status = ParseStatus::PartialParse {
warning: "partial".into(),
};
index.update_file("src/partial.rs".into(), partial);
let mut failed = make_indexed_file_for_mutation("src/failed.rs");
failed.parse_status = ParseStatus::Failed {
error: "boom".into(),
};
index.update_file("src/failed.rs".into(), failed);
let manifest_entry = |path: &str, size: u64, disposition: FileDisposition| CatalogEntry {
path: crate::domain::CatalogPath {
public_id: path.to_string(),
normalized_utf8: Some(path.to_string()),
},
size,
language: None,
classification: FileClassification {
class: crate::domain::FileClass::Text,
is_generated: false,
is_test: false,
is_vendor: false,
is_config: false,
},
disposition,
content_hash: None,
};
index.manifest_entries.extend([
manifest_entry(
"vendor.lock",
100,
FileDisposition::MetadataOnly {
reason: MetadataOnlyReason::Lockfile,
},
),
manifest_entry(
"tests/foo.tcl",
50,
FileDisposition::MetadataOnly {
reason: MetadataOnlyReason::UnsupportedTextEncoding,
},
),
manifest_entry(
"huge.bin",
200_000_000,
FileDisposition::HardSkip {
reason: HardSkipReason::PerFileCeiling,
},
),
]);
let (tier1, tier2, tier3) = index.tier_counts();
let discovered = index.file_count() + index.compatibility_skipped_files().len();
assert_eq!(
tier1 + tier2 + tier3,
discovered,
"tier sum ({}+{}+{}) must equal discovered records ({})",
tier1,
tier2,
tier3,
discovered
);
assert_eq!(tier1, index.file_count(), "Tier-1 must equal indexed files");
assert_eq!((tier1, tier2, tier3), (3, 2, 1));
let stats = index.health_stats();
assert_eq!(
stats.parsed_count + stats.partial_parse_count + stats.failed_count,
stats.file_count,
"parsed+partial+failed must equal indexed file count"
);
assert_eq!(stats.file_count, 3);
assert_eq!(stats.parsed_count, 1);
assert_eq!(stats.partial_parse_count, 1);
assert_eq!(stats.failed_count, 1);
}
}