use std::path::Path;
use rayon::prelude::*;
use crate::backpressure::FootprintGate;
use crate::chunk_drive::{ChunkCut, drive_chunks_governed};
use crate::config::MaxFootprint;
use crate::index::IndexDb;
use crate::intel::model::FileResolvedRefs;
use crate::intel::stage_budget::BudgetedWriter;
use crate::lang;
use crate::path::RelPath;
use crate::scanner_lanes::contain_panic;
use crate::store::Store;
const INDEX_COMMIT_BATCH: usize = 256;
const RESOLVE_CHUNK_FILES: usize = 1024;
const RESOLVE_CHUNK_SOURCE_BYTES: u64 = 16 * 1024 * 1024;
fn resolve_chunk_cut() -> ChunkCut {
ChunkCut::new(RESOLVE_CHUNK_FILES, RESOLVE_CHUNK_SOURCE_BYTES)
}
struct FileSnapshot {
rel: String,
hash_hex: String,
language: String,
size_bytes: u64,
}
pub(crate) fn resolve_pass(root: &Path, store: &Store, precise: bool, footprint: MaxFootprint) -> bool {
let Some(index_db) = store.index_db.as_ref() else {
return true;
};
let files: Vec<FileSnapshot> = store
.index
.files
.iter()
.map(|(rel, entry)| FileSnapshot {
rel: rel.to_str_lossy().into_owned(),
hash_hex: entry.hash_hex.clone(),
language: entry.language.clone(),
size_bytes: entry.size_bytes,
})
.collect();
#[cfg(any(feature = "code-intel-js", feature = "code-intel-stack"))]
let committed = {
if let Err(error) = index_db.set_import_probes_complete(false) {
tracing::warn!(%error, "resolve pass: could not withdraw the import-probe stamp");
}
let mut cross_file: ahash::AHashMap<String, crate::intel::xfile::FileFacts> = ahash::AHashMap::new();
let staged = resolve_in_chunks(root, store, index_db, &files, precise, footprint, |facts| {
harvest_cross_file_facts(facts, &mut cross_file);
});
let stitched = crate::intel::xfile::stitch_cross_file_edges(root, store, index_db, &cross_file);
if staged
&& stitched
&& let Err(error) = index_db.set_import_probes_complete(true)
{
tracing::warn!(%error, "resolve pass: could not stamp the import-probe index complete");
}
staged && stitched
};
#[cfg(not(any(feature = "code-intel-js", feature = "code-intel-stack")))]
let committed = resolve_in_chunks(root, store, index_db, &files, precise, footprint, drop);
committed
}
#[cfg_attr(not(any(feature = "code-intel-js", feature = "code-intel-stack")), allow(dead_code))]
pub(crate) struct PathDelta<'a> {
pub changed: &'a [String],
pub removed: &'a [String],
pub other: &'a [String],
}
pub(crate) fn probed_only(index_db: &IndexDb, candidates: Vec<String>) -> Vec<String> {
candidates.into_iter().filter(|rel| index_db.is_probed(rel)).collect()
}
pub(crate) fn resolve_pass_incremental(
root: &Path,
store: &Store,
delta: &PathDelta<'_>,
precise: bool,
footprint: MaxFootprint,
) {
let Some(index_db) = store.index_db.as_ref() else {
return;
};
#[cfg(any(feature = "code-intel-js", feature = "code-intel-stack"))]
if !index_db.import_probes_complete() {
tracing::info!("resolve pass: import-probe index not built yet — running the wholesale pass once");
resolve_pass(root, store, precise, footprint);
return;
}
let changed_snapshot = snapshot_of(store, delta.changed.iter());
#[cfg(any(feature = "code-intel-js", feature = "code-intel-stack"))]
{
let mut importers: ahash::AHashMap<String, crate::intel::xfile::FileFacts> = ahash::AHashMap::new();
resolve_in_chunks(root, store, index_db, &changed_snapshot, precise, footprint, |facts| {
harvest_cross_file_facts(facts, &mut importers);
});
let affected = xfile_incremental::affected_importers(index_db, store, delta);
let affected_snapshot = snapshot_of(store, affected.iter());
resolve_in_chunks(root, store, index_db, &affected_snapshot, precise, footprint, |facts| {
harvest_cross_file_facts(facts, &mut importers);
});
crate::intel::xfile::stitch_cross_file_edges(root, store, index_db, &importers);
}
#[cfg(not(any(feature = "code-intel-js", feature = "code-intel-stack")))]
resolve_in_chunks(root, store, index_db, &changed_snapshot, precise, footprint, drop);
}
fn snapshot_of<'a>(store: &Store, rels: impl Iterator<Item = &'a String>) -> Vec<FileSnapshot> {
rels.filter_map(|rel| {
let entry = store.lookup(rel.as_str())?;
Some(FileSnapshot {
rel: rel.clone(),
hash_hex: entry.hash_hex.clone(),
language: entry.language.clone(),
size_bytes: entry.size_bytes,
})
})
.collect()
}
fn resolve_in_chunks(
root: &Path,
store: &Store,
index_db: &IndexDb,
files: &[FileSnapshot],
precise: bool,
footprint: MaxFootprint,
mut harvest: impl FnMut(Vec<(String, FileResolvedRefs)>),
) -> bool {
let gate = FootprintGate::new(footprint);
let mut all_committed = true;
drive_chunks_governed(
&mut (),
files,
|| {
if index_db.fatal_fault().is_some() {
return None;
}
gate.admit();
Some(resolve_chunk_cut())
},
|(), snapshot| snapshot.size_bytes,
|(), chunk| compute_facts(root, store, chunk, precise),
|(), facts| {
all_committed &= stage_facts(index_db, &facts);
harvest(facts);
},
);
all_committed && index_db.fatal_fault().is_none()
}
fn compute_facts(root: &Path, store: &Store, files: &[FileSnapshot], precise: bool) -> Vec<(String, FileResolvedRefs)> {
files
.par_iter()
.filter_map(|snapshot| {
let rel_str = snapshot.rel.as_str();
let refs = match store.read_resolved_by_hex(&snapshot.hash_hex) {
Ok(Some(cached)) => cached,
_ => {
let lang = lang::intern(&snapshot.language)?;
let abs = root.join(rel_str);
let bytes = std::fs::read(&abs).ok()?;
let computed = match contain_panic(|| crate::intel::resolve::resolve_file(lang, &abs, &bytes, precise))
{
Ok(computed) => computed,
Err(reason) => {
tracing::warn!(
path = rel_str,
lang,
reason,
"resolve pass: resolver panicked on this file — skipping it; its navigation stays name-only"
);
return None;
}
};
if !computed.is_empty() {
let _ = store.write_resolved_hex(&snapshot.hash_hex, &computed);
}
computed
}
};
Some((rel_str.to_string(), refs))
})
.collect()
}
fn stage_facts(index_db: &IndexDb, facts: &[(String, FileResolvedRefs)]) -> bool {
let mut batch = BudgetedWriter::new(index_db, INDEX_COMMIT_BATCH, "resolve pass");
for (rel_str, refs) in facts {
let rel = RelPath::from(rel_str.as_str());
let staged_res = if refs.is_empty() {
batch.writer().remove_resolved_file(&rel)
} else {
batch.writer().upsert_resolved_file(&rel, refs)
};
if let Err(error) = staged_res {
tracing::warn!(path = %rel, %error, "resolve pass: failed to stage resolved edges — skipping file");
}
batch.item_staged();
}
batch.finish()
}
#[cfg(any(feature = "code-intel-js", feature = "code-intel-stack"))]
fn import_bound_edges(refs: &FileResolvedRefs) -> Vec<crate::intel::model::ResolvedEdge> {
if refs.imports.is_empty() || refs.intra.is_empty() {
return Vec::new();
}
let import_starts: ahash::AHashSet<u32> = refs.imports.iter().map(|i| i.local_start).collect();
refs.intra
.iter()
.filter(|e| import_starts.contains(&e.def_start))
.cloned()
.collect()
}
#[cfg(any(feature = "code-intel-js", feature = "code-intel-stack"))]
fn harvest_cross_file_facts(
facts: Vec<(String, FileResolvedRefs)>,
out: &mut ahash::AHashMap<String, crate::intel::xfile::FileFacts>,
) {
for (rel, refs) in facts {
if refs.imports.is_empty() && refs.exports.is_empty() {
continue;
}
let import_uses = import_bound_edges(&refs);
out.insert(
rel,
crate::intel::xfile::FileFacts {
imports: refs.imports,
exports: refs.exports,
import_uses,
},
);
}
}
#[cfg(any(feature = "code-intel-js", feature = "code-intel-stack"))]
mod xfile_incremental {
use ahash::AHashSet;
use super::PathDelta;
use crate::index::IndexDb;
use crate::intel::resolver::SpecifierResolver;
use crate::store::Store;
pub(super) fn affected_importers(index_db: &IndexDb, store: &Store, delta: &PathDelta<'_>) -> Vec<String> {
let changed: AHashSet<&str> = delta.changed.iter().map(String::as_str).collect();
let mut affected: AHashSet<String> = AHashSet::new();
for touched in delta.changed.iter().chain(delta.removed).chain(delta.other) {
for importer in index_db.importers_probing(touched) {
let Some(key) = importer.as_str() else {
continue;
};
if !changed.contains(key)
&& store
.lookup(key)
.is_some_and(|entry| SpecifierResolver::supports(&entry.language))
{
affected.insert(key.to_string());
}
}
}
affected.into_iter().collect()
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::chunk_drive::drive_chunks;
fn snapshot(rel: &str, size_bytes: u64) -> FileSnapshot {
FileSnapshot {
rel: rel.to_string(),
hash_hex: "0".repeat(64),
language: "typescript".to_string(),
size_bytes,
}
}
#[test]
fn the_pass_cut_is_byte_aware_and_not_only_count_aware() {
let heavy: Vec<FileSnapshot> = (0..8)
.map(|i| snapshot(&format!("src/generated_{i}.ts"), 8 * 1024 * 1024))
.collect();
let mut chunk_sizes = Vec::new();
drive_chunks(
&heavy,
resolve_chunk_cut(),
|s| s.size_bytes,
|chunk| {
chunk_sizes.push(chunk.len());
Vec::<()>::new()
},
|_| {},
);
assert!(
chunk_sizes.len() > 1,
"eight 8 MiB files must not ride in one chunk under a {RESOLVE_CHUNK_SOURCE_BYTES}-byte budget"
);
assert!(
heavy.len() < RESOLVE_CHUNK_FILES,
"the file-count bound must not be what cut these chunks"
);
assert_eq!(chunk_sizes.iter().sum::<usize>(), heavy.len());
}
#[test]
fn ordinary_files_still_ride_the_count_bound() {
let light: Vec<FileSnapshot> = (0..RESOLVE_CHUNK_FILES)
.map(|i| snapshot(&format!("src/hand_written_{i}.ts"), 4 * 1024))
.collect();
let mut chunk_sizes = Vec::new();
drive_chunks(
&light,
resolve_chunk_cut(),
|s| s.size_bytes,
|chunk| {
chunk_sizes.push(chunk.len());
Vec::<()>::new()
},
|_| {},
);
assert_eq!(chunk_sizes, vec![RESOLVE_CHUNK_FILES]);
}
#[cfg(any(feature = "code-intel-js", feature = "code-intel-stack"))]
#[test]
fn the_harvest_retains_only_import_bound_edges() {
use crate::intel::model::{ImportEdge, ResolvedEdge};
let mut refs = FileResolvedRefs::new("typescript");
refs.imports = vec![ImportEdge {
local: "useThing".to_string(),
specifier: "./thing".to_string(),
imported: Some("useThing".to_string()),
is_type: false,
local_start: 7,
}];
refs.intra = (0..256)
.map(|i| ResolvedEdge {
use_start: 1000 + i * 16,
use_end: 1008 + i * 16,
def_start: if i.is_multiple_of(128) { 7 } else { 500 },
def_end: 20,
})
.collect();
let mut harvested = ahash::AHashMap::new();
harvest_cross_file_facts(vec![("src/app.ts".to_string(), refs)], &mut harvested);
let facts = harvested.get("src/app.ts").expect("importer must be harvested");
assert_eq!(
facts.import_uses.len(),
2,
"only the edges bound to an import may survive the chunk"
);
assert!(facts.import_uses.iter().all(|e| e.def_start == 7));
}
#[cfg(any(feature = "code-intel-js", feature = "code-intel-stack"))]
#[test]
fn what_survives_the_chunks_scales_with_imports_not_with_resolved_edges() {
use crate::intel::model::{ImportEdge, ResolvedEdge};
const CHUNKS: usize = 8;
const FILES_PER_CHUNK: usize = 16;
const EDGES_PER_FILE: u32 = 4096;
let file_facts = |rel: String| {
let mut refs = FileResolvedRefs::new("typescript");
refs.imports = vec![ImportEdge {
local: "useThing".to_string(),
specifier: "./thing".to_string(),
imported: Some("useThing".to_string()),
is_type: false,
local_start: 7,
}];
refs.intra = (0..EDGES_PER_FILE)
.map(|i| ResolvedEdge {
use_start: 1000 + i * 16,
use_end: 1008 + i * 16,
def_start: if i == 0 { 7 } else { 500 },
def_end: 20,
})
.collect();
(rel, refs)
};
let mut retained: ahash::AHashMap<String, crate::intel::xfile::FileFacts> = ahash::AHashMap::new();
for chunk in 0..CHUNKS {
let computed: Vec<(String, FileResolvedRefs)> = (0..FILES_PER_CHUNK)
.map(|f| file_facts(format!("src/c{chunk}/f{f}.ts")))
.collect();
harvest_cross_file_facts(computed, &mut retained);
}
let files = CHUNKS * FILES_PER_CHUNK;
let computed_edges = files * EDGES_PER_FILE as usize;
let survived: usize = retained.values().map(|f| f.import_uses.len()).sum();
assert_eq!(retained.len(), files, "every importer must be harvested");
assert_eq!(
survived, files,
"exactly the one import-bound edge per file may outlive its chunk"
);
assert!(
survived * 1000 < computed_edges,
"{survived} edges survived {computed_edges} computed — the projection stopped being one"
);
let (_rel, facts) = retained.into_iter().next().expect("at least one importer");
let crate::intel::xfile::FileFacts {
imports,
exports,
import_uses,
} = facts;
assert_eq!(imports.len(), 1);
assert!(exports.is_empty());
assert_eq!(import_uses.len(), 1);
}
}