use std::path::Path;
use rayon::prelude::*;
use crate::chunk_drive::{ChunkCut, drive_chunks};
use crate::index::IndexDb;
use crate::intel::model::FileResolvedRefs;
use crate::intel::stage_budget::BudgetedWriter;
use crate::lang;
use crate::path::RelPath;
use crate::scanner_lanes::contain_panic;
use crate::store::Store;
const INDEX_COMMIT_BATCH: usize = 256;
const RESOLVE_CHUNK_FILES: usize = 1024;
const RESOLVE_CHUNK_SOURCE_BYTES: u64 = 16 * 1024 * 1024;
fn resolve_chunk_cut() -> ChunkCut {
ChunkCut::new(RESOLVE_CHUNK_FILES, RESOLVE_CHUNK_SOURCE_BYTES)
}
struct FileSnapshot {
rel: String,
hash_hex: String,
language: String,
size_bytes: u64,
}
pub(crate) fn resolve_pass(root: &Path, store: &Store, precise: bool) {
let Some(index_db) = store.index_db.as_ref() else {
return;
};
let files: Vec<FileSnapshot> = store
.index
.files
.iter()
.map(|(rel, entry)| FileSnapshot {
rel: rel.to_str_lossy().into_owned(),
hash_hex: entry.hash_hex.clone(),
language: entry.language.clone(),
size_bytes: entry.size_bytes,
})
.collect();
#[cfg(any(feature = "code-intel-js", feature = "code-intel-stack"))]
{
let mut cross_file: ahash::AHashMap<String, crate::intel::xfile::FileFacts> = ahash::AHashMap::new();
resolve_in_chunks(root, store, index_db, &files, precise, |facts| {
harvest_cross_file_facts(facts, &mut cross_file);
});
crate::intel::xfile::stitch_cross_file_edges(root, store, index_db, &cross_file);
}
#[cfg(not(any(feature = "code-intel-js", feature = "code-intel-stack")))]
resolve_in_chunks(root, store, index_db, &files, precise, drop);
}
pub(crate) fn resolve_pass_incremental(root: &Path, store: &Store, changed: &[String], precise: bool) {
let Some(index_db) = store.index_db.as_ref() else {
return;
};
let changed_snapshot: Vec<FileSnapshot> = changed
.iter()
.filter_map(|rel| {
let entry = store.lookup(rel.as_str())?;
Some(FileSnapshot {
rel: rel.clone(),
hash_hex: entry.hash_hex.clone(),
language: entry.language.clone(),
size_bytes: entry.size_bytes,
})
})
.collect();
resolve_in_chunks(root, store, index_db, &changed_snapshot, precise, drop);
#[cfg(any(feature = "code-intel-js", feature = "code-intel-stack"))]
xfile_incremental::restitch_affected(root, store, index_db, changed, precise);
}
fn resolve_in_chunks(
root: &Path,
store: &Store,
index_db: &IndexDb,
files: &[FileSnapshot],
precise: bool,
mut harvest: impl FnMut(Vec<(String, FileResolvedRefs)>),
) {
drive_chunks(
files,
resolve_chunk_cut(),
|snapshot| snapshot.size_bytes,
|chunk| compute_facts(root, store, chunk, precise),
|facts| {
stage_facts(index_db, &facts);
harvest(facts);
},
);
}
fn compute_facts(root: &Path, store: &Store, files: &[FileSnapshot], precise: bool) -> Vec<(String, FileResolvedRefs)> {
files
.par_iter()
.filter_map(|snapshot| {
let rel_str = snapshot.rel.as_str();
let refs = match store.read_resolved_by_hex(&snapshot.hash_hex) {
Ok(Some(cached)) => cached,
_ => {
let lang = lang::intern(&snapshot.language)?;
let abs = root.join(rel_str);
let bytes = std::fs::read(&abs).ok()?;
let computed = match contain_panic(|| crate::intel::resolve::resolve_file(lang, &abs, &bytes, precise))
{
Ok(computed) => computed,
Err(reason) => {
tracing::warn!(
path = rel_str,
lang,
reason,
"resolve pass: resolver panicked on this file — skipping it; its navigation stays name-only"
);
return None;
}
};
if !computed.is_empty() {
let _ = store.write_resolved_hex(&snapshot.hash_hex, &computed);
}
computed
}
};
Some((rel_str.to_string(), refs))
})
.collect()
}
fn stage_facts(index_db: &IndexDb, facts: &[(String, FileResolvedRefs)]) {
let mut batch = BudgetedWriter::new(index_db, INDEX_COMMIT_BATCH, "resolve pass");
for (rel_str, refs) in facts {
let rel = RelPath::from(rel_str.as_str());
let staged_res = if refs.is_empty() {
batch.writer().remove_resolved_file(&rel)
} else {
batch.writer().upsert_resolved_file(&rel, refs)
};
if let Err(error) = staged_res {
tracing::warn!(path = %rel, %error, "resolve pass: failed to stage resolved edges — skipping file");
}
batch.item_staged();
}
batch.finish();
}
#[cfg(any(feature = "code-intel-js", feature = "code-intel-stack"))]
fn import_bound_edges(refs: &FileResolvedRefs) -> Vec<crate::intel::model::ResolvedEdge> {
if refs.imports.is_empty() || refs.intra.is_empty() {
return Vec::new();
}
let import_starts: ahash::AHashSet<u32> = refs.imports.iter().map(|i| i.local_start).collect();
refs.intra
.iter()
.filter(|e| import_starts.contains(&e.def_start))
.cloned()
.collect()
}
#[cfg(any(feature = "code-intel-js", feature = "code-intel-stack"))]
fn harvest_cross_file_facts(
facts: Vec<(String, FileResolvedRefs)>,
out: &mut ahash::AHashMap<String, crate::intel::xfile::FileFacts>,
) {
for (rel, refs) in facts {
if refs.imports.is_empty() && refs.exports.is_empty() {
continue;
}
let import_uses = import_bound_edges(&refs);
out.insert(
rel,
crate::intel::xfile::FileFacts {
imports: refs.imports,
exports: refs.exports,
import_uses,
},
);
}
}
#[cfg(any(feature = "code-intel-js", feature = "code-intel-stack"))]
mod xfile_incremental {
use std::path::Path;
use ahash::{AHashMap, AHashSet};
use rayon::prelude::*;
use super::{FileSnapshot, resolve_in_chunks};
use crate::index::IndexDb;
use crate::intel::resolver::SpecifierResolver;
use crate::intel::xfile::{FileFacts, stitch_cross_file_edges};
use crate::store::Store;
fn has_resolver(language: &str) -> bool {
SpecifierResolver::supports(language)
}
struct FileEntry {
language: String,
facts: FileFacts,
}
type ResolverCache = AHashMap<String, Option<SpecifierResolver>>;
fn build_resolvers(entries: &AHashMap<String, FileEntry>) -> ResolverCache {
let mut cache = ResolverCache::new();
for entry in entries.values() {
if !cache.contains_key(&entry.language) {
cache.insert(entry.language.clone(), SpecifierResolver::for_language(&entry.language));
}
}
cache
}
fn resolve_targets(
root: &Path,
importer: &str,
entry: &FileEntry,
resolvers: &ResolverCache,
out: &mut Vec<String>,
) {
let Some(Some(resolver)) = resolvers.get(&entry.language) else {
return;
};
for import in &entry.facts.imports {
if import.is_type {
continue;
}
if let Some(target) = resolver.resolve(root, importer, import)
&& let Some(key) = target.as_str()
{
out.push(key.to_string());
}
}
}
pub(super) fn restitch_affected(root: &Path, store: &Store, index_db: &IndexDb, changed: &[String], precise: bool) {
let changed_set: AHashSet<&str> = changed
.iter()
.filter(|rel| store.lookup(rel.as_str()).is_some_and(|e| has_resolver(&e.language)))
.map(String::as_str)
.collect();
if changed_set.is_empty() {
return;
}
let candidate_files: Vec<(String, String, String)> = store
.index
.files
.iter()
.filter(|(_, e)| has_resolver(&e.language))
.map(|(rel, e)| (rel.to_str_lossy().into_owned(), e.hash_hex.clone(), e.language.clone()))
.collect();
let entries: AHashMap<String, FileEntry> = candidate_files
.par_iter()
.filter_map(|(rel, hash, language)| {
let refs = store.read_resolved_by_hex(hash).ok()??;
if refs.imports.is_empty() && refs.exports.is_empty() {
return None;
}
let import_uses = super::import_bound_edges(&refs);
Some((
rel.clone(),
FileEntry {
language: language.clone(),
facts: FileFacts {
imports: refs.imports,
exports: refs.exports,
import_uses,
},
},
))
})
.collect::<Vec<_>>()
.into_iter()
.collect();
let resolvers = build_resolvers(&entries);
let importers_of_changed: Vec<String> = entries
.par_iter()
.filter_map(|(importer, entry)| {
if entry.facts.imports.is_empty() {
return None;
}
let mut targets = Vec::new();
resolve_targets(root, importer, entry, &resolvers, &mut targets);
targets
.iter()
.any(|t| changed_set.contains(t.as_str()))
.then(|| importer.clone())
})
.collect();
let mut affected: AHashSet<String> = changed_set.iter().map(|s| (*s).to_string()).collect();
affected.extend(importers_of_changed);
let unchanged_affected: Vec<FileSnapshot> = affected
.iter()
.filter(|k| !changed_set.contains(k.as_str()))
.filter_map(|k| {
let entry = store.lookup(k.as_str())?;
Some(FileSnapshot {
rel: k.clone(),
hash_hex: entry.hash_hex.clone(),
language: entry.language.clone(),
size_bytes: entry.size_bytes,
})
})
.collect();
resolve_in_chunks(root, store, index_db, &unchanged_affected, precise, drop);
let mut stitch_facts: AHashMap<String, FileFacts> = AHashMap::with_capacity(affected.len());
for key in &affected {
if let Some(entry) = entries.get(key) {
stitch_facts.insert(
key.clone(),
FileFacts {
imports: entry.facts.imports.clone(),
exports: entry.facts.exports.clone(),
import_uses: entry.facts.import_uses.clone(),
},
);
}
}
let mut provider_targets: Vec<String> = Vec::new();
for key in &affected {
if let Some(entry) = entries.get(key) {
resolve_targets(root, key, entry, &resolvers, &mut provider_targets);
}
}
for target in provider_targets {
if stitch_facts.contains_key(&target) {
continue;
}
if let Some(entry) = entries.get(&target) {
stitch_facts.insert(
target,
FileFacts {
imports: Vec::new(),
exports: entry.facts.exports.clone(),
import_uses: Vec::new(),
},
);
}
}
stitch_cross_file_edges(root, store, index_db, &stitch_facts);
}
}
#[cfg(test)]
mod tests {
use super::*;
fn snapshot(rel: &str, size_bytes: u64) -> FileSnapshot {
FileSnapshot {
rel: rel.to_string(),
hash_hex: "0".repeat(64),
language: "typescript".to_string(),
size_bytes,
}
}
#[test]
fn the_pass_cut_is_byte_aware_and_not_only_count_aware() {
let heavy: Vec<FileSnapshot> = (0..8)
.map(|i| snapshot(&format!("src/generated_{i}.ts"), 8 * 1024 * 1024))
.collect();
let mut chunk_sizes = Vec::new();
drive_chunks(
&heavy,
resolve_chunk_cut(),
|s| s.size_bytes,
|chunk| {
chunk_sizes.push(chunk.len());
Vec::<()>::new()
},
|_| {},
);
assert!(
chunk_sizes.len() > 1,
"eight 8 MiB files must not ride in one chunk under a {RESOLVE_CHUNK_SOURCE_BYTES}-byte budget"
);
assert!(
heavy.len() < RESOLVE_CHUNK_FILES,
"the file-count bound must not be what cut these chunks"
);
assert_eq!(chunk_sizes.iter().sum::<usize>(), heavy.len());
}
#[test]
fn ordinary_files_still_ride_the_count_bound() {
let light: Vec<FileSnapshot> = (0..RESOLVE_CHUNK_FILES)
.map(|i| snapshot(&format!("src/hand_written_{i}.ts"), 4 * 1024))
.collect();
let mut chunk_sizes = Vec::new();
drive_chunks(
&light,
resolve_chunk_cut(),
|s| s.size_bytes,
|chunk| {
chunk_sizes.push(chunk.len());
Vec::<()>::new()
},
|_| {},
);
assert_eq!(chunk_sizes, vec![RESOLVE_CHUNK_FILES]);
}
#[cfg(any(feature = "code-intel-js", feature = "code-intel-stack"))]
#[test]
fn the_harvest_retains_only_import_bound_edges() {
use crate::intel::model::{ImportEdge, ResolvedEdge};
let mut refs = FileResolvedRefs::new("typescript");
refs.imports = vec![ImportEdge {
local: "useThing".to_string(),
specifier: "./thing".to_string(),
imported: Some("useThing".to_string()),
is_type: false,
local_start: 7,
}];
refs.intra = (0..256)
.map(|i| ResolvedEdge {
use_start: 1000 + i * 16,
use_end: 1008 + i * 16,
def_start: if i.is_multiple_of(128) { 7 } else { 500 },
def_end: 20,
})
.collect();
let mut harvested = ahash::AHashMap::new();
harvest_cross_file_facts(vec![("src/app.ts".to_string(), refs)], &mut harvested);
let facts = harvested.get("src/app.ts").expect("importer must be harvested");
assert_eq!(
facts.import_uses.len(),
2,
"only the edges bound to an import may survive the chunk"
);
assert!(facts.import_uses.iter().all(|e| e.def_start == 7));
}
#[cfg(any(feature = "code-intel-js", feature = "code-intel-stack"))]
#[test]
fn what_survives_the_chunks_scales_with_imports_not_with_resolved_edges() {
use crate::intel::model::{ImportEdge, ResolvedEdge};
const CHUNKS: usize = 8;
const FILES_PER_CHUNK: usize = 16;
const EDGES_PER_FILE: u32 = 4096;
let file_facts = |rel: String| {
let mut refs = FileResolvedRefs::new("typescript");
refs.imports = vec![ImportEdge {
local: "useThing".to_string(),
specifier: "./thing".to_string(),
imported: Some("useThing".to_string()),
is_type: false,
local_start: 7,
}];
refs.intra = (0..EDGES_PER_FILE)
.map(|i| ResolvedEdge {
use_start: 1000 + i * 16,
use_end: 1008 + i * 16,
def_start: if i == 0 { 7 } else { 500 },
def_end: 20,
})
.collect();
(rel, refs)
};
let mut retained: ahash::AHashMap<String, crate::intel::xfile::FileFacts> = ahash::AHashMap::new();
for chunk in 0..CHUNKS {
let computed: Vec<(String, FileResolvedRefs)> = (0..FILES_PER_CHUNK)
.map(|f| file_facts(format!("src/c{chunk}/f{f}.ts")))
.collect();
harvest_cross_file_facts(computed, &mut retained);
}
let files = CHUNKS * FILES_PER_CHUNK;
let computed_edges = files * EDGES_PER_FILE as usize;
let survived: usize = retained.values().map(|f| f.import_uses.len()).sum();
assert_eq!(retained.len(), files, "every importer must be harvested");
assert_eq!(
survived, files,
"exactly the one import-bound edge per file may outlive its chunk"
);
assert!(
survived * 1000 < computed_edges,
"{survived} edges survived {computed_edges} computed — the projection stopped being one"
);
let (_rel, facts) = retained.into_iter().next().expect("at least one importer");
let crate::intel::xfile::FileFacts {
imports,
exports,
import_uses,
} = facts;
assert_eq!(imports.len(), 1);
assert!(exports.is_empty());
assert_eq!(import_uses.len(), 1);
}
}