use std::path::Path;
use std::time::Instant;
use anyhow::{Context, Result};
use chrono::{DateTime, Utc};
use uuid::Uuid;
use ra_ap_ide::{Analysis, AnalysisHost, FileId};
use ra_ap_load_cargo::{load_workspace_at, LoadCargoConfig, ProcMacroServerChoice};
use ra_ap_project_model::CargoConfig;
use ra_ap_vfs::{Vfs, VfsPath};
use super::scip::{ScipRow, ScipScan};
fn gatling_map_owned<T, R, F>(items: Vec<T>, f: F) -> Vec<R>
where
T: Send,
R: Send,
F: Fn(T) -> R + Sync,
{
let n = items.len();
if n == 0 {
return Vec::new();
}
let workers = std::thread::available_parallelism()
.map(|w| w.get())
.unwrap_or(1)
.min(n);
if workers <= 1 {
return items.into_iter().map(f).collect();
}
use std::cell::UnsafeCell;
use std::mem::MaybeUninit;
use std::sync::atomic::{AtomicUsize, Ordering};
struct Slots<X>(Vec<UnsafeCell<MaybeUninit<X>>>);
unsafe impl<X: Send> Sync for Slots<X> {}
let inputs = Slots(
items
.into_iter()
.map(|t| UnsafeCell::new(MaybeUninit::new(t)))
.collect::<Vec<_>>(),
);
let outputs: Slots<R> =
Slots((0..n).map(|_| UnsafeCell::new(MaybeUninit::uninit())).collect());
let cursor = AtomicUsize::new(0);
let inputs_ref = &inputs;
let outputs_ref = &outputs;
let cursor_ref = &cursor;
let f_ref = &f;
std::thread::scope(|s| {
for _ in 0..workers {
s.spawn(move || loop {
let i = cursor_ref.fetch_add(1, Ordering::Relaxed);
if i >= n {
break;
}
let item = unsafe { (*inputs_ref.0[i].get()).assume_init_read() };
let r = f_ref(item);
unsafe { (*outputs_ref.0[i].get()).write(r) };
});
}
});
outputs
.0
.into_iter()
.map(|c| unsafe { c.into_inner().assume_init() })
.collect()
}
struct DefSite {
file_id: FileId,
offset: ra_ap_ide::TextSize,
label: String,
kind: String,
file: String,
line: u32,
col: u32,
enc_start_line: u32,
enc_end_line: u32,
}
pub fn ingest_in_process(
root: &Path,
repo: &str,
git_sha: &str,
snapshot_id: Uuid,
ts: DateTime<Utc>,
) -> Result<ScipScan> {
let single_pass = std::env::var("NORNIR_SCIP_SINGLEPASS").map(|v| v != "0").unwrap_or(true);
if single_pass {
return ingest_in_process_single_pass(root, repo, git_sha, snapshot_id, ts);
}
let cargo_config = CargoConfig::default();
let load_config = LoadCargoConfig {
load_out_dirs_from_check: true,
with_proc_macro_server: ProcMacroServerChoice::Sysroot,
prefill_caches: true,
num_worker_threads: std::thread::available_parallelism().map(|n| n.get()).unwrap_or(8),
proc_macro_processes: 0,
};
let (db, vfs, _proc_macro) = load_workspace_at(
root,
&cargo_config,
&load_config,
&|_progress| {},
)
.with_context(|| format!("loading cargo workspace at {} into rust-analyzer", root.display()))?;
let host = AnalysisHost::with_database(db);
let analysis = host.analysis();
let root_str = root.to_string_lossy().replace('\\', "/");
let files: Vec<(FileId, String)> = vfs
.iter()
.filter_map(|(file_id, path)| vfs_rel_path(path, &root_str).map(|p| (file_id, p)))
.filter(|(_, p)| p.ends_with(".rs"))
.collect();
let mut defs: Vec<DefSite> = Vec::new();
for (file_id, file) in &files {
let struct_cfg = ra_ap_ide::FileStructureConfig { exclude_locals: true };
let Ok(nodes) = analysis.file_structure(&struct_cfg, *file_id) else { continue };
let Ok(line_index) = analysis.file_line_index(*file_id) else { continue };
for node in nodes {
let off = node.navigation_range.start();
let lc = line_index.line_col(off);
let enc_lc_start = line_index.line_col(node.node_range.start());
let enc_lc_end = line_index.line_col(node.node_range.end());
defs.push(DefSite {
file_id: *file_id,
offset: off,
label: node.label,
kind: format!("{:?}", node.kind),
file: file.clone(),
line: lc.line + 1,
col: lc.col + 1,
enc_start_line: enc_lc_start.line + 1,
enc_end_line: enc_lc_end.line + 1,
});
}
}
let workers = std::thread::available_parallelism()
.map(|w| w.get())
.unwrap_or(1)
.max(1);
let target_chunks = workers * 8;
let chunk_len = defs.len().div_ceil(target_chunks).max(1);
let work: Vec<(&[DefSite], Analysis)> = defs
.chunks(chunk_len)
.map(|chunk| (chunk, host.analysis()))
.collect();
let vfs_ref = &vfs;
let root_ref = root_str.as_str();
let rows: Vec<ScipRow> = gatling_map_owned(work, move |(chunk, analysis)| {
chunk
.iter()
.flat_map(|def| def_and_refs(&analysis, vfs_ref, root_ref, def))
.collect::<Vec<ScipRow>>()
})
.into_iter()
.flatten()
.collect();
Ok(ScipScan {
snapshot_id,
ts,
repo: repo.to_string(),
git_sha: git_sha.to_string(),
rows,
})
}
pub fn ingest_in_process_single_pass(
root: &Path,
repo: &str,
git_sha: &str,
snapshot_id: Uuid,
ts: DateTime<Utc>,
) -> Result<ScipScan> {
let cargo_config = CargoConfig::default();
let load_config = LoadCargoConfig {
load_out_dirs_from_check: true,
with_proc_macro_server: ProcMacroServerChoice::Sysroot,
prefill_caches: true,
num_worker_threads: std::thread::available_parallelism().map(|n| n.get()).unwrap_or(8),
proc_macro_processes: 0,
};
let t_total = Instant::now();
let (db, vfs, _proc_macro) = load_workspace_at(
root,
&cargo_config,
&load_config,
&|_progress| {},
)
.with_context(|| format!("loading cargo workspace at {} into rust-analyzer", root.display()))?;
let host = AnalysisHost::with_database(db);
let analysis = host.analysis();
let root_str = root.to_string_lossy().replace('\\', "/");
let files: Vec<(FileId, String)> = vfs
.iter()
.filter_map(|(file_id, path)| vfs_rel_path(path, &root_str).map(|p| (file_id, p)))
.filter(|(_, p)| p.ends_with(".rs"))
.collect();
let mut def_rows: Vec<ScipRow> = Vec::new();
for (file_id, file) in &files {
let struct_cfg = ra_ap_ide::FileStructureConfig { exclude_locals: true };
let Ok(nodes) = analysis.file_structure(&struct_cfg, *file_id) else { continue };
let Ok(line_index) = analysis.file_line_index(*file_id) else { continue };
for node in nodes {
let off = node.navigation_range.start();
let lc = line_index.line_col(off);
let enc_lc_start = line_index.line_col(node.node_range.start());
let enc_lc_end = line_index.line_col(node.node_range.end());
let line = lc.line + 1;
let col = lc.col + 1;
def_rows.push(ScipRow {
symbol: format!("{}#L{}:C{}", file, line, col),
role: "definition".to_string(),
is_definition: true,
display_name: node.label,
kind: format!("{:?}", node.kind),
file: file.clone(),
start_line: line,
start_col: col,
enc_start_line: enc_lc_start.line + 1,
enc_end_line: enc_lc_end.line + 1,
});
}
}
let setup_secs = t_total.elapsed().as_secs_f64(); let n_defs = def_rows.len();
let t_refs = Instant::now();
let workers = std::thread::available_parallelism()
.map(|w| w.get())
.unwrap_or(1)
.max(1);
let target_chunks = workers * 8;
let chunk_len = files.len().div_ceil(target_chunks).max(1);
let work: Vec<(&[(FileId, String)], Analysis)> = files
.chunks(chunk_len)
.map(|chunk| (chunk, host.analysis()))
.collect();
let vfs_ref = &vfs;
let root_ref = root_str.as_str();
let ref_rows: Vec<ScipRow> = gatling_map_owned(work, move |(chunk, analysis)| {
chunk
.iter()
.flat_map(|(file_id, file)| refs_in_file(&analysis, vfs_ref, root_ref, *file_id, file))
.collect::<Vec<ScipRow>>()
})
.into_iter()
.flatten()
.collect();
let refs_secs = t_refs.elapsed().as_secs_f64();
let n_refs = ref_rows.len();
let t_merge = Instant::now();
let mut rows = def_rows;
rows.extend(ref_rows);
let merge_secs = t_merge.elapsed().as_secs_f64();
eprintln!(
"scip single-pass: setup(load+defs) {setup_secs:.1}s ({n_defs} defs) | \
refs {refs_secs:.1}s ({n_refs} refs) | merge {merge_secs:.3}s | \
{} rows | total {:.1}s",
rows.len(),
t_total.elapsed().as_secs_f64(),
);
Ok(ScipScan {
snapshot_id,
ts,
repo: repo.to_string(),
git_sha: git_sha.to_string(),
rows,
})
}
fn refs_in_file(
analysis: &Analysis,
vfs: &Vfs,
root_str: &str,
file_id: FileId,
occ_file: &str,
) -> Vec<ScipRow> {
use ra_ap_syntax::{ast, AstNode};
let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
let mut out = Vec::new();
let Ok(source) = analysis.parse(file_id) else { return out };
let Ok(occ_line_index) = analysis.file_line_index(file_id) else { return out };
let goto_cfg = ra_ap_ide::GotoDefinitionConfig {
ra_fixture: ra_ap_ide::RaFixtureConfig::default(),
};
for name_ref in source.syntax().descendants().filter_map(ast::NameRef::cast) {
let Some(tok) = name_ref.ident_token() else { continue };
let occ_off = tok.text_range().start();
let pos = ra_ap_ide::FilePosition { file_id, offset: occ_off };
let Ok(Some(nav_info)) = analysis.goto_definition(pos, &goto_cfg) else { continue };
let Some(nav) = nav_info.info.into_iter().next() else { continue };
let Some(def_file) = vfs_rel_path(vfs.file_path(nav.file_id), root_str) else {
continue;
};
let Ok(def_line_index) = analysis.file_line_index(nav.file_id) else { continue };
let def_off = nav.focus_range.unwrap_or(nav.full_range).start();
let def_lc = def_line_index.line_col(def_off);
let def_line = def_lc.line + 1;
let def_col = def_lc.col + 1;
let symbol = format!("{}#L{}:C{}", def_file, def_line, def_col);
let occ_lc = occ_line_index.line_col(occ_off);
out.push(ScipRow {
symbol,
role: "reference".to_string(),
is_definition: false,
display_name: String::new(),
kind: String::new(),
file: occ_file.to_string(),
start_line: occ_lc.line + 1,
start_col: occ_lc.col + 1,
enc_start_line: 0,
enc_end_line: 0,
});
}
out
}));
result.unwrap_or_default()
}
fn def_and_refs(
analysis: &Analysis,
vfs: &Vfs,
root_str: &str,
def: &DefSite,
) -> Vec<ScipRow> {
let symbol = format!("{}#L{}:C{}", def.file, def.line, def.col);
let mut out = Vec::new();
out.push(ScipRow {
symbol: symbol.clone(),
role: "definition".to_string(),
is_definition: true,
display_name: def.label.clone(),
kind: def.kind.clone(),
file: def.file.clone(),
start_line: def.line,
start_col: def.col,
enc_start_line: def.enc_start_line,
enc_end_line: def.enc_end_line,
});
let pos = ra_ap_ide::FilePosition { file_id: def.file_id, offset: def.offset };
let refs_cfg = ra_ap_ide::FindAllRefsConfig {
search_scope: None,
ra_fixture: ra_ap_ide::RaFixtureConfig::default(),
exclude_imports: false,
exclude_tests: false,
};
let Ok(Some(refs)) = analysis.find_all_refs(pos, &refs_cfg) else {
return out;
};
for res in refs {
for (file_id, ranges) in res.references {
let Some(path) = vfs_rel_path(vfs.file_path(file_id), root_str) else { continue };
let Ok(line_index) = analysis.file_line_index(file_id) else { continue };
for (range, category) in ranges {
let lc = line_index.line_col(range.start());
out.push(ScipRow {
symbol: symbol.clone(),
role: ref_category_label(category),
is_definition: false,
display_name: def.label.clone(),
kind: String::new(),
file: path.clone(),
start_line: lc.line + 1,
start_col: lc.col + 1,
enc_start_line: 0,
enc_end_line: 0,
});
}
}
}
out
}
fn ref_category_label(cat: ra_ap_ide::ReferenceCategory) -> String {
use ra_ap_ide::ReferenceCategory as C;
let (w, r, i, t) = (
cat.contains(C::WRITE),
cat.contains(C::READ),
cat.contains(C::IMPORT),
cat.contains(C::TEST),
);
let single: Option<&'static str> = match (w, r, i, t) {
(false, false, false, false) => Some("reference"),
(true, false, false, false) => Some("write"),
(false, true, false, false) => Some("read"),
(false, false, true, false) => Some("import"),
(false, false, false, true) => Some("test"),
_ => None,
};
if let Some(s) = single {
return s.to_string();
}
let mut labels: Vec<&str> = Vec::new();
if w {
labels.push("write");
}
if r {
labels.push("read");
}
if i {
labels.push("import");
}
if t {
labels.push("test");
}
labels.join("+")
}
fn vfs_rel_path(path: &VfsPath, root_str: &str) -> Option<String> {
let p = path.as_path()?;
let s = p.to_string();
let norm: std::borrow::Cow<'_, str> = if s.contains('\\') {
std::borrow::Cow::Owned(s.replace('\\', "/"))
} else {
std::borrow::Cow::Borrowed(s.as_str())
};
let rel = norm.strip_prefix(root_str)?;
Some(rel.trim_start_matches('/').to_string())
}