use std::path::Path;
use std::time::Instant;
use anyhow::{Context, Result};
use chrono::{DateTime, Utc};
use uuid::Uuid;
use ra_ap_ide::{Analysis, AnalysisHost, FileId};
use ra_ap_load_cargo::{load_workspace_at, LoadCargoConfig, ProcMacroServerChoice};
use ra_ap_project_model::CargoConfig;
use ra_ap_vfs::{Vfs, VfsPath};
use super::scip::{ScipRow, ScipScan};
fn canon_kind(kind: ra_ap_ide::StructureNodeKind) -> &'static str {
use ra_ap_ide::StructureNodeKind;
use ra_ap_ide_db::SymbolKind;
match kind {
StructureNodeKind::SymbolKind(sk) => match sk {
SymbolKind::Function => "Function",
SymbolKind::Method => "Method",
SymbolKind::Struct | SymbolKind::Union => "Struct",
SymbolKind::Trait => "Trait",
SymbolKind::Enum => "Enum",
SymbolKind::Variant => "EnumMember",
SymbolKind::Field => "Field",
SymbolKind::Module | SymbolKind::CrateRoot => "Module",
SymbolKind::Macro
| SymbolKind::ProcMacro
| SymbolKind::Derive
| SymbolKind::DeriveHelper
| SymbolKind::Attribute
| SymbolKind::BuiltinAttr => "Macro",
SymbolKind::TypeAlias => "TypeAlias",
SymbolKind::Const | SymbolKind::ConstParam => "Constant",
SymbolKind::Static | SymbolKind::Local => "Variable",
SymbolKind::ValueParam => "Parameter",
SymbolKind::TypeParam => "TypeParameter",
SymbolKind::SelfParam => "SelfParameter",
_ => "Other",
},
StructureNodeKind::ExternBlock | StructureNodeKind::Region => "Other",
}
}
use znippy_zoomies::gatling_forkjoin::gatling_map_owned;
struct DefSite {
file_id: FileId,
offset: ra_ap_ide::TextSize,
label: String,
kind: String,
file: String,
line: u32,
col: u32,
enc_start_line: u32,
enc_end_line: u32,
}
pub fn ingest_in_process(
root: &Path,
repo: &str,
git_sha: &str,
snapshot_id: Uuid,
ts: DateTime<Utc>,
) -> Result<ScipScan> {
let single_pass = std::env::var("NORNIR_SCIP_SINGLEPASS").map(|v| v != "0").unwrap_or(true);
if single_pass {
return ingest_in_process_single_pass(root, repo, git_sha, snapshot_id, ts);
}
let cargo_config = CargoConfig::default();
let load_config = LoadCargoConfig {
load_out_dirs_from_check: true,
with_proc_macro_server: ProcMacroServerChoice::Sysroot,
prefill_caches: true,
num_worker_threads: std::thread::available_parallelism().map(|n| n.get()).unwrap_or(8),
proc_macro_processes: 0,
};
let (db, vfs, _proc_macro) = load_workspace_at(
root,
&cargo_config,
&load_config,
&|_progress| {},
)
.with_context(|| format!("loading cargo workspace at {} into rust-analyzer", root.display()))?;
let host = AnalysisHost::with_database(db);
let analysis = host.analysis();
let root_str = root.to_string_lossy().replace('\\', "/");
let files: Vec<(FileId, String)> = vfs
.iter()
.filter_map(|(file_id, path)| vfs_rel_path(path, &root_str).map(|p| (file_id, p)))
.filter(|(_, p)| p.ends_with(".rs"))
.collect();
let mut defs: Vec<DefSite> = Vec::new();
for (file_id, file) in &files {
let struct_cfg = ra_ap_ide::FileStructureConfig { exclude_locals: true };
let Ok(nodes) = analysis.file_structure(&struct_cfg, *file_id) else { continue };
let Ok(line_index) = analysis.file_line_index(*file_id) else { continue };
for node in nodes {
let off = node.navigation_range.start();
let lc = line_index.line_col(off);
let enc_lc_start = line_index.line_col(node.node_range.start());
let enc_lc_end = line_index.line_col(node.node_range.end());
defs.push(DefSite {
file_id: *file_id,
offset: off,
label: node.label,
kind: canon_kind(node.kind).to_string(),
file: file.clone(),
line: lc.line + 1,
col: lc.col + 1,
enc_start_line: enc_lc_start.line + 1,
enc_end_line: enc_lc_end.line + 1,
});
}
}
let workers = std::thread::available_parallelism()
.map(|w| w.get())
.unwrap_or(1)
.max(1);
let target_chunks = workers * 8;
let chunk_len = defs.len().div_ceil(target_chunks).max(1);
let work: Vec<(&[DefSite], Analysis)> = defs
.chunks(chunk_len)
.map(|chunk| (chunk, host.analysis()))
.collect();
let vfs_ref = &vfs;
let root_ref = root_str.as_str();
let rows: Vec<ScipRow> = gatling_map_owned(work, move |(chunk, analysis)| {
chunk
.iter()
.flat_map(|def| def_and_refs(&analysis, vfs_ref, root_ref, def))
.collect::<Vec<ScipRow>>()
})
.into_iter()
.flatten()
.collect();
Ok(ScipScan {
snapshot_id,
ts,
repo: repo.to_string(),
git_sha: git_sha.to_string(),
rows,
})
}
pub fn ingest_in_process_single_pass(
root: &Path,
repo: &str,
git_sha: &str,
snapshot_id: Uuid,
ts: DateTime<Utc>,
) -> Result<ScipScan> {
let cargo_config = CargoConfig::default();
let load_config = LoadCargoConfig {
load_out_dirs_from_check: true,
with_proc_macro_server: ProcMacroServerChoice::Sysroot,
prefill_caches: true,
num_worker_threads: std::thread::available_parallelism().map(|n| n.get()).unwrap_or(8),
proc_macro_processes: 0,
};
let t_total = Instant::now();
let (db, vfs, _proc_macro) = load_workspace_at(
root,
&cargo_config,
&load_config,
&|_progress| {},
)
.with_context(|| format!("loading cargo workspace at {} into rust-analyzer", root.display()))?;
let host = AnalysisHost::with_database(db);
let analysis = host.analysis();
let root_str = root.to_string_lossy().replace('\\', "/");
let files: Vec<(FileId, String)> = vfs
.iter()
.filter_map(|(file_id, path)| vfs_rel_path(path, &root_str).map(|p| (file_id, p)))
.filter(|(_, p)| p.ends_with(".rs"))
.collect();
let mut def_rows: Vec<ScipRow> = Vec::new();
for (file_id, file) in &files {
let struct_cfg = ra_ap_ide::FileStructureConfig { exclude_locals: true };
let Ok(nodes) = analysis.file_structure(&struct_cfg, *file_id) else { continue };
let Ok(line_index) = analysis.file_line_index(*file_id) else { continue };
for node in nodes {
let off = node.navigation_range.start();
let lc = line_index.line_col(off);
let enc_lc_start = line_index.line_col(node.node_range.start());
let enc_lc_end = line_index.line_col(node.node_range.end());
let line = lc.line + 1;
let col = lc.col + 1;
def_rows.push(ScipRow {
symbol: format!("{}#L{}:C{}", file, line, col),
role: "definition".to_string(),
is_definition: true,
display_name: node.label,
kind: canon_kind(node.kind).to_string(),
file: file.clone(),
start_line: line,
start_col: col,
enc_start_line: enc_lc_start.line + 1,
enc_end_line: enc_lc_end.line + 1,
});
}
}
let setup_secs = t_total.elapsed().as_secs_f64(); let n_defs = def_rows.len();
let t_refs = Instant::now();
let workers = std::thread::available_parallelism()
.map(|w| w.get())
.unwrap_or(1)
.max(1);
let target_chunks = workers * 8;
let chunk_len = files.len().div_ceil(target_chunks).max(1);
let work: Vec<(&[(FileId, String)], Analysis)> = files
.chunks(chunk_len)
.map(|chunk| (chunk, host.analysis()))
.collect();
let vfs_ref = &vfs;
let root_ref = root_str.as_str();
let ref_rows: Vec<ScipRow> = gatling_map_owned(work, move |(chunk, analysis)| {
chunk
.iter()
.flat_map(|(file_id, file)| refs_in_file(&analysis, vfs_ref, root_ref, *file_id, file))
.collect::<Vec<ScipRow>>()
})
.into_iter()
.flatten()
.collect();
let refs_secs = t_refs.elapsed().as_secs_f64();
let n_refs = ref_rows.len();
let t_merge = Instant::now();
let mut rows = def_rows;
rows.extend(ref_rows);
let merge_secs = t_merge.elapsed().as_secs_f64();
eprintln!(
"scip single-pass: setup(load+defs) {setup_secs:.1}s ({n_defs} defs) | \
refs {refs_secs:.1}s ({n_refs} refs) | merge {merge_secs:.3}s | \
{} rows | total {:.1}s",
rows.len(),
t_total.elapsed().as_secs_f64(),
);
Ok(ScipScan {
snapshot_id,
ts,
repo: repo.to_string(),
git_sha: git_sha.to_string(),
rows,
})
}
fn refs_in_file(
analysis: &Analysis,
vfs: &Vfs,
root_str: &str,
file_id: FileId,
occ_file: &str,
) -> Vec<ScipRow> {
use ra_ap_syntax::{ast, AstNode};
let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
let mut out = Vec::new();
let Ok(source) = analysis.parse(file_id) else { return out };
let Ok(occ_line_index) = analysis.file_line_index(file_id) else { return out };
let goto_cfg = ra_ap_ide::GotoDefinitionConfig {
ra_fixture: ra_ap_ide::RaFixtureConfig::default(),
};
for name_ref in source.syntax().descendants().filter_map(ast::NameRef::cast) {
let Some(tok) = name_ref.ident_token() else { continue };
let occ_off = tok.text_range().start();
let pos = ra_ap_ide::FilePosition { file_id, offset: occ_off };
let Ok(Some(nav_info)) = analysis.goto_definition(pos, &goto_cfg) else { continue };
let Some(nav) = nav_info.info.into_iter().next() else { continue };
let Some(def_file) = vfs_rel_path(vfs.file_path(nav.file_id), root_str) else {
continue;
};
let Ok(def_line_index) = analysis.file_line_index(nav.file_id) else { continue };
let def_off = nav.focus_range.unwrap_or(nav.full_range).start();
let def_lc = def_line_index.line_col(def_off);
let def_line = def_lc.line + 1;
let def_col = def_lc.col + 1;
let symbol = format!("{}#L{}:C{}", def_file, def_line, def_col);
let occ_lc = occ_line_index.line_col(occ_off);
out.push(ScipRow {
symbol,
role: "reference".to_string(),
is_definition: false,
display_name: String::new(),
kind: String::new(),
file: occ_file.to_string(),
start_line: occ_lc.line + 1,
start_col: occ_lc.col + 1,
enc_start_line: 0,
enc_end_line: 0,
});
}
out
}));
result.unwrap_or_default()
}
fn def_and_refs(
analysis: &Analysis,
vfs: &Vfs,
root_str: &str,
def: &DefSite,
) -> Vec<ScipRow> {
let symbol = format!("{}#L{}:C{}", def.file, def.line, def.col);
let mut out = Vec::new();
out.push(ScipRow {
symbol: symbol.clone(),
role: "definition".to_string(),
is_definition: true,
display_name: def.label.clone(),
kind: def.kind.clone(),
file: def.file.clone(),
start_line: def.line,
start_col: def.col,
enc_start_line: def.enc_start_line,
enc_end_line: def.enc_end_line,
});
let pos = ra_ap_ide::FilePosition { file_id: def.file_id, offset: def.offset };
let refs_cfg = ra_ap_ide::FindAllRefsConfig {
search_scope: None,
ra_fixture: ra_ap_ide::RaFixtureConfig::default(),
exclude_imports: false,
exclude_tests: false,
};
let Ok(Some(refs)) = analysis.find_all_refs(pos, &refs_cfg) else {
return out;
};
for res in refs {
for (file_id, ranges) in res.references {
let Some(path) = vfs_rel_path(vfs.file_path(file_id), root_str) else { continue };
let Ok(line_index) = analysis.file_line_index(file_id) else { continue };
for (range, category) in ranges {
let lc = line_index.line_col(range.start());
out.push(ScipRow {
symbol: symbol.clone(),
role: ref_category_label(category),
is_definition: false,
display_name: def.label.clone(),
kind: String::new(),
file: path.clone(),
start_line: lc.line + 1,
start_col: lc.col + 1,
enc_start_line: 0,
enc_end_line: 0,
});
}
}
}
out
}
fn ref_category_label(cat: ra_ap_ide::ReferenceCategory) -> String {
use ra_ap_ide::ReferenceCategory as C;
let (w, r, i, t) = (
cat.contains(C::WRITE),
cat.contains(C::READ),
cat.contains(C::IMPORT),
cat.contains(C::TEST),
);
let single: Option<&'static str> = match (w, r, i, t) {
(false, false, false, false) => Some("reference"),
(true, false, false, false) => Some("write"),
(false, true, false, false) => Some("read"),
(false, false, true, false) => Some("import"),
(false, false, false, true) => Some("test"),
_ => None,
};
if let Some(s) = single {
return s.to_string();
}
let mut labels: Vec<&str> = Vec::new();
if w {
labels.push("write");
}
if r {
labels.push("read");
}
if i {
labels.push("import");
}
if t {
labels.push("test");
}
labels.join("+")
}
fn vfs_rel_path(path: &VfsPath, root_str: &str) -> Option<String> {
let p = path.as_path()?;
let s = p.to_string();
let norm: std::borrow::Cow<'_, str> = if s.contains('\\') {
std::borrow::Cow::Owned(s.replace('\\', "/"))
} else {
std::borrow::Cow::Borrowed(s.as_str())
};
let rel = norm.strip_prefix(root_str)?;
Some(rel.trim_start_matches('/').to_string())
}