1use crate::CliError;
36use code_extract::{
37 extract, resolve_call, resolve_import, resolve_mention, FileFacts, SymbolIndex, MAX_FILE_BYTES,
38};
39use core_api::repograph::rules::{about_rule, concept_sources_rule, ABOUT_LABELS};
40use core_api::{default_max_edges, BatchOp, Predicate, RuleDef, Value};
41use std::collections::{BTreeMap, BTreeSet};
42use std::path::Path;
43
44pub type Db = <core_api::WriteGuard<'static> as std::ops::Deref>::Target;
48
49pub const MAX_SYMBOLS_PER_FILE: usize = 2_000;
53
54const BATCH_FILES: usize = 500;
56
57pub const DEFINES_RULE: &str = "auto_fk_symbol_file_id";
62
63pub const FULLTEXT: [(&str, &str); 8] = [
70 ("Concept", "name"),
71 ("Concept", "summary"),
72 ("File", "body"),
73 ("File", "headings"),
74 ("File", "path"),
75 ("Note", "text"),
76 ("Symbol", "doc"),
77 ("Symbol", "name"),
78];
79
80#[derive(Debug, Default, Clone, PartialEq, Eq)]
83pub struct StructureReport {
84 pub files_scanned: usize,
86 pub symbols: usize,
88 pub imports: usize,
90 pub mentions: usize,
92 pub calls: usize,
94 pub skipped_large: usize,
97 pub symbols_capped: usize,
99}
100
101#[must_use]
112pub fn rules() -> Vec<RuleDef> {
113 let mut out = vec![
114 key_rule(DEFINES_RULE, "Symbol", "File", "file_id", "DEFINES"),
115 key_rule("imports", "File", "File", "imports", "IMPORTS"),
116 key_rule("calls", "Symbol", "Symbol", "calls_to", "CALLS"),
117 key_rule("mentions", "File", "File", "mentions", "MENTIONS"),
118 concept_sources_rule(),
119 ];
120 for label in ABOUT_LABELS {
121 out.push(about_rule(label));
122 }
123 out
124}
125
126fn key_rule(name: &str, src: &str, dst: &str, field: &str, edge: &str) -> RuleDef {
129 let predicate = Predicate::KeyMatch {
130 field: field.into(),
131 };
132 let max_edges = Some(default_max_edges(&predicate));
133 RuleDef {
134 name: name.into(),
135 src_label: src.into(),
136 dst_label: dst.into(),
137 predicate,
138 edge_type: edge.into(),
139 weight_prop: None,
140 max_edges,
141 approximate: false,
142 via_label: None,
143 via_edge: None,
144 via_dir: None,
145 }
146}
147
148pub fn ensure_rules_and_fulltext(w: &mut Db) -> Result<Vec<String>, CliError> {
156 let existing: BTreeSet<String> = w.rules().into_iter().map(|r| r.name).collect();
157 let mut created = Vec::new();
158 for def in rules() {
159 if existing.contains(&def.name) {
160 continue;
161 }
162 if def.src_label == "Note" && !label_present(w, &def.dst_label)? {
165 continue;
166 }
167 let name = def.name.clone();
168 w.create_rule(def)?;
169 created.push(name);
170 }
171 for (label, field) in FULLTEXT {
172 if !w
173 .fulltext_pairs()
174 .contains(&(label.to_string(), field.to_string()))
175 {
176 w.enable_fulltext(label, field)?;
177 }
178 }
179 Ok(created)
180}
181
182fn label_present(w: &Db, label: &str) -> Result<bool, CliError> {
185 let rs = w.query(
186 &format!("MATCH (n:{label}) RETURN n.id AS id LIMIT 1"),
187 &BTreeMap::new(),
188 )?;
189 Ok(!rs.is_empty())
190}
191
192const FILE_KEYS_QUERY: &str = "MATCH (f:File) WHERE startsWith(f.id, $prefix) RETURN f.id AS id";
196
197const SYMBOL_QUERY: &str =
199 "MATCH (s:Symbol) RETURN s.id AS id, s.name AS name, s.file_id AS file_id";
200
201const LINK_LISTS_QUERY: &str =
203 "MATCH (f:File) RETURN f.id AS id, f.imports AS imports, f.mentions AS mentions";
204
205pub fn refresh_all(
210 w: &mut Db,
211 repo: &Path,
212 prefix: &str,
213 with_docs: bool,
214) -> Result<StructureReport, CliError> {
215 refresh(w, repo, prefix, None, with_docs)
216}
217
218pub fn refresh_files(
224 w: &mut Db,
225 repo: &Path,
226 prefix: &str,
227 paths: &[String],
228 with_docs: bool,
229) -> Result<StructureReport, CliError> {
230 refresh(w, repo, prefix, Some(paths), with_docs)
231}
232
233pub fn importers_of(w: &Db, keys: &BTreeSet<String>) -> Result<Vec<String>, CliError> {
242 if keys.is_empty() {
243 return Ok(Vec::new());
244 }
245 let rs = w.query(LINK_LISTS_QUERY, &BTreeMap::new())?;
246 let mut out = BTreeSet::new();
247 for i in 0..rs.len() {
248 let Some(Value::Str(id)) = rs.get(i, "id") else {
249 continue;
250 };
251 let names = |field: &str| {
252 matches!(rs.get(i, field), Some(Value::List(l))
253 if l.iter().any(|v| matches!(v, Value::Str(s) if keys.contains(s))))
254 };
255 if names("imports") || names("mentions") {
256 out.insert(id.clone());
257 }
258 }
259 Ok(out.into_iter().collect())
260}
261
262#[derive(Default)]
270struct Tree {
271 files: BTreeSet<String>,
272 by_dir: BTreeMap<String, Vec<String>>,
273 by_base: BTreeMap<String, Vec<String>>,
274}
275
276impl Tree {
277 fn build(keys: impl IntoIterator<Item = String>) -> Tree {
278 let mut tree = Tree::default();
279 for key in keys {
280 let (dir, base) = match key.rsplit_once('/') {
281 Some((d, b)) => (d.to_string(), b.to_string()),
282 None => (String::new(), key.clone()),
283 };
284 tree.by_base.entry(base).or_default().push(key.clone());
285 tree.by_dir.entry(dir).or_default().push(key.clone());
286 tree.files.insert(key);
287 }
288 tree
289 }
290
291 fn known(&self, path: &str) -> bool {
292 self.files.contains(path)
293 }
294
295 fn files_in(&self, dir: &str) -> Vec<String> {
296 self.by_dir.get(dir).cloned().unwrap_or_default()
297 }
298
299 fn by_basename(&self, name: &str) -> Vec<String> {
300 self.by_base.get(name).cloned().unwrap_or_default()
301 }
302}
303
304struct SymbolWrite {
306 key: String,
307 name: String,
308 kind: &'static str,
309 line_start: u32,
310 line_end: u32,
311 signature: String,
312 doc: String,
313 calls: Vec<String>,
315 call_lines: Vec<String>,
317}
318
319struct FileWrite {
321 path: String,
322 hash: String,
323 lines: u32,
324 lang: &'static str,
325 imports: Vec<String>,
326 import_lines: Vec<String>,
327 mentions: Vec<String>,
328 headings: Vec<String>,
329 body: Option<String>,
330 symbols: Vec<SymbolWrite>,
331}
332
333fn list(items: &[String]) -> Value {
334 Value::List(items.iter().map(|s| Value::Str(s.clone())).collect())
335}
336
337fn some_list(items: &[String]) -> Option<Value> {
340 (!items.is_empty()).then(|| list(items))
341}
342
343impl FileWrite {
344 fn props(&self) -> Vec<(&'static str, Option<Value>)> {
347 vec![
348 ("hash", Some(Value::Str(self.hash.clone()))),
349 ("lines", Some(Value::Int(i64::from(self.lines)))),
350 ("lang", Some(Value::Str(self.lang.to_string()))),
351 ("symbols_n", Some(Value::Int(self.symbols.len() as i64))),
352 ("imports", some_list(&self.imports)),
353 ("import_lines", some_list(&self.import_lines)),
354 ("mentions", some_list(&self.mentions)),
355 ("headings", some_list(&self.headings)),
356 ("body", self.body.as_ref().map(|b| Value::Str(b.clone()))),
357 ]
358 }
359}
360
361impl SymbolWrite {
362 fn props(&self, file: &str) -> Vec<(&'static str, Option<Value>)> {
363 vec![
364 ("id", Some(Value::Str(self.key.clone()))),
365 ("name", Some(Value::Str(self.name.clone()))),
366 ("kind", Some(Value::Str(self.kind.to_string()))),
367 ("path", Some(Value::Str(file.to_string()))),
368 ("file_id", Some(Value::Str(file.to_string()))),
369 ("line_start", Some(Value::Int(i64::from(self.line_start)))),
370 ("line_end", Some(Value::Int(i64::from(self.line_end)))),
371 ("signature", Some(Value::Str(self.signature.clone()))),
372 ("doc", Some(Value::Str(self.doc.clone()))),
373 ("calls_to", some_list(&self.calls)),
374 ("call_lines", some_list(&self.call_lines)),
375 ]
376 }
377}
378
379fn symbol_keys(path: &str, facts: &FileFacts) -> (Vec<(String, usize)>, bool) {
383 let mut seen = BTreeSet::new();
384 let mut out = Vec::new();
385 let mut capped = false;
386 for (at, sym) in facts.symbols.iter().enumerate() {
387 let key = format!("{path}#{}", sym.name);
388 if !seen.insert(key.clone()) {
389 continue;
390 }
391 if out.len() == MAX_SYMBOLS_PER_FILE {
392 capped = true;
393 break;
394 }
395 out.push((key, at));
396 }
397 (out, capped)
398}
399
400fn refresh(
401 w: &mut Db,
402 repo: &Path,
403 prefix: &str,
404 only: Option<&[String]>,
405 with_docs: bool,
406) -> Result<StructureReport, CliError> {
407 let params = BTreeMap::from([("prefix".to_string(), Value::Str(prefix.to_string()))]);
409 let rs = w.query(FILE_KEYS_QUERY, ¶ms)?;
410 let mut candidates = Vec::new();
411 for i in 0..rs.len() {
412 if let Some(Value::Str(id)) = rs.get(i, "id") {
413 if repo.join(id).is_file() {
414 candidates.push(id.clone());
415 }
416 }
417 }
418 let tree = Tree::build(candidates.iter().cloned());
419
420 let targets: Vec<String> = match only {
422 None => candidates,
423 Some(paths) => {
424 let wanted: BTreeSet<&String> = paths.iter().collect();
425 candidates
426 .into_iter()
427 .filter(|p| wanted.contains(p))
428 .collect()
429 }
430 };
431
432 let mut facts: BTreeMap<String, FileFacts> = BTreeMap::new();
435 let mut hash_only: BTreeSet<String> = BTreeSet::new();
436 for path in &targets {
437 let Ok(bytes) = std::fs::read(repo.join(path)) else {
438 continue; };
440 if bytes.len() > MAX_FILE_BYTES || is_binary(&bytes) {
441 hash_only.insert(path.clone());
442 }
443 facts.insert(path.clone(), extract(path, &bytes));
444 }
445
446 let stored = w.query(SYMBOL_QUERY, &BTreeMap::new())?;
450 let mut by_file: BTreeMap<String, BTreeSet<String>> = BTreeMap::new();
451 let mut orphans: Vec<String> = Vec::new();
452 let mut index = SymbolIndex::new();
453 for i in 0..stored.len() {
454 let (Some(Value::Str(id)), Some(Value::Str(file))) =
455 (stored.get(i, "id"), stored.get(i, "file_id"))
456 else {
457 continue;
458 };
459 if !w.has_node(file.as_str()) {
460 orphans.push(id.clone());
461 continue;
462 }
463 by_file.entry(file.clone()).or_default().insert(id.clone());
464 if facts.contains_key(file) || !tree.known(file) {
465 continue; }
467 if let Some(Value::Str(name)) = stored.get(i, "name") {
468 index.insert(name, id);
469 }
470 }
471
472 let mut report = StructureReport::default();
475 let mut keyed: BTreeMap<&String, Vec<(String, usize)>> = BTreeMap::new();
476 for (path, f) in &facts {
477 let (keys, capped) = symbol_keys(path, f);
478 for (key, at) in &keys {
479 index.insert(&f.symbols[*at].name, key);
480 }
481 report.symbols_capped += usize::from(capped);
482 keyed.insert(path, keys);
483 }
484
485 let mut writes: Vec<FileWrite> = Vec::new();
486 for (path, f) in &facts {
487 let write = resolve_file(path, f, &tree, &index, &keyed[path], with_docs);
488 report.files_scanned += 1;
489 report.symbols += write.symbols.len();
490 report.imports += write.imports.len();
491 report.mentions += write.mentions.len();
492 report.calls += write.symbols.iter().map(|s| s.calls.len()).sum::<usize>();
493 report.skipped_large += usize::from(hash_only.contains(path));
494 writes.push(write);
495 }
496
497 if !orphans.is_empty() {
500 let ops = orphans
501 .iter()
502 .map(|key| BatchOp::DeleteNode { key: key.clone() })
503 .collect();
504 commit(w, ops)?;
505 }
506 for chunk in writes.chunks(BATCH_FILES) {
507 let mut ops = Vec::new();
508 for file in chunk {
509 plan_file(w, file, by_file.get(&file.path), &mut ops);
510 }
511 commit(w, ops)?;
512 }
513 Ok(report)
514}
515
516fn commit(w: &mut Db, ops: Vec<BatchOp>) -> Result<(), CliError> {
518 if ops.is_empty() {
519 return Ok(());
520 }
521 let (results, sync) = w.commit_group(vec![ops]);
522 for r in results {
523 r?;
524 }
525 match sync {
526 Some(e) => Err(CliError(e.to_string())),
527 None => Ok(()),
528 }
529}
530
531fn is_binary(bytes: &[u8]) -> bool {
534 bytes[..bytes.len().min(8 * 1024)].contains(&0)
535}
536
537fn resolve_file(
539 path: &str,
540 f: &FileFacts,
541 tree: &Tree,
542 index: &SymbolIndex,
543 keys: &[(String, usize)],
544 with_docs: bool,
545) -> FileWrite {
546 let known = |p: &str| tree.known(p);
547 let files_in = |d: &str| tree.files_in(d);
548 let by_base = |n: &str| tree.by_basename(n);
549
550 let mut imports = BTreeSet::new();
551 let mut import_lines = BTreeSet::new();
552 for imp in &f.imports {
553 for target in resolve_import(f.lang, path, &imp.raw, &known, &files_in) {
554 if target == path {
555 continue;
556 }
557 import_lines.insert(format!("{target}\t{}", imp.line));
558 imports.insert(target);
559 }
560 }
561
562 let mut mentions = BTreeSet::new();
563 if with_docs {
564 for token in &f.mentions {
565 if let Some(target) = resolve_mention(path, token, &known, &by_base) {
566 if target != path {
567 mentions.insert(target);
568 }
569 }
570 }
571 }
572
573 let mut symbols = Vec::with_capacity(keys.len());
574 for (key, at) in keys {
575 let fact = &f.symbols[*at];
576 let mut calls = BTreeSet::new();
577 let mut call_lines = BTreeSet::new();
578 for (callee, line) in &fact.calls {
579 let Some(target) = resolve_call(path, callee, index) else {
580 continue;
581 };
582 if &target == key {
583 continue; }
585 call_lines.insert(format!("{target}\t{line}"));
586 calls.insert(target);
587 }
588 symbols.push(SymbolWrite {
589 key: key.clone(),
590 name: fact.name.clone(),
591 kind: fact.kind,
592 line_start: fact.line_start,
593 line_end: fact.line_end,
594 signature: fact.signature.clone(),
595 doc: fact.doc.clone(),
596 calls: calls.into_iter().collect(),
597 call_lines: call_lines.into_iter().collect(),
598 });
599 }
600
601 FileWrite {
602 path: path.to_string(),
603 hash: f.hash.clone(),
604 lines: f.lines,
605 lang: f.lang.as_str(),
606 imports: imports.into_iter().collect(),
607 import_lines: import_lines.into_iter().collect(),
608 mentions: mentions.into_iter().collect(),
609 headings: if with_docs {
610 f.headings.clone()
611 } else {
612 Vec::new()
613 },
614 body: if with_docs { f.body.clone() } else { None },
615 symbols,
616 }
617}
618
619fn plan_file(w: &Db, file: &FileWrite, held: Option<&BTreeSet<String>>, ops: &mut Vec<BatchOp>) {
624 for (field, want) in file.props() {
625 diff_prop(w, &file.path, field, want, ops);
626 }
627
628 let wanted: BTreeSet<&String> = file.symbols.iter().map(|s| &s.key).collect();
629 for key in held.into_iter().flatten() {
630 if !wanted.contains(key) {
631 ops.push(BatchOp::DeleteNode { key: key.clone() });
632 }
633 }
634 for sym in &file.symbols {
635 let props = sym.props(&file.path);
636 match w.node_ref(&sym.key).map(|n| n.label().to_string()) {
637 Some(label) if label != "Symbol" => continue,
643 Some(_) => {
644 for (field, want) in props {
645 diff_prop(w, &sym.key, field, want, ops);
646 }
647 }
648 None => ops.push(BatchOp::InsertNode {
649 label: "Symbol".into(),
650 key: sym.key.clone(),
651 props: props
652 .into_iter()
653 .filter_map(|(f, v)| v.map(|v| (f.to_string(), v)))
654 .collect(),
655 }),
656 }
657 }
658}
659
660fn diff_prop(w: &Db, key: &str, field: &str, want: Option<Value>, ops: &mut Vec<BatchOp>) {
662 let current = w.node_ref(key).and_then(|n| n.prop(field));
663 if current == want {
664 return;
665 }
666 match want {
667 Some(value) => ops.push(BatchOp::SetProp {
668 key: key.to_string(),
669 field: field.to_string(),
670 value,
671 }),
672 None => ops.push(BatchOp::RemoveProp {
673 key: key.to_string(),
674 field: field.to_string(),
675 }),
676 }
677}
678
679#[cfg(test)]
680mod tests {
681 use super::*;
682
683 #[test]
684 fn rules_cover_every_derived_structure_edge() {
685 let names: Vec<String> = rules().into_iter().map(|r| r.name).collect();
686 for want in [
687 DEFINES_RULE,
688 "imports",
689 "calls",
690 "mentions",
691 "concept_sources",
692 "about_author",
693 "about_concept",
694 "about_file",
695 "about_note",
696 "about_symbol",
697 ] {
698 assert!(names.contains(&want.to_string()), "missing rule {want}");
699 }
700 for def in rules() {
701 assert_eq!(
702 def.max_edges,
703 Some(default_max_edges(&def.predicate)),
704 "{} must state its fan-out",
705 def.name
706 );
707 }
708 }
709
710 #[test]
711 fn the_tree_answers_every_lookup_the_resolvers_need() {
712 let tree = Tree::build([
713 "src/lib.rs".to_string(),
714 "src/net/mod.rs".to_string(),
715 "README.md".to_string(),
716 ]);
717 assert!(tree.known("src/lib.rs"));
718 assert!(!tree.known("src/gone.rs"));
719 assert_eq!(tree.files_in("src"), vec!["src/lib.rs".to_string()]);
720 assert_eq!(tree.files_in("nope"), Vec::<String>::new());
721 assert_eq!(
722 tree.by_basename("mod.rs"),
723 vec!["src/net/mod.rs".to_string()]
724 );
725 assert_eq!(tree.files_in(""), vec!["README.md".to_string()]);
726 }
727
728 #[test]
729 fn binary_probe_matches_the_extractors() {
730 assert!(!is_binary(b"pub fn a() {}"));
731 assert!(is_binary(b"pub fn a() {}\0"));
732 assert!(!is_binary(b""));
733 }
734}