1use sinter_core::{
2 Edge, Evidence, FieldBinding, Node, NodeId, Reference, Relation, Span, SymbolKind,
3};
4use streaming_iterator::StreamingIterator;
5use tree_sitter::{Node as TsNode, Parser, Query, QueryCursor};
6
7const DOC_CAP_BYTES: usize = 4096;
10
11use sinter_core::FileFacts;
12
13use crate::language::LanguageSpec;
14
15#[derive(Debug, thiserror::Error)]
16pub enum ExtractError {
17 #[error("bad grammar or query for {language}: {message}")]
18 Query {
19 language: &'static str,
20 message: String,
21 },
22 #[error("parser returned no tree for {0}")]
23 Parse(String),
24}
25
26pub struct Extractor {
29 spec: &'static LanguageSpec,
30 parser: Parser,
31 query: Query,
32 inline: Option<(Parser, Query)>,
36}
37
38struct RawEntry {
40 start: usize,
41 end: usize,
42 name: String,
43 kind: Option<SymbolKind>,
45 qualifier: Option<String>,
47 signature: String,
48 doc: Option<String>,
49}
50
51struct RawRef {
53 start: usize,
54 end: usize,
55 name: String,
56 path: Option<String>,
57 alias: Option<String>,
58 relation: Relation,
59}
60
61struct RawLocal {
63 start: usize,
64 end: usize,
65 name: String,
66 type_name: Option<String>,
67}
68
69#[derive(Default)]
71struct Collected {
72 refs: Vec<RawRef>,
73 locals: Vec<RawLocal>,
74 fields: Vec<(usize, usize, String, String)>,
76 embeds: Vec<(usize, usize, String)>,
78 trait_impls: Vec<(usize, usize, String)>,
80 alias_spans: Vec<(usize, usize)>,
83 def_name_spans: Vec<(usize, usize)>,
86 docs: Vec<(usize, usize, String)>,
90}
91
92impl Extractor {
93 pub fn new(spec: &'static LanguageSpec) -> Result<Self, ExtractError> {
94 let language = (spec.grammar)();
95 let mut parser = Parser::new();
96 parser
97 .set_language(&language)
98 .map_err(|e| ExtractError::Query {
99 language: spec.name,
100 message: e.to_string(),
101 })?;
102 let query = Query::new(&language, spec.query_source).map_err(|e| ExtractError::Query {
103 language: spec.name,
104 message: e.to_string(),
105 })?;
106 let inline = spec
107 .inline
108 .map(|i| {
109 let language = (i.grammar)();
110 let mut parser = Parser::new();
111 parser
112 .set_language(&language)
113 .map_err(|e| (spec.name, e.to_string()))?;
114 let query = Query::new(&language, i.query_source)
115 .map_err(|e| (spec.name, e.to_string()))?;
116 Ok((parser, query))
117 })
118 .transpose()
119 .map_err(
120 |(language, message): (&'static str, String)| ExtractError::Query {
121 language,
122 message,
123 },
124 )?;
125 Ok(Self {
126 spec,
127 parser,
128 query,
129 inline,
130 })
131 }
132
133 pub fn extract(&mut self, file: &str, source: &str) -> Result<FileFacts, ExtractError> {
135 let tree = self
136 .parser
137 .parse(source, None)
138 .ok_or_else(|| ExtractError::Parse(file.to_string()))?;
139 let root = tree.root_node();
140
141 let mut entries = Vec::new();
142 let mut collected = Collected::default();
143 collect(
144 &self.query,
145 self.spec,
146 root,
147 source,
148 &mut entries,
149 &mut collected,
150 );
151 if let (Some((parser, query)), Some(ispec)) = (&mut self.inline, self.spec.inline) {
155 let ranges = container_ranges(root, ispec.container_kinds);
156 if !ranges.is_empty() {
157 parser.set_included_ranges(&ranges).map_err(|e| {
163 ExtractError::Parse(format!("{} (inline ranges: {e})", self.spec.name))
164 })?;
165 let inline_tree = parser
166 .parse(source, None)
167 .ok_or_else(|| ExtractError::Parse(format!("{} (inline)", self.spec.name)))?;
168 collect(
169 query,
170 self.spec,
171 inline_tree.root_node(),
172 source,
173 &mut entries,
174 &mut collected,
175 );
176 }
177 }
178 entries.sort_by_key(|e| (e.start, usize::MAX - e.end));
179 let mut doc_set = vec![false; entries.len()];
184 for (d_start, d_end, text) in &collected.docs {
185 let owner = entries
186 .iter_mut()
187 .enumerate()
188 .filter(|(_, e)| e.kind.is_some() && e.start <= *d_start && *d_end <= e.end)
189 .min_by_key(|(_, e)| e.end - e.start);
190 if let Some((i, entry)) = owner {
191 let cleaned: Vec<&str> = text.lines().map(str::trim).collect();
192 let trimmed = cleaned.join("\n");
193 let trimmed = trimmed.trim_matches('\n');
194 if trimmed.is_empty() {
195 continue;
196 }
197 entry.doc = match entry.doc.take().filter(|_| doc_set[i]) {
198 Some(prev) if prev.len() >= DOC_CAP_BYTES => Some(prev),
201 Some(prev) => Some(format!("{prev}\n\n{trimmed}")),
202 None => Some(trimmed.to_string()),
203 };
204 doc_set[i] = true;
205 }
206 }
207 entries.dedup_by(|b, a| {
211 let same = a.start == b.start && a.end == b.end && a.name == b.name;
212 if same && a.kind == Some(SymbolKind::Variable) && b.kind.is_some() {
213 a.kind = b.kind;
214 }
215 same
216 });
217
218 let file_id = NodeId::new(file);
219 let mut nodes = vec![Node {
220 id: file_id.clone(),
221 kind: SymbolKind::File,
222 name: file.to_string(),
223 file: file.to_string(),
224 span: Span {
225 start: 0,
226 end: source.len().max(1) as u64,
227 },
228 signature: String::new(),
229 doc: None,
230 }];
231 let mut contains = Vec::new();
232
233 let mut stack: Vec<(usize, String, Option<NodeId>)> = Vec::new();
235 let mut def_spans: Vec<(usize, usize, NodeId, SymbolKind)> = Vec::new();
238
239 for entry in &entries {
240 while stack.last().is_some_and(|(end, _, _)| *end <= entry.start) {
241 stack.pop();
242 }
243 let mut path: Vec<&str> = stack.iter().map(|(_, name, _)| name.as_str()).collect();
244 if let Some(q) = &entry.qualifier {
245 path.push(q);
246 }
247 path.push(&entry.name);
248 let qualified = path.join("::");
249 let scope_segment = entry
251 .qualifier
252 .as_ref()
253 .map_or(entry.name.clone(), |q| format!("{q}::{}", entry.name));
254
255 let id = if let Some(kind) = entry.kind {
256 let id = NodeId::new(format!("{file}#{qualified}@{}", entry.start));
257 let parent = stack
258 .iter()
259 .rev()
260 .find_map(|(_, _, id)| id.clone())
261 .unwrap_or_else(|| file_id.clone());
262 nodes.push(Node {
263 id: id.clone(),
264 kind,
265 name: entry.name.clone(),
266 file: file.to_string(),
267 span: Span {
268 start: entry.start as u64,
269 end: entry.end as u64,
270 },
271 signature: entry.signature.clone(),
272 doc: entry.doc.clone(),
273 });
274 contains.push(Edge {
275 src: parent,
276 dst: id.clone(),
277 relation: Relation::Contains,
278 evidence: Evidence::Structural,
279 confidence: Evidence::Structural.confidence(),
280 site: None,
282 });
283 def_spans.push((entry.start, entry.end, id.clone(), kind));
284 Some(id)
285 } else {
286 None
287 };
288 stack.push((entry.end, scope_segment, id));
289 }
290
291 let mut refs = collected.refs;
294 refs.retain(|r| !collected.def_name_spans.contains(&(r.start, r.end)));
295 if self.spec.name == "rust" {
298 refs.retain(|r| {
299 r.path.is_some()
300 || !RUST_PRELUDE.contains(&r.name.as_str())
301 || entries.iter().any(|e| e.name == r.name)
302 });
303 }
304 refs.sort_by_key(|r| (r.start, r.end, r.path.is_none()));
305 refs.dedup_by_key(|r| (r.start, r.end));
306 let references = refs
307 .into_iter()
308 .map(|r| {
309 let enclosing = def_spans
310 .iter()
311 .filter(|(s, e, _, _)| *s <= r.start && r.end <= *e)
312 .min_by_key(|(s, e, _, _)| e - s)
313 .map(|(_, _, id, _)| id.clone());
314 Reference {
315 file: file.to_string(),
316 name: r.name,
317 path: r.path,
318 relation: r.relation,
319 span: Span {
320 start: r.start as u64,
321 end: r.end as u64,
322 },
323 enclosing,
324 alias: r.alias,
325 }
326 })
327 .collect();
328
329 let alias_spans = collected.alias_spans;
333 let embeds = collected
334 .embeds
335 .iter()
336 .filter_map(|(start, end, type_name)| {
337 let owner = def_spans
338 .iter()
339 .filter(|(s, e, _, _)| s <= start && end <= e)
340 .min_by_key(|(s, e, _, _)| e - s)
341 .map(|(_, _, id, _)| id.clone())?;
342 Some(sinter_core::Embed {
343 owner,
344 type_name: type_name.clone(),
345 })
346 })
347 .collect();
348 let mut raw_locals = collected
349 .locals
350 .into_iter()
351 .filter(|l| !alias_spans.contains(&(l.start, l.end)))
352 .collect::<Vec<_>>();
353 raw_locals.sort_by_key(|l| (l.start, l.end, l.name.clone(), l.type_name.is_none()));
356 raw_locals.dedup_by(|b, a| a.start == b.start && a.end == b.end && a.name == b.name);
357 let locals = raw_locals
358 .into_iter()
359 .map(|l| {
360 let scope_end = def_spans
361 .iter()
362 .filter(|(s, e, _, _)| *s <= l.start && l.end <= *e)
363 .min_by_key(|(s, e, _, _)| e - s)
364 .map_or(source.len() as u64, |(_, e, _, _)| *e as u64);
365 sinter_core::LocalBinding {
366 file: file.to_string(),
367 name: l.name,
368 span: Span {
369 start: l.start as u64,
370 end: l.end as u64,
371 },
372 scope_end,
373 type_name: l.type_name,
374 }
375 })
376 .collect();
377
378 let fields = collected
379 .fields
380 .iter()
381 .filter_map(|(start, end, name, type_name)| {
382 let owner = def_spans
383 .iter()
384 .filter(|(s, e, _, kind)| {
385 *s <= *start
386 && *end <= *e
387 && matches!(
388 kind,
389 SymbolKind::Struct
390 | SymbolKind::Class
391 | SymbolKind::Trait
392 | SymbolKind::Interface
393 )
394 })
395 .min_by_key(|(s, e, _, _)| e - s)
396 .map(|(_, _, id, _)| id.clone())?;
397 Some(FieldBinding {
398 owner,
399 name: name.clone(),
400 type_name: type_name.clone(),
401 })
402 })
403 .collect();
404
405 let trait_impls = collected
406 .trait_impls
407 .iter()
408 .map(|(start, end, trait_name)| sinter_core::TraitImpl {
409 file: file.to_string(),
410 trait_name: trait_name.clone(),
411 span: Span {
412 start: *start as u64,
413 end: *end as u64,
414 },
415 })
416 .collect();
417 let scopes = crate::scope::node_scopes(self.spec.name, source, &nodes);
418 let body_terms = crate::body_terms::body_terms(source, &nodes, &scopes);
419 Ok(FileFacts {
420 file: file.to_string(),
421 content_hash: blake3::hash(source.as_bytes()).to_hex().to_string(),
422 has_syntax_errors: root.has_error(),
423 nodes,
424 contains,
425 references,
426 locals,
427 fields,
428 embeds,
429 trait_impls,
430 scopes,
431 body_terms,
432 })
433 }
434}
435
436const RUST_PRELUDE: &[&str] = &[
439 "Ok", "Err", "Some", "None", "Vec", "String", "Box", "Option", "Result", "Self", "bool",
440 "char", "str", "u8", "u16", "u32", "u64", "u128", "usize", "i8", "i16", "i32", "i64", "i128",
441 "isize", "f32", "f64",
442];
443
444fn container_ranges(root: TsNode<'_>, kinds: &[&str]) -> Vec<tree_sitter::Range> {
448 let mut out = Vec::new();
449 let mut stack = vec![root];
450 while let Some(node) = stack.pop() {
451 if kinds.contains(&node.kind()) {
452 out.push(node.range());
453 } else {
454 for i in (0..node.child_count()).rev() {
455 stack.extend(node.child(i));
456 }
457 }
458 }
459 out.sort_by_key(|r| r.start_byte);
460 out
461}
462
463fn collect(
466 query: &Query,
467 spec: &LanguageSpec,
468 root: TsNode<'_>,
469 source: &str,
470 entries: &mut Vec<RawEntry>,
471 out: &mut Collected,
472) {
473 {
474 let mut cursor = QueryCursor::new();
475 let mut matches = cursor.matches(query, root, source.as_bytes());
476 while let Some(m) = matches.next() {
477 let mut def: Option<(TsNode, SymbolKind)> = None;
478 let mut scope: Option<TsNode> = None;
479 let mut name: Option<TsNode> = None;
480 let mut qualifier: Option<TsNode> = None;
481 let mut reference: Option<(TsNode, Relation)> = None;
482 let mut refpath: Option<TsNode> = None;
483 let mut import_path: Option<TsNode> = None;
484 let mut import_module: Option<TsNode> = None;
485 let mut import_name: Option<TsNode> = None;
486 let mut import_alias: Option<TsNode> = None;
487 let mut import_star = false;
488 let mut match_locals: Vec<TsNode> = Vec::new();
489 let mut local_type: Option<TsNode> = None;
490 let mut field_name: Option<TsNode> = None;
491 let mut field_type: Option<TsNode> = None;
492 let mut trait_name: Option<TsNode> = None;
493 let mut trait_impl: Option<TsNode> = None;
494 for cap in m.captures {
495 let cap_name = &query.capture_names()[cap.index as usize];
496 if let Some(kind_str) = cap_name.strip_prefix("def.") {
497 if let Some(kind) = SymbolKind::from_str_opt(kind_str) {
498 def = Some((cap.node, kind));
499 }
500 } else if let Some(rel) = cap_name.strip_prefix("ref.") {
501 let relation = match rel {
502 "use" => Relation::Uses,
503 _ => Relation::Calls,
504 };
505 reference = Some((cap.node, relation));
506 } else {
507 match *cap_name {
508 "scope" => scope = Some(cap.node),
509 "name" => name = Some(cap.node),
510 "qualifier" => qualifier = Some(cap.node),
511 "refpath" => refpath = Some(cap.node),
512 "import" => import_path = Some(cap.node),
513 "import.module" => import_module = Some(cap.node),
514 "import.name" => import_name = Some(cap.node),
515 "import.alias" => import_alias = Some(cap.node),
516 "import.star" => import_star = true,
517 "local" => match_locals.push(cap.node),
518 "local.type" => local_type = Some(cap.node),
519 "field.name" => field_name = Some(cap.node),
520 "field.type" => field_type = Some(cap.node),
521 "trait" => trait_name = Some(cap.node),
522 "trait.impl" => trait_impl = Some(cap.node),
523 "doc" => out.docs.push((
524 cap.node.start_byte(),
525 cap.node.end_byte(),
526 text(cap.node, source).to_string(),
527 )),
528 "embed" => out.embeds.push((
529 cap.node.start_byte(),
530 cap.node.end_byte(),
531 text(cap.node, source).to_string(),
532 )),
533 _ => {}
534 }
535 }
536 }
537 let sep = spec.path_separators.first().copied().unwrap_or(".");
538 if let (Some(t), Some(block)) = (trait_name, trait_impl) {
539 out.trait_impls.push((
540 block.start_byte(),
541 block.end_byte(),
542 text(t, source).to_string(),
543 ));
544 }
545 if let (Some(name), Some(ty)) = (field_name, field_type) {
546 out.fields.push((
547 name.start_byte(),
548 ty.end_byte(),
549 text(name, source).to_string(),
550 text(ty, source).to_string(),
551 ));
552 }
553 if let Some(a) = import_alias {
554 out.alias_spans.push((a.start_byte(), a.end_byte()));
555 }
556 let alias = import_alias.map(|a| text(a, source).to_string());
557 for l in &match_locals {
558 out.locals.push(RawLocal {
559 start: l.start_byte(),
560 end: l.end_byte(),
561 name: text(*l, source).to_string(),
562 type_name: local_type.map(|t| text(t, source).to_string()),
563 });
564 }
565 if let Some(path_node) = import_path {
566 out.refs.push(RawRef {
570 start: path_node.start_byte(),
571 end: path_node.end_byte(),
572 name: text(path_node, source)
573 .trim_matches(['"', '\'', '`'])
574 .to_string(),
575 path: None,
576 alias: alias.or_else(|| import_star.then(|| "*".to_string())),
577 relation: Relation::Imports,
578 });
579 } else if let (Some(module), Some(item)) = (import_module, import_name) {
580 let module_text = text(module, source).trim_matches(['"', '\'', '`']);
583 out.refs.push(RawRef {
584 start: module.start_byte().min(item.start_byte()),
585 end: item.end_byte().max(module.end_byte()),
586 name: format!("{module_text}{sep}{}", text(item, source)),
587 path: None,
588 alias,
589 relation: Relation::Imports,
590 });
591 } else if let (Some(module), true) = (import_module, import_star) {
592 let module_text = text(module, source).trim_matches(['"', '\'', '`']);
594 out.refs.push(RawRef {
595 start: module.start_byte(),
596 end: module.end_byte(),
597 name: format!("{module_text}{sep}*"),
598 path: None,
599 alias: Some("*".to_string()),
600 relation: Relation::Imports,
601 });
602 }
603 if let Some((node, relation)) = reference {
604 out.refs.push(RawRef {
605 start: node.start_byte(),
606 end: node.end_byte(),
607 name: text(node, source).to_string(),
608 path: refpath.map(|p| qualified_path(p, node, source)),
609 alias: None,
610 relation,
611 });
612 }
613 let container = def.map(|(n, _)| n).or(scope);
614 if let (Some(container), Some(name_node)) = (container, name) {
615 if def.is_some() {
616 out.def_name_spans
617 .push((name_node.start_byte(), name_node.end_byte()));
618 }
619 entries.push(RawEntry {
620 start: container.start_byte(),
621 end: container.end_byte(),
622 name: text(name_node, source).to_string(),
623 kind: def.map(|(_, k)| k),
624 qualifier: qualifier.map(|q| text(q, source).to_string()),
625 signature: signature(container, source),
626 doc: doc_comment(container, source, spec.comment_kinds, spec.doc_skip_kinds),
627 });
628 }
629 }
630 }
631}
632
633fn qualified_path(refpath: TsNode<'_>, name: TsNode<'_>, source: &str) -> String {
639 let receiver = refpath
640 .named_child(0)
641 .filter(|c| refpath.id() != name.id() && c.end_byte() <= name.start_byte());
642 let Some(receiver) = receiver else {
643 return text(refpath, source).to_string();
644 };
645 let sep = source[receiver.end_byte()..name.start_byte()].trim();
646 format!("{}{sep}{}", text(receiver, source), text(name, source))
647}
648
649fn text<'a>(node: TsNode<'_>, source: &'a str) -> &'a str {
650 &source[node.start_byte()..node.end_byte()]
651}
652
653fn signature(node: TsNode<'_>, source: &str) -> String {
656 let t = text(node, source);
657 let first_line = t.lines().next().unwrap_or(t);
658 let head = if first_line.trim_end().ends_with(':') || first_line.contains('{') {
659 first_line.split('{').next().unwrap_or(first_line)
660 } else {
661 let up_to_brace = t.split('{').next().unwrap_or(t);
662 if up_to_brace.len() == t.len() {
663 first_line
664 } else {
665 up_to_brace
666 }
667 };
668 head.split_whitespace().collect::<Vec<_>>().join(" ")
669}
670
671fn doc_comment(
675 node: TsNode<'_>,
676 source: &str,
677 comment_kinds: &[&str],
678 skip_kinds: &[&str],
679) -> Option<String> {
680 let comments = preceding_comments(node, comment_kinds, skip_kinds).or_else(|| {
681 node.parent()
682 .and_then(|p| preceding_comments(p, comment_kinds, skip_kinds))
683 })?;
684 let mut lines = Vec::new();
685 for c in comments {
686 for line in text(c, source).lines() {
687 let mut l = line.trim();
688 for marker in ["///", "//!", "//", "/**", "/*", "*/", "--"] {
689 if let Some(stripped) = l.strip_prefix(marker) {
690 l = stripped;
691 break;
692 }
693 }
694 if let Some(rest) = l.strip_prefix('*')
697 && !l.starts_with("**")
698 {
699 l = rest;
700 }
701 l = l.strip_suffix("*/").unwrap_or(l);
702 lines.push(l.trim());
703 }
704 }
705 while lines.first().is_some_and(|l| l.is_empty()) {
706 lines.remove(0);
707 }
708 while lines.last().is_some_and(|l| l.is_empty()) {
709 lines.pop();
710 }
711 if lines.is_empty() {
712 None
713 } else {
714 Some(lines.join("\n"))
715 }
716}
717
718fn preceding_comments<'t>(
719 node: TsNode<'t>,
720 comment_kinds: &[&str],
721 skip_kinds: &[&str],
722) -> Option<Vec<TsNode<'t>>> {
723 let mut comments = Vec::new();
724 let mut cur = node.prev_named_sibling();
725 let mut skips = 0;
726 while let Some(sib) = cur {
727 if !comment_kinds.contains(&sib.kind()) {
728 if skips < 2 && comments.is_empty() && skip_kinds.contains(&sib.kind()) {
731 skips += 1;
732 cur = sib.prev_named_sibling();
733 continue;
734 }
735 break;
736 }
737 comments.push(sib);
738 cur = sib.prev_named_sibling();
739 }
740 comments.reverse();
741 if comments.is_empty() {
742 None
743 } else {
744 Some(comments)
745 }
746}