sinter-extract 0.54.0

Language-agnostic tree-sitter extraction: languages are data, the engine never forks
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
use sinter_core::{
    Edge, Evidence, FieldBinding, Node, NodeId, Reference, Relation, Span, SymbolKind,
};
use streaming_iterator::StreamingIterator;
use tree_sitter::{Node as TsNode, Parser, Query, QueryCursor};

/// Upper bound on a stored doc body; markdown sections and long
/// docstrings are truncated here, keeping the beginning.
const DOC_CAP_BYTES: usize = 4096;

use sinter_core::FileFacts;

use crate::language::LanguageSpec;

#[derive(Debug, thiserror::Error)]
pub enum ExtractError {
    #[error("bad grammar or query for {language}: {message}")]
    Query {
        language: &'static str,
        message: String,
    },
    #[error("parser returned no tree for {0}")]
    Parse(String),
}

/// One reusable extractor per (language, thread): parser and compiled query
/// are pooled here, never rebuilt per file.
pub struct Extractor {
    spec: &'static LanguageSpec,
    parser: Parser,
    query: Query,
    /// Secondary inline grammar (spec.inline): parses designated
    /// container-node ranges of the primary tree; captures merge into
    /// the same facts through the same contract.
    inline: Option<(Parser, Query)>,
}

/// A definition or scope-only entry, pre-qualification.
struct RawEntry {
    start: usize,
    end: usize,
    name: String,
    /// None for scope-only entries (e.g. impl blocks).
    kind: Option<SymbolKind>,
    /// Extra scope prefix from the same match (e.g. Go receiver type).
    qualifier: Option<String>,
    signature: String,
    doc: Option<String>,
}

/// A reference site, pre-enclosure.
struct RawRef {
    start: usize,
    end: usize,
    name: String,
    path: Option<String>,
    alias: Option<String>,
    relation: Relation,
}

/// A local binding site, pre-scoping.
struct RawLocal {
    start: usize,
    end: usize,
    name: String,
    type_name: Option<String>,
}

/// Everything collect() gathers besides definitions.
#[derive(Default)]
struct Collected {
    refs: Vec<RawRef>,
    locals: Vec<RawLocal>,
    /// (span, field name, written type) — owner resolved after entries exist.
    fields: Vec<(usize, usize, String, String)>,
    /// (span, embedded type name) — owner resolved after entries exist.
    embeds: Vec<(usize, usize, String)>,
    /// (impl block span, trait name) — trait-impl pairing facts.
    trait_impls: Vec<(usize, usize, String)>,
    /// Import-alias name spans: identical local captures are the import
    /// binding itself, not a shadow.
    alias_spans: Vec<(usize, usize)>,
    /// Definition name spans: a blanket type-reference capture also lands
    /// on `struct Foo`'s own name, which is a definition, not a use.
    def_name_spans: Vec<(usize, usize)>,
    /// Explicit doc captures (`@doc`, e.g. Python docstrings): (span, text).
    /// Attached to the smallest containing definition, overriding any
    /// sibling-comment doc.
    docs: Vec<(usize, usize, String)>,
}

impl Extractor {
    pub fn new(spec: &'static LanguageSpec) -> Result<Self, ExtractError> {
        let language = (spec.grammar)();
        let mut parser = Parser::new();
        parser
            .set_language(&language)
            .map_err(|e| ExtractError::Query {
                language: spec.name,
                message: e.to_string(),
            })?;
        let query = Query::new(&language, spec.query_source).map_err(|e| ExtractError::Query {
            language: spec.name,
            message: e.to_string(),
        })?;
        let inline = spec
            .inline
            .map(|i| {
                let language = (i.grammar)();
                let mut parser = Parser::new();
                parser
                    .set_language(&language)
                    .map_err(|e| (spec.name, e.to_string()))?;
                let query = Query::new(&language, i.query_source)
                    .map_err(|e| (spec.name, e.to_string()))?;
                Ok((parser, query))
            })
            .transpose()
            .map_err(
                |(language, message): (&'static str, String)| ExtractError::Query {
                    language,
                    message,
                },
            )?;
        Ok(Self {
            spec,
            parser,
            query,
            inline,
        })
    }

    /// Extract facts from one file. `file` is the repo-relative path.
    pub fn extract(&mut self, file: &str, source: &str) -> Result<FileFacts, ExtractError> {
        let tree = self
            .parser
            .parse(source, None)
            .ok_or_else(|| ExtractError::Parse(file.to_string()))?;
        let root = tree.root_node();

        let mut entries = Vec::new();
        let mut collected = Collected::default();
        collect(
            &self.query,
            self.spec,
            root,
            source,
            &mut entries,
            &mut collected,
        );
        // Secondary inline grammar (spec.inline): parse the container
        // nodes' ranges of the same source, so capture spans are already
        // file-absolute, and merge through the identical contract.
        if let (Some((parser, query)), Some(ispec)) = (&mut self.inline, self.spec.inline) {
            let ranges = container_ranges(root, ispec.container_kinds);
            if !ranges.is_empty() {
                // Both failure modes are broken invariants (container_ranges
                // yields sorted non-overlapping ranges; the primary parse
                // already succeeded) — swallowing them would silently drop
                // this file's inline refs and let the graph assert "no
                // links" without evidence. Fail loudly like the primary.
                parser.set_included_ranges(&ranges).map_err(|e| {
                    ExtractError::Parse(format!("{} (inline ranges: {e})", self.spec.name))
                })?;
                let inline_tree = parser
                    .parse(source, None)
                    .ok_or_else(|| ExtractError::Parse(format!("{} (inline)", self.spec.name)))?;
                collect(
                    query,
                    self.spec,
                    inline_tree.root_node(),
                    source,
                    &mut entries,
                    &mut collected,
                );
            }
        }
        entries.sort_by_key(|e| (e.start, usize::MAX - e.end));
        // Explicit @doc captures override sibling-comment docs on the
        // smallest definition containing them (Python docstrings). Several
        // captures on one definition (markdown section blocks) are joined
        // with blank lines in document order.
        let mut doc_set = vec![false; entries.len()];
        for (d_start, d_end, text) in &collected.docs {
            let owner = entries
                .iter_mut()
                .enumerate()
                .filter(|(_, e)| e.kind.is_some() && e.start <= *d_start && *d_end <= e.end)
                .min_by_key(|(_, e)| e.end - e.start);
            if let Some((i, entry)) = owner {
                let cleaned: Vec<&str> = text.lines().map(str::trim).collect();
                let trimmed = cleaned.join("\n");
                let trimmed = trimmed.trim_matches('\n');
                if trimmed.is_empty() {
                    continue;
                }
                entry.doc = match entry.doc.take().filter(|_| doc_set[i]) {
                    // Whole-section bodies can run long; keep the beginning
                    // (what ask/show surface) and stop appending past the cap.
                    Some(prev) if prev.len() >= DOC_CAP_BYTES => Some(prev),
                    Some(prev) => Some(format!("{prev}\n\n{trimmed}")),
                    None => Some(trimmed.to_string()),
                };
                doc_set[i] = true;
            }
        }
        // Two patterns may claim the same node (e.g. `const f = () => ...`
        // as variable and function): the more specific, non-variable kind
        // wins; sort puts identical spans adjacent.
        entries.dedup_by(|b, a| {
            let same = a.start == b.start && a.end == b.end && a.name == b.name;
            if same && a.kind == Some(SymbolKind::Variable) && b.kind.is_some() {
                a.kind = b.kind;
            }
            same
        });

        let file_id = NodeId::new(file);
        let mut nodes = vec![Node {
            id: file_id.clone(),
            kind: SymbolKind::File,
            name: file.to_string(),
            file: file.to_string(),
            span: Span {
                start: 0,
                end: source.len().max(1) as u64,
            },
            signature: String::new(),
            doc: None,
        }];
        let mut contains = Vec::new();

        // Containment stack: (end, scope name, node id if a real definition).
        let mut stack: Vec<(usize, String, Option<NodeId>)> = Vec::new();
        // (start, end, id, kind) of each definition, for enclosure and
        // declared-field owner lookup.
        let mut def_spans: Vec<(usize, usize, NodeId, SymbolKind)> = Vec::new();

        for entry in &entries {
            while stack.last().is_some_and(|(end, _, _)| *end <= entry.start) {
                stack.pop();
            }
            let mut path: Vec<&str> = stack.iter().map(|(_, name, _)| name.as_str()).collect();
            if let Some(q) = &entry.qualifier {
                path.push(q);
            }
            path.push(&entry.name);
            let qualified = path.join("::");
            // Children nest under the entry's qualified segment.
            let scope_segment = entry
                .qualifier
                .as_ref()
                .map_or(entry.name.clone(), |q| format!("{q}::{}", entry.name));

            let id = if let Some(kind) = entry.kind {
                let id = NodeId::new(format!("{file}#{qualified}@{}", entry.start));
                let parent = stack
                    .iter()
                    .rev()
                    .find_map(|(_, _, id)| id.clone())
                    .unwrap_or_else(|| file_id.clone());
                nodes.push(Node {
                    id: id.clone(),
                    kind,
                    name: entry.name.clone(),
                    file: file.to_string(),
                    span: Span {
                        start: entry.start as u64,
                        end: entry.end as u64,
                    },
                    signature: entry.signature.clone(),
                    doc: entry.doc.clone(),
                });
                contains.push(Edge {
                    src: parent,
                    dst: id.clone(),
                    relation: Relation::Contains,
                    evidence: Evidence::Structural,
                    confidence: Evidence::Structural.confidence(),
                    // Containment is structure, not a reference: no site.
                    site: None,
                });
                def_spans.push((entry.start, entry.end, id.clone(), kind));
                Some(id)
            } else {
                None
            };
            stack.push((entry.end, scope_segment, id));
        }

        // One span may be captured twice (a scoped type's name both with
        // its path and bare): keep the path-bearing one, never both.
        let mut refs = collected.refs;
        refs.retain(|r| !collected.def_name_spans.contains(&(r.start, r.end)));
        // Rust prelude/primitive names never name a corpus symbol unless
        // this file defines one of the same name (then it may shadow).
        if self.spec.name == "rust" {
            refs.retain(|r| {
                r.path.is_some()
                    || !RUST_PRELUDE.contains(&r.name.as_str())
                    || entries.iter().any(|e| e.name == r.name)
            });
        }
        refs.sort_by_key(|r| (r.start, r.end, r.path.is_none()));
        refs.dedup_by_key(|r| (r.start, r.end));
        let references = refs
            .into_iter()
            .map(|r| {
                let enclosing = def_spans
                    .iter()
                    .filter(|(s, e, _, _)| *s <= r.start && r.end <= *e)
                    .min_by_key(|(s, e, _, _)| e - s)
                    .map(|(_, _, id, _)| id.clone());
                Reference {
                    file: file.to_string(),
                    name: r.name,
                    path: r.path,
                    relation: r.relation,
                    span: Span {
                        start: r.start as u64,
                        end: r.end as u64,
                    },
                    enclosing,
                    alias: r.alias,
                }
            })
            .collect();

        // A local shadows from its introduction to the end of the innermost
        // definition containing it (file end at top level). A "local" whose
        // span is an import alias IS the import binding, not a shadow.
        let alias_spans = collected.alias_spans;
        let embeds = collected
            .embeds
            .iter()
            .filter_map(|(start, end, type_name)| {
                let owner = def_spans
                    .iter()
                    .filter(|(s, e, _, _)| s <= start && end <= e)
                    .min_by_key(|(s, e, _, _)| e - s)
                    .map(|(_, _, id, _)| id.clone())?;
                Some(sinter_core::Embed {
                    owner,
                    type_name: type_name.clone(),
                })
            })
            .collect();
        let mut raw_locals = collected
            .locals
            .into_iter()
            .filter(|l| !alias_spans.contains(&(l.start, l.end)))
            .collect::<Vec<_>>();
        // Typed and untyped query patterns can match the same binding.
        // Keep exactly one, preferring the typed capture.
        raw_locals.sort_by_key(|l| (l.start, l.end, l.name.clone(), l.type_name.is_none()));
        raw_locals.dedup_by(|b, a| a.start == b.start && a.end == b.end && a.name == b.name);
        let locals = raw_locals
            .into_iter()
            .map(|l| {
                let scope_end = def_spans
                    .iter()
                    .filter(|(s, e, _, _)| *s <= l.start && l.end <= *e)
                    .min_by_key(|(s, e, _, _)| e - s)
                    .map_or(source.len() as u64, |(_, e, _, _)| *e as u64);
                sinter_core::LocalBinding {
                    file: file.to_string(),
                    name: l.name,
                    span: Span {
                        start: l.start as u64,
                        end: l.end as u64,
                    },
                    scope_end,
                    type_name: l.type_name,
                }
            })
            .collect();

        let fields = collected
            .fields
            .iter()
            .filter_map(|(start, end, name, type_name)| {
                let owner = def_spans
                    .iter()
                    .filter(|(s, e, _, kind)| {
                        *s <= *start
                            && *end <= *e
                            && matches!(
                                kind,
                                SymbolKind::Struct
                                    | SymbolKind::Class
                                    | SymbolKind::Trait
                                    | SymbolKind::Interface
                            )
                    })
                    .min_by_key(|(s, e, _, _)| e - s)
                    .map(|(_, _, id, _)| id.clone())?;
                Some(FieldBinding {
                    owner,
                    name: name.clone(),
                    type_name: type_name.clone(),
                })
            })
            .collect();

        let trait_impls = collected
            .trait_impls
            .iter()
            .map(|(start, end, trait_name)| sinter_core::TraitImpl {
                file: file.to_string(),
                trait_name: trait_name.clone(),
                span: Span {
                    start: *start as u64,
                    end: *end as u64,
                },
            })
            .collect();
        let scopes = crate::scope::node_scopes(self.spec.name, source, &nodes);
        let body_terms = crate::body_terms::body_terms(source, &nodes, &scopes);
        Ok(FileFacts {
            file: file.to_string(),
            content_hash: blake3::hash(source.as_bytes()).to_hex().to_string(),
            has_syntax_errors: root.has_error(),
            nodes,
            contains,
            references,
            locals,
            fields,
            embeds,
            trait_impls,
            scopes,
            body_terms,
        })
    }
}

/// Bare Rust names that always come from the prelude or are primitive
/// types: a reference to one is noise unless the file shadows it.
const RUST_PRELUDE: &[&str] = &[
    "Ok", "Err", "Some", "None", "Vec", "String", "Box", "Option", "Result", "Self", "bool",
    "char", "str", "u8", "u16", "u32", "u64", "u128", "usize", "i8", "i16", "i32", "i64", "i128",
    "isize", "f32", "f64",
];

/// Byte ranges of every node of the given kinds, in document order —
/// the included-range input for a secondary inline parse. Matched nodes
/// are not descended into, so ranges never overlap.
fn container_ranges(root: TsNode<'_>, kinds: &[&str]) -> Vec<tree_sitter::Range> {
    let mut out = Vec::new();
    let mut stack = vec![root];
    while let Some(node) = stack.pop() {
        if kinds.contains(&node.kind()) {
            out.push(node.range());
        } else {
            for i in (0..node.child_count()).rev() {
                stack.extend(node.child(i));
            }
        }
    }
    out.sort_by_key(|r| r.start_byte);
    out
}

/// Run one query over one tree; group captures per match by the universal
/// contract, appending to `entries`/`out` (called once per grammar).
fn collect(
    query: &Query,
    spec: &LanguageSpec,
    root: TsNode<'_>,
    source: &str,
    entries: &mut Vec<RawEntry>,
    out: &mut Collected,
) {
    {
        let mut cursor = QueryCursor::new();
        let mut matches = cursor.matches(query, root, source.as_bytes());
        while let Some(m) = matches.next() {
            let mut def: Option<(TsNode, SymbolKind)> = None;
            let mut scope: Option<TsNode> = None;
            let mut name: Option<TsNode> = None;
            let mut qualifier: Option<TsNode> = None;
            let mut reference: Option<(TsNode, Relation)> = None;
            let mut refpath: Option<TsNode> = None;
            let mut import_path: Option<TsNode> = None;
            let mut import_module: Option<TsNode> = None;
            let mut import_name: Option<TsNode> = None;
            let mut import_alias: Option<TsNode> = None;
            let mut import_star = false;
            let mut match_locals: Vec<TsNode> = Vec::new();
            let mut local_type: Option<TsNode> = None;
            let mut field_name: Option<TsNode> = None;
            let mut field_type: Option<TsNode> = None;
            let mut trait_name: Option<TsNode> = None;
            let mut trait_impl: Option<TsNode> = None;
            for cap in m.captures {
                let cap_name = &query.capture_names()[cap.index as usize];
                if let Some(kind_str) = cap_name.strip_prefix("def.") {
                    if let Some(kind) = SymbolKind::from_str_opt(kind_str) {
                        def = Some((cap.node, kind));
                    }
                } else if let Some(rel) = cap_name.strip_prefix("ref.") {
                    let relation = match rel {
                        "use" => Relation::Uses,
                        _ => Relation::Calls,
                    };
                    reference = Some((cap.node, relation));
                } else {
                    match *cap_name {
                        "scope" => scope = Some(cap.node),
                        "name" => name = Some(cap.node),
                        "qualifier" => qualifier = Some(cap.node),
                        "refpath" => refpath = Some(cap.node),
                        "import" => import_path = Some(cap.node),
                        "import.module" => import_module = Some(cap.node),
                        "import.name" => import_name = Some(cap.node),
                        "import.alias" => import_alias = Some(cap.node),
                        "import.star" => import_star = true,
                        "local" => match_locals.push(cap.node),
                        "local.type" => local_type = Some(cap.node),
                        "field.name" => field_name = Some(cap.node),
                        "field.type" => field_type = Some(cap.node),
                        "trait" => trait_name = Some(cap.node),
                        "trait.impl" => trait_impl = Some(cap.node),
                        "doc" => out.docs.push((
                            cap.node.start_byte(),
                            cap.node.end_byte(),
                            text(cap.node, source).to_string(),
                        )),
                        "embed" => out.embeds.push((
                            cap.node.start_byte(),
                            cap.node.end_byte(),
                            text(cap.node, source).to_string(),
                        )),
                        _ => {}
                    }
                }
            }
            let sep = spec.path_separators.first().copied().unwrap_or(".");
            if let (Some(t), Some(block)) = (trait_name, trait_impl) {
                out.trait_impls.push((
                    block.start_byte(),
                    block.end_byte(),
                    text(t, source).to_string(),
                ));
            }
            if let (Some(name), Some(ty)) = (field_name, field_type) {
                out.fields.push((
                    name.start_byte(),
                    ty.end_byte(),
                    text(name, source).to_string(),
                    text(ty, source).to_string(),
                ));
            }
            if let Some(a) = import_alias {
                out.alias_spans.push((a.start_byte(), a.end_byte()));
            }
            let alias = import_alias.map(|a| text(a, source).to_string());
            for l in &match_locals {
                out.locals.push(RawLocal {
                    start: l.start_byte(),
                    end: l.end_byte(),
                    name: text(*l, source).to_string(),
                    type_name: local_type.map(|t| text(t, source).to_string()),
                });
            }
            if let Some(path_node) = import_path {
                // Whole-path import (`use a::b`, `import "pkg"`), possibly
                // with an alias, Go's dot form, or glob semantics
                // (`@import.star` alongside: bash `source` binds every name).
                out.refs.push(RawRef {
                    start: path_node.start_byte(),
                    end: path_node.end_byte(),
                    name: text(path_node, source)
                        .trim_matches(['"', '\'', '`'])
                        .to_string(),
                    path: None,
                    alias: alias.or_else(|| import_star.then(|| "*".to_string())),
                    relation: Relation::Imports,
                });
            } else if let (Some(module), Some(item)) = (import_module, import_name) {
                // From-style import: module and item joined so the import
                // binds the item itself. Alias renames the local binding.
                let module_text = text(module, source).trim_matches(['"', '\'', '`']);
                out.refs.push(RawRef {
                    start: module.start_byte().min(item.start_byte()),
                    end: item.end_byte().max(module.end_byte()),
                    name: format!("{module_text}{sep}{}", text(item, source)),
                    path: None,
                    alias,
                    relation: Relation::Imports,
                });
            } else if let (Some(module), true) = (import_module, import_star) {
                // Glob import: every top-level name of the module is bound.
                let module_text = text(module, source).trim_matches(['"', '\'', '`']);
                out.refs.push(RawRef {
                    start: module.start_byte(),
                    end: module.end_byte(),
                    name: format!("{module_text}{sep}*"),
                    path: None,
                    alias: Some("*".to_string()),
                    relation: Relation::Imports,
                });
            }
            if let Some((node, relation)) = reference {
                out.refs.push(RawRef {
                    start: node.start_byte(),
                    end: node.end_byte(),
                    name: text(node, source).to_string(),
                    path: refpath.map(|p| qualified_path(p, node, source)),
                    alias: None,
                    relation,
                });
            }
            let container = def.map(|(n, _)| n).or(scope);
            if let (Some(container), Some(name_node)) = (container, name) {
                if def.is_some() {
                    out.def_name_spans
                        .push((name_node.start_byte(), name_node.end_byte()));
                }
                entries.push(RawEntry {
                    start: container.start_byte(),
                    end: container.end_byte(),
                    name: text(name_node, source).to_string(),
                    kind: def.map(|(_, k)| k),
                    qualifier: qualifier.map(|q| text(q, source).to_string()),
                    signature: signature(container, source),
                    doc: doc_comment(container, source, spec.comment_kinds, spec.doc_skip_kinds),
                });
            }
        }
    }
}

/// Path text for a qualified reference: the immediate receiver joined to
/// the referenced name by the separator written between them. Using the
/// receiver child rather than the whole `refpath` text keeps a method
/// chain (`a.b().c()`) from smearing the entire chain into the path of
/// its tail, and drops the whitespace/comments around the separator.
fn qualified_path(refpath: TsNode<'_>, name: TsNode<'_>, source: &str) -> String {
    let receiver = refpath
        .named_child(0)
        .filter(|c| refpath.id() != name.id() && c.end_byte() <= name.start_byte());
    let Some(receiver) = receiver else {
        return text(refpath, source).to_string();
    };
    let sep = source[receiver.end_byte()..name.start_byte()].trim();
    format!("{}{sep}{}", text(receiver, source), text(name, source))
}

fn text<'a>(node: TsNode<'_>, source: &'a str) -> &'a str {
    &source[node.start_byte()..node.end_byte()]
}

/// Declaration text up to the body. Brace languages cut at the first `{`;
/// a first line ending in `:` (Python-style) is the whole signature.
fn signature(node: TsNode<'_>, source: &str) -> String {
    let t = text(node, source);
    let first_line = t.lines().next().unwrap_or(t);
    let head = if first_line.trim_end().ends_with(':') || first_line.contains('{') {
        first_line.split('{').next().unwrap_or(first_line)
    } else {
        let up_to_brace = t.split('{').next().unwrap_or(t);
        if up_to_brace.len() == t.len() {
            first_line
        } else {
            up_to_brace
        }
    };
    head.split_whitespace().collect::<Vec<_>>().join(" ")
}

/// Contiguous comment siblings immediately above the definition (or its
/// parent declaration), stripped of comment markers. Generic across
/// languages: comment node kinds come from the spec.
fn doc_comment(
    node: TsNode<'_>,
    source: &str,
    comment_kinds: &[&str],
    skip_kinds: &[&str],
) -> Option<String> {
    let comments = preceding_comments(node, comment_kinds, skip_kinds).or_else(|| {
        node.parent()
            .and_then(|p| preceding_comments(p, comment_kinds, skip_kinds))
    })?;
    let mut lines = Vec::new();
    for c in comments {
        for line in text(c, source).lines() {
            let mut l = line.trim();
            for marker in ["///", "//!", "//", "/**", "/*", "*/", "--"] {
                if let Some(stripped) = l.strip_prefix(marker) {
                    l = stripped;
                    break;
                }
            }
            // Block-comment continuation: a leading `*` (Javadoc/C-style
            // interior line) is decoration — but `**bold**` is markdown.
            if let Some(rest) = l.strip_prefix('*')
                && !l.starts_with("**")
            {
                l = rest;
            }
            l = l.strip_suffix("*/").unwrap_or(l);
            lines.push(l.trim());
        }
    }
    while lines.first().is_some_and(|l| l.is_empty()) {
        lines.remove(0);
    }
    while lines.last().is_some_and(|l| l.is_empty()) {
        lines.pop();
    }
    if lines.is_empty() {
        None
    } else {
        Some(lines.join("\n"))
    }
}

fn preceding_comments<'t>(
    node: TsNode<'t>,
    comment_kinds: &[&str],
    skip_kinds: &[&str],
) -> Option<Vec<TsNode<'t>>> {
    let mut comments = Vec::new();
    let mut cur = node.prev_named_sibling();
    let mut skips = 0;
    while let Some(sib) = cur {
        if !comment_kinds.contains(&sib.kind()) {
            // Step over decorator-style macro lines (UCLASS, UPROPERTY)
            // that sit between a definition and its doc comment.
            if skips < 2 && comments.is_empty() && skip_kinds.contains(&sib.kind()) {
                skips += 1;
                cur = sib.prev_named_sibling();
                continue;
            }
            break;
        }
        comments.push(sib);
        cur = sib.prev_named_sibling();
    }
    comments.reverse();
    if comments.is_empty() {
        None
    } else {
        Some(comments)
    }
}