Skip to main content

lockdocs_core/
extract.rs

1//! Turn a package's files into entries: API symbols (signature + doc comment)
2//! from TypeScript declarations, JavaScript, Python, Rust and Go sources, and
3//! prose sections from READMEs, changelogs and docs folders.
4
5use crate::locate::Source;
6use crate::markdown;
7use crate::Eco;
8use rayon::prelude::*;
9use serde::{Deserialize, Serialize};
10use std::cell::RefCell;
11use std::collections::{HashMap, HashSet};
12use std::path::{Path, PathBuf};
13use tree_sitter::{Node, Parser};
14
15#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
16pub enum Kind {
17    Function,
18    Method,
19    Class,
20    Interface,
21    Type,
22    Const,
23    Enum,
24    Struct,
25    Trait,
26    Macro,
27    Module,
28    Field,
29    Alias,
30    Prose,
31}
32
33impl Kind {
34    pub fn as_str(self) -> &'static str {
35        match self {
36            Kind::Function => "function",
37            Kind::Method => "method",
38            Kind::Class => "class",
39            Kind::Interface => "interface",
40            Kind::Type => "type",
41            Kind::Const => "const",
42            Kind::Enum => "enum",
43            Kind::Struct => "struct",
44            Kind::Trait => "trait",
45            Kind::Macro => "macro",
46            Kind::Module => "module",
47            Kind::Field => "field",
48            Kind::Alias => "re-export",
49            Kind::Prose => "doc",
50        }
51    }
52    pub fn is_container(self) -> bool {
53        matches!(self, Kind::Class | Kind::Interface | Kind::Struct | Kind::Trait | Kind::Enum | Kind::Module)
54    }
55}
56
57#[derive(Debug, Clone, Serialize, Deserialize)]
58pub struct Entry {
59    pub kind: Kind,
60    /// Short name (`object`, `spawn`), or the heading path for prose.
61    pub name: String,
62    /// Qualified path: `z.object`, `tokio::task::spawn`, `pydantic.main.BaseModel.model_dump`.
63    pub path: String,
64    /// File relative to the package root.
65    pub file: String,
66    pub line: u32,
67    /// Declaration text without the body.
68    pub sig: String,
69    /// Doc comment, or the prose section text.
70    pub doc: String,
71    /// For re-exports: the local name this entry points at.
72    pub alias_of: Option<String>,
73    /// Lives in a legacy compatibility copy (`zod/v3` inside zod 4,
74    /// `pydantic/v1` inside pydantic 2): ranked below the current API.
75    #[serde(default)]
76    pub legacy: bool,
77}
78
79/// A path segment `vN` with N below the package's major version.
80fn legacy_file(file: &str, major: u64) -> bool {
81    major > 1
82        && file.split('/').any(|seg| {
83            seg.len() >= 2 && seg.starts_with('v') && seg[1..].chars().all(|c| c.is_ascii_digit()) && seg[1..].parse::<u64>().is_ok_and(|n| n < major)
84        })
85}
86
87// ---------------------------------------------------------------- file selection
88
89const SKIP_DIRS: &[&str] = &[
90    "node_modules",
91    "compiled",
92    ".git",
93    "test",
94    "tests",
95    "__tests__",
96    "__test__",
97    "testdata",
98    "test_data",
99    "benches",
100    "bench",
101    "benchmarks",
102    "fixtures",
103    "__fixtures__",
104    "__pycache__",
105    "examples",
106    "example",
107    "_examples",
108    "coverage",
109    ".github",
110    "vendor",
111    "internal",
112    "__snapshots__",
113];
114
115const MAX_FILES: usize = 25_000;
116const MAX_FILE_BYTES: u64 = 3 * 1024 * 1024;
117
118fn is_doc_name(name: &str) -> bool {
119    let u = name.to_ascii_uppercase();
120    [
121        "README",
122        "CHANGELOG",
123        "CHANGES",
124        "HISTORY",
125        "MIGRATION",
126        "MIGRATING",
127        "UPGRADING",
128        "UPGRADE",
129        "RELEASES",
130        "NEWS",
131        "GUIDE",
132        "API",
133    ]
134    .iter()
135    .any(|p| u.starts_with(p))
136}
137
138fn doc_ext(ext: &str) -> bool {
139    matches!(ext, "md" | "mdx" | "markdown" | "rst")
140}
141
142fn walk(dir: &Path, rel: &Path, depth: usize, out: &mut Vec<PathBuf>) {
143    if depth > 14 || out.len() >= MAX_FILES {
144        return;
145    }
146    let Ok(rd) = std::fs::read_dir(dir) else { return };
147    let mut entries: Vec<_> = rd.flatten().collect();
148    entries.sort_by_key(|e| e.file_name());
149    for e in entries {
150        let name = e.file_name().to_string_lossy().to_string();
151        let Ok(ft) = e.file_type() else { continue };
152        let r = rel.join(&name);
153        if ft.is_dir() {
154            if SKIP_DIRS.contains(&name.as_str()) || name.starts_with('.') {
155                continue;
156            }
157            walk(&e.path(), &r, depth + 1, out);
158        } else if ft.is_file() || ft.is_symlink() {
159            out.push(r);
160        }
161        if out.len() >= MAX_FILES {
162            return;
163        }
164    }
165}
166
167/// Every file under `dir` (docs trees keep folders like `examples/` we want).
168fn walk_all(dir: &Path, rel: &Path, depth: usize, out: &mut Vec<PathBuf>) {
169    if depth > 16 || out.len() >= MAX_FILES {
170        return;
171    }
172    let Ok(rd) = std::fs::read_dir(dir) else { return };
173    let mut entries: Vec<_> = rd.flatten().collect();
174    entries.sort_by_key(|e| e.file_name());
175    for e in entries {
176        let name = e.file_name().to_string_lossy().to_string();
177        if name.starts_with('.') {
178            continue;
179        }
180        let r = rel.join(&name);
181        match e.file_type() {
182            Ok(t) if t.is_dir() => walk_all(&e.path(), &r, depth + 1, out),
183            Ok(_) => out.push(r),
184            _ => {}
185        }
186    }
187}
188
189fn ext_of(p: &Path) -> String {
190    let s = p.file_name().map(|f| f.to_string_lossy().to_ascii_lowercase()).unwrap_or_default();
191    for e in [".d.ts", ".d.mts", ".d.cts"] {
192        if s.ends_with(e) {
193            return e[1..].to_string();
194        }
195    }
196    p.extension().map(|e| e.to_string_lossy().to_ascii_lowercase()).unwrap_or_default()
197}
198
199fn in_docs_dir(p: &Path) -> bool {
200    p.components().any(|c| {
201        matches!(
202            c.as_os_str().to_string_lossy().to_ascii_lowercase().as_str(),
203            "docs" | "doc" | "documentation" | "guide" | "guides"
204        )
205    })
206}
207
208/// A unit of work: one file to read, how to parse it, and how to label it.
209struct Job {
210    abs: PathBuf,
211    rel: String,
212    how: How,
213}
214
215#[derive(Clone, Copy, PartialEq)]
216enum How {
217    Prose,
218    /// A docs page written as a JS/TSX component.
219    Page,
220    Example,
221    Metadata,
222    Ts,
223    Python,
224    Rust,
225    Go,
226}
227
228/// Plan which files to read for a package.
229fn plan(eco: Eco, src: &Source, extra_types: Option<&Path>, upstream: Option<&Path>) -> Vec<Job> {
230    let mut rels = Vec::new();
231    match &src.files {
232        Some(list) => rels.extend(list.iter().cloned()),
233        None => walk(&src.dir, Path::new(""), 0, &mut rels),
234    }
235    let mut jobs = Vec::new();
236    let names: HashSet<String> = rels.iter().map(|r| r.to_string_lossy().replace('\\', "/")).collect();
237    let mut has_dts = false;
238    for r in &rels {
239        let e = ext_of(r);
240        if e.starts_with("d.") {
241            has_dts = true;
242        }
243    }
244    let rs_crate_src = eco == Eco::Cargo;
245    for r in rels {
246        let rel = r.to_string_lossy().replace('\\', "/");
247        if rel.split('/').any(|seg| SKIP_DIRS.contains(&seg) && src.files.is_some()) {
248            continue;
249        }
250        let abs = src.dir.join(&r);
251        let fname = r.file_name().map(|f| f.to_string_lossy().to_string()).unwrap_or_default();
252        let e = ext_of(&r);
253        let depth = rel.matches('/').count();
254        let how = if doc_ext(&e) || (e.is_empty() || e == "txt") && is_doc_name(&fname) && depth == 0 {
255            if is_doc_name(&fname) || in_docs_dir(&r) || depth == 0 {
256                Some(How::Prose)
257            } else {
258                None
259            }
260        } else {
261            match eco {
262                Eco::Npm => match e.as_str() {
263                    "d.ts" => Some(How::Ts),
264                    "d.mts" | "d.cts" => {
265                        let base = rel.trim_end_matches(".d.mts").trim_end_matches(".d.cts");
266                        if names.contains(&format!("{base}.d.ts")) {
267                            None
268                        } else {
269                            Some(How::Ts)
270                        }
271                    }
272                    "ts" | "mts" | "cts" | "tsx" if !has_dts => Some(How::Ts),
273                    "js" | "mjs" | "cjs" if !has_dts && !fname.contains(".min.") => Some(How::Ts),
274                    _ => None,
275                },
276                Eco::PyPI => match e.as_str() {
277                    "py" => Some(How::Python),
278                    "pyi" => {
279                        let base = rel.trim_end_matches(".pyi");
280                        if names.contains(&format!("{base}.py")) {
281                            None
282                        } else {
283                            Some(How::Python)
284                        }
285                    }
286                    _ => None,
287                },
288                Eco::Cargo => (e == "rs" && rs_crate_src && rel.starts_with("src/")).then_some(How::Rust),
289                Eco::Go => (e == "go" && !fname.ends_with("_test.go")).then_some(How::Go),
290            }
291        };
292        if let Some(how) = how {
293            jobs.push(Job { abs, rel, how });
294        }
295    }
296    // Upstream docs fetched from the repository at this version's tag.
297    if let Some(up) = upstream {
298        let pages: HashSet<String> = std::fs::read_to_string(up.join(".lockdocs-upstream.json"))
299            .ok()
300            .and_then(|t| serde_json::from_str::<crate::upstream::Manifest>(&t).ok())
301            .map(|m| m.pages.into_iter().collect())
302            .unwrap_or_default();
303        let mut rels = Vec::new();
304        walk_all(up, Path::new(""), 0, &mut rels);
305        for r in rels {
306            let e = ext_of(&r);
307            let plain = r.to_string_lossy().replace('\\', "/");
308            let rel = format!("upstream:{plain}");
309            if pages.contains(&plain) {
310                jobs.push(Job {
311                    abs: up.join(&r),
312                    rel,
313                    how: How::Page,
314                });
315                continue;
316            }
317            // Django and others write reStructuredText in `.txt` files under docs/.
318            if doc_ext(&e) || e == "mdoc" || e == "txt" {
319                jobs.push(Job {
320                    abs: up.join(&r),
321                    rel,
322                    how: How::Prose,
323                });
324            } else if EXAMPLE_EXTS.contains(&e.as_str()) {
325                // Docs examples (`docs/examples/validators_simple.py`) carry the code the pages include.
326                jobs.push(Job {
327                    abs: up.join(&r),
328                    rel,
329                    how: How::Example,
330                });
331            }
332        }
333    }
334    if let Some(meta) = &src.metadata {
335        jobs.push(Job {
336            abs: meta.clone(),
337            rel: "README (package metadata)".into(),
338            how: How::Metadata,
339        });
340    }
341    // Types that ship separately (`@types/react` for `react`).
342    if let (Some(types), false) = (extra_types, has_dts) {
343        let mut rels = Vec::new();
344        walk(types, Path::new(""), 0, &mut rels);
345        let label = types
346            .iter()
347            .rev()
348            .take(2)
349            .collect::<Vec<_>>()
350            .into_iter()
351            .rev()
352            .map(|s| s.to_string_lossy())
353            .collect::<Vec<_>>()
354            .join("/");
355        for r in rels {
356            if ext_of(&r) == "d.ts" {
357                let rel = format!("{label}/{}", r.to_string_lossy().replace('\\', "/"));
358                jobs.push(Job {
359                    abs: types.join(&r),
360                    rel,
361                    how: How::Ts,
362                });
363            }
364        }
365    }
366    jobs
367}
368
369/// Extract every entry for a package.
370pub fn extract(eco: Eco, pkg_name: &str, src: &Source, extra_types: Option<&Path>, upstream: Option<&Path>) -> Vec<Entry> {
371    let major: u64 = src.version.trim_start_matches('v').split('.').next().and_then(|m| m.parse().ok()).unwrap_or(0);
372    let jobs = plan(eco, src, extra_types, upstream);
373    let crate_name = if eco == Eco::Cargo {
374        rust_crate_name(&src.dir).unwrap_or_else(|| pkg_name.replace('-', "_"))
375    } else {
376        String::new()
377    };
378    let includes = upstream.map(Includes::scan).unwrap_or_default();
379    let mut all: Vec<Entry> = jobs
380        .par_iter()
381        .flat_map_iter(|job| {
382            let Ok(meta) = std::fs::metadata(&job.abs) else { return Vec::new() };
383            if meta.len() > MAX_FILE_BYTES && job.how != How::Prose {
384                return Vec::new();
385            }
386            let Ok(bytes) = std::fs::read(&job.abs) else { return Vec::new() };
387            let text = String::from_utf8_lossy(&bytes);
388            let mut out = Vec::new();
389            match job.how {
390                How::Prose => {
391                    prose(&text, &job.rel, 0, &mut out);
392                    if job.rel.starts_with("upstream:") && !includes.files.is_empty() {
393                        for e in &mut out {
394                            e.doc = includes.expand(&e.doc);
395                        }
396                    }
397                }
398                How::Page => {
399                    if let Some((md, title)) = page_markdown(&text) {
400                        let title = title.unwrap_or_else(|| page_title(&job.rel));
401                        prose_titled(&md, &job.rel, 0, Some(title), &mut out);
402                    }
403                }
404                How::Example => example(&text, &job.rel, &mut out),
405                How::Metadata => {
406                    let (body, line) = markdown::metadata_body(&text);
407                    prose(&body, &job.rel, line - 1, &mut out)
408                }
409                How::Ts => ts_file(&text, &job.rel, &mut out),
410                How::Python => py_file(&text, &job.rel, &mut out),
411                How::Rust => rs_file(&text, &job.rel, &crate_name, &mut out),
412                How::Go => go_file(&text, &job.rel, pkg_name, &mut out),
413            }
414            out
415        })
416        .collect();
417    for e in &mut all {
418        e.legacy = legacy_file(&e.file, major);
419    }
420    // Drop exact duplicates (the same declaration shipped twice).
421    let mut seen = HashSet::new();
422    all.retain(|e| seen.insert((e.path.clone(), e.sig.clone(), e.doc.len())));
423    all
424}
425
426fn rust_crate_name(dir: &Path) -> Option<String> {
427    let t = std::fs::read_to_string(dir.join("Cargo.toml")).ok()?;
428    let v: toml::Table = toml::from_str(&t).ok()?;
429    let lib = v.get("lib").and_then(|l| l.get("name")).and_then(|n| n.as_str());
430    let pkg = v.get("package").and_then(|p| p.get("name")).and_then(|n| n.as_str());
431    lib.or(pkg).map(|n| n.replace('-', "_"))
432}
433
434/// Docs sites include example files by reference (`{!.tmp_examples/x.md!}`,
435/// `{* ../../docs_src/a/tutorial001.py *}`, `--8<-- "x.py"`). Inline them so
436/// the section carries its code.
437#[derive(Default)]
438struct Includes {
439    /// (lowercase path without extension, absolute path)
440    files: Vec<(String, PathBuf)>,
441}
442
443impl Includes {
444    fn scan(dir: &Path) -> Includes {
445        let mut rels = Vec::new();
446        walk_all(dir, Path::new(""), 0, &mut rels);
447        let files = rels
448            .into_iter()
449            .filter(|r| EXAMPLE_EXTS.contains(&ext_of(r).as_str()))
450            .map(|r| {
451                let s = r.to_string_lossy().replace('\\', "/").to_ascii_lowercase();
452                let stem = s.rsplit_once('.').map_or(s.clone(), |x| x.0.to_string());
453                (stem, dir.join(&r))
454            })
455            .collect();
456        Includes { files }
457    }
458
459    fn find(&self, reference: &str) -> Option<&PathBuf> {
460        let r = reference.trim().trim_matches(['"', '\'']).to_ascii_lowercase();
461        let r = r.split_whitespace().next().unwrap_or("");
462        let r = r.rsplit_once('.').map_or(r, |x| x.0);
463        let segs: Vec<&str> = r
464            .split('/')
465            .filter(|s| !s.is_empty() && *s != ".." && *s != "." && *s != ".tmp_examples")
466            .collect();
467        for take in (1..=segs.len().min(3)).rev() {
468            let suffix = segs[segs.len() - take..].join("/");
469            let hits: Vec<&(String, PathBuf)> = self.files.iter().filter(|(k, _)| *k == suffix || k.ends_with(&format!("/{suffix}"))).collect();
470            if hits.len() == 1 {
471                return Some(&hits[0].1);
472            }
473        }
474        None
475    }
476
477    fn expand(&self, text: &str) -> String {
478        if !(text.contains("{!") || text.contains("{*") || text.contains("--8<--")) {
479            return text.to_string();
480        }
481        let mut out = String::with_capacity(text.len());
482        for line in text.lines() {
483            let t = line.trim();
484            let reference = t
485                .strip_prefix("{!")
486                .and_then(|x| x.strip_suffix("!}"))
487                .or_else(|| t.strip_prefix("{*").and_then(|x| x.strip_suffix("*}")))
488                .or_else(|| t.strip_prefix("--8<--"));
489            match reference
490                .and_then(|r| self.find(r))
491                .and_then(|p| std::fs::read_to_string(p).ok().map(|c| (p, c)))
492            {
493                Some((p, code)) => {
494                    let lang = p.extension().map(|e| e.to_string_lossy().to_string()).unwrap_or_default();
495                    let code: String = code.chars().take(3000).collect();
496                    out.push_str(&format!("```{lang}\n{}\n```\n", code.trim_end()));
497                }
498                None => {
499                    out.push_str(line);
500                    out.push('\n');
501                }
502            }
503        }
504        out
505    }
506}
507
508const EXAMPLE_EXTS: &[&str] = &["py", "ts", "tsx", "js", "jsx", "mjs", "rs", "go"];
509
510/// A docs example file as one code section titled by its file name.
511fn example(text: &str, rel: &str, out: &mut Vec<Entry>) {
512    if text.len() > 12_000 || text.trim().is_empty() {
513        return;
514    }
515    let name = rel.rsplit('/').next().unwrap_or(rel);
516    let lang = name.rsplit('.').next().unwrap_or("");
517    let title = name.rsplit_once('.').map_or(name, |x| x.0).replace(['_', '-'], " ");
518    out.push(Entry {
519        kind: Kind::Prose,
520        name: format!("example › {title}"),
521        path: format!("example › {title}"),
522        file: rel.to_string(),
523        line: 1,
524        sig: String::new(),
525        doc: format!("```{lang}\n{}\n```", text.trim_end()),
526        alias_of: None,
527        legacy: false,
528    });
529}
530
531/// Title for a page file without one: its route (`docs/installation/using-vite`).
532fn page_title(rel: &str) -> String {
533    let segs: Vec<&str> = rel.split('/').filter(|s| !(s.starts_with('(') && s.ends_with(')'))).collect();
534    let mut segs: Vec<&str> = segs.iter().rev().take(3).rev().copied().collect();
535    if let Some(last) = segs.last() {
536        if last.starts_with("page.") || last.starts_with("index.") {
537            segs.pop();
538        }
539    }
540    segs.join("/")
541        .trim_end_matches(".tsx")
542        .trim_end_matches(".jsx")
543        .trim_end_matches(".js")
544        .to_string()
545}
546
547/// Markdown from a docs page written as a JS/TSX component: `title:` values
548/// of listed steps become headings, `description:` and JSX text become
549/// paragraphs, `code:` strings become fenced code (language from `lang:`),
550/// and JSX `<h1>`..`<h6>` stay headings. Lines keep their source position
551/// where they can. Returns (markdown, page title from `metadata.title`).
552fn page_markdown(text: &str) -> Option<(String, Option<String>)> {
553    if text.len() > 200_000 {
554        return None;
555    }
556    parse_with("tsx", text, |root, src| {
557        let mut p = PageOut::default();
558        page_walk(root, src, &mut p, false);
559        p.flush();
560        let mut md = String::new();
561        let mut n = 0u32;
562        for (line, t) in &p.out {
563            while n + 1 < *line {
564                md.push('\n');
565                n += 1;
566            }
567            md.push_str(t);
568            md.push('\n');
569            n += 1 + t.matches('\n').count() as u32;
570        }
571        (md, p.title)
572    })
573    .filter(|(md, _)| !md.trim().is_empty())
574}
575
576#[derive(Default)]
577struct PageOut {
578    out: Vec<(u32, String)>,
579    para: String,
580    para_line: u32,
581    title: Option<String>,
582}
583
584impl PageOut {
585    fn text(&mut self, line: u32, t: &str) {
586        if self.para.trim().is_empty() {
587            self.para_line = line;
588        }
589        self.para.push_str(t);
590    }
591    fn flush(&mut self) {
592        let t = compact(&self.para, usize::MAX);
593        if !t.is_empty() {
594            self.out.push((self.para_line, t));
595        }
596        self.para.clear();
597    }
598    fn block(&mut self, line: u32, t: String) {
599        self.flush();
600        self.out.push((line, t));
601    }
602}
603
604fn line_of(n: Node) -> u32 {
605    n.start_position().row as u32 + 1
606}
607
608/// The value of a string, template string or tagged template (`dedent`...``).
609fn js_string(n: Node, src: &[u8]) -> Option<String> {
610    match n.kind() {
611        "string" => {
612            let raw = txt(n, src);
613            let inner = raw.get(1..raw.len().saturating_sub(1)).unwrap_or("");
614            Some(inner.replace("\\n", "\n").replace("\\'", "'").replace("\\\"", "\"").replace("\\\\", "\\"))
615        }
616        "template_string" => {
617            let raw = txt(n, src);
618            let inner = raw.get(1..raw.len().saturating_sub(1)).unwrap_or("");
619            Some(dedent(&inner.trim_matches('\n').lines().collect::<Vec<_>>()))
620        }
621        "call_expression" => n
622            .child_by_field_name("arguments")
623            .filter(|a| a.kind() == "template_string")
624            .and_then(|a| js_string(a, src)),
625        "parenthesized_expression" => n.named_child(0).and_then(|c| js_string(c, src)),
626        _ => None,
627    }
628}
629
630fn jsx_name(n: Node, src: &[u8]) -> String {
631    let open = if n.kind() == "jsx_element" {
632        n.child_by_field_name("open_tag")
633    } else {
634        Some(n)
635    };
636    open.and_then(|o| o.child_by_field_name("name"))
637        .map(|x| txt(x, src).to_string())
638        .unwrap_or_default()
639}
640
641/// The visible text inside a JSX element.
642fn jsx_text(n: Node, src: &[u8], out: &mut String) {
643    match n.kind() {
644        "jsx_text" => out.push_str(txt(n, src)),
645        "jsx_expression" => {
646            if let Some(v) = n.named_child(0).and_then(|c| js_string(c, src)) {
647                out.push_str(&v);
648            }
649        }
650        "jsx_attribute" | "jsx_opening_element" | "jsx_closing_element" => {}
651        _ => {
652            let mut c = n.walk();
653            for ch in n.named_children(&mut c) {
654                jsx_text(ch, src, out);
655            }
656        }
657    }
658}
659
660fn page_walk(n: Node, src: &[u8], p: &mut PageOut, in_array: bool) {
661    match n.kind() {
662        "import_statement" | "comment" | "jsx_attribute" => {}
663        "pair" => {
664            let key = n
665                .child_by_field_name("key")
666                .map(|k| txt(k, src).trim_matches(['"', '\'']).to_string())
667                .unwrap_or_default();
668            let Some(value) = n.child_by_field_name("value") else { return };
669            match key.as_str() {
670                "title" | "heading" => {
671                    if let Some(t) = js_string(value, src) {
672                        if !in_array && p.title.is_none() {
673                            p.title = Some(t);
674                        } else if in_array {
675                            p.block(line_of(n), format!("## {}", compact(&t, 200)));
676                        }
677                    }
678                }
679                "description" => {
680                    if let Some(t) = js_string(value, src) {
681                        p.block(line_of(value), compact(&t, usize::MAX));
682                    }
683                }
684                "code" => match js_string(value, src) {
685                    Some(code) => {
686                        let lang = n
687                            .parent()
688                            .and_then(|obj| {
689                                let mut c = obj.walk();
690                                let found = obj.named_children(&mut c).find(|x| {
691                                    x.kind() == "pair" && x.child_by_field_name("key").is_some_and(|k| txt(k, src).trim_matches(['"', '\'']) == "lang")
692                                });
693                                found
694                            })
695                            .and_then(|x| x.child_by_field_name("value"))
696                            .and_then(|v| js_string(v, src))
697                            .unwrap_or_default();
698                        p.block(line_of(value), format!("```{lang}\n{}\n```", code.trim_end()));
699                    }
700                    None => page_walk(value, src, p, in_array),
701                },
702                "name" | "lang" | "className" | "href" | "id" | "url" | "src" | "image" | "images" | "icon" | "openGraph" | "twitter" | "alternates" => {}
703                _ => page_walk(value, src, p, in_array),
704            }
705        }
706        "jsx_element" | "jsx_self_closing_element" => {
707            let name = jsx_name(n, src);
708            let b = name.as_bytes();
709            if b.len() == 2 && b[0] == b'h' && (b'1'..=b'6').contains(&b[1]) {
710                let mut t = String::new();
711                jsx_text(n, src, &mut t);
712                let t = compact(&t, 200);
713                if !t.is_empty() {
714                    p.block(line_of(n), format!("{} {t}", "#".repeat((b[1] - b'0') as usize)));
715                }
716            } else if name == "code" {
717                let mut t = String::new();
718                jsx_text(n, src, &mut t);
719                p.text(line_of(n), &format!("`{}`", t.trim()));
720            } else {
721                let block = matches!(name.as_str(), "p" | "li" | "div" | "section" | "ul" | "ol" | "pre" | "table" | "tr");
722                if block {
723                    p.flush();
724                }
725                let mut c = n.walk();
726                for ch in n.named_children(&mut c) {
727                    page_walk(ch, src, p, in_array);
728                }
729                if block {
730                    p.flush();
731                }
732            }
733        }
734        "jsx_text" => p.text(line_of(n), txt(n, src)),
735        "jsx_expression" => match n.named_child(0) {
736            Some(c) if matches!(c.kind(), "string" | "template_string") => {
737                let v = js_string(c, src).unwrap_or_default();
738                p.text(line_of(n), &v);
739            }
740            Some(c) => page_walk(c, src, p, in_array),
741            None => {}
742        },
743        _ => {
744            let arr = in_array || n.kind() == "array";
745            let mut c = n.walk();
746            for ch in n.named_children(&mut c) {
747                page_walk(ch, src, p, arr);
748            }
749        }
750    }
751}
752
753fn prose(text: &str, rel: &str, line_off: u32, out: &mut Vec<Entry>) {
754    prose_titled(text, rel, line_off, None, out)
755}
756
757fn prose_titled(text: &str, rel: &str, line_off: u32, given: Option<String>, out: &mut Vec<Entry>) {
758    let (text, fm_title) = markdown::clean_mdx(text, rel.ends_with(".mdx"));
759    // A page without a title is named after its file: `model_config.md` -> `model config`.
760    let file_name = rel.rsplit('/').next().unwrap_or(rel);
761    let file_title = if rel.starts_with("upstream:") && !is_doc_name(file_name) {
762        file_name
763            .rsplit_once('.')
764            .map_or(file_name, |x| x.0)
765            .trim_start_matches(|c: char| c.is_ascii_digit() || c == '-')
766            .replace(['_', '-'], " ")
767    } else {
768        file_name.to_string()
769    };
770    let file_title = file_title.as_str();
771    let title = if let Some(t) = given {
772        t
773    } else if rel.contains("package metadata") {
774        "README".to_string()
775    } else if let Some(t) = fm_title {
776        t
777    } else {
778        file_title.to_string()
779    };
780    for s in markdown::split(&text, &title) {
781        out.push(Entry {
782            kind: Kind::Prose,
783            name: s.heading.clone(),
784            path: s.heading,
785            file: rel.to_string(),
786            line: s.line + line_off,
787            sig: String::new(),
788            doc: s.text,
789            alias_of: None,
790            legacy: false,
791        });
792    }
793}
794
795// ---------------------------------------------------------------- helpers
796
797thread_local! {
798    static PARSERS: RefCell<Vec<(&'static str, Parser)>> = const { RefCell::new(Vec::new()) };
799}
800
801/// Parse with a pooled parser. The parser is taken out of the pool while in
802/// use, so `f` may parse again (Rust macro bodies do).
803fn parse_with<R>(lang: &'static str, text: &str, f: impl FnOnce(Node, &[u8]) -> R) -> Option<R> {
804    let pooled = PARSERS.with(|ps| {
805        let mut ps = ps.borrow_mut();
806        ps.iter().position(|(l, _)| *l == lang).map(|i| ps.swap_remove(i).1)
807    });
808    let mut parser = match pooled {
809        Some(p) => p,
810        None => {
811            let mut p = Parser::new();
812            let language: tree_sitter::Language = match lang {
813                "ts" => tree_sitter_typescript::LANGUAGE_TYPESCRIPT.into(),
814                "tsx" => tree_sitter_typescript::LANGUAGE_TSX.into(),
815                "py" => tree_sitter_python::LANGUAGE.into(),
816                "rs" => tree_sitter_rust::LANGUAGE.into(),
817                _ => tree_sitter_go::LANGUAGE.into(),
818            };
819            p.set_language(&language).ok()?;
820            p
821        }
822    };
823    let tree = parser.parse(text, None);
824    PARSERS.with(|ps| ps.borrow_mut().push((lang, parser)));
825    let tree = tree?;
826    Some(f(tree.root_node(), text.as_bytes()))
827}
828
829fn txt<'a>(n: Node, src: &'a [u8]) -> &'a str {
830    n.utf8_text(src).unwrap_or("")
831}
832
833fn compact(s: &str, cap: usize) -> String {
834    let mut out = String::with_capacity(s.len().min(cap.saturating_add(8)));
835    let mut space = false;
836    for c in s.chars() {
837        if c.is_whitespace() {
838            space = true;
839            continue;
840        }
841        if space && !out.is_empty() {
842            out.push(' ');
843        }
844        space = false;
845        out.push(c);
846        if out.len() >= cap {
847            out.push_str(" …");
848            break;
849        }
850    }
851    out.replace("( ", "(").replace(", )", ")").replace(",)", ")").replace(" )", ")")
852}
853
854/// Drop `# comments` from Python source lines (outside string literals).
855fn strip_py_comments(s: &str) -> String {
856    let mut out = String::with_capacity(s.len());
857    for line in s.lines() {
858        let mut quote: Option<char> = None;
859        let mut cut = line.len();
860        for (i, c) in line.char_indices() {
861            match (quote, c) {
862                (None, '#') => {
863                    cut = i;
864                    break;
865                }
866                (None, '"' | '\'') => quote = Some(c),
867                (Some(q), c) if c == q => quote = None,
868                _ => {}
869            }
870        }
871        out.push_str(&line[..cut]);
872        out.push('\n');
873    }
874    out
875}
876
877/// Source text from `n` up to (not including) its body child.
878fn head(n: Node, src: &[u8], body_field: &str, cap: usize) -> String {
879    let end = n.child_by_field_name(body_field).map_or(n.end_byte(), |b| b.start_byte());
880    let s = std::str::from_utf8(&src[n.start_byte()..end]).unwrap_or("");
881    compact(s.trim_end().trim_end_matches(['{', ':', '=']).trim_end(), cap)
882}
883
884fn cap_doc(s: String) -> String {
885    const CAP: usize = 4000;
886    if s.len() <= CAP {
887        return s;
888    }
889    let mut i = CAP;
890    while !s.is_char_boundary(i) {
891        i -= 1;
892    }
893    format!("{} …", &s[..i])
894}
895
896fn dedent(lines: &[&str]) -> String {
897    let indent = lines
898        .iter()
899        .filter(|l| !l.trim().is_empty())
900        .map(|l| l.len() - l.trim_start().len())
901        .min()
902        .unwrap_or(0);
903    let v: Vec<&str> = lines.iter().map(|l| if l.len() >= indent { &l[indent..] } else { l.trim_start() }).collect();
904    v.join("\n").trim().to_string()
905}
906
907fn clean_block_comment(s: &str) -> String {
908    let s = s
909        .trim_start_matches("/**")
910        .trim_start_matches("/*!")
911        .trim_start_matches("/*")
912        .trim_end_matches("*/");
913    let lines: Vec<&str> = s
914        .lines()
915        .map(|l| {
916            let t = l.trim_start();
917            match t.strip_prefix('*') {
918                Some(r) => r.strip_prefix(' ').unwrap_or(r),
919                None => l,
920            }
921        })
922        .collect();
923    cap_doc(dedent(&lines))
924}
925
926fn entry(kind: Kind, name: &str, path: String, file: &str, line: u32, sig: String, doc: String) -> Entry {
927    Entry {
928        kind,
929        name: name.to_string(),
930        path,
931        file: file.to_string(),
932        line,
933        sig,
934        doc,
935        alias_of: None,
936        legacy: false,
937    }
938}
939
940// ---------------------------------------------------------------- TypeScript / JavaScript
941
942fn ts_file(text: &str, rel: &str, out: &mut Vec<Entry>) {
943    let lang = if rel.ends_with(".tsx") || rel.ends_with(".jsx") { "tsx" } else { "ts" };
944    let is_js = rel.ends_with(".js") || rel.ends_with(".mjs") || rel.ends_with(".cjs");
945    parse_with(lang, text, |root, src| {
946        let mut w = TsWalker {
947            src,
948            file: rel,
949            out,
950            js: is_js,
951        };
952        w.block(root, "");
953    });
954}
955
956struct TsWalker<'a> {
957    src: &'a [u8],
958    file: &'a str,
959    out: &'a mut Vec<Entry>,
960    js: bool,
961}
962
963impl TsWalker<'_> {
964    fn doc(&self, n: Node) -> String {
965        let mut p = n.prev_sibling();
966        while let Some(c) = p {
967            if c.kind() == "comment" {
968                let t = txt(c, self.src);
969                if t.starts_with("/**") && c.end_position().row + 2 >= n.start_position().row {
970                    return clean_block_comment(t);
971                }
972                if t.starts_with("//") {
973                    p = c.prev_sibling();
974                    continue;
975                }
976            }
977            break;
978        }
979        String::new()
980    }
981
982    fn line(&self, n: Node) -> u32 {
983        n.start_position().row as u32 + 1
984    }
985
986    fn block(&mut self, node: Node, prefix: &str) {
987        let mut cur = node.walk();
988        let children: Vec<Node> = node.named_children(&mut cur).collect();
989        for c in children {
990            match c.kind() {
991                "export_statement" => {
992                    let doc = self.doc(c);
993                    if let Some(d) = c.child_by_field_name("declaration") {
994                        self.decl(d, prefix, doc, c);
995                    } else {
996                        self.exports(c, prefix);
997                    }
998                }
999                "ambient_declaration" => {
1000                    let doc = self.doc(c);
1001                    let mut cc = c.walk();
1002                    let inner: Vec<Node> = c.named_children(&mut cc).collect();
1003                    for d in inner {
1004                        self.decl(d, prefix, doc.clone(), c);
1005                    }
1006                }
1007                "expression_statement" => {
1008                    let mut cc = c.walk();
1009                    let inner: Vec<Node> = c.named_children(&mut cc).collect();
1010                    for d in inner {
1011                        if matches!(d.kind(), "internal_module" | "module") {
1012                            let doc = self.doc(c);
1013                            self.decl(d, prefix, doc, c);
1014                        }
1015                    }
1016                }
1017                _ => {
1018                    let doc = self.doc(c);
1019                    // In JS only documented top-level declarations are API.
1020                    if !self.js || !doc.is_empty() || !prefix.is_empty() {
1021                        self.decl(c, prefix, doc, c);
1022                    }
1023                }
1024            }
1025        }
1026    }
1027
1028    fn exports(&mut self, n: Node, prefix: &str) {
1029        let mut cur = n.walk();
1030        let children: Vec<Node> = n.named_children(&mut cur).collect();
1031        for c in children {
1032            match c.kind() {
1033                "export_clause" => {
1034                    let mut cc = c.walk();
1035                    let specs: Vec<Node> = c.named_children(&mut cc).collect();
1036                    for s in specs {
1037                        let (Some(name), alias) = (s.child_by_field_name("name"), s.child_by_field_name("alias")) else {
1038                            continue;
1039                        };
1040                        let Some(alias) = alias else { continue };
1041                        let (from, to) = (txt(name, self.src), txt(alias, self.src));
1042                        if from == to || to == "default" {
1043                            continue;
1044                        }
1045                        let mut e = entry(
1046                            Kind::Alias,
1047                            to,
1048                            format!("{prefix}{to}"),
1049                            self.file,
1050                            self.line(s),
1051                            compact(txt(n, self.src), 200),
1052                            String::new(),
1053                        );
1054                        e.alias_of = Some(from.to_string());
1055                        self.out.push(e);
1056                    }
1057                }
1058                "namespace_export" => {
1059                    if let Some(id) = c.named_child(0) {
1060                        let name = txt(id, self.src);
1061                        self.out.push(entry(
1062                            Kind::Module,
1063                            name,
1064                            format!("{prefix}{name}"),
1065                            self.file,
1066                            self.line(n),
1067                            compact(txt(n, self.src), 200),
1068                            self.doc(n),
1069                        ));
1070                    }
1071                }
1072                _ => {}
1073            }
1074        }
1075    }
1076
1077    fn decl(&mut self, d: Node, prefix: &str, doc: String, outer: Node) {
1078        let src = self.src;
1079        let name_of = |n: Node| n.child_by_field_name("name").map(|x| txt(x, src).to_string());
1080        let line = self.line(outer);
1081        match d.kind() {
1082            "ambient_declaration" => {
1083                let mut cur = d.walk();
1084                let inner: Vec<Node> = d.named_children(&mut cur).collect();
1085                for x in inner {
1086                    self.decl(x, prefix, doc.clone(), outer);
1087                }
1088            }
1089            "function_signature" | "function_declaration" | "generator_function_declaration" => {
1090                let Some(name) = name_of(d) else { return };
1091                let sig = head(outer, src, "__none__", 600).trim_end_matches(';').to_string();
1092                let sig = if d.child_by_field_name("body").is_some() {
1093                    head(d, src, "body", 600)
1094                } else {
1095                    sig
1096                };
1097                self.out
1098                    .push(entry(Kind::Function, &name, format!("{prefix}{name}"), self.file, line, sig, doc));
1099            }
1100            "class_declaration" | "abstract_class_declaration" | "class" => {
1101                let Some(name) = name_of(d) else { return };
1102                self.out.push(entry(
1103                    Kind::Class,
1104                    &name,
1105                    format!("{prefix}{name}"),
1106                    self.file,
1107                    line,
1108                    head(d, src, "body", 400),
1109                    doc,
1110                ));
1111                if let Some(body) = d.child_by_field_name("body") {
1112                    self.members(body, &format!("{prefix}{name}."));
1113                }
1114            }
1115            "interface_declaration" => {
1116                let Some(name) = name_of(d) else { return };
1117                self.out.push(entry(
1118                    Kind::Interface,
1119                    &name,
1120                    format!("{prefix}{name}"),
1121                    self.file,
1122                    line,
1123                    head(d, src, "body", 400),
1124                    doc,
1125                ));
1126                if let Some(body) = d.child_by_field_name("body") {
1127                    self.members(body, &format!("{prefix}{name}."));
1128                }
1129            }
1130            "type_alias_declaration" => {
1131                let Some(name) = name_of(d) else { return };
1132                self.out.push(entry(
1133                    Kind::Type,
1134                    &name,
1135                    format!("{prefix}{name}"),
1136                    self.file,
1137                    line,
1138                    compact(txt(d, src), 700),
1139                    doc,
1140                ));
1141            }
1142            "enum_declaration" => {
1143                let Some(name) = name_of(d) else { return };
1144                self.out.push(entry(
1145                    Kind::Enum,
1146                    &name,
1147                    format!("{prefix}{name}"),
1148                    self.file,
1149                    line,
1150                    compact(txt(d, src), 500),
1151                    doc,
1152                ));
1153            }
1154            "lexical_declaration" | "variable_declaration" => {
1155                let kw = txt(d, src).split_whitespace().next().unwrap_or("const").to_string();
1156                let mut cur = d.walk();
1157                let decls: Vec<Node> = d.named_children(&mut cur).filter(|n| n.kind() == "variable_declarator").collect();
1158                for v in decls {
1159                    let Some(nn) = v.child_by_field_name("name") else { continue };
1160                    if nn.kind() != "identifier" {
1161                        continue;
1162                    }
1163                    let name = txt(nn, src).to_string();
1164                    let value = v.child_by_field_name("value");
1165                    let is_fn = value.is_some_and(|x| matches!(x.kind(), "arrow_function" | "function_expression" | "function"))
1166                        || v.child_by_field_name("type").is_some_and(|t| txt(t, src).contains("=>"));
1167                    let sig = match value {
1168                        Some(val) if matches!(val.kind(), "arrow_function" | "function_expression" | "function") => {
1169                            let body_start = val.child_by_field_name("body").map_or(val.end_byte(), |b| b.start_byte());
1170                            let s = std::str::from_utf8(&src[v.start_byte()..body_start]).unwrap_or("");
1171                            compact(&format!("{kw} {}", s.trim_end().trim_end_matches("=>").trim_end()), 600)
1172                        }
1173                        Some(val) if val.end_byte() - val.start_byte() > 300 => {
1174                            let s = std::str::from_utf8(&src[v.start_byte()..val.start_byte()]).unwrap_or("");
1175                            compact(&format!("{kw} {s} …"), 600)
1176                        }
1177                        _ => compact(&format!("{kw} {}", txt(v, src)), 600),
1178                    };
1179                    let kind = if is_fn { Kind::Function } else { Kind::Const };
1180                    self.out.push(entry(kind, &name, format!("{prefix}{name}"), self.file, line, sig, doc.clone()));
1181                }
1182            }
1183            "internal_module" | "module" => {
1184                let Some(nn) = d.child_by_field_name("name") else { return };
1185                let raw = txt(nn, src);
1186                let quoted = raw.starts_with('"') || raw.starts_with('\'');
1187                let name = raw.trim_matches(['"', '\'']).to_string();
1188                let new_prefix = if quoted { prefix.to_string() } else { format!("{prefix}{name}.") };
1189                if !quoted {
1190                    self.out.push(entry(
1191                        Kind::Module,
1192                        &name,
1193                        format!("{prefix}{name}"),
1194                        self.file,
1195                        line,
1196                        head(d, src, "body", 200),
1197                        doc,
1198                    ));
1199                }
1200                if let Some(body) = d.child_by_field_name("body") {
1201                    self.block(body, &new_prefix);
1202                }
1203            }
1204            _ => {}
1205        }
1206    }
1207
1208    fn members(&mut self, body: Node, prefix: &str) {
1209        let src = self.src;
1210        let mut cur = body.walk();
1211        let children: Vec<Node> = body.named_children(&mut cur).collect();
1212        for m in children {
1213            let kind = match m.kind() {
1214                "method_definition" | "method_signature" | "abstract_method_signature" => Kind::Method,
1215                "public_field_definition" | "property_signature" => Kind::Field,
1216                _ => continue,
1217            };
1218            let t = txt(m, src);
1219            if t.starts_with("private ") || t.starts_with("protected ") || t.contains(" private ") && kind == Kind::Field {
1220                continue;
1221            }
1222            let Some(nn) = m.child_by_field_name("name") else { continue };
1223            let name = txt(nn, src).trim_matches(['"', '\'']).to_string();
1224            if name.starts_with('#') || name.starts_with('_') || name.starts_with('[') {
1225                continue;
1226            }
1227            let doc = self.doc(m);
1228            let sig = if m.child_by_field_name("body").is_some() {
1229                head(m, src, "body", 500)
1230            } else {
1231                compact(t.trim_end_matches(';'), 500)
1232            };
1233            self.out.push(entry(
1234                kind,
1235                &name,
1236                format!("{prefix}{name}"),
1237                self.file,
1238                m.start_position().row as u32 + 1,
1239                sig,
1240                doc,
1241            ));
1242        }
1243    }
1244}
1245
1246// ---------------------------------------------------------------- Python
1247
1248fn py_module(rel: &str) -> String {
1249    let r = rel.trim_end_matches(".pyi").trim_end_matches(".py");
1250    let r = r.trim_end_matches("/__init__");
1251    r.replace('/', ".")
1252}
1253
1254fn py_file(text: &str, rel: &str, out: &mut Vec<Entry>) {
1255    let module = py_module(rel);
1256    parse_with("py", text, |root, src| {
1257        if let Some(doc) = py_docstring(root, src) {
1258            for s in markdown::split(&doc, &module) {
1259                out.push(Entry {
1260                    kind: Kind::Prose,
1261                    name: s.heading.clone(),
1262                    path: s.heading,
1263                    file: rel.to_string(),
1264                    line: s.line,
1265                    sig: String::new(),
1266                    doc: s.text,
1267                    alias_of: None,
1268                    legacy: false,
1269                });
1270            }
1271        }
1272        py_block(root, src, &module, false, rel, out);
1273    });
1274}
1275
1276fn py_docstring(block: Node, src: &[u8]) -> Option<String> {
1277    let first = block.named_child(0)?;
1278    let first = if first.kind() == "comment" { first.next_named_sibling()? } else { first };
1279    if first.kind() != "expression_statement" {
1280        return None;
1281    }
1282    let s = first.named_child(0)?;
1283    if s.kind() != "string" {
1284        return None;
1285    }
1286    let raw = txt(s, src);
1287    let raw = raw.trim_start_matches(['r', 'R', 'u', 'U', 'b', 'B']);
1288    let inner = raw
1289        .strip_prefix("\"\"\"")
1290        .and_then(|x| x.strip_suffix("\"\"\""))
1291        .or_else(|| raw.strip_prefix("'''").and_then(|x| x.strip_suffix("'''")))
1292        .or_else(|| raw.strip_prefix('"').and_then(|x| x.strip_suffix('"')))
1293        .or_else(|| raw.strip_prefix('\'').and_then(|x| x.strip_suffix('\'')))?;
1294    let lines: Vec<&str> = inner.lines().collect();
1295    if lines.is_empty() {
1296        return None;
1297    }
1298    // inspect.cleandoc: first line as is, the rest dedented.
1299    let first_line = lines[0].trim();
1300    let rest = dedent(&lines[1..]);
1301    let gap = if lines.get(1).is_some_and(|l| l.trim().is_empty()) { "\n\n" } else { "\n" };
1302    let doc = if rest.is_empty() {
1303        first_line.to_string()
1304    } else if first_line.is_empty() {
1305        rest
1306    } else {
1307        format!("{first_line}{gap}{rest}")
1308    };
1309    Some(cap_doc(doc))
1310}
1311
1312fn py_block(block: Node, src: &[u8], prefix: &str, in_class: bool, rel: &str, out: &mut Vec<Entry>) {
1313    let mut cur = block.walk();
1314    let children: Vec<Node> = block.named_children(&mut cur).collect();
1315    for c in children {
1316        let (def, decorators) = if c.kind() == "decorated_definition" {
1317            let mut dc = c.walk();
1318            let decs: Vec<String> = c
1319                .named_children(&mut dc)
1320                .filter(|n| n.kind() == "decorator")
1321                .map(|n| compact(txt(n, src), 80))
1322                .collect();
1323            match c.child_by_field_name("definition") {
1324                Some(d) => (d, decs),
1325                None => continue,
1326            }
1327        } else {
1328            (c, Vec::new())
1329        };
1330        let Some(nn) = def.child_by_field_name("name") else { continue };
1331        let name = txt(nn, src);
1332        let dunder_ok = matches!(name, "__init__" | "__call__");
1333        if name.starts_with('_') && !dunder_ok {
1334            continue;
1335        }
1336        let line = c.start_position().row as u32 + 1;
1337        let mut sig = decorators.join(" ");
1338        match def.kind() {
1339            "function_definition" => {
1340                if !sig.is_empty() {
1341                    sig.push(' ');
1342                }
1343                let end = def.child_by_field_name("body").map_or(def.end_byte(), |b| b.start_byte());
1344                let raw = strip_py_comments(std::str::from_utf8(&src[def.start_byte()..end]).unwrap_or(""));
1345                sig.push_str(&compact(raw.trim_end().trim_end_matches(':').trim_end(), 700));
1346                let doc = def.child_by_field_name("body").and_then(|b| py_docstring(b, src)).unwrap_or_default();
1347                let kind = if in_class { Kind::Method } else { Kind::Function };
1348                out.push(entry(kind, name, format!("{prefix}.{name}"), rel, line, sig, doc));
1349            }
1350            "class_definition" => {
1351                if !sig.is_empty() {
1352                    sig.push(' ');
1353                }
1354                sig.push_str(&head(def, src, "body", 400));
1355                let body = def.child_by_field_name("body");
1356                let doc = body.and_then(|b| py_docstring(b, src)).unwrap_or_default();
1357                let path = format!("{prefix}.{name}");
1358                out.push(entry(Kind::Class, name, path.clone(), rel, line, sig, doc));
1359                if let Some(b) = body {
1360                    py_block(b, src, &path, true, rel, out);
1361                }
1362            }
1363            _ => {}
1364        }
1365    }
1366}
1367
1368// ---------------------------------------------------------------- Rust
1369
1370fn rs_module(rel: &str, crate_name: &str) -> String {
1371    let r = rel.strip_prefix("src/").unwrap_or(rel).trim_end_matches(".rs");
1372    let mut segs: Vec<&str> = r.split('/').collect();
1373    if matches!(segs.last(), Some(&"mod") | Some(&"lib") | Some(&"main")) {
1374        segs.pop();
1375    }
1376    let mut p = crate_name.to_string();
1377    for s in segs {
1378        if !s.is_empty() {
1379            p.push_str("::");
1380            p.push_str(s);
1381        }
1382    }
1383    p
1384}
1385
1386fn rs_file(text: &str, rel: &str, crate_name: &str, out: &mut Vec<Entry>) {
1387    let module = rs_module(rel, crate_name);
1388    rs_parse(text, rel, &module, 0, 0, None, out);
1389}
1390
1391/// Docs and `#[macro_export]` a wrapper macro applies to the items it wraps
1392/// (tokio's `doc! { macro_rules! select { .. } }` pattern).
1393type Inherit = Option<(String, bool)>;
1394
1395fn rs_parse(text: &str, rel: &str, module: &str, line_off: u32, depth: u32, inherit: Inherit, out: &mut Vec<Entry>) {
1396    parse_with("rs", text, |root, src| {
1397        let mut inner_doc = Vec::new();
1398        let mut w = RsWalker {
1399            src,
1400            file: rel,
1401            line_off,
1402            depth,
1403            out: Vec::new(),
1404            wrappers: HashMap::new(),
1405            inherit,
1406        };
1407        w.block(root, module, None, &mut inner_doc);
1408        if !inner_doc.is_empty() {
1409            let doc = inner_doc.join("\n");
1410            for s in markdown::split(&doc, module) {
1411                out.push(Entry {
1412                    kind: Kind::Prose,
1413                    name: s.heading.clone(),
1414                    path: s.heading,
1415                    file: rel.to_string(),
1416                    line: s.line + line_off,
1417                    sig: String::new(),
1418                    doc: s.text,
1419                    alias_of: None,
1420                    legacy: false,
1421                });
1422            }
1423        }
1424        out.append(&mut w.out);
1425    });
1426}
1427
1428struct RsWalker<'a> {
1429    src: &'a [u8],
1430    file: &'a str,
1431    line_off: u32,
1432    depth: u32,
1433    out: Vec<Entry>,
1434    /// Local `macro_rules!` that wrap an `$item` with docs: name -> (doc, exported).
1435    wrappers: HashMap<String, (String, bool)>,
1436    inherit: Inherit,
1437}
1438
1439/// A local macro like `macro_rules! doc { ($x:item) => { /// docs  #[macro_export] $x } }`.
1440fn doc_wrapper(text: &str) -> Option<(String, bool)> {
1441    if !text.contains(":item") {
1442        return None;
1443    }
1444    let doc: Vec<&str> = text
1445        .lines()
1446        .filter_map(|l| strip_line_doc(l.trim()).filter(|(_, inner)| !inner).map(|(d, _)| d))
1447        .collect();
1448    if doc.is_empty() {
1449        return None;
1450    }
1451    Some((cap_doc(doc.join("\n").trim().to_string()), text.contains("macro_export")))
1452}
1453
1454fn is_pub(n: Node, src: &[u8]) -> bool {
1455    let mut cur = n.walk();
1456    let r = n
1457        .named_children(&mut cur)
1458        .any(|c| c.kind() == "visibility_modifier" && txt(c, src).trim() == "pub");
1459    r
1460}
1461
1462fn strip_line_doc(t: &str) -> Option<(&str, bool)> {
1463    if let Some(r) = t.strip_prefix("///") {
1464        if r.starts_with('/') {
1465            return None;
1466        }
1467        return Some((r.strip_prefix(' ').unwrap_or(r).trim_end(), false));
1468    }
1469    if let Some(r) = t.strip_prefix("//!") {
1470        return Some((r.strip_prefix(' ').unwrap_or(r).trim_end(), true));
1471    }
1472    None
1473}
1474
1475impl RsWalker<'_> {
1476    fn line(&self, n: Node) -> u32 {
1477        n.start_position().row as u32 + 1 + self.line_off
1478    }
1479
1480    /// `owner`: impl/trait type whose members we are in (`Some((name, is_trait))`).
1481    fn block(&mut self, node: Node, module: &str, owner: Option<(&str, bool)>, inner_doc: &mut Vec<String>) {
1482        let src = self.src;
1483        let mut cur = node.walk();
1484        let children: Vec<Node> = node.named_children(&mut cur).collect();
1485        let mut pending: Vec<String> = Vec::new();
1486        let mut hidden = false;
1487        let mut exported_macro = false;
1488        let mut first_line: Option<u32> = None;
1489        for c in children {
1490            let k = c.kind();
1491            match k {
1492                "line_comment" => {
1493                    let t = txt(c, src);
1494                    if let Some((d, inner)) = strip_line_doc(t) {
1495                        if inner {
1496                            inner_doc.push(d.to_string());
1497                        } else {
1498                            if pending.is_empty() {
1499                                first_line = Some(self.line(c));
1500                            }
1501                            pending.push(d.to_string());
1502                        }
1503                    }
1504                    continue;
1505                }
1506                "block_comment" => {
1507                    let t = txt(c, src);
1508                    if t.starts_with("/*!") {
1509                        inner_doc.push(clean_block_comment(t));
1510                    } else if t.starts_with("/**") && !t.starts_with("/***") {
1511                        pending.push(clean_block_comment(t));
1512                    }
1513                    continue;
1514                }
1515                "attribute_item" => {
1516                    let t = txt(c, src);
1517                    if t.contains("doc(hidden)") {
1518                        hidden = true;
1519                    }
1520                    if t.contains("macro_export") {
1521                        exported_macro = true;
1522                    }
1523                    // `#[doc = "..."]`
1524                    if let Some(rest) = t.strip_prefix("#[doc = \"") {
1525                        pending.push(rest.trim_end_matches("\"]").replace("\\n", "\n"));
1526                    }
1527                    continue;
1528                }
1529                "inner_attribute_item" => continue,
1530                _ => {}
1531            }
1532            let mut doc = cap_doc(pending.join("\n").trim().to_string());
1533            if doc.is_empty() {
1534                if let Some((d, _)) = &self.inherit {
1535                    doc = d.clone();
1536                }
1537            }
1538            let _ = first_line.take();
1539            pending.clear();
1540            let was_hidden = std::mem::take(&mut hidden);
1541            let was_macro_export = std::mem::take(&mut exported_macro) || self.inherit.as_ref().is_some_and(|i| i.1);
1542            if was_hidden {
1543                continue;
1544            }
1545            let line = self.line(c);
1546            let sep = "::";
1547            let in_trait = owner.is_some_and(|o| o.1);
1548            let public = in_trait || is_pub(c, src);
1549            let name = c.child_by_field_name("name").map(|n| txt(n, src).to_string());
1550            let qual = |name: &str| match owner {
1551                Some((o, _)) => format!("{module}{sep}{o}{sep}{name}"),
1552                None => format!("{module}{sep}{name}"),
1553            };
1554            match k {
1555                "function_item" | "function_signature_item" => {
1556                    let (Some(name), true) = (name, public) else { continue };
1557                    let kind = if owner.is_some() { Kind::Method } else { Kind::Function };
1558                    self.out.push(entry(
1559                        kind,
1560                        &name,
1561                        qual(&name),
1562                        self.file,
1563                        line,
1564                        head(c, src, "body", 600).trim_end_matches(';').to_string(),
1565                        doc,
1566                    ));
1567                }
1568                "struct_item" | "enum_item" | "union_item" => {
1569                    let (Some(name), true) = (name, public) else { continue };
1570                    let kind = if k == "enum_item" { Kind::Enum } else { Kind::Struct };
1571                    let full = txt(c, src);
1572                    let sig = if full.len() <= 900 {
1573                        compact(full, 900)
1574                    } else {
1575                        format!("{} {{ … }}", head(c, src, "body", 400))
1576                    };
1577                    self.out.push(entry(kind, &name, qual(&name), self.file, line, sig, doc));
1578                }
1579                "trait_item" => {
1580                    let (Some(name), true) = (name, public) else { continue };
1581                    self.out
1582                        .push(entry(Kind::Trait, &name, qual(&name), self.file, line, head(c, src, "body", 400), doc));
1583                    if let Some(b) = c.child_by_field_name("body") {
1584                        let mut ignore = Vec::new();
1585                        self.block(b, module, Some((&name, true)), &mut ignore);
1586                    }
1587                }
1588                "impl_item" => {
1589                    if c.child_by_field_name("trait").is_some() {
1590                        continue;
1591                    }
1592                    let Some(ty) = c.child_by_field_name("type") else { continue };
1593                    let ty = txt(ty, src);
1594                    let ty = ty.split('<').next().unwrap_or(ty).trim().to_string();
1595                    if let Some(b) = c.child_by_field_name("body") {
1596                        let mut ignore = Vec::new();
1597                        self.block(b, module, Some((&ty, false)), &mut ignore);
1598                    }
1599                }
1600                "mod_item" => {
1601                    let (Some(name), true) = (name, public) else { continue };
1602                    let path = format!("{module}{sep}{name}");
1603                    if !doc.is_empty() {
1604                        self.out
1605                            .push(entry(Kind::Module, &name, path.clone(), self.file, line, format!("pub mod {name}"), doc));
1606                    }
1607                    if let Some(b) = c.child_by_field_name("body") {
1608                        let mut inner = Vec::new();
1609                        self.block(b, &path, None, &mut inner);
1610                    }
1611                }
1612                "type_item" | "const_item" | "static_item" => {
1613                    let (Some(name), true) = (name, public) else { continue };
1614                    let kind = if k == "type_item" { Kind::Type } else { Kind::Const };
1615                    self.out.push(entry(
1616                        kind,
1617                        &name,
1618                        qual(&name),
1619                        self.file,
1620                        line,
1621                        compact(txt(c, src).trim_end_matches(';'), 400),
1622                        doc,
1623                    ));
1624                }
1625                "macro_definition" => {
1626                    let Some(name) = name else { continue };
1627                    if let Some(w) = doc_wrapper(txt(c, src)) {
1628                        self.wrappers.insert(name.clone(), w);
1629                        continue;
1630                    }
1631                    if !was_macro_export {
1632                        continue;
1633                    }
1634                    // Exported macros live at the crate root.
1635                    let root = module.split("::").next().unwrap_or(module);
1636                    let arms = compact(txt(c, src), 300);
1637                    self.out.push(entry(Kind::Macro, &name, format!("{root}::{name}!"), self.file, line, arms, doc));
1638                }
1639                "macro_invocation" if self.depth < 3 => {
1640                    // cfg_rt! { pub fn spawn ... } and friends: parse the body as items.
1641                    let Some(tt) = c.named_children(&mut c.walk()).find(|n| n.kind() == "token_tree") else {
1642                        continue;
1643                    };
1644                    let body = txt(tt, src);
1645                    let called = c.child_by_field_name("macro").map(|m| txt(m, src).to_string()).unwrap_or_default();
1646                    let wrapper = self.wrappers.get(&called).cloned();
1647                    let itemish =
1648                        body.contains("fn ") || body.contains("struct ") || body.contains("mod ") || body.contains("trait ") || body.contains("macro_rules!");
1649                    if body.len() < 4 || !itemish {
1650                        continue;
1651                    }
1652                    let inner = &body[1..body.len() - 1];
1653                    let off = tt.start_position().row as u32 + self.line_off;
1654                    let mut sub = Vec::new();
1655                    rs_parse(
1656                        inner,
1657                        self.file,
1658                        module,
1659                        off,
1660                        self.depth + 1,
1661                        wrapper.or_else(|| self.inherit.clone()),
1662                        &mut sub,
1663                    );
1664                    // Owner-less items only: prose from nested inner docs stays.
1665                    self.out.extend(sub);
1666                }
1667                _ => {}
1668            }
1669        }
1670    }
1671}
1672
1673// ---------------------------------------------------------------- Go
1674
1675fn exported(name: &str) -> bool {
1676    name.chars().next().is_some_and(|c| c.is_uppercase())
1677}
1678
1679fn go_doc(n: Node, src: &[u8]) -> String {
1680    let mut lines: Vec<String> = Vec::new();
1681    let mut p = n.prev_sibling();
1682    let mut next_row = n.start_position().row;
1683    while let Some(c) = p {
1684        if c.kind() != "comment" || c.end_position().row + 1 != next_row {
1685            break;
1686        }
1687        let t = txt(c, src);
1688        if let Some(r) = t.strip_prefix("//") {
1689            lines.push(r.strip_prefix(' ').unwrap_or(r).trim_end().to_string());
1690        } else {
1691            lines.push(clean_block_comment(t));
1692        }
1693        next_row = c.start_position().row;
1694        p = c.prev_sibling();
1695    }
1696    lines.reverse();
1697    cap_doc(lines.join("\n").trim().to_string())
1698}
1699
1700fn go_file(text: &str, rel: &str, module: &str, out: &mut Vec<Entry>) {
1701    parse_with("go", text, |root, src| {
1702        let mut cur = root.walk();
1703        let children: Vec<Node> = root.named_children(&mut cur).collect();
1704        let mut pkg = module.rsplit('/').next().unwrap_or(module).to_string();
1705        for c in children {
1706            let line = c.start_position().row as u32 + 1;
1707            match c.kind() {
1708                "package_clause" => {
1709                    if let Some(id) = c.named_child(0) {
1710                        pkg = txt(id, src).to_string();
1711                    }
1712                    if pkg == "main" {
1713                        return;
1714                    }
1715                    let doc = go_doc(c, src);
1716                    if !doc.is_empty() {
1717                        let dir = rel.rsplit_once('/').map_or("", |x| x.0);
1718                        let title = if dir.is_empty() {
1719                            format!("package {pkg}")
1720                        } else {
1721                            format!("package {pkg} ({dir})")
1722                        };
1723                        for s in markdown::split(&doc, &title) {
1724                            out.push(Entry {
1725                                kind: Kind::Prose,
1726                                name: s.heading.clone(),
1727                                path: s.heading,
1728                                file: rel.to_string(),
1729                                line: s.line,
1730                                sig: String::new(),
1731                                doc: s.text,
1732                                alias_of: None,
1733                                legacy: false,
1734                            });
1735                        }
1736                    }
1737                }
1738                "function_declaration" => {
1739                    let Some(name) = c.child_by_field_name("name").map(|n| txt(n, src)) else {
1740                        continue;
1741                    };
1742                    if !exported(name) {
1743                        continue;
1744                    }
1745                    out.push(entry(
1746                        Kind::Function,
1747                        name,
1748                        format!("{pkg}.{name}"),
1749                        rel,
1750                        line,
1751                        head(c, src, "body", 600),
1752                        go_doc(c, src),
1753                    ));
1754                }
1755                "method_declaration" => {
1756                    let Some(name) = c.child_by_field_name("name").map(|n| txt(n, src)) else {
1757                        continue;
1758                    };
1759                    let recv = c.child_by_field_name("receiver").map(|r| txt(r, src)).unwrap_or("");
1760                    let ty: String = recv
1761                        .trim_matches(['(', ')'])
1762                        .split_whitespace()
1763                        .last()
1764                        .unwrap_or("")
1765                        .trim_start_matches('*')
1766                        .split('[')
1767                        .next()
1768                        .unwrap_or("")
1769                        .to_string();
1770                    if !exported(name) || !exported(&ty) {
1771                        continue;
1772                    }
1773                    out.push(entry(
1774                        Kind::Method,
1775                        name,
1776                        format!("{pkg}.{ty}.{name}"),
1777                        rel,
1778                        line,
1779                        head(c, src, "body", 600),
1780                        go_doc(c, src),
1781                    ));
1782                }
1783                "type_declaration" | "const_declaration" | "var_declaration" => {
1784                    let outer_doc = go_doc(c, src);
1785                    let mut cc = c.walk();
1786                    let specs: Vec<Node> = c
1787                        .named_children(&mut cc)
1788                        .filter(|n| matches!(n.kind(), "type_spec" | "type_alias" | "const_spec" | "var_spec"))
1789                        .collect();
1790                    let single = specs.len() == 1;
1791                    for s in specs {
1792                        let names: Vec<String> = if s.kind().starts_with("type") {
1793                            s.child_by_field_name("name").map(|n| vec![txt(n, src).to_string()]).unwrap_or_default()
1794                        } else {
1795                            let mut sc = s.walk();
1796                            s.children_by_field_name("name", &mut sc).map(|n| txt(n, src).to_string()).collect()
1797                        };
1798                        let doc = if single { outer_doc.clone() } else { go_doc(s, src) };
1799                        let sline = s.start_position().row as u32 + 1;
1800                        for name in names.iter().filter(|n| exported(n)) {
1801                            if s.kind().starts_with("type") {
1802                                let full = txt(s, src);
1803                                let sig = if full.len() <= 900 {
1804                                    format!("type {}", compact(full, 900))
1805                                } else {
1806                                    format!("type {} …", compact(&full[..full.find('{').unwrap_or(200.min(full.len()))], 300))
1807                                };
1808                                let kind = if full.contains("interface") { Kind::Interface } else { Kind::Struct };
1809                                out.push(entry(kind, name, format!("{pkg}.{name}"), rel, sline, sig, doc.clone()));
1810                                // Interface methods
1811                                if let Some(t) = s.child_by_field_name("type").filter(|t| t.kind() == "interface_type") {
1812                                    let mut ic = t.walk();
1813                                    let elems: Vec<Node> = t.named_children(&mut ic).collect();
1814                                    for m in elems {
1815                                        if !matches!(m.kind(), "method_elem" | "method_spec") {
1816                                            continue;
1817                                        }
1818                                        let Some(mn) = m.child_by_field_name("name").map(|n| txt(n, src)) else {
1819                                            continue;
1820                                        };
1821                                        out.push(entry(
1822                                            Kind::Method,
1823                                            mn,
1824                                            format!("{pkg}.{name}.{mn}"),
1825                                            rel,
1826                                            m.start_position().row as u32 + 1,
1827                                            compact(txt(m, src), 400),
1828                                            go_doc(m, src),
1829                                        ));
1830                                    }
1831                                }
1832                            } else {
1833                                let kw = if c.kind() == "const_declaration" { "const" } else { "var" };
1834                                out.push(entry(
1835                                    Kind::Const,
1836                                    name,
1837                                    format!("{pkg}.{name}"),
1838                                    rel,
1839                                    sline,
1840                                    compact(&format!("{kw} {}", txt(s, src)), 300),
1841                                    doc.clone(),
1842                                ));
1843                            }
1844                        }
1845                    }
1846                }
1847                _ => {}
1848            }
1849        }
1850    });
1851}
1852
1853#[cfg(test)]
1854mod tests {
1855    use super::*;
1856
1857    fn run(f: fn(&str, &str, &mut Vec<Entry>), text: &str, rel: &str) -> Vec<Entry> {
1858        let mut out = Vec::new();
1859        f(text, rel, &mut out);
1860        out
1861    }
1862
1863    #[test]
1864    fn typescript_declarations() {
1865        let src = r#"
1866/** Creates an object schema. */
1867export declare function object<T extends ZodRawShape>(shape: T, params?: string): ZodObject<T>;
1868export declare namespace z {
1869  /** A string schema. */
1870  function string(): ZodString;
1871}
1872export interface ZodType<T> {
1873  /** Parse or throw. */
1874  parse(data: unknown): T;
1875  readonly _def: Def;
1876}
1877export declare class ZodObject<T> extends ZodType<T> {
1878  /** Disallow unknown keys. */
1879  strict(message?: string): ZodObject<T>;
1880  private _cached;
1881}
1882declare const objectType: <T>(shape: T) => ZodObject<T>;
1883export { objectType as obj };
1884export * as zz from "./external";
1885"#;
1886        let e = run(ts_file, src, "lib/index.d.ts");
1887        let get = |p: &str| {
1888            e.iter()
1889                .find(|x| x.path == p)
1890                .unwrap_or_else(|| panic!("missing {p}: {:#?}", e.iter().map(|x| &x.path).collect::<Vec<_>>()))
1891        };
1892        assert_eq!(get("object").doc, "Creates an object schema.");
1893        assert!(get("object").sig.contains("ZodObject<T>"));
1894        assert_eq!(get("z.string").doc, "A string schema.");
1895        assert_eq!(get("ZodType.parse").kind, Kind::Method);
1896        assert!(e.iter().all(|x| x.name != "_def" && x.name != "_cached"));
1897        assert_eq!(get("ZodObject.strict").doc, "Disallow unknown keys.");
1898        assert_eq!(get("objectType").kind, Kind::Function);
1899        assert_eq!(get("obj").alias_of.as_deref(), Some("objectType"));
1900        assert_eq!(get("zz").kind, Kind::Module);
1901        assert_eq!(get("object").line, 3);
1902    }
1903
1904    #[test]
1905    fn legacy_dirs() {
1906        assert!(legacy_file("v3/types.d.ts", 4));
1907        assert!(!legacy_file("v4/classic/schemas.d.ts", 4));
1908        assert!(legacy_file("pydantic/v1/main.py", 2));
1909        assert!(!legacy_file("pydantic/main.py", 2));
1910        assert!(!legacy_file("src/v2/x.rs", 1));
1911    }
1912
1913    #[test]
1914    fn python_definitions() {
1915        let src = "\"\"\"Module docs.\"\"\"\n\nclass BaseModel:\n    \"\"\"Base class.\n\n    More.\n    \"\"\"\n\n    def model_dump(self, *, mode: str = 'python') -> dict:\n        \"\"\"Dump the model.\"\"\"\n        return {}\n\n    @classmethod\n    def model_validate(cls, obj):\n        '''Validate.'''\n\n    def _private(self): pass\n\ndef create_model(name: str, **fields) -> type:\n    return None\n";
1916        let e = run(py_file, src, "pydantic/main.py");
1917        let get = |p: &str| e.iter().find(|x| x.path == p).unwrap_or_else(|| panic!("missing {p}"));
1918        assert_eq!(get("pydantic.main.BaseModel").doc, "Base class.\n\nMore.");
1919        assert_eq!(
1920            get("pydantic.main.BaseModel.model_dump").sig,
1921            "def model_dump(self, *, mode: str = 'python') -> dict"
1922        );
1923        assert!(get("pydantic.main.BaseModel.model_validate").sig.starts_with("@classmethod def model_validate"));
1924        assert!(e.iter().all(|x| x.name != "_private"));
1925        assert!(e.iter().any(|x| x.kind == Kind::Prose && x.doc == "Module docs."));
1926        assert_eq!(get("pydantic.main.create_model").kind, Kind::Function);
1927    }
1928
1929    #[test]
1930    fn docs_pages_written_as_components() {
1931        let src = r#"import dedent from "dedent";
1932export const metadata = { title: "Installing with Vite", description: "Use the Vite plugin.", openGraph: { title: "x" } };
1933const steps = [
1934  {
1935    title: "Import Tailwind CSS",
1936    body: (
1937      <p>
1938        Add an <code>@import</code> to your CSS file.
1939      </p>
1940    ),
1941    code: { name: "CSS", lang: "css", code: dedent`
1942        @import "tailwindcss";
1943      ` },
1944  },
1945  { title: 'Old way', body: () => <p>Use directives{' '}here.</p>, code: { lang: 'css', code: '@tailwind base;\n@tailwind utilities;' } },
1946];
1947export default function Page() { return <div className="prose"><h3 className="sr-only">Overview</h3><p>Fast.</p></div> }
1948"#;
1949        let (md, title) = page_markdown(src).unwrap();
1950        assert_eq!(title.as_deref(), Some("Installing with Vite"));
1951        assert!(md.contains("## Import Tailwind CSS"), "{md}");
1952        assert!(md.contains("Add an `@import` to your CSS file."), "{md}");
1953        assert!(md.contains("```css\n@import \"tailwindcss\";\n```"), "{md}");
1954        assert!(md.contains("@tailwind base;\n@tailwind utilities;"), "{md}");
1955        assert!(md.contains("### Overview"), "{md}");
1956        assert!(!md.contains("dedent") && !md.contains("prose"), "{md}");
1957        let mut out = Vec::new();
1958        prose_titled(&md, "upstream:site/docs/installation/page.tsx", 0, title, &mut out);
1959        let s = out.iter().find(|e| e.name == "Installing with Vite › Import Tailwind CSS").unwrap();
1960        assert_eq!(s.line, 5);
1961        assert_eq!(
1962            page_title("upstream:site/src/app/(docs)/docs/installation/(tabs)/using-vite/page.tsx"),
1963            "installation/using-vite"
1964        );
1965    }
1966
1967    #[test]
1968    fn rust_items_and_macro_bodies() {
1969        let src = "//! Crate docs.\n//!\n//! # Usage\n//! Call spawn.\n\ncfg_rt! {\n    /// Spawns a new task.\n    #[track_caller]\n    pub fn spawn<F>(future: F) -> JoinHandle<F::Output>\n    where F: Future + Send + 'static {\n        todo!()\n    }\n}\n\n/// A router.\npub struct Router<S = ()> { inner: Inner }\n\nimpl<S> Router<S> {\n    /// Add a route.\n    pub fn route(self, path: &str, m: MethodRouter<S>) -> Self { self }\n    fn private(&self) {}\n}\n\n/// Hidden\n#[doc(hidden)]\npub fn hidden() {}\n\n/// Macro.\n#[macro_export]\nmacro_rules! select { () => {} }\n";
1970        let mut e = Vec::new();
1971        rs_file(src, "src/task/spawn.rs", "tokio", &mut e);
1972        let get = |p: &str| {
1973            e.iter()
1974                .find(|x| x.path == p)
1975                .unwrap_or_else(|| panic!("missing {p}: {:#?}", e.iter().map(|x| &x.path).collect::<Vec<_>>()))
1976        };
1977        let spawn = get("tokio::task::spawn::spawn");
1978        assert_eq!(spawn.doc, "Spawns a new task.");
1979        assert_eq!(spawn.line, 9);
1980        assert!(spawn.sig.contains("JoinHandle"));
1981        assert_eq!(get("tokio::task::spawn::Router::route").doc, "Add a route.");
1982        assert!(e.iter().all(|x| x.name != "private" && x.name != "hidden"));
1983        assert_eq!(get("tokio::select!").kind, Kind::Macro);
1984        assert!(e.iter().any(|x| x.kind == Kind::Prose && x.path.ends_with("Usage")));
1985
1986        // tokio's doc-wrapper pattern
1987        let src = "macro_rules! doc {\n    ($select:item) => {\n        /// Waits on multiple branches.\n        #[macro_export]\n        $select\n    };\n}\n\ndoc! {macro_rules! select {\n    () => {};\n}}\n";
1988        let mut e = Vec::new();
1989        rs_file(src, "src/macros/select.rs", "tokio", &mut e);
1990        let sel = e.iter().find(|x| x.path == "tokio::select!").expect("select! extracted");
1991        assert_eq!(sel.doc, "Waits on multiple branches.");
1992        assert!(e.iter().all(|x| x.name != "doc"));
1993    }
1994
1995    #[test]
1996    fn go_declarations() {
1997        let src = "// Package gin implements a web framework.\npackage gin\n\n// Context is the request context.\ntype Context struct {\n\tKeys map[string]any\n}\n\n// JSON serializes obj.\nfunc (c *Context) JSON(code int, obj any) {\n}\n\n// Default returns an Engine.\nfunc Default(opts ...Option) *Engine {\n\treturn nil\n}\n\nfunc internal() {}\n\ntype Handler interface {\n\t// Serve serves.\n\tServe(c *Context)\n}\n";
1998        let mut e = Vec::new();
1999        go_file(src, "context.go", "github.com/gin-gonic/gin", &mut e);
2000        let get = |p: &str| {
2001            e.iter()
2002                .find(|x| x.path == p)
2003                .unwrap_or_else(|| panic!("missing {p}: {:#?}", e.iter().map(|x| &x.path).collect::<Vec<_>>()))
2004        };
2005        assert_eq!(get("gin.Context.JSON").doc, "JSON serializes obj.");
2006        assert_eq!(get("gin.Default").sig, "func Default(opts ...Option) *Engine");
2007        assert_eq!(get("gin.Context").doc, "Context is the request context.");
2008        assert_eq!(get("gin.Handler.Serve").doc, "Serve serves.");
2009        assert!(e.iter().all(|x| x.name != "internal"));
2010        assert!(e.iter().any(|x| x.kind == Kind::Prose && x.doc.contains("web framework")));
2011    }
2012}