Skip to main content

lockdocs_core/
extract.rs

1//! Turn a package's files into entries: API symbols (signature + doc comment)
2//! from TypeScript declarations, JavaScript, Python, Rust and Go sources, and
3//! prose sections from READMEs, changelogs and docs folders.
4
5use crate::locate::Source;
6use crate::markdown;
7use crate::Eco;
8use rayon::prelude::*;
9use serde::{Deserialize, Serialize};
10use std::cell::RefCell;
11use std::collections::{HashMap, HashSet};
12use std::path::{Path, PathBuf};
13use tree_sitter::{Node, Parser};
14
15#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
16pub enum Kind {
17    Function,
18    Method,
19    Class,
20    Interface,
21    Type,
22    Const,
23    Enum,
24    Struct,
25    Trait,
26    Macro,
27    Module,
28    Field,
29    Alias,
30    Prose,
31}
32
33impl Kind {
34    pub fn as_str(self) -> &'static str {
35        match self {
36            Kind::Function => "function",
37            Kind::Method => "method",
38            Kind::Class => "class",
39            Kind::Interface => "interface",
40            Kind::Type => "type",
41            Kind::Const => "const",
42            Kind::Enum => "enum",
43            Kind::Struct => "struct",
44            Kind::Trait => "trait",
45            Kind::Macro => "macro",
46            Kind::Module => "module",
47            Kind::Field => "field",
48            Kind::Alias => "re-export",
49            Kind::Prose => "doc",
50        }
51    }
52    pub fn is_container(self) -> bool {
53        matches!(self, Kind::Class | Kind::Interface | Kind::Struct | Kind::Trait | Kind::Enum | Kind::Module)
54    }
55}
56
57#[derive(Debug, Clone, Serialize, Deserialize)]
58pub struct Entry {
59    pub kind: Kind,
60    /// Short name (`object`, `spawn`), or the heading path for prose.
61    pub name: String,
62    /// Qualified path: `z.object`, `tokio::task::spawn`, `pydantic.main.BaseModel.model_dump`.
63    pub path: String,
64    /// File relative to the package root.
65    pub file: String,
66    pub line: u32,
67    /// Declaration text without the body.
68    pub sig: String,
69    /// Doc comment, or the prose section text.
70    pub doc: String,
71    /// For re-exports: the local name this entry points at.
72    pub alias_of: Option<String>,
73    /// Lives in a legacy compatibility copy (`zod/v3` inside zod 4,
74    /// `pydantic/v1` inside pydantic 2): ranked below the current API.
75    #[serde(default)]
76    pub legacy: bool,
77    /// Effective module imports in this prose section and its setup examples.
78    #[serde(default)]
79    pub prose_imports: Vec<String>,
80    /// Citation lines of the logical ancestor sections, before chunking.
81    #[serde(default)]
82    pub prose_parents: Vec<u32>,
83}
84
85/// A path segment `vN` with N below the package's major version.
86fn legacy_file(file: &str, major: u64) -> bool {
87    major > 1
88        && file.split('/').any(|seg| {
89            seg.len() >= 2 && seg.starts_with('v') && seg[1..].chars().all(|c| c.is_ascii_digit()) && seg[1..].parse::<u64>().is_ok_and(|n| n < major)
90        })
91}
92
93// ---------------------------------------------------------------- file selection
94
95const SKIP_DIRS: &[&str] = &[
96    "node_modules",
97    "compiled",
98    ".git",
99    "test",
100    "tests",
101    "__tests__",
102    "__test__",
103    "testdata",
104    "test_data",
105    "benches",
106    "bench",
107    "benchmarks",
108    "fixtures",
109    "__fixtures__",
110    "__pycache__",
111    "examples",
112    "example",
113    "_examples",
114    "coverage",
115    ".github",
116    "vendor",
117    "internal",
118    "__snapshots__",
119];
120
121const MAX_FILES: usize = 25_000;
122const MAX_FILE_BYTES: u64 = 3 * 1024 * 1024;
123
124fn is_doc_name(name: &str) -> bool {
125    let u = name.to_ascii_uppercase();
126    [
127        "README",
128        "CHANGELOG",
129        "CHANGES",
130        "HISTORY",
131        "MIGRATION",
132        "MIGRATING",
133        "UPGRADING",
134        "UPGRADE",
135        "RELEASES",
136        "NEWS",
137        "GUIDE",
138        "API",
139    ]
140    .iter()
141    .any(|p| u.starts_with(p))
142}
143
144fn doc_ext(ext: &str) -> bool {
145    matches!(ext, "md" | "mdx" | "markdown" | "rst")
146}
147
148fn walk(dir: &Path, rel: &Path, depth: usize, out: &mut Vec<PathBuf>, public_npm: bool) {
149    if depth > 14 || out.len() >= MAX_FILES {
150        return;
151    }
152    let Ok(rd) = std::fs::read_dir(dir) else { return };
153    let mut entries: Vec<_> = rd.flatten().collect();
154    entries.sort_by_key(|e| e.file_name());
155    for e in entries {
156        let name = e.file_name().to_string_lossy().to_string();
157        let Ok(ft) = e.file_type() else { continue };
158        let r = rel.join(&name);
159        if ft.is_dir() {
160            if (SKIP_DIRS.contains(&name.as_str()) && !(public_npm && name == "internal")) || name.starts_with('.') {
161                continue;
162            }
163            walk(&e.path(), &r, depth + 1, out, public_npm);
164        } else if ft.is_file() || ft.is_symlink() {
165            out.push(r);
166        }
167        if out.len() >= MAX_FILES {
168            return;
169        }
170    }
171}
172
173/// Every file under `dir` (docs trees keep folders like `examples/` we want).
174fn walk_all(dir: &Path, rel: &Path, depth: usize, out: &mut Vec<PathBuf>) {
175    if depth > 16 || out.len() >= MAX_FILES {
176        return;
177    }
178    let Ok(rd) = std::fs::read_dir(dir) else { return };
179    let mut entries: Vec<_> = rd.flatten().collect();
180    entries.sort_by_key(|e| e.file_name());
181    for e in entries {
182        let name = e.file_name().to_string_lossy().to_string();
183        if name.starts_with('.') {
184            continue;
185        }
186        let r = rel.join(&name);
187        match e.file_type() {
188            Ok(t) if t.is_dir() => walk_all(&e.path(), &r, depth + 1, out),
189            Ok(_) => out.push(r),
190            _ => {}
191        }
192    }
193}
194
195fn ext_of(p: &Path) -> String {
196    let s = p.file_name().map(|f| f.to_string_lossy().to_ascii_lowercase()).unwrap_or_default();
197    for e in [".d.ts", ".d.mts", ".d.cts"] {
198        if s.ends_with(e) {
199            return e[1..].to_string();
200        }
201    }
202    p.extension().map(|e| e.to_string_lossy().to_ascii_lowercase()).unwrap_or_default()
203}
204
205fn in_docs_dir(p: &Path) -> bool {
206    p.components().any(|c| {
207        matches!(
208            c.as_os_str().to_string_lossy().to_ascii_lowercase().as_str(),
209            "docs" | "doc" | "documentation" | "guide" | "guides"
210        )
211    })
212}
213
214/// A unit of work: one file to read, how to parse it, and how to label it.
215struct Job {
216    abs: PathBuf,
217    rel: String,
218    how: How,
219}
220
221#[derive(Clone, Copy, PartialEq)]
222enum How {
223    Prose,
224    /// A docs page written as a JS/TSX component.
225    Page,
226    Example,
227    Metadata,
228    Ts,
229    Python,
230    Rust,
231    Go,
232}
233
234/// Plan which files to read for a package.
235fn plan(eco: Eco, src: &Source, extra_types: Option<&Path>, upstream: Option<&Path>, public_api: bool) -> Vec<Job> {
236    let mut rels = Vec::new();
237    match &src.files {
238        Some(list) => rels.extend(list.iter().cloned()),
239        None => walk(&src.dir, Path::new(""), 0, &mut rels, public_api),
240    }
241    let mut jobs = Vec::new();
242    let names: HashSet<String> = rels.iter().map(|r| r.to_string_lossy().replace('\\', "/")).collect();
243    let mut has_dts = false;
244    for r in &rels {
245        let e = ext_of(r);
246        if e.starts_with("d.") {
247            has_dts = true;
248        }
249    }
250    let rs_crate_src = eco == Eco::Cargo;
251    for r in rels {
252        let rel = r.to_string_lossy().replace('\\', "/");
253        if rel
254            .split('/')
255            .any(|seg| SKIP_DIRS.contains(&seg) && !(public_api && seg == "internal") && src.files.is_some())
256        {
257            continue;
258        }
259        let abs = src.dir.join(&r);
260        let fname = r.file_name().map(|f| f.to_string_lossy().to_string()).unwrap_or_default();
261        let e = ext_of(&r);
262        let depth = rel.matches('/').count();
263        let how = if doc_ext(&e) || (e.is_empty() || e == "txt") && is_doc_name(&fname) && depth == 0 {
264            if is_doc_name(&fname) || in_docs_dir(&r) || depth == 0 {
265                Some(How::Prose)
266            } else {
267                None
268            }
269        } else {
270            match eco {
271                Eco::Npm => match e.as_str() {
272                    "d.ts" => Some(How::Ts),
273                    "d.mts" | "d.cts" => {
274                        let base = rel.trim_end_matches(".d.mts").trim_end_matches(".d.cts");
275                        (public_api || !names.contains(&format!("{base}.d.ts"))).then_some(How::Ts)
276                    }
277                    "ts" | "mts" | "cts" | "tsx" if !has_dts => Some(How::Ts),
278                    "js" | "mjs" | "cjs" if !has_dts && !fname.contains(".min.") => Some(How::Ts),
279                    _ => None,
280                },
281                Eco::PyPI => match e.as_str() {
282                    "py" => Some(How::Python),
283                    "pyi" => {
284                        let base = rel.trim_end_matches(".pyi");
285                        if names.contains(&format!("{base}.py")) {
286                            None
287                        } else {
288                            Some(How::Python)
289                        }
290                    }
291                    _ => None,
292                },
293                Eco::Cargo => (e == "rs" && rs_crate_src && rel.starts_with("src/")).then_some(How::Rust),
294                Eco::Go => (e == "go" && !fname.ends_with("_test.go")).then_some(How::Go),
295            }
296        };
297        if let Some(how) = how {
298            jobs.push(Job { abs, rel, how });
299        }
300    }
301    // Upstream docs fetched from the repository at this version's tag.
302    if let Some(up) = upstream {
303        let pages: HashSet<String> = std::fs::read_to_string(up.join(".lockdocs-upstream.json"))
304            .ok()
305            .and_then(|t| serde_json::from_str::<crate::upstream::Manifest>(&t).ok())
306            .map(|m| m.pages.into_iter().collect())
307            .unwrap_or_default();
308        let mut rels = Vec::new();
309        walk_all(up, Path::new(""), 0, &mut rels);
310        for r in rels {
311            let e = ext_of(&r);
312            let plain = r.to_string_lossy().replace('\\', "/");
313            let rel = format!("upstream:{plain}");
314            if pages.contains(&plain) {
315                jobs.push(Job {
316                    abs: up.join(&r),
317                    rel,
318                    how: How::Page,
319                });
320                continue;
321            }
322            // Django and others write reStructuredText in `.txt` files under docs/.
323            if doc_ext(&e) || e == "mdoc" || e == "txt" {
324                jobs.push(Job {
325                    abs: up.join(&r),
326                    rel,
327                    how: How::Prose,
328                });
329            } else if EXAMPLE_EXTS.contains(&e.as_str()) {
330                // Docs examples (`docs/examples/validators_simple.py`) carry the code the pages include.
331                jobs.push(Job {
332                    abs: up.join(&r),
333                    rel,
334                    how: How::Example,
335                });
336            }
337        }
338    }
339    if let Some(meta) = &src.metadata {
340        jobs.push(Job {
341            abs: meta.clone(),
342            rel: "README (package metadata)".into(),
343            how: How::Metadata,
344        });
345    }
346    // Types that ship separately (`@types/react` for `react`).
347    if let (Some(types), false) = (extra_types, has_dts) {
348        let mut rels = Vec::new();
349        walk(types, Path::new(""), 0, &mut rels, public_api);
350        let label = types
351            .iter()
352            .rev()
353            .take(2)
354            .collect::<Vec<_>>()
355            .into_iter()
356            .rev()
357            .map(|s| s.to_string_lossy())
358            .collect::<Vec<_>>()
359            .join("/");
360        for r in rels {
361            if ext_of(&r) == "d.ts" || public_api && matches!(ext_of(&r).as_str(), "d.cts" | "d.mts") {
362                let rel = format!("{label}/{}", r.to_string_lossy().replace('\\', "/"));
363                jobs.push(Job {
364                    abs: types.join(&r),
365                    rel,
366                    how: How::Ts,
367                });
368            }
369        }
370    }
371    jobs
372}
373
374/// Extract every entry for a package.
375pub fn extract(eco: Eco, pkg_name: &str, src: &Source, extra_types: Option<&Path>, upstream: Option<&Path>) -> Vec<Entry> {
376    let major: u64 = src.version.trim_start_matches('v').split('.').next().and_then(|m| m.parse().ok()).unwrap_or(0);
377    let jobs = plan(eco, src, extra_types, upstream, false);
378    let crate_name = if eco == Eco::Cargo {
379        rust_crate_name(&src.dir).unwrap_or_else(|| pkg_name.replace('-', "_"))
380    } else {
381        String::new()
382    };
383    let includes = upstream.map(Includes::scan).unwrap_or_default();
384    let mut all: Vec<Entry> = jobs
385        .par_iter()
386        .flat_map_iter(|job| {
387            let Ok(meta) = std::fs::metadata(&job.abs) else { return Vec::new() };
388            if meta.len() > MAX_FILE_BYTES && job.how != How::Prose {
389                return Vec::new();
390            }
391            let Ok(bytes) = std::fs::read(&job.abs) else { return Vec::new() };
392            let text = String::from_utf8_lossy(&bytes);
393            let mut out = Vec::new();
394            match job.how {
395                How::Prose => {
396                    prose(&text, &job.rel, 0, &mut out);
397                    if job.rel.starts_with("upstream:") && !includes.files.is_empty() {
398                        for e in &mut out {
399                            e.doc = includes.expand(&e.doc);
400                        }
401                    }
402                }
403                How::Page => {
404                    if let Some((md, title)) = page_markdown(&text) {
405                        let title = title.unwrap_or_else(|| page_title(&job.rel));
406                        prose_titled(&md, &job.rel, 0, Some(title), &mut out);
407                    }
408                }
409                How::Example => example(&text, &job.rel, &mut out),
410                How::Metadata => {
411                    let (body, line) = markdown::metadata_body(&text);
412                    prose(&body, &job.rel, line - 1, &mut out)
413                }
414                How::Ts => ts_file(&text, &job.rel, &mut out),
415                How::Python => py_file(&text, &job.rel, &mut out),
416                How::Rust => rs_file(&text, &job.rel, &crate_name, &mut out),
417                How::Go => go_file(&text, &job.rel, pkg_name, &mut out),
418            }
419            out
420        })
421        .collect();
422    for e in &mut all {
423        e.legacy = legacy_file(&e.file, major);
424    }
425    // Drop exact duplicates (the same declaration shipped twice).
426    let mut seen = HashSet::new();
427    all.retain(|e| seen.insert((e.path.clone(), e.sig.clone(), e.doc.len())));
428    all
429}
430
431/// Declaration graph inputs and cache identity use the same file selection as extraction.
432pub(crate) fn npm_sources(src: &Source, extra_types: Option<&Path>) -> std::collections::BTreeMap<String, String> {
433    plan(Eco::Npm, src, extra_types, None, true)
434        .into_iter()
435        .filter(|j| j.how == How::Ts)
436        .filter_map(|j| {
437            if std::fs::metadata(&j.abs).ok()?.len() > MAX_FILE_BYTES {
438                return None;
439            }
440            Some((j.rel, std::fs::read_to_string(j.abs).ok()?))
441        })
442        .collect()
443}
444
445/// Keep declaration identities until the requested import graph is selected.
446/// Mirrored declarations are deduplicated within that search scope.
447pub(crate) fn npm_entries(sources: &std::collections::BTreeMap<String, String>) -> Vec<Entry> {
448    let mut entries = Vec::new();
449    for (file, text) in sources {
450        ts_file(text, file, &mut entries);
451    }
452    entries
453}
454
455/// Content identity, including names, catches edits, additions and removals even
456/// when a package's version and metadata timestamps did not change.
457pub(crate) fn npm_stamp(src: &Source, extra_types: Option<&Path>) -> String {
458    let mut parts = Vec::new();
459    for j in plan(Eco::Npm, src, extra_types, None, true) {
460        parts.push(j.rel);
461        if let Ok(bytes) = std::fs::read(&j.abs) {
462            parts.push(crate::cache::hash(&[&String::from_utf8_lossy(&bytes)]));
463        }
464    }
465    crate::cache::hash(&parts.iter().map(String::as_str).collect::<Vec<_>>())
466}
467
468fn rust_crate_name(dir: &Path) -> Option<String> {
469    let t = std::fs::read_to_string(dir.join("Cargo.toml")).ok()?;
470    let v: toml::Table = toml::from_str(&t).ok()?;
471    let lib = v.get("lib").and_then(|l| l.get("name")).and_then(|n| n.as_str());
472    let pkg = v.get("package").and_then(|p| p.get("name")).and_then(|n| n.as_str());
473    lib.or(pkg).map(|n| n.replace('-', "_"))
474}
475
476/// Docs sites include example files by reference (`{!.tmp_examples/x.md!}`,
477/// `{* ../../docs_src/a/tutorial001.py *}`, `--8<-- "x.py"`). Inline them so
478/// the section carries its code.
479#[derive(Default)]
480struct Includes {
481    /// (lowercase path without extension, absolute path)
482    files: Vec<(String, PathBuf)>,
483}
484
485impl Includes {
486    fn scan(dir: &Path) -> Includes {
487        let mut rels = Vec::new();
488        walk_all(dir, Path::new(""), 0, &mut rels);
489        let files = rels
490            .into_iter()
491            .filter(|r| EXAMPLE_EXTS.contains(&ext_of(r).as_str()))
492            .map(|r| {
493                let s = r.to_string_lossy().replace('\\', "/").to_ascii_lowercase();
494                let stem = s.rsplit_once('.').map_or(s.clone(), |x| x.0.to_string());
495                (stem, dir.join(&r))
496            })
497            .collect();
498        Includes { files }
499    }
500
501    fn find(&self, reference: &str) -> Option<&PathBuf> {
502        let r = reference.trim().trim_matches(['"', '\'']).to_ascii_lowercase();
503        let r = r.split_whitespace().next().unwrap_or("");
504        let r = r.rsplit_once('.').map_or(r, |x| x.0);
505        let segs: Vec<&str> = r
506            .split('/')
507            .filter(|s| !s.is_empty() && *s != ".." && *s != "." && *s != ".tmp_examples")
508            .collect();
509        for take in (1..=segs.len().min(3)).rev() {
510            let suffix = segs[segs.len() - take..].join("/");
511            let hits: Vec<&(String, PathBuf)> = self.files.iter().filter(|(k, _)| *k == suffix || k.ends_with(&format!("/{suffix}"))).collect();
512            if hits.len() == 1 {
513                return Some(&hits[0].1);
514            }
515        }
516        None
517    }
518
519    fn expand(&self, text: &str) -> String {
520        if !(text.contains("{!") || text.contains("{*") || text.contains("--8<--")) {
521            return text.to_string();
522        }
523        let mut out = String::with_capacity(text.len());
524        for line in text.lines() {
525            let t = line.trim();
526            let reference = t
527                .strip_prefix("{!")
528                .and_then(|x| x.strip_suffix("!}"))
529                .or_else(|| t.strip_prefix("{*").and_then(|x| x.strip_suffix("*}")))
530                .or_else(|| t.strip_prefix("--8<--"));
531            match reference
532                .and_then(|r| self.find(r))
533                .and_then(|p| std::fs::read_to_string(p).ok().map(|c| (p, c)))
534            {
535                Some((p, code)) => {
536                    let lang = p.extension().map(|e| e.to_string_lossy().to_string()).unwrap_or_default();
537                    let code: String = code.chars().take(3000).collect();
538                    out.push_str(&format!("```{lang}\n{}\n```\n", code.trim_end()));
539                }
540                None => {
541                    out.push_str(line);
542                    out.push('\n');
543                }
544            }
545        }
546        out
547    }
548}
549
550const EXAMPLE_EXTS: &[&str] = &["py", "ts", "tsx", "js", "jsx", "mjs", "rs", "go"];
551
552/// A docs example file as one code section titled by its file name.
553fn example(text: &str, rel: &str, out: &mut Vec<Entry>) {
554    if text.len() > 12_000 || text.trim().is_empty() {
555        return;
556    }
557    let name = rel.rsplit('/').next().unwrap_or(rel);
558    let lang = name.rsplit('.').next().unwrap_or("");
559    let title = name.rsplit_once('.').map_or(name, |x| x.0).replace(['_', '-'], " ");
560    out.push(Entry {
561        kind: Kind::Prose,
562        name: format!("example › {title}"),
563        path: format!("example › {title}"),
564        file: rel.to_string(),
565        line: 1,
566        sig: String::new(),
567        doc: format!("```{lang}\n{}\n```", text.trim_end()),
568        alias_of: None,
569        legacy: false,
570        prose_imports: Vec::new(),
571        prose_parents: Vec::new(),
572    });
573}
574
575/// Title for a page file without one: its route (`docs/installation/using-vite`).
576fn page_title(rel: &str) -> String {
577    let segs: Vec<&str> = rel.split('/').filter(|s| !(s.starts_with('(') && s.ends_with(')'))).collect();
578    let mut segs: Vec<&str> = segs.iter().rev().take(3).rev().copied().collect();
579    if let Some(last) = segs.last() {
580        if last.starts_with("page.") || last.starts_with("index.") {
581            segs.pop();
582        }
583    }
584    segs.join("/")
585        .trim_end_matches(".tsx")
586        .trim_end_matches(".jsx")
587        .trim_end_matches(".js")
588        .to_string()
589}
590
591/// Markdown from a docs page written as a JS/TSX component: `title:` values
592/// of listed steps become headings, `description:` and JSX text become
593/// paragraphs, `code:` strings become fenced code (language from `lang:`),
594/// and JSX `<h1>`..`<h6>` stay headings. Lines keep their source position
595/// where they can. Returns (markdown, page title from `metadata.title`).
596fn page_markdown(text: &str) -> Option<(String, Option<String>)> {
597    if text.len() > 200_000 {
598        return None;
599    }
600    parse_with("tsx", text, |root, src| {
601        let mut p = PageOut::default();
602        page_walk(root, src, &mut p, false);
603        p.flush();
604        let mut md = String::new();
605        let mut n = 0u32;
606        for (line, t) in &p.out {
607            while n + 1 < *line {
608                md.push('\n');
609                n += 1;
610            }
611            md.push_str(t);
612            md.push('\n');
613            n += 1 + t.matches('\n').count() as u32;
614        }
615        (md, p.title)
616    })
617    .filter(|(md, _)| !md.trim().is_empty())
618}
619
620#[derive(Default)]
621struct PageOut {
622    out: Vec<(u32, String)>,
623    para: String,
624    para_line: u32,
625    title: Option<String>,
626}
627
628impl PageOut {
629    fn text(&mut self, line: u32, t: &str) {
630        if self.para.trim().is_empty() {
631            self.para_line = line;
632        }
633        self.para.push_str(t);
634    }
635    fn flush(&mut self) {
636        let t = compact(&self.para, usize::MAX);
637        if !t.is_empty() {
638            self.out.push((self.para_line, t));
639        }
640        self.para.clear();
641    }
642    fn block(&mut self, line: u32, t: String) {
643        self.flush();
644        self.out.push((line, t));
645    }
646}
647
648fn line_of(n: Node) -> u32 {
649    n.start_position().row as u32 + 1
650}
651
652/// The value of a string, template string or tagged template (`dedent`...``).
653fn js_string(n: Node, src: &[u8]) -> Option<String> {
654    match n.kind() {
655        "string" => {
656            let raw = txt(n, src);
657            let inner = raw.get(1..raw.len().saturating_sub(1)).unwrap_or("");
658            Some(inner.replace("\\n", "\n").replace("\\'", "'").replace("\\\"", "\"").replace("\\\\", "\\"))
659        }
660        "template_string" => {
661            let raw = txt(n, src);
662            let inner = raw.get(1..raw.len().saturating_sub(1)).unwrap_or("");
663            Some(dedent(&inner.trim_matches('\n').lines().collect::<Vec<_>>()))
664        }
665        "call_expression" => n
666            .child_by_field_name("arguments")
667            .filter(|a| a.kind() == "template_string")
668            .and_then(|a| js_string(a, src)),
669        "parenthesized_expression" => n.named_child(0).and_then(|c| js_string(c, src)),
670        _ => None,
671    }
672}
673
674fn jsx_name(n: Node, src: &[u8]) -> String {
675    let open = if n.kind() == "jsx_element" {
676        n.child_by_field_name("open_tag")
677    } else {
678        Some(n)
679    };
680    open.and_then(|o| o.child_by_field_name("name"))
681        .map(|x| txt(x, src).to_string())
682        .unwrap_or_default()
683}
684
685/// The visible text inside a JSX element.
686fn jsx_text(n: Node, src: &[u8], out: &mut String) {
687    match n.kind() {
688        "jsx_text" => out.push_str(txt(n, src)),
689        "jsx_expression" => {
690            if let Some(v) = n.named_child(0).and_then(|c| js_string(c, src)) {
691                out.push_str(&v);
692            }
693        }
694        "jsx_attribute" | "jsx_opening_element" | "jsx_closing_element" => {}
695        _ => {
696            let mut c = n.walk();
697            for ch in n.named_children(&mut c) {
698                jsx_text(ch, src, out);
699            }
700        }
701    }
702}
703
704fn page_walk(n: Node, src: &[u8], p: &mut PageOut, in_array: bool) {
705    match n.kind() {
706        "import_statement" | "comment" | "jsx_attribute" => {}
707        "pair" => {
708            let key = n
709                .child_by_field_name("key")
710                .map(|k| txt(k, src).trim_matches(['"', '\'']).to_string())
711                .unwrap_or_default();
712            let Some(value) = n.child_by_field_name("value") else { return };
713            match key.as_str() {
714                "title" | "heading" => {
715                    if let Some(t) = js_string(value, src) {
716                        if !in_array && p.title.is_none() {
717                            p.title = Some(t);
718                        } else if in_array {
719                            p.block(line_of(n), format!("## {}", compact(&t, 200)));
720                        }
721                    }
722                }
723                "description" => {
724                    if let Some(t) = js_string(value, src) {
725                        p.block(line_of(value), compact(&t, usize::MAX));
726                    }
727                }
728                "code" => match js_string(value, src) {
729                    Some(code) => {
730                        let lang = n
731                            .parent()
732                            .and_then(|obj| {
733                                let mut c = obj.walk();
734                                let found = obj.named_children(&mut c).find(|x| {
735                                    x.kind() == "pair" && x.child_by_field_name("key").is_some_and(|k| txt(k, src).trim_matches(['"', '\'']) == "lang")
736                                });
737                                found
738                            })
739                            .and_then(|x| x.child_by_field_name("value"))
740                            .and_then(|v| js_string(v, src))
741                            .unwrap_or_default();
742                        p.block(line_of(value), format!("```{lang}\n{}\n```", code.trim_end()));
743                    }
744                    None => page_walk(value, src, p, in_array),
745                },
746                "name" | "lang" | "className" | "href" | "id" | "url" | "src" | "image" | "images" | "icon" | "openGraph" | "twitter" | "alternates" => {}
747                _ => page_walk(value, src, p, in_array),
748            }
749        }
750        "jsx_element" | "jsx_self_closing_element" => {
751            let name = jsx_name(n, src);
752            let b = name.as_bytes();
753            if b.len() == 2 && b[0] == b'h' && (b'1'..=b'6').contains(&b[1]) {
754                let mut t = String::new();
755                jsx_text(n, src, &mut t);
756                let t = compact(&t, 200);
757                if !t.is_empty() {
758                    p.block(line_of(n), format!("{} {t}", "#".repeat((b[1] - b'0') as usize)));
759                }
760            } else if name == "code" {
761                let mut t = String::new();
762                jsx_text(n, src, &mut t);
763                p.text(line_of(n), &format!("`{}`", t.trim()));
764            } else {
765                let block = matches!(name.as_str(), "p" | "li" | "div" | "section" | "ul" | "ol" | "pre" | "table" | "tr");
766                if block {
767                    p.flush();
768                }
769                let mut c = n.walk();
770                for ch in n.named_children(&mut c) {
771                    page_walk(ch, src, p, in_array);
772                }
773                if block {
774                    p.flush();
775                }
776            }
777        }
778        "jsx_text" => p.text(line_of(n), txt(n, src)),
779        "jsx_expression" => match n.named_child(0) {
780            Some(c) if matches!(c.kind(), "string" | "template_string") => {
781                let v = js_string(c, src).unwrap_or_default();
782                p.text(line_of(n), &v);
783            }
784            Some(c) => page_walk(c, src, p, in_array),
785            None => {}
786        },
787        _ => {
788            let arr = in_array || n.kind() == "array";
789            let mut c = n.walk();
790            for ch in n.named_children(&mut c) {
791                page_walk(ch, src, p, arr);
792            }
793        }
794    }
795}
796
797fn prose(text: &str, rel: &str, line_off: u32, out: &mut Vec<Entry>) {
798    prose_titled(text, rel, line_off, None, out)
799}
800
801fn prose_titled(text: &str, rel: &str, line_off: u32, given: Option<String>, out: &mut Vec<Entry>) {
802    let (text, fm_title) = markdown::clean_mdx(text, rel.ends_with(".mdx"));
803    // A page without a title is named after its file: `model_config.md` -> `model config`.
804    let file_name = rel.rsplit('/').next().unwrap_or(rel);
805    let file_title = if rel.starts_with("upstream:") && !is_doc_name(file_name) {
806        file_name
807            .rsplit_once('.')
808            .map_or(file_name, |x| x.0)
809            .trim_start_matches(|c: char| c.is_ascii_digit() || c == '-')
810            .replace(['_', '-'], " ")
811    } else {
812        file_name.to_string()
813    };
814    let file_title = file_title.as_str();
815    let title = if let Some(t) = given {
816        t
817    } else if rel.contains("package metadata") {
818        "README".to_string()
819    } else if let Some(t) = fm_title {
820        t
821    } else {
822        file_title.to_string()
823    };
824    for s in markdown::split(&text, &title) {
825        out.push(prose_section(s, rel, line_off));
826    }
827}
828
829// ---------------------------------------------------------------- helpers
830
831thread_local! {
832    static PARSERS: RefCell<Vec<(&'static str, Parser)>> = const { RefCell::new(Vec::new()) };
833}
834
835/// Parse with a pooled parser. The parser is taken out of the pool while in
836/// use, so `f` may parse again (Rust macro bodies do).
837fn parse_with<R>(lang: &'static str, text: &str, f: impl FnOnce(Node, &[u8]) -> R) -> Option<R> {
838    let pooled = PARSERS.with(|ps| {
839        let mut ps = ps.borrow_mut();
840        ps.iter().position(|(l, _)| *l == lang).map(|i| ps.swap_remove(i).1)
841    });
842    let mut parser = match pooled {
843        Some(p) => p,
844        None => {
845            let mut p = Parser::new();
846            let language: tree_sitter::Language = match lang {
847                "ts" => tree_sitter_typescript::LANGUAGE_TYPESCRIPT.into(),
848                "tsx" => tree_sitter_typescript::LANGUAGE_TSX.into(),
849                "py" => tree_sitter_python::LANGUAGE.into(),
850                "rs" => tree_sitter_rust::LANGUAGE.into(),
851                _ => tree_sitter_go::LANGUAGE.into(),
852            };
853            p.set_language(&language).ok()?;
854            p
855        }
856    };
857    let tree = parser.parse(text, None);
858    PARSERS.with(|ps| ps.borrow_mut().push((lang, parser)));
859    let tree = tree?;
860    Some(f(tree.root_node(), text.as_bytes()))
861}
862
863fn txt<'a>(n: Node, src: &'a [u8]) -> &'a str {
864    n.utf8_text(src).unwrap_or("")
865}
866
867fn compact(s: &str, cap: usize) -> String {
868    let mut out = String::with_capacity(s.len().min(cap.saturating_add(8)));
869    let mut space = false;
870    for c in s.chars() {
871        if c.is_whitespace() {
872            space = true;
873            continue;
874        }
875        if space && !out.is_empty() {
876            out.push(' ');
877        }
878        space = false;
879        out.push(c);
880        if out.len() >= cap {
881            out.push_str(" …");
882            break;
883        }
884    }
885    out.replace("( ", "(").replace(", )", ")").replace(",)", ")").replace(" )", ")")
886}
887
888/// Drop `# comments` from Python source lines (outside string literals).
889fn strip_py_comments(s: &str) -> String {
890    let mut out = String::with_capacity(s.len());
891    for line in s.lines() {
892        let mut quote: Option<char> = None;
893        let mut cut = line.len();
894        for (i, c) in line.char_indices() {
895            match (quote, c) {
896                (None, '#') => {
897                    cut = i;
898                    break;
899                }
900                (None, '"' | '\'') => quote = Some(c),
901                (Some(q), c) if c == q => quote = None,
902                _ => {}
903            }
904        }
905        out.push_str(&line[..cut]);
906        out.push('\n');
907    }
908    out
909}
910
911/// Source text from `n` up to (not including) its body child.
912fn head(n: Node, src: &[u8], body_field: &str, cap: usize) -> String {
913    let end = n.child_by_field_name(body_field).map_or(n.end_byte(), |b| b.start_byte());
914    let s = std::str::from_utf8(&src[n.start_byte()..end]).unwrap_or("");
915    compact(s.trim_end().trim_end_matches(['{', ':', '=']).trim_end(), cap)
916}
917
918fn cap_doc(s: String) -> String {
919    const CAP: usize = 4000;
920    if s.len() <= CAP {
921        return s;
922    }
923    let mut i = CAP;
924    while !s.is_char_boundary(i) {
925        i -= 1;
926    }
927    format!("{} …", &s[..i])
928}
929
930fn dedent(lines: &[&str]) -> String {
931    let indent = lines
932        .iter()
933        .filter(|l| !l.trim().is_empty())
934        .map(|l| l.len() - l.trim_start().len())
935        .min()
936        .unwrap_or(0);
937    let v: Vec<&str> = lines.iter().map(|l| if l.len() >= indent { &l[indent..] } else { l.trim_start() }).collect();
938    v.join("\n").trim().to_string()
939}
940
941fn clean_block_comment(s: &str) -> String {
942    let s = s
943        .trim_start_matches("/**")
944        .trim_start_matches("/*!")
945        .trim_start_matches("/*")
946        .trim_end_matches("*/");
947    let lines: Vec<&str> = s
948        .lines()
949        .map(|l| {
950            let t = l.trim_start();
951            match t.strip_prefix('*') {
952                Some(r) => r.strip_prefix(' ').unwrap_or(r),
953                None => l,
954            }
955        })
956        .collect();
957    cap_doc(dedent(&lines))
958}
959
960fn entry(kind: Kind, name: &str, path: String, file: &str, line: u32, sig: String, doc: String) -> Entry {
961    Entry {
962        kind,
963        name: name.to_string(),
964        path,
965        file: file.to_string(),
966        line,
967        sig,
968        doc,
969        alias_of: None,
970        legacy: false,
971        prose_imports: Vec::new(),
972        prose_parents: Vec::new(),
973    }
974}
975
976/// A Prose entry for one Markdown section: the heading names it and is its
977/// path, and `line_off` shifts the section line into the surrounding file.
978fn prose_section(s: crate::markdown::Section, rel: &str, line_off: u32) -> Entry {
979    Entry {
980        kind: Kind::Prose,
981        name: s.heading.clone(),
982        path: s.heading,
983        file: rel.to_string(),
984        line: s.line + line_off,
985        sig: String::new(),
986        doc: s.text,
987        alias_of: None,
988        legacy: false,
989        prose_imports: s.imports,
990        prose_parents: s.parents.into_iter().map(|line| line + line_off).collect(),
991    }
992}
993
994// ---------------------------------------------------------------- TypeScript / JavaScript
995
996fn ts_file(text: &str, rel: &str, out: &mut Vec<Entry>) {
997    let lang = if rel.ends_with(".tsx") || rel.ends_with(".jsx") { "tsx" } else { "ts" };
998    let is_js = rel.ends_with(".js") || rel.ends_with(".mjs") || rel.ends_with(".cjs");
999    parse_with(lang, text, |root, src| {
1000        let mut w = TsWalker {
1001            src,
1002            file: rel,
1003            out,
1004            js: is_js,
1005        };
1006        w.block(root, "");
1007    });
1008}
1009
1010struct TsWalker<'a> {
1011    src: &'a [u8],
1012    file: &'a str,
1013    out: &'a mut Vec<Entry>,
1014    js: bool,
1015}
1016
1017impl TsWalker<'_> {
1018    fn doc(&self, n: Node) -> String {
1019        let mut p = n.prev_sibling();
1020        while let Some(c) = p {
1021            if c.kind() == "comment" {
1022                let t = txt(c, self.src);
1023                if t.starts_with("/**") && c.end_position().row + 2 >= n.start_position().row {
1024                    return clean_block_comment(t);
1025                }
1026                if t.starts_with("//") {
1027                    p = c.prev_sibling();
1028                    continue;
1029                }
1030            }
1031            break;
1032        }
1033        String::new()
1034    }
1035
1036    fn line(&self, n: Node) -> u32 {
1037        n.start_position().row as u32 + 1
1038    }
1039
1040    fn block(&mut self, node: Node, prefix: &str) {
1041        let mut cur = node.walk();
1042        let children: Vec<Node> = node.named_children(&mut cur).collect();
1043        for c in children {
1044            match c.kind() {
1045                "export_statement" => {
1046                    let doc = self.doc(c);
1047                    if let Some(d) = c.child_by_field_name("declaration") {
1048                        self.decl(d, prefix, doc, c);
1049                    } else {
1050                        self.exports(c, prefix);
1051                    }
1052                }
1053                "ambient_declaration" => {
1054                    let doc = self.doc(c);
1055                    let mut cc = c.walk();
1056                    let inner: Vec<Node> = c.named_children(&mut cc).collect();
1057                    for d in inner {
1058                        self.decl(d, prefix, doc.clone(), c);
1059                    }
1060                }
1061                "expression_statement" => {
1062                    let mut cc = c.walk();
1063                    let inner: Vec<Node> = c.named_children(&mut cc).collect();
1064                    for d in inner {
1065                        if matches!(d.kind(), "internal_module" | "module") {
1066                            let doc = self.doc(c);
1067                            self.decl(d, prefix, doc, c);
1068                        }
1069                    }
1070                }
1071                _ => {
1072                    let doc = self.doc(c);
1073                    // In JS only documented top-level declarations are API.
1074                    if !self.js || !doc.is_empty() || !prefix.is_empty() {
1075                        self.decl(c, prefix, doc, c);
1076                    }
1077                }
1078            }
1079        }
1080    }
1081
1082    fn exports(&mut self, n: Node, prefix: &str) {
1083        let mut cur = n.walk();
1084        let children: Vec<Node> = n.named_children(&mut cur).collect();
1085        for c in children {
1086            match c.kind() {
1087                "export_clause" => {
1088                    let mut cc = c.walk();
1089                    let specs: Vec<Node> = c.named_children(&mut cc).collect();
1090                    for s in specs {
1091                        let (Some(name), alias) = (s.child_by_field_name("name"), s.child_by_field_name("alias")) else {
1092                            continue;
1093                        };
1094                        let Some(alias) = alias else { continue };
1095                        let (from, to) = (txt(name, self.src), txt(alias, self.src));
1096                        if from == to || to == "default" {
1097                            continue;
1098                        }
1099                        let mut e = entry(
1100                            Kind::Alias,
1101                            to,
1102                            format!("{prefix}{to}"),
1103                            self.file,
1104                            self.line(s),
1105                            compact(txt(n, self.src), 200),
1106                            String::new(),
1107                        );
1108                        e.alias_of = Some(from.to_string());
1109                        self.out.push(e);
1110                    }
1111                }
1112                "namespace_export" => {
1113                    if let Some(id) = c.named_child(0) {
1114                        let name = txt(id, self.src);
1115                        self.out.push(entry(
1116                            Kind::Module,
1117                            name,
1118                            format!("{prefix}{name}"),
1119                            self.file,
1120                            self.line(n),
1121                            compact(txt(n, self.src), 200),
1122                            self.doc(n),
1123                        ));
1124                    }
1125                }
1126                _ => {}
1127            }
1128        }
1129    }
1130
1131    fn decl(&mut self, d: Node, prefix: &str, doc: String, outer: Node) {
1132        let src = self.src;
1133        let name_of = |n: Node| n.child_by_field_name("name").map(|x| txt(x, src).to_string());
1134        let line = self.line(outer);
1135        match d.kind() {
1136            "ambient_declaration" => {
1137                let mut cur = d.walk();
1138                let inner: Vec<Node> = d.named_children(&mut cur).collect();
1139                for x in inner {
1140                    self.decl(x, prefix, doc.clone(), outer);
1141                }
1142            }
1143            "function_signature" | "function_declaration" | "generator_function_declaration" => {
1144                let Some(name) = name_of(d) else { return };
1145                let sig = head(outer, src, "__none__", 600).trim_end_matches(';').to_string();
1146                let sig = if d.child_by_field_name("body").is_some() {
1147                    head(d, src, "body", 600)
1148                } else {
1149                    sig
1150                };
1151                self.out
1152                    .push(entry(Kind::Function, &name, format!("{prefix}{name}"), self.file, line, sig, doc));
1153            }
1154            "class_declaration" | "abstract_class_declaration" | "class" => {
1155                let Some(name) = name_of(d) else { return };
1156                self.out.push(entry(
1157                    Kind::Class,
1158                    &name,
1159                    format!("{prefix}{name}"),
1160                    self.file,
1161                    line,
1162                    head(d, src, "body", 400),
1163                    doc,
1164                ));
1165                if let Some(body) = d.child_by_field_name("body") {
1166                    self.members(body, &format!("{prefix}{name}."));
1167                }
1168            }
1169            "interface_declaration" => {
1170                let Some(name) = name_of(d) else { return };
1171                self.out.push(entry(
1172                    Kind::Interface,
1173                    &name,
1174                    format!("{prefix}{name}"),
1175                    self.file,
1176                    line,
1177                    head(d, src, "body", 400),
1178                    doc,
1179                ));
1180                if let Some(body) = d.child_by_field_name("body") {
1181                    self.members(body, &format!("{prefix}{name}."));
1182                }
1183            }
1184            "type_alias_declaration" => {
1185                let Some(name) = name_of(d) else { return };
1186                self.out.push(entry(
1187                    Kind::Type,
1188                    &name,
1189                    format!("{prefix}{name}"),
1190                    self.file,
1191                    line,
1192                    compact(txt(d, src), 700),
1193                    doc,
1194                ));
1195            }
1196            "enum_declaration" => {
1197                let Some(name) = name_of(d) else { return };
1198                self.out.push(entry(
1199                    Kind::Enum,
1200                    &name,
1201                    format!("{prefix}{name}"),
1202                    self.file,
1203                    line,
1204                    compact(txt(d, src), 500),
1205                    doc,
1206                ));
1207            }
1208            "lexical_declaration" | "variable_declaration" => {
1209                let kw = txt(d, src).split_whitespace().next().unwrap_or("const").to_string();
1210                let mut cur = d.walk();
1211                let decls: Vec<Node> = d.named_children(&mut cur).filter(|n| n.kind() == "variable_declarator").collect();
1212                for v in decls {
1213                    let Some(nn) = v.child_by_field_name("name") else { continue };
1214                    if nn.kind() != "identifier" {
1215                        continue;
1216                    }
1217                    let name = txt(nn, src).to_string();
1218                    let value = v.child_by_field_name("value");
1219                    let is_fn = value.is_some_and(|x| matches!(x.kind(), "arrow_function" | "function_expression" | "function"))
1220                        || v.child_by_field_name("type").is_some_and(|t| txt(t, src).contains("=>"));
1221                    let sig = match value {
1222                        Some(val) if matches!(val.kind(), "arrow_function" | "function_expression" | "function") => {
1223                            let body_start = val.child_by_field_name("body").map_or(val.end_byte(), |b| b.start_byte());
1224                            let s = std::str::from_utf8(&src[v.start_byte()..body_start]).unwrap_or("");
1225                            compact(&format!("{kw} {}", s.trim_end().trim_end_matches("=>").trim_end()), 600)
1226                        }
1227                        Some(val) if val.end_byte() - val.start_byte() > 300 => {
1228                            let s = std::str::from_utf8(&src[v.start_byte()..val.start_byte()]).unwrap_or("");
1229                            compact(&format!("{kw} {s} …"), 600)
1230                        }
1231                        _ => compact(&format!("{kw} {}", txt(v, src)), 600),
1232                    };
1233                    let kind = if is_fn { Kind::Function } else { Kind::Const };
1234                    self.out.push(entry(kind, &name, format!("{prefix}{name}"), self.file, line, sig, doc.clone()));
1235                }
1236            }
1237            "internal_module" | "module" => {
1238                let Some(nn) = d.child_by_field_name("name") else { return };
1239                let raw = txt(nn, src);
1240                let quoted = raw.starts_with('"') || raw.starts_with('\'');
1241                let name = raw.trim_matches(['"', '\'']).to_string();
1242                let new_prefix = if quoted { prefix.to_string() } else { format!("{prefix}{name}.") };
1243                if !quoted {
1244                    self.out.push(entry(
1245                        Kind::Module,
1246                        &name,
1247                        format!("{prefix}{name}"),
1248                        self.file,
1249                        line,
1250                        head(d, src, "body", 200),
1251                        doc,
1252                    ));
1253                }
1254                if let Some(body) = d.child_by_field_name("body") {
1255                    self.block(body, &new_prefix);
1256                }
1257            }
1258            _ => {}
1259        }
1260    }
1261
1262    fn members(&mut self, body: Node, prefix: &str) {
1263        let src = self.src;
1264        let mut cur = body.walk();
1265        let children: Vec<Node> = body.named_children(&mut cur).collect();
1266        for m in children {
1267            let kind = match m.kind() {
1268                "method_definition" | "method_signature" | "abstract_method_signature" => Kind::Method,
1269                "public_field_definition" | "property_signature" => Kind::Field,
1270                _ => continue,
1271            };
1272            let t = txt(m, src);
1273            if t.starts_with("private ") || t.starts_with("protected ") || t.contains(" private ") && kind == Kind::Field {
1274                continue;
1275            }
1276            let Some(nn) = m.child_by_field_name("name") else { continue };
1277            let name = txt(nn, src).trim_matches(['"', '\'']).to_string();
1278            if name.starts_with('#') || name.starts_with('_') || name.starts_with('[') {
1279                continue;
1280            }
1281            let doc = self.doc(m);
1282            let sig = if m.child_by_field_name("body").is_some() {
1283                head(m, src, "body", 500)
1284            } else {
1285                compact(t.trim_end_matches(';'), 500)
1286            };
1287            self.out.push(entry(
1288                kind,
1289                &name,
1290                format!("{prefix}{name}"),
1291                self.file,
1292                m.start_position().row as u32 + 1,
1293                sig,
1294                doc,
1295            ));
1296        }
1297    }
1298}
1299
1300// ---------------------------------------------------------------- Python
1301
1302fn py_module(rel: &str) -> String {
1303    let r = rel.trim_end_matches(".pyi").trim_end_matches(".py");
1304    let r = r.trim_end_matches("/__init__");
1305    r.replace('/', ".")
1306}
1307
1308fn py_file(text: &str, rel: &str, out: &mut Vec<Entry>) {
1309    let module = py_module(rel);
1310    parse_with("py", text, |root, src| {
1311        if let Some(doc) = py_docstring(root, src) {
1312            for s in markdown::split(&doc, &module) {
1313                out.push(prose_section(s, rel, 0));
1314            }
1315        }
1316        py_block(root, src, &module, false, rel, out);
1317    });
1318}
1319
1320fn py_docstring(block: Node, src: &[u8]) -> Option<String> {
1321    let first = block.named_child(0)?;
1322    let first = if first.kind() == "comment" { first.next_named_sibling()? } else { first };
1323    if first.kind() != "expression_statement" {
1324        return None;
1325    }
1326    let s = first.named_child(0)?;
1327    if s.kind() != "string" {
1328        return None;
1329    }
1330    let raw = txt(s, src);
1331    let raw = raw.trim_start_matches(['r', 'R', 'u', 'U', 'b', 'B']);
1332    let inner = raw
1333        .strip_prefix("\"\"\"")
1334        .and_then(|x| x.strip_suffix("\"\"\""))
1335        .or_else(|| raw.strip_prefix("'''").and_then(|x| x.strip_suffix("'''")))
1336        .or_else(|| raw.strip_prefix('"').and_then(|x| x.strip_suffix('"')))
1337        .or_else(|| raw.strip_prefix('\'').and_then(|x| x.strip_suffix('\'')))?;
1338    let lines: Vec<&str> = inner.lines().collect();
1339    if lines.is_empty() {
1340        return None;
1341    }
1342    // inspect.cleandoc: first line as is, the rest dedented.
1343    let first_line = lines[0].trim();
1344    let rest = dedent(&lines[1..]);
1345    let gap = if lines.get(1).is_some_and(|l| l.trim().is_empty()) { "\n\n" } else { "\n" };
1346    let doc = if rest.is_empty() {
1347        first_line.to_string()
1348    } else if first_line.is_empty() {
1349        rest
1350    } else {
1351        format!("{first_line}{gap}{rest}")
1352    };
1353    Some(cap_doc(doc))
1354}
1355
1356fn py_block(block: Node, src: &[u8], prefix: &str, in_class: bool, rel: &str, out: &mut Vec<Entry>) {
1357    let mut cur = block.walk();
1358    let children: Vec<Node> = block.named_children(&mut cur).collect();
1359    for c in children {
1360        let (def, decorators) = if c.kind() == "decorated_definition" {
1361            let mut dc = c.walk();
1362            let decs: Vec<String> = c
1363                .named_children(&mut dc)
1364                .filter(|n| n.kind() == "decorator")
1365                .map(|n| compact(txt(n, src), 80))
1366                .collect();
1367            match c.child_by_field_name("definition") {
1368                Some(d) => (d, decs),
1369                None => continue,
1370            }
1371        } else {
1372            (c, Vec::new())
1373        };
1374        let Some(nn) = def.child_by_field_name("name") else { continue };
1375        let name = txt(nn, src);
1376        let dunder_ok = matches!(name, "__init__" | "__call__");
1377        if name.starts_with('_') && !dunder_ok {
1378            continue;
1379        }
1380        let line = c.start_position().row as u32 + 1;
1381        let mut sig = decorators.join(" ");
1382        match def.kind() {
1383            "function_definition" => {
1384                if !sig.is_empty() {
1385                    sig.push(' ');
1386                }
1387                let end = def.child_by_field_name("body").map_or(def.end_byte(), |b| b.start_byte());
1388                let raw = strip_py_comments(std::str::from_utf8(&src[def.start_byte()..end]).unwrap_or(""));
1389                sig.push_str(&compact(raw.trim_end().trim_end_matches(':').trim_end(), 700));
1390                let doc = def.child_by_field_name("body").and_then(|b| py_docstring(b, src)).unwrap_or_default();
1391                let kind = if in_class { Kind::Method } else { Kind::Function };
1392                out.push(entry(kind, name, format!("{prefix}.{name}"), rel, line, sig, doc));
1393            }
1394            "class_definition" => {
1395                if !sig.is_empty() {
1396                    sig.push(' ');
1397                }
1398                sig.push_str(&head(def, src, "body", 400));
1399                let body = def.child_by_field_name("body");
1400                let doc = body.and_then(|b| py_docstring(b, src)).unwrap_or_default();
1401                let path = format!("{prefix}.{name}");
1402                out.push(entry(Kind::Class, name, path.clone(), rel, line, sig, doc));
1403                if let Some(b) = body {
1404                    py_block(b, src, &path, true, rel, out);
1405                }
1406            }
1407            _ => {}
1408        }
1409    }
1410}
1411
1412// ---------------------------------------------------------------- Rust
1413
1414fn rs_module(rel: &str, crate_name: &str) -> String {
1415    let r = rel.strip_prefix("src/").unwrap_or(rel).trim_end_matches(".rs");
1416    let mut segs: Vec<&str> = r.split('/').collect();
1417    if matches!(segs.last(), Some(&"mod") | Some(&"lib") | Some(&"main")) {
1418        segs.pop();
1419    }
1420    let mut p = crate_name.to_string();
1421    for s in segs {
1422        if !s.is_empty() {
1423            p.push_str("::");
1424            p.push_str(s);
1425        }
1426    }
1427    p
1428}
1429
1430fn rs_file(text: &str, rel: &str, crate_name: &str, out: &mut Vec<Entry>) {
1431    let module = rs_module(rel, crate_name);
1432    rs_parse(text, rel, &module, 0, 0, None, out);
1433}
1434
1435/// Docs and `#[macro_export]` a wrapper macro applies to the items it wraps
1436/// (tokio's `doc! { macro_rules! select { .. } }` pattern).
1437type Inherit = Option<(String, bool)>;
1438
1439fn rs_parse(text: &str, rel: &str, module: &str, line_off: u32, depth: u32, inherit: Inherit, out: &mut Vec<Entry>) {
1440    parse_with("rs", text, |root, src| {
1441        let mut inner_doc = Vec::new();
1442        let mut w = RsWalker {
1443            src,
1444            file: rel,
1445            line_off,
1446            depth,
1447            out: Vec::new(),
1448            wrappers: HashMap::new(),
1449            inherit,
1450        };
1451        w.block(root, module, None, &mut inner_doc);
1452        if !inner_doc.is_empty() {
1453            let doc = inner_doc.join("\n");
1454            for s in markdown::split(&doc, module) {
1455                out.push(prose_section(s, rel, line_off));
1456            }
1457        }
1458        out.append(&mut w.out);
1459    });
1460}
1461
1462struct RsWalker<'a> {
1463    src: &'a [u8],
1464    file: &'a str,
1465    line_off: u32,
1466    depth: u32,
1467    out: Vec<Entry>,
1468    /// Local `macro_rules!` that wrap an `$item` with docs: name -> (doc, exported).
1469    wrappers: HashMap<String, (String, bool)>,
1470    inherit: Inherit,
1471}
1472
1473/// A local macro like `macro_rules! doc { ($x:item) => { /// docs  #[macro_export] $x } }`.
1474fn doc_wrapper(text: &str) -> Option<(String, bool)> {
1475    if !text.contains(":item") {
1476        return None;
1477    }
1478    let doc: Vec<&str> = text
1479        .lines()
1480        .filter_map(|l| strip_line_doc(l.trim()).filter(|(_, inner)| !inner).map(|(d, _)| d))
1481        .collect();
1482    if doc.is_empty() {
1483        return None;
1484    }
1485    Some((cap_doc(doc.join("\n").trim().to_string()), text.contains("macro_export")))
1486}
1487
1488fn is_pub(n: Node, src: &[u8]) -> bool {
1489    let mut cur = n.walk();
1490    let r = n
1491        .named_children(&mut cur)
1492        .any(|c| c.kind() == "visibility_modifier" && txt(c, src).trim() == "pub");
1493    r
1494}
1495
1496fn strip_line_doc(t: &str) -> Option<(&str, bool)> {
1497    if let Some(r) = t.strip_prefix("///") {
1498        if r.starts_with('/') {
1499            return None;
1500        }
1501        return Some((r.strip_prefix(' ').unwrap_or(r).trim_end(), false));
1502    }
1503    if let Some(r) = t.strip_prefix("//!") {
1504        return Some((r.strip_prefix(' ').unwrap_or(r).trim_end(), true));
1505    }
1506    None
1507}
1508
1509impl RsWalker<'_> {
1510    fn line(&self, n: Node) -> u32 {
1511        n.start_position().row as u32 + 1 + self.line_off
1512    }
1513
1514    /// `owner`: impl/trait type whose members we are in (`Some((name, is_trait))`).
1515    fn block(&mut self, node: Node, module: &str, owner: Option<(&str, bool)>, inner_doc: &mut Vec<String>) {
1516        let src = self.src;
1517        let mut cur = node.walk();
1518        let children: Vec<Node> = node.named_children(&mut cur).collect();
1519        let mut pending: Vec<String> = Vec::new();
1520        let mut hidden = false;
1521        let mut exported_macro = false;
1522        let mut first_line: Option<u32> = None;
1523        for c in children {
1524            let k = c.kind();
1525            match k {
1526                "line_comment" => {
1527                    let t = txt(c, src);
1528                    if let Some((d, inner)) = strip_line_doc(t) {
1529                        if inner {
1530                            inner_doc.push(d.to_string());
1531                        } else {
1532                            if pending.is_empty() {
1533                                first_line = Some(self.line(c));
1534                            }
1535                            pending.push(d.to_string());
1536                        }
1537                    }
1538                    continue;
1539                }
1540                "block_comment" => {
1541                    let t = txt(c, src);
1542                    if t.starts_with("/*!") {
1543                        inner_doc.push(clean_block_comment(t));
1544                    } else if t.starts_with("/**") && !t.starts_with("/***") {
1545                        pending.push(clean_block_comment(t));
1546                    }
1547                    continue;
1548                }
1549                "attribute_item" => {
1550                    let t = txt(c, src);
1551                    if t.contains("doc(hidden)") {
1552                        hidden = true;
1553                    }
1554                    if t.contains("macro_export") {
1555                        exported_macro = true;
1556                    }
1557                    // `#[doc = "..."]`
1558                    if let Some(rest) = t.strip_prefix("#[doc = \"") {
1559                        pending.push(rest.trim_end_matches("\"]").replace("\\n", "\n"));
1560                    }
1561                    continue;
1562                }
1563                "inner_attribute_item" => continue,
1564                _ => {}
1565            }
1566            let mut doc = cap_doc(pending.join("\n").trim().to_string());
1567            if doc.is_empty() {
1568                if let Some((d, _)) = &self.inherit {
1569                    doc = d.clone();
1570                }
1571            }
1572            let _ = first_line.take();
1573            pending.clear();
1574            let was_hidden = std::mem::take(&mut hidden);
1575            let was_macro_export = std::mem::take(&mut exported_macro) || self.inherit.as_ref().is_some_and(|i| i.1);
1576            if was_hidden {
1577                continue;
1578            }
1579            let line = self.line(c);
1580            let sep = "::";
1581            let in_trait = owner.is_some_and(|o| o.1);
1582            let public = in_trait || is_pub(c, src);
1583            let name = c.child_by_field_name("name").map(|n| txt(n, src).to_string());
1584            let qual = |name: &str| match owner {
1585                Some((o, _)) => format!("{module}{sep}{o}{sep}{name}"),
1586                None => format!("{module}{sep}{name}"),
1587            };
1588            match k {
1589                "function_item" | "function_signature_item" => {
1590                    let (Some(name), true) = (name, public) else { continue };
1591                    let kind = if owner.is_some() { Kind::Method } else { Kind::Function };
1592                    self.out.push(entry(
1593                        kind,
1594                        &name,
1595                        qual(&name),
1596                        self.file,
1597                        line,
1598                        head(c, src, "body", 600).trim_end_matches(';').to_string(),
1599                        doc,
1600                    ));
1601                }
1602                "struct_item" | "enum_item" | "union_item" => {
1603                    let (Some(name), true) = (name, public) else { continue };
1604                    let kind = if k == "enum_item" { Kind::Enum } else { Kind::Struct };
1605                    let full = txt(c, src);
1606                    let sig = if full.len() <= 900 {
1607                        compact(full, 900)
1608                    } else {
1609                        format!("{} {{ … }}", head(c, src, "body", 400))
1610                    };
1611                    self.out.push(entry(kind, &name, qual(&name), self.file, line, sig, doc));
1612                }
1613                "trait_item" => {
1614                    let (Some(name), true) = (name, public) else { continue };
1615                    self.out
1616                        .push(entry(Kind::Trait, &name, qual(&name), self.file, line, head(c, src, "body", 400), doc));
1617                    if let Some(b) = c.child_by_field_name("body") {
1618                        let mut ignore = Vec::new();
1619                        self.block(b, module, Some((&name, true)), &mut ignore);
1620                    }
1621                }
1622                "impl_item" => {
1623                    if c.child_by_field_name("trait").is_some() {
1624                        continue;
1625                    }
1626                    let Some(ty) = c.child_by_field_name("type") else { continue };
1627                    let ty = txt(ty, src);
1628                    let ty = ty.split('<').next().unwrap_or(ty).trim().to_string();
1629                    if let Some(b) = c.child_by_field_name("body") {
1630                        let mut ignore = Vec::new();
1631                        self.block(b, module, Some((&ty, false)), &mut ignore);
1632                    }
1633                }
1634                "mod_item" => {
1635                    let (Some(name), true) = (name, public) else { continue };
1636                    let path = format!("{module}{sep}{name}");
1637                    if !doc.is_empty() {
1638                        self.out
1639                            .push(entry(Kind::Module, &name, path.clone(), self.file, line, format!("pub mod {name}"), doc));
1640                    }
1641                    if let Some(b) = c.child_by_field_name("body") {
1642                        let mut inner = Vec::new();
1643                        self.block(b, &path, None, &mut inner);
1644                    }
1645                }
1646                "type_item" | "const_item" | "static_item" => {
1647                    let (Some(name), true) = (name, public) else { continue };
1648                    let kind = if k == "type_item" { Kind::Type } else { Kind::Const };
1649                    self.out.push(entry(
1650                        kind,
1651                        &name,
1652                        qual(&name),
1653                        self.file,
1654                        line,
1655                        compact(txt(c, src).trim_end_matches(';'), 400),
1656                        doc,
1657                    ));
1658                }
1659                "macro_definition" => {
1660                    let Some(name) = name else { continue };
1661                    if let Some(w) = doc_wrapper(txt(c, src)) {
1662                        self.wrappers.insert(name.clone(), w);
1663                        continue;
1664                    }
1665                    if !was_macro_export {
1666                        continue;
1667                    }
1668                    // Exported macros live at the crate root.
1669                    let root = module.split("::").next().unwrap_or(module);
1670                    let arms = compact(txt(c, src), 300);
1671                    self.out.push(entry(Kind::Macro, &name, format!("{root}::{name}!"), self.file, line, arms, doc));
1672                }
1673                "macro_invocation" if self.depth < 3 => {
1674                    // cfg_rt! { pub fn spawn ... } and friends: parse the body as items.
1675                    let Some(tt) = c.named_children(&mut c.walk()).find(|n| n.kind() == "token_tree") else {
1676                        continue;
1677                    };
1678                    let body = txt(tt, src);
1679                    let called = c.child_by_field_name("macro").map(|m| txt(m, src).to_string()).unwrap_or_default();
1680                    let wrapper = self.wrappers.get(&called).cloned();
1681                    let itemish =
1682                        body.contains("fn ") || body.contains("struct ") || body.contains("mod ") || body.contains("trait ") || body.contains("macro_rules!");
1683                    if body.len() < 4 || !itemish {
1684                        continue;
1685                    }
1686                    let inner = &body[1..body.len() - 1];
1687                    let off = tt.start_position().row as u32 + self.line_off;
1688                    let mut sub = Vec::new();
1689                    rs_parse(
1690                        inner,
1691                        self.file,
1692                        module,
1693                        off,
1694                        self.depth + 1,
1695                        wrapper.or_else(|| self.inherit.clone()),
1696                        &mut sub,
1697                    );
1698                    // Owner-less items only: prose from nested inner docs stays.
1699                    self.out.extend(sub);
1700                }
1701                _ => {}
1702            }
1703        }
1704    }
1705}
1706
1707// ---------------------------------------------------------------- Go
1708
1709fn exported(name: &str) -> bool {
1710    name.chars().next().is_some_and(|c| c.is_uppercase())
1711}
1712
1713fn go_doc(n: Node, src: &[u8]) -> String {
1714    let mut lines: Vec<String> = Vec::new();
1715    let mut p = n.prev_sibling();
1716    let mut next_row = n.start_position().row;
1717    while let Some(c) = p {
1718        if c.kind() != "comment" || c.end_position().row + 1 != next_row {
1719            break;
1720        }
1721        let t = txt(c, src);
1722        if let Some(r) = t.strip_prefix("//") {
1723            lines.push(r.strip_prefix(' ').unwrap_or(r).trim_end().to_string());
1724        } else {
1725            lines.push(clean_block_comment(t));
1726        }
1727        next_row = c.start_position().row;
1728        p = c.prev_sibling();
1729    }
1730    lines.reverse();
1731    cap_doc(lines.join("\n").trim().to_string())
1732}
1733
1734fn go_file(text: &str, rel: &str, module: &str, out: &mut Vec<Entry>) {
1735    parse_with("go", text, |root, src| {
1736        let mut cur = root.walk();
1737        let children: Vec<Node> = root.named_children(&mut cur).collect();
1738        let mut pkg = module.rsplit('/').next().unwrap_or(module).to_string();
1739        for c in children {
1740            let line = c.start_position().row as u32 + 1;
1741            match c.kind() {
1742                "package_clause" => {
1743                    if let Some(id) = c.named_child(0) {
1744                        pkg = txt(id, src).to_string();
1745                    }
1746                    if pkg == "main" {
1747                        return;
1748                    }
1749                    let doc = go_doc(c, src);
1750                    if !doc.is_empty() {
1751                        let dir = rel.rsplit_once('/').map_or("", |x| x.0);
1752                        let title = if dir.is_empty() {
1753                            format!("package {pkg}")
1754                        } else {
1755                            format!("package {pkg} ({dir})")
1756                        };
1757                        for s in markdown::split(&doc, &title) {
1758                            out.push(prose_section(s, rel, 0));
1759                        }
1760                    }
1761                }
1762                "function_declaration" => {
1763                    let Some(name) = c.child_by_field_name("name").map(|n| txt(n, src)) else {
1764                        continue;
1765                    };
1766                    if !exported(name) {
1767                        continue;
1768                    }
1769                    out.push(entry(
1770                        Kind::Function,
1771                        name,
1772                        format!("{pkg}.{name}"),
1773                        rel,
1774                        line,
1775                        head(c, src, "body", 600),
1776                        go_doc(c, src),
1777                    ));
1778                }
1779                "method_declaration" => {
1780                    let Some(name) = c.child_by_field_name("name").map(|n| txt(n, src)) else {
1781                        continue;
1782                    };
1783                    let recv = c.child_by_field_name("receiver").map(|r| txt(r, src)).unwrap_or("");
1784                    let ty: String = recv
1785                        .trim_matches(['(', ')'])
1786                        .split_whitespace()
1787                        .last()
1788                        .unwrap_or("")
1789                        .trim_start_matches('*')
1790                        .split('[')
1791                        .next()
1792                        .unwrap_or("")
1793                        .to_string();
1794                    if !exported(name) || !exported(&ty) {
1795                        continue;
1796                    }
1797                    out.push(entry(
1798                        Kind::Method,
1799                        name,
1800                        format!("{pkg}.{ty}.{name}"),
1801                        rel,
1802                        line,
1803                        head(c, src, "body", 600),
1804                        go_doc(c, src),
1805                    ));
1806                }
1807                "type_declaration" | "const_declaration" | "var_declaration" => {
1808                    let outer_doc = go_doc(c, src);
1809                    let mut cc = c.walk();
1810                    let specs: Vec<Node> = c
1811                        .named_children(&mut cc)
1812                        .filter(|n| matches!(n.kind(), "type_spec" | "type_alias" | "const_spec" | "var_spec"))
1813                        .collect();
1814                    let single = specs.len() == 1;
1815                    for s in specs {
1816                        let names: Vec<String> = if s.kind().starts_with("type") {
1817                            s.child_by_field_name("name").map(|n| vec![txt(n, src).to_string()]).unwrap_or_default()
1818                        } else {
1819                            let mut sc = s.walk();
1820                            s.children_by_field_name("name", &mut sc).map(|n| txt(n, src).to_string()).collect()
1821                        };
1822                        let doc = if single { outer_doc.clone() } else { go_doc(s, src) };
1823                        let sline = s.start_position().row as u32 + 1;
1824                        for name in names.iter().filter(|n| exported(n)) {
1825                            if s.kind().starts_with("type") {
1826                                let full = txt(s, src);
1827                                let sig = if full.len() <= 900 {
1828                                    format!("type {}", compact(full, 900))
1829                                } else {
1830                                    format!("type {} …", compact(&full[..full.find('{').unwrap_or(200.min(full.len()))], 300))
1831                                };
1832                                let kind = if full.contains("interface") { Kind::Interface } else { Kind::Struct };
1833                                out.push(entry(kind, name, format!("{pkg}.{name}"), rel, sline, sig, doc.clone()));
1834                                // Interface methods
1835                                if let Some(t) = s.child_by_field_name("type").filter(|t| t.kind() == "interface_type") {
1836                                    let mut ic = t.walk();
1837                                    let elems: Vec<Node> = t.named_children(&mut ic).collect();
1838                                    for m in elems {
1839                                        if !matches!(m.kind(), "method_elem" | "method_spec") {
1840                                            continue;
1841                                        }
1842                                        let Some(mn) = m.child_by_field_name("name").map(|n| txt(n, src)) else {
1843                                            continue;
1844                                        };
1845                                        out.push(entry(
1846                                            Kind::Method,
1847                                            mn,
1848                                            format!("{pkg}.{name}.{mn}"),
1849                                            rel,
1850                                            m.start_position().row as u32 + 1,
1851                                            compact(txt(m, src), 400),
1852                                            go_doc(m, src),
1853                                        ));
1854                                    }
1855                                }
1856                            } else {
1857                                let kw = if c.kind() == "const_declaration" { "const" } else { "var" };
1858                                out.push(entry(
1859                                    Kind::Const,
1860                                    name,
1861                                    format!("{pkg}.{name}"),
1862                                    rel,
1863                                    sline,
1864                                    compact(&format!("{kw} {}", txt(s, src)), 300),
1865                                    doc.clone(),
1866                                ));
1867                            }
1868                        }
1869                    }
1870                }
1871                _ => {}
1872            }
1873        }
1874    });
1875}
1876
1877#[cfg(test)]
1878mod tests {
1879    use super::*;
1880
1881    fn run(f: fn(&str, &str, &mut Vec<Entry>), text: &str, rel: &str) -> Vec<Entry> {
1882        let mut out = Vec::new();
1883        f(text, rel, &mut out);
1884        out
1885    }
1886
1887    #[test]
1888    fn typescript_declarations() {
1889        let src = r#"
1890/** Creates an object schema. */
1891export declare function object<T extends ZodRawShape>(shape: T, params?: string): ZodObject<T>;
1892export declare namespace z {
1893  /** A string schema. */
1894  function string(): ZodString;
1895}
1896export interface ZodType<T> {
1897  /** Parse or throw. */
1898  parse(data: unknown): T;
1899  readonly _def: Def;
1900}
1901export declare class ZodObject<T> extends ZodType<T> {
1902  /** Disallow unknown keys. */
1903  strict(message?: string): ZodObject<T>;
1904  private _cached;
1905}
1906declare const objectType: <T>(shape: T) => ZodObject<T>;
1907export { objectType as obj };
1908export * as zz from "./external";
1909"#;
1910        let e = run(ts_file, src, "lib/index.d.ts");
1911        let get = |p: &str| {
1912            e.iter()
1913                .find(|x| x.path == p)
1914                .unwrap_or_else(|| panic!("missing {p}: {:#?}", e.iter().map(|x| &x.path).collect::<Vec<_>>()))
1915        };
1916        assert_eq!(get("object").doc, "Creates an object schema.");
1917        assert!(get("object").sig.contains("ZodObject<T>"));
1918        assert_eq!(get("z.string").doc, "A string schema.");
1919        assert_eq!(get("ZodType.parse").kind, Kind::Method);
1920        assert!(e.iter().all(|x| x.name != "_def" && x.name != "_cached"));
1921        assert_eq!(get("ZodObject.strict").doc, "Disallow unknown keys.");
1922        assert_eq!(get("objectType").kind, Kind::Function);
1923        assert_eq!(get("obj").alias_of.as_deref(), Some("objectType"));
1924        assert_eq!(get("zz").kind, Kind::Module);
1925        assert_eq!(get("object").line, 3);
1926    }
1927
1928    #[test]
1929    fn legacy_dirs() {
1930        assert!(legacy_file("v3/types.d.ts", 4));
1931        assert!(!legacy_file("v4/classic/schemas.d.ts", 4));
1932        assert!(legacy_file("pydantic/v1/main.py", 2));
1933        assert!(!legacy_file("pydantic/main.py", 2));
1934        assert!(!legacy_file("src/v2/x.rs", 1));
1935    }
1936
1937    #[test]
1938    fn python_definitions() {
1939        let src = "\"\"\"Module docs.\"\"\"\n\nclass BaseModel:\n    \"\"\"Base class.\n\n    More.\n    \"\"\"\n\n    def model_dump(self, *, mode: str = 'python') -> dict:\n        \"\"\"Dump the model.\"\"\"\n        return {}\n\n    @classmethod\n    def model_validate(cls, obj):\n        '''Validate.'''\n\n    def _private(self): pass\n\ndef create_model(name: str, **fields) -> type:\n    return None\n";
1940        let e = run(py_file, src, "pydantic/main.py");
1941        let get = |p: &str| e.iter().find(|x| x.path == p).unwrap_or_else(|| panic!("missing {p}"));
1942        assert_eq!(get("pydantic.main.BaseModel").doc, "Base class.\n\nMore.");
1943        assert_eq!(
1944            get("pydantic.main.BaseModel.model_dump").sig,
1945            "def model_dump(self, *, mode: str = 'python') -> dict"
1946        );
1947        assert!(get("pydantic.main.BaseModel.model_validate").sig.starts_with("@classmethod def model_validate"));
1948        assert!(e.iter().all(|x| x.name != "_private"));
1949        assert!(e.iter().any(|x| x.kind == Kind::Prose && x.doc == "Module docs."));
1950        assert_eq!(get("pydantic.main.create_model").kind, Kind::Function);
1951    }
1952
1953    #[test]
1954    fn docs_pages_written_as_components() {
1955        let src = r#"import dedent from "dedent";
1956export const metadata = { title: "Installing with Vite", description: "Use the Vite plugin.", openGraph: { title: "x" } };
1957const steps = [
1958  {
1959    title: "Import Tailwind CSS",
1960    body: (
1961      <p>
1962        Add an <code>@import</code> to your CSS file.
1963      </p>
1964    ),
1965    code: { name: "CSS", lang: "css", code: dedent`
1966        @import "tailwindcss";
1967      ` },
1968  },
1969  { title: 'Old way', body: () => <p>Use directives{' '}here.</p>, code: { lang: 'css', code: '@tailwind base;\n@tailwind utilities;' } },
1970];
1971export default function Page() { return <div className="prose"><h3 className="sr-only">Overview</h3><p>Fast.</p></div> }
1972"#;
1973        let (md, title) = page_markdown(src).unwrap();
1974        assert_eq!(title.as_deref(), Some("Installing with Vite"));
1975        assert!(md.contains("## Import Tailwind CSS"), "{md}");
1976        assert!(md.contains("Add an `@import` to your CSS file."), "{md}");
1977        assert!(md.contains("```css\n@import \"tailwindcss\";\n```"), "{md}");
1978        assert!(md.contains("@tailwind base;\n@tailwind utilities;"), "{md}");
1979        assert!(md.contains("### Overview"), "{md}");
1980        assert!(!md.contains("dedent") && !md.contains("prose"), "{md}");
1981        let mut out = Vec::new();
1982        prose_titled(&md, "upstream:site/docs/installation/page.tsx", 0, title, &mut out);
1983        let s = out.iter().find(|e| e.name == "Installing with Vite › Import Tailwind CSS").unwrap();
1984        assert_eq!(s.line, 5);
1985        assert_eq!(
1986            page_title("upstream:site/src/app/(docs)/docs/installation/(tabs)/using-vite/page.tsx"),
1987            "installation/using-vite"
1988        );
1989    }
1990
1991    #[test]
1992    fn rust_items_and_macro_bodies() {
1993        let src = "//! Crate docs.\n//!\n//! # Usage\n//! Call spawn.\n\ncfg_rt! {\n    /// Spawns a new task.\n    #[track_caller]\n    pub fn spawn<F>(future: F) -> JoinHandle<F::Output>\n    where F: Future + Send + 'static {\n        todo!()\n    }\n}\n\n/// A router.\npub struct Router<S = ()> { inner: Inner }\n\nimpl<S> Router<S> {\n    /// Add a route.\n    pub fn route(self, path: &str, m: MethodRouter<S>) -> Self { self }\n    fn private(&self) {}\n}\n\n/// Hidden\n#[doc(hidden)]\npub fn hidden() {}\n\n/// Macro.\n#[macro_export]\nmacro_rules! select { () => {} }\n";
1994        let mut e = Vec::new();
1995        rs_file(src, "src/task/spawn.rs", "tokio", &mut e);
1996        let get = |p: &str| {
1997            e.iter()
1998                .find(|x| x.path == p)
1999                .unwrap_or_else(|| panic!("missing {p}: {:#?}", e.iter().map(|x| &x.path).collect::<Vec<_>>()))
2000        };
2001        let spawn = get("tokio::task::spawn::spawn");
2002        assert_eq!(spawn.doc, "Spawns a new task.");
2003        assert_eq!(spawn.line, 9);
2004        assert!(spawn.sig.contains("JoinHandle"));
2005        assert_eq!(get("tokio::task::spawn::Router::route").doc, "Add a route.");
2006        assert!(e.iter().all(|x| x.name != "private" && x.name != "hidden"));
2007        assert_eq!(get("tokio::select!").kind, Kind::Macro);
2008        assert!(e.iter().any(|x| x.kind == Kind::Prose && x.path.ends_with("Usage")));
2009
2010        // tokio's doc-wrapper pattern
2011        let src = "macro_rules! doc {\n    ($select:item) => {\n        /// Waits on multiple branches.\n        #[macro_export]\n        $select\n    };\n}\n\ndoc! {macro_rules! select {\n    () => {};\n}}\n";
2012        let mut e = Vec::new();
2013        rs_file(src, "src/macros/select.rs", "tokio", &mut e);
2014        let sel = e.iter().find(|x| x.path == "tokio::select!").expect("select! extracted");
2015        assert_eq!(sel.doc, "Waits on multiple branches.");
2016        assert!(e.iter().all(|x| x.name != "doc"));
2017    }
2018
2019    #[test]
2020    fn go_declarations() {
2021        let src = "// Package gin implements a web framework.\npackage gin\n\n// Context is the request context.\ntype Context struct {\n\tKeys map[string]any\n}\n\n// JSON serializes obj.\nfunc (c *Context) JSON(code int, obj any) {\n}\n\n// Default returns an Engine.\nfunc Default(opts ...Option) *Engine {\n\treturn nil\n}\n\nfunc internal() {}\n\ntype Handler interface {\n\t// Serve serves.\n\tServe(c *Context)\n}\n";
2022        let mut e = Vec::new();
2023        go_file(src, "context.go", "github.com/gin-gonic/gin", &mut e);
2024        let get = |p: &str| {
2025            e.iter()
2026                .find(|x| x.path == p)
2027                .unwrap_or_else(|| panic!("missing {p}: {:#?}", e.iter().map(|x| &x.path).collect::<Vec<_>>()))
2028        };
2029        assert_eq!(get("gin.Context.JSON").doc, "JSON serializes obj.");
2030        assert_eq!(get("gin.Default").sig, "func Default(opts ...Option) *Engine");
2031        assert_eq!(get("gin.Context").doc, "Context is the request context.");
2032        assert_eq!(get("gin.Handler.Serve").doc, "Serve serves.");
2033        assert!(e.iter().all(|x| x.name != "internal"));
2034        assert!(e.iter().any(|x| x.kind == Kind::Prose && x.doc.contains("web framework")));
2035    }
2036}