Skip to main content

lockdocs_core/
extract.rs

1//! Turn a package's files into entries: API symbols (signature + doc comment)
2//! from TypeScript declarations, JavaScript, Python, Rust and Go sources, and
3//! prose sections from READMEs, changelogs and docs folders.
4
5use crate::locate::Source;
6use crate::markdown;
7use crate::Eco;
8use rayon::prelude::*;
9use serde::{Deserialize, Serialize};
10use std::cell::RefCell;
11use std::collections::{HashMap, HashSet};
12use std::path::{Path, PathBuf};
13use tree_sitter::{Node, Parser};
14
15#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
16pub enum Kind {
17    Function,
18    Method,
19    Class,
20    Interface,
21    Type,
22    Const,
23    Enum,
24    Struct,
25    Trait,
26    Macro,
27    Module,
28    Field,
29    Alias,
30    Prose,
31}
32
33impl Kind {
34    pub fn as_str(self) -> &'static str {
35        match self {
36            Kind::Function => "function",
37            Kind::Method => "method",
38            Kind::Class => "class",
39            Kind::Interface => "interface",
40            Kind::Type => "type",
41            Kind::Const => "const",
42            Kind::Enum => "enum",
43            Kind::Struct => "struct",
44            Kind::Trait => "trait",
45            Kind::Macro => "macro",
46            Kind::Module => "module",
47            Kind::Field => "field",
48            Kind::Alias => "re-export",
49            Kind::Prose => "doc",
50        }
51    }
52    pub fn is_container(self) -> bool {
53        matches!(self, Kind::Class | Kind::Interface | Kind::Struct | Kind::Trait | Kind::Enum | Kind::Module)
54    }
55}
56
57#[derive(Debug, Clone, Serialize, Deserialize)]
58pub struct Entry {
59    pub kind: Kind,
60    /// Short name (`object`, `spawn`), or the heading path for prose.
61    pub name: String,
62    /// Qualified path: `z.object`, `tokio::task::spawn`, `pydantic.main.BaseModel.model_dump`.
63    pub path: String,
64    /// File relative to the package root.
65    pub file: String,
66    pub line: u32,
67    /// Declaration text without the body.
68    pub sig: String,
69    /// Doc comment, or the prose section text.
70    pub doc: String,
71    /// For re-exports: the local name this entry points at.
72    pub alias_of: Option<String>,
73    /// Lives in a legacy compatibility copy (`zod/v3` inside zod 4,
74    /// `pydantic/v1` inside pydantic 2): ranked below the current API.
75    #[serde(default)]
76    pub legacy: bool,
77}
78
79/// A path segment `vN` with N below the package's major version.
80fn legacy_file(file: &str, major: u64) -> bool {
81    major > 1
82        && file.split('/').any(|seg| {
83            seg.len() >= 2 && seg.starts_with('v') && seg[1..].chars().all(|c| c.is_ascii_digit()) && seg[1..].parse::<u64>().is_ok_and(|n| n < major)
84        })
85}
86
87// ---------------------------------------------------------------- file selection
88
89const SKIP_DIRS: &[&str] = &[
90    "node_modules",
91    "compiled",
92    ".git",
93    "test",
94    "tests",
95    "__tests__",
96    "__test__",
97    "testdata",
98    "test_data",
99    "benches",
100    "bench",
101    "benchmarks",
102    "fixtures",
103    "__fixtures__",
104    "__pycache__",
105    "examples",
106    "example",
107    "_examples",
108    "coverage",
109    ".github",
110    "vendor",
111    "internal",
112    "__snapshots__",
113];
114
115const MAX_FILES: usize = 25_000;
116const MAX_FILE_BYTES: u64 = 3 * 1024 * 1024;
117
118fn is_doc_name(name: &str) -> bool {
119    let u = name.to_ascii_uppercase();
120    [
121        "README",
122        "CHANGELOG",
123        "CHANGES",
124        "HISTORY",
125        "MIGRATION",
126        "MIGRATING",
127        "UPGRADING",
128        "UPGRADE",
129        "RELEASES",
130        "NEWS",
131        "GUIDE",
132        "API",
133    ]
134    .iter()
135    .any(|p| u.starts_with(p))
136}
137
138fn doc_ext(ext: &str) -> bool {
139    matches!(ext, "md" | "mdx" | "markdown" | "rst")
140}
141
142fn walk(dir: &Path, rel: &Path, depth: usize, out: &mut Vec<PathBuf>, public_npm: bool) {
143    if depth > 14 || out.len() >= MAX_FILES {
144        return;
145    }
146    let Ok(rd) = std::fs::read_dir(dir) else { return };
147    let mut entries: Vec<_> = rd.flatten().collect();
148    entries.sort_by_key(|e| e.file_name());
149    for e in entries {
150        let name = e.file_name().to_string_lossy().to_string();
151        let Ok(ft) = e.file_type() else { continue };
152        let r = rel.join(&name);
153        if ft.is_dir() {
154            if (SKIP_DIRS.contains(&name.as_str()) && !(public_npm && name == "internal")) || name.starts_with('.') {
155                continue;
156            }
157            walk(&e.path(), &r, depth + 1, out, public_npm);
158        } else if ft.is_file() || ft.is_symlink() {
159            out.push(r);
160        }
161        if out.len() >= MAX_FILES {
162            return;
163        }
164    }
165}
166
167/// Every file under `dir` (docs trees keep folders like `examples/` we want).
168fn walk_all(dir: &Path, rel: &Path, depth: usize, out: &mut Vec<PathBuf>) {
169    if depth > 16 || out.len() >= MAX_FILES {
170        return;
171    }
172    let Ok(rd) = std::fs::read_dir(dir) else { return };
173    let mut entries: Vec<_> = rd.flatten().collect();
174    entries.sort_by_key(|e| e.file_name());
175    for e in entries {
176        let name = e.file_name().to_string_lossy().to_string();
177        if name.starts_with('.') {
178            continue;
179        }
180        let r = rel.join(&name);
181        match e.file_type() {
182            Ok(t) if t.is_dir() => walk_all(&e.path(), &r, depth + 1, out),
183            Ok(_) => out.push(r),
184            _ => {}
185        }
186    }
187}
188
189fn ext_of(p: &Path) -> String {
190    let s = p.file_name().map(|f| f.to_string_lossy().to_ascii_lowercase()).unwrap_or_default();
191    for e in [".d.ts", ".d.mts", ".d.cts"] {
192        if s.ends_with(e) {
193            return e[1..].to_string();
194        }
195    }
196    p.extension().map(|e| e.to_string_lossy().to_ascii_lowercase()).unwrap_or_default()
197}
198
199fn in_docs_dir(p: &Path) -> bool {
200    p.components().any(|c| {
201        matches!(
202            c.as_os_str().to_string_lossy().to_ascii_lowercase().as_str(),
203            "docs" | "doc" | "documentation" | "guide" | "guides"
204        )
205    })
206}
207
208/// A unit of work: one file to read, how to parse it, and how to label it.
209struct Job {
210    abs: PathBuf,
211    rel: String,
212    how: How,
213}
214
215#[derive(Clone, Copy, PartialEq)]
216enum How {
217    Prose,
218    /// A docs page written as a JS/TSX component.
219    Page,
220    Example,
221    Metadata,
222    Ts,
223    Python,
224    Rust,
225    Go,
226}
227
228/// Plan which files to read for a package.
229fn plan(eco: Eco, src: &Source, extra_types: Option<&Path>, upstream: Option<&Path>, public_api: bool) -> Vec<Job> {
230    let mut rels = Vec::new();
231    match &src.files {
232        Some(list) => rels.extend(list.iter().cloned()),
233        None => walk(&src.dir, Path::new(""), 0, &mut rels, public_api),
234    }
235    let mut jobs = Vec::new();
236    let names: HashSet<String> = rels.iter().map(|r| r.to_string_lossy().replace('\\', "/")).collect();
237    let mut has_dts = false;
238    for r in &rels {
239        let e = ext_of(r);
240        if e.starts_with("d.") {
241            has_dts = true;
242        }
243    }
244    let rs_crate_src = eco == Eco::Cargo;
245    for r in rels {
246        let rel = r.to_string_lossy().replace('\\', "/");
247        if rel
248            .split('/')
249            .any(|seg| SKIP_DIRS.contains(&seg) && !(public_api && seg == "internal") && src.files.is_some())
250        {
251            continue;
252        }
253        let abs = src.dir.join(&r);
254        let fname = r.file_name().map(|f| f.to_string_lossy().to_string()).unwrap_or_default();
255        let e = ext_of(&r);
256        let depth = rel.matches('/').count();
257        let how = if doc_ext(&e) || (e.is_empty() || e == "txt") && is_doc_name(&fname) && depth == 0 {
258            if is_doc_name(&fname) || in_docs_dir(&r) || depth == 0 {
259                Some(How::Prose)
260            } else {
261                None
262            }
263        } else {
264            match eco {
265                Eco::Npm => match e.as_str() {
266                    "d.ts" => Some(How::Ts),
267                    "d.mts" | "d.cts" => {
268                        let base = rel.trim_end_matches(".d.mts").trim_end_matches(".d.cts");
269                        (public_api || !names.contains(&format!("{base}.d.ts"))).then_some(How::Ts)
270                    }
271                    "ts" | "mts" | "cts" | "tsx" if !has_dts => Some(How::Ts),
272                    "js" | "mjs" | "cjs" if !has_dts && !fname.contains(".min.") => Some(How::Ts),
273                    _ => None,
274                },
275                Eco::PyPI => match e.as_str() {
276                    "py" => Some(How::Python),
277                    "pyi" => {
278                        let base = rel.trim_end_matches(".pyi");
279                        if names.contains(&format!("{base}.py")) {
280                            None
281                        } else {
282                            Some(How::Python)
283                        }
284                    }
285                    _ => None,
286                },
287                Eco::Cargo => (e == "rs" && rs_crate_src && rel.starts_with("src/")).then_some(How::Rust),
288                Eco::Go => (e == "go" && !fname.ends_with("_test.go")).then_some(How::Go),
289            }
290        };
291        if let Some(how) = how {
292            jobs.push(Job { abs, rel, how });
293        }
294    }
295    // Upstream docs fetched from the repository at this version's tag.
296    if let Some(up) = upstream {
297        let pages: HashSet<String> = std::fs::read_to_string(up.join(".lockdocs-upstream.json"))
298            .ok()
299            .and_then(|t| serde_json::from_str::<crate::upstream::Manifest>(&t).ok())
300            .map(|m| m.pages.into_iter().collect())
301            .unwrap_or_default();
302        let mut rels = Vec::new();
303        walk_all(up, Path::new(""), 0, &mut rels);
304        for r in rels {
305            let e = ext_of(&r);
306            let plain = r.to_string_lossy().replace('\\', "/");
307            let rel = format!("upstream:{plain}");
308            if pages.contains(&plain) {
309                jobs.push(Job {
310                    abs: up.join(&r),
311                    rel,
312                    how: How::Page,
313                });
314                continue;
315            }
316            // Django and others write reStructuredText in `.txt` files under docs/.
317            if doc_ext(&e) || e == "mdoc" || e == "txt" {
318                jobs.push(Job {
319                    abs: up.join(&r),
320                    rel,
321                    how: How::Prose,
322                });
323            } else if EXAMPLE_EXTS.contains(&e.as_str()) {
324                // Docs examples (`docs/examples/validators_simple.py`) carry the code the pages include.
325                jobs.push(Job {
326                    abs: up.join(&r),
327                    rel,
328                    how: How::Example,
329                });
330            }
331        }
332    }
333    if let Some(meta) = &src.metadata {
334        jobs.push(Job {
335            abs: meta.clone(),
336            rel: "README (package metadata)".into(),
337            how: How::Metadata,
338        });
339    }
340    // Types that ship separately (`@types/react` for `react`).
341    if let (Some(types), false) = (extra_types, has_dts) {
342        let mut rels = Vec::new();
343        walk(types, Path::new(""), 0, &mut rels, public_api);
344        let label = types
345            .iter()
346            .rev()
347            .take(2)
348            .collect::<Vec<_>>()
349            .into_iter()
350            .rev()
351            .map(|s| s.to_string_lossy())
352            .collect::<Vec<_>>()
353            .join("/");
354        for r in rels {
355            if ext_of(&r) == "d.ts" || public_api && matches!(ext_of(&r).as_str(), "d.cts" | "d.mts") {
356                let rel = format!("{label}/{}", r.to_string_lossy().replace('\\', "/"));
357                jobs.push(Job {
358                    abs: types.join(&r),
359                    rel,
360                    how: How::Ts,
361                });
362            }
363        }
364    }
365    jobs
366}
367
368/// Extract every entry for a package.
369pub fn extract(eco: Eco, pkg_name: &str, src: &Source, extra_types: Option<&Path>, upstream: Option<&Path>) -> Vec<Entry> {
370    let major: u64 = src.version.trim_start_matches('v').split('.').next().and_then(|m| m.parse().ok()).unwrap_or(0);
371    let jobs = plan(eco, src, extra_types, upstream, false);
372    let crate_name = if eco == Eco::Cargo {
373        rust_crate_name(&src.dir).unwrap_or_else(|| pkg_name.replace('-', "_"))
374    } else {
375        String::new()
376    };
377    let includes = upstream.map(Includes::scan).unwrap_or_default();
378    let mut all: Vec<Entry> = jobs
379        .par_iter()
380        .flat_map_iter(|job| {
381            let Ok(meta) = std::fs::metadata(&job.abs) else { return Vec::new() };
382            if meta.len() > MAX_FILE_BYTES && job.how != How::Prose {
383                return Vec::new();
384            }
385            let Ok(bytes) = std::fs::read(&job.abs) else { return Vec::new() };
386            let text = String::from_utf8_lossy(&bytes);
387            let mut out = Vec::new();
388            match job.how {
389                How::Prose => {
390                    prose(&text, &job.rel, 0, &mut out);
391                    if job.rel.starts_with("upstream:") && !includes.files.is_empty() {
392                        for e in &mut out {
393                            e.doc = includes.expand(&e.doc);
394                        }
395                    }
396                }
397                How::Page => {
398                    if let Some((md, title)) = page_markdown(&text) {
399                        let title = title.unwrap_or_else(|| page_title(&job.rel));
400                        prose_titled(&md, &job.rel, 0, Some(title), &mut out);
401                    }
402                }
403                How::Example => example(&text, &job.rel, &mut out),
404                How::Metadata => {
405                    let (body, line) = markdown::metadata_body(&text);
406                    prose(&body, &job.rel, line - 1, &mut out)
407                }
408                How::Ts => ts_file(&text, &job.rel, &mut out),
409                How::Python => py_file(&text, &job.rel, &mut out),
410                How::Rust => rs_file(&text, &job.rel, &crate_name, &mut out),
411                How::Go => go_file(&text, &job.rel, pkg_name, &mut out),
412            }
413            out
414        })
415        .collect();
416    for e in &mut all {
417        e.legacy = legacy_file(&e.file, major);
418    }
419    // Drop exact duplicates (the same declaration shipped twice).
420    let mut seen = HashSet::new();
421    all.retain(|e| seen.insert((e.path.clone(), e.sig.clone(), e.doc.len())));
422    all
423}
424
425/// Declaration graph inputs and cache identity use the same file selection as extraction.
426pub(crate) fn npm_sources(src: &Source, extra_types: Option<&Path>) -> std::collections::BTreeMap<String, String> {
427    plan(Eco::Npm, src, extra_types, None, true)
428        .into_iter()
429        .filter(|j| j.how == How::Ts)
430        .filter_map(|j| {
431            if std::fs::metadata(&j.abs).ok()?.len() > MAX_FILE_BYTES {
432                return None;
433            }
434            Some((j.rel, std::fs::read_to_string(j.abs).ok()?))
435        })
436        .collect()
437}
438
439/// Keep declaration identities until the requested import graph is selected.
440/// Mirrored declarations are deduplicated within that search scope.
441pub(crate) fn npm_entries(sources: &std::collections::BTreeMap<String, String>) -> Vec<Entry> {
442    let mut entries = Vec::new();
443    for (file, text) in sources {
444        ts_file(text, file, &mut entries);
445    }
446    entries
447}
448
449/// Content identity, including names, catches edits, additions and removals even
450/// when a package's version and metadata timestamps did not change.
451pub(crate) fn npm_stamp(src: &Source, extra_types: Option<&Path>) -> String {
452    let mut parts = Vec::new();
453    for j in plan(Eco::Npm, src, extra_types, None, true) {
454        parts.push(j.rel);
455        if let Ok(bytes) = std::fs::read(&j.abs) {
456            parts.push(crate::cache::hash(&[&String::from_utf8_lossy(&bytes)]));
457        }
458    }
459    crate::cache::hash(&parts.iter().map(String::as_str).collect::<Vec<_>>())
460}
461
462fn rust_crate_name(dir: &Path) -> Option<String> {
463    let t = std::fs::read_to_string(dir.join("Cargo.toml")).ok()?;
464    let v: toml::Table = toml::from_str(&t).ok()?;
465    let lib = v.get("lib").and_then(|l| l.get("name")).and_then(|n| n.as_str());
466    let pkg = v.get("package").and_then(|p| p.get("name")).and_then(|n| n.as_str());
467    lib.or(pkg).map(|n| n.replace('-', "_"))
468}
469
470/// Docs sites include example files by reference (`{!.tmp_examples/x.md!}`,
471/// `{* ../../docs_src/a/tutorial001.py *}`, `--8<-- "x.py"`). Inline them so
472/// the section carries its code.
473#[derive(Default)]
474struct Includes {
475    /// (lowercase path without extension, absolute path)
476    files: Vec<(String, PathBuf)>,
477}
478
479impl Includes {
480    fn scan(dir: &Path) -> Includes {
481        let mut rels = Vec::new();
482        walk_all(dir, Path::new(""), 0, &mut rels);
483        let files = rels
484            .into_iter()
485            .filter(|r| EXAMPLE_EXTS.contains(&ext_of(r).as_str()))
486            .map(|r| {
487                let s = r.to_string_lossy().replace('\\', "/").to_ascii_lowercase();
488                let stem = s.rsplit_once('.').map_or(s.clone(), |x| x.0.to_string());
489                (stem, dir.join(&r))
490            })
491            .collect();
492        Includes { files }
493    }
494
495    fn find(&self, reference: &str) -> Option<&PathBuf> {
496        let r = reference.trim().trim_matches(['"', '\'']).to_ascii_lowercase();
497        let r = r.split_whitespace().next().unwrap_or("");
498        let r = r.rsplit_once('.').map_or(r, |x| x.0);
499        let segs: Vec<&str> = r
500            .split('/')
501            .filter(|s| !s.is_empty() && *s != ".." && *s != "." && *s != ".tmp_examples")
502            .collect();
503        for take in (1..=segs.len().min(3)).rev() {
504            let suffix = segs[segs.len() - take..].join("/");
505            let hits: Vec<&(String, PathBuf)> = self.files.iter().filter(|(k, _)| *k == suffix || k.ends_with(&format!("/{suffix}"))).collect();
506            if hits.len() == 1 {
507                return Some(&hits[0].1);
508            }
509        }
510        None
511    }
512
513    fn expand(&self, text: &str) -> String {
514        if !(text.contains("{!") || text.contains("{*") || text.contains("--8<--")) {
515            return text.to_string();
516        }
517        let mut out = String::with_capacity(text.len());
518        for line in text.lines() {
519            let t = line.trim();
520            let reference = t
521                .strip_prefix("{!")
522                .and_then(|x| x.strip_suffix("!}"))
523                .or_else(|| t.strip_prefix("{*").and_then(|x| x.strip_suffix("*}")))
524                .or_else(|| t.strip_prefix("--8<--"));
525            match reference
526                .and_then(|r| self.find(r))
527                .and_then(|p| std::fs::read_to_string(p).ok().map(|c| (p, c)))
528            {
529                Some((p, code)) => {
530                    let lang = p.extension().map(|e| e.to_string_lossy().to_string()).unwrap_or_default();
531                    let code: String = code.chars().take(3000).collect();
532                    out.push_str(&format!("```{lang}\n{}\n```\n", code.trim_end()));
533                }
534                None => {
535                    out.push_str(line);
536                    out.push('\n');
537                }
538            }
539        }
540        out
541    }
542}
543
544const EXAMPLE_EXTS: &[&str] = &["py", "ts", "tsx", "js", "jsx", "mjs", "rs", "go"];
545
546/// A docs example file as one code section titled by its file name.
547fn example(text: &str, rel: &str, out: &mut Vec<Entry>) {
548    if text.len() > 12_000 || text.trim().is_empty() {
549        return;
550    }
551    let name = rel.rsplit('/').next().unwrap_or(rel);
552    let lang = name.rsplit('.').next().unwrap_or("");
553    let title = name.rsplit_once('.').map_or(name, |x| x.0).replace(['_', '-'], " ");
554    out.push(Entry {
555        kind: Kind::Prose,
556        name: format!("example › {title}"),
557        path: format!("example › {title}"),
558        file: rel.to_string(),
559        line: 1,
560        sig: String::new(),
561        doc: format!("```{lang}\n{}\n```", text.trim_end()),
562        alias_of: None,
563        legacy: false,
564    });
565}
566
567/// Title for a page file without one: its route (`docs/installation/using-vite`).
568fn page_title(rel: &str) -> String {
569    let segs: Vec<&str> = rel.split('/').filter(|s| !(s.starts_with('(') && s.ends_with(')'))).collect();
570    let mut segs: Vec<&str> = segs.iter().rev().take(3).rev().copied().collect();
571    if let Some(last) = segs.last() {
572        if last.starts_with("page.") || last.starts_with("index.") {
573            segs.pop();
574        }
575    }
576    segs.join("/")
577        .trim_end_matches(".tsx")
578        .trim_end_matches(".jsx")
579        .trim_end_matches(".js")
580        .to_string()
581}
582
583/// Markdown from a docs page written as a JS/TSX component: `title:` values
584/// of listed steps become headings, `description:` and JSX text become
585/// paragraphs, `code:` strings become fenced code (language from `lang:`),
586/// and JSX `<h1>`..`<h6>` stay headings. Lines keep their source position
587/// where they can. Returns (markdown, page title from `metadata.title`).
588fn page_markdown(text: &str) -> Option<(String, Option<String>)> {
589    if text.len() > 200_000 {
590        return None;
591    }
592    parse_with("tsx", text, |root, src| {
593        let mut p = PageOut::default();
594        page_walk(root, src, &mut p, false);
595        p.flush();
596        let mut md = String::new();
597        let mut n = 0u32;
598        for (line, t) in &p.out {
599            while n + 1 < *line {
600                md.push('\n');
601                n += 1;
602            }
603            md.push_str(t);
604            md.push('\n');
605            n += 1 + t.matches('\n').count() as u32;
606        }
607        (md, p.title)
608    })
609    .filter(|(md, _)| !md.trim().is_empty())
610}
611
612#[derive(Default)]
613struct PageOut {
614    out: Vec<(u32, String)>,
615    para: String,
616    para_line: u32,
617    title: Option<String>,
618}
619
620impl PageOut {
621    fn text(&mut self, line: u32, t: &str) {
622        if self.para.trim().is_empty() {
623            self.para_line = line;
624        }
625        self.para.push_str(t);
626    }
627    fn flush(&mut self) {
628        let t = compact(&self.para, usize::MAX);
629        if !t.is_empty() {
630            self.out.push((self.para_line, t));
631        }
632        self.para.clear();
633    }
634    fn block(&mut self, line: u32, t: String) {
635        self.flush();
636        self.out.push((line, t));
637    }
638}
639
640fn line_of(n: Node) -> u32 {
641    n.start_position().row as u32 + 1
642}
643
644/// The value of a string, template string or tagged template (`dedent`...``).
645fn js_string(n: Node, src: &[u8]) -> Option<String> {
646    match n.kind() {
647        "string" => {
648            let raw = txt(n, src);
649            let inner = raw.get(1..raw.len().saturating_sub(1)).unwrap_or("");
650            Some(inner.replace("\\n", "\n").replace("\\'", "'").replace("\\\"", "\"").replace("\\\\", "\\"))
651        }
652        "template_string" => {
653            let raw = txt(n, src);
654            let inner = raw.get(1..raw.len().saturating_sub(1)).unwrap_or("");
655            Some(dedent(&inner.trim_matches('\n').lines().collect::<Vec<_>>()))
656        }
657        "call_expression" => n
658            .child_by_field_name("arguments")
659            .filter(|a| a.kind() == "template_string")
660            .and_then(|a| js_string(a, src)),
661        "parenthesized_expression" => n.named_child(0).and_then(|c| js_string(c, src)),
662        _ => None,
663    }
664}
665
666fn jsx_name(n: Node, src: &[u8]) -> String {
667    let open = if n.kind() == "jsx_element" {
668        n.child_by_field_name("open_tag")
669    } else {
670        Some(n)
671    };
672    open.and_then(|o| o.child_by_field_name("name"))
673        .map(|x| txt(x, src).to_string())
674        .unwrap_or_default()
675}
676
677/// The visible text inside a JSX element.
678fn jsx_text(n: Node, src: &[u8], out: &mut String) {
679    match n.kind() {
680        "jsx_text" => out.push_str(txt(n, src)),
681        "jsx_expression" => {
682            if let Some(v) = n.named_child(0).and_then(|c| js_string(c, src)) {
683                out.push_str(&v);
684            }
685        }
686        "jsx_attribute" | "jsx_opening_element" | "jsx_closing_element" => {}
687        _ => {
688            let mut c = n.walk();
689            for ch in n.named_children(&mut c) {
690                jsx_text(ch, src, out);
691            }
692        }
693    }
694}
695
696fn page_walk(n: Node, src: &[u8], p: &mut PageOut, in_array: bool) {
697    match n.kind() {
698        "import_statement" | "comment" | "jsx_attribute" => {}
699        "pair" => {
700            let key = n
701                .child_by_field_name("key")
702                .map(|k| txt(k, src).trim_matches(['"', '\'']).to_string())
703                .unwrap_or_default();
704            let Some(value) = n.child_by_field_name("value") else { return };
705            match key.as_str() {
706                "title" | "heading" => {
707                    if let Some(t) = js_string(value, src) {
708                        if !in_array && p.title.is_none() {
709                            p.title = Some(t);
710                        } else if in_array {
711                            p.block(line_of(n), format!("## {}", compact(&t, 200)));
712                        }
713                    }
714                }
715                "description" => {
716                    if let Some(t) = js_string(value, src) {
717                        p.block(line_of(value), compact(&t, usize::MAX));
718                    }
719                }
720                "code" => match js_string(value, src) {
721                    Some(code) => {
722                        let lang = n
723                            .parent()
724                            .and_then(|obj| {
725                                let mut c = obj.walk();
726                                let found = obj.named_children(&mut c).find(|x| {
727                                    x.kind() == "pair" && x.child_by_field_name("key").is_some_and(|k| txt(k, src).trim_matches(['"', '\'']) == "lang")
728                                });
729                                found
730                            })
731                            .and_then(|x| x.child_by_field_name("value"))
732                            .and_then(|v| js_string(v, src))
733                            .unwrap_or_default();
734                        p.block(line_of(value), format!("```{lang}\n{}\n```", code.trim_end()));
735                    }
736                    None => page_walk(value, src, p, in_array),
737                },
738                "name" | "lang" | "className" | "href" | "id" | "url" | "src" | "image" | "images" | "icon" | "openGraph" | "twitter" | "alternates" => {}
739                _ => page_walk(value, src, p, in_array),
740            }
741        }
742        "jsx_element" | "jsx_self_closing_element" => {
743            let name = jsx_name(n, src);
744            let b = name.as_bytes();
745            if b.len() == 2 && b[0] == b'h' && (b'1'..=b'6').contains(&b[1]) {
746                let mut t = String::new();
747                jsx_text(n, src, &mut t);
748                let t = compact(&t, 200);
749                if !t.is_empty() {
750                    p.block(line_of(n), format!("{} {t}", "#".repeat((b[1] - b'0') as usize)));
751                }
752            } else if name == "code" {
753                let mut t = String::new();
754                jsx_text(n, src, &mut t);
755                p.text(line_of(n), &format!("`{}`", t.trim()));
756            } else {
757                let block = matches!(name.as_str(), "p" | "li" | "div" | "section" | "ul" | "ol" | "pre" | "table" | "tr");
758                if block {
759                    p.flush();
760                }
761                let mut c = n.walk();
762                for ch in n.named_children(&mut c) {
763                    page_walk(ch, src, p, in_array);
764                }
765                if block {
766                    p.flush();
767                }
768            }
769        }
770        "jsx_text" => p.text(line_of(n), txt(n, src)),
771        "jsx_expression" => match n.named_child(0) {
772            Some(c) if matches!(c.kind(), "string" | "template_string") => {
773                let v = js_string(c, src).unwrap_or_default();
774                p.text(line_of(n), &v);
775            }
776            Some(c) => page_walk(c, src, p, in_array),
777            None => {}
778        },
779        _ => {
780            let arr = in_array || n.kind() == "array";
781            let mut c = n.walk();
782            for ch in n.named_children(&mut c) {
783                page_walk(ch, src, p, arr);
784            }
785        }
786    }
787}
788
789fn prose(text: &str, rel: &str, line_off: u32, out: &mut Vec<Entry>) {
790    prose_titled(text, rel, line_off, None, out)
791}
792
793fn prose_titled(text: &str, rel: &str, line_off: u32, given: Option<String>, out: &mut Vec<Entry>) {
794    let (text, fm_title) = markdown::clean_mdx(text, rel.ends_with(".mdx"));
795    // A page without a title is named after its file: `model_config.md` -> `model config`.
796    let file_name = rel.rsplit('/').next().unwrap_or(rel);
797    let file_title = if rel.starts_with("upstream:") && !is_doc_name(file_name) {
798        file_name
799            .rsplit_once('.')
800            .map_or(file_name, |x| x.0)
801            .trim_start_matches(|c: char| c.is_ascii_digit() || c == '-')
802            .replace(['_', '-'], " ")
803    } else {
804        file_name.to_string()
805    };
806    let file_title = file_title.as_str();
807    let title = if let Some(t) = given {
808        t
809    } else if rel.contains("package metadata") {
810        "README".to_string()
811    } else if let Some(t) = fm_title {
812        t
813    } else {
814        file_title.to_string()
815    };
816    for s in markdown::split(&text, &title) {
817        out.push(prose_section(s, rel, line_off));
818    }
819}
820
821// ---------------------------------------------------------------- helpers
822
823thread_local! {
824    static PARSERS: RefCell<Vec<(&'static str, Parser)>> = const { RefCell::new(Vec::new()) };
825}
826
827/// Parse with a pooled parser. The parser is taken out of the pool while in
828/// use, so `f` may parse again (Rust macro bodies do).
829fn parse_with<R>(lang: &'static str, text: &str, f: impl FnOnce(Node, &[u8]) -> R) -> Option<R> {
830    let pooled = PARSERS.with(|ps| {
831        let mut ps = ps.borrow_mut();
832        ps.iter().position(|(l, _)| *l == lang).map(|i| ps.swap_remove(i).1)
833    });
834    let mut parser = match pooled {
835        Some(p) => p,
836        None => {
837            let mut p = Parser::new();
838            let language: tree_sitter::Language = match lang {
839                "ts" => tree_sitter_typescript::LANGUAGE_TYPESCRIPT.into(),
840                "tsx" => tree_sitter_typescript::LANGUAGE_TSX.into(),
841                "py" => tree_sitter_python::LANGUAGE.into(),
842                "rs" => tree_sitter_rust::LANGUAGE.into(),
843                _ => tree_sitter_go::LANGUAGE.into(),
844            };
845            p.set_language(&language).ok()?;
846            p
847        }
848    };
849    let tree = parser.parse(text, None);
850    PARSERS.with(|ps| ps.borrow_mut().push((lang, parser)));
851    let tree = tree?;
852    Some(f(tree.root_node(), text.as_bytes()))
853}
854
855fn txt<'a>(n: Node, src: &'a [u8]) -> &'a str {
856    n.utf8_text(src).unwrap_or("")
857}
858
859fn compact(s: &str, cap: usize) -> String {
860    let mut out = String::with_capacity(s.len().min(cap.saturating_add(8)));
861    let mut space = false;
862    for c in s.chars() {
863        if c.is_whitespace() {
864            space = true;
865            continue;
866        }
867        if space && !out.is_empty() {
868            out.push(' ');
869        }
870        space = false;
871        out.push(c);
872        if out.len() >= cap {
873            out.push_str(" …");
874            break;
875        }
876    }
877    out.replace("( ", "(").replace(", )", ")").replace(",)", ")").replace(" )", ")")
878}
879
880/// Drop `# comments` from Python source lines (outside string literals).
881fn strip_py_comments(s: &str) -> String {
882    let mut out = String::with_capacity(s.len());
883    for line in s.lines() {
884        let mut quote: Option<char> = None;
885        let mut cut = line.len();
886        for (i, c) in line.char_indices() {
887            match (quote, c) {
888                (None, '#') => {
889                    cut = i;
890                    break;
891                }
892                (None, '"' | '\'') => quote = Some(c),
893                (Some(q), c) if c == q => quote = None,
894                _ => {}
895            }
896        }
897        out.push_str(&line[..cut]);
898        out.push('\n');
899    }
900    out
901}
902
903/// Source text from `n` up to (not including) its body child.
904fn head(n: Node, src: &[u8], body_field: &str, cap: usize) -> String {
905    let end = n.child_by_field_name(body_field).map_or(n.end_byte(), |b| b.start_byte());
906    let s = std::str::from_utf8(&src[n.start_byte()..end]).unwrap_or("");
907    compact(s.trim_end().trim_end_matches(['{', ':', '=']).trim_end(), cap)
908}
909
910fn cap_doc(s: String) -> String {
911    const CAP: usize = 4000;
912    if s.len() <= CAP {
913        return s;
914    }
915    let mut i = CAP;
916    while !s.is_char_boundary(i) {
917        i -= 1;
918    }
919    format!("{} …", &s[..i])
920}
921
922fn dedent(lines: &[&str]) -> String {
923    let indent = lines
924        .iter()
925        .filter(|l| !l.trim().is_empty())
926        .map(|l| l.len() - l.trim_start().len())
927        .min()
928        .unwrap_or(0);
929    let v: Vec<&str> = lines.iter().map(|l| if l.len() >= indent { &l[indent..] } else { l.trim_start() }).collect();
930    v.join("\n").trim().to_string()
931}
932
933fn clean_block_comment(s: &str) -> String {
934    let s = s
935        .trim_start_matches("/**")
936        .trim_start_matches("/*!")
937        .trim_start_matches("/*")
938        .trim_end_matches("*/");
939    let lines: Vec<&str> = s
940        .lines()
941        .map(|l| {
942            let t = l.trim_start();
943            match t.strip_prefix('*') {
944                Some(r) => r.strip_prefix(' ').unwrap_or(r),
945                None => l,
946            }
947        })
948        .collect();
949    cap_doc(dedent(&lines))
950}
951
952fn entry(kind: Kind, name: &str, path: String, file: &str, line: u32, sig: String, doc: String) -> Entry {
953    Entry {
954        kind,
955        name: name.to_string(),
956        path,
957        file: file.to_string(),
958        line,
959        sig,
960        doc,
961        alias_of: None,
962        legacy: false,
963    }
964}
965
966/// A Prose entry for one Markdown section: the heading names it and is its
967/// path, and `line_off` shifts the section line into the surrounding file.
968fn prose_section(s: crate::markdown::Section, rel: &str, line_off: u32) -> Entry {
969    Entry {
970        kind: Kind::Prose,
971        name: s.heading.clone(),
972        path: s.heading,
973        file: rel.to_string(),
974        line: s.line + line_off,
975        sig: String::new(),
976        doc: s.text,
977        alias_of: None,
978        legacy: false,
979    }
980}
981
982// ---------------------------------------------------------------- TypeScript / JavaScript
983
984fn ts_file(text: &str, rel: &str, out: &mut Vec<Entry>) {
985    let lang = if rel.ends_with(".tsx") || rel.ends_with(".jsx") { "tsx" } else { "ts" };
986    let is_js = rel.ends_with(".js") || rel.ends_with(".mjs") || rel.ends_with(".cjs");
987    parse_with(lang, text, |root, src| {
988        let mut w = TsWalker {
989            src,
990            file: rel,
991            out,
992            js: is_js,
993        };
994        w.block(root, "");
995    });
996}
997
998struct TsWalker<'a> {
999    src: &'a [u8],
1000    file: &'a str,
1001    out: &'a mut Vec<Entry>,
1002    js: bool,
1003}
1004
1005impl TsWalker<'_> {
1006    fn doc(&self, n: Node) -> String {
1007        let mut p = n.prev_sibling();
1008        while let Some(c) = p {
1009            if c.kind() == "comment" {
1010                let t = txt(c, self.src);
1011                if t.starts_with("/**") && c.end_position().row + 2 >= n.start_position().row {
1012                    return clean_block_comment(t);
1013                }
1014                if t.starts_with("//") {
1015                    p = c.prev_sibling();
1016                    continue;
1017                }
1018            }
1019            break;
1020        }
1021        String::new()
1022    }
1023
1024    fn line(&self, n: Node) -> u32 {
1025        n.start_position().row as u32 + 1
1026    }
1027
1028    fn block(&mut self, node: Node, prefix: &str) {
1029        let mut cur = node.walk();
1030        let children: Vec<Node> = node.named_children(&mut cur).collect();
1031        for c in children {
1032            match c.kind() {
1033                "export_statement" => {
1034                    let doc = self.doc(c);
1035                    if let Some(d) = c.child_by_field_name("declaration") {
1036                        self.decl(d, prefix, doc, c);
1037                    } else {
1038                        self.exports(c, prefix);
1039                    }
1040                }
1041                "ambient_declaration" => {
1042                    let doc = self.doc(c);
1043                    let mut cc = c.walk();
1044                    let inner: Vec<Node> = c.named_children(&mut cc).collect();
1045                    for d in inner {
1046                        self.decl(d, prefix, doc.clone(), c);
1047                    }
1048                }
1049                "expression_statement" => {
1050                    let mut cc = c.walk();
1051                    let inner: Vec<Node> = c.named_children(&mut cc).collect();
1052                    for d in inner {
1053                        if matches!(d.kind(), "internal_module" | "module") {
1054                            let doc = self.doc(c);
1055                            self.decl(d, prefix, doc, c);
1056                        }
1057                    }
1058                }
1059                _ => {
1060                    let doc = self.doc(c);
1061                    // In JS only documented top-level declarations are API.
1062                    if !self.js || !doc.is_empty() || !prefix.is_empty() {
1063                        self.decl(c, prefix, doc, c);
1064                    }
1065                }
1066            }
1067        }
1068    }
1069
1070    fn exports(&mut self, n: Node, prefix: &str) {
1071        let mut cur = n.walk();
1072        let children: Vec<Node> = n.named_children(&mut cur).collect();
1073        for c in children {
1074            match c.kind() {
1075                "export_clause" => {
1076                    let mut cc = c.walk();
1077                    let specs: Vec<Node> = c.named_children(&mut cc).collect();
1078                    for s in specs {
1079                        let (Some(name), alias) = (s.child_by_field_name("name"), s.child_by_field_name("alias")) else {
1080                            continue;
1081                        };
1082                        let Some(alias) = alias else { continue };
1083                        let (from, to) = (txt(name, self.src), txt(alias, self.src));
1084                        if from == to || to == "default" {
1085                            continue;
1086                        }
1087                        let mut e = entry(
1088                            Kind::Alias,
1089                            to,
1090                            format!("{prefix}{to}"),
1091                            self.file,
1092                            self.line(s),
1093                            compact(txt(n, self.src), 200),
1094                            String::new(),
1095                        );
1096                        e.alias_of = Some(from.to_string());
1097                        self.out.push(e);
1098                    }
1099                }
1100                "namespace_export" => {
1101                    if let Some(id) = c.named_child(0) {
1102                        let name = txt(id, self.src);
1103                        self.out.push(entry(
1104                            Kind::Module,
1105                            name,
1106                            format!("{prefix}{name}"),
1107                            self.file,
1108                            self.line(n),
1109                            compact(txt(n, self.src), 200),
1110                            self.doc(n),
1111                        ));
1112                    }
1113                }
1114                _ => {}
1115            }
1116        }
1117    }
1118
1119    fn decl(&mut self, d: Node, prefix: &str, doc: String, outer: Node) {
1120        let src = self.src;
1121        let name_of = |n: Node| n.child_by_field_name("name").map(|x| txt(x, src).to_string());
1122        let line = self.line(outer);
1123        match d.kind() {
1124            "ambient_declaration" => {
1125                let mut cur = d.walk();
1126                let inner: Vec<Node> = d.named_children(&mut cur).collect();
1127                for x in inner {
1128                    self.decl(x, prefix, doc.clone(), outer);
1129                }
1130            }
1131            "function_signature" | "function_declaration" | "generator_function_declaration" => {
1132                let Some(name) = name_of(d) else { return };
1133                let sig = head(outer, src, "__none__", 600).trim_end_matches(';').to_string();
1134                let sig = if d.child_by_field_name("body").is_some() {
1135                    head(d, src, "body", 600)
1136                } else {
1137                    sig
1138                };
1139                self.out
1140                    .push(entry(Kind::Function, &name, format!("{prefix}{name}"), self.file, line, sig, doc));
1141            }
1142            "class_declaration" | "abstract_class_declaration" | "class" => {
1143                let Some(name) = name_of(d) else { return };
1144                self.out.push(entry(
1145                    Kind::Class,
1146                    &name,
1147                    format!("{prefix}{name}"),
1148                    self.file,
1149                    line,
1150                    head(d, src, "body", 400),
1151                    doc,
1152                ));
1153                if let Some(body) = d.child_by_field_name("body") {
1154                    self.members(body, &format!("{prefix}{name}."));
1155                }
1156            }
1157            "interface_declaration" => {
1158                let Some(name) = name_of(d) else { return };
1159                self.out.push(entry(
1160                    Kind::Interface,
1161                    &name,
1162                    format!("{prefix}{name}"),
1163                    self.file,
1164                    line,
1165                    head(d, src, "body", 400),
1166                    doc,
1167                ));
1168                if let Some(body) = d.child_by_field_name("body") {
1169                    self.members(body, &format!("{prefix}{name}."));
1170                }
1171            }
1172            "type_alias_declaration" => {
1173                let Some(name) = name_of(d) else { return };
1174                self.out.push(entry(
1175                    Kind::Type,
1176                    &name,
1177                    format!("{prefix}{name}"),
1178                    self.file,
1179                    line,
1180                    compact(txt(d, src), 700),
1181                    doc,
1182                ));
1183            }
1184            "enum_declaration" => {
1185                let Some(name) = name_of(d) else { return };
1186                self.out.push(entry(
1187                    Kind::Enum,
1188                    &name,
1189                    format!("{prefix}{name}"),
1190                    self.file,
1191                    line,
1192                    compact(txt(d, src), 500),
1193                    doc,
1194                ));
1195            }
1196            "lexical_declaration" | "variable_declaration" => {
1197                let kw = txt(d, src).split_whitespace().next().unwrap_or("const").to_string();
1198                let mut cur = d.walk();
1199                let decls: Vec<Node> = d.named_children(&mut cur).filter(|n| n.kind() == "variable_declarator").collect();
1200                for v in decls {
1201                    let Some(nn) = v.child_by_field_name("name") else { continue };
1202                    if nn.kind() != "identifier" {
1203                        continue;
1204                    }
1205                    let name = txt(nn, src).to_string();
1206                    let value = v.child_by_field_name("value");
1207                    let is_fn = value.is_some_and(|x| matches!(x.kind(), "arrow_function" | "function_expression" | "function"))
1208                        || v.child_by_field_name("type").is_some_and(|t| txt(t, src).contains("=>"));
1209                    let sig = match value {
1210                        Some(val) if matches!(val.kind(), "arrow_function" | "function_expression" | "function") => {
1211                            let body_start = val.child_by_field_name("body").map_or(val.end_byte(), |b| b.start_byte());
1212                            let s = std::str::from_utf8(&src[v.start_byte()..body_start]).unwrap_or("");
1213                            compact(&format!("{kw} {}", s.trim_end().trim_end_matches("=>").trim_end()), 600)
1214                        }
1215                        Some(val) if val.end_byte() - val.start_byte() > 300 => {
1216                            let s = std::str::from_utf8(&src[v.start_byte()..val.start_byte()]).unwrap_or("");
1217                            compact(&format!("{kw} {s} …"), 600)
1218                        }
1219                        _ => compact(&format!("{kw} {}", txt(v, src)), 600),
1220                    };
1221                    let kind = if is_fn { Kind::Function } else { Kind::Const };
1222                    self.out.push(entry(kind, &name, format!("{prefix}{name}"), self.file, line, sig, doc.clone()));
1223                }
1224            }
1225            "internal_module" | "module" => {
1226                let Some(nn) = d.child_by_field_name("name") else { return };
1227                let raw = txt(nn, src);
1228                let quoted = raw.starts_with('"') || raw.starts_with('\'');
1229                let name = raw.trim_matches(['"', '\'']).to_string();
1230                let new_prefix = if quoted { prefix.to_string() } else { format!("{prefix}{name}.") };
1231                if !quoted {
1232                    self.out.push(entry(
1233                        Kind::Module,
1234                        &name,
1235                        format!("{prefix}{name}"),
1236                        self.file,
1237                        line,
1238                        head(d, src, "body", 200),
1239                        doc,
1240                    ));
1241                }
1242                if let Some(body) = d.child_by_field_name("body") {
1243                    self.block(body, &new_prefix);
1244                }
1245            }
1246            _ => {}
1247        }
1248    }
1249
1250    fn members(&mut self, body: Node, prefix: &str) {
1251        let src = self.src;
1252        let mut cur = body.walk();
1253        let children: Vec<Node> = body.named_children(&mut cur).collect();
1254        for m in children {
1255            let kind = match m.kind() {
1256                "method_definition" | "method_signature" | "abstract_method_signature" => Kind::Method,
1257                "public_field_definition" | "property_signature" => Kind::Field,
1258                _ => continue,
1259            };
1260            let t = txt(m, src);
1261            if t.starts_with("private ") || t.starts_with("protected ") || t.contains(" private ") && kind == Kind::Field {
1262                continue;
1263            }
1264            let Some(nn) = m.child_by_field_name("name") else { continue };
1265            let name = txt(nn, src).trim_matches(['"', '\'']).to_string();
1266            if name.starts_with('#') || name.starts_with('_') || name.starts_with('[') {
1267                continue;
1268            }
1269            let doc = self.doc(m);
1270            let sig = if m.child_by_field_name("body").is_some() {
1271                head(m, src, "body", 500)
1272            } else {
1273                compact(t.trim_end_matches(';'), 500)
1274            };
1275            self.out.push(entry(
1276                kind,
1277                &name,
1278                format!("{prefix}{name}"),
1279                self.file,
1280                m.start_position().row as u32 + 1,
1281                sig,
1282                doc,
1283            ));
1284        }
1285    }
1286}
1287
1288// ---------------------------------------------------------------- Python
1289
1290fn py_module(rel: &str) -> String {
1291    let r = rel.trim_end_matches(".pyi").trim_end_matches(".py");
1292    let r = r.trim_end_matches("/__init__");
1293    r.replace('/', ".")
1294}
1295
1296fn py_file(text: &str, rel: &str, out: &mut Vec<Entry>) {
1297    let module = py_module(rel);
1298    parse_with("py", text, |root, src| {
1299        if let Some(doc) = py_docstring(root, src) {
1300            for s in markdown::split(&doc, &module) {
1301                out.push(prose_section(s, rel, 0));
1302            }
1303        }
1304        py_block(root, src, &module, false, rel, out);
1305    });
1306}
1307
1308fn py_docstring(block: Node, src: &[u8]) -> Option<String> {
1309    let first = block.named_child(0)?;
1310    let first = if first.kind() == "comment" { first.next_named_sibling()? } else { first };
1311    if first.kind() != "expression_statement" {
1312        return None;
1313    }
1314    let s = first.named_child(0)?;
1315    if s.kind() != "string" {
1316        return None;
1317    }
1318    let raw = txt(s, src);
1319    let raw = raw.trim_start_matches(['r', 'R', 'u', 'U', 'b', 'B']);
1320    let inner = raw
1321        .strip_prefix("\"\"\"")
1322        .and_then(|x| x.strip_suffix("\"\"\""))
1323        .or_else(|| raw.strip_prefix("'''").and_then(|x| x.strip_suffix("'''")))
1324        .or_else(|| raw.strip_prefix('"').and_then(|x| x.strip_suffix('"')))
1325        .or_else(|| raw.strip_prefix('\'').and_then(|x| x.strip_suffix('\'')))?;
1326    let lines: Vec<&str> = inner.lines().collect();
1327    if lines.is_empty() {
1328        return None;
1329    }
1330    // inspect.cleandoc: first line as is, the rest dedented.
1331    let first_line = lines[0].trim();
1332    let rest = dedent(&lines[1..]);
1333    let gap = if lines.get(1).is_some_and(|l| l.trim().is_empty()) { "\n\n" } else { "\n" };
1334    let doc = if rest.is_empty() {
1335        first_line.to_string()
1336    } else if first_line.is_empty() {
1337        rest
1338    } else {
1339        format!("{first_line}{gap}{rest}")
1340    };
1341    Some(cap_doc(doc))
1342}
1343
1344fn py_block(block: Node, src: &[u8], prefix: &str, in_class: bool, rel: &str, out: &mut Vec<Entry>) {
1345    let mut cur = block.walk();
1346    let children: Vec<Node> = block.named_children(&mut cur).collect();
1347    for c in children {
1348        let (def, decorators) = if c.kind() == "decorated_definition" {
1349            let mut dc = c.walk();
1350            let decs: Vec<String> = c
1351                .named_children(&mut dc)
1352                .filter(|n| n.kind() == "decorator")
1353                .map(|n| compact(txt(n, src), 80))
1354                .collect();
1355            match c.child_by_field_name("definition") {
1356                Some(d) => (d, decs),
1357                None => continue,
1358            }
1359        } else {
1360            (c, Vec::new())
1361        };
1362        let Some(nn) = def.child_by_field_name("name") else { continue };
1363        let name = txt(nn, src);
1364        let dunder_ok = matches!(name, "__init__" | "__call__");
1365        if name.starts_with('_') && !dunder_ok {
1366            continue;
1367        }
1368        let line = c.start_position().row as u32 + 1;
1369        let mut sig = decorators.join(" ");
1370        match def.kind() {
1371            "function_definition" => {
1372                if !sig.is_empty() {
1373                    sig.push(' ');
1374                }
1375                let end = def.child_by_field_name("body").map_or(def.end_byte(), |b| b.start_byte());
1376                let raw = strip_py_comments(std::str::from_utf8(&src[def.start_byte()..end]).unwrap_or(""));
1377                sig.push_str(&compact(raw.trim_end().trim_end_matches(':').trim_end(), 700));
1378                let doc = def.child_by_field_name("body").and_then(|b| py_docstring(b, src)).unwrap_or_default();
1379                let kind = if in_class { Kind::Method } else { Kind::Function };
1380                out.push(entry(kind, name, format!("{prefix}.{name}"), rel, line, sig, doc));
1381            }
1382            "class_definition" => {
1383                if !sig.is_empty() {
1384                    sig.push(' ');
1385                }
1386                sig.push_str(&head(def, src, "body", 400));
1387                let body = def.child_by_field_name("body");
1388                let doc = body.and_then(|b| py_docstring(b, src)).unwrap_or_default();
1389                let path = format!("{prefix}.{name}");
1390                out.push(entry(Kind::Class, name, path.clone(), rel, line, sig, doc));
1391                if let Some(b) = body {
1392                    py_block(b, src, &path, true, rel, out);
1393                }
1394            }
1395            _ => {}
1396        }
1397    }
1398}
1399
1400// ---------------------------------------------------------------- Rust
1401
1402fn rs_module(rel: &str, crate_name: &str) -> String {
1403    let r = rel.strip_prefix("src/").unwrap_or(rel).trim_end_matches(".rs");
1404    let mut segs: Vec<&str> = r.split('/').collect();
1405    if matches!(segs.last(), Some(&"mod") | Some(&"lib") | Some(&"main")) {
1406        segs.pop();
1407    }
1408    let mut p = crate_name.to_string();
1409    for s in segs {
1410        if !s.is_empty() {
1411            p.push_str("::");
1412            p.push_str(s);
1413        }
1414    }
1415    p
1416}
1417
1418fn rs_file(text: &str, rel: &str, crate_name: &str, out: &mut Vec<Entry>) {
1419    let module = rs_module(rel, crate_name);
1420    rs_parse(text, rel, &module, 0, 0, None, out);
1421}
1422
1423/// Docs and `#[macro_export]` a wrapper macro applies to the items it wraps
1424/// (tokio's `doc! { macro_rules! select { .. } }` pattern).
1425type Inherit = Option<(String, bool)>;
1426
1427fn rs_parse(text: &str, rel: &str, module: &str, line_off: u32, depth: u32, inherit: Inherit, out: &mut Vec<Entry>) {
1428    parse_with("rs", text, |root, src| {
1429        let mut inner_doc = Vec::new();
1430        let mut w = RsWalker {
1431            src,
1432            file: rel,
1433            line_off,
1434            depth,
1435            out: Vec::new(),
1436            wrappers: HashMap::new(),
1437            inherit,
1438        };
1439        w.block(root, module, None, &mut inner_doc);
1440        if !inner_doc.is_empty() {
1441            let doc = inner_doc.join("\n");
1442            for s in markdown::split(&doc, module) {
1443                out.push(prose_section(s, rel, line_off));
1444            }
1445        }
1446        out.append(&mut w.out);
1447    });
1448}
1449
1450struct RsWalker<'a> {
1451    src: &'a [u8],
1452    file: &'a str,
1453    line_off: u32,
1454    depth: u32,
1455    out: Vec<Entry>,
1456    /// Local `macro_rules!` that wrap an `$item` with docs: name -> (doc, exported).
1457    wrappers: HashMap<String, (String, bool)>,
1458    inherit: Inherit,
1459}
1460
1461/// A local macro like `macro_rules! doc { ($x:item) => { /// docs  #[macro_export] $x } }`.
1462fn doc_wrapper(text: &str) -> Option<(String, bool)> {
1463    if !text.contains(":item") {
1464        return None;
1465    }
1466    let doc: Vec<&str> = text
1467        .lines()
1468        .filter_map(|l| strip_line_doc(l.trim()).filter(|(_, inner)| !inner).map(|(d, _)| d))
1469        .collect();
1470    if doc.is_empty() {
1471        return None;
1472    }
1473    Some((cap_doc(doc.join("\n").trim().to_string()), text.contains("macro_export")))
1474}
1475
1476fn is_pub(n: Node, src: &[u8]) -> bool {
1477    let mut cur = n.walk();
1478    let r = n
1479        .named_children(&mut cur)
1480        .any(|c| c.kind() == "visibility_modifier" && txt(c, src).trim() == "pub");
1481    r
1482}
1483
1484fn strip_line_doc(t: &str) -> Option<(&str, bool)> {
1485    if let Some(r) = t.strip_prefix("///") {
1486        if r.starts_with('/') {
1487            return None;
1488        }
1489        return Some((r.strip_prefix(' ').unwrap_or(r).trim_end(), false));
1490    }
1491    if let Some(r) = t.strip_prefix("//!") {
1492        return Some((r.strip_prefix(' ').unwrap_or(r).trim_end(), true));
1493    }
1494    None
1495}
1496
1497impl RsWalker<'_> {
1498    fn line(&self, n: Node) -> u32 {
1499        n.start_position().row as u32 + 1 + self.line_off
1500    }
1501
1502    /// `owner`: impl/trait type whose members we are in (`Some((name, is_trait))`).
1503    fn block(&mut self, node: Node, module: &str, owner: Option<(&str, bool)>, inner_doc: &mut Vec<String>) {
1504        let src = self.src;
1505        let mut cur = node.walk();
1506        let children: Vec<Node> = node.named_children(&mut cur).collect();
1507        let mut pending: Vec<String> = Vec::new();
1508        let mut hidden = false;
1509        let mut exported_macro = false;
1510        let mut first_line: Option<u32> = None;
1511        for c in children {
1512            let k = c.kind();
1513            match k {
1514                "line_comment" => {
1515                    let t = txt(c, src);
1516                    if let Some((d, inner)) = strip_line_doc(t) {
1517                        if inner {
1518                            inner_doc.push(d.to_string());
1519                        } else {
1520                            if pending.is_empty() {
1521                                first_line = Some(self.line(c));
1522                            }
1523                            pending.push(d.to_string());
1524                        }
1525                    }
1526                    continue;
1527                }
1528                "block_comment" => {
1529                    let t = txt(c, src);
1530                    if t.starts_with("/*!") {
1531                        inner_doc.push(clean_block_comment(t));
1532                    } else if t.starts_with("/**") && !t.starts_with("/***") {
1533                        pending.push(clean_block_comment(t));
1534                    }
1535                    continue;
1536                }
1537                "attribute_item" => {
1538                    let t = txt(c, src);
1539                    if t.contains("doc(hidden)") {
1540                        hidden = true;
1541                    }
1542                    if t.contains("macro_export") {
1543                        exported_macro = true;
1544                    }
1545                    // `#[doc = "..."]`
1546                    if let Some(rest) = t.strip_prefix("#[doc = \"") {
1547                        pending.push(rest.trim_end_matches("\"]").replace("\\n", "\n"));
1548                    }
1549                    continue;
1550                }
1551                "inner_attribute_item" => continue,
1552                _ => {}
1553            }
1554            let mut doc = cap_doc(pending.join("\n").trim().to_string());
1555            if doc.is_empty() {
1556                if let Some((d, _)) = &self.inherit {
1557                    doc = d.clone();
1558                }
1559            }
1560            let _ = first_line.take();
1561            pending.clear();
1562            let was_hidden = std::mem::take(&mut hidden);
1563            let was_macro_export = std::mem::take(&mut exported_macro) || self.inherit.as_ref().is_some_and(|i| i.1);
1564            if was_hidden {
1565                continue;
1566            }
1567            let line = self.line(c);
1568            let sep = "::";
1569            let in_trait = owner.is_some_and(|o| o.1);
1570            let public = in_trait || is_pub(c, src);
1571            let name = c.child_by_field_name("name").map(|n| txt(n, src).to_string());
1572            let qual = |name: &str| match owner {
1573                Some((o, _)) => format!("{module}{sep}{o}{sep}{name}"),
1574                None => format!("{module}{sep}{name}"),
1575            };
1576            match k {
1577                "function_item" | "function_signature_item" => {
1578                    let (Some(name), true) = (name, public) else { continue };
1579                    let kind = if owner.is_some() { Kind::Method } else { Kind::Function };
1580                    self.out.push(entry(
1581                        kind,
1582                        &name,
1583                        qual(&name),
1584                        self.file,
1585                        line,
1586                        head(c, src, "body", 600).trim_end_matches(';').to_string(),
1587                        doc,
1588                    ));
1589                }
1590                "struct_item" | "enum_item" | "union_item" => {
1591                    let (Some(name), true) = (name, public) else { continue };
1592                    let kind = if k == "enum_item" { Kind::Enum } else { Kind::Struct };
1593                    let full = txt(c, src);
1594                    let sig = if full.len() <= 900 {
1595                        compact(full, 900)
1596                    } else {
1597                        format!("{} {{ … }}", head(c, src, "body", 400))
1598                    };
1599                    self.out.push(entry(kind, &name, qual(&name), self.file, line, sig, doc));
1600                }
1601                "trait_item" => {
1602                    let (Some(name), true) = (name, public) else { continue };
1603                    self.out
1604                        .push(entry(Kind::Trait, &name, qual(&name), self.file, line, head(c, src, "body", 400), doc));
1605                    if let Some(b) = c.child_by_field_name("body") {
1606                        let mut ignore = Vec::new();
1607                        self.block(b, module, Some((&name, true)), &mut ignore);
1608                    }
1609                }
1610                "impl_item" => {
1611                    if c.child_by_field_name("trait").is_some() {
1612                        continue;
1613                    }
1614                    let Some(ty) = c.child_by_field_name("type") else { continue };
1615                    let ty = txt(ty, src);
1616                    let ty = ty.split('<').next().unwrap_or(ty).trim().to_string();
1617                    if let Some(b) = c.child_by_field_name("body") {
1618                        let mut ignore = Vec::new();
1619                        self.block(b, module, Some((&ty, false)), &mut ignore);
1620                    }
1621                }
1622                "mod_item" => {
1623                    let (Some(name), true) = (name, public) else { continue };
1624                    let path = format!("{module}{sep}{name}");
1625                    if !doc.is_empty() {
1626                        self.out
1627                            .push(entry(Kind::Module, &name, path.clone(), self.file, line, format!("pub mod {name}"), doc));
1628                    }
1629                    if let Some(b) = c.child_by_field_name("body") {
1630                        let mut inner = Vec::new();
1631                        self.block(b, &path, None, &mut inner);
1632                    }
1633                }
1634                "type_item" | "const_item" | "static_item" => {
1635                    let (Some(name), true) = (name, public) else { continue };
1636                    let kind = if k == "type_item" { Kind::Type } else { Kind::Const };
1637                    self.out.push(entry(
1638                        kind,
1639                        &name,
1640                        qual(&name),
1641                        self.file,
1642                        line,
1643                        compact(txt(c, src).trim_end_matches(';'), 400),
1644                        doc,
1645                    ));
1646                }
1647                "macro_definition" => {
1648                    let Some(name) = name else { continue };
1649                    if let Some(w) = doc_wrapper(txt(c, src)) {
1650                        self.wrappers.insert(name.clone(), w);
1651                        continue;
1652                    }
1653                    if !was_macro_export {
1654                        continue;
1655                    }
1656                    // Exported macros live at the crate root.
1657                    let root = module.split("::").next().unwrap_or(module);
1658                    let arms = compact(txt(c, src), 300);
1659                    self.out.push(entry(Kind::Macro, &name, format!("{root}::{name}!"), self.file, line, arms, doc));
1660                }
1661                "macro_invocation" if self.depth < 3 => {
1662                    // cfg_rt! { pub fn spawn ... } and friends: parse the body as items.
1663                    let Some(tt) = c.named_children(&mut c.walk()).find(|n| n.kind() == "token_tree") else {
1664                        continue;
1665                    };
1666                    let body = txt(tt, src);
1667                    let called = c.child_by_field_name("macro").map(|m| txt(m, src).to_string()).unwrap_or_default();
1668                    let wrapper = self.wrappers.get(&called).cloned();
1669                    let itemish =
1670                        body.contains("fn ") || body.contains("struct ") || body.contains("mod ") || body.contains("trait ") || body.contains("macro_rules!");
1671                    if body.len() < 4 || !itemish {
1672                        continue;
1673                    }
1674                    let inner = &body[1..body.len() - 1];
1675                    let off = tt.start_position().row as u32 + self.line_off;
1676                    let mut sub = Vec::new();
1677                    rs_parse(
1678                        inner,
1679                        self.file,
1680                        module,
1681                        off,
1682                        self.depth + 1,
1683                        wrapper.or_else(|| self.inherit.clone()),
1684                        &mut sub,
1685                    );
1686                    // Owner-less items only: prose from nested inner docs stays.
1687                    self.out.extend(sub);
1688                }
1689                _ => {}
1690            }
1691        }
1692    }
1693}
1694
1695// ---------------------------------------------------------------- Go
1696
1697fn exported(name: &str) -> bool {
1698    name.chars().next().is_some_and(|c| c.is_uppercase())
1699}
1700
1701fn go_doc(n: Node, src: &[u8]) -> String {
1702    let mut lines: Vec<String> = Vec::new();
1703    let mut p = n.prev_sibling();
1704    let mut next_row = n.start_position().row;
1705    while let Some(c) = p {
1706        if c.kind() != "comment" || c.end_position().row + 1 != next_row {
1707            break;
1708        }
1709        let t = txt(c, src);
1710        if let Some(r) = t.strip_prefix("//") {
1711            lines.push(r.strip_prefix(' ').unwrap_or(r).trim_end().to_string());
1712        } else {
1713            lines.push(clean_block_comment(t));
1714        }
1715        next_row = c.start_position().row;
1716        p = c.prev_sibling();
1717    }
1718    lines.reverse();
1719    cap_doc(lines.join("\n").trim().to_string())
1720}
1721
1722fn go_file(text: &str, rel: &str, module: &str, out: &mut Vec<Entry>) {
1723    parse_with("go", text, |root, src| {
1724        let mut cur = root.walk();
1725        let children: Vec<Node> = root.named_children(&mut cur).collect();
1726        let mut pkg = module.rsplit('/').next().unwrap_or(module).to_string();
1727        for c in children {
1728            let line = c.start_position().row as u32 + 1;
1729            match c.kind() {
1730                "package_clause" => {
1731                    if let Some(id) = c.named_child(0) {
1732                        pkg = txt(id, src).to_string();
1733                    }
1734                    if pkg == "main" {
1735                        return;
1736                    }
1737                    let doc = go_doc(c, src);
1738                    if !doc.is_empty() {
1739                        let dir = rel.rsplit_once('/').map_or("", |x| x.0);
1740                        let title = if dir.is_empty() {
1741                            format!("package {pkg}")
1742                        } else {
1743                            format!("package {pkg} ({dir})")
1744                        };
1745                        for s in markdown::split(&doc, &title) {
1746                            out.push(prose_section(s, rel, 0));
1747                        }
1748                    }
1749                }
1750                "function_declaration" => {
1751                    let Some(name) = c.child_by_field_name("name").map(|n| txt(n, src)) else {
1752                        continue;
1753                    };
1754                    if !exported(name) {
1755                        continue;
1756                    }
1757                    out.push(entry(
1758                        Kind::Function,
1759                        name,
1760                        format!("{pkg}.{name}"),
1761                        rel,
1762                        line,
1763                        head(c, src, "body", 600),
1764                        go_doc(c, src),
1765                    ));
1766                }
1767                "method_declaration" => {
1768                    let Some(name) = c.child_by_field_name("name").map(|n| txt(n, src)) else {
1769                        continue;
1770                    };
1771                    let recv = c.child_by_field_name("receiver").map(|r| txt(r, src)).unwrap_or("");
1772                    let ty: String = recv
1773                        .trim_matches(['(', ')'])
1774                        .split_whitespace()
1775                        .last()
1776                        .unwrap_or("")
1777                        .trim_start_matches('*')
1778                        .split('[')
1779                        .next()
1780                        .unwrap_or("")
1781                        .to_string();
1782                    if !exported(name) || !exported(&ty) {
1783                        continue;
1784                    }
1785                    out.push(entry(
1786                        Kind::Method,
1787                        name,
1788                        format!("{pkg}.{ty}.{name}"),
1789                        rel,
1790                        line,
1791                        head(c, src, "body", 600),
1792                        go_doc(c, src),
1793                    ));
1794                }
1795                "type_declaration" | "const_declaration" | "var_declaration" => {
1796                    let outer_doc = go_doc(c, src);
1797                    let mut cc = c.walk();
1798                    let specs: Vec<Node> = c
1799                        .named_children(&mut cc)
1800                        .filter(|n| matches!(n.kind(), "type_spec" | "type_alias" | "const_spec" | "var_spec"))
1801                        .collect();
1802                    let single = specs.len() == 1;
1803                    for s in specs {
1804                        let names: Vec<String> = if s.kind().starts_with("type") {
1805                            s.child_by_field_name("name").map(|n| vec![txt(n, src).to_string()]).unwrap_or_default()
1806                        } else {
1807                            let mut sc = s.walk();
1808                            s.children_by_field_name("name", &mut sc).map(|n| txt(n, src).to_string()).collect()
1809                        };
1810                        let doc = if single { outer_doc.clone() } else { go_doc(s, src) };
1811                        let sline = s.start_position().row as u32 + 1;
1812                        for name in names.iter().filter(|n| exported(n)) {
1813                            if s.kind().starts_with("type") {
1814                                let full = txt(s, src);
1815                                let sig = if full.len() <= 900 {
1816                                    format!("type {}", compact(full, 900))
1817                                } else {
1818                                    format!("type {} …", compact(&full[..full.find('{').unwrap_or(200.min(full.len()))], 300))
1819                                };
1820                                let kind = if full.contains("interface") { Kind::Interface } else { Kind::Struct };
1821                                out.push(entry(kind, name, format!("{pkg}.{name}"), rel, sline, sig, doc.clone()));
1822                                // Interface methods
1823                                if let Some(t) = s.child_by_field_name("type").filter(|t| t.kind() == "interface_type") {
1824                                    let mut ic = t.walk();
1825                                    let elems: Vec<Node> = t.named_children(&mut ic).collect();
1826                                    for m in elems {
1827                                        if !matches!(m.kind(), "method_elem" | "method_spec") {
1828                                            continue;
1829                                        }
1830                                        let Some(mn) = m.child_by_field_name("name").map(|n| txt(n, src)) else {
1831                                            continue;
1832                                        };
1833                                        out.push(entry(
1834                                            Kind::Method,
1835                                            mn,
1836                                            format!("{pkg}.{name}.{mn}"),
1837                                            rel,
1838                                            m.start_position().row as u32 + 1,
1839                                            compact(txt(m, src), 400),
1840                                            go_doc(m, src),
1841                                        ));
1842                                    }
1843                                }
1844                            } else {
1845                                let kw = if c.kind() == "const_declaration" { "const" } else { "var" };
1846                                out.push(entry(
1847                                    Kind::Const,
1848                                    name,
1849                                    format!("{pkg}.{name}"),
1850                                    rel,
1851                                    sline,
1852                                    compact(&format!("{kw} {}", txt(s, src)), 300),
1853                                    doc.clone(),
1854                                ));
1855                            }
1856                        }
1857                    }
1858                }
1859                _ => {}
1860            }
1861        }
1862    });
1863}
1864
1865#[cfg(test)]
1866mod tests {
1867    use super::*;
1868
1869    fn run(f: fn(&str, &str, &mut Vec<Entry>), text: &str, rel: &str) -> Vec<Entry> {
1870        let mut out = Vec::new();
1871        f(text, rel, &mut out);
1872        out
1873    }
1874
1875    #[test]
1876    fn typescript_declarations() {
1877        let src = r#"
1878/** Creates an object schema. */
1879export declare function object<T extends ZodRawShape>(shape: T, params?: string): ZodObject<T>;
1880export declare namespace z {
1881  /** A string schema. */
1882  function string(): ZodString;
1883}
1884export interface ZodType<T> {
1885  /** Parse or throw. */
1886  parse(data: unknown): T;
1887  readonly _def: Def;
1888}
1889export declare class ZodObject<T> extends ZodType<T> {
1890  /** Disallow unknown keys. */
1891  strict(message?: string): ZodObject<T>;
1892  private _cached;
1893}
1894declare const objectType: <T>(shape: T) => ZodObject<T>;
1895export { objectType as obj };
1896export * as zz from "./external";
1897"#;
1898        let e = run(ts_file, src, "lib/index.d.ts");
1899        let get = |p: &str| {
1900            e.iter()
1901                .find(|x| x.path == p)
1902                .unwrap_or_else(|| panic!("missing {p}: {:#?}", e.iter().map(|x| &x.path).collect::<Vec<_>>()))
1903        };
1904        assert_eq!(get("object").doc, "Creates an object schema.");
1905        assert!(get("object").sig.contains("ZodObject<T>"));
1906        assert_eq!(get("z.string").doc, "A string schema.");
1907        assert_eq!(get("ZodType.parse").kind, Kind::Method);
1908        assert!(e.iter().all(|x| x.name != "_def" && x.name != "_cached"));
1909        assert_eq!(get("ZodObject.strict").doc, "Disallow unknown keys.");
1910        assert_eq!(get("objectType").kind, Kind::Function);
1911        assert_eq!(get("obj").alias_of.as_deref(), Some("objectType"));
1912        assert_eq!(get("zz").kind, Kind::Module);
1913        assert_eq!(get("object").line, 3);
1914    }
1915
1916    #[test]
1917    fn legacy_dirs() {
1918        assert!(legacy_file("v3/types.d.ts", 4));
1919        assert!(!legacy_file("v4/classic/schemas.d.ts", 4));
1920        assert!(legacy_file("pydantic/v1/main.py", 2));
1921        assert!(!legacy_file("pydantic/main.py", 2));
1922        assert!(!legacy_file("src/v2/x.rs", 1));
1923    }
1924
1925    #[test]
1926    fn python_definitions() {
1927        let src = "\"\"\"Module docs.\"\"\"\n\nclass BaseModel:\n    \"\"\"Base class.\n\n    More.\n    \"\"\"\n\n    def model_dump(self, *, mode: str = 'python') -> dict:\n        \"\"\"Dump the model.\"\"\"\n        return {}\n\n    @classmethod\n    def model_validate(cls, obj):\n        '''Validate.'''\n\n    def _private(self): pass\n\ndef create_model(name: str, **fields) -> type:\n    return None\n";
1928        let e = run(py_file, src, "pydantic/main.py");
1929        let get = |p: &str| e.iter().find(|x| x.path == p).unwrap_or_else(|| panic!("missing {p}"));
1930        assert_eq!(get("pydantic.main.BaseModel").doc, "Base class.\n\nMore.");
1931        assert_eq!(
1932            get("pydantic.main.BaseModel.model_dump").sig,
1933            "def model_dump(self, *, mode: str = 'python') -> dict"
1934        );
1935        assert!(get("pydantic.main.BaseModel.model_validate").sig.starts_with("@classmethod def model_validate"));
1936        assert!(e.iter().all(|x| x.name != "_private"));
1937        assert!(e.iter().any(|x| x.kind == Kind::Prose && x.doc == "Module docs."));
1938        assert_eq!(get("pydantic.main.create_model").kind, Kind::Function);
1939    }
1940
1941    #[test]
1942    fn docs_pages_written_as_components() {
1943        let src = r#"import dedent from "dedent";
1944export const metadata = { title: "Installing with Vite", description: "Use the Vite plugin.", openGraph: { title: "x" } };
1945const steps = [
1946  {
1947    title: "Import Tailwind CSS",
1948    body: (
1949      <p>
1950        Add an <code>@import</code> to your CSS file.
1951      </p>
1952    ),
1953    code: { name: "CSS", lang: "css", code: dedent`
1954        @import "tailwindcss";
1955      ` },
1956  },
1957  { title: 'Old way', body: () => <p>Use directives{' '}here.</p>, code: { lang: 'css', code: '@tailwind base;\n@tailwind utilities;' } },
1958];
1959export default function Page() { return <div className="prose"><h3 className="sr-only">Overview</h3><p>Fast.</p></div> }
1960"#;
1961        let (md, title) = page_markdown(src).unwrap();
1962        assert_eq!(title.as_deref(), Some("Installing with Vite"));
1963        assert!(md.contains("## Import Tailwind CSS"), "{md}");
1964        assert!(md.contains("Add an `@import` to your CSS file."), "{md}");
1965        assert!(md.contains("```css\n@import \"tailwindcss\";\n```"), "{md}");
1966        assert!(md.contains("@tailwind base;\n@tailwind utilities;"), "{md}");
1967        assert!(md.contains("### Overview"), "{md}");
1968        assert!(!md.contains("dedent") && !md.contains("prose"), "{md}");
1969        let mut out = Vec::new();
1970        prose_titled(&md, "upstream:site/docs/installation/page.tsx", 0, title, &mut out);
1971        let s = out.iter().find(|e| e.name == "Installing with Vite › Import Tailwind CSS").unwrap();
1972        assert_eq!(s.line, 5);
1973        assert_eq!(
1974            page_title("upstream:site/src/app/(docs)/docs/installation/(tabs)/using-vite/page.tsx"),
1975            "installation/using-vite"
1976        );
1977    }
1978
1979    #[test]
1980    fn rust_items_and_macro_bodies() {
1981        let src = "//! Crate docs.\n//!\n//! # Usage\n//! Call spawn.\n\ncfg_rt! {\n    /// Spawns a new task.\n    #[track_caller]\n    pub fn spawn<F>(future: F) -> JoinHandle<F::Output>\n    where F: Future + Send + 'static {\n        todo!()\n    }\n}\n\n/// A router.\npub struct Router<S = ()> { inner: Inner }\n\nimpl<S> Router<S> {\n    /// Add a route.\n    pub fn route(self, path: &str, m: MethodRouter<S>) -> Self { self }\n    fn private(&self) {}\n}\n\n/// Hidden\n#[doc(hidden)]\npub fn hidden() {}\n\n/// Macro.\n#[macro_export]\nmacro_rules! select { () => {} }\n";
1982        let mut e = Vec::new();
1983        rs_file(src, "src/task/spawn.rs", "tokio", &mut e);
1984        let get = |p: &str| {
1985            e.iter()
1986                .find(|x| x.path == p)
1987                .unwrap_or_else(|| panic!("missing {p}: {:#?}", e.iter().map(|x| &x.path).collect::<Vec<_>>()))
1988        };
1989        let spawn = get("tokio::task::spawn::spawn");
1990        assert_eq!(spawn.doc, "Spawns a new task.");
1991        assert_eq!(spawn.line, 9);
1992        assert!(spawn.sig.contains("JoinHandle"));
1993        assert_eq!(get("tokio::task::spawn::Router::route").doc, "Add a route.");
1994        assert!(e.iter().all(|x| x.name != "private" && x.name != "hidden"));
1995        assert_eq!(get("tokio::select!").kind, Kind::Macro);
1996        assert!(e.iter().any(|x| x.kind == Kind::Prose && x.path.ends_with("Usage")));
1997
1998        // tokio's doc-wrapper pattern
1999        let src = "macro_rules! doc {\n    ($select:item) => {\n        /// Waits on multiple branches.\n        #[macro_export]\n        $select\n    };\n}\n\ndoc! {macro_rules! select {\n    () => {};\n}}\n";
2000        let mut e = Vec::new();
2001        rs_file(src, "src/macros/select.rs", "tokio", &mut e);
2002        let sel = e.iter().find(|x| x.path == "tokio::select!").expect("select! extracted");
2003        assert_eq!(sel.doc, "Waits on multiple branches.");
2004        assert!(e.iter().all(|x| x.name != "doc"));
2005    }
2006
2007    #[test]
2008    fn go_declarations() {
2009        let src = "// Package gin implements a web framework.\npackage gin\n\n// Context is the request context.\ntype Context struct {\n\tKeys map[string]any\n}\n\n// JSON serializes obj.\nfunc (c *Context) JSON(code int, obj any) {\n}\n\n// Default returns an Engine.\nfunc Default(opts ...Option) *Engine {\n\treturn nil\n}\n\nfunc internal() {}\n\ntype Handler interface {\n\t// Serve serves.\n\tServe(c *Context)\n}\n";
2010        let mut e = Vec::new();
2011        go_file(src, "context.go", "github.com/gin-gonic/gin", &mut e);
2012        let get = |p: &str| {
2013            e.iter()
2014                .find(|x| x.path == p)
2015                .unwrap_or_else(|| panic!("missing {p}: {:#?}", e.iter().map(|x| &x.path).collect::<Vec<_>>()))
2016        };
2017        assert_eq!(get("gin.Context.JSON").doc, "JSON serializes obj.");
2018        assert_eq!(get("gin.Default").sig, "func Default(opts ...Option) *Engine");
2019        assert_eq!(get("gin.Context").doc, "Context is the request context.");
2020        assert_eq!(get("gin.Handler.Serve").doc, "Serve serves.");
2021        assert!(e.iter().all(|x| x.name != "internal"));
2022        assert!(e.iter().any(|x| x.kind == Kind::Prose && x.doc.contains("web framework")));
2023    }
2024}