1use crate::locate::Source;
6use crate::markdown;
7use crate::Eco;
8use rayon::prelude::*;
9use serde::{Deserialize, Serialize};
10use std::cell::RefCell;
11use std::collections::{HashMap, HashSet};
12use std::path::{Path, PathBuf};
13use tree_sitter::{Node, Parser};
14
15#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
16pub enum Kind {
17 Function,
18 Method,
19 Class,
20 Interface,
21 Type,
22 Const,
23 Enum,
24 Struct,
25 Trait,
26 Macro,
27 Module,
28 Field,
29 Alias,
30 Prose,
31}
32
33impl Kind {
34 pub fn as_str(self) -> &'static str {
35 match self {
36 Kind::Function => "function",
37 Kind::Method => "method",
38 Kind::Class => "class",
39 Kind::Interface => "interface",
40 Kind::Type => "type",
41 Kind::Const => "const",
42 Kind::Enum => "enum",
43 Kind::Struct => "struct",
44 Kind::Trait => "trait",
45 Kind::Macro => "macro",
46 Kind::Module => "module",
47 Kind::Field => "field",
48 Kind::Alias => "re-export",
49 Kind::Prose => "doc",
50 }
51 }
52 pub fn is_container(self) -> bool {
53 matches!(self, Kind::Class | Kind::Interface | Kind::Struct | Kind::Trait | Kind::Enum | Kind::Module)
54 }
55}
56
57#[derive(Debug, Clone, Serialize, Deserialize)]
58pub struct Entry {
59 pub kind: Kind,
60 pub name: String,
62 pub path: String,
64 pub file: String,
66 pub line: u32,
67 pub sig: String,
69 pub doc: String,
71 pub alias_of: Option<String>,
73 #[serde(default)]
76 pub legacy: bool,
77}
78
79fn legacy_file(file: &str, major: u64) -> bool {
81 major > 1
82 && file.split('/').any(|seg| {
83 seg.len() >= 2 && seg.starts_with('v') && seg[1..].chars().all(|c| c.is_ascii_digit()) && seg[1..].parse::<u64>().is_ok_and(|n| n < major)
84 })
85}
86
87const SKIP_DIRS: &[&str] = &[
90 "node_modules",
91 "compiled",
92 ".git",
93 "test",
94 "tests",
95 "__tests__",
96 "__test__",
97 "testdata",
98 "test_data",
99 "benches",
100 "bench",
101 "benchmarks",
102 "fixtures",
103 "__fixtures__",
104 "__pycache__",
105 "examples",
106 "example",
107 "_examples",
108 "coverage",
109 ".github",
110 "vendor",
111 "internal",
112 "__snapshots__",
113];
114
115const MAX_FILES: usize = 25_000;
116const MAX_FILE_BYTES: u64 = 3 * 1024 * 1024;
117
118fn is_doc_name(name: &str) -> bool {
119 let u = name.to_ascii_uppercase();
120 [
121 "README",
122 "CHANGELOG",
123 "CHANGES",
124 "HISTORY",
125 "MIGRATION",
126 "MIGRATING",
127 "UPGRADING",
128 "UPGRADE",
129 "RELEASES",
130 "NEWS",
131 "GUIDE",
132 "API",
133 ]
134 .iter()
135 .any(|p| u.starts_with(p))
136}
137
138fn doc_ext(ext: &str) -> bool {
139 matches!(ext, "md" | "mdx" | "markdown" | "rst")
140}
141
142fn walk(dir: &Path, rel: &Path, depth: usize, out: &mut Vec<PathBuf>) {
143 if depth > 14 || out.len() >= MAX_FILES {
144 return;
145 }
146 let Ok(rd) = std::fs::read_dir(dir) else { return };
147 let mut entries: Vec<_> = rd.flatten().collect();
148 entries.sort_by_key(|e| e.file_name());
149 for e in entries {
150 let name = e.file_name().to_string_lossy().to_string();
151 let Ok(ft) = e.file_type() else { continue };
152 let r = rel.join(&name);
153 if ft.is_dir() {
154 if SKIP_DIRS.contains(&name.as_str()) || name.starts_with('.') {
155 continue;
156 }
157 walk(&e.path(), &r, depth + 1, out);
158 } else if ft.is_file() || ft.is_symlink() {
159 out.push(r);
160 }
161 if out.len() >= MAX_FILES {
162 return;
163 }
164 }
165}
166
167fn walk_all(dir: &Path, rel: &Path, depth: usize, out: &mut Vec<PathBuf>) {
169 if depth > 16 || out.len() >= MAX_FILES {
170 return;
171 }
172 let Ok(rd) = std::fs::read_dir(dir) else { return };
173 let mut entries: Vec<_> = rd.flatten().collect();
174 entries.sort_by_key(|e| e.file_name());
175 for e in entries {
176 let name = e.file_name().to_string_lossy().to_string();
177 if name.starts_with('.') {
178 continue;
179 }
180 let r = rel.join(&name);
181 match e.file_type() {
182 Ok(t) if t.is_dir() => walk_all(&e.path(), &r, depth + 1, out),
183 Ok(_) => out.push(r),
184 _ => {}
185 }
186 }
187}
188
189fn ext_of(p: &Path) -> String {
190 let s = p.file_name().map(|f| f.to_string_lossy().to_ascii_lowercase()).unwrap_or_default();
191 for e in [".d.ts", ".d.mts", ".d.cts"] {
192 if s.ends_with(e) {
193 return e[1..].to_string();
194 }
195 }
196 p.extension().map(|e| e.to_string_lossy().to_ascii_lowercase()).unwrap_or_default()
197}
198
199fn in_docs_dir(p: &Path) -> bool {
200 p.components().any(|c| {
201 matches!(
202 c.as_os_str().to_string_lossy().to_ascii_lowercase().as_str(),
203 "docs" | "doc" | "documentation" | "guide" | "guides"
204 )
205 })
206}
207
208struct Job {
210 abs: PathBuf,
211 rel: String,
212 how: How,
213}
214
215#[derive(Clone, Copy, PartialEq)]
216enum How {
217 Prose,
218 Page,
220 Example,
221 Metadata,
222 Ts,
223 Python,
224 Rust,
225 Go,
226}
227
228fn plan(eco: Eco, src: &Source, extra_types: Option<&Path>, upstream: Option<&Path>) -> Vec<Job> {
230 let mut rels = Vec::new();
231 match &src.files {
232 Some(list) => rels.extend(list.iter().cloned()),
233 None => walk(&src.dir, Path::new(""), 0, &mut rels),
234 }
235 let mut jobs = Vec::new();
236 let names: HashSet<String> = rels.iter().map(|r| r.to_string_lossy().replace('\\', "/")).collect();
237 let mut has_dts = false;
238 for r in &rels {
239 let e = ext_of(r);
240 if e.starts_with("d.") {
241 has_dts = true;
242 }
243 }
244 let rs_crate_src = eco == Eco::Cargo;
245 for r in rels {
246 let rel = r.to_string_lossy().replace('\\', "/");
247 if rel.split('/').any(|seg| SKIP_DIRS.contains(&seg) && src.files.is_some()) {
248 continue;
249 }
250 let abs = src.dir.join(&r);
251 let fname = r.file_name().map(|f| f.to_string_lossy().to_string()).unwrap_or_default();
252 let e = ext_of(&r);
253 let depth = rel.matches('/').count();
254 let how = if doc_ext(&e) || (e.is_empty() || e == "txt") && is_doc_name(&fname) && depth == 0 {
255 if is_doc_name(&fname) || in_docs_dir(&r) || depth == 0 {
256 Some(How::Prose)
257 } else {
258 None
259 }
260 } else {
261 match eco {
262 Eco::Npm => match e.as_str() {
263 "d.ts" => Some(How::Ts),
264 "d.mts" | "d.cts" => {
265 let base = rel.trim_end_matches(".d.mts").trim_end_matches(".d.cts");
266 if names.contains(&format!("{base}.d.ts")) {
267 None
268 } else {
269 Some(How::Ts)
270 }
271 }
272 "ts" | "mts" | "cts" | "tsx" if !has_dts => Some(How::Ts),
273 "js" | "mjs" | "cjs" if !has_dts && !fname.contains(".min.") => Some(How::Ts),
274 _ => None,
275 },
276 Eco::PyPI => match e.as_str() {
277 "py" => Some(How::Python),
278 "pyi" => {
279 let base = rel.trim_end_matches(".pyi");
280 if names.contains(&format!("{base}.py")) {
281 None
282 } else {
283 Some(How::Python)
284 }
285 }
286 _ => None,
287 },
288 Eco::Cargo => (e == "rs" && rs_crate_src && rel.starts_with("src/")).then_some(How::Rust),
289 Eco::Go => (e == "go" && !fname.ends_with("_test.go")).then_some(How::Go),
290 }
291 };
292 if let Some(how) = how {
293 jobs.push(Job { abs, rel, how });
294 }
295 }
296 if let Some(up) = upstream {
298 let pages: HashSet<String> = std::fs::read_to_string(up.join(".lockdocs-upstream.json"))
299 .ok()
300 .and_then(|t| serde_json::from_str::<crate::upstream::Manifest>(&t).ok())
301 .map(|m| m.pages.into_iter().collect())
302 .unwrap_or_default();
303 let mut rels = Vec::new();
304 walk_all(up, Path::new(""), 0, &mut rels);
305 for r in rels {
306 let e = ext_of(&r);
307 let plain = r.to_string_lossy().replace('\\', "/");
308 let rel = format!("upstream:{plain}");
309 if pages.contains(&plain) {
310 jobs.push(Job {
311 abs: up.join(&r),
312 rel,
313 how: How::Page,
314 });
315 continue;
316 }
317 if doc_ext(&e) || e == "mdoc" || e == "txt" {
319 jobs.push(Job {
320 abs: up.join(&r),
321 rel,
322 how: How::Prose,
323 });
324 } else if EXAMPLE_EXTS.contains(&e.as_str()) {
325 jobs.push(Job {
327 abs: up.join(&r),
328 rel,
329 how: How::Example,
330 });
331 }
332 }
333 }
334 if let Some(meta) = &src.metadata {
335 jobs.push(Job {
336 abs: meta.clone(),
337 rel: "README (package metadata)".into(),
338 how: How::Metadata,
339 });
340 }
341 if let (Some(types), false) = (extra_types, has_dts) {
343 let mut rels = Vec::new();
344 walk(types, Path::new(""), 0, &mut rels);
345 let label = types
346 .iter()
347 .rev()
348 .take(2)
349 .collect::<Vec<_>>()
350 .into_iter()
351 .rev()
352 .map(|s| s.to_string_lossy())
353 .collect::<Vec<_>>()
354 .join("/");
355 for r in rels {
356 if ext_of(&r) == "d.ts" {
357 let rel = format!("{label}/{}", r.to_string_lossy().replace('\\', "/"));
358 jobs.push(Job {
359 abs: types.join(&r),
360 rel,
361 how: How::Ts,
362 });
363 }
364 }
365 }
366 jobs
367}
368
369pub fn extract(eco: Eco, pkg_name: &str, src: &Source, extra_types: Option<&Path>, upstream: Option<&Path>) -> Vec<Entry> {
371 let major: u64 = src.version.trim_start_matches('v').split('.').next().and_then(|m| m.parse().ok()).unwrap_or(0);
372 let jobs = plan(eco, src, extra_types, upstream);
373 let crate_name = if eco == Eco::Cargo {
374 rust_crate_name(&src.dir).unwrap_or_else(|| pkg_name.replace('-', "_"))
375 } else {
376 String::new()
377 };
378 let includes = upstream.map(Includes::scan).unwrap_or_default();
379 let mut all: Vec<Entry> = jobs
380 .par_iter()
381 .flat_map_iter(|job| {
382 let Ok(meta) = std::fs::metadata(&job.abs) else { return Vec::new() };
383 if meta.len() > MAX_FILE_BYTES && job.how != How::Prose {
384 return Vec::new();
385 }
386 let Ok(bytes) = std::fs::read(&job.abs) else { return Vec::new() };
387 let text = String::from_utf8_lossy(&bytes);
388 let mut out = Vec::new();
389 match job.how {
390 How::Prose => {
391 prose(&text, &job.rel, 0, &mut out);
392 if job.rel.starts_with("upstream:") && !includes.files.is_empty() {
393 for e in &mut out {
394 e.doc = includes.expand(&e.doc);
395 }
396 }
397 }
398 How::Page => {
399 if let Some((md, title)) = page_markdown(&text) {
400 let title = title.unwrap_or_else(|| page_title(&job.rel));
401 prose_titled(&md, &job.rel, 0, Some(title), &mut out);
402 }
403 }
404 How::Example => example(&text, &job.rel, &mut out),
405 How::Metadata => {
406 let (body, line) = markdown::metadata_body(&text);
407 prose(&body, &job.rel, line - 1, &mut out)
408 }
409 How::Ts => ts_file(&text, &job.rel, &mut out),
410 How::Python => py_file(&text, &job.rel, &mut out),
411 How::Rust => rs_file(&text, &job.rel, &crate_name, &mut out),
412 How::Go => go_file(&text, &job.rel, pkg_name, &mut out),
413 }
414 out
415 })
416 .collect();
417 for e in &mut all {
418 e.legacy = legacy_file(&e.file, major);
419 }
420 let mut seen = HashSet::new();
422 all.retain(|e| seen.insert((e.path.clone(), e.sig.clone(), e.doc.len())));
423 all
424}
425
426fn rust_crate_name(dir: &Path) -> Option<String> {
427 let t = std::fs::read_to_string(dir.join("Cargo.toml")).ok()?;
428 let v: toml::Table = toml::from_str(&t).ok()?;
429 let lib = v.get("lib").and_then(|l| l.get("name")).and_then(|n| n.as_str());
430 let pkg = v.get("package").and_then(|p| p.get("name")).and_then(|n| n.as_str());
431 lib.or(pkg).map(|n| n.replace('-', "_"))
432}
433
434#[derive(Default)]
438struct Includes {
439 files: Vec<(String, PathBuf)>,
441}
442
443impl Includes {
444 fn scan(dir: &Path) -> Includes {
445 let mut rels = Vec::new();
446 walk_all(dir, Path::new(""), 0, &mut rels);
447 let files = rels
448 .into_iter()
449 .filter(|r| EXAMPLE_EXTS.contains(&ext_of(r).as_str()))
450 .map(|r| {
451 let s = r.to_string_lossy().replace('\\', "/").to_ascii_lowercase();
452 let stem = s.rsplit_once('.').map_or(s.clone(), |x| x.0.to_string());
453 (stem, dir.join(&r))
454 })
455 .collect();
456 Includes { files }
457 }
458
459 fn find(&self, reference: &str) -> Option<&PathBuf> {
460 let r = reference.trim().trim_matches(['"', '\'']).to_ascii_lowercase();
461 let r = r.split_whitespace().next().unwrap_or("");
462 let r = r.rsplit_once('.').map_or(r, |x| x.0);
463 let segs: Vec<&str> = r
464 .split('/')
465 .filter(|s| !s.is_empty() && *s != ".." && *s != "." && *s != ".tmp_examples")
466 .collect();
467 for take in (1..=segs.len().min(3)).rev() {
468 let suffix = segs[segs.len() - take..].join("/");
469 let hits: Vec<&(String, PathBuf)> = self.files.iter().filter(|(k, _)| *k == suffix || k.ends_with(&format!("/{suffix}"))).collect();
470 if hits.len() == 1 {
471 return Some(&hits[0].1);
472 }
473 }
474 None
475 }
476
477 fn expand(&self, text: &str) -> String {
478 if !(text.contains("{!") || text.contains("{*") || text.contains("--8<--")) {
479 return text.to_string();
480 }
481 let mut out = String::with_capacity(text.len());
482 for line in text.lines() {
483 let t = line.trim();
484 let reference = t
485 .strip_prefix("{!")
486 .and_then(|x| x.strip_suffix("!}"))
487 .or_else(|| t.strip_prefix("{*").and_then(|x| x.strip_suffix("*}")))
488 .or_else(|| t.strip_prefix("--8<--"));
489 match reference
490 .and_then(|r| self.find(r))
491 .and_then(|p| std::fs::read_to_string(p).ok().map(|c| (p, c)))
492 {
493 Some((p, code)) => {
494 let lang = p.extension().map(|e| e.to_string_lossy().to_string()).unwrap_or_default();
495 let code: String = code.chars().take(3000).collect();
496 out.push_str(&format!("```{lang}\n{}\n```\n", code.trim_end()));
497 }
498 None => {
499 out.push_str(line);
500 out.push('\n');
501 }
502 }
503 }
504 out
505 }
506}
507
508const EXAMPLE_EXTS: &[&str] = &["py", "ts", "tsx", "js", "jsx", "mjs", "rs", "go"];
509
510fn example(text: &str, rel: &str, out: &mut Vec<Entry>) {
512 if text.len() > 12_000 || text.trim().is_empty() {
513 return;
514 }
515 let name = rel.rsplit('/').next().unwrap_or(rel);
516 let lang = name.rsplit('.').next().unwrap_or("");
517 let title = name.rsplit_once('.').map_or(name, |x| x.0).replace(['_', '-'], " ");
518 out.push(Entry {
519 kind: Kind::Prose,
520 name: format!("example › {title}"),
521 path: format!("example › {title}"),
522 file: rel.to_string(),
523 line: 1,
524 sig: String::new(),
525 doc: format!("```{lang}\n{}\n```", text.trim_end()),
526 alias_of: None,
527 legacy: false,
528 });
529}
530
531fn page_title(rel: &str) -> String {
533 let segs: Vec<&str> = rel.split('/').filter(|s| !(s.starts_with('(') && s.ends_with(')'))).collect();
534 let mut segs: Vec<&str> = segs.iter().rev().take(3).rev().copied().collect();
535 if let Some(last) = segs.last() {
536 if last.starts_with("page.") || last.starts_with("index.") {
537 segs.pop();
538 }
539 }
540 segs.join("/")
541 .trim_end_matches(".tsx")
542 .trim_end_matches(".jsx")
543 .trim_end_matches(".js")
544 .to_string()
545}
546
547fn page_markdown(text: &str) -> Option<(String, Option<String>)> {
553 if text.len() > 200_000 {
554 return None;
555 }
556 parse_with("tsx", text, |root, src| {
557 let mut p = PageOut::default();
558 page_walk(root, src, &mut p, false);
559 p.flush();
560 let mut md = String::new();
561 let mut n = 0u32;
562 for (line, t) in &p.out {
563 while n + 1 < *line {
564 md.push('\n');
565 n += 1;
566 }
567 md.push_str(t);
568 md.push('\n');
569 n += 1 + t.matches('\n').count() as u32;
570 }
571 (md, p.title)
572 })
573 .filter(|(md, _)| !md.trim().is_empty())
574}
575
576#[derive(Default)]
577struct PageOut {
578 out: Vec<(u32, String)>,
579 para: String,
580 para_line: u32,
581 title: Option<String>,
582}
583
584impl PageOut {
585 fn text(&mut self, line: u32, t: &str) {
586 if self.para.trim().is_empty() {
587 self.para_line = line;
588 }
589 self.para.push_str(t);
590 }
591 fn flush(&mut self) {
592 let t = compact(&self.para, usize::MAX);
593 if !t.is_empty() {
594 self.out.push((self.para_line, t));
595 }
596 self.para.clear();
597 }
598 fn block(&mut self, line: u32, t: String) {
599 self.flush();
600 self.out.push((line, t));
601 }
602}
603
604fn line_of(n: Node) -> u32 {
605 n.start_position().row as u32 + 1
606}
607
608fn js_string(n: Node, src: &[u8]) -> Option<String> {
610 match n.kind() {
611 "string" => {
612 let raw = txt(n, src);
613 let inner = raw.get(1..raw.len().saturating_sub(1)).unwrap_or("");
614 Some(inner.replace("\\n", "\n").replace("\\'", "'").replace("\\\"", "\"").replace("\\\\", "\\"))
615 }
616 "template_string" => {
617 let raw = txt(n, src);
618 let inner = raw.get(1..raw.len().saturating_sub(1)).unwrap_or("");
619 Some(dedent(&inner.trim_matches('\n').lines().collect::<Vec<_>>()))
620 }
621 "call_expression" => n
622 .child_by_field_name("arguments")
623 .filter(|a| a.kind() == "template_string")
624 .and_then(|a| js_string(a, src)),
625 "parenthesized_expression" => n.named_child(0).and_then(|c| js_string(c, src)),
626 _ => None,
627 }
628}
629
630fn jsx_name(n: Node, src: &[u8]) -> String {
631 let open = if n.kind() == "jsx_element" {
632 n.child_by_field_name("open_tag")
633 } else {
634 Some(n)
635 };
636 open.and_then(|o| o.child_by_field_name("name"))
637 .map(|x| txt(x, src).to_string())
638 .unwrap_or_default()
639}
640
641fn jsx_text(n: Node, src: &[u8], out: &mut String) {
643 match n.kind() {
644 "jsx_text" => out.push_str(txt(n, src)),
645 "jsx_expression" => {
646 if let Some(v) = n.named_child(0).and_then(|c| js_string(c, src)) {
647 out.push_str(&v);
648 }
649 }
650 "jsx_attribute" | "jsx_opening_element" | "jsx_closing_element" => {}
651 _ => {
652 let mut c = n.walk();
653 for ch in n.named_children(&mut c) {
654 jsx_text(ch, src, out);
655 }
656 }
657 }
658}
659
660fn page_walk(n: Node, src: &[u8], p: &mut PageOut, in_array: bool) {
661 match n.kind() {
662 "import_statement" | "comment" | "jsx_attribute" => {}
663 "pair" => {
664 let key = n
665 .child_by_field_name("key")
666 .map(|k| txt(k, src).trim_matches(['"', '\'']).to_string())
667 .unwrap_or_default();
668 let Some(value) = n.child_by_field_name("value") else { return };
669 match key.as_str() {
670 "title" | "heading" => {
671 if let Some(t) = js_string(value, src) {
672 if !in_array && p.title.is_none() {
673 p.title = Some(t);
674 } else if in_array {
675 p.block(line_of(n), format!("## {}", compact(&t, 200)));
676 }
677 }
678 }
679 "description" => {
680 if let Some(t) = js_string(value, src) {
681 p.block(line_of(value), compact(&t, usize::MAX));
682 }
683 }
684 "code" => match js_string(value, src) {
685 Some(code) => {
686 let lang = n
687 .parent()
688 .and_then(|obj| {
689 let mut c = obj.walk();
690 let found = obj.named_children(&mut c).find(|x| {
691 x.kind() == "pair" && x.child_by_field_name("key").is_some_and(|k| txt(k, src).trim_matches(['"', '\'']) == "lang")
692 });
693 found
694 })
695 .and_then(|x| x.child_by_field_name("value"))
696 .and_then(|v| js_string(v, src))
697 .unwrap_or_default();
698 p.block(line_of(value), format!("```{lang}\n{}\n```", code.trim_end()));
699 }
700 None => page_walk(value, src, p, in_array),
701 },
702 "name" | "lang" | "className" | "href" | "id" | "url" | "src" | "image" | "images" | "icon" | "openGraph" | "twitter" | "alternates" => {}
703 _ => page_walk(value, src, p, in_array),
704 }
705 }
706 "jsx_element" | "jsx_self_closing_element" => {
707 let name = jsx_name(n, src);
708 let b = name.as_bytes();
709 if b.len() == 2 && b[0] == b'h' && (b'1'..=b'6').contains(&b[1]) {
710 let mut t = String::new();
711 jsx_text(n, src, &mut t);
712 let t = compact(&t, 200);
713 if !t.is_empty() {
714 p.block(line_of(n), format!("{} {t}", "#".repeat((b[1] - b'0') as usize)));
715 }
716 } else if name == "code" {
717 let mut t = String::new();
718 jsx_text(n, src, &mut t);
719 p.text(line_of(n), &format!("`{}`", t.trim()));
720 } else {
721 let block = matches!(name.as_str(), "p" | "li" | "div" | "section" | "ul" | "ol" | "pre" | "table" | "tr");
722 if block {
723 p.flush();
724 }
725 let mut c = n.walk();
726 for ch in n.named_children(&mut c) {
727 page_walk(ch, src, p, in_array);
728 }
729 if block {
730 p.flush();
731 }
732 }
733 }
734 "jsx_text" => p.text(line_of(n), txt(n, src)),
735 "jsx_expression" => match n.named_child(0) {
736 Some(c) if matches!(c.kind(), "string" | "template_string") => {
737 let v = js_string(c, src).unwrap_or_default();
738 p.text(line_of(n), &v);
739 }
740 Some(c) => page_walk(c, src, p, in_array),
741 None => {}
742 },
743 _ => {
744 let arr = in_array || n.kind() == "array";
745 let mut c = n.walk();
746 for ch in n.named_children(&mut c) {
747 page_walk(ch, src, p, arr);
748 }
749 }
750 }
751}
752
753fn prose(text: &str, rel: &str, line_off: u32, out: &mut Vec<Entry>) {
754 prose_titled(text, rel, line_off, None, out)
755}
756
757fn prose_titled(text: &str, rel: &str, line_off: u32, given: Option<String>, out: &mut Vec<Entry>) {
758 let (text, fm_title) = markdown::clean_mdx(text, rel.ends_with(".mdx"));
759 let file_name = rel.rsplit('/').next().unwrap_or(rel);
761 let file_title = if rel.starts_with("upstream:") && !is_doc_name(file_name) {
762 file_name
763 .rsplit_once('.')
764 .map_or(file_name, |x| x.0)
765 .trim_start_matches(|c: char| c.is_ascii_digit() || c == '-')
766 .replace(['_', '-'], " ")
767 } else {
768 file_name.to_string()
769 };
770 let file_title = file_title.as_str();
771 let title = if let Some(t) = given {
772 t
773 } else if rel.contains("package metadata") {
774 "README".to_string()
775 } else if let Some(t) = fm_title {
776 t
777 } else {
778 file_title.to_string()
779 };
780 for s in markdown::split(&text, &title) {
781 out.push(Entry {
782 kind: Kind::Prose,
783 name: s.heading.clone(),
784 path: s.heading,
785 file: rel.to_string(),
786 line: s.line + line_off,
787 sig: String::new(),
788 doc: s.text,
789 alias_of: None,
790 legacy: false,
791 });
792 }
793}
794
795thread_local! {
798 static PARSERS: RefCell<Vec<(&'static str, Parser)>> = const { RefCell::new(Vec::new()) };
799}
800
801fn parse_with<R>(lang: &'static str, text: &str, f: impl FnOnce(Node, &[u8]) -> R) -> Option<R> {
804 let pooled = PARSERS.with(|ps| {
805 let mut ps = ps.borrow_mut();
806 ps.iter().position(|(l, _)| *l == lang).map(|i| ps.swap_remove(i).1)
807 });
808 let mut parser = match pooled {
809 Some(p) => p,
810 None => {
811 let mut p = Parser::new();
812 let language: tree_sitter::Language = match lang {
813 "ts" => tree_sitter_typescript::LANGUAGE_TYPESCRIPT.into(),
814 "tsx" => tree_sitter_typescript::LANGUAGE_TSX.into(),
815 "py" => tree_sitter_python::LANGUAGE.into(),
816 "rs" => tree_sitter_rust::LANGUAGE.into(),
817 _ => tree_sitter_go::LANGUAGE.into(),
818 };
819 p.set_language(&language).ok()?;
820 p
821 }
822 };
823 let tree = parser.parse(text, None);
824 PARSERS.with(|ps| ps.borrow_mut().push((lang, parser)));
825 let tree = tree?;
826 Some(f(tree.root_node(), text.as_bytes()))
827}
828
829fn txt<'a>(n: Node, src: &'a [u8]) -> &'a str {
830 n.utf8_text(src).unwrap_or("")
831}
832
833fn compact(s: &str, cap: usize) -> String {
834 let mut out = String::with_capacity(s.len().min(cap.saturating_add(8)));
835 let mut space = false;
836 for c in s.chars() {
837 if c.is_whitespace() {
838 space = true;
839 continue;
840 }
841 if space && !out.is_empty() {
842 out.push(' ');
843 }
844 space = false;
845 out.push(c);
846 if out.len() >= cap {
847 out.push_str(" …");
848 break;
849 }
850 }
851 out.replace("( ", "(").replace(", )", ")").replace(",)", ")").replace(" )", ")")
852}
853
854fn strip_py_comments(s: &str) -> String {
856 let mut out = String::with_capacity(s.len());
857 for line in s.lines() {
858 let mut quote: Option<char> = None;
859 let mut cut = line.len();
860 for (i, c) in line.char_indices() {
861 match (quote, c) {
862 (None, '#') => {
863 cut = i;
864 break;
865 }
866 (None, '"' | '\'') => quote = Some(c),
867 (Some(q), c) if c == q => quote = None,
868 _ => {}
869 }
870 }
871 out.push_str(&line[..cut]);
872 out.push('\n');
873 }
874 out
875}
876
877fn head(n: Node, src: &[u8], body_field: &str, cap: usize) -> String {
879 let end = n.child_by_field_name(body_field).map_or(n.end_byte(), |b| b.start_byte());
880 let s = std::str::from_utf8(&src[n.start_byte()..end]).unwrap_or("");
881 compact(s.trim_end().trim_end_matches(['{', ':', '=']).trim_end(), cap)
882}
883
884fn cap_doc(s: String) -> String {
885 const CAP: usize = 4000;
886 if s.len() <= CAP {
887 return s;
888 }
889 let mut i = CAP;
890 while !s.is_char_boundary(i) {
891 i -= 1;
892 }
893 format!("{} …", &s[..i])
894}
895
896fn dedent(lines: &[&str]) -> String {
897 let indent = lines
898 .iter()
899 .filter(|l| !l.trim().is_empty())
900 .map(|l| l.len() - l.trim_start().len())
901 .min()
902 .unwrap_or(0);
903 let v: Vec<&str> = lines.iter().map(|l| if l.len() >= indent { &l[indent..] } else { l.trim_start() }).collect();
904 v.join("\n").trim().to_string()
905}
906
907fn clean_block_comment(s: &str) -> String {
908 let s = s
909 .trim_start_matches("/**")
910 .trim_start_matches("/*!")
911 .trim_start_matches("/*")
912 .trim_end_matches("*/");
913 let lines: Vec<&str> = s
914 .lines()
915 .map(|l| {
916 let t = l.trim_start();
917 match t.strip_prefix('*') {
918 Some(r) => r.strip_prefix(' ').unwrap_or(r),
919 None => l,
920 }
921 })
922 .collect();
923 cap_doc(dedent(&lines))
924}
925
926fn entry(kind: Kind, name: &str, path: String, file: &str, line: u32, sig: String, doc: String) -> Entry {
927 Entry {
928 kind,
929 name: name.to_string(),
930 path,
931 file: file.to_string(),
932 line,
933 sig,
934 doc,
935 alias_of: None,
936 legacy: false,
937 }
938}
939
940fn ts_file(text: &str, rel: &str, out: &mut Vec<Entry>) {
943 let lang = if rel.ends_with(".tsx") || rel.ends_with(".jsx") { "tsx" } else { "ts" };
944 let is_js = rel.ends_with(".js") || rel.ends_with(".mjs") || rel.ends_with(".cjs");
945 parse_with(lang, text, |root, src| {
946 let mut w = TsWalker {
947 src,
948 file: rel,
949 out,
950 js: is_js,
951 };
952 w.block(root, "");
953 });
954}
955
956struct TsWalker<'a> {
957 src: &'a [u8],
958 file: &'a str,
959 out: &'a mut Vec<Entry>,
960 js: bool,
961}
962
963impl TsWalker<'_> {
964 fn doc(&self, n: Node) -> String {
965 let mut p = n.prev_sibling();
966 while let Some(c) = p {
967 if c.kind() == "comment" {
968 let t = txt(c, self.src);
969 if t.starts_with("/**") && c.end_position().row + 2 >= n.start_position().row {
970 return clean_block_comment(t);
971 }
972 if t.starts_with("//") {
973 p = c.prev_sibling();
974 continue;
975 }
976 }
977 break;
978 }
979 String::new()
980 }
981
982 fn line(&self, n: Node) -> u32 {
983 n.start_position().row as u32 + 1
984 }
985
986 fn block(&mut self, node: Node, prefix: &str) {
987 let mut cur = node.walk();
988 let children: Vec<Node> = node.named_children(&mut cur).collect();
989 for c in children {
990 match c.kind() {
991 "export_statement" => {
992 let doc = self.doc(c);
993 if let Some(d) = c.child_by_field_name("declaration") {
994 self.decl(d, prefix, doc, c);
995 } else {
996 self.exports(c, prefix);
997 }
998 }
999 "ambient_declaration" => {
1000 let doc = self.doc(c);
1001 let mut cc = c.walk();
1002 let inner: Vec<Node> = c.named_children(&mut cc).collect();
1003 for d in inner {
1004 self.decl(d, prefix, doc.clone(), c);
1005 }
1006 }
1007 "expression_statement" => {
1008 let mut cc = c.walk();
1009 let inner: Vec<Node> = c.named_children(&mut cc).collect();
1010 for d in inner {
1011 if matches!(d.kind(), "internal_module" | "module") {
1012 let doc = self.doc(c);
1013 self.decl(d, prefix, doc, c);
1014 }
1015 }
1016 }
1017 _ => {
1018 let doc = self.doc(c);
1019 if !self.js || !doc.is_empty() || !prefix.is_empty() {
1021 self.decl(c, prefix, doc, c);
1022 }
1023 }
1024 }
1025 }
1026 }
1027
1028 fn exports(&mut self, n: Node, prefix: &str) {
1029 let mut cur = n.walk();
1030 let children: Vec<Node> = n.named_children(&mut cur).collect();
1031 for c in children {
1032 match c.kind() {
1033 "export_clause" => {
1034 let mut cc = c.walk();
1035 let specs: Vec<Node> = c.named_children(&mut cc).collect();
1036 for s in specs {
1037 let (Some(name), alias) = (s.child_by_field_name("name"), s.child_by_field_name("alias")) else {
1038 continue;
1039 };
1040 let Some(alias) = alias else { continue };
1041 let (from, to) = (txt(name, self.src), txt(alias, self.src));
1042 if from == to || to == "default" {
1043 continue;
1044 }
1045 let mut e = entry(
1046 Kind::Alias,
1047 to,
1048 format!("{prefix}{to}"),
1049 self.file,
1050 self.line(s),
1051 compact(txt(n, self.src), 200),
1052 String::new(),
1053 );
1054 e.alias_of = Some(from.to_string());
1055 self.out.push(e);
1056 }
1057 }
1058 "namespace_export" => {
1059 if let Some(id) = c.named_child(0) {
1060 let name = txt(id, self.src);
1061 self.out.push(entry(
1062 Kind::Module,
1063 name,
1064 format!("{prefix}{name}"),
1065 self.file,
1066 self.line(n),
1067 compact(txt(n, self.src), 200),
1068 self.doc(n),
1069 ));
1070 }
1071 }
1072 _ => {}
1073 }
1074 }
1075 }
1076
1077 fn decl(&mut self, d: Node, prefix: &str, doc: String, outer: Node) {
1078 let src = self.src;
1079 let name_of = |n: Node| n.child_by_field_name("name").map(|x| txt(x, src).to_string());
1080 let line = self.line(outer);
1081 match d.kind() {
1082 "ambient_declaration" => {
1083 let mut cur = d.walk();
1084 let inner: Vec<Node> = d.named_children(&mut cur).collect();
1085 for x in inner {
1086 self.decl(x, prefix, doc.clone(), outer);
1087 }
1088 }
1089 "function_signature" | "function_declaration" | "generator_function_declaration" => {
1090 let Some(name) = name_of(d) else { return };
1091 let sig = head(outer, src, "__none__", 600).trim_end_matches(';').to_string();
1092 let sig = if d.child_by_field_name("body").is_some() {
1093 head(d, src, "body", 600)
1094 } else {
1095 sig
1096 };
1097 self.out
1098 .push(entry(Kind::Function, &name, format!("{prefix}{name}"), self.file, line, sig, doc));
1099 }
1100 "class_declaration" | "abstract_class_declaration" | "class" => {
1101 let Some(name) = name_of(d) else { return };
1102 self.out.push(entry(
1103 Kind::Class,
1104 &name,
1105 format!("{prefix}{name}"),
1106 self.file,
1107 line,
1108 head(d, src, "body", 400),
1109 doc,
1110 ));
1111 if let Some(body) = d.child_by_field_name("body") {
1112 self.members(body, &format!("{prefix}{name}."));
1113 }
1114 }
1115 "interface_declaration" => {
1116 let Some(name) = name_of(d) else { return };
1117 self.out.push(entry(
1118 Kind::Interface,
1119 &name,
1120 format!("{prefix}{name}"),
1121 self.file,
1122 line,
1123 head(d, src, "body", 400),
1124 doc,
1125 ));
1126 if let Some(body) = d.child_by_field_name("body") {
1127 self.members(body, &format!("{prefix}{name}."));
1128 }
1129 }
1130 "type_alias_declaration" => {
1131 let Some(name) = name_of(d) else { return };
1132 self.out.push(entry(
1133 Kind::Type,
1134 &name,
1135 format!("{prefix}{name}"),
1136 self.file,
1137 line,
1138 compact(txt(d, src), 700),
1139 doc,
1140 ));
1141 }
1142 "enum_declaration" => {
1143 let Some(name) = name_of(d) else { return };
1144 self.out.push(entry(
1145 Kind::Enum,
1146 &name,
1147 format!("{prefix}{name}"),
1148 self.file,
1149 line,
1150 compact(txt(d, src), 500),
1151 doc,
1152 ));
1153 }
1154 "lexical_declaration" | "variable_declaration" => {
1155 let kw = txt(d, src).split_whitespace().next().unwrap_or("const").to_string();
1156 let mut cur = d.walk();
1157 let decls: Vec<Node> = d.named_children(&mut cur).filter(|n| n.kind() == "variable_declarator").collect();
1158 for v in decls {
1159 let Some(nn) = v.child_by_field_name("name") else { continue };
1160 if nn.kind() != "identifier" {
1161 continue;
1162 }
1163 let name = txt(nn, src).to_string();
1164 let value = v.child_by_field_name("value");
1165 let is_fn = value.is_some_and(|x| matches!(x.kind(), "arrow_function" | "function_expression" | "function"))
1166 || v.child_by_field_name("type").is_some_and(|t| txt(t, src).contains("=>"));
1167 let sig = match value {
1168 Some(val) if matches!(val.kind(), "arrow_function" | "function_expression" | "function") => {
1169 let body_start = val.child_by_field_name("body").map_or(val.end_byte(), |b| b.start_byte());
1170 let s = std::str::from_utf8(&src[v.start_byte()..body_start]).unwrap_or("");
1171 compact(&format!("{kw} {}", s.trim_end().trim_end_matches("=>").trim_end()), 600)
1172 }
1173 Some(val) if val.end_byte() - val.start_byte() > 300 => {
1174 let s = std::str::from_utf8(&src[v.start_byte()..val.start_byte()]).unwrap_or("");
1175 compact(&format!("{kw} {s} …"), 600)
1176 }
1177 _ => compact(&format!("{kw} {}", txt(v, src)), 600),
1178 };
1179 let kind = if is_fn { Kind::Function } else { Kind::Const };
1180 self.out.push(entry(kind, &name, format!("{prefix}{name}"), self.file, line, sig, doc.clone()));
1181 }
1182 }
1183 "internal_module" | "module" => {
1184 let Some(nn) = d.child_by_field_name("name") else { return };
1185 let raw = txt(nn, src);
1186 let quoted = raw.starts_with('"') || raw.starts_with('\'');
1187 let name = raw.trim_matches(['"', '\'']).to_string();
1188 let new_prefix = if quoted { prefix.to_string() } else { format!("{prefix}{name}.") };
1189 if !quoted {
1190 self.out.push(entry(
1191 Kind::Module,
1192 &name,
1193 format!("{prefix}{name}"),
1194 self.file,
1195 line,
1196 head(d, src, "body", 200),
1197 doc,
1198 ));
1199 }
1200 if let Some(body) = d.child_by_field_name("body") {
1201 self.block(body, &new_prefix);
1202 }
1203 }
1204 _ => {}
1205 }
1206 }
1207
1208 fn members(&mut self, body: Node, prefix: &str) {
1209 let src = self.src;
1210 let mut cur = body.walk();
1211 let children: Vec<Node> = body.named_children(&mut cur).collect();
1212 for m in children {
1213 let kind = match m.kind() {
1214 "method_definition" | "method_signature" | "abstract_method_signature" => Kind::Method,
1215 "public_field_definition" | "property_signature" => Kind::Field,
1216 _ => continue,
1217 };
1218 let t = txt(m, src);
1219 if t.starts_with("private ") || t.starts_with("protected ") || t.contains(" private ") && kind == Kind::Field {
1220 continue;
1221 }
1222 let Some(nn) = m.child_by_field_name("name") else { continue };
1223 let name = txt(nn, src).trim_matches(['"', '\'']).to_string();
1224 if name.starts_with('#') || name.starts_with('_') || name.starts_with('[') {
1225 continue;
1226 }
1227 let doc = self.doc(m);
1228 let sig = if m.child_by_field_name("body").is_some() {
1229 head(m, src, "body", 500)
1230 } else {
1231 compact(t.trim_end_matches(';'), 500)
1232 };
1233 self.out.push(entry(
1234 kind,
1235 &name,
1236 format!("{prefix}{name}"),
1237 self.file,
1238 m.start_position().row as u32 + 1,
1239 sig,
1240 doc,
1241 ));
1242 }
1243 }
1244}
1245
1246fn py_module(rel: &str) -> String {
1249 let r = rel.trim_end_matches(".pyi").trim_end_matches(".py");
1250 let r = r.trim_end_matches("/__init__");
1251 r.replace('/', ".")
1252}
1253
1254fn py_file(text: &str, rel: &str, out: &mut Vec<Entry>) {
1255 let module = py_module(rel);
1256 parse_with("py", text, |root, src| {
1257 if let Some(doc) = py_docstring(root, src) {
1258 for s in markdown::split(&doc, &module) {
1259 out.push(Entry {
1260 kind: Kind::Prose,
1261 name: s.heading.clone(),
1262 path: s.heading,
1263 file: rel.to_string(),
1264 line: s.line,
1265 sig: String::new(),
1266 doc: s.text,
1267 alias_of: None,
1268 legacy: false,
1269 });
1270 }
1271 }
1272 py_block(root, src, &module, false, rel, out);
1273 });
1274}
1275
1276fn py_docstring(block: Node, src: &[u8]) -> Option<String> {
1277 let first = block.named_child(0)?;
1278 let first = if first.kind() == "comment" { first.next_named_sibling()? } else { first };
1279 if first.kind() != "expression_statement" {
1280 return None;
1281 }
1282 let s = first.named_child(0)?;
1283 if s.kind() != "string" {
1284 return None;
1285 }
1286 let raw = txt(s, src);
1287 let raw = raw.trim_start_matches(['r', 'R', 'u', 'U', 'b', 'B']);
1288 let inner = raw
1289 .strip_prefix("\"\"\"")
1290 .and_then(|x| x.strip_suffix("\"\"\""))
1291 .or_else(|| raw.strip_prefix("'''").and_then(|x| x.strip_suffix("'''")))
1292 .or_else(|| raw.strip_prefix('"').and_then(|x| x.strip_suffix('"')))
1293 .or_else(|| raw.strip_prefix('\'').and_then(|x| x.strip_suffix('\'')))?;
1294 let lines: Vec<&str> = inner.lines().collect();
1295 if lines.is_empty() {
1296 return None;
1297 }
1298 let first_line = lines[0].trim();
1300 let rest = dedent(&lines[1..]);
1301 let gap = if lines.get(1).is_some_and(|l| l.trim().is_empty()) { "\n\n" } else { "\n" };
1302 let doc = if rest.is_empty() {
1303 first_line.to_string()
1304 } else if first_line.is_empty() {
1305 rest
1306 } else {
1307 format!("{first_line}{gap}{rest}")
1308 };
1309 Some(cap_doc(doc))
1310}
1311
1312fn py_block(block: Node, src: &[u8], prefix: &str, in_class: bool, rel: &str, out: &mut Vec<Entry>) {
1313 let mut cur = block.walk();
1314 let children: Vec<Node> = block.named_children(&mut cur).collect();
1315 for c in children {
1316 let (def, decorators) = if c.kind() == "decorated_definition" {
1317 let mut dc = c.walk();
1318 let decs: Vec<String> = c
1319 .named_children(&mut dc)
1320 .filter(|n| n.kind() == "decorator")
1321 .map(|n| compact(txt(n, src), 80))
1322 .collect();
1323 match c.child_by_field_name("definition") {
1324 Some(d) => (d, decs),
1325 None => continue,
1326 }
1327 } else {
1328 (c, Vec::new())
1329 };
1330 let Some(nn) = def.child_by_field_name("name") else { continue };
1331 let name = txt(nn, src);
1332 let dunder_ok = matches!(name, "__init__" | "__call__");
1333 if name.starts_with('_') && !dunder_ok {
1334 continue;
1335 }
1336 let line = c.start_position().row as u32 + 1;
1337 let mut sig = decorators.join(" ");
1338 match def.kind() {
1339 "function_definition" => {
1340 if !sig.is_empty() {
1341 sig.push(' ');
1342 }
1343 let end = def.child_by_field_name("body").map_or(def.end_byte(), |b| b.start_byte());
1344 let raw = strip_py_comments(std::str::from_utf8(&src[def.start_byte()..end]).unwrap_or(""));
1345 sig.push_str(&compact(raw.trim_end().trim_end_matches(':').trim_end(), 700));
1346 let doc = def.child_by_field_name("body").and_then(|b| py_docstring(b, src)).unwrap_or_default();
1347 let kind = if in_class { Kind::Method } else { Kind::Function };
1348 out.push(entry(kind, name, format!("{prefix}.{name}"), rel, line, sig, doc));
1349 }
1350 "class_definition" => {
1351 if !sig.is_empty() {
1352 sig.push(' ');
1353 }
1354 sig.push_str(&head(def, src, "body", 400));
1355 let body = def.child_by_field_name("body");
1356 let doc = body.and_then(|b| py_docstring(b, src)).unwrap_or_default();
1357 let path = format!("{prefix}.{name}");
1358 out.push(entry(Kind::Class, name, path.clone(), rel, line, sig, doc));
1359 if let Some(b) = body {
1360 py_block(b, src, &path, true, rel, out);
1361 }
1362 }
1363 _ => {}
1364 }
1365 }
1366}
1367
1368fn rs_module(rel: &str, crate_name: &str) -> String {
1371 let r = rel.strip_prefix("src/").unwrap_or(rel).trim_end_matches(".rs");
1372 let mut segs: Vec<&str> = r.split('/').collect();
1373 if matches!(segs.last(), Some(&"mod") | Some(&"lib") | Some(&"main")) {
1374 segs.pop();
1375 }
1376 let mut p = crate_name.to_string();
1377 for s in segs {
1378 if !s.is_empty() {
1379 p.push_str("::");
1380 p.push_str(s);
1381 }
1382 }
1383 p
1384}
1385
1386fn rs_file(text: &str, rel: &str, crate_name: &str, out: &mut Vec<Entry>) {
1387 let module = rs_module(rel, crate_name);
1388 rs_parse(text, rel, &module, 0, 0, None, out);
1389}
1390
1391type Inherit = Option<(String, bool)>;
1394
1395fn rs_parse(text: &str, rel: &str, module: &str, line_off: u32, depth: u32, inherit: Inherit, out: &mut Vec<Entry>) {
1396 parse_with("rs", text, |root, src| {
1397 let mut inner_doc = Vec::new();
1398 let mut w = RsWalker {
1399 src,
1400 file: rel,
1401 line_off,
1402 depth,
1403 out: Vec::new(),
1404 wrappers: HashMap::new(),
1405 inherit,
1406 };
1407 w.block(root, module, None, &mut inner_doc);
1408 if !inner_doc.is_empty() {
1409 let doc = inner_doc.join("\n");
1410 for s in markdown::split(&doc, module) {
1411 out.push(Entry {
1412 kind: Kind::Prose,
1413 name: s.heading.clone(),
1414 path: s.heading,
1415 file: rel.to_string(),
1416 line: s.line + line_off,
1417 sig: String::new(),
1418 doc: s.text,
1419 alias_of: None,
1420 legacy: false,
1421 });
1422 }
1423 }
1424 out.append(&mut w.out);
1425 });
1426}
1427
1428struct RsWalker<'a> {
1429 src: &'a [u8],
1430 file: &'a str,
1431 line_off: u32,
1432 depth: u32,
1433 out: Vec<Entry>,
1434 wrappers: HashMap<String, (String, bool)>,
1436 inherit: Inherit,
1437}
1438
1439fn doc_wrapper(text: &str) -> Option<(String, bool)> {
1441 if !text.contains(":item") {
1442 return None;
1443 }
1444 let doc: Vec<&str> = text
1445 .lines()
1446 .filter_map(|l| strip_line_doc(l.trim()).filter(|(_, inner)| !inner).map(|(d, _)| d))
1447 .collect();
1448 if doc.is_empty() {
1449 return None;
1450 }
1451 Some((cap_doc(doc.join("\n").trim().to_string()), text.contains("macro_export")))
1452}
1453
1454fn is_pub(n: Node, src: &[u8]) -> bool {
1455 let mut cur = n.walk();
1456 let r = n
1457 .named_children(&mut cur)
1458 .any(|c| c.kind() == "visibility_modifier" && txt(c, src).trim() == "pub");
1459 r
1460}
1461
1462fn strip_line_doc(t: &str) -> Option<(&str, bool)> {
1463 if let Some(r) = t.strip_prefix("///") {
1464 if r.starts_with('/') {
1465 return None;
1466 }
1467 return Some((r.strip_prefix(' ').unwrap_or(r).trim_end(), false));
1468 }
1469 if let Some(r) = t.strip_prefix("//!") {
1470 return Some((r.strip_prefix(' ').unwrap_or(r).trim_end(), true));
1471 }
1472 None
1473}
1474
1475impl RsWalker<'_> {
1476 fn line(&self, n: Node) -> u32 {
1477 n.start_position().row as u32 + 1 + self.line_off
1478 }
1479
1480 fn block(&mut self, node: Node, module: &str, owner: Option<(&str, bool)>, inner_doc: &mut Vec<String>) {
1482 let src = self.src;
1483 let mut cur = node.walk();
1484 let children: Vec<Node> = node.named_children(&mut cur).collect();
1485 let mut pending: Vec<String> = Vec::new();
1486 let mut hidden = false;
1487 let mut exported_macro = false;
1488 let mut first_line: Option<u32> = None;
1489 for c in children {
1490 let k = c.kind();
1491 match k {
1492 "line_comment" => {
1493 let t = txt(c, src);
1494 if let Some((d, inner)) = strip_line_doc(t) {
1495 if inner {
1496 inner_doc.push(d.to_string());
1497 } else {
1498 if pending.is_empty() {
1499 first_line = Some(self.line(c));
1500 }
1501 pending.push(d.to_string());
1502 }
1503 }
1504 continue;
1505 }
1506 "block_comment" => {
1507 let t = txt(c, src);
1508 if t.starts_with("/*!") {
1509 inner_doc.push(clean_block_comment(t));
1510 } else if t.starts_with("/**") && !t.starts_with("/***") {
1511 pending.push(clean_block_comment(t));
1512 }
1513 continue;
1514 }
1515 "attribute_item" => {
1516 let t = txt(c, src);
1517 if t.contains("doc(hidden)") {
1518 hidden = true;
1519 }
1520 if t.contains("macro_export") {
1521 exported_macro = true;
1522 }
1523 if let Some(rest) = t.strip_prefix("#[doc = \"") {
1525 pending.push(rest.trim_end_matches("\"]").replace("\\n", "\n"));
1526 }
1527 continue;
1528 }
1529 "inner_attribute_item" => continue,
1530 _ => {}
1531 }
1532 let mut doc = cap_doc(pending.join("\n").trim().to_string());
1533 if doc.is_empty() {
1534 if let Some((d, _)) = &self.inherit {
1535 doc = d.clone();
1536 }
1537 }
1538 let _ = first_line.take();
1539 pending.clear();
1540 let was_hidden = std::mem::take(&mut hidden);
1541 let was_macro_export = std::mem::take(&mut exported_macro) || self.inherit.as_ref().is_some_and(|i| i.1);
1542 if was_hidden {
1543 continue;
1544 }
1545 let line = self.line(c);
1546 let sep = "::";
1547 let in_trait = owner.is_some_and(|o| o.1);
1548 let public = in_trait || is_pub(c, src);
1549 let name = c.child_by_field_name("name").map(|n| txt(n, src).to_string());
1550 let qual = |name: &str| match owner {
1551 Some((o, _)) => format!("{module}{sep}{o}{sep}{name}"),
1552 None => format!("{module}{sep}{name}"),
1553 };
1554 match k {
1555 "function_item" | "function_signature_item" => {
1556 let (Some(name), true) = (name, public) else { continue };
1557 let kind = if owner.is_some() { Kind::Method } else { Kind::Function };
1558 self.out.push(entry(
1559 kind,
1560 &name,
1561 qual(&name),
1562 self.file,
1563 line,
1564 head(c, src, "body", 600).trim_end_matches(';').to_string(),
1565 doc,
1566 ));
1567 }
1568 "struct_item" | "enum_item" | "union_item" => {
1569 let (Some(name), true) = (name, public) else { continue };
1570 let kind = if k == "enum_item" { Kind::Enum } else { Kind::Struct };
1571 let full = txt(c, src);
1572 let sig = if full.len() <= 900 {
1573 compact(full, 900)
1574 } else {
1575 format!("{} {{ … }}", head(c, src, "body", 400))
1576 };
1577 self.out.push(entry(kind, &name, qual(&name), self.file, line, sig, doc));
1578 }
1579 "trait_item" => {
1580 let (Some(name), true) = (name, public) else { continue };
1581 self.out
1582 .push(entry(Kind::Trait, &name, qual(&name), self.file, line, head(c, src, "body", 400), doc));
1583 if let Some(b) = c.child_by_field_name("body") {
1584 let mut ignore = Vec::new();
1585 self.block(b, module, Some((&name, true)), &mut ignore);
1586 }
1587 }
1588 "impl_item" => {
1589 if c.child_by_field_name("trait").is_some() {
1590 continue;
1591 }
1592 let Some(ty) = c.child_by_field_name("type") else { continue };
1593 let ty = txt(ty, src);
1594 let ty = ty.split('<').next().unwrap_or(ty).trim().to_string();
1595 if let Some(b) = c.child_by_field_name("body") {
1596 let mut ignore = Vec::new();
1597 self.block(b, module, Some((&ty, false)), &mut ignore);
1598 }
1599 }
1600 "mod_item" => {
1601 let (Some(name), true) = (name, public) else { continue };
1602 let path = format!("{module}{sep}{name}");
1603 if !doc.is_empty() {
1604 self.out
1605 .push(entry(Kind::Module, &name, path.clone(), self.file, line, format!("pub mod {name}"), doc));
1606 }
1607 if let Some(b) = c.child_by_field_name("body") {
1608 let mut inner = Vec::new();
1609 self.block(b, &path, None, &mut inner);
1610 }
1611 }
1612 "type_item" | "const_item" | "static_item" => {
1613 let (Some(name), true) = (name, public) else { continue };
1614 let kind = if k == "type_item" { Kind::Type } else { Kind::Const };
1615 self.out.push(entry(
1616 kind,
1617 &name,
1618 qual(&name),
1619 self.file,
1620 line,
1621 compact(txt(c, src).trim_end_matches(';'), 400),
1622 doc,
1623 ));
1624 }
1625 "macro_definition" => {
1626 let Some(name) = name else { continue };
1627 if let Some(w) = doc_wrapper(txt(c, src)) {
1628 self.wrappers.insert(name.clone(), w);
1629 continue;
1630 }
1631 if !was_macro_export {
1632 continue;
1633 }
1634 let root = module.split("::").next().unwrap_or(module);
1636 let arms = compact(txt(c, src), 300);
1637 self.out.push(entry(Kind::Macro, &name, format!("{root}::{name}!"), self.file, line, arms, doc));
1638 }
1639 "macro_invocation" if self.depth < 3 => {
1640 let Some(tt) = c.named_children(&mut c.walk()).find(|n| n.kind() == "token_tree") else {
1642 continue;
1643 };
1644 let body = txt(tt, src);
1645 let called = c.child_by_field_name("macro").map(|m| txt(m, src).to_string()).unwrap_or_default();
1646 let wrapper = self.wrappers.get(&called).cloned();
1647 let itemish =
1648 body.contains("fn ") || body.contains("struct ") || body.contains("mod ") || body.contains("trait ") || body.contains("macro_rules!");
1649 if body.len() < 4 || !itemish {
1650 continue;
1651 }
1652 let inner = &body[1..body.len() - 1];
1653 let off = tt.start_position().row as u32 + self.line_off;
1654 let mut sub = Vec::new();
1655 rs_parse(
1656 inner,
1657 self.file,
1658 module,
1659 off,
1660 self.depth + 1,
1661 wrapper.or_else(|| self.inherit.clone()),
1662 &mut sub,
1663 );
1664 self.out.extend(sub);
1666 }
1667 _ => {}
1668 }
1669 }
1670 }
1671}
1672
1673fn exported(name: &str) -> bool {
1676 name.chars().next().is_some_and(|c| c.is_uppercase())
1677}
1678
1679fn go_doc(n: Node, src: &[u8]) -> String {
1680 let mut lines: Vec<String> = Vec::new();
1681 let mut p = n.prev_sibling();
1682 let mut next_row = n.start_position().row;
1683 while let Some(c) = p {
1684 if c.kind() != "comment" || c.end_position().row + 1 != next_row {
1685 break;
1686 }
1687 let t = txt(c, src);
1688 if let Some(r) = t.strip_prefix("//") {
1689 lines.push(r.strip_prefix(' ').unwrap_or(r).trim_end().to_string());
1690 } else {
1691 lines.push(clean_block_comment(t));
1692 }
1693 next_row = c.start_position().row;
1694 p = c.prev_sibling();
1695 }
1696 lines.reverse();
1697 cap_doc(lines.join("\n").trim().to_string())
1698}
1699
1700fn go_file(text: &str, rel: &str, module: &str, out: &mut Vec<Entry>) {
1701 parse_with("go", text, |root, src| {
1702 let mut cur = root.walk();
1703 let children: Vec<Node> = root.named_children(&mut cur).collect();
1704 let mut pkg = module.rsplit('/').next().unwrap_or(module).to_string();
1705 for c in children {
1706 let line = c.start_position().row as u32 + 1;
1707 match c.kind() {
1708 "package_clause" => {
1709 if let Some(id) = c.named_child(0) {
1710 pkg = txt(id, src).to_string();
1711 }
1712 if pkg == "main" {
1713 return;
1714 }
1715 let doc = go_doc(c, src);
1716 if !doc.is_empty() {
1717 let dir = rel.rsplit_once('/').map_or("", |x| x.0);
1718 let title = if dir.is_empty() {
1719 format!("package {pkg}")
1720 } else {
1721 format!("package {pkg} ({dir})")
1722 };
1723 for s in markdown::split(&doc, &title) {
1724 out.push(Entry {
1725 kind: Kind::Prose,
1726 name: s.heading.clone(),
1727 path: s.heading,
1728 file: rel.to_string(),
1729 line: s.line,
1730 sig: String::new(),
1731 doc: s.text,
1732 alias_of: None,
1733 legacy: false,
1734 });
1735 }
1736 }
1737 }
1738 "function_declaration" => {
1739 let Some(name) = c.child_by_field_name("name").map(|n| txt(n, src)) else {
1740 continue;
1741 };
1742 if !exported(name) {
1743 continue;
1744 }
1745 out.push(entry(
1746 Kind::Function,
1747 name,
1748 format!("{pkg}.{name}"),
1749 rel,
1750 line,
1751 head(c, src, "body", 600),
1752 go_doc(c, src),
1753 ));
1754 }
1755 "method_declaration" => {
1756 let Some(name) = c.child_by_field_name("name").map(|n| txt(n, src)) else {
1757 continue;
1758 };
1759 let recv = c.child_by_field_name("receiver").map(|r| txt(r, src)).unwrap_or("");
1760 let ty: String = recv
1761 .trim_matches(['(', ')'])
1762 .split_whitespace()
1763 .last()
1764 .unwrap_or("")
1765 .trim_start_matches('*')
1766 .split('[')
1767 .next()
1768 .unwrap_or("")
1769 .to_string();
1770 if !exported(name) || !exported(&ty) {
1771 continue;
1772 }
1773 out.push(entry(
1774 Kind::Method,
1775 name,
1776 format!("{pkg}.{ty}.{name}"),
1777 rel,
1778 line,
1779 head(c, src, "body", 600),
1780 go_doc(c, src),
1781 ));
1782 }
1783 "type_declaration" | "const_declaration" | "var_declaration" => {
1784 let outer_doc = go_doc(c, src);
1785 let mut cc = c.walk();
1786 let specs: Vec<Node> = c
1787 .named_children(&mut cc)
1788 .filter(|n| matches!(n.kind(), "type_spec" | "type_alias" | "const_spec" | "var_spec"))
1789 .collect();
1790 let single = specs.len() == 1;
1791 for s in specs {
1792 let names: Vec<String> = if s.kind().starts_with("type") {
1793 s.child_by_field_name("name").map(|n| vec![txt(n, src).to_string()]).unwrap_or_default()
1794 } else {
1795 let mut sc = s.walk();
1796 s.children_by_field_name("name", &mut sc).map(|n| txt(n, src).to_string()).collect()
1797 };
1798 let doc = if single { outer_doc.clone() } else { go_doc(s, src) };
1799 let sline = s.start_position().row as u32 + 1;
1800 for name in names.iter().filter(|n| exported(n)) {
1801 if s.kind().starts_with("type") {
1802 let full = txt(s, src);
1803 let sig = if full.len() <= 900 {
1804 format!("type {}", compact(full, 900))
1805 } else {
1806 format!("type {} …", compact(&full[..full.find('{').unwrap_or(200.min(full.len()))], 300))
1807 };
1808 let kind = if full.contains("interface") { Kind::Interface } else { Kind::Struct };
1809 out.push(entry(kind, name, format!("{pkg}.{name}"), rel, sline, sig, doc.clone()));
1810 if let Some(t) = s.child_by_field_name("type").filter(|t| t.kind() == "interface_type") {
1812 let mut ic = t.walk();
1813 let elems: Vec<Node> = t.named_children(&mut ic).collect();
1814 for m in elems {
1815 if !matches!(m.kind(), "method_elem" | "method_spec") {
1816 continue;
1817 }
1818 let Some(mn) = m.child_by_field_name("name").map(|n| txt(n, src)) else {
1819 continue;
1820 };
1821 out.push(entry(
1822 Kind::Method,
1823 mn,
1824 format!("{pkg}.{name}.{mn}"),
1825 rel,
1826 m.start_position().row as u32 + 1,
1827 compact(txt(m, src), 400),
1828 go_doc(m, src),
1829 ));
1830 }
1831 }
1832 } else {
1833 let kw = if c.kind() == "const_declaration" { "const" } else { "var" };
1834 out.push(entry(
1835 Kind::Const,
1836 name,
1837 format!("{pkg}.{name}"),
1838 rel,
1839 sline,
1840 compact(&format!("{kw} {}", txt(s, src)), 300),
1841 doc.clone(),
1842 ));
1843 }
1844 }
1845 }
1846 }
1847 _ => {}
1848 }
1849 }
1850 });
1851}
1852
1853#[cfg(test)]
1854mod tests {
1855 use super::*;
1856
1857 fn run(f: fn(&str, &str, &mut Vec<Entry>), text: &str, rel: &str) -> Vec<Entry> {
1858 let mut out = Vec::new();
1859 f(text, rel, &mut out);
1860 out
1861 }
1862
1863 #[test]
1864 fn typescript_declarations() {
1865 let src = r#"
1866/** Creates an object schema. */
1867export declare function object<T extends ZodRawShape>(shape: T, params?: string): ZodObject<T>;
1868export declare namespace z {
1869 /** A string schema. */
1870 function string(): ZodString;
1871}
1872export interface ZodType<T> {
1873 /** Parse or throw. */
1874 parse(data: unknown): T;
1875 readonly _def: Def;
1876}
1877export declare class ZodObject<T> extends ZodType<T> {
1878 /** Disallow unknown keys. */
1879 strict(message?: string): ZodObject<T>;
1880 private _cached;
1881}
1882declare const objectType: <T>(shape: T) => ZodObject<T>;
1883export { objectType as obj };
1884export * as zz from "./external";
1885"#;
1886 let e = run(ts_file, src, "lib/index.d.ts");
1887 let get = |p: &str| {
1888 e.iter()
1889 .find(|x| x.path == p)
1890 .unwrap_or_else(|| panic!("missing {p}: {:#?}", e.iter().map(|x| &x.path).collect::<Vec<_>>()))
1891 };
1892 assert_eq!(get("object").doc, "Creates an object schema.");
1893 assert!(get("object").sig.contains("ZodObject<T>"));
1894 assert_eq!(get("z.string").doc, "A string schema.");
1895 assert_eq!(get("ZodType.parse").kind, Kind::Method);
1896 assert!(e.iter().all(|x| x.name != "_def" && x.name != "_cached"));
1897 assert_eq!(get("ZodObject.strict").doc, "Disallow unknown keys.");
1898 assert_eq!(get("objectType").kind, Kind::Function);
1899 assert_eq!(get("obj").alias_of.as_deref(), Some("objectType"));
1900 assert_eq!(get("zz").kind, Kind::Module);
1901 assert_eq!(get("object").line, 3);
1902 }
1903
1904 #[test]
1905 fn legacy_dirs() {
1906 assert!(legacy_file("v3/types.d.ts", 4));
1907 assert!(!legacy_file("v4/classic/schemas.d.ts", 4));
1908 assert!(legacy_file("pydantic/v1/main.py", 2));
1909 assert!(!legacy_file("pydantic/main.py", 2));
1910 assert!(!legacy_file("src/v2/x.rs", 1));
1911 }
1912
1913 #[test]
1914 fn python_definitions() {
1915 let src = "\"\"\"Module docs.\"\"\"\n\nclass BaseModel:\n \"\"\"Base class.\n\n More.\n \"\"\"\n\n def model_dump(self, *, mode: str = 'python') -> dict:\n \"\"\"Dump the model.\"\"\"\n return {}\n\n @classmethod\n def model_validate(cls, obj):\n '''Validate.'''\n\n def _private(self): pass\n\ndef create_model(name: str, **fields) -> type:\n return None\n";
1916 let e = run(py_file, src, "pydantic/main.py");
1917 let get = |p: &str| e.iter().find(|x| x.path == p).unwrap_or_else(|| panic!("missing {p}"));
1918 assert_eq!(get("pydantic.main.BaseModel").doc, "Base class.\n\nMore.");
1919 assert_eq!(
1920 get("pydantic.main.BaseModel.model_dump").sig,
1921 "def model_dump(self, *, mode: str = 'python') -> dict"
1922 );
1923 assert!(get("pydantic.main.BaseModel.model_validate").sig.starts_with("@classmethod def model_validate"));
1924 assert!(e.iter().all(|x| x.name != "_private"));
1925 assert!(e.iter().any(|x| x.kind == Kind::Prose && x.doc == "Module docs."));
1926 assert_eq!(get("pydantic.main.create_model").kind, Kind::Function);
1927 }
1928
1929 #[test]
1930 fn docs_pages_written_as_components() {
1931 let src = r#"import dedent from "dedent";
1932export const metadata = { title: "Installing with Vite", description: "Use the Vite plugin.", openGraph: { title: "x" } };
1933const steps = [
1934 {
1935 title: "Import Tailwind CSS",
1936 body: (
1937 <p>
1938 Add an <code>@import</code> to your CSS file.
1939 </p>
1940 ),
1941 code: { name: "CSS", lang: "css", code: dedent`
1942 @import "tailwindcss";
1943 ` },
1944 },
1945 { title: 'Old way', body: () => <p>Use directives{' '}here.</p>, code: { lang: 'css', code: '@tailwind base;\n@tailwind utilities;' } },
1946];
1947export default function Page() { return <div className="prose"><h3 className="sr-only">Overview</h3><p>Fast.</p></div> }
1948"#;
1949 let (md, title) = page_markdown(src).unwrap();
1950 assert_eq!(title.as_deref(), Some("Installing with Vite"));
1951 assert!(md.contains("## Import Tailwind CSS"), "{md}");
1952 assert!(md.contains("Add an `@import` to your CSS file."), "{md}");
1953 assert!(md.contains("```css\n@import \"tailwindcss\";\n```"), "{md}");
1954 assert!(md.contains("@tailwind base;\n@tailwind utilities;"), "{md}");
1955 assert!(md.contains("### Overview"), "{md}");
1956 assert!(!md.contains("dedent") && !md.contains("prose"), "{md}");
1957 let mut out = Vec::new();
1958 prose_titled(&md, "upstream:site/docs/installation/page.tsx", 0, title, &mut out);
1959 let s = out.iter().find(|e| e.name == "Installing with Vite › Import Tailwind CSS").unwrap();
1960 assert_eq!(s.line, 5);
1961 assert_eq!(
1962 page_title("upstream:site/src/app/(docs)/docs/installation/(tabs)/using-vite/page.tsx"),
1963 "installation/using-vite"
1964 );
1965 }
1966
1967 #[test]
1968 fn rust_items_and_macro_bodies() {
1969 let src = "//! Crate docs.\n//!\n//! # Usage\n//! Call spawn.\n\ncfg_rt! {\n /// Spawns a new task.\n #[track_caller]\n pub fn spawn<F>(future: F) -> JoinHandle<F::Output>\n where F: Future + Send + 'static {\n todo!()\n }\n}\n\n/// A router.\npub struct Router<S = ()> { inner: Inner }\n\nimpl<S> Router<S> {\n /// Add a route.\n pub fn route(self, path: &str, m: MethodRouter<S>) -> Self { self }\n fn private(&self) {}\n}\n\n/// Hidden\n#[doc(hidden)]\npub fn hidden() {}\n\n/// Macro.\n#[macro_export]\nmacro_rules! select { () => {} }\n";
1970 let mut e = Vec::new();
1971 rs_file(src, "src/task/spawn.rs", "tokio", &mut e);
1972 let get = |p: &str| {
1973 e.iter()
1974 .find(|x| x.path == p)
1975 .unwrap_or_else(|| panic!("missing {p}: {:#?}", e.iter().map(|x| &x.path).collect::<Vec<_>>()))
1976 };
1977 let spawn = get("tokio::task::spawn::spawn");
1978 assert_eq!(spawn.doc, "Spawns a new task.");
1979 assert_eq!(spawn.line, 9);
1980 assert!(spawn.sig.contains("JoinHandle"));
1981 assert_eq!(get("tokio::task::spawn::Router::route").doc, "Add a route.");
1982 assert!(e.iter().all(|x| x.name != "private" && x.name != "hidden"));
1983 assert_eq!(get("tokio::select!").kind, Kind::Macro);
1984 assert!(e.iter().any(|x| x.kind == Kind::Prose && x.path.ends_with("Usage")));
1985
1986 let src = "macro_rules! doc {\n ($select:item) => {\n /// Waits on multiple branches.\n #[macro_export]\n $select\n };\n}\n\ndoc! {macro_rules! select {\n () => {};\n}}\n";
1988 let mut e = Vec::new();
1989 rs_file(src, "src/macros/select.rs", "tokio", &mut e);
1990 let sel = e.iter().find(|x| x.path == "tokio::select!").expect("select! extracted");
1991 assert_eq!(sel.doc, "Waits on multiple branches.");
1992 assert!(e.iter().all(|x| x.name != "doc"));
1993 }
1994
1995 #[test]
1996 fn go_declarations() {
1997 let src = "// Package gin implements a web framework.\npackage gin\n\n// Context is the request context.\ntype Context struct {\n\tKeys map[string]any\n}\n\n// JSON serializes obj.\nfunc (c *Context) JSON(code int, obj any) {\n}\n\n// Default returns an Engine.\nfunc Default(opts ...Option) *Engine {\n\treturn nil\n}\n\nfunc internal() {}\n\ntype Handler interface {\n\t// Serve serves.\n\tServe(c *Context)\n}\n";
1998 let mut e = Vec::new();
1999 go_file(src, "context.go", "github.com/gin-gonic/gin", &mut e);
2000 let get = |p: &str| {
2001 e.iter()
2002 .find(|x| x.path == p)
2003 .unwrap_or_else(|| panic!("missing {p}: {:#?}", e.iter().map(|x| &x.path).collect::<Vec<_>>()))
2004 };
2005 assert_eq!(get("gin.Context.JSON").doc, "JSON serializes obj.");
2006 assert_eq!(get("gin.Default").sig, "func Default(opts ...Option) *Engine");
2007 assert_eq!(get("gin.Context").doc, "Context is the request context.");
2008 assert_eq!(get("gin.Handler.Serve").doc, "Serve serves.");
2009 assert!(e.iter().all(|x| x.name != "internal"));
2010 assert!(e.iter().any(|x| x.kind == Kind::Prose && x.doc.contains("web framework")));
2011 }
2012}