1use crate::locate::Source;
6use crate::markdown;
7use crate::Eco;
8use rayon::prelude::*;
9use serde::{Deserialize, Serialize};
10use std::cell::RefCell;
11use std::collections::{HashMap, HashSet};
12use std::path::{Path, PathBuf};
13use tree_sitter::{Node, Parser};
14
15#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
16pub enum Kind {
17 Function,
18 Method,
19 Class,
20 Interface,
21 Type,
22 Const,
23 Enum,
24 Struct,
25 Trait,
26 Macro,
27 Module,
28 Field,
29 Alias,
30 Prose,
31}
32
33impl Kind {
34 pub fn as_str(self) -> &'static str {
35 match self {
36 Kind::Function => "function",
37 Kind::Method => "method",
38 Kind::Class => "class",
39 Kind::Interface => "interface",
40 Kind::Type => "type",
41 Kind::Const => "const",
42 Kind::Enum => "enum",
43 Kind::Struct => "struct",
44 Kind::Trait => "trait",
45 Kind::Macro => "macro",
46 Kind::Module => "module",
47 Kind::Field => "field",
48 Kind::Alias => "re-export",
49 Kind::Prose => "doc",
50 }
51 }
52 pub fn is_container(self) -> bool {
53 matches!(self, Kind::Class | Kind::Interface | Kind::Struct | Kind::Trait | Kind::Enum | Kind::Module)
54 }
55}
56
57#[derive(Debug, Clone, Serialize, Deserialize)]
58pub struct Entry {
59 pub kind: Kind,
60 pub name: String,
62 pub path: String,
64 pub file: String,
66 pub line: u32,
67 pub sig: String,
69 pub doc: String,
71 pub alias_of: Option<String>,
73 #[serde(default)]
76 pub legacy: bool,
77 #[serde(default)]
79 pub prose_imports: Vec<String>,
80 #[serde(default)]
82 pub prose_parents: Vec<u32>,
83}
84
85fn legacy_file(file: &str, major: u64) -> bool {
87 major > 1
88 && file.split('/').any(|seg| {
89 seg.len() >= 2 && seg.starts_with('v') && seg[1..].chars().all(|c| c.is_ascii_digit()) && seg[1..].parse::<u64>().is_ok_and(|n| n < major)
90 })
91}
92
93const SKIP_DIRS: &[&str] = &[
96 "node_modules",
97 "compiled",
98 ".git",
99 "test",
100 "tests",
101 "__tests__",
102 "__test__",
103 "testdata",
104 "test_data",
105 "benches",
106 "bench",
107 "benchmarks",
108 "fixtures",
109 "__fixtures__",
110 "__pycache__",
111 "examples",
112 "example",
113 "_examples",
114 "coverage",
115 ".github",
116 "vendor",
117 "internal",
118 "__snapshots__",
119];
120
121const MAX_FILES: usize = 25_000;
122const MAX_FILE_BYTES: u64 = 3 * 1024 * 1024;
123
124fn is_doc_name(name: &str) -> bool {
125 let u = name.to_ascii_uppercase();
126 [
127 "README",
128 "CHANGELOG",
129 "CHANGES",
130 "HISTORY",
131 "MIGRATION",
132 "MIGRATING",
133 "UPGRADING",
134 "UPGRADE",
135 "RELEASES",
136 "NEWS",
137 "GUIDE",
138 "API",
139 ]
140 .iter()
141 .any(|p| u.starts_with(p))
142}
143
144fn doc_ext(ext: &str) -> bool {
145 matches!(ext, "md" | "mdx" | "markdown" | "rst")
146}
147
148fn walk(dir: &Path, rel: &Path, depth: usize, out: &mut Vec<PathBuf>, public_npm: bool) {
149 if depth > 14 || out.len() >= MAX_FILES {
150 return;
151 }
152 let Ok(rd) = std::fs::read_dir(dir) else { return };
153 let mut entries: Vec<_> = rd.flatten().collect();
154 entries.sort_by_key(|e| e.file_name());
155 for e in entries {
156 let name = e.file_name().to_string_lossy().to_string();
157 let Ok(ft) = e.file_type() else { continue };
158 let r = rel.join(&name);
159 if ft.is_dir() {
160 if (SKIP_DIRS.contains(&name.as_str()) && !(public_npm && name == "internal")) || name.starts_with('.') {
161 continue;
162 }
163 walk(&e.path(), &r, depth + 1, out, public_npm);
164 } else if ft.is_file() || ft.is_symlink() {
165 out.push(r);
166 }
167 if out.len() >= MAX_FILES {
168 return;
169 }
170 }
171}
172
173fn walk_all(dir: &Path, rel: &Path, depth: usize, out: &mut Vec<PathBuf>) {
175 if depth > 16 || out.len() >= MAX_FILES {
176 return;
177 }
178 let Ok(rd) = std::fs::read_dir(dir) else { return };
179 let mut entries: Vec<_> = rd.flatten().collect();
180 entries.sort_by_key(|e| e.file_name());
181 for e in entries {
182 let name = e.file_name().to_string_lossy().to_string();
183 if name.starts_with('.') {
184 continue;
185 }
186 let r = rel.join(&name);
187 match e.file_type() {
188 Ok(t) if t.is_dir() => walk_all(&e.path(), &r, depth + 1, out),
189 Ok(_) => out.push(r),
190 _ => {}
191 }
192 }
193}
194
195fn ext_of(p: &Path) -> String {
196 let s = p.file_name().map(|f| f.to_string_lossy().to_ascii_lowercase()).unwrap_or_default();
197 for e in [".d.ts", ".d.mts", ".d.cts"] {
198 if s.ends_with(e) {
199 return e[1..].to_string();
200 }
201 }
202 p.extension().map(|e| e.to_string_lossy().to_ascii_lowercase()).unwrap_or_default()
203}
204
205fn in_docs_dir(p: &Path) -> bool {
206 p.components().any(|c| {
207 matches!(
208 c.as_os_str().to_string_lossy().to_ascii_lowercase().as_str(),
209 "docs" | "doc" | "documentation" | "guide" | "guides"
210 )
211 })
212}
213
214struct Job {
216 abs: PathBuf,
217 rel: String,
218 how: How,
219}
220
221#[derive(Clone, Copy, PartialEq)]
222enum How {
223 Prose,
224 Page,
226 Example,
227 Metadata,
228 Ts,
229 Python,
230 Rust,
231 Go,
232}
233
234fn plan(eco: Eco, src: &Source, extra_types: Option<&Path>, upstream: Option<&Path>, public_api: bool) -> Vec<Job> {
236 let mut rels = Vec::new();
237 match &src.files {
238 Some(list) => rels.extend(list.iter().cloned()),
239 None => walk(&src.dir, Path::new(""), 0, &mut rels, public_api),
240 }
241 let mut jobs = Vec::new();
242 let names: HashSet<String> = rels.iter().map(|r| r.to_string_lossy().replace('\\', "/")).collect();
243 let mut has_dts = false;
244 for r in &rels {
245 let e = ext_of(r);
246 if e.starts_with("d.") {
247 has_dts = true;
248 }
249 }
250 let rs_crate_src = eco == Eco::Cargo;
251 for r in rels {
252 let rel = r.to_string_lossy().replace('\\', "/");
253 if rel
254 .split('/')
255 .any(|seg| SKIP_DIRS.contains(&seg) && !(public_api && seg == "internal") && src.files.is_some())
256 {
257 continue;
258 }
259 let abs = src.dir.join(&r);
260 let fname = r.file_name().map(|f| f.to_string_lossy().to_string()).unwrap_or_default();
261 let e = ext_of(&r);
262 let depth = rel.matches('/').count();
263 let how = if doc_ext(&e) || (e.is_empty() || e == "txt") && is_doc_name(&fname) && depth == 0 {
264 if is_doc_name(&fname) || in_docs_dir(&r) || depth == 0 {
265 Some(How::Prose)
266 } else {
267 None
268 }
269 } else {
270 match eco {
271 Eco::Npm => match e.as_str() {
272 "d.ts" => Some(How::Ts),
273 "d.mts" | "d.cts" => {
274 let base = rel.trim_end_matches(".d.mts").trim_end_matches(".d.cts");
275 (public_api || !names.contains(&format!("{base}.d.ts"))).then_some(How::Ts)
276 }
277 "ts" | "mts" | "cts" | "tsx" if !has_dts => Some(How::Ts),
278 "js" | "mjs" | "cjs" if !has_dts && !fname.contains(".min.") => Some(How::Ts),
279 _ => None,
280 },
281 Eco::PyPI => match e.as_str() {
282 "py" => Some(How::Python),
283 "pyi" => {
284 let base = rel.trim_end_matches(".pyi");
285 if names.contains(&format!("{base}.py")) {
286 None
287 } else {
288 Some(How::Python)
289 }
290 }
291 _ => None,
292 },
293 Eco::Cargo => (e == "rs" && rs_crate_src && rel.starts_with("src/")).then_some(How::Rust),
294 Eco::Go => (e == "go" && !fname.ends_with("_test.go")).then_some(How::Go),
295 }
296 };
297 if let Some(how) = how {
298 jobs.push(Job { abs, rel, how });
299 }
300 }
301 if let Some(up) = upstream {
303 let pages: HashSet<String> = std::fs::read_to_string(up.join(".lockdocs-upstream.json"))
304 .ok()
305 .and_then(|t| serde_json::from_str::<crate::upstream::Manifest>(&t).ok())
306 .map(|m| m.pages.into_iter().collect())
307 .unwrap_or_default();
308 let mut rels = Vec::new();
309 walk_all(up, Path::new(""), 0, &mut rels);
310 for r in rels {
311 let e = ext_of(&r);
312 let plain = r.to_string_lossy().replace('\\', "/");
313 let rel = format!("upstream:{plain}");
314 if pages.contains(&plain) {
315 jobs.push(Job {
316 abs: up.join(&r),
317 rel,
318 how: How::Page,
319 });
320 continue;
321 }
322 if doc_ext(&e) || e == "mdoc" || e == "txt" {
324 jobs.push(Job {
325 abs: up.join(&r),
326 rel,
327 how: How::Prose,
328 });
329 } else if EXAMPLE_EXTS.contains(&e.as_str()) {
330 jobs.push(Job {
332 abs: up.join(&r),
333 rel,
334 how: How::Example,
335 });
336 }
337 }
338 }
339 if let Some(meta) = &src.metadata {
340 jobs.push(Job {
341 abs: meta.clone(),
342 rel: "README (package metadata)".into(),
343 how: How::Metadata,
344 });
345 }
346 if let (Some(types), false) = (extra_types, has_dts) {
348 let mut rels = Vec::new();
349 walk(types, Path::new(""), 0, &mut rels, public_api);
350 let label = types
351 .iter()
352 .rev()
353 .take(2)
354 .collect::<Vec<_>>()
355 .into_iter()
356 .rev()
357 .map(|s| s.to_string_lossy())
358 .collect::<Vec<_>>()
359 .join("/");
360 for r in rels {
361 if ext_of(&r) == "d.ts" || public_api && matches!(ext_of(&r).as_str(), "d.cts" | "d.mts") {
362 let rel = format!("{label}/{}", r.to_string_lossy().replace('\\', "/"));
363 jobs.push(Job {
364 abs: types.join(&r),
365 rel,
366 how: How::Ts,
367 });
368 }
369 }
370 }
371 jobs
372}
373
374pub fn extract(eco: Eco, pkg_name: &str, src: &Source, extra_types: Option<&Path>, upstream: Option<&Path>) -> Vec<Entry> {
376 let major: u64 = src.version.trim_start_matches('v').split('.').next().and_then(|m| m.parse().ok()).unwrap_or(0);
377 let jobs = plan(eco, src, extra_types, upstream, false);
378 let crate_name = if eco == Eco::Cargo {
379 rust_crate_name(&src.dir).unwrap_or_else(|| pkg_name.replace('-', "_"))
380 } else {
381 String::new()
382 };
383 let includes = upstream.map(Includes::scan).unwrap_or_default();
384 let mut all: Vec<Entry> = jobs
385 .par_iter()
386 .flat_map_iter(|job| {
387 let Ok(meta) = std::fs::metadata(&job.abs) else { return Vec::new() };
388 if meta.len() > MAX_FILE_BYTES && job.how != How::Prose {
389 return Vec::new();
390 }
391 let Ok(bytes) = std::fs::read(&job.abs) else { return Vec::new() };
392 let text = String::from_utf8_lossy(&bytes);
393 let mut out = Vec::new();
394 match job.how {
395 How::Prose => {
396 prose(&text, &job.rel, 0, &mut out);
397 if job.rel.starts_with("upstream:") && !includes.files.is_empty() {
398 for e in &mut out {
399 e.doc = includes.expand(&e.doc);
400 }
401 }
402 }
403 How::Page => {
404 if let Some((md, title)) = page_markdown(&text) {
405 let title = title.unwrap_or_else(|| page_title(&job.rel));
406 prose_titled(&md, &job.rel, 0, Some(title), &mut out);
407 }
408 }
409 How::Example => example(&text, &job.rel, &mut out),
410 How::Metadata => {
411 let (body, line) = markdown::metadata_body(&text);
412 prose(&body, &job.rel, line - 1, &mut out)
413 }
414 How::Ts => ts_file(&text, &job.rel, &mut out),
415 How::Python => py_file(&text, &job.rel, &mut out),
416 How::Rust => rs_file(&text, &job.rel, &crate_name, &mut out),
417 How::Go => go_file(&text, &job.rel, pkg_name, &mut out),
418 }
419 out
420 })
421 .collect();
422 for e in &mut all {
423 e.legacy = legacy_file(&e.file, major);
424 }
425 let mut seen = HashSet::new();
427 all.retain(|e| seen.insert((e.path.clone(), e.sig.clone(), e.doc.len())));
428 all
429}
430
431pub(crate) fn npm_sources(src: &Source, extra_types: Option<&Path>) -> std::collections::BTreeMap<String, String> {
433 plan(Eco::Npm, src, extra_types, None, true)
434 .into_iter()
435 .filter(|j| j.how == How::Ts)
436 .filter_map(|j| {
437 if std::fs::metadata(&j.abs).ok()?.len() > MAX_FILE_BYTES {
438 return None;
439 }
440 Some((j.rel, std::fs::read_to_string(j.abs).ok()?))
441 })
442 .collect()
443}
444
445pub(crate) fn npm_entries(sources: &std::collections::BTreeMap<String, String>) -> Vec<Entry> {
448 let mut entries = Vec::new();
449 for (file, text) in sources {
450 ts_file(text, file, &mut entries);
451 }
452 entries
453}
454
455pub(crate) fn npm_stamp(src: &Source, extra_types: Option<&Path>) -> String {
458 let mut parts = Vec::new();
459 for j in plan(Eco::Npm, src, extra_types, None, true) {
460 parts.push(j.rel);
461 if let Ok(bytes) = std::fs::read(&j.abs) {
462 parts.push(crate::cache::hash(&[&String::from_utf8_lossy(&bytes)]));
463 }
464 }
465 crate::cache::hash(&parts.iter().map(String::as_str).collect::<Vec<_>>())
466}
467
468fn rust_crate_name(dir: &Path) -> Option<String> {
469 let t = std::fs::read_to_string(dir.join("Cargo.toml")).ok()?;
470 let v: toml::Table = toml::from_str(&t).ok()?;
471 let lib = v.get("lib").and_then(|l| l.get("name")).and_then(|n| n.as_str());
472 let pkg = v.get("package").and_then(|p| p.get("name")).and_then(|n| n.as_str());
473 lib.or(pkg).map(|n| n.replace('-', "_"))
474}
475
476#[derive(Default)]
480struct Includes {
481 files: Vec<(String, PathBuf)>,
483}
484
485impl Includes {
486 fn scan(dir: &Path) -> Includes {
487 let mut rels = Vec::new();
488 walk_all(dir, Path::new(""), 0, &mut rels);
489 let files = rels
490 .into_iter()
491 .filter(|r| EXAMPLE_EXTS.contains(&ext_of(r).as_str()))
492 .map(|r| {
493 let s = r.to_string_lossy().replace('\\', "/").to_ascii_lowercase();
494 let stem = s.rsplit_once('.').map_or(s.clone(), |x| x.0.to_string());
495 (stem, dir.join(&r))
496 })
497 .collect();
498 Includes { files }
499 }
500
501 fn find(&self, reference: &str) -> Option<&PathBuf> {
502 let r = reference.trim().trim_matches(['"', '\'']).to_ascii_lowercase();
503 let r = r.split_whitespace().next().unwrap_or("");
504 let r = r.rsplit_once('.').map_or(r, |x| x.0);
505 let segs: Vec<&str> = r
506 .split('/')
507 .filter(|s| !s.is_empty() && *s != ".." && *s != "." && *s != ".tmp_examples")
508 .collect();
509 for take in (1..=segs.len().min(3)).rev() {
510 let suffix = segs[segs.len() - take..].join("/");
511 let hits: Vec<&(String, PathBuf)> = self.files.iter().filter(|(k, _)| *k == suffix || k.ends_with(&format!("/{suffix}"))).collect();
512 if hits.len() == 1 {
513 return Some(&hits[0].1);
514 }
515 }
516 None
517 }
518
519 fn expand(&self, text: &str) -> String {
520 if !(text.contains("{!") || text.contains("{*") || text.contains("--8<--")) {
521 return text.to_string();
522 }
523 let mut out = String::with_capacity(text.len());
524 for line in text.lines() {
525 let t = line.trim();
526 let reference = t
527 .strip_prefix("{!")
528 .and_then(|x| x.strip_suffix("!}"))
529 .or_else(|| t.strip_prefix("{*").and_then(|x| x.strip_suffix("*}")))
530 .or_else(|| t.strip_prefix("--8<--"));
531 match reference
532 .and_then(|r| self.find(r))
533 .and_then(|p| std::fs::read_to_string(p).ok().map(|c| (p, c)))
534 {
535 Some((p, code)) => {
536 let lang = p.extension().map(|e| e.to_string_lossy().to_string()).unwrap_or_default();
537 let code: String = code.chars().take(3000).collect();
538 out.push_str(&format!("```{lang}\n{}\n```\n", code.trim_end()));
539 }
540 None => {
541 out.push_str(line);
542 out.push('\n');
543 }
544 }
545 }
546 out
547 }
548}
549
550const EXAMPLE_EXTS: &[&str] = &["py", "ts", "tsx", "js", "jsx", "mjs", "rs", "go"];
551
552fn example(text: &str, rel: &str, out: &mut Vec<Entry>) {
554 if text.len() > 12_000 || text.trim().is_empty() {
555 return;
556 }
557 let name = rel.rsplit('/').next().unwrap_or(rel);
558 let lang = name.rsplit('.').next().unwrap_or("");
559 let title = name.rsplit_once('.').map_or(name, |x| x.0).replace(['_', '-'], " ");
560 out.push(Entry {
561 kind: Kind::Prose,
562 name: format!("example › {title}"),
563 path: format!("example › {title}"),
564 file: rel.to_string(),
565 line: 1,
566 sig: String::new(),
567 doc: format!("```{lang}\n{}\n```", text.trim_end()),
568 alias_of: None,
569 legacy: false,
570 prose_imports: Vec::new(),
571 prose_parents: Vec::new(),
572 });
573}
574
575fn page_title(rel: &str) -> String {
577 let segs: Vec<&str> = rel.split('/').filter(|s| !(s.starts_with('(') && s.ends_with(')'))).collect();
578 let mut segs: Vec<&str> = segs.iter().rev().take(3).rev().copied().collect();
579 if let Some(last) = segs.last() {
580 if last.starts_with("page.") || last.starts_with("index.") {
581 segs.pop();
582 }
583 }
584 segs.join("/")
585 .trim_end_matches(".tsx")
586 .trim_end_matches(".jsx")
587 .trim_end_matches(".js")
588 .to_string()
589}
590
591fn page_markdown(text: &str) -> Option<(String, Option<String>)> {
597 if text.len() > 200_000 {
598 return None;
599 }
600 parse_with("tsx", text, |root, src| {
601 let mut p = PageOut::default();
602 page_walk(root, src, &mut p, false);
603 p.flush();
604 let mut md = String::new();
605 let mut n = 0u32;
606 for (line, t) in &p.out {
607 while n + 1 < *line {
608 md.push('\n');
609 n += 1;
610 }
611 md.push_str(t);
612 md.push('\n');
613 n += 1 + t.matches('\n').count() as u32;
614 }
615 (md, p.title)
616 })
617 .filter(|(md, _)| !md.trim().is_empty())
618}
619
620#[derive(Default)]
621struct PageOut {
622 out: Vec<(u32, String)>,
623 para: String,
624 para_line: u32,
625 title: Option<String>,
626}
627
628impl PageOut {
629 fn text(&mut self, line: u32, t: &str) {
630 if self.para.trim().is_empty() {
631 self.para_line = line;
632 }
633 self.para.push_str(t);
634 }
635 fn flush(&mut self) {
636 let t = compact(&self.para, usize::MAX);
637 if !t.is_empty() {
638 self.out.push((self.para_line, t));
639 }
640 self.para.clear();
641 }
642 fn block(&mut self, line: u32, t: String) {
643 self.flush();
644 self.out.push((line, t));
645 }
646}
647
648fn line_of(n: Node) -> u32 {
649 n.start_position().row as u32 + 1
650}
651
652fn js_string(n: Node, src: &[u8]) -> Option<String> {
654 match n.kind() {
655 "string" => {
656 let raw = txt(n, src);
657 let inner = raw.get(1..raw.len().saturating_sub(1)).unwrap_or("");
658 Some(inner.replace("\\n", "\n").replace("\\'", "'").replace("\\\"", "\"").replace("\\\\", "\\"))
659 }
660 "template_string" => {
661 let raw = txt(n, src);
662 let inner = raw.get(1..raw.len().saturating_sub(1)).unwrap_or("");
663 Some(dedent(&inner.trim_matches('\n').lines().collect::<Vec<_>>()))
664 }
665 "call_expression" => n
666 .child_by_field_name("arguments")
667 .filter(|a| a.kind() == "template_string")
668 .and_then(|a| js_string(a, src)),
669 "parenthesized_expression" => n.named_child(0).and_then(|c| js_string(c, src)),
670 _ => None,
671 }
672}
673
674fn jsx_name(n: Node, src: &[u8]) -> String {
675 let open = if n.kind() == "jsx_element" {
676 n.child_by_field_name("open_tag")
677 } else {
678 Some(n)
679 };
680 open.and_then(|o| o.child_by_field_name("name"))
681 .map(|x| txt(x, src).to_string())
682 .unwrap_or_default()
683}
684
685fn jsx_text(n: Node, src: &[u8], out: &mut String) {
687 match n.kind() {
688 "jsx_text" => out.push_str(txt(n, src)),
689 "jsx_expression" => {
690 if let Some(v) = n.named_child(0).and_then(|c| js_string(c, src)) {
691 out.push_str(&v);
692 }
693 }
694 "jsx_attribute" | "jsx_opening_element" | "jsx_closing_element" => {}
695 _ => {
696 let mut c = n.walk();
697 for ch in n.named_children(&mut c) {
698 jsx_text(ch, src, out);
699 }
700 }
701 }
702}
703
704fn page_walk(n: Node, src: &[u8], p: &mut PageOut, in_array: bool) {
705 match n.kind() {
706 "import_statement" | "comment" | "jsx_attribute" => {}
707 "pair" => {
708 let key = n
709 .child_by_field_name("key")
710 .map(|k| txt(k, src).trim_matches(['"', '\'']).to_string())
711 .unwrap_or_default();
712 let Some(value) = n.child_by_field_name("value") else { return };
713 match key.as_str() {
714 "title" | "heading" => {
715 if let Some(t) = js_string(value, src) {
716 if !in_array && p.title.is_none() {
717 p.title = Some(t);
718 } else if in_array {
719 p.block(line_of(n), format!("## {}", compact(&t, 200)));
720 }
721 }
722 }
723 "description" => {
724 if let Some(t) = js_string(value, src) {
725 p.block(line_of(value), compact(&t, usize::MAX));
726 }
727 }
728 "code" => match js_string(value, src) {
729 Some(code) => {
730 let lang = n
731 .parent()
732 .and_then(|obj| {
733 let mut c = obj.walk();
734 let found = obj.named_children(&mut c).find(|x| {
735 x.kind() == "pair" && x.child_by_field_name("key").is_some_and(|k| txt(k, src).trim_matches(['"', '\'']) == "lang")
736 });
737 found
738 })
739 .and_then(|x| x.child_by_field_name("value"))
740 .and_then(|v| js_string(v, src))
741 .unwrap_or_default();
742 p.block(line_of(value), format!("```{lang}\n{}\n```", code.trim_end()));
743 }
744 None => page_walk(value, src, p, in_array),
745 },
746 "name" | "lang" | "className" | "href" | "id" | "url" | "src" | "image" | "images" | "icon" | "openGraph" | "twitter" | "alternates" => {}
747 _ => page_walk(value, src, p, in_array),
748 }
749 }
750 "jsx_element" | "jsx_self_closing_element" => {
751 let name = jsx_name(n, src);
752 let b = name.as_bytes();
753 if b.len() == 2 && b[0] == b'h' && (b'1'..=b'6').contains(&b[1]) {
754 let mut t = String::new();
755 jsx_text(n, src, &mut t);
756 let t = compact(&t, 200);
757 if !t.is_empty() {
758 p.block(line_of(n), format!("{} {t}", "#".repeat((b[1] - b'0') as usize)));
759 }
760 } else if name == "code" {
761 let mut t = String::new();
762 jsx_text(n, src, &mut t);
763 p.text(line_of(n), &format!("`{}`", t.trim()));
764 } else {
765 let block = matches!(name.as_str(), "p" | "li" | "div" | "section" | "ul" | "ol" | "pre" | "table" | "tr");
766 if block {
767 p.flush();
768 }
769 let mut c = n.walk();
770 for ch in n.named_children(&mut c) {
771 page_walk(ch, src, p, in_array);
772 }
773 if block {
774 p.flush();
775 }
776 }
777 }
778 "jsx_text" => p.text(line_of(n), txt(n, src)),
779 "jsx_expression" => match n.named_child(0) {
780 Some(c) if matches!(c.kind(), "string" | "template_string") => {
781 let v = js_string(c, src).unwrap_or_default();
782 p.text(line_of(n), &v);
783 }
784 Some(c) => page_walk(c, src, p, in_array),
785 None => {}
786 },
787 _ => {
788 let arr = in_array || n.kind() == "array";
789 let mut c = n.walk();
790 for ch in n.named_children(&mut c) {
791 page_walk(ch, src, p, arr);
792 }
793 }
794 }
795}
796
797fn prose(text: &str, rel: &str, line_off: u32, out: &mut Vec<Entry>) {
798 prose_titled(text, rel, line_off, None, out)
799}
800
801fn prose_titled(text: &str, rel: &str, line_off: u32, given: Option<String>, out: &mut Vec<Entry>) {
802 let (text, fm_title) = markdown::clean_mdx(text, rel.ends_with(".mdx"));
803 let file_name = rel.rsplit('/').next().unwrap_or(rel);
805 let file_title = if rel.starts_with("upstream:") && !is_doc_name(file_name) {
806 file_name
807 .rsplit_once('.')
808 .map_or(file_name, |x| x.0)
809 .trim_start_matches(|c: char| c.is_ascii_digit() || c == '-')
810 .replace(['_', '-'], " ")
811 } else {
812 file_name.to_string()
813 };
814 let file_title = file_title.as_str();
815 let title = if let Some(t) = given {
816 t
817 } else if rel.contains("package metadata") {
818 "README".to_string()
819 } else if let Some(t) = fm_title {
820 t
821 } else {
822 file_title.to_string()
823 };
824 for s in markdown::split(&text, &title) {
825 out.push(prose_section(s, rel, line_off));
826 }
827}
828
829thread_local! {
832 static PARSERS: RefCell<Vec<(&'static str, Parser)>> = const { RefCell::new(Vec::new()) };
833}
834
835fn parse_with<R>(lang: &'static str, text: &str, f: impl FnOnce(Node, &[u8]) -> R) -> Option<R> {
838 let pooled = PARSERS.with(|ps| {
839 let mut ps = ps.borrow_mut();
840 ps.iter().position(|(l, _)| *l == lang).map(|i| ps.swap_remove(i).1)
841 });
842 let mut parser = match pooled {
843 Some(p) => p,
844 None => {
845 let mut p = Parser::new();
846 let language: tree_sitter::Language = match lang {
847 "ts" => tree_sitter_typescript::LANGUAGE_TYPESCRIPT.into(),
848 "tsx" => tree_sitter_typescript::LANGUAGE_TSX.into(),
849 "py" => tree_sitter_python::LANGUAGE.into(),
850 "rs" => tree_sitter_rust::LANGUAGE.into(),
851 _ => tree_sitter_go::LANGUAGE.into(),
852 };
853 p.set_language(&language).ok()?;
854 p
855 }
856 };
857 let tree = parser.parse(text, None);
858 PARSERS.with(|ps| ps.borrow_mut().push((lang, parser)));
859 let tree = tree?;
860 Some(f(tree.root_node(), text.as_bytes()))
861}
862
863fn txt<'a>(n: Node, src: &'a [u8]) -> &'a str {
864 n.utf8_text(src).unwrap_or("")
865}
866
867fn compact(s: &str, cap: usize) -> String {
868 let mut out = String::with_capacity(s.len().min(cap.saturating_add(8)));
869 let mut space = false;
870 for c in s.chars() {
871 if c.is_whitespace() {
872 space = true;
873 continue;
874 }
875 if space && !out.is_empty() {
876 out.push(' ');
877 }
878 space = false;
879 out.push(c);
880 if out.len() >= cap {
881 out.push_str(" …");
882 break;
883 }
884 }
885 out.replace("( ", "(").replace(", )", ")").replace(",)", ")").replace(" )", ")")
886}
887
888fn strip_py_comments(s: &str) -> String {
890 let mut out = String::with_capacity(s.len());
891 for line in s.lines() {
892 let mut quote: Option<char> = None;
893 let mut cut = line.len();
894 for (i, c) in line.char_indices() {
895 match (quote, c) {
896 (None, '#') => {
897 cut = i;
898 break;
899 }
900 (None, '"' | '\'') => quote = Some(c),
901 (Some(q), c) if c == q => quote = None,
902 _ => {}
903 }
904 }
905 out.push_str(&line[..cut]);
906 out.push('\n');
907 }
908 out
909}
910
911fn head(n: Node, src: &[u8], body_field: &str, cap: usize) -> String {
913 let end = n.child_by_field_name(body_field).map_or(n.end_byte(), |b| b.start_byte());
914 let s = std::str::from_utf8(&src[n.start_byte()..end]).unwrap_or("");
915 compact(s.trim_end().trim_end_matches(['{', ':', '=']).trim_end(), cap)
916}
917
918fn cap_doc(s: String) -> String {
919 const CAP: usize = 4000;
920 if s.len() <= CAP {
921 return s;
922 }
923 let mut i = CAP;
924 while !s.is_char_boundary(i) {
925 i -= 1;
926 }
927 format!("{} …", &s[..i])
928}
929
930fn dedent(lines: &[&str]) -> String {
931 let indent = lines
932 .iter()
933 .filter(|l| !l.trim().is_empty())
934 .map(|l| l.len() - l.trim_start().len())
935 .min()
936 .unwrap_or(0);
937 let v: Vec<&str> = lines.iter().map(|l| if l.len() >= indent { &l[indent..] } else { l.trim_start() }).collect();
938 v.join("\n").trim().to_string()
939}
940
941fn clean_block_comment(s: &str) -> String {
942 let s = s
943 .trim_start_matches("/**")
944 .trim_start_matches("/*!")
945 .trim_start_matches("/*")
946 .trim_end_matches("*/");
947 let lines: Vec<&str> = s
948 .lines()
949 .map(|l| {
950 let t = l.trim_start();
951 match t.strip_prefix('*') {
952 Some(r) => r.strip_prefix(' ').unwrap_or(r),
953 None => l,
954 }
955 })
956 .collect();
957 cap_doc(dedent(&lines))
958}
959
960fn entry(kind: Kind, name: &str, path: String, file: &str, line: u32, sig: String, doc: String) -> Entry {
961 Entry {
962 kind,
963 name: name.to_string(),
964 path,
965 file: file.to_string(),
966 line,
967 sig,
968 doc,
969 alias_of: None,
970 legacy: false,
971 prose_imports: Vec::new(),
972 prose_parents: Vec::new(),
973 }
974}
975
976fn prose_section(s: crate::markdown::Section, rel: &str, line_off: u32) -> Entry {
979 Entry {
980 kind: Kind::Prose,
981 name: s.heading.clone(),
982 path: s.heading,
983 file: rel.to_string(),
984 line: s.line + line_off,
985 sig: String::new(),
986 doc: s.text,
987 alias_of: None,
988 legacy: false,
989 prose_imports: s.imports,
990 prose_parents: s.parents.into_iter().map(|line| line + line_off).collect(),
991 }
992}
993
994fn ts_file(text: &str, rel: &str, out: &mut Vec<Entry>) {
997 let lang = if rel.ends_with(".tsx") || rel.ends_with(".jsx") { "tsx" } else { "ts" };
998 let is_js = rel.ends_with(".js") || rel.ends_with(".mjs") || rel.ends_with(".cjs");
999 parse_with(lang, text, |root, src| {
1000 let mut w = TsWalker {
1001 src,
1002 file: rel,
1003 out,
1004 js: is_js,
1005 };
1006 w.block(root, "");
1007 });
1008}
1009
1010struct TsWalker<'a> {
1011 src: &'a [u8],
1012 file: &'a str,
1013 out: &'a mut Vec<Entry>,
1014 js: bool,
1015}
1016
1017impl TsWalker<'_> {
1018 fn doc(&self, n: Node) -> String {
1019 let mut p = n.prev_sibling();
1020 while let Some(c) = p {
1021 if c.kind() == "comment" {
1022 let t = txt(c, self.src);
1023 if t.starts_with("/**") && c.end_position().row + 2 >= n.start_position().row {
1024 return clean_block_comment(t);
1025 }
1026 if t.starts_with("//") {
1027 p = c.prev_sibling();
1028 continue;
1029 }
1030 }
1031 break;
1032 }
1033 String::new()
1034 }
1035
1036 fn line(&self, n: Node) -> u32 {
1037 n.start_position().row as u32 + 1
1038 }
1039
1040 fn block(&mut self, node: Node, prefix: &str) {
1041 let mut cur = node.walk();
1042 let children: Vec<Node> = node.named_children(&mut cur).collect();
1043 for c in children {
1044 match c.kind() {
1045 "export_statement" => {
1046 let doc = self.doc(c);
1047 if let Some(d) = c.child_by_field_name("declaration") {
1048 self.decl(d, prefix, doc, c);
1049 } else {
1050 self.exports(c, prefix);
1051 }
1052 }
1053 "ambient_declaration" => {
1054 let doc = self.doc(c);
1055 let mut cc = c.walk();
1056 let inner: Vec<Node> = c.named_children(&mut cc).collect();
1057 for d in inner {
1058 self.decl(d, prefix, doc.clone(), c);
1059 }
1060 }
1061 "expression_statement" => {
1062 let mut cc = c.walk();
1063 let inner: Vec<Node> = c.named_children(&mut cc).collect();
1064 for d in inner {
1065 if matches!(d.kind(), "internal_module" | "module") {
1066 let doc = self.doc(c);
1067 self.decl(d, prefix, doc, c);
1068 }
1069 }
1070 }
1071 _ => {
1072 let doc = self.doc(c);
1073 if !self.js || !doc.is_empty() || !prefix.is_empty() {
1075 self.decl(c, prefix, doc, c);
1076 }
1077 }
1078 }
1079 }
1080 }
1081
1082 fn exports(&mut self, n: Node, prefix: &str) {
1083 let mut cur = n.walk();
1084 let children: Vec<Node> = n.named_children(&mut cur).collect();
1085 for c in children {
1086 match c.kind() {
1087 "export_clause" => {
1088 let mut cc = c.walk();
1089 let specs: Vec<Node> = c.named_children(&mut cc).collect();
1090 for s in specs {
1091 let (Some(name), alias) = (s.child_by_field_name("name"), s.child_by_field_name("alias")) else {
1092 continue;
1093 };
1094 let Some(alias) = alias else { continue };
1095 let (from, to) = (txt(name, self.src), txt(alias, self.src));
1096 if from == to || to == "default" {
1097 continue;
1098 }
1099 let mut e = entry(
1100 Kind::Alias,
1101 to,
1102 format!("{prefix}{to}"),
1103 self.file,
1104 self.line(s),
1105 compact(txt(n, self.src), 200),
1106 String::new(),
1107 );
1108 e.alias_of = Some(from.to_string());
1109 self.out.push(e);
1110 }
1111 }
1112 "namespace_export" => {
1113 if let Some(id) = c.named_child(0) {
1114 let name = txt(id, self.src);
1115 self.out.push(entry(
1116 Kind::Module,
1117 name,
1118 format!("{prefix}{name}"),
1119 self.file,
1120 self.line(n),
1121 compact(txt(n, self.src), 200),
1122 self.doc(n),
1123 ));
1124 }
1125 }
1126 _ => {}
1127 }
1128 }
1129 }
1130
1131 fn decl(&mut self, d: Node, prefix: &str, doc: String, outer: Node) {
1132 let src = self.src;
1133 let name_of = |n: Node| n.child_by_field_name("name").map(|x| txt(x, src).to_string());
1134 let line = self.line(outer);
1135 match d.kind() {
1136 "ambient_declaration" => {
1137 let mut cur = d.walk();
1138 let inner: Vec<Node> = d.named_children(&mut cur).collect();
1139 for x in inner {
1140 self.decl(x, prefix, doc.clone(), outer);
1141 }
1142 }
1143 "function_signature" | "function_declaration" | "generator_function_declaration" => {
1144 let Some(name) = name_of(d) else { return };
1145 let sig = head(outer, src, "__none__", 600).trim_end_matches(';').to_string();
1146 let sig = if d.child_by_field_name("body").is_some() {
1147 head(d, src, "body", 600)
1148 } else {
1149 sig
1150 };
1151 self.out
1152 .push(entry(Kind::Function, &name, format!("{prefix}{name}"), self.file, line, sig, doc));
1153 }
1154 "class_declaration" | "abstract_class_declaration" | "class" => {
1155 let Some(name) = name_of(d) else { return };
1156 self.out.push(entry(
1157 Kind::Class,
1158 &name,
1159 format!("{prefix}{name}"),
1160 self.file,
1161 line,
1162 head(d, src, "body", 400),
1163 doc,
1164 ));
1165 if let Some(body) = d.child_by_field_name("body") {
1166 self.members(body, &format!("{prefix}{name}."));
1167 }
1168 }
1169 "interface_declaration" => {
1170 let Some(name) = name_of(d) else { return };
1171 self.out.push(entry(
1172 Kind::Interface,
1173 &name,
1174 format!("{prefix}{name}"),
1175 self.file,
1176 line,
1177 head(d, src, "body", 400),
1178 doc,
1179 ));
1180 if let Some(body) = d.child_by_field_name("body") {
1181 self.members(body, &format!("{prefix}{name}."));
1182 }
1183 }
1184 "type_alias_declaration" => {
1185 let Some(name) = name_of(d) else { return };
1186 self.out.push(entry(
1187 Kind::Type,
1188 &name,
1189 format!("{prefix}{name}"),
1190 self.file,
1191 line,
1192 compact(txt(d, src), 700),
1193 doc,
1194 ));
1195 }
1196 "enum_declaration" => {
1197 let Some(name) = name_of(d) else { return };
1198 self.out.push(entry(
1199 Kind::Enum,
1200 &name,
1201 format!("{prefix}{name}"),
1202 self.file,
1203 line,
1204 compact(txt(d, src), 500),
1205 doc,
1206 ));
1207 }
1208 "lexical_declaration" | "variable_declaration" => {
1209 let kw = txt(d, src).split_whitespace().next().unwrap_or("const").to_string();
1210 let mut cur = d.walk();
1211 let decls: Vec<Node> = d.named_children(&mut cur).filter(|n| n.kind() == "variable_declarator").collect();
1212 for v in decls {
1213 let Some(nn) = v.child_by_field_name("name") else { continue };
1214 if nn.kind() != "identifier" {
1215 continue;
1216 }
1217 let name = txt(nn, src).to_string();
1218 let value = v.child_by_field_name("value");
1219 let is_fn = value.is_some_and(|x| matches!(x.kind(), "arrow_function" | "function_expression" | "function"))
1220 || v.child_by_field_name("type").is_some_and(|t| txt(t, src).contains("=>"));
1221 let sig = match value {
1222 Some(val) if matches!(val.kind(), "arrow_function" | "function_expression" | "function") => {
1223 let body_start = val.child_by_field_name("body").map_or(val.end_byte(), |b| b.start_byte());
1224 let s = std::str::from_utf8(&src[v.start_byte()..body_start]).unwrap_or("");
1225 compact(&format!("{kw} {}", s.trim_end().trim_end_matches("=>").trim_end()), 600)
1226 }
1227 Some(val) if val.end_byte() - val.start_byte() > 300 => {
1228 let s = std::str::from_utf8(&src[v.start_byte()..val.start_byte()]).unwrap_or("");
1229 compact(&format!("{kw} {s} …"), 600)
1230 }
1231 _ => compact(&format!("{kw} {}", txt(v, src)), 600),
1232 };
1233 let kind = if is_fn { Kind::Function } else { Kind::Const };
1234 self.out.push(entry(kind, &name, format!("{prefix}{name}"), self.file, line, sig, doc.clone()));
1235 }
1236 }
1237 "internal_module" | "module" => {
1238 let Some(nn) = d.child_by_field_name("name") else { return };
1239 let raw = txt(nn, src);
1240 let quoted = raw.starts_with('"') || raw.starts_with('\'');
1241 let name = raw.trim_matches(['"', '\'']).to_string();
1242 let new_prefix = if quoted { prefix.to_string() } else { format!("{prefix}{name}.") };
1243 if !quoted {
1244 self.out.push(entry(
1245 Kind::Module,
1246 &name,
1247 format!("{prefix}{name}"),
1248 self.file,
1249 line,
1250 head(d, src, "body", 200),
1251 doc,
1252 ));
1253 }
1254 if let Some(body) = d.child_by_field_name("body") {
1255 self.block(body, &new_prefix);
1256 }
1257 }
1258 _ => {}
1259 }
1260 }
1261
1262 fn members(&mut self, body: Node, prefix: &str) {
1263 let src = self.src;
1264 let mut cur = body.walk();
1265 let children: Vec<Node> = body.named_children(&mut cur).collect();
1266 for m in children {
1267 let kind = match m.kind() {
1268 "method_definition" | "method_signature" | "abstract_method_signature" => Kind::Method,
1269 "public_field_definition" | "property_signature" => Kind::Field,
1270 _ => continue,
1271 };
1272 let t = txt(m, src);
1273 if t.starts_with("private ") || t.starts_with("protected ") || t.contains(" private ") && kind == Kind::Field {
1274 continue;
1275 }
1276 let Some(nn) = m.child_by_field_name("name") else { continue };
1277 let name = txt(nn, src).trim_matches(['"', '\'']).to_string();
1278 if name.starts_with('#') || name.starts_with('_') || name.starts_with('[') {
1279 continue;
1280 }
1281 let doc = self.doc(m);
1282 let sig = if m.child_by_field_name("body").is_some() {
1283 head(m, src, "body", 500)
1284 } else {
1285 compact(t.trim_end_matches(';'), 500)
1286 };
1287 self.out.push(entry(
1288 kind,
1289 &name,
1290 format!("{prefix}{name}"),
1291 self.file,
1292 m.start_position().row as u32 + 1,
1293 sig,
1294 doc,
1295 ));
1296 }
1297 }
1298}
1299
1300fn py_module(rel: &str) -> String {
1303 let r = rel.trim_end_matches(".pyi").trim_end_matches(".py");
1304 let r = r.trim_end_matches("/__init__");
1305 r.replace('/', ".")
1306}
1307
1308fn py_file(text: &str, rel: &str, out: &mut Vec<Entry>) {
1309 let module = py_module(rel);
1310 parse_with("py", text, |root, src| {
1311 if let Some(doc) = py_docstring(root, src) {
1312 for s in markdown::split(&doc, &module) {
1313 out.push(prose_section(s, rel, 0));
1314 }
1315 }
1316 py_block(root, src, &module, false, rel, out);
1317 });
1318}
1319
1320fn py_docstring(block: Node, src: &[u8]) -> Option<String> {
1321 let first = block.named_child(0)?;
1322 let first = if first.kind() == "comment" { first.next_named_sibling()? } else { first };
1323 if first.kind() != "expression_statement" {
1324 return None;
1325 }
1326 let s = first.named_child(0)?;
1327 if s.kind() != "string" {
1328 return None;
1329 }
1330 let raw = txt(s, src);
1331 let raw = raw.trim_start_matches(['r', 'R', 'u', 'U', 'b', 'B']);
1332 let inner = raw
1333 .strip_prefix("\"\"\"")
1334 .and_then(|x| x.strip_suffix("\"\"\""))
1335 .or_else(|| raw.strip_prefix("'''").and_then(|x| x.strip_suffix("'''")))
1336 .or_else(|| raw.strip_prefix('"').and_then(|x| x.strip_suffix('"')))
1337 .or_else(|| raw.strip_prefix('\'').and_then(|x| x.strip_suffix('\'')))?;
1338 let lines: Vec<&str> = inner.lines().collect();
1339 if lines.is_empty() {
1340 return None;
1341 }
1342 let first_line = lines[0].trim();
1344 let rest = dedent(&lines[1..]);
1345 let gap = if lines.get(1).is_some_and(|l| l.trim().is_empty()) { "\n\n" } else { "\n" };
1346 let doc = if rest.is_empty() {
1347 first_line.to_string()
1348 } else if first_line.is_empty() {
1349 rest
1350 } else {
1351 format!("{first_line}{gap}{rest}")
1352 };
1353 Some(cap_doc(doc))
1354}
1355
1356fn py_block(block: Node, src: &[u8], prefix: &str, in_class: bool, rel: &str, out: &mut Vec<Entry>) {
1357 let mut cur = block.walk();
1358 let children: Vec<Node> = block.named_children(&mut cur).collect();
1359 for c in children {
1360 let (def, decorators) = if c.kind() == "decorated_definition" {
1361 let mut dc = c.walk();
1362 let decs: Vec<String> = c
1363 .named_children(&mut dc)
1364 .filter(|n| n.kind() == "decorator")
1365 .map(|n| compact(txt(n, src), 80))
1366 .collect();
1367 match c.child_by_field_name("definition") {
1368 Some(d) => (d, decs),
1369 None => continue,
1370 }
1371 } else {
1372 (c, Vec::new())
1373 };
1374 let Some(nn) = def.child_by_field_name("name") else { continue };
1375 let name = txt(nn, src);
1376 let dunder_ok = matches!(name, "__init__" | "__call__");
1377 if name.starts_with('_') && !dunder_ok {
1378 continue;
1379 }
1380 let line = c.start_position().row as u32 + 1;
1381 let mut sig = decorators.join(" ");
1382 match def.kind() {
1383 "function_definition" => {
1384 if !sig.is_empty() {
1385 sig.push(' ');
1386 }
1387 let end = def.child_by_field_name("body").map_or(def.end_byte(), |b| b.start_byte());
1388 let raw = strip_py_comments(std::str::from_utf8(&src[def.start_byte()..end]).unwrap_or(""));
1389 sig.push_str(&compact(raw.trim_end().trim_end_matches(':').trim_end(), 700));
1390 let doc = def.child_by_field_name("body").and_then(|b| py_docstring(b, src)).unwrap_or_default();
1391 let kind = if in_class { Kind::Method } else { Kind::Function };
1392 out.push(entry(kind, name, format!("{prefix}.{name}"), rel, line, sig, doc));
1393 }
1394 "class_definition" => {
1395 if !sig.is_empty() {
1396 sig.push(' ');
1397 }
1398 sig.push_str(&head(def, src, "body", 400));
1399 let body = def.child_by_field_name("body");
1400 let doc = body.and_then(|b| py_docstring(b, src)).unwrap_or_default();
1401 let path = format!("{prefix}.{name}");
1402 out.push(entry(Kind::Class, name, path.clone(), rel, line, sig, doc));
1403 if let Some(b) = body {
1404 py_block(b, src, &path, true, rel, out);
1405 }
1406 }
1407 _ => {}
1408 }
1409 }
1410}
1411
1412fn rs_module(rel: &str, crate_name: &str) -> String {
1415 let r = rel.strip_prefix("src/").unwrap_or(rel).trim_end_matches(".rs");
1416 let mut segs: Vec<&str> = r.split('/').collect();
1417 if matches!(segs.last(), Some(&"mod") | Some(&"lib") | Some(&"main")) {
1418 segs.pop();
1419 }
1420 let mut p = crate_name.to_string();
1421 for s in segs {
1422 if !s.is_empty() {
1423 p.push_str("::");
1424 p.push_str(s);
1425 }
1426 }
1427 p
1428}
1429
1430fn rs_file(text: &str, rel: &str, crate_name: &str, out: &mut Vec<Entry>) {
1431 let module = rs_module(rel, crate_name);
1432 rs_parse(text, rel, &module, 0, 0, None, out);
1433}
1434
1435type Inherit = Option<(String, bool)>;
1438
1439fn rs_parse(text: &str, rel: &str, module: &str, line_off: u32, depth: u32, inherit: Inherit, out: &mut Vec<Entry>) {
1440 parse_with("rs", text, |root, src| {
1441 let mut inner_doc = Vec::new();
1442 let mut w = RsWalker {
1443 src,
1444 file: rel,
1445 line_off,
1446 depth,
1447 out: Vec::new(),
1448 wrappers: HashMap::new(),
1449 inherit,
1450 };
1451 w.block(root, module, None, &mut inner_doc);
1452 if !inner_doc.is_empty() {
1453 let doc = inner_doc.join("\n");
1454 for s in markdown::split(&doc, module) {
1455 out.push(prose_section(s, rel, line_off));
1456 }
1457 }
1458 out.append(&mut w.out);
1459 });
1460}
1461
1462struct RsWalker<'a> {
1463 src: &'a [u8],
1464 file: &'a str,
1465 line_off: u32,
1466 depth: u32,
1467 out: Vec<Entry>,
1468 wrappers: HashMap<String, (String, bool)>,
1470 inherit: Inherit,
1471}
1472
1473fn doc_wrapper(text: &str) -> Option<(String, bool)> {
1475 if !text.contains(":item") {
1476 return None;
1477 }
1478 let doc: Vec<&str> = text
1479 .lines()
1480 .filter_map(|l| strip_line_doc(l.trim()).filter(|(_, inner)| !inner).map(|(d, _)| d))
1481 .collect();
1482 if doc.is_empty() {
1483 return None;
1484 }
1485 Some((cap_doc(doc.join("\n").trim().to_string()), text.contains("macro_export")))
1486}
1487
1488fn is_pub(n: Node, src: &[u8]) -> bool {
1489 let mut cur = n.walk();
1490 let r = n
1491 .named_children(&mut cur)
1492 .any(|c| c.kind() == "visibility_modifier" && txt(c, src).trim() == "pub");
1493 r
1494}
1495
1496fn strip_line_doc(t: &str) -> Option<(&str, bool)> {
1497 if let Some(r) = t.strip_prefix("///") {
1498 if r.starts_with('/') {
1499 return None;
1500 }
1501 return Some((r.strip_prefix(' ').unwrap_or(r).trim_end(), false));
1502 }
1503 if let Some(r) = t.strip_prefix("//!") {
1504 return Some((r.strip_prefix(' ').unwrap_or(r).trim_end(), true));
1505 }
1506 None
1507}
1508
1509impl RsWalker<'_> {
1510 fn line(&self, n: Node) -> u32 {
1511 n.start_position().row as u32 + 1 + self.line_off
1512 }
1513
1514 fn block(&mut self, node: Node, module: &str, owner: Option<(&str, bool)>, inner_doc: &mut Vec<String>) {
1516 let src = self.src;
1517 let mut cur = node.walk();
1518 let children: Vec<Node> = node.named_children(&mut cur).collect();
1519 let mut pending: Vec<String> = Vec::new();
1520 let mut hidden = false;
1521 let mut exported_macro = false;
1522 let mut first_line: Option<u32> = None;
1523 for c in children {
1524 let k = c.kind();
1525 match k {
1526 "line_comment" => {
1527 let t = txt(c, src);
1528 if let Some((d, inner)) = strip_line_doc(t) {
1529 if inner {
1530 inner_doc.push(d.to_string());
1531 } else {
1532 if pending.is_empty() {
1533 first_line = Some(self.line(c));
1534 }
1535 pending.push(d.to_string());
1536 }
1537 }
1538 continue;
1539 }
1540 "block_comment" => {
1541 let t = txt(c, src);
1542 if t.starts_with("/*!") {
1543 inner_doc.push(clean_block_comment(t));
1544 } else if t.starts_with("/**") && !t.starts_with("/***") {
1545 pending.push(clean_block_comment(t));
1546 }
1547 continue;
1548 }
1549 "attribute_item" => {
1550 let t = txt(c, src);
1551 if t.contains("doc(hidden)") {
1552 hidden = true;
1553 }
1554 if t.contains("macro_export") {
1555 exported_macro = true;
1556 }
1557 if let Some(rest) = t.strip_prefix("#[doc = \"") {
1559 pending.push(rest.trim_end_matches("\"]").replace("\\n", "\n"));
1560 }
1561 continue;
1562 }
1563 "inner_attribute_item" => continue,
1564 _ => {}
1565 }
1566 let mut doc = cap_doc(pending.join("\n").trim().to_string());
1567 if doc.is_empty() {
1568 if let Some((d, _)) = &self.inherit {
1569 doc = d.clone();
1570 }
1571 }
1572 let _ = first_line.take();
1573 pending.clear();
1574 let was_hidden = std::mem::take(&mut hidden);
1575 let was_macro_export = std::mem::take(&mut exported_macro) || self.inherit.as_ref().is_some_and(|i| i.1);
1576 if was_hidden {
1577 continue;
1578 }
1579 let line = self.line(c);
1580 let sep = "::";
1581 let in_trait = owner.is_some_and(|o| o.1);
1582 let public = in_trait || is_pub(c, src);
1583 let name = c.child_by_field_name("name").map(|n| txt(n, src).to_string());
1584 let qual = |name: &str| match owner {
1585 Some((o, _)) => format!("{module}{sep}{o}{sep}{name}"),
1586 None => format!("{module}{sep}{name}"),
1587 };
1588 match k {
1589 "function_item" | "function_signature_item" => {
1590 let (Some(name), true) = (name, public) else { continue };
1591 let kind = if owner.is_some() { Kind::Method } else { Kind::Function };
1592 self.out.push(entry(
1593 kind,
1594 &name,
1595 qual(&name),
1596 self.file,
1597 line,
1598 head(c, src, "body", 600).trim_end_matches(';').to_string(),
1599 doc,
1600 ));
1601 }
1602 "struct_item" | "enum_item" | "union_item" => {
1603 let (Some(name), true) = (name, public) else { continue };
1604 let kind = if k == "enum_item" { Kind::Enum } else { Kind::Struct };
1605 let full = txt(c, src);
1606 let sig = if full.len() <= 900 {
1607 compact(full, 900)
1608 } else {
1609 format!("{} {{ … }}", head(c, src, "body", 400))
1610 };
1611 self.out.push(entry(kind, &name, qual(&name), self.file, line, sig, doc));
1612 }
1613 "trait_item" => {
1614 let (Some(name), true) = (name, public) else { continue };
1615 self.out
1616 .push(entry(Kind::Trait, &name, qual(&name), self.file, line, head(c, src, "body", 400), doc));
1617 if let Some(b) = c.child_by_field_name("body") {
1618 let mut ignore = Vec::new();
1619 self.block(b, module, Some((&name, true)), &mut ignore);
1620 }
1621 }
1622 "impl_item" => {
1623 if c.child_by_field_name("trait").is_some() {
1624 continue;
1625 }
1626 let Some(ty) = c.child_by_field_name("type") else { continue };
1627 let ty = txt(ty, src);
1628 let ty = ty.split('<').next().unwrap_or(ty).trim().to_string();
1629 if let Some(b) = c.child_by_field_name("body") {
1630 let mut ignore = Vec::new();
1631 self.block(b, module, Some((&ty, false)), &mut ignore);
1632 }
1633 }
1634 "mod_item" => {
1635 let (Some(name), true) = (name, public) else { continue };
1636 let path = format!("{module}{sep}{name}");
1637 if !doc.is_empty() {
1638 self.out
1639 .push(entry(Kind::Module, &name, path.clone(), self.file, line, format!("pub mod {name}"), doc));
1640 }
1641 if let Some(b) = c.child_by_field_name("body") {
1642 let mut inner = Vec::new();
1643 self.block(b, &path, None, &mut inner);
1644 }
1645 }
1646 "type_item" | "const_item" | "static_item" => {
1647 let (Some(name), true) = (name, public) else { continue };
1648 let kind = if k == "type_item" { Kind::Type } else { Kind::Const };
1649 self.out.push(entry(
1650 kind,
1651 &name,
1652 qual(&name),
1653 self.file,
1654 line,
1655 compact(txt(c, src).trim_end_matches(';'), 400),
1656 doc,
1657 ));
1658 }
1659 "macro_definition" => {
1660 let Some(name) = name else { continue };
1661 if let Some(w) = doc_wrapper(txt(c, src)) {
1662 self.wrappers.insert(name.clone(), w);
1663 continue;
1664 }
1665 if !was_macro_export {
1666 continue;
1667 }
1668 let root = module.split("::").next().unwrap_or(module);
1670 let arms = compact(txt(c, src), 300);
1671 self.out.push(entry(Kind::Macro, &name, format!("{root}::{name}!"), self.file, line, arms, doc));
1672 }
1673 "macro_invocation" if self.depth < 3 => {
1674 let Some(tt) = c.named_children(&mut c.walk()).find(|n| n.kind() == "token_tree") else {
1676 continue;
1677 };
1678 let body = txt(tt, src);
1679 let called = c.child_by_field_name("macro").map(|m| txt(m, src).to_string()).unwrap_or_default();
1680 let wrapper = self.wrappers.get(&called).cloned();
1681 let itemish =
1682 body.contains("fn ") || body.contains("struct ") || body.contains("mod ") || body.contains("trait ") || body.contains("macro_rules!");
1683 if body.len() < 4 || !itemish {
1684 continue;
1685 }
1686 let inner = &body[1..body.len() - 1];
1687 let off = tt.start_position().row as u32 + self.line_off;
1688 let mut sub = Vec::new();
1689 rs_parse(
1690 inner,
1691 self.file,
1692 module,
1693 off,
1694 self.depth + 1,
1695 wrapper.or_else(|| self.inherit.clone()),
1696 &mut sub,
1697 );
1698 self.out.extend(sub);
1700 }
1701 _ => {}
1702 }
1703 }
1704 }
1705}
1706
1707fn exported(name: &str) -> bool {
1710 name.chars().next().is_some_and(|c| c.is_uppercase())
1711}
1712
1713fn go_doc(n: Node, src: &[u8]) -> String {
1714 let mut lines: Vec<String> = Vec::new();
1715 let mut p = n.prev_sibling();
1716 let mut next_row = n.start_position().row;
1717 while let Some(c) = p {
1718 if c.kind() != "comment" || c.end_position().row + 1 != next_row {
1719 break;
1720 }
1721 let t = txt(c, src);
1722 if let Some(r) = t.strip_prefix("//") {
1723 lines.push(r.strip_prefix(' ').unwrap_or(r).trim_end().to_string());
1724 } else {
1725 lines.push(clean_block_comment(t));
1726 }
1727 next_row = c.start_position().row;
1728 p = c.prev_sibling();
1729 }
1730 lines.reverse();
1731 cap_doc(lines.join("\n").trim().to_string())
1732}
1733
1734fn go_file(text: &str, rel: &str, module: &str, out: &mut Vec<Entry>) {
1735 parse_with("go", text, |root, src| {
1736 let mut cur = root.walk();
1737 let children: Vec<Node> = root.named_children(&mut cur).collect();
1738 let mut pkg = module.rsplit('/').next().unwrap_or(module).to_string();
1739 for c in children {
1740 let line = c.start_position().row as u32 + 1;
1741 match c.kind() {
1742 "package_clause" => {
1743 if let Some(id) = c.named_child(0) {
1744 pkg = txt(id, src).to_string();
1745 }
1746 if pkg == "main" {
1747 return;
1748 }
1749 let doc = go_doc(c, src);
1750 if !doc.is_empty() {
1751 let dir = rel.rsplit_once('/').map_or("", |x| x.0);
1752 let title = if dir.is_empty() {
1753 format!("package {pkg}")
1754 } else {
1755 format!("package {pkg} ({dir})")
1756 };
1757 for s in markdown::split(&doc, &title) {
1758 out.push(prose_section(s, rel, 0));
1759 }
1760 }
1761 }
1762 "function_declaration" => {
1763 let Some(name) = c.child_by_field_name("name").map(|n| txt(n, src)) else {
1764 continue;
1765 };
1766 if !exported(name) {
1767 continue;
1768 }
1769 out.push(entry(
1770 Kind::Function,
1771 name,
1772 format!("{pkg}.{name}"),
1773 rel,
1774 line,
1775 head(c, src, "body", 600),
1776 go_doc(c, src),
1777 ));
1778 }
1779 "method_declaration" => {
1780 let Some(name) = c.child_by_field_name("name").map(|n| txt(n, src)) else {
1781 continue;
1782 };
1783 let recv = c.child_by_field_name("receiver").map(|r| txt(r, src)).unwrap_or("");
1784 let ty: String = recv
1785 .trim_matches(['(', ')'])
1786 .split_whitespace()
1787 .last()
1788 .unwrap_or("")
1789 .trim_start_matches('*')
1790 .split('[')
1791 .next()
1792 .unwrap_or("")
1793 .to_string();
1794 if !exported(name) || !exported(&ty) {
1795 continue;
1796 }
1797 out.push(entry(
1798 Kind::Method,
1799 name,
1800 format!("{pkg}.{ty}.{name}"),
1801 rel,
1802 line,
1803 head(c, src, "body", 600),
1804 go_doc(c, src),
1805 ));
1806 }
1807 "type_declaration" | "const_declaration" | "var_declaration" => {
1808 let outer_doc = go_doc(c, src);
1809 let mut cc = c.walk();
1810 let specs: Vec<Node> = c
1811 .named_children(&mut cc)
1812 .filter(|n| matches!(n.kind(), "type_spec" | "type_alias" | "const_spec" | "var_spec"))
1813 .collect();
1814 let single = specs.len() == 1;
1815 for s in specs {
1816 let names: Vec<String> = if s.kind().starts_with("type") {
1817 s.child_by_field_name("name").map(|n| vec![txt(n, src).to_string()]).unwrap_or_default()
1818 } else {
1819 let mut sc = s.walk();
1820 s.children_by_field_name("name", &mut sc).map(|n| txt(n, src).to_string()).collect()
1821 };
1822 let doc = if single { outer_doc.clone() } else { go_doc(s, src) };
1823 let sline = s.start_position().row as u32 + 1;
1824 for name in names.iter().filter(|n| exported(n)) {
1825 if s.kind().starts_with("type") {
1826 let full = txt(s, src);
1827 let sig = if full.len() <= 900 {
1828 format!("type {}", compact(full, 900))
1829 } else {
1830 format!("type {} …", compact(&full[..full.find('{').unwrap_or(200.min(full.len()))], 300))
1831 };
1832 let kind = if full.contains("interface") { Kind::Interface } else { Kind::Struct };
1833 out.push(entry(kind, name, format!("{pkg}.{name}"), rel, sline, sig, doc.clone()));
1834 if let Some(t) = s.child_by_field_name("type").filter(|t| t.kind() == "interface_type") {
1836 let mut ic = t.walk();
1837 let elems: Vec<Node> = t.named_children(&mut ic).collect();
1838 for m in elems {
1839 if !matches!(m.kind(), "method_elem" | "method_spec") {
1840 continue;
1841 }
1842 let Some(mn) = m.child_by_field_name("name").map(|n| txt(n, src)) else {
1843 continue;
1844 };
1845 out.push(entry(
1846 Kind::Method,
1847 mn,
1848 format!("{pkg}.{name}.{mn}"),
1849 rel,
1850 m.start_position().row as u32 + 1,
1851 compact(txt(m, src), 400),
1852 go_doc(m, src),
1853 ));
1854 }
1855 }
1856 } else {
1857 let kw = if c.kind() == "const_declaration" { "const" } else { "var" };
1858 out.push(entry(
1859 Kind::Const,
1860 name,
1861 format!("{pkg}.{name}"),
1862 rel,
1863 sline,
1864 compact(&format!("{kw} {}", txt(s, src)), 300),
1865 doc.clone(),
1866 ));
1867 }
1868 }
1869 }
1870 }
1871 _ => {}
1872 }
1873 }
1874 });
1875}
1876
1877#[cfg(test)]
1878mod tests {
1879 use super::*;
1880
1881 fn run(f: fn(&str, &str, &mut Vec<Entry>), text: &str, rel: &str) -> Vec<Entry> {
1882 let mut out = Vec::new();
1883 f(text, rel, &mut out);
1884 out
1885 }
1886
1887 #[test]
1888 fn typescript_declarations() {
1889 let src = r#"
1890/** Creates an object schema. */
1891export declare function object<T extends ZodRawShape>(shape: T, params?: string): ZodObject<T>;
1892export declare namespace z {
1893 /** A string schema. */
1894 function string(): ZodString;
1895}
1896export interface ZodType<T> {
1897 /** Parse or throw. */
1898 parse(data: unknown): T;
1899 readonly _def: Def;
1900}
1901export declare class ZodObject<T> extends ZodType<T> {
1902 /** Disallow unknown keys. */
1903 strict(message?: string): ZodObject<T>;
1904 private _cached;
1905}
1906declare const objectType: <T>(shape: T) => ZodObject<T>;
1907export { objectType as obj };
1908export * as zz from "./external";
1909"#;
1910 let e = run(ts_file, src, "lib/index.d.ts");
1911 let get = |p: &str| {
1912 e.iter()
1913 .find(|x| x.path == p)
1914 .unwrap_or_else(|| panic!("missing {p}: {:#?}", e.iter().map(|x| &x.path).collect::<Vec<_>>()))
1915 };
1916 assert_eq!(get("object").doc, "Creates an object schema.");
1917 assert!(get("object").sig.contains("ZodObject<T>"));
1918 assert_eq!(get("z.string").doc, "A string schema.");
1919 assert_eq!(get("ZodType.parse").kind, Kind::Method);
1920 assert!(e.iter().all(|x| x.name != "_def" && x.name != "_cached"));
1921 assert_eq!(get("ZodObject.strict").doc, "Disallow unknown keys.");
1922 assert_eq!(get("objectType").kind, Kind::Function);
1923 assert_eq!(get("obj").alias_of.as_deref(), Some("objectType"));
1924 assert_eq!(get("zz").kind, Kind::Module);
1925 assert_eq!(get("object").line, 3);
1926 }
1927
1928 #[test]
1929 fn legacy_dirs() {
1930 assert!(legacy_file("v3/types.d.ts", 4));
1931 assert!(!legacy_file("v4/classic/schemas.d.ts", 4));
1932 assert!(legacy_file("pydantic/v1/main.py", 2));
1933 assert!(!legacy_file("pydantic/main.py", 2));
1934 assert!(!legacy_file("src/v2/x.rs", 1));
1935 }
1936
1937 #[test]
1938 fn python_definitions() {
1939 let src = "\"\"\"Module docs.\"\"\"\n\nclass BaseModel:\n \"\"\"Base class.\n\n More.\n \"\"\"\n\n def model_dump(self, *, mode: str = 'python') -> dict:\n \"\"\"Dump the model.\"\"\"\n return {}\n\n @classmethod\n def model_validate(cls, obj):\n '''Validate.'''\n\n def _private(self): pass\n\ndef create_model(name: str, **fields) -> type:\n return None\n";
1940 let e = run(py_file, src, "pydantic/main.py");
1941 let get = |p: &str| e.iter().find(|x| x.path == p).unwrap_or_else(|| panic!("missing {p}"));
1942 assert_eq!(get("pydantic.main.BaseModel").doc, "Base class.\n\nMore.");
1943 assert_eq!(
1944 get("pydantic.main.BaseModel.model_dump").sig,
1945 "def model_dump(self, *, mode: str = 'python') -> dict"
1946 );
1947 assert!(get("pydantic.main.BaseModel.model_validate").sig.starts_with("@classmethod def model_validate"));
1948 assert!(e.iter().all(|x| x.name != "_private"));
1949 assert!(e.iter().any(|x| x.kind == Kind::Prose && x.doc == "Module docs."));
1950 assert_eq!(get("pydantic.main.create_model").kind, Kind::Function);
1951 }
1952
1953 #[test]
1954 fn docs_pages_written_as_components() {
1955 let src = r#"import dedent from "dedent";
1956export const metadata = { title: "Installing with Vite", description: "Use the Vite plugin.", openGraph: { title: "x" } };
1957const steps = [
1958 {
1959 title: "Import Tailwind CSS",
1960 body: (
1961 <p>
1962 Add an <code>@import</code> to your CSS file.
1963 </p>
1964 ),
1965 code: { name: "CSS", lang: "css", code: dedent`
1966 @import "tailwindcss";
1967 ` },
1968 },
1969 { title: 'Old way', body: () => <p>Use directives{' '}here.</p>, code: { lang: 'css', code: '@tailwind base;\n@tailwind utilities;' } },
1970];
1971export default function Page() { return <div className="prose"><h3 className="sr-only">Overview</h3><p>Fast.</p></div> }
1972"#;
1973 let (md, title) = page_markdown(src).unwrap();
1974 assert_eq!(title.as_deref(), Some("Installing with Vite"));
1975 assert!(md.contains("## Import Tailwind CSS"), "{md}");
1976 assert!(md.contains("Add an `@import` to your CSS file."), "{md}");
1977 assert!(md.contains("```css\n@import \"tailwindcss\";\n```"), "{md}");
1978 assert!(md.contains("@tailwind base;\n@tailwind utilities;"), "{md}");
1979 assert!(md.contains("### Overview"), "{md}");
1980 assert!(!md.contains("dedent") && !md.contains("prose"), "{md}");
1981 let mut out = Vec::new();
1982 prose_titled(&md, "upstream:site/docs/installation/page.tsx", 0, title, &mut out);
1983 let s = out.iter().find(|e| e.name == "Installing with Vite › Import Tailwind CSS").unwrap();
1984 assert_eq!(s.line, 5);
1985 assert_eq!(
1986 page_title("upstream:site/src/app/(docs)/docs/installation/(tabs)/using-vite/page.tsx"),
1987 "installation/using-vite"
1988 );
1989 }
1990
1991 #[test]
1992 fn rust_items_and_macro_bodies() {
1993 let src = "//! Crate docs.\n//!\n//! # Usage\n//! Call spawn.\n\ncfg_rt! {\n /// Spawns a new task.\n #[track_caller]\n pub fn spawn<F>(future: F) -> JoinHandle<F::Output>\n where F: Future + Send + 'static {\n todo!()\n }\n}\n\n/// A router.\npub struct Router<S = ()> { inner: Inner }\n\nimpl<S> Router<S> {\n /// Add a route.\n pub fn route(self, path: &str, m: MethodRouter<S>) -> Self { self }\n fn private(&self) {}\n}\n\n/// Hidden\n#[doc(hidden)]\npub fn hidden() {}\n\n/// Macro.\n#[macro_export]\nmacro_rules! select { () => {} }\n";
1994 let mut e = Vec::new();
1995 rs_file(src, "src/task/spawn.rs", "tokio", &mut e);
1996 let get = |p: &str| {
1997 e.iter()
1998 .find(|x| x.path == p)
1999 .unwrap_or_else(|| panic!("missing {p}: {:#?}", e.iter().map(|x| &x.path).collect::<Vec<_>>()))
2000 };
2001 let spawn = get("tokio::task::spawn::spawn");
2002 assert_eq!(spawn.doc, "Spawns a new task.");
2003 assert_eq!(spawn.line, 9);
2004 assert!(spawn.sig.contains("JoinHandle"));
2005 assert_eq!(get("tokio::task::spawn::Router::route").doc, "Add a route.");
2006 assert!(e.iter().all(|x| x.name != "private" && x.name != "hidden"));
2007 assert_eq!(get("tokio::select!").kind, Kind::Macro);
2008 assert!(e.iter().any(|x| x.kind == Kind::Prose && x.path.ends_with("Usage")));
2009
2010 let src = "macro_rules! doc {\n ($select:item) => {\n /// Waits on multiple branches.\n #[macro_export]\n $select\n };\n}\n\ndoc! {macro_rules! select {\n () => {};\n}}\n";
2012 let mut e = Vec::new();
2013 rs_file(src, "src/macros/select.rs", "tokio", &mut e);
2014 let sel = e.iter().find(|x| x.path == "tokio::select!").expect("select! extracted");
2015 assert_eq!(sel.doc, "Waits on multiple branches.");
2016 assert!(e.iter().all(|x| x.name != "doc"));
2017 }
2018
2019 #[test]
2020 fn go_declarations() {
2021 let src = "// Package gin implements a web framework.\npackage gin\n\n// Context is the request context.\ntype Context struct {\n\tKeys map[string]any\n}\n\n// JSON serializes obj.\nfunc (c *Context) JSON(code int, obj any) {\n}\n\n// Default returns an Engine.\nfunc Default(opts ...Option) *Engine {\n\treturn nil\n}\n\nfunc internal() {}\n\ntype Handler interface {\n\t// Serve serves.\n\tServe(c *Context)\n}\n";
2022 let mut e = Vec::new();
2023 go_file(src, "context.go", "github.com/gin-gonic/gin", &mut e);
2024 let get = |p: &str| {
2025 e.iter()
2026 .find(|x| x.path == p)
2027 .unwrap_or_else(|| panic!("missing {p}: {:#?}", e.iter().map(|x| &x.path).collect::<Vec<_>>()))
2028 };
2029 assert_eq!(get("gin.Context.JSON").doc, "JSON serializes obj.");
2030 assert_eq!(get("gin.Default").sig, "func Default(opts ...Option) *Engine");
2031 assert_eq!(get("gin.Context").doc, "Context is the request context.");
2032 assert_eq!(get("gin.Handler.Serve").doc, "Serve serves.");
2033 assert!(e.iter().all(|x| x.name != "internal"));
2034 assert!(e.iter().any(|x| x.kind == Kind::Prose && x.doc.contains("web framework")));
2035 }
2036}