1use crate::locate::Source;
6use crate::markdown;
7use crate::Eco;
8use rayon::prelude::*;
9use serde::{Deserialize, Serialize};
10use std::cell::RefCell;
11use std::collections::{HashMap, HashSet};
12use std::path::{Path, PathBuf};
13use tree_sitter::{Node, Parser};
14
15#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
16pub enum Kind {
17 Function,
18 Method,
19 Class,
20 Interface,
21 Type,
22 Const,
23 Enum,
24 Struct,
25 Trait,
26 Macro,
27 Module,
28 Field,
29 Alias,
30 Prose,
31}
32
33impl Kind {
34 pub fn as_str(self) -> &'static str {
35 match self {
36 Kind::Function => "function",
37 Kind::Method => "method",
38 Kind::Class => "class",
39 Kind::Interface => "interface",
40 Kind::Type => "type",
41 Kind::Const => "const",
42 Kind::Enum => "enum",
43 Kind::Struct => "struct",
44 Kind::Trait => "trait",
45 Kind::Macro => "macro",
46 Kind::Module => "module",
47 Kind::Field => "field",
48 Kind::Alias => "re-export",
49 Kind::Prose => "doc",
50 }
51 }
52 pub fn is_container(self) -> bool {
53 matches!(self, Kind::Class | Kind::Interface | Kind::Struct | Kind::Trait | Kind::Enum | Kind::Module)
54 }
55}
56
57#[derive(Debug, Clone, Serialize, Deserialize)]
58pub struct Entry {
59 pub kind: Kind,
60 pub name: String,
62 pub path: String,
64 pub file: String,
66 pub line: u32,
67 pub sig: String,
69 pub doc: String,
71 pub alias_of: Option<String>,
73 #[serde(default)]
76 pub legacy: bool,
77}
78
79fn legacy_file(file: &str, major: u64) -> bool {
81 major > 1
82 && file.split('/').any(|seg| {
83 seg.len() >= 2 && seg.starts_with('v') && seg[1..].chars().all(|c| c.is_ascii_digit()) && seg[1..].parse::<u64>().is_ok_and(|n| n < major)
84 })
85}
86
87const SKIP_DIRS: &[&str] = &[
90 "node_modules",
91 "compiled",
92 ".git",
93 "test",
94 "tests",
95 "__tests__",
96 "__test__",
97 "testdata",
98 "test_data",
99 "benches",
100 "bench",
101 "benchmarks",
102 "fixtures",
103 "__fixtures__",
104 "__pycache__",
105 "examples",
106 "example",
107 "_examples",
108 "coverage",
109 ".github",
110 "vendor",
111 "internal",
112 "__snapshots__",
113];
114
115const MAX_FILES: usize = 25_000;
116const MAX_FILE_BYTES: u64 = 3 * 1024 * 1024;
117
118fn is_doc_name(name: &str) -> bool {
119 let u = name.to_ascii_uppercase();
120 [
121 "README",
122 "CHANGELOG",
123 "CHANGES",
124 "HISTORY",
125 "MIGRATION",
126 "MIGRATING",
127 "UPGRADING",
128 "UPGRADE",
129 "RELEASES",
130 "NEWS",
131 "GUIDE",
132 "API",
133 ]
134 .iter()
135 .any(|p| u.starts_with(p))
136}
137
138fn doc_ext(ext: &str) -> bool {
139 matches!(ext, "md" | "mdx" | "markdown" | "rst")
140}
141
142fn walk(dir: &Path, rel: &Path, depth: usize, out: &mut Vec<PathBuf>, public_npm: bool) {
143 if depth > 14 || out.len() >= MAX_FILES {
144 return;
145 }
146 let Ok(rd) = std::fs::read_dir(dir) else { return };
147 let mut entries: Vec<_> = rd.flatten().collect();
148 entries.sort_by_key(|e| e.file_name());
149 for e in entries {
150 let name = e.file_name().to_string_lossy().to_string();
151 let Ok(ft) = e.file_type() else { continue };
152 let r = rel.join(&name);
153 if ft.is_dir() {
154 if (SKIP_DIRS.contains(&name.as_str()) && !(public_npm && name == "internal")) || name.starts_with('.') {
155 continue;
156 }
157 walk(&e.path(), &r, depth + 1, out, public_npm);
158 } else if ft.is_file() || ft.is_symlink() {
159 out.push(r);
160 }
161 if out.len() >= MAX_FILES {
162 return;
163 }
164 }
165}
166
167fn walk_all(dir: &Path, rel: &Path, depth: usize, out: &mut Vec<PathBuf>) {
169 if depth > 16 || out.len() >= MAX_FILES {
170 return;
171 }
172 let Ok(rd) = std::fs::read_dir(dir) else { return };
173 let mut entries: Vec<_> = rd.flatten().collect();
174 entries.sort_by_key(|e| e.file_name());
175 for e in entries {
176 let name = e.file_name().to_string_lossy().to_string();
177 if name.starts_with('.') {
178 continue;
179 }
180 let r = rel.join(&name);
181 match e.file_type() {
182 Ok(t) if t.is_dir() => walk_all(&e.path(), &r, depth + 1, out),
183 Ok(_) => out.push(r),
184 _ => {}
185 }
186 }
187}
188
189fn ext_of(p: &Path) -> String {
190 let s = p.file_name().map(|f| f.to_string_lossy().to_ascii_lowercase()).unwrap_or_default();
191 for e in [".d.ts", ".d.mts", ".d.cts"] {
192 if s.ends_with(e) {
193 return e[1..].to_string();
194 }
195 }
196 p.extension().map(|e| e.to_string_lossy().to_ascii_lowercase()).unwrap_or_default()
197}
198
199fn in_docs_dir(p: &Path) -> bool {
200 p.components().any(|c| {
201 matches!(
202 c.as_os_str().to_string_lossy().to_ascii_lowercase().as_str(),
203 "docs" | "doc" | "documentation" | "guide" | "guides"
204 )
205 })
206}
207
208struct Job {
210 abs: PathBuf,
211 rel: String,
212 how: How,
213}
214
215#[derive(Clone, Copy, PartialEq)]
216enum How {
217 Prose,
218 Page,
220 Example,
221 Metadata,
222 Ts,
223 Python,
224 Rust,
225 Go,
226}
227
228fn plan(eco: Eco, src: &Source, extra_types: Option<&Path>, upstream: Option<&Path>, public_api: bool) -> Vec<Job> {
230 let mut rels = Vec::new();
231 match &src.files {
232 Some(list) => rels.extend(list.iter().cloned()),
233 None => walk(&src.dir, Path::new(""), 0, &mut rels, public_api),
234 }
235 let mut jobs = Vec::new();
236 let names: HashSet<String> = rels.iter().map(|r| r.to_string_lossy().replace('\\', "/")).collect();
237 let mut has_dts = false;
238 for r in &rels {
239 let e = ext_of(r);
240 if e.starts_with("d.") {
241 has_dts = true;
242 }
243 }
244 let rs_crate_src = eco == Eco::Cargo;
245 for r in rels {
246 let rel = r.to_string_lossy().replace('\\', "/");
247 if rel
248 .split('/')
249 .any(|seg| SKIP_DIRS.contains(&seg) && !(public_api && seg == "internal") && src.files.is_some())
250 {
251 continue;
252 }
253 let abs = src.dir.join(&r);
254 let fname = r.file_name().map(|f| f.to_string_lossy().to_string()).unwrap_or_default();
255 let e = ext_of(&r);
256 let depth = rel.matches('/').count();
257 let how = if doc_ext(&e) || (e.is_empty() || e == "txt") && is_doc_name(&fname) && depth == 0 {
258 if is_doc_name(&fname) || in_docs_dir(&r) || depth == 0 {
259 Some(How::Prose)
260 } else {
261 None
262 }
263 } else {
264 match eco {
265 Eco::Npm => match e.as_str() {
266 "d.ts" => Some(How::Ts),
267 "d.mts" | "d.cts" => {
268 let base = rel.trim_end_matches(".d.mts").trim_end_matches(".d.cts");
269 (public_api || !names.contains(&format!("{base}.d.ts"))).then_some(How::Ts)
270 }
271 "ts" | "mts" | "cts" | "tsx" if !has_dts => Some(How::Ts),
272 "js" | "mjs" | "cjs" if !has_dts && !fname.contains(".min.") => Some(How::Ts),
273 _ => None,
274 },
275 Eco::PyPI => match e.as_str() {
276 "py" => Some(How::Python),
277 "pyi" => {
278 let base = rel.trim_end_matches(".pyi");
279 if names.contains(&format!("{base}.py")) {
280 None
281 } else {
282 Some(How::Python)
283 }
284 }
285 _ => None,
286 },
287 Eco::Cargo => (e == "rs" && rs_crate_src && rel.starts_with("src/")).then_some(How::Rust),
288 Eco::Go => (e == "go" && !fname.ends_with("_test.go")).then_some(How::Go),
289 }
290 };
291 if let Some(how) = how {
292 jobs.push(Job { abs, rel, how });
293 }
294 }
295 if let Some(up) = upstream {
297 let pages: HashSet<String> = std::fs::read_to_string(up.join(".lockdocs-upstream.json"))
298 .ok()
299 .and_then(|t| serde_json::from_str::<crate::upstream::Manifest>(&t).ok())
300 .map(|m| m.pages.into_iter().collect())
301 .unwrap_or_default();
302 let mut rels = Vec::new();
303 walk_all(up, Path::new(""), 0, &mut rels);
304 for r in rels {
305 let e = ext_of(&r);
306 let plain = r.to_string_lossy().replace('\\', "/");
307 let rel = format!("upstream:{plain}");
308 if pages.contains(&plain) {
309 jobs.push(Job {
310 abs: up.join(&r),
311 rel,
312 how: How::Page,
313 });
314 continue;
315 }
316 if doc_ext(&e) || e == "mdoc" || e == "txt" {
318 jobs.push(Job {
319 abs: up.join(&r),
320 rel,
321 how: How::Prose,
322 });
323 } else if EXAMPLE_EXTS.contains(&e.as_str()) {
324 jobs.push(Job {
326 abs: up.join(&r),
327 rel,
328 how: How::Example,
329 });
330 }
331 }
332 }
333 if let Some(meta) = &src.metadata {
334 jobs.push(Job {
335 abs: meta.clone(),
336 rel: "README (package metadata)".into(),
337 how: How::Metadata,
338 });
339 }
340 if let (Some(types), false) = (extra_types, has_dts) {
342 let mut rels = Vec::new();
343 walk(types, Path::new(""), 0, &mut rels, public_api);
344 let label = types
345 .iter()
346 .rev()
347 .take(2)
348 .collect::<Vec<_>>()
349 .into_iter()
350 .rev()
351 .map(|s| s.to_string_lossy())
352 .collect::<Vec<_>>()
353 .join("/");
354 for r in rels {
355 if ext_of(&r) == "d.ts" || public_api && matches!(ext_of(&r).as_str(), "d.cts" | "d.mts") {
356 let rel = format!("{label}/{}", r.to_string_lossy().replace('\\', "/"));
357 jobs.push(Job {
358 abs: types.join(&r),
359 rel,
360 how: How::Ts,
361 });
362 }
363 }
364 }
365 jobs
366}
367
368pub fn extract(eco: Eco, pkg_name: &str, src: &Source, extra_types: Option<&Path>, upstream: Option<&Path>) -> Vec<Entry> {
370 let major: u64 = src.version.trim_start_matches('v').split('.').next().and_then(|m| m.parse().ok()).unwrap_or(0);
371 let jobs = plan(eco, src, extra_types, upstream, false);
372 let crate_name = if eco == Eco::Cargo {
373 rust_crate_name(&src.dir).unwrap_or_else(|| pkg_name.replace('-', "_"))
374 } else {
375 String::new()
376 };
377 let includes = upstream.map(Includes::scan).unwrap_or_default();
378 let mut all: Vec<Entry> = jobs
379 .par_iter()
380 .flat_map_iter(|job| {
381 let Ok(meta) = std::fs::metadata(&job.abs) else { return Vec::new() };
382 if meta.len() > MAX_FILE_BYTES && job.how != How::Prose {
383 return Vec::new();
384 }
385 let Ok(bytes) = std::fs::read(&job.abs) else { return Vec::new() };
386 let text = String::from_utf8_lossy(&bytes);
387 let mut out = Vec::new();
388 match job.how {
389 How::Prose => {
390 prose(&text, &job.rel, 0, &mut out);
391 if job.rel.starts_with("upstream:") && !includes.files.is_empty() {
392 for e in &mut out {
393 e.doc = includes.expand(&e.doc);
394 }
395 }
396 }
397 How::Page => {
398 if let Some((md, title)) = page_markdown(&text) {
399 let title = title.unwrap_or_else(|| page_title(&job.rel));
400 prose_titled(&md, &job.rel, 0, Some(title), &mut out);
401 }
402 }
403 How::Example => example(&text, &job.rel, &mut out),
404 How::Metadata => {
405 let (body, line) = markdown::metadata_body(&text);
406 prose(&body, &job.rel, line - 1, &mut out)
407 }
408 How::Ts => ts_file(&text, &job.rel, &mut out),
409 How::Python => py_file(&text, &job.rel, &mut out),
410 How::Rust => rs_file(&text, &job.rel, &crate_name, &mut out),
411 How::Go => go_file(&text, &job.rel, pkg_name, &mut out),
412 }
413 out
414 })
415 .collect();
416 for e in &mut all {
417 e.legacy = legacy_file(&e.file, major);
418 }
419 let mut seen = HashSet::new();
421 all.retain(|e| seen.insert((e.path.clone(), e.sig.clone(), e.doc.len())));
422 all
423}
424
425pub(crate) fn npm_sources(src: &Source, extra_types: Option<&Path>) -> std::collections::BTreeMap<String, String> {
427 plan(Eco::Npm, src, extra_types, None, true)
428 .into_iter()
429 .filter(|j| j.how == How::Ts)
430 .filter_map(|j| {
431 if std::fs::metadata(&j.abs).ok()?.len() > MAX_FILE_BYTES {
432 return None;
433 }
434 Some((j.rel, std::fs::read_to_string(j.abs).ok()?))
435 })
436 .collect()
437}
438
439pub(crate) fn npm_entries(sources: &std::collections::BTreeMap<String, String>) -> Vec<Entry> {
442 let mut entries = Vec::new();
443 for (file, text) in sources {
444 ts_file(text, file, &mut entries);
445 }
446 entries
447}
448
449pub(crate) fn npm_stamp(src: &Source, extra_types: Option<&Path>) -> String {
452 let mut parts = Vec::new();
453 for j in plan(Eco::Npm, src, extra_types, None, true) {
454 parts.push(j.rel);
455 if let Ok(bytes) = std::fs::read(&j.abs) {
456 parts.push(crate::cache::hash(&[&String::from_utf8_lossy(&bytes)]));
457 }
458 }
459 crate::cache::hash(&parts.iter().map(String::as_str).collect::<Vec<_>>())
460}
461
462fn rust_crate_name(dir: &Path) -> Option<String> {
463 let t = std::fs::read_to_string(dir.join("Cargo.toml")).ok()?;
464 let v: toml::Table = toml::from_str(&t).ok()?;
465 let lib = v.get("lib").and_then(|l| l.get("name")).and_then(|n| n.as_str());
466 let pkg = v.get("package").and_then(|p| p.get("name")).and_then(|n| n.as_str());
467 lib.or(pkg).map(|n| n.replace('-', "_"))
468}
469
470#[derive(Default)]
474struct Includes {
475 files: Vec<(String, PathBuf)>,
477}
478
479impl Includes {
480 fn scan(dir: &Path) -> Includes {
481 let mut rels = Vec::new();
482 walk_all(dir, Path::new(""), 0, &mut rels);
483 let files = rels
484 .into_iter()
485 .filter(|r| EXAMPLE_EXTS.contains(&ext_of(r).as_str()))
486 .map(|r| {
487 let s = r.to_string_lossy().replace('\\', "/").to_ascii_lowercase();
488 let stem = s.rsplit_once('.').map_or(s.clone(), |x| x.0.to_string());
489 (stem, dir.join(&r))
490 })
491 .collect();
492 Includes { files }
493 }
494
495 fn find(&self, reference: &str) -> Option<&PathBuf> {
496 let r = reference.trim().trim_matches(['"', '\'']).to_ascii_lowercase();
497 let r = r.split_whitespace().next().unwrap_or("");
498 let r = r.rsplit_once('.').map_or(r, |x| x.0);
499 let segs: Vec<&str> = r
500 .split('/')
501 .filter(|s| !s.is_empty() && *s != ".." && *s != "." && *s != ".tmp_examples")
502 .collect();
503 for take in (1..=segs.len().min(3)).rev() {
504 let suffix = segs[segs.len() - take..].join("/");
505 let hits: Vec<&(String, PathBuf)> = self.files.iter().filter(|(k, _)| *k == suffix || k.ends_with(&format!("/{suffix}"))).collect();
506 if hits.len() == 1 {
507 return Some(&hits[0].1);
508 }
509 }
510 None
511 }
512
513 fn expand(&self, text: &str) -> String {
514 if !(text.contains("{!") || text.contains("{*") || text.contains("--8<--")) {
515 return text.to_string();
516 }
517 let mut out = String::with_capacity(text.len());
518 for line in text.lines() {
519 let t = line.trim();
520 let reference = t
521 .strip_prefix("{!")
522 .and_then(|x| x.strip_suffix("!}"))
523 .or_else(|| t.strip_prefix("{*").and_then(|x| x.strip_suffix("*}")))
524 .or_else(|| t.strip_prefix("--8<--"));
525 match reference
526 .and_then(|r| self.find(r))
527 .and_then(|p| std::fs::read_to_string(p).ok().map(|c| (p, c)))
528 {
529 Some((p, code)) => {
530 let lang = p.extension().map(|e| e.to_string_lossy().to_string()).unwrap_or_default();
531 let code: String = code.chars().take(3000).collect();
532 out.push_str(&format!("```{lang}\n{}\n```\n", code.trim_end()));
533 }
534 None => {
535 out.push_str(line);
536 out.push('\n');
537 }
538 }
539 }
540 out
541 }
542}
543
544const EXAMPLE_EXTS: &[&str] = &["py", "ts", "tsx", "js", "jsx", "mjs", "rs", "go"];
545
546fn example(text: &str, rel: &str, out: &mut Vec<Entry>) {
548 if text.len() > 12_000 || text.trim().is_empty() {
549 return;
550 }
551 let name = rel.rsplit('/').next().unwrap_or(rel);
552 let lang = name.rsplit('.').next().unwrap_or("");
553 let title = name.rsplit_once('.').map_or(name, |x| x.0).replace(['_', '-'], " ");
554 out.push(Entry {
555 kind: Kind::Prose,
556 name: format!("example › {title}"),
557 path: format!("example › {title}"),
558 file: rel.to_string(),
559 line: 1,
560 sig: String::new(),
561 doc: format!("```{lang}\n{}\n```", text.trim_end()),
562 alias_of: None,
563 legacy: false,
564 });
565}
566
567fn page_title(rel: &str) -> String {
569 let segs: Vec<&str> = rel.split('/').filter(|s| !(s.starts_with('(') && s.ends_with(')'))).collect();
570 let mut segs: Vec<&str> = segs.iter().rev().take(3).rev().copied().collect();
571 if let Some(last) = segs.last() {
572 if last.starts_with("page.") || last.starts_with("index.") {
573 segs.pop();
574 }
575 }
576 segs.join("/")
577 .trim_end_matches(".tsx")
578 .trim_end_matches(".jsx")
579 .trim_end_matches(".js")
580 .to_string()
581}
582
583fn page_markdown(text: &str) -> Option<(String, Option<String>)> {
589 if text.len() > 200_000 {
590 return None;
591 }
592 parse_with("tsx", text, |root, src| {
593 let mut p = PageOut::default();
594 page_walk(root, src, &mut p, false);
595 p.flush();
596 let mut md = String::new();
597 let mut n = 0u32;
598 for (line, t) in &p.out {
599 while n + 1 < *line {
600 md.push('\n');
601 n += 1;
602 }
603 md.push_str(t);
604 md.push('\n');
605 n += 1 + t.matches('\n').count() as u32;
606 }
607 (md, p.title)
608 })
609 .filter(|(md, _)| !md.trim().is_empty())
610}
611
612#[derive(Default)]
613struct PageOut {
614 out: Vec<(u32, String)>,
615 para: String,
616 para_line: u32,
617 title: Option<String>,
618}
619
620impl PageOut {
621 fn text(&mut self, line: u32, t: &str) {
622 if self.para.trim().is_empty() {
623 self.para_line = line;
624 }
625 self.para.push_str(t);
626 }
627 fn flush(&mut self) {
628 let t = compact(&self.para, usize::MAX);
629 if !t.is_empty() {
630 self.out.push((self.para_line, t));
631 }
632 self.para.clear();
633 }
634 fn block(&mut self, line: u32, t: String) {
635 self.flush();
636 self.out.push((line, t));
637 }
638}
639
640fn line_of(n: Node) -> u32 {
641 n.start_position().row as u32 + 1
642}
643
644fn js_string(n: Node, src: &[u8]) -> Option<String> {
646 match n.kind() {
647 "string" => {
648 let raw = txt(n, src);
649 let inner = raw.get(1..raw.len().saturating_sub(1)).unwrap_or("");
650 Some(inner.replace("\\n", "\n").replace("\\'", "'").replace("\\\"", "\"").replace("\\\\", "\\"))
651 }
652 "template_string" => {
653 let raw = txt(n, src);
654 let inner = raw.get(1..raw.len().saturating_sub(1)).unwrap_or("");
655 Some(dedent(&inner.trim_matches('\n').lines().collect::<Vec<_>>()))
656 }
657 "call_expression" => n
658 .child_by_field_name("arguments")
659 .filter(|a| a.kind() == "template_string")
660 .and_then(|a| js_string(a, src)),
661 "parenthesized_expression" => n.named_child(0).and_then(|c| js_string(c, src)),
662 _ => None,
663 }
664}
665
666fn jsx_name(n: Node, src: &[u8]) -> String {
667 let open = if n.kind() == "jsx_element" {
668 n.child_by_field_name("open_tag")
669 } else {
670 Some(n)
671 };
672 open.and_then(|o| o.child_by_field_name("name"))
673 .map(|x| txt(x, src).to_string())
674 .unwrap_or_default()
675}
676
677fn jsx_text(n: Node, src: &[u8], out: &mut String) {
679 match n.kind() {
680 "jsx_text" => out.push_str(txt(n, src)),
681 "jsx_expression" => {
682 if let Some(v) = n.named_child(0).and_then(|c| js_string(c, src)) {
683 out.push_str(&v);
684 }
685 }
686 "jsx_attribute" | "jsx_opening_element" | "jsx_closing_element" => {}
687 _ => {
688 let mut c = n.walk();
689 for ch in n.named_children(&mut c) {
690 jsx_text(ch, src, out);
691 }
692 }
693 }
694}
695
696fn page_walk(n: Node, src: &[u8], p: &mut PageOut, in_array: bool) {
697 match n.kind() {
698 "import_statement" | "comment" | "jsx_attribute" => {}
699 "pair" => {
700 let key = n
701 .child_by_field_name("key")
702 .map(|k| txt(k, src).trim_matches(['"', '\'']).to_string())
703 .unwrap_or_default();
704 let Some(value) = n.child_by_field_name("value") else { return };
705 match key.as_str() {
706 "title" | "heading" => {
707 if let Some(t) = js_string(value, src) {
708 if !in_array && p.title.is_none() {
709 p.title = Some(t);
710 } else if in_array {
711 p.block(line_of(n), format!("## {}", compact(&t, 200)));
712 }
713 }
714 }
715 "description" => {
716 if let Some(t) = js_string(value, src) {
717 p.block(line_of(value), compact(&t, usize::MAX));
718 }
719 }
720 "code" => match js_string(value, src) {
721 Some(code) => {
722 let lang = n
723 .parent()
724 .and_then(|obj| {
725 let mut c = obj.walk();
726 let found = obj.named_children(&mut c).find(|x| {
727 x.kind() == "pair" && x.child_by_field_name("key").is_some_and(|k| txt(k, src).trim_matches(['"', '\'']) == "lang")
728 });
729 found
730 })
731 .and_then(|x| x.child_by_field_name("value"))
732 .and_then(|v| js_string(v, src))
733 .unwrap_or_default();
734 p.block(line_of(value), format!("```{lang}\n{}\n```", code.trim_end()));
735 }
736 None => page_walk(value, src, p, in_array),
737 },
738 "name" | "lang" | "className" | "href" | "id" | "url" | "src" | "image" | "images" | "icon" | "openGraph" | "twitter" | "alternates" => {}
739 _ => page_walk(value, src, p, in_array),
740 }
741 }
742 "jsx_element" | "jsx_self_closing_element" => {
743 let name = jsx_name(n, src);
744 let b = name.as_bytes();
745 if b.len() == 2 && b[0] == b'h' && (b'1'..=b'6').contains(&b[1]) {
746 let mut t = String::new();
747 jsx_text(n, src, &mut t);
748 let t = compact(&t, 200);
749 if !t.is_empty() {
750 p.block(line_of(n), format!("{} {t}", "#".repeat((b[1] - b'0') as usize)));
751 }
752 } else if name == "code" {
753 let mut t = String::new();
754 jsx_text(n, src, &mut t);
755 p.text(line_of(n), &format!("`{}`", t.trim()));
756 } else {
757 let block = matches!(name.as_str(), "p" | "li" | "div" | "section" | "ul" | "ol" | "pre" | "table" | "tr");
758 if block {
759 p.flush();
760 }
761 let mut c = n.walk();
762 for ch in n.named_children(&mut c) {
763 page_walk(ch, src, p, in_array);
764 }
765 if block {
766 p.flush();
767 }
768 }
769 }
770 "jsx_text" => p.text(line_of(n), txt(n, src)),
771 "jsx_expression" => match n.named_child(0) {
772 Some(c) if matches!(c.kind(), "string" | "template_string") => {
773 let v = js_string(c, src).unwrap_or_default();
774 p.text(line_of(n), &v);
775 }
776 Some(c) => page_walk(c, src, p, in_array),
777 None => {}
778 },
779 _ => {
780 let arr = in_array || n.kind() == "array";
781 let mut c = n.walk();
782 for ch in n.named_children(&mut c) {
783 page_walk(ch, src, p, arr);
784 }
785 }
786 }
787}
788
789fn prose(text: &str, rel: &str, line_off: u32, out: &mut Vec<Entry>) {
790 prose_titled(text, rel, line_off, None, out)
791}
792
793fn prose_titled(text: &str, rel: &str, line_off: u32, given: Option<String>, out: &mut Vec<Entry>) {
794 let (text, fm_title) = markdown::clean_mdx(text, rel.ends_with(".mdx"));
795 let file_name = rel.rsplit('/').next().unwrap_or(rel);
797 let file_title = if rel.starts_with("upstream:") && !is_doc_name(file_name) {
798 file_name
799 .rsplit_once('.')
800 .map_or(file_name, |x| x.0)
801 .trim_start_matches(|c: char| c.is_ascii_digit() || c == '-')
802 .replace(['_', '-'], " ")
803 } else {
804 file_name.to_string()
805 };
806 let file_title = file_title.as_str();
807 let title = if let Some(t) = given {
808 t
809 } else if rel.contains("package metadata") {
810 "README".to_string()
811 } else if let Some(t) = fm_title {
812 t
813 } else {
814 file_title.to_string()
815 };
816 for s in markdown::split(&text, &title) {
817 out.push(prose_section(s, rel, line_off));
818 }
819}
820
821thread_local! {
824 static PARSERS: RefCell<Vec<(&'static str, Parser)>> = const { RefCell::new(Vec::new()) };
825}
826
827fn parse_with<R>(lang: &'static str, text: &str, f: impl FnOnce(Node, &[u8]) -> R) -> Option<R> {
830 let pooled = PARSERS.with(|ps| {
831 let mut ps = ps.borrow_mut();
832 ps.iter().position(|(l, _)| *l == lang).map(|i| ps.swap_remove(i).1)
833 });
834 let mut parser = match pooled {
835 Some(p) => p,
836 None => {
837 let mut p = Parser::new();
838 let language: tree_sitter::Language = match lang {
839 "ts" => tree_sitter_typescript::LANGUAGE_TYPESCRIPT.into(),
840 "tsx" => tree_sitter_typescript::LANGUAGE_TSX.into(),
841 "py" => tree_sitter_python::LANGUAGE.into(),
842 "rs" => tree_sitter_rust::LANGUAGE.into(),
843 _ => tree_sitter_go::LANGUAGE.into(),
844 };
845 p.set_language(&language).ok()?;
846 p
847 }
848 };
849 let tree = parser.parse(text, None);
850 PARSERS.with(|ps| ps.borrow_mut().push((lang, parser)));
851 let tree = tree?;
852 Some(f(tree.root_node(), text.as_bytes()))
853}
854
855fn txt<'a>(n: Node, src: &'a [u8]) -> &'a str {
856 n.utf8_text(src).unwrap_or("")
857}
858
859fn compact(s: &str, cap: usize) -> String {
860 let mut out = String::with_capacity(s.len().min(cap.saturating_add(8)));
861 let mut space = false;
862 for c in s.chars() {
863 if c.is_whitespace() {
864 space = true;
865 continue;
866 }
867 if space && !out.is_empty() {
868 out.push(' ');
869 }
870 space = false;
871 out.push(c);
872 if out.len() >= cap {
873 out.push_str(" …");
874 break;
875 }
876 }
877 out.replace("( ", "(").replace(", )", ")").replace(",)", ")").replace(" )", ")")
878}
879
880fn strip_py_comments(s: &str) -> String {
882 let mut out = String::with_capacity(s.len());
883 for line in s.lines() {
884 let mut quote: Option<char> = None;
885 let mut cut = line.len();
886 for (i, c) in line.char_indices() {
887 match (quote, c) {
888 (None, '#') => {
889 cut = i;
890 break;
891 }
892 (None, '"' | '\'') => quote = Some(c),
893 (Some(q), c) if c == q => quote = None,
894 _ => {}
895 }
896 }
897 out.push_str(&line[..cut]);
898 out.push('\n');
899 }
900 out
901}
902
903fn head(n: Node, src: &[u8], body_field: &str, cap: usize) -> String {
905 let end = n.child_by_field_name(body_field).map_or(n.end_byte(), |b| b.start_byte());
906 let s = std::str::from_utf8(&src[n.start_byte()..end]).unwrap_or("");
907 compact(s.trim_end().trim_end_matches(['{', ':', '=']).trim_end(), cap)
908}
909
910fn cap_doc(s: String) -> String {
911 const CAP: usize = 4000;
912 if s.len() <= CAP {
913 return s;
914 }
915 let mut i = CAP;
916 while !s.is_char_boundary(i) {
917 i -= 1;
918 }
919 format!("{} …", &s[..i])
920}
921
922fn dedent(lines: &[&str]) -> String {
923 let indent = lines
924 .iter()
925 .filter(|l| !l.trim().is_empty())
926 .map(|l| l.len() - l.trim_start().len())
927 .min()
928 .unwrap_or(0);
929 let v: Vec<&str> = lines.iter().map(|l| if l.len() >= indent { &l[indent..] } else { l.trim_start() }).collect();
930 v.join("\n").trim().to_string()
931}
932
933fn clean_block_comment(s: &str) -> String {
934 let s = s
935 .trim_start_matches("/**")
936 .trim_start_matches("/*!")
937 .trim_start_matches("/*")
938 .trim_end_matches("*/");
939 let lines: Vec<&str> = s
940 .lines()
941 .map(|l| {
942 let t = l.trim_start();
943 match t.strip_prefix('*') {
944 Some(r) => r.strip_prefix(' ').unwrap_or(r),
945 None => l,
946 }
947 })
948 .collect();
949 cap_doc(dedent(&lines))
950}
951
952fn entry(kind: Kind, name: &str, path: String, file: &str, line: u32, sig: String, doc: String) -> Entry {
953 Entry {
954 kind,
955 name: name.to_string(),
956 path,
957 file: file.to_string(),
958 line,
959 sig,
960 doc,
961 alias_of: None,
962 legacy: false,
963 }
964}
965
966fn prose_section(s: crate::markdown::Section, rel: &str, line_off: u32) -> Entry {
969 Entry {
970 kind: Kind::Prose,
971 name: s.heading.clone(),
972 path: s.heading,
973 file: rel.to_string(),
974 line: s.line + line_off,
975 sig: String::new(),
976 doc: s.text,
977 alias_of: None,
978 legacy: false,
979 }
980}
981
982fn ts_file(text: &str, rel: &str, out: &mut Vec<Entry>) {
985 let lang = if rel.ends_with(".tsx") || rel.ends_with(".jsx") { "tsx" } else { "ts" };
986 let is_js = rel.ends_with(".js") || rel.ends_with(".mjs") || rel.ends_with(".cjs");
987 parse_with(lang, text, |root, src| {
988 let mut w = TsWalker {
989 src,
990 file: rel,
991 out,
992 js: is_js,
993 };
994 w.block(root, "");
995 });
996}
997
998struct TsWalker<'a> {
999 src: &'a [u8],
1000 file: &'a str,
1001 out: &'a mut Vec<Entry>,
1002 js: bool,
1003}
1004
1005impl TsWalker<'_> {
1006 fn doc(&self, n: Node) -> String {
1007 let mut p = n.prev_sibling();
1008 while let Some(c) = p {
1009 if c.kind() == "comment" {
1010 let t = txt(c, self.src);
1011 if t.starts_with("/**") && c.end_position().row + 2 >= n.start_position().row {
1012 return clean_block_comment(t);
1013 }
1014 if t.starts_with("//") {
1015 p = c.prev_sibling();
1016 continue;
1017 }
1018 }
1019 break;
1020 }
1021 String::new()
1022 }
1023
1024 fn line(&self, n: Node) -> u32 {
1025 n.start_position().row as u32 + 1
1026 }
1027
1028 fn block(&mut self, node: Node, prefix: &str) {
1029 let mut cur = node.walk();
1030 let children: Vec<Node> = node.named_children(&mut cur).collect();
1031 for c in children {
1032 match c.kind() {
1033 "export_statement" => {
1034 let doc = self.doc(c);
1035 if let Some(d) = c.child_by_field_name("declaration") {
1036 self.decl(d, prefix, doc, c);
1037 } else {
1038 self.exports(c, prefix);
1039 }
1040 }
1041 "ambient_declaration" => {
1042 let doc = self.doc(c);
1043 let mut cc = c.walk();
1044 let inner: Vec<Node> = c.named_children(&mut cc).collect();
1045 for d in inner {
1046 self.decl(d, prefix, doc.clone(), c);
1047 }
1048 }
1049 "expression_statement" => {
1050 let mut cc = c.walk();
1051 let inner: Vec<Node> = c.named_children(&mut cc).collect();
1052 for d in inner {
1053 if matches!(d.kind(), "internal_module" | "module") {
1054 let doc = self.doc(c);
1055 self.decl(d, prefix, doc, c);
1056 }
1057 }
1058 }
1059 _ => {
1060 let doc = self.doc(c);
1061 if !self.js || !doc.is_empty() || !prefix.is_empty() {
1063 self.decl(c, prefix, doc, c);
1064 }
1065 }
1066 }
1067 }
1068 }
1069
1070 fn exports(&mut self, n: Node, prefix: &str) {
1071 let mut cur = n.walk();
1072 let children: Vec<Node> = n.named_children(&mut cur).collect();
1073 for c in children {
1074 match c.kind() {
1075 "export_clause" => {
1076 let mut cc = c.walk();
1077 let specs: Vec<Node> = c.named_children(&mut cc).collect();
1078 for s in specs {
1079 let (Some(name), alias) = (s.child_by_field_name("name"), s.child_by_field_name("alias")) else {
1080 continue;
1081 };
1082 let Some(alias) = alias else { continue };
1083 let (from, to) = (txt(name, self.src), txt(alias, self.src));
1084 if from == to || to == "default" {
1085 continue;
1086 }
1087 let mut e = entry(
1088 Kind::Alias,
1089 to,
1090 format!("{prefix}{to}"),
1091 self.file,
1092 self.line(s),
1093 compact(txt(n, self.src), 200),
1094 String::new(),
1095 );
1096 e.alias_of = Some(from.to_string());
1097 self.out.push(e);
1098 }
1099 }
1100 "namespace_export" => {
1101 if let Some(id) = c.named_child(0) {
1102 let name = txt(id, self.src);
1103 self.out.push(entry(
1104 Kind::Module,
1105 name,
1106 format!("{prefix}{name}"),
1107 self.file,
1108 self.line(n),
1109 compact(txt(n, self.src), 200),
1110 self.doc(n),
1111 ));
1112 }
1113 }
1114 _ => {}
1115 }
1116 }
1117 }
1118
1119 fn decl(&mut self, d: Node, prefix: &str, doc: String, outer: Node) {
1120 let src = self.src;
1121 let name_of = |n: Node| n.child_by_field_name("name").map(|x| txt(x, src).to_string());
1122 let line = self.line(outer);
1123 match d.kind() {
1124 "ambient_declaration" => {
1125 let mut cur = d.walk();
1126 let inner: Vec<Node> = d.named_children(&mut cur).collect();
1127 for x in inner {
1128 self.decl(x, prefix, doc.clone(), outer);
1129 }
1130 }
1131 "function_signature" | "function_declaration" | "generator_function_declaration" => {
1132 let Some(name) = name_of(d) else { return };
1133 let sig = head(outer, src, "__none__", 600).trim_end_matches(';').to_string();
1134 let sig = if d.child_by_field_name("body").is_some() {
1135 head(d, src, "body", 600)
1136 } else {
1137 sig
1138 };
1139 self.out
1140 .push(entry(Kind::Function, &name, format!("{prefix}{name}"), self.file, line, sig, doc));
1141 }
1142 "class_declaration" | "abstract_class_declaration" | "class" => {
1143 let Some(name) = name_of(d) else { return };
1144 self.out.push(entry(
1145 Kind::Class,
1146 &name,
1147 format!("{prefix}{name}"),
1148 self.file,
1149 line,
1150 head(d, src, "body", 400),
1151 doc,
1152 ));
1153 if let Some(body) = d.child_by_field_name("body") {
1154 self.members(body, &format!("{prefix}{name}."));
1155 }
1156 }
1157 "interface_declaration" => {
1158 let Some(name) = name_of(d) else { return };
1159 self.out.push(entry(
1160 Kind::Interface,
1161 &name,
1162 format!("{prefix}{name}"),
1163 self.file,
1164 line,
1165 head(d, src, "body", 400),
1166 doc,
1167 ));
1168 if let Some(body) = d.child_by_field_name("body") {
1169 self.members(body, &format!("{prefix}{name}."));
1170 }
1171 }
1172 "type_alias_declaration" => {
1173 let Some(name) = name_of(d) else { return };
1174 self.out.push(entry(
1175 Kind::Type,
1176 &name,
1177 format!("{prefix}{name}"),
1178 self.file,
1179 line,
1180 compact(txt(d, src), 700),
1181 doc,
1182 ));
1183 }
1184 "enum_declaration" => {
1185 let Some(name) = name_of(d) else { return };
1186 self.out.push(entry(
1187 Kind::Enum,
1188 &name,
1189 format!("{prefix}{name}"),
1190 self.file,
1191 line,
1192 compact(txt(d, src), 500),
1193 doc,
1194 ));
1195 }
1196 "lexical_declaration" | "variable_declaration" => {
1197 let kw = txt(d, src).split_whitespace().next().unwrap_or("const").to_string();
1198 let mut cur = d.walk();
1199 let decls: Vec<Node> = d.named_children(&mut cur).filter(|n| n.kind() == "variable_declarator").collect();
1200 for v in decls {
1201 let Some(nn) = v.child_by_field_name("name") else { continue };
1202 if nn.kind() != "identifier" {
1203 continue;
1204 }
1205 let name = txt(nn, src).to_string();
1206 let value = v.child_by_field_name("value");
1207 let is_fn = value.is_some_and(|x| matches!(x.kind(), "arrow_function" | "function_expression" | "function"))
1208 || v.child_by_field_name("type").is_some_and(|t| txt(t, src).contains("=>"));
1209 let sig = match value {
1210 Some(val) if matches!(val.kind(), "arrow_function" | "function_expression" | "function") => {
1211 let body_start = val.child_by_field_name("body").map_or(val.end_byte(), |b| b.start_byte());
1212 let s = std::str::from_utf8(&src[v.start_byte()..body_start]).unwrap_or("");
1213 compact(&format!("{kw} {}", s.trim_end().trim_end_matches("=>").trim_end()), 600)
1214 }
1215 Some(val) if val.end_byte() - val.start_byte() > 300 => {
1216 let s = std::str::from_utf8(&src[v.start_byte()..val.start_byte()]).unwrap_or("");
1217 compact(&format!("{kw} {s} …"), 600)
1218 }
1219 _ => compact(&format!("{kw} {}", txt(v, src)), 600),
1220 };
1221 let kind = if is_fn { Kind::Function } else { Kind::Const };
1222 self.out.push(entry(kind, &name, format!("{prefix}{name}"), self.file, line, sig, doc.clone()));
1223 }
1224 }
1225 "internal_module" | "module" => {
1226 let Some(nn) = d.child_by_field_name("name") else { return };
1227 let raw = txt(nn, src);
1228 let quoted = raw.starts_with('"') || raw.starts_with('\'');
1229 let name = raw.trim_matches(['"', '\'']).to_string();
1230 let new_prefix = if quoted { prefix.to_string() } else { format!("{prefix}{name}.") };
1231 if !quoted {
1232 self.out.push(entry(
1233 Kind::Module,
1234 &name,
1235 format!("{prefix}{name}"),
1236 self.file,
1237 line,
1238 head(d, src, "body", 200),
1239 doc,
1240 ));
1241 }
1242 if let Some(body) = d.child_by_field_name("body") {
1243 self.block(body, &new_prefix);
1244 }
1245 }
1246 _ => {}
1247 }
1248 }
1249
1250 fn members(&mut self, body: Node, prefix: &str) {
1251 let src = self.src;
1252 let mut cur = body.walk();
1253 let children: Vec<Node> = body.named_children(&mut cur).collect();
1254 for m in children {
1255 let kind = match m.kind() {
1256 "method_definition" | "method_signature" | "abstract_method_signature" => Kind::Method,
1257 "public_field_definition" | "property_signature" => Kind::Field,
1258 _ => continue,
1259 };
1260 let t = txt(m, src);
1261 if t.starts_with("private ") || t.starts_with("protected ") || t.contains(" private ") && kind == Kind::Field {
1262 continue;
1263 }
1264 let Some(nn) = m.child_by_field_name("name") else { continue };
1265 let name = txt(nn, src).trim_matches(['"', '\'']).to_string();
1266 if name.starts_with('#') || name.starts_with('_') || name.starts_with('[') {
1267 continue;
1268 }
1269 let doc = self.doc(m);
1270 let sig = if m.child_by_field_name("body").is_some() {
1271 head(m, src, "body", 500)
1272 } else {
1273 compact(t.trim_end_matches(';'), 500)
1274 };
1275 self.out.push(entry(
1276 kind,
1277 &name,
1278 format!("{prefix}{name}"),
1279 self.file,
1280 m.start_position().row as u32 + 1,
1281 sig,
1282 doc,
1283 ));
1284 }
1285 }
1286}
1287
1288fn py_module(rel: &str) -> String {
1291 let r = rel.trim_end_matches(".pyi").trim_end_matches(".py");
1292 let r = r.trim_end_matches("/__init__");
1293 r.replace('/', ".")
1294}
1295
1296fn py_file(text: &str, rel: &str, out: &mut Vec<Entry>) {
1297 let module = py_module(rel);
1298 parse_with("py", text, |root, src| {
1299 if let Some(doc) = py_docstring(root, src) {
1300 for s in markdown::split(&doc, &module) {
1301 out.push(prose_section(s, rel, 0));
1302 }
1303 }
1304 py_block(root, src, &module, false, rel, out);
1305 });
1306}
1307
1308fn py_docstring(block: Node, src: &[u8]) -> Option<String> {
1309 let first = block.named_child(0)?;
1310 let first = if first.kind() == "comment" { first.next_named_sibling()? } else { first };
1311 if first.kind() != "expression_statement" {
1312 return None;
1313 }
1314 let s = first.named_child(0)?;
1315 if s.kind() != "string" {
1316 return None;
1317 }
1318 let raw = txt(s, src);
1319 let raw = raw.trim_start_matches(['r', 'R', 'u', 'U', 'b', 'B']);
1320 let inner = raw
1321 .strip_prefix("\"\"\"")
1322 .and_then(|x| x.strip_suffix("\"\"\""))
1323 .or_else(|| raw.strip_prefix("'''").and_then(|x| x.strip_suffix("'''")))
1324 .or_else(|| raw.strip_prefix('"').and_then(|x| x.strip_suffix('"')))
1325 .or_else(|| raw.strip_prefix('\'').and_then(|x| x.strip_suffix('\'')))?;
1326 let lines: Vec<&str> = inner.lines().collect();
1327 if lines.is_empty() {
1328 return None;
1329 }
1330 let first_line = lines[0].trim();
1332 let rest = dedent(&lines[1..]);
1333 let gap = if lines.get(1).is_some_and(|l| l.trim().is_empty()) { "\n\n" } else { "\n" };
1334 let doc = if rest.is_empty() {
1335 first_line.to_string()
1336 } else if first_line.is_empty() {
1337 rest
1338 } else {
1339 format!("{first_line}{gap}{rest}")
1340 };
1341 Some(cap_doc(doc))
1342}
1343
1344fn py_block(block: Node, src: &[u8], prefix: &str, in_class: bool, rel: &str, out: &mut Vec<Entry>) {
1345 let mut cur = block.walk();
1346 let children: Vec<Node> = block.named_children(&mut cur).collect();
1347 for c in children {
1348 let (def, decorators) = if c.kind() == "decorated_definition" {
1349 let mut dc = c.walk();
1350 let decs: Vec<String> = c
1351 .named_children(&mut dc)
1352 .filter(|n| n.kind() == "decorator")
1353 .map(|n| compact(txt(n, src), 80))
1354 .collect();
1355 match c.child_by_field_name("definition") {
1356 Some(d) => (d, decs),
1357 None => continue,
1358 }
1359 } else {
1360 (c, Vec::new())
1361 };
1362 let Some(nn) = def.child_by_field_name("name") else { continue };
1363 let name = txt(nn, src);
1364 let dunder_ok = matches!(name, "__init__" | "__call__");
1365 if name.starts_with('_') && !dunder_ok {
1366 continue;
1367 }
1368 let line = c.start_position().row as u32 + 1;
1369 let mut sig = decorators.join(" ");
1370 match def.kind() {
1371 "function_definition" => {
1372 if !sig.is_empty() {
1373 sig.push(' ');
1374 }
1375 let end = def.child_by_field_name("body").map_or(def.end_byte(), |b| b.start_byte());
1376 let raw = strip_py_comments(std::str::from_utf8(&src[def.start_byte()..end]).unwrap_or(""));
1377 sig.push_str(&compact(raw.trim_end().trim_end_matches(':').trim_end(), 700));
1378 let doc = def.child_by_field_name("body").and_then(|b| py_docstring(b, src)).unwrap_or_default();
1379 let kind = if in_class { Kind::Method } else { Kind::Function };
1380 out.push(entry(kind, name, format!("{prefix}.{name}"), rel, line, sig, doc));
1381 }
1382 "class_definition" => {
1383 if !sig.is_empty() {
1384 sig.push(' ');
1385 }
1386 sig.push_str(&head(def, src, "body", 400));
1387 let body = def.child_by_field_name("body");
1388 let doc = body.and_then(|b| py_docstring(b, src)).unwrap_or_default();
1389 let path = format!("{prefix}.{name}");
1390 out.push(entry(Kind::Class, name, path.clone(), rel, line, sig, doc));
1391 if let Some(b) = body {
1392 py_block(b, src, &path, true, rel, out);
1393 }
1394 }
1395 _ => {}
1396 }
1397 }
1398}
1399
1400fn rs_module(rel: &str, crate_name: &str) -> String {
1403 let r = rel.strip_prefix("src/").unwrap_or(rel).trim_end_matches(".rs");
1404 let mut segs: Vec<&str> = r.split('/').collect();
1405 if matches!(segs.last(), Some(&"mod") | Some(&"lib") | Some(&"main")) {
1406 segs.pop();
1407 }
1408 let mut p = crate_name.to_string();
1409 for s in segs {
1410 if !s.is_empty() {
1411 p.push_str("::");
1412 p.push_str(s);
1413 }
1414 }
1415 p
1416}
1417
1418fn rs_file(text: &str, rel: &str, crate_name: &str, out: &mut Vec<Entry>) {
1419 let module = rs_module(rel, crate_name);
1420 rs_parse(text, rel, &module, 0, 0, None, out);
1421}
1422
1423type Inherit = Option<(String, bool)>;
1426
1427fn rs_parse(text: &str, rel: &str, module: &str, line_off: u32, depth: u32, inherit: Inherit, out: &mut Vec<Entry>) {
1428 parse_with("rs", text, |root, src| {
1429 let mut inner_doc = Vec::new();
1430 let mut w = RsWalker {
1431 src,
1432 file: rel,
1433 line_off,
1434 depth,
1435 out: Vec::new(),
1436 wrappers: HashMap::new(),
1437 inherit,
1438 };
1439 w.block(root, module, None, &mut inner_doc);
1440 if !inner_doc.is_empty() {
1441 let doc = inner_doc.join("\n");
1442 for s in markdown::split(&doc, module) {
1443 out.push(prose_section(s, rel, line_off));
1444 }
1445 }
1446 out.append(&mut w.out);
1447 });
1448}
1449
1450struct RsWalker<'a> {
1451 src: &'a [u8],
1452 file: &'a str,
1453 line_off: u32,
1454 depth: u32,
1455 out: Vec<Entry>,
1456 wrappers: HashMap<String, (String, bool)>,
1458 inherit: Inherit,
1459}
1460
1461fn doc_wrapper(text: &str) -> Option<(String, bool)> {
1463 if !text.contains(":item") {
1464 return None;
1465 }
1466 let doc: Vec<&str> = text
1467 .lines()
1468 .filter_map(|l| strip_line_doc(l.trim()).filter(|(_, inner)| !inner).map(|(d, _)| d))
1469 .collect();
1470 if doc.is_empty() {
1471 return None;
1472 }
1473 Some((cap_doc(doc.join("\n").trim().to_string()), text.contains("macro_export")))
1474}
1475
1476fn is_pub(n: Node, src: &[u8]) -> bool {
1477 let mut cur = n.walk();
1478 let r = n
1479 .named_children(&mut cur)
1480 .any(|c| c.kind() == "visibility_modifier" && txt(c, src).trim() == "pub");
1481 r
1482}
1483
1484fn strip_line_doc(t: &str) -> Option<(&str, bool)> {
1485 if let Some(r) = t.strip_prefix("///") {
1486 if r.starts_with('/') {
1487 return None;
1488 }
1489 return Some((r.strip_prefix(' ').unwrap_or(r).trim_end(), false));
1490 }
1491 if let Some(r) = t.strip_prefix("//!") {
1492 return Some((r.strip_prefix(' ').unwrap_or(r).trim_end(), true));
1493 }
1494 None
1495}
1496
1497impl RsWalker<'_> {
1498 fn line(&self, n: Node) -> u32 {
1499 n.start_position().row as u32 + 1 + self.line_off
1500 }
1501
1502 fn block(&mut self, node: Node, module: &str, owner: Option<(&str, bool)>, inner_doc: &mut Vec<String>) {
1504 let src = self.src;
1505 let mut cur = node.walk();
1506 let children: Vec<Node> = node.named_children(&mut cur).collect();
1507 let mut pending: Vec<String> = Vec::new();
1508 let mut hidden = false;
1509 let mut exported_macro = false;
1510 let mut first_line: Option<u32> = None;
1511 for c in children {
1512 let k = c.kind();
1513 match k {
1514 "line_comment" => {
1515 let t = txt(c, src);
1516 if let Some((d, inner)) = strip_line_doc(t) {
1517 if inner {
1518 inner_doc.push(d.to_string());
1519 } else {
1520 if pending.is_empty() {
1521 first_line = Some(self.line(c));
1522 }
1523 pending.push(d.to_string());
1524 }
1525 }
1526 continue;
1527 }
1528 "block_comment" => {
1529 let t = txt(c, src);
1530 if t.starts_with("/*!") {
1531 inner_doc.push(clean_block_comment(t));
1532 } else if t.starts_with("/**") && !t.starts_with("/***") {
1533 pending.push(clean_block_comment(t));
1534 }
1535 continue;
1536 }
1537 "attribute_item" => {
1538 let t = txt(c, src);
1539 if t.contains("doc(hidden)") {
1540 hidden = true;
1541 }
1542 if t.contains("macro_export") {
1543 exported_macro = true;
1544 }
1545 if let Some(rest) = t.strip_prefix("#[doc = \"") {
1547 pending.push(rest.trim_end_matches("\"]").replace("\\n", "\n"));
1548 }
1549 continue;
1550 }
1551 "inner_attribute_item" => continue,
1552 _ => {}
1553 }
1554 let mut doc = cap_doc(pending.join("\n").trim().to_string());
1555 if doc.is_empty() {
1556 if let Some((d, _)) = &self.inherit {
1557 doc = d.clone();
1558 }
1559 }
1560 let _ = first_line.take();
1561 pending.clear();
1562 let was_hidden = std::mem::take(&mut hidden);
1563 let was_macro_export = std::mem::take(&mut exported_macro) || self.inherit.as_ref().is_some_and(|i| i.1);
1564 if was_hidden {
1565 continue;
1566 }
1567 let line = self.line(c);
1568 let sep = "::";
1569 let in_trait = owner.is_some_and(|o| o.1);
1570 let public = in_trait || is_pub(c, src);
1571 let name = c.child_by_field_name("name").map(|n| txt(n, src).to_string());
1572 let qual = |name: &str| match owner {
1573 Some((o, _)) => format!("{module}{sep}{o}{sep}{name}"),
1574 None => format!("{module}{sep}{name}"),
1575 };
1576 match k {
1577 "function_item" | "function_signature_item" => {
1578 let (Some(name), true) = (name, public) else { continue };
1579 let kind = if owner.is_some() { Kind::Method } else { Kind::Function };
1580 self.out.push(entry(
1581 kind,
1582 &name,
1583 qual(&name),
1584 self.file,
1585 line,
1586 head(c, src, "body", 600).trim_end_matches(';').to_string(),
1587 doc,
1588 ));
1589 }
1590 "struct_item" | "enum_item" | "union_item" => {
1591 let (Some(name), true) = (name, public) else { continue };
1592 let kind = if k == "enum_item" { Kind::Enum } else { Kind::Struct };
1593 let full = txt(c, src);
1594 let sig = if full.len() <= 900 {
1595 compact(full, 900)
1596 } else {
1597 format!("{} {{ … }}", head(c, src, "body", 400))
1598 };
1599 self.out.push(entry(kind, &name, qual(&name), self.file, line, sig, doc));
1600 }
1601 "trait_item" => {
1602 let (Some(name), true) = (name, public) else { continue };
1603 self.out
1604 .push(entry(Kind::Trait, &name, qual(&name), self.file, line, head(c, src, "body", 400), doc));
1605 if let Some(b) = c.child_by_field_name("body") {
1606 let mut ignore = Vec::new();
1607 self.block(b, module, Some((&name, true)), &mut ignore);
1608 }
1609 }
1610 "impl_item" => {
1611 if c.child_by_field_name("trait").is_some() {
1612 continue;
1613 }
1614 let Some(ty) = c.child_by_field_name("type") else { continue };
1615 let ty = txt(ty, src);
1616 let ty = ty.split('<').next().unwrap_or(ty).trim().to_string();
1617 if let Some(b) = c.child_by_field_name("body") {
1618 let mut ignore = Vec::new();
1619 self.block(b, module, Some((&ty, false)), &mut ignore);
1620 }
1621 }
1622 "mod_item" => {
1623 let (Some(name), true) = (name, public) else { continue };
1624 let path = format!("{module}{sep}{name}");
1625 if !doc.is_empty() {
1626 self.out
1627 .push(entry(Kind::Module, &name, path.clone(), self.file, line, format!("pub mod {name}"), doc));
1628 }
1629 if let Some(b) = c.child_by_field_name("body") {
1630 let mut inner = Vec::new();
1631 self.block(b, &path, None, &mut inner);
1632 }
1633 }
1634 "type_item" | "const_item" | "static_item" => {
1635 let (Some(name), true) = (name, public) else { continue };
1636 let kind = if k == "type_item" { Kind::Type } else { Kind::Const };
1637 self.out.push(entry(
1638 kind,
1639 &name,
1640 qual(&name),
1641 self.file,
1642 line,
1643 compact(txt(c, src).trim_end_matches(';'), 400),
1644 doc,
1645 ));
1646 }
1647 "macro_definition" => {
1648 let Some(name) = name else { continue };
1649 if let Some(w) = doc_wrapper(txt(c, src)) {
1650 self.wrappers.insert(name.clone(), w);
1651 continue;
1652 }
1653 if !was_macro_export {
1654 continue;
1655 }
1656 let root = module.split("::").next().unwrap_or(module);
1658 let arms = compact(txt(c, src), 300);
1659 self.out.push(entry(Kind::Macro, &name, format!("{root}::{name}!"), self.file, line, arms, doc));
1660 }
1661 "macro_invocation" if self.depth < 3 => {
1662 let Some(tt) = c.named_children(&mut c.walk()).find(|n| n.kind() == "token_tree") else {
1664 continue;
1665 };
1666 let body = txt(tt, src);
1667 let called = c.child_by_field_name("macro").map(|m| txt(m, src).to_string()).unwrap_or_default();
1668 let wrapper = self.wrappers.get(&called).cloned();
1669 let itemish =
1670 body.contains("fn ") || body.contains("struct ") || body.contains("mod ") || body.contains("trait ") || body.contains("macro_rules!");
1671 if body.len() < 4 || !itemish {
1672 continue;
1673 }
1674 let inner = &body[1..body.len() - 1];
1675 let off = tt.start_position().row as u32 + self.line_off;
1676 let mut sub = Vec::new();
1677 rs_parse(
1678 inner,
1679 self.file,
1680 module,
1681 off,
1682 self.depth + 1,
1683 wrapper.or_else(|| self.inherit.clone()),
1684 &mut sub,
1685 );
1686 self.out.extend(sub);
1688 }
1689 _ => {}
1690 }
1691 }
1692 }
1693}
1694
1695fn exported(name: &str) -> bool {
1698 name.chars().next().is_some_and(|c| c.is_uppercase())
1699}
1700
1701fn go_doc(n: Node, src: &[u8]) -> String {
1702 let mut lines: Vec<String> = Vec::new();
1703 let mut p = n.prev_sibling();
1704 let mut next_row = n.start_position().row;
1705 while let Some(c) = p {
1706 if c.kind() != "comment" || c.end_position().row + 1 != next_row {
1707 break;
1708 }
1709 let t = txt(c, src);
1710 if let Some(r) = t.strip_prefix("//") {
1711 lines.push(r.strip_prefix(' ').unwrap_or(r).trim_end().to_string());
1712 } else {
1713 lines.push(clean_block_comment(t));
1714 }
1715 next_row = c.start_position().row;
1716 p = c.prev_sibling();
1717 }
1718 lines.reverse();
1719 cap_doc(lines.join("\n").trim().to_string())
1720}
1721
1722fn go_file(text: &str, rel: &str, module: &str, out: &mut Vec<Entry>) {
1723 parse_with("go", text, |root, src| {
1724 let mut cur = root.walk();
1725 let children: Vec<Node> = root.named_children(&mut cur).collect();
1726 let mut pkg = module.rsplit('/').next().unwrap_or(module).to_string();
1727 for c in children {
1728 let line = c.start_position().row as u32 + 1;
1729 match c.kind() {
1730 "package_clause" => {
1731 if let Some(id) = c.named_child(0) {
1732 pkg = txt(id, src).to_string();
1733 }
1734 if pkg == "main" {
1735 return;
1736 }
1737 let doc = go_doc(c, src);
1738 if !doc.is_empty() {
1739 let dir = rel.rsplit_once('/').map_or("", |x| x.0);
1740 let title = if dir.is_empty() {
1741 format!("package {pkg}")
1742 } else {
1743 format!("package {pkg} ({dir})")
1744 };
1745 for s in markdown::split(&doc, &title) {
1746 out.push(prose_section(s, rel, 0));
1747 }
1748 }
1749 }
1750 "function_declaration" => {
1751 let Some(name) = c.child_by_field_name("name").map(|n| txt(n, src)) else {
1752 continue;
1753 };
1754 if !exported(name) {
1755 continue;
1756 }
1757 out.push(entry(
1758 Kind::Function,
1759 name,
1760 format!("{pkg}.{name}"),
1761 rel,
1762 line,
1763 head(c, src, "body", 600),
1764 go_doc(c, src),
1765 ));
1766 }
1767 "method_declaration" => {
1768 let Some(name) = c.child_by_field_name("name").map(|n| txt(n, src)) else {
1769 continue;
1770 };
1771 let recv = c.child_by_field_name("receiver").map(|r| txt(r, src)).unwrap_or("");
1772 let ty: String = recv
1773 .trim_matches(['(', ')'])
1774 .split_whitespace()
1775 .last()
1776 .unwrap_or("")
1777 .trim_start_matches('*')
1778 .split('[')
1779 .next()
1780 .unwrap_or("")
1781 .to_string();
1782 if !exported(name) || !exported(&ty) {
1783 continue;
1784 }
1785 out.push(entry(
1786 Kind::Method,
1787 name,
1788 format!("{pkg}.{ty}.{name}"),
1789 rel,
1790 line,
1791 head(c, src, "body", 600),
1792 go_doc(c, src),
1793 ));
1794 }
1795 "type_declaration" | "const_declaration" | "var_declaration" => {
1796 let outer_doc = go_doc(c, src);
1797 let mut cc = c.walk();
1798 let specs: Vec<Node> = c
1799 .named_children(&mut cc)
1800 .filter(|n| matches!(n.kind(), "type_spec" | "type_alias" | "const_spec" | "var_spec"))
1801 .collect();
1802 let single = specs.len() == 1;
1803 for s in specs {
1804 let names: Vec<String> = if s.kind().starts_with("type") {
1805 s.child_by_field_name("name").map(|n| vec![txt(n, src).to_string()]).unwrap_or_default()
1806 } else {
1807 let mut sc = s.walk();
1808 s.children_by_field_name("name", &mut sc).map(|n| txt(n, src).to_string()).collect()
1809 };
1810 let doc = if single { outer_doc.clone() } else { go_doc(s, src) };
1811 let sline = s.start_position().row as u32 + 1;
1812 for name in names.iter().filter(|n| exported(n)) {
1813 if s.kind().starts_with("type") {
1814 let full = txt(s, src);
1815 let sig = if full.len() <= 900 {
1816 format!("type {}", compact(full, 900))
1817 } else {
1818 format!("type {} …", compact(&full[..full.find('{').unwrap_or(200.min(full.len()))], 300))
1819 };
1820 let kind = if full.contains("interface") { Kind::Interface } else { Kind::Struct };
1821 out.push(entry(kind, name, format!("{pkg}.{name}"), rel, sline, sig, doc.clone()));
1822 if let Some(t) = s.child_by_field_name("type").filter(|t| t.kind() == "interface_type") {
1824 let mut ic = t.walk();
1825 let elems: Vec<Node> = t.named_children(&mut ic).collect();
1826 for m in elems {
1827 if !matches!(m.kind(), "method_elem" | "method_spec") {
1828 continue;
1829 }
1830 let Some(mn) = m.child_by_field_name("name").map(|n| txt(n, src)) else {
1831 continue;
1832 };
1833 out.push(entry(
1834 Kind::Method,
1835 mn,
1836 format!("{pkg}.{name}.{mn}"),
1837 rel,
1838 m.start_position().row as u32 + 1,
1839 compact(txt(m, src), 400),
1840 go_doc(m, src),
1841 ));
1842 }
1843 }
1844 } else {
1845 let kw = if c.kind() == "const_declaration" { "const" } else { "var" };
1846 out.push(entry(
1847 Kind::Const,
1848 name,
1849 format!("{pkg}.{name}"),
1850 rel,
1851 sline,
1852 compact(&format!("{kw} {}", txt(s, src)), 300),
1853 doc.clone(),
1854 ));
1855 }
1856 }
1857 }
1858 }
1859 _ => {}
1860 }
1861 }
1862 });
1863}
1864
1865#[cfg(test)]
1866mod tests {
1867 use super::*;
1868
1869 fn run(f: fn(&str, &str, &mut Vec<Entry>), text: &str, rel: &str) -> Vec<Entry> {
1870 let mut out = Vec::new();
1871 f(text, rel, &mut out);
1872 out
1873 }
1874
1875 #[test]
1876 fn typescript_declarations() {
1877 let src = r#"
1878/** Creates an object schema. */
1879export declare function object<T extends ZodRawShape>(shape: T, params?: string): ZodObject<T>;
1880export declare namespace z {
1881 /** A string schema. */
1882 function string(): ZodString;
1883}
1884export interface ZodType<T> {
1885 /** Parse or throw. */
1886 parse(data: unknown): T;
1887 readonly _def: Def;
1888}
1889export declare class ZodObject<T> extends ZodType<T> {
1890 /** Disallow unknown keys. */
1891 strict(message?: string): ZodObject<T>;
1892 private _cached;
1893}
1894declare const objectType: <T>(shape: T) => ZodObject<T>;
1895export { objectType as obj };
1896export * as zz from "./external";
1897"#;
1898 let e = run(ts_file, src, "lib/index.d.ts");
1899 let get = |p: &str| {
1900 e.iter()
1901 .find(|x| x.path == p)
1902 .unwrap_or_else(|| panic!("missing {p}: {:#?}", e.iter().map(|x| &x.path).collect::<Vec<_>>()))
1903 };
1904 assert_eq!(get("object").doc, "Creates an object schema.");
1905 assert!(get("object").sig.contains("ZodObject<T>"));
1906 assert_eq!(get("z.string").doc, "A string schema.");
1907 assert_eq!(get("ZodType.parse").kind, Kind::Method);
1908 assert!(e.iter().all(|x| x.name != "_def" && x.name != "_cached"));
1909 assert_eq!(get("ZodObject.strict").doc, "Disallow unknown keys.");
1910 assert_eq!(get("objectType").kind, Kind::Function);
1911 assert_eq!(get("obj").alias_of.as_deref(), Some("objectType"));
1912 assert_eq!(get("zz").kind, Kind::Module);
1913 assert_eq!(get("object").line, 3);
1914 }
1915
1916 #[test]
1917 fn legacy_dirs() {
1918 assert!(legacy_file("v3/types.d.ts", 4));
1919 assert!(!legacy_file("v4/classic/schemas.d.ts", 4));
1920 assert!(legacy_file("pydantic/v1/main.py", 2));
1921 assert!(!legacy_file("pydantic/main.py", 2));
1922 assert!(!legacy_file("src/v2/x.rs", 1));
1923 }
1924
1925 #[test]
1926 fn python_definitions() {
1927 let src = "\"\"\"Module docs.\"\"\"\n\nclass BaseModel:\n \"\"\"Base class.\n\n More.\n \"\"\"\n\n def model_dump(self, *, mode: str = 'python') -> dict:\n \"\"\"Dump the model.\"\"\"\n return {}\n\n @classmethod\n def model_validate(cls, obj):\n '''Validate.'''\n\n def _private(self): pass\n\ndef create_model(name: str, **fields) -> type:\n return None\n";
1928 let e = run(py_file, src, "pydantic/main.py");
1929 let get = |p: &str| e.iter().find(|x| x.path == p).unwrap_or_else(|| panic!("missing {p}"));
1930 assert_eq!(get("pydantic.main.BaseModel").doc, "Base class.\n\nMore.");
1931 assert_eq!(
1932 get("pydantic.main.BaseModel.model_dump").sig,
1933 "def model_dump(self, *, mode: str = 'python') -> dict"
1934 );
1935 assert!(get("pydantic.main.BaseModel.model_validate").sig.starts_with("@classmethod def model_validate"));
1936 assert!(e.iter().all(|x| x.name != "_private"));
1937 assert!(e.iter().any(|x| x.kind == Kind::Prose && x.doc == "Module docs."));
1938 assert_eq!(get("pydantic.main.create_model").kind, Kind::Function);
1939 }
1940
1941 #[test]
1942 fn docs_pages_written_as_components() {
1943 let src = r#"import dedent from "dedent";
1944export const metadata = { title: "Installing with Vite", description: "Use the Vite plugin.", openGraph: { title: "x" } };
1945const steps = [
1946 {
1947 title: "Import Tailwind CSS",
1948 body: (
1949 <p>
1950 Add an <code>@import</code> to your CSS file.
1951 </p>
1952 ),
1953 code: { name: "CSS", lang: "css", code: dedent`
1954 @import "tailwindcss";
1955 ` },
1956 },
1957 { title: 'Old way', body: () => <p>Use directives{' '}here.</p>, code: { lang: 'css', code: '@tailwind base;\n@tailwind utilities;' } },
1958];
1959export default function Page() { return <div className="prose"><h3 className="sr-only">Overview</h3><p>Fast.</p></div> }
1960"#;
1961 let (md, title) = page_markdown(src).unwrap();
1962 assert_eq!(title.as_deref(), Some("Installing with Vite"));
1963 assert!(md.contains("## Import Tailwind CSS"), "{md}");
1964 assert!(md.contains("Add an `@import` to your CSS file."), "{md}");
1965 assert!(md.contains("```css\n@import \"tailwindcss\";\n```"), "{md}");
1966 assert!(md.contains("@tailwind base;\n@tailwind utilities;"), "{md}");
1967 assert!(md.contains("### Overview"), "{md}");
1968 assert!(!md.contains("dedent") && !md.contains("prose"), "{md}");
1969 let mut out = Vec::new();
1970 prose_titled(&md, "upstream:site/docs/installation/page.tsx", 0, title, &mut out);
1971 let s = out.iter().find(|e| e.name == "Installing with Vite › Import Tailwind CSS").unwrap();
1972 assert_eq!(s.line, 5);
1973 assert_eq!(
1974 page_title("upstream:site/src/app/(docs)/docs/installation/(tabs)/using-vite/page.tsx"),
1975 "installation/using-vite"
1976 );
1977 }
1978
1979 #[test]
1980 fn rust_items_and_macro_bodies() {
1981 let src = "//! Crate docs.\n//!\n//! # Usage\n//! Call spawn.\n\ncfg_rt! {\n /// Spawns a new task.\n #[track_caller]\n pub fn spawn<F>(future: F) -> JoinHandle<F::Output>\n where F: Future + Send + 'static {\n todo!()\n }\n}\n\n/// A router.\npub struct Router<S = ()> { inner: Inner }\n\nimpl<S> Router<S> {\n /// Add a route.\n pub fn route(self, path: &str, m: MethodRouter<S>) -> Self { self }\n fn private(&self) {}\n}\n\n/// Hidden\n#[doc(hidden)]\npub fn hidden() {}\n\n/// Macro.\n#[macro_export]\nmacro_rules! select { () => {} }\n";
1982 let mut e = Vec::new();
1983 rs_file(src, "src/task/spawn.rs", "tokio", &mut e);
1984 let get = |p: &str| {
1985 e.iter()
1986 .find(|x| x.path == p)
1987 .unwrap_or_else(|| panic!("missing {p}: {:#?}", e.iter().map(|x| &x.path).collect::<Vec<_>>()))
1988 };
1989 let spawn = get("tokio::task::spawn::spawn");
1990 assert_eq!(spawn.doc, "Spawns a new task.");
1991 assert_eq!(spawn.line, 9);
1992 assert!(spawn.sig.contains("JoinHandle"));
1993 assert_eq!(get("tokio::task::spawn::Router::route").doc, "Add a route.");
1994 assert!(e.iter().all(|x| x.name != "private" && x.name != "hidden"));
1995 assert_eq!(get("tokio::select!").kind, Kind::Macro);
1996 assert!(e.iter().any(|x| x.kind == Kind::Prose && x.path.ends_with("Usage")));
1997
1998 let src = "macro_rules! doc {\n ($select:item) => {\n /// Waits on multiple branches.\n #[macro_export]\n $select\n };\n}\n\ndoc! {macro_rules! select {\n () => {};\n}}\n";
2000 let mut e = Vec::new();
2001 rs_file(src, "src/macros/select.rs", "tokio", &mut e);
2002 let sel = e.iter().find(|x| x.path == "tokio::select!").expect("select! extracted");
2003 assert_eq!(sel.doc, "Waits on multiple branches.");
2004 assert!(e.iter().all(|x| x.name != "doc"));
2005 }
2006
2007 #[test]
2008 fn go_declarations() {
2009 let src = "// Package gin implements a web framework.\npackage gin\n\n// Context is the request context.\ntype Context struct {\n\tKeys map[string]any\n}\n\n// JSON serializes obj.\nfunc (c *Context) JSON(code int, obj any) {\n}\n\n// Default returns an Engine.\nfunc Default(opts ...Option) *Engine {\n\treturn nil\n}\n\nfunc internal() {}\n\ntype Handler interface {\n\t// Serve serves.\n\tServe(c *Context)\n}\n";
2010 let mut e = Vec::new();
2011 go_file(src, "context.go", "github.com/gin-gonic/gin", &mut e);
2012 let get = |p: &str| {
2013 e.iter()
2014 .find(|x| x.path == p)
2015 .unwrap_or_else(|| panic!("missing {p}: {:#?}", e.iter().map(|x| &x.path).collect::<Vec<_>>()))
2016 };
2017 assert_eq!(get("gin.Context.JSON").doc, "JSON serializes obj.");
2018 assert_eq!(get("gin.Default").sig, "func Default(opts ...Option) *Engine");
2019 assert_eq!(get("gin.Context").doc, "Context is the request context.");
2020 assert_eq!(get("gin.Handler.Serve").doc, "Serve serves.");
2021 assert!(e.iter().all(|x| x.name != "internal"));
2022 assert!(e.iter().any(|x| x.kind == Kind::Prose && x.doc.contains("web framework")));
2023 }
2024}