1#[derive(Debug, Clone, PartialEq)]
5pub struct Section {
6 pub heading: String,
8 pub line: u32,
10 pub text: String,
11 pub imports: Vec<String>,
13 pub parents: Vec<u32>,
15}
16
17const MAX_CHARS: usize = 1800;
18const RST_UNDERLINE: &[char] = &['=', '-', '~', '^', '*', '+', '#', '"', '`', '\''];
19
20fn is_underline(line: &str, prev: &str) -> Option<char> {
21 let t = line.trim_end();
22 let c = t.chars().next()?;
23 if !RST_UNDERLINE.contains(&c) || t.len() < 3 || !t.chars().all(|x| x == c) {
24 return None;
25 }
26 let p = prev.trim();
27 if p.is_empty() || p.len() > 120 || t.len() + 2 < p.chars().count() {
28 return None;
29 }
30 Some(c)
31}
32
33fn noise(line: &str) -> bool {
35 let t = line.trim();
36 (t.starts_with("[![") || t.starts_with("<img") || t.starts_with("<a href") && t.contains("<img"))
37 || (t.starts_with("<p") || t.starts_with("</p") || t.starts_with("<div") || t.starts_with("</div") || t.starts_with("<br")) && t.len() < 60
38 || t.starts_with("<!--") && t.ends_with("-->")
39}
40
41struct Scope {
42 level: usize,
43 title: String,
44 imports: Vec<String>,
45 line: Option<u32>,
46}
47
48pub(crate) fn import_base(import: &str) -> &str {
49 let skip = usize::from(import.starts_with('@'));
50 import.match_indices('/').nth(skip).map_or(import, |(i, _)| &import[..i])
51}
52
53fn example_imports(body: &str) -> Vec<String> {
55 fn walk(n: tree_sitter::Node, src: &[u8], out: &mut Vec<String>) {
56 let source = if matches!(n.kind(), "import_statement" | "export_statement") {
57 n.child_by_field_name("source")
58 } else if n.kind() == "call_expression"
59 && n.child_by_field_name("function")
60 .and_then(|f| f.utf8_text(src).ok())
61 .is_some_and(|f| f == "require" || f == "import")
62 {
63 n.child_by_field_name("arguments")
64 .and_then(|a| a.named_child(0))
65 .filter(|s| s.kind() == "string")
66 } else {
67 None
68 };
69 if let Some(source) = source {
70 if let Ok(s) = source.utf8_text(src) {
71 out.push(s.trim_matches(['\'', '"']).to_string());
72 }
73 }
74 for child in n.named_children(&mut n.walk()) {
75 walk(child, src, out);
76 }
77 }
78 let mut out = Vec::new();
79 let mut fence: Option<String> = None;
80 let mut code = String::new();
81 let mut javascript = false;
82 let mut parser = tree_sitter::Parser::new();
83 let _ = parser.set_language(&tree_sitter_typescript::LANGUAGE_TSX.into());
84 for line in body.lines() {
85 let t = line.trim_start();
86 if let Some(marker) = &fence {
87 if t.starts_with(marker) {
88 if javascript {
89 if let Some(tree) = parser.parse(&code, None) {
90 walk(tree.root_node(), code.as_bytes(), &mut out);
91 }
92 }
93 fence = None;
94 code.clear();
95 } else {
96 code.push_str(line);
97 code.push('\n');
98 }
99 } else if t.starts_with("```") || t.starts_with("~~~") {
100 fence = Some(t[..3].to_string());
101 let lang = t[3..].split_whitespace().next().unwrap_or("");
102 javascript = ["", "js", "jsx", "ts", "tsx", "javascript", "typescript", "mjs", "cjs"].contains(&lang);
103 }
104 }
105 if fence.is_some() && javascript {
106 if let Some(tree) = parser.parse(&code, None) {
107 walk(tree.root_node(), code.as_bytes(), &mut out);
108 }
109 }
110 out
111}
112
113pub fn split(text: &str, title: &str) -> Vec<Section> {
114 let lines: Vec<&str> = text.lines().collect();
115 let mut out: Vec<Section> = Vec::new();
116 let mut stack = vec![Scope {
117 level: 0,
118 title: title.to_string(),
119 imports: Vec::new(),
120 line: None,
121 }];
122 let mut rst_levels: Vec<char> = Vec::new();
123 let mut cur = String::new();
124 let mut cur_line = 1u32;
125 let mut fence: Option<String> = None;
126 let flush = |out: &mut Vec<Section>, cur: &mut String, stack: &mut Vec<Scope>, line: u32| {
127 let body = cur.trim();
128 if !body.is_empty() {
129 let own = example_imports(body);
130 let scope = stack.last_mut().unwrap();
131 for import in &own {
132 scope.imports.retain(|old| import_base(old) != import_base(import));
133 }
134 scope.imports.extend(own);
135 scope.imports.sort();
136 scope.imports.dedup();
137 let section = Section {
138 heading: stack.iter().map(|s| s.title.as_str()).collect::<Vec<_>>().join(" › "),
139 line,
140 text: body.to_string(),
141 imports: stack.last().unwrap().imports.clone(),
142 parents: stack[..stack.len() - 1].iter().filter_map(|s| s.line).collect(),
143 };
144 push_sized(out, §ion);
145 stack.last_mut().unwrap().line = Some(line);
146 }
147 cur.clear();
148 };
149 let is_md = lines.iter().any(|l| l.starts_with("# ") || l.starts_with("## "));
150 let mut i = 0;
151 while i < lines.len() {
152 let line = lines[i];
153 let t = line.trim_start();
154 if let Some(f) = &fence {
155 if t.starts_with(f.as_str()) {
156 fence = None;
157 }
158 cur.push_str(line);
159 cur.push('\n');
160 i += 1;
161 continue;
162 }
163 if t.starts_with("```") || t.starts_with("~~~") {
164 fence = Some(t[..3].to_string());
165 cur.push_str(line);
166 cur.push('\n');
167 i += 1;
168 continue;
169 }
170 if line.starts_with('#') {
172 let level = line.chars().take_while(|c| *c == '#').count();
173 let rest = &line[level..];
174 if level <= 6 && (rest.is_empty() || rest.starts_with(' ')) {
175 let h = rest.trim().trim_end_matches('#').trim().to_string();
176 flush(&mut out, &mut cur, &mut stack, cur_line);
177 stack.retain(|s| s.level < level);
178 stack.push(Scope {
179 level,
180 title: clean_heading(&h),
181 imports: stack.last().unwrap().imports.clone(),
182 line: None,
183 });
184 cur_line = i as u32 + 1;
185 i += 1;
186 continue;
187 }
188 }
189 if i + 1 < lines.len() && !line.trim().is_empty() {
191 let under = is_underline(lines[i + 1], line).filter(|c| !is_md || *c == '=' || *c == '-');
192 if let Some(c) = under {
193 let level = if is_md {
194 if c == '=' {
195 1
196 } else {
197 2
198 }
199 } else {
200 match rst_levels.iter().position(|x| *x == c) {
201 Some(p) => p + 1,
202 None => {
203 rst_levels.push(c);
204 rst_levels.len()
205 }
206 }
207 };
208 flush(&mut out, &mut cur, &mut stack, cur_line);
209 stack.retain(|s| s.level < level);
210 stack.push(Scope {
211 level,
212 title: clean_heading(line.trim()),
213 imports: stack.last().unwrap().imports.clone(),
214 line: None,
215 });
216 cur_line = i as u32 + 1;
217 i += 2;
218 continue;
219 }
220 }
221 if !noise(line) {
222 cur.push_str(line);
223 cur.push('\n');
224 }
225 i += 1;
226 }
227 flush(&mut out, &mut cur, &mut stack, cur_line);
228 out
229}
230
231pub fn generic_heading(h: &str) -> bool {
234 let h = h.trim().trim_end_matches(':').to_ascii_lowercase();
235 [
236 "parameters",
237 "parameter",
238 "params",
239 "arguments",
240 "args",
241 "returns",
242 "return value",
243 "return type",
244 "props",
245 "options",
246 "examples",
247 "example",
248 "usage",
249 "reference",
250 "overview",
251 "notes",
252 "caveats",
253 "good to know",
254 "type",
255 "types",
256 "description",
257 "syntax",
258 "signature",
259 "see also",
260 "troubleshooting",
261 "introduction",
262 "summary",
263 "details",
264 ]
265 .contains(&h.as_str())
266}
267
268pub fn topic_heading(name: &str) -> &str {
271 let mut it = name.rsplit(" › ");
272 let last = it.next().unwrap_or("");
273 if generic_heading(last) {
274 it.next().unwrap_or(last)
275 } else {
276 last
277 }
278}
279
280fn clean_heading(h: &str) -> String {
281 let mut h = h.to_string();
283 for (open, close) in [("{/*", "*/}"), ("{#", "}")] {
284 while let Some(i) = h.find(open) {
285 match h[i..].find(close) {
286 Some(j) => h.replace_range(i..i + j + close.len(), ""),
287 None => break,
288 }
289 }
290 }
291 let h = h.as_str();
292 let mut s = String::new();
294 let mut in_tag = false;
295 let mut in_url = false;
296 let mut prev = ' ';
297 for c in h.chars() {
298 match c {
299 '<' => in_tag = true,
300 '>' if in_tag => in_tag = false,
301 '(' if prev == ']' => in_url = true,
302 ')' if in_url => in_url = false,
303 '[' | ']' | '`' | '*' => {}
304 _ if in_tag || in_url => {}
305 _ => s.push(c),
306 }
307 prev = c;
308 }
309 s.trim().to_string()
310}
311
312#[allow(clippy::explicit_counter_loop)]
314fn push_sized(out: &mut Vec<Section>, section: &Section) {
315 let body = §ion.text;
316 let line = section.line;
317 if body.len() <= MAX_CHARS {
318 out.push(Section {
319 heading: section.heading.clone(),
320 line,
321 text: body.to_string(),
322 imports: section.imports.clone(),
323 parents: section.parents.clone(),
324 });
325 return;
326 }
327 let mut buf = String::new();
328 let mut start = line;
329 let mut ln = line;
330 let mut in_fence = false;
331 for l in body.lines() {
332 let t = l.trim_start();
333 if t.starts_with("```") || t.starts_with("~~~") {
334 in_fence = !in_fence;
335 }
336 if !in_fence && l.trim().is_empty() && buf.len() > MAX_CHARS * 2 / 3 {
337 out.push(Section {
338 heading: section.heading.clone(),
339 line: start,
340 text: buf.trim().to_string(),
341 imports: section.imports.clone(),
342 parents: section.parents.clone(),
343 });
344 buf.clear();
345 start = ln + 1;
346 } else if buf.len() > MAX_CHARS * 3 {
347 out.push(Section {
349 heading: section.heading.clone(),
350 line: start,
351 text: buf.trim().to_string(),
352 imports: section.imports.clone(),
353 parents: section.parents.clone(),
354 });
355 buf.clear();
356 start = ln;
357 }
358 buf.push_str(l);
359 buf.push('\n');
360 ln += 1;
361 }
362 if !buf.trim().is_empty() {
363 out.push(Section {
364 heading: section.heading.clone(),
365 line: start,
366 text: buf.trim().to_string(),
367 imports: section.imports.clone(),
368 parents: section.parents.clone(),
369 });
370 }
371}
372
373pub fn clean_mdx(text: &str, mdx: bool) -> (String, Option<String>) {
379 let lines: Vec<&str> = text.lines().collect();
380 let mut out: Vec<String> = Vec::with_capacity(lines.len());
381 let mut title = None;
382 let mut description = None;
383 let mut i = 0;
384 if lines.first().map(|l| l.trim_end()) == Some("---") {
385 if let Some(end) = lines.iter().skip(1).position(|l| l.trim_end() == "---") {
386 for l in &lines[1..=end] {
387 if let Some(t) = l.strip_prefix("title:") {
388 title = Some(t.trim().trim_matches(['"', '\'']).to_string()).filter(|t| !t.is_empty());
389 }
390 if let Some(t) = l.strip_prefix("description:") {
391 description = Some(t.trim().trim_matches(['"', '\'']).to_string()).filter(|t| !t.is_empty());
392 }
393 }
394 out.push(description.take().unwrap_or_default());
396 for _ in 0..end + 1 {
397 out.push(String::new());
398 }
399 i = end + 2;
400 }
401 }
402 let mut fence = false;
403 let mut j = i.min(lines.len());
404 while j < lines.len() {
405 let l = lines[j];
406 let t = l.trim();
407 if t.starts_with("```") || t.starts_with("~~~") {
408 fence = !fence;
409 }
410 if !fence {
411 if let Some((level, text, used)) = html_heading(&lines[j..]) {
412 out.push(format!("{} {text}", "#".repeat(level)));
413 out.extend(std::iter::repeat_n(String::new(), used - 1));
414 j += used;
415 continue;
416 }
417 }
418 if mdx && !fence {
419 if let Some(v) = export_const(t, "title") {
420 title = title.or(Some(v));
421 out.push(String::new());
422 j += 1;
423 continue;
424 }
425 if let Some(v) = export_const(t, "description") {
426 out.push(v);
427 j += 1;
428 continue;
429 }
430 }
431 let drop = mdx
432 && !fence
433 && (t.starts_with("import ")
434 || t.starts_with("export ")
435 || (t.starts_with('<') && t.ends_with('>') && !t.starts_with("<!--") && !t.contains("</code>")));
436 out.push(if drop { String::new() } else { l.to_string() });
437 j += 1;
438 }
439 (out.join("\n"), title)
440}
441
442fn export_const(line: &str, name: &str) -> Option<String> {
444 let rest = line.strip_prefix("export const ")?.strip_prefix(name)?.trim_start().strip_prefix('=')?.trim();
445 let rest = rest.trim_end_matches(';').trim();
446 let q = rest.chars().next().filter(|c| matches!(c, '"' | '\'' | '`'))?;
447 let v = rest.strip_prefix(q)?.strip_suffix(q)?;
448 (!v.is_empty()).then(|| v.to_string())
449}
450
451fn html_heading(lines: &[&str]) -> Option<(usize, String, usize)> {
454 let t = lines.first()?.trim_start();
455 let b = t.as_bytes();
456 if b.len() < 4 || b[0] != b'<' || !matches!(b[1], b'h' | b'H') || !(b'1'..=b'6').contains(&b[2]) || !matches!(b[3], b'>' | b' ') {
457 return None;
458 }
459 let level = (b[2] - b'0') as usize;
460 let close = format!("</h{level}>");
461 let mut joined = String::new();
462 for (n, l) in lines.iter().take(4).enumerate() {
463 joined.push_str(l.trim());
464 joined.push(' ');
465 if l.to_ascii_lowercase().contains(&close) {
466 let text = clean_heading(&joined);
467 return (!text.is_empty()).then_some((level, text, n + 1));
468 }
469 }
470 None
471}
472
473pub fn metadata_body(text: &str) -> (String, u32) {
476 let mut line = 1u32;
477 let mut rest = text;
478 while let Some(i) = rest.find('\n') {
479 let l = &rest[..i];
480 rest = &rest[i + 1..];
481 line += 1;
482 if l.trim().is_empty() {
483 return (rest.to_string(), line);
484 }
485 }
486 (String::new(), line)
488}
489
490#[cfg(test)]
491mod tests {
492 use super::*;
493
494 #[test]
495 fn imports_and_ancestry_follow_logical_sections_before_chunking() {
496 let md = format!("# Guide\n\nCommon introduction.\n\n## Setup\n\n```ts\nimport {{ build }} from '@sample/forms/next';\n```\n\n{}\n\n### Recipe\n\nInherited recipe.\n\n## Setup\n\nUnrelated setup.\n\n### Recipe\n\nIndependent recipe.\n", ("Long paragraph. ".repeat(100) + "\n\n").repeat(3));
497 let sections = split(&md, "README");
498 let chunks: Vec<_> = sections
499 .iter()
500 .filter(|s| s.heading.ends_with("Setup") && !s.text.contains("Unrelated"))
501 .collect();
502 assert!(chunks.len() > 1);
503 assert!(chunks.iter().all(|s| s.imports == ["@sample/forms/next"]));
504 let inherited = sections.iter().find(|s| s.text == "Inherited recipe.").unwrap();
505 assert_eq!(inherited.imports, ["@sample/forms/next"]);
506 assert!(inherited.parents.contains(&chunks[0].line));
507 let independent = sections.iter().find(|s| s.text == "Independent recipe.").unwrap();
508 assert!(independent.imports.is_empty());
509 assert!(!independent.parents.contains(&chunks[0].line));
510 }
511
512 #[test]
513 fn imports_come_from_examples_and_override_only_their_package() {
514 let md = "import X from 'component/plumbing';\n\n# Setup\n\n```js\nconst f = require('@sample/forms/next');\nconst other = import('other/setup');\n```\n\n## Override\n\n```ts\nimport { build } from '@sample/forms';\nexport { build } from '@sample/forms/unknown';\n// import x from 'ignored/comment';\nconst label = \"import x from 'ignored/string'\";\n```\n\n### Child\n\nAn inherited example.\n";
515 let sections = split(md, "README");
516 assert!(sections[0].imports.is_empty());
517 let child = sections.last().unwrap();
518 assert_eq!(child.imports, ["@sample/forms", "@sample/forms/unknown", "other/setup"]);
519 assert_eq!(example_imports("```ts\nimport f from 'unfinished/example';"), ["unfinished/example"]);
520 }
521
522 #[test]
523 fn sections_follow_headings_and_skip_fences() {
524 let md = "# Zod\n\nIntro.\n\n## Objects\n\n```ts\n# not a heading\nz.object({})\n```\n\n### strict\n\nText.\n\nSetext\n------\n\nMore.\n";
525 let s = split(md, "README");
526 let heads: Vec<&str> = s.iter().map(|x| x.heading.as_str()).collect();
527 assert_eq!(
528 heads,
529 vec![
530 "README › Zod",
531 "README › Zod › Objects",
532 "README › Zod › Objects › strict",
533 "README › Zod › Setext"
534 ]
535 );
536 assert!(s[1].text.contains("# not a heading"));
537 assert_eq!(s[1].line, 5);
538 }
539
540 #[test]
541 fn mdx_exports_and_html_headings() {
542 let mdx = "import { X } from \"x\";\n\nexport const title = \"Functions and directives\";\nexport const description = \"Custom directives.\";\n\n## Directives\n\nIntro.\n\n<h3 id=\"import-directive\">\n <a href=\"#import-directive\">@import</a>\n</h3>\n\nUse `@import`.\n\n<h3 id=\"theme\"><a href=\"#theme\">@theme</a></h3>\n\nTokens.\n";
543 let (text, title) = clean_mdx(mdx, true);
544 assert_eq!(title.as_deref(), Some("Functions and directives"));
545 assert_eq!(text.lines().count(), mdx.lines().count());
546 let s = split(&text, title.as_deref().unwrap());
547 let heads: Vec<&str> = s.iter().map(|x| x.heading.as_str()).collect();
548 assert_eq!(
549 heads,
550 vec![
551 "Functions and directives",
552 "Functions and directives › Directives",
553 "Functions and directives › Directives › @import",
554 "Functions and directives › Directives › @theme"
555 ]
556 );
557 assert!(s[0].text.contains("Custom directives."));
558 assert_eq!(s[2].line, 10);
559 }
560
561 #[test]
562 fn heading_ids_and_generic_headings() {
563 let s = split("# forwardRef\n\n## Usage {/*usage*/}\n\nText.\n\n### Props {#props}\n\nMore.\n", "doc");
564 assert_eq!(s[0].heading, "doc › forwardRef › Usage");
565 assert_eq!(topic_heading(&s[1].heading), "Usage");
566 assert_eq!(topic_heading("generateMetadata › Parameters"), "generateMetadata");
567 }
568
569 #[test]
570 fn rst_headings() {
571 let rst = "Title\n=====\n\nBody\n\nSub\n---\n\nx\n";
572 let s = split(rst, "doc");
573 assert_eq!(s.len(), 2);
574 assert_eq!(s[1].heading, "doc › Title › Sub");
575 }
576}