1#[derive(Debug, Clone, PartialEq)]
5pub struct Section {
6 pub heading: String,
8 pub line: u32,
10 pub text: String,
11}
12
13const MAX_CHARS: usize = 1800;
14const RST_UNDERLINE: &[char] = &['=', '-', '~', '^', '*', '+', '#', '"', '`', '\''];
15
16fn is_underline(line: &str, prev: &str) -> Option<char> {
17 let t = line.trim_end();
18 let c = t.chars().next()?;
19 if !RST_UNDERLINE.contains(&c) || t.len() < 3 || !t.chars().all(|x| x == c) {
20 return None;
21 }
22 let p = prev.trim();
23 if p.is_empty() || p.len() > 120 || t.len() + 2 < p.chars().count() {
24 return None;
25 }
26 Some(c)
27}
28
29fn noise(line: &str) -> bool {
31 let t = line.trim();
32 (t.starts_with("[![") || t.starts_with("<img") || t.starts_with("<a href") && t.contains("<img"))
33 || (t.starts_with("<p") || t.starts_with("</p") || t.starts_with("<div") || t.starts_with("</div") || t.starts_with("<br")) && t.len() < 60
34 || t.starts_with("<!--") && t.ends_with("-->")
35}
36
37pub fn split(text: &str, title: &str) -> Vec<Section> {
38 let lines: Vec<&str> = text.lines().collect();
39 let mut out: Vec<Section> = Vec::new();
40 let mut stack: Vec<(usize, String)> = Vec::new();
41 let mut rst_levels: Vec<char> = Vec::new();
42 let mut cur = String::new();
43 let mut cur_line = 1u32;
44 let mut fence: Option<String> = None;
45 let heading = |stack: &Vec<(usize, String)>| {
46 let mut parts: Vec<&str> = vec![title];
47 parts.extend(stack.iter().map(|(_, h)| h.as_str()));
48 parts.join(" › ")
49 };
50 let flush = |out: &mut Vec<Section>, cur: &mut String, head: String, line: u32| {
51 let body = cur.trim();
52 if !body.is_empty() {
53 push_sized(out, &head, line, body);
54 }
55 cur.clear();
56 };
57 let is_md = lines.iter().any(|l| l.starts_with("# ") || l.starts_with("## "));
58 let mut i = 0;
59 while i < lines.len() {
60 let line = lines[i];
61 let t = line.trim_start();
62 if let Some(f) = &fence {
63 if t.starts_with(f.as_str()) {
64 fence = None;
65 }
66 cur.push_str(line);
67 cur.push('\n');
68 i += 1;
69 continue;
70 }
71 if t.starts_with("```") || t.starts_with("~~~") {
72 fence = Some(t[..3].to_string());
73 cur.push_str(line);
74 cur.push('\n');
75 i += 1;
76 continue;
77 }
78 if line.starts_with('#') {
80 let level = line.chars().take_while(|c| *c == '#').count();
81 let rest = &line[level..];
82 if level <= 6 && (rest.is_empty() || rest.starts_with(' ')) {
83 let h = rest.trim().trim_end_matches('#').trim().to_string();
84 flush(&mut out, &mut cur, heading(&stack), cur_line);
85 stack.retain(|(l, _)| *l < level);
86 stack.push((level, clean_heading(&h)));
87 cur_line = i as u32 + 1;
88 i += 1;
89 continue;
90 }
91 }
92 if i + 1 < lines.len() && !line.trim().is_empty() {
94 let under = is_underline(lines[i + 1], line).filter(|c| !is_md || *c == '=' || *c == '-');
95 if let Some(c) = under {
96 let level = if is_md {
97 if c == '=' {
98 1
99 } else {
100 2
101 }
102 } else {
103 match rst_levels.iter().position(|x| *x == c) {
104 Some(p) => p + 1,
105 None => {
106 rst_levels.push(c);
107 rst_levels.len()
108 }
109 }
110 };
111 flush(&mut out, &mut cur, heading(&stack), cur_line);
112 stack.retain(|(l, _)| *l < level);
113 stack.push((level, clean_heading(line.trim())));
114 cur_line = i as u32 + 1;
115 i += 2;
116 continue;
117 }
118 }
119 if !noise(line) {
120 cur.push_str(line);
121 cur.push('\n');
122 }
123 i += 1;
124 }
125 flush(&mut out, &mut cur, heading(&stack), cur_line);
126 out
127}
128
129pub fn generic_heading(h: &str) -> bool {
132 let h = h.trim().trim_end_matches(':').to_ascii_lowercase();
133 [
134 "parameters",
135 "parameter",
136 "params",
137 "arguments",
138 "args",
139 "returns",
140 "return value",
141 "return type",
142 "props",
143 "options",
144 "examples",
145 "example",
146 "usage",
147 "reference",
148 "overview",
149 "notes",
150 "caveats",
151 "good to know",
152 "type",
153 "types",
154 "description",
155 "syntax",
156 "signature",
157 "see also",
158 "troubleshooting",
159 "introduction",
160 "summary",
161 "details",
162 ]
163 .contains(&h.as_str())
164}
165
166pub fn topic_heading(name: &str) -> &str {
169 let mut it = name.rsplit(" › ");
170 let last = it.next().unwrap_or("");
171 if generic_heading(last) {
172 it.next().unwrap_or(last)
173 } else {
174 last
175 }
176}
177
178fn clean_heading(h: &str) -> String {
179 let mut h = h.to_string();
181 for (open, close) in [("{/*", "*/}"), ("{#", "}")] {
182 while let Some(i) = h.find(open) {
183 match h[i..].find(close) {
184 Some(j) => h.replace_range(i..i + j + close.len(), ""),
185 None => break,
186 }
187 }
188 }
189 let h = h.as_str();
190 let mut s = String::new();
192 let mut in_tag = false;
193 let mut in_url = false;
194 let mut prev = ' ';
195 for c in h.chars() {
196 match c {
197 '<' => in_tag = true,
198 '>' if in_tag => in_tag = false,
199 '(' if prev == ']' => in_url = true,
200 ')' if in_url => in_url = false,
201 '[' | ']' | '`' | '*' => {}
202 _ if in_tag || in_url => {}
203 _ => s.push(c),
204 }
205 prev = c;
206 }
207 s.trim().to_string()
208}
209
210#[allow(clippy::explicit_counter_loop)]
212fn push_sized(out: &mut Vec<Section>, head: &str, line: u32, body: &str) {
213 if body.len() <= MAX_CHARS {
214 out.push(Section {
215 heading: head.to_string(),
216 line,
217 text: body.to_string(),
218 });
219 return;
220 }
221 let mut buf = String::new();
222 let mut start = line;
223 let mut ln = line;
224 let mut in_fence = false;
225 for l in body.lines() {
226 let t = l.trim_start();
227 if t.starts_with("```") || t.starts_with("~~~") {
228 in_fence = !in_fence;
229 }
230 if !in_fence && l.trim().is_empty() && buf.len() > MAX_CHARS * 2 / 3 {
231 out.push(Section {
232 heading: head.to_string(),
233 line: start,
234 text: buf.trim().to_string(),
235 });
236 buf.clear();
237 start = ln + 1;
238 } else if buf.len() > MAX_CHARS * 3 {
239 out.push(Section {
241 heading: head.to_string(),
242 line: start,
243 text: buf.trim().to_string(),
244 });
245 buf.clear();
246 start = ln;
247 }
248 buf.push_str(l);
249 buf.push('\n');
250 ln += 1;
251 }
252 if !buf.trim().is_empty() {
253 out.push(Section {
254 heading: head.to_string(),
255 line: start,
256 text: buf.trim().to_string(),
257 });
258 }
259}
260
261pub fn clean_mdx(text: &str, mdx: bool) -> (String, Option<String>) {
267 let lines: Vec<&str> = text.lines().collect();
268 let mut out: Vec<String> = Vec::with_capacity(lines.len());
269 let mut title = None;
270 let mut description = None;
271 let mut i = 0;
272 if lines.first().map(|l| l.trim_end()) == Some("---") {
273 if let Some(end) = lines.iter().skip(1).position(|l| l.trim_end() == "---") {
274 for l in &lines[1..=end] {
275 if let Some(t) = l.strip_prefix("title:") {
276 title = Some(t.trim().trim_matches(['"', '\'']).to_string()).filter(|t| !t.is_empty());
277 }
278 if let Some(t) = l.strip_prefix("description:") {
279 description = Some(t.trim().trim_matches(['"', '\'']).to_string()).filter(|t| !t.is_empty());
280 }
281 }
282 out.push(description.take().unwrap_or_default());
284 for _ in 0..end + 1 {
285 out.push(String::new());
286 }
287 i = end + 2;
288 }
289 }
290 let mut fence = false;
291 let mut j = i.min(lines.len());
292 while j < lines.len() {
293 let l = lines[j];
294 let t = l.trim();
295 if t.starts_with("```") || t.starts_with("~~~") {
296 fence = !fence;
297 }
298 if !fence {
299 if let Some((level, text, used)) = html_heading(&lines[j..]) {
300 out.push(format!("{} {text}", "#".repeat(level)));
301 out.extend(std::iter::repeat_n(String::new(), used - 1));
302 j += used;
303 continue;
304 }
305 }
306 if mdx && !fence {
307 if let Some(v) = export_const(t, "title") {
308 title = title.or(Some(v));
309 out.push(String::new());
310 j += 1;
311 continue;
312 }
313 if let Some(v) = export_const(t, "description") {
314 out.push(v);
315 j += 1;
316 continue;
317 }
318 }
319 let drop = mdx
320 && !fence
321 && (t.starts_with("import ")
322 || t.starts_with("export ")
323 || (t.starts_with('<') && t.ends_with('>') && !t.starts_with("<!--") && !t.contains("</code>")));
324 out.push(if drop { String::new() } else { l.to_string() });
325 j += 1;
326 }
327 (out.join("\n"), title)
328}
329
330fn export_const(line: &str, name: &str) -> Option<String> {
332 let rest = line.strip_prefix("export const ")?.strip_prefix(name)?.trim_start().strip_prefix('=')?.trim();
333 let rest = rest.trim_end_matches(';').trim();
334 let q = rest.chars().next().filter(|c| matches!(c, '"' | '\'' | '`'))?;
335 let v = rest.strip_prefix(q)?.strip_suffix(q)?;
336 (!v.is_empty()).then(|| v.to_string())
337}
338
339fn html_heading(lines: &[&str]) -> Option<(usize, String, usize)> {
342 let t = lines.first()?.trim_start();
343 let b = t.as_bytes();
344 if b.len() < 4 || b[0] != b'<' || !matches!(b[1], b'h' | b'H') || !(b'1'..=b'6').contains(&b[2]) || !matches!(b[3], b'>' | b' ') {
345 return None;
346 }
347 let level = (b[2] - b'0') as usize;
348 let close = format!("</h{level}>");
349 let mut joined = String::new();
350 for (n, l) in lines.iter().take(4).enumerate() {
351 joined.push_str(l.trim());
352 joined.push(' ');
353 if l.to_ascii_lowercase().contains(&close) {
354 let text = clean_heading(&joined);
355 return (!text.is_empty()).then_some((level, text, n + 1));
356 }
357 }
358 None
359}
360
361pub fn metadata_body(text: &str) -> (String, u32) {
364 let mut line = 1u32;
365 let mut rest = text;
366 while let Some(i) = rest.find('\n') {
367 let l = &rest[..i];
368 rest = &rest[i + 1..];
369 line += 1;
370 if l.trim().is_empty() {
371 return (rest.to_string(), line);
372 }
373 }
374 (String::new(), line)
376}
377
378#[cfg(test)]
379mod tests {
380 use super::*;
381
382 #[test]
383 fn sections_follow_headings_and_skip_fences() {
384 let md = "# Zod\n\nIntro.\n\n## Objects\n\n```ts\n# not a heading\nz.object({})\n```\n\n### strict\n\nText.\n\nSetext\n------\n\nMore.\n";
385 let s = split(md, "README");
386 let heads: Vec<&str> = s.iter().map(|x| x.heading.as_str()).collect();
387 assert_eq!(
388 heads,
389 vec![
390 "README › Zod",
391 "README › Zod › Objects",
392 "README › Zod › Objects › strict",
393 "README › Zod › Setext"
394 ]
395 );
396 assert!(s[1].text.contains("# not a heading"));
397 assert_eq!(s[1].line, 5);
398 }
399
400 #[test]
401 fn mdx_exports_and_html_headings() {
402 let mdx = "import { X } from \"x\";\n\nexport const title = \"Functions and directives\";\nexport const description = \"Custom directives.\";\n\n## Directives\n\nIntro.\n\n<h3 id=\"import-directive\">\n <a href=\"#import-directive\">@import</a>\n</h3>\n\nUse `@import`.\n\n<h3 id=\"theme\"><a href=\"#theme\">@theme</a></h3>\n\nTokens.\n";
403 let (text, title) = clean_mdx(mdx, true);
404 assert_eq!(title.as_deref(), Some("Functions and directives"));
405 assert_eq!(text.lines().count(), mdx.lines().count());
406 let s = split(&text, title.as_deref().unwrap());
407 let heads: Vec<&str> = s.iter().map(|x| x.heading.as_str()).collect();
408 assert_eq!(
409 heads,
410 vec![
411 "Functions and directives",
412 "Functions and directives › Directives",
413 "Functions and directives › Directives › @import",
414 "Functions and directives › Directives › @theme"
415 ]
416 );
417 assert!(s[0].text.contains("Custom directives."));
418 assert_eq!(s[2].line, 10);
419 }
420
421 #[test]
422 fn heading_ids_and_generic_headings() {
423 let s = split("# forwardRef\n\n## Usage {/*usage*/}\n\nText.\n\n### Props {#props}\n\nMore.\n", "doc");
424 assert_eq!(s[0].heading, "doc › forwardRef › Usage");
425 assert_eq!(topic_heading(&s[1].heading), "Usage");
426 assert_eq!(topic_heading("generateMetadata › Parameters"), "generateMetadata");
427 }
428
429 #[test]
430 fn rst_headings() {
431 let rst = "Title\n=====\n\nBody\n\nSub\n---\n\nx\n";
432 let s = split(rst, "doc");
433 assert_eq!(s.len(), 2);
434 assert_eq!(s[1].heading, "doc › Title › Sub");
435 }
436}