Skip to main content

lockdocs_core/
markdown.rs

1//! Split Markdown, reStructuredText and plain text into heading-scoped
2//! sections small enough to rank and quote.
3
4#[derive(Debug, Clone, PartialEq)]
5pub struct Section {
6    /// Heading path, e.g. `Basic usage › Objects`.
7    pub heading: String,
8    /// 1-based line of the section start.
9    pub line: u32,
10    pub text: String,
11}
12
13const MAX_CHARS: usize = 1800;
14const RST_UNDERLINE: &[char] = &['=', '-', '~', '^', '*', '+', '#', '"', '`', '\''];
15
16fn is_underline(line: &str, prev: &str) -> Option<char> {
17    let t = line.trim_end();
18    let c = t.chars().next()?;
19    if !RST_UNDERLINE.contains(&c) || t.len() < 3 || !t.chars().all(|x| x == c) {
20        return None;
21    }
22    let p = prev.trim();
23    if p.is_empty() || p.len() > 120 || t.len() + 2 < p.chars().count() {
24        return None;
25    }
26    Some(c)
27}
28
29/// Drop lines that are only badges or HTML wrappers: they cost tokens and carry no API.
30fn noise(line: &str) -> bool {
31    let t = line.trim();
32    (t.starts_with("[![") || t.starts_with("<img") || t.starts_with("<a href") && t.contains("<img"))
33        || (t.starts_with("<p") || t.starts_with("</p") || t.starts_with("<div") || t.starts_with("</div") || t.starts_with("<br")) && t.len() < 60
34        || t.starts_with("<!--") && t.ends_with("-->")
35}
36
37pub fn split(text: &str, title: &str) -> Vec<Section> {
38    let lines: Vec<&str> = text.lines().collect();
39    let mut out: Vec<Section> = Vec::new();
40    let mut stack: Vec<(usize, String)> = Vec::new();
41    let mut rst_levels: Vec<char> = Vec::new();
42    let mut cur = String::new();
43    let mut cur_line = 1u32;
44    let mut fence: Option<String> = None;
45    let heading = |stack: &Vec<(usize, String)>| {
46        let mut parts: Vec<&str> = vec![title];
47        parts.extend(stack.iter().map(|(_, h)| h.as_str()));
48        parts.join(" › ")
49    };
50    let flush = |out: &mut Vec<Section>, cur: &mut String, head: String, line: u32| {
51        let body = cur.trim();
52        if !body.is_empty() {
53            push_sized(out, &head, line, body);
54        }
55        cur.clear();
56    };
57    let is_md = lines.iter().any(|l| l.starts_with("# ") || l.starts_with("## "));
58    let mut i = 0;
59    while i < lines.len() {
60        let line = lines[i];
61        let t = line.trim_start();
62        if let Some(f) = &fence {
63            if t.starts_with(f.as_str()) {
64                fence = None;
65            }
66            cur.push_str(line);
67            cur.push('\n');
68            i += 1;
69            continue;
70        }
71        if t.starts_with("```") || t.starts_with("~~~") {
72            fence = Some(t[..3].to_string());
73            cur.push_str(line);
74            cur.push('\n');
75            i += 1;
76            continue;
77        }
78        // ATX heading
79        if line.starts_with('#') {
80            let level = line.chars().take_while(|c| *c == '#').count();
81            let rest = &line[level..];
82            if level <= 6 && (rest.is_empty() || rest.starts_with(' ')) {
83                let h = rest.trim().trim_end_matches('#').trim().to_string();
84                flush(&mut out, &mut cur, heading(&stack), cur_line);
85                stack.retain(|(l, _)| *l < level);
86                stack.push((level, clean_heading(&h)));
87                cur_line = i as u32 + 1;
88                i += 1;
89                continue;
90            }
91        }
92        // Setext / reST underline heading
93        if i + 1 < lines.len() && !line.trim().is_empty() {
94            let under = is_underline(lines[i + 1], line).filter(|c| !is_md || *c == '=' || *c == '-');
95            if let Some(c) = under {
96                let level = if is_md {
97                    if c == '=' {
98                        1
99                    } else {
100                        2
101                    }
102                } else {
103                    match rst_levels.iter().position(|x| *x == c) {
104                        Some(p) => p + 1,
105                        None => {
106                            rst_levels.push(c);
107                            rst_levels.len()
108                        }
109                    }
110                };
111                flush(&mut out, &mut cur, heading(&stack), cur_line);
112                stack.retain(|(l, _)| *l < level);
113                stack.push((level, clean_heading(line.trim())));
114                cur_line = i as u32 + 1;
115                i += 2;
116                continue;
117            }
118        }
119        if !noise(line) {
120            cur.push_str(line);
121            cur.push('\n');
122        }
123        i += 1;
124    }
125    flush(&mut out, &mut cur, heading(&stack), cur_line);
126    out
127}
128
129/// Headings that name a part of a page, not a topic (`Parameters`,
130/// `Returns`, `Examples`): the topic is the heading above them.
131pub fn generic_heading(h: &str) -> bool {
132    let h = h.trim().trim_end_matches(':').to_ascii_lowercase();
133    [
134        "parameters",
135        "parameter",
136        "params",
137        "arguments",
138        "args",
139        "returns",
140        "return value",
141        "return type",
142        "props",
143        "options",
144        "examples",
145        "example",
146        "usage",
147        "reference",
148        "overview",
149        "notes",
150        "caveats",
151        "good to know",
152        "type",
153        "types",
154        "description",
155        "syntax",
156        "signature",
157        "see also",
158        "troubleshooting",
159        "introduction",
160        "summary",
161        "details",
162    ]
163    .contains(&h.as_str())
164}
165
166/// The heading an entry is about: its last heading, or the one above it
167/// when the last is generic (`generateMetadata › Parameters` -> `generateMetadata`).
168pub fn topic_heading(name: &str) -> &str {
169    let mut it = name.rsplit(" › ");
170    let last = it.next().unwrap_or("");
171    if generic_heading(last) {
172        it.next().unwrap_or(last)
173    } else {
174        last
175    }
176}
177
178fn clean_heading(h: &str) -> String {
179    // MDX heading ids: `Usage {/*usage*/}`, `Usage {#usage}`.
180    let mut h = h.to_string();
181    for (open, close) in [("{/*", "*/}"), ("{#", "}")] {
182        while let Some(i) = h.find(open) {
183            match h[i..].find(close) {
184                Some(j) => h.replace_range(i..i + j + close.len(), ""),
185                None => break,
186            }
187        }
188    }
189    let h = h.as_str();
190    // `[text](url)` -> text, drop inline code ticks and HTML tags.
191    let mut s = String::new();
192    let mut in_tag = false;
193    let mut in_url = false;
194    let mut prev = ' ';
195    for c in h.chars() {
196        match c {
197            '<' => in_tag = true,
198            '>' if in_tag => in_tag = false,
199            '(' if prev == ']' => in_url = true,
200            ')' if in_url => in_url = false,
201            '[' | ']' | '`' | '*' => {}
202            _ if in_tag || in_url => {}
203            _ => s.push(c),
204        }
205        prev = c;
206    }
207    s.trim().to_string()
208}
209
210/// Split long sections at blank lines outside code fences.
211#[allow(clippy::explicit_counter_loop)]
212fn push_sized(out: &mut Vec<Section>, head: &str, line: u32, body: &str) {
213    if body.len() <= MAX_CHARS {
214        out.push(Section {
215            heading: head.to_string(),
216            line,
217            text: body.to_string(),
218        });
219        return;
220    }
221    let mut buf = String::new();
222    let mut start = line;
223    let mut ln = line;
224    let mut in_fence = false;
225    for l in body.lines() {
226        let t = l.trim_start();
227        if t.starts_with("```") || t.starts_with("~~~") {
228            in_fence = !in_fence;
229        }
230        if !in_fence && l.trim().is_empty() && buf.len() > MAX_CHARS * 2 / 3 {
231            out.push(Section {
232                heading: head.to_string(),
233                line: start,
234                text: buf.trim().to_string(),
235            });
236            buf.clear();
237            start = ln + 1;
238        } else if buf.len() > MAX_CHARS * 3 {
239            // A giant fence or paragraph: cut hard.
240            out.push(Section {
241                heading: head.to_string(),
242                line: start,
243                text: buf.trim().to_string(),
244            });
245            buf.clear();
246            start = ln;
247        }
248        buf.push_str(l);
249        buf.push('\n');
250        ln += 1;
251    }
252    if !buf.trim().is_empty() {
253        out.push(Section {
254            heading: head.to_string(),
255            line: start,
256            text: buf.trim().to_string(),
257        });
258    }
259}
260
261/// Blank out front matter (keeping its `title`), and for MDX the `import` /
262/// `export` lines and JSX-only lines. `export const title = "..."` names the
263/// page and `export const description` keeps its text. HTML headings
264/// (`<h3 id="x"><a href="#x">@import</a></h3>`, one or more lines) become
265/// Markdown headings so sections split at them. Line numbers are preserved.
266pub fn clean_mdx(text: &str, mdx: bool) -> (String, Option<String>) {
267    let lines: Vec<&str> = text.lines().collect();
268    let mut out: Vec<String> = Vec::with_capacity(lines.len());
269    let mut title = None;
270    let mut description = None;
271    let mut i = 0;
272    if lines.first().map(|l| l.trim_end()) == Some("---") {
273        if let Some(end) = lines.iter().skip(1).position(|l| l.trim_end() == "---") {
274            for l in &lines[1..=end] {
275                if let Some(t) = l.strip_prefix("title:") {
276                    title = Some(t.trim().trim_matches(['"', '\'']).to_string()).filter(|t| !t.is_empty());
277                }
278                if let Some(t) = l.strip_prefix("description:") {
279                    description = Some(t.trim().trim_matches(['"', '\'']).to_string()).filter(|t| !t.is_empty());
280                }
281            }
282            // Keep line numbers: the description takes the front matter's first line.
283            out.push(description.take().unwrap_or_default());
284            for _ in 0..end + 1 {
285                out.push(String::new());
286            }
287            i = end + 2;
288        }
289    }
290    let mut fence = false;
291    let mut j = i.min(lines.len());
292    while j < lines.len() {
293        let l = lines[j];
294        let t = l.trim();
295        if t.starts_with("```") || t.starts_with("~~~") {
296            fence = !fence;
297        }
298        if !fence {
299            if let Some((level, text, used)) = html_heading(&lines[j..]) {
300                out.push(format!("{} {text}", "#".repeat(level)));
301                out.extend(std::iter::repeat_n(String::new(), used - 1));
302                j += used;
303                continue;
304            }
305        }
306        if mdx && !fence {
307            if let Some(v) = export_const(t, "title") {
308                title = title.or(Some(v));
309                out.push(String::new());
310                j += 1;
311                continue;
312            }
313            if let Some(v) = export_const(t, "description") {
314                out.push(v);
315                j += 1;
316                continue;
317            }
318        }
319        let drop = mdx
320            && !fence
321            && (t.starts_with("import ")
322                || t.starts_with("export ")
323                || (t.starts_with('<') && t.ends_with('>') && !t.starts_with("<!--") && !t.contains("</code>")));
324        out.push(if drop { String::new() } else { l.to_string() });
325        j += 1;
326    }
327    (out.join("\n"), title)
328}
329
330/// `export const <name> = "value";` -> value.
331fn export_const(line: &str, name: &str) -> Option<String> {
332    let rest = line.strip_prefix("export const ")?.strip_prefix(name)?.trim_start().strip_prefix('=')?.trim();
333    let rest = rest.trim_end_matches(';').trim();
334    let q = rest.chars().next().filter(|c| matches!(c, '"' | '\'' | '`'))?;
335    let v = rest.strip_prefix(q)?.strip_suffix(q)?;
336    (!v.is_empty()).then(|| v.to_string())
337}
338
339/// An HTML heading starting at `lines[0]`, spanning up to 4 lines:
340/// (level, plain text, lines used).
341fn html_heading(lines: &[&str]) -> Option<(usize, String, usize)> {
342    let t = lines.first()?.trim_start();
343    let b = t.as_bytes();
344    if b.len() < 4 || b[0] != b'<' || !matches!(b[1], b'h' | b'H') || !(b'1'..=b'6').contains(&b[2]) || !matches!(b[3], b'>' | b' ') {
345        return None;
346    }
347    let level = (b[2] - b'0') as usize;
348    let close = format!("</h{level}>");
349    let mut joined = String::new();
350    for (n, l) in lines.iter().take(4).enumerate() {
351        joined.push_str(l.trim());
352        joined.push(' ');
353        if l.to_ascii_lowercase().contains(&close) {
354            let text = clean_heading(&joined);
355            return (!text.is_empty()).then_some((level, text, n + 1));
356        }
357    }
358    None
359}
360
361/// The body of a Python METADATA file (after the RFC 822 headers), which is
362/// the package README.
363pub fn metadata_body(text: &str) -> (String, u32) {
364    let mut line = 1u32;
365    let mut rest = text;
366    while let Some(i) = rest.find('\n') {
367        let l = &rest[..i];
368        rest = &rest[i + 1..];
369        line += 1;
370        if l.trim().is_empty() {
371            return (rest.to_string(), line);
372        }
373    }
374    // Older metadata keeps the README in a Description: header.
375    (String::new(), line)
376}
377
378#[cfg(test)]
379mod tests {
380    use super::*;
381
382    #[test]
383    fn sections_follow_headings_and_skip_fences() {
384        let md = "# Zod\n\nIntro.\n\n## Objects\n\n```ts\n# not a heading\nz.object({})\n```\n\n### strict\n\nText.\n\nSetext\n------\n\nMore.\n";
385        let s = split(md, "README");
386        let heads: Vec<&str> = s.iter().map(|x| x.heading.as_str()).collect();
387        assert_eq!(
388            heads,
389            vec![
390                "README › Zod",
391                "README › Zod › Objects",
392                "README › Zod › Objects › strict",
393                "README › Zod › Setext"
394            ]
395        );
396        assert!(s[1].text.contains("# not a heading"));
397        assert_eq!(s[1].line, 5);
398    }
399
400    #[test]
401    fn mdx_exports_and_html_headings() {
402        let mdx = "import { X } from \"x\";\n\nexport const title = \"Functions and directives\";\nexport const description = \"Custom directives.\";\n\n## Directives\n\nIntro.\n\n<h3 id=\"import-directive\">\n  <a href=\"#import-directive\">@import</a>\n</h3>\n\nUse `@import`.\n\n<h3 id=\"theme\"><a href=\"#theme\">@theme</a></h3>\n\nTokens.\n";
403        let (text, title) = clean_mdx(mdx, true);
404        assert_eq!(title.as_deref(), Some("Functions and directives"));
405        assert_eq!(text.lines().count(), mdx.lines().count());
406        let s = split(&text, title.as_deref().unwrap());
407        let heads: Vec<&str> = s.iter().map(|x| x.heading.as_str()).collect();
408        assert_eq!(
409            heads,
410            vec![
411                "Functions and directives",
412                "Functions and directives › Directives",
413                "Functions and directives › Directives › @import",
414                "Functions and directives › Directives › @theme"
415            ]
416        );
417        assert!(s[0].text.contains("Custom directives."));
418        assert_eq!(s[2].line, 10);
419    }
420
421    #[test]
422    fn heading_ids_and_generic_headings() {
423        let s = split("# forwardRef\n\n## Usage {/*usage*/}\n\nText.\n\n### Props {#props}\n\nMore.\n", "doc");
424        assert_eq!(s[0].heading, "doc › forwardRef › Usage");
425        assert_eq!(topic_heading(&s[1].heading), "Usage");
426        assert_eq!(topic_heading("generateMetadata › Parameters"), "generateMetadata");
427    }
428
429    #[test]
430    fn rst_headings() {
431        let rst = "Title\n=====\n\nBody\n\nSub\n---\n\nx\n";
432        let s = split(rst, "doc");
433        assert_eq!(s.len(), 2);
434        assert_eq!(s[1].heading, "doc › Title › Sub");
435    }
436}