Skip to main content

lockdocs_core/
markdown.rs

1//! Split Markdown, reStructuredText and plain text into heading-scoped
2//! sections small enough to rank and quote.
3
4#[derive(Debug, Clone, PartialEq)]
5pub struct Section {
6    /// Heading path, e.g. `Basic usage › Objects`.
7    pub heading: String,
8    /// 1-based line of the section start.
9    pub line: u32,
10    pub text: String,
11    /// Effective example imports, inherited by package before section chunking.
12    pub imports: Vec<String>,
13    /// Lines of nonempty ancestor sections in this page, outermost first.
14    pub parents: Vec<u32>,
15}
16
17const MAX_CHARS: usize = 1800;
18const RST_UNDERLINE: &[char] = &['=', '-', '~', '^', '*', '+', '#', '"', '`', '\''];
19
20fn is_underline(line: &str, prev: &str) -> Option<char> {
21    let t = line.trim_end();
22    let c = t.chars().next()?;
23    if !RST_UNDERLINE.contains(&c) || t.len() < 3 || !t.chars().all(|x| x == c) {
24        return None;
25    }
26    let p = prev.trim();
27    if p.is_empty() || p.len() > 120 || t.len() + 2 < p.chars().count() {
28        return None;
29    }
30    Some(c)
31}
32
33/// Drop lines that are only badges or HTML wrappers: they cost tokens and carry no API.
34fn noise(line: &str) -> bool {
35    let t = line.trim();
36    (t.starts_with("[![") || t.starts_with("<img") || t.starts_with("<a href") && t.contains("<img"))
37        || (t.starts_with("<p") || t.starts_with("</p") || t.starts_with("<div") || t.starts_with("</div") || t.starts_with("<br")) && t.len() < 60
38        || t.starts_with("<!--") && t.ends_with("-->")
39}
40
41struct Scope {
42    level: usize,
43    title: String,
44    imports: Vec<String>,
45    line: Option<u32>,
46}
47
48pub(crate) fn import_base(import: &str) -> &str {
49    let skip = usize::from(import.starts_with('@'));
50    import.match_indices('/').nth(skip).map_or(import, |(i, _)| &import[..i])
51}
52
53/// Read module specifiers from actual JS/TS examples, not prose or MDX imports.
54fn example_imports(body: &str) -> Vec<String> {
55    fn walk(n: tree_sitter::Node, src: &[u8], out: &mut Vec<String>) {
56        let source = if matches!(n.kind(), "import_statement" | "export_statement") {
57            n.child_by_field_name("source")
58        } else if n.kind() == "call_expression"
59            && n.child_by_field_name("function")
60                .and_then(|f| f.utf8_text(src).ok())
61                .is_some_and(|f| f == "require" || f == "import")
62        {
63            n.child_by_field_name("arguments")
64                .and_then(|a| a.named_child(0))
65                .filter(|s| s.kind() == "string")
66        } else {
67            None
68        };
69        if let Some(source) = source {
70            if let Ok(s) = source.utf8_text(src) {
71                out.push(s.trim_matches(['\'', '"']).to_string());
72            }
73        }
74        for child in n.named_children(&mut n.walk()) {
75            walk(child, src, out);
76        }
77    }
78    let mut out = Vec::new();
79    let mut fence: Option<String> = None;
80    let mut code = String::new();
81    let mut javascript = false;
82    let mut parser = tree_sitter::Parser::new();
83    let _ = parser.set_language(&tree_sitter_typescript::LANGUAGE_TSX.into());
84    for line in body.lines() {
85        let t = line.trim_start();
86        if let Some(marker) = &fence {
87            if t.starts_with(marker) {
88                if javascript {
89                    if let Some(tree) = parser.parse(&code, None) {
90                        walk(tree.root_node(), code.as_bytes(), &mut out);
91                    }
92                }
93                fence = None;
94                code.clear();
95            } else {
96                code.push_str(line);
97                code.push('\n');
98            }
99        } else if t.starts_with("```") || t.starts_with("~~~") {
100            fence = Some(t[..3].to_string());
101            let lang = t[3..].split_whitespace().next().unwrap_or("");
102            javascript = ["", "js", "jsx", "ts", "tsx", "javascript", "typescript", "mjs", "cjs"].contains(&lang);
103        }
104    }
105    if fence.is_some() && javascript {
106        if let Some(tree) = parser.parse(&code, None) {
107            walk(tree.root_node(), code.as_bytes(), &mut out);
108        }
109    }
110    out
111}
112
113pub fn split(text: &str, title: &str) -> Vec<Section> {
114    let lines: Vec<&str> = text.lines().collect();
115    let mut out: Vec<Section> = Vec::new();
116    let mut stack = vec![Scope {
117        level: 0,
118        title: title.to_string(),
119        imports: Vec::new(),
120        line: None,
121    }];
122    let mut rst_levels: Vec<char> = Vec::new();
123    let mut cur = String::new();
124    let mut cur_line = 1u32;
125    let mut fence: Option<String> = None;
126    let flush = |out: &mut Vec<Section>, cur: &mut String, stack: &mut Vec<Scope>, line: u32| {
127        let body = cur.trim();
128        if !body.is_empty() {
129            let own = example_imports(body);
130            let scope = stack.last_mut().unwrap();
131            for import in &own {
132                scope.imports.retain(|old| import_base(old) != import_base(import));
133            }
134            scope.imports.extend(own);
135            scope.imports.sort();
136            scope.imports.dedup();
137            let section = Section {
138                heading: stack.iter().map(|s| s.title.as_str()).collect::<Vec<_>>().join(" › "),
139                line,
140                text: body.to_string(),
141                imports: stack.last().unwrap().imports.clone(),
142                parents: stack[..stack.len() - 1].iter().filter_map(|s| s.line).collect(),
143            };
144            push_sized(out, &section);
145            stack.last_mut().unwrap().line = Some(line);
146        }
147        cur.clear();
148    };
149    let is_md = lines.iter().any(|l| l.starts_with("# ") || l.starts_with("## "));
150    let mut i = 0;
151    while i < lines.len() {
152        let line = lines[i];
153        let t = line.trim_start();
154        if let Some(f) = &fence {
155            if t.starts_with(f.as_str()) {
156                fence = None;
157            }
158            cur.push_str(line);
159            cur.push('\n');
160            i += 1;
161            continue;
162        }
163        if t.starts_with("```") || t.starts_with("~~~") {
164            fence = Some(t[..3].to_string());
165            cur.push_str(line);
166            cur.push('\n');
167            i += 1;
168            continue;
169        }
170        // ATX heading
171        if line.starts_with('#') {
172            let level = line.chars().take_while(|c| *c == '#').count();
173            let rest = &line[level..];
174            if level <= 6 && (rest.is_empty() || rest.starts_with(' ')) {
175                let h = rest.trim().trim_end_matches('#').trim().to_string();
176                flush(&mut out, &mut cur, &mut stack, cur_line);
177                stack.retain(|s| s.level < level);
178                stack.push(Scope {
179                    level,
180                    title: clean_heading(&h),
181                    imports: stack.last().unwrap().imports.clone(),
182                    line: None,
183                });
184                cur_line = i as u32 + 1;
185                i += 1;
186                continue;
187            }
188        }
189        // Setext / reST underline heading
190        if i + 1 < lines.len() && !line.trim().is_empty() {
191            let under = is_underline(lines[i + 1], line).filter(|c| !is_md || *c == '=' || *c == '-');
192            if let Some(c) = under {
193                let level = if is_md {
194                    if c == '=' {
195                        1
196                    } else {
197                        2
198                    }
199                } else {
200                    match rst_levels.iter().position(|x| *x == c) {
201                        Some(p) => p + 1,
202                        None => {
203                            rst_levels.push(c);
204                            rst_levels.len()
205                        }
206                    }
207                };
208                flush(&mut out, &mut cur, &mut stack, cur_line);
209                stack.retain(|s| s.level < level);
210                stack.push(Scope {
211                    level,
212                    title: clean_heading(line.trim()),
213                    imports: stack.last().unwrap().imports.clone(),
214                    line: None,
215                });
216                cur_line = i as u32 + 1;
217                i += 2;
218                continue;
219            }
220        }
221        if !noise(line) {
222            cur.push_str(line);
223            cur.push('\n');
224        }
225        i += 1;
226    }
227    flush(&mut out, &mut cur, &mut stack, cur_line);
228    out
229}
230
231/// Headings that name a part of a page, not a topic (`Parameters`,
232/// `Returns`, `Examples`): the topic is the heading above them.
233pub fn generic_heading(h: &str) -> bool {
234    let h = h.trim().trim_end_matches(':').to_ascii_lowercase();
235    [
236        "parameters",
237        "parameter",
238        "params",
239        "arguments",
240        "args",
241        "returns",
242        "return value",
243        "return type",
244        "props",
245        "options",
246        "examples",
247        "example",
248        "usage",
249        "reference",
250        "overview",
251        "notes",
252        "caveats",
253        "good to know",
254        "type",
255        "types",
256        "description",
257        "syntax",
258        "signature",
259        "see also",
260        "troubleshooting",
261        "introduction",
262        "summary",
263        "details",
264    ]
265    .contains(&h.as_str())
266}
267
268/// The heading an entry is about: its last heading, or the one above it
269/// when the last is generic (`generateMetadata › Parameters` -> `generateMetadata`).
270pub fn topic_heading(name: &str) -> &str {
271    let mut it = name.rsplit(" › ");
272    let last = it.next().unwrap_or("");
273    if generic_heading(last) {
274        it.next().unwrap_or(last)
275    } else {
276        last
277    }
278}
279
280fn clean_heading(h: &str) -> String {
281    // MDX heading ids: `Usage {/*usage*/}`, `Usage {#usage}`.
282    let mut h = h.to_string();
283    for (open, close) in [("{/*", "*/}"), ("{#", "}")] {
284        while let Some(i) = h.find(open) {
285            match h[i..].find(close) {
286                Some(j) => h.replace_range(i..i + j + close.len(), ""),
287                None => break,
288            }
289        }
290    }
291    let h = h.as_str();
292    // `[text](url)` -> text, drop inline code ticks and HTML tags.
293    let mut s = String::new();
294    let mut in_tag = false;
295    let mut in_url = false;
296    let mut prev = ' ';
297    for c in h.chars() {
298        match c {
299            '<' => in_tag = true,
300            '>' if in_tag => in_tag = false,
301            '(' if prev == ']' => in_url = true,
302            ')' if in_url => in_url = false,
303            '[' | ']' | '`' | '*' => {}
304            _ if in_tag || in_url => {}
305            _ => s.push(c),
306        }
307        prev = c;
308    }
309    s.trim().to_string()
310}
311
312/// Split long sections at blank lines outside code fences.
313#[allow(clippy::explicit_counter_loop)]
314fn push_sized(out: &mut Vec<Section>, section: &Section) {
315    let body = &section.text;
316    let line = section.line;
317    if body.len() <= MAX_CHARS {
318        out.push(Section {
319            heading: section.heading.clone(),
320            line,
321            text: body.to_string(),
322            imports: section.imports.clone(),
323            parents: section.parents.clone(),
324        });
325        return;
326    }
327    let mut buf = String::new();
328    let mut start = line;
329    let mut ln = line;
330    let mut in_fence = false;
331    for l in body.lines() {
332        let t = l.trim_start();
333        if t.starts_with("```") || t.starts_with("~~~") {
334            in_fence = !in_fence;
335        }
336        if !in_fence && l.trim().is_empty() && buf.len() > MAX_CHARS * 2 / 3 {
337            out.push(Section {
338                heading: section.heading.clone(),
339                line: start,
340                text: buf.trim().to_string(),
341                imports: section.imports.clone(),
342                parents: section.parents.clone(),
343            });
344            buf.clear();
345            start = ln + 1;
346        } else if buf.len() > MAX_CHARS * 3 {
347            // A giant fence or paragraph: cut hard.
348            out.push(Section {
349                heading: section.heading.clone(),
350                line: start,
351                text: buf.trim().to_string(),
352                imports: section.imports.clone(),
353                parents: section.parents.clone(),
354            });
355            buf.clear();
356            start = ln;
357        }
358        buf.push_str(l);
359        buf.push('\n');
360        ln += 1;
361    }
362    if !buf.trim().is_empty() {
363        out.push(Section {
364            heading: section.heading.clone(),
365            line: start,
366            text: buf.trim().to_string(),
367            imports: section.imports.clone(),
368            parents: section.parents.clone(),
369        });
370    }
371}
372
373/// Blank out front matter (keeping its `title`), and for MDX the `import` /
374/// `export` lines and JSX-only lines. `export const title = "..."` names the
375/// page and `export const description` keeps its text. HTML headings
376/// (`<h3 id="x"><a href="#x">@import</a></h3>`, one or more lines) become
377/// Markdown headings so sections split at them. Line numbers are preserved.
378pub fn clean_mdx(text: &str, mdx: bool) -> (String, Option<String>) {
379    let lines: Vec<&str> = text.lines().collect();
380    let mut out: Vec<String> = Vec::with_capacity(lines.len());
381    let mut title = None;
382    let mut description = None;
383    let mut i = 0;
384    if lines.first().map(|l| l.trim_end()) == Some("---") {
385        if let Some(end) = lines.iter().skip(1).position(|l| l.trim_end() == "---") {
386            for l in &lines[1..=end] {
387                if let Some(t) = l.strip_prefix("title:") {
388                    title = Some(t.trim().trim_matches(['"', '\'']).to_string()).filter(|t| !t.is_empty());
389                }
390                if let Some(t) = l.strip_prefix("description:") {
391                    description = Some(t.trim().trim_matches(['"', '\'']).to_string()).filter(|t| !t.is_empty());
392                }
393            }
394            // Keep line numbers: the description takes the front matter's first line.
395            out.push(description.take().unwrap_or_default());
396            for _ in 0..end + 1 {
397                out.push(String::new());
398            }
399            i = end + 2;
400        }
401    }
402    let mut fence = false;
403    let mut j = i.min(lines.len());
404    while j < lines.len() {
405        let l = lines[j];
406        let t = l.trim();
407        if t.starts_with("```") || t.starts_with("~~~") {
408            fence = !fence;
409        }
410        if !fence {
411            if let Some((level, text, used)) = html_heading(&lines[j..]) {
412                out.push(format!("{} {text}", "#".repeat(level)));
413                out.extend(std::iter::repeat_n(String::new(), used - 1));
414                j += used;
415                continue;
416            }
417        }
418        if mdx && !fence {
419            if let Some(v) = export_const(t, "title") {
420                title = title.or(Some(v));
421                out.push(String::new());
422                j += 1;
423                continue;
424            }
425            if let Some(v) = export_const(t, "description") {
426                out.push(v);
427                j += 1;
428                continue;
429            }
430        }
431        let drop = mdx
432            && !fence
433            && (t.starts_with("import ")
434                || t.starts_with("export ")
435                || (t.starts_with('<') && t.ends_with('>') && !t.starts_with("<!--") && !t.contains("</code>")));
436        out.push(if drop { String::new() } else { l.to_string() });
437        j += 1;
438    }
439    (out.join("\n"), title)
440}
441
442/// `export const <name> = "value";` -> value.
443fn export_const(line: &str, name: &str) -> Option<String> {
444    let rest = line.strip_prefix("export const ")?.strip_prefix(name)?.trim_start().strip_prefix('=')?.trim();
445    let rest = rest.trim_end_matches(';').trim();
446    let q = rest.chars().next().filter(|c| matches!(c, '"' | '\'' | '`'))?;
447    let v = rest.strip_prefix(q)?.strip_suffix(q)?;
448    (!v.is_empty()).then(|| v.to_string())
449}
450
451/// An HTML heading starting at `lines[0]`, spanning up to 4 lines:
452/// (level, plain text, lines used).
453fn html_heading(lines: &[&str]) -> Option<(usize, String, usize)> {
454    let t = lines.first()?.trim_start();
455    let b = t.as_bytes();
456    if b.len() < 4 || b[0] != b'<' || !matches!(b[1], b'h' | b'H') || !(b'1'..=b'6').contains(&b[2]) || !matches!(b[3], b'>' | b' ') {
457        return None;
458    }
459    let level = (b[2] - b'0') as usize;
460    let close = format!("</h{level}>");
461    let mut joined = String::new();
462    for (n, l) in lines.iter().take(4).enumerate() {
463        joined.push_str(l.trim());
464        joined.push(' ');
465        if l.to_ascii_lowercase().contains(&close) {
466            let text = clean_heading(&joined);
467            return (!text.is_empty()).then_some((level, text, n + 1));
468        }
469    }
470    None
471}
472
473/// The body of a Python METADATA file (after the RFC 822 headers), which is
474/// the package README.
475pub fn metadata_body(text: &str) -> (String, u32) {
476    let mut line = 1u32;
477    let mut rest = text;
478    while let Some(i) = rest.find('\n') {
479        let l = &rest[..i];
480        rest = &rest[i + 1..];
481        line += 1;
482        if l.trim().is_empty() {
483            return (rest.to_string(), line);
484        }
485    }
486    // Older metadata keeps the README in a Description: header.
487    (String::new(), line)
488}
489
490#[cfg(test)]
491mod tests {
492    use super::*;
493
494    #[test]
495    fn imports_and_ancestry_follow_logical_sections_before_chunking() {
496        let md = format!("# Guide\n\nCommon introduction.\n\n## Setup\n\n```ts\nimport {{ build }} from '@sample/forms/next';\n```\n\n{}\n\n### Recipe\n\nInherited recipe.\n\n## Setup\n\nUnrelated setup.\n\n### Recipe\n\nIndependent recipe.\n", ("Long paragraph. ".repeat(100) + "\n\n").repeat(3));
497        let sections = split(&md, "README");
498        let chunks: Vec<_> = sections
499            .iter()
500            .filter(|s| s.heading.ends_with("Setup") && !s.text.contains("Unrelated"))
501            .collect();
502        assert!(chunks.len() > 1);
503        assert!(chunks.iter().all(|s| s.imports == ["@sample/forms/next"]));
504        let inherited = sections.iter().find(|s| s.text == "Inherited recipe.").unwrap();
505        assert_eq!(inherited.imports, ["@sample/forms/next"]);
506        assert!(inherited.parents.contains(&chunks[0].line));
507        let independent = sections.iter().find(|s| s.text == "Independent recipe.").unwrap();
508        assert!(independent.imports.is_empty());
509        assert!(!independent.parents.contains(&chunks[0].line));
510    }
511
512    #[test]
513    fn imports_come_from_examples_and_override_only_their_package() {
514        let md = "import X from 'component/plumbing';\n\n# Setup\n\n```js\nconst f = require('@sample/forms/next');\nconst other = import('other/setup');\n```\n\n## Override\n\n```ts\nimport { build } from '@sample/forms';\nexport { build } from '@sample/forms/unknown';\n// import x from 'ignored/comment';\nconst label = \"import x from 'ignored/string'\";\n```\n\n### Child\n\nAn inherited example.\n";
515        let sections = split(md, "README");
516        assert!(sections[0].imports.is_empty());
517        let child = sections.last().unwrap();
518        assert_eq!(child.imports, ["@sample/forms", "@sample/forms/unknown", "other/setup"]);
519        assert_eq!(example_imports("```ts\nimport f from 'unfinished/example';"), ["unfinished/example"]);
520    }
521
522    #[test]
523    fn sections_follow_headings_and_skip_fences() {
524        let md = "# Zod\n\nIntro.\n\n## Objects\n\n```ts\n# not a heading\nz.object({})\n```\n\n### strict\n\nText.\n\nSetext\n------\n\nMore.\n";
525        let s = split(md, "README");
526        let heads: Vec<&str> = s.iter().map(|x| x.heading.as_str()).collect();
527        assert_eq!(
528            heads,
529            vec![
530                "README › Zod",
531                "README › Zod › Objects",
532                "README › Zod › Objects › strict",
533                "README › Zod › Setext"
534            ]
535        );
536        assert!(s[1].text.contains("# not a heading"));
537        assert_eq!(s[1].line, 5);
538    }
539
540    #[test]
541    fn mdx_exports_and_html_headings() {
542        let mdx = "import { X } from \"x\";\n\nexport const title = \"Functions and directives\";\nexport const description = \"Custom directives.\";\n\n## Directives\n\nIntro.\n\n<h3 id=\"import-directive\">\n  <a href=\"#import-directive\">@import</a>\n</h3>\n\nUse `@import`.\n\n<h3 id=\"theme\"><a href=\"#theme\">@theme</a></h3>\n\nTokens.\n";
543        let (text, title) = clean_mdx(mdx, true);
544        assert_eq!(title.as_deref(), Some("Functions and directives"));
545        assert_eq!(text.lines().count(), mdx.lines().count());
546        let s = split(&text, title.as_deref().unwrap());
547        let heads: Vec<&str> = s.iter().map(|x| x.heading.as_str()).collect();
548        assert_eq!(
549            heads,
550            vec![
551                "Functions and directives",
552                "Functions and directives › Directives",
553                "Functions and directives › Directives › @import",
554                "Functions and directives › Directives › @theme"
555            ]
556        );
557        assert!(s[0].text.contains("Custom directives."));
558        assert_eq!(s[2].line, 10);
559    }
560
561    #[test]
562    fn heading_ids_and_generic_headings() {
563        let s = split("# forwardRef\n\n## Usage {/*usage*/}\n\nText.\n\n### Props {#props}\n\nMore.\n", "doc");
564        assert_eq!(s[0].heading, "doc › forwardRef › Usage");
565        assert_eq!(topic_heading(&s[1].heading), "Usage");
566        assert_eq!(topic_heading("generateMetadata › Parameters"), "generateMetadata");
567    }
568
569    #[test]
570    fn rst_headings() {
571        let rst = "Title\n=====\n\nBody\n\nSub\n---\n\nx\n";
572        let s = split(rst, "doc");
573        assert_eq!(s.len(), 2);
574        assert_eq!(s[1].heading, "doc › Title › Sub");
575    }
576}