ailint_core/parser/
markdown.rs

1//! Markdown parsing via `pulldown-cmark`.
2
3use std::ops::Range;
4
5use pulldown_cmark::{CodeBlockKind, Event, HeadingLevel, Options, Parser, Tag, TagEnd};
6
7/// Structural index of a Markdown document, built in one parse pass.
8#[derive(Debug, Clone, Default)]
9pub struct MarkdownDoc {
10    /// YAML frontmatter block, if the file starts with `---`.
11    pub frontmatter: Option<Frontmatter>,
12    /// All headings, in document order.
13    pub headings: Vec<Heading>,
14    /// Heading-delimited spans covering the whole body.
15    pub sections: Vec<Section>,
16    /// Fenced and indented code blocks.
17    pub code_blocks: Vec<CodeBlock>,
18    /// Bullet and ordered list items, flattened.
19    pub list_items: Vec<ListItem>,
20    /// Top-level prose paragraphs (excludes headings, list items, code blocks).
21    pub paragraphs: Vec<Paragraph>,
22    /// All hyperlinks.
23    pub links: Vec<Link>,
24}
25
26/// Raw YAML frontmatter (`---` fenced) at the top of a document.
27#[derive(Debug, Clone)]
28pub struct Frontmatter {
29    /// Frontmatter body without the `---` fences.
30    pub raw: String,
31    /// Byte span in the original input, fences included.
32    pub byte_range: Range<usize>,
33}
34
35/// A Markdown heading.
36#[derive(Debug, Clone)]
37pub struct Heading {
38    /// Heading depth, 1–6.
39    pub level: u8,
40    /// Heading text with inline markup stripped.
41    pub text: String,
42    /// Byte span in the original input.
43    pub byte_range: Range<usize>,
44    /// 1-based line number.
45    pub line: usize,
46}
47
48/// A span of the body owned by one heading (or the preamble).
49#[derive(Debug, Clone)]
50pub struct Section {
51    /// Index into [`MarkdownDoc::headings`]; `None` for pre-heading content.
52    pub heading_index: Option<usize>,
53    /// Byte span in the original input.
54    pub byte_range: Range<usize>,
55}
56
57/// A fenced or indented code block.
58#[derive(Debug, Clone)]
59pub struct CodeBlock {
60    /// Info-string language tag, if any.
61    pub lang: Option<String>,
62    /// Code block contents.
63    pub text: String,
64    /// Byte span in the original input.
65    pub byte_range: Range<usize>,
66    /// 1-based line number.
67    pub line: usize,
68}
69
70/// A single list item (bullet or ordered), flattened.
71#[derive(Debug, Clone)]
72pub struct ListItem {
73    /// Item text with inline markup stripped.
74    pub text: String,
75    /// Byte span in the original input.
76    pub byte_range: Range<usize>,
77    /// 1-based line number.
78    pub line: usize,
79}
80
81/// A top-level prose paragraph (not inside a list item or code block).
82#[derive(Debug, Clone)]
83pub struct Paragraph {
84    /// Paragraph text with inline markup stripped.
85    pub text: String,
86    /// Byte span in the original input.
87    pub byte_range: Range<usize>,
88    /// 1-based line number.
89    pub line: usize,
90}
91
92/// A hyperlink discovered in the document (`[text](url)` or reference form).
93#[derive(Debug, Clone)]
94pub struct Link {
95    /// Link destination as written.
96    pub url: String,
97    /// Link text.
98    pub text: String,
99    /// Byte span in the original input.
100    pub byte_range: Range<usize>,
101    /// 1-based line number.
102    pub line: usize,
103}
104
105/// Parse `input` into a [`MarkdownDoc`].
106pub fn parse(input: &str) -> MarkdownDoc {
107    let (frontmatter, body_start) = split_frontmatter(input);
108    let body = &input[body_start..];
109    let line_starts = compute_line_starts(input);
110
111    let mut doc = MarkdownDoc {
112        frontmatter,
113        headings: Vec::new(),
114        sections: Vec::new(),
115        code_blocks: Vec::new(),
116        list_items: Vec::new(),
117        paragraphs: Vec::new(),
118        links: Vec::new(),
119    };
120
121    let mut heading_stack: Vec<PendingHeading> = Vec::new();
122    let mut code_stack: Vec<PendingCodeBlock> = Vec::new();
123    let mut item_stack: Vec<PendingListItem> = Vec::new();
124    let mut paragraph_stack: Vec<PendingParagraph> = Vec::new();
125    let mut link_stack: Vec<PendingLink> = Vec::new();
126
127    for (event, raw_range) in Parser::new_ext(body, Options::empty()).into_offset_iter() {
128        let range = (raw_range.start + body_start)..(raw_range.end + body_start);
129        match event {
130            Event::Start(Tag::Heading { level, .. }) => {
131                heading_stack.push(PendingHeading {
132                    level: heading_level_u8(level),
133                    text: String::new(),
134                    start: range.start,
135                });
136            }
137            Event::End(TagEnd::Heading(_)) => {
138                if let Some(pending) = heading_stack.pop() {
139                    let line = offset_to_line(&line_starts, pending.start);
140                    doc.headings.push(Heading {
141                        level: pending.level,
142                        text: pending.text.trim().to_string(),
143                        byte_range: pending.start..range.end,
144                        line,
145                    });
146                }
147            }
148            Event::Start(Tag::CodeBlock(kind)) => {
149                let lang = match kind {
150                    CodeBlockKind::Fenced(l) => {
151                        let s = l.to_string();
152                        if s.is_empty() {
153                            None
154                        } else {
155                            Some(s)
156                        }
157                    }
158                    CodeBlockKind::Indented => None,
159                };
160                code_stack.push(PendingCodeBlock {
161                    lang,
162                    text: String::new(),
163                    start: range.start,
164                });
165            }
166            Event::End(TagEnd::CodeBlock) => {
167                if let Some(pending) = code_stack.pop() {
168                    let line = offset_to_line(&line_starts, pending.start);
169                    doc.code_blocks.push(CodeBlock {
170                        lang: pending.lang,
171                        text: pending.text,
172                        byte_range: pending.start..range.end,
173                        line,
174                    });
175                }
176            }
177            Event::Start(Tag::Item) => {
178                item_stack.push(PendingListItem {
179                    text: String::new(),
180                    start: range.start,
181                });
182            }
183            Event::End(TagEnd::Item) => {
184                if let Some(pending) = item_stack.pop() {
185                    let line = offset_to_line(&line_starts, pending.start);
186                    doc.list_items.push(ListItem {
187                        text: pending.text.trim().to_string(),
188                        byte_range: pending.start..range.end,
189                        line,
190                    });
191                }
192            }
193            Event::Start(Tag::Paragraph) => {
194                // Nested paragraphs (inside list items, blockquotes) don't count as top-level prose.
195                if item_stack.is_empty() {
196                    paragraph_stack.push(PendingParagraph {
197                        text: String::new(),
198                        start: range.start,
199                    });
200                }
201            }
202            Event::End(TagEnd::Paragraph) => {
203                if let Some(pending) = paragraph_stack.pop() {
204                    let line = offset_to_line(&line_starts, pending.start);
205                    doc.paragraphs.push(Paragraph {
206                        text: pending.text.trim().to_string(),
207                        byte_range: pending.start..range.end,
208                        line,
209                    });
210                }
211            }
212            Event::Start(Tag::Link { dest_url, .. }) => {
213                link_stack.push(PendingLink {
214                    url: dest_url.to_string(),
215                    text: String::new(),
216                    start: range.start,
217                });
218            }
219            Event::End(TagEnd::Link) => {
220                if let Some(pending) = link_stack.pop() {
221                    let line = offset_to_line(&line_starts, pending.start);
222                    doc.links.push(Link {
223                        url: pending.url,
224                        text: pending.text.trim().to_string(),
225                        byte_range: pending.start..range.end,
226                        line,
227                    });
228                }
229            }
230            Event::Text(t) => {
231                if let Some(c) = code_stack.last_mut() {
232                    c.text.push_str(&t);
233                } else {
234                    if let Some(h) = heading_stack.last_mut() {
235                        h.text.push_str(&t);
236                    }
237                    if let Some(li) = item_stack.last_mut() {
238                        li.text.push_str(&t);
239                    }
240                    if let Some(p) = paragraph_stack.last_mut() {
241                        p.text.push_str(&t);
242                    }
243                    if let Some(lk) = link_stack.last_mut() {
244                        lk.text.push_str(&t);
245                    }
246                }
247            }
248            Event::Code(t) => {
249                if let Some(h) = heading_stack.last_mut() {
250                    h.text.push_str(&t);
251                }
252                if let Some(li) = item_stack.last_mut() {
253                    li.text.push_str(&t);
254                }
255                if let Some(p) = paragraph_stack.last_mut() {
256                    p.text.push_str(&t);
257                }
258                if let Some(lk) = link_stack.last_mut() {
259                    lk.text.push_str(&t);
260                }
261            }
262            _ => {}
263        }
264    }
265
266    doc.sections = build_sections(&doc.headings, body_start, input.len());
267    doc
268}
269
270struct PendingHeading {
271    level: u8,
272    text: String,
273    start: usize,
274}
275
276struct PendingCodeBlock {
277    lang: Option<String>,
278    text: String,
279    start: usize,
280}
281
282struct PendingListItem {
283    text: String,
284    start: usize,
285}
286
287struct PendingParagraph {
288    text: String,
289    start: usize,
290}
291
292struct PendingLink {
293    url: String,
294    text: String,
295    start: usize,
296}
297
298fn heading_level_u8(level: HeadingLevel) -> u8 {
299    match level {
300        HeadingLevel::H1 => 1,
301        HeadingLevel::H2 => 2,
302        HeadingLevel::H3 => 3,
303        HeadingLevel::H4 => 4,
304        HeadingLevel::H5 => 5,
305        HeadingLevel::H6 => 6,
306    }
307}
308
309fn build_sections(headings: &[Heading], body_start: usize, file_end: usize) -> Vec<Section> {
310    let mut sections = Vec::new();
311    let first_start = headings
312        .first()
313        .map(|h| h.byte_range.start)
314        .unwrap_or(file_end);
315    if body_start < first_start {
316        sections.push(Section {
317            heading_index: None,
318            byte_range: body_start..first_start,
319        });
320    }
321    for (i, h) in headings.iter().enumerate() {
322        let end = headings
323            .get(i + 1)
324            .map(|nh| nh.byte_range.start)
325            .unwrap_or(file_end);
326        sections.push(Section {
327            heading_index: Some(i),
328            byte_range: h.byte_range.end..end,
329        });
330    }
331    sections
332}
333
334fn compute_line_starts(input: &str) -> Vec<usize> {
335    let mut v = Vec::with_capacity(64);
336    v.push(0);
337    for (i, b) in input.bytes().enumerate() {
338        if b == b'\n' {
339            v.push(i + 1);
340        }
341    }
342    v
343}
344
345fn offset_to_line(starts: &[usize], offset: usize) -> usize {
346    match starts.binary_search(&offset) {
347        Ok(i) => i + 1,
348        Err(i) => i.max(1),
349    }
350}
351
352fn split_frontmatter(input: &str) -> (Option<Frontmatter>, usize) {
353    let bom_len = if input.starts_with('\u{FEFF}') {
354        '\u{FEFF}'.len_utf8()
355    } else {
356        0
357    };
358    let after_bom = &input[bom_len..];
359
360    let open_len = if after_bom.starts_with("---\r\n") {
361        5
362    } else if after_bom.starts_with("---\n") {
363        4
364    } else {
365        return (None, 0);
366    };
367
368    let body_after_open = &after_bom[open_len..];
369    let mut cursor = 0usize;
370    while cursor <= body_after_open.len() {
371        let rest = &body_after_open[cursor..];
372        let (line, consumed) = match rest.find('\n') {
373            Some(nl) => (&rest[..nl], nl + 1),
374            None => (rest, rest.len()),
375        };
376        let content = line.strip_suffix('\r').unwrap_or(line);
377        if content == "---" {
378            let close_end = cursor + consumed;
379            let raw_slice = &body_after_open[..cursor];
380            let raw_trimmed = raw_slice.strip_suffix('\n').unwrap_or(raw_slice);
381            let raw_trimmed = raw_trimmed.strip_suffix('\r').unwrap_or(raw_trimmed);
382            let full_end = bom_len + open_len + close_end;
383            return (
384                Some(Frontmatter {
385                    raw: raw_trimmed.to_string(),
386                    byte_range: bom_len..full_end,
387                }),
388                full_end,
389            );
390        }
391        if consumed == 0 {
392            break;
393        }
394        cursor += consumed;
395    }
396    (None, 0)
397}
398
399#[cfg(test)]
400mod tests {
401    use super::*;
402
403    #[test]
404    fn parses_frontmatter_when_present() {
405        let src = "---\ntitle: Hi\nkey: val\n---\n# Heading\n";
406        let doc = parse(src);
407        let fm = doc.frontmatter.expect("frontmatter");
408        assert_eq!(fm.raw, "title: Hi\nkey: val");
409        assert_eq!(fm.byte_range.start, 0);
410        assert_eq!(
411            &src[fm.byte_range.clone()],
412            "---\ntitle: Hi\nkey: val\n---\n"
413        );
414        assert_eq!(doc.headings.len(), 1);
415        assert_eq!(doc.headings[0].text, "Heading");
416    }
417
418    #[test]
419    fn no_frontmatter_when_missing_closing_fence() {
420        let src = "---\ntitle: unterminated\n# Heading\n";
421        let doc = parse(src);
422        assert!(doc.frontmatter.is_none());
423    }
424
425    #[test]
426    fn no_frontmatter_when_not_delimited() {
427        let src = "# Heading\n\nBody text.\n";
428        let doc = parse(src);
429        assert!(doc.frontmatter.is_none());
430        assert_eq!(doc.headings.len(), 1);
431    }
432
433    #[test]
434    fn extracts_headings_with_line_numbers() {
435        let src = "# One\n\n## Two\n\nBody\n\n### Three\n";
436        let doc = parse(src);
437        assert_eq!(doc.headings.len(), 3);
438        assert_eq!(doc.headings[0].level, 1);
439        assert_eq!(doc.headings[0].text, "One");
440        assert_eq!(doc.headings[0].line, 1);
441        assert_eq!(doc.headings[1].level, 2);
442        assert_eq!(doc.headings[1].line, 3);
443        assert_eq!(doc.headings[2].level, 3);
444        assert_eq!(doc.headings[2].line, 7);
445    }
446
447    #[test]
448    fn sections_span_between_headings() {
449        let src = "prelude\n\n# One\nbody-one\n\n# Two\nbody-two\n";
450        let doc = parse(src);
451        assert_eq!(doc.sections.len(), 3);
452        assert!(doc.sections[0].heading_index.is_none());
453        assert_eq!(doc.sections[1].heading_index, Some(0));
454        assert_eq!(doc.sections[2].heading_index, Some(1));
455        let s1 = &src[doc.sections[1].byte_range.clone()];
456        assert!(s1.contains("body-one"));
457        assert!(!s1.contains("body-two"));
458    }
459
460    #[test]
461    fn code_blocks_capture_language() {
462        let src = "text\n\n```rust\nfn main() {}\n```\n\n```\nplain\n```\n";
463        let doc = parse(src);
464        assert_eq!(doc.code_blocks.len(), 2);
465        assert_eq!(doc.code_blocks[0].lang.as_deref(), Some("rust"));
466        assert!(doc.code_blocks[0].text.contains("fn main"));
467        assert_eq!(doc.code_blocks[1].lang, None);
468    }
469
470    #[test]
471    fn handles_crlf_line_endings() {
472        let src = "---\r\ntitle: crlf\r\n---\r\n# Heading\r\n\r\n- item one\r\n- item two\r\n";
473        let doc = parse(src);
474        let fm = doc.frontmatter.expect("frontmatter");
475        assert_eq!(fm.raw, "title: crlf");
476        assert_eq!(doc.headings.len(), 1);
477        assert_eq!(doc.headings[0].text, "Heading");
478        assert_eq!(doc.list_items.len(), 2);
479        assert_eq!(doc.list_items[0].text, "item one");
480        assert_eq!(doc.list_items[1].text, "item two");
481    }
482}
ailint_core/parser/markdown.rs

ailint_core/parser/
markdown.rs
