ailint_core/parser/
markdown.rs

1//! Markdown parsing via `pulldown-cmark`.
2
3use std::ops::Range;
4
5use pulldown_cmark::{CodeBlockKind, Event, HeadingLevel, Options, Parser, Tag, TagEnd};
6
7/// Structural index of a Markdown document, built in one parse pass.
8#[derive(Debug, Clone, Default)]
9pub struct MarkdownDoc {
10    /// YAML frontmatter block, if the file starts with `---`.
11    pub frontmatter: Option<Frontmatter>,
12    /// All headings, in document order.
13    pub headings: Vec<Heading>,
14    /// Heading-delimited spans covering the whole body.
15    pub sections: Vec<Section>,
16    /// Fenced and indented code blocks.
17    pub code_blocks: Vec<CodeBlock>,
18    /// Bullet and ordered list items, flattened.
19    pub list_items: Vec<ListItem>,
20    /// All hyperlinks.
21    pub links: Vec<Link>,
22}
23
24/// Raw YAML frontmatter (`---` fenced) at the top of a document.
25#[derive(Debug, Clone)]
26pub struct Frontmatter {
27    /// Frontmatter body without the `---` fences.
28    pub raw: String,
29    /// Byte span in the original input, fences included.
30    pub byte_range: Range<usize>,
31}
32
33/// A Markdown heading.
34#[derive(Debug, Clone)]
35pub struct Heading {
36    /// Heading depth, 1–6.
37    pub level: u8,
38    /// Heading text with inline markup stripped.
39    pub text: String,
40    /// Byte span in the original input.
41    pub byte_range: Range<usize>,
42    /// 1-based line number.
43    pub line: usize,
44}
45
46/// A span of the body owned by one heading (or the preamble).
47#[derive(Debug, Clone)]
48pub struct Section {
49    /// Index into [`MarkdownDoc::headings`]; `None` for pre-heading content.
50    pub heading_index: Option<usize>,
51    /// Byte span in the original input.
52    pub byte_range: Range<usize>,
53}
54
55/// A fenced or indented code block.
56#[derive(Debug, Clone)]
57pub struct CodeBlock {
58    /// Info-string language tag, if any.
59    pub lang: Option<String>,
60    /// Code block contents.
61    pub text: String,
62    /// Byte span in the original input.
63    pub byte_range: Range<usize>,
64    /// 1-based line number.
65    pub line: usize,
66}
67
68/// A single list item (bullet or ordered), flattened.
69#[derive(Debug, Clone)]
70pub struct ListItem {
71    /// Item text with inline markup stripped.
72    pub text: String,
73    /// Byte span in the original input.
74    pub byte_range: Range<usize>,
75    /// 1-based line number.
76    pub line: usize,
77}
78
79/// A hyperlink discovered in the document (`[text](url)` or reference form).
80#[derive(Debug, Clone)]
81pub struct Link {
82    /// Link destination as written.
83    pub url: String,
84    /// Link text.
85    pub text: String,
86    /// Byte span in the original input.
87    pub byte_range: Range<usize>,
88    /// 1-based line number.
89    pub line: usize,
90}
91
92/// Parse `input` into a [`MarkdownDoc`].
93pub fn parse(input: &str) -> MarkdownDoc {
94    let (frontmatter, body_start) = split_frontmatter(input);
95    let body = &input[body_start..];
96    let line_starts = compute_line_starts(input);
97
98    let mut doc = MarkdownDoc {
99        frontmatter,
100        headings: Vec::new(),
101        sections: Vec::new(),
102        code_blocks: Vec::new(),
103        list_items: Vec::new(),
104        links: Vec::new(),
105    };
106
107    let mut heading_stack: Vec<PendingHeading> = Vec::new();
108    let mut code_stack: Vec<PendingCodeBlock> = Vec::new();
109    let mut item_stack: Vec<PendingListItem> = Vec::new();
110    let mut link_stack: Vec<PendingLink> = Vec::new();
111
112    for (event, raw_range) in Parser::new_ext(body, Options::empty()).into_offset_iter() {
113        let range = (raw_range.start + body_start)..(raw_range.end + body_start);
114        match event {
115            Event::Start(Tag::Heading { level, .. }) => {
116                heading_stack.push(PendingHeading {
117                    level: heading_level_u8(level),
118                    text: String::new(),
119                    start: range.start,
120                });
121            }
122            Event::End(TagEnd::Heading(_)) => {
123                if let Some(pending) = heading_stack.pop() {
124                    let line = offset_to_line(&line_starts, pending.start);
125                    doc.headings.push(Heading {
126                        level: pending.level,
127                        text: pending.text.trim().to_string(),
128                        byte_range: pending.start..range.end,
129                        line,
130                    });
131                }
132            }
133            Event::Start(Tag::CodeBlock(kind)) => {
134                let lang = match kind {
135                    CodeBlockKind::Fenced(l) => {
136                        let s = l.to_string();
137                        if s.is_empty() {
138                            None
139                        } else {
140                            Some(s)
141                        }
142                    }
143                    CodeBlockKind::Indented => None,
144                };
145                code_stack.push(PendingCodeBlock {
146                    lang,
147                    text: String::new(),
148                    start: range.start,
149                });
150            }
151            Event::End(TagEnd::CodeBlock) => {
152                if let Some(pending) = code_stack.pop() {
153                    let line = offset_to_line(&line_starts, pending.start);
154                    doc.code_blocks.push(CodeBlock {
155                        lang: pending.lang,
156                        text: pending.text,
157                        byte_range: pending.start..range.end,
158                        line,
159                    });
160                }
161            }
162            Event::Start(Tag::Item) => {
163                item_stack.push(PendingListItem {
164                    text: String::new(),
165                    start: range.start,
166                });
167            }
168            Event::End(TagEnd::Item) => {
169                if let Some(pending) = item_stack.pop() {
170                    let line = offset_to_line(&line_starts, pending.start);
171                    doc.list_items.push(ListItem {
172                        text: pending.text.trim().to_string(),
173                        byte_range: pending.start..range.end,
174                        line,
175                    });
176                }
177            }
178            Event::Start(Tag::Link { dest_url, .. }) => {
179                link_stack.push(PendingLink {
180                    url: dest_url.to_string(),
181                    text: String::new(),
182                    start: range.start,
183                });
184            }
185            Event::End(TagEnd::Link) => {
186                if let Some(pending) = link_stack.pop() {
187                    let line = offset_to_line(&line_starts, pending.start);
188                    doc.links.push(Link {
189                        url: pending.url,
190                        text: pending.text.trim().to_string(),
191                        byte_range: pending.start..range.end,
192                        line,
193                    });
194                }
195            }
196            Event::Text(t) => {
197                if let Some(c) = code_stack.last_mut() {
198                    c.text.push_str(&t);
199                } else {
200                    if let Some(h) = heading_stack.last_mut() {
201                        h.text.push_str(&t);
202                    }
203                    if let Some(li) = item_stack.last_mut() {
204                        li.text.push_str(&t);
205                    }
206                    if let Some(lk) = link_stack.last_mut() {
207                        lk.text.push_str(&t);
208                    }
209                }
210            }
211            Event::Code(t) => {
212                if let Some(h) = heading_stack.last_mut() {
213                    h.text.push_str(&t);
214                }
215                if let Some(li) = item_stack.last_mut() {
216                    li.text.push_str(&t);
217                }
218                if let Some(lk) = link_stack.last_mut() {
219                    lk.text.push_str(&t);
220                }
221            }
222            _ => {}
223        }
224    }
225
226    doc.sections = build_sections(&doc.headings, body_start, input.len());
227    doc
228}
229
230struct PendingHeading {
231    level: u8,
232    text: String,
233    start: usize,
234}
235
236struct PendingCodeBlock {
237    lang: Option<String>,
238    text: String,
239    start: usize,
240}
241
242struct PendingListItem {
243    text: String,
244    start: usize,
245}
246
247struct PendingLink {
248    url: String,
249    text: String,
250    start: usize,
251}
252
253fn heading_level_u8(level: HeadingLevel) -> u8 {
254    match level {
255        HeadingLevel::H1 => 1,
256        HeadingLevel::H2 => 2,
257        HeadingLevel::H3 => 3,
258        HeadingLevel::H4 => 4,
259        HeadingLevel::H5 => 5,
260        HeadingLevel::H6 => 6,
261    }
262}
263
264fn build_sections(headings: &[Heading], body_start: usize, file_end: usize) -> Vec<Section> {
265    let mut sections = Vec::new();
266    let first_start = headings
267        .first()
268        .map(|h| h.byte_range.start)
269        .unwrap_or(file_end);
270    if body_start < first_start {
271        sections.push(Section {
272            heading_index: None,
273            byte_range: body_start..first_start,
274        });
275    }
276    for (i, h) in headings.iter().enumerate() {
277        let end = headings
278            .get(i + 1)
279            .map(|nh| nh.byte_range.start)
280            .unwrap_or(file_end);
281        sections.push(Section {
282            heading_index: Some(i),
283            byte_range: h.byte_range.end..end,
284        });
285    }
286    sections
287}
288
289fn compute_line_starts(input: &str) -> Vec<usize> {
290    let mut v = Vec::with_capacity(64);
291    v.push(0);
292    for (i, b) in input.bytes().enumerate() {
293        if b == b'\n' {
294            v.push(i + 1);
295        }
296    }
297    v
298}
299
300fn offset_to_line(starts: &[usize], offset: usize) -> usize {
301    match starts.binary_search(&offset) {
302        Ok(i) => i + 1,
303        Err(i) => i.max(1),
304    }
305}
306
307fn split_frontmatter(input: &str) -> (Option<Frontmatter>, usize) {
308    let bom_len = if input.starts_with('\u{FEFF}') {
309        '\u{FEFF}'.len_utf8()
310    } else {
311        0
312    };
313    let after_bom = &input[bom_len..];
314
315    let open_len = if after_bom.starts_with("---\r\n") {
316        5
317    } else if after_bom.starts_with("---\n") {
318        4
319    } else {
320        return (None, 0);
321    };
322
323    let body_after_open = &after_bom[open_len..];
324    let mut cursor = 0usize;
325    while cursor <= body_after_open.len() {
326        let rest = &body_after_open[cursor..];
327        let (line, consumed) = match rest.find('\n') {
328            Some(nl) => (&rest[..nl], nl + 1),
329            None => (rest, rest.len()),
330        };
331        let content = line.strip_suffix('\r').unwrap_or(line);
332        if content == "---" {
333            let close_end = cursor + consumed;
334            let raw_slice = &body_after_open[..cursor];
335            let raw_trimmed = raw_slice.strip_suffix('\n').unwrap_or(raw_slice);
336            let raw_trimmed = raw_trimmed.strip_suffix('\r').unwrap_or(raw_trimmed);
337            let full_end = bom_len + open_len + close_end;
338            return (
339                Some(Frontmatter {
340                    raw: raw_trimmed.to_string(),
341                    byte_range: bom_len..full_end,
342                }),
343                full_end,
344            );
345        }
346        if consumed == 0 {
347            break;
348        }
349        cursor += consumed;
350    }
351    (None, 0)
352}
353
354#[cfg(test)]
355mod tests {
356    use super::*;
357
358    #[test]
359    fn parses_frontmatter_when_present() {
360        let src = "---\ntitle: Hi\nkey: val\n---\n# Heading\n";
361        let doc = parse(src);
362        let fm = doc.frontmatter.expect("frontmatter");
363        assert_eq!(fm.raw, "title: Hi\nkey: val");
364        assert_eq!(fm.byte_range.start, 0);
365        assert_eq!(
366            &src[fm.byte_range.clone()],
367            "---\ntitle: Hi\nkey: val\n---\n"
368        );
369        assert_eq!(doc.headings.len(), 1);
370        assert_eq!(doc.headings[0].text, "Heading");
371    }
372
373    #[test]
374    fn no_frontmatter_when_missing_closing_fence() {
375        let src = "---\ntitle: unterminated\n# Heading\n";
376        let doc = parse(src);
377        assert!(doc.frontmatter.is_none());
378    }
379
380    #[test]
381    fn no_frontmatter_when_not_delimited() {
382        let src = "# Heading\n\nBody text.\n";
383        let doc = parse(src);
384        assert!(doc.frontmatter.is_none());
385        assert_eq!(doc.headings.len(), 1);
386    }
387
388    #[test]
389    fn extracts_headings_with_line_numbers() {
390        let src = "# One\n\n## Two\n\nBody\n\n### Three\n";
391        let doc = parse(src);
392        assert_eq!(doc.headings.len(), 3);
393        assert_eq!(doc.headings[0].level, 1);
394        assert_eq!(doc.headings[0].text, "One");
395        assert_eq!(doc.headings[0].line, 1);
396        assert_eq!(doc.headings[1].level, 2);
397        assert_eq!(doc.headings[1].line, 3);
398        assert_eq!(doc.headings[2].level, 3);
399        assert_eq!(doc.headings[2].line, 7);
400    }
401
402    #[test]
403    fn sections_span_between_headings() {
404        let src = "prelude\n\n# One\nbody-one\n\n# Two\nbody-two\n";
405        let doc = parse(src);
406        assert_eq!(doc.sections.len(), 3);
407        assert!(doc.sections[0].heading_index.is_none());
408        assert_eq!(doc.sections[1].heading_index, Some(0));
409        assert_eq!(doc.sections[2].heading_index, Some(1));
410        let s1 = &src[doc.sections[1].byte_range.clone()];
411        assert!(s1.contains("body-one"));
412        assert!(!s1.contains("body-two"));
413    }
414
415    #[test]
416    fn code_blocks_capture_language() {
417        let src = "text\n\n```rust\nfn main() {}\n```\n\n```\nplain\n```\n";
418        let doc = parse(src);
419        assert_eq!(doc.code_blocks.len(), 2);
420        assert_eq!(doc.code_blocks[0].lang.as_deref(), Some("rust"));
421        assert!(doc.code_blocks[0].text.contains("fn main"));
422        assert_eq!(doc.code_blocks[1].lang, None);
423    }
424
425    #[test]
426    fn handles_crlf_line_endings() {
427        let src = "---\r\ntitle: crlf\r\n---\r\n# Heading\r\n\r\n- item one\r\n- item two\r\n";
428        let doc = parse(src);
429        let fm = doc.frontmatter.expect("frontmatter");
430        assert_eq!(fm.raw, "title: crlf");
431        assert_eq!(doc.headings.len(), 1);
432        assert_eq!(doc.headings[0].text, "Heading");
433        assert_eq!(doc.list_items.len(), 2);
434        assert_eq!(doc.list_items[0].text, "item one");
435        assert_eq!(doc.list_items[1].text, "item two");
436    }
437}
ailint_core/parser/markdown.rs

ailint_core/parser/
markdown.rs
