snapper_fmt/parser/
markdown.rs

1use regex::Regex;
2use std::sync::LazyLock;
3
4use crate::parser::{FormatParser, Region, flush_prose};
5
6static HEADING_RE: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"^(#{1,6}\s+)(.*)$").unwrap());
7
8static FENCED_CODE_RE: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"^(`{3,}|~{3,})").unwrap());
9
10/// Capture the language token immediately after a fence marker.
11/// `lang` is `[A-Za-z0-9_+.-]+`; anything past it (info string) is ignored.
12static FENCED_LANG_RE: LazyLock<Regex> =
13    LazyLock::new(|| Regex::new(r"^(?:`{3,}|~{3,})\s*([A-Za-z0-9_+.\-]+)").unwrap());
14
15static LIST_ITEM_RE: LazyLock<Regex> =
16    LazyLock::new(|| Regex::new(r"^(\s*(?:[-*+]|\d+[.)]) )(.*)$").unwrap());
17
18/// Match a markdown table row: line whose trimmed form starts and ends with `|`.
19/// Also matches separator rows like `|---|---|`.
20static TABLE_ROW_RE: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"^\s*\|.*\|\s*$").unwrap());
21
22/// CommonMark setext underline: one or more `=` (level 1) or `-` (level 2),
23/// optional leading indent up to three spaces, optional trailing spaces.
24static SETEXT_UNDERLINE_RE: LazyLock<Regex> =
25    LazyLock::new(|| Regex::new(r"^ {0,3}(?:=+|-+)\s*$").unwrap());
26
27pub struct MarkdownParser;
28
29/// Close an open list item: flush accumulated prose and emit the trailing newline.
30fn close_list_item(in_list_item: &mut bool, current_prose: &mut String, regions: &mut Vec<Region>) {
31    if *in_list_item {
32        flush_prose(current_prose, regions);
33        regions.push(Region::Structure("\n".to_string()));
34        *in_list_item = false;
35    }
36}
37
38/// True when `line` is a CommonMark setext underline (`===` or `---`).
39fn is_setext_underline(line: &str) -> bool {
40    let trimmed = line.trim_end();
41    if trimmed.is_empty() {
42        return false;
43    }
44    SETEXT_UNDERLINE_RE.is_match(trimmed)
45}
46
47/// True when `line` may be the text of a setext heading (non-empty, not an ATX
48/// marker line, not a table row, not a list item, not a fence opener).
49fn is_setext_title_line(line: &str) -> bool {
50    let trimmed = line.trim();
51    if trimmed.is_empty() {
52        return false;
53    }
54    if HEADING_RE.is_match(line) {
55        return false;
56    }
57    if TABLE_ROW_RE.is_match(line) {
58        return false;
59    }
60    if LIST_ITEM_RE.is_match(line) {
61        return false;
62    }
63    if FENCED_CODE_RE.is_match(line.trim_start()) {
64        return false;
65    }
66    true
67}
68
69impl FormatParser for MarkdownParser {
70    fn parse(&self, input: &str) -> Vec<Region> {
71        let mut regions: Vec<Region> = Vec::new();
72        let mut current_prose = String::new();
73        let mut in_fenced_code = false;
74        let mut fence_marker = String::new();
75        // Buffer for the running code block: header line, body lines, lang
76        let mut code_header = String::new();
77        let mut code_body = String::new();
78        let mut code_lang: Option<String> = None;
79        let mut in_frontmatter = false;
80        let mut frontmatter_fence = String::new();
81        let mut in_list_item = false;
82        let mut pragma_off = false;
83
84        let lines: Vec<&str> = input.lines().collect();
85        let total = lines.len();
86        let mut i = 0;
87
88        while i < total {
89            let line = lines[i];
90            let line_number = i + 1;
91
92            // Check for snapper:off/on pragmas. Inside a fenced code block,
93            // the markdown parser does NOT short-circuit on pragmas; the
94            // code-block reflow handles them per-language (the markers
95            // `#`, `//`, `--`, `;` are all valid pragma prefixes inside
96            // their respective languages).
97            if !in_fenced_code {
98                if let Some(on) = super::check_pragma(line) {
99                    close_list_item(&mut in_list_item, &mut current_prose, &mut regions);
100                    flush_prose(&mut current_prose, &mut regions);
101                    pragma_off = !on;
102                    regions.push(Region::Structure(format!("{line}\n")));
103                    i += 1;
104                    continue;
105                }
106
107                if pragma_off {
108                    close_list_item(&mut in_list_item, &mut current_prose, &mut regions);
109                    flush_prose(&mut current_prose, &mut regions);
110                    regions.push(Region::Structure(format!("{line}\n")));
111                    i += 1;
112                    continue;
113                }
114            }
115
116            // Front matter detection (only at start of file)
117            if line_number == 1 && (line.trim() == "---" || line.trim() == "+++") {
118                in_frontmatter = true;
119                frontmatter_fence = line.trim().to_string();
120                regions.push(Region::Structure(format!("{line}\n")));
121                i += 1;
122                continue;
123            }
124
125            if in_frontmatter {
126                if line.trim() == frontmatter_fence {
127                    in_frontmatter = false;
128                }
129                regions.push(Region::Structure(format!("{line}\n")));
130                i += 1;
131                continue;
132            }
133
134            // Inside fenced code block
135            if in_fenced_code {
136                close_list_item(&mut in_list_item, &mut current_prose, &mut regions);
137                flush_prose(&mut current_prose, &mut regions);
138                let mut closed = false;
139                if let Some(caps) = FENCED_CODE_RE.captures(line.trim_start()) {
140                    let marker = caps.get(1).unwrap().as_str();
141                    if marker.chars().next() == fence_marker.chars().next()
142                        && marker.len() >= fence_marker.len()
143                    {
144                        closed = true;
145                    }
146                }
147                if closed {
148                    in_fenced_code = false;
149                    regions.push(Region::Code {
150                        lang: code_lang.take(),
151                        header: std::mem::take(&mut code_header),
152                        body: std::mem::take(&mut code_body),
153                        footer: format!("{line}\n"),
154                    });
155                } else {
156                    code_body.push_str(line);
157                    code_body.push('\n');
158                }
159                i += 1;
160                continue;
161            }
162
163            // Fenced code block start
164            if let Some(caps) = FENCED_CODE_RE.captures(line.trim_start()) {
165                close_list_item(&mut in_list_item, &mut current_prose, &mut regions);
166                flush_prose(&mut current_prose, &mut regions);
167                fence_marker = caps.get(1).unwrap().as_str().to_string();
168                in_fenced_code = true;
169                code_lang = FENCED_LANG_RE
170                    .captures(line.trim_start())
171                    .map(|c| c.get(1).unwrap().as_str().to_string());
172                code_header = format!("{line}\n");
173                code_body.clear();
174                i += 1;
175                continue;
176            }
177
178            // Blank line
179            if line.trim().is_empty() {
180                close_list_item(&mut in_list_item, &mut current_prose, &mut regions);
181                flush_prose(&mut current_prose, &mut regions);
182                regions.push(Region::BlankLines(format!("{line}\n")));
183                i += 1;
184                continue;
185            }
186
187            // Heading — keep the entire ATX line as Structure.
188            // Splitting into Structure("### ") + Prose(title) let the sentence
189            // reflow engine break titles after "1." or mid-phrase, producing
190            // orphan headings like:
191            //   ### 1.
192            //   `cargo binstall` (preferred binary install)
193            // CommonMark ATX headings are single-line; do not reflow them.
194            if HEADING_RE.is_match(line) {
195                close_list_item(&mut in_list_item, &mut current_prose, &mut regions);
196                flush_prose(&mut current_prose, &mut regions);
197                regions.push(Region::Structure(format!("{line}\n")));
198                i += 1;
199                continue;
200            }
201
202            // Setext heading: title line + underline of `=` or `-`.
203            // Without this, title text is Prose and the underline is glued on
204            // (or mid-title periods reflow), collapsing the heading.
205            if i + 1 < total && is_setext_title_line(line) && is_setext_underline(lines[i + 1]) {
206                close_list_item(&mut in_list_item, &mut current_prose, &mut regions);
207                flush_prose(&mut current_prose, &mut regions);
208                regions.push(Region::Structure(format!("{line}\n")));
209                regions.push(Region::Structure(format!("{}\n", lines[i + 1])));
210                i += 2;
211                continue;
212            }
213
214            // Table row (pipe-delimited)
215            if TABLE_ROW_RE.is_match(line) {
216                close_list_item(&mut in_list_item, &mut current_prose, &mut regions);
217                flush_prose(&mut current_prose, &mut regions);
218                regions.push(Region::Structure(format!("{line}\n")));
219                i += 1;
220                continue;
221            }
222
223            // List item: emit marker as Structure, start accumulating text as prose.
224            // Continuation lines are appended until a block boundary.
225            if let Some(caps) = LIST_ITEM_RE.captures(line) {
226                close_list_item(&mut in_list_item, &mut current_prose, &mut regions);
227                flush_prose(&mut current_prose, &mut regions);
228                let marker = caps.get(1).unwrap().as_str();
229                let text = caps.get(2).unwrap().as_str();
230                regions.push(Region::Structure(marker.to_string()));
231                in_list_item = true;
232                if !text.is_empty() {
233                    current_prose.push_str(text);
234                }
235                i += 1;
236                continue;
237            }
238
239            // Regular prose (also serves as list-item continuation when in_list_item)
240            if !current_prose.is_empty() {
241                current_prose.push(' ');
242            }
243            current_prose.push_str(line.trim());
244            i += 1;
245        }
246
247        close_list_item(&mut in_list_item, &mut current_prose, &mut regions);
248        flush_prose(&mut current_prose, &mut regions);
249        // Unclosed fence at EOF: emit a code region with empty footer.
250        if in_fenced_code {
251            regions.push(Region::Code {
252                lang: code_lang.take(),
253                header: std::mem::take(&mut code_header),
254                body: std::mem::take(&mut code_body),
255                footer: String::new(),
256            });
257        }
258        regions
259    }
260}
261
262#[cfg(test)]
263mod tests {
264    use super::*;
265
266    #[test]
267    fn simple_prose() {
268        let input = "Hello world. This is a test.\nAnother line here.";
269        let regions = MarkdownParser.parse(input);
270        assert_eq!(
271            regions,
272            vec![Region::Prose(
273                "Hello world. This is a test. Another line here.".to_string()
274            )]
275        );
276    }
277
278    #[test]
279    fn fenced_code_preserved() {
280        let input = "Some text.\n```python\nprint('hello')\n```\nMore text.";
281        let regions = MarkdownParser.parse(input);
282        assert!(matches!(&regions[0], Region::Prose(_)));
283        // Code blocks now collapse into a single Region::Code carrying
284        // header, body, and footer.
285        match &regions[1] {
286            Region::Code {
287                lang,
288                header,
289                body,
290                footer,
291            } => {
292                assert_eq!(lang.as_deref(), Some("python"));
293                assert_eq!(header, "```python\n");
294                assert_eq!(body, "print('hello')\n");
295                assert_eq!(footer, "```\n");
296            }
297            other => panic!("expected Region::Code, got {other:?}"),
298        }
299        assert!(matches!(&regions[2], Region::Prose(_)));
300    }
301
302    #[test]
303    fn frontmatter_preserved() {
304        let input = "---\ntitle: Test\nauthor: Someone\n---\n\nSome text.";
305        let regions = MarkdownParser.parse(input);
306        // First 4 lines are structure (frontmatter)
307        assert!(matches!(&regions[0], Region::Structure(_)));
308        assert!(matches!(&regions[1], Region::Structure(_)));
309        assert!(matches!(&regions[2], Region::Structure(_)));
310        assert!(matches!(&regions[3], Region::Structure(_)));
311    }
312
313    #[test]
314    fn table_preserved() {
315        let input = "| Feature | Why |\n|---------|-----|\n| `Foo` | Bar |";
316        let regions = MarkdownParser.parse(input);
317        assert!(
318            regions.iter().all(|r| matches!(r, Region::Structure(_))),
319            "all table rows should be Structure, got: {:?}",
320            regions
321        );
322    }
323
324    #[test]
325    fn table_with_surrounding_prose() {
326        let input = "Some text before.\n\n| A | B |\n|---|---|\n| 1 | 2 |\n\nSome text after.";
327        let regions = MarkdownParser.parse(input);
328        // Should have: Prose, Blank, 3x Structure (table rows), Blank, Prose
329        let prose_count = regions
330            .iter()
331            .filter(|r| matches!(r, Region::Prose(_)))
332            .count();
333        let structure_count = regions
334            .iter()
335            .filter(|r| matches!(r, Region::Structure(_)))
336            .count();
337        assert_eq!(prose_count, 2);
338        assert_eq!(structure_count, 3);
339    }
340
341    #[test]
342    fn wide_table_preserved_verbatim() {
343        let input = "| Feature                         | Why excluded                                          | Follow-up article type     |\n|---------------------------------|-------------------------------------------------------|----------------------------|\n| `DraftValidation`               | LLM-assisted; needs API key, not production-reliable  | Step-by-Step Project       |";
344        let regions = MarkdownParser.parse(input);
345        assert_eq!(regions.len(), 3);
346        assert!(regions.iter().all(|r| matches!(r, Region::Structure(_))));
347        // Verify each line is preserved exactly (with trailing newline)
348        for r in &regions {
349            if let Region::Structure(s) = r {
350                assert!(s.starts_with('|'));
351                assert!(s.ends_with("|\n"));
352            }
353        }
354    }
355
356    #[test]
357    fn list_item_continuation_joined() {
358        let input = "1. First line of item\ncontinuation text here.\nAnother sentence.";
359        let regions = MarkdownParser.parse(input);
360        assert_eq!(regions[0], Region::Structure("1. ".to_string()));
361        // All three lines should be joined into one Prose region
362        assert_eq!(
363            regions[1],
364            Region::Prose(
365                "First line of item continuation text here. Another sentence.".to_string()
366            )
367        );
368        assert_eq!(regions[2], Region::Structure("\n".to_string()));
369        assert_eq!(regions.len(), 3);
370    }
371
372    #[test]
373    fn list_item_continuation_stops_at_blank() {
374        let input = "- Item one text.\ncontinuation.\n\nParagraph after.";
375        let regions = MarkdownParser.parse(input);
376        assert_eq!(regions[0], Region::Structure("- ".to_string()));
377        assert_eq!(
378            regions[1],
379            Region::Prose("Item one text. continuation.".to_string())
380        );
381        assert_eq!(regions[2], Region::Structure("\n".to_string()));
382        assert!(matches!(&regions[3], Region::BlankLines(_)));
383        assert_eq!(regions[4], Region::Prose("Paragraph after.".to_string()));
384    }
385
386    #[test]
387    fn list_item_continuation_stops_at_next_item() {
388        let input = "- First item\ncontinuation.\n- Second item";
389        let regions = MarkdownParser.parse(input);
390        // First item
391        assert_eq!(regions[0], Region::Structure("- ".to_string()));
392        assert_eq!(
393            regions[1],
394            Region::Prose("First item continuation.".to_string())
395        );
396        assert_eq!(regions[2], Region::Structure("\n".to_string()));
397        // Second item
398        assert_eq!(regions[3], Region::Structure("- ".to_string()));
399        assert_eq!(regions[4], Region::Prose("Second item".to_string()));
400        assert_eq!(regions[5], Region::Structure("\n".to_string()));
401    }
402
403    #[test]
404    fn numbered_list_with_backtick_continuation() {
405        // The exact bug from the user report
406        let input = "1. **Quality gates:** `Thresholds(warning=0.1)`\nlets you express failure rates. Replaces binary assert.";
407        let regions = MarkdownParser.parse(input);
408        assert_eq!(regions[0], Region::Structure("1. ".to_string()));
409        assert_eq!(
410            regions[1],
411            Region::Prose(
412                "**Quality gates:** `Thresholds(warning=0.1)` lets you express failure rates. Replaces binary assert.".to_string()
413            )
414        );
415        assert_eq!(regions[2], Region::Structure("\n".to_string()));
416    }
417
418    #[test]
419    fn heading_is_structure_not_prose() {
420        let input = "## My Heading";
421        let regions = MarkdownParser.parse(input);
422        assert_eq!(regions.len(), 1);
423        assert_eq!(regions[0], Region::Structure("## My Heading\n".to_string()));
424    }
425
426    #[test]
427    fn numbered_atx_heading_with_code_stays_one_line() {
428        // Regression: rtrash README / snapper-25kc — snapper -i turned
429        // `### 1. \`cargo binstall\` (preferred binary install)` into an orphan
430        // `### 1.` plus a reflowed title paragraph.
431        let input = "### 1. `cargo binstall` (preferred binary install)\n\nBody sentence one. Body sentence two.\n";
432        let regions = MarkdownParser.parse(input);
433        assert!(
434            matches!(&regions[0], Region::Structure(s) if s == "### 1. `cargo binstall` (preferred binary install)\n"),
435            "expected full ATX line as Structure, got: {:?}",
436            regions[0]
437        );
438        // Title must not appear as Prose (would be sentence-reflowed).
439        assert!(
440            !regions
441                .iter()
442                .any(|r| matches!(r, Region::Prose(p) if p.contains("cargo binstall"))),
443            "heading title must not be Prose: {regions:?}"
444        );
445    }
446
447    #[test]
448    fn atx_heading_levels_preserved_verbatim() {
449        for hashes in 1..=6 {
450            let marks = "#".repeat(hashes);
451            let line = format!("{marks} Title with `code` and (parens)");
452            let regions = MarkdownParser.parse(&line);
453            assert_eq!(
454                regions,
455                vec![Region::Structure(format!("{line}\n"))],
456                "level {hashes}"
457            );
458        }
459    }
460
461    #[test]
462    fn setext_heading_equals_is_structure() {
463        let input = "Setext Title With Period. Still Title\n=====================================\n\nBody after setext.\n";
464        let regions = MarkdownParser.parse(input);
465        assert!(
466            matches!(&regions[0], Region::Structure(s) if s == "Setext Title With Period. Still Title\n"),
467            "setext title must be Structure, got: {:?}",
468            regions[0]
469        );
470        assert!(
471            matches!(&regions[1], Region::Structure(s) if s.starts_with('=')),
472            "setext underline must be Structure, got: {:?}",
473            regions[1]
474        );
475        assert!(
476            !regions
477                .iter()
478                .any(|r| matches!(r, Region::Prose(p) if p.contains("Still Title"))),
479            "setext title must not be Prose: {regions:?}"
480        );
481    }
482
483    #[test]
484    fn setext_heading_dashes_is_structure() {
485        let input = "Secondary Setext Title\n----------------------\n\nParagraph text here.\n";
486        let regions = MarkdownParser.parse(input);
487        assert_eq!(
488            regions[0],
489            Region::Structure("Secondary Setext Title\n".to_string())
490        );
491        assert!(matches!(&regions[1], Region::Structure(s) if s.starts_with('-')));
492    }
493
494    #[test]
495    fn multi_sentence_setext_title_stays_one_line() {
496        use crate::format::Format;
497        use crate::{FormatConfig, format_text};
498
499        let input = "Setext Title With Period. Still Title\n=====================================\n\nBody after setext. Second body.\n";
500        let cfg = FormatConfig {
501            format: Format::Markdown,
502            ..Default::default()
503        };
504        let out = format_text(input, &cfg).unwrap();
505        assert!(
506            out.starts_with(
507                "Setext Title With Period. Still Title\n=====================================\n"
508            ),
509            "setext title+underline must stay intact, got:\n{out}"
510        );
511        assert!(
512            !out.contains("Still Title =====") && !out.contains("Still Title\nStill"),
513            "must not glue underline onto reflowed title:\n{out}"
514        );
515        assert_eq!(format_text(&out, &cfg).unwrap(), out);
516    }
517
518    #[test]
519    fn setext_after_prose_flushes_body() {
520        let input = "Body sentence one. Body two.\n\nHeading Here\n============\n";
521        let regions = MarkdownParser.parse(input);
522        let prose: Vec<_> = regions
523            .iter()
524            .filter_map(|r| match r {
525                Region::Prose(p) => Some(p.as_str()),
526                _ => None,
527            })
528            .collect();
529        assert!(prose.iter().any(|p| p.contains("Body sentence one")));
530        assert!(
531            regions
532                .iter()
533                .any(|r| matches!(r, Region::Structure(s) if s == "Heading Here\n"))
534        );
535    }
536}
snapper_fmt/parser/markdown.rs

snapper_fmt/parser/
markdown.rs
