basalt_tui/note_editor/
parser.rs

1use std::ops::{Deref, DerefMut};
2
3use pulldown_cmark::{CodeBlockKind, Event, Options, Tag, TagEnd};
4
5use crate::note_editor::{
6    ast::{self, Node, SourceRange, TaskKind},
7    rich_text::{RichText, Style, TextSegment},
8};
9
10pub struct Parser<'a> {
11    events: pulldown_cmark::TextMergeWithOffset<'a, pulldown_cmark::OffsetIter<'a>>,
12    source: &'a str,
13}
14
15impl<'a> Deref for Parser<'a> {
16    type Target = pulldown_cmark::TextMergeWithOffset<'a, pulldown_cmark::OffsetIter<'a>>;
17    fn deref(&self) -> &Self::Target {
18        &self.events
19    }
20}
21
22impl DerefMut for Parser<'_> {
23    fn deref_mut(&mut self) -> &mut Self::Target {
24        &mut self.events
25    }
26}
27
28impl<'a> Iterator for Parser<'a> {
29    type Item = (Event<'a>, SourceRange<usize>);
30    fn next(&mut self) -> Option<Self::Item> {
31        self.deref_mut().next()
32    }
33}
34
35#[derive(Clone, Debug, PartialEq, Default)]
36pub struct ParserState {
37    task_kind: Vec<ast::TaskKind>,
38    item_kind: Vec<ast::ItemKind>,
39}
40
41impl<'a> Parser<'a> {
42    /// Creates a new [`Parser`] from a Markdown input string.
43    ///
44    /// The parser uses [`pulldown_cmark::Parser::new_ext`] with [`Options::all()`] and
45    /// [`pulldown_cmark::TextMergeWithOffset`] internally.
46    ///
47    /// The offset is required to know where the node appears in the provided source text.
48    pub fn new(text: &'a str) -> Self {
49        let mut options = Options::all();
50
51        // Smart punctuation is excluded because it converts ASCII characters (e.g. ", ') to
52        // multi-byte Unicode (“, ‘), making the rendered text longer than the source, causing the
53        // source offset to overlap in some cases and causing unexpected behavior.
54        //
55        // TODO: Holistic approach to support smart punctation. Potentially need to do this on a
56        // different layer of the app to only do the smart punctuation effect visually using
57        // virtual elements or such, but keeping the original source content unchanged.
58        options.remove(Options::ENABLE_SMART_PUNCTUATION);
59
60        let parser = pulldown_cmark::TextMergeWithOffset::new(
61            pulldown_cmark::Parser::new_ext(text, options).into_offset_iter(),
62        );
63
64        Self {
65            events: parser,
66            source: text,
67        }
68    }
69
70    /// A thematic break (`---`, or a table's leftover dash row once the pipes are gone) is kept as
71    /// a plain paragraph so its source stays visible and editable. Without this it parses as a
72    /// [`pulldown_cmark::Event::Rule`], which carries no text and would leave that line uncovered
73    /// by any node — invisible and impossible to fix. Horizontal rules are not rendered specially.
74    fn rule_node(&self, source_range: SourceRange<usize>) -> Node {
75        let text = self
76            .source
77            .get(source_range.clone())
78            .unwrap_or("")
79            .trim_end_matches('\n');
80        Node::Paragraph {
81            text: RichText::from(vec![TextSegment::plain(text)]),
82            source_range,
83        }
84    }
85
86    pub fn parse(mut self) -> Vec<Node> {
87        let mut result = Vec::new();
88        let mut state = ParserState::default();
89
90        while let Some((event, source_range)) = self.next() {
91            match event {
92                Event::Start(Tag::Table(alignments)) => {
93                    result.push(self.parse_table(alignments, source_range));
94                }
95                Event::Start(tag) if Self::is_container_tag(&tag) => {
96                    if let Some(node) = self.parse_container(tag, &mut state) {
97                        result.push(node);
98                    }
99                }
100                Event::Rule => result.push(self.rule_node(source_range)),
101                _ => {}
102            }
103        }
104
105        result
106    }
107
108    pub fn parse_container(&mut self, tag: Tag, state: &mut ParserState) -> Option<Node> {
109        let mut nodes = Vec::new();
110        let mut text_segments = Vec::new();
111        let mut inline_styles = Vec::new();
112
113        match tag {
114            Tag::List(Some(start)) => {
115                state.item_kind.push(ast::ItemKind::Ordered(start));
116            }
117            Tag::List(..) => {
118                state.item_kind.push(ast::ItemKind::Unordered);
119            }
120            _ => {}
121        };
122
123        while let Some((event, source_range)) = self.next() {
124            match event {
125                Event::Start(Tag::Table(alignments)) => {
126                    nodes.push(self.parse_table(alignments, source_range));
127                }
128
129                Event::Start(inner_tag) if Self::is_container_tag(&inner_tag) => {
130                    if let Some(node) = self.parse_container(inner_tag, state) {
131                        nodes.push(node);
132                    }
133                }
134
135                Event::Start(inner_tag) if Self::is_inline_tag(&inner_tag) => {
136                    if let Some(style) = Self::tag_to_style(&inner_tag) {
137                        inline_styles.push(style);
138                    }
139                }
140
141                Event::Rule => nodes.push(self.rule_node(source_range)),
142
143                Event::TaskListMarker(checked) => {
144                    state.task_kind.push(if checked {
145                        TaskKind::Checked
146                    } else {
147                        TaskKind::Unchecked
148                    });
149                }
150
151                Event::Code(text) => {
152                    let text_segment = TextSegment::styled(&text, Style::Code);
153                    text_segments.push(text_segment);
154                }
155
156                Event::Text(text) => {
157                    let mut text_segment = TextSegment::plain(&text);
158                    inline_styles.iter().for_each(|style| {
159                        text_segment.add_style(style);
160                    });
161                    text_segments.push(text_segment);
162                }
163
164                Event::SoftBreak => {
165                    let text_segment = TextSegment::empty_line();
166                    text_segments.push(text_segment);
167                }
168
169                Event::End(tag_end) if Self::tags_match(&tag, &tag_end) => {
170                    let text = if !text_segments.is_empty() {
171                        RichText::from(text_segments)
172                    } else {
173                        RichText::empty()
174                    };
175
176                    return match tag {
177                        Tag::Heading { level, .. } => Some(Node::Heading {
178                            level: level.into(),
179                            text,
180                            source_range,
181                        }),
182                        Tag::Item => {
183                            // This is required since in block quotes list items are considered
184                            // "tight", thus the text is not stored in a paragraph directly.
185                            // TODO: Think if wrapping this into a paragraph is a good idea or not.
186                            // Potentially storing a RichText here is better.
187                            if !text.is_empty() {
188                                nodes.insert(
189                                    0,
190                                    Node::Paragraph {
191                                        text,
192                                        source_range: source_range.clone(),
193                                    },
194                                );
195                            }
196
197                            let item = if let Some(kind) = state.task_kind.pop() {
198                                Some(Node::Task {
199                                    kind,
200                                    nodes,
201                                    source_range,
202                                })
203                            } else {
204                                Some(Node::Item {
205                                    kind: state
206                                        .item_kind
207                                        .last()
208                                        .cloned()
209                                        .unwrap_or(ast::ItemKind::Unordered),
210                                    nodes,
211                                    source_range,
212                                })
213                            };
214
215                            if let Some(ast::ItemKind::Ordered(start)) = state.item_kind.last_mut()
216                            {
217                                *start += 1;
218                            };
219
220                            item
221                        }
222                        Tag::List(..) => {
223                            state.item_kind.pop();
224
225                            Some(Node::List {
226                                nodes,
227                                source_range,
228                            })
229                        }
230                        Tag::CodeBlock(kind) => Some(Node::CodeBlock {
231                            lang: match kind {
232                                CodeBlockKind::Fenced(lang) => Some(lang.to_string()),
233                                _ => None,
234                            },
235                            text,
236                            source_range,
237                        }),
238                        Tag::BlockQuote(kind) => {
239                            let (kind, title, nodes) =
240                                resolve_callout(kind.map(|kind| kind.into()), nodes);
241                            Some(Node::BlockQuote {
242                                kind,
243                                title,
244                                nodes,
245                                source_range,
246                            })
247                        }
248                        Tag::Paragraph => Some(Node::Paragraph { text, source_range }),
249                        _ => None,
250                    };
251                }
252                _ => {}
253            }
254        }
255
256        None
257    }
258
259    /// Parses a table from the events between `Start(Table)` and `End(Table)`.
260    ///
261    /// Cells before the first `TableRow` belong to the header; the rest form body rows. Inline
262    /// styles are tracked per cell so emphasis inside a cell is preserved.
263    fn parse_table(
264        &mut self,
265        alignments: Vec<pulldown_cmark::Alignment>,
266        source_range: SourceRange<usize>,
267    ) -> Node {
268        let alignments = alignments.into_iter().map(Into::into).collect();
269        let mut head = Vec::new();
270        let mut rows = Vec::new();
271        let mut row = Vec::new();
272        let mut cell = Vec::new();
273        let mut inline_styles = Vec::new();
274        let mut in_head = false;
275
276        for (event, _) in self.by_ref() {
277            match event {
278                Event::Start(Tag::TableHead) => in_head = true,
279                Event::Start(Tag::TableRow) => in_head = false,
280                Event::Start(Tag::TableCell) => {
281                    cell = Vec::new();
282                    inline_styles.clear();
283                }
284                Event::Start(inner_tag) if Self::is_inline_tag(&inner_tag) => {
285                    if let Some(style) = Self::tag_to_style(&inner_tag) {
286                        inline_styles.push(style);
287                    }
288                }
289                Event::End(TagEnd::Emphasis | TagEnd::Strong | TagEnd::Strikethrough) => {
290                    inline_styles.pop();
291                }
292                Event::Code(text) => cell.push(TextSegment::styled(&text, Style::Code)),
293                Event::Text(text) => {
294                    let mut text_segment = TextSegment::plain(&text);
295                    inline_styles
296                        .iter()
297                        .for_each(|style| text_segment.add_style(style));
298                    cell.push(text_segment);
299                }
300                Event::End(TagEnd::TableCell) => {
301                    let text = RichText::from(std::mem::take(&mut cell));
302                    if in_head {
303                        head.push(text);
304                    } else {
305                        row.push(text);
306                    }
307                }
308                Event::End(TagEnd::TableRow) => rows.push(std::mem::take(&mut row)),
309                Event::End(TagEnd::Table) => break,
310                _ => {}
311            }
312        }
313
314        Node::Table {
315            alignments,
316            head,
317            rows,
318            source_range,
319        }
320    }
321
322    fn is_container_tag(tag: &Tag) -> bool {
323        matches!(
324            tag,
325            Tag::Paragraph
326                | Tag::Item
327                | Tag::List(..)
328                | Tag::BlockQuote(..)
329                | Tag::CodeBlock(..)
330                | Tag::Heading { .. }
331        )
332    }
333
334    fn is_inline_tag(tag: &Tag) -> bool {
335        matches!(tag, Tag::Emphasis | Tag::Strong | Tag::Strikethrough)
336    }
337
338    fn tags_match(start: &Tag, end: &TagEnd) -> bool {
339        fn tag_to_end(tag: &Tag) -> Option<TagEnd> {
340            match tag {
341                Tag::Heading { level, .. } => Some(TagEnd::Heading(*level)),
342                Tag::List(ordered) => Some(TagEnd::List(ordered.is_some())),
343                Tag::Item => Some(TagEnd::Item),
344                Tag::BlockQuote(kind) => Some(TagEnd::BlockQuote(*kind)),
345                Tag::CodeBlock(..) => Some(TagEnd::CodeBlock),
346                Tag::Paragraph => Some(TagEnd::Paragraph),
347                _ => None,
348            }
349        }
350
351        if let Some(start) = tag_to_end(start) {
352            std::mem::discriminant(&start) == std::mem::discriminant(end)
353        } else {
354            false
355        }
356    }
357
358    fn tag_to_style(tag: &Tag) -> Option<Style> {
359        match tag {
360            Tag::Emphasis => Some(Style::Emphasis),
361            Tag::Strong => Some(Style::Strong),
362            Tag::Strikethrough => Some(Style::Strikethrough),
363            _ => None,
364        }
365    }
366}
367
368/// Resolves a quote's callout kind and title. `pulldown_cmark` handles bare
369/// GitHub alerts (`kind` is `Some`); otherwise we look for an Obsidian-style
370/// marker on the first line and strip that line from the body.
371fn resolve_callout(
372    kind: Option<ast::BlockQuoteKind>,
373    mut nodes: Vec<Node>,
374) -> (Option<ast::BlockQuoteKind>, Option<String>, Vec<Node>) {
375    if kind.is_some() {
376        return (kind, None, nodes);
377    }
378
379    let stripped = match nodes.first() {
380        Some(Node::Paragraph { text, source_range }) => {
381            let segments = text.segments();
382            let break_index = segments.iter().position(|segment| segment.content == "\n");
383            let first_line: String = segments[..break_index.unwrap_or(segments.len())]
384                .iter()
385                .map(|segment| segment.content.as_str())
386                .collect();
387            ast::parse_callout_marker(&first_line).map(|marker| {
388                let body = break_index
389                    .map(|index| segments[index + 1..].to_vec())
390                    .unwrap_or_default();
391                (marker, body, source_range.clone())
392            })
393        }
394        _ => None,
395    };
396
397    let Some((marker, body, source_range)) = stripped else {
398        return (None, None, nodes);
399    };
400
401    if body.is_empty() {
402        nodes.remove(0);
403    } else {
404        nodes[0] = Node::Paragraph {
405            text: RichText::from(body),
406            source_range,
407        };
408    }
409
410    (Some(marker.kind), marker.title, nodes)
411}
412
413pub fn from_str(text: &str) -> Vec<Node> {
414    Parser::new(text).parse()
415}
416
417#[cfg(test)]
418mod tests {
419    use indoc::indoc;
420    use insta::assert_snapshot;
421
422    use super::*;
423
424    #[test]
425    fn test_parser() {
426        let tests = [
427            (
428                "paragraphs",
429                indoc! { r#"## Paragraphs
430                To create paragraphs in Markdown, use a **blank line** to separate blocks of text. Each block of text separated by a blank line is treated as a distinct paragraph.
431
432                This is a paragraph.
433
434                This is another paragraph.
435
436                A blank line between lines of text creates separate paragraphs. This is the default behavior in Markdown.
437                "#},
438            ),
439            (
440                "headings",
441                indoc! { r#"## Headings
442                To create a heading, add up to six `#` symbols before your heading text. The number of `#` symbols determines the size of the heading.
443
444                # This is a heading 1
445                ## This is a heading 2
446                ### This is a heading 3
447                #### This is a heading 4
448                ##### This is a heading 5
449                ###### This is a heading 6
450                "#},
451            ),
452            (
453                "lists",
454                indoc! { r#"## Lists
455                You can create an unordered list by adding a `-`, `*`, or `+` before the text.
456
457                - First list item
458                - Second list item
459                - Third list item
460
461                To create an ordered list, start each line with a number followed by a `.` or `)` symbol.
462
463                1. First list item
464                2. Second list item
465                3. Third list item
466
467                1) First list item
468                2) Second list item
469                3) Third list item
470                "#},
471            ),
472            (
473                "lists_line_breaks",
474                indoc! { r#"## Lists with line breaks
475                You can use line breaks within an ordered list without altering the numbering.
476
477                1. First list item
478
479                2. Second list item
480                3. Third list item
481
482                4. Fourth list item
483                5. Fifth list item
484                6. Sixth list item
485                "#},
486            ),
487            (
488                "task_lists",
489                indoc! { r#"## Task lists
490                To create a task list, start each list item with a hyphen and space followed by `[ ]`.
491
492                - [x] This is a completed task.
493                - [ ] This is an incomplete task.
494
495                You can toggle a task in Reading view by selecting the checkbox.
496
497                > [!tip]
498                > You can use any character inside the brackets to mark it as complete.
499                >
500                > - [x] Milk
501                > - [?] Eggs
502                > - [-] Eggs
503                "#},
504            ),
505            (
506                "callouts",
507                indoc! { r#"## Callouts
508
509                > [!NOTE]
510                > Strict GitHub form.
511
512                > [!summary] Aliased to abstract
513                > Body line.
514
515                > [!danger]- Foldable with a title
516                > Body line.
517
518                > [!custom] Unknown defaults to note
519                > Body line.
520
521                > A plain quote, not a callout.
522                "#},
523            ),
524            (
525                "nesting_lists",
526                indoc! { r#"## Nesting lists
527                You can nest any type of list—ordered, unordered, or task lists—under any other type of list.
528
529                To create a nested list, indent one or more list items. You can mix list types within a nested structure:
530
531                1. First list item
532                   1. Ordered nested list item
533                2. Second list item
534                   - Unordered nested list item
535                "#},
536            ),
537            (
538                "nesting_task_lists",
539                indoc! { r#"## Nesting task lists
540                Similarly, you can create a nested task list by indenting one or more list items:
541
542                - [ ] Task item 1
543                  - [ ] Subtask 1
544                - [ ] Task item 2
545                  - [ ] Subtask 2
546                "#},
547            ),
548            (
549                // A thematic break (and a table's leftover dash row once the pipes are
550                // deleted) is kept as a plain paragraph so its source stays visible and
551                // editable rather than vanishing.
552                "horizontal_rule",
553                indoc! { r#"## Horizontal rule
554
555                ---
556
557                A broken table degrades to a dash row, which must stay visible:
558
559                First Header | Second Header
560                ------------ ------------
561                Content | Content
562                "#},
563            ),
564            (
565                "code_blocks",
566                indoc! { r#"## Code blocks
567                To format code as a block, enclose it with three backticks or three tildes.
568
569                ```md
570                cd ~/Desktop
571                ```
572
573                You can also create a code block by indenting the text using `Tab` or 4 blank spaces.
574
575                    cd ~/Desktop
576
577                "#},
578            ),
579            (
580                "tables",
581                indoc! { r#"## Tables
582                You can create a table by separating columns with `|` and the header from the body with a row of dashes.
583
584                | Name  | Role      | Notes                       |
585                | :---- | :-------: | --------------------------: |
586                | Alice | Maintainer | Writes most of the **core** code |
587                | Bob   | Reviewer  | `reviews`                   |
588                "#},
589            ),
590            (
591                "code_syntax_highlighting_in_blocks",
592                indoc! { r#"## Code syntax highlighting in blocks
593                You can add syntax highlighting to a code block, by adding a language code after the first set of backticks.
594
595                ```js
596                function fancyAlert(arg) {
597                  if(arg) {
598                    $.facebox({div:'#foo'})
599                  }
600                }
601                ```
602                "#},
603            ),
604        ];
605
606        tests.into_iter().for_each(|(name, text)| {
607            assert_snapshot!(
608                name,
609                format!(
610                    "{}\n ---\n\n{}",
611                    text,
612                    ast::nodes_to_sexp(&from_str(text), 0)
613                )
614            );
615        });
616    }
617}
basalt_tui/note_editor/parser.rs

basalt_tui/note_editor/
parser.rs
