cdoc_parser/ast/
parser.rs

1use crate::ast::*;
2
3use crate::raw;
4use crate::raw::{Child, ComposedMarkdown, Special};
5use anyhow::anyhow;
6use cowstr::ToCowStr;
7use pulldown_cmark::{Event, HeadingLevel, Parser as MdParser, Tag, TagEnd};
8use regex::Regex;
9use std::str::FromStr;
10use std::sync::LazyLock;
11
12/// Matches the `elem-N` placeholders emitted by [`ComposedMarkdown`] so inline children can be
13/// spliced back in. Compiled once — this conversion runs for every document and every nested
14/// command body, so recompiling the regex each call was pure overhead.
15static ELEM_PLACEHOLDER: LazyLock<Regex> =
16    LazyLock::new(|| Regex::new(r"elem-([0-9]+)").expect("invalid regex expression"));
17
18pub(crate) enum InnerContent {
19    Blocks(Vec<Block>),
20    Inlines(Vec<Inline>),
21}
22
23impl InnerContent {
24    pub(crate) fn into_blocks(self) -> Vec<Block> {
25        if let InnerContent::Blocks(b) = self {
26            b
27        } else {
28            panic!("Expected blocks")
29        }
30    }
31
32    pub(crate) fn into_inlines(self) -> Vec<Inline> {
33        if let InnerContent::Inlines(i) = self {
34            i
35        } else {
36            panic!("Expected inlines")
37        }
38    }
39
40    pub(crate) fn blocks_mut(&mut self) -> anyhow::Result<&mut Vec<Block>> {
41        if let InnerContent::Blocks(b) = self {
42            Ok(b)
43        } else {
44            Err(anyhow!("Expected block element"))
45        }
46    }
47
48    #[allow(unused)]
49    fn inlines_mut(&mut self) -> anyhow::Result<&mut Vec<Inline>> {
50        if let InnerContent::Inlines(i) = self {
51            Ok(i)
52        } else {
53            Err(anyhow!("Expected inline element"))
54        }
55    }
56
57    pub(crate) fn push_inline(&mut self, item: Inline) {
58        match self {
59            InnerContent::Blocks(b) => b.push(Block::Plain(vec![item])),
60            InnerContent::Inlines(i) => i.push(item),
61        }
62    }
63}
64
65impl From<raw::Value> for Value {
66    fn from(value: raw::Value) -> Self {
67        match value {
68            raw::Value::Flag(f) => Value::Flag(f),
69            raw::Value::Content(c) => Value::Content(ComposedMarkdown::from(c).into()),
70            raw::Value::String(s) => Value::String(s),
71        }
72    }
73}
74
75impl From<raw::Parameter> for Parameter {
76    fn from(value: raw::Parameter) -> Self {
77        Parameter {
78            key: value.key,
79            value: value.value.into(),
80            span: value.span,
81        }
82    }
83}
84
85// impl From<raw::Reference> for Reference {
86//     fn from(value: raw::Reference) -> Self {
87//         match value {
88//             raw::Reference::Math(s) => Reference::Math(s),
89//             raw::Reference::Code(s) => Reference::Code(s),
90//             raw::Reference::Command(name, val) => Reference::Command {
91//                 function: name,
92//                 parameters: val.into_iter().map(|p| p.into()).collect(),
93//             },
94//         }
95//     }
96// }
97
98impl From<Child> for Inline {
99    fn from(value: Child) -> Self {
100        match value.elem {
101            Special::Math { inner, is_block } => Inline::Math(Math {
102                label: value.label,
103                source: inner,
104                display_block: is_block,
105                span: value.span,
106            }),
107            Special::CodeBlock {
108                inner, attributes, ..
109            } => Inline::CodeBlock(CodeBlock {
110                label: value.label,
111                source: inner,
112                attributes,
113                display_cell: false,
114                global_idx: value.identifier,
115                span: value.span,
116            }),
117            Special::CodeInline { inner } => Inline::Code(inner),
118            Special::Command {
119                function,
120                parameters,
121                body,
122            } => {
123                let parameters = parameters.into_iter().map(|p| p.into()).collect();
124                let body = body.map(|b| ComposedMarkdown::from(b).into());
125
126                Inline::Command(Command {
127                    function,
128                    label: value.label,
129                    parameters,
130                    body,
131                    span: value.span,
132                    global_idx: value.identifier,
133                })
134            }
135            Special::Verbatim { inner } => Inline::Text(inner),
136        }
137    }
138}
139
140impl From<ComposedMarkdown> for Vec<Block> {
141    fn from(composed: ComposedMarkdown) -> Self {
142        let parser: MdParser = MdParser::new(&composed.src);
143        let r = &*ELEM_PLACEHOLDER;
144        let mut inners = vec![InnerContent::Blocks(Vec::new())];
145        // Mirrors `inners`: holds the `Tag` that opened each pushed frame, since `TagEnd` (unlike
146        // the old unified `Tag` used for both start and end in pulldown-cmark 0.9) no longer
147        // carries the heading id/classes, list start index, or link/image url and title.
148        let mut open_tags: Vec<Tag> = Vec::new();
149
150        for event in parser {
151            match event {
152                Event::Start(t) => {
153                    let opened = match &t {
154                        Tag::Paragraph
155                        | Tag::Heading { .. }
156                        | Tag::BlockQuote(_)
157                        | Tag::CodeBlock(_)
158                        | Tag::TableHead
159                        | Tag::TableRow
160                        | Tag::TableCell
161                        | Tag::Emphasis
162                        | Tag::Strong
163                        | Tag::Strikethrough
164                        | Tag::Image { .. }
165                        | Tag::Link { .. } => {
166                            inners.push(InnerContent::Inlines(Vec::new()));
167                            true
168                        }
169                        Tag::List(_) | Tag::Item | Tag::Table(_) | Tag::FootnoteDefinition(_) => {
170                            inners.push(InnerContent::Blocks(Vec::new()));
171                            true
172                        }
173                        _ => false, // TODO: Implement rest (html blocks, definition lists, etc.)
174                    };
175
176                    if opened {
177                        open_tags.push(t);
178                    }
179                }
180                Event::End(t) => {
181                    let closed = matches!(
182                        t,
183                        TagEnd::Paragraph
184                            | TagEnd::Heading(_)
185                            | TagEnd::BlockQuote(_)
186                            | TagEnd::CodeBlock
187                            | TagEnd::TableHead
188                            | TagEnd::TableRow
189                            | TagEnd::TableCell
190                            | TagEnd::Emphasis
191                            | TagEnd::Strong
192                            | TagEnd::Strikethrough
193                            | TagEnd::Image
194                            | TagEnd::Link
195                            | TagEnd::List(_)
196                            | TagEnd::Item
197                            | TagEnd::Table
198                            | TagEnd::FootnoteDefinition
199                    );
200
201                    if !closed {
202                        continue;
203                    }
204
205                    let inner = inners.pop().expect("No inner content");
206                    let start = open_tags.pop().expect("No matching open tag");
207
208                    match (t, start) {
209                        (TagEnd::Paragraph, _) => inners
210                            .last_mut()
211                            .unwrap()
212                            .blocks_mut()
213                            .expect("for paragraph")
214                            .push(Block::Paragraph(inner.into_inlines())),
215                        (TagEnd::Heading(lvl), Tag::Heading { id, classes, .. }) => inners
216                            .last_mut()
217                            .unwrap()
218                            .blocks_mut()
219                            .expect("for heading")
220                            .push(Block::Heading {
221                                lvl: heading_to_lvl(lvl),
222                                id: id.map(|s| s.to_cowstr()),
223                                classes: classes.into_iter().map(|s| s.to_cowstr()).collect(),
224                                inner: inner.into_inlines(),
225                            }),
226                        (TagEnd::BlockQuote(_), _) => inners
227                            .last_mut()
228                            .unwrap()
229                            .blocks_mut()
230                            .expect("for blockquote")
231                            .push(Block::BlockQuote(inner.into_inlines())),
232                        (TagEnd::List(_), Tag::List(idx)) => inners
233                            .last_mut()
234                            .unwrap()
235                            .blocks_mut()
236                            .expect("for list")
237                            .push(Block::List(idx, inner.into_blocks())),
238                        (TagEnd::Item, _) => inners
239                            .last_mut()
240                            .unwrap()
241                            .blocks_mut()
242                            .expect("for item")
243                            .push(Block::ListItem(inner.into_blocks())),
244                        (TagEnd::Emphasis, _) => {
245                            let src = inner.into_inlines();
246
247                            inners
248                                .last_mut()
249                                .unwrap()
250                                .push_inline(Inline::Styled(src, Style::Emphasis))
251                        }
252                        (TagEnd::Strong, _) => inners
253                            .last_mut()
254                            .unwrap()
255                            .push_inline(Inline::Styled(inner.into_inlines(), Style::Strong)),
256                        (TagEnd::Strikethrough, _) => inners.last_mut().unwrap().push_inline(
257                            Inline::Styled(inner.into_inlines(), Style::Strikethrough),
258                        ),
259                        (
260                            TagEnd::Link,
261                            Tag::Link {
262                                link_type,
263                                dest_url,
264                                title,
265                                ..
266                            },
267                        ) => inners.last_mut().unwrap().push_inline(Inline::Link(
268                            link_type,
269                            dest_url.to_cowstr(),
270                            title.to_cowstr(),
271                            inner.into_inlines(),
272                        )),
273                        (
274                            TagEnd::Image,
275                            Tag::Image {
276                                link_type,
277                                dest_url,
278                                title,
279                                ..
280                            },
281                        ) => inners.last_mut().unwrap().push_inline(Inline::Image(
282                            link_type,
283                            dest_url.to_cowstr(),
284                            title.to_cowstr(),
285                            inner.into_inlines(),
286                        )),
287                        _ => {} // TODO: Implement rest
288                    }
289                }
290                Event::Html(src) | Event::InlineHtml(src) => {
291                    let is_insert = r.captures(src.as_ref()).and_then(|c| c.get(1));
292
293                    if let Some(match_) = is_insert {
294                        let idx = usize::from_str(match_.as_str()).unwrap();
295                        let elem = composed.children[idx].clone();
296                        inners.last_mut().unwrap().push_inline(elem.into());
297                    } else {
298                        inners
299                            .last_mut()
300                            .unwrap()
301                            .push_inline(Inline::Html(src.to_cowstr()));
302                    }
303                }
304                other => {
305                    let inner = match other {
306                        Event::Text(s) => Inline::Text(s.to_cowstr()),
307                        Event::Code(s) => Inline::Code(s.to_cowstr()),
308                        Event::SoftBreak => Inline::SoftBreak,
309                        Event::HardBreak => Inline::HardBreak,
310                        Event::Rule => Inline::Rule,
311                        _ => unreachable!(),
312                    };
313
314                    let c = inners.last_mut().unwrap();
315                    c.push_inline(inner);
316                }
317            }
318        }
319        inners.remove(0).into_blocks()
320    }
321}
322
323fn heading_to_lvl(value: HeadingLevel) -> u8 {
324    match value {
325        HeadingLevel::H1 => 1,
326        HeadingLevel::H2 => 2,
327        HeadingLevel::H3 => 3,
328        HeadingLevel::H4 => 4,
329        HeadingLevel::H5 => 5,
330        HeadingLevel::H6 => 6,
331    }
332}
333
334#[cfg(test)]
335mod tests {
336    use crate::ast;
337    use crate::ast::Block::ListItem;
338    use crate::ast::{Block, Command, Inline, Math, Parameter, Style, Value};
339    use crate::code_ast::types::{CodeContent, CodeElem};
340    use crate::common::Span;
341    use crate::raw::{parse_to_doc, ComposedMarkdown, Element, ElementInfo, Special};
342
343    use pulldown_cmark::LinkType;
344
345    #[test]
346    fn simple_command() {
347        let stuff = vec![
348            ElementInfo {
349                element: Element::Markdown("regular stuff ".into()),
350                span: Span::new(0, 0),
351            },
352            ElementInfo {
353                element: Element::Special(
354                    None,
355                    Special::Command {
356                        function: "func".into(),
357                        parameters: vec![],
358                        body: Some(vec![ElementInfo {
359                            element: Element::Markdown("x".into()),
360                            span: Span::new(0, 0),
361                        }]),
362                    },
363                ),
364                span: Span::new(0, 0),
365            },
366        ];
367
368        let composed = ComposedMarkdown::from(stuff);
369        let doc = Vec::from(composed);
370
371        let expected = vec![Block::Paragraph(vec![
372            Inline::Text("regular stuff ".into()),
373            Inline::Command(Command {
374                function: "func".into(),
375                label: None,
376                parameters: vec![],
377                body: Some(vec![Block::Paragraph(vec![Inline::Text("x".into())])]),
378                span: Span::new(0, 0),
379                global_idx: 0,
380            }),
381        ])];
382
383        assert_eq!(expected, doc);
384    }
385
386    #[test]
387    fn markdown_elements() {
388        let input = include_str!("../../resources/tests/markdown_elems.md");
389        let input_doc = parse_to_doc(input).expect("rawdoc parse error");
390        let composed = ComposedMarkdown::from(input_doc.src);
391        let output_doc = Vec::from(composed);
392
393        let expected = vec![
394            Block::Heading {
395                lvl: 1,
396                id: None,
397                classes: vec![],
398                inner: vec![Inline::Text("Heading".into())],
399            },
400            Block::Heading {
401                lvl: 2,
402                id: None,
403                classes: vec![],
404                inner: vec![Inline::Text("Subheading".into())],
405            },
406            Block::List(
407                None,
408                vec![
409                    ListItem(vec![Block::Plain(vec![Inline::Text(
410                        "unordered list".into(),
411                    )])]),
412                    ListItem(vec![Block::Plain(vec![Inline::Text("item 2".into())])]),
413                ],
414            ),
415            Block::List(
416                Some(1),
417                vec![
418                    ListItem(vec![Block::Plain(vec![Inline::Text(
419                        "ordered list".into(),
420                    )])]),
421                    ListItem(vec![Block::Plain(vec![Inline::Text("item 2".into())])]),
422                ],
423            ),
424            Block::Paragraph(vec![
425                Inline::Link(
426                    LinkType::Inline,
427                    "path/is/here".into(),
428                    "".into(),
429                    vec![Inline::Text("link".into())],
430                ),
431                Inline::SoftBreak,
432                Inline::Image(
433                    LinkType::Inline,
434                    "path/is/here".into(),
435                    "".into(),
436                    vec![Inline::Text("image".into())],
437                ),
438            ]),
439            Block::Paragraph(vec![
440                Inline::Styled(vec![Inline::Text("emph".into())], Style::Emphasis),
441                Inline::SoftBreak,
442                Inline::Styled(vec![Inline::Text("strong".into())], Style::Strong),
443            ]),
444            Block::Plain(vec![Inline::Code("code inline".into())]),
445            Block::Plain(vec![Inline::CodeBlock(ast::CodeBlock {
446                label: None,
447                source: CodeContent::Parsed {
448                    blocks: vec![CodeElem::Src("\ncode block\n\n".to_string())],
449                    meta: Default::default(),
450                    hash: 8014072465408005981,
451                },
452
453                display_cell: false,
454                global_idx: 0,
455                span: Span::new(180, 198),
456                attributes: vec![],
457            })]),
458            Block::Plain(vec![Inline::Math(Math {
459                label: None,
460                source: "math inline".into(),
461                display_block: false,
462                span: Span::new(200, 213),
463            })]),
464            Block::Plain(vec![Inline::Math(Math {
465                label: None,
466                source: "\nmath block\n".into(),
467                display_block: true,
468                span: Span::new(215, 231),
469            })]),
470        ];
471
472        assert_eq!(expected, output_doc);
473    }
474
475    #[test]
476    fn commands() {
477        let input = include_str!("../../resources/tests/commands.md");
478        let input_doc = parse_to_doc(input).expect("rawdoc parse error");
479        let composed = ComposedMarkdown::from(input_doc.src);
480        let output_doc = Vec::from(composed);
481
482        let expected = vec![
483            Block::Plain(vec![Inline::Command(Command {
484                function: "func".into(),
485                label: None,
486                parameters: vec![],
487                body: None,
488                span: Span::new(0, 5),
489                global_idx: 0,
490            })]),
491            Block::Plain(vec![Inline::Command(Command {
492                function: "func_param".into(),
493                label: None,
494                parameters: vec![
495                    Parameter {
496                        key: None,
497                        value: Value::String("p1".into()),
498                        span: Span::new(19, 21),
499                    },
500                    Parameter {
501                        key: Some("x".into()),
502                        value: Value::String("p2".into()),
503                        span: Span::new(23, 27),
504                    },
505                ],
506                body: None,
507                span: Span::new(7, 28),
508
509                global_idx: 1,
510            })]),
511            Block::Plain(vec![Inline::Command(Command {
512                function: "func_body".into(),
513                label: None,
514                parameters: vec![],
515                body: Some(vec![Block::Paragraph(vec![Inline::Text(
516                    "hello there".into(),
517                )])]),
518                span: Span::new(30, 55),
519                global_idx: 2,
520            })]),
521            Block::Plain(vec![Inline::Command(Command {
522                function: "func_all".into(),
523                label: None,
524                parameters: vec![
525                    Parameter {
526                        key: None,
527                        value: Value::String("p1".into()),
528                        span: Span::new(67, 69),
529                    },
530                    Parameter {
531                        key: Some("x".into()),
532                        value: Value::String("p2".into()),
533                        span: Span::new(71, 75),
534                    },
535                ],
536                body: Some(vec![Block::Paragraph(vec![Inline::Text(
537                    "hello there".into(),
538                )])]),
539                span: Span::new(57, 91),
540                global_idx: 3,
541            })]),
542            Block::Plain(vec![Inline::Command(Command {
543                function: "func_inner".into(),
544                label: None,
545                parameters: vec![],
546                body: Some(vec![
547                    Block::Plain(vec![Inline::Code("#func".into())]),
548                    Block::Plain(vec![Inline::Command(Command {
549                        function: "inner".into(),
550                        label: None,
551                        parameters: vec![],
552                        body: Some(vec![Block::Plain(vec![Inline::Math(Math {
553                            label: None,
554                            source: "math".into(),
555                            display_block: false,
556                            span: Span::new(122, 128),
557                        })])]),
558                        span: Span::new(114, 130),
559                        global_idx: 0,
560                    })]),
561                ]),
562                span: Span::new(93, 132),
563                global_idx: 4,
564            })]),
565        ];
566
567        assert_eq!(expected, output_doc);
568    }
569}
cdoc_parser/ast/parser.rs

cdoc_parser/ast/
parser.rs
