cdoc_parser/ast/
parser.rs

1use crate::ast::*;
2
3use crate::raw;
4use crate::raw::{Child, ComposedMarkdown, Special};
5use anyhow::anyhow;
6use cowstr::ToCowStr;
7use pulldown_cmark::{Event, HeadingLevel, Parser as MdParser, Tag, TagEnd};
8use regex::Regex;
9use std::str::FromStr;
10
11pub(crate) enum InnerContent {
12    Blocks(Vec<Block>),
13    Inlines(Vec<Inline>),
14}
15
16impl InnerContent {
17    pub(crate) fn into_blocks(self) -> Vec<Block> {
18        if let InnerContent::Blocks(b) = self {
19            b
20        } else {
21            panic!("Expected blocks")
22        }
23    }
24
25    pub(crate) fn into_inlines(self) -> Vec<Inline> {
26        if let InnerContent::Inlines(i) = self {
27            i
28        } else {
29            panic!("Expected inlines")
30        }
31    }
32
33    pub(crate) fn blocks_mut(&mut self) -> anyhow::Result<&mut Vec<Block>> {
34        if let InnerContent::Blocks(b) = self {
35            Ok(b)
36        } else {
37            Err(anyhow!("Expected block element"))
38        }
39    }
40
41    #[allow(unused)]
42    fn inlines_mut(&mut self) -> anyhow::Result<&mut Vec<Inline>> {
43        if let InnerContent::Inlines(i) = self {
44            Ok(i)
45        } else {
46            Err(anyhow!("Expected inline element"))
47        }
48    }
49
50    pub(crate) fn push_inline(&mut self, item: Inline) {
51        match self {
52            InnerContent::Blocks(b) => b.push(Block::Plain(vec![item])),
53            InnerContent::Inlines(i) => i.push(item),
54        }
55    }
56}
57
58impl From<raw::Value> for Value {
59    fn from(value: raw::Value) -> Self {
60        match value {
61            raw::Value::Flag(f) => Value::Flag(f),
62            raw::Value::Content(c) => Value::Content(ComposedMarkdown::from(c).into()),
63            raw::Value::String(s) => Value::String(s),
64        }
65    }
66}
67
68impl From<raw::Parameter> for Parameter {
69    fn from(value: raw::Parameter) -> Self {
70        Parameter {
71            key: value.key,
72            value: value.value.into(),
73            span: value.span,
74        }
75    }
76}
77
78// impl From<raw::Reference> for Reference {
79//     fn from(value: raw::Reference) -> Self {
80//         match value {
81//             raw::Reference::Math(s) => Reference::Math(s),
82//             raw::Reference::Code(s) => Reference::Code(s),
83//             raw::Reference::Command(name, val) => Reference::Command {
84//                 function: name,
85//                 parameters: val.into_iter().map(|p| p.into()).collect(),
86//             },
87//         }
88//     }
89// }
90
91impl From<Child> for Inline {
92    fn from(value: Child) -> Self {
93        match value.elem {
94            Special::Math { inner, is_block } => Inline::Math(Math {
95                label: value.label,
96                source: inner,
97                display_block: is_block,
98                span: value.span,
99            }),
100            Special::CodeBlock {
101                inner, attributes, ..
102            } => Inline::CodeBlock(CodeBlock {
103                label: value.label,
104                source: inner,
105                attributes,
106                display_cell: false,
107                global_idx: value.identifier,
108                span: value.span,
109            }),
110            Special::CodeInline { inner } => Inline::Code(inner),
111            Special::Command {
112                function,
113                parameters,
114                body,
115            } => {
116                let parameters = parameters.into_iter().map(|p| p.into()).collect();
117                let body = body.map(|b| ComposedMarkdown::from(b).into());
118
119                Inline::Command(Command {
120                    function,
121                    label: value.label,
122                    parameters,
123                    body,
124                    span: value.span,
125                    global_idx: value.identifier,
126                })
127            }
128            Special::Verbatim { inner } => Inline::Text(inner),
129        }
130    }
131}
132
133impl From<ComposedMarkdown> for Vec<Block> {
134    fn from(composed: ComposedMarkdown) -> Self {
135        let parser: MdParser = MdParser::new(&composed.src);
136        let r = Regex::new(r"elem-([0-9]+)").expect("invalid regex expression");
137        let mut inners = vec![InnerContent::Blocks(Vec::new())];
138        // Mirrors `inners`: holds the `Tag` that opened each pushed frame, since `TagEnd` (unlike
139        // the old unified `Tag` used for both start and end in pulldown-cmark 0.9) no longer
140        // carries the heading id/classes, list start index, or link/image url and title.
141        let mut open_tags: Vec<Tag> = Vec::new();
142
143        for event in parser {
144            match event {
145                Event::Start(t) => {
146                    let opened = match &t {
147                        Tag::Paragraph
148                        | Tag::Heading { .. }
149                        | Tag::BlockQuote(_)
150                        | Tag::CodeBlock(_)
151                        | Tag::TableHead
152                        | Tag::TableRow
153                        | Tag::TableCell
154                        | Tag::Emphasis
155                        | Tag::Strong
156                        | Tag::Strikethrough
157                        | Tag::Image { .. }
158                        | Tag::Link { .. } => {
159                            inners.push(InnerContent::Inlines(Vec::new()));
160                            true
161                        }
162                        Tag::List(_) | Tag::Item | Tag::Table(_) | Tag::FootnoteDefinition(_) => {
163                            inners.push(InnerContent::Blocks(Vec::new()));
164                            true
165                        }
166                        _ => false, // TODO: Implement rest (html blocks, definition lists, etc.)
167                    };
168
169                    if opened {
170                        open_tags.push(t);
171                    }
172                }
173                Event::End(t) => {
174                    let closed = matches!(
175                        t,
176                        TagEnd::Paragraph
177                            | TagEnd::Heading(_)
178                            | TagEnd::BlockQuote(_)
179                            | TagEnd::CodeBlock
180                            | TagEnd::TableHead
181                            | TagEnd::TableRow
182                            | TagEnd::TableCell
183                            | TagEnd::Emphasis
184                            | TagEnd::Strong
185                            | TagEnd::Strikethrough
186                            | TagEnd::Image
187                            | TagEnd::Link
188                            | TagEnd::List(_)
189                            | TagEnd::Item
190                            | TagEnd::Table
191                            | TagEnd::FootnoteDefinition
192                    );
193
194                    if !closed {
195                        continue;
196                    }
197
198                    let inner = inners.pop().expect("No inner content");
199                    let start = open_tags.pop().expect("No matching open tag");
200
201                    match (t, start) {
202                        (TagEnd::Paragraph, _) => inners
203                            .last_mut()
204                            .unwrap()
205                            .blocks_mut()
206                            .expect("for paragraph")
207                            .push(Block::Paragraph(inner.into_inlines())),
208                        (TagEnd::Heading(lvl), Tag::Heading { id, classes, .. }) => inners
209                            .last_mut()
210                            .unwrap()
211                            .blocks_mut()
212                            .expect("for heading")
213                            .push(Block::Heading {
214                                lvl: heading_to_lvl(lvl),
215                                id: id.map(|s| s.to_cowstr()),
216                                classes: classes.into_iter().map(|s| s.to_cowstr()).collect(),
217                                inner: inner.into_inlines(),
218                            }),
219                        (TagEnd::BlockQuote(_), _) => inners
220                            .last_mut()
221                            .unwrap()
222                            .blocks_mut()
223                            .expect("for blockquote")
224                            .push(Block::BlockQuote(inner.into_inlines())),
225                        (TagEnd::List(_), Tag::List(idx)) => inners
226                            .last_mut()
227                            .unwrap()
228                            .blocks_mut()
229                            .expect("for list")
230                            .push(Block::List(idx, inner.into_blocks())),
231                        (TagEnd::Item, _) => inners
232                            .last_mut()
233                            .unwrap()
234                            .blocks_mut()
235                            .expect("for item")
236                            .push(Block::ListItem(inner.into_blocks())),
237                        (TagEnd::Emphasis, _) => {
238                            let src = inner.into_inlines();
239
240                            inners
241                                .last_mut()
242                                .unwrap()
243                                .push_inline(Inline::Styled(src, Style::Emphasis))
244                        }
245                        (TagEnd::Strong, _) => inners
246                            .last_mut()
247                            .unwrap()
248                            .push_inline(Inline::Styled(inner.into_inlines(), Style::Strong)),
249                        (TagEnd::Strikethrough, _) => inners.last_mut().unwrap().push_inline(
250                            Inline::Styled(inner.into_inlines(), Style::Strikethrough),
251                        ),
252                        (
253                            TagEnd::Link,
254                            Tag::Link {
255                                link_type,
256                                dest_url,
257                                title,
258                                ..
259                            },
260                        ) => inners.last_mut().unwrap().push_inline(Inline::Link(
261                            link_type,
262                            dest_url.to_cowstr(),
263                            title.to_cowstr(),
264                            inner.into_inlines(),
265                        )),
266                        (
267                            TagEnd::Image,
268                            Tag::Image {
269                                link_type,
270                                dest_url,
271                                title,
272                                ..
273                            },
274                        ) => inners.last_mut().unwrap().push_inline(Inline::Image(
275                            link_type,
276                            dest_url.to_cowstr(),
277                            title.to_cowstr(),
278                            inner.into_inlines(),
279                        )),
280                        _ => {} // TODO: Implement rest
281                    }
282                }
283                Event::Html(src) | Event::InlineHtml(src) => {
284                    let is_insert = r.captures(src.as_ref()).and_then(|c| c.get(1));
285
286                    if let Some(match_) = is_insert {
287                        let idx = usize::from_str(match_.as_str()).unwrap();
288                        let elem = composed.children[idx].clone();
289                        inners.last_mut().unwrap().push_inline(elem.into());
290                    } else {
291                        inners
292                            .last_mut()
293                            .unwrap()
294                            .push_inline(Inline::Html(src.to_cowstr()));
295                    }
296                }
297                other => {
298                    let inner = match other {
299                        Event::Text(s) => Inline::Text(s.to_cowstr()),
300                        Event::Code(s) => Inline::Code(s.to_cowstr()),
301                        Event::SoftBreak => Inline::SoftBreak,
302                        Event::HardBreak => Inline::HardBreak,
303                        Event::Rule => Inline::Rule,
304                        _ => unreachable!(),
305                    };
306
307                    let c = inners.last_mut().unwrap();
308                    c.push_inline(inner);
309                }
310            }
311        }
312        let b = inners.remove(0).into_blocks();
313        b.clone()
314    }
315}
316
317fn heading_to_lvl(value: HeadingLevel) -> u8 {
318    match value {
319        HeadingLevel::H1 => 1,
320        HeadingLevel::H2 => 2,
321        HeadingLevel::H3 => 3,
322        HeadingLevel::H4 => 4,
323        HeadingLevel::H5 => 5,
324        HeadingLevel::H6 => 6,
325    }
326}
327
328#[cfg(test)]
329mod tests {
330    use crate::ast;
331    use crate::ast::Block::ListItem;
332    use crate::ast::{Block, Command, Inline, Math, Parameter, Style, Value};
333    use crate::code_ast::types::{CodeContent, CodeElem};
334    use crate::common::Span;
335    use crate::raw::{parse_to_doc, ComposedMarkdown, Element, ElementInfo, Special};
336
337    use pulldown_cmark::LinkType;
338
339    #[test]
340    fn simple_command() {
341        let stuff = vec![
342            ElementInfo {
343                element: Element::Markdown("regular stuff ".into()),
344                span: Span::new(0, 0),
345            },
346            ElementInfo {
347                element: Element::Special(
348                    None,
349                    Special::Command {
350                        function: "func".into(),
351                        parameters: vec![],
352                        body: Some(vec![ElementInfo {
353                            element: Element::Markdown("x".into()),
354                            span: Span::new(0, 0),
355                        }]),
356                    },
357                ),
358                span: Span::new(0, 0),
359            },
360        ];
361
362        let composed = ComposedMarkdown::from(stuff);
363        let doc = Vec::from(composed);
364
365        let expected = vec![Block::Paragraph(vec![
366            Inline::Text("regular stuff ".into()),
367            Inline::Command(Command {
368                function: "func".into(),
369                label: None,
370                parameters: vec![],
371                body: Some(vec![Block::Paragraph(vec![Inline::Text("x".into())])]),
372                span: Span::new(0, 0),
373                global_idx: 0,
374            }),
375        ])];
376
377        assert_eq!(expected, doc);
378    }
379
380    #[test]
381    fn markdown_elements() {
382        let input = include_str!("../../resources/tests/markdown_elems.md");
383        let input_doc = parse_to_doc(input).expect("rawdoc parse error");
384        let composed = ComposedMarkdown::from(input_doc.src);
385        let output_doc = Vec::from(composed);
386
387        let expected = vec![
388            Block::Heading {
389                lvl: 1,
390                id: None,
391                classes: vec![],
392                inner: vec![Inline::Text("Heading".into())],
393            },
394            Block::Heading {
395                lvl: 2,
396                id: None,
397                classes: vec![],
398                inner: vec![Inline::Text("Subheading".into())],
399            },
400            Block::List(
401                None,
402                vec![
403                    ListItem(vec![Block::Plain(vec![Inline::Text(
404                        "unordered list".into(),
405                    )])]),
406                    ListItem(vec![Block::Plain(vec![Inline::Text("item 2".into())])]),
407                ],
408            ),
409            Block::List(
410                Some(1),
411                vec![
412                    ListItem(vec![Block::Plain(vec![Inline::Text(
413                        "ordered list".into(),
414                    )])]),
415                    ListItem(vec![Block::Plain(vec![Inline::Text("item 2".into())])]),
416                ],
417            ),
418            Block::Paragraph(vec![
419                Inline::Link(
420                    LinkType::Inline,
421                    "path/is/here".into(),
422                    "".into(),
423                    vec![Inline::Text("link".into())],
424                ),
425                Inline::SoftBreak,
426                Inline::Image(
427                    LinkType::Inline,
428                    "path/is/here".into(),
429                    "".into(),
430                    vec![Inline::Text("image".into())],
431                ),
432            ]),
433            Block::Paragraph(vec![
434                Inline::Styled(vec![Inline::Text("emph".into())], Style::Emphasis),
435                Inline::SoftBreak,
436                Inline::Styled(vec![Inline::Text("strong".into())], Style::Strong),
437            ]),
438            Block::Plain(vec![Inline::Code("code inline".into())]),
439            Block::Plain(vec![Inline::CodeBlock(ast::CodeBlock {
440                label: None,
441                source: CodeContent::Parsed {
442                    blocks: vec![CodeElem::Src("\ncode block\n\n".to_string())],
443                    meta: Default::default(),
444                    hash: 8014072465408005981,
445                },
446
447                display_cell: false,
448                global_idx: 0,
449                span: Span::new(180, 198),
450                attributes: vec![],
451            })]),
452            Block::Plain(vec![Inline::Math(Math {
453                label: None,
454                source: "math inline".into(),
455                display_block: false,
456                span: Span::new(200, 213),
457            })]),
458            Block::Plain(vec![Inline::Math(Math {
459                label: None,
460                source: "\nmath block\n".into(),
461                display_block: true,
462                span: Span::new(215, 231),
463            })]),
464        ];
465
466        assert_eq!(expected, output_doc);
467    }
468
469    #[test]
470    fn commands() {
471        let input = include_str!("../../resources/tests/commands.md");
472        let input_doc = parse_to_doc(input).expect("rawdoc parse error");
473        let composed = ComposedMarkdown::from(input_doc.src);
474        let output_doc = Vec::from(composed);
475
476        let expected = vec![
477            Block::Plain(vec![Inline::Command(Command {
478                function: "func".into(),
479                label: None,
480                parameters: vec![],
481                body: None,
482                span: Span::new(0, 5),
483                global_idx: 0,
484            })]),
485            Block::Plain(vec![Inline::Command(Command {
486                function: "func_param".into(),
487                label: None,
488                parameters: vec![
489                    Parameter {
490                        key: None,
491                        value: Value::String("p1".into()),
492                        span: Span::new(19, 21),
493                    },
494                    Parameter {
495                        key: Some("x".into()),
496                        value: Value::String("p2".into()),
497                        span: Span::new(23, 27),
498                    },
499                ],
500                body: None,
501                span: Span::new(7, 28),
502
503                global_idx: 1,
504            })]),
505            Block::Plain(vec![Inline::Command(Command {
506                function: "func_body".into(),
507                label: None,
508                parameters: vec![],
509                body: Some(vec![Block::Paragraph(vec![Inline::Text(
510                    "hello there".into(),
511                )])]),
512                span: Span::new(30, 55),
513                global_idx: 2,
514            })]),
515            Block::Plain(vec![Inline::Command(Command {
516                function: "func_all".into(),
517                label: None,
518                parameters: vec![
519                    Parameter {
520                        key: None,
521                        value: Value::String("p1".into()),
522                        span: Span::new(67, 69),
523                    },
524                    Parameter {
525                        key: Some("x".into()),
526                        value: Value::String("p2".into()),
527                        span: Span::new(71, 75),
528                    },
529                ],
530                body: Some(vec![Block::Paragraph(vec![Inline::Text(
531                    "hello there".into(),
532                )])]),
533                span: Span::new(57, 91),
534                global_idx: 3,
535            })]),
536            Block::Plain(vec![Inline::Command(Command {
537                function: "func_inner".into(),
538                label: None,
539                parameters: vec![],
540                body: Some(vec![
541                    Block::Plain(vec![Inline::Code("#func".into())]),
542                    Block::Plain(vec![Inline::Command(Command {
543                        function: "inner".into(),
544                        label: None,
545                        parameters: vec![],
546                        body: Some(vec![Block::Plain(vec![Inline::Math(Math {
547                            label: None,
548                            source: "math".into(),
549                            display_block: false,
550                            span: Span::new(122, 128),
551                        })])]),
552                        span: Span::new(114, 130),
553                        global_idx: 0,
554                    })]),
555                ]),
556                span: Span::new(93, 132),
557                global_idx: 4,
558            })]),
559        ];
560
561        assert_eq!(expected, output_doc);
562    }
563}
cdoc_parser/ast/parser.rs

cdoc_parser/ast/
parser.rs
