sim_codec_doc/
markdown.rs

1//! Markdown backend implementation over `pulldown-cmark`.
2
3use std::collections::BTreeMap;
4use std::mem;
5use std::ops::Range;
6
7use pulldown_cmark::{CodeBlockKind, Event, HeadingLevel, Options, Parser, Tag, TagEnd};
8
9use crate::backend::{
10    MarkupBackend, MarkupDecodeOptions, MarkupEncodeOptions, MarkupError, MarkupFidelity,
11    MarkupLoss,
12};
13use crate::markdown_writer::MarkdownEncoder;
14use crate::markup::{BackendId, Inline, MarkupBlock, MarkupDoc, MathSource, SourceDoc, Span};
15
16type MarkdownEvent = (Event<'static>, Range<usize>);
17
18/// CommonMark/GFM-compatible Markdown backend.
19#[derive(Clone, Debug, Default)]
20pub struct MarkdownBackend;
21
22impl MarkupBackend for MarkdownBackend {
23    fn id(&self) -> BackendId {
24        markdown_id()
25    }
26
27    fn decode(
28        &self,
29        input: &str,
30        opts: &MarkupDecodeOptions,
31    ) -> Result<(MarkupDoc, MarkupFidelity), MarkupError> {
32        let events = Parser::new_ext(input, markdown_options())
33            .into_offset_iter()
34            .map(|(event, range)| (event.into_static(), range))
35            .collect();
36        let mut parser = MarkdownParser::new(input, events, opts);
37        let blocks = parser.parse_blocks_until(|_| false);
38        let title = blocks.iter().find_map(|block| match block {
39            MarkupBlock::Heading { level: 1, text, .. } => Some(inline_plain_text(text)),
40            _ => None,
41        });
42        let source = opts.preserve_source.then(|| SourceDoc {
43            backend: markdown_id(),
44            text: input.to_owned(),
45        });
46        Ok((
47            MarkupDoc {
48                title,
49                blocks,
50                attrs: BTreeMap::new(),
51                source,
52            },
53            parser.fidelity,
54        ))
55    }
56
57    fn encode(
58        &self,
59        doc: &MarkupDoc,
60        opts: &MarkupEncodeOptions,
61    ) -> Result<(String, MarkupFidelity), MarkupError> {
62        let mut encoder = MarkdownEncoder::new(opts);
63        let source = encoder.write_doc(doc);
64        if opts.fail_on_loss && !encoder.fidelity.dropped.is_empty() {
65            return Err(MarkupError::Encode(format!(
66                "markdown encode dropped {} raw fragment(s)",
67                encoder.fidelity.dropped.len()
68            )));
69        }
70        Ok((source, encoder.fidelity))
71    }
72}
73
74struct MarkdownParser<'a> {
75    input: &'a str,
76    events: Vec<MarkdownEvent>,
77    index: usize,
78    preserve_raw: bool,
79    fidelity: MarkupFidelity,
80}
81
82impl<'a> MarkdownParser<'a> {
83    fn new(input: &'a str, events: Vec<MarkdownEvent>, opts: &MarkupDecodeOptions) -> Self {
84        Self {
85            input,
86            events,
87            index: 0,
88            preserve_raw: opts.preserve_raw,
89            fidelity: MarkupFidelity::exact(markdown_id()),
90        }
91    }
92
93    fn parse_blocks_until<F>(&mut self, stop: F) -> Vec<MarkupBlock>
94    where
95        F: Fn(&TagEnd) -> bool + Copy,
96    {
97        let mut blocks = Vec::new();
98        let mut loose = Vec::new();
99        while let Some((event, range)) = self.next() {
100            match event {
101                Event::End(end) if stop(&end) => {
102                    self.index -= 1;
103                    break;
104                }
105                Event::End(_) => self.flush_loose(&mut blocks, &mut loose),
106                Event::Start(tag) => {
107                    self.flush_loose(&mut blocks, &mut loose);
108                    self.push_block(tag, range, &mut blocks);
109                }
110                Event::DisplayMath(text) => {
111                    self.flush_loose(&mut blocks, &mut loose);
112                    blocks.push(MarkupBlock::MathBlock {
113                        source: tex_math(text),
114                        span: Some(span(range.start, range.end)),
115                    });
116                }
117                Event::Rule => {
118                    self.flush_loose(&mut blocks, &mut loose);
119                    if let Some(raw) = self.raw_block(self.slice(&range), "rule", &range) {
120                        blocks.push(raw);
121                    }
122                }
123                other => self.push_inline_event(other, range, &mut loose),
124            }
125        }
126        self.flush_loose(&mut blocks, &mut loose);
127        blocks
128    }
129
130    fn push_block(
131        &mut self,
132        tag: Tag<'static>,
133        range: Range<usize>,
134        blocks: &mut Vec<MarkupBlock>,
135    ) {
136        match tag {
137            Tag::Paragraph => blocks.push(self.parse_paragraph(range, Vec::new())),
138            Tag::Heading { level, id, .. } => blocks.push(self.parse_heading(level, id, range)),
139            Tag::CodeBlock(kind) => blocks.push(self.parse_code_block(kind, range)),
140            Tag::BlockQuote(_) => blocks.push(self.parse_quote(range)),
141            Tag::List(start) => blocks.push(self.parse_list(start.is_some(), range)),
142            Tag::Table(_) => blocks.push(self.parse_table(range)),
143            Tag::HtmlBlock => {
144                if let Some(raw) = self.raw_container(range, TagEnd::HtmlBlock, "html-block") {
145                    blocks.push(raw);
146                }
147            }
148            Tag::FootnoteDefinition(_) => {
149                if let Some(raw) =
150                    self.raw_container(range, TagEnd::FootnoteDefinition, "footnote-definition")
151                {
152                    blocks.push(raw);
153                }
154            }
155            other => {
156                let end = other.to_end();
157                if let Some(raw) = self.raw_container(range, end, "unsupported-block") {
158                    blocks.push(raw);
159                }
160            }
161        }
162    }
163
164    fn parse_paragraph(&mut self, start: Range<usize>, mut prefix: Vec<Inline>) -> MarkupBlock {
165        let (mut content, end) = self.collect_inlines_until(|end| *end == TagEnd::Paragraph);
166        prefix.append(&mut content);
167        if prefix.len() == 1
168            && matches!(prefix[0], Inline::Math(_))
169            && self
170                .input
171                .get(start.start..end.end)
172                .map(str::trim)
173                .is_some_and(|source| source.starts_with("$$") && source.ends_with("$$"))
174        {
175            let Inline::Math(source) = prefix.remove(0) else {
176                unreachable!();
177            };
178            return MarkupBlock::MathBlock {
179                source,
180                span: Some(span(start.start, end.end)),
181            };
182        }
183        MarkupBlock::Paragraph {
184            content: prefix,
185            span: Some(span(start.start, end.end)),
186        }
187    }
188
189    fn parse_heading(
190        &mut self,
191        level: HeadingLevel,
192        id: Option<pulldown_cmark::CowStr<'static>>,
193        start: Range<usize>,
194    ) -> MarkupBlock {
195        let (text, end) = self.collect_inlines_until(|end| matches!(end, TagEnd::Heading(_)));
196        MarkupBlock::Heading {
197            level: heading_level(level),
198            text,
199            id: id.map(|value| value.to_string()),
200            span: Some(span(start.start, end.end)),
201        }
202    }
203
204    fn parse_code_block(
205        &mut self,
206        kind: CodeBlockKind<'static>,
207        start: Range<usize>,
208    ) -> MarkupBlock {
209        let mut code = String::new();
210        let mut end = start.clone();
211        while let Some((event, range)) = self.next() {
212            end = range.clone();
213            match event {
214                Event::End(TagEnd::CodeBlock) => break,
215                Event::Text(text) => code.push_str(&text),
216                Event::Code(text) => code.push_str(&text),
217                Event::SoftBreak | Event::HardBreak => code.push('\n'),
218                _ => {}
219            }
220        }
221        let lang = match kind {
222            CodeBlockKind::Fenced(info) => info.split_whitespace().next().map(str::to_owned),
223            CodeBlockKind::Indented => None,
224        };
225        if matches!(lang.as_deref(), Some("math" | "tex")) {
226            MarkupBlock::MathBlock {
227                source: MathSource {
228                    notation: "tex".to_owned(),
229                    text: code,
230                },
231                span: Some(span(start.start, end.end)),
232            }
233        } else {
234            MarkupBlock::CodeBlock {
235                lang,
236                code,
237                span: Some(span(start.start, end.end)),
238            }
239        }
240    }
241
242    fn parse_quote(&mut self, start: Range<usize>) -> MarkupBlock {
243        let blocks = self.parse_blocks_until(|end| matches!(end, TagEnd::BlockQuote(_)));
244        let end = self.consume_end(|end| matches!(end, TagEnd::BlockQuote(_)), &start);
245        MarkupBlock::Quote {
246            blocks,
247            span: Some(span(start.start, end.end)),
248        }
249    }
250
251    fn parse_list(&mut self, ordered: bool, start: Range<usize>) -> MarkupBlock {
252        let mut items = Vec::new();
253        let mut end = start.clone();
254        while let Some((event, range)) = self.next() {
255            end = range.clone();
256            match event {
257                Event::End(TagEnd::List(_)) => break,
258                Event::Start(Tag::Item) => {
259                    let item = self.parse_blocks_until(|end| *end == TagEnd::Item);
260                    end = self.consume_end(|end| *end == TagEnd::Item, &range);
261                    items.push(item);
262                }
263                _ => {}
264            }
265        }
266        MarkupBlock::List {
267            ordered,
268            items,
269            span: Some(span(start.start, end.end)),
270        }
271    }
272
273    fn parse_table(&mut self, start: Range<usize>) -> MarkupBlock {
274        let mut header = Vec::new();
275        let mut rows = Vec::new();
276        let mut current_row: Option<Vec<Vec<Inline>>> = None;
277        let mut in_head = false;
278        let mut end = start.clone();
279        while let Some((event, range)) = self.next() {
280            end = range.clone();
281            match event {
282                Event::End(TagEnd::Table) => break,
283                Event::Start(Tag::TableHead) => {
284                    in_head = true;
285                    current_row = Some(Vec::new());
286                }
287                Event::End(TagEnd::TableHead) => {
288                    if let Some(row) = current_row.take() {
289                        header = row;
290                    }
291                    in_head = false;
292                }
293                Event::Start(Tag::TableRow) => current_row = Some(Vec::new()),
294                Event::End(TagEnd::TableRow) => {
295                    if let Some(row) = current_row.take() {
296                        if in_head {
297                            header = row;
298                        } else {
299                            rows.push(row);
300                        }
301                    }
302                }
303                Event::Start(Tag::TableCell) => {
304                    let (cell, cell_end) =
305                        self.collect_inlines_until(|end| *end == TagEnd::TableCell);
306                    end = cell_end;
307                    current_row.get_or_insert_with(Vec::new).push(cell);
308                }
309                _ => {}
310            }
311        }
312        MarkupBlock::Table {
313            header,
314            rows,
315            span: Some(span(start.start, end.end)),
316        }
317    }
318
319    fn collect_inlines_until<F>(&mut self, stop: F) -> (Vec<Inline>, Range<usize>)
320    where
321        F: Fn(&TagEnd) -> bool + Copy,
322    {
323        let mut items = Vec::new();
324        let mut end = self.current_end();
325        while let Some((event, range)) = self.next() {
326            end = range.clone();
327            match event {
328                Event::End(tag_end) if stop(&tag_end) => break,
329                other => self.push_inline_event(other, range, &mut items),
330            }
331        }
332        (items, end)
333    }
334
335    fn push_inline_event(
336        &mut self,
337        event: Event<'static>,
338        range: Range<usize>,
339        items: &mut Vec<Inline>,
340    ) {
341        match event {
342            Event::Text(text) => items.push(Inline::Text(text.to_string())),
343            Event::Code(text) => items.push(Inline::Code(text.to_string())),
344            Event::InlineMath(text) | Event::DisplayMath(text) => {
345                items.push(Inline::Math(tex_math(text)));
346            }
347            Event::SoftBreak | Event::HardBreak => items.push(Inline::Text("\n".to_owned())),
348            Event::Html(text) | Event::InlineHtml(text) => {
349                if let Some(raw) = self.raw_inline(text.to_string(), "html") {
350                    items.push(raw);
351                }
352            }
353            Event::FootnoteReference(label) => {
354                if let Some(raw) = self.raw_inline(format!("[^{label}]"), "footnote-reference") {
355                    items.push(raw);
356                }
357            }
358            Event::TaskListMarker(checked) => {
359                let marker = if checked { "[x] " } else { "[ ] " };
360                if let Some(raw) = self.raw_inline(marker.to_owned(), "task-list-marker") {
361                    items.push(raw);
362                }
363            }
364            Event::Rule => {
365                if let Some(raw) = self.raw_inline(self.slice(&range), "rule") {
366                    items.push(raw);
367                }
368            }
369            Event::Start(Tag::Emphasis) => {
370                let (children, _) = self.collect_inlines_until(|end| *end == TagEnd::Emphasis);
371                items.push(Inline::Emph(children));
372            }
373            Event::Start(Tag::Strong) => {
374                let (children, _) = self.collect_inlines_until(|end| *end == TagEnd::Strong);
375                items.push(Inline::Strong(children));
376            }
377            Event::Start(Tag::Link { dest_url, .. }) => {
378                let (label, _) = self.collect_inlines_until(|end| *end == TagEnd::Link);
379                items.push(Inline::Link {
380                    label,
381                    target: dest_url.to_string(),
382                });
383            }
384            Event::Start(tag) => {
385                let end = tag.to_end();
386                if let Some(raw) = self.raw_inline_container(range, end, "unsupported-inline") {
387                    items.push(raw);
388                }
389            }
390            Event::End(_) => {}
391        }
392    }
393
394    fn raw_container(
395        &mut self,
396        start: Range<usize>,
397        target: TagEnd,
398        path: &str,
399    ) -> Option<MarkupBlock> {
400        let (raw, end) = self.consume_raw_container(start.clone(), target);
401        self.raw_block(raw, path, &(start.start..end))
402    }
403
404    fn raw_inline_container(
405        &mut self,
406        start: Range<usize>,
407        target: TagEnd,
408        path: &str,
409    ) -> Option<Inline> {
410        let (raw, _) = self.consume_raw_container(start, target);
411        self.raw_inline(raw, path)
412    }
413
414    fn consume_raw_container(&mut self, start: Range<usize>, target: TagEnd) -> (String, usize) {
415        let mut depth = 1usize;
416        let mut end = start.end;
417        while let Some((event, range)) = self.next() {
418            end = range.end;
419            match event {
420                Event::Start(tag) if tag.to_end() == target => depth += 1,
421                Event::End(tag_end) if tag_end == target => {
422                    depth -= 1;
423                    if depth == 0 {
424                        break;
425                    }
426                }
427                _ => {}
428            }
429        }
430        (
431            self.input.get(start.start..end).unwrap_or("").to_owned(),
432            end,
433        )
434    }
435
436    fn raw_block(&mut self, raw: String, path: &str, range: &Range<usize>) -> Option<MarkupBlock> {
437        if self.preserve_raw {
438            self.fidelity.preserved_raw.push(raw.clone());
439            Some(MarkupBlock::Raw {
440                backend: markdown_id(),
441                text: raw,
442                span: Some(span(range.start, range.end)),
443            })
444        } else {
445            self.drop_raw(path, "unsupported markdown block");
446            None
447        }
448    }
449
450    fn raw_inline(&mut self, raw: String, path: &str) -> Option<Inline> {
451        if self.preserve_raw {
452            self.fidelity.preserved_raw.push(raw.clone());
453            Some(Inline::Raw {
454                backend: markdown_id(),
455                text: raw,
456            })
457        } else {
458            self.drop_raw(path, "unsupported markdown inline");
459            None
460        }
461    }
462
463    fn drop_raw(&mut self, path: &str, reason: &str) {
464        self.fidelity.dropped.push(MarkupLoss {
465            path: path.to_owned(),
466            reason: reason.to_owned(),
467        });
468    }
469
470    fn flush_loose(&mut self, blocks: &mut Vec<MarkupBlock>, loose: &mut Vec<Inline>) {
471        if !loose.is_empty() {
472            blocks.push(MarkupBlock::Paragraph {
473                content: mem::take(loose),
474                span: None,
475            });
476        }
477    }
478
479    fn consume_end<F>(&mut self, stop: F, fallback: &Range<usize>) -> Range<usize>
480    where
481        F: Fn(&TagEnd) -> bool,
482    {
483        match self.next() {
484            Some((Event::End(end), range)) if stop(&end) => range,
485            Some(_) => fallback.clone(),
486            None => fallback.clone(),
487        }
488    }
489
490    fn next(&mut self) -> Option<MarkdownEvent> {
491        let event = self.events.get(self.index).cloned();
492        if event.is_some() {
493            self.index += 1;
494        }
495        event
496    }
497
498    fn current_end(&self) -> Range<usize> {
499        self.events
500            .get(self.index.saturating_sub(1))
501            .map(|(_, range)| range.clone())
502            .unwrap_or(0..0)
503    }
504
505    fn slice(&self, range: &Range<usize>) -> String {
506        self.input.get(range.clone()).unwrap_or("").to_owned()
507    }
508}
509
510fn markdown_options() -> Options {
511    let mut options = Options::empty();
512    options.insert(Options::ENABLE_TABLES);
513    options.insert(Options::ENABLE_FOOTNOTES);
514    options.insert(Options::ENABLE_TASKLISTS);
515    options.insert(Options::ENABLE_HEADING_ATTRIBUTES);
516    options.insert(Options::ENABLE_MATH);
517    options
518}
519
520fn markdown_id() -> BackendId {
521    BackendId::new("markdown")
522}
523
524fn heading_level(level: HeadingLevel) -> u8 {
525    match level {
526        HeadingLevel::H1 => 1,
527        HeadingLevel::H2 => 2,
528        HeadingLevel::H3 => 3,
529        HeadingLevel::H4 => 4,
530        HeadingLevel::H5 => 5,
531        HeadingLevel::H6 => 6,
532    }
533}
534
535fn tex_math(text: pulldown_cmark::CowStr<'static>) -> MathSource {
536    MathSource {
537        notation: "tex".to_owned(),
538        text: text.trim_matches('\n').to_owned(),
539    }
540}
541
542fn span(start: usize, end: usize) -> Span {
543    Span {
544        start,
545        end,
546        state: crate::SpanState::Preserved,
547    }
548}
549
550fn inline_plain_text(items: &[Inline]) -> String {
551    let mut text = String::new();
552    for item in items {
553        match item {
554            Inline::Text(value) | Inline::Code(value) => text.push_str(value),
555            Inline::Emph(children) | Inline::Strong(children) => {
556                text.push_str(&inline_plain_text(children));
557            }
558            Inline::Link { label, .. } => text.push_str(&inline_plain_text(label)),
559            Inline::Math(source) => text.push_str(&source.text),
560            Inline::Raw { text: raw, .. } => text.push_str(raw),
561        }
562    }
563    text
564}
sim_codec_doc/markdown.rs

sim_codec_doc/
markdown.rs
