sim_codec_doc/
typst_backend.rs

1//! Typst backend implementation over `typst-syntax`.
2
3use std::collections::BTreeMap;
4use std::iter::Peekable;
5
6use typst_syntax::ast::{self, AstNode};
7use typst_syntax::{SyntaxNode, parse};
8
9use crate::backend::{
10    MarkupBackend, MarkupDecodeOptions, MarkupEncodeOptions, MarkupError, MarkupFidelity,
11    MarkupLoss,
12};
13use crate::markup::{
14    BackendId, Inline, MarkupBlock, MarkupDoc, MathSource, SourceDoc, Span, SpanState,
15};
16
17#[path = "typst_writer.rs"]
18mod typst_writer;
19
20use typst_writer::TypstEncoder;
21
22/// Safe Typst markup backend.
23#[derive(Clone, Debug, Default)]
24pub struct TypstBackend;
25
26impl MarkupBackend for TypstBackend {
27    fn id(&self) -> BackendId {
28        typst_id()
29    }
30
31    fn decode(
32        &self,
33        input: &str,
34        opts: &MarkupDecodeOptions,
35    ) -> Result<(MarkupDoc, MarkupFidelity), MarkupError> {
36        let root = parse(input);
37        let (errors, warnings) = root.errors_and_warnings();
38        if !errors.is_empty() {
39            return Err(MarkupError::Decode(format!(
40                "typst syntax contains {} error(s)",
41                errors.len()
42            )));
43        }
44
45        let markup = root
46            .cast::<ast::Markup>()
47            .ok_or_else(|| MarkupError::Decode("typst root is not markup".to_owned()))?;
48        let mut parser = TypstParser::new(input, opts);
49        parser.fidelity.warnings.extend(
50            warnings
51                .into_iter()
52                .map(|_| "typst syntax warning".to_owned()),
53        );
54        let blocks = parser.parse_markup(markup);
55        let title = blocks.iter().find_map(|block| match block {
56            MarkupBlock::Heading { level: 1, text, .. } => Some(inline_plain_text(text)),
57            _ => None,
58        });
59        let source = opts.preserve_source.then(|| SourceDoc {
60            backend: typst_id(),
61            text: input.to_owned(),
62        });
63        Ok((
64            MarkupDoc {
65                title,
66                blocks,
67                attrs: BTreeMap::new(),
68                source,
69            },
70            parser.fidelity,
71        ))
72    }
73
74    fn encode(
75        &self,
76        doc: &MarkupDoc,
77        opts: &MarkupEncodeOptions,
78    ) -> Result<(String, MarkupFidelity), MarkupError> {
79        let mut encoder = TypstEncoder::new(opts);
80        let source = encoder.write_doc(doc);
81        if opts.fail_on_loss && !encoder.fidelity().dropped.is_empty() {
82            return Err(MarkupError::Encode(format!(
83                "typst encode dropped {} unsupported fragment(s)",
84                encoder.fidelity().dropped.len()
85            )));
86        }
87        Ok((source, encoder.into_fidelity()))
88    }
89}
90
91struct TypstParser<'a> {
92    input: &'a str,
93    cursor: usize,
94    preserve_raw: bool,
95    fidelity: MarkupFidelity,
96}
97
98impl<'a> TypstParser<'a> {
99    fn new(input: &'a str, opts: &MarkupDecodeOptions) -> Self {
100        Self {
101            input,
102            cursor: 0,
103            preserve_raw: opts.preserve_raw,
104            fidelity: MarkupFidelity::exact(typst_id()),
105        }
106    }
107
108    fn parse_markup(&mut self, markup: ast::Markup<'_>) -> Vec<MarkupBlock> {
109        let mut blocks = Vec::new();
110        let mut paragraph = PendingParagraph::default();
111        let mut list: Option<PendingList> = None;
112        for expr in markup.exprs() {
113            let span = self.span_for(expr.to_untyped());
114            match expr {
115                ast::Expr::Parbreak(_) => {
116                    self.flush_paragraph(&mut blocks, &mut paragraph);
117                    self.flush_list(&mut blocks, &mut list);
118                }
119                ast::Expr::Heading(heading) => {
120                    self.flush_paragraph(&mut blocks, &mut paragraph);
121                    self.flush_list(&mut blocks, &mut list);
122                    blocks.push(MarkupBlock::Heading {
123                        level: heading.depth().get().min(u8::MAX as usize) as u8,
124                        text: self.inlines_from_markup(heading.body()),
125                        id: None,
126                        span,
127                    });
128                }
129                ast::Expr::Raw(raw) if raw.block() => {
130                    self.flush_paragraph(&mut blocks, &mut paragraph);
131                    self.flush_list(&mut blocks, &mut list);
132                    blocks.push(MarkupBlock::CodeBlock {
133                        lang: raw.lang().map(|lang| lang.get().to_string()),
134                        code: raw_text(raw),
135                        span,
136                    });
137                }
138                ast::Expr::Equation(equation) if equation.block() => {
139                    self.flush_paragraph(&mut blocks, &mut paragraph);
140                    self.flush_list(&mut blocks, &mut list);
141                    blocks.push(MarkupBlock::MathBlock {
142                        source: typst_math(equation.body()),
143                        span,
144                    });
145                }
146                ast::Expr::ListItem(item) => {
147                    self.flush_paragraph(&mut blocks, &mut paragraph);
148                    self.push_list_item(&mut blocks, &mut list, false, item.body(), span);
149                }
150                ast::Expr::EnumItem(item) => {
151                    self.flush_paragraph(&mut blocks, &mut paragraph);
152                    self.push_list_item(&mut blocks, &mut list, true, item.body(), span);
153                }
154                ast::Expr::FuncCall(call) if self.call_name(call.callee()) == Some("table") => {
155                    self.flush_paragraph(&mut blocks, &mut paragraph);
156                    self.flush_list(&mut blocks, &mut list);
157                    if let Some(table) = self.table_from_call(call, span.clone()) {
158                        blocks.push(table);
159                    } else if let Some(raw) =
160                        self.raw_block(expr_source(expr), "table", span, "unsupported typst table")
161                    {
162                        blocks.push(raw);
163                    }
164                }
165                ast::Expr::FuncCall(call) if self.call_name(call.callee()) == Some("figure") => {
166                    self.flush_paragraph(&mut blocks, &mut paragraph);
167                    self.flush_list(&mut blocks, &mut list);
168                    if let Some(figure) = self.figure_from_call(call, span.clone()) {
169                        blocks.push(figure);
170                    } else if let Some(raw) = self.raw_block(
171                        expr_source(expr),
172                        "figure",
173                        span,
174                        "unsupported typst figure",
175                    ) {
176                        blocks.push(raw);
177                    }
178                }
179                ast::Expr::FuncCall(call) if self.call_name(call.callee()) != Some("link") => {
180                    self.flush_paragraph(&mut blocks, &mut paragraph);
181                    self.flush_list(&mut blocks, &mut list);
182                    let path = self
183                        .call_name(call.callee())
184                        .unwrap_or("function")
185                        .to_owned();
186                    if let Some(raw) = self.raw_block(
187                        call.to_untyped().full_text().to_string(),
188                        &path,
189                        span,
190                        "unsupported typst function is preserved but not executed",
191                    ) {
192                        blocks.push(raw);
193                    }
194                }
195                ast::Expr::ModuleInclude(_) => {
196                    self.flush_paragraph(&mut blocks, &mut paragraph);
197                    self.flush_list(&mut blocks, &mut list);
198                    if let Some(raw) = self.raw_block(
199                        expr_source(expr),
200                        "include",
201                        span,
202                        "typst include is preserved but not executed",
203                    ) {
204                        blocks.push(raw);
205                    }
206                }
207                ast::Expr::ModuleImport(_) => {
208                    self.flush_paragraph(&mut blocks, &mut paragraph);
209                    self.flush_list(&mut blocks, &mut list);
210                    if let Some(raw) = self.raw_block(
211                        expr_source(expr),
212                        "import",
213                        span,
214                        "typst import is preserved but not executed",
215                    ) {
216                        blocks.push(raw);
217                    }
218                }
219                other => {
220                    self.flush_list(&mut blocks, &mut list);
221                    if let Some(inline) = self.inline_from_expr(other, "inline") {
222                        paragraph.push(inline, span);
223                    }
224                }
225            }
226        }
227        self.flush_paragraph(&mut blocks, &mut paragraph);
228        self.flush_list(&mut blocks, &mut list);
229        blocks
230    }
231
232    fn push_list_item(
233        &mut self,
234        blocks: &mut Vec<MarkupBlock>,
235        list: &mut Option<PendingList>,
236        ordered: bool,
237        body: ast::Markup<'_>,
238        span: Option<Span>,
239    ) {
240        if list.as_ref().is_some_and(|list| list.ordered != ordered) {
241            self.flush_list(blocks, list);
242        }
243        let item = self.blocks_from_item_body(body);
244        list.get_or_insert_with(|| PendingList::new(ordered))
245            .push(item, span);
246    }
247
248    fn blocks_from_item_body<'b>(&mut self, body: ast::Markup<'b>) -> Vec<MarkupBlock> {
249        let content = self.inlines_from_markup(body);
250        if content.is_empty() {
251            Vec::new()
252        } else {
253            vec![MarkupBlock::Paragraph {
254                content,
255                span: None,
256            }]
257        }
258    }
259
260    fn inlines_from_markup<'b>(&mut self, markup: ast::Markup<'b>) -> Vec<Inline> {
261        let mut items = Vec::new();
262        let mut exprs = markup.exprs().peekable();
263        while let Some(expr) = exprs.next() {
264            if let Some(link) = self.link_from_call_with_body(expr, &mut exprs) {
265                items.push(link);
266                continue;
267            }
268            if let Some(inline) = self.inline_from_expr(expr, "inline") {
269                items.push(inline);
270            }
271        }
272        items
273    }
274
275    fn inline_from_expr<'b>(&mut self, expr: ast::Expr<'b>, path: &str) -> Option<Inline> {
276        match expr {
277            ast::Expr::Text(text) => Some(Inline::Text(text.get().to_string())),
278            ast::Expr::Space(_) => Some(Inline::Text(" ".to_owned())),
279            ast::Expr::Linebreak(_) => Some(Inline::Text("\n".to_owned())),
280            ast::Expr::Escape(escape) => Some(Inline::Text(escape.get().to_string())),
281            ast::Expr::Shorthand(shorthand) => Some(Inline::Text(shorthand.get().to_string())),
282            ast::Expr::SmartQuote(quote) => Some(Inline::Text(
283                if quote.double() { "\"" } else { "'" }.to_owned(),
284            )),
285            ast::Expr::Strong(strong) => {
286                Some(Inline::Strong(self.inlines_from_markup(strong.body())))
287            }
288            ast::Expr::Emph(emph) => Some(Inline::Emph(self.inlines_from_markup(emph.body()))),
289            ast::Expr::Raw(raw) if !raw.block() => Some(Inline::Code(raw_text(raw))),
290            ast::Expr::Link(link) => {
291                let target = link.get().to_string();
292                Some(Inline::Link {
293                    label: vec![Inline::Text(target.clone())],
294                    target,
295                })
296            }
297            ast::Expr::Equation(equation) if !equation.block() => {
298                Some(Inline::Math(typst_math(equation.body())))
299            }
300            ast::Expr::FuncCall(call) if self.call_name(call.callee()) == Some("link") => {
301                let target = self.first_string_arg(call)?;
302                Some(Inline::Link {
303                    label: vec![Inline::Text(target.clone())],
304                    target,
305                })
306            }
307            ast::Expr::Parbreak(_) => None,
308            other => self.raw_inline(expr_source(other), path, "unsupported typst inline"),
309        }
310    }
311
312    fn link_from_call_with_body<'b, I>(
313        &mut self,
314        expr: ast::Expr<'b>,
315        exprs: &mut Peekable<I>,
316    ) -> Option<Inline>
317    where
318        I: Iterator<Item = ast::Expr<'b>>,
319    {
320        let ast::Expr::FuncCall(call) = expr else {
321            return None;
322        };
323        if self.call_name(call.callee()) != Some("link") {
324            return None;
325        }
326        let target = self.first_string_arg(call)?;
327        let label = match exprs.peek().copied() {
328            Some(ast::Expr::ContentBlock(block)) => {
329                exprs.next();
330                self.inlines_from_markup(block.body())
331            }
332            _ => vec![Inline::Text(target.clone())],
333        };
334        Some(Inline::Link { label, target })
335    }
336
337    fn table_from_call(
338        &mut self,
339        call: ast::FuncCall<'_>,
340        span: Option<Span>,
341    ) -> Option<MarkupBlock> {
342        let mut columns = None;
343        let mut cells = Vec::new();
344        for arg in call.args().items() {
345            match arg {
346                ast::Arg::Named(named) if named.name().as_str() == "columns" => {
347                    if let ast::Expr::Int(value) = named.expr() {
348                        columns = usize::try_from(value.get()).ok().filter(|value| *value > 0);
349                    }
350                }
351                ast::Arg::Pos(ast::Expr::ContentBlock(block)) => {
352                    cells.push(self.inlines_from_markup(block.body()));
353                }
354                _ => self
355                    .fidelity
356                    .warnings
357                    .push("unsupported typst table argument preserved by source span".to_owned()),
358            }
359        }
360        if cells.is_empty() {
361            return None;
362        }
363        let width = columns.unwrap_or(cells.len()).max(1);
364        let mut rows = cells
365            .chunks(width)
366            .map(|chunk| {
367                let mut row = chunk.to_vec();
368                row.resize_with(width, Vec::new);
369                row
370            })
371            .collect::<Vec<_>>();
372        let header = rows.drain(..1).next().unwrap_or_default();
373        Some(MarkupBlock::Table { header, rows, span })
374    }
375
376    fn figure_from_call(
377        &mut self,
378        call: ast::FuncCall<'_>,
379        span: Option<Span>,
380    ) -> Option<MarkupBlock> {
381        let mut src = None;
382        let mut caption = Vec::new();
383        for arg in call.args().items() {
384            match arg {
385                ast::Arg::Pos(ast::Expr::FuncCall(image))
386                    if self.call_name(image.callee()) == Some("image") =>
387                {
388                    src = self.first_string_arg(image);
389                }
390                ast::Arg::Named(named) if named.name().as_str() == "caption" => {
391                    if let ast::Expr::ContentBlock(block) = named.expr() {
392                        caption = self.inlines_from_markup(block.body());
393                    }
394                }
395                ast::Arg::Pos(ast::Expr::ContentBlock(block)) if caption.is_empty() => {
396                    caption = self.inlines_from_markup(block.body());
397                }
398                _ => self
399                    .fidelity
400                    .warnings
401                    .push("unsupported typst figure argument preserved by source span".to_owned()),
402            }
403        }
404        Some(MarkupBlock::Figure {
405            src: src?,
406            caption,
407            span,
408        })
409    }
410
411    fn first_string_arg(&self, call: ast::FuncCall<'_>) -> Option<String> {
412        call.args().items().find_map(|arg| match arg {
413            ast::Arg::Pos(ast::Expr::Str(value)) => Some(value.get().to_string()),
414            _ => None,
415        })
416    }
417
418    fn call_name<'b>(&self, expr: ast::Expr<'b>) -> Option<&'b str> {
419        match expr {
420            ast::Expr::Ident(ident) => Some(ident.as_str()),
421            _ => None,
422        }
423    }
424
425    fn flush_paragraph(&mut self, blocks: &mut Vec<MarkupBlock>, paragraph: &mut PendingParagraph) {
426        if paragraph.content.is_empty() {
427            return;
428        }
429
430        let content = std::mem::take(&mut paragraph.content);
431        let span = paragraph.span.take();
432        if inline_plain_text(&content).trim().is_empty() {
433            return;
434        }
435
436        blocks.push(MarkupBlock::Paragraph { content, span });
437    }
438
439    fn flush_list(&mut self, blocks: &mut Vec<MarkupBlock>, list: &mut Option<PendingList>) {
440        if let Some(list) = list.take() {
441            blocks.push(MarkupBlock::List {
442                ordered: list.ordered,
443                items: list.items,
444                span: list.span,
445            });
446        }
447    }
448
449    fn raw_block(
450        &mut self,
451        text: String,
452        path: &str,
453        span: Option<Span>,
454        reason: &str,
455    ) -> Option<MarkupBlock> {
456        if self.preserve_raw {
457            self.fidelity.preserved_raw.push(text.clone());
458            Some(MarkupBlock::Raw {
459                backend: typst_id(),
460                text,
461                span,
462            })
463        } else {
464            self.drop_raw(path, reason);
465            None
466        }
467    }
468
469    fn raw_inline(&mut self, text: String, path: &str, reason: &str) -> Option<Inline> {
470        if self.preserve_raw {
471            self.fidelity.preserved_raw.push(text.clone());
472            Some(Inline::Raw {
473                backend: typst_id(),
474                text,
475            })
476        } else {
477            self.drop_raw(path, reason);
478            None
479        }
480    }
481
482    fn drop_raw(&mut self, path: &str, reason: &str) {
483        self.fidelity.dropped.push(MarkupLoss {
484            path: path.to_owned(),
485            reason: reason.to_owned(),
486        });
487    }
488
489    fn span_for(&mut self, node: &SyntaxNode) -> Option<Span> {
490        let text = node.full_text();
491        if text.is_empty() {
492            return None;
493        }
494        let haystack = self.input.get(self.cursor..)?;
495        let start = haystack.find(text.as_str())? + self.cursor;
496        let end = start + text.len();
497        self.cursor = end;
498        Some(span(start, end))
499    }
500}
501
502#[derive(Default)]
503struct PendingParagraph {
504    content: Vec<Inline>,
505    span: Option<Span>,
506}
507
508impl PendingParagraph {
509    fn push(&mut self, inline: Inline, span: Option<Span>) {
510        merge_span(&mut self.span, &span);
511        self.content.push(inline);
512    }
513}
514
515struct PendingList {
516    ordered: bool,
517    items: Vec<Vec<MarkupBlock>>,
518    span: Option<Span>,
519}
520
521impl PendingList {
522    fn new(ordered: bool) -> Self {
523        Self {
524            ordered,
525            items: Vec::new(),
526            span: None,
527        }
528    }
529
530    fn push(&mut self, item: Vec<MarkupBlock>, span: Option<Span>) {
531        merge_span(&mut self.span, &span);
532        self.items.push(item);
533    }
534}
535
536fn typst_id() -> BackendId {
537    BackendId::new("typst")
538}
539
540fn expr_source(expr: ast::Expr<'_>) -> String {
541    expr.to_untyped().full_text().to_string()
542}
543
544fn raw_text(raw: ast::Raw<'_>) -> String {
545    let lines = raw
546        .lines()
547        .map(|line| line.get().to_string())
548        .collect::<Vec<_>>();
549    if lines.is_empty() {
550        raw.to_untyped().full_text().to_string()
551    } else {
552        lines.join("\n")
553    }
554}
555
556fn typst_math(math: ast::Math<'_>) -> MathSource {
557    MathSource {
558        notation: "typst".to_owned(),
559        text: math.to_untyped().full_text().trim().to_owned(),
560    }
561}
562
563fn span(start: usize, end: usize) -> Span {
564    Span {
565        start,
566        end,
567        state: SpanState::Preserved,
568    }
569}
570
571fn merge_span(target: &mut Option<Span>, next: &Option<Span>) {
572    let Some(next) = next else {
573        return;
574    };
575    match target {
576        Some(current) => {
577            current.start = current.start.min(next.start);
578            current.end = current.end.max(next.end);
579        }
580        None => *target = Some(next.clone()),
581    }
582}
583
584fn inline_plain_text(items: &[Inline]) -> String {
585    let mut text = String::new();
586    for item in items {
587        match item {
588            Inline::Text(value) | Inline::Code(value) => text.push_str(value),
589            Inline::Emph(children) | Inline::Strong(children) => {
590                text.push_str(&inline_plain_text(children));
591            }
592            Inline::Link { label, .. } => text.push_str(&inline_plain_text(label)),
593            Inline::Math(source) => text.push_str(&source.text),
594            Inline::Raw { text: raw, .. } => text.push_str(raw),
595        }
596    }
597    text
598}
sim_codec_doc/typst_backend.rs

sim_codec_doc/
typst_backend.rs
