sim_codec_doc/
markup.rs

1//! Shared semantic markup document IR and `Expr` projection.
2
3use std::collections::BTreeMap;
4use std::fmt;
5
6use sim_kernel::{Error, Expr, Result};
7use sim_value::build::{entry, list, map, sym, text, uint};
8
9use crate::document::{DocBlockKind, DocValue, decode_document};
10
11mod expr;
12
13use expr::*;
14
15/// A semantic markup document independent of its concrete source backend.
16#[derive(Clone, Debug, PartialEq)]
17pub struct MarkupDoc {
18    /// Optional document title.
19    pub title: Option<String>,
20    /// Semantic block sequence.
21    pub blocks: Vec<MarkupBlock>,
22    /// Open document attributes carried as ordinary SIM expressions.
23    pub attrs: BTreeMap<String, Expr>,
24    /// Original source text, when a backend can preserve it.
25    pub source: Option<SourceDoc>,
26}
27
28/// A concrete source document preserved alongside the semantic IR.
29#[derive(Clone, Debug, PartialEq, Eq)]
30pub struct SourceDoc {
31    /// Backend that produced the source text.
32    pub backend: BackendId,
33    /// Verbatim source text.
34    pub text: String,
35}
36
37/// A backend identifier such as `markdown`, `typst`, `asciidoc`, or `latex`.
38#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
39pub struct BackendId(
40    /// Backend id text.
41    pub String,
42);
43
44impl BackendId {
45    /// Create a backend id from stable id text.
46    pub fn new(id: impl Into<String>) -> Self {
47        Self(id.into())
48    }
49
50    /// Borrow the backend id text.
51    pub fn as_str(&self) -> &str {
52        &self.0
53    }
54}
55
56impl fmt::Display for BackendId {
57    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
58        f.write_str(&self.0)
59    }
60}
61
62/// A source span in byte offsets.
63#[derive(Clone, Debug, PartialEq, Eq)]
64pub struct Span {
65    /// Inclusive start byte offset.
66    pub start: usize,
67    /// Exclusive end byte offset.
68    pub end: usize,
69    /// Whether the semantic node still matches the original source span.
70    pub state: SpanState,
71}
72
73/// Source-span freshness after reversible document edits.
74#[derive(Clone, Debug, PartialEq, Eq)]
75pub enum SpanState {
76    /// The span still points at untouched source bytes.
77    Preserved,
78    /// The span's source bytes must be regenerated.
79    Dirty,
80}
81
82/// Markup math source with a notation label.
83#[derive(Clone, Debug, PartialEq, Eq)]
84pub struct MathSource {
85    /// Math notation, such as `tex`, `typst`, `asciimath`, or `unknown`.
86    pub notation: String,
87    /// Source text for the math expression.
88    pub text: String,
89}
90
91/// A semantic block in a markup document.
92#[derive(Clone, Debug, PartialEq)]
93pub enum MarkupBlock {
94    /// A heading with inline text.
95    Heading {
96        /// Heading level, normally 1 through 6.
97        level: u8,
98        /// Inline heading content.
99        text: Vec<Inline>,
100        /// Optional stable heading id.
101        id: Option<String>,
102        /// Optional source span.
103        span: Option<Span>,
104    },
105    /// A paragraph of inline content.
106    Paragraph {
107        /// Inline paragraph content.
108        content: Vec<Inline>,
109        /// Optional source span.
110        span: Option<Span>,
111    },
112    /// A fenced or literal code block.
113    CodeBlock {
114        /// Optional language tag.
115        lang: Option<String>,
116        /// Code text.
117        code: String,
118        /// Optional source span.
119        span: Option<Span>,
120    },
121    /// A display math block.
122    MathBlock {
123        /// Math source.
124        source: MathSource,
125        /// Optional source span.
126        span: Option<Span>,
127    },
128    /// A block quote.
129    Quote {
130        /// Quoted blocks.
131        blocks: Vec<MarkupBlock>,
132        /// Optional source span.
133        span: Option<Span>,
134    },
135    /// An ordered or unordered list.
136    List {
137        /// Whether the list is ordered.
138        ordered: bool,
139        /// List items, each item containing block content.
140        items: Vec<Vec<MarkupBlock>>,
141        /// Optional source span.
142        span: Option<Span>,
143    },
144    /// A simple table.
145    Table {
146        /// Header cells.
147        header: Vec<Vec<Inline>>,
148        /// Row cells.
149        rows: Vec<Vec<Vec<Inline>>>,
150        /// Optional source span.
151        span: Option<Span>,
152    },
153    /// A figure with source and caption.
154    Figure {
155        /// Image/media source.
156        src: String,
157        /// Figure caption.
158        caption: Vec<Inline>,
159        /// Optional source span.
160        span: Option<Span>,
161    },
162    /// Backend-specific raw text.
163    Raw {
164        /// Backend that owns the raw text.
165        backend: BackendId,
166        /// Raw source text.
167        text: String,
168        /// Optional source span.
169        span: Option<Span>,
170    },
171}
172
173/// Inline content in a markup document.
174#[derive(Clone, Debug, PartialEq, Eq)]
175pub enum Inline {
176    /// Plain text.
177    Text(String),
178    /// Emphasized inline content.
179    Emph(Vec<Inline>),
180    /// Strong inline content.
181    Strong(Vec<Inline>),
182    /// Inline code.
183    Code(String),
184    /// Link with label and target.
185    Link {
186        /// Link label.
187        label: Vec<Inline>,
188        /// Link target.
189        target: String,
190    },
191    /// Inline math.
192    Math(MathSource),
193    /// Backend-specific raw inline text.
194    Raw {
195        /// Backend that owns the raw text.
196        backend: BackendId,
197        /// Raw source text.
198        text: String,
199    },
200}
201
202/// Decode source text into the shared markup IR using the current lightweight
203/// document parser.
204pub fn decode_markup_doc(source: &str) -> MarkupDoc {
205    MarkupDoc::from_doc_value(&decode_document(source))
206}
207
208impl MarkupDoc {
209    /// Project this markup document into ordinary SIM data.
210    pub fn as_expr(&self) -> Expr {
211        let mut entries = vec![
212            entry("kind", sym("markup-doc")),
213            entry(
214                "blocks",
215                list(self.blocks.iter().map(MarkupBlock::as_expr).collect()),
216            ),
217            entry("attrs", attrs_expr(&self.attrs)),
218        ];
219        if let Some(title) = &self.title {
220            entries.push(entry("title", text(title)));
221        }
222        if let Some(source) = &self.source {
223            entries.push(entry("source", source.as_expr()));
224        }
225        Expr::Map(entries)
226    }
227
228    /// Reconstruct a markup document from ordinary SIM data.
229    pub fn from_expr(expr: &Expr) -> Result<Self> {
230        let entries = map_entries(expr, "markup document")?;
231        require_kind(entries, "markup-doc", "markup document")?;
232        let title = optional_string(entries, "title")?.map(str::to_owned);
233        let blocks = required_list(entries, "blocks", "markup document")?
234            .iter()
235            .map(MarkupBlock::from_expr)
236            .collect::<Result<Vec<_>>>()?;
237        let attrs = match field(entries, "attrs") {
238            Some(Expr::Map(attrs)) => attrs_from_entries(attrs.as_slice())?,
239            Some(_) => return Err(Error::Eval("markup attrs must be a map".to_owned())),
240            None => BTreeMap::new(),
241        };
242        let source = match field(entries, "source") {
243            Some(expr) => Some(SourceDoc::from_expr(expr)?),
244            None => None,
245        };
246        Ok(Self {
247            title,
248            blocks,
249            attrs,
250            source,
251        })
252    }
253
254    /// Render this document to a deterministic source string.
255    ///
256    /// When verbatim source is present, it wins. Documents constructed directly
257    /// from semantic blocks use a small Markdown-like writer.
258    pub fn to_source_text(&self) -> String {
259        if let Some(source) = &self.source {
260            return source.text.clone();
261        }
262        let mut out = String::new();
263        for (index, block) in self.blocks.iter().enumerate() {
264            if index > 0 {
265                out.push_str("\n\n");
266            }
267            block.write_source(&mut out);
268        }
269        out
270    }
271
272    pub(crate) fn from_doc_value(doc: &DocValue) -> Self {
273        let title = doc
274            .blocks
275            .iter()
276            .find(|block| block.kind == DocBlockKind::Heading && block.level == Some(1))
277            .map(|block| block.text.clone());
278        let blocks = doc
279            .blocks
280            .iter()
281            .map(|block| {
282                let span = Some(Span {
283                    start: block.start,
284                    end: block.end,
285                    state: SpanState::Preserved,
286                });
287                match block.kind {
288                    DocBlockKind::Heading => MarkupBlock::Heading {
289                        level: block.level.unwrap_or(1).clamp(1, 6) as u8,
290                        text: vec![Inline::Text(block.text.clone())],
291                        id: None,
292                        span,
293                    },
294                    DocBlockKind::Paragraph => MarkupBlock::Paragraph {
295                        content: vec![Inline::Text(block.text.clone())],
296                        span,
297                    },
298                }
299            })
300            .collect();
301        Self {
302            title,
303            blocks,
304            attrs: BTreeMap::new(),
305            source: Some(SourceDoc {
306                backend: BackendId(format_name(doc.format).to_owned()),
307                text: doc.text.clone(),
308            }),
309        }
310    }
311}
312
313impl SourceDoc {
314    fn as_expr(&self) -> Expr {
315        map(vec![
316            ("backend", text(&self.backend.0)),
317            ("text", text(&self.text)),
318        ])
319    }
320
321    fn from_expr(expr: &Expr) -> Result<Self> {
322        let entries = map_entries(expr, "markup source")?;
323        Ok(Self {
324            backend: BackendId(required_string(entries, "backend", "markup source")?.to_owned()),
325            text: required_string(entries, "text", "markup source")?.to_owned(),
326        })
327    }
328}
329
330impl Span {
331    fn as_expr(&self) -> Expr {
332        map(vec![
333            ("start", uint(self.start as u64)),
334            ("end", uint(self.end as u64)),
335            ("state", sym(self.state.as_str())),
336        ])
337    }
338
339    fn from_expr(expr: &Expr) -> Result<Self> {
340        let entries = map_entries(expr, "span")?;
341        Ok(Self {
342            start: required_usize(entries, "start", "span")?,
343            end: required_usize(entries, "end", "span")?,
344            state: match field(entries, "state") {
345                Some(value) => SpanState::from_expr(value)?,
346                None => SpanState::Preserved,
347            },
348        })
349    }
350}
351
352impl SpanState {
353    fn as_str(&self) -> &'static str {
354        match self {
355            Self::Preserved => "preserved",
356            Self::Dirty => "dirty",
357        }
358    }
359
360    fn from_expr(expr: &Expr) -> Result<Self> {
361        match expr {
362            Expr::Symbol(symbol) if symbol.namespace.is_none() => match symbol.name.as_ref() {
363                "preserved" => Ok(Self::Preserved),
364                "dirty" => Ok(Self::Dirty),
365                other => Err(Error::Eval(format!("unknown span state {other}"))),
366            },
367            Expr::String(value) => match value.as_str() {
368                "preserved" => Ok(Self::Preserved),
369                "dirty" => Ok(Self::Dirty),
370                other => Err(Error::Eval(format!("unknown span state {other}"))),
371            },
372            _ => Err(Error::Eval("span state must be a symbol".to_owned())),
373        }
374    }
375}
376
377impl MathSource {
378    fn as_expr(&self) -> Expr {
379        map(vec![
380            ("notation", text(&self.notation)),
381            ("text", text(&self.text)),
382        ])
383    }
384
385    fn from_expr(expr: &Expr) -> Result<Self> {
386        let entries = map_entries(expr, "math source")?;
387        Ok(Self {
388            notation: required_string(entries, "notation", "math source")?.to_owned(),
389            text: required_string(entries, "text", "math source")?.to_owned(),
390        })
391    }
392}
393
394impl MarkupBlock {
395    /// Project this block into ordinary SIM data.
396    pub fn as_expr(&self) -> Expr {
397        match self {
398            Self::Heading {
399                level,
400                text: heading,
401                id,
402                span,
403            } => {
404                let mut entries = vec![
405                    entry("kind", sym("heading")),
406                    entry("level", uint(u64::from(*level))),
407                    entry("text", inline_list(heading)),
408                ];
409                push_optional_string(&mut entries, "id", id);
410                push_optional_span(&mut entries, span);
411                Expr::Map(entries)
412            }
413            Self::Paragraph { content, span } => {
414                let mut entries = vec![
415                    entry("kind", sym("paragraph")),
416                    entry("content", inline_list(content)),
417                ];
418                push_optional_span(&mut entries, span);
419                Expr::Map(entries)
420            }
421            Self::CodeBlock { lang, code, span } => {
422                let mut entries = vec![entry("kind", sym("code-block")), entry("code", text(code))];
423                push_optional_string(&mut entries, "lang", lang);
424                push_optional_span(&mut entries, span);
425                Expr::Map(entries)
426            }
427            Self::MathBlock { source, span } => {
428                let mut entries = vec![
429                    entry("kind", sym("math-block")),
430                    entry("source", source.as_expr()),
431                ];
432                push_optional_span(&mut entries, span);
433                Expr::Map(entries)
434            }
435            Self::Quote { blocks, span } => {
436                let mut entries = vec![
437                    entry("kind", sym("quote")),
438                    entry("blocks", block_list(blocks)),
439                ];
440                push_optional_span(&mut entries, span);
441                Expr::Map(entries)
442            }
443            Self::List {
444                ordered,
445                items,
446                span,
447            } => {
448                let mut entries = vec![
449                    entry("kind", sym("list")),
450                    entry("ordered", Expr::Bool(*ordered)),
451                    entry(
452                        "items",
453                        list(items.iter().map(|item| block_list(item)).collect()),
454                    ),
455                ];
456                push_optional_span(&mut entries, span);
457                Expr::Map(entries)
458            }
459            Self::Table { header, rows, span } => {
460                let mut entries = vec![
461                    entry("kind", sym("table")),
462                    entry(
463                        "header",
464                        list(header.iter().map(|cell| inline_list(cell)).collect()),
465                    ),
466                    entry(
467                        "rows",
468                        list(
469                            rows.iter()
470                                .map(|row| list(row.iter().map(|cell| inline_list(cell)).collect()))
471                                .collect(),
472                        ),
473                    ),
474                ];
475                push_optional_span(&mut entries, span);
476                Expr::Map(entries)
477            }
478            Self::Figure { src, caption, span } => {
479                let mut entries = vec![
480                    entry("kind", sym("figure")),
481                    entry("src", text(src)),
482                    entry("caption", inline_list(caption)),
483                ];
484                push_optional_span(&mut entries, span);
485                Expr::Map(entries)
486            }
487            Self::Raw {
488                backend,
489                text: raw,
490                span,
491            } => {
492                let mut entries = vec![
493                    entry("kind", sym("raw")),
494                    entry("backend", text(&backend.0)),
495                    entry("text", text(raw)),
496                ];
497                push_optional_span(&mut entries, span);
498                Expr::Map(entries)
499            }
500        }
501    }
502
503    /// Reconstruct a block from ordinary SIM data.
504    pub fn from_expr(expr: &Expr) -> Result<Self> {
505        let entries = map_entries(expr, "markup block")?;
506        match required_kind(entries, "markup block")?.as_str() {
507            "heading" => Ok(Self::Heading {
508                level: required_u8(entries, "level", "heading")?,
509                text: inline_vec(required_list(entries, "text", "heading")?)?,
510                id: optional_string(entries, "id")?.map(str::to_owned),
511                span: optional_span(entries)?,
512            }),
513            "paragraph" => Ok(Self::Paragraph {
514                content: inline_vec(required_list(entries, "content", "paragraph")?)?,
515                span: optional_span(entries)?,
516            }),
517            "code-block" => Ok(Self::CodeBlock {
518                lang: optional_string(entries, "lang")?.map(str::to_owned),
519                code: required_string(entries, "code", "code block")?.to_owned(),
520                span: optional_span(entries)?,
521            }),
522            "math-block" => Ok(Self::MathBlock {
523                source: MathSource::from_expr(required_field(entries, "source", "math block")?)?,
524                span: optional_span(entries)?,
525            }),
526            "quote" => Ok(Self::Quote {
527                blocks: block_vec(required_list(entries, "blocks", "quote")?)?,
528                span: optional_span(entries)?,
529            }),
530            "list" => {
531                let items = required_list(entries, "items", "list")?
532                    .iter()
533                    .map(|item| block_vec(as_list(item, "list item")?))
534                    .collect::<Result<Vec<_>>>()?;
535                Ok(Self::List {
536                    ordered: required_bool(entries, "ordered", "list")?,
537                    items,
538                    span: optional_span(entries)?,
539                })
540            }
541            "table" => {
542                let header = required_list(entries, "header", "table")?
543                    .iter()
544                    .map(|cell| inline_vec(as_list(cell, "table header cell")?))
545                    .collect::<Result<Vec<_>>>()?;
546                let rows = required_list(entries, "rows", "table")?
547                    .iter()
548                    .map(|row| {
549                        as_list(row, "table row")?
550                            .iter()
551                            .map(|cell| inline_vec(as_list(cell, "table cell")?))
552                            .collect::<Result<Vec<_>>>()
553                    })
554                    .collect::<Result<Vec<_>>>()?;
555                Ok(Self::Table {
556                    header,
557                    rows,
558                    span: optional_span(entries)?,
559                })
560            }
561            "figure" => Ok(Self::Figure {
562                src: required_string(entries, "src", "figure")?.to_owned(),
563                caption: inline_vec(required_list(entries, "caption", "figure")?)?,
564                span: optional_span(entries)?,
565            }),
566            "raw" => Ok(Self::Raw {
567                backend: BackendId(required_string(entries, "backend", "raw block")?.to_owned()),
568                text: required_string(entries, "text", "raw block")?.to_owned(),
569                span: optional_span(entries)?,
570            }),
571            other => Err(Error::Eval(format!("unknown markup block kind {other}"))),
572        }
573    }
574
575    fn write_source(&self, out: &mut String) {
576        match self {
577            Self::Heading { level, text, .. } => {
578                out.push_str(&"#".repeat(usize::from(*level).max(1)));
579                out.push(' ');
580                write_inlines(out, text);
581            }
582            Self::Paragraph { content, .. } => write_inlines(out, content),
583            Self::CodeBlock { lang, code, .. } => {
584                out.push_str("```");
585                if let Some(lang) = lang {
586                    out.push_str(lang);
587                }
588                out.push('\n');
589                out.push_str(code);
590                if !code.ends_with('\n') {
591                    out.push('\n');
592                }
593                out.push_str("```");
594            }
595            Self::MathBlock { source, .. } => {
596                out.push_str("$$\n");
597                out.push_str(&source.text);
598                out.push_str("\n$$");
599            }
600            Self::Quote { blocks, .. } => {
601                let text = blocks_to_source(blocks);
602                for (index, line) in text.lines().enumerate() {
603                    if index > 0 {
604                        out.push('\n');
605                    }
606                    out.push_str("> ");
607                    out.push_str(line);
608                }
609            }
610            Self::List { ordered, items, .. } => {
611                for (index, item) in items.iter().enumerate() {
612                    if index > 0 {
613                        out.push('\n');
614                    }
615                    if *ordered {
616                        out.push_str(&format!("{}. ", index + 1));
617                    } else {
618                        out.push_str("- ");
619                    }
620                    out.push_str(&blocks_to_source(item).replace('\n', "\n  "));
621                }
622            }
623            Self::Table { header, rows, .. } => {
624                write_table_row(out, header);
625                out.push('\n');
626                write_table_separator(out, header.len());
627                for row in rows {
628                    out.push('\n');
629                    write_table_row(out, row);
630                }
631            }
632            Self::Figure { src, caption, .. } => {
633                out.push_str("![");
634                write_inlines(out, caption);
635                out.push_str("](");
636                out.push_str(src);
637                out.push(')');
638            }
639            Self::Raw { text, .. } => out.push_str(text),
640        }
641    }
642}
643
644impl Inline {
645    fn as_expr(&self) -> Expr {
646        match self {
647            Self::Text(value) => map(vec![("kind", sym("text")), ("text", text(value))]),
648            Self::Emph(items) => map(vec![("kind", sym("emph")), ("content", inline_list(items))]),
649            Self::Strong(items) => map(vec![
650                ("kind", sym("strong")),
651                ("content", inline_list(items)),
652            ]),
653            Self::Code(value) => map(vec![("kind", sym("code")), ("text", text(value))]),
654            Self::Link { label, target } => map(vec![
655                ("kind", sym("link")),
656                ("label", inline_list(label)),
657                ("target", text(target)),
658            ]),
659            Self::Math(source) => map(vec![("kind", sym("math")), ("source", source.as_expr())]),
660            Self::Raw { backend, text: raw } => map(vec![
661                ("kind", sym("raw")),
662                ("backend", text(&backend.0)),
663                ("text", text(raw)),
664            ]),
665        }
666    }
667
668    fn from_expr(expr: &Expr) -> Result<Self> {
669        if let Expr::String(value) = expr {
670            return Ok(Self::Text(value.clone()));
671        }
672        let entries = map_entries(expr, "inline")?;
673        match required_kind(entries, "inline")?.as_str() {
674            "text" => Ok(Self::Text(
675                required_string(entries, "text", "inline")?.to_owned(),
676            )),
677            "emph" => Ok(Self::Emph(inline_vec(required_list(
678                entries, "content", "inline",
679            )?)?)),
680            "strong" => Ok(Self::Strong(inline_vec(required_list(
681                entries, "content", "inline",
682            )?)?)),
683            "code" => Ok(Self::Code(
684                required_string(entries, "text", "inline")?.to_owned(),
685            )),
686            "link" => Ok(Self::Link {
687                label: inline_vec(required_list(entries, "label", "inline")?)?,
688                target: required_string(entries, "target", "inline")?.to_owned(),
689            }),
690            "math" => Ok(Self::Math(MathSource::from_expr(required_field(
691                entries, "source", "inline",
692            )?)?)),
693            "raw" => Ok(Self::Raw {
694                backend: BackendId(required_string(entries, "backend", "inline")?.to_owned()),
695                text: required_string(entries, "text", "inline")?.to_owned(),
696            }),
697            other => Err(Error::Eval(format!("unknown inline kind {other}"))),
698        }
699    }
700}
sim_codec_doc/markup.rs

sim_codec_doc/
markup.rs
