Skip to main content

markdoc_pdf/
pipeline.rs

1//! From Markdoc source to the tree the renderer lays out.
2//!
3//! The renderer reads documents through markdoc (a port of upstream Markdoc
4//! 0.5.11) and the extension layers a renderer needs (`markdoc::ext`).
5//! Everything that decides how a document is read is here, so that the binary
6//! and the tests read it the same way:
7//!
8//! - **Schema.** markdoc's built-in tag bundle, then markdoc-pdf's own overlay
9//!   ([`OVERLAY`]: `toc`, `footnote`, fence `align`, image `size`/`width`/`side`,
10//!   and the parse options below), then the caller's overlays (a corpus's
11//!   `--schema` files), in that order ([`bundle`]). Upstream renders only what
12//!   the config declares: an undeclared tag renders its children, an
13//!   undeclared attribute is dropped.
14//! - **Parsing** ([`parse_options`]): the bundle's `[parse]` and `[render]`
15//!   tables (indentation and comments allowed, a byte-order mark stripped, the
16//!   typographer on), with the frontmatter always read as YAML and markdoc's
17//!   own positions (`spans`) on, which the syntax-error warnings report
18//!   ([`syntax_errors`]). Source that is only rendered parses without them.
19//! - **Partials** are spliced into the tree before the renderer's own passes
20//!   (`ext::partials`, MIGRATION-MAP D12), from the files the caller's
21//!   resolver finds.
22//! - **Transform** ([`config`], [`transform`]): upstream's `if`/`else`,
23//!   `switch`/`case`/`for` from `ext::control_flow`, `include`/`includes` from
24//!   `ext::functions` beside upstream's six functions, and the variables:
25//!   `$markdoc.frontmatter` and the build-time `--var`/`--variables`
26//!   (`ext::variables`). Truthiness is upstream's. A schema file's
27//!   `[functions.*]` declarations and `[validation]` table reach the config
28//!   too (`Bundle::apply`); a declared function the renderer has no code for
29//!   is named per document ([`unknown_functions`]), not refused.
30
31use markdoc::IndexMap;
32use markdoc::ast::Node;
33use markdoc::ext::bundle::Bundle;
34use markdoc::ext::partials::{PartialResolver, expand_partials, load_partials};
35use markdoc::ext::variables::Bindings;
36use markdoc::ext::{Frontmatter, ParseOptions, control_flow, functions};
37use markdoc::schema::Config;
38use markdoc::types::{
39    MarkdocError, RenderableTreeNode, Result, Scalar, ValidationError, ValidationLevel,
40};
41
42/// markdoc-pdf's own schema overlay (`schema/markdoc-pdf.overlay.toml`).
43pub const OVERLAY: &str = include_str!("../schema/markdoc-pdf.overlay.toml");
44
45/// The schema: markdoc's built-in bundle, markdoc-pdf's [`OVERLAY`], then each of
46/// `overlays` in order (a later file wins, field by field). A corpus's
47/// `markdoc.toml` loads whole, its `[validation]` and `[functions.*]` tables
48/// included. `Err` names the first overlay that does not load, by its index in
49/// `overlays`.
50pub fn bundle(overlays: &[&str]) -> std::result::Result<Bundle, (usize, String)> {
51    let mut bundle = Bundle::builtin();
52    bundle
53        .overlay(OVERLAY)
54        .expect("markdoc-pdf's own overlay loads");
55    for (i, text) in overlays.iter().enumerate() {
56        bundle.overlay(text).map_err(|e| (i, e.to_string()))?;
57    }
58    Ok(bundle)
59}
60
61/// How the renderer parses a document and its partials: the bundle's render
62/// options, with the frontmatter read as YAML and markdoc's own positions on
63/// (`spans`, which markdoc leaves off by default), so that [`syntax_errors`]
64/// can name the file and line of an error. Source that is only rendered (the
65/// generated document history, [`render_tree`]) is parsed with
66/// `.with_spans(false)`: nothing reads its positions.
67pub fn parse_options(bundle: &Bundle) -> ParseOptions {
68    let mut options = bundle.render_options();
69    options.frontmatter = Frontmatter::Yaml;
70    options.args.spans = true;
71    options
72}
73
74/// `source` parsed with `options` (see [`parse_options`]).
75pub fn parse(source: &str, options: &ParseOptions) -> Result<Node> {
76    markdoc::ext::parse(source, options)
77}
78
79/// `doc` with every `{% partial file="..." /%}` replaced by the file's content,
80/// recursively: each file is read once through `resolver`, parsed with
81/// `options`, and spliced without its frontmatter. A missing file, a partial
82/// without a `file`, and a cycle are an `Err`.
83pub fn expand(doc: &Node, resolver: &dyn PartialResolver, options: &ParseOptions) -> Result<Node> {
84    let partials = load_partials(doc, resolver, options)?;
85    if partials.is_empty() {
86        return Ok(doc.clone());
87    }
88    expand_partials(doc, &partials)
89}
90
91/// The syntax errors markdoc recorded in `doc` (and the partials spliced into
92/// it), as `file:line: message`, `file` being `input` for the document itself.
93///
94/// Markdoc does not fail a parse over a syntax error (only past its depth
95/// limits): an unclosed tag, a stray closing tag, a tag it cannot read, or
96/// frontmatter that is not YAML stay in the tree as errors, and what they
97/// held renders as Markdoc decides (an unreadable tag not at all). The
98/// renderer says so on stderr; a corpus's lint is what fails a build on them.
99pub fn syntax_errors(doc: &Node, input: &str) -> Vec<String> {
100    std::iter::once(doc)
101        .chain(doc.walk())
102        .flat_map(|node| node.errors.iter().map(move |error| (node, error)))
103        .filter(|(_, error)| {
104            matches!(
105                error.level,
106                ValidationLevel::Error | ValidationLevel::Critical
107            )
108        })
109        .map(|(node, error)| {
110            let (file, line) = position(node, error);
111            let file = file.unwrap_or(input);
112            match line {
113                Some(line) => format!("{file}:{line}: {}", error.message),
114                None => format!("{file}: {}", error.message),
115            }
116        })
117        .collect()
118}
119
120/// Where an error is: its own span, else its node's (the file a partial was
121/// parsed as, and the 1-based line). The spans are markdoc's own positions,
122/// there when the tree was parsed with them on ([`parse_options`]);
123/// `location` and `lines` are upstream's. The one place that reads positions.
124fn position<'a>(node: &'a Node, error: &'a ValidationError) -> (Option<&'a str>, Option<usize>) {
125    let span = error.span.as_ref().or(node.span.as_ref());
126    match span {
127        Some(span) => (span.file.as_deref(), Some(span.start.line)),
128        None => (None, None),
129    }
130}
131
132/// The config a document is transformed with: the renderer's functions
133/// (`include`/`includes`) with the bundle applied over them (its tags and
134/// nodes, `[validation]`, and its function declarations, which keep the code
135/// registered under them), `switch`/`case`/`for`, and the variables:
136/// `$markdoc.frontmatter` bound to `frontmatter`, and each of `variables` as a
137/// top-level `$key` (a variable named `markdoc` replaces the frontmatter's).
138pub fn config(
139    bundle: &Bundle,
140    frontmatter: Option<&Scalar>,
141    variables: &IndexMap<String, Scalar>,
142) -> Config {
143    let mut config = Config::default();
144    config.functions.extend(functions::all());
145    bundle.apply(&mut config);
146    control_flow::apply_bundle(bundle, &mut config);
147    let mut bindings = Bindings::new();
148    if let Some(frontmatter) = frontmatter {
149        bindings = bindings.with_frontmatter(frontmatter);
150    }
151    bindings.with_all(variables).apply(&mut config);
152    config
153}
154
155/// The functions `doc` calls (in attributes, conditions and interpolations,
156/// arguments included) that `config` has no code for, in first-use order.
157///
158/// Markdoc resolves such a call to null without a word (MIGRATION-MAP R3):
159/// an `{% if %}` it guards hides its content, an interpolation prints
160/// nothing. A corpus may declare functions its site registers (`higher`,
161/// `lower`) that this renderer does not, so the renderer names the ones a
162/// document calls rather than refusing every document of the corpus
163/// (`Bundle::check_functions`).
164pub fn unknown_functions(doc: &Node, config: &Config) -> Vec<String> {
165    let known = markdoc::merge_config(config).functions;
166    let has_code = |name: &str| {
167        known
168            .get(name)
169            .and_then(Option::as_ref)
170            .is_some_and(|f| f.transform.is_some())
171    };
172    let mut unknown: Vec<String> = Vec::new();
173    let mut stack: Vec<&Scalar> = std::iter::once(doc)
174        .chain(doc.walk())
175        .flat_map(|node| node.attributes.values())
176        .collect();
177    while let Some(value) = stack.pop() {
178        match value {
179            Scalar::Function(function) => {
180                if !has_code(&function.name) && !unknown.contains(&function.name) {
181                    unknown.push(function.name.clone());
182                }
183                stack.extend(function.parameters.values());
184            }
185            Scalar::Array(items) => stack.extend(items),
186            Scalar::Object(members) => stack.extend(members.values()),
187            _ => {}
188        }
189    }
190    unknown
191}
192
193/// `doc` transformed into the render tree the layout walks.
194pub fn transform(doc: &Node, config: &Config) -> Result<RenderableTreeNode> {
195    markdoc::transform(doc, config)?
196        .to_render()
197        .map_err(MarkdocError::TransformError)
198}
199
200/// The render tree of a standalone `source` (no partials, no build-time
201/// variables), read as the binary reads a document with no `--schema`, without
202/// the positions only [`syntax_errors`] reads.
203pub fn render_tree(source: &str) -> Result<RenderableTreeNode> {
204    let bundle = bundle(&[]).expect("no overlays to fail");
205    let doc = parse(source, &parse_options(&bundle).with_spans(false))?;
206    let config = config(&bundle, doc.attributes.get("frontmatter"), &IndexMap::new());
207    transform(&doc, &config)
208}
209
210#[cfg(test)]
211mod tests {
212    use super::*;
213    use markdoc::ext::partials::InMemoryPartialResolver;
214
215    fn html_names(node: &RenderableTreeNode, out: &mut Vec<String>) {
216        if let RenderableTreeNode::Tag(t) = node {
217            out.push(t.name.clone());
218            for c in &t.children {
219                html_names(c, out);
220            }
221        }
222    }
223
224    #[test]
225    fn own_overlay_declares_what_only_this_renderer_reads() {
226        let tree = render_tree("{% toc /%}\n\nText{% footnote %}A note.{% /footnote %}\n").unwrap();
227        let mut names = Vec::new();
228        html_names(&tree, &mut names);
229        assert_eq!(names, ["article", "toc", "p", "footnote"]);
230    }
231
232    #[test]
233    fn parse_options_are_the_renderers() {
234        let options = parse_options(&bundle(&[]).unwrap());
235        assert!(options.tokenizer.allow_indentation);
236        assert!(options.tokenizer.allow_comments);
237        assert!(options.strip_bom);
238        assert_eq!(options.frontmatter, Frontmatter::Yaml);
239        // The syntax-error warnings name a file and line (`syntax_errors`).
240        assert!(options.args.spans && options.args.location);
241        assert_eq!(
242            options.tokenizer.markdown_it.get("typographer"),
243            Some(&serde_json::Value::Bool(true))
244        );
245        // A corpus that says otherwise wins, except for the frontmatter.
246        let corpus = "[parse]\nallow_comments = false\nfrontmatter = \"raw\"\n";
247        let options = parse_options(&bundle(&[corpus]).unwrap());
248        assert!(!options.tokenizer.allow_comments);
249        assert_eq!(options.frontmatter, Frontmatter::Yaml);
250    }
251
252    #[test]
253    fn a_corpus_files_validation_and_function_tables_reach_the_config() {
254        let markdoc_toml = "[parse]\nallow_indentation = true\n\n\
255                            [validation]\nvalidate_functions = true\n\n\
256                            [functions.higher]\ndescription = \"x\"\n\
257                            [functions.higher.parameters.0]\ntype = [\"String\"]\n\
258                            [functions.include]\ndescription = \"In the list\"\n";
259        let bundle = bundle(&[markdoc_toml]).unwrap();
260        assert_eq!(
261            bundle.function_names().collect::<Vec<_>>(),
262            ["higher", "include"]
263        );
264        let config = config(&bundle, None, &IndexMap::new());
265        assert!(config.validation.validate_functions);
266        // A declaration keeps the code registered under it; one without code
267        // is there, and named as such.
268        let include = config.functions["include"].as_ref().unwrap();
269        assert!(include.transform.is_some());
270        assert_eq!(include.description.as_deref(), Some("In the list"));
271        assert_eq!(bundle.functions_without_code(&config), ["higher"]);
272        // Anything else unknown still fails, and names the file.
273        assert_eq!(super::bundle(&["", "[tagz.x]\n"]).unwrap_err().0, 1);
274    }
275
276    #[test]
277    fn frontmatter_and_build_variables_are_bound() {
278        let bundle = bundle(&[]).unwrap();
279        let doc = parse(
280            "---\ntitle: Book\n---\n# {% $markdoc.frontmatter.title %} {% $v %}\n",
281            &parse_options(&bundle),
282        )
283        .unwrap();
284        let vars = IndexMap::from([("v".to_string(), Scalar::String("2".into()))]);
285        let config = config(&bundle, doc.attributes.get("frontmatter"), &vars);
286        let html =
287            markdoc::renderers::html::render(&markdoc::transform(&doc, &config).unwrap()).unwrap();
288        assert_eq!(html, "<article><h1>Book 2</h1></article>");
289    }
290
291    #[test]
292    fn partials_are_spliced_with_the_documents_options() {
293        let resolver = InMemoryPartialResolver::new()
294            .with(
295                "a.md",
296                "\u{FEFF}---\ntitle: dropped\n---\n# A\n\n{% partial file=\"b.md\" /%}\n",
297            )
298            .with("b.md", "<!-- hidden -->\nB\n");
299        let options = parse_options(&bundle(&[]).unwrap());
300        let doc = parse("{% partial file=\"a.md\" /%}\n", &options).unwrap();
301        let doc = expand(&doc, &resolver, &options).unwrap();
302        let kinds: Vec<&str> = doc.children.iter().map(|c| c.node_type.as_str()).collect();
303        // The BOM stripped (a heading, not a paragraph), the frontmatter
304        // dropped, the comment a comment (which renders nothing).
305        assert_eq!(kinds, ["heading", "comment", "paragraph"]);
306        assert!(doc.attributes.get("frontmatter").is_none());
307        let cycle = InMemoryPartialResolver::new().with("a.md", "{% partial file=\"a.md\" /%}\n");
308        let err = expand(&doc_with("a.md", &options), &cycle, &options).unwrap_err();
309        assert!(err.to_string().contains("cycle"), "{err}");
310    }
311
312    #[test]
313    fn syntax_errors_are_reported_with_their_file_and_line() {
314        let resolver = InMemoryPartialResolver::new().with("p.md", "Text\n\n{% callout %}\nopen\n");
315        let options = parse_options(&bundle(&[]).unwrap());
316        let doc = parse("# Title\n\n{% partial file=\"p.md\" /%}\n", &options).unwrap();
317        let doc = expand(&doc, &resolver, &options).unwrap();
318        let errors = syntax_errors(&doc, "book.mdoc");
319        assert_eq!(errors.len(), 1, "{errors:?}");
320        assert!(errors[0].starts_with("p.md:3: "), "{errors:?}");
321        let clean = parse("# Fine\n", &options).unwrap();
322        assert!(syntax_errors(&clean, "x.mdoc").is_empty());
323    }
324
325    #[test]
326    fn functions_without_code_are_named() {
327        let bundle = bundle(&[]).unwrap();
328        let doc = parse(
329            "{% if higher($v, [\"1.4\"]) %}\nA {% upper(\"x\") %}\n{% /if %}\n\n\
330             {% if and(include($p, \"a\"), equals(lower(1), 2)) %}\nB\n{% /if %}\n",
331            &parse_options(&bundle),
332        )
333        .unwrap();
334        let config = config(&bundle, None, &IndexMap::new());
335        let mut names = unknown_functions(&doc, &config);
336        names.sort();
337        assert_eq!(names, ["higher", "lower", "upper"]);
338    }
339
340    fn doc_with(file: &str, options: &ParseOptions) -> Node {
341        parse(&format!("{{% partial file=\"{file}\" /%}}\n"), options).unwrap()
342    }
343}