marksonnet/

lib.rs

//! This crate provides a `Parser` which wraps a `pulldown_cmark::Parser`, while evaluating
//! embedded Jsonnet code blocks. These code blocks are identified using the `marksonnet` language
//! tag, and they are substituted in the resulting `Iterator` for the result of their evaluation.
//!
//! `marksonnet` blocks can be used to run any Jsonnet program and parse Markdown as though the
//! result of that program were included in the text.
//!
//! ```
//!   # use pretty_assertions::assert_eq;
//!   use indoc;           // multiline string literals
//!   use pulldown_cmark;  // illustrate equivalent sources
//!
//!   let input: &str = indoc::indoc! {r#"
//!     The first five fibonacci numbers are:
//!
//!     ```marksonnet
//!     local fib(n) = if (n <= 2) then 1 else (fib(n-1) + fib(n-2));
//!     [fib(n) for n in std.range(1, 5)]
//!     ```
//!   "#};
//!
//!   let expected: &str = indoc::indoc! {r#"
//!     The first five fibonacci numbers are:
//!
//!     ```json
//!     [
//!         1,
//!         1,
//!         2,
//!         3,
//!         5
//!     ]
//!     ```
//!   "#};
//!
//!   assert_eq!(
//!     marksonnet::Parser::new(input).collect::<Vec<_>>(),
//!     pulldown_cmark::Parser::new(expected).collect::<Vec<_>>()
//!   )
//! ```
//!
//! If the Jsonnet evaluates to a string, then it is substituted inline, rather than being JSON
//! encoded and placed inside of a codeblock.
//!
//! ```
//!   # use pretty_assertions::assert_eq;
//!   use indoc;           // multiline string literals
//!   use pulldown_cmark;  // illustrate equivalent sources
//!
//!   let input: &str = indoc::indoc! {r#"
//!     My favorite greeting is:
//!     ```marksonnet
//!     std.format('> %s, %s!', ['Hello', 'World'])
//!     ```
//!   "#};
//!
//!   let expected: &str = indoc::indoc! {r#"
//!     My favorite greeting is:
//!
//!      > Hello, World!"#};
//!
//!
//!   assert_eq!(
//!     marksonnet::Parser::new(input).collect::<Vec<_>>(),
//!     pulldown_cmark::Parser::new(expected).collect::<Vec<_>>()
//!   )
//! ```
//!
//! Marksonnet also supports imports relative to the file directory.
//!
//! ```
//!   # use pretty_assertions::assert_eq;
//!   use indoc;           // multiline string literals
//!   use pulldown_cmark;  // illustrate equivalent sources
//!
//!   let input: &str = indoc::indoc! {r#"
//!     This is the content of `example/sample.json`:
//!
//!     ```marksonnet
//!     import 'example/sample.json'
//!     ```
//!   "#};
//!
//!   let expected: &str = indoc::indoc! {r#"
//!     This is the content of `example/sample.json`:
//!
//!     ```json
//!     {
//!         "bar": "baz",
//!         "foo": "bar"
//!     }
//!     ```
//!     "#};
//!
//!   assert_eq!(
//!     marksonnet::Parser::new(input).collect::<Vec<_>>(),
//!     pulldown_cmark::Parser::new(expected).collect::<Vec<_>>()
//!   )
//! ```
//!
//! Note that in this case, `example/sample.json` was imported by Jsonnet and parsed, then
//! re-serialized.

use jrsonnet_evaluator::{self, Val, manifest::JsonFormat, val::StrValue};
use pulldown_cmark::{self, CodeBlockKind, CowStr, Event, Tag, TagEnd};
use std::{collections::VecDeque, iter::Peekable, path::PathBuf};

pub struct Parser<'a> {
    /// A `pulldown_cmark::Parser` containing the Markdown source being processed.
    source_parser: Peekable<pulldown_cmark::Parser<'a>>,

    /// Contains the currently-emitting evaluation result if any.
    eval_result: VecDeque<Event<'a>>,

    /// Contains a list of Jsonnet library_paths (JPaths) to use for imports. The current directory
    /// is implicitly included by Jrsonnet.
    library_paths: Vec<PathBuf>,
}

impl<'a> Parser<'a> {
    pub fn new(text: &'a str) -> Self {
        Parser {
            source_parser: pulldown_cmark::Parser::new(text).peekable(),
            eval_result: VecDeque::new(),
            library_paths: vec![],
        }
    }

    pub fn with_library_paths(mut self, jpaths: &Vec<PathBuf>) -> Self {
        self.library_paths = jpaths.clone();
        return self;
    }

    pub fn add_library_path(&mut self, jpath: PathBuf) {
        self.library_paths.push(jpath);
    }

    /// Whether the next Event is from eval_result.
    fn has_eval_result(&self) -> bool {
        self.eval_result.len() > 0
    }

    /// Check whether the next Event in the `source_parser` is a Marksonnet block start.
    fn peek_marksonnet_block_start(&mut self) -> bool {
        Self::is_marksonnet_block_start(&self.source_parser.peek())
    }

    /// Determine whether a given event is a Marksonnet block start.
    fn is_marksonnet_block_start(event: &Option<&Event>) -> bool {
        matches!(
            event,
            Some(Event::Start(Tag::CodeBlock(CodeBlockKind::Fenced(
                CowStr::Borrowed("marksonnet")
            ))))
        )
    }

    /// eval_block consumes the upcoming marksonnet code block from `source_parser`, and stores the
    /// resulting events in `eval_result`.
    ///
    /// Will panic using `debug_assert!` if:
    /// - eval_result is nonempty
    /// - the next event is not a marksonnet code block start
    fn eval_block(&mut self) -> Result<(), jrsonnet_evaluator::Error> {
        debug_assert!(
            !self.has_eval_result(),
            "eval_block called when eval_result is nonempty; it is {:?}",
            self.eval_result
        );

        // Consume and drop the block start marker.
        let block_start = self.source_parser.next();
        debug_assert!(
            Self::is_marksonnet_block_start(&block_start.as_ref()),
            "eval_block called expecting next event to be a Marksonnet block start; it is {:?}",
            block_start
        );
        drop(block_start);

        let mut contents = String::new();

        loop {
            let event = self.source_parser.next();
            match event {
                // Add text events to the Jsonnet contents buffer.
                Some(Event::Text(CowStr::Borrowed(text))) => contents.push_str(text),

                // If we reach the end of the code block, break out of the loop.
                Some(Event::End(TagEnd::CodeBlock)) => break,

                // If we reach anything else, do something else
                _ => unimplemented!("handle malformatted code block (encountered {:?})", event),
            }
        }

        // Initialize a jrsonnet evaluator State using a builder pattern.
        let mut s = jrsonnet_evaluator::State::builder();
        s.import_resolver(jrsonnet_evaluator::FileImportResolver::new(
            self.library_paths.clone(),
        ));
        s.context_initializer(jrsonnet_stdlib::ContextInitializer::new(
            jrsonnet_evaluator::trace::PathResolver::new_cwd_fallback(),
        ));
        let s = s.build(); // shadow the builder with an instance
        let val = Some(s.evaluate_snippet("<marksonnet>", contents));

        match val {
            Some(Ok(Val::Str(StrValue::Flat(text)))) => {
                for event in pulldown_cmark::Parser::new(text.as_str()) {
                    // TODO: investigate whether into_static is a reasonable approach here.
                    //
                    // Because we're trying to be as lazily-evaluated an iterator as possible, we
                    // only act when `next()` is called. When we encounter a Marksonnet block, we
                    // must evaluate the whole thing. This gives us a result from jrsonnet in the
                    // form of interned IStr values. The pulldown_cmark Parser operates on an
                    // immutable reference to what we give it, and the Events that it emits may
                    // contain CowStr::Borrowed strings. Ultimately, these are references to a &str
                    // we converted in the scope of this function, so we need them to stay alive
                    // somehow.
                    //
                    // I don't know if into_static is appropriate for copying strings out of the
                    // function scope, or if there's something else to do (considering that
                    // self-referential structs are not allowed by the borrow checker.)
                    self.eval_result.push_back(event.into_static());
                }
            }
            Some(Ok(val)) => {
                let manifested = val.manifest(JsonFormat::cli(4))? + "\n";
                self.eval_result = vec![
                    Event::Start(Tag::CodeBlock(CodeBlockKind::Fenced(CowStr::Borrowed(
                        "json",
                    )))),
                    Event::Text(CowStr::Boxed(manifested.into())),
                    Event::End(TagEnd::CodeBlock),
                ]
                .into();
            }
            _ => todo!("error or unsupported Val variant: {:?}", val),
        }
        Ok(())
    }
}

impl<'a> Iterator for Parser<'a> {
    type Item = Event<'a>;

    /// Get the next Event.
    fn next(&mut self) -> Option<Self::Item> {
        if self.has_eval_result() {
            self.eval_result.pop_front()
        } else if self.peek_marksonnet_block_start() {
            // TODO: eval_block can currently fail and leave mangled event streams.
            let _ = self.eval_block();
            self.next()
        } else {
            self.source_parser.next()
        }
    }
}

#[cfg(test)]
mod test {
    use super::*;
    use indoc::indoc;
    use pretty_assertions::assert_eq;
    use pulldown_cmark_to_cmark::cmark;

    macro_rules! test {
        // 3-argument version to check pulldown-cmark-to-cmark equivalence. This dodges some of the
        // issues about non-equivalence of CowStr variants (e.g. Borrowed and Boxed strings).
        ($name:ident, $input:expr, $expected:expr) => {
            #[test]
            fn $name() {
                // Parse the input using our parser and convert to CommonMark.
                let mut input = String::new();
                let _ = cmark(Parser::new($input), &mut input);

                // Parse the expected Event stream using the upstream parser and convert to
                // CommonMark.
                let mut expected = String::new();
                let _ = cmark(pulldown_cmark::Parser::new($expected), &mut expected);

                // Compare equivalence as strings.
                assert_eq!(input, expected);
            }
        };
        // 4-argument version to check cmark and event equivalence
        ($name:ident, $input:expr, $expected:expr, $expected_events:expr) => {
            #[test]
            fn $name() {
                // Parse the input using our parser and collect it into a Vec for later comparison.
                let input_events: Vec<_> = Parser::new($input).collect();
                let mut input_cmark = String::new();
                let _ = cmark(input_events.iter(), &mut input_cmark); // convert cmark using the Vec
                let input = (input_cmark, input_events);

                // Parse the expected Event stream using the upstream parser and convert to
                // CommonMark.
                let mut expected_cmark = String::new();
                let _ = cmark(pulldown_cmark::Parser::new($expected), &mut expected_cmark);
                let expected = (expected_cmark, $expected_events);

                // Compare simultaneously, for easier debugging, both pairs:
                // - our parsed input serialized as cmark with the expected output parsed and
                // re-serialized as cmark
                // - our parsed input Events with the expected raw Events
                assert_eq!(input, expected);
            }
        };
    }

    test!(
        no_marksonnet,
        indoc! {r#"This is a simple markdown document containing no marksonnet."#},
        indoc! {r#"This is a simple markdown document containing no marksonnet."#}
    );
    test!(
        empty_marksonnet_object,
        indoc! {r#"
          This is a simple markdown document containing a marksonnet object.

          ```marksonnet
          {}
          ```

          There is also text after it."#},
        // NB: jrsonnet serializes `{}` as `"{ }"` for pretty-printing.
        indoc! {r#"
            This is a simple markdown document containing a marksonnet object.

            ```json
            { }
            ```

            There is also text after it."#}
    );
    test!(
        simple_marksonnet_object_calculation,
        indoc! {r#"```marksonnet
            {
                "value": 2 + 2
            }
            ```"#},
        // NB: pulldown_cmark_to_cmark seems to prefix a newline when a codeblock is the first
        // event. The Vec<Event> below shows the output insofar as marksonnet is concerned.
        indoc! {r#"

            ```json
            {
                "value": 4
            }
            ```"#},
        vec![
            Event::Start(Tag::CodeBlock(CodeBlockKind::Fenced(CowStr::Borrowed(
                "json"
            )))),
            Event::Text(CowStr::Boxed("{\n    \"value\": 4\n}\n".into())),
            Event::End(TagEnd::CodeBlock)
        ]
    );
    test!(
        simple_marksonnet_string,
        indoc! {r#"```marksonnet
            "Hello, world!"
            ```"#},
        indoc! {r#"Hello, world!"#}
    );
    test!(
        sample_import,
        indoc! {r#"```marksonnet
            import 'example/sample.json'
            ```"#},
        indoc! {r#"
            ```json
            {
                "bar": "baz",
                "foo": "bar"
            }
            ```"#}
    );
    test!(
        sample_importstr,
        indoc! {r#"```marksonnet
            importstr 'example/sample.md'
            ```"#},
        indoc! {r#"
            # Sample!

            This file is a sample.
            "#}
    );
    test!(
        sample_importstr_with_prefix,
        indoc! {r#"
            Here we have a prefix.

            ```marksonnet
            importstr 'example/sample.md'
            ```"#},
        indoc! {r#"
            Here we have a prefix.

            # Sample!

            This file is a sample.
            "#}
    );
}