pulldown_cmark_mdcat/

lib.rs

// Copyright 2018-2020 Sebastian Wiesner <sebastian@swsnr.de>

// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

//! Write markdown to TTYs.
//!
//! See [`push_tty`] for the main entry point.
//!
//! ## MSRV
//!
//! This library generally supports only the latest stable Rust version.
//!
//! ## Features
//!
//! - `default` enables `svg` and `image-processing`.
//!
//! - `svg` includes support for rendering SVG images to PNG for terminals which do not support SVG
//!   images natively.  This feature adds a dependency on `resvg`.
//!
//! - `image-processing` enables processing of pixel images before rendering.  This feature adds
//!   a dependency on `image`.  If disabled mdcat will not be able to render inline images on some
//!   terminals, or render images incorrectly or at wrong sizes on other terminals.
//!
//!   Do not disable this feature unless you are sure that you won't use inline images, or accept
//!   incomplete rendering of images.  Please do not report issues with inline images with this
//!   feature disabled.
//!
//!   This feature only exists to allow building with minimal dependencies for use cases where
//!   inline image support is not used or required.  Do not disable this feature unless you know
//!   you won't use inline images, or can accept buggy inline image rendering.
//!
//!   Please **do not report bugs** about inline image rendering with this feature disabled, unless
//!   the issue can also be reproduced if the feature is enabled.
//!
//! - `ratatui` enables rendering markdown into Ratatui `Text` and stateful widgets.  This feature
//!   always depends on `image`, independently of `image-processing`, because it uses the
//!   `ratatui-image` crate to decode and draw images as overlays over the rendered text; disabling
//!   `image-processing` does not remove this dependency.

#![deny(warnings, missing_docs, clippy::all)]
#![forbid(unsafe_code)]

use std::io::{Error, ErrorKind, Result, Write};
use std::path::Path;

use gethostname::gethostname;
use pulldown_cmark::{Event, Options};
use syntect::highlighting::Theme as SyntectTheme;
use syntect::parsing::SyntaxSet;
use tracing::instrument;
use url::Url;

pub use crate::resources::ResourceUrlHandler;
pub use crate::terminal::capabilities::TerminalCapabilities;
pub use crate::terminal::{TerminalProgram, TerminalSize};
pub use crate::theme::Theme;

#[cfg(feature = "ratatui")]
pub mod ratatui;
mod references;
pub mod resources;
pub mod terminal;
mod theme;

mod render;

/// Settings for markdown rendering.
#[derive(Debug)]
pub struct Settings<'a> {
    /// Capabilities of the terminal mdcat writes to.
    pub terminal_capabilities: TerminalCapabilities,
    /// The size of the terminal mdcat writes to.
    pub terminal_size: TerminalSize,
    /// Syntax set for syntax highlighting of code blocks.
    pub syntax_set: &'a SyntaxSet,
    /// Colour theme for mdcat
    pub theme: Theme,
    /// Syntect theme for syntax-highlighted code blocks.
    ///
    /// When set, code blocks are rendered with 24-bit RGB colors from this theme.
    /// When absent, falls back to the built-in Solarized Dark → ANSI color mapping.
    pub syntax_theme: Option<SyntectTheme>,
}

/// The environment to render markdown in.
#[derive(Debug, Clone)]
pub struct Environment {
    /// The base URL to resolve relative URLs with.
    pub base_url: Url,
    /// The local host name.
    pub hostname: String,
}

impl Environment {
    /// Create an environment for the local host with the given `base_url`.
    ///
    /// Take the local hostname from `gethostname`.
    pub fn for_localhost(base_url: Url) -> Result<Self> {
        gethostname()
            .into_string()
            .map_err(|raw| {
                Error::new(
                    ErrorKind::InvalidData,
                    format!("gethostname() returned invalid unicode data: {raw:?}"),
                )
            })
            .map(|hostname| Environment { base_url, hostname })
    }

    /// Create an environment for a local directory.
    ///
    /// Convert the directory to a directory URL, and obtain the hostname from `gethostname`.
    ///
    /// `base_dir` must be an absolute path; return an IO error with `ErrorKind::InvalidInput`
    /// otherwise.
    pub fn for_local_directory<P: AsRef<Path>>(base_dir: &P) -> Result<Self> {
        Url::from_directory_path(base_dir)
            .map_err(|_| {
                Error::new(
                    ErrorKind::InvalidInput,
                    format!(
                        "Base directory {} must be an absolute path",
                        base_dir.as_ref().display()
                    ),
                )
            })
            .and_then(Self::for_localhost)
    }
}

/// Return the pulldown-cmark options mdcat uses for Markdown parsing.
///
/// If `smart_punctuation` is `true`, straight quotes, `--`/`---`, and `...` are rendered as their
/// typographic equivalents (curly quotes, en/em dashes, ellipsis).
pub fn markdown_options(smart_punctuation: bool) -> Options {
    let mut options = Options::ENABLE_TASKLISTS
        | Options::ENABLE_STRIKETHROUGH
        | Options::ENABLE_TABLES
        | Options::ENABLE_FOOTNOTES
        | Options::ENABLE_MATH
        | Options::ENABLE_GFM
        | Options::ENABLE_DEFINITION_LIST;
    if smart_punctuation {
        options |= Options::ENABLE_SMART_PUNCTUATION;
    }
    options
}

/// Strip YAML frontmatter from the beginning of a Markdown document.
///
/// Frontmatter is a `---` block at the very start of the input, closed by another `---` or `...`
/// line. If no valid frontmatter block is found, return the input unchanged.
pub fn strip_frontmatter(input: &str) -> &str {
    let after_open = match input
        .strip_prefix("---\n")
        .or_else(|| input.strip_prefix("---\r\n"))
    {
        Some(s) => s,
        None => return input,
    };

    let mut start = 0;
    while start < after_open.len() {
        let end = after_open[start..]
            .find('\n')
            .map_or(after_open.len(), |i| start + i);
        let line = after_open[start..end].trim_end_matches('\r');
        let next = (end + 1).min(after_open.len());
        if line == "---" || line == "..." {
            return &after_open[next..];
        }
        start = end + 1;
    }

    input
}

/// Expand literal tab characters in `input` to spaces, using a tab stop width of `tab_width`.
///
/// CommonMark treats tabs specially only for block structure (e.g. list/code indentation),
/// internally assuming a tab stop of 4; a literal tab inside text content (a paragraph, inline
/// code, a fenced code block, ...) passes through parsing untouched. Since mdcat's line-wrapping
/// and alignment treat every character as one column wide, such a leftover tab throws off width
/// calculations downstream, as terminals render it as jumping to the next tab stop rather than
/// occupying a single column. Expanding tabs to spaces before parsing avoids that mismatch.
///
/// Tracks the current column per line, resetting after each `\n`, and inserts enough spaces to
/// reach the next multiple of `tab_width`. Column tracking counts one column per `char`; wide
/// characters (e.g. CJK) are not accounted for, matching the rest of mdcat's width handling.
///
/// Returns `input` unchanged, without allocating, if `tab_width` is `0` or `input` has no tabs.
pub fn expand_tabs(input: &str, tab_width: u16) -> std::borrow::Cow<'_, str> {
    if tab_width == 0 || !input.contains('\t') {
        return std::borrow::Cow::Borrowed(input);
    }

    let tab_width = usize::from(tab_width);
    let mut output = String::with_capacity(input.len());
    let mut column = 0;
    for c in input.chars() {
        match c {
            '\t' => {
                let spaces = tab_width - (column % tab_width);
                output.extend(std::iter::repeat_n(' ', spaces));
                column += spaces;
            }
            '\n' => {
                output.push('\n');
                column = 0;
            }
            _ => {
                output.push(c);
                column += 1;
            }
        }
    }
    std::borrow::Cow::Owned(output)
}

/// Write markdown to a TTY.
///
/// Iterate over Markdown AST `events`, format each event for TTY output and
/// write the result to a `writer`, using the given `settings` and `environment`
/// for rendering and resource access.
///
/// `push_tty` tries to limit output to the given number of TTY `columns` but
/// does not guarantee that output stays within the column limit.
#[instrument(level = "debug", skip_all, fields(environment.hostname = environment.hostname.as_str(), environment.base_url = &environment.base_url.as_str()))]
pub fn push_tty<'a, 'e, W, I>(
    settings: &Settings,
    environment: &Environment,
    resource_handler: &dyn ResourceUrlHandler,
    writer: &'a mut W,
    mut events: I,
) -> Result<()>
where
    I: Iterator<Item = Event<'e>>,
    W: Write,
{
    use render::*;
    let StateAndData(final_state, final_data) = events.try_fold(
        StateAndData(State::default(), StateData::default()),
        |StateAndData(state, data), event| {
            write_event(
                writer,
                settings,
                environment,
                &resource_handler,
                state,
                data,
                event,
            )
        },
    )?;
    finish(writer, settings, environment, final_state, final_data)
}

#[cfg(test)]
mod tests {
    use pulldown_cmark::Parser;

    use crate::resources::NoopResourceHandler;

    use super::*;

    fn render_string(input: &str, settings: &Settings) -> Result<String> {
        let source = Parser::new(input);
        let mut sink = Vec::new();
        let env =
            Environment::for_local_directory(&std::env::current_dir().expect("Working directory"))?;
        push_tty(settings, &env, &NoopResourceHandler, &mut sink, source)?;
        Ok(String::from_utf8_lossy(&sink).into())
    }

    fn render_string_dumb(markup: &str) -> Result<String> {
        render_string(
            markup,
            &Settings {
                syntax_set: &SyntaxSet::default(),
                terminal_capabilities: TerminalProgram::Dumb.capabilities(),
                terminal_size: TerminalSize::default(),
                theme: Theme::default(),
                syntax_theme: None,
            },
        )
    }

    #[test]
    fn markdown_options_smart_punctuation_toggle() {
        assert!(!markdown_options(false).contains(Options::ENABLE_SMART_PUNCTUATION));
        assert!(markdown_options(true).contains(Options::ENABLE_SMART_PUNCTUATION));
    }

    #[test]
    fn expand_tabs_zero_width_leaves_input_unchanged() {
        assert_eq!(expand_tabs("a\tb", 0), "a\tb");
    }

    #[test]
    fn expand_tabs_without_tabs_does_not_allocate() {
        assert!(matches!(
            expand_tabs("no tabs here", 4),
            std::borrow::Cow::Borrowed(_)
        ));
    }

    #[test]
    fn expand_tabs_advances_to_next_tab_stop() {
        assert_eq!(expand_tabs("a\tb", 4), "a   b");
        assert_eq!(expand_tabs("ab\tc", 4), "ab  c");
        assert_eq!(expand_tabs("abcd\te", 4), "abcd    e");
    }

    #[test]
    fn expand_tabs_resets_column_at_newline() {
        assert_eq!(expand_tabs("a\tb\nc\td", 4), "a   b\nc   d");
    }

    #[test]
    fn expand_tabs_handles_consecutive_tabs() {
        assert_eq!(expand_tabs("a\t\tb", 4), "a       b");
    }

    fn render_definition_list(markup: &str) -> Result<String> {
        let source = Parser::new_ext(markup, markdown_options(false));
        let mut sink = Vec::new();
        let env =
            Environment::for_local_directory(&std::env::current_dir().expect("Working directory"))?;
        push_tty(
            &Settings {
                syntax_set: &SyntaxSet::default(),
                terminal_capabilities: TerminalProgram::Dumb.capabilities(),
                terminal_size: TerminalSize::default(),
                theme: Theme::default(),
                syntax_theme: None,
            },
            &env,
            &NoopResourceHandler,
            &mut sink,
            source,
        )?;
        Ok(String::from_utf8_lossy(&sink).into())
    }

    #[test]
    fn definition_list_tight() {
        assert_eq!(
            render_definition_list("Apple\n: A fruit.\n: A tech company.\n\nBanana\n: A fruit.\n")
                .unwrap(),
            "Apple\n    A fruit.\n    A tech company.\nBanana\n    A fruit.\n"
        );
    }

    #[test]
    fn definition_list_with_inline_markup_does_not_panic() {
        // Regression test: bold/code/link/emphasis inside a term or description must not hit
        // the "impossible state" panic in `write_event`.
        let output = render_definition_list(
            "Term with `code` and **bold**\n: Def with [a link](https://example.com) and _italics_.\n",
        )
        .unwrap();
        assert!(output.contains("Term with code and bold"));
        assert!(output.contains("Def with a link"));
        assert!(output.contains("https://example.com"));
    }

    #[test]
    fn definition_list_nested_blocks_do_not_panic() {
        // Regression test: a loose definition (blank line before it) may contain nested
        // paragraphs, lists, and code blocks; none of these must hit the panic either.
        render_definition_list(
            "Term\n\n: First paragraph.\n\n  Second paragraph.\n\n  - a nested item\n\n  ```\n  code\n  ```\n",
        )
        .unwrap();
    }

    mod layout {
        use super::render_string_dumb;
        use insta::assert_snapshot;

        #[test]
        #[allow(non_snake_case)]
        fn GH_49_format_no_colour_simple() {
            assert_eq!(
                render_string_dumb("_lorem_ **ipsum** dolor **sit** _amet_").unwrap(),
                "lorem ipsum dolor sit amet\n",
            )
        }

        #[test]
        fn begins_with_rule() {
            assert_snapshot!(render_string_dumb("----").unwrap())
        }

        #[test]
        fn begins_with_block_quote() {
            assert_snapshot!(render_string_dumb("> Hello World").unwrap());
        }

        #[test]
        fn rule_in_block_quote() {
            assert_snapshot!(render_string_dumb(
                "> Hello World

> ----"
            )
            .unwrap());
        }

        #[test]
        fn heading_in_block_quote() {
            assert_snapshot!(render_string_dumb(
                "> Hello World

> # Hello World"
            )
            .unwrap())
        }

        #[test]
        fn heading_levels() {
            assert_snapshot!(render_string_dumb(
                "
# First

## Second

### Third"
            )
            .unwrap())
        }

        #[test]
        fn autolink_creates_no_reference() {
            assert_eq!(
                render_string_dumb("Hello <http://example.com>").unwrap(),
                "Hello http://example.com\n"
            )
        }

        #[test]
        fn flush_ref_links_before_toplevel_heading() {
            assert_snapshot!(render_string_dumb(
                "> Hello [World](http://example.com/world)

> # No refs before this headline

# But before this"
            )
            .unwrap())
        }

        #[test]
        fn flush_ref_links_at_end() {
            assert_snapshot!(render_string_dumb(
                "Hello [World](http://example.com/world)

# Headline

Hello [Donald](http://example.com/Donald)"
            )
            .unwrap())
        }
    }

    mod disabled_features {
        use insta::assert_snapshot;

        use super::render_string_dumb;

        #[test]
        #[allow(non_snake_case)]
        fn GH_155_do_not_choke_on_footnotes() {
            assert_snapshot!(render_string_dumb(
                "A footnote [^1]

[^1: We do not support footnotes."
            )
            .unwrap())
        }
    }
}