pulldown-cmark-mdcat 2.19.0

// Copyright 2018-2020 Sebastian Wiesner <sebastian@swsnr.de>

// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

//! Write markdown to TTYs.
//!
//! See [`push_tty`] for the main entry point.
//!
//! ## MSRV
//!
//! This library generally supports only the latest stable Rust version.
//!
//! ## Features
//!
//! - `default` enables `svg` and `image-processing`.
//!
//! - `svg` includes support for rendering SVG images to PNG for terminals which do not support SVG
//!   images natively.  This feature adds a dependency on `resvg`.
//!
//! - `image-processing` enables processing of pixel images before rendering.  This feature adds
//!   a dependency on `image`.  If disabled mdcat will not be able to render inline images on some
//!   terminals, or render images incorrectly or at wrong sizes on other terminals.
//!
//!   Do not disable this feature unless you are sure that you won't use inline images, or accept
//!   incomplete rendering of images.  Please do not report issues with inline images with this
//!   feature disabled.
//!
//!   This feature only exists to allow building with minimal dependencies for use cases where
//!   inline image support is not used or required.  Do not disable this feature unless you know
//!   you won't use inline images, or can accept buggy inline image rendering.
//!
//!   Please **do not report bugs** about inline image rendering with this feature disabled, unless
//!   the issue can also be reproduced if the feature is enabled.
//!
//! - `ratatui` enables rendering markdown into Ratatui `Text` and stateful widgets.  This feature
//!   always depends on `image`, independently of `image-processing`, because it uses the
//!   `ratatui-image` crate to decode and draw images as overlays over the rendered text; disabling
//!   `image-processing` does not remove this dependency.

#![deny(warnings, missing_docs, clippy::all)]
#![forbid(unsafe_code)]

use std::io::{Error, ErrorKind, Result, Write};
use std::path::Path;

use gethostname::gethostname;
use pulldown_cmark::{Event, Options, Tag, TagEnd};
use syntect::highlighting::Theme as SyntectTheme;
use syntect::parsing::SyntaxSet;
use tracing::instrument;
use url::Url;

pub use crate::frontmatter::{frontmatter_table_markdown, split_frontmatter, FrontmatterFormat};
pub use crate::resources::ResourceUrlHandler;
pub use crate::terminal::capabilities::TerminalCapabilities;
pub use crate::terminal::{TerminalProgram, TerminalSize};
pub use crate::theme::Theme;

mod frontmatter;
#[cfg(feature = "ratatui")]
pub mod ratatui;
mod references;
pub mod resources;
pub mod terminal;
mod theme;

mod render;

/// Settings for markdown rendering.
#[derive(Debug)]
pub struct Settings<'a> {
    /// Capabilities of the terminal mdcat writes to.
    pub terminal_capabilities: TerminalCapabilities,
    /// The size of the terminal mdcat writes to.
    pub terminal_size: TerminalSize,
    /// Syntax set for syntax highlighting of code blocks.
    pub syntax_set: &'a SyntaxSet,
    /// Colour theme for mdcat
    pub theme: Theme,
    /// Syntect theme for syntax-highlighted code blocks.
    ///
    /// When set, code blocks are rendered with 24-bit RGB colors from this theme.
    /// When absent, falls back to the built-in Solarized Dark → ANSI color mapping.
    pub syntax_theme: Option<SyntectTheme>,
}

/// The environment to render markdown in.
#[derive(Debug, Clone)]
pub struct Environment {
    /// The base URL to resolve relative URLs with.
    pub base_url: Url,
    /// The local host name.
    pub hostname: String,
}

impl Environment {
    /// Create an environment for the local host with the given `base_url`.
    ///
    /// Take the local hostname from `gethostname`.
    pub fn for_localhost(base_url: Url) -> Result<Self> {
        gethostname()
            .into_string()
            .map_err(|raw| {
                Error::new(
                    ErrorKind::InvalidData,
                    format!("gethostname() returned invalid unicode data: {raw:?}"),
                )
            })
            .map(|hostname| Environment { base_url, hostname })
    }

    /// Create an environment for a local directory.
    ///
    /// Convert the directory to a directory URL, and obtain the hostname from `gethostname`.
    ///
    /// `base_dir` must be an absolute path; return an IO error with `ErrorKind::InvalidInput`
    /// otherwise.
    pub fn for_local_directory<P: AsRef<Path>>(base_dir: &P) -> Result<Self> {
        Url::from_directory_path(base_dir)
            .map_err(|_| {
                Error::new(
                    ErrorKind::InvalidInput,
                    format!(
                        "Base directory {} must be an absolute path",
                        base_dir.as_ref().display()
                    ),
                )
            })
            .and_then(Self::for_localhost)
    }
}

/// Return the pulldown-cmark options mdcat uses for Markdown parsing.
///
/// If `smart_punctuation` is `true`, straight quotes, `--`/`---`, and `...` are rendered as their
/// typographic equivalents (curly quotes, en/em dashes, ellipsis).
pub fn markdown_options(smart_punctuation: bool) -> Options {
    let mut options = Options::ENABLE_TASKLISTS
        | Options::ENABLE_STRIKETHROUGH
        | Options::ENABLE_TABLES
        | Options::ENABLE_FOOTNOTES
        | Options::ENABLE_MATH
        | Options::ENABLE_GFM
        | Options::ENABLE_DEFINITION_LIST;
    if smart_punctuation {
        options |= Options::ENABLE_SMART_PUNCTUATION;
    }
    options
}

/// Strip YAML or TOML frontmatter from the beginning of a Markdown document.
///
/// YAML frontmatter is a `---` block at the very start of the input, closed by another `---` or
/// `...` line. TOML frontmatter is a `+++` block closed by another `+++` line. A leading UTF-8
/// byte order mark and trailing whitespace on the delimiter lines are ignored. If no valid
/// frontmatter block is found, return the input unchanged.
///
/// See [`split_frontmatter`] to inspect the stripped frontmatter instead of discarding it.
pub fn strip_frontmatter(input: &str) -> &str {
    frontmatter::split_frontmatter(input).0
}

/// Expand literal tab characters in `input` to spaces, using a tab stop width of `tab_width`.
///
/// CommonMark treats tabs specially only for block structure (e.g. list/code indentation),
/// internally assuming a tab stop of 4; a literal tab inside text content (a paragraph, inline
/// code, a fenced code block, ...) passes through parsing untouched. Since mdcat's line-wrapping
/// and alignment treat every character as one column wide, such a leftover tab throws off width
/// calculations downstream, as terminals render it as jumping to the next tab stop rather than
/// occupying a single column. Expanding tabs to spaces before parsing avoids that mismatch.
///
/// Tracks the current column per line, resetting after each `\n`, and inserts enough spaces to
/// reach the next multiple of `tab_width`. Column tracking counts one column per `char`; wide
/// characters (e.g. CJK) are not accounted for, matching the rest of mdcat's width handling.
///
/// Returns `input` unchanged, without allocating, if `tab_width` is `0` or `input` has no tabs.
pub fn expand_tabs(input: &str, tab_width: u16) -> std::borrow::Cow<'_, str> {
    if tab_width == 0 || !input.contains('\t') {
        return std::borrow::Cow::Borrowed(input);
    }

    let tab_width = usize::from(tab_width);
    let mut output = String::with_capacity(input.len());
    let mut column = 0;
    for c in input.chars() {
        match c {
            '\t' => {
                let spaces = tab_width - (column % tab_width);
                output.extend(std::iter::repeat_n(' ', spaces));
                column += spaces;
            }
            '\n' => {
                output.push('\n');
                column = 0;
            }
            _ => {
                output.push(c);
                column += 1;
            }
        }
    }
    std::borrow::Cow::Owned(output)
}

/// Replace GitHub-style `:emoji:` shortcodes with Unicode emoji in text content.
///
/// Only rewrites [`Event::Text`] content, and only outside fenced/indented code blocks; it never
/// touches [`Event::Code`] (inline code spans), [`Event::Html`], or text inside a
/// [`Tag::CodeBlock`], so emoji shortcodes in code are always left untouched -- matching GitHub's
/// own behaviour. Unknown shortcodes (no matching emoji name) are left as-is.
pub fn substitute_emoji<'e>(
    events: impl Iterator<Item = Event<'e>> + 'e,
) -> impl Iterator<Item = Event<'e>> {
    let replacer = gh_emoji::Replacer::new();
    let mut code_block_depth = 0u32;
    events.map(move |event| match event {
        Event::Start(Tag::CodeBlock(_)) => {
            code_block_depth += 1;
            event
        }
        Event::End(TagEnd::CodeBlock) => {
            code_block_depth = code_block_depth.saturating_sub(1);
            event
        }
        Event::Text(text) if code_block_depth == 0 => match replacer.replace_all(&text) {
            std::borrow::Cow::Borrowed(_) => Event::Text(text),
            std::borrow::Cow::Owned(s) => Event::Text(s.into()),
        },
        other => other,
    })
}

/// Write markdown to a TTY.
///
/// Iterate over Markdown AST `events`, format each event for TTY output and
/// write the result to a `writer`, using the given `settings` and `environment`
/// for rendering and resource access.
///
/// `push_tty` tries to limit output to the given number of TTY `columns` but
/// does not guarantee that output stays within the column limit.
#[instrument(level = "debug", skip_all, fields(environment.hostname = environment.hostname.as_str(), environment.base_url = &environment.base_url.as_str()))]
pub fn push_tty<'a, 'e, W, I>(
    settings: &Settings,
    environment: &Environment,
    resource_handler: &dyn ResourceUrlHandler,
    writer: &'a mut W,
    mut events: I,
) -> Result<()>
where
    I: Iterator<Item = Event<'e>>,
    W: Write,
{
    use render::*;
    let StateAndData(final_state, final_data) = events.try_fold(
        StateAndData(State::default(), StateData::default()),
        |StateAndData(state, data), event| {
            write_event(
                writer,
                settings,
                environment,
                &resource_handler,
                state,
                data,
                event,
            )
        },
    )?;
    finish(writer, settings, environment, final_state, final_data)
}

#[cfg(test)]
mod tests {
    use pulldown_cmark::Parser;

    use crate::resources::NoopResourceHandler;

    use super::*;

    fn render_string(input: &str, settings: &Settings) -> Result<String> {
        let source = Parser::new(input);
        let mut sink = Vec::new();
        let env =
            Environment::for_local_directory(&std::env::current_dir().expect("Working directory"))?;
        push_tty(settings, &env, &NoopResourceHandler, &mut sink, source)?;
        Ok(String::from_utf8_lossy(&sink).into())
    }

    fn render_string_dumb(markup: &str) -> Result<String> {
        render_string(
            markup,
            &Settings {
                syntax_set: &SyntaxSet::default(),
                terminal_capabilities: TerminalProgram::Dumb.capabilities(),
                terminal_size: TerminalSize::default(),
                theme: Theme::default(),
                syntax_theme: None,
            },
        )
    }

    #[test]
    fn markdown_options_smart_punctuation_toggle() {
        assert!(!markdown_options(false).contains(Options::ENABLE_SMART_PUNCTUATION));
        assert!(markdown_options(true).contains(Options::ENABLE_SMART_PUNCTUATION));
    }

    #[test]
    fn strip_frontmatter_yaml() {
        assert_eq!(strip_frontmatter("---\na: 1\n---\n# H\n"), "# H\n");
        assert_eq!(strip_frontmatter("---\r\na: 1\r\n...\r\n# H\n"), "# H\n");
    }

    #[test]
    fn strip_frontmatter_toml() {
        assert_eq!(strip_frontmatter("+++\na = 1\n+++\n# H\n"), "# H\n");
        assert_eq!(strip_frontmatter("+++\na = 1\n---\n"), "+++\na = 1\n---\n");
    }

    #[test]
    fn strip_frontmatter_bom_and_trailing_whitespace() {
        assert_eq!(strip_frontmatter("\u{feff}---\na: 1\n---\n# H\n"), "# H\n");
        assert_eq!(strip_frontmatter("--- \na: 1\n---\t\n# H\n"), "# H\n");
    }

    #[test]
    fn strip_frontmatter_without_closing_delimiter_leaves_input_unchanged() {
        assert_eq!(strip_frontmatter("---\na: 1\n"), "---\na: 1\n");
        assert_eq!(strip_frontmatter("# H\n---\n"), "# H\n---\n");
        assert_eq!(strip_frontmatter("---"), "---");
    }

    #[test]
    fn strip_frontmatter_closing_delimiter_at_eof() {
        assert_eq!(strip_frontmatter("---\na: 1\n---"), "");
    }

    #[test]
    fn expand_tabs_zero_width_leaves_input_unchanged() {
        assert_eq!(expand_tabs("a\tb", 0), "a\tb");
    }

    #[test]
    fn expand_tabs_without_tabs_does_not_allocate() {
        assert!(matches!(
            expand_tabs("no tabs here", 4),
            std::borrow::Cow::Borrowed(_)
        ));
    }

    #[test]
    fn expand_tabs_advances_to_next_tab_stop() {
        assert_eq!(expand_tabs("a\tb", 4), "a   b");
        assert_eq!(expand_tabs("ab\tc", 4), "ab  c");
        assert_eq!(expand_tabs("abcd\te", 4), "abcd    e");
    }

    #[test]
    fn expand_tabs_resets_column_at_newline() {
        assert_eq!(expand_tabs("a\tb\nc\td", 4), "a   b\nc   d");
    }

    #[test]
    fn expand_tabs_handles_consecutive_tabs() {
        assert_eq!(expand_tabs("a\t\tb", 4), "a       b");
    }

    #[test]
    fn substitute_emoji_replaces_known_shortcodes_in_text() {
        let events = vec![Event::Text("Hello :+1: and :smile:!".into())];
        let result: Vec<_> = substitute_emoji(events.into_iter()).collect();
        assert_eq!(result, vec![Event::Text("Hello 👍 and 😄!".into())]);
    }

    #[test]
    fn substitute_emoji_leaves_unknown_shortcodes_untouched() {
        let events = vec![Event::Text("before :not_a_real_emoji_xyz: after".into())];
        let result: Vec<_> = substitute_emoji(events.into_iter()).collect();
        assert_eq!(
            result,
            vec![Event::Text("before :not_a_real_emoji_xyz: after".into())]
        );
    }

    #[test]
    fn substitute_emoji_never_touches_code_events() {
        let events = vec![Event::Code(":+1:".into())];
        let result: Vec<_> = substitute_emoji(events.into_iter()).collect();
        assert_eq!(result, vec![Event::Code(":+1:".into())]);
    }

    #[test]
    fn substitute_emoji_never_touches_fenced_code_block_text() {
        let events = vec![
            Event::Start(Tag::CodeBlock(pulldown_cmark::CodeBlockKind::Fenced(
                "".into(),
            ))),
            Event::Text(":+1:".into()),
            Event::End(TagEnd::CodeBlock),
        ];
        let result: Vec<_> = substitute_emoji(events.clone().into_iter()).collect();
        assert_eq!(result, events);
    }

    fn render_definition_list(markup: &str) -> Result<String> {
        let source = Parser::new_ext(markup, markdown_options(false));
        let mut sink = Vec::new();
        let env =
            Environment::for_local_directory(&std::env::current_dir().expect("Working directory"))?;
        push_tty(
            &Settings {
                syntax_set: &SyntaxSet::default(),
                terminal_capabilities: TerminalProgram::Dumb.capabilities(),
                terminal_size: TerminalSize::default(),
                theme: Theme::default(),
                syntax_theme: None,
            },
            &env,
            &NoopResourceHandler,
            &mut sink,
            source,
        )?;
        Ok(String::from_utf8_lossy(&sink).into())
    }

    #[test]
    fn definition_list_tight() {
        assert_eq!(
            render_definition_list("Apple\n: A fruit.\n: A tech company.\n\nBanana\n: A fruit.\n")
                .unwrap(),
            "Apple\n    A fruit.\n    A tech company.\nBanana\n    A fruit.\n"
        );
    }

    #[test]
    fn definition_list_with_inline_markup_does_not_panic() {
        // Regression test: bold/code/link/emphasis inside a term or description must not hit
        // the "impossible state" panic in `write_event`.
        let output = render_definition_list(
            "Term with `code` and **bold**\n: Def with [a link](https://example.com) and _italics_.\n",
        )
        .unwrap();
        assert!(output.contains("Term with code and bold"));
        assert!(output.contains("Def with a link"));
        assert!(output.contains("https://example.com"));
    }

    #[test]
    fn definition_list_nested_blocks_do_not_panic() {
        // Regression test: a loose definition (blank line before it) may contain nested
        // paragraphs, lists, and code blocks; none of these must hit the panic either.
        render_definition_list(
            "Term\n\n: First paragraph.\n\n  Second paragraph.\n\n  - a nested item\n\n  ```\n  code\n  ```\n",
        )
        .unwrap();
    }

    mod layout {
        use super::render_string_dumb;
        use insta::assert_snapshot;

        #[test]
        #[allow(non_snake_case)]
        fn GH_49_format_no_colour_simple() {
            assert_eq!(
                render_string_dumb("_lorem_ **ipsum** dolor **sit** _amet_").unwrap(),
                "lorem ipsum dolor sit amet\n",
            )
        }

        #[test]
        fn begins_with_rule() {
            assert_snapshot!(render_string_dumb("----").unwrap())
        }

        #[test]
        fn begins_with_block_quote() {
            assert_snapshot!(render_string_dumb("> Hello World").unwrap());
        }

        #[test]
        fn rule_in_block_quote() {
            assert_snapshot!(render_string_dumb(
                "> Hello World

> ----"
            )
            .unwrap());
        }

        #[test]
        fn heading_in_block_quote() {
            assert_snapshot!(render_string_dumb(
                "> Hello World

> # Hello World"
            )
            .unwrap())
        }

        #[test]
        fn heading_levels() {
            assert_snapshot!(render_string_dumb(
                "
# First

## Second

### Third"
            )
            .unwrap())
        }

        #[test]
        fn autolink_creates_no_reference() {
            assert_eq!(
                render_string_dumb("Hello <http://example.com>").unwrap(),
                "Hello http://example.com\n"
            )
        }

        #[test]
        fn flush_ref_links_before_toplevel_heading() {
            assert_snapshot!(render_string_dumb(
                "> Hello [World](http://example.com/world)

> # No refs before this headline

# But before this"
            )
            .unwrap())
        }

        #[test]
        fn flush_ref_links_at_end() {
            assert_snapshot!(render_string_dumb(
                "Hello [World](http://example.com/world)

# Headline

Hello [Donald](http://example.com/Donald)"
            )
            .unwrap())
        }
    }

    mod disabled_features {
        use insta::assert_snapshot;

        use super::render_string_dumb;

        #[test]
        #[allow(non_snake_case)]
        fn GH_155_do_not_choke_on_footnotes() {
            assert_snapshot!(render_string_dumb(
                "A footnote [^1]

[^1: We do not support footnotes."
            )
            .unwrap())
        }
    }
}