mdtext 0.1.2

mod block;
mod generated_entities;
mod inline;
mod tagfilter;
mod utils;

pub mod html;

use std::borrow::Cow;
use std::collections::VecDeque;

/// A parse event, yielded by [`Parser`].
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum Event<'a> {
    /// Signals the beginning of a new markdown block or inline element.
    Start(Tag<'a>),
    /// Signals that the most recently opened element has just closed (i.e. last-in first-out).
    End,
    /// A span of text within the currently active element. It is **never** multi-line.
    Text(Cow<'a, str>),
    /// A span of code content within the currently active code span/block.
    Code(Cow<'a, str>),
    /// A line break that must be interpreted in HTML as either a literal `\n` or as a `U+0020` space.
    /// It may only appear after a `Text` element.
    SoftBreak,
    /// A line break that must be interpreted in HTML as `<br />`. It may only appear after a `Text`
    /// element.
    HardBreak,
    /// Must be interpreted in HTML as `<hr />`.
    ThematicBreak,
    /// A soan of HTML within the current block or inline HTML.
    /// Should be emitted literally.
    Html(Cow<'a, str>),
    /// Marker for a task-list item. `true` means checked. May only appear on list items.
    TaskListMarker(bool),
    /// GitHub/LaTeX math syntax (`$...$`, `` $`...`$ ``, or `\(...\)`).
    InlineMath(Cow<'a, str>),
    /// A span of display math content within [`Tag::DisplayMath`].
    DisplayMath(Cow<'a, str>),
}

/// The kind of element that is being opened. See: [`Event::Start`]
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum Tag<'a> {
    /// `<p>...</p>`
    Paragraph,
    /// `<h1>...</h1>` from 1-6 levels (same as HTML).
    Heading(u8),
    /// Either an indented or fenced code block with an optional info string.
    /// The info string should be included in HTML as a `class="language-XXX"` atribute.
    CodeBlock(Option<Cow<'a, str>>),
    /// An inline `<code>...</code>` span.
    CodeSpan,
    /// A GitHub/LaTeX display-math construct.
    DisplayMath,
    /// A CommonMark §4.6 HTML block.
    HtmlBlock,
    /// §5.1 Block quotes.
    Quote,
    /// Either a `<ul>` or `<ol>` depending on [`ListKind`]
    List(ListKind),
    /// An `<li>` within the current list.
    Item,
    /// Emphasis (usually interpreted as italic)
    Emphasis,
    /// Strong emphasis (usually interpreted as bold).
    Strong,
    /// A destination `url` and an optional (i.e. empty == `None`) title.
    Link {
        url: Cow<'a, str>,
        title: Option<Cow<'a, str>>,
    },
    /// A destination `url` and an optional (i.e. empty == `None`) title.
    /// Description content will be emitted as child events until [`Event::End`].
    Image {
        url: Cow<'a, str>,
        title: Option<Cow<'a, str>>,
    },

    // ------------------
    // ### Extensions ###
    // ------------------
    /// `<del>...</del>`. Requires [`Options::STRIKETHROUGH`]
    Strikethrough,
    /// `<td>...</td>`. Requires [`Options::TABLES`]
    Table(Vec<Alignment>),
    /// `<th>...</th>`. Requires [`Options::TABLES`]
    TableHead,
    /// `<tbody>...</tbody>`. Requires [`Options::TABLES`]
    TableBody,
    /// `<tr>...</tr>`. Requires [`Options::TABLES`]
    TableRow,
    /// Signals the start of content contained within a table cell. Requires [`Options::TABLES`]
    TableCell,
}

#[derive(Debug, Clone, PartialEq, Eq)]
pub enum ListKind {
    /// Numbered list with the `u32` field representing the starting item number.
    Ordered(u32),
    /// Ordered list
    Unordered,
}

/// Text alignment of all cells within the currently active table column. Returned only for the table head.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Alignment {
    None,
    Left,
    Center,
    Right,
}

pub(crate) enum Action<'a> {
    Event(Event<'a>),
    /// Drain one detached root from the reusable inline-parser arena.
    InlineParse(inline::InlineRoot),
}

// NOTE: Left as u64 for future expansion
/// Bitflag parser extensions. Example: [`Options::GFM`] enables GitHub-flavored markdown.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Default)]
pub struct Options(u64);

/*
    TODO: To extend IMMEDIATE_MODE to the normal path (i.e. non-root), changes need to be done to the architecture w.r.t. leaf
    parsing and event emission.

    Broad Idea:
    - Split out content into its own field. Remains empty for deferred output and tables.
    - Avoid emitting events directly after parse. Instead, queue the parsed component (which could be provably constant-sized).
    - At the end of Scanner::parse_line, run a function to take what was queued and emit or defer.

    It could also be investigated whether LineStart/BlockStart can be reused to simplify this process.

    This should cleanly enforce the invariants of this process (ensuring correctness both in terms of Rust and markdown parsing)
    and make it easier to reason about what the code is actually doing. The changes involved are likely mostly around
    moving and encapsulating the existing code along with some state machinery to represent the transition from
    parsing -> decision -> emission | deferral. Likely a minimal change for the future.
*/
impl Options {
    /// GFM Tables extension.
    pub const TABLES: Self = Self(1 << 0);
    /// GFM Task list items extension.
    pub const TASK_LISTS: Self = Self(1 << 1);
    /// GFM Strikethrough extension.
    pub const STRIKETHROUGH: Self = Self(1 << 2);
    /// GFM Autolinks extension.
    pub const EXTENDED_AUTOLINKS: Self = Self(1 << 3);
    /// GFM Disallowed Raw HTML extension (tagfilter).
    pub const TAGFILTER: Self = Self(1 << 4);
    /// GitHub math syntax: `$...$` and `$$...$$`.
    pub const MATH_DOLLARS: Self = Self(1 << 5);
    /// GitHub math syntax: `` $`...`$ ``.
    pub const MATH_CODE: Self = Self(1 << 6);
    /// LaTeX math syntax: `\(...\)` inline and `\[...\]` display.
    pub const MATH_LATEX: Self = Self(1 << 7);
    /// CommonMark and GFM diverge in classifying emphasis runs due to differing definitions of
    /// Unicode punctuation and how emphasis runs are minimized. This enables the GFM definition.
    pub const GFM_DIALECT: Self = Self(1 << 8);
    /// Emit top-level fenced code and HTML blocks line-by-line instead of waiting for their closer
    /// or the end of the document. This works because none of the lines have to be parsed inline and
    /// are **not** dependent on future lines -- which is why indented code block can't be added (needs
    /// lookahead to know whether a blank line closes it or is a continuation).
    pub const IMMEDIATE_MODE: Self = Self(1 << 9);

    pub const GFM: Self = Self(
        Self::TABLES.0
            | Self::TASK_LISTS.0
            | Self::STRIKETHROUGH.0
            | Self::EXTENDED_AUTOLINKS.0
            | Self::TAGFILTER.0
            | Self::GFM_DIALECT.0,
    );
    pub const MATH: Self = Self(Self::MATH_DOLLARS.0 | Self::MATH_CODE.0 | Self::MATH_LATEX.0);

    pub const fn empty() -> Self {
        Self(0)
    }

    pub const fn from_bits(bits: u64) -> Self {
        Self(bits)
    }

    pub const fn bits(self) -> u64 {
        self.0
    }

    pub const fn contains(self, other: Self) -> bool {
        (self.0 & other.0) == other.0
    }

    pub fn insert(&mut self, other: Self) {
        self.0 |= other.0;
    }

    pub fn remove(&mut self, other: Self) {
        self.0 &= !other.0;
    }
}

impl std::ops::BitOr for Options {
    type Output = Self;

    fn bitor(self, rhs: Self) -> Self::Output {
        Self(self.0 | rhs.0)
    }
}

impl std::ops::BitOrAssign for Options {
    fn bitor_assign(&mut self, rhs: Self) {
        self.0 |= rhs.0;
    }
}

impl std::ops::BitAnd for Options {
    type Output = Self;

    fn bitand(self, rhs: Self) -> Self::Output {
        Self(self.0 & rhs.0)
    }
}

pub use tagfilter::{filter_disallowed_html, filter_disallowed_html_into};

use crate::block::BufferedLeafEvents;

/// A streaming markdown parser. Text should be fed with [`Parser::feed`] until input exhaustion, where
/// upon you must call [`Parser::finish`].
pub struct Parser {
    block_parser: block::BlockParser,
    /// The byte index corresponding to the start of the new line we're currently reading.
    line_start: usize,
    /// The byte index of the byte we'd last read.
    current_pos: usize,
    finished: bool,
}

impl Parser {
    /// Create a new parser with default options.
    pub fn new() -> Self {
        Self {
            block_parser: block::BlockParser::new(Options::empty()),
            line_start: 0,
            current_pos: 0,
            finished: false,
        }
    }

    /// Create a parser with runtime-selected extensions.
    pub fn with_options(options: Options) -> Self {
        Self {
            block_parser: block::BlockParser::new(options),
            line_start: 0,
            current_pos: 0,
            finished: false,
        }
    }

    /// Feed a chunk of text to the parser and receive `(events, read)`.
    /// - `events` is non-empty IFF a block has been fully parsed.
    /// - `read` is non-zero IFF `events` is non-empty. Subsequent calls must then feed `&s[read..]` for valid parsing.
    pub fn feed_chunk<'a>(&mut self, s: &'a str) -> (VecDeque<Event<'a>>, usize) {
        let mut iter = self.feed(s);
        let events = iter.by_ref().collect();
        let read = iter.consumed();
        (events, read)
    }

    /// When the end of the input stream has been reached, this method must be called to flush any remaining
    /// markdown events and close all open blocks.
    pub fn finish<'a>(&mut self, s: &'a str) -> VecDeque<Event<'a>> {
        self.finish_iter(s).collect()
    }

    /// Convenience: parse a complete markdown string into a `Vec` of events.
    pub fn parse_str(markdown: &str, options: Options) -> Vec<Event<'_>> {
        let mut parser = Self::with_options(options);
        let (events, consumed) = parser.feed_chunk(markdown);
        let mut all: Vec<Event<'_>> = events.into_iter().collect();
        let remaining = if consumed > 0 {
            &markdown[consumed..]
        } else {
            markdown
        };
        all.extend(parser.finish(remaining));
        all
    }

    /// Feed a chunk `s` of text. Returns a lazy iterator of zero or more events. Upon iterator exhaustion
    /// you **must** call [`EventIterator::consumed`] to know how many bytes of the input `s` have been
    /// parsed and **must** no longer be included in future calls to this method. Failure to do so will
    /// lead to invalid parsing (and likely panics).
    pub fn feed<'p, 'a>(&'p mut self, s: &'a str) -> EventIterator<'p, 'a> {
        let pos = self.current_pos;
        EventIterator {
            inner: InnerEventIterator {
                parser: self,
                buf: s.as_bytes(),
                actions: VecDeque::new(),
                active_event_source: None,
                mode: IteratorMode::Feed {
                    pos,
                    output_line_start: 0,
                },
            },
        }
    }

    /// Feed any final text (or an empty str slice) to flush remaining events. Necessary for
    /// closing any remaining blocks.
    pub fn finish_iter<'p, 'a>(&'p mut self, s: &'a str) -> EventIterator<'p, 'a> {
        EventIterator {
            inner: InnerEventIterator {
                parser: self,
                buf: s.as_bytes(),
                actions: VecDeque::new(),
                active_event_source: None,
                mode: IteratorMode::Finish {
                    state: FinishState::Tail,
                    batch_active: false,
                },
            },
        }
    }
}

/// Iterator over the events produced by one chunk fed via [`Parser::feed`].
// It is a wrapper b.c. it improves performance with inlining. Dunno why.
pub struct EventIterator<'p, 'a> {
    inner: InnerEventIterator<'p, 'a>,
}

struct InnerEventIterator<'p, 'a> {
    parser: &'p mut Parser,
    buf: &'a [u8],
    actions: VecDeque<Action<'a>>,
    active_event_source: Option<EventOutputSource<'a>>,
    mode: IteratorMode,
}

enum IteratorMode {
    Feed {
        pos: usize,
        /// Start of the most recent physical line whose parse produced output.
        output_line_start: usize,
    },
    Finish {
        state: FinishState,
        batch_active: bool,
    },
}

enum FinishState {
    Tail,
    CloseBlocks,
    Done,
}

enum EventOutputSource<'a> {
    Buffered(block::BufferedLeafEvents<'a>),
    Inline(inline::InlineCursor),
}

impl<'p, 'a> EventIterator<'p, 'a> {
    /// Returns the number of bytes that were read **AND** parsed. The caller MUST exclude these bytes from future reads for valid parsing.
    /// i.e. `buf[consumed..]`. ALWAYS call this after consuming the iterator!
    pub fn consumed(&self) -> usize {
        let IteratorMode::Feed {
            output_line_start, ..
        } = self.inner.mode
        else {
            // Only returned when we're finishing!
            // Entire input is consumed.
            return self.inner.buf.len();
        };

        if self.inner.parser.block_parser.leaf_is_open() {
            output_line_start
        } else {
            self.inner.parser.line_start
        }
    }
}

impl<'p, 'a> Iterator for EventIterator<'p, 'a> {
    type Item = Event<'a>;

    #[inline(always)]
    fn next(&mut self) -> Option<Event<'a>> {
        self.inner.next()
    }
}

#[inline(always)]
fn check_buffered<'a>(
    source: &mut Option<EventOutputSource<'a>>,
    buffered: Option<BufferedLeafEvents<'a>>,
) -> Option<Event<'a>> {
    let mut events = buffered?;

    // SAFETY: `BufferedLeafEvents` is created IFF there it has parsed a leaf (i.e. it has events to emit).
    let event = unsafe { events.next().unwrap_unchecked() };
    *source = Some(EventOutputSource::Buffered(events));
    Some(event)
}

impl<'p, 'a> Iterator for InnerEventIterator<'p, 'a> {
    type Item = Event<'a>;

    #[inline(always)]
    fn next(&mut self) -> Option<Event<'a>> {
        loop {
            // Return any buffered events.
            if let Some(source) = &mut self.active_event_source {
                let event = match source {
                    EventOutputSource::Buffered(events) => events.next(),
                    EventOutputSource::Inline(cursor) => {
                        self.parser.block_parser.next_inline_event(cursor)
                    }
                };

                if let Some(event) = event {
                    return Some(event);
                } // Buffer exhausted

                self.active_event_source = None;
            }

            // Fetch the next set of buffered events.
            // TODO: not truly lazy yet since inline parsing has already been done in block parser
            if let Some(action) = self.actions.pop_front() {
                match action {
                    Action::Event(event) => return Some(event),
                    Action::InlineParse(root) => {
                        if let IteratorMode::Finish { batch_active, .. } = &mut self.mode {
                            *batch_active = true;
                        }
                        let mut cursor = inline::InlineCursor::new(root);
                        if let Some(event) = self.parser.block_parser.next_inline_event(&mut cursor)
                        {
                            self.active_event_source = Some(EventOutputSource::Inline(cursor));
                            return Some(event);
                        }
                        continue;
                    }
                }
            }

            if let IteratorMode::Finish {
                state,
                batch_active,
            } = &mut self.mode
            {
                if *batch_active {
                    self.parser.block_parser.reset_inline();
                    *batch_active = false;
                }
                let buf = self.buf;
                match state {
                    FinishState::Tail => {
                        *state = FinishState::CloseBlocks;

                        // NOTE: line_start instead of pos is used since we want to ensure all remaining lines are flushed
                        if !self.parser.finished && self.parser.line_start < buf.len() {
                            let pos = self.parser.line_start;

                            // Ensure that we only get the end of the line content (excluding newlines)
                            let mut end = buf.len();
                            if buf[end - 1] == b'\n' {
                                end -= 1;

                                // Handle \r\n
                                if end > pos && buf[end - 1] == b'\r' {
                                    end -= 1;
                                }
                            } else if buf[end - 1] == b'\r' {
                                end -= 1;
                            }

                            let mut buffered = None;
                            self.parser.block_parser.parse_line_for_iter(
                                buf,
                                pos..end,
                                pos..buf.len(),
                                &mut self.actions,
                                &mut buffered,
                            );

                            if let Some(event) =
                                check_buffered(&mut self.active_event_source, buffered)
                            {
                                return Some(event);
                            }
                        }
                        continue;
                    }
                    FinishState::CloseBlocks => {
                        *state = FinishState::Done;
                        self.parser.finished = true;
                        let mut buffered = None;
                        self.parser.block_parser.finish_for_iter(
                            buf,
                            &mut self.actions,
                            &mut buffered,
                        );
                        if let Some(event) = check_buffered(&mut self.active_event_source, buffered)
                        {
                            return Some(event);
                        }
                        continue;
                    }
                    FinishState::Done => return None,
                }
            }

            let IteratorMode::Feed {
                pos,
                output_line_start,
            } = &mut self.mode
            else {
                unreachable!()
            };

            // We've exhausted all buffered content. Therefore, we must continue reading any remaining text
            // in the chunk and buffer more content. First, reset all arenas for reuse.
            self.parser.block_parser.reset_inline();

            // Multiple lines may need to be read before the parser is ready to emit events.
            loop {
                if *pos >= self.buf.len() {
                    return None;
                }

                // Locate the next complete line (terminated by LF/CR/CRLF).
                // NOTE: LLVM automatically removed bounds checking from my testing
                let new_bytes = &self.buf[*pos..];
                let line_end = if new_bytes.len() >= 16 {
                    match memchr::memchr2(b'\n', b'\r', new_bytes) {
                        Some(offset) => *pos + offset,
                        None => self.buf.len(),
                    }
                } else {
                    match new_bytes.iter().position(|&b| b == b'\n' || b == b'\r') {
                        Some(offset) => *pos + offset,
                        None => self.buf.len(),
                    }
                };

                // The new content still does not contain a line ending to terminate the current line.
                // Thus, we must wait for the consumer to provide further data.
                if line_end >= self.buf.len() {
                    *pos = self.buf.len();
                    return None;
                }

                let mut next_line_start = line_end + 1;

                // CommonMark considers `\r`, `\n`, `\r\n` valid line endings. If we blindly accept any `\r`, though, `\r\n`
                // would get decomposed into two separate lines leading to invalid parsing. So, we must check if the
                // next byte is `\n` or await further input.
                if self.buf[line_end] == b'\r' {
                    if line_end + 1 == self.buf.len() {
                        *pos = line_end; // to include `\r` for the next round
                        return None;
                    }

                    if self.buf.get(line_end + 1) == Some(&b'\n') {
                        next_line_start += 1;
                    }
                }

                let line_start = self.parser.line_start;
                let line = line_start..line_end;
                let line_with_ending = line_start..next_line_start;
                let mut buffered = None;
                self.parser.block_parser.parse_line_for_iter(
                    self.buf,
                    line,
                    line_with_ending,
                    &mut self.actions,
                    &mut buffered,
                );

                self.parser.line_start = next_line_start;
                *pos = next_line_start;

                if let Some(mut events) = buffered {
                    // Created events; drop old text
                    *output_line_start = line_start;
                    self.parser.current_pos = *pos;

                    // SAFETY: `BufferedLeafEvents` is created IFF there it has parsed a leaf (i.e. it has
                    // events to emit).
                    let event = unsafe { events.next().unwrap_unchecked() };
                    self.active_event_source = Some(EventOutputSource::Buffered(events));
                    return Some(event);
                }

                if !self.actions.is_empty() {
                    // Created events; drop old text
                    *output_line_start = line_start;
                    self.parser.current_pos = *pos;

                    // We have inline events to return!
                    break;
                }
            }
        }
    }
}

impl Drop for InnerEventIterator<'_, '_> {
    fn drop(&mut self) {
        if matches!(self.mode, IteratorMode::Feed { .. }) {
            // We can't drop until we consume the entire iterator. The block parser's state has already changed,
            // and so we must also change the user-facing parser!
            while self.next().is_some() {}

            self.parser.block_parser.reset_inline();

            let IteratorMode::Feed {
                pos,
                output_line_start,
            } = self.mode
            else {
                unreachable!()
            };

            // Update the parent parser's position so that it knows what content it needs for future feeds.
            if self.parser.block_parser.leaf_is_open() {
                let consumed = output_line_start;
                self.parser.block_parser.update_leaf_spans(consumed);
                self.parser.line_start -= consumed;
                self.parser.current_pos = pos - consumed;
            } else {
                // We consumed a block and thus we must prep the parser to be ready for the next block!
                let consumed = self.parser.line_start;
                self.parser.line_start = 0;

                // this will always be valid since the consumed bytes must be included in the new block
                // TODO: fix bad comment above...
                self.parser.current_pos = pos - consumed;
            }
        } else {
            self.parser.block_parser.reset_inline();
        }
    }
}

impl Default for Parser {
    fn default() -> Self {
        Self::new()
    }
}