wikioxide 0.1.0

//! Block-level line machine: headings, lists, indent-pre, horizontal rules,
//! paragraphs, and the master loop that interlocks with the table machine.
//!
//! A single forward pass over the lines of the flattened marker text. Table
//! nesting runs on an explicit stack of [`TableFrame`]s — a table opened from
//! a block sink captures that sink as its `parent` and gives it back when the
//! table closes, so cells-in-blocks-in-cells never recurse on input depth.

use crate::engine::ast::{
    Block, DocTree, InlineContent, ListItem, ListItemKind, ListKind, TextRange,
};
use crate::engine::strip;
use crate::warn::{Warning, WarningCollector};

use super::table::{TableCtx, TableOutcome};

/// Converts byte offsets into a [`TextRange`], saturating past `u32::MAX`
/// (unreachable in practice: hygiene caps input size well below that).
pub(super) fn range(start: usize, end: usize) -> TextRange {
    TextRange::new(
        u32::try_from(start).unwrap_or(u32::MAX),
        u32::try_from(end).unwrap_or(u32::MAX),
    )
}

/// Trims spaces and tabs from both ends of `flat[start..end]`; `None` when
/// nothing remains.
pub(super) fn trim_range(flat: &str, start: usize, end: usize) -> Option<TextRange> {
    let bytes = flat.as_bytes();
    let mut s = start;
    let mut e = end;
    while s < e && matches!(bytes[s], b' ' | b'\t') {
        s += 1;
    }
    while e > s && matches!(bytes[e - 1], b' ' | b'\t') {
        e -= 1;
    }
    (s < e).then_some(range(s, e))
}

/// True when `flat[ls..le]` is empty or only spaces/tabs.
pub(super) fn is_blank(flat: &str, ls: usize, le: usize) -> bool {
    flat.as_bytes()[ls..le]
        .iter()
        .all(|b| matches!(b, b' ' | b'\t'))
}

/// True for the four list prefix bytes.
fn is_list_marker(b: u8) -> bool {
    matches!(b, b'*' | b'#' | b';' | b':')
}

/// List family of a prefix byte (`;` and `:` share one definition list).
fn list_kind(marker: u8) -> ListKind {
    match marker {
        b'*' => ListKind::Bullet,
        b'#' => ListKind::Ordered,
        _ => ListKind::Definition,
    }
}

/// Item classification of a prefix byte.
fn item_kind(marker: u8) -> ListItemKind {
    match marker {
        b'*' | b'#' => ListItemKind::Item,
        b';' => ListItemKind::Term,
        _ => ListItemKind::Definition,
    }
}

/// One open list level. Deeper open levels attach to `items.last()` when they
/// close, which reproduces "a nested list lives in the last shallower item".
struct OpenLevel {
    /// List family at this depth.
    kind: ListKind,
    /// The marker byte that last opened an item here — implicit items created
    /// for depth jumps inherit it.
    marker: u8,
    /// Items collected at this depth so far.
    items: Vec<ListItem>,
}

/// What a dispatched line asked the master loop to do.
enum LineOutcome {
    /// Fully handled inside the block state.
    Done,
    /// The line opened a `{|` table; the master loop pushes a table frame.
    OpenTable {
        /// Trimmed attribute text after `{|`.
        attributes: Option<TextRange>,
    },
}

/// Accumulating block-machine state for one sink: the document, or one table
/// cell's continuation lines.
#[derive(Default)]
struct BlockState {
    /// Pending paragraph lines.
    para: Vec<InlineContent>,
    /// Pending indent-pre lines.
    pre: Vec<InlineContent>,
    /// Open list levels, shallowest first.
    spine: Vec<OpenLevel>,
    /// Finished blocks.
    blocks: Vec<Block>,
}

impl BlockState {
    /// Closes the pending paragraph, if any.
    fn flush_para(&mut self) {
        if !self.para.is_empty() {
            let lines = std::mem::take(&mut self.para);
            self.blocks.push(Block::Paragraph { lines });
        }
    }

    /// Closes the pending indent-pre block, if any.
    fn flush_pre(&mut self) {
        if !self.pre.is_empty() {
            let lines = std::mem::take(&mut self.pre);
            self.blocks.push(Block::IndentPre { lines });
        }
    }

    /// Closes open list levels, deepest first, down to `depth` levels.
    fn close_lists_to(&mut self, depth: usize) {
        while self.spine.len() > depth {
            // `len > depth` makes this infallible: `swap_remove` of the last
            // element is `pop` without the `Option`.
            let level = self.spine.swap_remove(self.spine.len() - 1);
            let block = Block::List {
                kind: level.kind,
                items: level.items,
            };
            match self.spine.last_mut() {
                Some(parent) => {
                    // `open_level` guaranteed the parent has an item.
                    if let Some(item) = parent.items.last_mut() {
                        item.children.push(block);
                    }
                }
                None => self.blocks.push(block),
            }
        }
    }

    /// Opens one list level under the current deepest one, creating an
    /// implicit parent item (content `None`) when the parent has none yet —
    /// the depth-jump case (`***x` from nothing).
    fn open_level(&mut self, marker: u8) {
        if let Some(parent) = self.spine.last_mut() {
            if parent.items.is_empty() {
                parent.items.push(ListItem {
                    kind: item_kind(parent.marker),
                    content: None,
                    children: Vec::new(),
                });
            }
        }
        self.spine.push(OpenLevel {
            kind: list_kind(marker),
            marker,
            items: Vec::new(),
        });
    }

    /// Appends an item at the deepest open level.
    fn push_item(&mut self, marker: u8, content: Option<InlineContent>) {
        if let Some(level) = self.spine.last_mut() {
            level.marker = marker;
            level.items.push(ListItem {
                kind: item_kind(marker),
                content,
                children: Vec::new(),
            });
        }
    }

    /// Aligns the open spine with the line's `*#;:` prefix: shared levels stay
    /// open (`;`/`:` count as the same definition list, which is what lets a
    /// term switch to a definition without closing the list), deeper levels
    /// close, missing levels open.
    fn align_lists(&mut self, prefix: &[u8]) {
        let mut shared = 0;
        while shared < prefix.len()
            && shared < self.spine.len()
            && list_kind(prefix[shared]) == list_kind(self.spine[shared].marker)
        {
            shared += 1;
        }
        self.close_lists_to(shared);
        for &marker in &prefix[shared..] {
            self.open_level(marker);
        }
    }

    /// Handles one list line; `prefix_end` is where its `*#;:` run stops.
    fn list_line(&mut self, flat: &str, ls: usize, prefix_end: usize, le: usize) {
        let prefix = &flat.as_bytes()[ls..prefix_end];
        self.align_lists(prefix);
        let deepest = prefix[prefix.len() - 1];
        if deepest == b';' {
            if let Some(colon) = find_colon(flat, prefix_end, le) {
                self.push_item(b';', Some(InlineContent::Raw(range(prefix_end, colon))));
                self.push_item(b':', Some(InlineContent::Raw(range(colon + 1, le))));
                return;
            }
        }
        self.push_item(deepest, Some(InlineContent::Raw(range(prefix_end, le))));
    }

    /// Opens the definition-list chain for a `::{|` line; the table block will
    /// attach to the content-less item this pushes.
    fn colon_list_for_table(&mut self, flat: &str, ls: usize, prefix_end: usize) {
        let prefix = &flat.as_bytes()[ls..prefix_end];
        self.align_lists(prefix);
        self.push_item(b':', None);
    }

    /// Attaches a finished block: into the deepest open list item when a list
    /// is open (the `::{|` wrapping case), otherwise as a sibling block.
    fn attach_block(&mut self, block: Block) {
        match self
            .spine
            .last_mut()
            .and_then(|level| level.items.last_mut())
        {
            Some(item) => item.children.push(block),
            None => self.blocks.push(block),
        }
    }

    /// Flushes everything still open and returns the finished blocks.
    fn finish_blocks(mut self) -> Vec<Block> {
        self.flush_para();
        self.flush_pre();
        self.close_lists_to(0);
        self.blocks
    }

    /// Dispatches one line. `ls..le` excludes the newline.
    fn dispatch_line(&mut self, flat: &str, ls: usize, le: usize) -> LineOutcome {
        let bytes = flat.as_bytes();
        let line = &bytes[ls..le];
        if line.starts_with(b"{|") {
            self.flush_para();
            self.flush_pre();
            self.close_lists_to(0);
            return LineOutcome::OpenTable {
                attributes: trim_range(flat, ls + 2, le),
            };
        }
        if line.first() == Some(&b':') {
            let mut colons = 1;
            while colons < line.len() && line[colons] == b':' {
                colons += 1;
            }
            if line[colons..].starts_with(b"{|") {
                self.flush_para();
                self.flush_pre();
                self.colon_list_for_table(flat, ls, ls + colons);
                return LineOutcome::OpenTable {
                    attributes: trim_range(flat, ls + colons + 2, le),
                };
            }
        }
        if is_blank(flat, ls, le) {
            if !self.pre.is_empty() && line.first() == Some(&b' ') {
                // A space-only line between pre lines continues the pre.
                self.pre.push(InlineContent::Raw(range(ls + 1, le)));
            } else {
                self.flush_para();
                self.flush_pre();
                self.close_lists_to(0);
            }
            return LineOutcome::Done;
        }
        // Non-blank from here on, so the line has a first byte.
        let first = line[0];
        if is_list_marker(first) {
            let mut prefix_end = ls;
            while prefix_end < le && is_list_marker(bytes[prefix_end]) {
                prefix_end += 1;
            }
            self.flush_para();
            self.flush_pre();
            self.list_line(flat, ls, prefix_end, le);
            return LineOutcome::Done;
        }
        if first == b' ' {
            self.flush_para();
            self.close_lists_to(0);
            self.pre.push(InlineContent::Raw(range(ls + 1, le)));
            return LineOutcome::Done;
        }
        if first == b'=' {
            if let Some((level, content)) = try_heading(flat, ls, le) {
                self.flush_para();
                self.flush_pre();
                self.close_lists_to(0);
                self.blocks.push(Block::Heading {
                    level,
                    line: range(ls, le),
                    inner: content,
                    content: InlineContent::Raw(content),
                });
                return LineOutcome::Done;
            }
        }
        if line.starts_with(b"----") {
            let mut dashes_end = ls + 4;
            while dashes_end < le && bytes[dashes_end] == b'-' {
                dashes_end += 1;
            }
            self.flush_para();
            self.flush_pre();
            self.close_lists_to(0);
            self.blocks.push(Block::HorizontalRule);
            if dashes_end < le {
                self.para.push(InlineContent::Raw(range(dashes_end, le)));
            }
            return LineOutcome::Done;
        }
        self.flush_pre();
        self.close_lists_to(0);
        self.para.push(InlineContent::Raw(range(ls, le)));
        LineOutcome::Done
    }
}

/// Parses `= heading =` lines.
///
/// Both `=` runs must exist with content between them; the level is
/// `min(left, right)` clamped to 6 and surplus `=` on the longer side stay
/// literal inside the content; trailing spaces/tabs after the closing run are
/// allowed (a trailing strip marker is not — it simply leaves no closing run).
fn try_heading(flat: &str, ls: usize, le: usize) -> Option<(u8, TextRange)> {
    let bytes = flat.as_bytes();
    let mut left = 0;
    while ls + left < le && bytes[ls + left] == b'=' {
        left += 1;
    }
    let mut end = le;
    while end > ls && matches!(bytes[end - 1], b' ' | b'\t') {
        end -= 1;
    }
    let mut right = 0;
    while end - right > ls && bytes[end - right - 1] == b'=' {
        right += 1;
    }
    if right == 0 || left + right >= end - ls {
        return None;
    }
    let level = left.min(right).min(6);
    let mut cs = ls + level;
    let mut ce = end - level;
    while cs < ce && matches!(bytes[cs], b' ' | b'\t') {
        cs += 1;
    }
    while ce > cs && matches!(bytes[ce - 1], b' ' | b'\t') {
        ce -= 1;
    }
    Some((u8::try_from(level).unwrap_or(6), range(cs, ce)))
}

/// Finds the first `:` in `flat[from..to]` outside `[[…]]` brackets, outside
/// `<…>` angle pairs, and not inside a strip marker — the lexical scan behind
/// `; term : definition` splitting (MediaWiki's `findColonNoLinks`).
fn find_colon(flat: &str, from: usize, to: usize) -> Option<usize> {
    let bytes = flat.as_bytes();
    let mut links = 0usize;
    let mut angles = 0usize;
    let mut i = from;
    while i < to {
        match bytes[i] {
            0x7f => {
                // Whole markers are opaque atoms; skip a decoded one entirely.
                if let Some(marker) = strip::parse_marker(&flat[i..]) {
                    i += marker.len;
                    continue;
                }
            }
            b'[' if i + 1 < to && bytes[i + 1] == b'[' => {
                links += 1;
                i += 2;
                continue;
            }
            b']' if i + 1 < to && bytes[i + 1] == b']' => {
                links = links.saturating_sub(1);
                i += 2;
                continue;
            }
            b'<' => angles += 1,
            b'>' => angles = angles.saturating_sub(1),
            b':' if links == 0 && angles == 0 => return Some(i),
            _ => {}
        }
        i += 1;
    }
    None
}

/// One open table plus the block sinks around it: `parent` is the sink the
/// finished table attaches to, `cont` the sink for the current cell's
/// continuation lines.
struct TableFrame {
    /// Block state this table was opened from; restored on close.
    parent: BlockState,
    /// The table under construction.
    ctx: TableCtx,
    /// Open continuation sink for the current cell, if any.
    cont: Option<BlockState>,
}

/// One master-loop decision about the current line.
enum Step {
    /// Line consumed.
    Done,
    /// State changed (frame popped into or flushed); run the line again.
    Reprocess,
    /// Push a new table frame capturing `parent` as its return sink.
    Push {
        /// The block sink the `{|` line came from.
        parent: BlockState,
        /// Trimmed `{|` attribute text.
        attributes: Option<TextRange>,
    },
    /// `|}` closed the top table; `rest` continues as a synthetic line.
    PopClosed {
        /// Remainder after `|}`, when not blank.
        rest: Option<(usize, usize)>,
    },
}

/// Routes one line to the innermost active machine, flushing / popping /
/// pushing table frames as needed. May set `pending` to a synthetic remainder
/// line (the text after `|}`).
fn process_line(
    flat: &str,
    ls: usize,
    le: usize,
    bottom: &mut BlockState,
    stack: &mut Vec<TableFrame>,
    pending: &mut Option<(usize, usize)>,
) {
    loop {
        let step = match stack.last_mut() {
            None => match bottom.dispatch_line(flat, ls, le) {
                LineOutcome::Done => Step::Done,
                LineOutcome::OpenTable { attributes } => Step::Push {
                    parent: std::mem::take(bottom),
                    attributes,
                },
            },
            Some(frame) => match frame.cont.take() {
                Some(mut cont) => {
                    let first = if ls < le {
                        Some(flat.as_bytes()[ls])
                    } else {
                        None
                    };
                    if matches!(first, Some(b'|' | b'!')) {
                        // Table markup ends the cell continuation; the blocks
                        // built so far land in the current cell.
                        frame.ctx.attach_continuation(ls, cont.finish_blocks());
                        Step::Reprocess
                    } else {
                        match cont.dispatch_line(flat, ls, le) {
                            LineOutcome::Done => {
                                frame.cont = Some(cont);
                                Step::Done
                            }
                            LineOutcome::OpenTable { attributes } => Step::Push {
                                parent: cont,
                                attributes,
                            },
                        }
                    }
                }
                None => match frame.ctx.handle_line(flat, ls, le) {
                    TableOutcome::Consumed => Step::Done,
                    TableOutcome::Continuation => {
                        frame.cont = Some(BlockState::default());
                        Step::Reprocess
                    }
                    TableOutcome::Closed { rest } => Step::PopClosed { rest },
                },
            },
        };
        match step {
            Step::Done => return,
            Step::Reprocess => {}
            Step::Push { parent, attributes } => {
                stack.push(TableFrame {
                    parent,
                    ctx: TableCtx::new(attributes),
                    cont: None,
                });
                return;
            }
            Step::PopClosed { rest } => {
                if let Some(frame) = stack.pop() {
                    pop_table(bottom, stack, frame, ls, true, None);
                }
                *pending = rest;
                return;
            }
        }
    }
}

/// Retires a table frame the caller just popped: flushes its open continuation
/// sink into the current cell, finalizes the table (warning when auto-closed
/// at end of input), attaches it into the parent sink, and reinstates that
/// sink as the active one (the frame below's continuation, or the bottom).
fn pop_table(
    bottom: &mut BlockState,
    stack: &mut [TableFrame],
    mut frame: TableFrame,
    at: usize,
    closed: bool,
    warnings: Option<&mut WarningCollector>,
) {
    if let Some(cont) = frame.cont.take() {
        frame.ctx.attach_continuation(at, cont.finish_blocks());
    }
    if let Some(collector) = warnings {
        collector.emit(Warning::UnclosedTable);
    }
    let table = frame.ctx.finish(closed);
    let mut parent = frame.parent;
    parent.attach_block(Block::Table(table));
    match stack.last_mut() {
        Some(below) => below.cont = Some(parent),
        None => *bottom = parent,
    }
}

/// Parses the flattened marker text into the block-level [`DocTree`].
///
/// Inline payloads stay [`InlineContent::Raw`] — the inline pass resolves them
/// per line later. All emitted ranges index `flat` (byte offsets on char
/// boundaries). Tables left open at end of input are auto-closed with
/// `closed = false` and a [`Warning::UnclosedTable`] emission each (the
/// collector dedups the identical emissions into one entry).
pub(crate) fn parse_blocks(flat: &str, warnings: &mut WarningCollector) -> DocTree {
    let mut lines: Vec<(usize, usize)> = Vec::new();
    let mut start = 0usize;
    for line in flat.split('\n') {
        lines.push((start, start + line.len()));
        start += line.len() + 1;
    }

    let mut bottom = BlockState::default();
    let mut stack: Vec<TableFrame> = Vec::new();
    let mut pending: Option<(usize, usize)> = None;
    let mut i = 0usize;
    loop {
        let (ls, le) = if let Some(synthetic) = pending.take() {
            synthetic
        } else if i < lines.len() {
            i += 1;
            lines[i - 1]
        } else {
            break;
        };
        process_line(flat, ls, le, &mut bottom, &mut stack, &mut pending);
    }
    while let Some(frame) = stack.pop() {
        pop_table(
            &mut bottom,
            &mut stack,
            frame,
            flat.len(),
            false,
            Some(&mut *warnings),
        );
    }
    DocTree {
        blocks: bottom.finish_blocks(),
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    fn parse(flat: &str) -> (DocTree, Vec<Warning>) {
        let mut warnings = WarningCollector::new();
        let doc = parse_blocks(flat, &mut warnings);
        (doc, warnings.into_vec())
    }

    fn blocks(flat: &str) -> Vec<Block> {
        let (doc, warnings) = parse(flat);
        assert_eq!(warnings, vec![]);
        doc.blocks
    }

    fn r(flat: &str, sub: &str) -> TextRange {
        let start = flat.find(sub).unwrap();
        range(start, start + sub.len())
    }

    fn raw(flat: &str, sub: &str) -> InlineContent {
        InlineContent::Raw(r(flat, sub))
    }

    /// A `Block::Heading` whose `inner` is the (unique) `name` substring and whose `line`
    /// is that substring's whole physical line — matching what the parser records.
    fn heading(flat: &str, level: u8, name: &str) -> Block {
        let inner = r(flat, name);
        let bytes = flat.as_bytes();
        let mut ls = inner.start as usize;
        while ls > 0 && bytes[ls - 1] != b'\n' {
            ls -= 1;
        }
        let mut le = inner.end as usize;
        while le < flat.len() && bytes[le] != b'\n' {
            le += 1;
        }
        Block::Heading {
            level,
            line: range(ls, le),
            inner,
            content: InlineContent::Raw(inner),
        }
    }

    fn item(kind: ListItemKind, content: Option<InlineContent>, children: Vec<Block>) -> ListItem {
        ListItem {
            kind,
            content,
            children,
        }
    }

    #[test]
    fn headings_h1_through_h6() {
        let flat = "=a=\n==b==\n===c===\n====d====\n=====e=====\n======f======";
        let expected: Vec<Block> = [("a", 1u8), ("b", 2), ("c", 3), ("d", 4), ("e", 5), ("f", 6)]
            .into_iter()
            .map(|(name, level)| heading(flat, level, name))
            .collect();
        assert_eq!(blocks(flat), expected);
    }

    #[test]
    fn heading_surplus_equals_stay_in_content() {
        let flat = "==h===\n===i==";
        assert_eq!(
            blocks(flat),
            vec![heading(flat, 2, "h="), heading(flat, 2, "=i")]
        );
    }

    #[test]
    fn heading_without_closing_run_is_a_paragraph() {
        let flat = "==h";
        assert_eq!(
            blocks(flat),
            vec![Block::Paragraph {
                lines: vec![raw(flat, "==h")],
            }]
        );
    }

    #[test]
    fn heading_allows_trailing_spaces_and_tabs() {
        let flat = "==h== \t ";
        assert_eq!(blocks(flat), vec![heading(flat, 2, "h")]);
    }

    #[test]
    fn heading_rejects_a_trailing_strip_marker() {
        let mut flat = String::from("=h=");
        strip::write_marker(&mut flat, strip::MarkerKind::ExtTag, strip::MarkerId(0));
        assert_eq!(
            blocks(&flat),
            vec![Block::Paragraph {
                lines: vec![InlineContent::Raw(range(0, flat.len()))],
            }]
        );
    }

    #[test]
    fn heading_level_clamps_at_six_and_all_equals_is_a_paragraph() {
        let flat = "========g========\n====";
        assert_eq!(
            blocks(flat),
            vec![
                heading(flat, 6, "==g=="),
                Block::Paragraph {
                    lines: vec![InlineContent::Raw(range(18, 22))],
                },
            ]
        );
    }

    #[test]
    fn heading_content_trims_surrounding_spaces() {
        let flat = "== h ==";
        assert_eq!(blocks(flat), vec![heading(flat, 2, "h")]);
    }

    #[test]
    fn paragraphs_group_lines_and_split_on_blank_lines() {
        let flat = "a\nb\n\nc\n \nd";
        assert_eq!(
            blocks(flat),
            vec![
                Block::Paragraph {
                    lines: vec![raw(flat, "a"), raw(flat, "b")],
                },
                Block::Paragraph {
                    lines: vec![raw(flat, "c")],
                },
                Block::Paragraph {
                    lines: vec![raw(flat, "d")],
                },
            ]
        );
    }

    #[test]
    fn behavior_switch_lines_are_plain_paragraph_text() {
        let flat = "__NOTOC__";
        assert_eq!(
            blocks(flat),
            vec![Block::Paragraph {
                lines: vec![raw(flat, "__NOTOC__")],
            }]
        );
    }

    #[test]
    fn empty_and_whitespace_inputs_produce_no_blocks() {
        assert_eq!(blocks(""), vec![]);
        assert_eq!(blocks("\n\t\n"), vec![]);
    }

    #[test]
    fn horizontal_rule_consumes_the_dash_run() {
        let flat = "----\n-----";
        assert_eq!(
            blocks(flat),
            vec![Block::HorizontalRule, Block::HorizontalRule]
        );
    }

    #[test]
    fn horizontal_rule_trailing_text_becomes_a_paragraph() {
        let flat = "----tail\nmore";
        assert_eq!(
            blocks(flat),
            vec![
                Block::HorizontalRule,
                Block::Paragraph {
                    lines: vec![raw(flat, "tail"), raw(flat, "more")],
                },
            ]
        );
    }

    #[test]
    fn three_dashes_are_not_a_rule() {
        let flat = "---";
        assert_eq!(
            blocks(flat),
            vec![Block::Paragraph {
                lines: vec![raw(flat, "---")],
            }]
        );
    }

    #[test]
    fn bullet_list_nests_in_the_last_shallower_item() {
        let flat = "*a\n**b\n*c\n**d";
        let sub = |content| Block::List {
            kind: ListKind::Bullet,
            items: vec![item(ListItemKind::Item, Some(content), vec![])],
        };
        assert_eq!(
            blocks(flat),
            vec![Block::List {
                kind: ListKind::Bullet,
                items: vec![
                    item(
                        ListItemKind::Item,
                        Some(raw(flat, "a")),
                        vec![sub(raw(flat, "b"))],
                    ),
                    item(
                        ListItemKind::Item,
                        Some(raw(flat, "c")),
                        vec![sub(raw(flat, "d"))],
                    ),
                ],
            }]
        );
    }

    #[test]
    fn mixed_ordered_sublist() {
        let flat = "*a\n*#b\n*#c";
        assert_eq!(
            blocks(flat),
            vec![Block::List {
                kind: ListKind::Bullet,
                items: vec![item(
                    ListItemKind::Item,
                    Some(raw(flat, "a")),
                    vec![Block::List {
                        kind: ListKind::Ordered,
                        items: vec![
                            item(ListItemKind::Item, Some(raw(flat, "b")), vec![]),
                            item(ListItemKind::Item, Some(raw(flat, "c")), vec![]),
                        ],
                    }],
                )],
            }]
        );
    }

    #[test]
    fn term_definition_split_on_one_line() {
        let flat = ";t:d";
        assert_eq!(
            blocks(flat),
            vec![Block::List {
                kind: ListKind::Definition,
                items: vec![
                    item(ListItemKind::Term, Some(raw(flat, "t")), vec![]),
                    item(ListItemKind::Definition, Some(raw(flat, "d")), vec![]),
                ],
            }]
        );
    }

    #[test]
    fn colon_inside_link_does_not_split_term() {
        let flat = ";x[[a:b]]y";
        assert_eq!(
            blocks(flat),
            vec![Block::List {
                kind: ListKind::Definition,
                items: vec![item(
                    ListItemKind::Term,
                    Some(raw(flat, "x[[a:b]]y")),
                    vec![]
                )],
            }]
        );
    }

    #[test]
    fn colon_after_closed_link_splits() {
        let flat = ";[[a:b]]:d";
        assert_eq!(
            blocks(flat),
            vec![Block::List {
                kind: ListKind::Definition,
                items: vec![
                    item(ListItemKind::Term, Some(raw(flat, "[[a:b]]")), vec![]),
                    item(ListItemKind::Definition, Some(raw(flat, "d")), vec![]),
                ],
            }]
        );
    }

    #[test]
    fn colon_inside_angle_pair_does_not_split() {
        let flat = ";t<a:b>u:d";
        assert_eq!(
            blocks(flat),
            vec![Block::List {
                kind: ListKind::Definition,
                items: vec![
                    item(ListItemKind::Term, Some(raw(flat, "t<a:b>u")), vec![]),
                    item(ListItemKind::Definition, Some(raw(flat, "d")), vec![]),
                ],
            }]
        );
    }

    #[test]
    fn colon_scan_skips_whole_markers_and_keeps_stray_sentinels_literal() {
        let mut flat = String::from(";a");
        strip::write_marker(&mut flat, strip::MarkerKind::Template, strip::MarkerId(1));
        flat.push_str(":d");
        let colon = flat.rfind(':').unwrap();
        assert_eq!(
            blocks(&flat),
            vec![Block::List {
                kind: ListKind::Definition,
                items: vec![
                    item(
                        ListItemKind::Term,
                        Some(InlineContent::Raw(range(1, colon))),
                        vec![],
                    ),
                    item(
                        ListItemKind::Definition,
                        Some(InlineContent::Raw(range(colon + 1, flat.len()))),
                        vec![],
                    ),
                ],
            }]
        );

        let stray = ";a\u{7f}z:d";
        assert_eq!(
            blocks(stray),
            vec![Block::List {
                kind: ListKind::Definition,
                items: vec![
                    item(ListItemKind::Term, Some(raw(stray, "a\u{7f}z")), vec![]),
                    item(ListItemKind::Definition, Some(raw(stray, "d")), vec![]),
                ],
            }]
        );
    }

    #[test]
    fn unbalanced_closers_do_not_underflow_the_colon_scan() {
        let flat = ";a]]>:d";
        assert_eq!(
            blocks(flat),
            vec![Block::List {
                kind: ListKind::Definition,
                items: vec![
                    item(ListItemKind::Term, Some(raw(flat, "a]]>")), vec![]),
                    item(ListItemKind::Definition, Some(raw(flat, "d")), vec![]),
                ],
            }]
        );
    }

    #[test]
    fn semicolon_line_without_colon_is_a_plain_term() {
        let flat = ";t";
        assert_eq!(
            blocks(flat),
            vec![Block::List {
                kind: ListKind::Definition,
                items: vec![item(ListItemKind::Term, Some(raw(flat, "t")), vec![])],
            }]
        );
    }

    #[test]
    fn term_switches_to_definition_without_closing_the_list() {
        let flat = ";t\n:d";
        assert_eq!(
            blocks(flat),
            vec![Block::List {
                kind: ListKind::Definition,
                items: vec![
                    item(ListItemKind::Term, Some(raw(flat, "t")), vec![]),
                    item(ListItemKind::Definition, Some(raw(flat, "d")), vec![]),
                ],
            }]
        );
    }

    #[test]
    fn plain_colon_lines_build_a_definition_list() {
        let flat = ":d1\n::d2";
        assert_eq!(
            blocks(flat),
            vec![Block::List {
                kind: ListKind::Definition,
                items: vec![item(
                    ListItemKind::Definition,
                    Some(raw(flat, "d1")),
                    vec![Block::List {
                        kind: ListKind::Definition,
                        items: vec![item(
                            ListItemKind::Definition,
                            Some(raw(flat, "d2")),
                            vec![]
                        )],
                    }],
                )],
            }]
        );
    }

    #[test]
    fn depth_jump_creates_implicit_parent_items() {
        let flat = "***x";
        assert_eq!(
            blocks(flat),
            vec![Block::List {
                kind: ListKind::Bullet,
                items: vec![item(
                    ListItemKind::Item,
                    None,
                    vec![Block::List {
                        kind: ListKind::Bullet,
                        items: vec![item(
                            ListItemKind::Item,
                            None,
                            vec![Block::List {
                                kind: ListKind::Bullet,
                                items: vec![item(ListItemKind::Item, Some(raw(flat, "x")), vec![])],
                            }],
                        )],
                    }],
                )],
            }]
        );
    }

    #[test]
    fn blank_line_closes_all_lists() {
        let flat = "*a\n\n*b";
        assert_eq!(
            blocks(flat),
            vec![
                Block::List {
                    kind: ListKind::Bullet,
                    items: vec![item(ListItemKind::Item, Some(raw(flat, "a")), vec![])],
                },
                Block::List {
                    kind: ListKind::Bullet,
                    items: vec![item(ListItemKind::Item, Some(raw(flat, "b")), vec![])],
                },
            ]
        );
    }

    #[test]
    fn kind_change_at_top_level_closes_and_reopens() {
        let flat = "*a\n#b";
        assert_eq!(
            blocks(flat),
            vec![
                Block::List {
                    kind: ListKind::Bullet,
                    items: vec![item(ListItemKind::Item, Some(raw(flat, "a")), vec![])],
                },
                Block::List {
                    kind: ListKind::Ordered,
                    items: vec![item(ListItemKind::Item, Some(raw(flat, "b")), vec![])],
                },
            ]
        );
    }

    #[test]
    fn deeper_kind_change_closes_only_that_level() {
        let flat = "*a\n*#b\n*:c";
        assert_eq!(
            blocks(flat),
            vec![Block::List {
                kind: ListKind::Bullet,
                items: vec![item(
                    ListItemKind::Item,
                    Some(raw(flat, "a")),
                    vec![
                        Block::List {
                            kind: ListKind::Ordered,
                            items: vec![item(ListItemKind::Item, Some(raw(flat, "b")), vec![])],
                        },
                        Block::List {
                            kind: ListKind::Definition,
                            items: vec![item(
                                ListItemKind::Definition,
                                Some(raw(flat, "c")),
                                vec![],
                            )],
                        },
                    ],
                )],
            }]
        );
    }

    #[test]
    fn item_content_keeps_the_raw_range_including_leading_space() {
        let flat = "* a";
        assert_eq!(
            blocks(flat),
            vec![Block::List {
                kind: ListKind::Bullet,
                items: vec![item(ListItemKind::Item, Some(raw(flat, " a")), vec![])],
            }]
        );
    }

    #[test]
    fn indent_pre_merges_consecutive_lines() {
        let flat = " a\n b";
        assert_eq!(
            blocks(flat),
            vec![Block::IndentPre {
                lines: vec![raw(flat, "a"), raw(flat, "b")],
            }]
        );
    }

    #[test]
    fn space_only_line_between_pre_lines_continues_with_an_empty_entry() {
        let flat = " a\n \n b";
        assert_eq!(
            blocks(flat),
            vec![Block::IndentPre {
                lines: vec![
                    raw(flat, "a"),
                    InlineContent::Raw(range(4, 4)),
                    raw(flat, "b")
                ],
            }]
        );
    }

    #[test]
    fn blank_line_splits_pre_blocks() {
        let flat = " a\n\n b";
        assert_eq!(
            blocks(flat),
            vec![
                Block::IndentPre {
                    lines: vec![raw(flat, "a")],
                },
                Block::IndentPre {
                    lines: vec![raw(flat, "b")],
                },
            ]
        );
    }

    #[test]
    fn pre_and_lists_close_each_other() {
        let flat = "*a\n b\n*c";
        assert_eq!(
            blocks(flat),
            vec![
                Block::List {
                    kind: ListKind::Bullet,
                    items: vec![item(ListItemKind::Item, Some(raw(flat, "a")), vec![])],
                },
                Block::IndentPre {
                    lines: vec![raw(flat, "b")],
                },
                Block::List {
                    kind: ListKind::Bullet,
                    items: vec![item(ListItemKind::Item, Some(raw(flat, "c")), vec![])],
                },
            ]
        );
    }

    #[test]
    fn pre_after_paragraph_flushes_the_paragraph() {
        let flat = "p\n x";
        assert_eq!(
            blocks(flat),
            vec![
                Block::Paragraph {
                    lines: vec![raw(flat, "p")],
                },
                Block::IndentPre {
                    lines: vec![raw(flat, "x")],
                },
            ]
        );
    }
}