wikioxide 0.1.0

//! Plain-text rendering: flattens the parsed block/inline tree into readable text.
//!
//! Labels win over targets, footnote `<ref>`s drop, `nowiki`/`pre` bodies surface,
//! table cells join with tabs and rows with newlines. Both walks (blocks and
//! inlines) use explicit work stacks — no input-depth recursion.

use crate::engine::ast::{Block, DocTree, Inline, InlineContent, Table};
use crate::engine::pp::PpDom;
use crate::engine::source::SourceStore;
use crate::engine::strip::{MarkerId, StripPayload, StripState};
use crate::title::Title;

/// Renders the whole document to plain text: paragraphs, headings and list items
/// separated by `\n`, table cells by `\t` and rows by `\n`, `IndentPre` lines
/// verbatim; no leading/trailing whitespace and no runs of more than two newlines.
pub(crate) fn plain_text(
    doc: &DocTree,
    flat: &str,
    strips: &StripState,
    store: &SourceStore,
    dom: &PpDom,
) -> String {
    let mut out = String::new();
    let mut stack: Vec<BlockWork<'_>> = doc.blocks.iter().rev().map(BlockWork::Block).collect();
    while let Some(work) = stack.pop() {
        match work {
            BlockWork::Block(block) => push_block(&mut stack, block),
            BlockWork::Content(content) => {
                // `Raw` (pre-inline-pass) slots render nothing: the extractor
                // contract runs after the inline pass, so this is defensive only.
                if let InlineContent::Parsed(nodes) = content {
                    render_inlines(&mut out, nodes, flat, strips, Some((store, dom)));
                }
            }
            BlockWork::Sep(sep) => out.push_str(sep),
        }
    }
    tidy(&out)
}

/// Renders inline nodes to plain text with full strip-marker resolution; also used
/// for anchor/display text wherever the source store is available.
#[allow(
    dead_code,
    reason = "consumed by the expansion/enrichment stages landing next"
)]
pub(crate) fn inline_text(
    nodes: &[Inline],
    flat: &str,
    strips: &StripState,
    store: &SourceStore,
    dom: &PpDom,
) -> String {
    let mut out = String::new();
    render_inlines(&mut out, nodes, flat, strips, Some((store, dom)));
    out
}

/// Renders label/caption nodes without a source store: strip markers contribute
/// nothing (a `<nowiki>` body inside a label is unavailable here because the
/// `document::collect` contract carries no store). Everything else renders exactly
/// like [`inline_text`].
pub(crate) fn label_text(nodes: &[Inline], flat: &str, strips: &StripState) -> String {
    let mut out = String::new();
    render_inlines(&mut out, nodes, flat, strips, None);
    out
}

/// One pending unit of block-level rendering work.
enum BlockWork<'a> {
    /// A block still to expand into content and separators.
    Block(&'a Block),
    /// An inline slot to render.
    Content(&'a InlineContent),
    /// A literal separator (`\n` between line units, `\t` between table cells).
    Sep(&'static str),
}

/// Expands one block into forward-ordered content/separator work and pushes it
/// reversed (the stack is LIFO).
fn push_block<'a>(stack: &mut Vec<BlockWork<'a>>, block: &'a Block) {
    let mut seq: Vec<BlockWork<'a>> = Vec::new();
    match block {
        Block::Paragraph { lines } | Block::IndentPre { lines } => {
            for line in lines {
                seq.push(BlockWork::Content(line));
                seq.push(BlockWork::Sep("\n"));
            }
        }
        Block::Heading { content, .. } => {
            seq.push(BlockWork::Content(content));
            seq.push(BlockWork::Sep("\n"));
        }
        Block::HorizontalRule => {}
        Block::List { items, .. } => {
            for item in items {
                if let Some(content) = &item.content {
                    seq.push(BlockWork::Content(content));
                    seq.push(BlockWork::Sep("\n"));
                }
                seq.extend(item.children.iter().map(BlockWork::Block));
            }
        }
        Block::Table(table) => push_table(&mut seq, table),
    }
    stack.extend(seq.into_iter().rev());
}

/// Caption first, then rows: cells joined with tabs, each row ending in a newline;
/// a cell's block continuations render right after that cell's own content.
fn push_table<'a>(seq: &mut Vec<BlockWork<'a>>, table: &'a Table) {
    if let Some(caption) = &table.caption {
        seq.push(BlockWork::Content(&caption.content));
        seq.push(BlockWork::Sep("\n"));
    }
    for row in &table.rows {
        for (i, cell) in row.cells.iter().enumerate() {
            if i > 0 {
                seq.push(BlockWork::Sep("\t"));
            }
            seq.push(BlockWork::Content(&cell.content));
            seq.extend(cell.children.iter().map(BlockWork::Block));
        }
        seq.push(BlockWork::Sep("\n"));
    }
}

/// One pending unit of inline rendering work.
enum InlineWork<'a> {
    /// An inline node still to render.
    Node(&'a Inline),
    /// Literal text emitted verbatim (link-trail letters).
    Str(&'a str),
}

/// The shared iterative inline renderer. `src` is `None` for label/caption
/// rendering (see [`label_text`]), which makes every strip marker render as nothing.
fn render_inlines<'a>(
    out: &mut String,
    nodes: &'a [Inline],
    flat: &'a str,
    strips: &StripState,
    src: Option<(&SourceStore, &PpDom)>,
) {
    let mut stack: Vec<InlineWork<'a>> = nodes.iter().rev().map(InlineWork::Node).collect();
    while let Some(work) = stack.pop() {
        let node = match work {
            InlineWork::Str(text) => {
                out.push_str(text);
                continue;
            }
            InlineWork::Node(node) => node,
        };
        match node {
            Inline::Text(range) => out.push_str(range.slice(flat)),
            Inline::Entity { decoded, .. } => out.push(*decoded),
            Inline::Marker { id, .. } => render_marker(out, *id, strips, src),
            Inline::Link {
                target,
                written,
                label,
                trail,
                pipe_trick,
                ..
            } => {
                // The trail is pushed first so it pops after whatever the link body
                // renders as (label nodes are pushed later and pop earlier).
                if let Some(trail) = trail {
                    stack.push(InlineWork::Str(trail.slice(flat)));
                }
                if *pipe_trick {
                    out.push_str(&pipe_trick_text(target));
                } else if let Some(label) = label {
                    stack.extend(label.iter().rev().map(InlineWork::Node));
                } else {
                    // an unlabeled link shows its target exactly as written
                    // (`[[кот]]ами` → “котами”, not the normalized “Котами”)
                    out.push_str(written.slice(flat));
                }
            }
            Inline::InterLanguage {
                prefix,
                target,
                escaped,
            } => {
                // only the `[[:fr:X]]` escape renders in the body
                if *escaped {
                    out.push_str(prefix);
                    out.push(':');
                    out.push_str(target);
                }
            }
            Inline::Category { .. } | Inline::BehaviorSwitch { .. } | Inline::HtmlTag { .. } => {}
            Inline::Image { caption, .. } => {
                if let Some(caption) = caption {
                    stack.extend(caption.iter().rev().map(InlineWork::Node));
                }
            }
            Inline::ExternalLink { url, label } => match label {
                Some(label) => stack.extend(label.iter().rev().map(InlineWork::Node)),
                None => out.push_str(url),
            },
            Inline::AutoUrl { url, .. } => out.push_str(url),
            Inline::MagicLink { range, .. } => out.push_str(range.slice(flat)),
            Inline::Bold(children) | Inline::Italic(children) => {
                stack.extend(children.iter().rev().map(InlineWork::Node));
            }
        }
    }
}

/// Renders one strip marker: `nowiki` and `pre` surface their raw bodies, `ref`
/// renders as nothing (footnote policy), every other extension tag renders as
/// nothing, an unexpanded `{{{argument}}}` surfaces its original source text (it stays
/// literal in the parse-only view), and an unexpanded template call renders as
/// nothing. Without a store/dom, bodies cannot be sliced, so every marker renders as
/// nothing.
fn render_marker(
    out: &mut String,
    id: MarkerId,
    strips: &StripState,
    src: Option<(&SourceStore, &PpDom)>,
) {
    let (Some(payload), Some((store, dom))) = (strips.get(id), src) else {
        return;
    };
    match payload {
        StripPayload::ExtTag { name, body, .. } => {
            if let Some(body) = body {
                if name.eq_ignore_ascii_case("nowiki") || name.eq_ignore_ascii_case("pre") {
                    out.push_str(store.slice(*body));
                }
            }
        }
        StripPayload::TplArg(node) => out.push_str(store.slice(dom.node(*node).span)),
        StripPayload::Template(_) => {}
    }
}

/// Derives the pipe-trick label of `[[X|]]`: the target's page name without its
/// namespace prefix, with one trailing ` (…)` parenthetical removed — or, failing
/// that, everything from the first `, ` on.
fn pipe_trick_text(target: &Title) -> String {
    // The dbkey is exactly the prefix-less page name (spaces as underscores).
    let name = target.dbkey().replace('_', " ");
    match name.rfind(" (") {
        Some(open) if name.ends_with(')') => name[..open].to_owned(),
        _ => match name.find(", ") {
            Some(comma) => name[..comma].to_owned(),
            None => name,
        },
    }
}

/// Collapses runs of three or more newlines to exactly two and trims both ends.
fn tidy(raw: &str) -> String {
    let mut out = String::with_capacity(raw.len());
    let mut newlines = 0_u32;
    for c in raw.chars() {
        if c == '\n' {
            newlines += 1;
            if newlines > 2 {
                continue;
            }
        } else {
            newlines = 0;
        }
        out.push(c);
    }
    out.trim().to_owned()
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::config::{NamespaceDef, SiteConfig};
    use crate::engine::ast::{
        ListItem, ListItemKind, ListKind, MagicLinkKind, TableCaption, TableCell, TableRow,
        TextRange,
    };
    use crate::engine::pp::{PpKind, PpNode, PpNodeId};
    use crate::engine::source::{SourceId, Span};

    fn cfg() -> SiteConfig {
        SiteConfig::builder("ru", "Википедия")
            .namespace(NamespaceDef::main())
            .namespace(NamespaceDef::new(10, "Template", "Шаблон", ["Ш"], true))
            .namespace(NamespaceDef::new(
                14,
                "Category",
                "Категория",
                Vec::<String>::new(),
                true,
            ))
            .namespace(NamespaceDef::new(6, "File", "Файл", ["Изображение"], true))
            .build()
            .unwrap()
    }

    fn t(text: &str) -> Title {
        Title::parse(text, &cfg()).unwrap()
    }

    /// Grows the flattened text and hands out ranges into it.
    #[derive(Default)]
    struct Flat {
        text: String,
    }

    impl Flat {
        fn r(&mut self, s: &str) -> TextRange {
            let start = self.text.len() as u32;
            self.text.push_str(s);
            TextRange::new(start, self.text.len() as u32)
        }
    }

    /// Byte span of the first occurrence of `needle` inside the store buffer.
    fn span_of(text: &str, src: SourceId, needle: &str) -> Span {
        let start = text.find(needle).unwrap();
        Span::new(src, start as u32, (start + needle.len()) as u32)
    }

    fn text_inline(flat: &mut Flat, s: &str) -> Inline {
        Inline::Text(flat.r(s))
    }

    /// An unlabeled link whose as-written text goes into the flattened buffer.
    fn bare_link(flat: &mut Flat, target: &str, written: &str) -> Inline {
        Inline::Link {
            target: t(target),
            written: flat.r(written),
            fragment: None,
            label: None,
            trail: None,
            pipe_trick: false,
        }
    }

    fn pipe_trick_link(target: &str, trail: Option<TextRange>) -> Inline {
        Inline::Link {
            target: t(target),
            written: TextRange::new(0, 0),
            fragment: None,
            label: None,
            trail,
            pipe_trick: true,
        }
    }

    /// A data cell with plain-text content and optional block continuations.
    fn cell(flat: &mut Flat, s: &str, children: Vec<Block>) -> TableCell {
        TableCell {
            header: false,
            attributes: None,
            content: InlineContent::Parsed(vec![text_inline(flat, s)]),
            children,
        }
    }

    /// The strip state used across tests plus the store its ext-tag bodies point to.
    struct Strips {
        store: SourceStore,
        strips: StripState,
        nowiki: MarkerId,
        pre: MarkerId,
        reference: MarkerId,
        math: MarkerId,
        bodyless: MarkerId,
        template: MarkerId,
        tplarg: MarkerId,
        dom: PpDom,
    }

    fn strips() -> Strips {
        let stext = "тело nowiki|pre body|{{{арг|деф}}}";
        let mut store = SourceStore::new();
        let (src, _) = store.push(stext, 1 << 20);
        let nw = span_of(stext, src, "тело nowiki");
        let pr = span_of(stext, src, "pre body");
        let ta = span_of(stext, src, "{{{арг|деф}}}");
        let mut strips = StripState::new();
        let mut scratch = String::new();
        let tag = |name: &str, body: Option<Span>| StripPayload::ExtTag {
            name: name.to_owned(),
            attributes: None,
            body,
        };
        let nowiki = strips.add(&mut scratch, tag("nowiki", Some(nw)));
        let pre = strips.add(&mut scratch, tag("PRE", Some(pr)));
        let reference = strips.add(&mut scratch, tag("ref", Some(nw)));
        let math = strips.add(&mut scratch, tag("math", Some(pr)));
        let bodyless = strips.add(&mut scratch, tag("nowiki", None));
        let template = strips.add(&mut scratch, StripPayload::Template(PpNodeId(0)));
        let mut dom = PpDom::new();
        let arg_node = dom.push(PpNode {
            kind: PpKind::TplArg { parts: Vec::new() },
            span: ta,
        });
        let tplarg = strips.add(&mut scratch, StripPayload::TplArg(arg_node));
        Strips {
            store,
            strips,
            nowiki,
            pre,
            reference,
            math,
            bodyless,
            template,
            tplarg,
            dom,
        }
    }

    fn marker(id: MarkerId) -> Inline {
        let kind = crate::engine::strip::MarkerKind::ExtTag;
        Inline::Marker { id, kind }
    }

    #[test]
    fn whole_document_renders_to_the_exact_plain_text() {
        let s = strips();
        let mut flat = Flat::default();

        let heading = Block::Heading {
            level: 2,
            line: TextRange::new(0, 0),
            inner: TextRange::new(0, 0),
            content: InlineContent::Parsed(vec![text_inline(&mut flat, "История")]),
        };

        let para_quotes = Block::Paragraph {
            lines: vec![
                InlineContent::Parsed(vec![
                    text_inline(&mut flat, "Быстрый "),
                    Inline::Bold(vec![text_inline(&mut flat, "смелый")]),
                    text_inline(&mut flat, " и "),
                    Inline::Italic(vec![text_inline(&mut flat, "косой")]),
                    Inline::Entity {
                        range: flat.r("&amp;"),
                        decoded: '&',
                    },
                    text_inline(&mut flat, " конец"),
                ]),
                InlineContent::Parsed(vec![
                    // the as-written (lowercase) target renders, not the normalized one
                    Inline::Link {
                        target: t("Собака"),
                        written: flat.r("собака"),
                        fragment: None,
                        label: None,
                        trail: Some(flat.r("ми")),
                        pipe_trick: false,
                    },
                    text_inline(&mut flat, " гуляли"),
                ]),
            ],
        };

        let sep1 = flat.r(" / ");
        let sep2 = flat.r(" / ");
        let sep3 = flat.r(" / ");
        let sep4 = flat.r(" / ");
        let trail_x = flat.r("х");
        let para_pipe_trick = Block::Paragraph {
            lines: vec![InlineContent::Parsed(vec![
                // `pipe_trick` wins even over an (empty) label vector.
                Inline::Link {
                    target: t("Boston, Massachusetts"),
                    written: TextRange::new(0, 0),
                    fragment: None,
                    label: Some(vec![]),
                    trail: None,
                    pipe_trick: true,
                },
                Inline::Text(sep1),
                pipe_trick_link("Пушкин (станция)", None),
                Inline::Text(sep2),
                pipe_trick_link("Шаблон:Правка (тест)", Some(trail_x)),
                Inline::Text(sep3),
                pipe_trick_link("A (b) c", None),
                Inline::Text(sep4),
                pipe_trick_link("Просто", None),
            ])],
        };

        let para_markers = Block::Paragraph {
            lines: vec![InlineContent::Parsed(vec![
                Inline::Link {
                    target: t("Цель"),
                    written: TextRange::new(0, 0),
                    fragment: Some("Сек".to_owned()),
                    label: Some(vec![text_inline(&mut flat, "метка")]),
                    trail: Some(flat.r("й")),
                    pipe_trick: false,
                },
                text_inline(&mut flat, " "),
                bare_link(&mut flat, "Голая", "Голая"),
                text_inline(&mut flat, " "),
                marker(s.nowiki),
                text_inline(&mut flat, " "),
                marker(s.pre),
                marker(s.reference),
                marker(s.math),
                marker(s.bodyless),
                marker(s.template),
                marker(s.tplarg),
                marker(MarkerId(9999)),
                text_inline(&mut flat, " "),
                Inline::ExternalLink {
                    url: "https://e.org/1".to_owned(),
                    label: Some(vec![text_inline(&mut flat, "внешняя")]),
                },
                text_inline(&mut flat, " "),
                Inline::ExternalLink {
                    url: "https://e.org/2".to_owned(),
                    label: None,
                },
                text_inline(&mut flat, " "),
                Inline::AutoUrl {
                    url: "https://auto.example".to_owned(),
                    range: flat.r("https://auto.example"),
                },
                text_inline(&mut flat, " "),
                Inline::MagicLink {
                    kind: MagicLinkKind::Isbn,
                    range: flat.r("ISBN 978-5"),
                },
                Inline::Category {
                    category: t("Категория:Люди"),
                    sortkey: None,
                },
                Inline::InterLanguage {
                    prefix: "de".to_owned(),
                    target: "Hund".to_owned(),
                    escaped: false,
                },
                Inline::InterLanguage {
                    prefix: "fr".to_owned(),
                    target: "Chat".to_owned(),
                    escaped: true,
                },
                Inline::BehaviorSwitch {
                    id: "toc".to_owned(),
                    range: flat.r("__TOC__"),
                },
                Inline::HtmlTag {
                    name: "br".to_owned(),
                    closing: false,
                    self_closing: true,
                    attributes: None,
                },
                Inline::Image {
                    target: t("Файл:Ф.jpg"),
                    params: vec![flat.r("thumb")],
                    caption: Some(vec![text_inline(&mut flat, "подпись")]),
                },
                Inline::Image {
                    target: t("Файл:Б.jpg"),
                    params: vec![],
                    caption: None,
                },
            ])],
        };

        let list = Block::List {
            kind: ListKind::Bullet,
            items: vec![
                ListItem {
                    kind: ListItemKind::Item,
                    content: Some(InlineContent::Parsed(vec![text_inline(
                        &mut flat,
                        "первый",
                    )])),
                    children: vec![Block::List {
                        kind: ListKind::Ordered,
                        items: vec![ListItem {
                            kind: ListItemKind::Item,
                            content: Some(InlineContent::Parsed(vec![text_inline(
                                &mut flat,
                                "вложенный",
                            )])),
                            children: vec![],
                        }],
                    }],
                },
                ListItem {
                    kind: ListItemKind::Definition,
                    content: None,
                    children: vec![Block::HorizontalRule],
                },
                ListItem {
                    kind: ListItemKind::Item,
                    content: Some(InlineContent::Parsed(vec![text_inline(
                        &mut flat,
                        "второй",
                    )])),
                    children: vec![],
                },
            ],
        };

        let nested_table = Table {
            attributes: None,
            caption: None,
            rows: vec![TableRow {
                attributes: None,
                cells: vec![cell(&mut flat, "N", vec![])],
            }],
            closed: true,
        };
        let attrs = flat.r("class=\"wikitable\"");
        let caption_range = flat.r("шапка");
        let table = Block::Table(Table {
            attributes: Some(attrs),
            caption: Some(TableCaption {
                attributes: None,
                content: InlineContent::Parsed(vec![Inline::Text(caption_range)]),
            }),
            rows: vec![
                TableRow {
                    attributes: None,
                    cells: vec![cell(&mut flat, "A", vec![]), cell(&mut flat, "B", vec![])],
                },
                TableRow {
                    attributes: None,
                    cells: vec![
                        cell(&mut flat, "C", vec![Block::Table(nested_table)]),
                        cell(&mut flat, "D", vec![]),
                    ],
                },
            ],
            closed: true,
        });

        let para_blank = Block::Paragraph {
            lines: vec![
                InlineContent::Parsed(vec![text_inline(&mut flat, "до")]),
                InlineContent::Parsed(vec![]),
                InlineContent::Parsed(vec![]),
                InlineContent::Parsed(vec![text_inline(&mut flat, "после")]),
            ],
        };

        let para_raw = Block::Paragraph {
            lines: vec![InlineContent::Raw(TextRange::new(0, 0))],
        };

        let pre = Block::IndentPre {
            lines: vec![
                InlineContent::Parsed(vec![text_inline(&mut flat, " код 1")]),
                InlineContent::Parsed(vec![text_inline(&mut flat, " код 2")]),
            ],
        };

        let doc = DocTree {
            blocks: vec![
                heading,
                para_quotes,
                para_pipe_trick,
                para_markers,
                list,
                table,
                para_blank,
                para_raw,
                pre,
                Block::HorizontalRule,
            ],
        };

        let rendered = plain_text(&doc, &flat.text, &s.strips, &s.store, &s.dom);
        let expected = concat!(
            "История\n",
            "Быстрый смелый и косой& конец\n",
            "собаками гуляли\n",
            "Boston / Пушкин / Правках / A (b) c / Просто\n",
            "меткай Голая тело nowiki pre body{{{арг|деф}}} внешняя https://e.org/2 ",
            "https://auto.example ISBN 978-5fr:Chatподпись\n",
            "первый\n",
            "вложенный\n",
            "второй\n",
            "шапка\n",
            "A\tB\n",
            "CN\n",
            "\tD\n",
            "до\n",
            "\n",
            "после\n",
            "\n",
            " код 1\n",
            " код 2",
        );
        assert_eq!(rendered, expected);
    }

    #[test]
    fn inline_text_resolves_markers_but_label_text_does_not() {
        let s = strips();
        let mut flat = Flat::default();
        let nodes = vec![marker(s.nowiki), text_inline(&mut flat, "X")];
        assert_eq!(
            inline_text(&nodes, &flat.text, &s.strips, &s.store, &s.dom),
            "тело nowikiX"
        );
        assert_eq!(label_text(&nodes, &flat.text, &s.strips), "X");
    }

    #[test]
    fn leading_junk_and_edges_trim_away() {
        let mut flat = Flat::default();
        let doc = DocTree {
            blocks: vec![
                Block::HorizontalRule,
                Block::Paragraph {
                    lines: vec![InlineContent::Parsed(vec![text_inline(&mut flat, " x ")])],
                },
            ],
        };
        let s = strips();
        assert_eq!(
            plain_text(&doc, &flat.text, &s.strips, &s.store, &s.dom),
            "x"
        );
    }

    #[test]
    fn empty_document_renders_empty() {
        let s = strips();
        assert_eq!(
            plain_text(&DocTree::default(), "", &s.strips, &s.store, &s.dom),
            ""
        );
    }
}