wikioxide 0.1.0

//! Document-level fact collection: one iterative walk over the parsed block/inline
//! tree gathering internal links, categories, interlanguage links, external links
//! and structural statistics — in document order, without deduplication (the facade
//! dedups for API-parity lists later).

use crate::engine::ast::{Block, DocTree, Inline, InlineContent, ListItem, Table};
use crate::engine::strip::{StripPayload, StripState};
use crate::model::{CategoryLink, ExternalLink, InternalLink, StructuralStats};

use super::plaintext;

/// Everything the extractor reads off one parsed document in a single pass.
#[derive(Debug, Default)]
pub(crate) struct DocumentFacts {
    /// Internal links in document order, not deduplicated; spans are `None` in M2.
    pub(crate) links: Vec<InternalLink>,
    /// Category memberships with raw sort keys (`""` when unspecified —
    /// `DEFAULTSORT` awareness arrives with expansion).
    pub(crate) categories: Vec<CategoryLink>,
    /// Interlanguage links as `(prefix, target)` pairs.
    pub(crate) interlanguage: Vec<(String, String)>,
    /// External links: bracketed ones carry their label as `display`, bare
    /// auto-links carry `None`.
    pub(crate) external_links: Vec<ExternalLink>,
    /// Structural counters; `parser_errors` stays 0 until expansion lands.
    pub(crate) stats: StructuralStats,
}

/// One pending unit of walk work.
enum Work<'a> {
    /// A block still to visit.
    Block(&'a Block),
    /// An inline node still to visit.
    Inline(&'a Inline),
}

/// Walks every block and inline node of `doc` (explicit stack — nested labels,
/// captions, list children and table-cell children included) and collects the
/// document facts.
///
/// Anchor/display text renders through the store-less
/// [`plaintext::label_text`] helper — the contract carries no [`SourceStore`]
/// (`crate::engine::source::SourceStore`), so strip-marker bodies (e.g. a
/// `<nowiki>` inside a label) contribute nothing to those strings in M2.
pub(crate) fn collect(doc: &DocTree, flat: &str, strips: &StripState) -> DocumentFacts {
    let mut facts = DocumentFacts::default();
    let mut stack: Vec<Work<'_>> = doc.blocks.iter().rev().map(Work::Block).collect();
    while let Some(work) = stack.pop() {
        match work {
            Work::Block(block) => visit_block(block, &mut stack, &mut facts),
            Work::Inline(inline) => visit_inline(inline, flat, strips, &mut stack, &mut facts),
        }
    }
    facts
}

/// Visits one block: counts it where relevant and schedules its content (in
/// document order — the stack is LIFO, so everything is pushed reversed).
fn visit_block<'a>(block: &'a Block, stack: &mut Vec<Work<'a>>, facts: &mut DocumentFacts) {
    match block {
        Block::Paragraph { lines } | Block::IndentPre { lines } => {
            for line in lines.iter().rev() {
                push_inlines(stack, content_inlines(line));
            }
        }
        Block::Heading { content, .. } => {
            facts.stats.headings += 1;
            push_inlines(stack, content_inlines(content));
        }
        Block::HorizontalRule => {}
        Block::List { items, .. } => push_list_items(stack, items, facts),
        Block::Table(table) => push_table(stack, table, facts),
    }
}

/// Counts the items of one list level and schedules their content and nested
/// blocks; nested lists count their own items when their block is visited.
fn push_list_items<'a>(
    stack: &mut Vec<Work<'a>>,
    items: &'a [ListItem],
    facts: &mut DocumentFacts,
) {
    for item in items.iter().rev() {
        facts.stats.list_items += 1;
        stack.extend(item.children.iter().rev().map(Work::Block));
        if let Some(content) = &item.content {
            push_inlines(stack, content_inlines(content));
        }
    }
}

/// Counts one table (rows and cells included) and schedules caption, cell content
/// and cell block continuations; nested tables count when their block is visited.
fn push_table<'a>(stack: &mut Vec<Work<'a>>, table: &'a Table, facts: &mut DocumentFacts) {
    facts.stats.tables += 1;
    for row in table.rows.iter().rev() {
        facts.stats.table_rows += 1;
        for cell in row.cells.iter().rev() {
            facts.stats.table_cells += 1;
            stack.extend(cell.children.iter().rev().map(Work::Block));
            push_inlines(stack, content_inlines(&cell.content));
        }
    }
    if let Some(caption) = &table.caption {
        push_inlines(stack, content_inlines(&caption.content));
    }
}

/// Visits one inline node: records its fact (if any) and schedules nested inline
/// vectors (labels, captions, bold/italic children) for the same walk.
fn visit_inline<'a>(
    inline: &'a Inline,
    flat: &str,
    strips: &StripState,
    stack: &mut Vec<Work<'a>>,
    facts: &mut DocumentFacts,
) {
    match inline {
        Inline::Text(_)
        | Inline::Entity { .. }
        | Inline::MagicLink { .. }
        | Inline::HtmlTag { .. }
        | Inline::BehaviorSwitch { .. } => {}
        Inline::Marker { id, .. } => {
            if let Some(StripPayload::ExtTag { name, .. }) = strips.get(*id) {
                facts.stats.refs += u32::from(name.eq_ignore_ascii_case("ref"));
            }
        }
        Inline::Link {
            target,
            fragment,
            label,
            ..
        } => {
            facts.stats.internal_links += 1;
            let anchor = label
                .as_deref()
                .map(|nodes| plaintext::label_text(nodes, flat, strips));
            facts.links.push(InternalLink::new(
                target.clone(),
                fragment.clone(),
                anchor,
                None,
            ));
            if let Some(label) = label {
                push_inlines(stack, label);
            }
        }
        Inline::Category { category, sortkey } => {
            facts.categories.push(CategoryLink::new(
                category.clone(),
                sortkey.clone().unwrap_or_default(),
                None,
            ));
        }
        Inline::InterLanguage {
            prefix,
            target,
            escaped,
        } => {
            // `[[:fr:X]]` renders inline and is neither a langlink nor a local link
            if !*escaped {
                facts.interlanguage.push((prefix.clone(), target.clone()));
            }
        }
        Inline::Image { caption, .. } => {
            facts.stats.images += 1;
            if let Some(caption) = caption {
                push_inlines(stack, caption);
            }
        }
        Inline::ExternalLink { url, label } => {
            facts.stats.external_links += 1;
            let display = label
                .as_deref()
                .map(|nodes| plaintext::label_text(nodes, flat, strips));
            facts
                .external_links
                .push(ExternalLink::new(url.clone(), display, None));
            if let Some(label) = label {
                push_inlines(stack, label);
            }
        }
        Inline::AutoUrl { url, .. } => {
            facts.stats.external_links += 1;
            facts
                .external_links
                .push(ExternalLink::new(url.clone(), None, None));
        }
        Inline::Bold(children) | Inline::Italic(children) => push_inlines(stack, children),
    }
}

/// The parsed nodes of an inline slot; `Raw` (pre-inline-pass) content is ignored
/// defensively — the extractor contract runs after the inline pass.
fn content_inlines(content: &InlineContent) -> &[Inline] {
    match content {
        InlineContent::Parsed(nodes) => nodes,
        InlineContent::Raw(_) => &[],
    }
}

/// Pushes inline nodes reversed so the LIFO walk visits them in document order.
fn push_inlines<'a>(stack: &mut Vec<Work<'a>>, nodes: &'a [Inline]) {
    stack.extend(nodes.iter().rev().map(Work::Inline));
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::config::{NamespaceDef, SiteConfig};
    use crate::engine::ast::{
        ListItemKind, ListKind, MagicLinkKind, TableCaption, TableCell, TableRow, TextRange,
    };
    use crate::engine::pp::PpNodeId;
    use crate::engine::source::{SourceId, Span};
    use crate::engine::strip::{MarkerId, MarkerKind};
    use crate::title::Title;

    fn cfg() -> SiteConfig {
        SiteConfig::builder("ru", "Википедия")
            .namespace(NamespaceDef::main())
            .namespace(NamespaceDef::new(
                14,
                "Category",
                "Категория",
                Vec::<String>::new(),
                true,
            ))
            .namespace(NamespaceDef::new(6, "File", "Файл", ["Изображение"], true))
            .build()
            .unwrap()
    }

    fn t(text: &str) -> Title {
        Title::parse(text, &cfg()).unwrap()
    }

    /// Grows the flattened text and hands out ranges into it.
    #[derive(Default)]
    struct Flat {
        text: String,
    }

    impl Flat {
        fn r(&mut self, s: &str) -> TextRange {
            let start = self.text.len() as u32;
            self.text.push_str(s);
            TextRange::new(start, self.text.len() as u32)
        }
    }

    fn text_inline(flat: &mut Flat, s: &str) -> Inline {
        Inline::Text(flat.r(s))
    }

    fn bare_link(target: &str) -> Inline {
        Inline::Link {
            target: t(target),
            written: TextRange::new(0, 0),
            fragment: None,
            label: None,
            trail: None,
            pipe_trick: false,
        }
    }

    fn marker(id: MarkerId, kind: MarkerKind) -> Inline {
        Inline::Marker { id, kind }
    }

    fn ext_tag(name: &str) -> StripPayload {
        StripPayload::ExtTag {
            name: name.to_owned(),
            attributes: None,
            body: Some(Span::new(SourceId(0), 0, 0)),
        }
    }

    #[test]
    fn one_walk_collects_every_fact_in_document_order() {
        let mut strips = StripState::new();
        let mut scratch = String::new();
        let ref1 = strips.add(&mut scratch, ext_tag("ref"));
        let ref2 = strips.add(&mut scratch, ext_tag("REF"));
        let math = strips.add(&mut scratch, ext_tag("math"));
        let tpl = strips.add(&mut scratch, StripPayload::Template(PpNodeId(0)));

        let mut flat = Flat::default();

        let heading = Block::Heading {
            level: 2,
            line: TextRange::new(0, 0),
            inner: TextRange::new(0, 0),
            content: InlineContent::Parsed(vec![text_inline(&mut flat, "Заголовок")]),
        };

        let para_links = Block::Paragraph {
            lines: vec![
                InlineContent::Parsed(vec![
                    Inline::Link {
                        target: t("Цель"),
                        written: TextRange::new(0, 0),
                        fragment: Some("Сек".to_owned()),
                        label: Some(vec![text_inline(&mut flat, "якорь")]),
                        trail: None,
                        pipe_trick: false,
                    },
                    text_inline(&mut flat, " и "),
                    Inline::Link {
                        target: t("Другая"),
                        written: TextRange::new(0, 0),
                        fragment: None,
                        label: None,
                        trail: Some(flat.r("ми")),
                        pipe_trick: false,
                    },
                    Inline::Category {
                        category: t("Категория:Люди"),
                        sortkey: Some("ключ".to_owned()),
                    },
                    Inline::Bold(vec![Inline::Category {
                        category: t("Категория:Города"),
                        sortkey: None,
                    }]),
                    Inline::InterLanguage {
                        prefix: "de".to_owned(),
                        target: "Hund".to_owned(),
                        escaped: false,
                    },
                    // escaped: renders inline, so it must NOT become a langlink entry
                    Inline::InterLanguage {
                        prefix: "fr".to_owned(),
                        target: "Chat".to_owned(),
                        escaped: true,
                    },
                ]),
                InlineContent::Raw(TextRange::new(0, 0)),
            ],
        };

        let para_misc = Block::Paragraph {
            lines: vec![InlineContent::Parsed(vec![
                Inline::ExternalLink {
                    url: "https://example.org/a".to_owned(),
                    label: Some(vec![text_inline(&mut flat, "метка")]),
                },
                Inline::ExternalLink {
                    url: "https://example.org/b".to_owned(),
                    label: None,
                },
                Inline::AutoUrl {
                    url: "https://auto.example".to_owned(),
                    range: flat.r("https://auto.example"),
                },
                marker(ref1, MarkerKind::ExtTag),
                marker(math, MarkerKind::ExtTag),
                marker(tpl, MarkerKind::Template),
                marker(MarkerId(99), MarkerKind::ExtTag),
                Inline::Image {
                    target: t("Файл:Пример.jpg"),
                    params: vec![flat.r("thumb")],
                    caption: Some(vec![
                        text_inline(&mut flat, "подпись "),
                        bare_link("Вложенная"),
                    ]),
                },
                Inline::Image {
                    target: t("Файл:Пустой.jpg"),
                    params: vec![],
                    caption: None,
                },
                Inline::HtmlTag {
                    name: "sup".to_owned(),
                    closing: false,
                    self_closing: false,
                    attributes: None,
                },
                Inline::BehaviorSwitch {
                    id: "toc".to_owned(),
                    range: flat.r("__TOC__"),
                },
                Inline::Entity {
                    range: flat.r("&nbsp;"),
                    decoded: '\u{a0}',
                },
                Inline::MagicLink {
                    kind: MagicLinkKind::Rfc,
                    range: flat.r("RFC 1"),
                },
            ])],
        };

        let list = Block::List {
            kind: ListKind::Bullet,
            items: vec![
                ListItem {
                    kind: ListItemKind::Item,
                    content: Some(InlineContent::Parsed(vec![text_inline(&mut flat, "один")])),
                    children: vec![Block::List {
                        kind: ListKind::Ordered,
                        items: vec![ListItem {
                            kind: ListItemKind::Item,
                            content: Some(InlineContent::Parsed(vec![marker(
                                ref2,
                                MarkerKind::ExtTag,
                            )])),
                            children: vec![],
                        }],
                    }],
                },
                ListItem {
                    kind: ListItemKind::Definition,
                    content: None,
                    children: vec![],
                },
            ],
        };

        let nested_table = Table {
            attributes: None,
            caption: None,
            rows: vec![TableRow {
                attributes: None,
                cells: vec![TableCell {
                    header: false,
                    attributes: None,
                    content: InlineContent::Parsed(vec![text_inline(&mut flat, "N")]),
                    children: vec![],
                }],
            }],
            closed: true,
        };
        let table = Block::Table(Table {
            attributes: None,
            caption: Some(TableCaption {
                attributes: None,
                content: InlineContent::Parsed(vec![bare_link("Из подписи")]),
            }),
            rows: vec![TableRow {
                attributes: None,
                cells: vec![
                    TableCell {
                        header: true,
                        attributes: None,
                        content: InlineContent::Parsed(vec![text_inline(&mut flat, "A")]),
                        children: vec![],
                    },
                    TableCell {
                        header: false,
                        attributes: None,
                        content: InlineContent::Parsed(vec![text_inline(&mut flat, "B")]),
                        children: vec![Block::Table(nested_table)],
                    },
                ],
            }],
            closed: true,
        });

        let doc = DocTree {
            blocks: vec![
                heading,
                para_links,
                para_misc,
                list,
                table,
                Block::HorizontalRule,
            ],
        };

        let facts = collect(&doc, &flat.text, &strips);

        let link_keys: Vec<&str> = facts.links.iter().map(|l| l.target().dbkey()).collect();
        assert_eq!(link_keys, ["Цель", "Другая", "Вложенная", "Из_подписи"]);
        assert_eq!(facts.links[0].fragment(), Some("Сек"));
        assert_eq!(facts.links[0].anchor_text(), Some("якорь"));
        assert_eq!(facts.links[1].anchor_text(), None);
        assert_eq!(facts.links[0].span(), None);

        let cat_keys: Vec<(&str, &str)> = facts
            .categories
            .iter()
            .map(|c| (c.category().dbkey(), c.sortkey()))
            .collect();
        assert_eq!(cat_keys, [("Люди", "ключ"), ("Города", "")]);

        assert_eq!(facts.interlanguage, [("de".to_owned(), "Hund".to_owned())]);

        let ext: Vec<(&str, Option<&str>)> = facts
            .external_links
            .iter()
            .map(|e| (e.url(), e.display()))
            .collect();
        assert_eq!(
            ext,
            [
                ("https://example.org/a", Some("метка")),
                ("https://example.org/b", None),
                ("https://auto.example", None),
            ]
        );

        assert_eq!(facts.stats.headings(), 1);
        assert_eq!(facts.stats.internal_links(), 4);
        assert_eq!(facts.stats.external_links(), 3);
        assert_eq!(facts.stats.images(), 2);
        assert_eq!(facts.stats.refs(), 2);
        assert_eq!(facts.stats.list_items(), 3);
        assert_eq!(facts.stats.tables(), 2);
        assert_eq!(facts.stats.table_rows(), 2);
        assert_eq!(facts.stats.table_cells(), 3);
        assert_eq!(facts.stats.parser_errors(), 0);
    }

    #[test]
    fn anchor_text_skips_marker_bodies_without_a_store() {
        let mut strips = StripState::new();
        let mut scratch = String::new();
        let nowiki = strips.add(&mut scratch, ext_tag("nowiki"));

        let mut flat = Flat::default();
        let doc = DocTree {
            blocks: vec![Block::Paragraph {
                lines: vec![InlineContent::Parsed(vec![Inline::Link {
                    target: t("Цель"),
                    written: TextRange::new(0, 0),
                    fragment: None,
                    label: Some(vec![
                        text_inline(&mut flat, "см"),
                        marker(nowiki, MarkerKind::ExtTag),
                    ]),
                    trail: None,
                    pipe_trick: false,
                }])],
            }],
        };

        let facts = collect(&doc, &flat.text, &strips);
        assert_eq!(facts.links[0].anchor_text(), Some("см"));
    }

    #[test]
    fn empty_document_collects_nothing() {
        let facts = collect(&DocTree::default(), "", &StripState::new());
        assert!(facts.links.is_empty());
        assert!(facts.categories.is_empty());
        assert!(facts.interlanguage.is_empty());
        assert!(facts.external_links.is_empty());
        assert_eq!(facts.stats, StructuralStats::default());
    }
}