wikioxide 0.1.0

//! Pipeline orchestration: hygiene → preprocess → flatten → block parse → inline pass →
//! extraction, assembled into a [`ParsedDocument`].

use crate::clock::Timestamp;
use crate::config::{ExtTagContent, Limits, SiteConfig};
use crate::doc::{
    DocumentParts, ExpandOutcome, ExpandedDocument, ExpandedParts, MissingTemplates, ParsedDocument,
};
use crate::engine::ast::{Block, DocTree, InlineContent};
use crate::engine::expand::{ExpandContext, UnresolvedPolicy, expand};
use crate::engine::pp::{self, InclusionMode, PpDom};
use crate::engine::source::{HygieneReport, SourceStore};
use crate::engine::strip::{StripPayload, StripState};
use crate::engine::{extract, inline, parse};
use crate::model::InterLanguageLink;
use crate::source::{ExistenceProvider, TemplateSource};
use crate::title::Title;
use crate::warn::{Warning, WarningCollector};

/// Runs the parse-only pipeline (no template expansion) over one page.
///
/// `_title` and `_now` are threaded through for the expansion stage
/// (`{{PAGENAME}}`, `{{CURRENT*}}`) and are unused until it lands.
pub(crate) fn parse_only(
    config: &SiteConfig,
    limits: &Limits,
    wikitext: &str,
    _title: Option<&Title>,
    _now: Timestamp,
) -> ParsedDocument {
    let mut warnings = WarningCollector::new();

    let mut store = SourceStore::new();
    let (src, hygiene) = store.push(wikitext, limits.max_source_bytes());
    if hygiene.truncated {
        warnings.emit(Warning::InputTruncated {
            limit_bytes: limits.max_source_bytes(),
        });
    }
    if hygiene.sanitized_controls > 0 {
        warnings.emit(Warning::ControlCharactersSanitized {
            count: hygiene.sanitized_controls,
        });
    }

    let dom = pp::build(&store, src, config, InclusionMode::TopLevel, &mut warnings);
    let mut strips = StripState::new();
    let pp::Flattened { text: flat, prov } =
        pp::flatten(&dom, &store, src, &mut strips, &mut warnings);

    let mut tree = parse::parse_blocks(&flat, &mut warnings);
    fill_inline(&mut tree, &flat, config, &mut warnings);

    let (templates, modules) =
        extract::aswritten::template_calls(&dom, &store, config, &mut warnings);
    let facts = extract::document::collect(&tree, &flat, &strips);
    let sections = extract::sections::build(&tree, &flat, &strips, &prov);
    let plain_text = extract::plaintext::plain_text(&tree, &flat, &strips, &store, &dom);
    let redirect = extract::redirect::detect(store.text(src), config);

    ParsedDocument::from_parts(
        wikitext.to_owned(),
        DocumentParts {
            templates,
            modules,
            links: facts.links,
            categories: facts.categories,
            interlanguage: facts
                .interlanguage
                .into_iter()
                .map(|(prefix, target)| InterLanguageLink::new(prefix, target))
                .collect(),
            external_links: facts.external_links,
            warnings: warnings.into_vec(),
            stats: facts.stats,
            plain_text,
            redirect,
            sections,
        },
    )
}

/// Replaces every [`InlineContent::Raw`] slot in the tree with parsed inline nodes.
/// Iterative — an explicit stack of block references, no input-depth recursion.
fn fill_inline(
    tree: &mut DocTree,
    flat: &str,
    config: &SiteConfig,
    warnings: &mut WarningCollector,
) {
    let mut stack: Vec<&mut Block> = tree.blocks.iter_mut().collect();
    while let Some(block) = stack.pop() {
        match block {
            Block::Paragraph { lines } | Block::IndentPre { lines } => {
                for slot in lines {
                    fill_slot(slot, flat, config, warnings);
                }
            }
            Block::Heading { content, .. } => fill_slot(content, flat, config, warnings),
            Block::HorizontalRule => {}
            Block::List { items, .. } => {
                for item in items {
                    if let Some(slot) = item.content.as_mut() {
                        fill_slot(slot, flat, config, warnings);
                    }
                    stack.extend(item.children.iter_mut());
                }
            }
            Block::Table(table) => {
                if let Some(caption) = table.caption.as_mut() {
                    fill_slot(&mut caption.content, flat, config, warnings);
                }
                for row in &mut table.rows {
                    for cell in &mut row.cells {
                        fill_slot(&mut cell.content, flat, config, warnings);
                        stack.extend(cell.children.iter_mut());
                    }
                }
            }
        }
    }
}

fn fill_slot(
    slot: &mut InlineContent,
    flat: &str,
    config: &SiteConfig,
    warnings: &mut WarningCollector,
) {
    if let InlineContent::Raw(range) = *slot {
        *slot = InlineContent::Parsed(inline::parse_inline(flat, range, config, warnings));
    }
}

/// Re-parses the bodies of inline-wikitext extension tags (`<ref>`, `<poem>`) and folds
/// their links, categories, interlanguage and external links into `facts` — the API counts
/// a citation's internal/external links even though they live inside `<ref>`. Iterating all
/// such payloads once covers nested tags too, since every tag registered during expansion
/// is already in `strips`. Structural counters are left to the main walk.
fn collect_inline_tag_facts(
    strips: &StripState,
    store: &SourceStore,
    config: &SiteConfig,
    warnings: &mut WarningCollector,
    facts: &mut extract::document::DocumentFacts,
) {
    for payload in strips.payloads() {
        let StripPayload::ExtTag {
            name,
            body: Some(body),
            ..
        } = payload
        else {
            continue;
        };
        if config.ext_tag(name) != Some(ExtTagContent::InlineWikitext) {
            continue;
        }
        let text = store.slice(*body).to_owned();
        let mut tree = parse::parse_blocks(&text, warnings);
        fill_inline(&mut tree, &text, config, warnings);
        let sub = extract::document::collect(&tree, &text, strips);
        facts.links.extend(sub.links);
        facts.categories.extend(sub.categories);
        facts.interlanguage.extend(sub.interlanguage);
        facts.external_links.extend(sub.external_links);
    }
}

/// Emits the S0 hygiene warnings (input truncation, control-character sanitization).
fn emit_hygiene(warnings: &mut WarningCollector, hygiene: &HygieneReport, limits: &Limits) {
    if hygiene.truncated {
        warnings.emit(Warning::InputTruncated {
            limit_bytes: limits.max_source_bytes(),
        });
    }
    if hygiene.sanitized_controls > 0 {
        warnings.emit(Warning::ControlCharactersSanitized {
            count: hygiene.sanitized_controls,
        });
    }
}

/// Runs the full pipeline WITH template expansion over one page: hygiene → preprocess →
/// expand → block parse → inline pass → post-expansion extraction. Under
/// [`UnresolvedPolicy::Halt`] a pass that hit unresolved templates returns
/// [`ExpandOutcome::MissingTemplates`] so the caller can prefetch and re-run.
#[allow(
    clippy::too_many_arguments,
    reason = "the expand entry threads the page context and both provider seams"
)]
pub(crate) fn expand_page(
    config: &SiteConfig,
    limits: &Limits,
    wikitext: &str,
    title: &Title,
    now: Timestamp,
    source: &dyn TemplateSource,
    existence: Option<&dyn ExistenceProvider>,
    policy: UnresolvedPolicy,
) -> ExpandOutcome {
    let mut warnings = WarningCollector::new();
    let mut store = SourceStore::new();
    let (src, hygiene) = store.push(wikitext, limits.max_source_bytes());
    emit_hygiene(&mut warnings, &hygiene, limits);

    let page_dom = pp::build(&store, src, config, InclusionMode::TopLevel, &mut warnings);
    // As-written top-level calls (with parameters) and the raw-page redirect are captured
    // before the DOM is consumed by expansion.
    let (templates, _as_written_modules) =
        extract::aswritten::template_calls(&page_dom, &store, config, &mut warnings);
    let redirect = extract::redirect::detect(store.text(src), config);

    let ctx = ExpandContext {
        title: title.clone(),
        now,
        source,
        existence,
        policy,
    };
    let mut strips = StripState::new();
    let expansion = expand(
        &mut store,
        page_dom,
        config,
        limits,
        &ctx,
        &mut strips,
        &mut warnings,
    );

    if policy == UnresolvedPolicy::Halt && !expansion.missing.is_empty() {
        return ExpandOutcome::MissingTemplates(MissingTemplates::new(expansion.missing));
    }

    let expanded = expansion.text;
    let mut tree = parse::parse_blocks(&expanded, &mut warnings);
    fill_inline(&mut tree, &expanded, config, &mut warnings);
    let mut facts = extract::document::collect(&tree, &expanded, &strips);
    // Inline-wikitext extension tags (`<ref>`, `<poem>`) hold parsed content: their bodies'
    // links/categories/external links count post-expansion. The main walk sees them only as
    // opaque markers, so re-parse each such body and fold in its facts.
    collect_inline_tag_facts(&strips, &store, config, &mut warnings, &mut facts);
    // The expanded text carries only extension-tag strip markers (templates and arguments
    // are gone), so an empty preprocessor DOM suffices for plaintext rendering.
    let empty_dom = PpDom::new();
    let plain_text = extract::plaintext::plain_text(&tree, &expanded, &strips, &store, &empty_dom);

    // Note: DEFAULTSORT is captured as a document-level side effect (`default_sort`) but
    // NOT folded into per-category sort keys — the live API's `prop=categories` sortkey
    // reflects only the explicit `[[Category:X|key]]` prefix, so a bare category keeps an
    // empty sortkey even when DEFAULTSORT is set.
    ExpandOutcome::Complete(Box::new(ExpandedDocument::from_parts(ExpandedParts {
        source: wikitext.to_owned(),
        templates,
        closure: expansion.used,
        modules: expansion.modules,
        links: facts.links,
        categories: facts.categories,
        interlanguage: facts
            .interlanguage
            .into_iter()
            .map(|(prefix, target)| InterLanguageLink::new(prefix, target))
            .collect(),
        external_links: facts.external_links,
        warnings: warnings.into_vec(),
        stats: facts.stats,
        plain_text,
        redirect,
        default_sort: expansion.default_sort,
        display_title: expansion.display_title,
        wanted_existence: expansion.wanted_existence,
    })))
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::config::{ExtTagContent, MagicWordDef, NamespaceDef};
    use crate::doc::DocumentView;
    use crate::model::Span;
    use crate::title::Namespace;

    fn config() -> SiteConfig {
        SiteConfig::builder("ru", "Вики")
            .namespace(NamespaceDef::main())
            .namespace(NamespaceDef::new(10, "Template", "Шаблон", ["Ш"], true))
            .namespace(NamespaceDef::new(
                14,
                "Category",
                "Категория",
                Vec::<String>::new(),
                true,
            ))
            .namespace(NamespaceDef::new(6, "File", "Файл", ["Изображение"], true))
            .namespace(NamespaceDef::new(
                828,
                "Module",
                "Модуль",
                Vec::<String>::new(),
                true,
            ))
            .magic_word(MagicWordDef::new(
                "redirect",
                ["#REDIRECT", "#перенаправление"],
                false,
            ))
            .magic_word(MagicWordDef::new("invoke", ["invoke"], false))
            .ext_tag("ref", ExtTagContent::InlineWikitext)
            .ext_tag("nowiki", ExtTagContent::Opaque)
            .link_trail("a-zа-яё")
            .build()
            .unwrap()
    }

    fn parse(text: &str) -> ParsedDocument {
        parse_only(
            &config(),
            &Limits::default(),
            text,
            None,
            Timestamp::from_unix(0),
        )
    }

    #[test]
    fn kitchen_sink_flows_through_every_stage() {
        let doc = parse(concat!(
            "== Заголовок ==\n",
            "Абзац с [[ссылкой|меткой]] и {{шаблон|имя=значение|поз}}.\n",
            "* пункт с [[Категория:Тест|ключ]]\n",
            "{| class=\"wikitable\"\n|-\n! шапка\n| ячейка с [[таблице]]\n|}\n",
            "[https://example.com внешняя]\n",
        ));

        assert_eq!(doc.templates().len(), 1);
        assert_eq!(doc.templates()[0].title().text(), "Шаблон:Шаблон");
        assert_eq!(doc.templates()[0].param("имя"), Some("значение"));
        assert_eq!(doc.templates()[0].param("1"), Some("поз"));

        let link_targets: Vec<String> = doc.links().iter().map(|l| l.target().text()).collect();
        assert!(
            link_targets.contains(&"Ссылкой".to_owned()),
            "{link_targets:?}"
        );
        assert!(
            link_targets.contains(&"Таблице".to_owned()),
            "table cell went through the inline pass"
        );

        assert_eq!(doc.categories().len(), 1);
        assert_eq!(doc.categories()[0].sortkey(), "ключ");

        assert_eq!(doc.external_links().len(), 1);
        assert_eq!(doc.external_links()[0].url(), "https://example.com");

        assert_eq!(doc.stats().headings(), 1);
        assert_eq!(doc.stats().tables(), 1);
        assert_eq!(doc.stats().list_items(), 1);
        assert!(doc.plain_text().contains("меткой"));
        assert!(doc.redirect().is_none());
        assert!(doc.warnings().is_empty(), "{:?}", doc.warnings());
        assert!(doc.source().contains("{{шаблон"));
    }

    #[test]
    fn hygiene_warnings_surface() {
        let doc = parse("плохой байт \u{7f} текст");
        assert!(
            doc.warnings()
                .iter()
                .any(|w| matches!(w, Warning::ControlCharactersSanitized { count: 1 }))
        );

        let tiny = parse_only(
            &config(),
            &Limits::default().with_max_source_bytes(10),
            "очень длинный текст который не влезает",
            None,
            Timestamp::from_unix(0),
        );
        assert!(
            tiny.warnings()
                .iter()
                .any(|w| matches!(w, Warning::InputTruncated { limit_bytes: 10 }))
        );
    }

    #[test]
    fn redirect_and_modules_are_detected() {
        let doc = parse("#перенаправление [[Цель#Раздел]]");
        let redirect = doc.redirect().unwrap();
        assert_eq!(redirect.target().text(), "Цель");
        assert_eq!(redirect.fragment(), Some("Раздел"));

        let with_invoke = parse("{{#invoke:Строка|replace|а|б}}");
        assert_eq!(with_invoke.modules().len(), 1);
        assert_eq!(
            with_invoke.modules()[0].module().namespace(),
            Namespace::MODULE
        );
        assert_eq!(with_invoke.modules()[0].function(), "replace");
        assert!(with_invoke.templates().is_empty());
    }

    #[test]
    fn interlanguage_links_map_into_the_model() {
        let cfg = SiteConfig::builder("en", "Wiki")
            .namespace(NamespaceDef::main())
            .interwiki(["fr"])
            .build()
            .unwrap();
        let doc = parse_only(
            &cfg,
            &Limits::default(),
            "See [[fr:Chat]] too.",
            None,
            Timestamp::from_unix(0),
        );
        assert_eq!(doc.interlanguage_links().len(), 1);
        assert_eq!(doc.interlanguage_links()[0].prefix(), "fr");
        assert_eq!(doc.interlanguage_links()[0].target(), "Chat");
    }

    #[test]
    fn nested_structures_get_inline_parsed_without_recursion() {
        // A list item whose sublist holds a link, plus a table cell whose
        // continuation holds a nested table — both must reach the inline pass.
        let doc = parse(concat!(
            "* верх\n** глубже с [[глубокой]]\n",
            "{|\n|-\n| ячейка\n{|\n|-\n| вложенная с [[самой глубокой]]\n|}\n|}\n",
        ));
        let targets: Vec<String> = doc.links().iter().map(|l| l.target().text()).collect();
        assert!(targets.contains(&"Глубокой".to_owned()), "{targets:?}");
        assert!(
            targets.contains(&"Самой глубокой".to_owned()),
            "{targets:?}"
        );
        assert_eq!(doc.stats().tables(), 2);
    }

    #[test]
    fn empty_input_yields_an_empty_document() {
        let doc = parse("");
        assert!(doc.templates().is_empty());
        assert!(doc.links().is_empty());
        assert!(doc.plain_text().is_empty());
        assert!(doc.sections().is_empty());
        assert_eq!(doc.stats(), &crate::model::StructuralStats::default());
        assert!(format!("{doc:?}").contains("ParsedDocument"));
    }

    #[test]
    fn sections_nest_by_level_with_contiguous_source_spans() {
        let src = "== A ==\ntext\n=== B ===\n{{t}}<!--c-->z\n== C ==\nend";
        let doc = parse(src);
        let roots = doc.sections();
        assert_eq!(roots.len(), 2, "two top-level sections");

        let a = &roots[0];
        assert_eq!(a.level(), 2);
        assert_eq!(a.title_text(), "A");
        // Spans are original-source byte offsets, not flat-text offsets.
        assert_eq!(
            &src[a.heading_span().start..a.heading_span().end],
            "== A =="
        );
        assert_eq!(&src[a.title_span().start..a.title_span().end], "A");
        // A's body runs up to (not into) the next same-level heading `== C ==`.
        assert_eq!(a.body_span(), Span { start: 0, end: 38 });

        assert_eq!(a.children().len(), 1, "B nests under A");
        let b = &a.children()[0];
        assert_eq!(b.level(), 3);
        assert_eq!(b.title_text(), "B");
        // B's body is source-contiguous: it keeps the template and comment verbatim,
        // even though both vanish from the flattened text.
        let body = &src[b.body_span().start..b.body_span().end];
        assert!(
            body.contains("{{t}}") && body.contains("<!--c-->"),
            "{body:?}"
        );

        let c = &roots[1];
        assert_eq!(c.level(), 2);
        assert_eq!(c.title_text(), "C");
        assert!(c.children().is_empty());
        // The last section extends to end of source.
        assert_eq!(
            c.body_span(),
            Span {
                start: 38,
                end: src.len()
            }
        );
    }

    #[test]
    fn heading_titles_render_links_and_expose_raw_markup() {
        let src = "== <s>Old</s> [[Nom]] ==\nbody";
        let doc = parse(src);
        let section = &doc.sections()[0];
        // The link contributes its text to the rendered title.
        assert!(section.title_text().contains("Nom"));
        // The raw title slice lets a caller detect the struck (closed) marker.
        let raw = &src[section.title_span().start..section.title_span().end];
        assert_eq!(raw, "<s>Old</s> [[Nom]]");
    }

    /// Both arms are exercised across the tests below (a completing expand and a halting
    /// one), so neither is left uncovered.
    fn to_result(outcome: ExpandOutcome) -> Result<Box<ExpandedDocument>, MissingTemplates> {
        match outcome {
            ExpandOutcome::Complete(doc) => Ok(doc),
            ExpandOutcome::MissingTemplates(missing) => Err(missing),
        }
    }

    fn expand_complete(wikitext: &str, limits: &Limits) -> Box<ExpandedDocument> {
        use crate::source::EmptyTemplateSource;
        let cfg = config();
        let title = Title::parse("Тест", &cfg).unwrap();
        to_result(expand_page(
            &cfg,
            limits,
            wikitext,
            &title,
            Timestamp::from_unix(0),
            &EmptyTemplateSource,
            None,
            UnresolvedPolicy::RedLink,
        ))
        .expect("expected a complete doc")
    }

    #[test]
    fn expand_page_halts_on_unresolved_templates() {
        use crate::source::EmptyTemplateSource;
        let cfg = config();
        let title = Title::parse("Страница", &cfg).unwrap();
        let missing = to_result(expand_page(
            &cfg,
            &Limits::default(),
            "{{Тест}}",
            &title,
            Timestamp::from_unix(0),
            &EmptyTemplateSource,
            None,
            UnresolvedPolicy::Halt,
        ))
        .expect_err("expected a halt");
        assert_eq!(
            missing.titles().iter().map(Title::text).collect::<Vec<_>>(),
            ["Шаблон:Тест"]
        );
    }

    #[test]
    fn expand_page_emits_hygiene_warnings() {
        let sanitized = expand_complete("плохой \u{7f} текст", &Limits::default());
        assert!(
            sanitized
                .warnings()
                .iter()
                .any(|w| matches!(w, Warning::ControlCharactersSanitized { .. }))
        );
        let truncated = expand_complete(
            "очень длинный текст",
            &Limits::default().with_max_source_bytes(6),
        );
        assert!(
            truncated
                .warnings()
                .iter()
                .any(|w| matches!(w, Warning::InputTruncated { .. }))
        );
    }
}