wikioxide 0.1.0

//! MediaWiki parser-test harness.
//!
//! Reads the official `parserTests` corpus (downloaded by `cargo xtask fetch-parsertests`
//! into the gitignored `.parsertests/` — the corpus is GPL-2.0+ and is never committed) and
//! exercises our parser against it two ways:
//!
//! 1. **Totality** — every test's `!! wikitext` block is run through
//!    [`WikitextParser::parse`]; the only assertion is that it never panics and terminates.
//!    Across the whole corpus this is a robustness check against ~1000+ real, often
//!    pathological, MediaWiki inputs.
//! 2. **Semantic subset** — a curated allowlist of link-focused tests whose expected `!! html`
//!    unambiguously maps to our domain models: we extract the `<a href="/wiki/…">` internal
//!    links from the expected HTML and assert our `links()` extraction agrees. parserTests
//!    emit HTML and we emit typed models, so a full HTML comparison is impossible; this checks
//!    the slice we can map faithfully.
//!
//! Both tests self-skip (loudly) when `.parsertests/` is absent, so the GPL corpus is never a
//! hard build dependency.
#![cfg(feature = "presets")]
#![allow(clippy::expect_used)]

use std::collections::BTreeSet;
use std::fs;
use std::path::PathBuf;

use wikioxide::{DocumentView, PageContext, SiteConfig, WikitextParser};

fn corpus_dir() -> PathBuf {
    PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../../.parsertests")
}

/// The topic files this harness reads (a subset of the full split corpus).
const FILES: &[&str] = &[
    "parserTests.txt",
    "quotes.txt",
    "tables.txt",
    "extLinks.txt",
    "magicLinks.txt",
    "headings.txt",
    "indentPre.txt",
    "comments.txt",
    "definitionLists.txt",
    "redirects.txt",
    "pWrapping.txt",
];

/// One parsed `!! test` record.
#[derive(Debug, Default, Clone)]
struct Case {
    name: String,
    wikitext: Option<String>,
    html: Option<String>,
    options: String,
}

/// Splits one corpus file into its `!! test … !! end` records. `!! article` and other
/// block types are ignored (we do not model page existence for parse-only tests).
fn parse_cases(text: &str) -> Vec<Case> {
    let mut cases = Vec::new();
    let mut lines = text.lines();
    while let Some(line) = lines.next() {
        if line.trim_end() != "!! test" {
            continue;
        }
        let mut case = Case::default();
        let mut field = "name".to_owned();
        let mut buf = String::new();
        for inner in lines.by_ref() {
            let trimmed = inner.trim_end();
            if trimmed == "!! end" {
                store_field(&mut case, &field, &buf);
                break;
            }
            match trimmed.strip_prefix("!! ") {
                Some(key) => {
                    store_field(&mut case, &field, &buf);
                    buf.clear();
                    field = key.trim().to_owned();
                }
                None => {
                    buf.push_str(inner);
                    buf.push('\n');
                }
            }
        }
        cases.push(case);
    }
    cases
}

/// Assigns an accumulated block `buf` to the right field of `case`.
fn store_field(case: &mut Case, field: &str, buf: &str) {
    let value = buf.trim_end_matches('\n').to_owned();
    match field {
        "name" => case.name = value.trim().to_owned(),
        "wikitext" => case.wikitext = Some(value),
        "options" => case.options = value,
        // Legacy PHP HTML (`html` / `html/php`) always wins over a Parsoid-only block.
        "html" | "html/php" => case.html = Some(value),
        f if f.starts_with("html/") => {
            case.html.get_or_insert(value);
        }
        _ => {}
    }
}

/// Loads every available corpus file's test cases from `dir`, or `None` if `dir` holds no
/// readable corpus file (absent directory, or none of [`FILES`] present).
fn load_corpus_from(dir: &std::path::Path) -> Option<Vec<Case>> {
    let mut cases = Vec::new();
    let mut found = false;
    for file in FILES {
        if let Ok(text) = fs::read_to_string(dir.join(file)) {
            found = true;
            cases.extend(parse_cases(&text));
        }
    }
    found.then_some(cases)
}

/// Loads the corpus from the standard `.parsertests/` location.
fn load_corpus() -> Option<Vec<Case>> {
    load_corpus_from(&corpus_dir())
}

#[test]
fn parser_tests_totality() {
    let Some(cases) = load_corpus() else {
        eprintln!(
            "SKIP parser_tests_totality: .parsertests/ absent — run `cargo xtask fetch-parsertests`"
        );
        return;
    };
    let parser = WikitextParser::builder()
        .site_config(SiteConfig::enwiki().clone())
        .build()
        .expect("parser builds");
    let ctx = PageContext::default();

    let mut ran = 0;
    for case in &cases {
        let Some(wikitext) = &case.wikitext else {
            continue;
        };
        ran += 1;
        // The only contract: parse() is total — it must not panic on any real input.
        let doc = parser.parse(wikitext, &ctx);
        // Touch every accessor so nothing lazy hides a panic.
        let _ = (
            doc.templates().len(),
            doc.links().len(),
            doc.categories().len(),
            doc.external_links().len(),
            doc.warnings().len(),
            doc.stats().tables(),
            doc.plain_text().len(),
            doc.redirect().is_some(),
        );
    }
    assert!(ran >= 300, "expected a substantial corpus, ran only {ran}");
    eprintln!("parser_tests_totality: parsed {ran} parserTests wikitext blocks without panic");
}

#[test]
fn parser_tests_internal_links_semantic() {
    let Some(cases) = load_corpus() else {
        eprintln!(
            "SKIP parser_tests_internal_links_semantic: .parsertests/ absent — run `cargo xtask fetch-parsertests`"
        );
        return;
    };
    let parser = WikitextParser::builder()
        .site_config(SiteConfig::enwiki().clone())
        .build()
        .expect("parser builds");
    let ctx = PageContext::default();

    let mut checked = 0;
    let mut mismatches = Vec::new();
    for case in &cases {
        let (Some(wikitext), Some(html)) = (&case.wikitext, &case.html) else {
            continue;
        };
        if !is_link_semantic_candidate(case, wikitext, html) {
            continue;
        }
        let expected = expected_internal_links(html);
        // Restrict to the cleanest signal — plain main-namespace wikilinks. Any expected
        // target with a colon involves a namespace/Special/interwiki whose handling is
        // config-dependent (e.g. ISBN magic links point at `Special:BookSources`, which are
        // disabled on enwiki; `Meta:`/`MemoryAlpha:` are interwiki prefixes there), not a
        // parse question.
        if expected.is_empty() || expected.iter().any(|l| l.contains(':')) {
            continue;
        }
        let doc = parser.parse(wikitext, &ctx);
        let got: BTreeSet<String> = doc.links().iter().map(|l| l.target().text()).collect();
        checked += 1;
        // Our set must contain every internal link MediaWiki rendered. (We may legitimately
        // list more — e.g. a link the test's HTML shortened — so this is a superset check.)
        for want in &expected {
            if !got.contains(want) {
                mismatches.push(format!(
                    "[{}] missing link {want:?}; got {:?}",
                    case.name, got
                ));
            }
        }
    }
    assert!(
        mismatches.is_empty(),
        "{} internal-link mismatches over {checked} checked cases:\n{}",
        mismatches.len(),
        mismatches.join("\n")
    );
    eprintln!("parser_tests_internal_links_semantic: {checked} link cases agree with MediaWiki");
}

/// Whether a case is a safe internal-link semantic check: it has real HTML (not only
/// parsoid), uses no templates/parser functions or pre-save transform, and does not opt into
/// behaviors we intentionally diverge on.
fn is_link_semantic_candidate(case: &Case, wikitext: &str, html: &str) -> bool {
    let opts = case.options.to_lowercase();
    !wikitext.contains("{{")
        && !wikitext.contains("<ref")
        // A nested `[[…[[…]]…]]` (e.g. `[[Hi|[[Ho]]]]`) parses by rules we intentionally
        // diverge on; `|[[` is its signature.
        && !wikitext.contains("|[[")
        && !opts.contains("pst")
        && !opts.contains("subpage")
        && !opts.contains("title=")
        && !opts.contains("djvu")
        && html.contains("href=\"/wiki/")
        && !html.contains("redlink=1")
        && !html.contains("class=\"external")
        && !html.contains("interwiki")
}

/// Extracts the internal-link targets from expected HTML: every `href="/wiki/Title"` becomes
/// the decoded, space-normalized title MediaWiki would report.
fn expected_internal_links(html: &str) -> BTreeSet<String> {
    let mut out = BTreeSet::new();
    let needle = "href=\"/wiki/";
    let mut rest = html;
    while let Some(pos) = rest.find(needle) {
        rest = &rest[pos + needle.len()..];
        let end = rest.find('"').unwrap_or(rest.len());
        let raw = &rest[..end];
        rest = &rest[end..];
        // Drop fragments and skip red links / query-string hrefs.
        let target = raw.split('#').next().unwrap_or(raw);
        if target.is_empty() || target.contains('?') {
            continue;
        }
        out.insert(decode_wiki_href(target));
    }
    out
}

/// Turns a `/wiki/`-path segment into a display title: percent-decode, underscores → spaces.
fn decode_wiki_href(path: &str) -> String {
    let bytes = path.as_bytes();
    let mut decoded = Vec::with_capacity(bytes.len());
    let mut i = 0;
    while i < bytes.len() {
        if bytes[i] == b'%' && i + 2 < bytes.len() {
            if let Ok(byte) = u8::from_str_radix(&path[i + 1..i + 3], 16) {
                decoded.push(byte);
                i += 3;
                continue;
            }
        }
        decoded.push(bytes[i]);
        i += 1;
    }
    String::from_utf8_lossy(&decoded).replace('_', " ")
}

#[cfg(test)]
mod meta_tests {
    use super::*;

    #[test]
    fn case_parser_splits_a_record() {
        let text = "\
!! test
Sample
!! wikitext
Hello [[World]]
!! html
<p>Hello <a href=\"/wiki/World\">World</a></p>
!! end
";
        let cases = parse_cases(text);
        assert_eq!(cases.len(), 1);
        assert_eq!(cases[0].name, "Sample");
        assert_eq!(cases[0].wikitext.as_deref(), Some("Hello [[World]]"));
        assert!(cases[0].html.as_deref().unwrap().contains("/wiki/World"));
    }

    #[test]
    fn html_href_extraction_and_decoding() {
        let html = "<a href=\"/wiki/Main_Page\">x</a> <a href=\"/wiki/Caf%C3%A9#top\">y</a>";
        let links = expected_internal_links(html);
        assert!(links.contains("Main Page"));
        assert!(links.contains("Café"), "{links:?}");
    }

    #[test]
    fn load_corpus_from_absent_dir_is_none() {
        let missing = corpus_dir().join("does-not-exist-subdir");
        assert!(load_corpus_from(&missing).is_none());
    }

    #[test]
    fn php_html_is_preferred_over_parsoid() {
        let text = "\
!! test
Prefer
!! wikitext
x
!! html/parsoid
<p>parsoid</p>
!! html/php
<p>php</p>
!! end
";
        let cases = parse_cases(text);
        assert_eq!(cases[0].html.as_deref(), Some("<p>php</p>"));
    }
}