ruwex 0.1.0

//! Port of wikiextractor's `extractPage`: scans a dump line by line and
//! prints the raw XML of matching pages. Faithful to the original's
//! line-oriented logic, including its quirks: any `<id>` inside the page
//! (page id, revision id, contributor id) matches `--id`, and in
//! `--template` mode all `Template:` pages are printed while their
//! non-matching `<id>` lines are omitted.

use std::io::{BufRead, Write};
use std::sync::LazyLock;

use regex::Regex;

static TAG: LazyLock<Regex> =
    LazyLock::new(|| Regex::new(r"(?s)(.*?)<(/?\w+)[^>]*>(?:([^<]*)(<.*?>)?)?").unwrap());

/// Scans `input`, writing the raw XML of matching pages to `out`.
/// Without `templates`, the first page containing `<id>id</id>` is printed
/// and scanning stops; with `templates`, every `Template:` page is printed.
pub fn extract_page(
    mut input: impl BufRead,
    out: &mut impl Write,
    id: &str,
    templates: bool,
) -> std::io::Result<()> {
    let mut page: Vec<String> = Vec::new();
    let mut in_article = false;
    let mut line = String::new();
    loop {
        line.clear();
        if input.read_line(&mut line)? == 0 {
            return Ok(());
        }
        if !line.contains('<') {
            if !page.is_empty() {
                page.push(line.clone());
            }
            continue;
        }
        let Some(caps) = TAG.captures(&line) else {
            continue;
        };
        let tag = caps.get(2).expect("tag group").as_str();
        let content = caps.get(3).map(|m| m.as_str()).unwrap_or("");
        match tag {
            "page" => {
                page.clear();
                page.push(line.clone());
                in_article = false;
            }
            "id" => {
                if id == content {
                    page.push(line.clone());
                    in_article = true;
                } else if !in_article && !templates {
                    page.clear();
                }
            }
            "title" => {
                if templates {
                    if content.starts_with("Template:") {
                        page.push(line.clone());
                    } else {
                        page.clear();
                    }
                } else {
                    page.push(line.clone());
                }
            }
            "/page" => {
                if !page.is_empty() {
                    page.push(line.clone());
                    for page_line in &page {
                        out.write_all(page_line.as_bytes())?;
                    }
                    out.write_all(b"\n")?; // print() newline
                    if !templates {
                        return Ok(());
                    }
                }
                page.clear();
            }
            _ => {
                if !page.is_empty() {
                    page.push(line.clone());
                }
            }
        }
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    const DUMP: &str = "<mediawiki>\n  <siteinfo>\n    <base>http://x/wiki/Main</base>\n  </siteinfo>\n\
        \x20 <page>\n    <title>First</title>\n    <ns>0</ns>\n    <id>1</id>\n    <revision>\n      \
        <id>100</id>\n      <text>first text</text>\n    </revision>\n  </page>\n\
        \x20 <page>\n    <title>Second</title>\n    <ns>0</ns>\n    <id>2</id>\n    <revision>\n      \
        <id>200</id>\n      <text>second text</text>\n    </revision>\n  </page>\n\
        \x20 <page>\n    <title>Template:Box</title>\n    <ns>10</ns>\n    <id>3</id>\n    <revision>\n      \
        <id>300</id>\n      <text>box body</text>\n    </revision>\n  </page>\n</mediawiki>\n";

    fn run(id: &str, templates: bool) -> String {
        let mut out = Vec::new();
        extract_page(DUMP.as_bytes(), &mut out, id, templates).unwrap();
        String::from_utf8(out).unwrap()
    }

    #[test]
    fn extracts_page_by_id_and_stops() {
        let out = run("2", false);
        assert!(out.contains("<title>Second</title>"));
        assert!(out.contains("second text"));
        assert!(!out.contains("First"));
        assert!(!out.contains("Template:Box"));
        assert!(out.ends_with("</page>\n\n"));
    }

    #[test]
    fn matches_revision_ids_too() {
        // like the original: any <id> inside the page matches, but the
        // earlier non-matching page id reset the buffer, so the printed
        // fragment starts at the matching <id> line
        let out = run("100", false);
        assert!(out.contains("<id>100</id>"));
        assert!(out.contains("first text"));
        assert!(out.trim_end().ends_with("</page>"));
        assert!(!out.contains("<title>First</title>"));
    }

    #[test]
    fn unknown_id_prints_nothing() {
        assert_eq!(run("999", false), "");
    }

    #[test]
    fn template_mode_prints_all_template_pages() {
        let out = run("1", true);
        assert!(out.contains("<title>Template:Box</title>"));
        assert!(out.contains("box body"));
        assert!(!out.contains("<title>Second</title>"));
    }
}