ruwex 0.1.0

//! Wikitext cleaning: markup → plain text (or simple HTML), a faithful port
//! of wikiextractor's `clean()` and `compact()`.
//!
//! One deliberate divergence: the original collects HTML-comment spans
//! *before* substituting `<br>`/`<hr>` tags but removes them *after*, so any
//! line-break tag preceding a comment shifts the recorded offsets and
//! corrupts nearby text. We substitute line breaks first — the evidently
//! intended order.

pub mod entities;
mod entities_table;
pub mod links;
pub mod nested;
pub mod tags;

use std::collections::BTreeMap;
use std::sync::LazyLock;

use regex::Regex;

use crate::config::ExtractorConfig;
use crate::dump::Page;
use crate::expand::{Expander, TemplateSource};
use nested::drop_nested;

/// Behavioral switches are simply removed from the text.
const MAGIC_SWITCHES: &[&str] = &[
    "__NOTOC__",
    "__FORCETOC__",
    "__TOC__",
    "__NEWSECTIONLINK__",
    "__NONEWSECTIONLINK__",
    "__NOGALLERY__",
    "__HIDDENCAT__",
    "__NOCONTENTCONVERT__",
    "__NOCC__",
    "__NOTITLECONVERT__",
    "__NOTC__",
    "__START__",
    "__END__",
    "__INDEX__",
    "__NOINDEX__",
    "__STATICREDIRECT__",
    "__DISAMBIG__",
    "__NOEDITSECTION__",
];

static TABLE_OPEN: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"\{\|").unwrap());
static TABLE_CLOSE: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"\|\}").unwrap());

static BOLD_ITALIC: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"'''''(.*?)'''''").unwrap());
static BOLD: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"'''(.*?)'''").unwrap());
static ITALIC_QUOTE: LazyLock<Regex> = LazyLock::new(|| Regex::new(r#"''"([^"]*?)"''"#).unwrap());
static ITALIC: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"''(.*?)''").unwrap());
static QUOTE_QUOTE: LazyLock<Regex> = LazyLock::new(|| Regex::new("\"\"([^\"]*?)\"\"").unwrap());

static SPACES: LazyLock<Regex> = LazyLock::new(|| Regex::new(r" {2,}").unwrap());
static DOTS: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"\.{4,}").unwrap());
static PUNCT_ONLY_LINES: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"\n\W+?\n").unwrap());

/// Cleans a page's wikitext and splits it into output paragraphs.
pub fn extract_paragraphs(
    page: &Page,
    config: &ExtractorConfig,
    templates: &dyn TemplateSource,
) -> Vec<String> {
    let mut expander = Expander::new(&page.title, templates);
    let cleaned = clean_wikitext(&mut expander, &page.text, config);
    compact(&cleaned, config, false)
}

/// Port of `clean()`: transforms wiki markup into text. Templates are
/// expanded (against the — for now — empty template database), tables and
/// markup dropped, entities decoded, whitespace normalized.
pub fn clean_wikitext(expander: &mut Expander, text: &str, config: &ExtractorConfig) -> String {
    let text = expander.expand(text);
    let text = drop_nested(&text, &TABLE_OPEN, &TABLE_CLOSE);
    let text = links::replace_external_links(&text, config);
    let mut text = links::replace_internal_links(&text, config);

    for switch in MAGIC_SWITCHES {
        if text.contains(switch) {
            text = text.replace(switch, "");
        }
    }

    let (mut text, syntax_blocks) = tags::protect_syntaxhighlight(&text);

    // bold/italic/quotes
    if config.html {
        text = BOLD_ITALIC.replace_all(&text, "<b>${1}</b>").into_owned();
        text = BOLD.replace_all(&text, "<b>${1}</b>").into_owned();
        text = ITALIC.replace_all(&text, "<i>${1}</i>").into_owned();
    } else {
        text = BOLD_ITALIC.replace_all(&text, "${1}").into_owned();
        text = BOLD.replace_all(&text, "${1}").into_owned();
        text = ITALIC_QUOTE.replace_all(&text, "\"${1}\"").into_owned();
        text = ITALIC.replace_all(&text, "\"${1}\"").into_owned();
        text = QUOTE_QUOTE.replace_all(&text, "\"${1}\"").into_owned();
    }
    // residuals of unbalanced quotes
    text = text.replace("'''", "").replace("''", "\"");

    text = tags::substitute_line_break_tags(&text);
    text = tags::drop_tag_spans(&text, config.keep_links);
    text = tags::drop_discarded_elements(text);

    if !config.html {
        // turn what is left into text (&nbsp;, &ndash;, …)
        text = entities::unescape(&text);
    }
    text = tags::restore_syntaxhighlight(&text, &syntax_blocks);
    text = tags::expand_placeholders(&text);
    text = text.replace("<<", "«").replace(">>", "»");

    // cleanup
    text = text.replace('\t', " ");
    text = SPACES.replace_all(&text, " ").into_owned();
    text = DOTS.replace_all(&text, "...").into_owned();
    // these two literal fix-ups are what the original's (mis-grouped)
    // punctuation regexes actually do
    text = text.replace(" ,:.)]»", ",:.)]»");
    text = text.replace("[(« ", "[(«");
    text = PUNCT_ONLY_LINES.replace_all(&text, "\n").into_owned();
    text = text.replace(",,", ",").replace(",.", ".");

    if config.html_safe {
        text = entities::html_escape(&text).into_owned();
    }
    text
}

/// Port of `compact()`: handles headers, lists, empty sections, and
/// residuals of tables, producing the final list of output paragraphs.
pub fn compact(text: &str, config: &ExtractorConfig, mark_headers: bool) -> Vec<String> {
    let mut page: Vec<String> = Vec::new();
    let mut headers: BTreeMap<usize, String> = BTreeMap::new(); // level → unfilled section title
    let mut empty_section = false;
    let mut list_level: Vec<u8> = Vec::new(); // nesting of lists (HTML mode)

    let close_lists = |page: &mut Vec<String>, list_level: &mut Vec<u8>| {
        for &c in list_level.iter().rev() {
            page.push(list_close(c).to_string());
        }
        list_level.clear();
    };

    for line in text.split('\n') {
        if line.is_empty() {
            if !list_level.is_empty() {
                close_lists(&mut page, &mut list_level);
            }
            continue;
        }

        // section titles
        if let Some((level, title)) = match_section(line) {
            let mut title = title.to_string();
            if config.html {
                page.push(format!("<h{level}>{title}</h{level}>"));
            }
            if !title.is_empty() && !title.ends_with(['!', '?']) {
                title.push('.');
            }
            if mark_headers {
                title.insert_str(0, "## ");
            }
            headers.insert(level, title);
            headers.retain(|&k, _| k <= level); // drop deeper previous headers
            empty_section = true;
            continue;
        }

        let first = line.chars().next().expect("line is non-empty");
        let last = line.chars().next_back().expect("line is non-empty");
        if let Some(rest) = line.strip_prefix("++") {
            // page title line
            let title: String = {
                let count = rest.chars().count();
                rest.chars().take(count.saturating_sub(2)).collect()
            };
            if !title.is_empty() {
                let mut title = title;
                if !title.ends_with(['!', '?']) {
                    title.push('.');
                }
                page.push(title);
            }
        } else if first == ':' {
            // indent: emitted immediately, even before a pending section
            // header — a quirk inherited from the original
            page.push(line.trim_start_matches(':').to_string());
        } else if matches!(first, '*' | '#' | ';') {
            if config.html {
                compact_html_list_line(line, &mut page, &mut list_level);
            }
            // plain text output drops list items entirely
        } else if !list_level.is_empty() {
            // first line after a list (HTML mode): closes it, line dropped
            close_lists(&mut page, &mut list_level);
        } else if matches!(first, '{' | '|') || last == '}' {
            // residuals of tables
        } else if (first == '(' && last == ')')
            || line.trim_matches(['.', '-'].as_slice()).is_empty()
        {
            // irrelevant/punctuation-only lines
        } else if !headers.is_empty() {
            for title in headers.values() {
                page.push(title.clone());
            }
            headers.clear();
            page.push(line.to_string());
            empty_section = false;
        } else if !empty_section {
            page.push(line.to_string());
        }
    }
    page
}

/// HTML-mode list handling (only ever called with `config.html`).
/// The `#` item template's `</<li>` typo in the original is not reproduced.
fn compact_html_list_line(line: &str, page: &mut Vec<String>, list_level: &mut Vec<u8>) {
    let bytes = line.as_bytes();
    // close extra levels
    let mut l = 0;
    for (i, &c) in list_level.iter().enumerate() {
        if l < bytes.len() && c != bytes[l] {
            for &extra in list_level[i..].iter().rev() {
                page.push(list_close(extra).to_string());
            }
            list_level.truncate(i);
            break;
        }
        l += 1;
    }
    let item_type;
    let rest;
    if l < bytes.len() && matches!(bytes[l], b'*' | b'#' | b';' | b':') {
        // add new level (only one, no jumps)
        item_type = bytes[l];
        page.push(list_open(item_type).to_string());
        list_level.push(item_type);
        rest = line[l + 1..].trim();
    } else {
        // continue on the same level
        item_type = bytes[l - 1];
        rest = line[l..].trim();
    }
    page.push(list_item(item_type, rest));
}

fn list_open(marker: u8) -> &'static str {
    match marker {
        b'*' => "<ul>",
        b'#' => "<ol>",
        _ => "<dl>",
    }
}

fn list_close(marker: u8) -> &'static str {
    match marker {
        b'*' => "</ul>",
        b'#' => "</ol>",
        _ => "</dl>",
    }
}

fn list_item(marker: u8, text: &str) -> String {
    match marker {
        b'*' | b'#' => format!("<li>{text}</li>"),
        b';' => format!("<dt>{text}</dt>"),
        _ => format!("<dd>{text}</dd>"),
    }
}

/// Matches a section heading like the original's `(==+)\s*(.*?)\s*\1`
/// (anchored at line start, trailing text after the closing run ignored,
/// greedy opening run with backtracking).
fn match_section(line: &str) -> Option<(usize, &str)> {
    let run = line.bytes().take_while(|&b| b == b'=').count();
    if run < 2 {
        return None;
    }
    for open_len in (2..=run).rev() {
        let rest = &line[open_len..];
        let close = "=".repeat(open_len);
        if let Some(j) = rest.find(&close) {
            return Some((open_len, rest[..j].trim()));
        }
    }
    None
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::expand::TemplateDb;

    fn paragraphs(wikitext: &str) -> Vec<String> {
        let page = Page {
            id: 1,
            revid: 2,
            ns: 0,
            title: "Test".to_string(),
            redirect: None,
            text: wikitext.to_string(),
        };
        extract_paragraphs(&page, &ExtractorConfig::default(), &TemplateDb::default())
    }

    #[test]
    fn section_matcher_follows_python_backtracking() {
        assert_eq!(match_section("== Etymology =="), Some((2, "Etymology")));
        assert_eq!(match_section("=== Usage ==="), Some((3, "Usage")));
        assert_eq!(match_section("== a == b =="), Some((2, "a")));
        assert_eq!(match_section("===x=="), Some((2, "=x")));
        assert_eq!(match_section("===="), Some((2, "")));
        assert_eq!(match_section("= one ="), None);
        assert_eq!(match_section("no heading"), None);
    }

    #[test]
    fn headers_only_emitted_for_filled_sections() {
        let out = paragraphs("intro\n== Empty ==\n== Full ==\nbody text\n");
        assert_eq!(out, vec!["intro", "Full.", "body text"]);
    }

    #[test]
    fn nested_headers_flush_in_level_order() {
        let out = paragraphs("== A ==\n=== B ===\nbody\n");
        assert_eq!(out, vec!["A.", "B.", "body"]);
    }

    #[test]
    fn lists_indents_and_tables_are_handled() {
        let out = paragraphs("* item\n# num\n; def\n: indented\n{| table |}\nreal text\n");
        assert_eq!(out, vec![" indented", "real text"]);
    }

    #[test]
    fn bold_italic_and_quotes_become_plain() {
        let out = paragraphs("'''Bold''' and ''italic'' and '''''both'''''.");
        assert_eq!(out, vec!["Bold and \"italic\" and both."]);
    }

    #[test]
    fn guillemets_dots_and_spaces_normalize() {
        let out = paragraphs("<<q>> and....... too   many  spaces\t.");
        assert_eq!(out, vec!["«q» and... too many spaces ."]);
    }

    #[test]
    fn html_safe_escapes_output() {
        let out = paragraphs("AT&T rocks");
        assert_eq!(out, vec!["AT&amp;T rocks"]);
        let config = ExtractorConfig {
            html_safe: false,
            ..Default::default()
        };
        let page = Page {
            title: "T".into(),
            text: "AT&T rocks".into(),
            ..Default::default()
        };
        assert_eq!(
            extract_paragraphs(&page, &config, &TemplateDb::default()),
            vec!["AT&T rocks"]
        );
    }

    #[test]
    fn entities_decode_once_then_escape() {
        // author wrote &ndash; and &nbsp; in the wikitext
        let out = paragraphs("1&ndash;5 and 100&nbsp;km");
        assert_eq!(out, vec!["1\u{2013}5 and 100\u{a0}km"]);
    }

    #[test]
    fn refs_and_comments_disappear() {
        let out = paragraphs(
            "Fact<ref name=\"a\">cite</ref> and<ref name=\"b\"/> more<!-- hidden -->text.",
        );
        assert_eq!(out, vec!["Fact and moretext."]);
    }

    #[test]
    fn templates_vanish_but_parser_functions_evaluate() {
        let out = paragraphs("A {{fake template}} B {{#ifeq:x|x|C}} D {{PAGENAME}}.");
        assert_eq!(out, vec!["A B C D Test."]);
    }

    #[test]
    fn html_mode_keeps_formatting() {
        let config = ExtractorConfig {
            html: true,
            keep_links: true,
            html_safe: false,
            ..Default::default()
        };
        let page = Page {
            title: "T".into(),
            text: "== Head ==\n'''bold''' and [[Target|linked]]\n* item one\n* item two\n\ntail"
                .into(),
            ..Default::default()
        };
        let out = extract_paragraphs(&page, &config, &TemplateDb::default());
        // NOTE: upstream quirk preserved — the <b>/<i> tags that HTML mode
        // inserts are immediately stripped again by its own ignored-tags
        // pass (`b` and `i` are in ignoredTags), so only headings, lists,
        // and <a> links survive.
        assert_eq!(
            out,
            vec![
                "<h2>Head</h2>",
                "Head.",
                "bold and <a href=\"Target\">linked</a>",
                "<ul>",
                "<li>item one</li>",
                "<li>item two</li>",
                "</ul>",
                "tail"
            ]
        );
    }
}