ruwex 0.1.0

//! External and internal link replacement, ported from wikiextractor's
//! `replaceExternalLinks` / `replaceInternalLinks`.

use std::sync::LazyLock;

use regex::Regex;

use crate::clean::entities::urlencode;
use crate::config::ExtractorConfig;
use crate::expand::braces::find_balanced;

const URL_PROTOCOLS: &[&str] = &[
    "bitcoin:",
    "ftp://",
    "ftps://",
    "geo:",
    "git://",
    "gopher://",
    "http://",
    "https://",
    "irc://",
    "ircs://",
    "magnet:",
    "mailto:",
    "mms://",
    "news:",
    "nntp://",
    "redis://",
    "sftp://",
    "sip:",
    "sips:",
    "sms:",
    "ssh://",
    "svn://",
    "tel:",
    "telnet://",
    "urn:",
    "worldwind://",
    "xmpp:",
    "//",
];

static EXT_LINK_BRACKETED: LazyLock<Regex> = LazyLock::new(|| {
    let protocols = URL_PROTOCOLS.join("|");
    // everything except bracket, space, or control characters in the URL
    Regex::new(&format!(
        r#"(?is)\[(({protocols})[^\]\[<>"\x00-\x20\x7F\s]+)\s*([^\]\x00-\x08\x0a-\x1F]*?)\]"#
    ))
    .unwrap()
});

static EXT_IMAGE: LazyLock<Regex> = LazyLock::new(|| {
    Regex::new(
        r#"(?is)^(http://|https://)([^\]\[<>"\x00-\x20\x7F\s]+)/([A-Za-z0-9_.,~%\-+&;#*?!=()@\x80-\xFF]+)\.(gif|png|jpg|jpeg)$"#,
    )
    .unwrap()
});

static TAIL: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"^\w+").unwrap());

pub fn replace_external_links(text: &str, config: &ExtractorConfig) -> String {
    let mut result = String::with_capacity(text.len());
    let mut cur = 0;
    for caps in EXT_LINK_BRACKETED.captures_iter(text) {
        let whole = caps.get(0).expect("match");
        result.push_str(&text[cur..whole.start()]);
        cur = whole.end();

        let url = caps.get(1).expect("url").as_str();
        let label = caps.get(3).expect("label").as_str();

        // If the link text is an image URL, it becomes an <img> tag (or
        // nothing in plain-text mode) — an accident of the original parser
        // that people used extensively.
        if EXT_IMAGE.is_match(label) {
            result.push_str(&make_external_image(label, "", config));
        } else {
            result.push_str(&make_external_link(url, label, config));
        }
    }
    result.push_str(&text[cur..]);
    result
}

fn make_external_link(url: &str, anchor: &str, config: &ExtractorConfig) -> String {
    if config.keep_links {
        format!("<a href=\"{}\">{}</a>", urlencode(url), anchor)
    } else {
        anchor.to_string()
    }
}

fn make_external_image(url: &str, alt: &str, config: &ExtractorConfig) -> String {
    if config.keep_links {
        format!("<img src=\"{url}\" alt=\"{alt}\">")
    } else {
        alt.to_string()
    }
}

/// Replaces `[[title|…|label]]trail` with the label (or an `<a>` element),
/// concatenated with the trail (e.g. the `s` making a link plural).
pub fn replace_internal_links(text: &str, config: &ExtractorConfig) -> String {
    // called after external-link removal, so no triple closing ]]] worries
    let mut result = String::with_capacity(text.len());
    let mut cur = 0;
    for (s, e) in find_balanced(text, "[[", "]]") {
        let (trail, end) = match TAIL.find(&text[e..]) {
            Some(m) => (m.as_str(), e + m.end()),
            None => ("", e),
        };
        let inner = &text[s + 2..e - 2];
        let (title, label) = match inner.find('|') {
            None => (inner, inner),
            Some(first_pipe) => {
                let title = inner[..first_pipe].trim_end();
                // the label is what follows the last | outside nested links
                let mut pipe = first_pipe;
                let mut curp = first_pipe + 1;
                for (s1, e1) in find_balanced(inner, "[[", "]]") {
                    if s1 > curp
                        && let Some(last) = inner[curp..s1].rfind('|')
                    {
                        pipe = curp + last; // advance
                    }
                    curp = e1;
                }
                (title, inner[pipe + 1..].trim())
            }
        };
        result.push_str(&text[cur..s]);
        result.push_str(&make_internal_link(title, label, config));
        result.push_str(trail);
        cur = end;
    }
    result.push_str(&text[cur..]);
    result
}

fn make_internal_link(title: &str, label: &str, config: &ExtractorConfig) -> String {
    let accepted = |ns: &str| config.namespaces.iter().any(|a| a == ns);
    if let Some(colon) = title.find(':') {
        // links into namespaces other than the accepted ones are dropped
        // entirely, label included (File:, Category:, interwiki, …)
        if colon > 0 && !accepted(&title[..colon]) {
            return String::new();
        }
        if colon == 0 {
            // drop also :File:…
            if let Some(colon2) = title[1..].find(':').map(|i| i + 1)
                && colon2 > 1
                && !accepted(&title[1..colon2])
            {
                return String::new();
            }
        }
    }
    if config.keep_links {
        format!("<a href=\"{}\">{}</a>", urlencode(title), label)
    } else {
        label.to_string()
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    fn config() -> ExtractorConfig {
        ExtractorConfig::default()
    }

    #[test]
    fn external_links_keep_the_anchor() {
        assert_eq!(
            replace_external_links("see [https://example.org/p?a=b the site] end", &config()),
            "see the site end"
        );
        assert_eq!(
            replace_external_links("bare [https://example.org] link", &config()),
            "bare  link"
        );
        assert_eq!(
            replace_external_links("plain http://example.org stays", &config()),
            "plain http://example.org stays"
        );
    }

    #[test]
    fn external_links_as_anchors_when_kept() {
        let config = ExtractorConfig {
            keep_links: true,
            ..config()
        };
        assert_eq!(
            replace_external_links("[https://e.org/a b c]", &config),
            "<a href=\"https%3A//e.org/a\">b c</a>"
        );
    }

    #[test]
    fn internal_links_keep_label_and_trail() {
        assert_eq!(
            replace_internal_links(
                "[[political philosophy]] and [[Movement|movement]]s",
                &config()
            ),
            "political philosophy and movements"
        );
    }

    #[test]
    fn namespace_links_are_dropped_entirely() {
        assert_eq!(
            replace_internal_links("a [[File:X.svg|thumb|caption [[inner]]]] b", &config()),
            "a  b"
        );
        assert_eq!(
            replace_internal_links("a [[Category:Y]] b", &config()),
            "a  b"
        );
        assert_eq!(
            replace_internal_links("a [[:File:X.svg|img]] b", &config()),
            "a  b"
        );
        // w: and wikt: are accepted by default
        assert_eq!(
            replace_internal_links("a [[wikt:anarchy|anarchy]]s b", &config()),
            "a anarchys b"
        );
    }

    #[test]
    fn internal_links_as_anchors_when_kept() {
        let config = ExtractorConfig {
            keep_links: true,
            ..config()
        };
        assert_eq!(
            replace_internal_links("[[Some Page|label]]", &config),
            "<a href=\"Some%20Page\">label</a>"
        );
    }
}