ruwex 0.1.0

Fast Rust rewrite of wikiextractor: extract and clean text from Wikimedia XML dumps
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118

//! Entity decoding/encoding matching Python wikiextractor exactly:
//! `unescape()` (HTML 4.01 named entities + numeric references),
//! `html.escape(quote=False)`, and `urllib.parse.quote(safe='/')`.

use std::borrow::Cow;
use std::sync::LazyLock;

use regex::{Captures, Regex};

use super::entities_table::NAME2CODEPOINT;

static ENTITY_RE: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"&#?(\w+);").unwrap());

/// Port of wikiextractor's `unescape()`: decodes `&#nnn;`, `&#xhh;` and the
/// HTML 4.01 named entities; anything unresolvable is left untouched.
pub fn unescape(text: &str) -> String {
    ENTITY_RE
        .replace_all(text, |caps: &Captures| {
            let whole = caps.get(0).expect("match").as_str();
            let code = caps.get(1).expect("group").as_str();
            let decoded = if whole.as_bytes()[1] == b'#' {
                // character reference; Python checks for lowercase 'x' only
                if code.as_bytes().first() == Some(&b'x') {
                    u32::from_str_radix(&code[1..], 16).ok()
                } else {
                    code.parse::<u32>().ok()
                }
            } else {
                NAME2CODEPOINT
                    .binary_search_by_key(&code, |&(name, _)| name)
                    .ok()
                    .map(|i| NAME2CODEPOINT[i].1)
            };
            match decoded.and_then(char::from_u32) {
                Some(c) => c.to_string(),
                None => whole.to_string(), // leave as is
            }
        })
        .into_owned()
}

/// Python `html.escape(text, quote=False)`: `&`, `<`, `>` become entities.
pub fn html_escape(text: &str) -> Cow<'_, str> {
    if !text.contains(['&', '<', '>']) {
        return Cow::Borrowed(text);
    }
    let mut out = String::with_capacity(text.len() + 8);
    for c in text.chars() {
        match c {
            '&' => out.push_str("&amp;"),
            '<' => out.push_str("&lt;"),
            '>' => out.push_str("&gt;"),
            _ => out.push(c),
        }
    }
    Cow::Owned(out)
}

/// Python `urllib.parse.quote(s)` with the default `safe='/'`: everything
/// except unreserved characters and `/` is percent-encoded (UTF-8, uppercase
/// hex).
pub fn urlencode(text: &str) -> String {
    let mut out = String::with_capacity(text.len());
    for byte in text.bytes() {
        match byte {
            b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'_' | b'.' | b'-' | b'~' | b'/' => {
                out.push(byte as char);
            }
            _ => out.push_str(&format!("%{byte:02X}")),
        }
    }
    out
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn entity_table_is_sorted_for_binary_search() {
        assert!(NAME2CODEPOINT.windows(2).all(|w| w[0].0 < w[1].0));
        assert_eq!(NAME2CODEPOINT.len(), 252);
    }

    #[test]
    fn decodes_named_and_numeric_entities() {
        assert_eq!(
            unescape("1&ndash;5 and 100&nbsp;km"),
            "1\u{2013}5 and 100\u{a0}km"
        );
        assert_eq!(unescape("&#65;&#x42;"), "AB");
        assert_eq!(unescape("&amp;lt;"), "&lt;"); // decodes exactly one level
    }

    #[test]
    fn leaves_unknown_entities_untouched() {
        assert_eq!(
            unescape("&bogus; &#xZZ; &#99999999999;"),
            "&bogus; &#xZZ; &#99999999999;"
        );
        // Python checks lowercase 'x' only, so &#X41; stays
        assert_eq!(unescape("&#X41;"), "&#X41;");
    }

    #[test]
    fn escapes_html_without_quotes() {
        assert_eq!(
            html_escape("a & b < c > \"d\""),
            "a &amp; b &lt; c &gt; \"d\""
        );
    }

    #[test]
    fn urlencodes_like_python_quote() {
        assert_eq!(urlencode("a b&c/d~e"), "a%20b%26c/d~e");
        assert_eq!(urlencode("naïve"), "na%C3%AFve");
    }
}