ruwex 0.1.0

Fast Rust rewrite of wikiextractor: extract and clean text from Wikimedia XML dumps
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142

//! Rendering an extracted page into its output representation, byte-matching
//! wikiextractor: `<doc>` blocks repeat the title as the first text line and
//! end with a blank line; JSON lines carry ids as strings and use Python's
//! `json.dumps` defaults (`", "` / `": "` separators, `ensure_ascii`).

use crate::clean;
use crate::config::{ExtractorConfig, OutputFormat};
use crate::dump::{Page, SiteInfo};
use crate::expand::TemplateSource;

/// Renders one page as a complete output document, trailing newline included.
pub fn render_page(
    page: &Page,
    site: &SiteInfo,
    config: &ExtractorConfig,
    templates: &dyn TemplateSource,
) -> String {
    let paragraphs = clean::extract_paragraphs(page, config, templates);
    let url = site.page_url(page.id);
    let text = paragraphs.join("\n");
    match config.format {
        OutputFormat::Doc => format!(
            "<doc id=\"{}\" url=\"{}\" title=\"{}\">\n{}\n\n{}\n\n</doc>\n",
            page.id, url, page.title, page.title, text
        ),
        OutputFormat::Json => format!(
            "{{\"id\": {}, \"revid\": {}, \"url\": {}, \"title\": {}, \"text\": {}}}\n",
            json_string(&page.id.to_string()),
            json_string(&page.revid.to_string()),
            json_string(&url),
            json_string(&page.title),
            json_string(&text),
        ),
    }
}

/// Encodes a string like Python's `json.dumps` with `ensure_ascii=True`:
/// short escapes for the usual control characters, `\uXXXX` (lowercase hex,
/// surrogate pairs for astral characters) for everything non-ASCII.
fn json_string(s: &str) -> String {
    let mut out = String::with_capacity(s.len() + 2);
    out.push('"');
    for c in s.chars() {
        match c {
            '"' => out.push_str("\\\""),
            '\\' => out.push_str("\\\\"),
            '\n' => out.push_str("\\n"),
            '\r' => out.push_str("\\r"),
            '\t' => out.push_str("\\t"),
            '\u{8}' => out.push_str("\\b"),
            '\u{c}' => out.push_str("\\f"),
            c if (c as u32) < 0x20 => {
                out.push_str(&format!("\\u{:04x}", c as u32));
            }
            c if (c as u32) < 0x7f => out.push(c),
            c => {
                let cp = c as u32;
                if cp > 0xFFFF {
                    // surrogate pair
                    let v = cp - 0x10000;
                    out.push_str(&format!(
                        "\\u{:04x}\\u{:04x}",
                        0xD800 + (v >> 10),
                        0xDC00 + (v & 0x3FF)
                    ));
                } else {
                    out.push_str(&format!("\\u{cp:04x}"));
                }
            }
        }
    }
    out.push('"');
    out
}

#[cfg(test)]
mod tests {
    use std::collections::HashMap;

    use crate::expand::TemplateDb;

    use super::*;

    fn fixtures() -> (Page, SiteInfo) {
        let page = Page {
            id: 12,
            revid: 34,
            ns: 0,
            title: "Anarchism".to_string(),
            redirect: None,
            text: "'''Anarchism''' is a [[political philosophy]].".to_string(),
        };
        let site = SiteInfo {
            site_name: "Wikipedia".to_string(),
            base: "https://en.wikipedia.org/wiki/Main_Page".to_string(),
            namespaces: HashMap::new(),
        };
        (page, site)
    }

    #[test]
    fn doc_format_matches_wikiextractor() {
        let (page, site) = fixtures();
        let doc = render_page(
            &page,
            &site,
            &ExtractorConfig::default(),
            &TemplateDb::default(),
        );
        assert_eq!(
            doc,
            "<doc id=\"12\" url=\"https://en.wikipedia.org/wiki?curid=12\" title=\"Anarchism\">\n\
             Anarchism\n\n\
             Anarchism is a political philosophy.\n\n\
             </doc>\n"
        );
    }

    #[test]
    fn json_format_matches_python_dumps() {
        let (mut page, site) = fixtures();
        page.text = "Beta follows Alpha \u{2014} naturally.".to_string();
        let config = ExtractorConfig {
            format: OutputFormat::Json,
            ..Default::default()
        };
        let line = render_page(&page, &site, &config, &TemplateDb::default());
        assert_eq!(
            line,
            "{\"id\": \"12\", \"revid\": \"34\", \"url\": \"https://en.wikipedia.org/wiki?curid=12\", \
             \"title\": \"Anarchism\", \"text\": \"Beta follows Alpha \\u2014 naturally.\"}\n"
        );
    }

    #[test]
    fn json_string_escapes_like_python() {
        assert_eq!(json_string("a\"b\\c\nd"), "\"a\\\"b\\\\c\\nd\"");
        assert_eq!(json_string("caf\u{e9}"), "\"caf\\u00e9\"");
        assert_eq!(json_string("\u{1F600}"), "\"\\ud83d\\ude00\""); // astral → pair
        assert_eq!(json_string("\u{7f}"), "\"\\u007f\"");
    }
}