ruwex 0.1.0

Fast Rust rewrite of wikiextractor: extract and clean text from Wikimedia XML dumps
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90

//! Input layer: reading pages out of MediaWiki XML dumps in their various
//! packagings (plain XML, single-stream bz2, seekable multistream bz2).

pub mod multistream;
pub mod reader;
pub mod xml;

use std::collections::HashMap;

/// One `<page>` from a dump. For multi-revision (history) dumps the last
/// revision wins, matching what pages-articles dumps contain anyway.
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct Page {
    pub id: u64,
    pub revid: u64,
    pub ns: i32,
    pub title: String,
    /// Target title if this page is a redirect.
    pub redirect: Option<String>,
    /// Raw wikitext of the page body.
    pub text: String,
}

/// The dump's `<siteinfo>` header.
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct SiteInfo {
    pub site_name: String,
    /// URL of the main page, e.g. `https://en.wikipedia.org/wiki/Main_Page`.
    pub base: String,
    /// Namespace id → local name; the main namespace (0) has an empty name.
    pub namespaces: HashMap<i32, String>,
}

impl SiteInfo {
    /// `base` with its last path segment removed, wikiextractor's `urlbase`:
    /// `https://en.wikipedia.org/wiki/Main_Page` → `https://en.wikipedia.org/wiki`.
    pub fn url_base(&self) -> &str {
        match self.base.rfind('/') {
            Some(i) => &self.base[..i],
            None => "",
        }
    }

    /// Permanent URL of a page, e.g. `https://en.wikipedia.org/wiki?curid=12`.
    pub fn page_url(&self, id: u64) -> String {
        format!("{}?curid={}", self.url_base(), id)
    }

    /// Resolves a namespace name (case-insensitive) to its id.
    pub fn namespace_id(&self, name: &str) -> Option<i32> {
        let wanted = name.trim().to_lowercase();
        self.namespaces
            .iter()
            .find(|(_, ns_name)| ns_name.to_lowercase() == wanted)
            .map(|(&id, _)| id)
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    fn site() -> SiteInfo {
        SiteInfo {
            site_name: "Wikipedia".to_string(),
            base: "https://en.wikipedia.org/wiki/Main_Page".to_string(),
            namespaces: HashMap::from([
                (0, String::new()),
                (1, "Talk".to_string()),
                (10, "Template".to_string()),
            ]),
        }
    }

    #[test]
    fn page_url_uses_urlbase() {
        assert_eq!(
            site().page_url(12),
            "https://en.wikipedia.org/wiki?curid=12"
        );
    }

    #[test]
    fn namespace_lookup_is_case_insensitive() {
        let site = site();
        assert_eq!(site.namespace_id("talk"), Some(1));
        assert_eq!(site.namespace_id(" Template "), Some(10));
        assert_eq!(site.namespace_id("Bogus"), None);
    }
}