wikipedia_infobox_analyzer/

lib.rs

#[doc = include_str!("../README.md")]
pub mod wikipedia_infobox_analyzer {
    use regex::Regex;
    use itertools::Itertools;

    /// Looks for a used template that does lists wikidata properties listing
    pub fn extract_used_properties_from_template(template: String) -> Vec<u64> {
        let mut lines = template.lines();
        let used = lines
            .find(|line| line.starts_with("{{") && line.contains("Wikidata|"))
            .expect("Template should have a line declaring which properties are used");

        // The properties are listed inside the template with their number: P1|P2|P...
        let re = Regex::new(r"P(\d+)").unwrap();

        // Find all matches and collect the properties into vector
        re.captures_iter(used)
          .map(|m| m[1].parse().unwrap())
          .collect()
    }

    /// Properties will be displayed in the right column if they are in the list for that column
    ///
    /// All items in the left columns will be displayed
    /// The remaining items in the right columns not in the left columns will trail in the right column
    /// ```md
    /// ┌───────┬───────┐
    /// │   1   │   1   │
    /// │   3   │       │
    /// │       │   2   │
    /// │       │   4   │
    /// └───────┴───────┘
    /// ```
    /// So for example, [1, 3] and [1, 2, 4] become [1, 2, 3, 4], as shown above
    pub fn join_columns<T: Copy + Eq>(vec1: &[T], vec2: &[T]) -> Vec<(Option<T>, Option<T>)> {
        vec1.iter()
            .flat_map(|n1| {
                if let Some(&n2) = vec2.iter().find(|&x| x == n1) {
                    std::iter::once((Some(*n1), Some(n2)))
                } else {
                    std::iter::once((Some(*n1), None))
                }
            })
            .chain(vec2.iter()
                   .filter(|&n2| !vec1.contains(n2))
                   .map(|n2| (None, Some(*n2))))
            .collect()
    }

    /// Fetches the wikidata label for the wiki property
    pub async fn fetch_name_for_wiki_property(pid: u64) -> String {
        let query = format!("
        SELECT ?label
        WHERE
        {{
          wd:P{pid} rdfs:label ?label.
          FILTER(lang(?label) = \"en\")
          SERVICE wikibase:label {{ bd:serviceParam wikibase:language 'en'.}}
        }}
        ");

        let api = mediawiki::api::Api::new("https://www.wikidata.org/w/api.php")
            .await
            .expect("Wikidata should return the api endpoint");
        let res = api.sparql_query(&query)
                     .await
                     .expect("Query should be able to retrieve label");
        let res_s = serde_json::to_string_pretty(&res).unwrap();

        let re = Regex::new("\"value\": \"(.+)\"").unwrap();
        let Some(label) = re.captures(&res_s) else { return "<unknown>".to_string() };

        label[1].to_string()
    }

    /// Fetches the properties which have claims for the given wiki item inside of wikidata
    pub fn fetch_properties_for_wiki_item(qid: u64) -> Vec<u64> {
        let mut ids: Vec<u64> = vec![];
        let uri = format!("https://www.wikidata.org/wiki/Special:EntityData/Q{qid}.json");
        let res = reqwest::blocking::get(uri).unwrap();
        let text = res.text().unwrap();
        if text.contains("<h1>Not Found</h1><p>No entity with ID ") {
            return vec![];
        }
        let ent = wikidata::Entity::from_json(serde_json::from_str(&text).unwrap()).unwrap();
        if ent.claims.is_empty() {
            return vec![];
        }

        for i in 0..ent.claims.len() {
            let (wikidata::Pid(pid), _) = &ent.claims[i];
            ids.push(*pid);
        }

        ids.sort();

        ids.iter()
           .map(|n| n.to_owned())
           .unique()
           .collect::<Vec<u64>>()
    }

    /// Fetches the wikidata identifier associated with the wikipedia article
    pub fn fetch_wiki_item_by_article_title(title: String, language_code: String) -> u64 {
        let uri = format!("https://{}.wikipedia.org/w/api.php?action=query&prop=pageprops&format=json&titles={}", &language_code, &title);

        let res = reqwest::blocking::get(uri).unwrap();
        let text = res.text().unwrap();
        if text.contains("-1") {
            return 0;
        }

        let re = Regex::new("\"wikibase_item\":\"Q(\\d+)\"").unwrap();
        let Some(qid) = re.captures(&text) else { return 0 };

        qid[1].parse().expect("Should be a numeric value")
    }
}

#[cfg(test)]
mod tests_lib {
    use super::wikipedia_infobox_analyzer::*;
    use std::fs;

    #[test]
    fn test_extract_used_properties_from_template() {
        let contents = fs::read_to_string("examples/templates/nl_infobox_bedrijf")
            .expect("Template file should exist");

        assert_eq!(
            extract_used_properties_from_template(contents),
            vec![
                18, 154, 1128, 4103, 2139, 2295, 2226, 856, 169,
                1448, 159, 749, 355, 17, 1056, 452, 576, 112, 127,
                856, 2096
            ]
        );

        let contents = fs::read_to_string("examples/templates/nl_infobox_software")
            .expect("Template file should exist");

        assert_eq!(
            extract_used_properties_from_template(contents),
            vec![
                18, 154, 170, 178, 275, 277, 306, 348, 400, 548,
                571, 577, 856, 1324, 2096
            ]
        );
    }

    #[tokio::test]
    async fn test_fetch_name_for_wiki_property() {
        let name = fetch_name_for_wiki_property(31).await;
        assert_eq!(name, "instance of")
    }

    #[test]
    fn test_fetch_properties_for_wiki_item() {
        let qid = 16639197; // identifier for GitLab on wikidata
        let properties = fetch_properties_for_wiki_item(qid);

        // Properties which should be present:
        assert!(properties.contains(&10));  // property: video
        assert!(properties.contains(&18));  // property: image
        assert!(properties.contains(&154)); // property: logo image
        assert!(properties.contains(&178)); // property: developer

        // Properties which should not be present:
        assert!(!properties.contains(&19)); // property: place of birth
        assert!(!properties.contains(&30)); // property: continent
    }

    #[test]
    fn test_fetch_wiki_item_by_article_title() {
        assert_eq!(fetch_wiki_item_by_article_title("Earth".to_string(), "en".to_string()), 2);
        assert_eq!(fetch_wiki_item_by_article_title("Train".to_string(), "en".to_string()), 870);
        assert_eq!(fetch_wiki_item_by_article_title("Baum".to_string(),  "de".to_string()), 10884);
    }
}