wikiwho 0.3.4

Fast Rust reimplementation of the WikiWho algorithm for fine-grained authorship attribution on large datasets. Optimized for easy integration in multi-threaded applications.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81

#![cfg(feature = "python-diff")]

mod common;

use common::prelude::*;

fn call_split_fn_py(py: Python<'_>, input: &str, fn_name: &str) -> Vec<String> {
    let builtins = py.import("builtins").unwrap();
    let split_fn = py
        .import("WikiWho.utils")
        .unwrap()
        .getattr(fn_name)
        .unwrap();

    let result_iterator = split_fn.call1((input,)).unwrap();
    builtins
        .call_method1("list", (result_iterator,))
        .unwrap()
        .extract::<Vec<String>>()
        .unwrap()
}

proptest! {
    #![proptest_config(ProptestConfig {
        cases: 10000,
        ..ProptestConfig::default()
    })]
    #[test]
    fn compare_split_into_paragraphs_python(input in ".*") {
        with_gil!(py, {
            let result_rust = wikiwho::utils::split_into_paragraphs_naive(&input);
            let result_py = call_split_fn_py(py, &input, "split_into_paragraphs");

            prop_assert_eq!(result_rust, result_py);
        })
    }
}

proptest! {
    #![proptest_config(ProptestConfig {
        cases: 10000,
        ..ProptestConfig::default()
    })]
    #[test]
    fn compare_split_into_sentences_python(input in ".*") {
        with_gil!(py, {
            let result_rust = wikiwho::utils::split_into_sentences_naive(&input);
            let result_py = call_split_fn_py(py, &input, "split_into_sentences");

            prop_assert_eq!(result_rust, result_py);
        })
    }
}

proptest! {
    #![proptest_config(ProptestConfig {
        cases: 10000,
        ..ProptestConfig::default()
    })]
    #[test]
    fn compare_split_into_tokens_python(input in ".*") {
        with_gil!(py, {
            let result_rust = wikiwho::utils::split_into_tokens_naive(&input);
            let result_py = call_split_fn_py(py, &input, "split_into_tokens");

            prop_assert_eq!(result_rust, result_py);
        })
    }
}

// individual test cases found by proptest for closer inspection
#[test]
fn test_case_1() {
    Python::attach(|py| {
        let tokens_rust = wikiwho::utils::split_into_tokens_naive("®\u{2000}");
        let tokens_py = call_split_fn_py(py, "®\u{2000}", "split_into_tokens");

        assert_eq!(tokens_rust, tokens_py);
        assert_eq!(tokens_rust, vec!["®", "\u{2000}"]); // this should be what Python produces
    })
}