khive-text 0.10.0

Text analysis primitives for khive: tokenization, normalization, filtering
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56

//! Behavior tests for the `stem` feature-gated SnowballStemmer.

#![cfg(feature = "stem")]

use khive_text::filter::SnowballStemmer;
use khive_text::TokenFilter;

#[test]
fn english_stemmer_reduces_common_suffixes() {
    let s = SnowballStemmer::english();
    assert_eq!(s.apply("running".to_string()), Some("run".to_string()));
    assert_eq!(
        s.apply("connections".to_string()),
        Some("connect".to_string())
    );
    assert_eq!(s.apply("easily".to_string()), Some("easili".to_string()));
}

#[test]
fn stemmer_passes_through_non_ascii_tokens() {
    let s = SnowballStemmer::english();
    // Non-ASCII tokens are returned unchanged
    assert_eq!(
        s.apply("caf\u{00e9}".to_string()),
        Some("caf\u{00e9}".to_string())
    );
    assert_eq!(
        s.apply("\u{4f60}\u{597d}".to_string()),
        Some("\u{4f60}\u{597d}".to_string())
    );
}

#[test]
fn stemmer_passes_through_mixed_ascii_nonalpha() {
    let s = SnowballStemmer::english();
    // Tokens with digits or punctuation are not purely ASCII-alphabetic
    assert_eq!(s.apply("v2".to_string()), Some("v2".to_string()));
    assert_eq!(
        s.apply("fine-tuning".to_string()),
        Some("fine-tuning".to_string())
    );
}

#[test]
fn stemmer_handles_already_stemmed() {
    let s = SnowballStemmer::english();
    // "run" is already a stem
    assert_eq!(s.apply("run".to_string()), Some("run".to_string()));
}

#[test]
fn stemmer_handles_empty_token() {
    let s = SnowballStemmer::english();
    // Empty string has no ASCII-alphabetic chars, passes through
    assert_eq!(s.apply(String::new()), Some(String::new()));
}