textprep 0.1.5

Text preprocessing primitives: normalization, tokenization, and fast keyword matching.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56

//! N-gram generation.

/// Generate character n-grams from text.
///
/// ```
/// use textprep::ngram::char_ngrams;
///
/// let trigrams = char_ngrams("hello", 3);
/// assert_eq!(trigrams, vec!["hel", "ell", "llo"]);
/// assert!(char_ngrams("hi", 3).is_empty()); // too short
/// ```
pub fn char_ngrams(text: &str, n: usize) -> Vec<String> {
    let chars: Vec<char> = text.chars().collect();
    if chars.len() < n {
        return Vec::new();
    }
    let mut result = Vec::with_capacity(chars.len() - n + 1);
    for window in chars.windows(n) {
        result.push(window.iter().collect());
    }
    result
}

/// Generate word n-grams by joining adjacent words with spaces.
///
/// ```
/// use textprep::ngram::word_ngrams;
///
/// let bigrams = word_ngrams(&["the", "quick", "fox"], 2);
/// assert_eq!(bigrams, vec!["the quick", "quick fox"]);
/// ```
pub fn word_ngrams(words: &[&str], n: usize) -> Vec<String> {
    if words.len() < n {
        return Vec::new();
    }
    let mut result = Vec::with_capacity(words.len() - n + 1);
    for window in words.windows(n) {
        result.push(window.join(" "));
    }
    result
}

/// Generate token n-grams as vectors of `&str`.
///
/// This is a convenience for callers that want to apply their own hashing/keying
/// scheme instead of joining tokens into a single string.
pub fn token_ngrams<'a>(words: &'a [&'a str], n: usize) -> Vec<Vec<&'a str>> {
    if n == 0 || words.len() < n {
        return Vec::new();
    }
    let mut result = Vec::with_capacity(words.len() - n + 1);
    for window in words.windows(n) {
        result.push(window.to_vec());
    }
    result
}