vtpl 0.1.0

Vector-Threaded Posting Lists — fused n-gram + vector search in a single index pass
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59

use std::collections::BTreeSet;

/// Extract character-level n-grams from text.
/// Lowercases and strips non-alphanumeric characters before extraction.
pub fn extract_ngrams(text: &str, n: usize) -> BTreeSet<String> {
    let normalized: String = text
        .to_lowercase()
        .chars()
        .map(|c| if c.is_alphanumeric() { c } else { ' ' })
        .collect();

    let mut grams = BTreeSet::new();
    for word in normalized.split_whitespace() {
        let chars: Vec<char> = word.chars().collect();
        if chars.len() < n {
            grams.insert(word.to_string());
            continue;
        }
        for window in chars.windows(n) {
            grams.insert(window.iter().collect());
        }
    }
    grams
}

#[inline]
pub fn trigrams(text: &str) -> BTreeSet<String> {
    extract_ngrams(text, 3)
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn trigrams_basic() {
        let gs = trigrams("concurrent");
        assert!(gs.contains("con"));
        assert!(gs.contains("onc"));
        assert!(gs.contains("ncu"));
        assert!(gs.contains("ent"));
    }

    #[test]
    fn short_words_kept_whole() {
        let gs = trigrams("go is ok");
        assert!(gs.contains("go"));
        assert!(gs.contains("is"));
        assert!(gs.contains("ok"));
    }

    #[test]
    fn punctuation_stripped() {
        let gs = trigrams("it's a test!");
        assert!(gs.contains("it"));
        assert!(gs.contains("tes"));
        assert!(gs.contains("est"));
    }
}