ruve-db 0.1.1

A hybrid vector and full-text search database with HNSW approximate nearest-neighbour indexing and BM25
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93

pub struct Tokenizer {
    stop_words: Vec<String>,
}

impl Tokenizer {
    pub fn new() -> Self {
        Tokenizer {
            // todo load from config
            stop_words: vec![
                "the".to_string(),
                "is".to_string(),
                "in".to_string(),
                "and".to_string(),
                "to".to_string(),
                "a".to_string(),
                "of".to_string(),
                "that".to_string(),
                "it".to_string(),
                "with".to_string(),
            ],
        }
    }

    // tokenize whitespace, punctuation, and stop words, and lowercase the tokens
    pub fn tokenize(&self, text: &str) -> Vec<String> {
        text.split(|c: char| c.is_whitespace() || c.is_ascii_punctuation())
            .filter(|token| !token.is_empty())
            .map(|token| token.to_lowercase())
            .filter(|token| !self.stop_words.contains(token))
            .collect()
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn splits_on_whitespace() {
        let t = Tokenizer::new();
        assert_eq!(t.tokenize("hello world"), vec!["hello", "world"]);
    }

    #[test]
    fn splits_on_punctuation() {
        let t = Tokenizer::new();
        let tokens = t.tokenize("hello, world!");
        assert!(tokens.contains(&"hello".to_string()));
        assert!(tokens.contains(&"world".to_string()));
    }

    #[test]
    fn lowercases_tokens() {
        let t = Tokenizer::new();
        assert_eq!(t.tokenize("Hello WORLD"), vec!["hello", "world"]);
    }

    #[test]
    fn filters_stop_words() {
        let t = Tokenizer::new();
        let tokens = t.tokenize("the quick brown fox");
        assert!(!tokens.contains(&"the".to_string()));
        assert!(tokens.contains(&"quick".to_string()));
        assert!(tokens.contains(&"brown".to_string()));
        assert!(tokens.contains(&"fox".to_string()));
    }

    #[test]
    fn stop_words_are_case_insensitive() {
        let t = Tokenizer::new();
        // "The" should be lowercased to "the" then filtered
        assert!(t.tokenize("The dog").iter().all(|tok| tok != "the"));
    }

    #[test]
    fn empty_string_returns_empty() {
        let t = Tokenizer::new();
        assert!(t.tokenize("").is_empty());
    }

    #[test]
    fn only_stop_words_returns_empty() {
        let t = Tokenizer::new();
        assert!(t.tokenize("the and to a").is_empty());
    }

    #[test]
    fn consecutive_delimiters_produce_no_empty_tokens() {
        let t = Tokenizer::new();
        let tokens = t.tokenize("hello   world");
        assert_eq!(tokens, vec!["hello", "world"]);
    }
}