reefdb 0.1.0

ReefDB is a minimalistic, in-memory and on-disk database management system written in Rust, implementing basic SQL query capabilities and full-text search.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143

use std::collections::HashMap;
use std::collections::HashSet;

use serde::Deserialize;
use serde::Serialize;

use super::search::Search;
use super::tokenizers::tokenizer::Tokenizer;

#[derive(Debug, Serialize, Deserialize, Clone)]
pub struct InvertedIndex<T: Tokenizer> {
    index: HashMap<String, HashMap<String, HashMap<String, HashSet<usize>>>>,
    tokenizer: T,
}

impl<T: Tokenizer> InvertedIndex<T> {
    pub fn new() -> Self {
        InvertedIndex {
            index: HashMap::new(),
            tokenizer: T::new(),
        }
    }

    pub fn update_document(&mut self, table: &str, column: &str, row_id: usize, text: &str) {
        self.remove_document(table, column, row_id);
        self.add_document(table, column, row_id, text);
    }

    pub fn add_column(&mut self, table: &str, column: &str) {
        let table_entry = self
            .index
            .entry(table.to_string())
            .or_insert(HashMap::new());
        table_entry.insert(column.to_string(), HashMap::new());
    }

    pub fn add_document(&mut self, table: &str, column: &str, row_id: usize, text: &str) {
        let table_entry = self
            .index
            .entry(table.to_string())
            .or_insert(HashMap::new());
        let column_entry = table_entry
            .entry(column.to_string())
            .or_insert(HashMap::new());

        for word in self.tokenizer.tokenize(text) {
            let word_entry = column_entry
                .entry(word.to_string())
                .or_insert(HashSet::new());
            word_entry.insert(row_id);
        }
    }

    pub fn remove_document(&mut self, table: &str, column: &str, row_id: usize) {
        if let Some(table_entry) = self.index.get_mut(table) {
            if let Some(column_entry) = table_entry.get_mut(column) {
                for value in column_entry.values_mut() {
                    value.remove(&row_id);
                }
            }
        }
    }

    pub fn search(&self, table: &str, column: &str, query: &str) -> HashSet<usize> {
        let mut results = HashSet::new();
        if let Some(table_entry) = self.index.get(table) {
            if let Some(column_entry) = table_entry.get(column) {
                for word in self.tokenizer.tokenize(query) {
                    if let Some(word_entry) = column_entry.get(word) {
                        results.extend(word_entry);
                    }
                }
            }
        }
        results
    }
}

impl<T: Tokenizer + Serialize + for<'de> Deserialize<'de>> Search for InvertedIndex<T> {
    type NewArgs = ();
    fn new(_: Self::NewArgs) -> Self {
        InvertedIndex::new()
    }

    fn add_column(&mut self, table: &str, column: &str) {
        InvertedIndex::add_column(self, table, column)
    }

    fn search(&self, table: &str, column: &str, query: &str) -> HashSet<usize> {
        InvertedIndex::search(self, table, column, query)
    }

    fn add_document(&mut self, table: &str, column: &str, row_id: usize, text: &str) {
        InvertedIndex::add_document(self, table, column, row_id, text)
    }

    fn remove_document(&mut self, table: &str, column: &str, row_id: usize) {
        InvertedIndex::remove_document(self, table, column, row_id)
    }

    fn update_document(&mut self, table: &str, column: &str, row_id: usize, text: &str) {
        InvertedIndex::update_document(self, table, column, row_id, text)
    }
}

#[cfg(test)]
mod tests {
    use std::collections::HashSet;

    use crate::indexes::fts::{memory::InvertedIndex, tokenizers::default::DefaultTokenizer};

    #[test]
    fn test_inverted_index() {
        let mut index: InvertedIndex<DefaultTokenizer> = InvertedIndex::new();

        // Add documents
        index.add_document("table1", "column1", 0, "hello world");
        index.add_document("table1", "column1", 1, "goodbye world");
        index.add_document("table1", "column2", 0, "rust programming");
        index.add_document("table2", "column1", 0, "world peace");

        // Search
        let results = index.search("table1", "column1", "world");
        let expected: HashSet<usize> = [0, 1].iter().cloned().collect();
        assert_eq!(results, expected);

        let results = index.search("table1", "column2", "rust");
        let expected: HashSet<usize> = [0].iter().cloned().collect();
        assert_eq!(results, expected);

        let results = index.search("table2", "column1", "world");
        let expected: HashSet<usize> = [0].iter().cloned().collect();
        assert_eq!(results, expected);

        // Remove document
        index.remove_document("table1", "column1", 0);

        // Search after removing document
        let results = index.search("table1", "column1", "world");
        let expected: HashSet<usize> = [1].iter().cloned().collect();
        assert_eq!(results, expected);
    }
}