Skip to main content

graphrt_rag/
memory.rs

1//! In-process vector store with a BM25 keyword index.
2//!
3//! Search is brute force: exact cosine similarity over every chunk of the
4//! requested datasets. That is fast enough for tens of thousands of chunks
5//! (one dot product per chunk, auto-vectorized) and keeps the store free of
6//! dependencies.
7//!
8//! Every dataset records the dimension of its vectors; upserting or searching
9//! with a different dimension is an error (re-ingest after changing the
10//! embedder), never a silent empty result.
11//!
12//! With the `fs` feature the store can persist to a single JSON file. Each
13//! change is written while the store's write lock is held, to a unique temporary
14//! file that is flushed to disk and then atomically renamed over the target, so
15//! concurrent writers in one process cannot interleave and a crash leaves either
16//! the old or the new file. Several *processes* writing the same file are not
17//! coordinated; use one writer per file. For bulk loads, turn off
18//! [`MemoryStore::set_autosave`] and call [`MemoryStore::flush`] at the end.
19
20use std::collections::{HashMap, HashSet};
21use std::sync::{Arc, RwLock};
22
23use futures::future::BoxFuture;
24
25use crate::error::RagError;
26use crate::filter::MetadataFilter;
27use crate::types::{Hit, KeywordQuery, StoredChunk, VectorQuery, VectorStore, check_vector};
28
29/// BM25 parameters.
30const BM25_K1: f32 = 1.2;
31const BM25_B: f32 = 0.75;
32
33#[derive(Debug)]
34struct Entry {
35    chunk: StoredChunk,
36    norm: f32,
37    /// Term -> frequency in this chunk.
38    terms: HashMap<String, u32>,
39    length: u32,
40}
41
42#[derive(Debug, Default)]
43struct Dataset {
44    entries: Vec<Entry>,
45    /// Chunk id -> index in `entries`.
46    by_id: HashMap<String, usize>,
47    /// Document id -> chunk ids.
48    by_document: HashMap<String, HashSet<String>>,
49    /// Content hash -> number of chunks with it.
50    hashes: HashMap<String, u32>,
51    /// Term -> number of chunks containing it.
52    doc_freq: HashMap<String, u32>,
53    total_length: u64,
54    /// Dimension shared by every vector of the dataset.
55    dimension: Option<usize>,
56}
57
58impl Dataset {
59    fn insert(&mut self, chunk: StoredChunk) {
60        if let Some(&ix) = self.by_id.get(&chunk.id) {
61            self.remove_at(ix);
62        }
63        let terms = term_frequencies(&chunk.text);
64        let length = terms.values().sum();
65        for term in terms.keys() {
66            *self.doc_freq.entry(term.clone()).or_insert(0) += 1;
67        }
68        self.total_length += u64::from(length);
69        self.dimension = Some(chunk.vector.len());
70        *self.hashes.entry(chunk.content_hash.clone()).or_insert(0) += 1;
71        self.by_document.entry(chunk.document_id.clone()).or_default().insert(chunk.id.clone());
72        let norm = norm(&chunk.vector);
73        self.by_id.insert(chunk.id.clone(), self.entries.len());
74        self.entries.push(Entry { chunk, norm, terms, length });
75    }
76
77    /// Remove the entry at `ix` (swap-remove, fixing up the moved entry's index).
78    fn remove_at(&mut self, ix: usize) {
79        if ix >= self.entries.len() {
80            return;
81        }
82        let removed = self.entries.swap_remove(ix);
83        self.by_id.remove(&removed.chunk.id);
84        if let Some(ids) = self.by_document.get_mut(&removed.chunk.document_id) {
85            ids.remove(&removed.chunk.id);
86            if ids.is_empty() {
87                self.by_document.remove(&removed.chunk.document_id);
88            }
89        }
90        if let Some(count) = self.hashes.get_mut(&removed.chunk.content_hash) {
91            *count = count.saturating_sub(1);
92            if *count == 0 {
93                self.hashes.remove(&removed.chunk.content_hash);
94            }
95        }
96        for term in removed.terms.keys() {
97            if let Some(df) = self.doc_freq.get_mut(term) {
98                *df = df.saturating_sub(1);
99                if *df == 0 {
100                    self.doc_freq.remove(term);
101                }
102            }
103        }
104        self.total_length = self.total_length.saturating_sub(u64::from(removed.length));
105        if let Some(moved) = self.entries.get(ix) {
106            self.by_id.insert(moved.chunk.id.clone(), ix);
107        }
108        if self.entries.is_empty() {
109            self.dimension = None;
110        }
111    }
112
113    fn remove_id(&mut self, id: &str) -> bool {
114        match self.by_id.get(id).copied() {
115            Some(ix) => {
116                self.remove_at(ix);
117                true
118            }
119            None => false,
120        }
121    }
122
123    fn remove_document(&mut self, document_id: &str) -> usize {
124        let ids: Vec<String> =
125            self.by_document.get(document_id).map(|s| s.iter().cloned().collect()).unwrap_or_default();
126        ids.iter().filter(|id| self.remove_id(id)).count()
127    }
128
129    fn check_dimension(&self, dataset_id: &str, len: usize, what: &str) -> Result<(), RagError> {
130        match self.dimension {
131            Some(d) if d != len => Err(RagError::Input(format!(
132                "dataset `{dataset_id}` holds {d}-dimensional vectors but {what} is {len}-dimensional; \
133                 use the original embedder or re-ingest the dataset"
134            ))),
135            _ => Ok(()),
136        }
137    }
138}
139
140type Data = HashMap<String, Dataset>;
141
142/// In-memory [`VectorStore`] with cosine similarity and BM25 keyword search.
143#[derive(Debug, Default, Clone)]
144pub struct MemoryStore {
145    inner: Arc<RwLock<Data>>,
146    #[cfg(feature = "fs")]
147    file: Option<Arc<persistence::FileState>>,
148}
149
150impl MemoryStore {
151    pub fn new() -> Self {
152        Self::default()
153    }
154
155    /// Number of chunks across all datasets (0 if the store is unusable after a panic).
156    pub fn len(&self) -> usize {
157        self.read(|d| d.values().map(|ds| ds.entries.len()).sum()).unwrap_or(0)
158    }
159
160    pub fn is_empty(&self) -> bool {
161        self.len() == 0
162    }
163
164    /// All stored chunks (for export / inspection).
165    pub fn chunks(&self) -> Result<Vec<StoredChunk>, RagError> {
166        self.read(all_chunks)
167    }
168
169    fn read<T>(&self, f: impl FnOnce(&Data) -> T) -> Result<T, RagError> {
170        let guard = self.inner.read().map_err(|_| poisoned())?;
171        Ok(f(&guard))
172    }
173
174    /// Apply a change, then persist it (with the `fs` feature) while the write lock
175    /// is still held, so writes and file updates happen in the same order.
176    fn write<T>(&self, f: impl FnOnce(&mut Data) -> Result<T, RagError>) -> Result<T, RagError> {
177        let mut guard = self.inner.write().map_err(|_| poisoned())?;
178        let result = f(&mut guard)?;
179        self.persist_locked(&guard)?;
180        Ok(result)
181    }
182
183    /// Insert or replace chunks. The whole batch is validated first, so an error
184    /// leaves the store unchanged.
185    pub(crate) fn insert_all(&self, chunks: Vec<StoredChunk>) -> Result<(), RagError> {
186        self.write(|data| insert_batch(data, chunks))
187    }
188
189    /// Replace the whole content with `chunks` (validated first; unchanged on error).
190    #[cfg_attr(not(feature = "sqlite"), allow(dead_code))]
191    pub(crate) fn replace_all(&self, chunks: Vec<StoredChunk>) -> Result<(), RagError> {
192        self.write(|data| {
193            let mut fresh = Data::new();
194            insert_batch(&mut fresh, chunks)?;
195            *data = fresh;
196            Ok(())
197        })
198    }
199
200    pub(crate) fn remove_document(&self, dataset_id: &str, document_id: &str) -> Result<usize, RagError> {
201        self.write(|data| Ok(data.get_mut(dataset_id).map_or(0, |ds| ds.remove_document(document_id))))
202    }
203
204    pub(crate) fn remove_dataset(&self, dataset_id: &str) -> Result<usize, RagError> {
205        self.write(|data| Ok(data.remove(dataset_id).map_or(0, |ds| ds.entries.len())))
206    }
207
208    pub(crate) fn remove_chunks(&self, dataset_id: &str, ids: &[String]) -> Result<usize, RagError> {
209        self.write(|data| Ok(data.get_mut(dataset_id).map_or(0, |ds| ids.iter().filter(|id| ds.remove_id(id)).count())))
210    }
211
212    pub(crate) fn find_document_chunks(
213        &self,
214        dataset_id: &str,
215        document_id: &str,
216    ) -> Result<Vec<StoredChunk>, RagError> {
217        self.read(|data| {
218            let Some(ds) = data.get(dataset_id) else { return Vec::new() };
219            let Some(ids) = ds.by_document.get(document_id) else { return Vec::new() };
220            ids.iter()
221                .filter_map(|id| ds.by_id.get(id).and_then(|&ix| ds.entries.get(ix)))
222                .map(|e| e.chunk.clone())
223                .collect()
224        })
225    }
226
227    /// Exact cosine search over every chunk of the requested datasets.
228    pub(crate) fn vector_search(&self, query: &VectorQuery<'_>) -> Result<Vec<Hit>, RagError> {
229        check_vector(query.vector, "the query")?;
230        self.read(|data| {
231            let q_norm = norm(query.vector);
232            let mut scored: Vec<(f32, &Entry)> = Vec::new();
233            for (id, ds) in query.datasets.iter().filter_map(|id| data.get(id).map(|ds| (id, ds))) {
234                ds.check_dimension(id, query.vector.len(), "the query")?;
235                for entry in &ds.entries {
236                    if !passes(query.filter, &entry.chunk) {
237                        continue;
238                    }
239                    let denom = q_norm * entry.norm;
240                    let score = if denom > 0.0 { dot(query.vector, &entry.chunk.vector) / denom } else { 0.0 };
241                    scored.push((score, entry));
242                }
243            }
244            Ok(top_hits(scored, query.top_k))
245        })?
246    }
247
248    /// BM25 search with per-dataset statistics.
249    pub(crate) fn bm25_search(&self, query: &KeywordQuery<'_>) -> Result<Vec<Hit>, RagError> {
250        let terms: Vec<String> = tokenize(query.text);
251        self.read(|data| {
252            let mut scored: Vec<(f32, &Entry)> = Vec::new();
253            for ds in query.datasets.iter().filter_map(|id| data.get(id)) {
254                let n = ds.entries.len();
255                if n == 0 {
256                    continue;
257                }
258                #[allow(clippy::cast_precision_loss)]
259                let (n_f, avgdl) = (n as f32, (ds.total_length as f32 / n as f32).max(1.0));
260                for entry in &ds.entries {
261                    if !passes(query.filter, &entry.chunk) {
262                        continue;
263                    }
264                    let mut score = 0.0f32;
265                    #[allow(clippy::cast_precision_loss)]
266                    let dl = entry.length as f32;
267                    for term in &terms {
268                        let Some(&tf) = entry.terms.get(term) else { continue };
269                        #[allow(clippy::cast_precision_loss)]
270                        let df = ds.doc_freq.get(term).copied().unwrap_or(0) as f32;
271                        let idf = ((n_f - df + 0.5) / (df + 0.5) + 1.0).ln();
272                        #[allow(clippy::cast_precision_loss)]
273                        let tf = tf as f32;
274                        score += idf * (tf * (BM25_K1 + 1.0)) / (tf + BM25_K1 * (1.0 - BM25_B + BM25_B * dl / avgdl));
275                    }
276                    if score > 0.0 {
277                        scored.push((score, entry));
278                    }
279                }
280            }
281            top_hits(scored, query.top_k)
282        })
283    }
284
285    pub(crate) fn has_hash(&self, dataset_id: &str, content_hash: &str) -> Result<bool, RagError> {
286        self.read(|data| data.get(dataset_id).is_some_and(|ds| ds.hashes.contains_key(content_hash)))
287    }
288
289    #[cfg(not(feature = "fs"))]
290    fn persist_locked(&self, _data: &Data) -> Result<(), RagError> {
291        Ok(())
292    }
293}
294
295/// Validate a batch (finite vectors, one dimension per dataset), then insert it.
296fn insert_batch(data: &mut Data, chunks: Vec<StoredChunk>) -> Result<(), RagError> {
297    let mut batch_dims: HashMap<&str, usize> = HashMap::new();
298    for chunk in &chunks {
299        check_vector(&chunk.vector, &format!("chunk `{}`", chunk.id))?;
300        let len = chunk.vector.len();
301        if let Some(ds) = data.get(&chunk.dataset_id) {
302            ds.check_dimension(&chunk.dataset_id, len, &format!("chunk `{}`", chunk.id))?;
303        }
304        match batch_dims.get(chunk.dataset_id.as_str()) {
305            Some(&d) if d != len => {
306                return Err(RagError::Input(format!(
307                    "chunks for dataset `{}` mix {d}- and {len}-dimensional vectors",
308                    chunk.dataset_id
309                )));
310            }
311            _ => {
312                batch_dims.insert(&chunk.dataset_id, len);
313            }
314        }
315    }
316    for chunk in chunks {
317        data.entry(chunk.dataset_id.clone()).or_default().insert(chunk);
318    }
319    Ok(())
320}
321
322fn poisoned() -> RagError {
323    RagError::Store("memory store lock poisoned by an earlier panic".into())
324}
325
326fn all_chunks(data: &Data) -> Vec<StoredChunk> {
327    data.values().flat_map(|ds| ds.entries.iter().map(|e| e.chunk.clone())).collect()
328}
329
330#[cfg(feature = "fs")]
331mod persistence {
332    use super::*;
333    use std::io::Write;
334    use std::path::{Path, PathBuf};
335    use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
336
337    #[derive(serde::Serialize, serde::Deserialize)]
338    struct FileFormat {
339        version: u32,
340        chunks: Vec<StoredChunk>,
341    }
342
343    /// Where and how the store persists itself.
344    #[derive(Debug)]
345    pub(super) struct FileState {
346        path: PathBuf,
347        autosave: AtomicBool,
348        /// Changes not yet written (autosave off, or a failed write).
349        dirty: AtomicBool,
350    }
351
352    static TEMP_COUNTER: AtomicU64 = AtomicU64::new(0);
353
354    impl MemoryStore {
355        /// Open a store backed by `path`: existing content is loaded, and every
356        /// change is written back (see the module docs for the guarantees).
357        pub fn open(path: impl Into<PathBuf>) -> Result<Self, RagError> {
358            let path = path.into();
359            let store = Self { inner: Arc::default(), file: None };
360            if path.exists() {
361                let text =
362                    std::fs::read_to_string(&path).map_err(|e| RagError::Io(format!("{}: {e}", path.display())))?;
363                let file: FileFormat =
364                    serde_json::from_str(&text).map_err(|e| RagError::Decode(format!("{}: {e}", path.display())))?;
365                if file.version != 1 {
366                    return Err(RagError::Decode(format!("unsupported store file version {}", file.version)));
367                }
368                store.insert_all(file.chunks)?;
369            }
370            let file = FileState { path, autosave: AtomicBool::new(true), dirty: AtomicBool::new(false) };
371            Ok(Self { file: Some(Arc::new(file)), ..store })
372        }
373
374        /// Write after every change (default `true`). With autosave off, changes stay
375        /// in memory until [`MemoryStore::flush`].
376        pub fn set_autosave(&self, autosave: bool) {
377            if let Some(file) = &self.file {
378                file.autosave.store(autosave, Ordering::SeqCst);
379            }
380        }
381
382        /// Write pending changes to the backing file (no-op for in-memory stores).
383        pub fn flush(&self) -> Result<(), RagError> {
384            let guard = self.inner.read().map_err(|_| poisoned())?;
385            match &self.file {
386                Some(file) if file.dirty.load(Ordering::SeqCst) => write_file(file, &guard),
387                _ => Ok(()),
388            }
389        }
390
391        /// Write the whole store to `path` (temp file + fsync + rename).
392        pub fn save_to(&self, path: &Path) -> Result<(), RagError> {
393            let guard = self.inner.read().map_err(|_| poisoned())?;
394            write_atomically(path, &all_chunks(&guard))
395        }
396
397        pub(super) fn persist_locked(&self, data: &Data) -> Result<(), RagError> {
398            let Some(file) = &self.file else { return Ok(()) };
399            file.dirty.store(true, Ordering::SeqCst);
400            if file.autosave.load(Ordering::SeqCst) { write_file(file, data) } else { Ok(()) }
401        }
402    }
403
404    fn write_file(file: &FileState, data: &Data) -> Result<(), RagError> {
405        write_atomically(&file.path, &all_chunks(data))?;
406        file.dirty.store(false, Ordering::SeqCst);
407        Ok(())
408    }
409
410    fn write_atomically(path: &Path, chunks: &[StoredChunk]) -> Result<(), RagError> {
411        #[derive(serde::Serialize)]
412        struct FileRef<'a> {
413            version: u32,
414            chunks: &'a [StoredChunk],
415        }
416        let text = serde_json::to_vec(&FileRef { version: 1, chunks })
417            .map_err(|e| RagError::Store(format!("cannot encode store: {e}")))?;
418        let n = TEMP_COUNTER.fetch_add(1, Ordering::SeqCst);
419        let name = path.file_name().map(|s| s.to_string_lossy().into_owned()).unwrap_or_else(|| "store".into());
420        let tmp = path.with_file_name(format!(".{name}.{}.{n}.tmp", std::process::id()));
421        let io = |p: &Path, e: std::io::Error| RagError::Io(format!("{}: {e}", p.display()));
422        let result = (|| {
423            let mut f = std::fs::File::create(&tmp).map_err(|e| io(&tmp, e))?;
424            f.write_all(&text).map_err(|e| io(&tmp, e))?;
425            f.sync_all().map_err(|e| io(&tmp, e))?;
426            std::fs::rename(&tmp, path).map_err(|e| io(path, e))?;
427            sync_parent_dir(path);
428            Ok(())
429        })();
430        if result.is_err() {
431            let _cleanup = std::fs::remove_file(&tmp);
432        }
433        result
434    }
435
436    /// Make the rename durable. Directories cannot be opened for syncing on every
437    /// platform; failure only weakens crash durability, so it is ignored.
438    fn sync_parent_dir(path: &Path) {
439        #[cfg(unix)]
440        if let Some(dir) = path.parent().filter(|p| !p.as_os_str().is_empty()) {
441            if let Ok(handle) = std::fs::File::open(dir) {
442                let _best_effort = handle.sync_all();
443            }
444        }
445        #[cfg(not(unix))]
446        let _ = path;
447    }
448}
449
450impl VectorStore for MemoryStore {
451    fn upsert(&self, chunks: Vec<StoredChunk>) -> BoxFuture<'_, Result<(), RagError>> {
452        Box::pin(futures::future::ready(self.insert_all(chunks)))
453    }
454
455    fn delete_document<'a>(
456        &'a self,
457        dataset_id: &'a str,
458        document_id: &'a str,
459    ) -> BoxFuture<'a, Result<usize, RagError>> {
460        Box::pin(futures::future::ready(self.remove_document(dataset_id, document_id)))
461    }
462
463    fn delete_dataset<'a>(&'a self, dataset_id: &'a str) -> BoxFuture<'a, Result<usize, RagError>> {
464        Box::pin(futures::future::ready(self.remove_dataset(dataset_id)))
465    }
466
467    fn document_chunks<'a>(
468        &'a self,
469        dataset_id: &'a str,
470        document_id: &'a str,
471    ) -> BoxFuture<'a, Result<Vec<StoredChunk>, RagError>> {
472        Box::pin(futures::future::ready(self.find_document_chunks(dataset_id, document_id)))
473    }
474
475    fn delete_chunks<'a>(&'a self, dataset_id: &'a str, ids: &'a [String]) -> BoxFuture<'a, Result<usize, RagError>> {
476        Box::pin(futures::future::ready(self.remove_chunks(dataset_id, ids)))
477    }
478
479    fn search<'a>(&'a self, query: VectorQuery<'a>) -> BoxFuture<'a, Result<Vec<Hit>, RagError>> {
480        Box::pin(futures::future::ready(self.vector_search(&query)))
481    }
482
483    fn supports_keyword(&self) -> bool {
484        true
485    }
486
487    fn keyword_search<'a>(&'a self, query: KeywordQuery<'a>) -> BoxFuture<'a, Result<Vec<Hit>, RagError>> {
488        Box::pin(futures::future::ready(self.bm25_search(&query)))
489    }
490
491    fn contains_hash<'a>(
492        &'a self,
493        dataset_id: &'a str,
494        content_hash: &'a str,
495    ) -> BoxFuture<'a, Result<bool, RagError>> {
496        Box::pin(futures::future::ready(self.has_hash(dataset_id, content_hash)))
497    }
498}
499
500fn passes(filter: Option<&MetadataFilter>, chunk: &StoredChunk) -> bool {
501    filter.is_none_or(|f| f.matches(&chunk.metadata))
502}
503
504fn top_hits(mut scored: Vec<(f32, &Entry)>, top_k: usize) -> Vec<Hit> {
505    scored.retain(|(score, _)| score.is_finite());
506    scored.sort_by(|a, b| b.0.total_cmp(&a.0));
507    scored.into_iter().take(top_k).map(|(score, e)| Hit::from_chunk(&e.chunk, score)).collect()
508}
509
510/// Dot product written as 8 independent accumulators so the compiler can vectorize it.
511pub fn dot(a: &[f32], b: &[f32]) -> f32 {
512    let mut acc = [0.0f32; 8];
513    let chunks_a = a.chunks_exact(8);
514    let chunks_b = b.chunks_exact(8);
515    let (rest_a, rest_b) = (chunks_a.remainder(), chunks_b.remainder());
516    for (x, y) in chunks_a.zip(chunks_b) {
517        for ((slot, xi), yi) in acc.iter_mut().zip(x).zip(y) {
518            *slot += xi * yi;
519        }
520    }
521    let tail: f32 = rest_a.iter().zip(rest_b).map(|(x, y)| x * y).sum();
522    acc.iter().sum::<f32>() + tail
523}
524
525/// Euclidean norm.
526pub fn norm(v: &[f32]) -> f32 {
527    dot(v, v).sqrt()
528}
529
530/// Cosine similarity (0 when either vector is zero or lengths differ).
531pub fn cosine(a: &[f32], b: &[f32]) -> f32 {
532    if a.len() != b.len() {
533        return 0.0;
534    }
535    let denom = norm(a) * norm(b);
536    if denom > 0.0 { dot(a, b) / denom } else { 0.0 }
537}
538
539/// Lower-cased alphanumeric tokens. CJK characters become one token each so
540/// languages without spaces still get a usable keyword index.
541pub fn tokenize(text: &str) -> Vec<String> {
542    let mut tokens = Vec::new();
543    let mut current = String::new();
544    for c in text.chars() {
545        if is_cjk(c) {
546            if !current.is_empty() {
547                tokens.push(std::mem::take(&mut current));
548            }
549            tokens.push(c.to_string());
550        } else if c.is_alphanumeric() {
551            current.extend(c.to_lowercase());
552        } else if !current.is_empty() {
553            tokens.push(std::mem::take(&mut current));
554        }
555    }
556    if !current.is_empty() {
557        tokens.push(current);
558    }
559    tokens
560}
561
562fn is_cjk(c: char) -> bool {
563    matches!(u32::from(c), 0x3040..=0x30FF | 0x3400..=0x4DBF | 0x4E00..=0x9FFF | 0xAC00..=0xD7AF | 0xF900..=0xFAFF)
564}
565
566fn term_frequencies(text: &str) -> HashMap<String, u32> {
567    let mut map = HashMap::new();
568    for t in tokenize(text) {
569        *map.entry(t).or_insert(0) += 1;
570    }
571    map
572}
573
574#[cfg(test)]
575mod tests {
576    use super::*;
577
578    fn chunk(id: &str, ds: &str, text: &str, vector: Vec<f32>) -> StoredChunk {
579        StoredChunk {
580            id: id.into(),
581            dataset_id: ds.into(),
582            document_id: format!("doc-{id}"),
583            title: String::new(),
584            text: text.into(),
585            metadata: serde_json::Map::new(),
586            content_hash: crate::types::content_hash(ds, text),
587            vector,
588        }
589    }
590
591    fn block_on<F: std::future::Future>(f: F) -> F::Output {
592        futures::executor::block_on(f)
593    }
594
595    #[test]
596    fn cosine_ranking_and_dataset_scoping() {
597        let store = MemoryStore::new();
598        block_on(store.upsert(vec![
599            chunk("a", "d1", "a", vec![1.0, 0.0, 0.0]),
600            chunk("b", "d1", "b", vec![0.7, 0.7, 0.0]),
601            chunk("c", "d2", "c", vec![1.0, 0.0, 0.0]),
602        ]))
603        .unwrap();
604        let ds = vec!["d1".to_owned()];
605        let hits =
606            block_on(store.search(VectorQuery { datasets: &ds, vector: &[1.0, 0.1, 0.0], top_k: 5, filter: None }))
607                .unwrap();
608        assert_eq!(hits.iter().map(|h| h.id.as_str()).collect::<Vec<_>>(), vec!["a", "b"]);
609        assert!(hits[0].score > hits[1].score);
610    }
611
612    #[test]
613    fn bm25_prefers_rare_terms() {
614        let store = MemoryStore::new();
615        block_on(store.upsert(vec![
616            chunk("1", "d", "the cat sat on the mat", vec![1.0]),
617            chunk("2", "d", "the dog chased the cat", vec![1.0]),
618            chunk("3", "d", "quantum entanglement explained", vec![1.0]),
619        ]))
620        .unwrap();
621        let ds = vec!["d".to_owned()];
622        let hits = block_on(store.keyword_search(KeywordQuery { datasets: &ds, text: "dog", top_k: 5, filter: None }))
623            .unwrap();
624        assert_eq!(hits.len(), 1);
625        assert_eq!(hits[0].id, "2");
626        let hits =
627            block_on(store.keyword_search(KeywordQuery { datasets: &ds, text: "cat quantum", top_k: 5, filter: None }))
628                .unwrap();
629        assert_eq!(hits[0].id, "3", "rare term outweighs common one");
630    }
631
632    #[test]
633    fn upsert_replaces_and_delete_updates_index() {
634        let store = MemoryStore::new();
635        block_on(store.upsert(vec![chunk("x", "d", "apple banana", vec![1.0]), chunk("y", "d", "apple", vec![1.0])]))
636            .unwrap();
637        block_on(store.upsert(vec![chunk("x", "d", "cherry", vec![1.0])])).unwrap();
638        assert_eq!(store.len(), 2);
639        let ds = vec!["d".to_owned()];
640        let hits =
641            block_on(store.keyword_search(KeywordQuery { datasets: &ds, text: "banana", top_k: 5, filter: None }))
642                .unwrap();
643        assert!(hits.is_empty());
644        assert_eq!(block_on(store.delete_document("d", "doc-y")).unwrap(), 1);
645        assert_eq!(store.len(), 1);
646        assert_eq!(block_on(store.delete_dataset("d")).unwrap(), 1);
647        assert!(store.is_empty());
648    }
649
650    #[test]
651    fn dot_matches_naive() {
652        let a: Vec<f32> = (0..37).map(|i| i as f32 * 0.5).collect();
653        let b: Vec<f32> = (0..37).map(|i| (37 - i) as f32).collect();
654        let naive: f32 = a.iter().zip(&b).map(|(x, y)| x * y).sum();
655        assert!((dot(&a, &b) - naive).abs() < 1e-3);
656        assert!((cosine(&[1.0, 0.0], &[0.0, 1.0])).abs() < 1e-6);
657    }
658
659    #[test]
660    fn tokenizer_handles_cjk() {
661        assert_eq!(tokenize("Hello, World! 日本"), vec!["hello", "world", "日", "本"]);
662    }
663
664    #[cfg(feature = "fs")]
665    #[test]
666    fn persistence_roundtrip() {
667        let dir = std::env::temp_dir().join(format!("graphrt-rag-test-{}", std::process::id()));
668        std::fs::create_dir_all(&dir).unwrap();
669        let path = dir.join("kb.json");
670        let _ = std::fs::remove_file(&path);
671        {
672            let store = MemoryStore::open(&path).unwrap();
673            block_on(store.upsert(vec![chunk("a", "d", "persist me", vec![0.5, 0.5])])).unwrap();
674        }
675        let reopened = MemoryStore::open(&path).unwrap();
676        assert_eq!(reopened.len(), 1);
677        let ds = vec!["d".to_owned()];
678        let hits =
679            block_on(reopened.keyword_search(KeywordQuery { datasets: &ds, text: "persist", top_k: 1, filter: None }))
680                .unwrap();
681        assert_eq!(hits[0].content, "persist me");
682        block_on(reopened.delete_dataset("d")).unwrap();
683        assert!(MemoryStore::open(&path).unwrap().is_empty());
684        let _ = std::fs::remove_dir_all(&dir);
685    }
686}