1use std::collections::{HashMap, HashSet};
21use std::sync::{Arc, RwLock};
22
23use futures::future::BoxFuture;
24
25use crate::error::RagError;
26use crate::filter::MetadataFilter;
27use crate::types::{Hit, KeywordQuery, StoredChunk, VectorQuery, VectorStore, check_vector};
28
29const BM25_K1: f32 = 1.2;
31const BM25_B: f32 = 0.75;
32
33#[derive(Debug)]
34struct Entry {
35 chunk: StoredChunk,
36 norm: f32,
37 terms: HashMap<String, u32>,
39 length: u32,
40}
41
42#[derive(Debug, Default)]
43struct Dataset {
44 entries: Vec<Entry>,
45 by_id: HashMap<String, usize>,
47 by_document: HashMap<String, HashSet<String>>,
49 hashes: HashMap<String, u32>,
51 doc_freq: HashMap<String, u32>,
53 total_length: u64,
54 dimension: Option<usize>,
56}
57
58impl Dataset {
59 fn insert(&mut self, chunk: StoredChunk) {
60 if let Some(&ix) = self.by_id.get(&chunk.id) {
61 self.remove_at(ix);
62 }
63 let terms = term_frequencies(&chunk.text);
64 let length = terms.values().sum();
65 for term in terms.keys() {
66 *self.doc_freq.entry(term.clone()).or_insert(0) += 1;
67 }
68 self.total_length += u64::from(length);
69 self.dimension = Some(chunk.vector.len());
70 *self.hashes.entry(chunk.content_hash.clone()).or_insert(0) += 1;
71 self.by_document.entry(chunk.document_id.clone()).or_default().insert(chunk.id.clone());
72 let norm = norm(&chunk.vector);
73 self.by_id.insert(chunk.id.clone(), self.entries.len());
74 self.entries.push(Entry { chunk, norm, terms, length });
75 }
76
77 fn remove_at(&mut self, ix: usize) {
79 if ix >= self.entries.len() {
80 return;
81 }
82 let removed = self.entries.swap_remove(ix);
83 self.by_id.remove(&removed.chunk.id);
84 if let Some(ids) = self.by_document.get_mut(&removed.chunk.document_id) {
85 ids.remove(&removed.chunk.id);
86 if ids.is_empty() {
87 self.by_document.remove(&removed.chunk.document_id);
88 }
89 }
90 if let Some(count) = self.hashes.get_mut(&removed.chunk.content_hash) {
91 *count = count.saturating_sub(1);
92 if *count == 0 {
93 self.hashes.remove(&removed.chunk.content_hash);
94 }
95 }
96 for term in removed.terms.keys() {
97 if let Some(df) = self.doc_freq.get_mut(term) {
98 *df = df.saturating_sub(1);
99 if *df == 0 {
100 self.doc_freq.remove(term);
101 }
102 }
103 }
104 self.total_length = self.total_length.saturating_sub(u64::from(removed.length));
105 if let Some(moved) = self.entries.get(ix) {
106 self.by_id.insert(moved.chunk.id.clone(), ix);
107 }
108 if self.entries.is_empty() {
109 self.dimension = None;
110 }
111 }
112
113 fn remove_id(&mut self, id: &str) -> bool {
114 match self.by_id.get(id).copied() {
115 Some(ix) => {
116 self.remove_at(ix);
117 true
118 }
119 None => false,
120 }
121 }
122
123 fn remove_document(&mut self, document_id: &str) -> usize {
124 let ids: Vec<String> =
125 self.by_document.get(document_id).map(|s| s.iter().cloned().collect()).unwrap_or_default();
126 ids.iter().filter(|id| self.remove_id(id)).count()
127 }
128
129 fn check_dimension(&self, dataset_id: &str, len: usize, what: &str) -> Result<(), RagError> {
130 match self.dimension {
131 Some(d) if d != len => Err(RagError::Input(format!(
132 "dataset `{dataset_id}` holds {d}-dimensional vectors but {what} is {len}-dimensional; \
133 use the original embedder or re-ingest the dataset"
134 ))),
135 _ => Ok(()),
136 }
137 }
138}
139
140type Data = HashMap<String, Dataset>;
141
142#[derive(Debug, Default, Clone)]
144pub struct MemoryStore {
145 inner: Arc<RwLock<Data>>,
146 #[cfg(feature = "fs")]
147 file: Option<Arc<persistence::FileState>>,
148}
149
150impl MemoryStore {
151 pub fn new() -> Self {
152 Self::default()
153 }
154
155 pub fn len(&self) -> usize {
157 self.read(|d| d.values().map(|ds| ds.entries.len()).sum()).unwrap_or(0)
158 }
159
160 pub fn is_empty(&self) -> bool {
161 self.len() == 0
162 }
163
164 pub fn chunks(&self) -> Result<Vec<StoredChunk>, RagError> {
166 self.read(all_chunks)
167 }
168
169 fn read<T>(&self, f: impl FnOnce(&Data) -> T) -> Result<T, RagError> {
170 let guard = self.inner.read().map_err(|_| poisoned())?;
171 Ok(f(&guard))
172 }
173
174 fn write<T>(&self, f: impl FnOnce(&mut Data) -> Result<T, RagError>) -> Result<T, RagError> {
177 let mut guard = self.inner.write().map_err(|_| poisoned())?;
178 let result = f(&mut guard)?;
179 self.persist_locked(&guard)?;
180 Ok(result)
181 }
182
183 pub(crate) fn insert_all(&self, chunks: Vec<StoredChunk>) -> Result<(), RagError> {
186 self.write(|data| insert_batch(data, chunks))
187 }
188
189 #[cfg_attr(not(feature = "sqlite"), allow(dead_code))]
191 pub(crate) fn replace_all(&self, chunks: Vec<StoredChunk>) -> Result<(), RagError> {
192 self.write(|data| {
193 let mut fresh = Data::new();
194 insert_batch(&mut fresh, chunks)?;
195 *data = fresh;
196 Ok(())
197 })
198 }
199
200 pub(crate) fn remove_document(&self, dataset_id: &str, document_id: &str) -> Result<usize, RagError> {
201 self.write(|data| Ok(data.get_mut(dataset_id).map_or(0, |ds| ds.remove_document(document_id))))
202 }
203
204 pub(crate) fn remove_dataset(&self, dataset_id: &str) -> Result<usize, RagError> {
205 self.write(|data| Ok(data.remove(dataset_id).map_or(0, |ds| ds.entries.len())))
206 }
207
208 pub(crate) fn remove_chunks(&self, dataset_id: &str, ids: &[String]) -> Result<usize, RagError> {
209 self.write(|data| Ok(data.get_mut(dataset_id).map_or(0, |ds| ids.iter().filter(|id| ds.remove_id(id)).count())))
210 }
211
212 pub(crate) fn find_document_chunks(
213 &self,
214 dataset_id: &str,
215 document_id: &str,
216 ) -> Result<Vec<StoredChunk>, RagError> {
217 self.read(|data| {
218 let Some(ds) = data.get(dataset_id) else { return Vec::new() };
219 let Some(ids) = ds.by_document.get(document_id) else { return Vec::new() };
220 ids.iter()
221 .filter_map(|id| ds.by_id.get(id).and_then(|&ix| ds.entries.get(ix)))
222 .map(|e| e.chunk.clone())
223 .collect()
224 })
225 }
226
227 pub(crate) fn vector_search(&self, query: &VectorQuery<'_>) -> Result<Vec<Hit>, RagError> {
229 check_vector(query.vector, "the query")?;
230 self.read(|data| {
231 let q_norm = norm(query.vector);
232 let mut scored: Vec<(f32, &Entry)> = Vec::new();
233 for (id, ds) in query.datasets.iter().filter_map(|id| data.get(id).map(|ds| (id, ds))) {
234 ds.check_dimension(id, query.vector.len(), "the query")?;
235 for entry in &ds.entries {
236 if !passes(query.filter, &entry.chunk) {
237 continue;
238 }
239 let denom = q_norm * entry.norm;
240 let score = if denom > 0.0 { dot(query.vector, &entry.chunk.vector) / denom } else { 0.0 };
241 scored.push((score, entry));
242 }
243 }
244 Ok(top_hits(scored, query.top_k))
245 })?
246 }
247
248 pub(crate) fn bm25_search(&self, query: &KeywordQuery<'_>) -> Result<Vec<Hit>, RagError> {
250 let terms: Vec<String> = tokenize(query.text);
251 self.read(|data| {
252 let mut scored: Vec<(f32, &Entry)> = Vec::new();
253 for ds in query.datasets.iter().filter_map(|id| data.get(id)) {
254 let n = ds.entries.len();
255 if n == 0 {
256 continue;
257 }
258 #[allow(clippy::cast_precision_loss)]
259 let (n_f, avgdl) = (n as f32, (ds.total_length as f32 / n as f32).max(1.0));
260 for entry in &ds.entries {
261 if !passes(query.filter, &entry.chunk) {
262 continue;
263 }
264 let mut score = 0.0f32;
265 #[allow(clippy::cast_precision_loss)]
266 let dl = entry.length as f32;
267 for term in &terms {
268 let Some(&tf) = entry.terms.get(term) else { continue };
269 #[allow(clippy::cast_precision_loss)]
270 let df = ds.doc_freq.get(term).copied().unwrap_or(0) as f32;
271 let idf = ((n_f - df + 0.5) / (df + 0.5) + 1.0).ln();
272 #[allow(clippy::cast_precision_loss)]
273 let tf = tf as f32;
274 score += idf * (tf * (BM25_K1 + 1.0)) / (tf + BM25_K1 * (1.0 - BM25_B + BM25_B * dl / avgdl));
275 }
276 if score > 0.0 {
277 scored.push((score, entry));
278 }
279 }
280 }
281 top_hits(scored, query.top_k)
282 })
283 }
284
285 pub(crate) fn has_hash(&self, dataset_id: &str, content_hash: &str) -> Result<bool, RagError> {
286 self.read(|data| data.get(dataset_id).is_some_and(|ds| ds.hashes.contains_key(content_hash)))
287 }
288
289 #[cfg(not(feature = "fs"))]
290 fn persist_locked(&self, _data: &Data) -> Result<(), RagError> {
291 Ok(())
292 }
293}
294
295fn insert_batch(data: &mut Data, chunks: Vec<StoredChunk>) -> Result<(), RagError> {
297 let mut batch_dims: HashMap<&str, usize> = HashMap::new();
298 for chunk in &chunks {
299 check_vector(&chunk.vector, &format!("chunk `{}`", chunk.id))?;
300 let len = chunk.vector.len();
301 if let Some(ds) = data.get(&chunk.dataset_id) {
302 ds.check_dimension(&chunk.dataset_id, len, &format!("chunk `{}`", chunk.id))?;
303 }
304 match batch_dims.get(chunk.dataset_id.as_str()) {
305 Some(&d) if d != len => {
306 return Err(RagError::Input(format!(
307 "chunks for dataset `{}` mix {d}- and {len}-dimensional vectors",
308 chunk.dataset_id
309 )));
310 }
311 _ => {
312 batch_dims.insert(&chunk.dataset_id, len);
313 }
314 }
315 }
316 for chunk in chunks {
317 data.entry(chunk.dataset_id.clone()).or_default().insert(chunk);
318 }
319 Ok(())
320}
321
322fn poisoned() -> RagError {
323 RagError::Store("memory store lock poisoned by an earlier panic".into())
324}
325
326fn all_chunks(data: &Data) -> Vec<StoredChunk> {
327 data.values().flat_map(|ds| ds.entries.iter().map(|e| e.chunk.clone())).collect()
328}
329
330#[cfg(feature = "fs")]
331mod persistence {
332 use super::*;
333 use std::io::Write;
334 use std::path::{Path, PathBuf};
335 use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
336
337 #[derive(serde::Serialize, serde::Deserialize)]
338 struct FileFormat {
339 version: u32,
340 chunks: Vec<StoredChunk>,
341 }
342
343 #[derive(Debug)]
345 pub(super) struct FileState {
346 path: PathBuf,
347 autosave: AtomicBool,
348 dirty: AtomicBool,
350 }
351
352 static TEMP_COUNTER: AtomicU64 = AtomicU64::new(0);
353
354 impl MemoryStore {
355 pub fn open(path: impl Into<PathBuf>) -> Result<Self, RagError> {
358 let path = path.into();
359 let store = Self { inner: Arc::default(), file: None };
360 if path.exists() {
361 let text =
362 std::fs::read_to_string(&path).map_err(|e| RagError::Io(format!("{}: {e}", path.display())))?;
363 let file: FileFormat =
364 serde_json::from_str(&text).map_err(|e| RagError::Decode(format!("{}: {e}", path.display())))?;
365 if file.version != 1 {
366 return Err(RagError::Decode(format!("unsupported store file version {}", file.version)));
367 }
368 store.insert_all(file.chunks)?;
369 }
370 let file = FileState { path, autosave: AtomicBool::new(true), dirty: AtomicBool::new(false) };
371 Ok(Self { file: Some(Arc::new(file)), ..store })
372 }
373
374 pub fn set_autosave(&self, autosave: bool) {
377 if let Some(file) = &self.file {
378 file.autosave.store(autosave, Ordering::SeqCst);
379 }
380 }
381
382 pub fn flush(&self) -> Result<(), RagError> {
384 let guard = self.inner.read().map_err(|_| poisoned())?;
385 match &self.file {
386 Some(file) if file.dirty.load(Ordering::SeqCst) => write_file(file, &guard),
387 _ => Ok(()),
388 }
389 }
390
391 pub fn save_to(&self, path: &Path) -> Result<(), RagError> {
393 let guard = self.inner.read().map_err(|_| poisoned())?;
394 write_atomically(path, &all_chunks(&guard))
395 }
396
397 pub(super) fn persist_locked(&self, data: &Data) -> Result<(), RagError> {
398 let Some(file) = &self.file else { return Ok(()) };
399 file.dirty.store(true, Ordering::SeqCst);
400 if file.autosave.load(Ordering::SeqCst) { write_file(file, data) } else { Ok(()) }
401 }
402 }
403
404 fn write_file(file: &FileState, data: &Data) -> Result<(), RagError> {
405 write_atomically(&file.path, &all_chunks(data))?;
406 file.dirty.store(false, Ordering::SeqCst);
407 Ok(())
408 }
409
410 fn write_atomically(path: &Path, chunks: &[StoredChunk]) -> Result<(), RagError> {
411 #[derive(serde::Serialize)]
412 struct FileRef<'a> {
413 version: u32,
414 chunks: &'a [StoredChunk],
415 }
416 let text = serde_json::to_vec(&FileRef { version: 1, chunks })
417 .map_err(|e| RagError::Store(format!("cannot encode store: {e}")))?;
418 let n = TEMP_COUNTER.fetch_add(1, Ordering::SeqCst);
419 let name = path.file_name().map(|s| s.to_string_lossy().into_owned()).unwrap_or_else(|| "store".into());
420 let tmp = path.with_file_name(format!(".{name}.{}.{n}.tmp", std::process::id()));
421 let io = |p: &Path, e: std::io::Error| RagError::Io(format!("{}: {e}", p.display()));
422 let result = (|| {
423 let mut f = std::fs::File::create(&tmp).map_err(|e| io(&tmp, e))?;
424 f.write_all(&text).map_err(|e| io(&tmp, e))?;
425 f.sync_all().map_err(|e| io(&tmp, e))?;
426 std::fs::rename(&tmp, path).map_err(|e| io(path, e))?;
427 sync_parent_dir(path);
428 Ok(())
429 })();
430 if result.is_err() {
431 let _cleanup = std::fs::remove_file(&tmp);
432 }
433 result
434 }
435
436 fn sync_parent_dir(path: &Path) {
439 #[cfg(unix)]
440 if let Some(dir) = path.parent().filter(|p| !p.as_os_str().is_empty()) {
441 if let Ok(handle) = std::fs::File::open(dir) {
442 let _best_effort = handle.sync_all();
443 }
444 }
445 #[cfg(not(unix))]
446 let _ = path;
447 }
448}
449
450impl VectorStore for MemoryStore {
451 fn upsert(&self, chunks: Vec<StoredChunk>) -> BoxFuture<'_, Result<(), RagError>> {
452 Box::pin(futures::future::ready(self.insert_all(chunks)))
453 }
454
455 fn delete_document<'a>(
456 &'a self,
457 dataset_id: &'a str,
458 document_id: &'a str,
459 ) -> BoxFuture<'a, Result<usize, RagError>> {
460 Box::pin(futures::future::ready(self.remove_document(dataset_id, document_id)))
461 }
462
463 fn delete_dataset<'a>(&'a self, dataset_id: &'a str) -> BoxFuture<'a, Result<usize, RagError>> {
464 Box::pin(futures::future::ready(self.remove_dataset(dataset_id)))
465 }
466
467 fn document_chunks<'a>(
468 &'a self,
469 dataset_id: &'a str,
470 document_id: &'a str,
471 ) -> BoxFuture<'a, Result<Vec<StoredChunk>, RagError>> {
472 Box::pin(futures::future::ready(self.find_document_chunks(dataset_id, document_id)))
473 }
474
475 fn delete_chunks<'a>(&'a self, dataset_id: &'a str, ids: &'a [String]) -> BoxFuture<'a, Result<usize, RagError>> {
476 Box::pin(futures::future::ready(self.remove_chunks(dataset_id, ids)))
477 }
478
479 fn search<'a>(&'a self, query: VectorQuery<'a>) -> BoxFuture<'a, Result<Vec<Hit>, RagError>> {
480 Box::pin(futures::future::ready(self.vector_search(&query)))
481 }
482
483 fn supports_keyword(&self) -> bool {
484 true
485 }
486
487 fn keyword_search<'a>(&'a self, query: KeywordQuery<'a>) -> BoxFuture<'a, Result<Vec<Hit>, RagError>> {
488 Box::pin(futures::future::ready(self.bm25_search(&query)))
489 }
490
491 fn contains_hash<'a>(
492 &'a self,
493 dataset_id: &'a str,
494 content_hash: &'a str,
495 ) -> BoxFuture<'a, Result<bool, RagError>> {
496 Box::pin(futures::future::ready(self.has_hash(dataset_id, content_hash)))
497 }
498}
499
500fn passes(filter: Option<&MetadataFilter>, chunk: &StoredChunk) -> bool {
501 filter.is_none_or(|f| f.matches(&chunk.metadata))
502}
503
504fn top_hits(mut scored: Vec<(f32, &Entry)>, top_k: usize) -> Vec<Hit> {
505 scored.retain(|(score, _)| score.is_finite());
506 scored.sort_by(|a, b| b.0.total_cmp(&a.0));
507 scored.into_iter().take(top_k).map(|(score, e)| Hit::from_chunk(&e.chunk, score)).collect()
508}
509
510pub fn dot(a: &[f32], b: &[f32]) -> f32 {
512 let mut acc = [0.0f32; 8];
513 let chunks_a = a.chunks_exact(8);
514 let chunks_b = b.chunks_exact(8);
515 let (rest_a, rest_b) = (chunks_a.remainder(), chunks_b.remainder());
516 for (x, y) in chunks_a.zip(chunks_b) {
517 for ((slot, xi), yi) in acc.iter_mut().zip(x).zip(y) {
518 *slot += xi * yi;
519 }
520 }
521 let tail: f32 = rest_a.iter().zip(rest_b).map(|(x, y)| x * y).sum();
522 acc.iter().sum::<f32>() + tail
523}
524
525pub fn norm(v: &[f32]) -> f32 {
527 dot(v, v).sqrt()
528}
529
530pub fn cosine(a: &[f32], b: &[f32]) -> f32 {
532 if a.len() != b.len() {
533 return 0.0;
534 }
535 let denom = norm(a) * norm(b);
536 if denom > 0.0 { dot(a, b) / denom } else { 0.0 }
537}
538
539pub fn tokenize(text: &str) -> Vec<String> {
542 let mut tokens = Vec::new();
543 let mut current = String::new();
544 for c in text.chars() {
545 if is_cjk(c) {
546 if !current.is_empty() {
547 tokens.push(std::mem::take(&mut current));
548 }
549 tokens.push(c.to_string());
550 } else if c.is_alphanumeric() {
551 current.extend(c.to_lowercase());
552 } else if !current.is_empty() {
553 tokens.push(std::mem::take(&mut current));
554 }
555 }
556 if !current.is_empty() {
557 tokens.push(current);
558 }
559 tokens
560}
561
562fn is_cjk(c: char) -> bool {
563 matches!(u32::from(c), 0x3040..=0x30FF | 0x3400..=0x4DBF | 0x4E00..=0x9FFF | 0xAC00..=0xD7AF | 0xF900..=0xFAFF)
564}
565
566fn term_frequencies(text: &str) -> HashMap<String, u32> {
567 let mut map = HashMap::new();
568 for t in tokenize(text) {
569 *map.entry(t).or_insert(0) += 1;
570 }
571 map
572}
573
574#[cfg(test)]
575mod tests {
576 use super::*;
577
578 fn chunk(id: &str, ds: &str, text: &str, vector: Vec<f32>) -> StoredChunk {
579 StoredChunk {
580 id: id.into(),
581 dataset_id: ds.into(),
582 document_id: format!("doc-{id}"),
583 title: String::new(),
584 text: text.into(),
585 metadata: serde_json::Map::new(),
586 content_hash: crate::types::content_hash(ds, text),
587 vector,
588 }
589 }
590
591 fn block_on<F: std::future::Future>(f: F) -> F::Output {
592 futures::executor::block_on(f)
593 }
594
595 #[test]
596 fn cosine_ranking_and_dataset_scoping() {
597 let store = MemoryStore::new();
598 block_on(store.upsert(vec![
599 chunk("a", "d1", "a", vec![1.0, 0.0, 0.0]),
600 chunk("b", "d1", "b", vec![0.7, 0.7, 0.0]),
601 chunk("c", "d2", "c", vec![1.0, 0.0, 0.0]),
602 ]))
603 .unwrap();
604 let ds = vec!["d1".to_owned()];
605 let hits =
606 block_on(store.search(VectorQuery { datasets: &ds, vector: &[1.0, 0.1, 0.0], top_k: 5, filter: None }))
607 .unwrap();
608 assert_eq!(hits.iter().map(|h| h.id.as_str()).collect::<Vec<_>>(), vec!["a", "b"]);
609 assert!(hits[0].score > hits[1].score);
610 }
611
612 #[test]
613 fn bm25_prefers_rare_terms() {
614 let store = MemoryStore::new();
615 block_on(store.upsert(vec![
616 chunk("1", "d", "the cat sat on the mat", vec![1.0]),
617 chunk("2", "d", "the dog chased the cat", vec![1.0]),
618 chunk("3", "d", "quantum entanglement explained", vec![1.0]),
619 ]))
620 .unwrap();
621 let ds = vec!["d".to_owned()];
622 let hits = block_on(store.keyword_search(KeywordQuery { datasets: &ds, text: "dog", top_k: 5, filter: None }))
623 .unwrap();
624 assert_eq!(hits.len(), 1);
625 assert_eq!(hits[0].id, "2");
626 let hits =
627 block_on(store.keyword_search(KeywordQuery { datasets: &ds, text: "cat quantum", top_k: 5, filter: None }))
628 .unwrap();
629 assert_eq!(hits[0].id, "3", "rare term outweighs common one");
630 }
631
632 #[test]
633 fn upsert_replaces_and_delete_updates_index() {
634 let store = MemoryStore::new();
635 block_on(store.upsert(vec![chunk("x", "d", "apple banana", vec![1.0]), chunk("y", "d", "apple", vec![1.0])]))
636 .unwrap();
637 block_on(store.upsert(vec![chunk("x", "d", "cherry", vec![1.0])])).unwrap();
638 assert_eq!(store.len(), 2);
639 let ds = vec!["d".to_owned()];
640 let hits =
641 block_on(store.keyword_search(KeywordQuery { datasets: &ds, text: "banana", top_k: 5, filter: None }))
642 .unwrap();
643 assert!(hits.is_empty());
644 assert_eq!(block_on(store.delete_document("d", "doc-y")).unwrap(), 1);
645 assert_eq!(store.len(), 1);
646 assert_eq!(block_on(store.delete_dataset("d")).unwrap(), 1);
647 assert!(store.is_empty());
648 }
649
650 #[test]
651 fn dot_matches_naive() {
652 let a: Vec<f32> = (0..37).map(|i| i as f32 * 0.5).collect();
653 let b: Vec<f32> = (0..37).map(|i| (37 - i) as f32).collect();
654 let naive: f32 = a.iter().zip(&b).map(|(x, y)| x * y).sum();
655 assert!((dot(&a, &b) - naive).abs() < 1e-3);
656 assert!((cosine(&[1.0, 0.0], &[0.0, 1.0])).abs() < 1e-6);
657 }
658
659 #[test]
660 fn tokenizer_handles_cjk() {
661 assert_eq!(tokenize("Hello, World! 日本"), vec!["hello", "world", "日", "本"]);
662 }
663
664 #[cfg(feature = "fs")]
665 #[test]
666 fn persistence_roundtrip() {
667 let dir = std::env::temp_dir().join(format!("graphrt-rag-test-{}", std::process::id()));
668 std::fs::create_dir_all(&dir).unwrap();
669 let path = dir.join("kb.json");
670 let _ = std::fs::remove_file(&path);
671 {
672 let store = MemoryStore::open(&path).unwrap();
673 block_on(store.upsert(vec![chunk("a", "d", "persist me", vec![0.5, 0.5])])).unwrap();
674 }
675 let reopened = MemoryStore::open(&path).unwrap();
676 assert_eq!(reopened.len(), 1);
677 let ds = vec!["d".to_owned()];
678 let hits =
679 block_on(reopened.keyword_search(KeywordQuery { datasets: &ds, text: "persist", top_k: 1, filter: None }))
680 .unwrap();
681 assert_eq!(hits[0].content, "persist me");
682 block_on(reopened.delete_dataset("d")).unwrap();
683 assert!(MemoryStore::open(&path).unwrap().is_empty());
684 let _ = std::fs::remove_dir_all(&dir);
685 }
686}