1use std::path::{Path, PathBuf};
21use std::sync::{Mutex, MutexGuard};
22use std::time::Duration;
23
24use futures::future::BoxFuture;
25use rusqlite::{Connection, OptionalExtension, Transaction, params};
26
27use crate::error::RagError;
28use crate::memory::MemoryStore;
29use crate::types::{Hit, KeywordQuery, Metadata, StoredChunk, VectorQuery, VectorStore};
30
31const SCHEMA_VERSION: i64 = 1;
33const BUSY_TIMEOUT: Duration = Duration::from_secs(5);
35
36const SCHEMA: &str = "
37CREATE TABLE IF NOT EXISTS graphrt_meta (key TEXT PRIMARY KEY, value TEXT NOT NULL);
38CREATE TABLE IF NOT EXISTS chunks (
39 dataset_id TEXT NOT NULL,
40 id TEXT NOT NULL,
41 document_id TEXT NOT NULL,
42 title TEXT NOT NULL,
43 text TEXT NOT NULL,
44 metadata TEXT NOT NULL,
45 content_hash TEXT NOT NULL,
46 vector BLOB NOT NULL,
47 PRIMARY KEY (dataset_id, id)
48) WITHOUT ROWID;
49CREATE INDEX IF NOT EXISTS chunks_by_document ON chunks (dataset_id, document_id);
50";
51
52#[derive(Debug)]
64pub struct SqliteStore {
65 path: PathBuf,
66 state: Mutex<State>,
67}
68
69#[derive(Debug)]
70struct State {
71 connection: Connection,
72 index: MemoryStore,
73 synced_version: i64,
75 stale: bool,
77}
78
79impl SqliteStore {
80 pub fn open(path: impl Into<PathBuf>) -> Result<Self, RagError> {
85 let path = path.into();
86 let connection =
87 Connection::open(&path).map_err(|e| RagError::Io(format!("cannot open {}: {e}", path.display())))?;
88 prepare_connection(&connection, &path)?;
89 let mut state = State { connection, index: MemoryStore::new(), synced_version: 0, stale: true };
90 state.refresh(&path)?;
91 Ok(Self { path, state: Mutex::new(state) })
92 }
93
94 pub fn path(&self) -> &Path {
96 &self.path
97 }
98
99 pub fn len(&self) -> Result<usize, RagError> {
101 let mut state = self.lock()?;
102 state.refresh(&self.path)?;
103 Ok(state.index.len())
104 }
105
106 pub fn is_empty(&self) -> Result<bool, RagError> {
108 Ok(self.len()? == 0)
109 }
110
111 fn lock(&self) -> Result<MutexGuard<'_, State>, RagError> {
112 self.state.lock().map_err(|_| RagError::Store("sqlite store lock poisoned by an earlier panic".into()))
113 }
114
115 fn read<T>(&self, f: impl FnOnce(&MemoryStore) -> Result<T, RagError>) -> Result<T, RagError> {
117 let mut state = self.lock()?;
118 state.refresh(&self.path)?;
119 f(&state.index)
120 }
121
122 fn write<T, R>(
127 &self,
128 change: impl FnOnce(&Transaction<'_>) -> Result<T, RagError>,
129 mirror: impl FnOnce(&MemoryStore, T) -> Result<R, RagError>,
130 ) -> Result<R, RagError> {
131 let mut guard = self.lock()?;
132 guard.refresh(&self.path)?;
133 let state = &mut *guard;
134 let transaction = state.connection.transaction().map_err(|e| store_error(&self.path, &e))?;
135 let changed = change(&transaction)?;
136 let result = mirror(&state.index, changed)?;
137 if let Err(e) = transaction.commit() {
138 state.stale = true;
139 return Err(store_error(&self.path, &e));
140 }
141 Ok(result)
142 }
143}
144
145impl State {
146 fn refresh(&mut self, path: &Path) -> Result<(), RagError> {
148 let version: i64 = self
149 .connection
150 .query_row("PRAGMA data_version", [], |row| row.get(0))
151 .map_err(|e| store_error(path, &e))?;
152 if !self.stale && version == self.synced_version {
153 return Ok(());
154 }
155 let chunks = load_chunks(&self.connection, path)?;
156 self.index.replace_all(chunks)?;
157 self.synced_version = version;
158 self.stale = false;
159 Ok(())
160 }
161}
162
163fn prepare_connection(connection: &Connection, path: &Path) -> Result<(), RagError> {
165 connection.busy_timeout(BUSY_TIMEOUT).map_err(|e| store_error(path, &e))?;
166 let _mode: String =
168 connection.query_row("PRAGMA journal_mode = WAL", [], |row| row.get(0)).map_err(|e| store_error(path, &e))?;
169 connection.execute_batch(SCHEMA).map_err(|e| store_error(path, &e))?;
170 let stored: Option<String> = connection
171 .query_row("SELECT value FROM graphrt_meta WHERE key = 'schema_version'", [], |row| row.get(0))
172 .optional()
173 .map_err(|e| store_error(path, &e))?;
174 match stored {
175 None => {
176 connection
177 .execute(
178 "INSERT INTO graphrt_meta (key, value) VALUES ('schema_version', ?1)",
179 params![SCHEMA_VERSION.to_string()],
180 )
181 .map_err(|e| store_error(path, &e))?;
182 Ok(())
183 }
184 Some(v) if v == SCHEMA_VERSION.to_string() => Ok(()),
185 Some(v) => Err(RagError::Decode(format!(
186 "{}: unsupported store schema version {v} (this build reads {SCHEMA_VERSION})",
187 path.display()
188 ))),
189 }
190}
191
192fn load_chunks(connection: &Connection, path: &Path) -> Result<Vec<StoredChunk>, RagError> {
193 let mut statement = connection
194 .prepare(
195 "SELECT dataset_id, id, document_id, title, text, metadata, content_hash, vector \
196 FROM chunks ORDER BY dataset_id, id",
197 )
198 .map_err(|e| store_error(path, &e))?;
199 let rows = statement
200 .query_map([], |row| {
201 Ok(RawChunk {
202 dataset_id: row.get(0)?,
203 id: row.get(1)?,
204 document_id: row.get(2)?,
205 title: row.get(3)?,
206 text: row.get(4)?,
207 metadata: row.get(5)?,
208 content_hash: row.get(6)?,
209 vector: row.get(7)?,
210 })
211 })
212 .map_err(|e| store_error(path, &e))?;
213 let mut chunks = Vec::new();
214 for row in rows {
215 let raw = row.map_err(|e| store_error(path, &e))?;
216 chunks.push(raw.decode(path)?);
217 }
218 Ok(chunks)
219}
220
221struct RawChunk {
223 dataset_id: String,
224 id: String,
225 document_id: String,
226 title: String,
227 text: String,
228 metadata: String,
229 content_hash: String,
230 vector: Vec<u8>,
231}
232
233impl RawChunk {
234 fn decode(self, path: &Path) -> Result<StoredChunk, RagError> {
235 let corrupt = |what: String| {
236 RagError::Decode(format!(
237 "{}: chunk `{}` of dataset `{}`: {what}",
238 path.display(),
239 self.id,
240 self.dataset_id
241 ))
242 };
243 let metadata: Metadata =
244 serde_json::from_str(&self.metadata).map_err(|e| corrupt(format!("invalid metadata: {e}")))?;
245 let vector = decode_vector(&self.vector).ok_or_else(|| {
246 corrupt(format!("vector blob of {} bytes is not a whole number of f32 values", self.vector.len()))
247 })?;
248 Ok(StoredChunk {
249 id: self.id,
250 dataset_id: self.dataset_id,
251 document_id: self.document_id,
252 title: self.title,
253 text: self.text,
254 metadata,
255 content_hash: self.content_hash,
256 vector,
257 })
258 }
259}
260
261fn encode_vector(vector: &[f32]) -> Vec<u8> {
262 vector.iter().flat_map(|value| value.to_le_bytes()).collect()
263}
264
265fn decode_vector(bytes: &[u8]) -> Option<Vec<f32>> {
267 let values = bytes.chunks_exact(4);
268 if !values.remainder().is_empty() {
269 return None;
270 }
271 values.map(|b| <[u8; 4]>::try_from(b).ok().map(f32::from_le_bytes)).collect()
272}
273
274fn store_error(path: &Path, error: &rusqlite::Error) -> RagError {
275 RagError::Store(format!("{}: {error}", path.display()))
276}
277
278fn insert_rows(transaction: &Transaction<'_>, path: &Path, chunks: &[StoredChunk]) -> Result<(), RagError> {
279 let mut statement = transaction
280 .prepare_cached(
281 "INSERT OR REPLACE INTO chunks \
282 (dataset_id, id, document_id, title, text, metadata, content_hash, vector) \
283 VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8)",
284 )
285 .map_err(|e| store_error(path, &e))?;
286 for chunk in chunks {
287 let metadata = serde_json::to_string(&chunk.metadata)
288 .map_err(|e| RagError::Store(format!("chunk `{}`: cannot encode metadata: {e}", chunk.id)))?;
289 statement
290 .execute(params![
291 chunk.dataset_id,
292 chunk.id,
293 chunk.document_id,
294 chunk.title,
295 chunk.text,
296 metadata,
297 chunk.content_hash,
298 encode_vector(&chunk.vector),
299 ])
300 .map_err(|e| store_error(path, &e))?;
301 }
302 Ok(())
303}
304
305impl VectorStore for SqliteStore {
306 fn upsert(&self, chunks: Vec<StoredChunk>) -> BoxFuture<'_, Result<(), RagError>> {
307 let result = self.write(
308 |tx| insert_rows(tx, &self.path, &chunks).map(|()| chunks),
309 |index, chunks| index.insert_all(chunks),
310 );
311 Box::pin(futures::future::ready(result))
312 }
313
314 fn delete_document<'a>(
315 &'a self,
316 dataset_id: &'a str,
317 document_id: &'a str,
318 ) -> BoxFuture<'a, Result<usize, RagError>> {
319 let result = self.write(
320 |tx| {
321 tx.execute(
322 "DELETE FROM chunks WHERE dataset_id = ?1 AND document_id = ?2",
323 params![dataset_id, document_id],
324 )
325 .map_err(|e| store_error(&self.path, &e))
326 },
327 |index, removed| index.remove_document(dataset_id, document_id).map(|_| removed),
328 );
329 Box::pin(futures::future::ready(result))
330 }
331
332 fn delete_dataset<'a>(&'a self, dataset_id: &'a str) -> BoxFuture<'a, Result<usize, RagError>> {
333 let result = self.write(
334 |tx| {
335 tx.execute("DELETE FROM chunks WHERE dataset_id = ?1", params![dataset_id])
336 .map_err(|e| store_error(&self.path, &e))
337 },
338 |index, removed| index.remove_dataset(dataset_id).map(|_| removed),
339 );
340 Box::pin(futures::future::ready(result))
341 }
342
343 fn document_chunks<'a>(
344 &'a self,
345 dataset_id: &'a str,
346 document_id: &'a str,
347 ) -> BoxFuture<'a, Result<Vec<StoredChunk>, RagError>> {
348 Box::pin(futures::future::ready(self.read(|index| index.find_document_chunks(dataset_id, document_id))))
349 }
350
351 fn delete_chunks<'a>(&'a self, dataset_id: &'a str, ids: &'a [String]) -> BoxFuture<'a, Result<usize, RagError>> {
352 let result = self.write(
353 |tx| {
354 let mut statement = tx
355 .prepare_cached("DELETE FROM chunks WHERE dataset_id = ?1 AND id = ?2")
356 .map_err(|e| store_error(&self.path, &e))?;
357 let mut removed = 0;
358 for id in ids {
359 removed += statement.execute(params![dataset_id, id]).map_err(|e| store_error(&self.path, &e))?;
360 }
361 Ok(removed)
362 },
363 |index, removed| index.remove_chunks(dataset_id, ids).map(|_| removed),
364 );
365 Box::pin(futures::future::ready(result))
366 }
367
368 fn search<'a>(&'a self, query: VectorQuery<'a>) -> BoxFuture<'a, Result<Vec<Hit>, RagError>> {
369 Box::pin(futures::future::ready(self.read(|index| index.vector_search(&query))))
370 }
371
372 fn supports_keyword(&self) -> bool {
373 true
374 }
375
376 fn keyword_search<'a>(&'a self, query: KeywordQuery<'a>) -> BoxFuture<'a, Result<Vec<Hit>, RagError>> {
377 Box::pin(futures::future::ready(self.read(|index| index.bm25_search(&query))))
378 }
379
380 fn contains_hash<'a>(
381 &'a self,
382 dataset_id: &'a str,
383 content_hash: &'a str,
384 ) -> BoxFuture<'a, Result<bool, RagError>> {
385 Box::pin(futures::future::ready(self.read(|index| index.has_hash(dataset_id, content_hash))))
386 }
387}
388
389#[cfg(test)]
390mod tests {
391 use super::*;
392
393 fn chunk(id: &str, document_id: &str, text: &str, vector: Vec<f32>) -> StoredChunk {
394 let mut metadata = Metadata::new();
395 metadata.insert("lang".into(), serde_json::Value::String("en".into()));
396 StoredChunk {
397 id: id.into(),
398 dataset_id: "d".into(),
399 document_id: document_id.into(),
400 title: format!("title {id}"),
401 text: text.into(),
402 metadata,
403 content_hash: crate::types::content_hash("d", text),
404 vector,
405 }
406 }
407
408 fn block_on<F: std::future::Future>(f: F) -> F::Output {
409 futures::executor::block_on(f)
410 }
411
412 struct ScratchDir(PathBuf);
414
415 impl ScratchDir {
416 fn new(name: &str) -> Self {
417 let dir = std::env::temp_dir().join(format!("graphrt-rag-sqlite-{name}-{}", std::process::id()));
418 let _ = std::fs::remove_dir_all(&dir);
419 std::fs::create_dir_all(&dir).unwrap();
420 Self(dir)
421 }
422
423 fn db(&self) -> PathBuf {
424 self.0.join("kb.db")
425 }
426 }
427
428 impl Drop for ScratchDir {
429 fn drop(&mut self) {
430 let _ = std::fs::remove_dir_all(&self.0);
431 }
432 }
433
434 fn datasets() -> Vec<String> {
435 vec!["d".to_owned()]
436 }
437
438 #[test]
439 fn chunks_survive_reopening_with_vectors_and_metadata() {
440 let scratch = ScratchDir::new("reopen");
441 let original = chunk("a", "doc", "persist me", vec![0.25, -1.5, 3.0]);
442 {
443 let store = SqliteStore::open(scratch.db()).unwrap();
444 block_on(store.upsert(vec![original.clone()])).unwrap();
445 }
446 let reopened = SqliteStore::open(scratch.db()).unwrap();
447 assert_eq!(reopened.len().unwrap(), 1);
448 assert_eq!(block_on(reopened.document_chunks("d", "doc")).unwrap(), vec![original]);
449 }
450
451 #[test]
452 fn search_results_match_memory_store() {
453 let scratch = ScratchDir::new("parity");
454 let chunks = vec![
455 chunk("1", "a", "the cat sat on the mat", vec![1.0, 0.0, 0.0]),
456 chunk("2", "b", "the dog chased the cat", vec![0.7, 0.7, 0.0]),
457 chunk("3", "c", "quantum entanglement explained", vec![0.0, 0.0, 1.0]),
458 ];
459 let sqlite = SqliteStore::open(scratch.db()).unwrap();
460 let memory = MemoryStore::new();
461 block_on(sqlite.upsert(chunks.clone())).unwrap();
462 block_on(memory.upsert(chunks)).unwrap();
463 let ds = datasets();
464
465 let vector = VectorQuery { datasets: &ds, vector: &[1.0, 0.1, 0.0], top_k: 3, filter: None };
466 assert_eq!(block_on(sqlite.search(vector.clone())).unwrap(), block_on(memory.search(vector)).unwrap());
467 let keyword = KeywordQuery { datasets: &ds, text: "cat quantum", top_k: 3, filter: None };
468 assert_eq!(
469 block_on(sqlite.keyword_search(keyword.clone())).unwrap(),
470 block_on(memory.keyword_search(keyword)).unwrap()
471 );
472 }
473
474 #[test]
475 fn deletes_are_persisted() {
476 let scratch = ScratchDir::new("deletes");
477 let store = SqliteStore::open(scratch.db()).unwrap();
478 block_on(store.upsert(vec![
479 chunk("1", "a", "one", vec![1.0]),
480 chunk("2", "a", "two", vec![1.0]),
481 chunk("3", "b", "three", vec![1.0]),
482 chunk("4", "c", "four", vec![1.0]),
483 ]))
484 .unwrap();
485 assert_eq!(block_on(store.delete_document("d", "a")).unwrap(), 2);
486 assert_eq!(block_on(store.delete_chunks("d", &["3".to_owned(), "missing".to_owned()])).unwrap(), 1);
487 drop(store);
488
489 let reopened = SqliteStore::open(scratch.db()).unwrap();
490 assert_eq!(reopened.len().unwrap(), 1);
491 assert_eq!(block_on(reopened.delete_dataset("d")).unwrap(), 1);
492 assert!(SqliteStore::open(scratch.db()).unwrap().is_empty().unwrap());
493 }
494
495 #[test]
496 fn upsert_replaces_by_id() {
497 let scratch = ScratchDir::new("replace");
498 let store = SqliteStore::open(scratch.db()).unwrap();
499 block_on(store.upsert(vec![chunk("x", "doc", "apple banana", vec![1.0, 0.0])])).unwrap();
500 block_on(store.upsert(vec![chunk("x", "doc", "cherry", vec![0.0, 1.0])])).unwrap();
501 drop(store);
502 let reopened = SqliteStore::open(scratch.db()).unwrap();
503 let stored = block_on(reopened.document_chunks("d", "doc")).unwrap();
504 assert_eq!(stored.len(), 1);
505 assert_eq!(stored[0].text, "cherry");
506 }
507
508 #[test]
509 fn invalid_batch_leaves_file_and_index_unchanged() {
510 let scratch = ScratchDir::new("invalid");
511 let store = SqliteStore::open(scratch.db()).unwrap();
512 block_on(store.upsert(vec![chunk("1", "a", "kept", vec![1.0, 0.0])])).unwrap();
513 let mixed = vec![chunk("2", "b", "ok", vec![1.0, 0.0]), chunk("3", "b", "wrong size", vec![1.0, 0.0, 0.0])];
514 let error = block_on(store.upsert(mixed)).unwrap_err();
515 assert_eq!(error.kind(), "invalid_input");
516 assert_eq!(store.len().unwrap(), 1);
517 drop(store);
518 assert_eq!(SqliteStore::open(scratch.db()).unwrap().len().unwrap(), 1);
519 }
520
521 #[test]
522 fn sees_writes_from_another_connection() {
523 let scratch = ScratchDir::new("shared");
524 let reader = SqliteStore::open(scratch.db()).unwrap();
525 let writer = SqliteStore::open(scratch.db()).unwrap();
526 block_on(writer.upsert(vec![chunk("1", "a", "written elsewhere", vec![1.0])])).unwrap();
527 let ds = datasets();
528 let hits =
529 block_on(reader.keyword_search(KeywordQuery { datasets: &ds, text: "elsewhere", top_k: 1, filter: None }))
530 .unwrap();
531 assert_eq!(hits.len(), 1, "reader reloads after another connection commits");
532 block_on(writer.delete_dataset("d")).unwrap();
533 assert!(reader.is_empty().unwrap());
534 }
535
536 #[test]
537 fn rejects_unknown_schema_version() {
538 let scratch = ScratchDir::new("schema");
539 drop(SqliteStore::open(scratch.db()).unwrap());
540 let connection = Connection::open(scratch.db()).unwrap();
541 connection.execute("UPDATE graphrt_meta SET value = '99' WHERE key = 'schema_version'", []).unwrap();
542 drop(connection);
543 let error = SqliteStore::open(scratch.db()).unwrap_err();
544 assert!(error.to_string().contains("schema version 99"), "{error}");
545 }
546
547 #[test]
548 fn rejects_corrupt_vector_blob() {
549 let scratch = ScratchDir::new("corrupt");
550 drop(SqliteStore::open(scratch.db()).unwrap());
551 let connection = Connection::open(scratch.db()).unwrap();
552 connection
553 .execute("INSERT INTO chunks VALUES ('d', 'bad', 'doc', '', 'x', '{}', 'h', ?1)", params![vec![0u8, 1, 2]])
554 .unwrap();
555 drop(connection);
556 let error = SqliteStore::open(scratch.db()).unwrap_err();
557 assert_eq!(error.kind(), "invalid_response");
558 assert!(error.to_string().contains("chunk `bad`"), "{error}");
559 }
560
561 #[test]
562 fn closing_the_store_checkpoints_the_write_ahead_log() {
563 let scratch = ScratchDir::new("checkpoint");
564 let store = SqliteStore::open(scratch.db()).unwrap();
565 block_on(store.upsert(vec![chunk("1", "a", "flushed on close", vec![1.0])])).unwrap();
566 drop(store);
567 let wal = scratch.0.join("kb.db-wal");
569 assert!(std::fs::metadata(&wal).map_or(true, |m| m.len() == 0), "{} still holds commits", wal.display());
570 }
571
572 #[test]
573 fn vector_encoding_roundtrip() {
574 let vector = vec![0.0, -0.0, 1.5, f32::MIN_POSITIVE, -123.456];
575 assert_eq!(decode_vector(&encode_vector(&vector)).unwrap(), vector);
576 assert!(decode_vector(&[1, 2, 3, 4, 5]).is_none());
577 }
578}