1use crate::{
10 Catalog, ManagedConnection, SQLiteBTreeIndexStore, SQLiteDocumentStore, SQLiteHNSWIndex,
11 SQLiteIVFIndex, SQLiteInvertedIndex, SQLiteVectorIndex,
12};
13use std::{collections::BTreeMap, sync::Arc};
14use uqa_analysis::Analyzer;
15use uqa_core::{DocId, Value};
16use uqa_storage::{
17 CatalogFacade, DocumentStore, InvertedIndex, PersistentStorageBackend,
18 PersistentStorageIdentity, PersistentStorageProvider, PersistentStorageSession,
19 StorageBackendResult, StorageSavepointId, VectorIndex, VectorIndexOpenMode, VectorIndexSpec,
20};
21
22#[derive(Clone)]
23pub struct SQLiteStorageBackend {
24 conn: ManagedConnection,
25 native_initial_restore: bool,
26}
27
28impl SQLiteStorageBackend {
29 pub fn new(conn: ManagedConnection) -> Self {
30 Self {
31 conn,
32 native_initial_restore: false,
33 }
34 }
35
36 pub fn connection(&self) -> ManagedConnection {
37 self.conn.clone()
38 }
39
40 #[must_use]
43 pub fn new_session(&self) -> Self {
44 Self::new(self.conn.new_session())
45 }
46}
47
48#[derive(Clone)]
50pub struct SQLiteStorageProvider {
51 connection: ManagedConnection,
52}
53
54impl SQLiteStorageProvider {
55 pub fn new(connection: ManagedConnection) -> Self {
56 Self { connection }
57 }
58
59 fn native_session(
60 &self,
61 cancellation: &uqa_core::CancellationToken,
62 ) -> StorageBackendResult<ManagedConnection> {
63 if !self.connection.is_native_record_session() {
64 self.connection
65 .bind_native_records(uqa_storage::mvcc::VersionedSessionOptions::default())?;
66 }
67 Ok(self.connection.new_session_with_cancellation(cancellation))
68 }
69}
70
71impl PersistentStorageProvider for SQLiteStorageProvider {
72 fn auxiliary_encryption_key(&self) -> Option<uqa_storage::StorageEncryptionKey> {
73 self.connection.auxiliary_encryption_key()
74 }
75
76 fn open_initial_session(&self) -> StorageBackendResult<PersistentStorageSession> {
77 let connection = self.connection.new_session();
78 let catalog: Arc<dyn CatalogFacade> =
79 Arc::new(Catalog::for_initial_restore(connection.clone()));
80 let backend: Arc<dyn PersistentStorageBackend> = Arc::new(SQLiteStorageBackend {
81 conn: connection,
82 native_initial_restore: true,
83 });
84 Ok(PersistentStorageSession::new(catalog, backend))
85 }
86
87 fn open_session(&self) -> StorageBackendResult<PersistentStorageSession> {
88 self.open_session_with_cancellation(&uqa_core::CancellationToken::new())
89 }
90
91 fn open_session_with_cancellation(
92 &self,
93 cancellation: &uqa_core::CancellationToken,
94 ) -> StorageBackendResult<PersistentStorageSession> {
95 cancellation.check()?;
96 let connection = self.native_session(cancellation)?;
97 let catalog: Arc<dyn CatalogFacade> = Arc::new(Catalog::open(connection.clone())?);
98 let backend: Arc<dyn PersistentStorageBackend> =
99 Arc::new(SQLiteStorageBackend::new(connection));
100 Ok(PersistentStorageSession::new(catalog, backend))
101 }
102
103 fn storage_identity(&self) -> StorageBackendResult<Option<PersistentStorageIdentity>> {
104 self.connection.storage_identity()
105 }
106}
107
108impl uqa_storage::mvcc::IdentifierAllocator for SQLiteStorageBackend {
109 fn identifier_watermark(&self, namespace: &[u8]) -> StorageBackendResult<Option<u64>> {
110 self.conn
111 .native_identifier_watermark(namespace)
112 .map_err(Into::into)
113 }
114
115 fn allocate_identifiers(
116 &self,
117 namespace: &[u8],
118 request: uqa_storage::mvcc::IdentifierRequest,
119 ) -> StorageBackendResult<uqa_storage::mvcc::IdentifierAllocation> {
120 self.conn
121 .allocate_native_identifiers(namespace, request)
122 .map_err(Into::into)
123 }
124
125 fn observe_identifier(
126 &self,
127 namespace: &[u8],
128 value: u64,
129 ) -> StorageBackendResult<uqa_storage::mvcc::ObservedIdentifier> {
130 self.conn
131 .observe_native_identifier(namespace, value)
132 .map_err(Into::into)
133 }
134}
135
136impl PersistentStorageBackend for SQLiteStorageBackend {
137 fn require_sequence_value_durability(&self) -> StorageBackendResult<()> {
138 self.conn
139 .require_sequence_value_durability()
140 .map_err(Into::into)
141 }
142
143 fn open_sequence_value_session(
144 &self,
145 cancellation: &uqa_core::CancellationToken,
146 ) -> StorageBackendResult<PersistentStorageSession> {
147 let connection = self.conn.new_sequence_value_session(cancellation)?;
148 let catalog: Arc<dyn CatalogFacade> = Arc::new(Catalog::open(connection.clone())?);
149 let backend: Arc<dyn PersistentStorageBackend> = Arc::new(Self::new(connection));
150 Ok(PersistentStorageSession::new(catalog, backend))
151 }
152
153 fn retention_control(&self) -> Option<uqa_storage::read_control::StorageReadControl> {
154 self.conn.retention_control()
155 }
156
157 fn notification_publications(
158 &self,
159 ) -> Option<&dyn uqa_storage::notifications::NotificationPublicationStore> {
160 Some(&self.conn)
161 }
162
163 fn serializable_session(&self) -> Option<&dyn uqa_storage::mvcc::SerializableSession> {
164 self.conn
165 .is_native_record_session()
166 .then_some(&self.conn as &dyn uqa_storage::mvcc::SerializableSession)
167 }
168
169 fn open_retained_read_session(
170 &self,
171 cancellation: &uqa_core::CancellationToken,
172 ) -> StorageBackendResult<PersistentStorageSession> {
173 let connection = self.conn.new_retained_read_session(cancellation)?;
174 let catalog: Arc<dyn CatalogFacade> = Arc::new(Catalog::open(connection.clone())?);
175 let backend: Arc<dyn PersistentStorageBackend> = Arc::new(Self::new(connection));
176 Ok(PersistentStorageSession::new(catalog, backend))
177 }
178
179 fn write_cancellation(&self) -> Option<uqa_core::CancellationToken> {
180 Some(self.conn.write_cancellation())
181 }
182
183 fn transaction_model(&self) -> uqa_storage::StorageTransactionModel {
184 self.conn.transaction_model()
185 }
186
187 fn identifier_allocator(&self) -> Option<&dyn uqa_storage::mvcc::IdentifierAllocator> {
188 self.conn.is_native_record_session().then_some(self)
189 }
190
191 fn transaction_affinity(&self) -> Option<uqa_storage::StorageSessionAffinity> {
192 Some(self.conn.transaction_affinity())
193 }
194
195 fn auxiliary_encryption_key(&self) -> Option<uqa_storage::StorageEncryptionKey> {
196 self.conn.auxiliary_encryption_key()
197 }
198
199 fn storage_identity(&self) -> StorageBackendResult<Option<PersistentStorageIdentity>> {
200 self.conn.storage_identity()
201 }
202
203 fn open_session(&self) -> StorageBackendResult<PersistentStorageSession> {
204 self.open_session_with_cancellation(&uqa_core::CancellationToken::new())
205 }
206
207 fn open_session_with_cancellation(
208 &self,
209 cancellation: &uqa_core::CancellationToken,
210 ) -> StorageBackendResult<PersistentStorageSession> {
211 cancellation.check()?;
212 let connection = self.conn.new_session_with_cancellation(cancellation);
213 let catalog: Arc<dyn CatalogFacade> = Arc::new(Catalog::open(connection.clone())?);
214 let backend: Arc<dyn PersistentStorageBackend> = Arc::new(Self::new(connection));
215 Ok(PersistentStorageSession::new(catalog, backend))
216 }
217
218 fn open_controlled_session(
219 &self,
220 control: &uqa_storage::read_control::StorageReadControl,
221 ) -> StorageBackendResult<PersistentStorageSession> {
222 let connection = self.conn.new_controlled_session(control)?;
223 let catalog: Arc<dyn CatalogFacade> = Arc::new(Catalog::open(connection.clone())?);
224 let backend: Arc<dyn PersistentStorageBackend> = Arc::new(Self::new(connection));
225 Ok(PersistentStorageSession::new(catalog, backend))
226 }
227
228 fn supports_concurrent_pinned_read_and_write(&self) -> bool {
229 self.conn.supports_concurrent_pinned_read_and_write()
230 }
231
232 fn document_store(&self, table: &str) -> Box<dyn DocumentStore> {
233 Box::new(SQLiteDocumentStore::new(self.conn.clone(), table))
234 }
235
236 fn inverted_index(&self, table: &str, analyzer: Analyzer) -> Box<dyn InvertedIndex> {
237 Box::new(SQLiteInvertedIndex::new(self.conn.clone(), table, analyzer))
238 }
239
240 fn vector_index(
241 &self,
242 table: &str,
243 field: &str,
244 dimensions: u32,
245 spec: VectorIndexSpec,
246 mode: VectorIndexOpenMode,
247 ) -> StorageBackendResult<Box<dyn VectorIndex>> {
248 let index: Box<dyn VectorIndex> = match spec {
249 VectorIndexSpec::DiskANN(_) => {
250 return Err(uqa_storage::StorageBackendError::Other(
251 "DiskANN requires an actual catalog binding".into(),
252 ))
253 }
254 VectorIndexSpec::BruteForce => Box::new(SQLiteVectorIndex::new(
255 self.conn.clone(),
256 table,
257 field,
258 dimensions,
259 )),
260 VectorIndexSpec::IVF(params) => {
261 params.validate()?;
262 match mode {
263 VectorIndexOpenMode::Create => Box::new(SQLiteIVFIndex::with_params(
264 self.conn.clone(),
265 table,
266 field,
267 dimensions,
268 params.nlist,
269 params.nprobe,
270 params.train_threshold,
271 )),
272 VectorIndexOpenMode::Restore => Box::new(SQLiteIVFIndex::open_existing(
273 self.conn.clone(),
274 table,
275 field,
276 dimensions,
277 params.nlist,
278 params.nprobe,
279 params.train_threshold,
280 )),
281 }
282 }
283 VectorIndexSpec::HNSW(params) => {
284 params.validate()?;
285 match mode {
286 VectorIndexOpenMode::Create => Box::new(SQLiteHNSWIndex::with_params(
287 self.conn.clone(),
288 table,
289 field,
290 dimensions,
291 params,
292 )),
293 VectorIndexOpenMode::Restore => {
294 let index = SQLiteHNSWIndex::open_existing(
295 self.conn.clone(),
296 table,
297 field,
298 dimensions,
299 params,
300 );
301 index.validate_existing()?;
302 Box::new(index)
303 }
304 }
305 }
306 };
307 Ok(index)
308 }
309
310 fn diskann_index(
311 &self,
312 binding: uqa_storage::diskann_index::DiskANNIndexBinding<'_>,
313 options: uqa_storage::diskann_index::DiskANNIndexOptions,
314 temporary: &uqa_storage::diskann_index::build::DiskANNTemporaryBudget,
315 mode: VectorIndexOpenMode,
316 ) -> StorageBackendResult<Box<dyn VectorIndex>> {
317 let canonical = crate::vector_index::SQLiteDiskANNCanonical::new(
318 self.conn.clone(),
319 binding.table,
320 binding.field,
321 binding.dimensions,
322 )?;
323 if mode == VectorIndexOpenMode::Create {
324 canonical.create_index(
325 binding.index,
326 &*binding.resolver,
327 options,
328 temporary,
329 binding.control,
330 )?;
331 }
332 let handle = canonical.bind(
333 binding.index.clone(),
334 binding.resolver,
335 options.read,
336 binding.control,
337 )?;
338 Ok(Box::new(
339 uqa_storage::diskann_index::PersistentDiskANNIndex::new(handle, options, temporary)?,
340 ))
341 }
342
343 fn diskann_journal_pruner(
344 &self,
345 binding: uqa_storage::diskann_index::DiskANNIndexBinding<'_>,
346 max_record_bytes: usize,
347 ) -> StorageBackendResult<Box<dyn uqa_storage::diskann_index::changes::DiskANNJournalPruner>>
348 {
349 crate::vector_index::SQLiteDiskANNCanonical::new(
350 self.conn.clone(),
351 binding.table,
352 binding.field,
353 binding.dimensions,
354 )?
355 .journal_pruner(
356 binding.index,
357 binding.resolver,
358 max_record_bytes,
359 binding.control,
360 )
361 }
362
363 fn diskann_maintenance_source(
364 &self,
365 binding: uqa_storage::diskann_index::DiskANNIndexBinding<'_>,
366 max_record_bytes: usize,
367 ) -> StorageBackendResult<
368 Box<dyn uqa_storage::diskann_index::maintenance::DiskANNMaintenanceSource>,
369 > {
370 crate::vector_index::SQLiteDiskANNCanonical::new(
371 self.conn.clone(),
372 binding.table,
373 binding.field,
374 binding.dimensions,
375 )?
376 .maintenance_source(
377 binding.index,
378 binding.resolver,
379 max_record_bytes,
380 binding.control,
381 )
382 }
383
384 fn retire_diskann_index(
385 &self,
386 binding: uqa_storage::diskann_index::DiskANNIndexBinding<'_>,
387 ) -> StorageBackendResult<()> {
388 crate::vector_index::SQLiteDiskANNCanonical::new(
389 self.conn.clone(),
390 binding.table,
391 binding.field,
392 binding.dimensions,
393 )?
394 .retire_index(binding.index, &*binding.resolver, binding.control)
395 }
396
397 fn drop_vector_index_metadata(&self, table: &str, field: &str) -> StorageBackendResult<()> {
398 SQLiteIVFIndex::drop_metadata(&self.conn, table, field)?;
399 SQLiteHNSWIndex::drop_metadata(&self.conn, table, field)?;
400 Ok(())
401 }
402
403 fn persists_btree_indexes(&self) -> bool {
404 true
405 }
406
407 fn load_btree_index(
408 &self,
409 table: &str,
410 field: &uqa_storage::ValueIndexKey,
411 ) -> StorageBackendResult<Option<Vec<(DocId, Value)>>> {
412 Ok(SQLiteBTreeIndexStore::new(self.conn.clone()).load(table, field)?)
413 }
414
415 fn btree_index_fields(
416 &self,
417 table: &str,
418 ) -> StorageBackendResult<Vec<uqa_storage::ValueIndexKey>> {
419 Ok(SQLiteBTreeIndexStore::new(self.conn.clone()).fields(table)?)
420 }
421
422 fn read_btree_index_entry(
423 &self,
424 table: &str,
425 field: &uqa_storage::ValueIndexKey,
426 doc_id: DocId,
427 ) -> StorageBackendResult<uqa_storage::ValueIndexEntry> {
428 Ok(SQLiteBTreeIndexStore::new(self.conn.clone()).read_entry(table, field, doc_id)?)
429 }
430
431 fn btree_index_repairs(
432 &self,
433 ) -> StorageBackendResult<Vec<(String, uqa_storage::ValueIndexKey)>> {
434 Ok(SQLiteBTreeIndexStore::new(self.conn.clone()).repairs()?)
435 }
436
437 fn clear_btree_index_repair(
438 &self,
439 table: &str,
440 field: &uqa_storage::ValueIndexKey,
441 ) -> StorageBackendResult<()> {
442 SQLiteBTreeIndexStore::new(self.conn.clone()).clear_repair(table, field)?;
443 Ok(())
444 }
445
446 fn replace_btree_index(
447 &self,
448 table: &str,
449 field: &uqa_storage::ValueIndexKey,
450 values: &[(DocId, Value)],
451 ) -> StorageBackendResult<()> {
452 SQLiteBTreeIndexStore::new(self.conn.clone()).replace(table, field, values)?;
453 Ok(())
454 }
455
456 fn repair_btree_index(
457 &self,
458 table: &str,
459 field: &uqa_storage::ValueIndexKey,
460 _complete: &[(DocId, Value)],
461 stale_doc_ids: &[DocId],
462 missing: &[(DocId, Value)],
463 ) -> StorageBackendResult<()> {
464 SQLiteBTreeIndexStore::new(self.conn.clone()).repair(
465 table,
466 field,
467 stale_doc_ids,
468 missing,
469 )?;
470 Ok(())
471 }
472
473 fn replace_btree_indexes(
474 &self,
475 table: &str,
476 indexes: &[(&uqa_storage::ValueIndexKey, &[(DocId, Value)])],
477 ) -> StorageBackendResult<()> {
478 SQLiteBTreeIndexStore::new(self.conn.clone()).replace_many(table, indexes)?;
479 Ok(())
480 }
481
482 fn apply_btree_index_write(
483 &self,
484 table: &str,
485 doc_id: DocId,
486 values: Option<&BTreeMap<uqa_storage::ValueIndexKey, Value>>,
487 ) -> StorageBackendResult<()> {
488 SQLiteBTreeIndexStore::new(self.conn.clone()).apply_write(table, doc_id, values)?;
489 Ok(())
490 }
491
492 fn apply_unused_btree_index_write(
493 &self,
494 table: &str,
495 doc_id: DocId,
496 values: &BTreeMap<uqa_storage::ValueIndexKey, Value>,
497 namespace: uqa_storage::document_store::identifiers::DocumentIdNamespace,
498 ) -> StorageBackendResult<()> {
499 SQLiteBTreeIndexStore::new(self.conn.clone())
500 .apply_unused_write(table, doc_id, values, namespace)?;
501 Ok(())
502 }
503
504 fn drop_btree_index(
505 &self,
506 table: &str,
507 field: &uqa_storage::ValueIndexKey,
508 ) -> StorageBackendResult<()> {
509 SQLiteBTreeIndexStore::new(self.conn.clone()).drop_index(table, field)?;
510 Ok(())
511 }
512
513 fn clear_btree_indexes(&self, table: &str) -> StorageBackendResult<()> {
514 SQLiteBTreeIndexStore::new(self.conn.clone()).clear_table(table)?;
515 Ok(())
516 }
517
518 fn vacuum(&self) -> StorageBackendResult<()> {
519 self.conn.vacuum()?;
520 Ok(())
521 }
522
523 fn reclaim_obsolete(&self) -> StorageBackendResult<()> {
524 self.conn.reclaim_obsolete().map_err(Into::into)
525 }
526
527 fn begin_transaction(&self) -> StorageBackendResult<()> {
528 self.conn.begin_transaction()?;
529 Ok(())
530 }
531
532 fn begin_read_transaction(&self) -> StorageBackendResult<()> {
533 self.conn.begin_deferred_transaction()?;
534 Ok(())
535 }
536
537 fn begin_upgradeable_transaction(&self) -> StorageBackendResult<()> {
538 if self.native_initial_restore && !self.conn.is_native_record_session() {
539 self.conn.begin_native_initial_restore()?;
540 } else {
541 self.conn.begin_deferred_transaction()?;
542 }
543 Ok(())
544 }
545
546 fn refresh_transaction_snapshot(
547 &self,
548 cancellation: &uqa_core::CancellationToken,
549 ) -> StorageBackendResult<()> {
550 self.conn.refresh_transaction_snapshot(cancellation)?;
551 Ok(())
552 }
553
554 fn in_transaction(&self) -> bool {
555 self.conn.in_transaction()
556 }
557
558 fn transaction_has_written(&self) -> StorageBackendResult<bool> {
559 Ok(self.conn.transaction_has_written()?)
560 }
561
562 fn change_version(&self) -> StorageBackendResult<Option<u64>> {
563 Ok(self.conn.data_version()?)
564 }
565
566 fn read_view_revision(
567 &self,
568 ) -> StorageBackendResult<Option<uqa_storage::key_value::KeyValueReadRevision>> {
569 Ok(self.conn.read_view_revision()?)
570 }
571
572 fn committed_data_revision(
573 &self,
574 ) -> StorageBackendResult<Option<uqa_storage::CommittedDataRevision>> {
575 Ok(self.conn.committed_data_revision())
576 }
577
578 fn commit_monitor_version(&self) -> StorageBackendResult<Option<u64>> {
579 Ok(self.conn.commit_monitor_version()?)
580 }
581
582 fn change_version_monitor_is_nonblocking(&self) -> StorageBackendResult<bool> {
583 Ok(self.conn.data_version_monitor_is_nonblocking()?)
584 }
585
586 fn pin_transaction_snapshot(&self) -> StorageBackendResult<()> {
587 self.conn.pin_transaction_snapshot()?;
588 Ok(())
589 }
590
591 fn commit_transaction(&self) -> StorageBackendResult<()> {
592 self.conn.commit_transaction()?;
593 Ok(())
594 }
595
596 fn rollback_transaction(&self) -> StorageBackendResult<()> {
597 self.conn.rollback_transaction()?;
598 Ok(())
599 }
600
601 fn savepoint(&self, id: StorageSavepointId) -> StorageBackendResult<()> {
602 self.conn.savepoint(&id.backend_name())?;
603 Ok(())
604 }
605
606 fn release_savepoint(&self, id: StorageSavepointId) -> StorageBackendResult<()> {
607 self.conn.release_savepoint(&id.backend_name())?;
608 Ok(())
609 }
610
611 fn rollback_to_savepoint(&self, id: StorageSavepointId) -> StorageBackendResult<()> {
612 self.conn.rollback_to_savepoint(&id.backend_name())?;
613 Ok(())
614 }
615}
616
617#[cfg(test)]
618mod tests {
619 use std::collections::BTreeMap;
620
621 use uqa_analysis::analyzer::standard_analyzer;
622 use uqa_core::Value;
623 use uqa_storage::StorageBackendError;
624
625 use super::*;
626 use crate::{Catalog, SQLiteError};
627
628 #[test]
629 fn session_factories_preserve_a_bound_native_sessions_retention_options() {
630 let connection = ManagedConnection::open_in_memory().unwrap();
631 connection
632 .bind_native_records(uqa_storage::mvcc::VersionedSessionOptions {
633 retained_bytes: 8 << 20,
634 })
635 .unwrap();
636 let provider = SQLiteStorageProvider::new(connection);
637 let initial = provider.open_initial_session().unwrap();
638 let sibling = provider.open_session().unwrap();
639 initial.validate_transaction_affinity().unwrap();
640 sibling.validate_transaction_affinity().unwrap();
641 assert!(initial.backend.transaction_model().is_versioned());
642 assert_eq!(
643 initial.backend.transaction_model(),
644 sibling.backend.transaction_model()
645 );
646 assert_ne!(
647 initial.backend.transaction_affinity(),
648 sibling.backend.transaction_affinity()
649 );
650 }
651
652 #[test]
653 fn session_factory_reads_current_catalog_while_a_sibling_holds_a_writer_reservation() {
654 let directory = tempfile::tempdir().unwrap();
655 let connection = ManagedConnection::open_compressed(
656 &directory.path().join("session-writer-reservation.db"),
657 crate::SQLiteCompressionOptions::default(),
658 )
659 .unwrap();
660 let provider = SQLiteStorageProvider::new(connection);
661 let writer = provider.open_session().unwrap();
662 writer.backend.begin_transaction().unwrap();
663 writer
664 .catalog
665 .set_metadata("private-write", "uncommitted")
666 .unwrap();
667
668 let reader = provider.open_session().unwrap();
669 assert_eq!(reader.catalog.get_metadata("private-write").unwrap(), None);
670 writer.backend.commit_transaction().unwrap();
671 assert_eq!(
672 reader
673 .catalog
674 .get_metadata("private-write")
675 .unwrap()
676 .as_deref(),
677 Some("uncommitted")
678 );
679 }
680
681 #[test]
682 fn sqlite_backend_builds_document_index_and_vector_stores() {
683 let conn = ManagedConnection::open_in_memory().unwrap();
684 let _catalog = Catalog::open(conn.clone()).unwrap();
685 let backend = SQLiteStorageBackend::new(conn);
686
687 let mut doc = BTreeMap::new();
688 doc.insert("title".to_string(), Value::Str("rust storage".into()));
689 let mut docs = backend.document_store("articles");
690 docs.put(1, doc).unwrap();
691 assert_eq!(
692 docs.get_field(1, "title").unwrap(),
693 Some(Value::Str("rust storage".into()))
694 );
695
696 let mut inv = backend.inverted_index("articles", standard_analyzer("english"));
697 inv.add_document(
698 1,
699 BTreeMap::from([("title".to_string(), "rust storage".to_string())]),
700 )
701 .unwrap();
702 assert_eq!(inv.doc_freq("title", "rust").unwrap(), 1);
703
704 let mut vectors = backend
705 .vector_index(
706 "articles",
707 "embedding",
708 2,
709 VectorIndexSpec::IVF(uqa_storage::IVFIndexParams {
710 nlist: 2,
711 nprobe: 1,
712 train_threshold: 2,
713 }),
714 VectorIndexOpenMode::Create,
715 )
716 .unwrap();
717 vectors.add(1, vec![1.0, 0.0]).unwrap();
718 let hits = vectors.search_knn(&[1.0, 0.0], 1).unwrap();
719 assert_eq!(hits.entries().len(), 1);
720 assert_eq!(hits.entries()[0].doc_id, 1);
721 }
722
723 #[test]
724 fn sqlite_backend_transaction_rolls_back_cross_store_writes() {
725 let conn = ManagedConnection::open_in_memory().unwrap();
726 let _catalog = Catalog::open(conn.clone()).unwrap();
727 let backend = SQLiteStorageBackend::new(conn);
728 let mut docs = backend.document_store("articles");
729 let mut inv = backend.inverted_index("articles", standard_analyzer("english"));
730
731 backend.begin_transaction().unwrap();
732 docs.put(
733 1,
734 BTreeMap::from([("title".to_string(), Value::Str("rollback".into()))]),
735 )
736 .unwrap();
737 inv.add_document(
738 1,
739 BTreeMap::from([("title".to_string(), "rollback".to_string())]),
740 )
741 .unwrap();
742 backend.rollback_transaction().unwrap();
743
744 assert_eq!(docs.len().unwrap(), 0);
745 assert_eq!(inv.doc_freq("title", "rollback").unwrap(), 0);
746 }
747
748 #[test]
749 fn sqlite_sessions_isolate_and_atomically_commit_cross_store_writes() {
750 let dir = tempfile::tempdir().unwrap();
751 let path = dir.path().join("cross-store-isolation.sqlite3");
752 let conn = ManagedConnection::open(&path).unwrap();
753 let catalog = Catalog::open(conn.clone()).unwrap();
754 let writer = SQLiteStorageBackend::new(conn.clone());
755 let observer_conn = conn.new_session();
756 let observer_catalog = Catalog::open(observer_conn.clone()).unwrap();
757 let observer = SQLiteStorageBackend::new(observer_conn);
758
759 let mut writer_docs = writer.document_store("articles");
760 let mut writer_inv = writer.inverted_index("articles", standard_analyzer("english"));
761 let mut writer_vectors = writer
762 .vector_index(
763 "articles",
764 "embedding",
765 2,
766 VectorIndexSpec::BruteForce,
767 VectorIndexOpenMode::Create,
768 )
769 .unwrap();
770 let observer_docs = observer.document_store("articles");
771 let observer_inv = observer.inverted_index("articles", standard_analyzer("english"));
772 let observer_vectors = observer
773 .vector_index(
774 "articles",
775 "embedding",
776 2,
777 VectorIndexSpec::BruteForce,
778 VectorIndexOpenMode::Restore,
779 )
780 .unwrap();
781
782 writer.begin_transaction().unwrap();
783 writer_docs
784 .put(
785 1,
786 BTreeMap::from([("title".to_string(), Value::Str("atomic rust".into()))]),
787 )
788 .unwrap();
789 writer_inv
790 .add_document(
791 1,
792 BTreeMap::from([("title".to_string(), "atomic rust".to_string())]),
793 )
794 .unwrap();
795 writer_vectors.add(1, vec![1.0, 0.0]).unwrap();
796 catalog
797 .save_scoring_params("transactional", r#"{"alpha":1.0}"#)
798 .unwrap();
799
800 assert_eq!(writer_docs.len().unwrap(), 1);
801 assert_eq!(writer_inv.doc_freq("title", "rust").unwrap(), 1);
802 assert_eq!(writer_vectors.count().unwrap(), 1);
803 assert!(catalog
804 .load_scoring_params("transactional")
805 .unwrap()
806 .is_some());
807
808 assert_eq!(observer_docs.len().unwrap(), 0);
809 assert_eq!(observer_inv.doc_freq("title", "rust").unwrap(), 0);
810 assert_eq!(observer_vectors.count().unwrap(), 0);
811 assert!(observer_catalog
812 .load_scoring_params("transactional")
813 .unwrap()
814 .is_none());
815
816 writer.commit_transaction().unwrap();
817 assert_eq!(observer_docs.len().unwrap(), 1);
818 assert_eq!(observer_inv.doc_freq("title", "rust").unwrap(), 1);
819 assert_eq!(observer_vectors.count().unwrap(), 1);
820 assert!(observer_catalog
821 .load_scoring_params("transactional")
822 .unwrap()
823 .is_some());
824 }
825
826 #[test]
827 fn ignored_legacy_index_error_cannot_commit_partial_document_write() {
828 let dir = tempfile::tempdir().unwrap();
829 let path = dir.path().join("ignored-index-error.sqlite3");
830 let conn = ManagedConnection::open(&path).unwrap();
831 let _catalog = Catalog::open(conn.clone()).unwrap();
832 let backend = SQLiteStorageBackend::new(conn.clone());
833 let observer = conn.new_session();
834 let mut docs = backend.document_store("articles");
835 let mut vectors = backend
836 .vector_index(
837 "articles",
838 "embedding",
839 2,
840 VectorIndexSpec::BruteForce,
841 VectorIndexOpenMode::Create,
842 )
843 .unwrap();
844
845 backend.begin_transaction().unwrap();
846 docs.put(
847 1,
848 BTreeMap::from([("title".to_string(), Value::Str("must roll back".into()))]),
849 )
850 .unwrap();
851 conn.with(|connection| {
852 connection.execute("DROP TABLE _vectors", [])?;
853 Ok(())
854 })
855 .unwrap();
856 let ignored = vectors.add(1, vec![1.0, 0.0]);
860 assert!(ignored.is_err());
861 assert!(matches!(
862 backend.commit_transaction(),
863 Err(StorageBackendError::Backend { source, .. })
864 if matches!(source.downcast_ref::<SQLiteError>(), Some(SQLiteError::TransactionAborted(_)))
865 ));
866
867 let stored_docs: i64 = observer
868 .with(|connection| {
869 Ok(connection.query_row(
870 "SELECT COUNT(*) FROM _documents WHERE table_name = 'articles'",
871 [],
872 |row| row.get(0),
873 )?)
874 })
875 .unwrap();
876 let vector_table_exists: i64 = observer
877 .with(|connection| {
878 Ok(connection.query_row(
879 "SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name = '_vectors'",
880 [],
881 |row| row.get(0),
882 )?)
883 })
884 .unwrap();
885 assert_eq!(stored_docs, 0);
886 assert_eq!(vector_table_exists, 1);
887 }
888}