Skip to main content

uqa_storage_sqlite/
backend.rs

1//
2// Unified Query Algebra
3//
4// Copyright (c) 2023-2026 Cognica, Inc.
5//
6
7//! Session factories and physical stores for the `SQLite` backend.
8
9use crate::{
10    Catalog, ManagedConnection, SQLiteBTreeIndexStore, SQLiteDocumentStore, SQLiteHNSWIndex,
11    SQLiteIVFIndex, SQLiteInvertedIndex, SQLiteVectorIndex,
12};
13use std::{collections::BTreeMap, sync::Arc};
14use uqa_analysis::Analyzer;
15use uqa_core::{DocId, Value};
16use uqa_storage::{
17    CatalogFacade, DocumentStore, InvertedIndex, PersistentStorageBackend,
18    PersistentStorageIdentity, PersistentStorageProvider, PersistentStorageSession,
19    StorageBackendResult, StorageSavepointId, VectorIndex, VectorIndexOpenMode, VectorIndexSpec,
20};
21
22#[derive(Clone)]
23pub struct SQLiteStorageBackend {
24    conn: ManagedConnection,
25    native_initial_restore: bool,
26}
27
28impl SQLiteStorageBackend {
29    pub fn new(conn: ManagedConnection) -> Self {
30        Self {
31            conn,
32            native_initial_restore: false,
33        }
34    }
35
36    pub fn connection(&self) -> ManagedConnection {
37        self.conn.clone()
38    }
39
40    /// Create a backend whose stores use an independent transaction session
41    /// over the same physical `SQLite` pool.
42    #[must_use]
43    pub fn new_session(&self) -> Self {
44        Self::new(self.conn.new_session())
45    }
46}
47
48/// Database-level owner that creates isolated `SQLite` engine sessions.
49#[derive(Clone)]
50pub struct SQLiteStorageProvider {
51    connection: ManagedConnection,
52}
53
54impl SQLiteStorageProvider {
55    pub fn new(connection: ManagedConnection) -> Self {
56        Self { connection }
57    }
58
59    fn native_session(
60        &self,
61        cancellation: &uqa_core::CancellationToken,
62    ) -> StorageBackendResult<ManagedConnection> {
63        if !self.connection.is_native_record_session() {
64            self.connection
65                .bind_native_records(uqa_storage::mvcc::VersionedSessionOptions::default())?;
66        }
67        Ok(self.connection.new_session_with_cancellation(cancellation))
68    }
69}
70
71impl PersistentStorageProvider for SQLiteStorageProvider {
72    fn auxiliary_encryption_key(&self) -> Option<uqa_storage::StorageEncryptionKey> {
73        self.connection.auxiliary_encryption_key()
74    }
75
76    fn open_initial_session(&self) -> StorageBackendResult<PersistentStorageSession> {
77        let connection = self.connection.new_session();
78        let catalog: Arc<dyn CatalogFacade> =
79            Arc::new(Catalog::for_initial_restore(connection.clone()));
80        let backend: Arc<dyn PersistentStorageBackend> = Arc::new(SQLiteStorageBackend {
81            conn: connection,
82            native_initial_restore: true,
83        });
84        Ok(PersistentStorageSession::new(catalog, backend))
85    }
86
87    fn open_session(&self) -> StorageBackendResult<PersistentStorageSession> {
88        self.open_session_with_cancellation(&uqa_core::CancellationToken::new())
89    }
90
91    fn open_session_with_cancellation(
92        &self,
93        cancellation: &uqa_core::CancellationToken,
94    ) -> StorageBackendResult<PersistentStorageSession> {
95        cancellation.check()?;
96        let connection = self.native_session(cancellation)?;
97        let catalog: Arc<dyn CatalogFacade> = Arc::new(Catalog::open(connection.clone())?);
98        let backend: Arc<dyn PersistentStorageBackend> =
99            Arc::new(SQLiteStorageBackend::new(connection));
100        Ok(PersistentStorageSession::new(catalog, backend))
101    }
102
103    fn storage_identity(&self) -> StorageBackendResult<Option<PersistentStorageIdentity>> {
104        self.connection.storage_identity()
105    }
106}
107
108impl uqa_storage::mvcc::IdentifierAllocator for SQLiteStorageBackend {
109    fn identifier_watermark(&self, namespace: &[u8]) -> StorageBackendResult<Option<u64>> {
110        self.conn
111            .native_identifier_watermark(namespace)
112            .map_err(Into::into)
113    }
114
115    fn allocate_identifiers(
116        &self,
117        namespace: &[u8],
118        request: uqa_storage::mvcc::IdentifierRequest,
119    ) -> StorageBackendResult<uqa_storage::mvcc::IdentifierAllocation> {
120        self.conn
121            .allocate_native_identifiers(namespace, request)
122            .map_err(Into::into)
123    }
124
125    fn observe_identifier(
126        &self,
127        namespace: &[u8],
128        value: u64,
129    ) -> StorageBackendResult<uqa_storage::mvcc::ObservedIdentifier> {
130        self.conn
131            .observe_native_identifier(namespace, value)
132            .map_err(Into::into)
133    }
134}
135
136impl PersistentStorageBackend for SQLiteStorageBackend {
137    fn require_sequence_value_durability(&self) -> StorageBackendResult<()> {
138        self.conn
139            .require_sequence_value_durability()
140            .map_err(Into::into)
141    }
142
143    fn open_sequence_value_session(
144        &self,
145        cancellation: &uqa_core::CancellationToken,
146    ) -> StorageBackendResult<PersistentStorageSession> {
147        let connection = self.conn.new_sequence_value_session(cancellation)?;
148        let catalog: Arc<dyn CatalogFacade> = Arc::new(Catalog::open(connection.clone())?);
149        let backend: Arc<dyn PersistentStorageBackend> = Arc::new(Self::new(connection));
150        Ok(PersistentStorageSession::new(catalog, backend))
151    }
152
153    fn retention_control(&self) -> Option<uqa_storage::read_control::StorageReadControl> {
154        self.conn.retention_control()
155    }
156
157    fn notification_publications(
158        &self,
159    ) -> Option<&dyn uqa_storage::notifications::NotificationPublicationStore> {
160        Some(&self.conn)
161    }
162
163    fn serializable_session(&self) -> Option<&dyn uqa_storage::mvcc::SerializableSession> {
164        self.conn
165            .is_native_record_session()
166            .then_some(&self.conn as &dyn uqa_storage::mvcc::SerializableSession)
167    }
168
169    fn open_retained_read_session(
170        &self,
171        cancellation: &uqa_core::CancellationToken,
172    ) -> StorageBackendResult<PersistentStorageSession> {
173        let connection = self.conn.new_retained_read_session(cancellation)?;
174        let catalog: Arc<dyn CatalogFacade> = Arc::new(Catalog::open(connection.clone())?);
175        let backend: Arc<dyn PersistentStorageBackend> = Arc::new(Self::new(connection));
176        Ok(PersistentStorageSession::new(catalog, backend))
177    }
178
179    fn write_cancellation(&self) -> Option<uqa_core::CancellationToken> {
180        Some(self.conn.write_cancellation())
181    }
182
183    fn transaction_model(&self) -> uqa_storage::StorageTransactionModel {
184        self.conn.transaction_model()
185    }
186
187    fn identifier_allocator(&self) -> Option<&dyn uqa_storage::mvcc::IdentifierAllocator> {
188        self.conn.is_native_record_session().then_some(self)
189    }
190
191    fn transaction_affinity(&self) -> Option<uqa_storage::StorageSessionAffinity> {
192        Some(self.conn.transaction_affinity())
193    }
194
195    fn auxiliary_encryption_key(&self) -> Option<uqa_storage::StorageEncryptionKey> {
196        self.conn.auxiliary_encryption_key()
197    }
198
199    fn storage_identity(&self) -> StorageBackendResult<Option<PersistentStorageIdentity>> {
200        self.conn.storage_identity()
201    }
202
203    fn open_session(&self) -> StorageBackendResult<PersistentStorageSession> {
204        self.open_session_with_cancellation(&uqa_core::CancellationToken::new())
205    }
206
207    fn open_session_with_cancellation(
208        &self,
209        cancellation: &uqa_core::CancellationToken,
210    ) -> StorageBackendResult<PersistentStorageSession> {
211        cancellation.check()?;
212        let connection = self.conn.new_session_with_cancellation(cancellation);
213        let catalog: Arc<dyn CatalogFacade> = Arc::new(Catalog::open(connection.clone())?);
214        let backend: Arc<dyn PersistentStorageBackend> = Arc::new(Self::new(connection));
215        Ok(PersistentStorageSession::new(catalog, backend))
216    }
217
218    fn open_controlled_session(
219        &self,
220        control: &uqa_storage::read_control::StorageReadControl,
221    ) -> StorageBackendResult<PersistentStorageSession> {
222        let connection = self.conn.new_controlled_session(control)?;
223        let catalog: Arc<dyn CatalogFacade> = Arc::new(Catalog::open(connection.clone())?);
224        let backend: Arc<dyn PersistentStorageBackend> = Arc::new(Self::new(connection));
225        Ok(PersistentStorageSession::new(catalog, backend))
226    }
227
228    fn supports_concurrent_pinned_read_and_write(&self) -> bool {
229        self.conn.supports_concurrent_pinned_read_and_write()
230    }
231
232    fn document_store(&self, table: &str) -> Box<dyn DocumentStore> {
233        Box::new(SQLiteDocumentStore::new(self.conn.clone(), table))
234    }
235
236    fn inverted_index(&self, table: &str, analyzer: Analyzer) -> Box<dyn InvertedIndex> {
237        Box::new(SQLiteInvertedIndex::new(self.conn.clone(), table, analyzer))
238    }
239
240    fn vector_index(
241        &self,
242        table: &str,
243        field: &str,
244        dimensions: u32,
245        spec: VectorIndexSpec,
246        mode: VectorIndexOpenMode,
247    ) -> StorageBackendResult<Box<dyn VectorIndex>> {
248        let index: Box<dyn VectorIndex> = match spec {
249            VectorIndexSpec::DiskANN(_) => {
250                return Err(uqa_storage::StorageBackendError::Other(
251                    "DiskANN requires an actual catalog binding".into(),
252                ))
253            }
254            VectorIndexSpec::BruteForce => Box::new(SQLiteVectorIndex::new(
255                self.conn.clone(),
256                table,
257                field,
258                dimensions,
259            )),
260            VectorIndexSpec::IVF(params) => {
261                params.validate()?;
262                match mode {
263                    VectorIndexOpenMode::Create => Box::new(SQLiteIVFIndex::with_params(
264                        self.conn.clone(),
265                        table,
266                        field,
267                        dimensions,
268                        params.nlist,
269                        params.nprobe,
270                        params.train_threshold,
271                    )),
272                    VectorIndexOpenMode::Restore => Box::new(SQLiteIVFIndex::open_existing(
273                        self.conn.clone(),
274                        table,
275                        field,
276                        dimensions,
277                        params.nlist,
278                        params.nprobe,
279                        params.train_threshold,
280                    )),
281                }
282            }
283            VectorIndexSpec::HNSW(params) => {
284                params.validate()?;
285                match mode {
286                    VectorIndexOpenMode::Create => Box::new(SQLiteHNSWIndex::with_params(
287                        self.conn.clone(),
288                        table,
289                        field,
290                        dimensions,
291                        params,
292                    )),
293                    VectorIndexOpenMode::Restore => {
294                        let index = SQLiteHNSWIndex::open_existing(
295                            self.conn.clone(),
296                            table,
297                            field,
298                            dimensions,
299                            params,
300                        );
301                        index.validate_existing()?;
302                        Box::new(index)
303                    }
304                }
305            }
306        };
307        Ok(index)
308    }
309
310    fn diskann_index(
311        &self,
312        binding: uqa_storage::diskann_index::DiskANNIndexBinding<'_>,
313        options: uqa_storage::diskann_index::DiskANNIndexOptions,
314        temporary: &uqa_storage::diskann_index::build::DiskANNTemporaryBudget,
315        mode: VectorIndexOpenMode,
316    ) -> StorageBackendResult<Box<dyn VectorIndex>> {
317        let canonical = crate::vector_index::SQLiteDiskANNCanonical::new(
318            self.conn.clone(),
319            binding.table,
320            binding.field,
321            binding.dimensions,
322        )?;
323        if mode == VectorIndexOpenMode::Create {
324            canonical.create_index(
325                binding.index,
326                &*binding.resolver,
327                options,
328                temporary,
329                binding.control,
330            )?;
331        }
332        let handle = canonical.bind(
333            binding.index.clone(),
334            binding.resolver,
335            options.read,
336            binding.control,
337        )?;
338        Ok(Box::new(
339            uqa_storage::diskann_index::PersistentDiskANNIndex::new(handle, options, temporary)?,
340        ))
341    }
342
343    fn diskann_journal_pruner(
344        &self,
345        binding: uqa_storage::diskann_index::DiskANNIndexBinding<'_>,
346        max_record_bytes: usize,
347    ) -> StorageBackendResult<Box<dyn uqa_storage::diskann_index::changes::DiskANNJournalPruner>>
348    {
349        crate::vector_index::SQLiteDiskANNCanonical::new(
350            self.conn.clone(),
351            binding.table,
352            binding.field,
353            binding.dimensions,
354        )?
355        .journal_pruner(
356            binding.index,
357            binding.resolver,
358            max_record_bytes,
359            binding.control,
360        )
361    }
362
363    fn diskann_maintenance_source(
364        &self,
365        binding: uqa_storage::diskann_index::DiskANNIndexBinding<'_>,
366        max_record_bytes: usize,
367    ) -> StorageBackendResult<
368        Box<dyn uqa_storage::diskann_index::maintenance::DiskANNMaintenanceSource>,
369    > {
370        crate::vector_index::SQLiteDiskANNCanonical::new(
371            self.conn.clone(),
372            binding.table,
373            binding.field,
374            binding.dimensions,
375        )?
376        .maintenance_source(
377            binding.index,
378            binding.resolver,
379            max_record_bytes,
380            binding.control,
381        )
382    }
383
384    fn retire_diskann_index(
385        &self,
386        binding: uqa_storage::diskann_index::DiskANNIndexBinding<'_>,
387    ) -> StorageBackendResult<()> {
388        crate::vector_index::SQLiteDiskANNCanonical::new(
389            self.conn.clone(),
390            binding.table,
391            binding.field,
392            binding.dimensions,
393        )?
394        .retire_index(binding.index, &*binding.resolver, binding.control)
395    }
396
397    fn drop_vector_index_metadata(&self, table: &str, field: &str) -> StorageBackendResult<()> {
398        SQLiteIVFIndex::drop_metadata(&self.conn, table, field)?;
399        SQLiteHNSWIndex::drop_metadata(&self.conn, table, field)?;
400        Ok(())
401    }
402
403    fn persists_btree_indexes(&self) -> bool {
404        true
405    }
406
407    fn load_btree_index(
408        &self,
409        table: &str,
410        field: &uqa_storage::ValueIndexKey,
411    ) -> StorageBackendResult<Option<Vec<(DocId, Value)>>> {
412        Ok(SQLiteBTreeIndexStore::new(self.conn.clone()).load(table, field)?)
413    }
414
415    fn btree_index_fields(
416        &self,
417        table: &str,
418    ) -> StorageBackendResult<Vec<uqa_storage::ValueIndexKey>> {
419        Ok(SQLiteBTreeIndexStore::new(self.conn.clone()).fields(table)?)
420    }
421
422    fn read_btree_index_entry(
423        &self,
424        table: &str,
425        field: &uqa_storage::ValueIndexKey,
426        doc_id: DocId,
427    ) -> StorageBackendResult<uqa_storage::ValueIndexEntry> {
428        Ok(SQLiteBTreeIndexStore::new(self.conn.clone()).read_entry(table, field, doc_id)?)
429    }
430
431    fn btree_index_repairs(
432        &self,
433    ) -> StorageBackendResult<Vec<(String, uqa_storage::ValueIndexKey)>> {
434        Ok(SQLiteBTreeIndexStore::new(self.conn.clone()).repairs()?)
435    }
436
437    fn clear_btree_index_repair(
438        &self,
439        table: &str,
440        field: &uqa_storage::ValueIndexKey,
441    ) -> StorageBackendResult<()> {
442        SQLiteBTreeIndexStore::new(self.conn.clone()).clear_repair(table, field)?;
443        Ok(())
444    }
445
446    fn replace_btree_index(
447        &self,
448        table: &str,
449        field: &uqa_storage::ValueIndexKey,
450        values: &[(DocId, Value)],
451    ) -> StorageBackendResult<()> {
452        SQLiteBTreeIndexStore::new(self.conn.clone()).replace(table, field, values)?;
453        Ok(())
454    }
455
456    fn repair_btree_index(
457        &self,
458        table: &str,
459        field: &uqa_storage::ValueIndexKey,
460        _complete: &[(DocId, Value)],
461        stale_doc_ids: &[DocId],
462        missing: &[(DocId, Value)],
463    ) -> StorageBackendResult<()> {
464        SQLiteBTreeIndexStore::new(self.conn.clone()).repair(
465            table,
466            field,
467            stale_doc_ids,
468            missing,
469        )?;
470        Ok(())
471    }
472
473    fn replace_btree_indexes(
474        &self,
475        table: &str,
476        indexes: &[(&uqa_storage::ValueIndexKey, &[(DocId, Value)])],
477    ) -> StorageBackendResult<()> {
478        SQLiteBTreeIndexStore::new(self.conn.clone()).replace_many(table, indexes)?;
479        Ok(())
480    }
481
482    fn apply_btree_index_write(
483        &self,
484        table: &str,
485        doc_id: DocId,
486        values: Option<&BTreeMap<uqa_storage::ValueIndexKey, Value>>,
487    ) -> StorageBackendResult<()> {
488        SQLiteBTreeIndexStore::new(self.conn.clone()).apply_write(table, doc_id, values)?;
489        Ok(())
490    }
491
492    fn apply_unused_btree_index_write(
493        &self,
494        table: &str,
495        doc_id: DocId,
496        values: &BTreeMap<uqa_storage::ValueIndexKey, Value>,
497        namespace: uqa_storage::document_store::identifiers::DocumentIdNamespace,
498    ) -> StorageBackendResult<()> {
499        SQLiteBTreeIndexStore::new(self.conn.clone())
500            .apply_unused_write(table, doc_id, values, namespace)?;
501        Ok(())
502    }
503
504    fn drop_btree_index(
505        &self,
506        table: &str,
507        field: &uqa_storage::ValueIndexKey,
508    ) -> StorageBackendResult<()> {
509        SQLiteBTreeIndexStore::new(self.conn.clone()).drop_index(table, field)?;
510        Ok(())
511    }
512
513    fn clear_btree_indexes(&self, table: &str) -> StorageBackendResult<()> {
514        SQLiteBTreeIndexStore::new(self.conn.clone()).clear_table(table)?;
515        Ok(())
516    }
517
518    fn vacuum(&self) -> StorageBackendResult<()> {
519        self.conn.vacuum()?;
520        Ok(())
521    }
522
523    fn reclaim_obsolete(&self) -> StorageBackendResult<()> {
524        self.conn.reclaim_obsolete().map_err(Into::into)
525    }
526
527    fn begin_transaction(&self) -> StorageBackendResult<()> {
528        self.conn.begin_transaction()?;
529        Ok(())
530    }
531
532    fn begin_read_transaction(&self) -> StorageBackendResult<()> {
533        self.conn.begin_deferred_transaction()?;
534        Ok(())
535    }
536
537    fn begin_upgradeable_transaction(&self) -> StorageBackendResult<()> {
538        if self.native_initial_restore && !self.conn.is_native_record_session() {
539            self.conn.begin_native_initial_restore()?;
540        } else {
541            self.conn.begin_deferred_transaction()?;
542        }
543        Ok(())
544    }
545
546    fn refresh_transaction_snapshot(
547        &self,
548        cancellation: &uqa_core::CancellationToken,
549    ) -> StorageBackendResult<()> {
550        self.conn.refresh_transaction_snapshot(cancellation)?;
551        Ok(())
552    }
553
554    fn in_transaction(&self) -> bool {
555        self.conn.in_transaction()
556    }
557
558    fn transaction_has_written(&self) -> StorageBackendResult<bool> {
559        Ok(self.conn.transaction_has_written()?)
560    }
561
562    fn change_version(&self) -> StorageBackendResult<Option<u64>> {
563        Ok(self.conn.data_version()?)
564    }
565
566    fn read_view_revision(
567        &self,
568    ) -> StorageBackendResult<Option<uqa_storage::key_value::KeyValueReadRevision>> {
569        Ok(self.conn.read_view_revision()?)
570    }
571
572    fn committed_data_revision(
573        &self,
574    ) -> StorageBackendResult<Option<uqa_storage::CommittedDataRevision>> {
575        Ok(self.conn.committed_data_revision())
576    }
577
578    fn commit_monitor_version(&self) -> StorageBackendResult<Option<u64>> {
579        Ok(self.conn.commit_monitor_version()?)
580    }
581
582    fn change_version_monitor_is_nonblocking(&self) -> StorageBackendResult<bool> {
583        Ok(self.conn.data_version_monitor_is_nonblocking()?)
584    }
585
586    fn pin_transaction_snapshot(&self) -> StorageBackendResult<()> {
587        self.conn.pin_transaction_snapshot()?;
588        Ok(())
589    }
590
591    fn commit_transaction(&self) -> StorageBackendResult<()> {
592        self.conn.commit_transaction()?;
593        Ok(())
594    }
595
596    fn rollback_transaction(&self) -> StorageBackendResult<()> {
597        self.conn.rollback_transaction()?;
598        Ok(())
599    }
600
601    fn savepoint(&self, id: StorageSavepointId) -> StorageBackendResult<()> {
602        self.conn.savepoint(&id.backend_name())?;
603        Ok(())
604    }
605
606    fn release_savepoint(&self, id: StorageSavepointId) -> StorageBackendResult<()> {
607        self.conn.release_savepoint(&id.backend_name())?;
608        Ok(())
609    }
610
611    fn rollback_to_savepoint(&self, id: StorageSavepointId) -> StorageBackendResult<()> {
612        self.conn.rollback_to_savepoint(&id.backend_name())?;
613        Ok(())
614    }
615}
616
617#[cfg(test)]
618mod tests {
619    use std::collections::BTreeMap;
620
621    use uqa_analysis::analyzer::standard_analyzer;
622    use uqa_core::Value;
623    use uqa_storage::StorageBackendError;
624
625    use super::*;
626    use crate::{Catalog, SQLiteError};
627
628    #[test]
629    fn session_factories_preserve_a_bound_native_sessions_retention_options() {
630        let connection = ManagedConnection::open_in_memory().unwrap();
631        connection
632            .bind_native_records(uqa_storage::mvcc::VersionedSessionOptions {
633                retained_bytes: 8 << 20,
634            })
635            .unwrap();
636        let provider = SQLiteStorageProvider::new(connection);
637        let initial = provider.open_initial_session().unwrap();
638        let sibling = provider.open_session().unwrap();
639        initial.validate_transaction_affinity().unwrap();
640        sibling.validate_transaction_affinity().unwrap();
641        assert!(initial.backend.transaction_model().is_versioned());
642        assert_eq!(
643            initial.backend.transaction_model(),
644            sibling.backend.transaction_model()
645        );
646        assert_ne!(
647            initial.backend.transaction_affinity(),
648            sibling.backend.transaction_affinity()
649        );
650    }
651
652    #[test]
653    fn session_factory_reads_current_catalog_while_a_sibling_holds_a_writer_reservation() {
654        let directory = tempfile::tempdir().unwrap();
655        let connection = ManagedConnection::open_compressed(
656            &directory.path().join("session-writer-reservation.db"),
657            crate::SQLiteCompressionOptions::default(),
658        )
659        .unwrap();
660        let provider = SQLiteStorageProvider::new(connection);
661        let writer = provider.open_session().unwrap();
662        writer.backend.begin_transaction().unwrap();
663        writer
664            .catalog
665            .set_metadata("private-write", "uncommitted")
666            .unwrap();
667
668        let reader = provider.open_session().unwrap();
669        assert_eq!(reader.catalog.get_metadata("private-write").unwrap(), None);
670        writer.backend.commit_transaction().unwrap();
671        assert_eq!(
672            reader
673                .catalog
674                .get_metadata("private-write")
675                .unwrap()
676                .as_deref(),
677            Some("uncommitted")
678        );
679    }
680
681    #[test]
682    fn sqlite_backend_builds_document_index_and_vector_stores() {
683        let conn = ManagedConnection::open_in_memory().unwrap();
684        let _catalog = Catalog::open(conn.clone()).unwrap();
685        let backend = SQLiteStorageBackend::new(conn);
686
687        let mut doc = BTreeMap::new();
688        doc.insert("title".to_string(), Value::Str("rust storage".into()));
689        let mut docs = backend.document_store("articles");
690        docs.put(1, doc).unwrap();
691        assert_eq!(
692            docs.get_field(1, "title").unwrap(),
693            Some(Value::Str("rust storage".into()))
694        );
695
696        let mut inv = backend.inverted_index("articles", standard_analyzer("english"));
697        inv.add_document(
698            1,
699            BTreeMap::from([("title".to_string(), "rust storage".to_string())]),
700        )
701        .unwrap();
702        assert_eq!(inv.doc_freq("title", "rust").unwrap(), 1);
703
704        let mut vectors = backend
705            .vector_index(
706                "articles",
707                "embedding",
708                2,
709                VectorIndexSpec::IVF(uqa_storage::IVFIndexParams {
710                    nlist: 2,
711                    nprobe: 1,
712                    train_threshold: 2,
713                }),
714                VectorIndexOpenMode::Create,
715            )
716            .unwrap();
717        vectors.add(1, vec![1.0, 0.0]).unwrap();
718        let hits = vectors.search_knn(&[1.0, 0.0], 1).unwrap();
719        assert_eq!(hits.entries().len(), 1);
720        assert_eq!(hits.entries()[0].doc_id, 1);
721    }
722
723    #[test]
724    fn sqlite_backend_transaction_rolls_back_cross_store_writes() {
725        let conn = ManagedConnection::open_in_memory().unwrap();
726        let _catalog = Catalog::open(conn.clone()).unwrap();
727        let backend = SQLiteStorageBackend::new(conn);
728        let mut docs = backend.document_store("articles");
729        let mut inv = backend.inverted_index("articles", standard_analyzer("english"));
730
731        backend.begin_transaction().unwrap();
732        docs.put(
733            1,
734            BTreeMap::from([("title".to_string(), Value::Str("rollback".into()))]),
735        )
736        .unwrap();
737        inv.add_document(
738            1,
739            BTreeMap::from([("title".to_string(), "rollback".to_string())]),
740        )
741        .unwrap();
742        backend.rollback_transaction().unwrap();
743
744        assert_eq!(docs.len().unwrap(), 0);
745        assert_eq!(inv.doc_freq("title", "rollback").unwrap(), 0);
746    }
747
748    #[test]
749    fn sqlite_sessions_isolate_and_atomically_commit_cross_store_writes() {
750        let dir = tempfile::tempdir().unwrap();
751        let path = dir.path().join("cross-store-isolation.sqlite3");
752        let conn = ManagedConnection::open(&path).unwrap();
753        let catalog = Catalog::open(conn.clone()).unwrap();
754        let writer = SQLiteStorageBackend::new(conn.clone());
755        let observer_conn = conn.new_session();
756        let observer_catalog = Catalog::open(observer_conn.clone()).unwrap();
757        let observer = SQLiteStorageBackend::new(observer_conn);
758
759        let mut writer_docs = writer.document_store("articles");
760        let mut writer_inv = writer.inverted_index("articles", standard_analyzer("english"));
761        let mut writer_vectors = writer
762            .vector_index(
763                "articles",
764                "embedding",
765                2,
766                VectorIndexSpec::BruteForce,
767                VectorIndexOpenMode::Create,
768            )
769            .unwrap();
770        let observer_docs = observer.document_store("articles");
771        let observer_inv = observer.inverted_index("articles", standard_analyzer("english"));
772        let observer_vectors = observer
773            .vector_index(
774                "articles",
775                "embedding",
776                2,
777                VectorIndexSpec::BruteForce,
778                VectorIndexOpenMode::Restore,
779            )
780            .unwrap();
781
782        writer.begin_transaction().unwrap();
783        writer_docs
784            .put(
785                1,
786                BTreeMap::from([("title".to_string(), Value::Str("atomic rust".into()))]),
787            )
788            .unwrap();
789        writer_inv
790            .add_document(
791                1,
792                BTreeMap::from([("title".to_string(), "atomic rust".to_string())]),
793            )
794            .unwrap();
795        writer_vectors.add(1, vec![1.0, 0.0]).unwrap();
796        catalog
797            .save_scoring_params("transactional", r#"{"alpha":1.0}"#)
798            .unwrap();
799
800        assert_eq!(writer_docs.len().unwrap(), 1);
801        assert_eq!(writer_inv.doc_freq("title", "rust").unwrap(), 1);
802        assert_eq!(writer_vectors.count().unwrap(), 1);
803        assert!(catalog
804            .load_scoring_params("transactional")
805            .unwrap()
806            .is_some());
807
808        assert_eq!(observer_docs.len().unwrap(), 0);
809        assert_eq!(observer_inv.doc_freq("title", "rust").unwrap(), 0);
810        assert_eq!(observer_vectors.count().unwrap(), 0);
811        assert!(observer_catalog
812            .load_scoring_params("transactional")
813            .unwrap()
814            .is_none());
815
816        writer.commit_transaction().unwrap();
817        assert_eq!(observer_docs.len().unwrap(), 1);
818        assert_eq!(observer_inv.doc_freq("title", "rust").unwrap(), 1);
819        assert_eq!(observer_vectors.count().unwrap(), 1);
820        assert!(observer_catalog
821            .load_scoring_params("transactional")
822            .unwrap()
823            .is_some());
824    }
825
826    #[test]
827    fn ignored_legacy_index_error_cannot_commit_partial_document_write() {
828        let dir = tempfile::tempdir().unwrap();
829        let path = dir.path().join("ignored-index-error.sqlite3");
830        let conn = ManagedConnection::open(&path).unwrap();
831        let _catalog = Catalog::open(conn.clone()).unwrap();
832        let backend = SQLiteStorageBackend::new(conn.clone());
833        let observer = conn.new_session();
834        let mut docs = backend.document_store("articles");
835        let mut vectors = backend
836            .vector_index(
837                "articles",
838                "embedding",
839                2,
840                VectorIndexSpec::BruteForce,
841                VectorIndexOpenMode::Create,
842            )
843            .unwrap();
844
845        backend.begin_transaction().unwrap();
846        docs.put(
847            1,
848            BTreeMap::from([("title".to_string(), Value::Str("must roll back".into()))]),
849        )
850        .unwrap();
851        conn.with(|connection| {
852            connection.execute("DROP TABLE _vectors", [])?;
853            Ok(())
854        })
855        .unwrap();
856        // The vector write reports its error directly. Even if a caller
857        // ignores that Result, the managed transaction is poisoned and the
858        // partial document write cannot commit.
859        let ignored = vectors.add(1, vec![1.0, 0.0]);
860        assert!(ignored.is_err());
861        assert!(matches!(
862            backend.commit_transaction(),
863            Err(StorageBackendError::Backend { source, .. })
864                if matches!(source.downcast_ref::<SQLiteError>(), Some(SQLiteError::TransactionAborted(_)))
865        ));
866
867        let stored_docs: i64 = observer
868            .with(|connection| {
869                Ok(connection.query_row(
870                    "SELECT COUNT(*) FROM _documents WHERE table_name = 'articles'",
871                    [],
872                    |row| row.get(0),
873                )?)
874            })
875            .unwrap();
876        let vector_table_exists: i64 = observer
877            .with(|connection| {
878                Ok(connection.query_row(
879                    "SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name = '_vectors'",
880                    [],
881                    |row| row.get(0),
882                )?)
883            })
884            .unwrap();
885        assert_eq!(stored_docs, 0);
886        assert_eq!(vector_table_exists, 1);
887    }
888}