Skip to main content

uqa_storage_sqlite/
backend.rs

1//
2// Unified Query Algebra
3//
4// Copyright (c) 2023-2026 Cognica, Inc.
5//
6
7//! Session factories and physical stores for the `SQLite` backend.
8
9use crate::{
10    Catalog, ManagedConnection, SQLiteBTreeIndexStore, SQLiteDocumentStore, SQLiteHNSWIndex,
11    SQLiteIVFIndex, SQLiteInvertedIndex, SQLiteVectorIndex,
12};
13use std::{collections::BTreeMap, sync::Arc};
14use uqa_analysis::Analyzer;
15use uqa_core::{DocId, Value};
16use uqa_storage::{
17    CatalogFacade, DocumentStore, InvertedIndex, PersistentStorageBackend,
18    PersistentStorageIdentity, PersistentStorageProvider, PersistentStorageSession,
19    StorageBackendResult, StorageSavepointId, VectorIndex, VectorIndexOpenMode, VectorIndexSpec,
20};
21
22#[derive(Clone)]
23pub struct SQLiteStorageBackend {
24    conn: ManagedConnection,
25    native_initial_restore: bool,
26}
27
28impl SQLiteStorageBackend {
29    pub fn new(conn: ManagedConnection) -> Self {
30        Self {
31            conn,
32            native_initial_restore: false,
33        }
34    }
35
36    pub fn connection(&self) -> ManagedConnection {
37        self.conn.clone()
38    }
39
40    /// Create a backend whose stores use an independent transaction session
41    /// over the same physical `SQLite` pool.
42    #[must_use]
43    pub fn new_session(&self) -> Self {
44        Self::new(self.conn.new_session())
45    }
46}
47
48/// Database-level owner that creates isolated `SQLite` engine sessions.
49#[derive(Clone)]
50pub struct SQLiteStorageProvider {
51    connection: ManagedConnection,
52}
53
54impl SQLiteStorageProvider {
55    pub fn new(connection: ManagedConnection) -> Self {
56        Self { connection }
57    }
58
59    fn native_session(
60        &self,
61        cancellation: &uqa_core::CancellationToken,
62    ) -> StorageBackendResult<ManagedConnection> {
63        if !self.connection.is_native_record_session() {
64            self.connection
65                .bind_native_records(uqa_storage::mvcc::VersionedSessionOptions::default())?;
66        }
67        Ok(self.connection.new_session_with_cancellation(cancellation))
68    }
69}
70
71impl PersistentStorageProvider for SQLiteStorageProvider {
72    fn auxiliary_encryption_key(&self) -> Option<uqa_storage::StorageEncryptionKey> {
73        self.connection.auxiliary_encryption_key()
74    }
75
76    fn open_initial_session(&self) -> StorageBackendResult<PersistentStorageSession> {
77        let connection = self.connection.new_session();
78        let catalog: Arc<dyn CatalogFacade> =
79            Arc::new(Catalog::for_initial_restore(connection.clone()));
80        let backend: Arc<dyn PersistentStorageBackend> = Arc::new(SQLiteStorageBackend {
81            conn: connection,
82            native_initial_restore: true,
83        });
84        Ok(PersistentStorageSession::new(catalog, backend))
85    }
86
87    fn open_session(&self) -> StorageBackendResult<PersistentStorageSession> {
88        self.open_session_with_cancellation(&uqa_core::CancellationToken::new())
89    }
90
91    fn open_session_with_cancellation(
92        &self,
93        cancellation: &uqa_core::CancellationToken,
94    ) -> StorageBackendResult<PersistentStorageSession> {
95        cancellation.check()?;
96        let connection = self.native_session(cancellation)?;
97        let catalog: Arc<dyn CatalogFacade> = Arc::new(Catalog::open(connection.clone())?);
98        let backend: Arc<dyn PersistentStorageBackend> =
99            Arc::new(SQLiteStorageBackend::new(connection));
100        Ok(PersistentStorageSession::new(catalog, backend))
101    }
102
103    fn storage_identity(&self) -> StorageBackendResult<Option<PersistentStorageIdentity>> {
104        self.connection.storage_identity()
105    }
106}
107
108impl uqa_storage::mvcc::IdentifierAllocator for SQLiteStorageBackend {
109    fn identifier_watermark(&self, namespace: &[u8]) -> StorageBackendResult<Option<u64>> {
110        self.conn
111            .native_identifier_watermark(namespace)
112            .map_err(Into::into)
113    }
114
115    fn allocate_identifiers(
116        &self,
117        namespace: &[u8],
118        request: uqa_storage::mvcc::IdentifierRequest,
119    ) -> StorageBackendResult<uqa_storage::mvcc::IdentifierAllocation> {
120        self.conn
121            .allocate_native_identifiers(namespace, request)
122            .map_err(Into::into)
123    }
124
125    fn observe_identifier(
126        &self,
127        namespace: &[u8],
128        value: u64,
129    ) -> StorageBackendResult<uqa_storage::mvcc::ObservedIdentifier> {
130        self.conn
131            .observe_native_identifier(namespace, value)
132            .map_err(Into::into)
133    }
134}
135
136impl PersistentStorageBackend for SQLiteStorageBackend {
137    fn require_sequence_value_durability(&self) -> StorageBackendResult<()> {
138        self.conn
139            .require_sequence_value_durability()
140            .map_err(Into::into)
141    }
142
143    fn open_sequence_value_session(
144        &self,
145        cancellation: &uqa_core::CancellationToken,
146    ) -> StorageBackendResult<PersistentStorageSession> {
147        let connection = self.conn.new_sequence_value_session(cancellation)?;
148        let catalog: Arc<dyn CatalogFacade> = Arc::new(Catalog::open(connection.clone())?);
149        let backend: Arc<dyn PersistentStorageBackend> = Arc::new(Self::new(connection));
150        Ok(PersistentStorageSession::new(catalog, backend))
151    }
152
153    fn retention_control(&self) -> Option<uqa_storage::read_control::StorageReadControl> {
154        self.conn.retention_control()
155    }
156
157    fn notification_publications(
158        &self,
159    ) -> Option<&dyn uqa_storage::notifications::NotificationPublicationStore> {
160        Some(&self.conn)
161    }
162
163    fn serializable_session(&self) -> Option<&dyn uqa_storage::mvcc::SerializableSession> {
164        self.conn
165            .is_native_record_session()
166            .then_some(&self.conn as &dyn uqa_storage::mvcc::SerializableSession)
167    }
168
169    fn open_retained_read_session(
170        &self,
171        cancellation: &uqa_core::CancellationToken,
172    ) -> StorageBackendResult<PersistentStorageSession> {
173        let connection = self.conn.new_retained_read_session(cancellation)?;
174        let catalog: Arc<dyn CatalogFacade> = Arc::new(Catalog::open(connection.clone())?);
175        let backend: Arc<dyn PersistentStorageBackend> = Arc::new(Self::new(connection));
176        Ok(PersistentStorageSession::new(catalog, backend))
177    }
178
179    fn capture_committed_read_boundary(
180        &self,
181    ) -> StorageBackendResult<Arc<uqa_storage::mvcc::CommittedReadBoundary>> {
182        self.conn.capture_committed_read_boundary()
183    }
184
185    fn open_read_session_with_private_changes(
186        &self,
187        boundary: &uqa_storage::mvcc::CommittedReadBoundary,
188        cancellation: &uqa_core::CancellationToken,
189    ) -> StorageBackendResult<PersistentStorageSession> {
190        let connection = self
191            .conn
192            .new_read_session_with_private_changes(boundary, cancellation)?;
193        let catalog: Arc<dyn CatalogFacade> = Arc::new(Catalog::open(connection.clone())?);
194        let backend: Arc<dyn PersistentStorageBackend> = Arc::new(Self::new(connection));
195        Ok(PersistentStorageSession::new(catalog, backend))
196    }
197
198    fn write_cancellation(&self) -> Option<uqa_core::CancellationToken> {
199        Some(self.conn.write_cancellation())
200    }
201
202    fn transaction_model(&self) -> uqa_storage::StorageTransactionModel {
203        self.conn.transaction_model()
204    }
205
206    fn identifier_allocator(&self) -> Option<&dyn uqa_storage::mvcc::IdentifierAllocator> {
207        self.conn.is_native_record_session().then_some(self)
208    }
209
210    fn transaction_affinity(&self) -> Option<uqa_storage::StorageSessionAffinity> {
211        Some(self.conn.transaction_affinity())
212    }
213
214    fn auxiliary_encryption_key(&self) -> Option<uqa_storage::StorageEncryptionKey> {
215        self.conn.auxiliary_encryption_key()
216    }
217
218    fn storage_identity(&self) -> StorageBackendResult<Option<PersistentStorageIdentity>> {
219        self.conn.storage_identity()
220    }
221
222    fn open_session(&self) -> StorageBackendResult<PersistentStorageSession> {
223        self.open_session_with_cancellation(&uqa_core::CancellationToken::new())
224    }
225
226    fn open_session_with_cancellation(
227        &self,
228        cancellation: &uqa_core::CancellationToken,
229    ) -> StorageBackendResult<PersistentStorageSession> {
230        cancellation.check()?;
231        let connection = self.conn.new_session_with_cancellation(cancellation);
232        let catalog: Arc<dyn CatalogFacade> = Arc::new(Catalog::open(connection.clone())?);
233        let backend: Arc<dyn PersistentStorageBackend> = Arc::new(Self::new(connection));
234        Ok(PersistentStorageSession::new(catalog, backend))
235    }
236
237    fn open_controlled_session(
238        &self,
239        control: &uqa_storage::read_control::StorageReadControl,
240    ) -> StorageBackendResult<PersistentStorageSession> {
241        let connection = self.conn.new_controlled_session(control)?;
242        let catalog: Arc<dyn CatalogFacade> = Arc::new(Catalog::open(connection.clone())?);
243        let backend: Arc<dyn PersistentStorageBackend> = Arc::new(Self::new(connection));
244        Ok(PersistentStorageSession::new(catalog, backend))
245    }
246
247    fn supports_concurrent_pinned_read_and_write(&self) -> bool {
248        self.conn.supports_concurrent_pinned_read_and_write()
249    }
250
251    fn document_store(&self, table: &str) -> Box<dyn DocumentStore> {
252        Box::new(SQLiteDocumentStore::new(self.conn.clone(), table))
253    }
254
255    fn inverted_index(&self, table: &str, analyzer: Analyzer) -> Box<dyn InvertedIndex> {
256        Box::new(SQLiteInvertedIndex::new(self.conn.clone(), table, analyzer))
257    }
258
259    fn vector_index(
260        &self,
261        table: &str,
262        field: &str,
263        dimensions: u32,
264        spec: VectorIndexSpec,
265        mode: VectorIndexOpenMode,
266    ) -> StorageBackendResult<Box<dyn VectorIndex>> {
267        let index: Box<dyn VectorIndex> = match spec {
268            VectorIndexSpec::DiskANN(_) => {
269                return Err(uqa_storage::StorageBackendError::Other(
270                    "DiskANN requires an actual catalog binding".into(),
271                ))
272            }
273            VectorIndexSpec::BruteForce => Box::new(SQLiteVectorIndex::new(
274                self.conn.clone(),
275                table,
276                field,
277                dimensions,
278            )),
279            VectorIndexSpec::IVF(params) => {
280                params.validate()?;
281                match mode {
282                    VectorIndexOpenMode::Create => Box::new(SQLiteIVFIndex::with_params(
283                        self.conn.clone(),
284                        table,
285                        field,
286                        dimensions,
287                        params.nlist,
288                        params.nprobe,
289                        params.train_threshold,
290                    )),
291                    VectorIndexOpenMode::Restore => Box::new(SQLiteIVFIndex::open_existing(
292                        self.conn.clone(),
293                        table,
294                        field,
295                        dimensions,
296                        params.nlist,
297                        params.nprobe,
298                        params.train_threshold,
299                    )),
300                }
301            }
302            VectorIndexSpec::HNSW(params) => {
303                params.validate()?;
304                match mode {
305                    VectorIndexOpenMode::Create => Box::new(SQLiteHNSWIndex::with_params(
306                        self.conn.clone(),
307                        table,
308                        field,
309                        dimensions,
310                        params,
311                    )),
312                    VectorIndexOpenMode::Restore => {
313                        let index = SQLiteHNSWIndex::open_existing(
314                            self.conn.clone(),
315                            table,
316                            field,
317                            dimensions,
318                            params,
319                        );
320                        index.validate_existing()?;
321                        Box::new(index)
322                    }
323                }
324            }
325        };
326        Ok(index)
327    }
328
329    fn diskann_index(
330        &self,
331        binding: uqa_storage::diskann_index::DiskANNIndexBinding<'_>,
332        options: uqa_storage::diskann_index::DiskANNIndexOptions,
333        temporary: &uqa_storage::diskann_index::build::DiskANNTemporaryBudget,
334        mode: VectorIndexOpenMode,
335    ) -> StorageBackendResult<Box<dyn VectorIndex>> {
336        let canonical = crate::vector_index::SQLiteDiskANNCanonical::new(
337            self.conn.clone(),
338            binding.table,
339            binding.field,
340            binding.dimensions,
341        )?;
342        if mode == VectorIndexOpenMode::Create {
343            canonical.create_index(
344                binding.index,
345                &*binding.resolver,
346                options,
347                temporary,
348                binding.control,
349            )?;
350        }
351        let handle = canonical.bind(
352            binding.index.clone(),
353            binding.resolver,
354            options.read,
355            binding.control,
356        )?;
357        Ok(Box::new(
358            uqa_storage::diskann_index::PersistentDiskANNIndex::new(handle, options, temporary)?,
359        ))
360    }
361
362    fn diskann_journal_pruner(
363        &self,
364        binding: uqa_storage::diskann_index::DiskANNIndexBinding<'_>,
365        max_record_bytes: usize,
366    ) -> StorageBackendResult<Box<dyn uqa_storage::diskann_index::changes::DiskANNJournalPruner>>
367    {
368        crate::vector_index::SQLiteDiskANNCanonical::new(
369            self.conn.clone(),
370            binding.table,
371            binding.field,
372            binding.dimensions,
373        )?
374        .journal_pruner(
375            binding.index,
376            binding.resolver,
377            max_record_bytes,
378            binding.control,
379        )
380    }
381
382    fn diskann_maintenance_source(
383        &self,
384        binding: uqa_storage::diskann_index::DiskANNIndexBinding<'_>,
385        max_record_bytes: usize,
386    ) -> StorageBackendResult<
387        Box<dyn uqa_storage::diskann_index::maintenance::DiskANNMaintenanceSource>,
388    > {
389        crate::vector_index::SQLiteDiskANNCanonical::new(
390            self.conn.clone(),
391            binding.table,
392            binding.field,
393            binding.dimensions,
394        )?
395        .maintenance_source(
396            binding.index,
397            binding.resolver,
398            max_record_bytes,
399            binding.control,
400        )
401    }
402
403    fn retire_diskann_index(
404        &self,
405        binding: uqa_storage::diskann_index::DiskANNIndexBinding<'_>,
406    ) -> StorageBackendResult<()> {
407        crate::vector_index::SQLiteDiskANNCanonical::new(
408            self.conn.clone(),
409            binding.table,
410            binding.field,
411            binding.dimensions,
412        )?
413        .retire_index(binding.index, &*binding.resolver, binding.control)
414    }
415
416    fn drop_vector_index_metadata(&self, table: &str, field: &str) -> StorageBackendResult<()> {
417        SQLiteIVFIndex::drop_metadata(&self.conn, table, field)?;
418        SQLiteHNSWIndex::drop_metadata(&self.conn, table, field)?;
419        Ok(())
420    }
421
422    fn persists_btree_indexes(&self) -> bool {
423        true
424    }
425
426    fn load_btree_index(
427        &self,
428        table: &str,
429        field: &uqa_storage::ValueIndexKey,
430    ) -> StorageBackendResult<Option<Vec<(DocId, Value)>>> {
431        Ok(SQLiteBTreeIndexStore::new(self.conn.clone()).load(table, field)?)
432    }
433
434    fn btree_index_fields(
435        &self,
436        table: &str,
437    ) -> StorageBackendResult<Vec<uqa_storage::ValueIndexKey>> {
438        Ok(SQLiteBTreeIndexStore::new(self.conn.clone()).fields(table)?)
439    }
440
441    fn probe_btree_index_equal(
442        &self,
443        table: &str,
444        field: &uqa_storage::ValueIndexKey,
445        value: &Value,
446    ) -> StorageBackendResult<Option<Vec<DocId>>> {
447        Ok(SQLiteBTreeIndexStore::new(self.conn.clone()).probe_equal(table, field, value)?)
448    }
449
450    fn read_btree_index_entry(
451        &self,
452        table: &str,
453        field: &uqa_storage::ValueIndexKey,
454        doc_id: DocId,
455    ) -> StorageBackendResult<uqa_storage::ValueIndexEntry> {
456        Ok(SQLiteBTreeIndexStore::new(self.conn.clone()).read_entry(table, field, doc_id)?)
457    }
458
459    fn btree_index_repairs(
460        &self,
461    ) -> StorageBackendResult<Vec<(String, uqa_storage::ValueIndexKey)>> {
462        Ok(SQLiteBTreeIndexStore::new(self.conn.clone()).repairs()?)
463    }
464
465    fn clear_btree_index_repair(
466        &self,
467        table: &str,
468        field: &uqa_storage::ValueIndexKey,
469    ) -> StorageBackendResult<()> {
470        SQLiteBTreeIndexStore::new(self.conn.clone()).clear_repair(table, field)?;
471        Ok(())
472    }
473
474    fn replace_btree_index(
475        &self,
476        table: &str,
477        field: &uqa_storage::ValueIndexKey,
478        values: &[(DocId, Value)],
479    ) -> StorageBackendResult<()> {
480        SQLiteBTreeIndexStore::new(self.conn.clone()).replace(table, field, values)?;
481        Ok(())
482    }
483
484    fn repair_btree_index(
485        &self,
486        table: &str,
487        field: &uqa_storage::ValueIndexKey,
488        _complete: &[(DocId, Value)],
489        stale_doc_ids: &[DocId],
490        missing: &[(DocId, Value)],
491    ) -> StorageBackendResult<()> {
492        SQLiteBTreeIndexStore::new(self.conn.clone()).repair(
493            table,
494            field,
495            stale_doc_ids,
496            missing,
497        )?;
498        Ok(())
499    }
500
501    fn replace_btree_indexes(
502        &self,
503        table: &str,
504        indexes: &[(&uqa_storage::ValueIndexKey, &[(DocId, Value)])],
505    ) -> StorageBackendResult<()> {
506        SQLiteBTreeIndexStore::new(self.conn.clone()).replace_many(table, indexes)?;
507        Ok(())
508    }
509
510    fn apply_btree_index_write(
511        &self,
512        table: &str,
513        doc_id: DocId,
514        values: Option<&BTreeMap<uqa_storage::ValueIndexKey, Value>>,
515    ) -> StorageBackendResult<()> {
516        SQLiteBTreeIndexStore::new(self.conn.clone()).apply_write(table, doc_id, values)?;
517        Ok(())
518    }
519
520    fn apply_unused_btree_index_write(
521        &self,
522        table: &str,
523        doc_id: DocId,
524        values: &BTreeMap<uqa_storage::ValueIndexKey, Value>,
525        namespace: uqa_storage::document_store::identifiers::DocumentIdNamespace,
526    ) -> StorageBackendResult<()> {
527        SQLiteBTreeIndexStore::new(self.conn.clone())
528            .apply_unused_write(table, doc_id, values, namespace)?;
529        Ok(())
530    }
531
532    fn drop_btree_index(
533        &self,
534        table: &str,
535        field: &uqa_storage::ValueIndexKey,
536    ) -> StorageBackendResult<()> {
537        SQLiteBTreeIndexStore::new(self.conn.clone()).drop_index(table, field)?;
538        Ok(())
539    }
540
541    fn clear_btree_indexes(&self, table: &str) -> StorageBackendResult<()> {
542        SQLiteBTreeIndexStore::new(self.conn.clone()).clear_table(table)?;
543        Ok(())
544    }
545
546    fn vacuum(&self) -> StorageBackendResult<()> {
547        self.conn.vacuum()?;
548        Ok(())
549    }
550
551    fn reclaim_obsolete(&self) -> StorageBackendResult<()> {
552        self.conn.reclaim_obsolete().map_err(Into::into)
553    }
554
555    fn begin_transaction(&self) -> StorageBackendResult<()> {
556        self.conn.begin_transaction()?;
557        Ok(())
558    }
559
560    fn begin_read_transaction(&self) -> StorageBackendResult<()> {
561        self.conn.begin_deferred_transaction()?;
562        Ok(())
563    }
564
565    fn begin_upgradeable_transaction(&self) -> StorageBackendResult<()> {
566        if self.native_initial_restore && !self.conn.is_native_record_session() {
567            self.conn.begin_native_initial_restore()?;
568        } else {
569            self.conn.begin_deferred_transaction()?;
570        }
571        Ok(())
572    }
573
574    fn refresh_transaction_snapshot(
575        &self,
576        cancellation: &uqa_core::CancellationToken,
577    ) -> StorageBackendResult<()> {
578        self.conn.refresh_transaction_snapshot(cancellation)?;
579        Ok(())
580    }
581
582    fn in_transaction(&self) -> bool {
583        self.conn.in_transaction()
584    }
585
586    fn transaction_has_written(&self) -> StorageBackendResult<bool> {
587        Ok(self.conn.transaction_has_written()?)
588    }
589
590    fn change_version(&self) -> StorageBackendResult<Option<u64>> {
591        Ok(self.conn.data_version()?)
592    }
593
594    fn read_view_revision(
595        &self,
596    ) -> StorageBackendResult<Option<uqa_storage::key_value::KeyValueReadRevision>> {
597        Ok(self.conn.read_view_revision()?)
598    }
599
600    fn committed_data_revision(
601        &self,
602    ) -> StorageBackendResult<Option<uqa_storage::CommittedDataRevision>> {
603        Ok(self.conn.committed_data_revision())
604    }
605
606    fn commit_monitor_version(&self) -> StorageBackendResult<Option<u64>> {
607        Ok(self.conn.commit_monitor_version()?)
608    }
609
610    fn change_version_monitor_is_nonblocking(&self) -> StorageBackendResult<bool> {
611        Ok(self.conn.data_version_monitor_is_nonblocking()?)
612    }
613
614    fn pin_transaction_snapshot(&self) -> StorageBackendResult<()> {
615        self.conn.pin_transaction_snapshot()?;
616        Ok(())
617    }
618
619    fn commit_transaction(&self) -> StorageBackendResult<()> {
620        self.conn.commit_transaction()?;
621        Ok(())
622    }
623
624    fn rollback_transaction(&self) -> StorageBackendResult<()> {
625        self.conn.rollback_transaction()?;
626        Ok(())
627    }
628
629    fn savepoint(&self, id: StorageSavepointId) -> StorageBackendResult<()> {
630        self.conn.savepoint(&id.backend_name())?;
631        Ok(())
632    }
633
634    fn release_savepoint(&self, id: StorageSavepointId) -> StorageBackendResult<()> {
635        self.conn.release_savepoint(&id.backend_name())?;
636        Ok(())
637    }
638
639    fn rollback_to_savepoint(&self, id: StorageSavepointId) -> StorageBackendResult<()> {
640        self.conn.rollback_to_savepoint(&id.backend_name())?;
641        Ok(())
642    }
643}
644
645#[cfg(test)]
646mod tests {
647    use std::collections::BTreeMap;
648
649    use uqa_analysis::analyzer::standard_analyzer;
650    use uqa_core::Value;
651    use uqa_storage::StorageBackendError;
652
653    use super::*;
654    use crate::{Catalog, SQLiteError};
655
656    #[test]
657    fn session_factories_preserve_a_bound_native_sessions_retention_options() {
658        let connection = ManagedConnection::open_in_memory().unwrap();
659        connection
660            .bind_native_records(uqa_storage::mvcc::VersionedSessionOptions {
661                retained_bytes: 8 << 20,
662            })
663            .unwrap();
664        let provider = SQLiteStorageProvider::new(connection);
665        let initial = provider.open_initial_session().unwrap();
666        let sibling = provider.open_session().unwrap();
667        initial.validate_transaction_affinity().unwrap();
668        sibling.validate_transaction_affinity().unwrap();
669        assert!(initial.backend.transaction_model().is_versioned());
670        assert_eq!(
671            initial.backend.transaction_model(),
672            sibling.backend.transaction_model()
673        );
674        assert_ne!(
675            initial.backend.transaction_affinity(),
676            sibling.backend.transaction_affinity()
677        );
678    }
679
680    #[test]
681    fn session_factory_reads_current_catalog_while_a_sibling_holds_a_writer_reservation() {
682        let directory = tempfile::tempdir().unwrap();
683        let connection = ManagedConnection::open_compressed(
684            &directory.path().join("session-writer-reservation.db"),
685            crate::SQLiteCompressionOptions::default(),
686        )
687        .unwrap();
688        let provider = SQLiteStorageProvider::new(connection);
689        let writer = provider.open_session().unwrap();
690        writer.backend.begin_transaction().unwrap();
691        writer
692            .catalog
693            .set_metadata("private-write", "uncommitted")
694            .unwrap();
695
696        let reader = provider.open_session().unwrap();
697        assert_eq!(reader.catalog.get_metadata("private-write").unwrap(), None);
698        writer.backend.commit_transaction().unwrap();
699        assert_eq!(
700            reader
701                .catalog
702                .get_metadata("private-write")
703                .unwrap()
704                .as_deref(),
705            Some("uncommitted")
706        );
707    }
708
709    #[test]
710    fn sqlite_backend_builds_document_index_and_vector_stores() {
711        let conn = ManagedConnection::open_in_memory().unwrap();
712        let _catalog = Catalog::open(conn.clone()).unwrap();
713        let backend = SQLiteStorageBackend::new(conn);
714
715        let mut doc = BTreeMap::new();
716        doc.insert("title".to_string(), Value::Str("rust storage".into()));
717        let mut docs = backend.document_store("articles");
718        docs.put(1, doc).unwrap();
719        assert_eq!(
720            docs.get_field(1, "title").unwrap(),
721            Some(Value::Str("rust storage".into()))
722        );
723
724        let mut inv = backend.inverted_index("articles", standard_analyzer("english"));
725        inv.add_document(
726            1,
727            BTreeMap::from([("title".to_string(), "rust storage".to_string())]),
728        )
729        .unwrap();
730        assert_eq!(inv.doc_freq("title", "rust").unwrap(), 1);
731
732        let mut vectors = backend
733            .vector_index(
734                "articles",
735                "embedding",
736                2,
737                VectorIndexSpec::IVF(uqa_storage::IVFIndexParams {
738                    nlist: 2,
739                    nprobe: 1,
740                    train_threshold: 2,
741                }),
742                VectorIndexOpenMode::Create,
743            )
744            .unwrap();
745        vectors.add(1, vec![1.0, 0.0]).unwrap();
746        let hits = vectors.search_knn(&[1.0, 0.0], 1).unwrap();
747        assert_eq!(hits.entries().len(), 1);
748        assert_eq!(hits.entries()[0].doc_id, 1);
749    }
750
751    #[test]
752    fn sqlite_backend_transaction_rolls_back_cross_store_writes() {
753        let conn = ManagedConnection::open_in_memory().unwrap();
754        let _catalog = Catalog::open(conn.clone()).unwrap();
755        let backend = SQLiteStorageBackend::new(conn);
756        let mut docs = backend.document_store("articles");
757        let mut inv = backend.inverted_index("articles", standard_analyzer("english"));
758
759        backend.begin_transaction().unwrap();
760        docs.put(
761            1,
762            BTreeMap::from([("title".to_string(), Value::Str("rollback".into()))]),
763        )
764        .unwrap();
765        inv.add_document(
766            1,
767            BTreeMap::from([("title".to_string(), "rollback".to_string())]),
768        )
769        .unwrap();
770        backend.rollback_transaction().unwrap();
771
772        assert_eq!(docs.len().unwrap(), 0);
773        assert_eq!(inv.doc_freq("title", "rollback").unwrap(), 0);
774    }
775
776    #[test]
777    fn sqlite_sessions_isolate_and_atomically_commit_cross_store_writes() {
778        let dir = tempfile::tempdir().unwrap();
779        let path = dir.path().join("cross-store-isolation.sqlite3");
780        let conn = ManagedConnection::open(&path).unwrap();
781        let catalog = Catalog::open(conn.clone()).unwrap();
782        let writer = SQLiteStorageBackend::new(conn.clone());
783        let observer_conn = conn.new_session();
784        let observer_catalog = Catalog::open(observer_conn.clone()).unwrap();
785        let observer = SQLiteStorageBackend::new(observer_conn);
786
787        let mut writer_docs = writer.document_store("articles");
788        let mut writer_inv = writer.inverted_index("articles", standard_analyzer("english"));
789        let mut writer_vectors = writer
790            .vector_index(
791                "articles",
792                "embedding",
793                2,
794                VectorIndexSpec::BruteForce,
795                VectorIndexOpenMode::Create,
796            )
797            .unwrap();
798        let observer_docs = observer.document_store("articles");
799        let observer_inv = observer.inverted_index("articles", standard_analyzer("english"));
800        let observer_vectors = observer
801            .vector_index(
802                "articles",
803                "embedding",
804                2,
805                VectorIndexSpec::BruteForce,
806                VectorIndexOpenMode::Restore,
807            )
808            .unwrap();
809
810        writer.begin_transaction().unwrap();
811        writer_docs
812            .put(
813                1,
814                BTreeMap::from([("title".to_string(), Value::Str("atomic rust".into()))]),
815            )
816            .unwrap();
817        writer_inv
818            .add_document(
819                1,
820                BTreeMap::from([("title".to_string(), "atomic rust".to_string())]),
821            )
822            .unwrap();
823        writer_vectors.add(1, vec![1.0, 0.0]).unwrap();
824        catalog
825            .save_scoring_params("transactional", r#"{"alpha":1.0}"#)
826            .unwrap();
827
828        assert_eq!(writer_docs.len().unwrap(), 1);
829        assert_eq!(writer_inv.doc_freq("title", "rust").unwrap(), 1);
830        assert_eq!(writer_vectors.count().unwrap(), 1);
831        assert!(catalog
832            .load_scoring_params("transactional")
833            .unwrap()
834            .is_some());
835
836        assert_eq!(observer_docs.len().unwrap(), 0);
837        assert_eq!(observer_inv.doc_freq("title", "rust").unwrap(), 0);
838        assert_eq!(observer_vectors.count().unwrap(), 0);
839        assert!(observer_catalog
840            .load_scoring_params("transactional")
841            .unwrap()
842            .is_none());
843
844        writer.commit_transaction().unwrap();
845        assert_eq!(observer_docs.len().unwrap(), 1);
846        assert_eq!(observer_inv.doc_freq("title", "rust").unwrap(), 1);
847        assert_eq!(observer_vectors.count().unwrap(), 1);
848        assert!(observer_catalog
849            .load_scoring_params("transactional")
850            .unwrap()
851            .is_some());
852    }
853
854    #[test]
855    fn ignored_legacy_index_error_cannot_commit_partial_document_write() {
856        let dir = tempfile::tempdir().unwrap();
857        let path = dir.path().join("ignored-index-error.sqlite3");
858        let conn = ManagedConnection::open(&path).unwrap();
859        let _catalog = Catalog::open(conn.clone()).unwrap();
860        let backend = SQLiteStorageBackend::new(conn.clone());
861        let observer = conn.new_session();
862        let mut docs = backend.document_store("articles");
863        let mut vectors = backend
864            .vector_index(
865                "articles",
866                "embedding",
867                2,
868                VectorIndexSpec::BruteForce,
869                VectorIndexOpenMode::Create,
870            )
871            .unwrap();
872
873        backend.begin_transaction().unwrap();
874        docs.put(
875            1,
876            BTreeMap::from([("title".to_string(), Value::Str("must roll back".into()))]),
877        )
878        .unwrap();
879        conn.with(|connection| {
880            connection.execute("DROP TABLE _vectors", [])?;
881            Ok(())
882        })
883        .unwrap();
884        // The vector write reports its error directly. Even if a caller
885        // ignores that Result, the managed transaction is poisoned and the
886        // partial document write cannot commit.
887        let ignored = vectors.add(1, vec![1.0, 0.0]);
888        assert!(ignored.is_err());
889        assert!(matches!(
890            backend.commit_transaction(),
891            Err(StorageBackendError::Backend { source, .. })
892                if matches!(source.downcast_ref::<SQLiteError>(), Some(SQLiteError::TransactionAborted(_)))
893        ));
894
895        let stored_docs: i64 = observer
896            .with(|connection| {
897                Ok(connection.query_row(
898                    "SELECT COUNT(*) FROM _documents WHERE table_name = 'articles'",
899                    [],
900                    |row| row.get(0),
901                )?)
902            })
903            .unwrap();
904        let vector_table_exists: i64 = observer
905            .with(|connection| {
906                Ok(connection.query_row(
907                    "SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name = '_vectors'",
908                    [],
909                    |row| row.get(0),
910                )?)
911            })
912            .unwrap();
913        assert_eq!(stored_docs, 0);
914        assert_eq!(vector_table_exists, 1);
915    }
916}