Skip to main content

uqa_storage_sqlite/
connection.rs

1//
2// Unified Query Algebra
3//
4// Copyright (c) 2023-2026 Cognica, Inc.
5//
6
7//! Pooled `SQLite` connections with explicit session and transaction affinity.
8//!
9//! A [`ManagedConnection`] is a logical session. Clones share that session so
10//! catalog, document, inverted-index, and vector stores participate in the
11//! same explicit transaction. [`ManagedConnection::new_session`] creates an
12//! isolated session over the same physical connection pool. Outside explicit
13//! transactions operations check out independent connections, allowing WAL
14//! readers to make real concurrent progress.
15
16use std::path::{Path, PathBuf};
17use std::sync::{Arc, OnceLock};
18
19use parking_lot::{Mutex, RwLock};
20use rusqlite::{Connection, OpenFlags};
21use uqa_storage::{mvcc::VersionedKeyValueStore, KeyValueStore, StorageEncryptionKey};
22
23use crate::compressed_vfs::{self, SQLiteCompressedContainerAnchor, SQLiteCompressionOptions};
24
25mod identity;
26mod logical;
27mod native;
28mod native_restore;
29pub(crate) mod ownership;
30mod pool;
31mod restore;
32use pool::ConnectionPool;
33pub(crate) use pool::PooledConnection;
34mod notifications;
35mod serializable;
36mod snapshot;
37mod source;
38use snapshot::PhysicalConnection;
39pub(crate) use snapshot::SnapshotIdentity;
40
41#[derive(Debug, thiserror::Error)]
42pub enum SQLiteError {
43    #[error(transparent)]
44    Memory(#[from] uqa_core::memory::MemoryError),
45    #[error(transparent)]
46    Cancelled(#[from] uqa_core::QueryCancelled),
47    #[error("text analysis failed: {0}")]
48    Analysis(#[from] uqa_analysis::AnalysisError),
49    #[error("sqlite error: {0}")]
50    SQLite(#[from] rusqlite::Error),
51    #[error("encryption key must not be empty")]
52    EmptyEncryptionKey,
53    #[error("database requires an encryption key")]
54    EncryptionKeyRequired,
55    #[error("database is not encrypted but an encryption key was provided")]
56    NotEncrypted,
57    #[error("auxiliary database requires DELETE journaling, found {0}")]
58    AuxiliaryJournalMode(String),
59    #[error("compressed sqlite container error: {0}")]
60    CompressedContainer(String),
61    #[error("io error: {0}")]
62    Io(#[from] std::io::Error),
63    #[error("catalog migration {version} failed: {source}")]
64    Migration {
65        version: u32,
66        #[source]
67        source: rusqlite::Error,
68    },
69    #[error("invalid persisted catalog schema version `{0}`")]
70    InvalidSchemaVersion(String),
71    #[error("catalog schema version {found} is newer than this engine supports ({supported})")]
72    UnsupportedSchemaVersion { found: u32, supported: u32 },
73    #[error("corrupt document blob for `{table}` doc {doc_id} field `{field}`: {reason}")]
74    CorruptDocumentBlob {
75        table: String,
76        doc_id: u64,
77        field: String,
78        reason: String,
79    },
80    #[error("payload serialization failed: {0}")]
81    Serde(#[from] serde_json::Error),
82    #[error("storage backend error: {0}")]
83    StorageBackend(String),
84    #[error(transparent)]
85    StorageSource(Box<uqa_storage::StorageBackendError>),
86    #[error("transaction already active for this sqlite session")]
87    TransactionAlreadyActive,
88    #[error("no active transaction for this sqlite session")]
89    NoActiveTransaction,
90    #[error("sqlite transaction was aborted by an earlier storage error: {0}")]
91    TransactionAborted(String),
92    #[error("sqlite session cleanup failed: {0}")]
93    SessionCleanupFailed(String),
94    #[error("sqlite connection-pool checkout lost its connection")]
95    MissingCheckedOutConnection,
96    #[error("versioned storage requires its logical session; use with_physical only for explicit physical maintenance")]
97    LogicalSessionRequired,
98    #[error("database restoration requires all existing owners to close")]
99    DatabaseRestoreBusy,
100    #[error("database restoration is incomplete; resume the original source/target request")]
101    DatabaseRestoreIncomplete,
102    #[error("the SQLite session already has a different retention limit")]
103    SessionOptionsMismatch,
104    #[error("the SQLite session is bound to a different record mapping")]
105    SessionMappingMismatch,
106    #[error("the retained SQLite database source changed; close its owners and reopen explicitly")]
107    DatabaseSourceChanged,
108}
109
110pub type Result<T> = std::result::Result<T, SQLiteError>;
111
112const MIN_POOL_CONNECTIONS: usize = 4;
113const MAX_POOL_CONNECTIONS: usize = 32;
114
115#[derive(Clone)]
116enum ConnectionSpec {
117    File {
118        path: PathBuf,
119        key: Option<StorageEncryptionKey>,
120    },
121    Auxiliary {
122        path: PathBuf,
123        key: Option<StorageEncryptionKey>,
124    },
125    Compressed {
126        path: PathBuf,
127        compression: SQLiteCompressionOptions,
128        key: Option<StorageEncryptionKey>,
129    },
130    Memory,
131}
132
133impl ConnectionSpec {
134    fn open(&self, initialize_database: bool) -> Result<Connection> {
135        let mut flags = OpenFlags::default();
136        if !initialize_database && matches!(self, Self::File { .. } | Self::Auxiliary { .. }) {
137            flags.remove(OpenFlags::SQLITE_OPEN_CREATE);
138        }
139        match self {
140            Self::File { path, key } | Self::Auxiliary { path, key } => {
141                let conn = Connection::open_with_flags(path, flags)?;
142                if let Some(key) = key {
143                    ManagedConnection::apply_encryption_key(&conn, key.expose_secret())?;
144                }
145                if matches!(self, Self::Auxiliary { .. }) {
146                    // Registry writers already serialize their transactions.
147                    // Retain the original journal mode instead of racing to
148                    // promote a new registry to WAL during concurrent opens.
149                    let mode: String =
150                        conn.pragma_query_value(None, "journal_mode", |row| row.get(0))?;
151                    if mode != "delete" {
152                        return Err(SQLiteError::AuxiliaryJournalMode(mode));
153                    }
154                    ManagedConnection::configure_rollback_connection(&conn)?;
155                } else {
156                    if initialize_database {
157                        ManagedConnection::enable_wal(&conn)?;
158                    }
159                    ManagedConnection::configure_wal_connection(&conn)?;
160                }
161                Ok(conn)
162            }
163            Self::Compressed {
164                path, compression, ..
165            } => {
166                let conn =
167                    Connection::open_with_flags_and_vfs(path, flags, compressed_vfs::VFS_NAME)?;
168                if initialize_database {
169                    conn.pragma_update(None, "page_size", compression.page_size)?;
170                    ManagedConnection::enable_compressed_journal(&conn)?;
171                }
172                ManagedConnection::configure_rollback_connection(&conn)?;
173                Ok(conn)
174            }
175            Self::Memory => {
176                let conn = Connection::open_in_memory()?;
177                if initialize_database {
178                    ManagedConnection::enable_wal(&conn)?;
179                }
180                ManagedConnection::configure_wal_connection(&conn)?;
181                Ok(conn)
182            }
183        }
184    }
185}
186
187struct SessionState {
188    write_cancellation: uqa_core::CancellationToken,
189    affinity: uqa_storage::StorageSessionAffinity,
190    /// Read guards cover ordinary operations. Transaction lifecycle calls take
191    /// the write guard, making BEGIN/COMMIT/ROLLBACK linearizable with respect
192    /// to every operation issued through the same logical session.
193    gate: RwLock<()>,
194    transaction: Mutex<Option<PooledConnection>>,
195    transaction_failure: Mutex<Option<String>>,
196    cleanup_failure: Mutex<Option<String>>,
197    logical: OnceLock<Arc<logical::BoundRecordSession>>,
198    native_restore: Mutex<Option<native_restore::NativeRestore>>,
199    snapshot_branch: Mutex<Arc<()>>,
200}
201
202impl SessionState {
203    fn new() -> Self {
204        Self::with_cancellation(uqa_core::CancellationToken::new())
205    }
206
207    fn with_cancellation(write_cancellation: uqa_core::CancellationToken) -> Self {
208        Self {
209            write_cancellation,
210            affinity: uqa_storage::StorageSessionAffinity::new(),
211            gate: RwLock::new(()),
212            transaction: Mutex::new(None),
213            transaction_failure: Mutex::new(None),
214            cleanup_failure: Mutex::new(None),
215            logical: OnceLock::new(),
216            native_restore: Mutex::new(None),
217            snapshot_branch: Mutex::new(Arc::new(())),
218        }
219    }
220}
221
222impl Drop for SessionState {
223    fn drop(&mut self) {
224        // `PooledConnection::drop` performs the rollback and discards the
225        // physical connection when rollback itself fails. Taking the pinned
226        // handle here therefore cannot return a broken connection to the pool.
227        self.native_restore.get_mut().take();
228        self.transaction.get_mut().take();
229    }
230}
231
232/// Logical `SQLite` session backed by a bounded physical connection pool.
233/// Cloning preserves session/transaction affinity; call [`Self::new_session`]
234/// for an independently isolated transaction context.
235#[derive(Clone)]
236pub struct ManagedConnection {
237    pool: Arc<ConnectionPool>,
238    session: Arc<SessionState>,
239    record_access: bool,
240}
241
242impl ManagedConnection {
243    fn surface_cleanup_failure(&self) -> Result<()> {
244        if let Some(error) = self.session.cleanup_failure.lock().take() {
245            return Err(SQLiteError::SessionCleanupFailed(error));
246        }
247        Ok(())
248    }
249
250    pub fn open(path: &Path) -> Result<Self> {
251        if path == Path::new(":memory:") {
252            return Self::open_in_memory();
253        }
254        Self::open_with_optional_key(path, None)
255    }
256
257    /// Retain the database credential for encrypted auxiliary storage, including
258    /// compressed containers whose main-file encryption lives in the VFS.
259    #[must_use]
260    pub fn auxiliary_encryption_key(&self) -> Option<StorageEncryptionKey> {
261        match &self.pool.spec {
262            ConnectionSpec::File { key, .. }
263            | ConnectionSpec::Auxiliary { key, .. }
264            | ConnectionSpec::Compressed { key, .. } => key.clone(),
265            ConnectionSpec::Memory => None,
266        }
267    }
268
269    /// Lease an independent physical connection without joining this logical
270    /// session's transaction. The pool retains encryption and connection policy.
271    pub fn lease_connection(&self) -> Result<crate::SQLiteConnectionLease> {
272        self.pool.checkout().map(crate::SQLiteConnectionLease)
273    }
274
275    /// Open database-owned auxiliary storage with its inherited credential and
276    /// original DELETE journaling. Concurrent first opens never change journal
277    /// modes. An incompatible existing mode is rejected without rewriting it.
278    pub fn open_auxiliary(path: &Path, key: Option<StorageEncryptionKey>) -> Result<Self> {
279        if path == Path::new(":memory:") || path.as_os_str().is_empty() {
280            return Err(SQLiteError::StorageBackend(
281                "auxiliary storage requires a database file".into(),
282            ));
283        }
284        Self::from_spec(
285            ConnectionSpec::Auxiliary {
286                path: path.to_path_buf(),
287                key,
288            },
289            default_pool_connections(),
290        )
291    }
292
293    pub fn open_encrypted(path: &Path, key: &str) -> Result<Self> {
294        Self::open_with_optional_key(path, Some(key))
295    }
296
297    pub fn open_compressed(path: &Path, compression: SQLiteCompressionOptions) -> Result<Self> {
298        Self::open_compressed_with_optional_key(path, compression, None, None)
299    }
300
301    pub fn open_compressed_encrypted(
302        path: &Path,
303        key: &str,
304        compression: SQLiteCompressionOptions,
305    ) -> Result<Self> {
306        if key.is_empty() {
307            return Err(SQLiteError::EmptyEncryptionKey);
308        }
309        Self::open_compressed_with_optional_key(path, compression, Some(key), None)
310    }
311
312    /// Open an encrypted compressed database while enforcing an exact trusted
313    /// external state anchor in the VFS main-file open path.
314    pub fn open_compressed_encrypted_with_anchor(
315        path: &Path,
316        key: &str,
317        compression: SQLiteCompressionOptions,
318        trusted_anchor: SQLiteCompressedContainerAnchor,
319    ) -> Result<Self> {
320        if key.is_empty() {
321            return Err(SQLiteError::EmptyEncryptionKey);
322        }
323        Self::open_compressed_with_optional_key(path, compression, Some(key), Some(trusted_anchor))
324    }
325
326    fn open_with_optional_key(path: &Path, key: Option<&str>) -> Result<Self> {
327        let spec = ConnectionSpec::File {
328            path: path.to_path_buf(),
329            key: key.map(StorageEncryptionKey::new),
330        };
331        Self::from_spec(spec, default_pool_connections())
332    }
333
334    fn open_compressed_with_optional_key(
335        path: &Path,
336        compression: SQLiteCompressionOptions,
337        key: Option<&str>,
338        trusted_anchor: Option<SQLiteCompressedContainerAnchor>,
339    ) -> Result<Self> {
340        let compression = compression
341            .validate()
342            .map_err(SQLiteError::CompressedContainer)?;
343        let owner = ownership::open(
344            path,
345            &uqa_storage::read_control::StorageReadControl::with_limit(
346                uqa_storage::mvcc::VersionedSessionOptions::default().retained_bytes,
347            ),
348        )?;
349        match trusted_anchor {
350            Some(anchor) => compressed_vfs::register_database_with_anchor(
351                path,
352                compression,
353                key.ok_or(SQLiteError::EncryptionKeyRequired)?,
354                anchor,
355            ),
356            None => compressed_vfs::register_database(path, compression, key),
357        }
358        .map_err(SQLiteError::CompressedContainer)?;
359        let spec = ConnectionSpec::Compressed {
360            path: path.to_path_buf(),
361            compression,
362            key: key.map(StorageEncryptionKey::new),
363        };
364        Self::from_spec_owned(spec, default_pool_connections(), Some(owner))
365    }
366
367    pub fn open_in_memory() -> Result<Self> {
368        // Independent `:memory:` connections do not share a database. Keep a
369        // one-connection pool for this special target; file-backed databases
370        // use the real multi-connection pool.
371        Self::from_spec(ConnectionSpec::Memory, 1)
372    }
373
374    fn from_spec(spec: ConnectionSpec, max_connections: usize) -> Result<Self> {
375        let owner = match &spec {
376            ConnectionSpec::File { path, .. }
377                if path.as_os_str().is_empty() || path == Path::new(":memory:") =>
378            {
379                None
380            }
381            ConnectionSpec::File { path, .. }
382            | ConnectionSpec::Auxiliary { path, .. }
383            | ConnectionSpec::Compressed { path, .. } => Some(ownership::open(
384                path,
385                &uqa_storage::read_control::StorageReadControl::with_limit(
386                    uqa_storage::mvcc::VersionedSessionOptions::default().retained_bytes,
387                ),
388            )?),
389            ConnectionSpec::Memory => None,
390        };
391        Self::from_spec_owned(spec, max_connections, owner)
392    }
393
394    fn from_spec_owned(
395        spec: ConnectionSpec,
396        max_connections: usize,
397        owner: Option<Arc<ownership::DatabaseOwner>>,
398    ) -> Result<Self> {
399        let initial = spec.open(true)?;
400        crate::mvcc::restore::reject_pending(&initial)?;
401        Ok(Self {
402            pool: ConnectionPool::new(spec, initial, max_connections, owner)?,
403            session: Arc::new(SessionState::new()),
404            record_access: false,
405        })
406    }
407
408    pub(crate) fn database_owner(&self) -> Option<Arc<ownership::DatabaseOwner>> {
409        self.pool.owner.clone()
410    }
411
412    fn apply_encryption_key(conn: &Connection, key: &str) -> Result<()> {
413        if key.is_empty() {
414            return Err(SQLiteError::EmptyEncryptionKey);
415        }
416        conn.pragma_update(None, "key", key)?;
417        Ok(())
418    }
419
420    fn enable_wal(conn: &Connection) -> Result<()> {
421        conn.pragma_update(None, "journal_mode", "WAL")?;
422        Ok(())
423    }
424
425    fn configure_wal_connection(conn: &Connection) -> Result<()> {
426        // Legacy physical operations retain a bounded busy wait. Versioned writes supply cancellable admission under a scoped timeout.
427        conn.busy_timeout(std::time::Duration::from_secs(5))?;
428        // Synchronous=NORMAL is the recommended pairing with WAL: safe
429        // against power loss, faster than FULL.
430        conn.pragma_update(None, "synchronous", "NORMAL")?;
431        // Foreign keys are off by default; turn on so per-table cleanup
432        // can use ON DELETE CASCADE later.
433        conn.pragma_update(None, "foreign_keys", "ON")?;
434        Ok(())
435    }
436
437    fn enable_compressed_journal(conn: &Connection) -> Result<()> {
438        // The compressed VFS implements the byte-addressed database file.
439        // Rollback journals stay raw because they are short-lived commit
440        // machinery; compressing them only adds autocommit write amplification.
441        // WAL requires shared-memory VFS methods, so compressed databases use
442        // SQLite's rollback journal and keep temp storage in memory.
443        conn.pragma_update(None, "journal_mode", "DELETE")?;
444        Ok(())
445    }
446
447    fn configure_rollback_connection(conn: &Connection) -> Result<()> {
448        conn.busy_timeout(std::time::Duration::from_secs(5))?;
449        conn.pragma_update(None, "synchronous", "FULL")?;
450        conn.pragma_update(None, "temp_store", "MEMORY")?;
451        conn.pragma_update(None, "foreign_keys", "ON")?;
452        Ok(())
453    }
454
455    /// Identity shared by handles using this session's native or logical transaction context.
456    pub fn transaction_affinity(&self) -> uqa_storage::StorageSessionAffinity {
457        let _gate = self.session.gate.read();
458        self.session.logical.get().map_or_else(
459            || self.session.affinity.clone(),
460            |logical| logical.session_affinity(),
461        )
462    }
463
464    /// Whether this session has an active native or logical record transaction.
465    #[must_use]
466    pub fn in_transaction(&self) -> bool {
467        let _gate = self.session.gate.read();
468        if let Some(logical) = self.session.logical.get() {
469            return logical.in_transaction();
470        }
471        self.session.transaction.lock().is_some()
472    }
473
474    /// Whether this session has staged writes, including private logical records. Callers use this to enforce read-only execution boundaries before COMMIT.
475    pub fn transaction_has_written(&self) -> Result<bool> {
476        let _gate = self.session.gate.read();
477        if let Some(logical) = self.session.logical.get() {
478            return logical.transaction_has_written().map_err(Into::into);
479        }
480        let transaction = self.session.transaction.lock();
481        let transaction = transaction
482            .as_ref()
483            .ok_or(SQLiteError::NoActiveTransaction)?;
484        Ok(matches!(
485            transaction.connection()?.transaction_state(Some("main"))?,
486            rusqlite::TransactionState::Write
487        ))
488    }
489
490    /// Whether this session's independent [`Self::data_version`] monitor can
491    /// read without contending with the currently pinned transaction.
492    ///
493    /// A rollback-journal writer's pending lock blocks new readers while waiting for existing readers to finish. A rollback-journal read transaction must therefore also avoid the independent monitor: its own shared lock may be preventing that waiting writer from proceeding. Callers refresh through the pinned connection instead. WAL sessions and sessions without a pinned transaction permit the independent monitor.
494    pub fn data_version_monitor_is_nonblocking(&self) -> Result<bool> {
495        let _gate = self.session.gate.read();
496        if self.session.logical.get().is_some() {
497            return Ok(true);
498        }
499        if !matches!(
500            &self.pool.spec,
501            ConnectionSpec::Compressed { .. } | ConnectionSpec::Auxiliary { .. }
502        ) {
503            return Ok(true);
504        }
505        Ok(self.session.transaction.lock().is_none())
506    }
507
508    /// Whether one pooled connection may retain a read snapshot while another writes. Ordinary plain and encrypted `SQLite` databases use WAL; compressed containers and auxiliary files use rollback journaling and therefore require a detached engine snapshot before writer promotion.
509    #[must_use]
510    pub fn supports_concurrent_pinned_read_and_write(&self) -> bool {
511        let _gate = self.session.gate.read();
512        self.session.logical.get().is_some()
513            || !matches!(
514                &self.pool.spec,
515                ConnectionSpec::Compressed { .. } | ConnectionSpec::Auxiliary { .. }
516            )
517    }
518
519    /// Database change counter observed on one stable connection shared by
520    /// every logical session over this pool. The value changes when another
521    /// `SQLite` connection commits. In-memory databases have no independent
522    /// connections and therefore return `None`.
523    pub fn data_version(&self) -> Result<Option<u64>> {
524        let _gate = self.session.gate.read();
525        self.pool.check_source()?;
526        if let Some(logical) = self.session.logical.get() {
527            return logical.change_version().map_err(Into::into);
528        }
529        if matches!(&self.pool.spec, ConnectionSpec::Memory) {
530            return Ok(None);
531        }
532        let mut monitor = self.pool.data_version_monitor.lock();
533        if monitor.is_none() {
534            *monitor = Some(self.pool.open_connection()?);
535        }
536        let monitor = monitor.as_ref().ok_or_else(|| {
537            SQLiteError::StorageBackend(
538                "data-version monitor was not initialized after opening it".into(),
539            )
540        })?;
541        let version: i64 = monitor.pragma_query_value(None, "data_version", |row| row.get(0))?;
542        let version = u64::try_from(version).map_err(|_| {
543            SQLiteError::StorageBackend(format!(
544                "SQLite returned a negative PRAGMA data_version: {version}"
545            ))
546        })?;
547        Ok(Some(version))
548    }
549
550    /// Establish the database snapshot for the active transaction without
551    /// depending on the caller's first user query. `BEGIN DEFERRED` alone does
552    /// not start a read transaction, so a writer could otherwise commit after
553    /// the engine checks its cache generations but before the first catalog
554    /// read. Reading `sqlite_schema` is database-wide and keeps the operation
555    /// independent of any application table.
556    pub fn pin_transaction_snapshot(&self) -> Result<()> {
557        self.surface_cleanup_failure()?;
558        let _gate = self.session.gate.read();
559        if let Some(logical) = self.session.logical.get() {
560            return if logical.in_transaction() {
561                Ok(())
562            } else {
563                Err(SQLiteError::NoActiveTransaction)
564            };
565        }
566        self.with_native(|connection| {
567            if connection.is_autocommit() {
568                return Err(SQLiteError::NoActiveTransaction);
569            }
570            let _: i64 =
571                connection.query_row("SELECT COUNT(*) FROM sqlite_schema", [], |row| row.get(0))?;
572            Ok(())
573        })
574    }
575
576    /// Run a closure using this session. Outside a transaction the closure
577    /// checks out a pooled connection; inside a transaction every clone is
578    /// routed to the session's pinned connection.
579    pub fn with<R>(&self, f: impl FnOnce(&Connection) -> Result<R>) -> Result<R> {
580        self.surface_cleanup_failure()?;
581        let _gate = self.session.gate.read();
582        if self.session.logical.get().is_some() {
583            return Err(SQLiteError::LogicalSessionRequired);
584        }
585        self.with_native(f)
586    }
587
588    /// The caller holds the session gate and has ruled out logical record access.
589    fn with_native<R>(&self, f: impl FnOnce(&Connection) -> Result<R>) -> Result<R> {
590        let transaction = self.session.transaction.lock();
591        if let Some(connection) = transaction.as_ref() {
592            if let Some(error) = self.session.transaction_failure.lock().as_ref() {
593                return Err(SQLiteError::TransactionAborted(error.clone()));
594            }
595            self.check_native_access(connection.connection()?)?;
596            let result = f(connection.connection()?);
597            if let Err(error) = &result {
598                let mut failure = self.session.transaction_failure.lock();
599                if failure.is_none() {
600                    *failure = Some(error.to_string());
601                }
602            }
603            return result;
604        }
605        drop(transaction);
606        let connection = self.pool.checkout()?;
607        self.check_native_access(connection.connection()?)?;
608        f(connection.connection()?)
609    }
610
611    pub fn with_mut<R>(&self, f: impl FnOnce(&mut Connection) -> Result<R>) -> Result<R> {
612        self.with_physical_mut(|physical| f(&mut physical.connection))
613    }
614
615    fn with_physical_mut<R>(
616        &self,
617        f: impl FnOnce(&mut PhysicalConnection) -> Result<R>,
618    ) -> Result<R> {
619        self.surface_cleanup_failure()?;
620        let _gate = self.session.gate.read();
621        if self.session.logical.get().is_some() {
622            return Err(SQLiteError::LogicalSessionRequired);
623        }
624        let mut transaction = self.session.transaction.lock();
625        if let Some(connection) = transaction.as_mut() {
626            if let Some(error) = self.session.transaction_failure.lock().as_ref() {
627                return Err(SQLiteError::TransactionAborted(error.clone()));
628            }
629            self.check_native_access(connection.connection()?)?;
630            let result = f(connection.physical_mut()?);
631            if let Err(error) = &result {
632                let mut failure = self.session.transaction_failure.lock();
633                if failure.is_none() {
634                    *failure = Some(error.to_string());
635                }
636            }
637            return result;
638        }
639        drop(transaction);
640        let mut connection = self.pool.checkout()?;
641        self.check_native_access(connection.connection()?)?;
642        f(connection.physical_mut()?)
643    }
644
645    /// Rewrite the `SQLite` database into its minimum-sized file. `SQLite` requires `VACUUM` to run in autocommit mode, so the session write gate makes the transaction check and maintenance command one atomic session operation.
646    pub fn vacuum(&self) -> Result<()> {
647        self.maintain_storage(true)
648    }
649
650    /// Reclaim obsolete logical records and history without executing a physical `SQLite` `VACUUM`.
651    pub fn reclaim_obsolete(&self) -> Result<()> {
652        self.maintain_storage(false)
653    }
654
655    fn maintain_storage(&self, compact: bool) -> Result<()> {
656        self.surface_cleanup_failure()?;
657        let _gate = self.session.gate.write();
658        if self.session.transaction.lock().is_some()
659            || self
660                .session
661                .logical
662                .get()
663                .is_some_and(|logical| logical.in_transaction())
664        {
665            return Err(SQLiteError::TransactionAlreadyActive);
666        }
667        if let Some(logical) = self.session.logical.get() {
668            let source: Arc<dyn KeyValueStore> = if let Some(namespace) = logical.native {
669                Arc::new(crate::diskann::Records::new(
670                    logical.store.clone(),
671                    namespace,
672                    self.auxiliary_encryption_key(),
673                )?)
674            } else {
675                logical.store.clone()
676            };
677            let retained = logical.retention_control();
678            let control = uqa_storage::read_control::StorageReadControl::new(
679                retained.memory(),
680                &self.write_cancellation(),
681            );
682            uqa_storage::key_value::KeyValueDiskANNMaintenance::run(&source, &control)?;
683            logical.reclaim_versions()?;
684        }
685        if compact {
686            let connection = self.pool.checkout()?;
687            connection.connection()?.execute_batch("VACUUM")?;
688        }
689        Ok(())
690    }
691
692    /// Open an explicit (non-deferred) transaction. Subsequent
693    /// auto-commit hosts (catalog writes, FTS index updates, ...) all
694    /// flow through the same connection so the transaction enclosing
695    /// them is honoured. Use [`Self::commit_transaction`] /
696    /// [`Self::rollback_transaction`] / [`Self::savepoint`] etc. for
697    /// the lifecycle.
698    pub fn begin_transaction(&self) -> Result<()> {
699        self.begin_transaction_with("BEGIN IMMEDIATE")
700    }
701
702    /// Open a deferred transaction. Read-only SQL statements use this mode
703    /// so WAL readers do not take the single writer reservation; if a scalar
704    /// routine performs a write, `SQLite` upgrades the same transaction and
705    /// still preserves the statement's atomic boundary.
706    pub fn begin_deferred_transaction(&self) -> Result<()> {
707        self.begin_transaction_with("BEGIN DEFERRED")
708    }
709
710    fn begin_transaction_with(&self, statement: &str) -> Result<()> {
711        self.surface_cleanup_failure()?;
712        let _gate = self.session.gate.write();
713        if let Some(logical) = self.session.logical.get() {
714            if logical.in_transaction() {
715                return Err(SQLiteError::TransactionAlreadyActive);
716            }
717            return if statement == "BEGIN DEFERRED" {
718                logical.begin_upgradeable_transaction()
719            } else {
720                logical.begin_transaction()
721            }
722            .map_err(Into::into);
723        }
724        let mut transaction = self.session.transaction.lock();
725        if transaction.is_some() {
726            return Err(SQLiteError::TransactionAlreadyActive);
727        }
728        let connection = self.pool.checkout()?;
729        self.check_native_access(connection.connection()?)?;
730        connection.connection()?.execute_batch(statement)?;
731        self.session.transaction_failure.lock().take();
732        *transaction = Some(connection);
733        Ok(())
734    }
735
736    pub fn commit_transaction(&self) -> Result<()> {
737        self.surface_cleanup_failure()?;
738        self.finish_transaction("COMMIT")
739    }
740
741    pub fn rollback_transaction(&self) -> Result<()> {
742        self.surface_cleanup_failure()?;
743        self.finish_transaction("ROLLBACK")
744    }
745
746    /// Drop-only transaction cleanup. A rollback error is recorded on the
747    /// logical session and is returned by its next operation instead of being
748    /// mistaken for a successful rollback.
749    pub(crate) fn rollback_transaction_on_drop(&self) {
750        if let Err(error) = self.finish_transaction("ROLLBACK") {
751            let mut failure = self.session.cleanup_failure.lock();
752            if failure.is_none() {
753                *failure = Some(error.to_string());
754            }
755        }
756    }
757
758    fn finish_transaction(&self, statement: &str) -> Result<()> {
759        let _gate = self.session.gate.write();
760        if let Some(logical) = self.session.logical.get() {
761            if !logical.in_transaction() {
762                return Err(SQLiteError::NoActiveTransaction);
763            }
764            return if statement == "COMMIT" {
765                logical.commit_transaction()
766            } else {
767                logical.rollback_transaction()
768            }
769            .map_err(Into::into);
770        }
771        let mut transaction = self.session.transaction.lock();
772        let connection = transaction
773            .as_ref()
774            .ok_or(SQLiteError::NoActiveTransaction)?;
775        *self.session.snapshot_branch.lock() = Arc::new(());
776        if statement == "COMMIT" {
777            // Materialize the failure before entering the branch. Holding the
778            // mutex guard created by an `if let` scrutinee until the end of
779            // the branch would deadlock when cleanup takes the same lock.
780            let transaction_failure = self.session.transaction_failure.lock().clone();
781            if let Some(error) = transaction_failure {
782                if let Err(rollback_error) = connection.connection()?.execute_batch("ROLLBACK") {
783                    self.session.native_restore.lock().take();
784                    transaction.take();
785                    self.session.transaction_failure.lock().take();
786                    return Err(SQLiteError::SQLite(rollback_error));
787                }
788                self.session.native_restore.lock().take();
789                transaction.take();
790                self.session.transaction_failure.lock().take();
791                return Err(SQLiteError::TransactionAborted(error));
792            }
793        }
794        let native = if statement == "COMMIT" && self.session.native_restore.lock().is_some() {
795            match self.prepare_initial_native_binding(connection.connection()?) {
796                Ok(native) => Some(native),
797                Err(error) => {
798                    *self.session.transaction_failure.lock() = Some(error.to_string());
799                    return Err(error);
800                }
801            }
802        } else {
803            None
804        };
805        if let Err(error) = connection.connection()?.execute_batch(statement) {
806            self.session.native_restore.lock().take();
807            transaction.take();
808            self.session.transaction_failure.lock().take();
809            return Err(SQLiteError::SQLite(error));
810        }
811        self.session.native_restore.lock().take();
812        transaction.take();
813        self.session.transaction_failure.lock().take();
814        if let Some(native) = native {
815            self.install_initial_native_binding(native);
816        }
817        Ok(())
818    }
819
820    fn with_transaction<R>(
821        &self,
822        logical: impl FnOnce(&VersionedKeyValueStore) -> uqa_storage::StorageBackendResult<R>,
823        native: impl FnOnce(&Connection) -> Result<R>,
824    ) -> Result<R> {
825        self.surface_cleanup_failure()?;
826        let _gate = self.session.gate.read();
827        if let Some(store) = self.session.logical.get() {
828            if !store.in_transaction() {
829                return Err(SQLiteError::NoActiveTransaction);
830            }
831            return logical(store).map_err(Into::into);
832        }
833        let transaction = self.session.transaction.lock();
834        let connection = transaction
835            .as_ref()
836            .ok_or(SQLiteError::NoActiveTransaction)?;
837        self.check_native_access(connection.connection()?)?;
838        native(connection.connection()?)
839    }
840
841    pub fn savepoint(&self, name: &str) -> Result<()> {
842        self.with_transaction(
843            |logical| logical.savepoint(name),
844            |connection| {
845                let stmt = format!("SAVEPOINT \"{}\"", name.replace('"', "\"\""));
846                connection.execute_batch(&stmt)?;
847                Ok(())
848            },
849        )
850    }
851
852    pub fn release_savepoint(&self, name: &str) -> Result<()> {
853        self.with_transaction(
854            |logical| logical.release_savepoint(name),
855            |connection| {
856                let stmt = format!("RELEASE SAVEPOINT \"{}\"", name.replace('"', "\"\""));
857                connection.execute_batch(&stmt)?;
858                Ok(())
859            },
860        )
861    }
862
863    pub fn rollback_to_savepoint(&self, name: &str) -> Result<()> {
864        self.with_transaction(
865            |logical| logical.rollback_to_savepoint(name),
866            |connection| {
867                let stmt = format!("ROLLBACK TO SAVEPOINT \"{}\"", name.replace('"', "\"\""));
868                connection.execute_batch(&stmt)?;
869                self.session.transaction_failure.lock().take();
870                *self.session.snapshot_branch.lock() = Arc::new(());
871                Ok(())
872            },
873        )
874    }
875}
876
877fn default_pool_connections() -> usize {
878    std::thread::available_parallelism()
879        .map_or(MIN_POOL_CONNECTIONS, |parallelism| parallelism.get() * 2)
880        .clamp(MIN_POOL_CONNECTIONS, MAX_POOL_CONNECTIONS)
881}
882
883#[cfg(test)]
884mod tests;
885
886impl From<uqa_storage::mvcc::VersionError> for SQLiteError {
887    fn from(error: uqa_storage::mvcc::VersionError) -> Self {
888        Self::from(error.into_storage_error())
889    }
890}
891
892impl From<SQLiteError> for uqa_storage::StorageBackendError {
893    fn from(source: SQLiteError) -> Self {
894        match source {
895            SQLiteError::Memory(error) => Self::Memory(error),
896            SQLiteError::Cancelled(error) => Self::Cancelled(error),
897            SQLiteError::StorageSource(error) => *error,
898            source => Self::backend("SQLite", source),
899        }
900    }
901}
902
903impl From<SQLiteError> for uqa_storage::TransactionError {
904    fn from(source: SQLiteError) -> Self {
905        Self::Storage(source.into())
906    }
907}
908
909impl From<uqa_storage::StorageBackendError> for SQLiteError {
910    fn from(error: uqa_storage::StorageBackendError) -> Self {
911        use uqa_storage::StorageBackendError;
912        match error {
913            StorageBackendError::Memory(error) => Self::Memory(error),
914            StorageBackendError::Cancelled(error) => Self::Cancelled(error),
915            StorageBackendError::Analysis(error) => Self::Analysis(error),
916            StorageBackendError::Serde(error) => Self::Serde(error),
917            StorageBackendError::Backend { backend, source } => match source.downcast::<Self>() {
918                Ok(error) => *error,
919                Err(source) => {
920                    Self::StorageSource(Box::new(StorageBackendError::Backend { backend, source }))
921                }
922            },
923            StorageBackendError::Other(message) => Self::StorageBackend(message),
924        }
925    }
926}