Skip to main content

uqa_storage_sqlite/
connection.rs

1//
2// Unified Query Algebra
3//
4// Copyright (c) 2023-2026 Cognica, Inc.
5//
6
7//! Pooled `SQLite` connections with explicit session and transaction affinity.
8//!
9//! A [`ManagedConnection`] is a logical session. Clones share that session so
10//! catalog, document, inverted-index, and vector stores participate in the
11//! same explicit transaction. [`ManagedConnection::new_session`] creates an
12//! isolated session over the same physical connection pool. Outside explicit
13//! transactions operations check out independent connections, allowing WAL
14//! readers to make real concurrent progress.
15
16use std::path::{Path, PathBuf};
17use std::sync::{Arc, OnceLock};
18
19use parking_lot::{Mutex, RwLock};
20use rusqlite::{Connection, OpenFlags};
21use uqa_storage::{mvcc::VersionedKeyValueStore, KeyValueStore, StorageEncryptionKey};
22
23use crate::compressed_vfs::{self, SQLiteCompressedContainerAnchor, SQLiteCompressionOptions};
24
25mod identity;
26mod logical;
27mod native;
28mod native_restore;
29pub(crate) mod ownership;
30mod pool;
31mod restore;
32use pool::ConnectionPool;
33pub(crate) use pool::PooledConnection;
34mod notifications;
35mod serializable;
36mod snapshot;
37mod source;
38use snapshot::PhysicalConnection;
39pub(crate) use snapshot::{SnapshotIdentity, ValidatedRead};
40
41#[derive(Debug, thiserror::Error)]
42pub enum SQLiteError {
43    #[error(transparent)]
44    Memory(#[from] uqa_core::memory::MemoryError),
45    #[error(transparent)]
46    Cancelled(#[from] uqa_core::QueryCancelled),
47    #[error("text analysis failed: {0}")]
48    Analysis(#[from] uqa_analysis::AnalysisError),
49    #[error("sqlite error: {0}")]
50    SQLite(#[from] rusqlite::Error),
51    #[error("encryption key must not be empty")]
52    EmptyEncryptionKey,
53    #[error("database requires an encryption key")]
54    EncryptionKeyRequired,
55    #[error("database is not encrypted but an encryption key was provided")]
56    NotEncrypted,
57    #[error("auxiliary database requires DELETE journaling, found {0}")]
58    AuxiliaryJournalMode(String),
59    #[error("compressed sqlite container error: {0}")]
60    CompressedContainer(String),
61    #[error("io error: {0}")]
62    Io(#[from] std::io::Error),
63    #[error("catalog migration {version} failed: {source}")]
64    Migration {
65        version: u32,
66        #[source]
67        source: rusqlite::Error,
68    },
69    #[error("invalid persisted catalog schema version `{0}`")]
70    InvalidSchemaVersion(String),
71    #[error("catalog schema version {found} is newer than this engine supports ({supported})")]
72    UnsupportedSchemaVersion { found: u32, supported: u32 },
73    #[error("corrupt document blob for `{table}` doc {doc_id} field `{field}`: {reason}")]
74    CorruptDocumentBlob {
75        table: String,
76        doc_id: u64,
77        field: String,
78        reason: String,
79    },
80    #[error("payload serialization failed: {0}")]
81    Serde(#[from] serde_json::Error),
82    #[error("storage backend error: {0}")]
83    StorageBackend(String),
84    #[error(transparent)]
85    StorageSource(Box<uqa_storage::StorageBackendError>),
86    #[error("transaction already active for this sqlite session")]
87    TransactionAlreadyActive,
88    #[error("no active transaction for this sqlite session")]
89    NoActiveTransaction,
90    #[error("sqlite transaction was aborted by an earlier storage error: {0}")]
91    TransactionAborted(String),
92    #[error("sqlite session cleanup failed: {0}")]
93    SessionCleanupFailed(String),
94    #[error("sqlite connection-pool checkout lost its connection")]
95    MissingCheckedOutConnection,
96    #[error("versioned storage requires its logical session; use with_physical only for explicit physical maintenance")]
97    LogicalSessionRequired,
98    #[error("database restoration requires all existing owners to close")]
99    DatabaseRestoreBusy,
100    #[error("record initialization requires incompatible database owners to close")]
101    DatabaseCoordinationUpgradeBusy,
102    #[error("database restoration is incomplete; resume the original source/target request")]
103    DatabaseRestoreIncomplete,
104    #[error("the SQLite session already has a different retention limit")]
105    SessionOptionsMismatch,
106    #[error("the SQLite session is bound to a different record mapping")]
107    SessionMappingMismatch,
108    #[error("the retained SQLite database source changed; close its owners and reopen explicitly")]
109    DatabaseSourceChanged,
110}
111
112pub type Result<T> = std::result::Result<T, SQLiteError>;
113
114const MIN_POOL_CONNECTIONS: usize = 4;
115const MAX_POOL_CONNECTIONS: usize = 32;
116
117/// Prepared statements each connection retains. Statements are built from record layouts and work-queue conditions, so one commit runs more distinct statements than a small cache holds, and preparing a projection upsert compiles every trigger of its table; a cache that evicts them prepares them again in each transaction.
118pub(crate) const PREPARED_STATEMENT_CACHE_CAPACITY: usize = 256;
119
120#[derive(Clone)]
121enum ConnectionSpec {
122    File {
123        path: PathBuf,
124        key: Option<StorageEncryptionKey>,
125    },
126    Auxiliary {
127        path: PathBuf,
128        key: Option<StorageEncryptionKey>,
129    },
130    Compressed {
131        path: PathBuf,
132        compression: SQLiteCompressionOptions,
133        key: Option<StorageEncryptionKey>,
134    },
135    Memory,
136}
137
138impl ConnectionSpec {
139    fn open(&self, initialize_database: bool) -> Result<Connection> {
140        let conn = self.open_configured(initialize_database)?;
141        conn.set_prepared_statement_cache_capacity(PREPARED_STATEMENT_CACHE_CAPACITY);
142        Ok(conn)
143    }
144
145    fn open_configured(&self, initialize_database: bool) -> Result<Connection> {
146        let mut flags = OpenFlags::default();
147        if !initialize_database && matches!(self, Self::File { .. } | Self::Auxiliary { .. }) {
148            flags.remove(OpenFlags::SQLITE_OPEN_CREATE);
149        }
150        match self {
151            Self::File { path, key } | Self::Auxiliary { path, key } => {
152                let conn = Connection::open_with_flags(path, flags)?;
153                if let Some(key) = key {
154                    ManagedConnection::apply_encryption_key(&conn, key.expose_secret())?;
155                }
156                if matches!(self, Self::Auxiliary { .. }) {
157                    // Registry writers already serialize their transactions.
158                    // Retain the original journal mode instead of racing to
159                    // promote a new registry to WAL during concurrent opens.
160                    let mode: String =
161                        conn.pragma_query_value(None, "journal_mode", |row| row.get(0))?;
162                    if mode != "delete" {
163                        return Err(SQLiteError::AuxiliaryJournalMode(mode));
164                    }
165                    ManagedConnection::configure_rollback_connection(&conn)?;
166                } else {
167                    if initialize_database {
168                        ManagedConnection::enable_wal(&conn)?;
169                    }
170                    ManagedConnection::configure_wal_connection(&conn)?;
171                }
172                Ok(conn)
173            }
174            Self::Compressed {
175                path, compression, ..
176            } => {
177                let conn =
178                    Connection::open_with_flags_and_vfs(path, flags, compressed_vfs::VFS_NAME)?;
179                if initialize_database {
180                    conn.pragma_update(None, "page_size", compression.page_size)?;
181                    ManagedConnection::enable_compressed_journal(&conn)?;
182                }
183                ManagedConnection::configure_rollback_connection(&conn)?;
184                Ok(conn)
185            }
186            Self::Memory => {
187                let conn = Connection::open_in_memory()?;
188                if initialize_database {
189                    ManagedConnection::enable_wal(&conn)?;
190                }
191                ManagedConnection::configure_wal_connection(&conn)?;
192                Ok(conn)
193            }
194        }
195    }
196}
197
198struct SessionState {
199    write_cancellation: uqa_core::CancellationToken,
200    affinity: uqa_storage::StorageSessionAffinity,
201    /// Read guards cover ordinary operations. Transaction lifecycle calls take
202    /// the write guard, making BEGIN/COMMIT/ROLLBACK linearizable with respect
203    /// to every operation issued through the same logical session.
204    gate: RwLock<()>,
205    transaction: Mutex<Option<PooledConnection>>,
206    transaction_failure: Mutex<Option<String>>,
207    cleanup_failure: Mutex<Option<String>>,
208    logical: OnceLock<Arc<logical::BoundRecordSession>>,
209    native_restore: Mutex<Option<native_restore::NativeRestore>>,
210    snapshot_branch: Mutex<Arc<()>>,
211}
212
213impl SessionState {
214    fn new() -> Self {
215        Self::with_cancellation(uqa_core::CancellationToken::new())
216    }
217
218    fn with_cancellation(write_cancellation: uqa_core::CancellationToken) -> Self {
219        Self {
220            write_cancellation,
221            affinity: uqa_storage::StorageSessionAffinity::new(),
222            gate: RwLock::new(()),
223            transaction: Mutex::new(None),
224            transaction_failure: Mutex::new(None),
225            cleanup_failure: Mutex::new(None),
226            logical: OnceLock::new(),
227            native_restore: Mutex::new(None),
228            snapshot_branch: Mutex::new(Arc::new(())),
229        }
230    }
231}
232
233impl Drop for SessionState {
234    fn drop(&mut self) {
235        // `PooledConnection::drop` performs the rollback and discards the
236        // physical connection when rollback itself fails. Taking the pinned
237        // handle here therefore cannot return a broken connection to the pool.
238        self.native_restore.get_mut().take();
239        self.transaction.get_mut().take();
240    }
241}
242
243/// Logical `SQLite` session backed by a bounded physical connection pool.
244/// Cloning preserves session/transaction affinity; call [`Self::new_session`]
245/// for an independently isolated transaction context.
246#[derive(Clone)]
247pub struct ManagedConnection {
248    pool: Arc<ConnectionPool>,
249    session: Arc<SessionState>,
250    record_access: bool,
251}
252
253impl ManagedConnection {
254    /// Decoded document columns shared by every session of this database.
255    pub(crate) fn decoded_columns(
256        &self,
257    ) -> &crate::document_store::decoded_columns::DecodedColumns {
258        &self.pool.decoded_columns
259    }
260
261    fn surface_cleanup_failure(&self) -> Result<()> {
262        if let Some(error) = self.session.cleanup_failure.lock().take() {
263            return Err(SQLiteError::SessionCleanupFailed(error));
264        }
265        Ok(())
266    }
267
268    pub fn open(path: &Path) -> Result<Self> {
269        if path == Path::new(":memory:") {
270            return Self::open_in_memory();
271        }
272        Self::open_with_optional_key(path, None)
273    }
274
275    /// Retain the database credential for encrypted auxiliary storage, including
276    /// compressed containers whose main-file encryption lives in the VFS.
277    #[must_use]
278    pub fn auxiliary_encryption_key(&self) -> Option<StorageEncryptionKey> {
279        match &self.pool.spec {
280            ConnectionSpec::File { key, .. }
281            | ConnectionSpec::Auxiliary { key, .. }
282            | ConnectionSpec::Compressed { key, .. } => key.clone(),
283            ConnectionSpec::Memory => None,
284        }
285    }
286
287    /// Lease an independent physical connection without joining this logical
288    /// session's transaction. The pool retains encryption and connection policy.
289    pub fn lease_connection(&self) -> Result<crate::SQLiteConnectionLease> {
290        self.pool.checkout().map(crate::SQLiteConnectionLease)
291    }
292
293    /// Open database-owned auxiliary storage with its inherited credential and
294    /// original DELETE journaling. Concurrent first opens never change journal
295    /// modes. An incompatible existing mode is rejected without rewriting it.
296    pub fn open_auxiliary(path: &Path, key: Option<StorageEncryptionKey>) -> Result<Self> {
297        if path == Path::new(":memory:") || path.as_os_str().is_empty() {
298            return Err(SQLiteError::StorageBackend(
299                "auxiliary storage requires a database file".into(),
300            ));
301        }
302        Self::from_spec(
303            ConnectionSpec::Auxiliary {
304                path: path.to_path_buf(),
305                key,
306            },
307            default_pool_connections(),
308        )
309    }
310
311    pub fn open_encrypted(path: &Path, key: &str) -> Result<Self> {
312        Self::open_with_optional_key(path, Some(key))
313    }
314
315    pub fn open_compressed(path: &Path, compression: SQLiteCompressionOptions) -> Result<Self> {
316        Self::open_compressed_with_optional_key(path, compression, None, None)
317    }
318
319    pub fn open_compressed_encrypted(
320        path: &Path,
321        key: &str,
322        compression: SQLiteCompressionOptions,
323    ) -> Result<Self> {
324        if key.is_empty() {
325            return Err(SQLiteError::EmptyEncryptionKey);
326        }
327        Self::open_compressed_with_optional_key(path, compression, Some(key), None)
328    }
329
330    /// Open an encrypted compressed database while enforcing an exact trusted
331    /// external state anchor in the VFS main-file open path.
332    pub fn open_compressed_encrypted_with_anchor(
333        path: &Path,
334        key: &str,
335        compression: SQLiteCompressionOptions,
336        trusted_anchor: SQLiteCompressedContainerAnchor,
337    ) -> Result<Self> {
338        if key.is_empty() {
339            return Err(SQLiteError::EmptyEncryptionKey);
340        }
341        Self::open_compressed_with_optional_key(path, compression, Some(key), Some(trusted_anchor))
342    }
343
344    fn open_with_optional_key(path: &Path, key: Option<&str>) -> Result<Self> {
345        let spec = ConnectionSpec::File {
346            path: path.to_path_buf(),
347            key: key.map(StorageEncryptionKey::new),
348        };
349        Self::from_spec(spec, default_pool_connections())
350    }
351
352    fn open_compressed_with_optional_key(
353        path: &Path,
354        compression: SQLiteCompressionOptions,
355        key: Option<&str>,
356        trusted_anchor: Option<SQLiteCompressedContainerAnchor>,
357    ) -> Result<Self> {
358        let compression = compression
359            .validate()
360            .map_err(SQLiteError::CompressedContainer)?;
361        let owner = ownership::open(
362            path,
363            &uqa_storage::read_control::StorageReadControl::with_limit(
364                uqa_storage::mvcc::VersionedSessionOptions::default().retained_bytes,
365            ),
366        )?;
367        match trusted_anchor {
368            Some(anchor) => compressed_vfs::register_database_with_anchor(
369                path,
370                compression,
371                key.ok_or(SQLiteError::EncryptionKeyRequired)?,
372                anchor,
373            ),
374            None => compressed_vfs::register_database(path, compression, key),
375        }
376        .map_err(SQLiteError::CompressedContainer)?;
377        let spec = ConnectionSpec::Compressed {
378            path: path.to_path_buf(),
379            compression,
380            key: key.map(StorageEncryptionKey::new),
381        };
382        Self::from_spec_owned(spec, default_pool_connections(), Some(owner))
383    }
384
385    pub fn open_in_memory() -> Result<Self> {
386        // Independent `:memory:` connections do not share a database. Keep a
387        // one-connection pool for this special target; file-backed databases
388        // use the real multi-connection pool.
389        Self::from_spec(ConnectionSpec::Memory, 1)
390    }
391
392    fn from_spec(spec: ConnectionSpec, max_connections: usize) -> Result<Self> {
393        let owner = match &spec {
394            ConnectionSpec::File { path, .. }
395                if path.as_os_str().is_empty() || path == Path::new(":memory:") =>
396            {
397                None
398            }
399            ConnectionSpec::File { path, .. }
400            | ConnectionSpec::Auxiliary { path, .. }
401            | ConnectionSpec::Compressed { path, .. } => Some(ownership::open(
402                path,
403                &uqa_storage::read_control::StorageReadControl::with_limit(
404                    uqa_storage::mvcc::VersionedSessionOptions::default().retained_bytes,
405                ),
406            )?),
407            ConnectionSpec::Memory => None,
408        };
409        Self::from_spec_owned(spec, max_connections, owner)
410    }
411
412    fn from_spec_owned(
413        spec: ConnectionSpec,
414        max_connections: usize,
415        owner: Option<Arc<ownership::DatabaseOwner>>,
416    ) -> Result<Self> {
417        let initial = spec.open(true)?;
418        crate::mvcc::restore::reject_pending(&initial)?;
419        Ok(Self {
420            pool: ConnectionPool::new(spec, initial, max_connections, owner)?,
421            session: Arc::new(SessionState::new()),
422            record_access: false,
423        })
424    }
425
426    pub(crate) fn database_owner(&self) -> Option<Arc<ownership::DatabaseOwner>> {
427        self.pool.owner.clone()
428    }
429
430    fn apply_encryption_key(conn: &Connection, key: &str) -> Result<()> {
431        if key.is_empty() {
432            return Err(SQLiteError::EmptyEncryptionKey);
433        }
434        conn.pragma_update(None, "key", key)?;
435        Ok(())
436    }
437
438    fn enable_wal(conn: &Connection) -> Result<()> {
439        conn.pragma_update(None, "journal_mode", "WAL")?;
440        Ok(())
441    }
442
443    fn configure_wal_connection(conn: &Connection) -> Result<()> {
444        // Legacy physical operations retain a bounded busy wait. Versioned writes supply cancellable admission under a scoped timeout.
445        conn.busy_timeout(std::time::Duration::from_secs(5))?;
446        // Synchronous=NORMAL is the recommended pairing with WAL: safe
447        // against power loss, faster than FULL.
448        conn.pragma_update(None, "synchronous", "NORMAL")?;
449        // Foreign keys are off by default; turn on so per-table cleanup
450        // can use ON DELETE CASCADE later.
451        conn.pragma_update(None, "foreign_keys", "ON")?;
452        Ok(())
453    }
454
455    fn enable_compressed_journal(conn: &Connection) -> Result<()> {
456        // The compressed VFS implements the byte-addressed database file.
457        // Rollback journals stay raw because they are short-lived commit
458        // machinery; compressing them only adds autocommit write amplification.
459        // WAL requires shared-memory VFS methods, so compressed databases use
460        // SQLite's rollback journal and keep temp storage in memory.
461        conn.pragma_update(None, "journal_mode", "DELETE")?;
462        Ok(())
463    }
464
465    fn configure_rollback_connection(conn: &Connection) -> Result<()> {
466        conn.busy_timeout(std::time::Duration::from_secs(5))?;
467        conn.pragma_update(None, "synchronous", "FULL")?;
468        conn.pragma_update(None, "temp_store", "MEMORY")?;
469        conn.pragma_update(None, "foreign_keys", "ON")?;
470        Ok(())
471    }
472
473    /// Identity shared by handles using this session's native or logical transaction context.
474    pub fn transaction_affinity(&self) -> uqa_storage::StorageSessionAffinity {
475        let _gate = self.session.gate.read();
476        self.session.logical.get().map_or_else(
477            || self.session.affinity.clone(),
478            |logical| logical.session_affinity(),
479        )
480    }
481
482    /// Whether this session has an active native or logical record transaction.
483    #[must_use]
484    pub fn in_transaction(&self) -> bool {
485        let _gate = self.session.gate.read();
486        if let Some(logical) = self.session.logical.get() {
487            return logical.in_transaction();
488        }
489        self.session.transaction.lock().is_some()
490    }
491
492    /// Whether this session has staged writes, including private logical records. Callers use this to enforce read-only execution boundaries before COMMIT.
493    pub fn transaction_has_written(&self) -> Result<bool> {
494        let _gate = self.session.gate.read();
495        if let Some(logical) = self.session.logical.get() {
496            return logical.transaction_has_written().map_err(Into::into);
497        }
498        let transaction = self.session.transaction.lock();
499        let transaction = transaction
500            .as_ref()
501            .ok_or(SQLiteError::NoActiveTransaction)?;
502        Ok(matches!(
503            transaction.connection()?.transaction_state(Some("main"))?,
504            rusqlite::TransactionState::Write
505        ))
506    }
507
508    /// Whether this session's independent [`Self::data_version`] monitor can
509    /// read without contending with the currently pinned transaction.
510    ///
511    /// A rollback-journal writer's pending lock blocks new readers while waiting for existing readers to finish. A rollback-journal read transaction must therefore also avoid the independent monitor: its own shared lock may be preventing that waiting writer from proceeding. Callers refresh through the pinned connection instead. WAL sessions and sessions without a pinned transaction permit the independent monitor.
512    pub fn data_version_monitor_is_nonblocking(&self) -> Result<bool> {
513        let _gate = self.session.gate.read();
514        if self.session.logical.get().is_some() {
515            return Ok(true);
516        }
517        if !matches!(
518            &self.pool.spec,
519            ConnectionSpec::Compressed { .. } | ConnectionSpec::Auxiliary { .. }
520        ) {
521            return Ok(true);
522        }
523        Ok(self.session.transaction.lock().is_none())
524    }
525
526    /// Whether one pooled connection may retain a read snapshot while another writes. Ordinary plain and encrypted `SQLite` databases use WAL; compressed containers and auxiliary files use rollback journaling and therefore require a detached engine snapshot before writer promotion.
527    #[must_use]
528    pub fn supports_concurrent_pinned_read_and_write(&self) -> bool {
529        let _gate = self.session.gate.read();
530        self.session.logical.get().is_some()
531            || !matches!(
532                &self.pool.spec,
533                ConnectionSpec::Compressed { .. } | ConnectionSpec::Auxiliary { .. }
534            )
535    }
536
537    /// Database change counter observed on one stable connection shared by
538    /// every logical session over this pool. The value changes when another
539    /// `SQLite` connection commits. In-memory databases have no independent
540    /// connections and therefore return `None`.
541    pub fn data_version(&self) -> Result<Option<u64>> {
542        let _gate = self.session.gate.read();
543        self.pool.check_source()?;
544        if let Some(logical) = self.session.logical.get() {
545            return logical.change_version().map_err(Into::into);
546        }
547        if matches!(&self.pool.spec, ConnectionSpec::Memory) {
548            return Ok(None);
549        }
550        let mut monitor = self.pool.data_version_monitor.lock();
551        if monitor.is_none() {
552            *monitor = Some(self.pool.open_connection()?);
553        }
554        let monitor = monitor.as_ref().ok_or_else(|| {
555            SQLiteError::StorageBackend(
556                "data-version monitor was not initialized after opening it".into(),
557            )
558        })?;
559        let version: i64 = monitor.pragma_query_value(None, "data_version", |row| row.get(0))?;
560        let version = u64::try_from(version).map_err(|_| {
561            SQLiteError::StorageBackend(format!(
562                "SQLite returned a negative PRAGMA data_version: {version}"
563            ))
564        })?;
565        Ok(Some(version))
566    }
567
568    /// `SQLite`'s change counter on the pool's monitor connection, which differs from an earlier read exactly when another connection of any session or process committed in between. `None` when the monitor may not read at any time: an in-memory database has no independent connection, and a rollback-journal reader waits for a pending writer, which may itself be waiting for a read this session has open.
569    pub fn commit_monitor_version(&self) -> Result<Option<u64>> {
570        if matches!(
571            &self.pool.spec,
572            ConnectionSpec::Memory
573                | ConnectionSpec::Compressed { .. }
574                | ConnectionSpec::Auxiliary { .. }
575        ) {
576            return Ok(None);
577        }
578        self.pool.check_source()?;
579        let mut monitor = self.pool.data_version_monitor.lock();
580        if monitor.is_none() {
581            *monitor = Some(self.pool.open_connection()?);
582        }
583        let monitor = monitor.as_ref().ok_or_else(|| {
584            SQLiteError::StorageBackend(
585                "data-version monitor was not initialized after opening it".into(),
586            )
587        })?;
588        let version: i64 = monitor
589            .prepare_cached("PRAGMA data_version")?
590            .query_row([], |row| row.get(0))?;
591        u64::try_from(version).map(Some).map_err(|_| {
592            SQLiteError::StorageBackend(format!(
593                "SQLite returned a negative PRAGMA data_version: {version}"
594            ))
595        })
596    }
597
598    /// Complete logical record visibility, including the private root restored by savepoint undo. Legacy physical sessions retain their existing data-version refresh protocol.
599    pub fn read_view_revision(
600        &self,
601    ) -> Result<Option<uqa_storage::key_value::KeyValueReadRevision>> {
602        self.surface_cleanup_failure()?;
603        let _gate = self.session.gate.read();
604        self.session.logical.get().map_or(Ok(None), |logical| {
605            logical.read_view_revision().map_err(Into::into)
606        })
607    }
608
609    /// Establish the database snapshot for the active transaction without
610    /// depending on the caller's first user query. `BEGIN DEFERRED` alone does
611    /// not start a read transaction, so a writer could otherwise commit after
612    /// the engine checks its cache generations but before the first catalog
613    /// read. Reading `sqlite_schema` is database-wide and keeps the operation
614    /// independent of any application table.
615    pub fn pin_transaction_snapshot(&self) -> Result<()> {
616        self.surface_cleanup_failure()?;
617        let _gate = self.session.gate.read();
618        if let Some(logical) = self.session.logical.get() {
619            return if logical.in_transaction() {
620                Ok(())
621            } else {
622                Err(SQLiteError::NoActiveTransaction)
623            };
624        }
625        self.with_native(|connection| {
626            if connection.is_autocommit() {
627                return Err(SQLiteError::NoActiveTransaction);
628            }
629            let _: i64 =
630                connection.query_row("SELECT COUNT(*) FROM sqlite_schema", [], |row| row.get(0))?;
631            Ok(())
632        })
633    }
634
635    /// Run a closure using this session. Outside a transaction the closure
636    /// checks out a pooled connection; inside a transaction every clone is
637    /// routed to the session's pinned connection.
638    pub fn with<R>(&self, f: impl FnOnce(&Connection) -> Result<R>) -> Result<R> {
639        self.surface_cleanup_failure()?;
640        let _gate = self.session.gate.read();
641        if self.session.logical.get().is_some() {
642            return Err(SQLiteError::LogicalSessionRequired);
643        }
644        self.with_native(f)
645    }
646
647    /// The caller holds the session gate and has ruled out logical record access.
648    fn with_native<R>(&self, f: impl FnOnce(&Connection) -> Result<R>) -> Result<R> {
649        let transaction = self.session.transaction.lock();
650        if let Some(connection) = transaction.as_ref() {
651            if let Some(error) = self.session.transaction_failure.lock().as_ref() {
652                return Err(SQLiteError::TransactionAborted(error.clone()));
653            }
654            self.check_native_access(connection.connection()?)?;
655            let result = f(connection.connection()?);
656            if let Err(error) = &result {
657                let mut failure = self.session.transaction_failure.lock();
658                if failure.is_none() {
659                    *failure = Some(error.to_string());
660                }
661            }
662            return result;
663        }
664        drop(transaction);
665        let connection = self.pool.checkout()?;
666        self.check_native_access(connection.connection()?)?;
667        f(connection.connection()?)
668    }
669
670    pub fn with_mut<R>(&self, f: impl FnOnce(&mut Connection) -> Result<R>) -> Result<R> {
671        self.with_physical_mut(|physical| f(&mut physical.connection))
672    }
673
674    fn with_physical_mut<R>(
675        &self,
676        f: impl FnOnce(&mut PhysicalConnection) -> Result<R>,
677    ) -> Result<R> {
678        self.surface_cleanup_failure()?;
679        let _gate = self.session.gate.read();
680        if self.session.logical.get().is_some() {
681            return Err(SQLiteError::LogicalSessionRequired);
682        }
683        let mut transaction = self.session.transaction.lock();
684        if let Some(connection) = transaction.as_mut() {
685            if let Some(error) = self.session.transaction_failure.lock().as_ref() {
686                return Err(SQLiteError::TransactionAborted(error.clone()));
687            }
688            self.check_native_access(connection.connection()?)?;
689            let result = f(connection.physical_mut()?);
690            if let Err(error) = &result {
691                let mut failure = self.session.transaction_failure.lock();
692                if failure.is_none() {
693                    *failure = Some(error.to_string());
694                }
695            }
696            return result;
697        }
698        drop(transaction);
699        let mut connection = self.pool.checkout()?;
700        self.check_native_access(connection.connection()?)?;
701        f(connection.physical_mut()?)
702    }
703
704    /// Rewrite the `SQLite` database into its minimum-sized file. `SQLite` requires `VACUUM` to run in autocommit mode, so the session write gate makes the transaction check and maintenance command one atomic session operation.
705    pub fn vacuum(&self) -> Result<()> {
706        self.maintain_storage(true)
707    }
708
709    /// Reclaim obsolete logical records and history without executing a physical `SQLite` `VACUUM`.
710    pub fn reclaim_obsolete(&self) -> Result<()> {
711        self.maintain_storage(false)
712    }
713
714    fn maintain_storage(&self, compact: bool) -> Result<()> {
715        self.surface_cleanup_failure()?;
716        let _gate = self.session.gate.write();
717        if self.session.transaction.lock().is_some()
718            || self
719                .session
720                .logical
721                .get()
722                .is_some_and(|logical| logical.in_transaction())
723        {
724            return Err(SQLiteError::TransactionAlreadyActive);
725        }
726        if let Some(logical) = self.session.logical.get() {
727            let source: Arc<dyn KeyValueStore> = if let Some(namespace) = logical.native {
728                Arc::new(crate::diskann::Records::new(
729                    logical.store.clone(),
730                    namespace,
731                    self.auxiliary_encryption_key(),
732                )?)
733            } else {
734                logical.store.clone()
735            };
736            let retained = logical.retention_control();
737            let control = uqa_storage::read_control::StorageReadControl::new(
738                retained.memory(),
739                &self.write_cancellation(),
740            );
741            uqa_storage::key_value::KeyValueDiskANNMaintenance::run(&source, &control)?;
742            logical.reclaim_versions()?;
743        }
744        if compact {
745            let connection = self.pool.checkout()?;
746            connection.connection()?.execute_batch("VACUUM")?;
747        }
748        Ok(())
749    }
750
751    /// Open an explicit (non-deferred) transaction. Subsequent
752    /// auto-commit hosts (catalog writes, FTS index updates, ...) all
753    /// flow through the same connection so the transaction enclosing
754    /// them is honoured. Use [`Self::commit_transaction`] /
755    /// [`Self::rollback_transaction`] / [`Self::savepoint`] etc. for
756    /// the lifecycle.
757    pub fn begin_transaction(&self) -> Result<()> {
758        self.begin_transaction_with("BEGIN IMMEDIATE")
759    }
760
761    /// Open a deferred transaction. Read-only SQL statements use this mode
762    /// so WAL readers do not take the single writer reservation; if a scalar
763    /// routine performs a write, `SQLite` upgrades the same transaction and
764    /// still preserves the statement's atomic boundary.
765    pub fn begin_deferred_transaction(&self) -> Result<()> {
766        self.begin_transaction_with("BEGIN DEFERRED")
767    }
768
769    fn begin_transaction_with(&self, statement: &str) -> Result<()> {
770        self.surface_cleanup_failure()?;
771        let _gate = self.session.gate.write();
772        if let Some(logical) = self.session.logical.get() {
773            if logical.in_transaction() {
774                return Err(SQLiteError::TransactionAlreadyActive);
775            }
776            return if statement == "BEGIN DEFERRED" {
777                logical.begin_upgradeable_transaction()
778            } else {
779                logical.begin_transaction()
780            }
781            .map_err(Into::into);
782        }
783        let mut transaction = self.session.transaction.lock();
784        if transaction.is_some() {
785            return Err(SQLiteError::TransactionAlreadyActive);
786        }
787        let connection = self.pool.checkout()?;
788        self.check_native_access(connection.connection()?)?;
789        connection.connection()?.execute_batch(statement)?;
790        self.session.transaction_failure.lock().take();
791        *transaction = Some(connection);
792        Ok(())
793    }
794
795    pub fn commit_transaction(&self) -> Result<()> {
796        self.surface_cleanup_failure()?;
797        self.finish_transaction("COMMIT")
798    }
799
800    pub fn rollback_transaction(&self) -> Result<()> {
801        self.surface_cleanup_failure()?;
802        self.finish_transaction("ROLLBACK")
803    }
804
805    /// Drop-only transaction cleanup. A rollback error is recorded on the
806    /// logical session and is returned by its next operation instead of being
807    /// mistaken for a successful rollback.
808    pub(crate) fn rollback_transaction_on_drop(&self) {
809        if let Err(error) = self.finish_transaction("ROLLBACK") {
810            let mut failure = self.session.cleanup_failure.lock();
811            if failure.is_none() {
812                *failure = Some(error.to_string());
813            }
814        }
815    }
816
817    fn finish_transaction(&self, statement: &str) -> Result<()> {
818        let _gate = self.session.gate.write();
819        if let Some(logical) = self.session.logical.get() {
820            if !logical.in_transaction() {
821                return Err(SQLiteError::NoActiveTransaction);
822            }
823            return if statement == "COMMIT" {
824                logical.commit_transaction()
825            } else {
826                logical.rollback_transaction()
827            }
828            .map_err(Into::into);
829        }
830        let mut transaction = self.session.transaction.lock();
831        let connection = transaction
832            .as_ref()
833            .ok_or(SQLiteError::NoActiveTransaction)?;
834        *self.session.snapshot_branch.lock() = Arc::new(());
835        if statement == "COMMIT" {
836            // Materialize the failure before entering the branch. Holding the
837            // mutex guard created by an `if let` scrutinee until the end of
838            // the branch would deadlock when cleanup takes the same lock.
839            let transaction_failure = self.session.transaction_failure.lock().clone();
840            if let Some(error) = transaction_failure {
841                if let Err(rollback_error) = connection.connection()?.execute_batch("ROLLBACK") {
842                    self.session.native_restore.lock().take();
843                    transaction.take();
844                    self.session.transaction_failure.lock().take();
845                    return Err(SQLiteError::SQLite(rollback_error));
846                }
847                self.session.native_restore.lock().take();
848                transaction.take();
849                self.session.transaction_failure.lock().take();
850                return Err(SQLiteError::TransactionAborted(error));
851            }
852        }
853        let native = if statement == "COMMIT" && self.session.native_restore.lock().is_some() {
854            match self.prepare_initial_native_binding(connection.connection()?) {
855                Ok(native) => Some(native),
856                Err(error) => {
857                    *self.session.transaction_failure.lock() = Some(error.to_string());
858                    return Err(error);
859                }
860            }
861        } else {
862            None
863        };
864        if let Err(error) = connection.connection()?.execute_batch(statement) {
865            self.session.native_restore.lock().take();
866            transaction.take();
867            self.session.transaction_failure.lock().take();
868            return Err(SQLiteError::SQLite(error));
869        }
870        self.session.native_restore.lock().take();
871        transaction.take();
872        self.session.transaction_failure.lock().take();
873        if let Some(native) = native {
874            self.install_initial_native_binding(native);
875        }
876        Ok(())
877    }
878
879    fn with_transaction<R>(
880        &self,
881        logical: impl FnOnce(&VersionedKeyValueStore) -> uqa_storage::StorageBackendResult<R>,
882        native: impl FnOnce(&Connection) -> Result<R>,
883    ) -> Result<R> {
884        self.surface_cleanup_failure()?;
885        let _gate = self.session.gate.read();
886        if let Some(store) = self.session.logical.get() {
887            if !store.in_transaction() {
888                return Err(SQLiteError::NoActiveTransaction);
889            }
890            return logical(store).map_err(Into::into);
891        }
892        let transaction = self.session.transaction.lock();
893        let connection = transaction
894            .as_ref()
895            .ok_or(SQLiteError::NoActiveTransaction)?;
896        self.check_native_access(connection.connection()?)?;
897        native(connection.connection()?)
898    }
899
900    pub fn savepoint(&self, name: &str) -> Result<()> {
901        self.with_transaction(
902            |logical| logical.savepoint(name),
903            |connection| {
904                let stmt = format!("SAVEPOINT \"{}\"", name.replace('"', "\"\""));
905                connection.execute_batch(&stmt)?;
906                Ok(())
907            },
908        )
909    }
910
911    pub fn release_savepoint(&self, name: &str) -> Result<()> {
912        self.with_transaction(
913            |logical| logical.release_savepoint(name),
914            |connection| {
915                let stmt = format!("RELEASE SAVEPOINT \"{}\"", name.replace('"', "\"\""));
916                connection.execute_batch(&stmt)?;
917                Ok(())
918            },
919        )
920    }
921
922    pub fn rollback_to_savepoint(&self, name: &str) -> Result<()> {
923        self.with_transaction(
924            |logical| logical.rollback_to_savepoint(name),
925            |connection| {
926                let stmt = format!("ROLLBACK TO SAVEPOINT \"{}\"", name.replace('"', "\"\""));
927                connection.execute_batch(&stmt)?;
928                self.session.transaction_failure.lock().take();
929                *self.session.snapshot_branch.lock() = Arc::new(());
930                Ok(())
931            },
932        )
933    }
934}
935
936fn default_pool_connections() -> usize {
937    std::thread::available_parallelism()
938        .map_or(MIN_POOL_CONNECTIONS, |parallelism| parallelism.get() * 2)
939        .clamp(MIN_POOL_CONNECTIONS, MAX_POOL_CONNECTIONS)
940}
941
942#[cfg(test)]
943mod tests;
944
945impl From<uqa_storage::mvcc::VersionError> for SQLiteError {
946    fn from(error: uqa_storage::mvcc::VersionError) -> Self {
947        Self::from(error.into_storage_error())
948    }
949}
950
951impl From<SQLiteError> for uqa_storage::StorageBackendError {
952    fn from(source: SQLiteError) -> Self {
953        match source {
954            SQLiteError::Memory(error) => Self::Memory(error),
955            SQLiteError::Cancelled(error) => Self::Cancelled(error),
956            SQLiteError::StorageSource(error) => *error,
957            source => Self::backend("SQLite", source),
958        }
959    }
960}
961
962impl From<SQLiteError> for uqa_storage::TransactionError {
963    fn from(source: SQLiteError) -> Self {
964        Self::Storage(source.into())
965    }
966}
967
968impl From<uqa_storage::StorageBackendError> for SQLiteError {
969    fn from(error: uqa_storage::StorageBackendError) -> Self {
970        use uqa_storage::StorageBackendError;
971        match error {
972            StorageBackendError::Memory(error) => Self::Memory(error),
973            StorageBackendError::Cancelled(error) => Self::Cancelled(error),
974            StorageBackendError::Analysis(error) => Self::Analysis(error),
975            StorageBackendError::Serde(error) => Self::Serde(error),
976            StorageBackendError::Backend { backend, source } => match source.downcast::<Self>() {
977                Ok(error) => *error,
978                Err(source) => {
979                    Self::StorageSource(Box::new(StorageBackendError::Backend { backend, source }))
980                }
981            },
982            StorageBackendError::Other(message) => Self::StorageBackend(message),
983        }
984    }
985}