Skip to main content

uqa_storage_sqlite/
connection.rs

1//
2// Unified Query Algebra
3//
4// Copyright (c) 2023-2026 Cognica, Inc.
5//
6
7//! Pooled `SQLite` connections with explicit session and transaction affinity.
8//!
9//! A [`ManagedConnection`] is a logical session. Clones share that session so
10//! catalog, document, inverted-index, and vector stores participate in the
11//! same explicit transaction. [`ManagedConnection::new_session`] creates an
12//! isolated session over the same physical connection pool. Outside explicit
13//! transactions operations check out independent connections, allowing WAL
14//! readers to make real concurrent progress.
15
16use std::path::{Path, PathBuf};
17use std::sync::{Arc, OnceLock};
18
19use parking_lot::{Mutex, RwLock};
20use rusqlite::{Connection, OpenFlags};
21use uqa_storage::{mvcc::VersionedKeyValueStore, KeyValueStore, StorageEncryptionKey};
22
23use crate::compressed_vfs::{self, SQLiteCompressedContainerAnchor, SQLiteCompressionOptions};
24
25#[cfg(not(target_os = "emscripten"))]
26mod checkpoint;
27mod identity;
28mod logical;
29mod native;
30mod native_restore;
31pub(crate) mod ownership;
32mod pool;
33mod restore;
34use pool::ConnectionPool;
35pub(crate) use pool::PooledConnection;
36mod notifications;
37mod serializable;
38mod snapshot;
39mod source;
40use snapshot::PhysicalConnection;
41pub(crate) use snapshot::{SnapshotIdentity, ValidatedRead};
42
43#[derive(Debug, thiserror::Error)]
44pub enum SQLiteError {
45    #[error(transparent)]
46    Memory(#[from] uqa_core::memory::MemoryError),
47    #[error(transparent)]
48    Cancelled(#[from] uqa_core::QueryCancelled),
49    #[error("text analysis failed: {0}")]
50    Analysis(#[from] uqa_analysis::AnalysisError),
51    #[error("sqlite error: {0}")]
52    SQLite(#[from] rusqlite::Error),
53    #[error("encryption key must not be empty")]
54    EmptyEncryptionKey,
55    #[error("database requires an encryption key")]
56    EncryptionKeyRequired,
57    #[error("database is not encrypted but an encryption key was provided")]
58    NotEncrypted,
59    #[error("auxiliary database requires DELETE journaling, found {0}")]
60    AuxiliaryJournalMode(String),
61    #[error("compressed sqlite container error: {0}")]
62    CompressedContainer(String),
63    #[error("io error: {0}")]
64    Io(#[from] std::io::Error),
65    #[error("catalog migration {version} failed: {source}")]
66    Migration {
67        version: u32,
68        #[source]
69        source: rusqlite::Error,
70    },
71    #[error("invalid persisted catalog schema version `{0}`")]
72    InvalidSchemaVersion(String),
73    #[error("catalog schema version {found} is newer than this engine supports ({supported})")]
74    UnsupportedSchemaVersion { found: u32, supported: u32 },
75    #[error("corrupt document blob for `{table}` doc {doc_id} field `{field}`: {reason}")]
76    CorruptDocumentBlob {
77        table: String,
78        doc_id: u64,
79        field: String,
80        reason: String,
81    },
82    #[error("payload serialization failed: {0}")]
83    Serde(#[from] serde_json::Error),
84    #[error("storage backend error: {0}")]
85    StorageBackend(String),
86    #[error(transparent)]
87    StorageSource(Box<uqa_storage::StorageBackendError>),
88    #[error("transaction already active for this sqlite session")]
89    TransactionAlreadyActive,
90    #[error("no active transaction for this sqlite session")]
91    NoActiveTransaction,
92    #[error("sqlite transaction was aborted by an earlier storage error: {0}")]
93    TransactionAborted(String),
94    #[error("sqlite session cleanup failed: {0}")]
95    SessionCleanupFailed(String),
96    #[error("sqlite connection-pool checkout lost its connection")]
97    MissingCheckedOutConnection,
98    #[error("versioned storage requires its logical session; use with_physical only for explicit physical maintenance")]
99    LogicalSessionRequired,
100    #[error("database restoration requires all existing owners to close")]
101    DatabaseRestoreBusy,
102    #[error("record initialization requires incompatible database owners to close")]
103    DatabaseCoordinationUpgradeBusy,
104    #[error("database restoration is incomplete; resume the original source/target request")]
105    DatabaseRestoreIncomplete,
106    #[error("the SQLite session already has a different retention limit")]
107    SessionOptionsMismatch,
108    #[error("the SQLite session is bound to a different record mapping")]
109    SessionMappingMismatch,
110    #[error("the retained SQLite database source changed; close its owners and reopen explicitly")]
111    DatabaseSourceChanged,
112}
113
114pub type Result<T> = std::result::Result<T, SQLiteError>;
115
116const MIN_POOL_CONNECTIONS: usize = 4;
117const MAX_POOL_CONNECTIONS: usize = 32;
118
119/// Prepared statements each connection retains. Statements are built from record layouts and work-queue conditions, so one commit runs more distinct statements than a small cache holds, and preparing a projection upsert compiles every trigger of its table; a cache that evicts them prepares them again in each transaction.
120pub(crate) const PREPARED_STATEMENT_CACHE_CAPACITY: usize = 256;
121
122#[derive(Clone)]
123enum ConnectionSpec {
124    File {
125        path: PathBuf,
126        key: Option<StorageEncryptionKey>,
127    },
128    Auxiliary {
129        path: PathBuf,
130        key: Option<StorageEncryptionKey>,
131    },
132    Compressed {
133        path: PathBuf,
134        compression: SQLiteCompressionOptions,
135        key: Option<StorageEncryptionKey>,
136    },
137    Memory,
138}
139
140impl ConnectionSpec {
141    fn open(&self, initialize_database: bool) -> Result<Connection> {
142        let conn = self.open_configured(initialize_database)?;
143        conn.set_prepared_statement_cache_capacity(PREPARED_STATEMENT_CACHE_CAPACITY);
144        Ok(conn)
145    }
146
147    fn open_configured(&self, initialize_database: bool) -> Result<Connection> {
148        let mut flags = OpenFlags::default();
149        if !initialize_database && matches!(self, Self::File { .. } | Self::Auxiliary { .. }) {
150            flags.remove(OpenFlags::SQLITE_OPEN_CREATE);
151        }
152        match self {
153            Self::File { path, key } | Self::Auxiliary { path, key } => {
154                let conn = Connection::open_with_flags(path, flags)?;
155                if let Some(key) = key {
156                    ManagedConnection::apply_encryption_key(&conn, key.expose_secret())?;
157                }
158                crate::btree_index::equality::register(&conn)?;
159                if matches!(self, Self::Auxiliary { .. }) {
160                    // Registry writers already serialize their transactions.
161                    // Retain the original journal mode instead of racing to
162                    // promote a new registry to WAL during concurrent opens.
163                    let mode: String =
164                        conn.pragma_query_value(None, "journal_mode", |row| row.get(0))?;
165                    if mode != "delete" {
166                        return Err(SQLiteError::AuxiliaryJournalMode(mode));
167                    }
168                    ManagedConnection::configure_rollback_connection(&conn)?;
169                } else {
170                    if initialize_database {
171                        ManagedConnection::enable_wal(&conn)?;
172                    }
173                    ManagedConnection::configure_wal_connection(&conn)?;
174                }
175                Ok(conn)
176            }
177            Self::Compressed {
178                path, compression, ..
179            } => {
180                let conn =
181                    Connection::open_with_flags_and_vfs(path, flags, compressed_vfs::VFS_NAME)?;
182                crate::btree_index::equality::register(&conn)?;
183                if initialize_database {
184                    conn.pragma_update(None, "page_size", compression.page_size)?;
185                    ManagedConnection::enable_compressed_journal(&conn)?;
186                }
187                ManagedConnection::configure_rollback_connection(&conn)?;
188                Ok(conn)
189            }
190            Self::Memory => {
191                let conn = Connection::open_in_memory()?;
192                crate::btree_index::equality::register(&conn)?;
193                if initialize_database {
194                    ManagedConnection::enable_wal(&conn)?;
195                }
196                ManagedConnection::configure_wal_connection(&conn)?;
197                Ok(conn)
198            }
199        }
200    }
201}
202
203struct SessionState {
204    write_cancellation: uqa_core::CancellationToken,
205    affinity: uqa_storage::StorageSessionAffinity,
206    /// Read guards cover ordinary operations. Transaction lifecycle calls take
207    /// the write guard, making BEGIN/COMMIT/ROLLBACK linearizable with respect
208    /// to every operation issued through the same logical session.
209    gate: RwLock<()>,
210    transaction: Mutex<Option<PooledConnection>>,
211    transaction_failure: Mutex<Option<String>>,
212    cleanup_failure: Mutex<Option<String>>,
213    logical: OnceLock<Arc<logical::BoundRecordSession>>,
214    native_restore: Mutex<Option<native_restore::NativeRestore>>,
215    snapshot_branch: Mutex<Arc<()>>,
216}
217
218impl SessionState {
219    fn new() -> Self {
220        Self::with_cancellation(uqa_core::CancellationToken::new())
221    }
222
223    fn with_cancellation(write_cancellation: uqa_core::CancellationToken) -> Self {
224        Self {
225            write_cancellation,
226            affinity: uqa_storage::StorageSessionAffinity::new(),
227            gate: RwLock::new(()),
228            transaction: Mutex::new(None),
229            transaction_failure: Mutex::new(None),
230            cleanup_failure: Mutex::new(None),
231            logical: OnceLock::new(),
232            native_restore: Mutex::new(None),
233            snapshot_branch: Mutex::new(Arc::new(())),
234        }
235    }
236}
237
238impl Drop for SessionState {
239    fn drop(&mut self) {
240        // `PooledConnection::drop` performs the rollback and discards the
241        // physical connection when rollback itself fails. Taking the pinned
242        // handle here therefore cannot return a broken connection to the pool.
243        self.native_restore.get_mut().take();
244        self.transaction.get_mut().take();
245    }
246}
247
248/// Logical `SQLite` session backed by a bounded physical connection pool.
249/// Cloning preserves session/transaction affinity; call [`Self::new_session`]
250/// for an independently isolated transaction context.
251#[derive(Clone)]
252pub struct ManagedConnection {
253    pool: Arc<ConnectionPool>,
254    session: Arc<SessionState>,
255    record_access: bool,
256}
257
258impl ManagedConnection {
259    /// Decoded document columns shared by every session of this database.
260    pub(crate) fn decoded_columns(
261        &self,
262    ) -> &crate::document_store::decoded_columns::DecodedColumns {
263        &self.pool.decoded_columns
264    }
265
266    fn surface_cleanup_failure(&self) -> Result<()> {
267        if let Some(error) = self.session.cleanup_failure.lock().take() {
268            return Err(SQLiteError::SessionCleanupFailed(error));
269        }
270        Ok(())
271    }
272
273    pub fn open(path: &Path) -> Result<Self> {
274        if path == Path::new(":memory:") {
275            return Self::open_in_memory();
276        }
277        Self::open_with_optional_key(path, None)
278    }
279
280    /// Retain the database credential for encrypted auxiliary storage, including
281    /// compressed containers whose main-file encryption lives in the VFS.
282    #[must_use]
283    pub fn auxiliary_encryption_key(&self) -> Option<StorageEncryptionKey> {
284        match &self.pool.spec {
285            ConnectionSpec::File { key, .. }
286            | ConnectionSpec::Auxiliary { key, .. }
287            | ConnectionSpec::Compressed { key, .. } => key.clone(),
288            ConnectionSpec::Memory => None,
289        }
290    }
291
292    /// Lease an independent physical connection without joining this logical
293    /// session's transaction. The pool retains encryption and connection policy.
294    pub fn lease_connection(&self) -> Result<crate::SQLiteConnectionLease> {
295        self.pool.checkout().map(crate::SQLiteConnectionLease)
296    }
297
298    /// Open database-owned auxiliary storage with its inherited credential and
299    /// original DELETE journaling. Concurrent first opens never change journal
300    /// modes. An incompatible existing mode is rejected without rewriting it.
301    pub fn open_auxiliary(path: &Path, key: Option<StorageEncryptionKey>) -> Result<Self> {
302        if path == Path::new(":memory:") || path.as_os_str().is_empty() {
303            return Err(SQLiteError::StorageBackend(
304                "auxiliary storage requires a database file".into(),
305            ));
306        }
307        Self::from_spec(
308            ConnectionSpec::Auxiliary {
309                path: path.to_path_buf(),
310                key,
311            },
312            default_pool_connections(),
313        )
314    }
315
316    pub fn open_encrypted(path: &Path, key: &str) -> Result<Self> {
317        Self::open_with_optional_key(path, Some(key))
318    }
319
320    pub fn open_compressed(path: &Path, compression: SQLiteCompressionOptions) -> Result<Self> {
321        Self::open_compressed_with_optional_key(path, compression, None, None)
322    }
323
324    pub fn open_compressed_encrypted(
325        path: &Path,
326        key: &str,
327        compression: SQLiteCompressionOptions,
328    ) -> Result<Self> {
329        if key.is_empty() {
330            return Err(SQLiteError::EmptyEncryptionKey);
331        }
332        Self::open_compressed_with_optional_key(path, compression, Some(key), None)
333    }
334
335    /// Open an encrypted compressed database while enforcing an exact trusted
336    /// external state anchor in the VFS main-file open path.
337    pub fn open_compressed_encrypted_with_anchor(
338        path: &Path,
339        key: &str,
340        compression: SQLiteCompressionOptions,
341        trusted_anchor: SQLiteCompressedContainerAnchor,
342    ) -> Result<Self> {
343        if key.is_empty() {
344            return Err(SQLiteError::EmptyEncryptionKey);
345        }
346        Self::open_compressed_with_optional_key(path, compression, Some(key), Some(trusted_anchor))
347    }
348
349    fn open_with_optional_key(path: &Path, key: Option<&str>) -> Result<Self> {
350        let spec = ConnectionSpec::File {
351            path: path.to_path_buf(),
352            key: key.map(StorageEncryptionKey::new),
353        };
354        Self::from_spec(spec, default_pool_connections())
355    }
356
357    fn open_compressed_with_optional_key(
358        path: &Path,
359        compression: SQLiteCompressionOptions,
360        key: Option<&str>,
361        trusted_anchor: Option<SQLiteCompressedContainerAnchor>,
362    ) -> Result<Self> {
363        let compression = compression
364            .validate()
365            .map_err(SQLiteError::CompressedContainer)?;
366        let owner = ownership::open(
367            path,
368            &uqa_storage::read_control::StorageReadControl::with_limit(
369                uqa_storage::mvcc::VersionedSessionOptions::default().retained_bytes,
370            ),
371        )?;
372        match trusted_anchor {
373            Some(anchor) => compressed_vfs::register_database_with_anchor(
374                path,
375                compression,
376                key.ok_or(SQLiteError::EncryptionKeyRequired)?,
377                anchor,
378            ),
379            None => compressed_vfs::register_database(path, compression, key),
380        }
381        .map_err(SQLiteError::CompressedContainer)?;
382        let spec = ConnectionSpec::Compressed {
383            path: path.to_path_buf(),
384            compression,
385            key: key.map(StorageEncryptionKey::new),
386        };
387        Self::from_spec_owned(spec, default_pool_connections(), Some(owner))
388    }
389
390    pub fn open_in_memory() -> Result<Self> {
391        // Independent `:memory:` connections do not share a database. Keep a
392        // one-connection pool for this special target; file-backed databases
393        // use the real multi-connection pool.
394        Self::from_spec(ConnectionSpec::Memory, 1)
395    }
396
397    fn from_spec(spec: ConnectionSpec, max_connections: usize) -> Result<Self> {
398        let owner = match &spec {
399            ConnectionSpec::File { path, .. }
400                if path.as_os_str().is_empty() || path == Path::new(":memory:") =>
401            {
402                None
403            }
404            ConnectionSpec::File { path, .. }
405            | ConnectionSpec::Auxiliary { path, .. }
406            | ConnectionSpec::Compressed { path, .. } => Some(ownership::open(
407                path,
408                &uqa_storage::read_control::StorageReadControl::with_limit(
409                    uqa_storage::mvcc::VersionedSessionOptions::default().retained_bytes,
410                ),
411            )?),
412            ConnectionSpec::Memory => None,
413        };
414        Self::from_spec_owned(spec, max_connections, owner)
415    }
416
417    fn from_spec_owned(
418        spec: ConnectionSpec,
419        max_connections: usize,
420        owner: Option<Arc<ownership::DatabaseOwner>>,
421    ) -> Result<Self> {
422        let initial = spec.open(true)?;
423        crate::mvcc::restore::reject_pending(&initial)?;
424        Ok(Self {
425            pool: ConnectionPool::new(spec, initial, max_connections, owner)?,
426            session: Arc::new(SessionState::new()),
427            record_access: false,
428        })
429    }
430
431    pub(crate) fn database_owner(&self) -> Option<Arc<ownership::DatabaseOwner>> {
432        self.pool.owner.clone()
433    }
434
435    fn apply_encryption_key(conn: &Connection, key: &str) -> Result<()> {
436        if key.is_empty() {
437            return Err(SQLiteError::EmptyEncryptionKey);
438        }
439        conn.pragma_update(None, "key", key)?;
440        Ok(())
441    }
442
443    fn enable_wal(conn: &Connection) -> Result<()> {
444        conn.pragma_update(None, "journal_mode", "WAL")?;
445        Ok(())
446    }
447
448    fn configure_wal_connection(conn: &Connection) -> Result<()> {
449        // Legacy physical operations retain a bounded busy wait. Versioned writes supply cancellable admission under a scoped timeout.
450        conn.busy_timeout(std::time::Duration::from_secs(5))?;
451        // Synchronous=NORMAL is the recommended pairing with WAL: safe
452        // against power loss, faster than FULL.
453        conn.pragma_update(None, "synchronous", "NORMAL")?;
454        // Reuse WAL space only up to 64 MiB once readers permit a reset. Active readers and a single larger transaction retain their required frames.
455        conn.pragma_update(None, "journal_size_limit", 64_i64 << 20)?;
456        // Foreign keys are off by default; turn on so per-table cleanup
457        // can use ON DELETE CASCADE later.
458        conn.pragma_update(None, "foreign_keys", "ON")?;
459        Ok(())
460    }
461
462    fn enable_compressed_journal(conn: &Connection) -> Result<()> {
463        // The compressed VFS implements the byte-addressed database file.
464        // Rollback journals stay raw because they are short-lived commit
465        // machinery; compressing them only adds autocommit write amplification.
466        // WAL requires shared-memory VFS methods, so compressed databases use
467        // SQLite's rollback journal and keep temp storage in memory.
468        conn.pragma_update(None, "journal_mode", "DELETE")?;
469        Ok(())
470    }
471
472    fn configure_rollback_connection(conn: &Connection) -> Result<()> {
473        conn.busy_timeout(std::time::Duration::from_secs(5))?;
474        conn.pragma_update(None, "synchronous", "FULL")?;
475        conn.pragma_update(None, "temp_store", "MEMORY")?;
476        conn.pragma_update(None, "foreign_keys", "ON")?;
477        Ok(())
478    }
479
480    /// Identity shared by handles using this session's native or logical transaction context.
481    pub fn transaction_affinity(&self) -> uqa_storage::StorageSessionAffinity {
482        let _gate = self.session.gate.read();
483        self.session.logical.get().map_or_else(
484            || self.session.affinity.clone(),
485            |logical| logical.session_affinity(),
486        )
487    }
488
489    /// Whether this session has an active native or logical record transaction.
490    #[must_use]
491    pub fn in_transaction(&self) -> bool {
492        let _gate = self.session.gate.read();
493        if let Some(logical) = self.session.logical.get() {
494            return logical.in_transaction();
495        }
496        self.session.transaction.lock().is_some()
497    }
498
499    /// Whether this session has staged writes, including private logical records. Callers use this to enforce read-only execution boundaries before COMMIT.
500    pub fn transaction_has_written(&self) -> Result<bool> {
501        let _gate = self.session.gate.read();
502        if let Some(logical) = self.session.logical.get() {
503            return logical.transaction_has_written().map_err(Into::into);
504        }
505        let transaction = self.session.transaction.lock();
506        let transaction = transaction
507            .as_ref()
508            .ok_or(SQLiteError::NoActiveTransaction)?;
509        Ok(matches!(
510            transaction.connection()?.transaction_state(Some("main"))?,
511            rusqlite::TransactionState::Write
512        ))
513    }
514
515    /// Whether this session's independent [`Self::data_version`] monitor can
516    /// read without contending with the currently pinned transaction.
517    ///
518    /// A rollback-journal writer's pending lock blocks new readers while waiting for existing readers to finish. A rollback-journal read transaction must therefore also avoid the independent monitor: its own shared lock may be preventing that waiting writer from proceeding. Callers refresh through the pinned connection instead. WAL sessions and sessions without a pinned transaction permit the independent monitor.
519    pub fn data_version_monitor_is_nonblocking(&self) -> Result<bool> {
520        let _gate = self.session.gate.read();
521        if self.session.logical.get().is_some() {
522            return Ok(true);
523        }
524        if !matches!(
525            &self.pool.spec,
526            ConnectionSpec::Compressed { .. } | ConnectionSpec::Auxiliary { .. }
527        ) {
528            return Ok(true);
529        }
530        Ok(self.session.transaction.lock().is_none())
531    }
532
533    /// Whether one pooled connection may retain a read snapshot while another writes. Ordinary plain and encrypted `SQLite` databases use WAL; compressed containers and auxiliary files use rollback journaling and therefore require a detached engine snapshot before writer promotion.
534    #[must_use]
535    pub fn supports_concurrent_pinned_read_and_write(&self) -> bool {
536        let _gate = self.session.gate.read();
537        self.session.logical.get().is_some()
538            || !matches!(
539                &self.pool.spec,
540                ConnectionSpec::Compressed { .. } | ConnectionSpec::Auxiliary { .. }
541            )
542    }
543
544    /// Database change counter observed on one stable connection shared by
545    /// every logical session over this pool. The value changes when another
546    /// `SQLite` connection commits. In-memory databases have no independent
547    /// connections and therefore return `None`.
548    pub fn data_version(&self) -> Result<Option<u64>> {
549        let _gate = self.session.gate.read();
550        self.pool.check_source()?;
551        if let Some(logical) = self.session.logical.get() {
552            return logical.change_version().map_err(Into::into);
553        }
554        if matches!(&self.pool.spec, ConnectionSpec::Memory) {
555            return Ok(None);
556        }
557        let mut monitor = self.pool.data_version_monitor.lock();
558        if monitor.is_none() {
559            *monitor = Some(self.pool.open_connection()?);
560        }
561        let monitor = monitor.as_ref().ok_or_else(|| {
562            SQLiteError::StorageBackend(
563                "data-version monitor was not initialized after opening it".into(),
564            )
565        })?;
566        let version: i64 = monitor.pragma_query_value(None, "data_version", |row| row.get(0))?;
567        let version = u64::try_from(version).map_err(|_| {
568            SQLiteError::StorageBackend(format!(
569                "SQLite returned a negative PRAGMA data_version: {version}"
570            ))
571        })?;
572        Ok(Some(version))
573    }
574
575    /// `SQLite`'s change counter on the pool's monitor connection, which differs from an earlier read exactly when another connection of any session or process committed in between. `None` when the monitor may not read at any time: an in-memory database has no independent connection, and a rollback-journal reader waits for a pending writer, which may itself be waiting for a read this session has open.
576    pub fn commit_monitor_version(&self) -> Result<Option<u64>> {
577        if matches!(
578            &self.pool.spec,
579            ConnectionSpec::Memory
580                | ConnectionSpec::Compressed { .. }
581                | ConnectionSpec::Auxiliary { .. }
582        ) {
583            return Ok(None);
584        }
585        self.pool.check_source()?;
586        let mut monitor = self.pool.data_version_monitor.lock();
587        if monitor.is_none() {
588            *monitor = Some(self.pool.open_connection()?);
589        }
590        let monitor = monitor.as_ref().ok_or_else(|| {
591            SQLiteError::StorageBackend(
592                "data-version monitor was not initialized after opening it".into(),
593            )
594        })?;
595        let version: i64 = monitor
596            .prepare_cached("PRAGMA data_version")?
597            .query_row([], |row| row.get(0))?;
598        u64::try_from(version).map(Some).map_err(|_| {
599            SQLiteError::StorageBackend(format!(
600                "SQLite returned a negative PRAGMA data_version: {version}"
601            ))
602        })
603    }
604
605    /// Complete logical record visibility, including the private root restored by savepoint undo. Legacy physical sessions retain their existing data-version refresh protocol.
606    pub fn read_view_revision(
607        &self,
608    ) -> Result<Option<uqa_storage::key_value::KeyValueReadRevision>> {
609        self.surface_cleanup_failure()?;
610        let _gate = self.session.gate.read();
611        self.session.logical.get().map_or(Ok(None), |logical| {
612            logical.read_view_revision().map_err(Into::into)
613        })
614    }
615
616    /// Establish the database snapshot for the active transaction without
617    /// depending on the caller's first user query. `BEGIN DEFERRED` alone does
618    /// not start a read transaction, so a writer could otherwise commit after
619    /// the engine checks its cache generations but before the first catalog
620    /// read. Reading `sqlite_schema` is database-wide and keeps the operation
621    /// independent of any application table.
622    pub fn pin_transaction_snapshot(&self) -> Result<()> {
623        self.surface_cleanup_failure()?;
624        let _gate = self.session.gate.read();
625        if let Some(logical) = self.session.logical.get() {
626            return if logical.in_transaction() {
627                Ok(())
628            } else {
629                Err(SQLiteError::NoActiveTransaction)
630            };
631        }
632        self.with_native(|connection| {
633            if connection.is_autocommit() {
634                return Err(SQLiteError::NoActiveTransaction);
635            }
636            let _: i64 =
637                connection.query_row("SELECT COUNT(*) FROM sqlite_schema", [], |row| row.get(0))?;
638            Ok(())
639        })
640    }
641
642    /// Run a closure using this session. Outside a transaction the closure
643    /// checks out a pooled connection; inside a transaction every clone is
644    /// routed to the session's pinned connection.
645    pub fn with<R>(&self, f: impl FnOnce(&Connection) -> Result<R>) -> Result<R> {
646        self.surface_cleanup_failure()?;
647        let _gate = self.session.gate.read();
648        if self.session.logical.get().is_some() {
649            return Err(SQLiteError::LogicalSessionRequired);
650        }
651        self.with_native(f)
652    }
653
654    /// The caller holds the session gate and has ruled out logical record access.
655    fn with_native<R>(&self, f: impl FnOnce(&Connection) -> Result<R>) -> Result<R> {
656        let transaction = self.session.transaction.lock();
657        if let Some(connection) = transaction.as_ref() {
658            if let Some(error) = self.session.transaction_failure.lock().as_ref() {
659                return Err(SQLiteError::TransactionAborted(error.clone()));
660            }
661            self.check_native_access(connection.connection()?)?;
662            let result = f(connection.connection()?);
663            if let Err(error) = &result {
664                let mut failure = self.session.transaction_failure.lock();
665                if failure.is_none() {
666                    *failure = Some(error.to_string());
667                }
668            }
669            return result;
670        }
671        drop(transaction);
672        let connection = self.pool.checkout()?;
673        self.check_native_access(connection.connection()?)?;
674        f(connection.connection()?)
675    }
676
677    pub fn with_mut<R>(&self, f: impl FnOnce(&mut Connection) -> Result<R>) -> Result<R> {
678        self.with_physical_mut(|physical| f(&mut physical.connection))
679    }
680
681    fn with_physical_mut<R>(
682        &self,
683        f: impl FnOnce(&mut PhysicalConnection) -> Result<R>,
684    ) -> Result<R> {
685        self.surface_cleanup_failure()?;
686        let _gate = self.session.gate.read();
687        if self.session.logical.get().is_some() {
688            return Err(SQLiteError::LogicalSessionRequired);
689        }
690        let mut transaction = self.session.transaction.lock();
691        if let Some(connection) = transaction.as_mut() {
692            if let Some(error) = self.session.transaction_failure.lock().as_ref() {
693                return Err(SQLiteError::TransactionAborted(error.clone()));
694            }
695            self.check_native_access(connection.connection()?)?;
696            let result = f(connection.physical_mut()?);
697            if let Err(error) = &result {
698                let mut failure = self.session.transaction_failure.lock();
699                if failure.is_none() {
700                    *failure = Some(error.to_string());
701                }
702            }
703            return result;
704        }
705        drop(transaction);
706        let mut connection = self.pool.checkout()?;
707        self.check_native_access(connection.connection()?)?;
708        f(connection.physical_mut()?)
709    }
710
711    /// Rewrite the `SQLite` database into its minimum-sized file. `SQLite` requires `VACUUM` to run in autocommit mode, so the session write gate makes the transaction check and maintenance command one atomic session operation.
712    pub fn vacuum(&self) -> Result<()> {
713        self.maintain_storage(true)
714    }
715
716    /// Reclaim obsolete logical records and history without executing a physical `SQLite` `VACUUM`.
717    pub fn reclaim_obsolete(&self) -> Result<()> {
718        self.maintain_storage(false)
719    }
720
721    fn maintain_storage(&self, compact: bool) -> Result<()> {
722        self.surface_cleanup_failure()?;
723        let _gate = self.session.gate.write();
724        if self.session.transaction.lock().is_some()
725            || self
726                .session
727                .logical
728                .get()
729                .is_some_and(|logical| logical.in_transaction())
730        {
731            return Err(SQLiteError::TransactionAlreadyActive);
732        }
733        if let Some(logical) = self.session.logical.get() {
734            let source: Arc<dyn KeyValueStore> = if let Some(namespace) = logical.native {
735                Arc::new(crate::diskann::Records::new(
736                    logical.store.clone(),
737                    namespace,
738                    self.auxiliary_encryption_key(),
739                )?)
740            } else {
741                logical.store.clone()
742            };
743            let retained = logical.retention_control();
744            let control = uqa_storage::read_control::StorageReadControl::new(
745                retained.memory(),
746                &self.write_cancellation(),
747            );
748            uqa_storage::key_value::KeyValueDiskANNMaintenance::run(&source, &control)?;
749            logical.reclaim_versions()?;
750        }
751        if compact {
752            let connection = self.pool.checkout()?;
753            connection.connection()?.execute_batch("VACUUM")?;
754        }
755        Ok(())
756    }
757
758    /// Open an explicit (non-deferred) transaction. Subsequent
759    /// auto-commit hosts (catalog writes, FTS index updates, ...) all
760    /// flow through the same connection so the transaction enclosing
761    /// them is honoured. Use [`Self::commit_transaction`] /
762    /// [`Self::rollback_transaction`] / [`Self::savepoint`] etc. for
763    /// the lifecycle.
764    pub fn begin_transaction(&self) -> Result<()> {
765        self.begin_transaction_with("BEGIN IMMEDIATE")
766    }
767
768    /// Open a deferred transaction. Read-only SQL statements use this mode
769    /// so WAL readers do not take the single writer reservation; if a scalar
770    /// routine performs a write, `SQLite` upgrades the same transaction and
771    /// still preserves the statement's atomic boundary.
772    pub fn begin_deferred_transaction(&self) -> Result<()> {
773        self.begin_transaction_with("BEGIN DEFERRED")
774    }
775
776    fn begin_transaction_with(&self, statement: &str) -> Result<()> {
777        self.surface_cleanup_failure()?;
778        let _gate = self.session.gate.write();
779        if let Some(logical) = self.session.logical.get() {
780            if logical.in_transaction() {
781                return Err(SQLiteError::TransactionAlreadyActive);
782            }
783            return if statement == "BEGIN DEFERRED" {
784                logical.begin_upgradeable_transaction()
785            } else {
786                logical.begin_transaction()
787            }
788            .map_err(Into::into);
789        }
790        let mut transaction = self.session.transaction.lock();
791        if transaction.is_some() {
792            return Err(SQLiteError::TransactionAlreadyActive);
793        }
794        let connection = self.pool.checkout()?;
795        self.check_native_access(connection.connection()?)?;
796        connection.connection()?.execute_batch(statement)?;
797        self.session.transaction_failure.lock().take();
798        *transaction = Some(connection);
799        Ok(())
800    }
801
802    pub fn commit_transaction(&self) -> Result<()> {
803        self.surface_cleanup_failure()?;
804        self.finish_transaction("COMMIT")
805    }
806
807    pub fn rollback_transaction(&self) -> Result<()> {
808        self.surface_cleanup_failure()?;
809        self.finish_transaction("ROLLBACK")
810    }
811
812    /// Drop-only transaction cleanup. A rollback error is recorded on the
813    /// logical session and is returned by its next operation instead of being
814    /// mistaken for a successful rollback.
815    pub(crate) fn rollback_transaction_on_drop(&self) {
816        if let Err(error) = self.finish_transaction("ROLLBACK") {
817            let mut failure = self.session.cleanup_failure.lock();
818            if failure.is_none() {
819                *failure = Some(error.to_string());
820            }
821        }
822    }
823
824    fn finish_transaction(&self, statement: &str) -> Result<()> {
825        let _gate = self.session.gate.write();
826        if let Some(logical) = self.session.logical.get() {
827            if !logical.in_transaction() {
828                return Err(SQLiteError::NoActiveTransaction);
829            }
830            return if statement == "COMMIT" {
831                logical.commit_transaction()
832            } else {
833                logical.rollback_transaction()
834            }
835            .map_err(Into::into);
836        }
837        let mut transaction = self.session.transaction.lock();
838        let connection = transaction
839            .as_ref()
840            .ok_or(SQLiteError::NoActiveTransaction)?;
841        *self.session.snapshot_branch.lock() = Arc::new(());
842        if statement == "COMMIT" {
843            // Materialize the failure before entering the branch. Holding the
844            // mutex guard created by an `if let` scrutinee until the end of
845            // the branch would deadlock when cleanup takes the same lock.
846            let transaction_failure = self.session.transaction_failure.lock().clone();
847            if let Some(error) = transaction_failure {
848                if let Err(rollback_error) = connection.connection()?.execute_batch("ROLLBACK") {
849                    self.session.native_restore.lock().take();
850                    transaction.take();
851                    self.session.transaction_failure.lock().take();
852                    return Err(SQLiteError::SQLite(rollback_error));
853                }
854                self.session.native_restore.lock().take();
855                transaction.take();
856                self.session.transaction_failure.lock().take();
857                return Err(SQLiteError::TransactionAborted(error));
858            }
859        }
860        let native = if statement == "COMMIT" && self.session.native_restore.lock().is_some() {
861            match self.prepare_initial_native_binding(connection.connection()?) {
862                Ok(native) => Some(native),
863                Err(error) => {
864                    *self.session.transaction_failure.lock() = Some(error.to_string());
865                    return Err(error);
866                }
867            }
868        } else {
869            None
870        };
871        if let Err(error) = connection.connection()?.execute_batch(statement) {
872            self.session.native_restore.lock().take();
873            transaction.take();
874            self.session.transaction_failure.lock().take();
875            return Err(SQLiteError::SQLite(error));
876        }
877        self.session.native_restore.lock().take();
878        transaction.take();
879        self.session.transaction_failure.lock().take();
880        if let Some(native) = native {
881            self.install_initial_native_binding(native);
882        }
883        Ok(())
884    }
885
886    fn with_transaction<R>(
887        &self,
888        logical: impl FnOnce(&VersionedKeyValueStore) -> uqa_storage::StorageBackendResult<R>,
889        native: impl FnOnce(&Connection) -> Result<R>,
890    ) -> Result<R> {
891        self.surface_cleanup_failure()?;
892        let _gate = self.session.gate.read();
893        if let Some(store) = self.session.logical.get() {
894            if !store.in_transaction() {
895                return Err(SQLiteError::NoActiveTransaction);
896            }
897            return logical(store).map_err(Into::into);
898        }
899        let transaction = self.session.transaction.lock();
900        let connection = transaction
901            .as_ref()
902            .ok_or(SQLiteError::NoActiveTransaction)?;
903        self.check_native_access(connection.connection()?)?;
904        native(connection.connection()?)
905    }
906
907    pub fn savepoint(&self, name: &str) -> Result<()> {
908        self.with_transaction(
909            |logical| logical.savepoint(name),
910            |connection| {
911                let stmt = format!("SAVEPOINT \"{}\"", name.replace('"', "\"\""));
912                connection.execute_batch(&stmt)?;
913                Ok(())
914            },
915        )
916    }
917
918    pub fn release_savepoint(&self, name: &str) -> Result<()> {
919        self.with_transaction(
920            |logical| logical.release_savepoint(name),
921            |connection| {
922                let stmt = format!("RELEASE SAVEPOINT \"{}\"", name.replace('"', "\"\""));
923                connection.execute_batch(&stmt)?;
924                Ok(())
925            },
926        )
927    }
928
929    pub fn rollback_to_savepoint(&self, name: &str) -> Result<()> {
930        self.with_transaction(
931            |logical| logical.rollback_to_savepoint(name),
932            |connection| {
933                let stmt = format!("ROLLBACK TO SAVEPOINT \"{}\"", name.replace('"', "\"\""));
934                connection.execute_batch(&stmt)?;
935                self.session.transaction_failure.lock().take();
936                *self.session.snapshot_branch.lock() = Arc::new(());
937                Ok(())
938            },
939        )
940    }
941}
942
943fn default_pool_connections() -> usize {
944    std::thread::available_parallelism()
945        .map_or(MIN_POOL_CONNECTIONS, |parallelism| parallelism.get() * 2)
946        .clamp(MIN_POOL_CONNECTIONS, MAX_POOL_CONNECTIONS)
947}
948
949#[cfg(test)]
950mod tests;
951
952impl From<uqa_storage::mvcc::VersionError> for SQLiteError {
953    fn from(error: uqa_storage::mvcc::VersionError) -> Self {
954        Self::from(error.into_storage_error())
955    }
956}
957
958impl From<SQLiteError> for uqa_storage::StorageBackendError {
959    fn from(source: SQLiteError) -> Self {
960        match source {
961            SQLiteError::Memory(error) => Self::Memory(error),
962            SQLiteError::Cancelled(error) => Self::Cancelled(error),
963            SQLiteError::StorageSource(error) => *error,
964            source => Self::backend("SQLite", source),
965        }
966    }
967}
968
969impl From<SQLiteError> for uqa_storage::TransactionError {
970    fn from(source: SQLiteError) -> Self {
971        Self::Storage(source.into())
972    }
973}
974
975impl From<uqa_storage::StorageBackendError> for SQLiteError {
976    fn from(error: uqa_storage::StorageBackendError) -> Self {
977        use uqa_storage::StorageBackendError;
978        match error {
979            StorageBackendError::Memory(error) => Self::Memory(error),
980            StorageBackendError::Cancelled(error) => Self::Cancelled(error),
981            StorageBackendError::Analysis(error) => Self::Analysis(error),
982            StorageBackendError::Serde(error) => Self::Serde(error),
983            StorageBackendError::Backend { backend, source } => match source.downcast::<Self>() {
984                Ok(error) => *error,
985                Err(source) => {
986                    Self::StorageSource(Box::new(StorageBackendError::Backend { backend, source }))
987                }
988            },
989            StorageBackendError::Other(message) => Self::StorageBackend(message),
990        }
991    }
992}