Skip to main content

uqa_storage_sqlite/
connection.rs

1//
2// Unified Query Algebra
3//
4// Copyright (c) 2023-2026 Cognica, Inc.
5//
6
7//! Pooled `SQLite` connections with explicit session and transaction affinity.
8//!
9//! A [`ManagedConnection`] is a logical session. Clones share that session so
10//! catalog, document, inverted-index, and vector stores participate in the
11//! same explicit transaction. [`ManagedConnection::new_session`] creates an
12//! isolated session over the same physical connection pool. Outside explicit
13//! transactions operations check out independent connections, allowing WAL
14//! readers to make real concurrent progress.
15
16use std::path::{Path, PathBuf};
17use std::sync::{Arc, OnceLock};
18
19use parking_lot::{Mutex, RwLock};
20use rusqlite::{Connection, OpenFlags};
21use uqa_storage::{mvcc::VersionedKeyValueStore, KeyValueStore, StorageEncryptionKey};
22
23use crate::compressed_vfs::{self, SQLiteCompressedContainerAnchor, SQLiteCompressionOptions};
24
25mod identity;
26mod logical;
27mod native;
28mod native_restore;
29pub(crate) mod ownership;
30mod pool;
31mod restore;
32use pool::ConnectionPool;
33pub(crate) use pool::PooledConnection;
34mod notifications;
35mod serializable;
36mod snapshot;
37mod source;
38use snapshot::PhysicalConnection;
39pub(crate) use snapshot::{SnapshotIdentity, ValidatedRead};
40
41#[derive(Debug, thiserror::Error)]
42pub enum SQLiteError {
43    #[error(transparent)]
44    Memory(#[from] uqa_core::memory::MemoryError),
45    #[error(transparent)]
46    Cancelled(#[from] uqa_core::QueryCancelled),
47    #[error("text analysis failed: {0}")]
48    Analysis(#[from] uqa_analysis::AnalysisError),
49    #[error("sqlite error: {0}")]
50    SQLite(#[from] rusqlite::Error),
51    #[error("encryption key must not be empty")]
52    EmptyEncryptionKey,
53    #[error("database requires an encryption key")]
54    EncryptionKeyRequired,
55    #[error("database is not encrypted but an encryption key was provided")]
56    NotEncrypted,
57    #[error("auxiliary database requires DELETE journaling, found {0}")]
58    AuxiliaryJournalMode(String),
59    #[error("compressed sqlite container error: {0}")]
60    CompressedContainer(String),
61    #[error("io error: {0}")]
62    Io(#[from] std::io::Error),
63    #[error("catalog migration {version} failed: {source}")]
64    Migration {
65        version: u32,
66        #[source]
67        source: rusqlite::Error,
68    },
69    #[error("invalid persisted catalog schema version `{0}`")]
70    InvalidSchemaVersion(String),
71    #[error("catalog schema version {found} is newer than this engine supports ({supported})")]
72    UnsupportedSchemaVersion { found: u32, supported: u32 },
73    #[error("corrupt document blob for `{table}` doc {doc_id} field `{field}`: {reason}")]
74    CorruptDocumentBlob {
75        table: String,
76        doc_id: u64,
77        field: String,
78        reason: String,
79    },
80    #[error("payload serialization failed: {0}")]
81    Serde(#[from] serde_json::Error),
82    #[error("storage backend error: {0}")]
83    StorageBackend(String),
84    #[error(transparent)]
85    StorageSource(Box<uqa_storage::StorageBackendError>),
86    #[error("transaction already active for this sqlite session")]
87    TransactionAlreadyActive,
88    #[error("no active transaction for this sqlite session")]
89    NoActiveTransaction,
90    #[error("sqlite transaction was aborted by an earlier storage error: {0}")]
91    TransactionAborted(String),
92    #[error("sqlite session cleanup failed: {0}")]
93    SessionCleanupFailed(String),
94    #[error("sqlite connection-pool checkout lost its connection")]
95    MissingCheckedOutConnection,
96    #[error("versioned storage requires its logical session; use with_physical only for explicit physical maintenance")]
97    LogicalSessionRequired,
98    #[error("database restoration requires all existing owners to close")]
99    DatabaseRestoreBusy,
100    #[error("database restoration is incomplete; resume the original source/target request")]
101    DatabaseRestoreIncomplete,
102    #[error("the SQLite session already has a different retention limit")]
103    SessionOptionsMismatch,
104    #[error("the SQLite session is bound to a different record mapping")]
105    SessionMappingMismatch,
106    #[error("the retained SQLite database source changed; close its owners and reopen explicitly")]
107    DatabaseSourceChanged,
108}
109
110pub type Result<T> = std::result::Result<T, SQLiteError>;
111
112const MIN_POOL_CONNECTIONS: usize = 4;
113const MAX_POOL_CONNECTIONS: usize = 32;
114
115/// Prepared statements each connection retains. Statements are built from record layouts and work-queue conditions, so one commit runs more distinct statements than a small cache holds, and preparing a projection upsert compiles every trigger of its table; a cache that evicts them prepares them again in each transaction.
116pub(crate) const PREPARED_STATEMENT_CACHE_CAPACITY: usize = 256;
117
118#[derive(Clone)]
119enum ConnectionSpec {
120    File {
121        path: PathBuf,
122        key: Option<StorageEncryptionKey>,
123    },
124    Auxiliary {
125        path: PathBuf,
126        key: Option<StorageEncryptionKey>,
127    },
128    Compressed {
129        path: PathBuf,
130        compression: SQLiteCompressionOptions,
131        key: Option<StorageEncryptionKey>,
132    },
133    Memory,
134}
135
136impl ConnectionSpec {
137    fn open(&self, initialize_database: bool) -> Result<Connection> {
138        let conn = self.open_configured(initialize_database)?;
139        conn.set_prepared_statement_cache_capacity(PREPARED_STATEMENT_CACHE_CAPACITY);
140        Ok(conn)
141    }
142
143    fn open_configured(&self, initialize_database: bool) -> Result<Connection> {
144        let mut flags = OpenFlags::default();
145        if !initialize_database && matches!(self, Self::File { .. } | Self::Auxiliary { .. }) {
146            flags.remove(OpenFlags::SQLITE_OPEN_CREATE);
147        }
148        match self {
149            Self::File { path, key } | Self::Auxiliary { path, key } => {
150                let conn = Connection::open_with_flags(path, flags)?;
151                if let Some(key) = key {
152                    ManagedConnection::apply_encryption_key(&conn, key.expose_secret())?;
153                }
154                if matches!(self, Self::Auxiliary { .. }) {
155                    // Registry writers already serialize their transactions.
156                    // Retain the original journal mode instead of racing to
157                    // promote a new registry to WAL during concurrent opens.
158                    let mode: String =
159                        conn.pragma_query_value(None, "journal_mode", |row| row.get(0))?;
160                    if mode != "delete" {
161                        return Err(SQLiteError::AuxiliaryJournalMode(mode));
162                    }
163                    ManagedConnection::configure_rollback_connection(&conn)?;
164                } else {
165                    if initialize_database {
166                        ManagedConnection::enable_wal(&conn)?;
167                    }
168                    ManagedConnection::configure_wal_connection(&conn)?;
169                }
170                Ok(conn)
171            }
172            Self::Compressed {
173                path, compression, ..
174            } => {
175                let conn =
176                    Connection::open_with_flags_and_vfs(path, flags, compressed_vfs::VFS_NAME)?;
177                if initialize_database {
178                    conn.pragma_update(None, "page_size", compression.page_size)?;
179                    ManagedConnection::enable_compressed_journal(&conn)?;
180                }
181                ManagedConnection::configure_rollback_connection(&conn)?;
182                Ok(conn)
183            }
184            Self::Memory => {
185                let conn = Connection::open_in_memory()?;
186                if initialize_database {
187                    ManagedConnection::enable_wal(&conn)?;
188                }
189                ManagedConnection::configure_wal_connection(&conn)?;
190                Ok(conn)
191            }
192        }
193    }
194}
195
196struct SessionState {
197    write_cancellation: uqa_core::CancellationToken,
198    affinity: uqa_storage::StorageSessionAffinity,
199    /// Read guards cover ordinary operations. Transaction lifecycle calls take
200    /// the write guard, making BEGIN/COMMIT/ROLLBACK linearizable with respect
201    /// to every operation issued through the same logical session.
202    gate: RwLock<()>,
203    transaction: Mutex<Option<PooledConnection>>,
204    transaction_failure: Mutex<Option<String>>,
205    cleanup_failure: Mutex<Option<String>>,
206    logical: OnceLock<Arc<logical::BoundRecordSession>>,
207    native_restore: Mutex<Option<native_restore::NativeRestore>>,
208    snapshot_branch: Mutex<Arc<()>>,
209}
210
211impl SessionState {
212    fn new() -> Self {
213        Self::with_cancellation(uqa_core::CancellationToken::new())
214    }
215
216    fn with_cancellation(write_cancellation: uqa_core::CancellationToken) -> Self {
217        Self {
218            write_cancellation,
219            affinity: uqa_storage::StorageSessionAffinity::new(),
220            gate: RwLock::new(()),
221            transaction: Mutex::new(None),
222            transaction_failure: Mutex::new(None),
223            cleanup_failure: Mutex::new(None),
224            logical: OnceLock::new(),
225            native_restore: Mutex::new(None),
226            snapshot_branch: Mutex::new(Arc::new(())),
227        }
228    }
229}
230
231impl Drop for SessionState {
232    fn drop(&mut self) {
233        // `PooledConnection::drop` performs the rollback and discards the
234        // physical connection when rollback itself fails. Taking the pinned
235        // handle here therefore cannot return a broken connection to the pool.
236        self.native_restore.get_mut().take();
237        self.transaction.get_mut().take();
238    }
239}
240
241/// Logical `SQLite` session backed by a bounded physical connection pool.
242/// Cloning preserves session/transaction affinity; call [`Self::new_session`]
243/// for an independently isolated transaction context.
244#[derive(Clone)]
245pub struct ManagedConnection {
246    pool: Arc<ConnectionPool>,
247    session: Arc<SessionState>,
248    record_access: bool,
249}
250
251impl ManagedConnection {
252    /// Decoded document columns shared by every session of this database.
253    pub(crate) fn decoded_columns(
254        &self,
255    ) -> &crate::document_store::decoded_columns::DecodedColumns {
256        &self.pool.decoded_columns
257    }
258
259    fn surface_cleanup_failure(&self) -> Result<()> {
260        if let Some(error) = self.session.cleanup_failure.lock().take() {
261            return Err(SQLiteError::SessionCleanupFailed(error));
262        }
263        Ok(())
264    }
265
266    pub fn open(path: &Path) -> Result<Self> {
267        if path == Path::new(":memory:") {
268            return Self::open_in_memory();
269        }
270        Self::open_with_optional_key(path, None)
271    }
272
273    /// Retain the database credential for encrypted auxiliary storage, including
274    /// compressed containers whose main-file encryption lives in the VFS.
275    #[must_use]
276    pub fn auxiliary_encryption_key(&self) -> Option<StorageEncryptionKey> {
277        match &self.pool.spec {
278            ConnectionSpec::File { key, .. }
279            | ConnectionSpec::Auxiliary { key, .. }
280            | ConnectionSpec::Compressed { key, .. } => key.clone(),
281            ConnectionSpec::Memory => None,
282        }
283    }
284
285    /// Lease an independent physical connection without joining this logical
286    /// session's transaction. The pool retains encryption and connection policy.
287    pub fn lease_connection(&self) -> Result<crate::SQLiteConnectionLease> {
288        self.pool.checkout().map(crate::SQLiteConnectionLease)
289    }
290
291    /// Open database-owned auxiliary storage with its inherited credential and
292    /// original DELETE journaling. Concurrent first opens never change journal
293    /// modes. An incompatible existing mode is rejected without rewriting it.
294    pub fn open_auxiliary(path: &Path, key: Option<StorageEncryptionKey>) -> Result<Self> {
295        if path == Path::new(":memory:") || path.as_os_str().is_empty() {
296            return Err(SQLiteError::StorageBackend(
297                "auxiliary storage requires a database file".into(),
298            ));
299        }
300        Self::from_spec(
301            ConnectionSpec::Auxiliary {
302                path: path.to_path_buf(),
303                key,
304            },
305            default_pool_connections(),
306        )
307    }
308
309    pub fn open_encrypted(path: &Path, key: &str) -> Result<Self> {
310        Self::open_with_optional_key(path, Some(key))
311    }
312
313    pub fn open_compressed(path: &Path, compression: SQLiteCompressionOptions) -> Result<Self> {
314        Self::open_compressed_with_optional_key(path, compression, None, None)
315    }
316
317    pub fn open_compressed_encrypted(
318        path: &Path,
319        key: &str,
320        compression: SQLiteCompressionOptions,
321    ) -> Result<Self> {
322        if key.is_empty() {
323            return Err(SQLiteError::EmptyEncryptionKey);
324        }
325        Self::open_compressed_with_optional_key(path, compression, Some(key), None)
326    }
327
328    /// Open an encrypted compressed database while enforcing an exact trusted
329    /// external state anchor in the VFS main-file open path.
330    pub fn open_compressed_encrypted_with_anchor(
331        path: &Path,
332        key: &str,
333        compression: SQLiteCompressionOptions,
334        trusted_anchor: SQLiteCompressedContainerAnchor,
335    ) -> Result<Self> {
336        if key.is_empty() {
337            return Err(SQLiteError::EmptyEncryptionKey);
338        }
339        Self::open_compressed_with_optional_key(path, compression, Some(key), Some(trusted_anchor))
340    }
341
342    fn open_with_optional_key(path: &Path, key: Option<&str>) -> Result<Self> {
343        let spec = ConnectionSpec::File {
344            path: path.to_path_buf(),
345            key: key.map(StorageEncryptionKey::new),
346        };
347        Self::from_spec(spec, default_pool_connections())
348    }
349
350    fn open_compressed_with_optional_key(
351        path: &Path,
352        compression: SQLiteCompressionOptions,
353        key: Option<&str>,
354        trusted_anchor: Option<SQLiteCompressedContainerAnchor>,
355    ) -> Result<Self> {
356        let compression = compression
357            .validate()
358            .map_err(SQLiteError::CompressedContainer)?;
359        let owner = ownership::open(
360            path,
361            &uqa_storage::read_control::StorageReadControl::with_limit(
362                uqa_storage::mvcc::VersionedSessionOptions::default().retained_bytes,
363            ),
364        )?;
365        match trusted_anchor {
366            Some(anchor) => compressed_vfs::register_database_with_anchor(
367                path,
368                compression,
369                key.ok_or(SQLiteError::EncryptionKeyRequired)?,
370                anchor,
371            ),
372            None => compressed_vfs::register_database(path, compression, key),
373        }
374        .map_err(SQLiteError::CompressedContainer)?;
375        let spec = ConnectionSpec::Compressed {
376            path: path.to_path_buf(),
377            compression,
378            key: key.map(StorageEncryptionKey::new),
379        };
380        Self::from_spec_owned(spec, default_pool_connections(), Some(owner))
381    }
382
383    pub fn open_in_memory() -> Result<Self> {
384        // Independent `:memory:` connections do not share a database. Keep a
385        // one-connection pool for this special target; file-backed databases
386        // use the real multi-connection pool.
387        Self::from_spec(ConnectionSpec::Memory, 1)
388    }
389
390    fn from_spec(spec: ConnectionSpec, max_connections: usize) -> Result<Self> {
391        let owner = match &spec {
392            ConnectionSpec::File { path, .. }
393                if path.as_os_str().is_empty() || path == Path::new(":memory:") =>
394            {
395                None
396            }
397            ConnectionSpec::File { path, .. }
398            | ConnectionSpec::Auxiliary { path, .. }
399            | ConnectionSpec::Compressed { path, .. } => Some(ownership::open(
400                path,
401                &uqa_storage::read_control::StorageReadControl::with_limit(
402                    uqa_storage::mvcc::VersionedSessionOptions::default().retained_bytes,
403                ),
404            )?),
405            ConnectionSpec::Memory => None,
406        };
407        Self::from_spec_owned(spec, max_connections, owner)
408    }
409
410    fn from_spec_owned(
411        spec: ConnectionSpec,
412        max_connections: usize,
413        owner: Option<Arc<ownership::DatabaseOwner>>,
414    ) -> Result<Self> {
415        let initial = spec.open(true)?;
416        crate::mvcc::restore::reject_pending(&initial)?;
417        Ok(Self {
418            pool: ConnectionPool::new(spec, initial, max_connections, owner)?,
419            session: Arc::new(SessionState::new()),
420            record_access: false,
421        })
422    }
423
424    pub(crate) fn database_owner(&self) -> Option<Arc<ownership::DatabaseOwner>> {
425        self.pool.owner.clone()
426    }
427
428    fn apply_encryption_key(conn: &Connection, key: &str) -> Result<()> {
429        if key.is_empty() {
430            return Err(SQLiteError::EmptyEncryptionKey);
431        }
432        conn.pragma_update(None, "key", key)?;
433        Ok(())
434    }
435
436    fn enable_wal(conn: &Connection) -> Result<()> {
437        conn.pragma_update(None, "journal_mode", "WAL")?;
438        Ok(())
439    }
440
441    fn configure_wal_connection(conn: &Connection) -> Result<()> {
442        // Legacy physical operations retain a bounded busy wait. Versioned writes supply cancellable admission under a scoped timeout.
443        conn.busy_timeout(std::time::Duration::from_secs(5))?;
444        // Synchronous=NORMAL is the recommended pairing with WAL: safe
445        // against power loss, faster than FULL.
446        conn.pragma_update(None, "synchronous", "NORMAL")?;
447        // Foreign keys are off by default; turn on so per-table cleanup
448        // can use ON DELETE CASCADE later.
449        conn.pragma_update(None, "foreign_keys", "ON")?;
450        Ok(())
451    }
452
453    fn enable_compressed_journal(conn: &Connection) -> Result<()> {
454        // The compressed VFS implements the byte-addressed database file.
455        // Rollback journals stay raw because they are short-lived commit
456        // machinery; compressing them only adds autocommit write amplification.
457        // WAL requires shared-memory VFS methods, so compressed databases use
458        // SQLite's rollback journal and keep temp storage in memory.
459        conn.pragma_update(None, "journal_mode", "DELETE")?;
460        Ok(())
461    }
462
463    fn configure_rollback_connection(conn: &Connection) -> Result<()> {
464        conn.busy_timeout(std::time::Duration::from_secs(5))?;
465        conn.pragma_update(None, "synchronous", "FULL")?;
466        conn.pragma_update(None, "temp_store", "MEMORY")?;
467        conn.pragma_update(None, "foreign_keys", "ON")?;
468        Ok(())
469    }
470
471    /// Identity shared by handles using this session's native or logical transaction context.
472    pub fn transaction_affinity(&self) -> uqa_storage::StorageSessionAffinity {
473        let _gate = self.session.gate.read();
474        self.session.logical.get().map_or_else(
475            || self.session.affinity.clone(),
476            |logical| logical.session_affinity(),
477        )
478    }
479
480    /// Whether this session has an active native or logical record transaction.
481    #[must_use]
482    pub fn in_transaction(&self) -> bool {
483        let _gate = self.session.gate.read();
484        if let Some(logical) = self.session.logical.get() {
485            return logical.in_transaction();
486        }
487        self.session.transaction.lock().is_some()
488    }
489
490    /// Whether this session has staged writes, including private logical records. Callers use this to enforce read-only execution boundaries before COMMIT.
491    pub fn transaction_has_written(&self) -> Result<bool> {
492        let _gate = self.session.gate.read();
493        if let Some(logical) = self.session.logical.get() {
494            return logical.transaction_has_written().map_err(Into::into);
495        }
496        let transaction = self.session.transaction.lock();
497        let transaction = transaction
498            .as_ref()
499            .ok_or(SQLiteError::NoActiveTransaction)?;
500        Ok(matches!(
501            transaction.connection()?.transaction_state(Some("main"))?,
502            rusqlite::TransactionState::Write
503        ))
504    }
505
506    /// Whether this session's independent [`Self::data_version`] monitor can
507    /// read without contending with the currently pinned transaction.
508    ///
509    /// A rollback-journal writer's pending lock blocks new readers while waiting for existing readers to finish. A rollback-journal read transaction must therefore also avoid the independent monitor: its own shared lock may be preventing that waiting writer from proceeding. Callers refresh through the pinned connection instead. WAL sessions and sessions without a pinned transaction permit the independent monitor.
510    pub fn data_version_monitor_is_nonblocking(&self) -> Result<bool> {
511        let _gate = self.session.gate.read();
512        if self.session.logical.get().is_some() {
513            return Ok(true);
514        }
515        if !matches!(
516            &self.pool.spec,
517            ConnectionSpec::Compressed { .. } | ConnectionSpec::Auxiliary { .. }
518        ) {
519            return Ok(true);
520        }
521        Ok(self.session.transaction.lock().is_none())
522    }
523
524    /// Whether one pooled connection may retain a read snapshot while another writes. Ordinary plain and encrypted `SQLite` databases use WAL; compressed containers and auxiliary files use rollback journaling and therefore require a detached engine snapshot before writer promotion.
525    #[must_use]
526    pub fn supports_concurrent_pinned_read_and_write(&self) -> bool {
527        let _gate = self.session.gate.read();
528        self.session.logical.get().is_some()
529            || !matches!(
530                &self.pool.spec,
531                ConnectionSpec::Compressed { .. } | ConnectionSpec::Auxiliary { .. }
532            )
533    }
534
535    /// Database change counter observed on one stable connection shared by
536    /// every logical session over this pool. The value changes when another
537    /// `SQLite` connection commits. In-memory databases have no independent
538    /// connections and therefore return `None`.
539    pub fn data_version(&self) -> Result<Option<u64>> {
540        let _gate = self.session.gate.read();
541        self.pool.check_source()?;
542        if let Some(logical) = self.session.logical.get() {
543            return logical.change_version().map_err(Into::into);
544        }
545        if matches!(&self.pool.spec, ConnectionSpec::Memory) {
546            return Ok(None);
547        }
548        let mut monitor = self.pool.data_version_monitor.lock();
549        if monitor.is_none() {
550            *monitor = Some(self.pool.open_connection()?);
551        }
552        let monitor = monitor.as_ref().ok_or_else(|| {
553            SQLiteError::StorageBackend(
554                "data-version monitor was not initialized after opening it".into(),
555            )
556        })?;
557        let version: i64 = monitor.pragma_query_value(None, "data_version", |row| row.get(0))?;
558        let version = u64::try_from(version).map_err(|_| {
559            SQLiteError::StorageBackend(format!(
560                "SQLite returned a negative PRAGMA data_version: {version}"
561            ))
562        })?;
563        Ok(Some(version))
564    }
565
566    /// `SQLite`'s change counter on the pool's monitor connection, which differs from an earlier read exactly when another connection of any session or process committed in between. `None` when the monitor may not read at any time: an in-memory database has no independent connection, and a rollback-journal reader waits for a pending writer, which may itself be waiting for a read this session has open.
567    pub fn commit_monitor_version(&self) -> Result<Option<u64>> {
568        if matches!(
569            &self.pool.spec,
570            ConnectionSpec::Memory
571                | ConnectionSpec::Compressed { .. }
572                | ConnectionSpec::Auxiliary { .. }
573        ) {
574            return Ok(None);
575        }
576        self.pool.check_source()?;
577        let mut monitor = self.pool.data_version_monitor.lock();
578        if monitor.is_none() {
579            *monitor = Some(self.pool.open_connection()?);
580        }
581        let monitor = monitor.as_ref().ok_or_else(|| {
582            SQLiteError::StorageBackend(
583                "data-version monitor was not initialized after opening it".into(),
584            )
585        })?;
586        let version: i64 = monitor
587            .prepare_cached("PRAGMA data_version")?
588            .query_row([], |row| row.get(0))?;
589        u64::try_from(version).map(Some).map_err(|_| {
590            SQLiteError::StorageBackend(format!(
591                "SQLite returned a negative PRAGMA data_version: {version}"
592            ))
593        })
594    }
595
596    /// Complete logical record visibility, including the private root restored by savepoint undo. Legacy physical sessions retain their existing data-version refresh protocol.
597    pub fn read_view_revision(
598        &self,
599    ) -> Result<Option<uqa_storage::key_value::KeyValueReadRevision>> {
600        self.surface_cleanup_failure()?;
601        let _gate = self.session.gate.read();
602        self.session.logical.get().map_or(Ok(None), |logical| {
603            logical.read_view_revision().map_err(Into::into)
604        })
605    }
606
607    /// Establish the database snapshot for the active transaction without
608    /// depending on the caller's first user query. `BEGIN DEFERRED` alone does
609    /// not start a read transaction, so a writer could otherwise commit after
610    /// the engine checks its cache generations but before the first catalog
611    /// read. Reading `sqlite_schema` is database-wide and keeps the operation
612    /// independent of any application table.
613    pub fn pin_transaction_snapshot(&self) -> Result<()> {
614        self.surface_cleanup_failure()?;
615        let _gate = self.session.gate.read();
616        if let Some(logical) = self.session.logical.get() {
617            return if logical.in_transaction() {
618                Ok(())
619            } else {
620                Err(SQLiteError::NoActiveTransaction)
621            };
622        }
623        self.with_native(|connection| {
624            if connection.is_autocommit() {
625                return Err(SQLiteError::NoActiveTransaction);
626            }
627            let _: i64 =
628                connection.query_row("SELECT COUNT(*) FROM sqlite_schema", [], |row| row.get(0))?;
629            Ok(())
630        })
631    }
632
633    /// Run a closure using this session. Outside a transaction the closure
634    /// checks out a pooled connection; inside a transaction every clone is
635    /// routed to the session's pinned connection.
636    pub fn with<R>(&self, f: impl FnOnce(&Connection) -> Result<R>) -> Result<R> {
637        self.surface_cleanup_failure()?;
638        let _gate = self.session.gate.read();
639        if self.session.logical.get().is_some() {
640            return Err(SQLiteError::LogicalSessionRequired);
641        }
642        self.with_native(f)
643    }
644
645    /// The caller holds the session gate and has ruled out logical record access.
646    fn with_native<R>(&self, f: impl FnOnce(&Connection) -> Result<R>) -> Result<R> {
647        let transaction = self.session.transaction.lock();
648        if let Some(connection) = transaction.as_ref() {
649            if let Some(error) = self.session.transaction_failure.lock().as_ref() {
650                return Err(SQLiteError::TransactionAborted(error.clone()));
651            }
652            self.check_native_access(connection.connection()?)?;
653            let result = f(connection.connection()?);
654            if let Err(error) = &result {
655                let mut failure = self.session.transaction_failure.lock();
656                if failure.is_none() {
657                    *failure = Some(error.to_string());
658                }
659            }
660            return result;
661        }
662        drop(transaction);
663        let connection = self.pool.checkout()?;
664        self.check_native_access(connection.connection()?)?;
665        f(connection.connection()?)
666    }
667
668    pub fn with_mut<R>(&self, f: impl FnOnce(&mut Connection) -> Result<R>) -> Result<R> {
669        self.with_physical_mut(|physical| f(&mut physical.connection))
670    }
671
672    fn with_physical_mut<R>(
673        &self,
674        f: impl FnOnce(&mut PhysicalConnection) -> Result<R>,
675    ) -> Result<R> {
676        self.surface_cleanup_failure()?;
677        let _gate = self.session.gate.read();
678        if self.session.logical.get().is_some() {
679            return Err(SQLiteError::LogicalSessionRequired);
680        }
681        let mut transaction = self.session.transaction.lock();
682        if let Some(connection) = transaction.as_mut() {
683            if let Some(error) = self.session.transaction_failure.lock().as_ref() {
684                return Err(SQLiteError::TransactionAborted(error.clone()));
685            }
686            self.check_native_access(connection.connection()?)?;
687            let result = f(connection.physical_mut()?);
688            if let Err(error) = &result {
689                let mut failure = self.session.transaction_failure.lock();
690                if failure.is_none() {
691                    *failure = Some(error.to_string());
692                }
693            }
694            return result;
695        }
696        drop(transaction);
697        let mut connection = self.pool.checkout()?;
698        self.check_native_access(connection.connection()?)?;
699        f(connection.physical_mut()?)
700    }
701
702    /// Rewrite the `SQLite` database into its minimum-sized file. `SQLite` requires `VACUUM` to run in autocommit mode, so the session write gate makes the transaction check and maintenance command one atomic session operation.
703    pub fn vacuum(&self) -> Result<()> {
704        self.maintain_storage(true)
705    }
706
707    /// Reclaim obsolete logical records and history without executing a physical `SQLite` `VACUUM`.
708    pub fn reclaim_obsolete(&self) -> Result<()> {
709        self.maintain_storage(false)
710    }
711
712    fn maintain_storage(&self, compact: bool) -> Result<()> {
713        self.surface_cleanup_failure()?;
714        let _gate = self.session.gate.write();
715        if self.session.transaction.lock().is_some()
716            || self
717                .session
718                .logical
719                .get()
720                .is_some_and(|logical| logical.in_transaction())
721        {
722            return Err(SQLiteError::TransactionAlreadyActive);
723        }
724        if let Some(logical) = self.session.logical.get() {
725            let source: Arc<dyn KeyValueStore> = if let Some(namespace) = logical.native {
726                Arc::new(crate::diskann::Records::new(
727                    logical.store.clone(),
728                    namespace,
729                    self.auxiliary_encryption_key(),
730                )?)
731            } else {
732                logical.store.clone()
733            };
734            let retained = logical.retention_control();
735            let control = uqa_storage::read_control::StorageReadControl::new(
736                retained.memory(),
737                &self.write_cancellation(),
738            );
739            uqa_storage::key_value::KeyValueDiskANNMaintenance::run(&source, &control)?;
740            logical.reclaim_versions()?;
741        }
742        if compact {
743            let connection = self.pool.checkout()?;
744            connection.connection()?.execute_batch("VACUUM")?;
745        }
746        Ok(())
747    }
748
749    /// Open an explicit (non-deferred) transaction. Subsequent
750    /// auto-commit hosts (catalog writes, FTS index updates, ...) all
751    /// flow through the same connection so the transaction enclosing
752    /// them is honoured. Use [`Self::commit_transaction`] /
753    /// [`Self::rollback_transaction`] / [`Self::savepoint`] etc. for
754    /// the lifecycle.
755    pub fn begin_transaction(&self) -> Result<()> {
756        self.begin_transaction_with("BEGIN IMMEDIATE")
757    }
758
759    /// Open a deferred transaction. Read-only SQL statements use this mode
760    /// so WAL readers do not take the single writer reservation; if a scalar
761    /// routine performs a write, `SQLite` upgrades the same transaction and
762    /// still preserves the statement's atomic boundary.
763    pub fn begin_deferred_transaction(&self) -> Result<()> {
764        self.begin_transaction_with("BEGIN DEFERRED")
765    }
766
767    fn begin_transaction_with(&self, statement: &str) -> Result<()> {
768        self.surface_cleanup_failure()?;
769        let _gate = self.session.gate.write();
770        if let Some(logical) = self.session.logical.get() {
771            if logical.in_transaction() {
772                return Err(SQLiteError::TransactionAlreadyActive);
773            }
774            return if statement == "BEGIN DEFERRED" {
775                logical.begin_upgradeable_transaction()
776            } else {
777                logical.begin_transaction()
778            }
779            .map_err(Into::into);
780        }
781        let mut transaction = self.session.transaction.lock();
782        if transaction.is_some() {
783            return Err(SQLiteError::TransactionAlreadyActive);
784        }
785        let connection = self.pool.checkout()?;
786        self.check_native_access(connection.connection()?)?;
787        connection.connection()?.execute_batch(statement)?;
788        self.session.transaction_failure.lock().take();
789        *transaction = Some(connection);
790        Ok(())
791    }
792
793    pub fn commit_transaction(&self) -> Result<()> {
794        self.surface_cleanup_failure()?;
795        self.finish_transaction("COMMIT")
796    }
797
798    pub fn rollback_transaction(&self) -> Result<()> {
799        self.surface_cleanup_failure()?;
800        self.finish_transaction("ROLLBACK")
801    }
802
803    /// Drop-only transaction cleanup. A rollback error is recorded on the
804    /// logical session and is returned by its next operation instead of being
805    /// mistaken for a successful rollback.
806    pub(crate) fn rollback_transaction_on_drop(&self) {
807        if let Err(error) = self.finish_transaction("ROLLBACK") {
808            let mut failure = self.session.cleanup_failure.lock();
809            if failure.is_none() {
810                *failure = Some(error.to_string());
811            }
812        }
813    }
814
815    fn finish_transaction(&self, statement: &str) -> Result<()> {
816        let _gate = self.session.gate.write();
817        if let Some(logical) = self.session.logical.get() {
818            if !logical.in_transaction() {
819                return Err(SQLiteError::NoActiveTransaction);
820            }
821            return if statement == "COMMIT" {
822                logical.commit_transaction()
823            } else {
824                logical.rollback_transaction()
825            }
826            .map_err(Into::into);
827        }
828        let mut transaction = self.session.transaction.lock();
829        let connection = transaction
830            .as_ref()
831            .ok_or(SQLiteError::NoActiveTransaction)?;
832        *self.session.snapshot_branch.lock() = Arc::new(());
833        if statement == "COMMIT" {
834            // Materialize the failure before entering the branch. Holding the
835            // mutex guard created by an `if let` scrutinee until the end of
836            // the branch would deadlock when cleanup takes the same lock.
837            let transaction_failure = self.session.transaction_failure.lock().clone();
838            if let Some(error) = transaction_failure {
839                if let Err(rollback_error) = connection.connection()?.execute_batch("ROLLBACK") {
840                    self.session.native_restore.lock().take();
841                    transaction.take();
842                    self.session.transaction_failure.lock().take();
843                    return Err(SQLiteError::SQLite(rollback_error));
844                }
845                self.session.native_restore.lock().take();
846                transaction.take();
847                self.session.transaction_failure.lock().take();
848                return Err(SQLiteError::TransactionAborted(error));
849            }
850        }
851        let native = if statement == "COMMIT" && self.session.native_restore.lock().is_some() {
852            match self.prepare_initial_native_binding(connection.connection()?) {
853                Ok(native) => Some(native),
854                Err(error) => {
855                    *self.session.transaction_failure.lock() = Some(error.to_string());
856                    return Err(error);
857                }
858            }
859        } else {
860            None
861        };
862        if let Err(error) = connection.connection()?.execute_batch(statement) {
863            self.session.native_restore.lock().take();
864            transaction.take();
865            self.session.transaction_failure.lock().take();
866            return Err(SQLiteError::SQLite(error));
867        }
868        self.session.native_restore.lock().take();
869        transaction.take();
870        self.session.transaction_failure.lock().take();
871        if let Some(native) = native {
872            self.install_initial_native_binding(native);
873        }
874        Ok(())
875    }
876
877    fn with_transaction<R>(
878        &self,
879        logical: impl FnOnce(&VersionedKeyValueStore) -> uqa_storage::StorageBackendResult<R>,
880        native: impl FnOnce(&Connection) -> Result<R>,
881    ) -> Result<R> {
882        self.surface_cleanup_failure()?;
883        let _gate = self.session.gate.read();
884        if let Some(store) = self.session.logical.get() {
885            if !store.in_transaction() {
886                return Err(SQLiteError::NoActiveTransaction);
887            }
888            return logical(store).map_err(Into::into);
889        }
890        let transaction = self.session.transaction.lock();
891        let connection = transaction
892            .as_ref()
893            .ok_or(SQLiteError::NoActiveTransaction)?;
894        self.check_native_access(connection.connection()?)?;
895        native(connection.connection()?)
896    }
897
898    pub fn savepoint(&self, name: &str) -> Result<()> {
899        self.with_transaction(
900            |logical| logical.savepoint(name),
901            |connection| {
902                let stmt = format!("SAVEPOINT \"{}\"", name.replace('"', "\"\""));
903                connection.execute_batch(&stmt)?;
904                Ok(())
905            },
906        )
907    }
908
909    pub fn release_savepoint(&self, name: &str) -> Result<()> {
910        self.with_transaction(
911            |logical| logical.release_savepoint(name),
912            |connection| {
913                let stmt = format!("RELEASE SAVEPOINT \"{}\"", name.replace('"', "\"\""));
914                connection.execute_batch(&stmt)?;
915                Ok(())
916            },
917        )
918    }
919
920    pub fn rollback_to_savepoint(&self, name: &str) -> Result<()> {
921        self.with_transaction(
922            |logical| logical.rollback_to_savepoint(name),
923            |connection| {
924                let stmt = format!("ROLLBACK TO SAVEPOINT \"{}\"", name.replace('"', "\"\""));
925                connection.execute_batch(&stmt)?;
926                self.session.transaction_failure.lock().take();
927                *self.session.snapshot_branch.lock() = Arc::new(());
928                Ok(())
929            },
930        )
931    }
932}
933
934fn default_pool_connections() -> usize {
935    std::thread::available_parallelism()
936        .map_or(MIN_POOL_CONNECTIONS, |parallelism| parallelism.get() * 2)
937        .clamp(MIN_POOL_CONNECTIONS, MAX_POOL_CONNECTIONS)
938}
939
940#[cfg(test)]
941mod tests;
942
943impl From<uqa_storage::mvcc::VersionError> for SQLiteError {
944    fn from(error: uqa_storage::mvcc::VersionError) -> Self {
945        Self::from(error.into_storage_error())
946    }
947}
948
949impl From<SQLiteError> for uqa_storage::StorageBackendError {
950    fn from(source: SQLiteError) -> Self {
951        match source {
952            SQLiteError::Memory(error) => Self::Memory(error),
953            SQLiteError::Cancelled(error) => Self::Cancelled(error),
954            SQLiteError::StorageSource(error) => *error,
955            source => Self::backend("SQLite", source),
956        }
957    }
958}
959
960impl From<SQLiteError> for uqa_storage::TransactionError {
961    fn from(source: SQLiteError) -> Self {
962        Self::Storage(source.into())
963    }
964}
965
966impl From<uqa_storage::StorageBackendError> for SQLiteError {
967    fn from(error: uqa_storage::StorageBackendError) -> Self {
968        use uqa_storage::StorageBackendError;
969        match error {
970            StorageBackendError::Memory(error) => Self::Memory(error),
971            StorageBackendError::Cancelled(error) => Self::Cancelled(error),
972            StorageBackendError::Analysis(error) => Self::Analysis(error),
973            StorageBackendError::Serde(error) => Self::Serde(error),
974            StorageBackendError::Backend { backend, source } => match source.downcast::<Self>() {
975                Ok(error) => *error,
976                Err(source) => {
977                    Self::StorageSource(Box::new(StorageBackendError::Backend { backend, source }))
978                }
979            },
980            StorageBackendError::Other(message) => Self::StorageBackend(message),
981        }
982    }
983}