Skip to main content

uqa_storage_sqlite/mvcc/
mod.rs

1//
2// Unified Query Algebra
3//
4// Copyright (c) 2023-2026 Cognica, Inc.
5//
6
7//! `SQLite` persistence for shared logical records, with short physical transactions and durable receipts.
8
9mod admission;
10mod codec;
11mod identifiers;
12mod key_value;
13#[cfg(any(windows, all(unix, not(target_os = "emscripten"))))]
14pub(crate) mod leases;
15pub mod native;
16mod read;
17pub(crate) mod receipts;
18mod reclamation;
19pub(crate) mod restore;
20mod retention;
21mod runs;
22mod schema;
23mod serializable;
24#[cfg(test)]
25mod tests;
26mod write;
27
28pub(crate) use schema::WritePermit;
29pub use serializable::SQLiteSerializableAdmission;
30
31use std::sync::Arc;
32
33use rusqlite::Connection;
34use uqa_storage::mvcc::{
35    CommitResult, CommitStatus, CommittedRecordSnapshot, DatabaseId, PreparedRecordCommit,
36    StorageTransactionId, VersionError, VersionResult, VersionedPersistence,
37};
38use uqa_storage::read_control::StorageReadControl;
39
40use crate::{ManagedConnection, SQLiteError};
41
42#[derive(Debug, thiserror::Error)]
43enum Error {
44    #[error(transparent)]
45    SQLite(#[from] rusqlite::Error),
46    #[error(transparent)]
47    Version(#[from] VersionError),
48}
49
50impl Error {
51    fn into_version(self) -> VersionError {
52        match self {
53            Self::SQLite(error) => sqlite_error(error),
54            Self::Version(error) => error,
55        }
56    }
57}
58
59impl From<SQLiteError> for Error {
60    fn from(error: SQLiteError) -> Self {
61        match error {
62            SQLiteError::Memory(error) => Self::Version(VersionError::Memory(error)),
63            SQLiteError::Cancelled(error) => Self::Version(VersionError::Cancelled(error)),
64            SQLiteError::SQLite(error) => Self::SQLite(error),
65            error => Self::Version(VersionError::Storage(error.into())),
66        }
67    }
68}
69
70type PhysicalResult<T> = std::result::Result<T, Error>;
71
72fn sqlite_error(error: rusqlite::Error) -> VersionError {
73    VersionError::Storage(SQLiteError::SQLite(error).into())
74}
75
76/// Record persistence over a managed `SQLite` pool, including `SQLCipher` and compressed connections. Snapshots retain logical sequences; this adapter does not retain physical transactions between operations.
77///
78/// Logical snapshot leases protect predecessor histories across bounded reads and native processes; reclamation preserves head tombstones and all commit receipts. The Key/Value provider uses these records directly. `Self::for_native` converts a native catalog and atomically maintains its current rows with their history; `ManagedConnection::bind_native_records` binds catalog and data handles to the same logical session.
79#[derive(Clone)]
80pub struct SQLiteRecordStore {
81    connection: ManagedConnection,
82    identity: DatabaseId,
83    native: Option<native::NativeRecordNamespace>,
84    snapshots: Arc<uqa_storage::mvcc::SnapshotRegistry>,
85}
86
87impl SQLiteRecordStore {
88    pub(crate) fn has_native_mapping(connection: &Connection) -> VersionResult<bool> {
89        native::present(connection).map_err(Error::into_version)
90    }
91
92    /// Prepare a native baseline on the initial restore's physical transaction. Its owner publishes both the restored catalog and this baseline with one COMMIT.
93    pub(crate) fn initialize_native_in(
94        connection: &ManagedConnection,
95        transaction: &Connection,
96        control: &StorageReadControl,
97    ) -> VersionResult<Self> {
98        let native::NativeMapping {
99            identity,
100            namespace,
101        } = native::initialize_in(transaction, control).map_err(Error::into_version)?;
102        Ok(Self {
103            snapshots: retention::registry(connection, identity)?,
104            connection: connection.record_connection(),
105            identity,
106            native: Some(namespace),
107        })
108    }
109
110    pub fn new(connection: &ManagedConnection) -> VersionResult<Self> {
111        if connection.in_transaction() {
112            return Err(VersionError::Storage(
113                SQLiteError::TransactionAlreadyActive.into(),
114            ));
115        }
116        let connection = connection.record_connection();
117        let identity = connection
118            .with(|connection| Ok(schema::initialize(connection)))
119            .map_err(|error| VersionError::Storage(error.into()))?
120            .map_err(Error::into_version)?;
121        Ok(Self {
122            snapshots: retention::registry(&connection, identity)?,
123            connection,
124            identity,
125            native: None,
126        })
127    }
128
129    /// Atomically initialize or upgrade a native catalog and import its baseline, or reopen its versioned materialization. Unbound native store handles are disabled after conversion; callers must bind a routed native session or prepare native records through the shared transaction contract.
130    pub fn for_native(
131        connection: &ManagedConnection,
132        control: &StorageReadControl,
133    ) -> VersionResult<Self> {
134        if connection.in_transaction() {
135            return Err(VersionError::Storage(
136                SQLiteError::TransactionAlreadyActive.into(),
137            ));
138        }
139        let connection = connection.record_connection();
140        let native::NativeMapping {
141            identity,
142            namespace,
143        } = connection
144            .with(|connection| Ok(native::initialize(connection, control)))
145            .map_err(|error| VersionError::Storage(error.into()))?
146            .map_err(Error::into_version)?;
147        Ok(Self {
148            snapshots: retention::registry(&connection, identity)?,
149            connection,
150            identity,
151            native: Some(namespace),
152        })
153    }
154
155    pub(crate) fn for_key_value(
156        connection: &ManagedConnection,
157        control: &StorageReadControl,
158    ) -> VersionResult<Self> {
159        let connection = connection.record_connection();
160        let identity = connection
161            .with(|connection| Ok(key_value::initialize(connection, control)))
162            .map_err(|error| VersionError::Storage(error.into()))?
163            .map_err(Error::into_version)?;
164        Ok(Self {
165            snapshots: retention::registry(&connection, identity)?,
166            connection,
167            identity,
168            native: None,
169        })
170    }
171
172    /// Immutable namespace for database-owned native record keys. Use it with `NativeRecordOwner::Database`; `VersionedPersistence::database_id` identifies the transaction history and may differ after restoration. Key/Value and raw record stores return `None`.
173    #[must_use]
174    pub fn native_namespace(&self) -> Option<DatabaseId> {
175        self.native.map(|namespace| namespace.0)
176    }
177
178    fn with<T>(
179        &self,
180        operation: impl FnOnce(&Connection) -> PhysicalResult<T>,
181    ) -> VersionResult<T> {
182        self.connection
183            .with(|connection| Ok(operation(connection)))
184            .map_err(|error| VersionError::Storage(error.into()))?
185            .map_err(Error::into_version)
186    }
187
188    fn with_write<T>(
189        &self,
190        control: &StorageReadControl,
191        operation: impl FnOnce(&Connection) -> PhysicalResult<T>,
192    ) -> VersionResult<T> {
193        let mut operation = Some(operation);
194        loop {
195            control.cancellation().check()?;
196            let result = self
197                .connection
198                .with_record_connection(control, |connection| {
199                    let _timeout = admission::BusyTimeout::new(connection)?;
200                    control.cancellation().check()?;
201                    Ok(operation.take().expect("record write runs once")(
202                        connection,
203                    ))
204                });
205            match result {
206                Err(SQLiteError::SQLite(error))
207                    if operation.is_some() && admission::is_busy(&error) =>
208                {
209                    // Checkout/configuration failed before the operation was invoked.
210                    admission::wait(control).map_err(Error::into_version)?;
211                }
212                result => {
213                    return result
214                        .map_err(|error| Error::from(error).into_version())?
215                        .map_err(Error::into_version);
216                }
217            }
218        }
219    }
220
221    fn check_transaction(&self, transaction: StorageTransactionId) -> VersionResult<()> {
222        if transaction.database() != self.identity {
223            return Err(VersionError::WrongDatabase);
224        }
225        Ok(())
226    }
227}
228
229impl VersionedPersistence for SQLiteRecordStore {
230    fn serializable_coordinator(&self) -> Option<&dyn uqa_storage::mvcc::SerializableCoordinator> {
231        Some(self)
232    }
233
234    fn identifier_watermark(
235        &self,
236        namespace: &[u8],
237        control: &StorageReadControl,
238    ) -> VersionResult<Option<u64>> {
239        control.cancellation().check()?;
240        self.with(|connection| {
241            identifiers::read(connection, self.identity, self.native, namespace, control)
242        })
243    }
244
245    fn allocate_identifiers(
246        &self,
247        namespace: &[u8],
248        request: uqa_storage::mvcc::IdentifierRequest,
249        control: &StorageReadControl,
250    ) -> VersionResult<uqa_storage::mvcc::IdentifierAllocation> {
251        control.cancellation().check()?;
252        self.with_write(control, |connection| {
253            identifiers::allocate(
254                connection,
255                self.identity,
256                self.native,
257                namespace,
258                request,
259                control,
260            )
261        })
262    }
263
264    fn database_id(&self) -> DatabaseId {
265        self.identity
266    }
267    fn graph_record_layout(&self) -> Option<&dyn uqa_storage::mvcc::GraphRecordLayout> {
268        match self.native.as_ref() {
269            Some(namespace) => Some(namespace),
270            None => Some(&uqa_storage::key_value::KeyValueGraphRecords),
271        }
272    }
273    fn occurrence_record_layout(&self) -> &dyn uqa_storage::mvcc::OccurrenceRecordLayout {
274        if self.native.is_some() {
275            &crate::inverted_index::NativeOccurrenceRecords
276        } else {
277            &uqa_storage::key_value::KeyValueOccurrenceRecords
278        }
279    }
280    fn notification_record_layout(&self) -> &dyn uqa_storage::mvcc::NotificationRecordLayout {
281        match self.native.as_ref() {
282            Some(namespace) => namespace,
283            None => &uqa_storage::key_value::KeyValueNotificationRecords,
284        }
285    }
286
287    fn maintenance_record_layout(&self) -> &dyn uqa_storage::mvcc::MaintenanceRecordLayout {
288        if self.native.is_some() {
289            &native::NativeMaintenanceRecords
290        } else {
291            &uqa_storage::key_value::KeyValueMaintenanceRecords
292        }
293    }
294    fn ivf_record_layout(&self) -> &dyn uqa_storage::mvcc::IVFRecordLayout {
295        if self.native.is_some() {
296            &crate::vector_index::NativeIVFRecords
297        } else {
298            &uqa_storage::key_value::KeyValueIVFRecords
299        }
300    }
301    fn hnsw_record_layout(&self) -> Option<&dyn uqa_storage::mvcc::HNSWRecordLayout> {
302        if self.native.is_some() {
303            Some(&crate::vector_index::NativeHNSWRecords)
304        } else {
305            Some(&uqa_storage::key_value::KeyValueHNSWRecords)
306        }
307    }
308    fn allocate_transaction(
309        &self,
310        control: &StorageReadControl,
311    ) -> VersionResult<StorageTransactionId> {
312        control.cancellation().check()?;
313        self.with_write(control, |connection| {
314            write::allocate(connection, self.identity, self.native, control)
315        })
316    }
317
318    fn allocate_managed_transaction(
319        &self,
320        control: &StorageReadControl,
321    ) -> VersionResult<uqa_storage::mvcc::RetainedTransactionAllocation> {
322        self.allocate_receipt_owner(control)
323    }
324    fn snapshot(
325        &self,
326        control: &StorageReadControl,
327    ) -> VersionResult<Arc<dyn CommittedRecordSnapshot>> {
328        control.cancellation().check()?;
329        let lease = self.snapshots.capture(control, || {
330            self.with(|connection| {
331                let read = connection.unchecked_transaction()?;
332                native::check_mapping(&read, self.native)?;
333                let sequence = codec::header(&read, self.identity)?.sequence;
334                read.commit()?;
335                Ok(sequence)
336            })
337        })?;
338        uqa_storage::mvcc::retain_record_snapshot(
339            read::Snapshot {
340                store: self.clone(),
341                sequence: lease.sequence(),
342                _lease: lease,
343            },
344            control,
345        )
346    }
347    fn reclaim_versions(&self, control: &StorageReadControl) -> VersionResult<u64> {
348        self.snapshots.reclaim(control, |oldest| {
349            self.with_write(control, |connection| {
350                reclamation::reclaim(connection, self.identity, self.native, oldest, control)
351            })
352        })
353    }
354
355    fn commit(
356        &self,
357        transaction: StorageTransactionId,
358        prepared: &PreparedRecordCommit,
359        control: &StorageReadControl,
360    ) -> CommitResult {
361        self.check_transaction(transaction)?;
362        control.cancellation().check().map_err(VersionError::from)?;
363        let _bindings = write::reserve_bindings(prepared, control)?;
364        self.with_write(control, |connection| {
365            Ok(write::commit(
366                connection,
367                transaction,
368                prepared,
369                self.native,
370                control,
371            ))
372        })?
373    }
374    fn commit_status(
375        &self,
376        transaction: StorageTransactionId,
377        control: &StorageReadControl,
378    ) -> VersionResult<CommitStatus> {
379        self.check_transaction(transaction)?;
380        control.cancellation().check()?;
381        self.with(|connection| {
382            let read = connection.unchecked_transaction()?;
383            native::check_mapping(&read, self.native)?;
384            codec::header(&read, self.identity)?;
385            let status = codec::status(&read, transaction)?;
386            control.cancellation().check().map_err(VersionError::from)?;
387            read.commit()?;
388            Ok(status)
389        })
390    }
391
392    fn acknowledge_transaction(
393        &self,
394        acknowledgement: uqa_storage::mvcc::ReceiptAcknowledgement,
395        control: &StorageReadControl,
396    ) -> VersionResult<()> {
397        self.check_transaction(acknowledgement.transaction())?;
398        self.with_write(control, |connection| {
399            receipts::acknowledge(connection, self.native, acknowledgement, control)
400        })
401    }
402
403    fn reclaim_transaction_receipts(&self, control: &StorageReadControl) -> VersionResult<u64> {
404        self.reclaim_receipts(control)
405    }
406    fn abort(
407        &self,
408        transaction: StorageTransactionId,
409        control: &StorageReadControl,
410    ) -> VersionResult<CommitStatus> {
411        self.check_transaction(transaction)?;
412        control.cancellation().check()?;
413        self.with(|connection| write::abort(connection, transaction, self.native, control))
414    }
415}