Skip to main content

uqa_storage_sqlite/mvcc/
mod.rs

1//
2// Unified Query Algebra
3//
4// Copyright (c) 2023-2026 Cognica, Inc.
5//
6
7//! `SQLite` persistence for shared logical records, with short physical transactions and durable receipts.
8
9mod admission;
10mod codec;
11mod identifiers;
12mod key_value;
13#[cfg(any(windows, all(unix, not(target_os = "emscripten"))))]
14pub(crate) mod leases;
15pub mod native;
16mod read;
17pub(crate) mod receipts;
18mod reclamation;
19mod resources;
20pub(crate) mod restore;
21mod retention;
22mod runs;
23mod schema;
24mod serializable;
25#[cfg(test)]
26mod tests;
27mod tombstones;
28mod write;
29
30pub(crate) use schema::WritePermit;
31pub use serializable::SQLiteSerializableAdmission;
32
33use std::sync::Arc;
34
35use rusqlite::Connection;
36use uqa_storage::mvcc::{
37    CommitResult, CommitStatus, CommittedRecordSnapshot, DatabaseId, PreparedRecordCommit,
38    StorageTransactionId, VersionError, VersionResult, VersionedPersistence,
39};
40use uqa_storage::read_control::StorageReadControl;
41
42use crate::{ManagedConnection, SQLiteError};
43
44#[derive(Debug, thiserror::Error)]
45enum Error {
46    #[error(transparent)]
47    SQLite(#[from] rusqlite::Error),
48    #[error(transparent)]
49    Version(#[from] VersionError),
50}
51
52impl Error {
53    fn into_version(self) -> VersionError {
54        match self {
55            Self::SQLite(error) => sqlite_error(error),
56            Self::Version(error) => error,
57        }
58    }
59}
60
61impl From<SQLiteError> for Error {
62    fn from(error: SQLiteError) -> Self {
63        match error {
64            SQLiteError::Memory(error) => Self::Version(VersionError::Memory(error)),
65            SQLiteError::Cancelled(error) => Self::Version(VersionError::Cancelled(error)),
66            SQLiteError::SQLite(error) => Self::SQLite(error),
67            error => Self::Version(VersionError::Storage(error.into())),
68        }
69    }
70}
71
72type PhysicalResult<T> = std::result::Result<T, Error>;
73
74fn sqlite_error(error: rusqlite::Error) -> VersionError {
75    VersionError::Storage(SQLiteError::SQLite(error).into())
76}
77
78/// Record persistence over a managed `SQLite` pool, including `SQLCipher` and compressed connections. Snapshots retain logical sequences; this adapter does not retain physical transactions between operations.
79///
80/// Logical snapshot leases protect predecessor histories across bounded reads and native processes; reclamation preserves head tombstones and all commit receipts. The Key/Value provider uses these records directly. `Self::for_native` converts a native catalog and atomically maintains its current rows with their history; `ManagedConnection::bind_native_records` binds catalog and data handles to the same logical session.
81#[derive(Clone)]
82pub struct SQLiteRecordStore {
83    connection: ManagedConnection,
84    identity: DatabaseId,
85    native: Option<native::NativeRecordNamespace>,
86    snapshots: Arc<uqa_storage::mvcc::SnapshotRegistry>,
87}
88
89impl SQLiteRecordStore {
90    pub(crate) fn has_native_mapping(connection: &Connection) -> VersionResult<bool> {
91        native::present(connection).map_err(Error::into_version)
92    }
93
94    /// Prepare a native baseline on the initial restore's physical transaction. Its owner publishes both the restored catalog and this baseline with one COMMIT.
95    pub(crate) fn initialize_native_in(
96        connection: &ManagedConnection,
97        transaction: &Connection,
98        control: &StorageReadControl,
99    ) -> VersionResult<Self> {
100        let native::NativeMapping {
101            identity,
102            namespace,
103        } = native::initialize_in(transaction, control).map_err(Error::into_version)?;
104        Ok(Self {
105            snapshots: retention::registry(connection, identity)?,
106            connection: connection.record_connection(),
107            identity,
108            native: Some(namespace),
109        })
110    }
111
112    pub fn new(connection: &ManagedConnection) -> VersionResult<Self> {
113        if connection.in_transaction() {
114            return Err(VersionError::Storage(
115                SQLiteError::TransactionAlreadyActive.into(),
116            ));
117        }
118        let connection = connection.record_connection();
119        let identity = connection
120            .with(|connection| Ok(schema::initialize(connection)))
121            .map_err(|error| VersionError::Storage(error.into()))?
122            .map_err(Error::into_version)?;
123        Ok(Self {
124            snapshots: retention::registry(&connection, identity)?,
125            connection,
126            identity,
127            native: None,
128        })
129    }
130
131    /// Atomically initialize or upgrade a native catalog and import its baseline, or reopen its versioned materialization. Unbound native store handles are disabled after conversion; callers must bind a routed native session or prepare native records through the shared transaction contract.
132    pub fn for_native(
133        connection: &ManagedConnection,
134        control: &StorageReadControl,
135    ) -> VersionResult<Self> {
136        if connection.in_transaction() {
137            return Err(VersionError::Storage(
138                SQLiteError::TransactionAlreadyActive.into(),
139            ));
140        }
141        let connection = connection.record_connection();
142        let native::NativeMapping {
143            identity,
144            namespace,
145        } = connection
146            .with(|connection| Ok(native::initialize(connection, control)))
147            .map_err(|error| VersionError::Storage(error.into()))?
148            .map_err(Error::into_version)?;
149        Ok(Self {
150            snapshots: retention::registry(&connection, identity)?,
151            connection,
152            identity,
153            native: Some(namespace),
154        })
155    }
156
157    pub(crate) fn for_key_value(
158        connection: &ManagedConnection,
159        control: &StorageReadControl,
160    ) -> VersionResult<Self> {
161        let connection = connection.record_connection();
162        let identity = connection
163            .with(|connection| Ok(key_value::initialize(connection, control)))
164            .map_err(|error| VersionError::Storage(error.into()))?
165            .map_err(Error::into_version)?;
166        Ok(Self {
167            snapshots: retention::registry(&connection, identity)?,
168            connection,
169            identity,
170            native: None,
171        })
172    }
173
174    /// Immutable namespace for database-owned native record keys. Use it with `NativeRecordOwner::Database`; `VersionedPersistence::database_id` identifies the transaction history and may differ after restoration. Key/Value and raw record stores return `None`.
175    #[must_use]
176    pub fn native_namespace(&self) -> Option<DatabaseId> {
177        self.native.map(|namespace| namespace.0)
178    }
179
180    fn with<T>(
181        &self,
182        operation: impl FnOnce(&Connection) -> PhysicalResult<T>,
183    ) -> VersionResult<T> {
184        self.connection
185            .with(|connection| Ok(operation(connection)))
186            .map_err(|error| VersionError::Storage(error.into()))?
187            .map_err(Error::into_version)
188    }
189
190    fn with_write<T>(
191        &self,
192        control: &StorageReadControl,
193        operation: impl FnOnce(&Connection) -> PhysicalResult<T>,
194    ) -> VersionResult<T> {
195        let mut operation = Some(operation);
196        loop {
197            control.cancellation().check()?;
198            let result = self
199                .connection
200                .with_record_connection(control, |connection| {
201                    let _timeout = admission::BusyTimeout::new(connection)?;
202                    control.cancellation().check()?;
203                    Ok(operation.take().expect("record write runs once")(
204                        connection,
205                    ))
206                });
207            match result {
208                Err(SQLiteError::SQLite(error))
209                    if operation.is_some() && admission::is_busy(&error) =>
210                {
211                    // Checkout/configuration failed before the operation was invoked.
212                    admission::wait(control).map_err(Error::into_version)?;
213                }
214                result => {
215                    return result
216                        .map_err(|error| Error::from(error).into_version())?
217                        .map_err(Error::into_version);
218                }
219            }
220        }
221    }
222
223    fn check_transaction(&self, transaction: StorageTransactionId) -> VersionResult<()> {
224        if transaction.database() != self.identity {
225            return Err(VersionError::WrongDatabase);
226        }
227        Ok(())
228    }
229}
230
231impl VersionedPersistence for SQLiteRecordStore {
232    fn diskann_population_record_layout(
233        &self,
234    ) -> Option<&dyn uqa_storage::mvcc::DiskANNPopulationRecordLayout> {
235        Some(match &self.native {
236            Some(namespace) => namespace,
237            None => &uqa_storage::key_value::KeyValueDiskANNPopulationRecords,
238        })
239    }
240
241    fn resource_leases(&self) -> Option<&dyn uqa_storage::mvcc::ResourceLeaseProvider> {
242        Some(self)
243    }
244
245    fn serializable_coordinator(&self) -> Option<&dyn uqa_storage::mvcc::SerializableCoordinator> {
246        Some(self)
247    }
248
249    fn identifier_watermark(
250        &self,
251        namespace: &[u8],
252        control: &StorageReadControl,
253    ) -> VersionResult<Option<u64>> {
254        control.cancellation().check()?;
255        self.with(|connection| {
256            identifiers::read(connection, self.identity, self.native, namespace, control)
257        })
258    }
259
260    fn allocate_identifiers(
261        &self,
262        namespace: &[u8],
263        request: uqa_storage::mvcc::IdentifierRequest,
264        control: &StorageReadControl,
265    ) -> VersionResult<uqa_storage::mvcc::IdentifierAllocation> {
266        control.cancellation().check()?;
267        self.with_write(control, |connection| {
268            identifiers::allocate(
269                connection,
270                self.identity,
271                self.native,
272                namespace,
273                request,
274                control,
275            )
276        })
277    }
278
279    fn database_id(&self) -> DatabaseId {
280        self.identity
281    }
282    fn graph_record_layout(&self) -> Option<&dyn uqa_storage::mvcc::GraphRecordLayout> {
283        match self.native.as_ref() {
284            Some(namespace) => Some(namespace),
285            None => Some(&uqa_storage::key_value::KeyValueGraphRecords),
286        }
287    }
288    fn occurrence_record_layout(&self) -> &dyn uqa_storage::mvcc::OccurrenceRecordLayout {
289        if self.native.is_some() {
290            &crate::inverted_index::NativeOccurrenceRecords
291        } else {
292            &uqa_storage::key_value::KeyValueOccurrenceRecords
293        }
294    }
295    fn notification_record_layout(&self) -> &dyn uqa_storage::mvcc::NotificationRecordLayout {
296        match self.native.as_ref() {
297            Some(namespace) => namespace,
298            None => &uqa_storage::key_value::KeyValueNotificationRecords,
299        }
300    }
301
302    fn maintenance_record_layout(&self) -> &dyn uqa_storage::mvcc::MaintenanceRecordLayout {
303        if self.native.is_some() {
304            &native::NativeMaintenanceRecords
305        } else {
306            &uqa_storage::key_value::KeyValueMaintenanceRecords
307        }
308    }
309    fn ivf_record_layout(&self) -> &dyn uqa_storage::mvcc::IVFRecordLayout {
310        if self.native.is_some() {
311            &crate::vector_index::NativeIVFRecords
312        } else {
313            &uqa_storage::key_value::KeyValueIVFRecords
314        }
315    }
316    fn hnsw_record_layout(&self) -> Option<&dyn uqa_storage::mvcc::HNSWRecordLayout> {
317        if self.native.is_some() {
318            Some(&crate::vector_index::NativeHNSWRecords)
319        } else {
320            Some(&uqa_storage::key_value::KeyValueHNSWRecords)
321        }
322    }
323    fn allocate_transaction(
324        &self,
325        control: &StorageReadControl,
326    ) -> VersionResult<StorageTransactionId> {
327        control.cancellation().check()?;
328        self.with_write(control, |connection| {
329            write::allocate(connection, self.identity, self.native, control)
330        })
331    }
332
333    fn allocate_managed_transaction(
334        &self,
335        control: &StorageReadControl,
336    ) -> VersionResult<uqa_storage::mvcc::RetainedTransactionAllocation> {
337        self.allocate_receipt_owner(control)
338    }
339    fn snapshot(
340        &self,
341        control: &StorageReadControl,
342    ) -> VersionResult<Arc<dyn CommittedRecordSnapshot>> {
343        control.cancellation().check()?;
344        let mut reclamation_epoch = 0;
345        let lease = self.snapshots.capture(control, || {
346            self.with(|connection| {
347                let read = connection.unchecked_transaction()?;
348                native::check_mapping(&read, self.native)?;
349                let sequence = codec::header(&read, self.identity)?.sequence;
350                reclamation_epoch = tombstones::epoch(&read)?;
351                read.commit()?;
352                Ok(sequence)
353            })
354        })?;
355        uqa_storage::mvcc::retain_record_snapshot(
356            read::Snapshot {
357                store: self.clone(),
358                sequence: lease.sequence(),
359                reclamation_epoch,
360                _lease: lease,
361            },
362            control,
363        )
364    }
365    fn reclaim_versions(&self, control: &StorageReadControl) -> VersionResult<u64> {
366        self.snapshots.reclaim(control, |oldest| {
367            self.with_write(control, |connection| {
368                reclamation::reclaim(connection, self.identity, self.native, oldest, control)
369            })
370        })
371    }
372
373    fn reclaim_tombstones(
374        &self,
375        request: &uqa_storage::mvcc::TombstoneReclamationRequest<'_>,
376        control: &StorageReadControl,
377    ) -> VersionResult<uqa_storage::mvcc::TombstoneReclamationStep> {
378        request.validate(control)?;
379        self.snapshots.reclaim(control, |oldest| {
380            if oldest.is_some() {
381                return Ok(uqa_storage::mvcc::TombstoneReclamationStep::Retained);
382            }
383            self.with_write(control, |connection| {
384                tombstones::reclaim(connection, self.identity, self.native, request, control)
385            })
386        })
387    }
388
389    fn reclaim_diskann_tombstones(&self, control: &StorageReadControl) -> VersionResult<()> {
390        if self.native.is_none() {
391            return uqa_storage::mvcc::reclaim_key_value_diskann_tombstones(self, control);
392        }
393        for family in [
394            native::NativeRecordFamily::DiskANNRecords,
395            native::NativeRecordFamily::VectorOrigins,
396            native::NativeRecordFamily::VectorChanges,
397            native::NativeRecordFamily::VectorPopulations,
398            native::NativeRecordFamily::VectorPopulationWitnesses,
399        ] {
400            let prefix = native::NativeRecordIdentity::family_prefix(family, control)?;
401            uqa_storage::mvcc::reclaim_tombstone_prefix(self, &prefix, control)?;
402        }
403        let prefix = self.vector_field_guard_layout().prefix(control)?;
404        uqa_storage::mvcc::reclaim_tombstone_prefix(self, &prefix, control)?;
405        Ok(())
406    }
407
408    fn vector_field_guard_layout(&self) -> &dyn uqa_storage::mvcc::VectorFieldGuardLayout {
409        self
410    }
411
412    fn commit(
413        &self,
414        transaction: StorageTransactionId,
415        prepared: &PreparedRecordCommit,
416        control: &StorageReadControl,
417    ) -> CommitResult {
418        self.check_transaction(transaction)?;
419        control.cancellation().check().map_err(VersionError::from)?;
420        let _bindings = write::reserve_bindings(prepared, control)?;
421        self.with_write(control, |connection| {
422            Ok(write::commit(
423                connection,
424                transaction,
425                prepared,
426                self.native,
427                control,
428            ))
429        })?
430    }
431    fn commit_status(
432        &self,
433        transaction: StorageTransactionId,
434        control: &StorageReadControl,
435    ) -> VersionResult<CommitStatus> {
436        self.check_transaction(transaction)?;
437        control.cancellation().check()?;
438        self.with(|connection| {
439            let read = connection.unchecked_transaction()?;
440            native::check_mapping(&read, self.native)?;
441            codec::header(&read, self.identity)?;
442            let status = codec::status(&read, transaction)?;
443            control.cancellation().check().map_err(VersionError::from)?;
444            read.commit()?;
445            Ok(status)
446        })
447    }
448
449    fn acknowledge_transaction(
450        &self,
451        acknowledgement: uqa_storage::mvcc::ReceiptAcknowledgement,
452        control: &StorageReadControl,
453    ) -> VersionResult<()> {
454        self.check_transaction(acknowledgement.transaction())?;
455        self.with_write(control, |connection| {
456            receipts::acknowledge(connection, self.native, acknowledgement, control)
457        })
458    }
459
460    fn reclaim_transaction_receipts(&self, control: &StorageReadControl) -> VersionResult<u64> {
461        self.reclaim_receipts(control)
462    }
463    fn abort(
464        &self,
465        transaction: StorageTransactionId,
466        control: &StorageReadControl,
467    ) -> VersionResult<CommitStatus> {
468        self.check_transaction(transaction)?;
469        control.cancellation().check()?;
470        self.with(|connection| write::abort(connection, transaction, self.native, control))
471    }
472}