Skip to main content

uqa_storage_sqlite/connection/
native.rs

1//
2// Unified Query Algebra
3//
4// Copyright (c) 2023-2026 Cognica, Inc.
5//
6
7//! Native row operations share one logical session, including the read boundary of each evaluated mutation.
8
9use uqa_storage::mvcc::{StorageMutationOrigin, VersionError, VersionedSessionOptions};
10use uqa_storage::read_control::StorageReadControl;
11use uqa_storage::KeyValueBatch;
12
13use super::logical::BoundRecordSession;
14use super::{Arc, KeyValueStore, ManagedConnection, Result, SQLiteError, VersionedKeyValueStore};
15use crate::mvcc::native::NativeSnapshot;
16
17pub(crate) struct NativeWriteSnapshot {
18    pub(crate) snapshot: NativeSnapshot,
19    pub(crate) committed: Option<uqa_storage::mvcc::CommitSequence>,
20}
21
22impl ManagedConnection {
23    /// Prune one bounded `DiskANN` journal page in this connection's evaluated transaction. The captured source guards the catalog; the current committed canonical view and selected physical coverage determine deletion. Advance the returned cursor only after committing the caller's transaction.
24    pub fn prune_diskann_changes(
25        &self,
26        source: &crate::vector_index::RetainedSQLiteDiskANNCanonical,
27        resolver: &dyn uqa_storage::diskann_index::catalog::DiskANNIndexResolver,
28        pruner: &uqa_storage::key_value::KeyValueDiskANNPruner,
29        request: uqa_storage::diskann_index::changes::DiskANNPruneRequest,
30        control: &StorageReadControl,
31    ) -> uqa_storage::StorageBackendResult<uqa_storage::diskann_index::changes::DiskANNPruneResult>
32    {
33        self.with_native_write(|snapshot, batch| {
34            Ok(source.prune_changes(resolver, pruner, (snapshot, batch), request, control)?)
35        })?
36        .ok_or_else(|| {
37            uqa_storage::StorageBackendError::Other(
38                "DiskANN pruning requires a bound native session".into(),
39            )
40        })
41    }
42
43    /// Install an actually sealed `DiskANN` generation and its complete coverage in this connection's native transaction. The build's retained source supplies catalog and expected-head evidence. Later private DDL must cancel or supersede this effect through the catalog lifecycle owner.
44    pub fn publish_diskann_generation(
45        &self,
46        coverage: &uqa_storage::diskann_index::build::DiskANNCanonicalCoverage<
47            crate::vector_index::RetainedSQLiteDiskANNCanonical,
48        >,
49        resolver: &dyn uqa_storage::diskann_index::catalog::DiskANNIndexResolver,
50        control: &StorageReadControl,
51    ) -> uqa_storage::StorageBackendResult<()> {
52        let sealed = self
53            .diskann_generations(control)?
54            .open_source(coverage.fingerprint().generation(), control)?;
55        self.with_native_write(|snapshot, batch| {
56            crate::vector_index::RetainedSQLiteDiskANNCanonical::publish_generation(
57                coverage, resolver, &sealed, snapshot, batch, control,
58            )?;
59            Ok(())
60        })?
61        .ok_or_else(|| {
62            uqa_storage::StorageBackendError::Other(
63                "DiskANN publication requires a bound native session".into(),
64            )
65        })
66    }
67
68    /// Connect physical `DiskANN` staging to an already bound native owner. The returned store opens a dedicated session; it cannot complete this connection's transaction or publish a SQL index.
69    pub fn diskann_generations(
70        &self,
71        control: &StorageReadControl,
72    ) -> uqa_storage::StorageBackendResult<uqa_storage::key_value::KeyValueDiskANNStore> {
73        let records: Arc<dyn KeyValueStore> = Arc::new(self.native_diskann_records()?);
74        uqa_storage::key_value::KeyValueDiskANNStore::connect(&records, control)
75    }
76
77    pub(crate) fn native_diskann_records(&self) -> Result<crate::diskann::Records> {
78        self.surface_cleanup_failure()?;
79        let _gate = self.session.gate.read();
80        let logical = self
81            .session
82            .logical
83            .get()
84            .ok_or(SQLiteError::LogicalSessionRequired)?;
85        let namespace = logical.native.ok_or(SQLiteError::SessionMappingMismatch)?;
86        crate::diskann::Records::new(
87            logical.store.clone(),
88            namespace,
89            self.auxiliary_encryption_key(),
90        )
91        .map_err(Into::into)
92    }
93
94    pub(crate) fn native_identifier_watermark(&self, namespace: &[u8]) -> Result<Option<u64>> {
95        self.surface_cleanup_failure()?;
96        let _gate = self.session.gate.read();
97        let logical = self
98            .session
99            .logical
100            .get()
101            .filter(|session| session.native.is_some())
102            .ok_or(SQLiteError::SessionMappingMismatch)?;
103        logical.identifier_watermark(namespace).map_err(Into::into)
104    }
105
106    pub(crate) fn allocate_native_identifiers(
107        &self,
108        namespace: &[u8],
109        request: uqa_storage::mvcc::IdentifierRequest,
110    ) -> Result<uqa_storage::mvcc::IdentifierAllocation> {
111        self.surface_cleanup_failure()?;
112        let _gate = self.session.gate.read();
113        let logical = self
114            .session
115            .logical
116            .get()
117            .filter(|session| session.native.is_some())
118            .ok_or(SQLiteError::SessionMappingMismatch)?;
119        logical
120            .allocate_identifiers(namespace, request)
121            .map_err(Into::into)
122    }
123
124    pub(crate) fn observe_native_identifier(
125        &self,
126        namespace: &[u8],
127        value: u64,
128    ) -> Result<uqa_storage::mvcc::ObservedIdentifier> {
129        self.surface_cleanup_failure()?;
130        let _gate = self.session.gate.read();
131        let logical = self
132            .session
133            .logical
134            .get()
135            .filter(|session| session.native.is_some())
136            .ok_or(SQLiteError::SessionMappingMismatch)?;
137        logical
138            .observe_identifier(namespace, value)
139            .map_err(Into::into)
140    }
141
142    pub(crate) fn is_native_record_session(&self) -> bool {
143        self.session
144            .logical
145            .get()
146            .is_some_and(|session| session.native.is_some())
147    }
148
149    /// Bind a new or existing native catalog or recognized standalone graph file to shared logical record transactions, initializing/upgrading and converting its physical format atomically if needed. Document/B-tree stores, exact/IVF/HNSW vectors, occurrences and catalog operations share this session. Native provider factories publish a legacy catalog restore and its baseline together before attaching this session; direct callers must bind before opening a catalog on an already converted file. Standalone graph handles retain their scoped records, including handles opened before binding. Direct physical access is rejected.
150    pub fn bind_native_records(&self, options: VersionedSessionOptions) -> Result<()> {
151        self.surface_cleanup_failure()?;
152        let _gate = self.session.gate.write();
153        if let Some(logical) = self.session.logical.get() {
154            if logical.native.is_none() {
155                return Err(SQLiteError::SessionMappingMismatch);
156            }
157            return if logical.options().retained_bytes == options.retained_bytes {
158                Ok(())
159            } else {
160                Err(SQLiteError::SessionOptionsMismatch)
161            };
162        }
163        if self.session.transaction.lock().is_some() {
164            return Err(SQLiteError::TransactionAlreadyActive);
165        }
166        let identity = self.storage_identity()?;
167        let records = crate::SQLiteRecordStore::for_native(
168            &self.record_connection(),
169            &StorageReadControl::with_limit(options.retained_bytes),
170        )
171        .map_err(VersionError::into_storage_error)?;
172        let namespace = records
173            .native_namespace()
174            .ok_or(SQLiteError::SessionMappingMismatch)?;
175        self.session
176            .logical
177            .set(Arc::new(BoundRecordSession {
178                store: Arc::new(VersionedKeyValueStore::new_with_cancellation(
179                    Arc::new(records),
180                    identity,
181                    options,
182                    self.write_cancellation(),
183                )),
184                native: Some(namespace),
185            }))
186            .map_err(|_| SQLiteError::SessionMappingMismatch)
187    }
188
189    pub(crate) fn native_snapshot(&self) -> Result<Option<Arc<NativeSnapshot>>> {
190        self.surface_cleanup_failure()?;
191        let _gate = self.session.gate.read();
192        self.session
193            .logical
194            .get()
195            .map(|logical| {
196                NativeSnapshot::capture(
197                    logical,
198                    logical.native.ok_or(SQLiteError::SessionMappingMismatch)?,
199                )
200                .map(Arc::new)
201            })
202            .transpose()
203    }
204
205    pub(crate) fn with_native_write<R>(
206        &self,
207        operation: impl FnOnce(&NativeSnapshot, &mut dyn KeyValueBatch) -> Result<R>,
208    ) -> Result<Option<R>> {
209        self.native_write_result(operation, false)
210            .map(|result| result.map(|(value, _)| value))
211    }
212
213    /// Retain the successfully staged boundary while the session gate is still held, before autocommit can rebase onto a peer's data. A later read cannot certify the evaluated candidate.
214    pub(crate) fn with_native_write_snapshot<R>(
215        &self,
216        operation: impl FnOnce(&NativeSnapshot, &mut dyn KeyValueBatch) -> Result<R>,
217    ) -> Result<Option<(R, Option<NativeWriteSnapshot>)>> {
218        self.native_write_result(operation, true)
219    }
220
221    fn native_write_result<R>(
222        &self,
223        operation: impl FnOnce(&NativeSnapshot, &mut dyn KeyValueBatch) -> Result<R>,
224        retain_snapshot: bool,
225    ) -> Result<Option<(R, Option<NativeWriteSnapshot>)>> {
226        self.surface_cleanup_failure()?;
227        // This gate covers read/evaluate/stage, not just each individual byte-store call.
228        let _gate = self.session.gate.write();
229        let Some(logical) = self.session.logical.get() else {
230            return Ok(None);
231        };
232        let database = logical.native.ok_or(SQLiteError::SessionMappingMismatch)?;
233        let own_transaction = !logical.in_transaction();
234        if own_transaction {
235            logical.begin_transaction()?;
236        }
237        let mut scope = WriteScope {
238            connection: self,
239            logical,
240            rollback: own_transaction,
241        };
242        let snapshot = NativeSnapshot::capture(logical, database)?;
243        let mut batch = logical.batch();
244        let result = operation(&snapshot, &mut *batch)?;
245        batch.commit()?;
246        // This capture is advisory cache evidence; failure must not turn an already staged mutation into an error.
247        let staged = retain_snapshot
248            .then(|| NativeSnapshot::capture(logical, database).ok())
249            .flatten();
250        // A failed publication keeps its evaluated attempt for explicit receipt resolution.
251        scope.rollback = false;
252        if own_transaction {
253            logical.commit_transaction()?;
254        }
255        let staged = staged.map(|snapshot| {
256            // Session serialization is still held. Only the immediately following committed sequence proves no peer changed the candidate's inputs during publication.
257            let committed = own_transaction
258                .then(|| logical.completed_commit())
259                .flatten()
260                .map(|receipt| receipt.sequence)
261                .filter(|sequence| snapshot.view.sequence().successor().ok() == Some(*sequence));
262            NativeWriteSnapshot {
263                snapshot,
264                committed,
265            }
266        });
267        Ok(Some((result, staged)))
268    }
269
270    /// Evaluate native records through the common origin scope. Its supplied snapshot avoids reentering the session and preserves common autocommit, abort-only cleanup and exact receipt retry.
271    pub(crate) fn with_native_versioned_write<R>(
272        &self,
273        operation: impl FnOnce(
274            StorageMutationOrigin,
275            &NativeSnapshot,
276            &mut dyn KeyValueBatch,
277        ) -> Result<R>,
278    ) -> Result<R> {
279        self.surface_cleanup_failure()?;
280        let _gate = self.session.gate.write();
281        let logical = self
282            .session
283            .logical
284            .get()
285            .ok_or(SQLiteError::LogicalSessionRequired)?;
286        let database = logical.native.ok_or(SQLiteError::SessionMappingMismatch)?;
287        let control = logical.retention_control();
288        Ok(
289            logical.with_versioned_record_mutation(|origin, view, batch| {
290                operation(
291                    origin,
292                    &NativeSnapshot {
293                        view,
294                        control,
295                        database,
296                        history: origin.transaction().database(),
297                    },
298                    batch,
299                )
300                .map_err(Into::into)
301            })?,
302        )
303    }
304}
305
306struct WriteScope<'a> {
307    connection: &'a ManagedConnection,
308    logical: &'a BoundRecordSession,
309    rollback: bool,
310}
311
312impl Drop for WriteScope<'_> {
313    fn drop(&mut self) {
314        if self.rollback {
315            if let Err(error) = self.logical.rollback_transaction() {
316                *self.connection.session.cleanup_failure.lock() = Some(error.to_string());
317            }
318        }
319    }
320}
321
322#[cfg(test)]
323mod tests {
324    use super::*;
325
326    #[test]
327    fn native_evaluation_unwind_rolls_back_only_its_own_transaction() {
328        let connection = ManagedConnection::open_in_memory().unwrap();
329        crate::Catalog::open(connection.clone()).unwrap();
330        connection
331            .bind_native_records(VersionedSessionOptions::default())
332            .unwrap();
333        for explicit in [false, true] {
334            if explicit {
335                connection.begin_transaction().unwrap();
336            }
337            let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
338                let _: Result<Option<()>> = connection.with_native_write(|snapshot, batch| {
339                    snapshot.ensure_table_owner("never-published", batch)?;
340                    panic!("injected evaluation unwind");
341                });
342            }));
343            assert!(result.is_err());
344            assert_eq!(connection.in_transaction(), explicit);
345            assert!(connection
346                .native_snapshot()
347                .unwrap()
348                .unwrap()
349                .table_owner("never-published")
350                .unwrap()
351                .is_none());
352            if explicit {
353                connection.rollback_transaction().unwrap();
354            }
355        }
356    }
357
358    #[test]
359    fn native_document_writes_respect_a_read_only_logical_transaction() {
360        use uqa_storage::DocumentStore;
361        let connection = ManagedConnection::open_in_memory().unwrap();
362        crate::Catalog::open(connection.clone()).unwrap();
363        connection
364            .bind_native_records(VersionedSessionOptions::default())
365            .unwrap();
366        connection.begin_record_read().unwrap();
367        let mut documents = crate::SQLiteDocumentStore::new(connection.clone(), "docs");
368        assert!(documents.put(1, std::collections::BTreeMap::new()).is_err());
369        assert!(!connection.transaction_has_written().unwrap());
370        connection.commit_transaction().unwrap();
371        assert!(!documents.contains_doc_id(1).unwrap());
372        connection
373            .with_physical(|sqlite| {
374                assert_eq!(
375                    sqlite.query_row("SELECT count(*) FROM _uqa_mvcc_identifiers", [], |row| {
376                        row.get::<_, i64>(0)
377                    })?,
378                    0
379                );
380                Ok(())
381            })
382            .unwrap();
383    }
384
385    #[test]
386    fn native_autocommit_pins_preconditions_before_evaluating_mutation() {
387        use crate::mvcc::native::{NativeRecord, NativeRecordFamily};
388        use uqa_storage::DocumentStore;
389        let connection = ManagedConnection::open_in_memory().unwrap();
390        crate::Catalog::open(connection.clone()).unwrap();
391        connection
392            .bind_native_records(VersionedSessionOptions::default())
393            .unwrap();
394        let mut a = crate::SQLiteDocumentStore::new(connection.clone(), "docs");
395        let fields = |n| std::collections::BTreeMap::from([("n".into(), uqa_core::Value::Int(n))]);
396        a.put(1, fields(1)).unwrap();
397        let mut b = crate::SQLiteDocumentStore::new(connection.new_session(), "docs");
398        let result = connection.with_native_write(|snapshot, batch| {
399            let owner = snapshot.table_owner("docs")?.unwrap();
400            let old = snapshot
401                .read_row(
402                    NativeRecordFamily::Documents,
403                    owner,
404                    &[rusqlite::types::ValueRef::Integer(1)],
405                    |row| {
406                        Ok(NativeRecord::encode(
407                            NativeRecordFamily::Documents,
408                            owner,
409                            row,
410                            &snapshot.control,
411                        )?)
412                    },
413                )?
414                .unwrap();
415            // B publishes after A's evaluation read but before A stages its result.
416            b.put(1, fields(2))?;
417            batch.put(old.key(), old.row())?;
418            Ok(())
419        });
420        assert!(result.is_err());
421        connection.rollback_transaction().unwrap();
422        assert_eq!(a.get(1).unwrap(), Some(fields(2)));
423    }
424}