Skip to main content

uqa_storage_sqlite/connection/
native.rs

1//
2// Unified Query Algebra
3//
4// Copyright (c) 2023-2026 Cognica, Inc.
5//
6
7//! Native row operations share one logical session, including the read boundary of each evaluated mutation.
8
9use uqa_storage::mvcc::{StorageMutationOrigin, VersionError, VersionedSessionOptions};
10use uqa_storage::read_control::StorageReadControl;
11use uqa_storage::KeyValueBatch;
12
13use super::logical::BoundRecordSession;
14use super::{Arc, KeyValueStore, ManagedConnection, Result, SQLiteError, VersionedKeyValueStore};
15use crate::mvcc::native::NativeSnapshot;
16
17impl ManagedConnection {
18    /// Prune one bounded `DiskANN` journal page in this connection's evaluated transaction. The captured source guards the catalog; the current committed canonical view and selected physical coverage determine deletion. Advance the returned cursor only after committing the caller's transaction.
19    pub fn prune_diskann_changes(
20        &self,
21        source: &crate::vector_index::RetainedSQLiteDiskANNCanonical,
22        resolver: &dyn uqa_storage::diskann_index::catalog::DiskANNIndexResolver,
23        pruner: &uqa_storage::key_value::KeyValueDiskANNPruner,
24        request: uqa_storage::diskann_index::changes::DiskANNPruneRequest,
25        control: &StorageReadControl,
26    ) -> uqa_storage::StorageBackendResult<uqa_storage::diskann_index::changes::DiskANNPruneResult>
27    {
28        self.with_native_write(|snapshot, batch| {
29            Ok(source.prune_changes(resolver, pruner, (snapshot, batch), request, control)?)
30        })?
31        .ok_or_else(|| {
32            uqa_storage::StorageBackendError::Other(
33                "DiskANN pruning requires a bound native session".into(),
34            )
35        })
36    }
37
38    /// Install an actually sealed `DiskANN` generation and its complete coverage in this connection's native transaction. The build's retained source supplies catalog and expected-head evidence. Later private DDL must cancel or supersede this effect through the catalog lifecycle owner.
39    pub fn publish_diskann_generation(
40        &self,
41        coverage: &uqa_storage::diskann_index::build::DiskANNCanonicalCoverage<
42            crate::vector_index::RetainedSQLiteDiskANNCanonical,
43        >,
44        resolver: &dyn uqa_storage::diskann_index::catalog::DiskANNIndexResolver,
45        control: &StorageReadControl,
46    ) -> uqa_storage::StorageBackendResult<()> {
47        let sealed = self
48            .diskann_generations(control)?
49            .open_source(coverage.fingerprint().generation(), control)?;
50        self.with_native_write(|snapshot, batch| {
51            crate::vector_index::RetainedSQLiteDiskANNCanonical::publish_generation(
52                coverage, resolver, &sealed, snapshot, batch, control,
53            )?;
54            Ok(())
55        })?
56        .ok_or_else(|| {
57            uqa_storage::StorageBackendError::Other(
58                "DiskANN publication requires a bound native session".into(),
59            )
60        })
61    }
62
63    /// Connect physical `DiskANN` staging to an already bound native owner. The returned store opens a dedicated session; it cannot complete this connection's transaction or publish a SQL index.
64    pub fn diskann_generations(
65        &self,
66        control: &StorageReadControl,
67    ) -> uqa_storage::StorageBackendResult<uqa_storage::key_value::KeyValueDiskANNStore> {
68        let records: Arc<dyn KeyValueStore> = Arc::new(self.native_diskann_records()?);
69        uqa_storage::key_value::KeyValueDiskANNStore::connect(&records, control)
70    }
71
72    pub(crate) fn native_diskann_records(&self) -> Result<crate::diskann::Records> {
73        self.surface_cleanup_failure()?;
74        let _gate = self.session.gate.read();
75        let logical = self
76            .session
77            .logical
78            .get()
79            .ok_or(SQLiteError::LogicalSessionRequired)?;
80        let namespace = logical.native.ok_or(SQLiteError::SessionMappingMismatch)?;
81        crate::diskann::Records::new(
82            logical.store.clone(),
83            namespace,
84            self.auxiliary_encryption_key(),
85        )
86        .map_err(Into::into)
87    }
88
89    pub(crate) fn native_identifier_watermark(&self, namespace: &[u8]) -> Result<Option<u64>> {
90        self.surface_cleanup_failure()?;
91        let _gate = self.session.gate.read();
92        let logical = self
93            .session
94            .logical
95            .get()
96            .filter(|session| session.native.is_some())
97            .ok_or(SQLiteError::SessionMappingMismatch)?;
98        logical.identifier_watermark(namespace).map_err(Into::into)
99    }
100
101    pub(crate) fn allocate_native_identifiers(
102        &self,
103        namespace: &[u8],
104        request: uqa_storage::mvcc::IdentifierRequest,
105    ) -> Result<uqa_storage::mvcc::IdentifierAllocation> {
106        self.surface_cleanup_failure()?;
107        let _gate = self.session.gate.read();
108        let logical = self
109            .session
110            .logical
111            .get()
112            .filter(|session| session.native.is_some())
113            .ok_or(SQLiteError::SessionMappingMismatch)?;
114        logical
115            .allocate_identifiers(namespace, request)
116            .map_err(Into::into)
117    }
118
119    pub(crate) fn observe_native_identifier(
120        &self,
121        namespace: &[u8],
122        value: u64,
123    ) -> Result<uqa_storage::mvcc::ObservedIdentifier> {
124        self.surface_cleanup_failure()?;
125        let _gate = self.session.gate.read();
126        let logical = self
127            .session
128            .logical
129            .get()
130            .filter(|session| session.native.is_some())
131            .ok_or(SQLiteError::SessionMappingMismatch)?;
132        logical
133            .observe_identifier(namespace, value)
134            .map_err(Into::into)
135    }
136
137    pub(crate) fn is_native_record_session(&self) -> bool {
138        self.session
139            .logical
140            .get()
141            .is_some_and(|session| session.native.is_some())
142    }
143
144    /// Bind a new or existing native catalog or recognized standalone graph file to shared logical record transactions, initializing/upgrading and converting its physical format atomically if needed. Document/B-tree stores, exact/IVF/HNSW vectors, occurrences and catalog operations share this session. Native provider factories publish a legacy catalog restore and its baseline together before attaching this session; direct callers must bind before opening a catalog on an already converted file. Standalone graph handles retain their scoped records, including handles opened before binding. Direct physical access is rejected.
145    pub fn bind_native_records(&self, options: VersionedSessionOptions) -> Result<()> {
146        self.surface_cleanup_failure()?;
147        let _gate = self.session.gate.write();
148        if let Some(logical) = self.session.logical.get() {
149            if logical.native.is_none() {
150                return Err(SQLiteError::SessionMappingMismatch);
151            }
152            return if logical.options().retained_bytes == options.retained_bytes {
153                Ok(())
154            } else {
155                Err(SQLiteError::SessionOptionsMismatch)
156            };
157        }
158        if self.session.transaction.lock().is_some() {
159            return Err(SQLiteError::TransactionAlreadyActive);
160        }
161        let identity = self.storage_identity()?;
162        let records = crate::SQLiteRecordStore::for_native(
163            &self.record_connection(),
164            &StorageReadControl::with_limit(options.retained_bytes),
165        )
166        .map_err(VersionError::into_storage_error)?;
167        let namespace = records
168            .native_namespace()
169            .ok_or(SQLiteError::SessionMappingMismatch)?;
170        self.session
171            .logical
172            .set(Arc::new(BoundRecordSession {
173                store: Arc::new(VersionedKeyValueStore::new_with_cancellation(
174                    Arc::new(records),
175                    identity,
176                    options,
177                    self.write_cancellation(),
178                )),
179                native: Some(namespace),
180            }))
181            .map_err(|_| SQLiteError::SessionMappingMismatch)
182    }
183
184    pub(crate) fn native_snapshot(&self) -> Result<Option<Arc<NativeSnapshot>>> {
185        self.surface_cleanup_failure()?;
186        let _gate = self.session.gate.read();
187        self.session
188            .logical
189            .get()
190            .map(|logical| {
191                NativeSnapshot::capture(
192                    logical,
193                    logical.native.ok_or(SQLiteError::SessionMappingMismatch)?,
194                )
195                .map(Arc::new)
196            })
197            .transpose()
198    }
199
200    pub(crate) fn with_native_write<R>(
201        &self,
202        operation: impl FnOnce(&NativeSnapshot, &mut dyn KeyValueBatch) -> Result<R>,
203    ) -> Result<Option<R>> {
204        self.surface_cleanup_failure()?;
205        // This gate covers read/evaluate/stage, not just each individual byte-store call.
206        let _gate = self.session.gate.write();
207        let Some(logical) = self.session.logical.get() else {
208            return Ok(None);
209        };
210        let database = logical.native.ok_or(SQLiteError::SessionMappingMismatch)?;
211        let own_transaction = !logical.in_transaction();
212        if own_transaction {
213            logical.begin_transaction()?;
214        }
215        let mut scope = WriteScope {
216            connection: self,
217            logical,
218            rollback: own_transaction,
219        };
220        let snapshot = NativeSnapshot::capture(logical, database)?;
221        let mut batch = logical.batch();
222        let result = operation(&snapshot, &mut *batch)?;
223        batch.commit()?;
224        // A failed publication keeps its evaluated attempt for explicit receipt resolution.
225        scope.rollback = false;
226        if own_transaction {
227            logical.commit_transaction()?;
228        }
229        Ok(Some(result))
230    }
231
232    /// Evaluate native records through the common origin scope. Its supplied snapshot avoids reentering the session and preserves common autocommit, abort-only cleanup and exact receipt retry.
233    pub(crate) fn with_native_versioned_write<R>(
234        &self,
235        operation: impl FnOnce(
236            StorageMutationOrigin,
237            &NativeSnapshot,
238            &mut dyn KeyValueBatch,
239        ) -> Result<R>,
240    ) -> Result<R> {
241        self.surface_cleanup_failure()?;
242        let _gate = self.session.gate.write();
243        let logical = self
244            .session
245            .logical
246            .get()
247            .ok_or(SQLiteError::LogicalSessionRequired)?;
248        let database = logical.native.ok_or(SQLiteError::SessionMappingMismatch)?;
249        let control = logical.retention_control();
250        Ok(
251            logical.with_versioned_record_mutation(|origin, view, batch| {
252                operation(
253                    origin,
254                    &NativeSnapshot {
255                        view,
256                        control,
257                        database,
258                        history: origin.transaction().database(),
259                    },
260                    batch,
261                )
262                .map_err(Into::into)
263            })?,
264        )
265    }
266}
267
268struct WriteScope<'a> {
269    connection: &'a ManagedConnection,
270    logical: &'a BoundRecordSession,
271    rollback: bool,
272}
273
274impl Drop for WriteScope<'_> {
275    fn drop(&mut self) {
276        if self.rollback {
277            if let Err(error) = self.logical.rollback_transaction() {
278                *self.connection.session.cleanup_failure.lock() = Some(error.to_string());
279            }
280        }
281    }
282}
283
284#[cfg(test)]
285mod tests {
286    use super::*;
287
288    #[test]
289    fn native_evaluation_unwind_rolls_back_only_its_own_transaction() {
290        let connection = ManagedConnection::open_in_memory().unwrap();
291        crate::Catalog::open(connection.clone()).unwrap();
292        connection
293            .bind_native_records(VersionedSessionOptions::default())
294            .unwrap();
295        for explicit in [false, true] {
296            if explicit {
297                connection.begin_transaction().unwrap();
298            }
299            let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
300                let _: Result<Option<()>> = connection.with_native_write(|snapshot, batch| {
301                    snapshot.ensure_table_owner("never-published", batch)?;
302                    panic!("injected evaluation unwind");
303                });
304            }));
305            assert!(result.is_err());
306            assert_eq!(connection.in_transaction(), explicit);
307            assert!(connection
308                .native_snapshot()
309                .unwrap()
310                .unwrap()
311                .table_owner("never-published")
312                .unwrap()
313                .is_none());
314            if explicit {
315                connection.rollback_transaction().unwrap();
316            }
317        }
318    }
319
320    #[test]
321    fn native_document_writes_respect_a_read_only_logical_transaction() {
322        use uqa_storage::DocumentStore;
323        let connection = ManagedConnection::open_in_memory().unwrap();
324        crate::Catalog::open(connection.clone()).unwrap();
325        connection
326            .bind_native_records(VersionedSessionOptions::default())
327            .unwrap();
328        connection.begin_record_read().unwrap();
329        let mut documents = crate::SQLiteDocumentStore::new(connection.clone(), "docs");
330        assert!(documents.put(1, std::collections::BTreeMap::new()).is_err());
331        assert!(!connection.transaction_has_written().unwrap());
332        connection.commit_transaction().unwrap();
333        assert!(!documents.contains_doc_id(1).unwrap());
334        connection
335            .with_physical(|sqlite| {
336                assert_eq!(
337                    sqlite.query_row("SELECT count(*) FROM _uqa_mvcc_identifiers", [], |row| {
338                        row.get::<_, i64>(0)
339                    })?,
340                    0
341                );
342                Ok(())
343            })
344            .unwrap();
345    }
346
347    #[test]
348    fn native_autocommit_pins_preconditions_before_evaluating_mutation() {
349        use crate::mvcc::native::{NativeRecord, NativeRecordFamily};
350        use uqa_storage::DocumentStore;
351        let connection = ManagedConnection::open_in_memory().unwrap();
352        crate::Catalog::open(connection.clone()).unwrap();
353        connection
354            .bind_native_records(VersionedSessionOptions::default())
355            .unwrap();
356        let mut a = crate::SQLiteDocumentStore::new(connection.clone(), "docs");
357        let fields = |n| std::collections::BTreeMap::from([("n".into(), uqa_core::Value::Int(n))]);
358        a.put(1, fields(1)).unwrap();
359        let mut b = crate::SQLiteDocumentStore::new(connection.new_session(), "docs");
360        let result = connection.with_native_write(|snapshot, batch| {
361            let owner = snapshot.table_owner("docs")?.unwrap();
362            let old = snapshot
363                .read_row(
364                    NativeRecordFamily::Documents,
365                    owner,
366                    &[rusqlite::types::ValueRef::Integer(1)],
367                    |row| {
368                        Ok(NativeRecord::encode(
369                            NativeRecordFamily::Documents,
370                            owner,
371                            row,
372                            &snapshot.control,
373                        )?)
374                    },
375                )?
376                .unwrap();
377            // B publishes after A's evaluation read but before A stages its result.
378            b.put(1, fields(2))?;
379            batch.put(old.key(), old.row())?;
380            Ok(())
381        });
382        assert!(result.is_err());
383        connection.rollback_transaction().unwrap();
384        assert_eq!(a.get(1).unwrap(), Some(fields(2)));
385    }
386}