Skip to main content

uqa_storage_sqlite/vector_index/
diskann.rs

1//
2// Unified Query Algebra
3//
4// Copyright (c) 2023-2026 Cognica, Inc.
5//
6
7//! Native canonical tensors and origins share one evaluated publishing mutation.
8
9mod lifecycle;
10mod live;
11mod maintenance;
12mod retained;
13#[cfg(test)]
14mod tests;
15
16pub use live::SQLiteDiskANNHandle;
17pub use retained::RetainedSQLiteDiskANNCanonical;
18
19use rusqlite::types::ValueRef;
20use uqa_core::{memory::BudgetedVec, DocId};
21use uqa_storage::{
22    diskann_index::format::{DiskANNCanonicalOrigin, DiskANNChangeIdentity, DiskANNVectorVersion},
23    mvcc::VersionError,
24    read_control::StorageReadControl,
25    vector_index::validate_vector_values_controlled,
26    StorageBackendResult,
27};
28
29use super::{encode_doc_id, SQLiteVectorIndex};
30use crate::{
31    mvcc::native::{NativeRecord, NativeRecordFamily as Family},
32    ManagedConnection,
33};
34
35/// Canonical tensor owner for a bound native `SQLite` session. Raw coordinates stay in `_vectors`; this does not enable a public `DiskANN` index or publish a generation.
36pub struct SQLiteDiskANNCanonical {
37    index: SQLiteVectorIndex,
38}
39
40impl SQLiteDiskANNCanonical {
41    pub fn new(
42        connection: ManagedConnection,
43        table: impl Into<String>,
44        field: impl Into<String>,
45        dimensions: u32,
46    ) -> StorageBackendResult<Self> {
47        if dimensions == 0 || !connection.is_native_record_session() {
48            return Err(invalid(
49                "canonical origins require dimensions and native records",
50            ));
51        }
52        Ok(Self {
53            index: SQLiteVectorIndex::new(connection, table, field, dimensions),
54        })
55    }
56
57    /// Replace every ordinal and the complete tensor's publishing origin in one batch, including explicit empty replacements. The callback is never replayed after a publication failure.
58    pub fn replace(
59        &self,
60        document: DocId,
61        vectors: &[Vec<f32>],
62        control: &StorageReadControl,
63    ) -> StorageBackendResult<DiskANNVectorVersion> {
64        self.replace_guarded(document, vectors, control, |_, _| Ok(()))
65    }
66
67    fn replace_guarded(
68        &self,
69        document: DocId,
70        vectors: &[Vec<f32>],
71        control: &StorageReadControl,
72        guard: impl FnOnce(
73            &crate::mvcc::native::NativeSnapshot,
74            &mut dyn uqa_storage::KeyValueBatch,
75        ) -> StorageBackendResult<()>,
76    ) -> StorageBackendResult<DiskANNVectorVersion> {
77        control.check()?;
78        let document = encode_doc_id(document)?;
79        let count =
80            u64::try_from(vectors.len()).map_err(|_| invalid("canonical count overflow"))?;
81        super::validate_vector_ordinal_count(count)?;
82        for vector in vectors {
83            validate_vector_values_controlled(self.index.dimensions, vector, Some(control))?;
84        }
85        Ok(self
86            .index
87            .conn
88            .with_native_versioned_write(|origin, snapshot, batch| {
89                control.check()?;
90                guard(snapshot, batch)?;
91                let owner = snapshot.ensure_table_owner(&self.index.table, batch)?;
92                let field = ValueRef::Text(self.index.field.as_bytes());
93                snapshot.coordinate_vector_field(batch, owner, field, false)?;
94                let version = DiskANNVectorVersion::new(origin.transaction(), origin.revision())?;
95                let record = DiskANNCanonicalOrigin::new(version, self.index.dimensions, count)?;
96                snapshot.delete_prefix(
97                    batch,
98                    Family::Vectors,
99                    owner,
100                    &[field, ValueRef::Integer(document)],
101                )?;
102                let mut bytes = BudgetedVec::new(control.memory());
103                for (ordinal, vector) in vectors.iter().enumerate() {
104                    control.check()?;
105                    bytes.clear();
106                    for chunk in vector.chunks(1024) {
107                        control.check()?;
108                        for coordinate in chunk {
109                            bytes.extend_from_slice(&coordinate.to_le_bytes())?;
110                        }
111                    }
112                    snapshot.put_row(
113                        batch,
114                        Family::Vectors,
115                        owner,
116                        &[
117                            ValueRef::Text(self.index.table.as_bytes()),
118                            field,
119                            ValueRef::Integer(document),
120                            ValueRef::Integer(ordinal as i64),
121                            ValueRef::Blob(&bytes),
122                        ],
123                    )?;
124                }
125                let origin = NativeRecord::encode(
126                    Family::VectorOrigins,
127                    owner,
128                    &[
129                        ValueRef::Text(self.index.table.as_bytes()),
130                        field,
131                        ValueRef::Integer(document),
132                        ValueRef::Blob(&record.encode()),
133                    ],
134                    control,
135                )?;
136                batch.replace_diskann_origin(origin.key(), origin.row())?;
137                snapshot.put_row(
138                    batch,
139                    Family::VectorChanges,
140                    owner,
141                    &[
142                        ValueRef::Text(self.index.table.as_bytes()),
143                        field,
144                        ValueRef::Blob(
145                            &DiskANNChangeIdentity::new(document as DocId, version).encode(),
146                        ),
147                        ValueRef::Blob(&record.encode()),
148                    ],
149                )?;
150                control.check()?;
151                Ok(version)
152            })?)
153    }
154
155    /// Retain the exact private/committed source and table incarnation without reading the vector corpus.
156    pub fn retain(
157        &self,
158        control: &StorageReadControl,
159    ) -> StorageBackendResult<RetainedSQLiteDiskANNCanonical> {
160        RetainedSQLiteDiskANNCanonical::capture(&self.index, control)
161    }
162
163    /// Capture the actual native table-name owner, table definition and index definition on the canonical source's fixed view.
164    pub fn retain_for_index(
165        &self,
166        index: &uqa_storage::RelationIdentity,
167        control: &StorageReadControl,
168    ) -> StorageBackendResult<RetainedSQLiteDiskANNCanonical> {
169        RetainedSQLiteDiskANNCanonical::capture_for_index(&self.index, index, control)
170    }
171}
172
173fn invalid(message: &'static str) -> uqa_storage::StorageBackendError {
174    VersionError::InvalidEncoding(message).into_storage_error()
175}