uqa_storage_sqlite/vector_index/
diskann.rs1mod lifecycle;
10mod live;
11mod maintenance;
12mod retained;
13#[cfg(test)]
14mod tests;
15
16pub use live::SQLiteDiskANNHandle;
17pub use retained::RetainedSQLiteDiskANNCanonical;
18
19use rusqlite::types::ValueRef;
20use uqa_core::{memory::BudgetedVec, DocId};
21use uqa_storage::{
22 diskann_index::format::{DiskANNCanonicalOrigin, DiskANNChangeIdentity, DiskANNVectorVersion},
23 mvcc::VersionError,
24 read_control::StorageReadControl,
25 vector_index::validate_vector_values_controlled,
26 StorageBackendResult,
27};
28
29use super::{encode_doc_id, SQLiteVectorIndex};
30use crate::{
31 mvcc::native::{NativeRecord, NativeRecordFamily as Family},
32 ManagedConnection,
33};
34
35pub struct SQLiteDiskANNCanonical {
37 index: SQLiteVectorIndex,
38}
39
40impl SQLiteDiskANNCanonical {
41 pub fn new(
42 connection: ManagedConnection,
43 table: impl Into<String>,
44 field: impl Into<String>,
45 dimensions: u32,
46 ) -> StorageBackendResult<Self> {
47 if dimensions == 0 || !connection.is_native_record_session() {
48 return Err(invalid(
49 "canonical origins require dimensions and native records",
50 ));
51 }
52 Ok(Self {
53 index: SQLiteVectorIndex::new(connection, table, field, dimensions),
54 })
55 }
56
57 pub fn replace(
59 &self,
60 document: DocId,
61 vectors: &[Vec<f32>],
62 control: &StorageReadControl,
63 ) -> StorageBackendResult<DiskANNVectorVersion> {
64 self.replace_guarded(document, vectors, control, |_, _| Ok(()))
65 }
66
67 fn replace_guarded(
68 &self,
69 document: DocId,
70 vectors: &[Vec<f32>],
71 control: &StorageReadControl,
72 guard: impl FnOnce(
73 &crate::mvcc::native::NativeSnapshot,
74 &mut dyn uqa_storage::KeyValueBatch,
75 ) -> StorageBackendResult<()>,
76 ) -> StorageBackendResult<DiskANNVectorVersion> {
77 control.check()?;
78 let document = encode_doc_id(document)?;
79 let count =
80 u64::try_from(vectors.len()).map_err(|_| invalid("canonical count overflow"))?;
81 super::validate_vector_ordinal_count(count)?;
82 for vector in vectors {
83 validate_vector_values_controlled(self.index.dimensions, vector, Some(control))?;
84 }
85 Ok(self
86 .index
87 .conn
88 .with_native_versioned_write(|origin, snapshot, batch| {
89 control.check()?;
90 guard(snapshot, batch)?;
91 let owner = snapshot.ensure_table_owner(&self.index.table, batch)?;
92 let field = ValueRef::Text(self.index.field.as_bytes());
93 snapshot.coordinate_vector_field(batch, owner, field, false)?;
94 let version = DiskANNVectorVersion::new(origin.transaction(), origin.revision())?;
95 let record = DiskANNCanonicalOrigin::new(version, self.index.dimensions, count)?;
96 snapshot.delete_prefix(
97 batch,
98 Family::Vectors,
99 owner,
100 &[field, ValueRef::Integer(document)],
101 )?;
102 let mut bytes = BudgetedVec::new(control.memory());
103 for (ordinal, vector) in vectors.iter().enumerate() {
104 control.check()?;
105 bytes.clear();
106 for chunk in vector.chunks(1024) {
107 control.check()?;
108 for coordinate in chunk {
109 bytes.extend_from_slice(&coordinate.to_le_bytes())?;
110 }
111 }
112 snapshot.put_row(
113 batch,
114 Family::Vectors,
115 owner,
116 &[
117 ValueRef::Text(self.index.table.as_bytes()),
118 field,
119 ValueRef::Integer(document),
120 ValueRef::Integer(ordinal as i64),
121 ValueRef::Blob(&bytes),
122 ],
123 )?;
124 }
125 let origin = NativeRecord::encode(
126 Family::VectorOrigins,
127 owner,
128 &[
129 ValueRef::Text(self.index.table.as_bytes()),
130 field,
131 ValueRef::Integer(document),
132 ValueRef::Blob(&record.encode()),
133 ],
134 control,
135 )?;
136 batch.replace_diskann_origin(origin.key(), origin.row())?;
137 snapshot.put_row(
138 batch,
139 Family::VectorChanges,
140 owner,
141 &[
142 ValueRef::Text(self.index.table.as_bytes()),
143 field,
144 ValueRef::Blob(
145 &DiskANNChangeIdentity::new(document as DocId, version).encode(),
146 ),
147 ValueRef::Blob(&record.encode()),
148 ],
149 )?;
150 control.check()?;
151 Ok(version)
152 })?)
153 }
154
155 pub fn retain(
157 &self,
158 control: &StorageReadControl,
159 ) -> StorageBackendResult<RetainedSQLiteDiskANNCanonical> {
160 RetainedSQLiteDiskANNCanonical::capture(&self.index, control)
161 }
162
163 pub fn retain_for_index(
165 &self,
166 index: &uqa_storage::RelationIdentity,
167 control: &StorageReadControl,
168 ) -> StorageBackendResult<RetainedSQLiteDiskANNCanonical> {
169 RetainedSQLiteDiskANNCanonical::capture_for_index(&self.index, index, control)
170 }
171}
172
173fn invalid(message: &'static str) -> uqa_storage::StorageBackendError {
174 VersionError::InvalidEncoding(message).into_storage_error()
175}