uqa_storage_sqlite/vector_index/diskann/
lifecycle.rs1use super::{invalid, RetainedSQLiteDiskANNCanonical, SQLiteDiskANNCanonical};
10use crate::{mvcc::native::NativeRecordFamily as Family, vector_index::native::records};
11use rusqlite::types::ValueRef;
12use uqa_storage::{
13 diskann_index::{
14 build::DiskANNTemporaryBudget,
15 catalog::DiskANNIndexResolver,
16 format::{DiskANNCanonicalOrigin, DiskANNChangeIdentity, DiskANNVectorVersion},
17 DiskANNIndexOptions,
18 },
19 read_control::StorageReadControl,
20 RelationIdentity, StorageBackendError, StorageBackendResult, StorageSavepointId,
21};
22
23impl SQLiteDiskANNCanonical {
24 pub fn retire_index(
26 &self,
27 index: &RelationIdentity,
28 resolver: &dyn DiskANNIndexResolver,
29 control: &StorageReadControl,
30 ) -> StorageBackendResult<()> {
31 self.structural_change(|| {
32 let source = self.retain_for_index(index, control)?;
33 self.index
34 .conn
35 .with_native_write(|snapshot, batch| {
36 source.retire_generation(resolver, snapshot, batch, control)?;
37 let owner = snapshot.ensure_table_owner(&self.index.table, batch)?;
38 snapshot.coordinate_vector_field(
39 batch,
40 owner,
41 ValueRef::Text(self.index.field.as_bytes()),
42 true,
43 )?;
44 control.check()?;
45 Ok(())
46 })?
47 .ok_or_else(|| invalid("retirement requires a native record session"))
48 })
49 }
50
51 pub fn create_index(
53 &self,
54 index: &RelationIdentity,
55 resolver: &dyn DiskANNIndexResolver,
56 options: DiskANNIndexOptions,
57 temporary: &DiskANNTemporaryBudget,
58 control: &StorageReadControl,
59 ) -> StorageBackendResult<()> {
60 self.structural_change(|| {
61 if self
62 .retain_for_index(index, control)?
63 .selected_source(resolver, control)?
64 .is_some()
65 {
66 return Err(invalid("index already has a published generation"));
67 }
68 self.adopt(index, control)?;
69 self.rebuild_index(index, resolver, options, temporary, control)
70 })
71 }
72
73 pub fn rebuild_index(
75 &self,
76 index: &RelationIdentity,
77 resolver: &dyn DiskANNIndexResolver,
78 options: DiskANNIndexOptions,
79 temporary: &DiskANNTemporaryBudget,
80 control: &StorageReadControl,
81 ) -> StorageBackendResult<()> {
82 self.rebuild_source(
83 self.retain_for_index(index, control)?,
84 resolver,
85 options,
86 temporary,
87 control,
88 )
89 }
90
91 pub(super) fn rebuild_source(
92 &self,
93 source: RetainedSQLiteDiskANNCanonical,
94 resolver: &dyn DiskANNIndexResolver,
95 options: DiskANNIndexOptions,
96 temporary: &DiskANNTemporaryBudget,
97 control: &StorageReadControl,
98 ) -> StorageBackendResult<()> {
99 self.require_transaction()?;
100 if source.index_parameters() != Some(options.parameters) {
101 return Err(invalid("construction parameters differ from the catalog"));
102 }
103 let scope = source.index_scope(resolver, control)?;
104 let repository = self.index.conn.diskann_generations(control)?;
105 repository.initialize(control)?;
106 let mut stage = repository.allocate_bound_stage(&scope, control)?;
107 let coverage = stage.build(source, options, temporary, control)?;
108 let sealed = repository.open_source(stage.generation(), control)?;
109 let _prepared = uqa_storage::diskann_index::DiskANNQuery::open(
110 coverage.source(),
111 sealed,
112 options.parameters,
113 options.read,
114 control,
115 )?;
116 self.index
117 .conn
118 .publish_diskann_generation(&coverage, resolver, control)
119 }
120
121 pub(super) fn clear_index(
122 &self,
123 source: &RetainedSQLiteDiskANNCanonical,
124 index: &RelationIdentity,
125 resolver: &dyn DiskANNIndexResolver,
126 options: DiskANNIndexOptions,
127 temporary: &DiskANNTemporaryBudget,
128 control: &StorageReadControl,
129 ) -> StorageBackendResult<()> {
130 self.structural_change(|| {
131 self.index
132 .conn
133 .with_native_write(|snapshot, batch| {
134 source.require_current_index(&snapshot.record_read(), batch, control)?;
135 crate::vector_index::native::NativeVectorRead::new(snapshot, &self.index)?
136 .clear_family(batch, Family::Vectors)?;
137 control.check()?;
138 Ok(())
139 })?
140 .ok_or_else(|| invalid("clear requires a native record session"))?;
141 self.rebuild_index(index, resolver, options, temporary, control)
142 })
143 }
144
145 fn adopt(
146 &self,
147 index: &RelationIdentity,
148 control: &StorageReadControl,
149 ) -> StorageBackendResult<()> {
150 self.index
151 .conn
152 .with_native_versioned_write(|origin, snapshot, batch| {
153 let binding = crate::catalog::DiskANNCatalogBinding::capture(
154 snapshot,
155 &self.index.table,
156 &self.index.field,
157 self.index.dimensions,
158 index,
159 control,
160 )?;
161 binding.require_current(&snapshot.record_read(), batch, control)?;
162 let version = DiskANNVectorVersion::new(origin.transaction(), origin.revision())?;
163 let owner = snapshot.ensure_table_owner(&self.index.table, batch)?;
164 let field = ValueRef::Text(self.index.field.as_bytes());
165 let table = ValueRef::Text(self.index.table.as_bytes());
166 snapshot.coordinate_vector_field(batch, owner, field, true)?;
167 for family in [Family::VectorOrigins, Family::VectorChanges] {
168 snapshot.delete_prefix(batch, family, owner, &[field])?;
169 }
170 let stamp = |document: u64,
171 count,
172 batch: &mut dyn uqa_storage::KeyValueBatch|
173 -> crate::Result<()> {
174 let record =
175 DiskANNCanonicalOrigin::new(version, self.index.dimensions, count)?;
176 snapshot.put_row(
177 batch,
178 Family::VectorOrigins,
179 owner,
180 &[
181 table,
182 field,
183 ValueRef::Integer(document as i64),
184 ValueRef::Blob(&record.encode()),
185 ],
186 )?;
187 snapshot.put_row(
188 batch,
189 Family::VectorChanges,
190 owner,
191 &[
192 table,
193 field,
194 ValueRef::Blob(&DiskANNChangeIdentity::new(document, version).encode()),
195 ValueRef::Blob(&record.encode()),
196 ],
197 )?;
198 Ok(())
199 };
200 let mut current = None;
201 let mut count = 0_u64;
202 snapshot.visit_rows(Family::Vectors, Some(owner), &[field], |row| {
203 control.check()?;
204 let document = records::unsigned(records::integer(row[2])?)?;
205 let ordinal = records::ordinal(records::integer(row[3])?)?;
206 let vector = records::vector(row[4], control)?;
207 uqa_storage::vector_index::validate_vector_values_controlled(
208 self.index.dimensions,
209 &vector,
210 Some(control),
211 )?;
212 if current != Some(document) {
213 if let Some(previous) = current {
214 stamp(previous, count, batch)?;
215 }
216 current = Some(document);
217 count = 0;
218 }
219 if u64::from(ordinal) != count {
220 return Err(invalid("canonical ordinal sequence has a gap").into());
221 }
222 count = count
223 .checked_add(1)
224 .ok_or_else(|| invalid("canonical count overflow"))?;
225 Ok(())
226 })?;
227 if let Some(document) = current {
228 stamp(document, count, batch)?;
229 }
230 control.check()?;
231 Ok(())
232 })?;
233 Ok(())
234 }
235
236 fn structural_change(
237 &self,
238 operation: impl FnOnce() -> StorageBackendResult<()>,
239 ) -> StorageBackendResult<()> {
240 self.require_transaction()?;
241 let connection = &self.index.conn;
242 let savepoint = StorageSavepointId::allocate().backend_name();
243 connection.savepoint(&savepoint)?;
244 match operation() {
245 Ok(()) => Ok(connection.release_savepoint(&savepoint)?),
246 Err(error) => {
247 connection.rollback_to_savepoint(&savepoint).and_then(|()| connection.release_savepoint(&savepoint))
248 .map_err(|rollback| StorageBackendError::Other(format!("DiskANN structural rollback failed: {rollback}; original error: {error}")))?;
249 Err(error)
250 }
251 }
252 }
253
254 fn require_transaction(&self) -> StorageBackendResult<()> {
255 if !self.index.conn.in_transaction() {
256 return Err(invalid(
257 "structural changes require the caller's active catalog transaction",
258 ));
259 }
260 Ok(())
261 }
262}