uqa_execution/maintenance/
diskann.rs1mod rebuild;
10#[cfg(test)]
11mod tests;
12mod transaction;
13
14pub use rebuild::{DiskANNMaintenanceCensus, DiskANNRebuildPolicy};
15
16use std::{collections::BTreeMap, ops::Bound, sync::Arc};
17use transaction::{Completed, Job};
18use uqa_core::memory::MemoryReservation;
19use uqa_sql::schema::indexes::vectors::VectorIndexCatalog;
20use uqa_storage::{
21 diskann_index::{build::DiskANNTemporaryBudget, DiskANNIndexBinding, DiskANNIndexOptions},
22 read_control::StorageReadControl,
23 CatalogIndexRow, PersistentStorageBackend, RelationIdentity, StorageBackendError,
24 StorageBackendResult,
25};
26
27#[derive(Clone, Copy, Debug, PartialEq, Eq)]
28pub enum DiskANNMaintenancePhase {
29 Counting,
30 Pruning,
31 Rebuilding,
32 Completing,
33}
34
35#[derive(Clone, Debug, Default, PartialEq, Eq)]
37pub struct DiskANNMaintenanceStatus {
38 pub completed_passes: u64,
39 pub examined: u64,
40 pub removed: u64,
41 pub completed_censuses: u64,
42 pub completed_rebuilds: u64,
43 pub last_census: Option<DiskANNMaintenanceCensus>,
44 pub phase: Option<DiskANNMaintenancePhase>,
45 pub pending_completion: bool,
46 pub last_error: Option<String>,
47}
48
49pub struct DiskANNJournalMaintenance {
51 indexes: Option<Arc<BTreeMap<RelationIdentity, CatalogIndexRow>>>,
52 last_version: Option<u64>,
53 after: Option<RelationIdentity>,
54 name_memory: MemoryReservation,
55 job: Option<Job>,
56 rebuild: Option<rebuild::Configuration>,
57 control: StorageReadControl,
58 status: DiskANNMaintenanceStatus,
59 _memory: MemoryReservation,
60}
61
62impl DiskANNJournalMaintenance {
63 pub fn new(control: &StorageReadControl) -> StorageBackendResult<Self> {
64 control.check()?;
65 Ok(Self {
66 indexes: None,
67 last_version: None,
68 after: None,
69 name_memory: control.memory().empty_reservation(),
70 job: None,
71 rebuild: None,
72 control: control.clone(),
73 status: DiskANNMaintenanceStatus::default(),
74 _memory: control.memory().reserve(size_of::<Self>())?,
75 })
76 }
77
78 pub fn with_rebuilds(
80 control: &StorageReadControl,
81 temporary: &DiskANNTemporaryBudget,
82 policy: DiskANNRebuildPolicy,
83 ) -> StorageBackendResult<Self> {
84 let mut maintenance = Self::new(control)?;
85 maintenance.rebuild = Some(rebuild::Configuration {
86 policy,
87 temporary: temporary.clone(),
88 });
89 Ok(maintenance)
90 }
91
92 pub fn set_rebuild_policy(&mut self, policy: DiskANNRebuildPolicy) -> StorageBackendResult<()> {
94 let configuration = self.rebuild.as_mut().ok_or_else(|| {
95 StorageBackendError::Other(
96 "DiskANN reconstruction was not enabled for this maintenance owner".into(),
97 )
98 })?;
99 if configuration.policy != policy {
100 configuration.policy = policy;
101 self.last_version = None;
102 }
103 Ok(())
104 }
105
106 pub fn status(&self) -> DiskANNMaintenanceStatus {
107 self.status.clone()
108 }
109
110 pub fn has_pending_work(&self) -> bool {
112 self.indexes.is_some() || self.job.is_some()
113 }
114
115 pub fn step(
117 &mut self,
118 indexes: Arc<BTreeMap<RelationIdentity, CatalogIndexRow>>,
119 version: Option<u64>,
120 vectors: &dyn VectorIndexCatalog,
121 backend: &dyn PersistentStorageBackend,
122 ) -> StorageBackendResult<()> {
123 let result = self.advance(indexes, version, vectors, backend);
124 if result.is_err() {
125 self.last_version = None;
126 }
127 self.status.pending_completion = self.job.as_ref().is_some_and(Job::pending);
128 self.status.phase = self.job.as_ref().and_then(Job::phase);
129 self.status.last_error = result.as_ref().err().map(ToString::to_string);
130 result
131 }
132
133 fn advance(
134 &mut self,
135 indexes: Arc<BTreeMap<RelationIdentity, CatalogIndexRow>>,
136 version: Option<u64>,
137 vectors: &dyn VectorIndexCatalog,
138 backend: &dyn PersistentStorageBackend,
139 ) -> StorageBackendResult<()> {
140 if let Some(job) = &mut self.job {
142 let result = job.step(&self.control);
143 if let Some(census) = job.take_census() {
144 self.status.completed_censuses = self.status.completed_censuses.saturating_add(1);
145 self.status.last_census = Some(census);
146 }
147 match job.take_completed() {
148 Some(Completed::Rebuilt) => {
149 self.status.completed_rebuilds =
150 self.status.completed_rebuilds.saturating_add(1);
151 self.last_version = None;
152 }
153 Some(Completed::Pruned(committed)) => {
154 self.status.examined = self
155 .status
156 .examined
157 .saturating_add(committed.examined as u64);
158 self.status.removed =
159 self.status.removed.saturating_add(committed.removed as u64);
160 if committed.next.is_none() {
161 self.status.completed_passes =
162 self.status.completed_passes.saturating_add(1);
163 }
164 }
165 None => {}
166 }
167 if job.finished() {
168 self.job = None;
169 }
170 return result;
171 }
172 self.control.check()?;
173 if self.indexes.is_none() {
174 if version.is_some() && version == self.last_version {
175 return Ok(());
176 }
177 self.last_version = version;
178 }
179 let catalog = self.indexes.get_or_insert(indexes);
180 let bound = self
181 .after
182 .as_ref()
183 .map_or(Bound::Unbounded, Bound::Excluded);
184 let Some((name, row)) = catalog.range((bound, Bound::Unbounded)).next() else {
185 self.indexes = None;
186 self.after = None;
187 self.name_memory = self.control.memory().empty_reservation();
188 return Ok(());
189 };
190 let memory = self.control.memory().reserve(
191 name.schema
192 .len()
193 .checked_add(name.name.len())
194 .ok_or(uqa_core::memory::MemoryError::SizeOverflow)?,
195 )?;
196 self.after = Some(name.clone());
197 self.name_memory = memory;
198 if !row.index_type.eq_ignore_ascii_case("diskann") {
199 return Ok(());
200 }
201 self.job = Some(admit(
202 row,
203 vectors,
204 backend,
205 &self.control,
206 self.rebuild.as_ref(),
207 )?);
208 Ok(())
209 }
210}
211
212fn admit(
213 row: &CatalogIndexRow,
214 vectors: &dyn VectorIndexCatalog,
215 backend: &dyn PersistentStorageBackend,
216 control: &StorageReadControl,
217 rebuild: Option<&rebuild::Configuration>,
218) -> StorageBackendResult<Job> {
219 let bytes = [
220 row.columns_json.len(),
221 row.parameters_json.len(),
222 row.definition_json.as_ref().map_or(0, String::len),
223 ]
224 .into_iter()
225 .try_fold(0_usize, usize::checked_add)
226 .and_then(|bytes| bytes.checked_mul(size_of::<uqa_sql::ast::Expr>() * 2))
227 .ok_or(uqa_core::memory::MemoryError::SizeOverflow)?;
228 let _decoding = control.memory().reserve(bytes)?;
229 let (field, dimensions, parameters) = crate::schema::indexes::diskann::target(vectors, row)?;
230 let session = backend.open_controlled_session(control)?;
231 session.validate_transaction_affinity()?;
232 if session.backend.in_transaction()
233 || session.backend.transaction_model() != backend.transaction_model()
234 || session.backend.transaction_affinity() == backend.transaction_affinity()
235 || session
236 .backend
237 .retention_control()
238 .is_none_or(|retained| !retained.memory().shares_allowance(control.memory()))
239 || session
240 .backend
241 .write_cancellation()
242 .is_none_or(|cancellation| !cancellation.shares_signal(control.cancellation()))
243 {
244 return Err(StorageBackendError::Other(
245 "DiskANN maintenance requires an independent inactive session with the same database, allowance and write cancellation"
246 .into(),
247 ));
248 }
249 let binding = DiskANNIndexBinding {
250 table: &row.table_name,
251 field: &field,
252 dimensions,
253 index: &row.relation,
254 resolver: Arc::new(crate::catalog::index::diskann::DiskANNIndexIdentityResolver),
255 control,
256 };
257 let options = DiskANNIndexOptions::for_parameters(parameters);
258 if let Some(configuration) = rebuild {
259 let source = session
260 .backend
261 .diskann_maintenance_source(binding, options.read.max_record_bytes)?;
262 Ok(Job::with_rebuild(
263 session.backend,
264 source,
265 options,
266 configuration.clone(),
267 ))
268 } else {
269 let pruner = session
270 .backend
271 .diskann_journal_pruner(binding, options.read.max_record_bytes)?;
272 Ok(Job::new(session.backend, pruner))
273 }
274}