Skip to main content

uqa_execution/maintenance/
diskann.rs

1//
2// Unified Query Algebra
3//
4// Copyright (c) 2023-2026 Cognica, Inc.
5//
6
7//! One admitted index maintenance pass per database, with finite catalog and change discovery.
8
9mod rebuild;
10#[cfg(test)]
11mod tests;
12mod transaction;
13
14pub use rebuild::{DiskANNMaintenanceCensus, DiskANNRebuildPolicy};
15
16use std::{collections::BTreeMap, ops::Bound, sync::Arc};
17use transaction::{Completed, Job};
18use uqa_core::memory::MemoryReservation;
19use uqa_sql::schema::indexes::vectors::VectorIndexCatalog;
20use uqa_storage::{
21    diskann_index::{build::DiskANNTemporaryBudget, DiskANNIndexBinding, DiskANNIndexOptions},
22    read_control::StorageReadControl,
23    CatalogIndexRow, PersistentStorageBackend, RelationIdentity, StorageBackendError,
24    StorageBackendResult,
25};
26
27#[derive(Clone, Copy, Debug, PartialEq, Eq)]
28pub enum DiskANNMaintenancePhase {
29    Counting,
30    Pruning,
31    Rebuilding,
32    Completing,
33}
34
35/// Census progress is read-only. Evaluated writes increment pruning/rebuild counters only after confirmed completion.
36#[derive(Clone, Debug, Default, PartialEq, Eq)]
37pub struct DiskANNMaintenanceStatus {
38    pub completed_passes: u64,
39    pub examined: u64,
40    pub removed: u64,
41    pub completed_censuses: u64,
42    pub completed_rebuilds: u64,
43    pub last_census: Option<DiskANNMaintenanceCensus>,
44    pub phase: Option<DiskANNMaintenancePhase>,
45    pub pending_completion: bool,
46    pub last_error: Option<String>,
47}
48
49/// The host supplies coalesced wakes; this owner admits one index and counts/prunes one 64-entry page or constructs one admitted generation per step. A retained catalog pass prevents new indexes from extending the current pass indefinitely.
50pub struct DiskANNJournalMaintenance {
51    indexes: Option<Arc<BTreeMap<RelationIdentity, CatalogIndexRow>>>,
52    last_version: Option<u64>,
53    after: Option<RelationIdentity>,
54    name_memory: MemoryReservation,
55    job: Option<Job>,
56    rebuild: Option<rebuild::Configuration>,
57    control: StorageReadControl,
58    status: DiskANNMaintenanceStatus,
59    _memory: MemoryReservation,
60}
61
62impl DiskANNJournalMaintenance {
63    pub fn new(control: &StorageReadControl) -> StorageBackendResult<Self> {
64        control.check()?;
65        Ok(Self {
66            indexes: None,
67            last_version: None,
68            after: None,
69            name_memory: control.memory().empty_reservation(),
70            job: None,
71            rebuild: None,
72            control: control.clone(),
73            status: DiskANNMaintenanceStatus::default(),
74            _memory: control.memory().reserve(size_of::<Self>())?,
75        })
76    }
77
78    /// Enable reconstruction using the supplied shared encrypted-temporary allowance. `new` remains available for standalone journal-only maintenance.
79    pub fn with_rebuilds(
80        control: &StorageReadControl,
81        temporary: &DiskANNTemporaryBudget,
82        policy: DiskANNRebuildPolicy,
83    ) -> StorageBackendResult<Self> {
84        let mut maintenance = Self::new(control)?;
85        maintenance.rebuild = Some(rebuild::Configuration {
86            policy,
87            temporary: temporary.clone(),
88        });
89        Ok(maintenance)
90    }
91
92    /// Applies to later admissions. Changing policy also revisits an otherwise unchanged database; an admitted capture keeps its original policy.
93    pub fn set_rebuild_policy(&mut self, policy: DiskANNRebuildPolicy) -> StorageBackendResult<()> {
94        let configuration = self.rebuild.as_mut().ok_or_else(|| {
95            StorageBackendError::Other(
96                "DiskANN reconstruction was not enabled for this maintenance owner".into(),
97            )
98        })?;
99        if configuration.policy != policy {
100            configuration.policy = policy;
101            self.last_version = None;
102        }
103        Ok(())
104    }
105
106    pub fn status(&self) -> DiskANNMaintenanceStatus {
107        self.status.clone()
108    }
109
110    /// A successful host step may continue immediately while this finite pass has work; failed steps retain the host's normal retry delay.
111    pub fn has_pending_work(&self) -> bool {
112        self.indexes.is_some() || self.job.is_some()
113    }
114
115    /// `version` is a commit generation already incorporated into the supplied catalog snapshot, never a newer independent observation. None disables unchanged-database coalescing. Commits during a finite pass trigger another pass instead of being consumed by its completion.
116    pub fn step(
117        &mut self,
118        indexes: Arc<BTreeMap<RelationIdentity, CatalogIndexRow>>,
119        version: Option<u64>,
120        vectors: &dyn VectorIndexCatalog,
121        backend: &dyn PersistentStorageBackend,
122    ) -> StorageBackendResult<()> {
123        let result = self.advance(indexes, version, vectors, backend);
124        if result.is_err() {
125            self.last_version = None;
126        }
127        self.status.pending_completion = self.job.as_ref().is_some_and(Job::pending);
128        self.status.phase = self.job.as_ref().and_then(Job::phase);
129        self.status.last_error = result.as_ref().err().map(ToString::to_string);
130        result
131    }
132
133    fn advance(
134        &mut self,
135        indexes: Arc<BTreeMap<RelationIdentity, CatalogIndexRow>>,
136        version: Option<u64>,
137        vectors: &dyn VectorIndexCatalog,
138        backend: &dyn PersistentStorageBackend,
139    ) -> StorageBackendResult<()> {
140        // Resolve original receipts even after cancellation; cancellation may forbid new work, never receipt acknowledgement.
141        if let Some(job) = &mut self.job {
142            let result = job.step(&self.control);
143            if let Some(census) = job.take_census() {
144                self.status.completed_censuses = self.status.completed_censuses.saturating_add(1);
145                self.status.last_census = Some(census);
146            }
147            match job.take_completed() {
148                Some(Completed::Rebuilt) => {
149                    self.status.completed_rebuilds =
150                        self.status.completed_rebuilds.saturating_add(1);
151                    self.last_version = None;
152                }
153                Some(Completed::Pruned(committed)) => {
154                    self.status.examined = self
155                        .status
156                        .examined
157                        .saturating_add(committed.examined as u64);
158                    self.status.removed =
159                        self.status.removed.saturating_add(committed.removed as u64);
160                    if committed.next.is_none() {
161                        self.status.completed_passes =
162                            self.status.completed_passes.saturating_add(1);
163                    }
164                }
165                None => {}
166            }
167            if job.finished() {
168                self.job = None;
169            }
170            return result;
171        }
172        self.control.check()?;
173        if self.indexes.is_none() {
174            if version.is_some() && version == self.last_version {
175                return Ok(());
176            }
177            self.last_version = version;
178        }
179        let catalog = self.indexes.get_or_insert(indexes);
180        let bound = self
181            .after
182            .as_ref()
183            .map_or(Bound::Unbounded, Bound::Excluded);
184        let Some((name, row)) = catalog.range((bound, Bound::Unbounded)).next() else {
185            self.indexes = None;
186            self.after = None;
187            self.name_memory = self.control.memory().empty_reservation();
188            return Ok(());
189        };
190        let memory = self.control.memory().reserve(
191            name.schema
192                .len()
193                .checked_add(name.name.len())
194                .ok_or(uqa_core::memory::MemoryError::SizeOverflow)?,
195        )?;
196        self.after = Some(name.clone());
197        self.name_memory = memory;
198        if !row.index_type.eq_ignore_ascii_case("diskann") {
199            return Ok(());
200        }
201        self.job = Some(admit(
202            row,
203            vectors,
204            backend,
205            &self.control,
206            self.rebuild.as_ref(),
207        )?);
208        Ok(())
209    }
210}
211
212fn admit(
213    row: &CatalogIndexRow,
214    vectors: &dyn VectorIndexCatalog,
215    backend: &dyn PersistentStorageBackend,
216    control: &StorageReadControl,
217    rebuild: Option<&rebuild::Configuration>,
218) -> StorageBackendResult<Job> {
219    let bytes = [
220        row.columns_json.len(),
221        row.parameters_json.len(),
222        row.definition_json.as_ref().map_or(0, String::len),
223    ]
224    .into_iter()
225    .try_fold(0_usize, usize::checked_add)
226    .and_then(|bytes| bytes.checked_mul(size_of::<uqa_sql::ast::Expr>() * 2))
227    .ok_or(uqa_core::memory::MemoryError::SizeOverflow)?;
228    let _decoding = control.memory().reserve(bytes)?;
229    let (field, dimensions, parameters) = crate::schema::indexes::diskann::target(vectors, row)?;
230    let session = backend.open_controlled_session(control)?;
231    session.validate_transaction_affinity()?;
232    if session.backend.in_transaction()
233        || session.backend.transaction_model() != backend.transaction_model()
234        || session.backend.transaction_affinity() == backend.transaction_affinity()
235        || session
236            .backend
237            .retention_control()
238            .is_none_or(|retained| !retained.memory().shares_allowance(control.memory()))
239        || session
240            .backend
241            .write_cancellation()
242            .is_none_or(|cancellation| !cancellation.shares_signal(control.cancellation()))
243    {
244        return Err(StorageBackendError::Other(
245            "DiskANN maintenance requires an independent inactive session with the same database, allowance and write cancellation"
246                .into(),
247        ));
248    }
249    let binding = DiskANNIndexBinding {
250        table: &row.table_name,
251        field: &field,
252        dimensions,
253        index: &row.relation,
254        resolver: Arc::new(crate::catalog::index::diskann::DiskANNIndexIdentityResolver),
255        control,
256    };
257    let options = DiskANNIndexOptions::for_parameters(parameters);
258    if let Some(configuration) = rebuild {
259        let source = session
260            .backend
261            .diskann_maintenance_source(binding, options.read.max_record_bytes)?;
262        Ok(Job::with_rebuild(
263            session.backend,
264            source,
265            options,
266            configuration.clone(),
267        ))
268    } else {
269        let pruner = session
270            .backend
271            .diskann_journal_pruner(binding, options.read.max_record_bytes)?;
272        Ok(Job::new(session.backend, pruner))
273    }
274}