Skip to main content

uqa_storage_sqlite/vector_index/diskann/
lifecycle.rs

1//
2// Unified Query Algebra
3//
4// Copyright (c) 2023-2026 Cognica, Inc.
5//
6
7//! Native creation and rebuilding use the common builder and the original catalog transaction.
8
9use super::{invalid, RetainedSQLiteDiskANNCanonical, SQLiteDiskANNCanonical};
10use crate::{mvcc::native::NativeRecordFamily as Family, vector_index::native::records};
11use rusqlite::types::ValueRef;
12use uqa_storage::{
13    diskann_index::{
14        build::DiskANNTemporaryBudget,
15        catalog::DiskANNIndexResolver,
16        format::{DiskANNCanonicalOrigin, DiskANNChangeIdentity, DiskANNVectorVersion},
17        DiskANNIndexOptions,
18    },
19    read_control::StorageReadControl,
20    RelationIdentity, StorageBackendError, StorageBackendResult, StorageSavepointId,
21};
22
23impl SQLiteDiskANNCanonical {
24    /// Retire the selected generation in the caller's transaction before its SQL catalog row is removed. Preserve raw vectors for the replacement exact index and fence obsolete writers.
25    pub fn retire_index(
26        &self,
27        index: &RelationIdentity,
28        resolver: &dyn DiskANNIndexResolver,
29        control: &StorageReadControl,
30    ) -> StorageBackendResult<()> {
31        self.structural_change(|| {
32            let source = self.retain_for_index(index, control)?;
33            self.index
34                .conn
35                .with_native_write(|snapshot, batch| {
36                    source.retire_generation(resolver, snapshot, batch, control)?;
37                    let owner = snapshot.ensure_table_owner(&self.index.table, batch)?;
38                    snapshot.coordinate_vector_field(
39                        batch,
40                        owner,
41                        ValueRef::Text(self.index.field.as_bytes()),
42                        true,
43                    )?;
44                    control.check()?;
45                    Ok(())
46                })?
47                .ok_or_else(|| invalid("retirement requires a native record session"))
48        })
49    }
50
51    /// Adopt the complete existing raw field and publish its first generation inside the caller's active catalog transaction. Its private table/index definitions must already exist.
52    pub fn create_index(
53        &self,
54        index: &RelationIdentity,
55        resolver: &dyn DiskANNIndexResolver,
56        options: DiskANNIndexOptions,
57        temporary: &DiskANNTemporaryBudget,
58        control: &StorageReadControl,
59    ) -> StorageBackendResult<()> {
60        self.structural_change(|| {
61            if self
62                .retain_for_index(index, control)?
63                .selected_source(resolver, control)?
64                .is_some()
65            {
66                return Err(invalid("index already has a published generation"));
67            }
68            self.adopt(index, control)?;
69            self.rebuild_index(index, resolver, options, temporary, control)
70        })
71    }
72
73    /// Retain one canonical/catalog view, construct through the common bounded builder, and install its original expected head in this native transaction.
74    pub fn rebuild_index(
75        &self,
76        index: &RelationIdentity,
77        resolver: &dyn DiskANNIndexResolver,
78        options: DiskANNIndexOptions,
79        temporary: &DiskANNTemporaryBudget,
80        control: &StorageReadControl,
81    ) -> StorageBackendResult<()> {
82        self.rebuild_source(
83            self.retain_for_index(index, control)?,
84            resolver,
85            options,
86            temporary,
87            control,
88        )
89    }
90
91    pub(super) fn rebuild_source(
92        &self,
93        source: RetainedSQLiteDiskANNCanonical,
94        resolver: &dyn DiskANNIndexResolver,
95        options: DiskANNIndexOptions,
96        temporary: &DiskANNTemporaryBudget,
97        control: &StorageReadControl,
98    ) -> StorageBackendResult<()> {
99        self.require_transaction()?;
100        if source.index_parameters() != Some(options.parameters) {
101            return Err(invalid("construction parameters differ from the catalog"));
102        }
103        let scope = source.index_scope(resolver, control)?;
104        let repository = self.index.conn.diskann_generations(control)?;
105        repository.initialize(control)?;
106        let mut stage = repository.allocate_bound_stage(&scope, control)?;
107        let coverage = stage.build(source, options, temporary, control)?;
108        let sealed = repository.open_source(stage.generation(), control)?;
109        let _prepared = uqa_storage::diskann_index::DiskANNQuery::open(
110            coverage.source(),
111            sealed,
112            options.parameters,
113            options.read,
114            control,
115        )?;
116        self.index
117            .conn
118            .publish_diskann_generation(&coverage, resolver, control)
119    }
120
121    pub(super) fn clear_index(
122        &self,
123        source: &RetainedSQLiteDiskANNCanonical,
124        index: &RelationIdentity,
125        resolver: &dyn DiskANNIndexResolver,
126        options: DiskANNIndexOptions,
127        temporary: &DiskANNTemporaryBudget,
128        control: &StorageReadControl,
129    ) -> StorageBackendResult<()> {
130        self.structural_change(|| {
131            self.index
132                .conn
133                .with_native_write(|snapshot, batch| {
134                    source.require_current_index(&snapshot.record_read(), batch, control)?;
135                    crate::vector_index::native::NativeVectorRead::new(snapshot, &self.index)?
136                        .clear_family(batch, Family::Vectors)?;
137                    control.check()?;
138                    Ok(())
139                })?
140                .ok_or_else(|| invalid("clear requires a native record session"))?;
141            self.rebuild_index(index, resolver, options, temporary, control)
142        })
143    }
144
145    fn adopt(
146        &self,
147        index: &RelationIdentity,
148        control: &StorageReadControl,
149    ) -> StorageBackendResult<()> {
150        self.index
151            .conn
152            .with_native_versioned_write(|origin, snapshot, batch| {
153                let binding = crate::catalog::DiskANNCatalogBinding::capture(
154                    snapshot,
155                    &self.index.table,
156                    &self.index.field,
157                    self.index.dimensions,
158                    index,
159                    control,
160                )?;
161                binding.require_current(&snapshot.record_read(), batch, control)?;
162                let version = DiskANNVectorVersion::new(origin.transaction(), origin.revision())?;
163                let owner = snapshot.ensure_table_owner(&self.index.table, batch)?;
164                let field = ValueRef::Text(self.index.field.as_bytes());
165                let table = ValueRef::Text(self.index.table.as_bytes());
166                snapshot.coordinate_vector_field(batch, owner, field, true)?;
167                for family in [Family::VectorOrigins, Family::VectorChanges] {
168                    snapshot.delete_prefix(batch, family, owner, &[field])?;
169                }
170                let stamp = |document: u64,
171                             count,
172                             batch: &mut dyn uqa_storage::KeyValueBatch|
173                 -> crate::Result<()> {
174                    let record =
175                        DiskANNCanonicalOrigin::new(version, self.index.dimensions, count)?;
176                    snapshot.put_row(
177                        batch,
178                        Family::VectorOrigins,
179                        owner,
180                        &[
181                            table,
182                            field,
183                            ValueRef::Integer(document as i64),
184                            ValueRef::Blob(&record.encode()),
185                        ],
186                    )?;
187                    snapshot.put_row(
188                        batch,
189                        Family::VectorChanges,
190                        owner,
191                        &[
192                            table,
193                            field,
194                            ValueRef::Blob(&DiskANNChangeIdentity::new(document, version).encode()),
195                            ValueRef::Blob(&record.encode()),
196                        ],
197                    )?;
198                    Ok(())
199                };
200                let mut current = None;
201                let mut count = 0_u64;
202                snapshot.visit_rows(Family::Vectors, Some(owner), &[field], |row| {
203                    control.check()?;
204                    let document = records::unsigned(records::integer(row[2])?)?;
205                    let ordinal = records::ordinal(records::integer(row[3])?)?;
206                    let vector = records::vector(row[4], control)?;
207                    uqa_storage::vector_index::validate_vector_values_controlled(
208                        self.index.dimensions,
209                        &vector,
210                        Some(control),
211                    )?;
212                    if current != Some(document) {
213                        if let Some(previous) = current {
214                            stamp(previous, count, batch)?;
215                        }
216                        current = Some(document);
217                        count = 0;
218                    }
219                    if u64::from(ordinal) != count {
220                        return Err(invalid("canonical ordinal sequence has a gap").into());
221                    }
222                    count = count
223                        .checked_add(1)
224                        .ok_or_else(|| invalid("canonical count overflow"))?;
225                    Ok(())
226                })?;
227                if let Some(document) = current {
228                    stamp(document, count, batch)?;
229                }
230                control.check()?;
231                Ok(())
232            })?;
233        Ok(())
234    }
235
236    fn structural_change(
237        &self,
238        operation: impl FnOnce() -> StorageBackendResult<()>,
239    ) -> StorageBackendResult<()> {
240        self.require_transaction()?;
241        let connection = &self.index.conn;
242        let savepoint = StorageSavepointId::allocate().backend_name();
243        connection.savepoint(&savepoint)?;
244        match operation() {
245            Ok(()) => Ok(connection.release_savepoint(&savepoint)?),
246            Err(error) => {
247                connection.rollback_to_savepoint(&savepoint).and_then(|()| connection.release_savepoint(&savepoint))
248                    .map_err(|rollback| StorageBackendError::Other(format!("DiskANN structural rollback failed: {rollback}; original error: {error}")))?;
249                Err(error)
250            }
251        }
252    }
253
254    fn require_transaction(&self) -> StorageBackendResult<()> {
255        if !self.index.conn.in_transaction() {
256            return Err(invalid(
257                "structural changes require the caller's active catalog transaction",
258            ));
259        }
260        Ok(())
261    }
262}