Skip to main content

uqa_execution/query/document_changes/
vectors.rs

1//
2// Unified Query Algebra
3//
4// Copyright (c) 2023-2026 Cognica, Inc.
5//
6
7//! Private rows retain the physical canonical sources captured with their evaluated versions.
8
9use super::{
10    Arc, Change, DocId, DocumentChanges, DocumentSelection, DocumentStore, StorageBackendResult,
11    StorageReadControl,
12};
13use uqa_core::memory::{Budgeted, BudgetedVec, MemoryReservation};
14use uqa_sql::ast::ColumnDef;
15use uqa_storage::{
16    diskann_index::{DiskANNReadChanges, DiskANNReadSnapshot},
17    vector_index::{
18        RetainedVectorIndexesBuilder, SelectedVectorRead, VectorIndexSource, VectorReadSnapshot,
19    },
20};
21
22struct VectorSource {
23    field: String,
24    column: Option<[u8; 16]>,
25    source: Option<DiskANNReadSnapshot>,
26    values: VectorReadSnapshot,
27    _memory: MemoryReservation,
28}
29
30/// The vector sources the indexes of a relation hold now, beside its rows `documents`; `None` when no index holds vectors.
31fn capture_vectors(
32    documents: &Arc<dyn DocumentStore>,
33    columns: &[ColumnDef],
34    indexes: &dyn VectorIndexSource,
35    control: &StorageReadControl,
36) -> StorageBackendResult<Option<Arc<Budgeted<CapturedRows>>>> {
37    let mut vectors = BudgetedVec::new(control.memory());
38    indexes.visit(&mut |field, index| {
39        let source = index.diskann_read_snapshot(control)?;
40        let values = if let Some(source) = &source {
41            Some(
42                Budgeted::new(source.clone(), control.memory().empty_reservation()).into_shared()?
43                    as VectorReadSnapshot,
44            )
45        } else {
46            index.vector_read_snapshot(control)?
47        };
48        let Some(values) = values else {
49            return Ok(());
50        };
51        vectors.reserve(1)?;
52        let column = columns
53            .iter()
54            .find(|column| column.name == field)
55            .and_then(|column| column.object_id);
56        let (field, memory) = RetainedVectorIndexesBuilder::copy_field(field, control)?;
57        vectors.push(VectorSource {
58            field,
59            column,
60            source,
61            values,
62            _memory: memory,
63        })?;
64        Ok(())
65    })?;
66    if vectors.is_empty() {
67        return Ok(None);
68    }
69    Ok(Some(
70        Budgeted::new(
71            CapturedRows {
72                documents: Arc::clone(documents),
73                vectors,
74            },
75            control.memory().empty_reservation(),
76        )
77        .into_shared()?,
78    ))
79}
80
81pub(super) struct CapturedRows {
82    pub(super) documents: Arc<dyn DocumentStore>,
83    vectors: BudgetedVec<VectorSource>,
84}
85
86impl CapturedRows {
87    fn vector(&self, field: &str, column: Option<&ColumnDef>) -> Option<&VectorSource> {
88        let identity = column.and_then(|column| column.object_id);
89        self.vectors
90            .iter()
91            .find(|source| match (source.column, identity) {
92                (Some(source), Some(target)) => source == target,
93                _ => source.field == field,
94            })
95    }
96}
97
98impl DocumentChanges {
99    /// Capture immutable rows and their actual canonical vector sources at the same selected table boundary. Each retained row, including a tombstone, keeps the source that supplied that version across later captures, schema renames and transaction completion. Raw sources carry values independently of physical mutation provenance.
100    pub fn with_retained_vectors(
101        mut self,
102        documents: Arc<dyn DocumentStore>,
103        desired: DocumentSelection,
104        columns: &[ColumnDef],
105        indexes: &dyn VectorIndexSource,
106        control: &StorageReadControl,
107    ) -> StorageBackendResult<Self> {
108        control.check()?;
109        let Some(source) = capture_vectors(&documents, columns, indexes, control)? else {
110            return self.with_retained(documents, desired, control);
111        };
112        let desired = desired.finish(control)?;
113        let mut newer = Self::default();
114        for (id, present) in desired.entries() {
115            control.check()?;
116            let present = present && source.documents.contains_doc_id(id)?;
117            newer.insert(id, Change::Captured(source.clone(), present), control)?;
118        }
119        self.extend(newer, control)?;
120        Ok(self)
121    }
122
123    /// These changes with the rows a transaction changed above them, as `changed` records them: each present one read from `documents`, the transaction's view of the relation, with the vector sources it holds now.
124    pub fn with_identities(
125        self,
126        changed: super::ChangedIdentitiesView,
127        documents: Arc<dyn DocumentStore>,
128        columns: &[ColumnDef],
129        indexes: &dyn VectorIndexSource,
130        control: &StorageReadControl,
131    ) -> StorageBackendResult<Self> {
132        control.check()?;
133        let rows = match capture_vectors(&documents, columns, indexes, control)? {
134            Some(source) => super::layers::IdentityRows::Captured(source),
135            None => super::layers::IdentityRows::Retained(documents),
136        };
137        self.with_identity_layer(changed, rows, control)
138    }
139
140    /// Select the actual source retained with each evaluated private row. None means this row source predates physical capture or has no matching column incarnation; callers must not substitute a later live source.
141    pub(in crate::query) fn diskann_read_changes(
142        &self,
143        field: &str,
144        column: Option<&ColumnDef>,
145        control: &StorageReadControl,
146    ) -> StorageBackendResult<Option<DiskANNReadChanges>> {
147        let physical = |change: &Change| matches!(change, Change::Captured(source, true) if source.vector(field, column).is_some_and(|source| source.source.is_some()));
148        let source = |document: DocId, change: &Change| {
149            if !change.present() {
150                return (document, None);
151            }
152            let Change::Captured(source, _) = change else {
153                unreachable!("validated private source");
154            };
155            (
156                document,
157                Some(
158                    source
159                        .vector(field, column)
160                        .expect("validated private column")
161                        .source
162                        .as_ref()
163                        .expect("validated private physical source")
164                        .clone(),
165                ),
166            )
167        };
168        if self.staged_views().is_empty() && !self.has_identities() {
169            for (_, change) in self.rows() {
170                control.check()?;
171                if change.present() && !physical(change) {
172                    return Ok(None);
173                }
174            }
175            return DiskANNReadChanges::capture(
176                self.rows()
177                    .iter()
178                    .map(|(document, change)| Ok(source(*document, change))),
179                control,
180            )
181            .map(Some);
182        }
183        // Rows that commands staged are evaluated fields, which no physical source supplies, so only their deletions leave a physical selection; the rows a transaction changed keep the sources captured with them.
184        for change in self.change_rows_after(None) {
185            control.check()?;
186            let (_, change) = change?;
187            if change.present() && !physical(&change) {
188                return Ok(None);
189            }
190        }
191        DiskANNReadChanges::capture(
192            self.change_rows_after(None)
193                .map(|change| change.map(|(document, change)| source(document, &change))),
194            control,
195        )
196        .map(Some)
197    }
198
199    pub(in crate::query) fn vector_read_selection(
200        &self,
201        base: Option<VectorReadSnapshot>,
202        dimensions: u32,
203        field: &str,
204        column: Option<&ColumnDef>,
205        control: &StorageReadControl,
206    ) -> StorageBackendResult<Option<VectorReadSnapshot>> {
207        let values = |change: &Change| match change {
208            Change::Captured(source, true) => source
209                .vector(field, column)
210                .map(|source| source.values.clone()),
211            _ => None,
212        };
213        if self.staged_views().is_empty() && !self.has_identities() {
214            for (_, change) in self.rows() {
215                control.check()?;
216                if change.present() && values(change).is_none() {
217                    return Ok(None);
218                }
219            }
220            return SelectedVectorRead::capture(
221                base,
222                dimensions,
223                self.rows()
224                    .iter()
225                    .map(|(document, change)| (*document, values(change))),
226                control,
227            )
228            .map(Some);
229        }
230        // Rows that commands staged are evaluated fields, which no physical source supplies, so only their deletions leave a physical selection; the rows a transaction changed keep the sources captured with them.
231        let mut selected = BudgetedVec::new(control.memory());
232        for change in self.change_rows_after(None) {
233            control.check()?;
234            let (document, change) = change?;
235            let source = values(&change);
236            if change.present() && source.is_none() {
237                return Ok(None);
238            }
239            selected.push((document, source))?;
240        }
241        let (selected, _memory) = selected.into_parts();
242        SelectedVectorRead::capture(base, dimensions, selected, control).map(Some)
243    }
244}