uqa_execution/query/document_changes/
vectors.rs1use super::{
10 Arc, Change, DocId, DocumentChanges, DocumentSelection, DocumentStore, StorageBackendResult,
11 StorageReadControl,
12};
13use uqa_core::memory::{Budgeted, BudgetedVec, MemoryReservation};
14use uqa_sql::ast::ColumnDef;
15use uqa_storage::{
16 diskann_index::{DiskANNReadChanges, DiskANNReadSnapshot},
17 vector_index::{
18 RetainedVectorIndexesBuilder, SelectedVectorRead, VectorIndexSource, VectorReadSnapshot,
19 },
20};
21
22struct VectorSource {
23 field: String,
24 column: Option<[u8; 16]>,
25 source: Option<DiskANNReadSnapshot>,
26 values: VectorReadSnapshot,
27 _memory: MemoryReservation,
28}
29
30fn capture_vectors(
32 documents: &Arc<dyn DocumentStore>,
33 columns: &[ColumnDef],
34 indexes: &dyn VectorIndexSource,
35 control: &StorageReadControl,
36) -> StorageBackendResult<Option<Arc<Budgeted<CapturedRows>>>> {
37 let mut vectors = BudgetedVec::new(control.memory());
38 indexes.visit(&mut |field, index| {
39 let source = index.diskann_read_snapshot(control)?;
40 let values = if let Some(source) = &source {
41 Some(
42 Budgeted::new(source.clone(), control.memory().empty_reservation()).into_shared()?
43 as VectorReadSnapshot,
44 )
45 } else {
46 index.vector_read_snapshot(control)?
47 };
48 let Some(values) = values else {
49 return Ok(());
50 };
51 vectors.reserve(1)?;
52 let column = columns
53 .iter()
54 .find(|column| column.name == field)
55 .and_then(|column| column.object_id);
56 let (field, memory) = RetainedVectorIndexesBuilder::copy_field(field, control)?;
57 vectors.push(VectorSource {
58 field,
59 column,
60 source,
61 values,
62 _memory: memory,
63 })?;
64 Ok(())
65 })?;
66 if vectors.is_empty() {
67 return Ok(None);
68 }
69 Ok(Some(
70 Budgeted::new(
71 CapturedRows {
72 documents: Arc::clone(documents),
73 vectors,
74 },
75 control.memory().empty_reservation(),
76 )
77 .into_shared()?,
78 ))
79}
80
81pub(super) struct CapturedRows {
82 pub(super) documents: Arc<dyn DocumentStore>,
83 vectors: BudgetedVec<VectorSource>,
84}
85
86impl CapturedRows {
87 fn vector(&self, field: &str, column: Option<&ColumnDef>) -> Option<&VectorSource> {
88 let identity = column.and_then(|column| column.object_id);
89 self.vectors
90 .iter()
91 .find(|source| match (source.column, identity) {
92 (Some(source), Some(target)) => source == target,
93 _ => source.field == field,
94 })
95 }
96}
97
98impl DocumentChanges {
99 pub fn with_retained_vectors(
101 mut self,
102 documents: Arc<dyn DocumentStore>,
103 desired: DocumentSelection,
104 columns: &[ColumnDef],
105 indexes: &dyn VectorIndexSource,
106 control: &StorageReadControl,
107 ) -> StorageBackendResult<Self> {
108 control.check()?;
109 let Some(source) = capture_vectors(&documents, columns, indexes, control)? else {
110 return self.with_retained(documents, desired, control);
111 };
112 let desired = desired.finish(control)?;
113 let mut newer = Self::default();
114 for (id, present) in desired.entries() {
115 control.check()?;
116 let present = present && source.documents.contains_doc_id(id)?;
117 newer.insert(id, Change::Captured(source.clone(), present), control)?;
118 }
119 self.extend(newer, control)?;
120 Ok(self)
121 }
122
123 pub fn with_identities(
125 self,
126 changed: super::ChangedIdentitiesView,
127 documents: Arc<dyn DocumentStore>,
128 columns: &[ColumnDef],
129 indexes: &dyn VectorIndexSource,
130 control: &StorageReadControl,
131 ) -> StorageBackendResult<Self> {
132 control.check()?;
133 let rows = match capture_vectors(&documents, columns, indexes, control)? {
134 Some(source) => super::layers::IdentityRows::Captured(source),
135 None => super::layers::IdentityRows::Retained(documents),
136 };
137 self.with_identity_layer(changed, rows, control)
138 }
139
140 pub(in crate::query) fn diskann_read_changes(
142 &self,
143 field: &str,
144 column: Option<&ColumnDef>,
145 control: &StorageReadControl,
146 ) -> StorageBackendResult<Option<DiskANNReadChanges>> {
147 let physical = |change: &Change| matches!(change, Change::Captured(source, true) if source.vector(field, column).is_some_and(|source| source.source.is_some()));
148 let source = |document: DocId, change: &Change| {
149 if !change.present() {
150 return (document, None);
151 }
152 let Change::Captured(source, _) = change else {
153 unreachable!("validated private source");
154 };
155 (
156 document,
157 Some(
158 source
159 .vector(field, column)
160 .expect("validated private column")
161 .source
162 .as_ref()
163 .expect("validated private physical source")
164 .clone(),
165 ),
166 )
167 };
168 if self.staged_views().is_empty() && !self.has_identities() {
169 for (_, change) in self.rows() {
170 control.check()?;
171 if change.present() && !physical(change) {
172 return Ok(None);
173 }
174 }
175 return DiskANNReadChanges::capture(
176 self.rows()
177 .iter()
178 .map(|(document, change)| Ok(source(*document, change))),
179 control,
180 )
181 .map(Some);
182 }
183 for change in self.change_rows_after(None) {
185 control.check()?;
186 let (_, change) = change?;
187 if change.present() && !physical(&change) {
188 return Ok(None);
189 }
190 }
191 DiskANNReadChanges::capture(
192 self.change_rows_after(None)
193 .map(|change| change.map(|(document, change)| source(document, &change))),
194 control,
195 )
196 .map(Some)
197 }
198
199 pub(in crate::query) fn vector_read_selection(
200 &self,
201 base: Option<VectorReadSnapshot>,
202 dimensions: u32,
203 field: &str,
204 column: Option<&ColumnDef>,
205 control: &StorageReadControl,
206 ) -> StorageBackendResult<Option<VectorReadSnapshot>> {
207 let values = |change: &Change| match change {
208 Change::Captured(source, true) => source
209 .vector(field, column)
210 .map(|source| source.values.clone()),
211 _ => None,
212 };
213 if self.staged_views().is_empty() && !self.has_identities() {
214 for (_, change) in self.rows() {
215 control.check()?;
216 if change.present() && values(change).is_none() {
217 return Ok(None);
218 }
219 }
220 return SelectedVectorRead::capture(
221 base,
222 dimensions,
223 self.rows()
224 .iter()
225 .map(|(document, change)| (*document, values(change))),
226 control,
227 )
228 .map(Some);
229 }
230 let mut selected = BudgetedVec::new(control.memory());
232 for change in self.change_rows_after(None) {
233 control.check()?;
234 let (document, change) = change?;
235 let source = values(&change);
236 if change.present() && source.is_none() {
237 return Ok(None);
238 }
239 selected.push((document, source))?;
240 }
241 let (selected, _memory) = selected.into_parts();
242 SelectedVectorRead::capture(base, dimensions, selected, control).map(Some)
243 }
244}