1use std::{ops::Bound::Included, sync::Arc};
10
11use uqa_core::{memory::Budgeted, DocId, PostingList};
12use uqa_sql::{ast::ColumnDef, SQLError};
13use uqa_storage::{
14 mvcc::{SerializableKeySpace, SerializablePredicate},
15 read_control::StorageReadControl,
16 vector_index::validate_vector_values,
17 StorageBackendError, StorageBackendResult, VectorIndex,
18};
19
20use super::{field::prefix as field_prefix, SerializableRelationRead, SerializableWrites};
21use crate::storage_errors::storage_error;
22
23pub fn observe_snapshot(
25 read: Option<&SerializableRelationRead>,
26 columns: &[ColumnDef],
27 field: &str,
28 index: Arc<dyn VectorIndex>,
29) -> StorageBackendResult<Arc<dyn VectorIndex>> {
30 let Some(read) = read else {
31 return Ok(index);
32 };
33 Ok(Arc::new(ObservedVectorIndex {
34 index,
35 observation: VectorObservation::bind(read, columns, field)?,
36 }))
37}
38
39pub fn search_knn(
41 index: &dyn VectorIndex,
42 read: Option<&SerializableRelationRead>,
43 columns: &[ColumnDef],
44 field: &str,
45 query: &[f32],
46 k: usize,
47) -> StorageBackendResult<PostingList> {
48 search_knn_with_diagnostics(index, read, columns, field, query, k, None)
49}
50
51pub fn search_knn_with_diagnostics(
52 index: &dyn VectorIndex,
53 read: Option<&SerializableRelationRead>,
54 columns: &[ColumnDef],
55 field: &str,
56 query: &[f32],
57 k: usize,
58 diagnostics: Option<(&crate::query::diagnostics::CapturedDiagnostics, &str)>,
59) -> StorageBackendResult<PostingList> {
60 if let Some(read) = read.filter(|_| k != 0) {
61 validate_vector_values(index.dimensions(), query)?;
62 VectorObservation::bind(read, columns, field)?.observe()?;
63 }
64 let Some((collector, relation)) = diagnostics else {
65 return index.search_knn(query, k);
66 };
67 let result = index.search_knn_with_statistics(query, k, Some(collector.control()))?;
68 collector.record(
69 relation,
70 field,
71 uqa_core::vector_execution::VectorSearchOperation::KNN { k },
72 &result,
73 )?;
74 Ok(result.postings)
75}
76
77#[derive(Clone)]
78struct ObservedVectorIndex {
79 index: Arc<dyn VectorIndex>,
80 observation: VectorObservation,
81}
82
83#[derive(Clone)]
84struct VectorObservation {
85 read: SerializableRelationRead,
86 upper: Arc<Budgeted<Vec<u8>>>,
87 prefix_len: usize,
88}
89
90impl VectorObservation {
91 fn bind(
92 read: &SerializableRelationRead,
93 columns: &[ColumnDef],
94 field: &str,
95 ) -> StorageBackendResult<Self> {
96 let mut upper = field_prefix(columns, field, &read.control)?;
97 let prefix_len = upper.len();
98 upper.extend_from_slice(&DocId::MAX.to_be_bytes())?;
99 let (upper, memory) = upper.into_parts();
100 Ok(Self {
101 read: read.clone(),
102 upper: Budgeted::new(upper, memory).into_shared()?,
103 prefix_len,
104 })
105 }
106
107 fn observe(&self) -> StorageBackendResult<()> {
108 self.read
110 .context
111 .observe_read(
112 SerializablePredicate::range(
113 self.read.object,
114 SerializableKeySpace::Vectors,
115 Included(&self.upper[..self.prefix_len]),
116 Included(&self.upper),
117 ),
118 &self.read.control,
119 )
120 .map_err(uqa_storage::mvcc::VersionError::into_storage_error)
121 }
122}
123
124impl VectorIndex for ObservedVectorIndex {
125 fn diskann_query_metadata(
126 &self,
127 control: &StorageReadControl,
128 ) -> StorageBackendResult<Option<uqa_storage::diskann_index::DiskANNQueryMetadata>> {
129 self.index.diskann_query_metadata(control)
130 }
131 fn dimensions(&self) -> u32 {
132 self.index.dimensions()
133 }
134 fn index_kind(&self) -> &'static str {
135 self.index.index_kind()
136 }
137 fn contains_document(&self, doc_id: DocId) -> StorageBackendResult<bool> {
138 self.index.contains_document(doc_id)
139 }
140 fn count(&self) -> StorageBackendResult<usize> {
141 self.index.count()
142 }
143 fn search_knn(&self, query: &[f32], k: usize) -> StorageBackendResult<PostingList> {
144 validate_vector_values(self.dimensions(), query)?;
145 if k != 0 {
146 self.observation.observe()?;
147 }
148 self.index.search_knn(query, k)
149 }
150 fn search_threshold(&self, query: &[f32], threshold: f32) -> StorageBackendResult<PostingList> {
151 validate_vector_values(self.dimensions(), query)?;
152 if !threshold.is_finite() {
153 return Err(StorageBackendError::Other(
154 "vector similarity threshold must be finite".into(),
155 ));
156 }
157 self.observation.observe()?;
158 self.index.search_threshold(query, threshold)
159 }
160 fn search_knn_with_control(
161 &self,
162 query: &[f32],
163 k: usize,
164 control: &StorageReadControl,
165 ) -> StorageBackendResult<PostingList> {
166 control.check()?;
167 validate_vector_values(self.dimensions(), query)?;
168 if k != 0 {
169 self.observation.observe()?;
170 }
171 self.index.search_knn_with_control(query, k, control)
172 }
173 fn search_threshold_with_control(
174 &self,
175 query: &[f32],
176 threshold: f32,
177 control: &StorageReadControl,
178 ) -> StorageBackendResult<PostingList> {
179 control.check()?;
180 validate_vector_values(self.dimensions(), query)?;
181 if !threshold.is_finite() {
182 return Err(StorageBackendError::Other(
183 "vector similarity threshold must be finite".into(),
184 ));
185 }
186 self.observation.observe()?;
187 self.index
188 .search_threshold_with_control(query, threshold, control)
189 }
190 fn search_knn_with_statistics(
191 &self,
192 query: &[f32],
193 k: usize,
194 control: Option<&StorageReadControl>,
195 ) -> StorageBackendResult<uqa_storage::vector_index::VectorQueryResult> {
196 if let Some(control) = control {
197 control.check()?;
198 }
199 validate_vector_values(self.dimensions(), query)?;
200 if k != 0 {
201 self.observation.observe()?;
202 }
203 self.index.search_knn_with_statistics(query, k, control)
204 }
205 fn search_threshold_with_statistics(
206 &self,
207 query: &[f32],
208 threshold: f32,
209 control: Option<&StorageReadControl>,
210 ) -> StorageBackendResult<uqa_storage::vector_index::VectorQueryResult> {
211 if let Some(control) = control {
212 control.check()?;
213 }
214 validate_vector_values(self.dimensions(), query)?;
215 if !threshold.is_finite() {
216 return Err(StorageBackendError::Other(
217 "vector similarity threshold must be finite".into(),
218 ));
219 }
220 self.observation.observe()?;
221 self.index
222 .search_threshold_with_statistics(query, threshold, control)
223 }
224 fn snapshot(&self) -> StorageBackendResult<Arc<dyn VectorIndex>> {
225 Ok(Arc::new(self.clone()))
226 }
227 fn diskann_read_snapshot(
228 &self,
229 control: &StorageReadControl,
230 ) -> StorageBackendResult<Option<uqa_storage::diskann_index::DiskANNReadSnapshot>> {
231 self.index.diskann_read_snapshot(control)
232 }
233 fn vector_read_snapshot(
234 &self,
235 control: &StorageReadControl,
236 ) -> StorageBackendResult<Option<uqa_storage::vector_index::VectorReadSnapshot>> {
237 self.index.vector_read_snapshot(control)
238 }
239 fn snapshot_with_vector_read(
240 &self,
241 source: uqa_storage::vector_index::VectorReadSnapshot,
242 control: &StorageReadControl,
243 ) -> StorageBackendResult<Option<Arc<dyn VectorIndex>>> {
244 Ok(self
245 .index
246 .snapshot_with_vector_read(source, control)?
247 .map(|index| {
248 Arc::new(Self {
249 index,
250 observation: self.observation.clone(),
251 }) as Arc<dyn VectorIndex>
252 }))
253 }
254 fn snapshot_with_diskann_changes(
255 &self,
256 changes: &uqa_storage::diskann_index::DiskANNReadChanges,
257 control: &StorageReadControl,
258 ) -> StorageBackendResult<Option<Arc<dyn VectorIndex>>> {
259 Ok(self
260 .index
261 .snapshot_with_diskann_changes(changes, control)?
262 .map(|index| {
263 Arc::new(Self {
264 index,
265 observation: self.observation.clone(),
266 }) as Arc<dyn VectorIndex>
267 }))
268 }
269 fn add(&mut self, _: DocId, _: Vec<f32>) -> StorageBackendResult<()> {
270 Err(read_only())
271 }
272 fn add_many(&mut self, _: DocId, _: Vec<Vec<f32>>) -> StorageBackendResult<()> {
273 Err(read_only())
274 }
275 fn delete(&mut self, _: DocId) -> StorageBackendResult<()> {
276 Err(read_only())
277 }
278 fn clear(&mut self) -> StorageBackendResult<()> {
279 Err(read_only())
280 }
281 fn initialize(&mut self) -> StorageBackendResult<()> {
282 Err(read_only())
283 }
284}
285
286fn read_only() -> StorageBackendError {
287 StorageBackendError::Other("cannot write a retained serializable vector snapshot".into())
288}
289
290pub enum VectorChange<'a> {
292 Single(&'a [f32]),
293 Tensor(&'a [Vec<f32>]),
294 Delete,
295}
296
297pub fn observe_write(
299 writes: &dyn SerializableWrites,
300 table: &str,
301 columns: &[ColumnDef],
302 field: &str,
303 index: &dyn VectorIndex,
304 doc_id: DocId,
305 change: VectorChange<'_>,
306) -> Result<(), SQLError> {
307 let Some(read) = SerializableRelationRead::for_mutation(writes, table)? else {
308 return Ok(());
309 };
310 let nonempty = (|| -> StorageBackendResult<bool> {
311 match change {
312 VectorChange::Single(vector) => {
313 validate_vector_values(index.dimensions(), vector)?;
314 Ok(true)
315 }
316 VectorChange::Tensor(vectors) => {
317 for vector in vectors {
318 read.control.check()?;
319 validate_vector_values(index.dimensions(), vector)?;
320 }
321 Ok(!vectors.is_empty())
322 }
323 VectorChange::Delete => Ok(false),
324 }
325 })()
326 .map_err(|error| storage_error("validate serializable vector write", &error))?;
327 if !nonempty
328 && !index
329 .contains_document(doc_id)
330 .map_err(|error| storage_error("read original vector membership", &error))?
331 {
332 return Ok(());
333 }
334 let mut key = field_prefix(columns, field, &read.control)
335 .map_err(|error| storage_error("bind serializable vector field", &error))?;
336 key.extend_from_slice(&doc_id.to_be_bytes())
337 .map_err(|error| storage_error("bind serializable vector candidate", &error.into()))?;
338 let session = writes.serializable_session().ok_or_else(|| {
339 SQLError::Internal("serializable vector writer lost its original session".into())
340 })?;
341 session
342 .observe_serializable_write(SerializablePredicate::point(
343 read.object,
344 SerializableKeySpace::Vectors,
345 &key,
346 ))
347 .map_err(|error| storage_error("observe serializable vector write", &error))
348}
349
350#[cfg(test)]
351mod tests {
352 use super::{field_prefix, storage_error};
353 use uqa_sql::{compile, Statement};
354 use uqa_storage::read_control::StorageReadControl;
355
356 #[test]
357 fn vector_field_addresses_preserve_incarnations_and_dynamic_name_boundaries() {
358 let Statement::CreateTable(mut table) =
359 compile("CREATE TABLE t (v VECTOR(2))").unwrap().remove(0)
360 else {
361 panic!("expected table declaration");
362 };
363 let control = StorageReadControl::with_limit(4096);
364 assert!(field_prefix(&table.columns, "v", &control).is_err());
365 table.columns[0].object_id = Some([1; 16]);
366 let old = field_prefix(&table.columns, "v", &control).unwrap();
367 table.columns[0].name = "renamed".into();
368 assert_eq!(
369 &*old,
370 &*field_prefix(&table.columns, "renamed", &control).unwrap()
371 );
372 table.columns[0].object_id = Some([2; 16]);
373 assert_ne!(
374 &*old,
375 &*field_prefix(&table.columns, "renamed", &control).unwrap()
376 );
377 let dynamic = field_prefix(&[], "v", &control).unwrap();
378 assert_ne!(&*old, &*dynamic);
379 assert_eq!(
380 &*dynamic,
381 &*field_prefix(&table.columns, "v", &control).unwrap()
382 );
383 for name in ["vv", "v\0", "v\0\0", "日本語"] {
384 let other = field_prefix(&[], name, &control).unwrap();
385 assert!(!other.starts_with(&dynamic));
386 assert!(!dynamic.starts_with(&other));
387 }
388 }
389
390 #[test]
391 fn vector_field_binding_uses_the_original_allowance_and_cancellation() {
392 let control = StorageReadControl::with_limit(32);
393 let retained = field_prefix(&[], "vector", &control).unwrap();
394 let held = control.memory().used();
395 let error = field_prefix(&[], &"long".repeat(32), &control).unwrap_err();
396 assert_eq!(
397 storage_error("vector field", &error).sqlstate(),
398 Some("53200")
399 );
400 assert_eq!(control.memory().used(), held);
401 control.cancellation().cancel();
402 let error = field_prefix(&[], "v", &control).unwrap_err();
403 assert_eq!(
404 storage_error("vector field", &error).sqlstate(),
405 Some("57014")
406 );
407 drop(retained);
408 assert_eq!(control.memory().used(), 0);
409 }
410}