uqa_execution/query/document_projection/
captured.rs1use crate::{
10 physical::physical_exec_error,
11 query::{
12 document_changes::DocumentChanges,
13 scored_input::{CapturedScoredEntries, ScoredEntryCapture},
14 SelectedIds,
15 },
16 spill::{SharedSpillReader, SpillBatchWriter, SpillRows},
17 PhysicalRow, RowSchema, SpillBuffer,
18};
19use uqa_core::{CancellationToken, DocId, Predicate, ScoredEntry, Value};
20use uqa_sql::{ast::ColumnDef, semantics::XMIN_COLUMN, SQLError};
21use uqa_storage::{DocumentMetadata, DocumentStore};
22
23#[cfg(test)]
24mod tests;
25
26pub struct FilterProjection<'a> {
27 pub source: &'a dyn DocumentStore,
28 pub changes: Option<&'a DocumentChanges>,
29 pub definitions: &'a [ColumnDef],
30}
31
32#[derive(Default)]
33struct Diagnostics {
34 missing: Option<DocId>,
35 malformed: Option<(DocId, usize)>,
36}
37
38impl FilterProjection<'_> {
39 pub fn capture(
40 &self,
41 table: &str,
42 ids: &SelectedIds,
43 field: &str,
44 predicate: &Predicate,
45 bytes: usize,
46 cancellation: &CancellationToken,
47 ) -> Result<CapturedScoredEntries, SQLError> {
48 cancellation.check()?;
49 let mut diagnostics = Diagnostics::default();
50 let (private, persisted) = ids.partition(bytes / 4, |id| {
52 cancellation.check()?;
53 uqa_sql::expr::check_statement_cancellation()?;
54 let presence = self
55 .changes
56 .map(|changes| changes.change_presence(id))
57 .transpose()
58 .map_err(|error| {
59 crate::storage_errors::storage_error("read private document projection", &error)
60 })?
61 .flatten();
62 Ok(match presence {
63 Some(false) => {
64 diagnostics.missing = Some(diagnostics.missing.map_or(id, |old| old.min(id)));
65 None
66 }
67 present => Some(present == Some(true)),
68 })
69 })?;
70 let matched = SelectedIds::capture_visit(bytes / 4, cancellation, |visit| {
71 for (selected, source) in [
72 (&persisted, Some(self.source)),
73 (
74 &private,
75 self.changes.map(|changes| changes as &dyn DocumentStore),
76 ),
77 ] {
78 let Some(source) = source else { continue };
79 let projected = Projection {
80 source,
81 definitions: self.definitions,
82 field,
83 cancellation,
84 };
85 projected.capture(
86 selected.cursor(),
87 predicate,
88 bytes / 4,
89 &mut diagnostics,
90 visit,
91 )?;
92 }
93 Ok(())
94 })?;
95 if let Some((id, width)) = diagnostics.malformed {
96 return Err(SQLError::Internal(format!(
97 "read document field returned {width} projected values for document {id}; expected 1"
98 )));
99 }
100 if let Some(id) = diagnostics.missing {
101 return Err(SQLError::Internal(format!(
102 "Filter consistency error: candidate {id} is missing from the document-field snapshot for table `{table}`"
103 )));
104 }
105 let mut output = ScoredEntryCapture::new(bytes / 4, cancellation.clone())?;
106 for id in matched.cursor() {
107 output.push(ScoredEntry {
108 doc_id: id?,
109 score: 0.0,
110 })?;
111 }
112 output.finish()
113 }
114}
115
116struct Projection<'a> {
117 source: &'a dyn DocumentStore,
118 definitions: &'a [ColumnDef],
119 field: &'a str,
120 cancellation: &'a CancellationToken,
121}
122
123impl Projection<'_> {
124 fn capture(
125 &self,
126 ids: impl Iterator<Item = Result<DocId, SQLError>>,
127 predicate: &Predicate,
128 bytes: usize,
129 diagnostics: &mut Diagnostics,
130 visit: &mut dyn FnMut(DocId) -> Result<(), SQLError>,
131 ) -> Result<(), SQLError> {
132 let requested = [self.field.to_owned()];
133 let generated = super::super::generated::projection_contains_virtual_generated_column(
134 self.definitions,
135 &requested,
136 );
137 let xmin = super::projection_uses_tuple_xmin(self.field, self.definitions);
138 if !generated && !xmin {
139 for id in ids {
141 let id = id?;
142 let Some(raw) = self.read_raw(id, false)? else {
143 diagnostics.missing = Some(diagnostics.missing.map_or(id, |old| old.min(id)));
144 continue;
145 };
146 if let Some(value) = self.stored_value(id, raw, false, diagnostics)? {
147 if predicate.evaluate(Some(&value)) {
148 visit(id)?;
149 }
150 }
151 }
152 return Ok(());
153 }
154 let raw = self.capture_raw(ids, generated, bytes, diagnostics)?;
155 for row in raw {
157 self.cancellation.check()?;
158 uqa_sql::expr::check_statement_cancellation()?;
159 let [id, raw] = row
160 .map_err(physical_exec_error)?
161 .row
162 .into_physical_values()
163 .try_into()
164 .map_err(|_| SQLError::Internal("invalid captured field width".into()))?;
165 let id = crate::mutation::prepared::decode_prepared_doc_id(id, "captured field")?;
166 let value = if generated {
167 let Value::Map(mut document) = raw else {
168 return Err(SQLError::Internal(
169 "invalid captured generated document".into(),
170 ));
171 };
172 super::super::generated::materialize_projected_virtual_generated_columns(
173 self.definitions,
174 &mut document,
175 &requested,
176 )?;
177 document.remove(self.field).unwrap_or(Value::Null)
178 } else {
179 let Some(value) = self.stored_value(id, raw, xmin, diagnostics)? else {
180 continue;
181 };
182 value
183 };
184 if predicate.evaluate(Some(&value)) {
185 visit(id)?;
186 }
187 }
188 Ok(())
189 }
190
191 fn capture_raw(
192 &self,
193 ids: impl Iterator<Item = Result<DocId, SQLError>>,
194 generated: bool,
195 bytes: usize,
196 diagnostics: &mut Diagnostics,
197 ) -> Result<SpillRows<SharedSpillReader>, SQLError> {
198 let schema = RowSchema::with_internal_relation_types(
199 uqa_sql::ast::InternalRelationId::allocate(),
200 vec![None, None],
201 );
202 let mut rows = SpillBuffer::new(bytes / 4);
203 let mut writer = SpillBatchWriter::new(schema.clone()).map_err(physical_exec_error)?;
204 for id in ids {
205 let id = id?;
206 let raw = self.read_raw(id, generated)?;
207 match raw {
208 Some(raw) => writer
209 .push(
210 &mut rows,
211 PhysicalRow::from_values(vec![
212 crate::mutation::prepared::encode_prepared_doc_id(id),
213 raw,
214 ]),
215 )
216 .map_err(physical_exec_error)?,
217 None => {
218 diagnostics.missing = Some(diagnostics.missing.map_or(id, |old| old.min(id)));
219 }
220 }
221 }
222 writer.finish(&mut rows).map_err(physical_exec_error)?;
223 rows.into_shared(schema)
224 .map_err(physical_exec_error)?
225 .read_rows()
226 .map_err(physical_exec_error)
227 }
228
229 fn read_raw(&self, id: DocId, generated: bool) -> Result<Option<Value>, SQLError> {
230 self.cancellation.check()?;
231 uqa_sql::expr::check_statement_cancellation()?;
232 if generated {
233 self.source
234 .get_stored(id)
235 .map_err(|error| {
236 crate::storage_errors::storage_error(
237 "read generated document projection",
238 &error,
239 )
240 })
241 .map(|document| document.map(|document| Value::Map(document.into_fields())))
242 } else {
243 self.source
244 .get_fields_multi(&[id], &[self.field])
245 .map_err(|error| {
246 crate::storage_errors::storage_error("read document projection", &error)
247 })
248 .map(|mut rows| rows.remove(&id).map(Value::List))
249 }
250 }
251
252 fn stored_value(
253 &self,
254 id: DocId,
255 raw: Value,
256 xmin: bool,
257 diagnostics: &mut Diagnostics,
258 ) -> Result<Option<Value>, SQLError> {
259 let Value::List(mut values) = raw else {
260 return Err(SQLError::Internal("invalid captured field values".into()));
261 };
262 if values.len() != 1 {
263 if xmin {
264 return Err(SQLError::Internal(format!(
265 "document {id} returned {} fields for a 1-field projection",
266 values.len()
267 )));
268 }
269 if diagnostics.malformed.is_none_or(|(old, _)| id < old) {
270 diagnostics.malformed = Some((id, values.len()));
271 }
272 return Ok(None);
273 }
274 if xmin {
275 self.tuple_xmin(id).map(Some)
276 } else {
277 Ok(Some(values.remove(0)))
278 }
279 }
280
281 fn tuple_xmin(&self, id: DocId) -> Result<Value, SQLError> {
282 let explicit = if self.definitions.is_empty() {
283 self.source.get_field(id, XMIN_COLUMN).map_err(|error| {
284 crate::storage_errors::storage_error("read dynamic xmin field", &error)
285 })?
286 } else {
287 None
288 };
289 match explicit {
290 Some(value) => Ok(value),
291 None => Ok(self
292 .source
293 .get_metadata(id)
294 .map_err(|error| {
295 crate::storage_errors::storage_error("read projected tuple metadata", &error)
296 })?
297 .and_then(DocumentMetadata::tuple_xmin)
298 .map_or(Value::Null, |xmin| Value::Int(i64::from(xmin)))),
299 }
300 }
301}