Skip to main content

uqa_execution/query/document_projection/
captured.rs

1//
2// Unified Query Algebra
3//
4// Copyright (c) 2023-2026 Cognica, Inc.
5//
6
7//! Capture raw projected inputs before generated evaluation and retain persisted/private phase order.
8
9use crate::{
10    physical::physical_exec_error,
11    query::{
12        document_changes::DocumentChanges,
13        scored_input::{CapturedScoredEntries, ScoredEntryCapture},
14        SelectedIds,
15    },
16    spill::{SharedSpillReader, SpillBatchWriter, SpillRows},
17    PhysicalRow, RowSchema, SpillBuffer,
18};
19use uqa_core::{CancellationToken, DocId, Predicate, ScoredEntry, Value};
20use uqa_sql::{ast::ColumnDef, semantics::XMIN_COLUMN, SQLError};
21use uqa_storage::{DocumentMetadata, DocumentStore};
22
23#[cfg(test)]
24mod tests;
25
26pub struct FilterProjection<'a> {
27    pub source: &'a dyn DocumentStore,
28    pub changes: Option<&'a DocumentChanges>,
29    pub definitions: &'a [ColumnDef],
30}
31
32#[derive(Default)]
33struct Diagnostics {
34    missing: Option<DocId>,
35    malformed: Option<(DocId, usize)>,
36}
37
38impl FilterProjection<'_> {
39    pub fn capture(
40        &self,
41        table: &str,
42        ids: &SelectedIds,
43        field: &str,
44        predicate: &Predicate,
45        bytes: usize,
46        cancellation: &CancellationToken,
47    ) -> Result<CapturedScoredEntries, SQLError> {
48        cancellation.check()?;
49        let mut diagnostics = Diagnostics::default();
50        // The ordinary projection classifies every identity before reading either source.
51        let (private, persisted) = ids.partition(bytes / 4, |id| {
52            cancellation.check()?;
53            uqa_sql::expr::check_statement_cancellation()?;
54            let presence = self
55                .changes
56                .map(|changes| changes.change_presence(id))
57                .transpose()
58                .map_err(|error| {
59                    crate::storage_errors::storage_error("read private document projection", &error)
60                })?
61                .flatten();
62            Ok(match presence {
63                Some(false) => {
64                    diagnostics.missing = Some(diagnostics.missing.map_or(id, |old| old.min(id)));
65                    None
66                }
67                present => Some(present == Some(true)),
68            })
69        })?;
70        let matched = SelectedIds::capture_visit(bytes / 4, cancellation, |visit| {
71            for (selected, source) in [
72                (&persisted, Some(self.source)),
73                (
74                    &private,
75                    self.changes.map(|changes| changes as &dyn DocumentStore),
76                ),
77            ] {
78                let Some(source) = source else { continue };
79                let projected = Projection {
80                    source,
81                    definitions: self.definitions,
82                    field,
83                    cancellation,
84                };
85                projected.capture(
86                    selected.cursor(),
87                    predicate,
88                    bytes / 4,
89                    &mut diagnostics,
90                    visit,
91                )?;
92            }
93            Ok(())
94        })?;
95        if let Some((id, width)) = diagnostics.malformed {
96            return Err(SQLError::Internal(format!(
97                "read document field returned {width} projected values for document {id}; expected 1"
98            )));
99        }
100        if let Some(id) = diagnostics.missing {
101            return Err(SQLError::Internal(format!(
102                "Filter consistency error: candidate {id} is missing from the document-field snapshot for table `{table}`"
103            )));
104        }
105        let mut output = ScoredEntryCapture::new(bytes / 4, cancellation.clone())?;
106        for id in matched.cursor() {
107            output.push(ScoredEntry {
108                doc_id: id?,
109                score: 0.0,
110            })?;
111        }
112        output.finish()
113    }
114}
115
116struct Projection<'a> {
117    source: &'a dyn DocumentStore,
118    definitions: &'a [ColumnDef],
119    field: &'a str,
120    cancellation: &'a CancellationToken,
121}
122
123impl Projection<'_> {
124    fn capture(
125        &self,
126        ids: impl Iterator<Item = Result<DocId, SQLError>>,
127        predicate: &Predicate,
128        bytes: usize,
129        diagnostics: &mut Diagnostics,
130        visit: &mut dyn FnMut(DocId) -> Result<(), SQLError>,
131    ) -> Result<(), SQLError> {
132        let requested = [self.field.to_owned()];
133        let generated = super::super::generated::projection_contains_virtual_generated_column(
134            self.definitions,
135            &requested,
136        );
137        let xmin = super::projection_uses_tuple_xmin(self.field, self.definitions);
138        if !generated && !xmin {
139            // Stored-value predicates are pure and infallible. Consume each value directly while retaining all output until both source phases finish.
140            for id in ids {
141                let id = id?;
142                let Some(raw) = self.read_raw(id, false)? else {
143                    diagnostics.missing = Some(diagnostics.missing.map_or(id, |old| old.min(id)));
144                    continue;
145                };
146                if let Some(value) = self.stored_value(id, raw, false, diagnostics)? {
147                    if predicate.evaluate(Some(&value)) {
148                        visit(id)?;
149                    }
150                }
151            }
152            return Ok(());
153        }
154        let raw = self.capture_raw(ids, generated, bytes, diagnostics)?;
155        // All storage reads precede generated expressions and tuple-metadata reads, as in read_document_projection.
156        for row in raw {
157            self.cancellation.check()?;
158            uqa_sql::expr::check_statement_cancellation()?;
159            let [id, raw] = row
160                .map_err(physical_exec_error)?
161                .row
162                .into_physical_values()
163                .try_into()
164                .map_err(|_| SQLError::Internal("invalid captured field width".into()))?;
165            let id = crate::mutation::prepared::decode_prepared_doc_id(id, "captured field")?;
166            let value = if generated {
167                let Value::Map(mut document) = raw else {
168                    return Err(SQLError::Internal(
169                        "invalid captured generated document".into(),
170                    ));
171                };
172                super::super::generated::materialize_projected_virtual_generated_columns(
173                    self.definitions,
174                    &mut document,
175                    &requested,
176                )?;
177                document.remove(self.field).unwrap_or(Value::Null)
178            } else {
179                let Some(value) = self.stored_value(id, raw, xmin, diagnostics)? else {
180                    continue;
181                };
182                value
183            };
184            if predicate.evaluate(Some(&value)) {
185                visit(id)?;
186            }
187        }
188        Ok(())
189    }
190
191    fn capture_raw(
192        &self,
193        ids: impl Iterator<Item = Result<DocId, SQLError>>,
194        generated: bool,
195        bytes: usize,
196        diagnostics: &mut Diagnostics,
197    ) -> Result<SpillRows<SharedSpillReader>, SQLError> {
198        let schema = RowSchema::with_internal_relation_types(
199            uqa_sql::ast::InternalRelationId::allocate(),
200            vec![None, None],
201        );
202        let mut rows = SpillBuffer::new(bytes / 4);
203        let mut writer = SpillBatchWriter::new(schema.clone()).map_err(physical_exec_error)?;
204        for id in ids {
205            let id = id?;
206            let raw = self.read_raw(id, generated)?;
207            match raw {
208                Some(raw) => writer
209                    .push(
210                        &mut rows,
211                        PhysicalRow::from_values(vec![
212                            crate::mutation::prepared::encode_prepared_doc_id(id),
213                            raw,
214                        ]),
215                    )
216                    .map_err(physical_exec_error)?,
217                None => {
218                    diagnostics.missing = Some(diagnostics.missing.map_or(id, |old| old.min(id)));
219                }
220            }
221        }
222        writer.finish(&mut rows).map_err(physical_exec_error)?;
223        rows.into_shared(schema)
224            .map_err(physical_exec_error)?
225            .read_rows()
226            .map_err(physical_exec_error)
227    }
228
229    fn read_raw(&self, id: DocId, generated: bool) -> Result<Option<Value>, SQLError> {
230        self.cancellation.check()?;
231        uqa_sql::expr::check_statement_cancellation()?;
232        if generated {
233            self.source
234                .get_stored(id)
235                .map_err(|error| {
236                    crate::storage_errors::storage_error(
237                        "read generated document projection",
238                        &error,
239                    )
240                })
241                .map(|document| document.map(|document| Value::Map(document.into_fields())))
242        } else {
243            self.source
244                .get_fields_multi(&[id], &[self.field])
245                .map_err(|error| {
246                    crate::storage_errors::storage_error("read document projection", &error)
247                })
248                .map(|mut rows| rows.remove(&id).map(Value::List))
249        }
250    }
251
252    fn stored_value(
253        &self,
254        id: DocId,
255        raw: Value,
256        xmin: bool,
257        diagnostics: &mut Diagnostics,
258    ) -> Result<Option<Value>, SQLError> {
259        let Value::List(mut values) = raw else {
260            return Err(SQLError::Internal("invalid captured field values".into()));
261        };
262        if values.len() != 1 {
263            if xmin {
264                return Err(SQLError::Internal(format!(
265                    "document {id} returned {} fields for a 1-field projection",
266                    values.len()
267                )));
268            }
269            if diagnostics.malformed.is_none_or(|(old, _)| id < old) {
270                diagnostics.malformed = Some((id, values.len()));
271            }
272            return Ok(None);
273        }
274        if xmin {
275            self.tuple_xmin(id).map(Some)
276        } else {
277            Ok(Some(values.remove(0)))
278        }
279    }
280
281    fn tuple_xmin(&self, id: DocId) -> Result<Value, SQLError> {
282        let explicit = if self.definitions.is_empty() {
283            self.source.get_field(id, XMIN_COLUMN).map_err(|error| {
284                crate::storage_errors::storage_error("read dynamic xmin field", &error)
285            })?
286        } else {
287            None
288        };
289        match explicit {
290            Some(value) => Ok(value),
291            None => Ok(self
292                .source
293                .get_metadata(id)
294                .map_err(|error| {
295                    crate::storage_errors::storage_error("read projected tuple metadata", &error)
296                })?
297                .and_then(DocumentMetadata::tuple_xmin)
298                .map_or(Value::Null, |xmin| Value::Int(i64::from(xmin)))),
299        }
300    }
301}