Skip to main content

uqa_execution/
project_set.rs

1//
2// Unified Query Algebra
3//
4// Copyright (c) 2023-2026 Cognica, Inc.
5//
6
7//! Streaming one-to-many projection (`ProjectSet`).
8
9use uqa_sql::ResultRow;
10
11use crate::batch::DEFAULT_BATCH_SIZE;
12use crate::{Batch, ExecResult, OwnedPhysicalRow, PhysicalOperator, PhysicalRow, RowSchema};
13
14/// Owned row stream produced for one input row.
15///
16/// The iterator yields errors at the exact row where production failed. This
17/// matters for generators backed by files or user code: eagerly collecting a
18/// `Vec` both defeated the Volcano memory boundary and made late failures
19/// impossible to represent without discarding already-produced rows.
20pub type ProjectRows = Box<dyn Iterator<Item = ExecResult<ResultRow>> + Send>;
21
22/// Engine seam for set-returning projection expressions.
23pub trait SetProjector: Send {
24    fn project(&mut self, row: &ResultRow) -> ExecResult<ProjectRows>;
25}
26
27impl<F> SetProjector for F
28where
29    F: FnMut(&ResultRow) -> ExecResult<ProjectRows> + Send,
30{
31    fn project(&mut self, row: &ResultRow) -> ExecResult<ProjectRows> {
32        self(row)
33    }
34}
35
36/// Pulls input rows and expands each through a set-returning projection.
37pub struct ProjectSet<'a> {
38    child: Box<dyn PhysicalOperator + 'a>,
39    projector: Box<dyn SetProjector + 'a>,
40    schema: RowSchema,
41    input: std::vec::IntoIter<ResultRow>,
42    projected: Option<ProjectRows>,
43    exhausted: bool,
44}
45
46impl<'a> ProjectSet<'a> {
47    pub fn new(
48        child: Box<dyn PhysicalOperator + 'a>,
49        output_schema: Vec<String>,
50        projector: Box<dyn SetProjector + 'a>,
51    ) -> Self {
52        Self {
53            child,
54            projector,
55            schema: RowSchema::new(output_schema),
56            input: Vec::new().into_iter(),
57            projected: None,
58            exhausted: false,
59        }
60    }
61
62    fn next_input(&mut self, demanded: bool) -> ExecResult<Option<ResultRow>> {
63        loop {
64            if let Some(row) = self.input.next() {
65                return Ok(Some(row));
66            }
67            let Some(batch) = (if demanded {
68                self.child.next_with_demand(crate::physical::demand::ONE)?
69            } else {
70                self.child.next()?
71            }) else {
72                return Ok(None);
73            };
74            self.input = batch.into_result_rows().into_iter();
75        }
76    }
77}
78
79impl ProjectSet<'_> {
80    fn pull(&mut self, demand: Option<std::num::NonZeroUsize>) -> ExecResult<Option<Batch>> {
81        let batch_size = demand.map_or(DEFAULT_BATCH_SIZE, |rows| {
82            rows.get().min(DEFAULT_BATCH_SIZE)
83        });
84        if self.exhausted && self.projected.is_none() {
85            return Ok(None);
86        }
87
88        let mut rows = Vec::with_capacity(batch_size);
89        while rows.len() < batch_size {
90            if let Some(projected) = self.projected.as_mut() {
91                match projected.next() {
92                    Some(row) => {
93                        rows.push(row?);
94                        continue;
95                    }
96                    None => self.projected = None,
97                }
98            }
99
100            match self.next_input(demand.is_some())? {
101                Some(row) => self.projected = Some(self.projector.project(&row)?),
102                None => {
103                    self.exhausted = true;
104                    break;
105                }
106            }
107        }
108        if rows.is_empty() {
109            return Ok(None);
110        }
111        Ok(Some(Batch::new(self.schema.clone(), rows)))
112    }
113}
114
115impl PhysicalOperator for ProjectSet<'_> {
116    fn row_schema(&self) -> &RowSchema {
117        &self.schema
118    }
119
120    fn open(&mut self) -> ExecResult<()> {
121        self.input = Vec::new().into_iter();
122        self.projected = None;
123        self.exhausted = false;
124        self.child.open()
125    }
126
127    fn next(&mut self) -> ExecResult<Option<Batch>> {
128        self.pull(None)
129    }
130
131    fn next_with_demand(&mut self, rows: std::num::NonZeroUsize) -> ExecResult<Option<Batch>> {
132        self.pull(Some(rows))
133    }
134
135    fn close(&mut self) -> ExecResult<()> {
136        self.input = Vec::new().into_iter();
137        self.projected = None;
138        self.exhausted = true;
139        self.child.close()
140    }
141}
142
143/// Owned physical row stream produced for one input row without materializing named maps.
144pub type PhysicalProjectRows = crate::SQLRowStream<'static, PhysicalRow>;
145
146/// Engine seam for set-returning projections that keep rows in their physical layout.
147pub trait PhysicalSetProjector: Send {
148    fn project(&mut self, row: OwnedPhysicalRow) -> ExecResult<PhysicalProjectRows>;
149}
150
151impl<F> PhysicalSetProjector for F
152where
153    F: FnMut(OwnedPhysicalRow) -> ExecResult<PhysicalProjectRows> + Send,
154{
155    fn project(&mut self, row: OwnedPhysicalRow) -> ExecResult<PhysicalProjectRows> {
156        self(row)
157    }
158}
159
160/// Pulls physical input rows and expands each without crossing the named-row materialization boundary.
161pub struct PhysicalProjectSet<'a> {
162    child: Box<dyn PhysicalOperator + 'a>,
163    projector: Box<dyn PhysicalSetProjector + 'a>,
164    schema: RowSchema,
165    input: std::vec::IntoIter<OwnedPhysicalRow>,
166    projected: Option<PhysicalProjectRows>,
167    exhausted: bool,
168}
169
170impl<'a> PhysicalProjectSet<'a> {
171    pub fn new(
172        child: Box<dyn PhysicalOperator + 'a>,
173        schema: RowSchema,
174        projector: Box<dyn PhysicalSetProjector + 'a>,
175    ) -> Self {
176        Self {
177            child,
178            projector,
179            schema,
180            input: Vec::new().into_iter(),
181            projected: None,
182            exhausted: false,
183        }
184    }
185
186    fn next_input(&mut self, demanded: bool) -> ExecResult<Option<OwnedPhysicalRow>> {
187        loop {
188            if let Some(row) = self.input.next() {
189                return Ok(Some(row));
190            }
191            let Some(batch) = (if demanded {
192                self.child.next_with_demand(crate::physical::demand::ONE)?
193            } else {
194                self.child.next()?
195            }) else {
196                return Ok(None);
197            };
198            self.input = batch.into_owned_rows().into_iter();
199        }
200    }
201}
202
203impl PhysicalProjectSet<'_> {
204    fn close_projection(&mut self) -> ExecResult<()> {
205        self.projected
206            .take()
207            .map_or(Ok(()), |mut rows| rows.close().map_err(Into::into))
208    }
209
210    fn pull(&mut self, demand: Option<std::num::NonZeroUsize>) -> ExecResult<Option<Batch>> {
211        let batch_size = demand.map_or(DEFAULT_BATCH_SIZE, |rows| {
212            rows.get().min(DEFAULT_BATCH_SIZE)
213        });
214        if self.exhausted && self.projected.is_none() {
215            return Ok(None);
216        }
217
218        let mut rows = Vec::with_capacity(batch_size);
219        while rows.len() < batch_size {
220            if let Some(projected) = self.projected.as_mut() {
221                match projected.next() {
222                    Some(row) => {
223                        rows.push(row?);
224                        continue;
225                    }
226                    None => self.projected = None,
227                }
228            }
229
230            match self.next_input(demand.is_some())? {
231                Some(row) => self.projected = Some(self.projector.project(row)?),
232                None => {
233                    self.exhausted = true;
234                    break;
235                }
236            }
237        }
238        if rows.is_empty() {
239            return Ok(None);
240        }
241        Ok(Some(Batch::from_physical_rows(self.schema.clone(), rows)))
242    }
243}
244
245impl PhysicalOperator for PhysicalProjectSet<'_> {
246    fn row_schema(&self) -> &RowSchema {
247        &self.schema
248    }
249
250    fn open(&mut self) -> ExecResult<()> {
251        self.input = Vec::new().into_iter();
252        self.close_projection()?;
253        self.exhausted = false;
254        self.child.open()
255    }
256
257    fn next(&mut self) -> ExecResult<Option<Batch>> {
258        self.pull(None)
259    }
260
261    fn next_with_demand(&mut self, rows: std::num::NonZeroUsize) -> ExecResult<Option<Batch>> {
262        self.pull(Some(rows))
263    }
264
265    fn close(&mut self) -> ExecResult<()> {
266        self.input = Vec::new().into_iter();
267        let projected = self.close_projection();
268        self.exhausted = true;
269        crate::physical::with_cleanup(projected, self.child.close(), "close projection input")
270    }
271}
272
273#[cfg(test)]
274mod tests {
275    use super::*;
276    use crate::physical::run_to_rows;
277    use crate::scan::TableScan;
278    use crate::RowSchemaExecution;
279    use crate::{RowLockOrigin, RowProjectionValue};
280    use uqa_core::Value;
281
282    #[test]
283    fn expands_each_input_row_in_child_order() {
284        let input: ResultRow = [("n".into(), Value::Int(2))].into_iter().collect();
285        let child = TableScan::from_rows(vec!["n".into()], vec![input]);
286        let projector = |row: &ResultRow| -> ExecResult<ProjectRows> {
287            let Value::Int(end) = row.get("n").cloned().unwrap_or(Value::Null) else {
288                return Ok(Box::new(std::iter::empty()));
289            };
290            Ok(Box::new((1..=end).map(|value| {
291                Ok([("value".into(), Value::Int(value))].into_iter().collect())
292            })))
293        };
294        let mut project =
295            ProjectSet::new(Box::new(child), vec!["value".into()], Box::new(projector));
296        let (_, rows) = run_to_rows(&mut project).unwrap();
297        assert_eq!(rows.len(), 2);
298        assert_eq!(rows[0].get("value"), Some(&Value::Int(1)));
299        assert_eq!(rows[1].get("value"), Some(&Value::Int(2)));
300    }
301
302    #[test]
303    fn one_input_with_many_outputs_never_builds_an_unbounded_pending_queue() {
304        let input: ResultRow = [("n".into(), Value::Int(10_000))].into_iter().collect();
305        let child = TableScan::from_rows(vec!["n".into()], vec![input]);
306        let projector = |row: &ResultRow| -> ExecResult<ProjectRows> {
307            let Value::Int(end) = row.get("n").cloned().unwrap_or(Value::Null) else {
308                return Ok(Box::new(std::iter::empty()));
309            };
310            Ok(Box::new((0..end).map(|value| {
311                Ok([("value".into(), Value::Int(value))].into_iter().collect())
312            })))
313        };
314        let mut project =
315            ProjectSet::new(Box::new(child), vec!["value".into()], Box::new(projector));
316        project.open().unwrap();
317        let first = project.next().unwrap().unwrap();
318        assert_eq!(first.len(), DEFAULT_BATCH_SIZE);
319        let second = project.next().unwrap().unwrap();
320        assert_eq!(second.len(), DEFAULT_BATCH_SIZE);
321        project.close().unwrap();
322    }
323
324    #[test]
325    fn late_projector_error_is_not_converted_to_end_of_stream() {
326        let child = TableScan::from_rows(vec!["n".into()], vec![ResultRow::new()]);
327        let projector = |_row: &ResultRow| -> ExecResult<ProjectRows> {
328            Ok(Box::new(
329                vec![
330                    Ok([("value".into(), Value::Int(1))].into_iter().collect()),
331                    Err(crate::ExecError::Other("injected projector failure".into())),
332                ]
333                .into_iter(),
334            ))
335        };
336        let mut project =
337            ProjectSet::new(Box::new(child), vec!["value".into()], Box::new(projector));
338        project.open().unwrap();
339        let error = project.next().unwrap_err();
340        assert!(error.to_string().contains("injected projector failure"));
341    }
342
343    #[test]
344    fn physical_projection_preserves_row_lineage_without_named_materialization() {
345        let input_schema = RowSchema::new(vec!["unused".into(), "value".into()]);
346        let input_row = PhysicalRow::from_values(vec![
347            Value::Str("unused payload".repeat(64)),
348            Value::Str("shared value".repeat(64)),
349        ])
350        .with_lock_origin(RowLockOrigin::new("source", "public.source", 7));
351        let child = TableScan::from_physical_rows(input_schema, vec![input_row]);
352        let projector = |row: OwnedPhysicalRow| -> ExecResult<PhysicalProjectRows> {
353            let slot = row.schema.physical_slot(1).unwrap();
354            Ok(crate::SQLRowStream::new(std::iter::once(Ok(row
355                .row
356                .project_with_values([RowProjectionValue::InputSlot(
357                    slot,
358                )])))))
359        };
360        let output_schema = RowSchema::new(vec!["value".into()]);
361        let mut project =
362            PhysicalProjectSet::new(Box::new(child), output_schema.clone(), Box::new(projector));
363
364        project.open().unwrap();
365        let output = project.next().unwrap().unwrap();
366        assert_eq!(output.rows.len(), 1);
367        assert_eq!(
368            output_schema.view(&output.rows[0]).value_at(0),
369            Some(&Value::Str("shared value".repeat(64)))
370        );
371        assert_eq!(output.rows[0].lock_origins()[0].doc_id, 7);
372        assert!(project.next().unwrap().is_none());
373        project.close().unwrap();
374    }
375}