Skip to main content

uqa_execution/relational/
project.rs

1//
2// Unified Query Algebra
3//
4// Copyright (c) 2023-2026 Cognica, Inc.
5//
6
7//! Scalar projection and star expansion.
8
9use uqa_core::Value;
10
11use super::{
12    BackwardScanSupport, Batch, DefaultExpressionEvaluator, ExecResult, PhysicalOperator,
13    PhysicalScanDirection, RowSchema, SQLParam, ScalarExpr, SharedExpressionEvaluator,
14};
15use crate::batch::ProjectedSlot;
16use crate::spill::{SpillBatchWriter, SpillDrain};
17
18pub use uqa_sql::plan::ProjectionTarget;
19
20mod production;
21#[cfg(test)]
22mod tests;
23
24/// Per-row scalar projection. Each `(alias, expr)` pair is evaluated
25/// against the input row and written under `alias` in the output. The
26/// child schema is replaced with the output aliases.
27pub struct Project<'a> {
28    child: Box<dyn PhysicalOperator + 'a>,
29    input_pending: crate::physical::demand::PendingRows,
30    computed: crate::scalar::PreparedExpressions<uqa_core::memory::Produced<Vec<ScalarExpr>>>,
31    evaluator: SharedExpressionEvaluator<'a>,
32    schema: RowSchema,
33    ordering: Vec<crate::PhysicalOrder>,
34    work_mem: Option<usize>,
35    pending: Option<SpillDrain>,
36}
37
38fn projection_layout(
39    input: &RowSchema,
40    projections: Vec<(ProjectionTarget, ScalarExpr)>,
41    evaluator: &SharedExpressionEvaluator<'_>,
42    pass_through: bool,
43) -> (RowSchema, Vec<ScalarExpr>) {
44    let mut projected = Vec::new();
45    let mut projected_internal = Vec::new();
46    let mut computed = Vec::new();
47    for (target, expression) in projections {
48        if let ScalarExpr::QualifiedStar(qualifier) = &expression {
49            let ProjectionTarget::Column(_) = target else {
50                unreachable!("an internal projection target cannot expand a qualified star");
51            };
52            if pass_through {
53                continue;
54            }
55            for (column, logical, _, ty) in input.qualified_star_position_layout(qualifier) {
56                if logical.is_some_and(|position| !evaluator.star_position_visible(input, position))
57                {
58                    continue;
59                }
60                let slot = logical
61                    .and_then(|logical| input.physical_slot(logical))
62                    .or_else(|| {
63                        input.physical_slot_for_identity(&crate::ColumnIdentity::qualified(
64                            qualifier, &column,
65                        ))
66                    });
67                let collation = logical.map_or_else(
68                    || input.qualified_collation(qualifier, &column),
69                    |position| input.column_collation(position),
70                );
71                projected.push((
72                    column,
73                    ty,
74                    ProjectedSlot::BoundInput {
75                        physical: slot,
76                        collation,
77                    },
78                ));
79            }
80            continue;
81        }
82        if matches!(expression, ScalarExpr::Star) {
83            let ProjectionTarget::Column(_) = target else {
84                unreachable!("an internal projection target cannot expand a star");
85            };
86            if pass_through {
87                continue;
88            }
89            for (logical, column) in input.iter().enumerate() {
90                if !evaluator.star_position_visible(input, logical) {
91                    continue;
92                }
93                projected.push((
94                    input.public_name(logical).unwrap_or(column).to_string(),
95                    input.column_type(logical).cloned(),
96                    ProjectedSlot::BoundInput {
97                        physical: input.physical_slot(logical),
98                        collation: input.column_collation(logical),
99                    },
100                ));
101            }
102            continue;
103        }
104
105        let ty = evaluator.expression_type(&expression, input).ok().flatten();
106        let source = if let Some(source) = input.projection_input(&expression) {
107            source
108        } else {
109            let position = computed.len();
110            let collation = input.input_collation(&expression);
111            computed.push(expression);
112            ProjectedSlot::Computed {
113                position,
114                collation,
115            }
116        };
117        match target {
118            ProjectionTarget::Column(name) => projected.push((name, ty, source)),
119            ProjectionTarget::Internal(column) => {
120                projected_internal.push((column, ty, source));
121            }
122        }
123    }
124    let computed_count = computed.len();
125    (
126        RowSchema::project_with_sources(
127            input,
128            projected,
129            projected_internal,
130            computed_count,
131            pass_through,
132        ),
133        computed,
134    )
135}
136
137impl Project<'static> {
138    pub fn new(
139        child: Box<dyn PhysicalOperator>,
140        projections: Vec<(String, ScalarExpr)>,
141        params: Vec<SQLParam>,
142    ) -> Self {
143        Self::with_evaluator(
144            child,
145            projections,
146            DefaultExpressionEvaluator::shared(params),
147        )
148    }
149
150    pub fn with_targets(
151        child: Box<dyn PhysicalOperator>,
152        projections: Vec<(ProjectionTarget, ScalarExpr)>,
153        params: Vec<SQLParam>,
154    ) -> Self {
155        Self::with_target_evaluator(
156            child,
157            projections,
158            DefaultExpressionEvaluator::shared(params),
159        )
160    }
161
162    /// Variant that keeps every input column in the output and appends
163    /// the projections at the end. Used by aggregate / window paths.
164    pub fn appending(
165        child: Box<dyn PhysicalOperator>,
166        projections: Vec<(String, ScalarExpr)>,
167        params: Vec<SQLParam>,
168    ) -> Self {
169        Self::appending_with_evaluator(
170            child,
171            projections,
172            DefaultExpressionEvaluator::shared(params),
173        )
174    }
175
176    pub fn appending_targets(
177        child: Box<dyn PhysicalOperator>,
178        projections: Vec<(ProjectionTarget, ScalarExpr)>,
179        params: Vec<SQLParam>,
180    ) -> Self {
181        Self::appending_target_evaluator(
182            child,
183            projections,
184            DefaultExpressionEvaluator::shared(params),
185        )
186    }
187}
188
189impl<'a> Project<'a> {
190    pub fn with_evaluator(
191        child: Box<dyn PhysicalOperator + 'a>,
192        projections: Vec<(String, ScalarExpr)>,
193        evaluator: SharedExpressionEvaluator<'a>,
194    ) -> Self {
195        Self::with_target_evaluator(
196            child,
197            projections
198                .into_iter()
199                .map(|(name, expression)| (ProjectionTarget::Column(name), expression))
200                .collect(),
201            evaluator,
202        )
203    }
204
205    pub fn with_target_evaluator(
206        child: Box<dyn PhysicalOperator + 'a>,
207        projections: Vec<(ProjectionTarget, ScalarExpr)>,
208        evaluator: SharedExpressionEvaluator<'a>,
209    ) -> Self {
210        let projections = projections
211            .into_iter()
212            .map(|(target, expression)| {
213                (
214                    target,
215                    evaluator.bind_type_introspection(expression, child.row_schema()),
216                )
217            })
218            .collect::<Vec<_>>();
219        let (schema, computed) =
220            projection_layout(child.row_schema(), projections, &evaluator, false);
221        Self {
222            child,
223            input_pending: crate::physical::demand::PendingRows::default(),
224            computed: crate::scalar::PreparedExpressions::produced_scalars(
225                uqa_core::memory::ProductionControl::uncontrolled()
226                    .finish(computed, None)
227                    .expect("ordinary projection expressions"),
228                &uqa_core::memory::ProductionControl::uncontrolled(),
229            )
230            .expect("ordinary projection call state"),
231            evaluator,
232            schema,
233            ordering: Vec::new(),
234            work_mem: None,
235            pending: None,
236        }
237    }
238
239    pub fn appending_with_evaluator(
240        child: Box<dyn PhysicalOperator + 'a>,
241        projections: Vec<(String, ScalarExpr)>,
242        evaluator: SharedExpressionEvaluator<'a>,
243    ) -> Self {
244        Self::appending_target_evaluator(
245            child,
246            projections
247                .into_iter()
248                .map(|(name, expression)| (ProjectionTarget::Column(name), expression))
249                .collect(),
250            evaluator,
251        )
252    }
253
254    pub fn appending_target_evaluator(
255        child: Box<dyn PhysicalOperator + 'a>,
256        projections: Vec<(ProjectionTarget, ScalarExpr)>,
257        evaluator: SharedExpressionEvaluator<'a>,
258    ) -> Self {
259        let projections = projections
260            .into_iter()
261            .map(|(target, expression)| {
262                (
263                    target,
264                    evaluator.bind_type_introspection(expression, child.row_schema()),
265                )
266            })
267            .collect::<Vec<_>>();
268        let ordering = child
269            .output_ordering()
270            .iter()
271            .take_while(|order| {
272                projections.iter().all(|(target, expression)| {
273                    let ProjectionTarget::Column(name) = target else {
274                        return true;
275                    };
276                    child
277                        .row_schema()
278                        .columns()
279                        .iter()
280                        .position(|column| column == name)
281                        != Some(order.position)
282                        || matches!(expression, ScalarExpr::Star | ScalarExpr::QualifiedStar(_))
283                        || crate::order_expression_position(child.row_schema(), expression)
284                            == Some(order.position)
285                })
286            })
287            .cloned()
288            .collect();
289        let (schema, computed) =
290            projection_layout(child.row_schema(), projections, &evaluator, true);
291        Self {
292            child,
293            input_pending: crate::physical::demand::PendingRows::default(),
294            computed: crate::scalar::PreparedExpressions::produced_scalars(
295                uqa_core::memory::ProductionControl::uncontrolled()
296                    .finish(computed, None)
297                    .expect("ordinary projection expressions"),
298                &uqa_core::memory::ProductionControl::uncontrolled(),
299            )
300            .expect("ordinary projection call state"),
301            evaluator,
302            schema,
303            ordering,
304            work_mem: None,
305            pending: None,
306        }
307    }
308
309    /// Retain a scalar evaluation boundary without splitting batches for a slot-only projection.
310    pub(crate) fn with_row_at_a_time_input(mut self) -> Self {
311        if !self.computed.is_empty() {
312            self.child = Box::new(crate::query::row_at_a_time::RowAtATime::new(self.child));
313        }
314        self
315    }
316
317    /// Bound evaluated output while retaining the original input-batch evaluation boundary.
318    pub fn with_memory_budget(mut self, bytes: usize) -> Self {
319        self.work_mem = Some(bytes);
320        self
321    }
322
323    fn prepare_batch(&mut self, batch: Batch) -> ExecResult<Option<Batch>> {
324        let Some(budget) = self
325            .work_mem
326            .filter(|_| !self.computed.is_empty() && batch.rows.len() > 1)
327        else {
328            return self.project_batch(batch).map(Some);
329        };
330        self.project_spilled_batch(batch, budget)
331    }
332
333    fn project_spilled_batch(&mut self, batch: Batch, budget: usize) -> ExecResult<Option<Batch>> {
334        // Finish every expression in this input batch before exposing its first result. Only the storage of those evaluated rows changes.
335        let mut output = crate::SpillBuffer::new(budget / 2);
336        let mut writer = SpillBatchWriter::new(self.schema.clone())?;
337        for row in batch.rows {
338            let values = self.project_values(&batch.schema, &row)?;
339            writer.push(&mut output, row.append_values(values))?;
340        }
341        writer.finish(&mut output)?;
342        self.pending = Some(output.drain()?);
343        self.next_pending()
344    }
345
346    fn next_pending(&mut self) -> ExecResult<Option<Batch>> {
347        if self.pending.is_some() {
348            if let Err(error) = uqa_sql::expr::check_statement_cancellation() {
349                self.pending = None;
350                return Err(uqa_sql::SQLError::from(error).into());
351            }
352        }
353        let batch = self.pending.as_mut().and_then(Iterator::next).transpose();
354        if !matches!(batch, Ok(Some(_))) {
355            self.pending = None;
356        }
357        batch
358    }
359
360    fn project_values(
361        &self,
362        schema: &RowSchema,
363        row: &crate::PhysicalRow,
364    ) -> ExecResult<Vec<Value>> {
365        let mut values = Vec::with_capacity(self.computed.len());
366        for expression in self.computed.iter() {
367            values.push(self.evaluator.evaluate_physical_with_function_states(
368                expression,
369                schema,
370                row,
371                self.computed.calls(),
372            )?);
373        }
374        Ok(values)
375    }
376
377    fn project_batch(&self, batch: Batch) -> ExecResult<Batch> {
378        if self.computed.is_empty() {
379            return Ok(Batch::from_physical_rows(self.schema.clone(), batch.rows));
380        }
381        let mut out = Vec::with_capacity(batch.rows.len());
382        for row in batch.rows {
383            let values = self.project_values(&batch.schema, &row)?;
384            out.push(row.append_values(values));
385        }
386        Ok(Batch::from_physical_rows(self.schema.clone(), out))
387    }
388}
389
390impl Project<'_> {
391    fn pull(&mut self, demand: Option<std::num::NonZeroUsize>) -> ExecResult<Option<Batch>> {
392        if let Some(batch) = self.next_pending()? {
393            return Ok(Some(batch));
394        }
395        let Some(batch) = self.input_pending.pull(self.child.as_mut(), demand)? else {
396            return Ok(None);
397        };
398        self.prepare_batch(batch)
399    }
400}
401
402impl PhysicalOperator for Project<'_> {
403    fn row_schema(&self) -> &RowSchema {
404        &self.schema
405    }
406
407    fn supports_batched_consumption(&self) -> bool {
408        self.computed.is_empty() && self.child.supports_batched_consumption()
409    }
410
411    fn collect_materialized(
412        &mut self,
413        sink: &mut crate::query::output::MaterializedResultSink<'_>,
414    ) -> ExecResult<bool> {
415        if !self.computed.is_empty()
416            || self.schema.physical_width() != self.child.row_schema().physical_width()
417        {
418            return Ok(false);
419        }
420        self.child.collect_materialized(sink)
421    }
422
423    fn output_ordering(&self) -> &[crate::PhysicalOrder] {
424        &self.ordering
425    }
426
427    fn backward_scan_support(&self) -> BackwardScanSupport {
428        let child = self.child.backward_scan_support();
429        if child == BackwardScanSupport::Native || self.computed.is_empty() {
430            child
431        } else {
432            BackwardScanSupport::Unsupported
433        }
434    }
435
436    fn open(&mut self) -> ExecResult<()> {
437        self.input_pending.clear();
438        self.pending = None;
439        self.child.open()
440    }
441
442    fn next(&mut self) -> ExecResult<Option<Batch>> {
443        self.pull(None)
444    }
445
446    fn next_with_demand(&mut self, rows: std::num::NonZeroUsize) -> ExecResult<Option<Batch>> {
447        self.pull(Some(rows))
448    }
449
450    fn next_direction(&mut self, direction: PhysicalScanDirection) -> ExecResult<Option<Batch>> {
451        if let Some(batch) = self.next_pending()? {
452            return Ok(Some(batch));
453        }
454        let Some(batch) = self.child.next_direction(direction)? else {
455            return Ok(None);
456        };
457        self.prepare_batch(batch)
458    }
459
460    fn rewind(&mut self) -> ExecResult<()> {
461        self.input_pending.clear();
462        self.pending = None;
463        self.child.rewind()
464    }
465
466    fn close(&mut self) -> ExecResult<()> {
467        self.input_pending.clear();
468        self.pending = None;
469        self.child.close()
470    }
471}
472
473// -------------------------------------------------------------------------
474// Sort
475// -------------------------------------------------------------------------