rudb_bind/binder.rs
1//! From an `Ast` to a `Plan`.
2//!
3//! The binder walks the written query once, in the order the operators end up in rather than the
4//! order the clauses are written in, which is `FROM`, `WHERE`, `GROUP BY`, `HAVING`, `SELECT`,
5//! `DISTINCT`, `ORDER BY`, `LIMIT`. That order is not a stylistic choice: it is the reason `WHERE`
6//! cannot see an output alias and `HAVING` cannot see a column that was not grouped, and doing it
7//! in any other order means special casing both of those instead of getting them for free.
8//!
9//! Two things leave here settled that nothing downstream reconsiders. Every column is a table index
10//! and a position rather than a name, so the optimizer never has to ask which `id` a name meant.
11//! And every expression has a type, with the casts that make the types line up already written into
12//! the plan as [`Expr::Cast`] nodes, so an executor never has to decide what a comparison between
13//! an `INTEGER` and a `BIGINT` does.
14
15use std::sync::Arc;
16
17use rudb_catalog::{Catalog, Entry, FileStamp, QualifiedName, same_name};
18use rudb_common::bounds::Zones;
19use rudb_common::{
20 Error, Field, LogicalType, Result, Semantics, Session, ShowBehavior, Span, Stat, Value,
21};
22use rudb_functions::{
23 Columns, FILE_ROW_NUMBER, Footers, FunctionKind, Given, Resolved, TableFunction, csv_fields,
24 csv_given, files, is_file, is_pattern, kind_of, parquet_footers, parquet_outline, resolve,
25 resolve_pragma, resolve_table,
26};
27use rudb_kernels::{percentage, row_count};
28use rudb_parse::ast::{self, Ast, Distinct, LiteralKind, Nulls, Order, Quantifier, SetOp};
29use rudb_parse::{NONE, identifier_parts, parse_ast_with_case};
30use rudb_plan::{
31 Bound, BuildSide, ColumnBinding, ConjunctionOp, Expr, ExprRef, JoinKind, Node, NodeRef, Plan,
32 SetOpKind, Share, SortKey, WindowBound, WindowExclude, WindowFrame, WindowUnit,
33};
34
35use crate::expr::{describe, has_aggregate};
36use crate::fold;
37use crate::parameters::Parameters;
38use crate::scope::{Scope, Visible};
39
40/// Binds a parsed statement against a catalog.
41///
42/// # Errors
43///
44/// If the script does not hold exactly one statement, if a name does not resolve, if a type does
45/// not work out, or if the query uses something M0 does not bind yet.
46pub fn bind(ast: &Ast, catalog: &Catalog) -> Result<Plan> {
47 bind_with(ast, catalog, &Parameters::new(), &Session::new())
48}
49
50/// Binds a parsed query against a catalog, with values for its parameters and its settings.
51///
52/// The session is what `current_setting()` reads, and a caller with no database behind it passes an
53/// empty one, which makes every setting name unrecognized rather than making up an answer.
54///
55/// # Errors
56///
57/// Everything [`bind`] reports, plus an error for a parameter that was given no value.
58pub fn bind_with(
59 ast: &Ast,
60 catalog: &Catalog,
61 parameters: &Parameters,
62 session: &Session,
63) -> Result<Plan> {
64 let query = match ast.statements.as_slice() {
65 [ast::Statement::Query(query)] => *query,
66 [] => return Err(Error::binder("no statement to bind")),
67 // One statement that is not a query is its own answer. Reporting it as a script of several
68 // reads as a count being wrong, and the count is right.
69 [_] => return Err(Error::not_implemented("a statement that is not a query")),
70 _ => return Err(Error::not_implemented("a script of more than one statement")),
71 };
72 let mut binder = Binder::with(catalog, parameters, session);
73 let (root, _) = binder.bind_query(ast, query)?;
74 let mut plan = binder.into_plan();
75 plan.set_root(root);
76 plan.validate()?;
77 Ok(plan)
78}
79
80/// Parses and binds one query, which is the whole front end in one call.
81///
82/// # Errors
83///
84/// Anything the parser or the binder reports.
85pub fn bind_sql(query: &str, catalog: &Catalog) -> Result<Plan> {
86 bind_sql_with(query, catalog, &Session::new())
87}
88
89/// Parses and binds one query, with the settings a call to `current_setting()` reads.
90///
91/// # Errors
92///
93/// Anything the parser or the binder reports.
94pub fn bind_sql_with(query: &str, catalog: &Catalog, session: &Session) -> Result<Plan> {
95 let ast = parse_ast_with_case(query, session.semantics().identifier_case())?;
96 bind_with(&ast, catalog, &Parameters::new(), session)
97}
98
99/// What an aggregating select block has decided so far.
100#[derive(Debug)]
101pub(crate) struct Aggregation {
102 /// The table index the aggregate's output binds against.
103 pub(crate) index: u32,
104 /// The group expressions, over the input, which are the first output columns.
105 pub(crate) groups: Vec<ExprRef>,
106 /// The aggregate calls found so far, which follow the groups in the output.
107 pub(crate) aggregates: Vec<ExprRef>,
108}
109
110/// One run of window calls that agree on where the rows come from and in what order.
111///
112/// The run is the unit the plan has an operator for, so two calls that write the same partition,
113/// the same order and the same frame are one operator and one sort, and a third that writes a
114/// different order is a second operator stacked on the first. Nothing here merges runs that only
115/// look compatible, because a window is evaluated over the rows the operator below it produced and
116/// deciding two runs are the same is the optimizer's job rather than the binder's.
117#[derive(Debug)]
118pub(crate) struct WindowRun {
119 /// The table index the run's result columns bind against.
120 index: u32,
121 /// What divides the input into independent partitions.
122 partition: Vec<ExprRef>,
123 /// The order within a partition.
124 order: Vec<SortKey>,
125 /// The frame every call in the run shares.
126 frame: WindowFrame,
127 /// The calls, in the order their columns are appended.
128 calls: Vec<ExprRef>,
129}
130
131/// One window call as it was written, before any of it has been bound.
132///
133/// These six travel together from the parser all the way to the run they end up filed under, and
134/// carrying them as one thing keeps the call that binds them readable.
135pub(crate) struct WindowCall<'a> {
136 /// The function name, as written and not yet resolved.
137 pub(crate) name: &'a str,
138 /// The arguments, which may include a star that only `count` is allowed to be given.
139 pub(crate) args: &'a [ast::ExprRef],
140 /// Whether `DISTINCT` was written inside the parens.
141 pub(crate) distinct: bool,
142 /// The `FILTER (WHERE ...)` predicate, which is written before the `OVER`, or `NONE`.
143 pub(crate) filter: ast::ExprRef,
144 /// Whether `IGNORE NULLS` was written inside the parens, which is where DuckDB puts it.
145 pub(crate) ignore_nulls: bool,
146 /// The `ORDER BY` written inside the parens, which says what order the call reads the rows of
147 /// its frame in and is a different clause from the one in the `OVER`.
148 pub(crate) order: ast::Slice,
149 /// The `OVER`, which the parser has already resolved against any `WINDOW` clause.
150 pub(crate) spec: ast::WindowRef,
151}
152
153/// Everything inside one window call once it is bound, which is what decides its run.
154struct WindowParts {
155 /// The arguments, before the casts the resolved signature asks for.
156 args: Vec<ExprRef>,
157 /// What divides the input into independent partitions.
158 partition: Vec<ExprRef>,
159 /// The order within a partition.
160 order: Vec<SortKey>,
161 /// The order the call reads the rows of its frame in, which is the `ORDER BY` written inside
162 /// the brackets rather than the one in the `OVER` and is empty far more often than not.
163 inner: Vec<SortKey>,
164 /// The frame, with both ends and the exclusion.
165 frame: WindowFrame,
166}
167
168/// What opening the files behind a table function call said about them.
169///
170/// The answers travel together because they come out of the same footer. A Parquet file states its
171/// columns, its row count and its statistics in the same few kilobytes at the end of it, so a
172/// binder that has read one has read all of them, and splitting them into four arguments would
173/// mean four ways to forget one.
174#[derive(Debug)]
175struct Read {
176 /// The columns the call produces, in the order the file stores them.
177 fields: Vec<Field>,
178 /// How many rows all of the files hold, where anybody counted.
179 rows: Stat<u64>,
180 /// How many distinct values a column holds, by name, for the columns anybody counted.
181 distincts: Vec<(String, Stat<u64>)>,
182 /// The bounds the files keep per part of themselves, where anything can answer for them.
183 zones: Option<Arc<dyn Zones>>,
184}
185
186impl Read {
187 /// Columns that came from somewhere other than a file, so nothing counted anything.
188 fn uncounted(fields: Vec<Field>) -> Self {
189 Self { fields, rows: Stat::Unknown, distincts: Vec::new(), zones: None }
190 }
191}
192
193/// A materialised `WITH` definition that has been bound and can be read by name.
194#[derive(Debug)]
195struct Materialized {
196 /// Which written definition this is, as an index into `Ast::ctes`.
197 written: u32,
198 /// The number the plan uses to pair a read with what it reads.
199 cte: u32,
200 /// The name it was written with, which is the table name a read is reachable through.
201 name: String,
202 /// What it produces, in order, under the declared names when a column list was written.
203 fields: Vec<Field>,
204}
205
206#[derive(Debug)]
207pub(crate) struct PendingSubquery {
208 pub(crate) node: NodeRef,
209 pub(crate) kind: JoinKind,
210 pub(crate) conditions: Vec<ExprRef>,
211 pub(crate) dependent: bool,
212 /// The outer columns the query's body read, which is what `dependent` counts.
213 ///
214 /// Kept rather than reduced to the flag because a join's `ON` has to decide which of its two
215 /// inputs the query is joined into, and the answer is the side those columns come from. A
216 /// query that reads neither side can go on either.
217 pub(crate) reads: Vec<ColumnBinding>,
218 /// The table index this query's join adds to the rows it is joined into.
219 ///
220 /// Kept so that a `HAVING` which reads one of these can say which columns came from a query
221 /// joined above the grouping rather than from the table underneath it. Those columns are not
222 /// the table's and the grouping rule has nothing to say about them.
223 pub(crate) index: u32,
224 /// Whether the query was written inside an aggregate call's argument or its `FILTER`.
225 ///
226 /// One written there is read once per row going into the aggregate, so it has to be joined in
227 /// underneath the grouping however uncorrelated it is. Every other query a grouped block writes
228 /// is one row for the whole block and is lifted over the grouping instead, which is what
229 /// [`Binder::lift_over_aggregate`] decides.
230 pub(crate) inside_aggregate: bool,
231}
232
233/// Which input of a join a query written in that join's `ON` is joined into.
234#[derive(Debug, Clone, Copy, PartialEq, Eq)]
235enum Side {
236 Left,
237 Right,
238}
239
240/// The state one binding run carries.
241#[derive(Debug)]
242pub(crate) struct Binder<'a> {
243 catalog: &'a Catalog,
244 /// What the parameters were given, empty for a statement that is not prepared.
245 pub(crate) parameters: &'a Parameters,
246 /// What the settings are now, which is what `current_setting()` folds to.
247 pub(crate) session: &'a Session,
248 /// Meaning-changing choices copied once and resolved into the plan above execution.
249 pub(crate) semantics: Semantics,
250 plan: Plan,
251 next_index: u32,
252 /// Source range inherited by plan objects built for the current AST expression or query.
253 pub(crate) current_span: Span,
254 /// Set while a select block aggregates, which changes what a bare column means.
255 pub(crate) aggregation: Option<Aggregation>,
256 /// A grouped block may need stored column order to close groups while it scans. Other queries
257 /// leave the summaries in the file instead of reading every column's section while binding.
258 want_ascending: bool,
259 /// Set while an aggregate's own arguments are being bound, so nesting is caught.
260 pub(crate) in_aggregate: bool,
261 /// Set while an aggregate's `FILTER` is being bound, which is refused its own aggregate.
262 pub(crate) in_filter: bool,
263 /// The window runs this select block has collected, in the order they were first written.
264 pub(crate) windows: Vec<WindowRun>,
265 /// Set while a window call's own arguments and keys are being bound, so nesting is caught.
266 pub(crate) in_window: bool,
267 /// Uncorrelated scalar queries waiting to be joined into the select block that uses them.
268 pub(crate) scalar_subqueries: Vec<PendingSubquery>,
269 /// Table indices of the queries this block will join in above its grouping, not below it.
270 ///
271 /// Only ever set while a `HAVING` is being rewritten over the aggregate. A column from one of
272 /// these is not a column of the grouped table, so the rule about grouping every column does not
273 /// reach it, and the join that produces it goes on top of the `Aggregate` rather than under it.
274 pub(crate) joined_above: Vec<u32>,
275 pub(crate) outer_scopes: Vec<Scope>,
276 /// Which of the outer scopes are a FROM entry's left neighbours rather than an enclosing query.
277 ///
278 /// The two are resolved the same way and refused differently. An aggregate may read a column of
279 /// the query it is written in and may not read one a LATERAL brought in from the left, so the
280 /// check needs to know which scope the name came out of. Each entry is a position in
281 /// `outer_scopes`.
282 pub(crate) lateral_scopes: Vec<usize>,
283 pub(crate) correlations: Vec<Vec<ColumnBinding>>,
284 /// The lambdas whose bodies are being bound, innermost last. See `crate::lambda`.
285 pub(crate) lambda_frames: Vec<crate::lambda::Frame>,
286 /// Where we are, for an error message that says which clause the writer should look at.
287 pub(crate) clause: &'static str,
288 /// Whether a Parquet file that could be read through a native mirror is bound from its outline
289 /// alone, which is the columns and the row count and none of the row groups.
290 ///
291 /// Set by a bind whose plan is thrown away: a `CREATE VIEW`, and the first bind of a query that
292 /// may be bound again once its mirrors are in. A plan bound this way knows no bounds and no
293 /// distinct counts for the file, so the caller must not run it, and every read it did this for
294 /// asked for a mirror, which is how the caller knows to bind again. See
295 /// [`rudb_parquet::Outline`].
296 pub(crate) outlined: bool,
297 /// The views whose bodies are open on the stack, which is what catches a cycle.
298 expanding: Vec<String>,
299 /// The materialised `WITH` definitions whose bodies are being bound, innermost last.
300 ///
301 /// A stack rather than a map from what was written, because a plain `WITH` is put into every
302 /// place it is named, so a materialised one written inside a plain one is bound once per use
303 /// and each of those is a materialisation of its own with a number of its own.
304 materialized: Vec<Materialized>,
305 /// How many materialisations have been numbered, which is where the next number comes from.
306 next_cte: u32,
307 /// When this statement started, read once and kept, which is what `now()` folds to.
308 started: Option<i64>,
309}
310
311impl<'a> Binder<'a> {
312 pub(crate) fn with(
313 catalog: &'a Catalog,
314 parameters: &'a Parameters,
315 session: &'a Session,
316 ) -> Self {
317 Self {
318 catalog,
319 parameters,
320 session,
321 semantics: session.semantics(),
322 plan: Plan::new(),
323 next_index: 0,
324 current_span: Span::new(0, 0),
325 aggregation: None,
326 want_ascending: false,
327 in_aggregate: false,
328 in_filter: false,
329 windows: Vec::new(),
330 in_window: false,
331 scalar_subqueries: Vec::new(),
332 joined_above: Vec::new(),
333 outer_scopes: Vec::new(),
334 lateral_scopes: Vec::new(),
335 correlations: Vec::new(),
336 lambda_frames: Vec::new(),
337 clause: "SELECT clause",
338 outlined: false,
339 expanding: Vec::new(),
340 materialized: Vec::new(),
341 next_cte: 0,
342 started: None,
343 }
344 }
345
346 pub(crate) fn catalog(&self) -> &Catalog {
347 self.catalog
348 }
349
350 /// When this statement started, in microseconds since the epoch.
351 ///
352 /// Read from the clock the first time something asks and kept after that, so a query that
353 /// writes `now()` twice gets one answer for both. That is what the pin does and what it reports
354 /// in the `stability` column of `duckdb_functions()`, where every one of these is
355 /// `CONSISTENT_WITHIN_QUERY`. A query that never asks never reads the clock.
356 pub(crate) fn instant(&mut self) -> i64 {
357 *self.started.get_or_insert_with(crate::context::micros_now)
358 }
359
360 pub(crate) fn plan(&self) -> &Plan {
361 &self.plan
362 }
363
364 pub(crate) fn plan_mut(&mut self) -> &mut Plan {
365 &mut self.plan
366 }
367
368 pub(crate) fn add_expr(&mut self, expr: Expr, ty: LogicalType) -> ExprRef {
369 self.plan.add_expr_at(expr, ty, self.current_span)
370 }
371
372 pub(crate) fn add_constant(&mut self, value: Value) -> ExprRef {
373 let ty = value.logical_type();
374 let reference = self.plan.add_value(value);
375 self.plan.add_expr_at(Expr::Constant(reference), ty, self.current_span)
376 }
377
378 pub(crate) fn add_node(&mut self, node: Node) -> NodeRef {
379 self.plan.add_node_at(node, self.current_span)
380 }
381
382 pub(crate) fn into_plan(self) -> Plan {
383 self.plan
384 }
385
386 /// A table index nothing else has.
387 pub(crate) fn fresh_index(&mut self) -> u32 {
388 let index = self.next_index;
389 self.next_index += 1;
390 index
391 }
392
393 /// A reference to one column of an operator's output.
394 fn column(&mut self, index: u32, position: usize, ty: LogicalType) -> ExprRef {
395 let binding = ColumnBinding::new(index, position as u32);
396 self.plan.add_expr(Expr::Column(binding), ty)
397 }
398
399 /// Joins scalar query results into the row stream that contains their expressions.
400 fn attach_scalar_subqueries(&mut self, mut input: NodeRef) -> NodeRef {
401 let subqueries = std::mem::take(&mut self.scalar_subqueries);
402 for pending in subqueries {
403 input = self.attach_subquery(input, pending);
404 }
405 input
406 }
407
408 /// Joins one query's result into a row stream, which is where its columns come from.
409 ///
410 /// Split out from [`Self::attach_scalar_subqueries`] because a join's `ON` does not attach its
411 /// queries to the rows the whole `FROM` produced. It attaches them to one of the join's two
412 /// inputs, since a join condition is evaluated by the join and can only read what the join was
413 /// given.
414 fn attach_subquery(&mut self, input: NodeRef, pending: PendingSubquery) -> NodeRef {
415 let PendingSubquery {
416 node: mut right,
417 kind,
418 conditions,
419 dependent,
420 reads: _,
421 index: _,
422 inside_aggregate: _,
423 } = pending;
424 if kind == JoinKind::Single && !self.semantics.scalar_subquery_error_on_multiple_rows() {
425 right = self.add_node(Node::Limit {
426 input: right,
427 count: Bound::Rows(1),
428 offset: Bound::Rows(0),
429 });
430 }
431 let conditions = self.plan.add_expr_list(&conditions);
432 if dependent {
433 self.add_node(Node::DependentJoin { left: input, right, kind, conditions })
434 } else {
435 self.add_node(Node::Join {
436 left: input,
437 right,
438 kind,
439 conditions,
440 build: BuildSide::default(),
441 })
442 }
443 }
444
445 // ---------------------------------------------------------------- queries
446
447 pub(crate) fn bind_query(
448 &mut self,
449 ast: &Ast,
450 query: ast::QueryRef,
451 ) -> Result<(NodeRef, Scope)> {
452 let span = ast.query_span(query);
453 let outer = std::mem::replace(&mut self.current_span, span);
454 let result =
455 self.bind_query_inner(ast, query).map_err(|error| error.with_fallback_span(span));
456 self.current_span = outer;
457 result
458 }
459
460 fn bind_query_inner(&mut self, ast: &Ast, query: ast::QueryRef) -> Result<(NodeRef, Scope)> {
461 let written = ast.query(query);
462 if written.ctes.is_empty() {
463 return self.bind_body(ast, &written);
464 }
465 // The names a query introduces are gone again once it is bound, and they go whether the
466 // binding worked or not, which is why the stack is cut back here rather than at the end of
467 // the call that pushed onto it.
468 let depth = self.materialized.len();
469 let result = self.bind_materialized(ast, &written);
470 self.materialized.truncate(depth);
471 result
472 }
473
474 /// A query with materialised `WITH` definitions in front of it.
475 ///
476 /// The definitions are bound first and in the order they were written, so that a later one can
477 /// read an earlier one, and then the body. The wrapping runs backwards so that the first
478 /// definition ends up outermost, which is the order they have to be filled in.
479 fn bind_materialized(&mut self, ast: &Ast, written: &ast::Query) -> Result<(NodeRef, Scope)> {
480 let depth = self.materialized.len();
481 let held = ast.cte_list(written.ctes).to_vec();
482 let mut definitions = Vec::with_capacity(held.len());
483 for &index in &held {
484 definitions.push(self.bind_definition(ast, index)?);
485 }
486 let (mut node, scope) = self.bind_body(ast, written)?;
487 for (at, definition) in definitions.into_iter().enumerate().rev() {
488 let entry = &self.materialized[depth + at];
489 let cte = entry.cte;
490 let name = entry.name.clone();
491 let fields = entry.fields.clone();
492 let name = self.plan.intern(&name);
493 let columns = self.plan.add_fields(&fields);
494 node =
495 self.add_node(Node::MaterializedCte { definition, body: node, name, cte, columns });
496 }
497 Ok((node, scope))
498 }
499
500 /// Binds one materialised `WITH` definition and makes its name readable from there on.
501 ///
502 /// The definition is projected onto exactly the columns a read of it sees, under the names the
503 /// column list declared when there was one. That projection is not decoration: what is held is
504 /// what a read gets back, so the held rows have to be the rows of the definition's own select
505 /// list and nothing it happened to carry along underneath.
506 ///
507 /// A column list with more names in it than the definition has columns is not an error here,
508 /// which is the pinned build's rule and is written out on [`Scope::rename_prefix`].
509 fn bind_definition(&mut self, ast: &Ast, index: u32) -> Result<NodeRef> {
510 let held = ast.cte(index);
511 let name = ast.string(held.name).to_string();
512 let (node, mut scope) = self.bind_query(ast, held.query)?;
513 if !held.columns.is_empty() {
514 let names: Vec<&str> = ast.name(held.columns).collect();
515 scope.rename_prefix(&names);
516 }
517 let table = self.fresh_index();
518 let mut exprs = Vec::with_capacity(scope.len());
519 let mut names = Vec::with_capacity(scope.len());
520 for column in &scope.columns {
521 exprs.push(self.plan.add_expr(Expr::Column(column.binding), column.ty.clone()));
522 names.push(self.plan.intern(&column.name));
523 }
524 let exprs = self.plan.add_expr_list(&exprs);
525 let names = self.plan.add_name_list(&names);
526 let node = self.add_node(Node::Project { input: node, index: table, exprs, names });
527 let cte = self.next_cte;
528 self.next_cte += 1;
529 self.materialized.push(Materialized { written: index, cte, name, fields: scope.fields() });
530 Ok(node)
531 }
532
533 fn bind_body(&mut self, ast: &Ast, written: &ast::Query) -> Result<(NodeRef, Scope)> {
534 match written.body {
535 ast::QueryBody::Select(select) => self.bind_select(ast, select, written),
536 ast::QueryBody::SetOp { op, quantifier, by_name, left, right } => {
537 let operator = Operator { op, quantifier, by_name };
538 self.bind_set_op(ast, written, operator, left, right)
539 }
540 ast::QueryBody::Values(rows) => self.bind_values(ast, written, rows),
541 ast::QueryBody::Describe(inner) => self.bind_describe(ast, written, inner),
542 ast::QueryBody::Show { name, relation } => self.bind_show(ast, written, name, relation),
543 }
544 }
545
546 /// `SHOW name`, resolved while binding so execution receives an ordinary constant plan.
547 fn bind_show(
548 &mut self,
549 ast: &Ast,
550 query: &ast::Query,
551 name: ast::Slice,
552 relation: ast::QueryRef,
553 ) -> Result<(NodeRef, Scope)> {
554 let text = ast.name_text(name);
555 let parts: Vec<&str> = ast.name(name).collect();
556 let table_exists = self.catalog.resolve(&parts).is_ok();
557 let as_table = match self.semantics.show_behavior() {
558 ShowBehavior::Auto => table_exists,
559 ShowBehavior::Setting => false,
560 ShowBehavior::Table => true,
561 };
562 if as_table {
563 return self.bind_describe(ast, query, relation);
564 }
565 // A name the session has no answer for is either a setting rudb has and DuckDB does not, in
566 // which case [`Binder::beyond`] reads it, or it is nothing, in which case that says so in
567 // upstream's words. `SHOW` prints and printing is text, so a rule's boolean comes back here
568 // as the word it reads back as rather than as a boolean column.
569 let shown = match self.session.iter().find(|(name, _)| name.eq_ignore_ascii_case(&text)) {
570 Some((_, value)) => value.to_string(),
571 None => match self.beyond(&text)? {
572 Some(Value::Varchar(declared)) => declared,
573 Some(other) => other.to_string(),
574 None => {
575 return Err(Error::catalog(format!(
576 "Setting with name \"{text}\" does not exist"
577 )));
578 }
579 },
580 };
581 let field = Field::new(text, LogicalType::Varchar);
582 let expr = self.plan.add_constant(Value::Varchar(shown));
583 let row = self.plan.add_expr_list(&[expr]);
584 let rows = self.plan.add_rows(&[row]);
585 let columns = self.plan.add_fields(std::slice::from_ref(&field));
586 let index = self.fresh_index();
587 let node = self.add_node(Node::Values { index, columns, rows });
588 let mut scope = Scope::empty();
589 scope.push(Visible {
590 table: String::new(),
591 name: field.name,
592 binding: ColumnBinding::new(index, 0),
593 ty: LogicalType::Varchar,
594 not_null: false,
595 also: None,
596 });
597 Ok((node, scope))
598 }
599
600 /// `DESCRIBE <query>`, which is six VARCHAR columns saying what the query returns.
601 ///
602 /// The query is bound and never run, because binding is the whole of the answer: the names and
603 /// the types of a query's columns are settled by the time the binder is done with it, so the
604 /// rows of a describe are a constant from there on. That is why this comes out as a `VALUES`
605 /// whose rows were computed here rather than as an operator of its own, and it is what makes
606 /// `SELECT column_name FROM (DESCRIBE ...) WHERE ...` an ordinary query over an ordinary
607 /// relation with no special case above it.
608 ///
609 /// The six columns, their order and their types are the reference binary's. `key`, `default`
610 /// and `extra` are null for everything this engine can declare, since `PRIMARY KEY`, `UNIQUE`
611 /// and `DEFAULT` are all refused by `CREATE TABLE` today and there is nothing for the first two
612 /// to hold, and `extra` is empty upstream as well on every table it was asked about. They are
613 /// here rather than left out because the width of a result is part of the result, and a program
614 /// that reads the fifth column has to find one.
615 fn bind_describe(
616 &mut self,
617 ast: &Ast,
618 query: &ast::Query,
619 inner: ast::QueryRef,
620 ) -> Result<(NodeRef, Scope)> {
621 let (_, described) = self.bind_query(ast, inner)?;
622 let fields: Vec<Field> = ["column_name", "column_type", "null", "key", "default", "extra"]
623 .iter()
624 .map(|name| Field::new(*name, LogicalType::Varchar))
625 .collect();
626 let mut slices = Vec::with_capacity(described.columns.len());
627 for column in described.columns.clone() {
628 // `NO` and `YES` and not a boolean, because the column is VARCHAR upstream and a
629 // client that prints the result has to get the same four or three characters.
630 let written = [
631 column.name.clone(),
632 column.ty.to_string(),
633 if column.not_null { "NO" } else { "YES" }.to_owned(),
634 ];
635 let mut items: Vec<ExprRef> = written
636 .into_iter()
637 .map(|text| self.plan.add_constant(Value::Varchar(text)))
638 .collect();
639 for _ in 0..3 {
640 let empty = self.plan.add_constant(Value::Null);
641 items.push(self.cast_to(empty, &LogicalType::Varchar));
642 }
643 slices.push(self.plan.add_expr_list(&items));
644 }
645 let rows = self.plan.add_rows(&slices);
646 let columns = self.plan.add_fields(&fields);
647 let index = self.fresh_index();
648 let mut node = self.add_node(Node::Values { index, columns, rows });
649 let mut scope = Scope::empty();
650 for (at, field) in fields.iter().enumerate() {
651 scope.push(Visible {
652 table: String::new(),
653 name: field.name.clone(),
654 binding: ColumnBinding::new(index, at as u32),
655 ty: field.ty.clone(),
656 not_null: false,
657 also: None,
658 });
659 }
660 let keys = self.sort_keys(ast, query, &scope, &[])?;
661 if !keys.is_empty() {
662 let keys = self.plan.add_sort_keys(&keys);
663 node = self.add_node(Node::Sort { input: node, keys });
664 }
665 node = self.apply_limit(ast, query, node, &mut scope)?;
666 Ok((node, scope))
667 }
668
669 /// Whether a projected expression is a column passed straight through from below.
670 ///
671 /// Only `DESCRIBE` asks, and only to decide whether the `null` column says `NO`. Anything that
672 /// is computed is nullable however strict its inputs were, which is both the safe reading and
673 /// the one the reference binary gives.
674 fn passes_through(&self, expr: ExprRef, input: &Scope) -> bool {
675 let Expr::Column(binding) = *self.plan.expr(expr) else { return false };
676 input.columns.iter().any(|column| column.binding == binding && column.not_null)
677 }
678
679 /// `VALUES (1, 'a'), (2, 'b')`, as a query in its own right.
680 ///
681 /// The column names are `col0`, `col1` and so on, which is what DuckDB calls them, and the
682 /// column types are what every row in that position promotes to. Promotion is the same rule a
683 /// set operation uses, and for the same reason: a column has one type and the rows have to
684 /// agree on it before anything downstream can read the column.
685 fn bind_values(
686 &mut self,
687 ast: &Ast,
688 query: &ast::Query,
689 rows: ast::Slice,
690 ) -> Result<(NodeRef, Scope)> {
691 let written = ast.rows(rows).to_vec();
692 let Some(first) = written.first() else {
693 return Err(Error::binder("VALUES needs at least one row"));
694 };
695 let width = first.len as usize;
696 for (at, row) in written.iter().enumerate() {
697 if row.len as usize != width {
698 return Err(Error::binder(format!(
699 "VALUES lists must all be the same length, expected {width} columns but row {} has {}",
700 at + 1,
701 row.len
702 )));
703 }
704 }
705 // A row of a `VALUES` cannot see a column, because there is nothing under it to see.
706 let empty = Scope::empty();
707 let previous = std::mem::replace(&mut self.clause, "VALUES clause");
708 let mut bound: Vec<Vec<ExprRef>> = Vec::with_capacity(written.len());
709 for row in &written {
710 let mut items = Vec::with_capacity(width);
711 for &expr in ast.expr_list(*row) {
712 items.push(self.bind_expr(ast, expr, &empty)?);
713 }
714 bound.push(items);
715 }
716 self.clause = previous;
717 let mut types = Vec::with_capacity(width);
718 for at in 0..width {
719 let mut ty = self.plan.expr_type(bound[0][at]).clone();
720 for row in &bound[1..] {
721 let other = self.plan.expr_type(row[at]).clone();
722 ty = ty.promote(&other).ok_or_else(|| {
723 Error::binder(format!(
724 "Cannot combine a value of type {ty} with a value of type {other} in column {} of a VALUES",
725 at + 1
726 ))
727 })?;
728 }
729 types.push(ty);
730 }
731 let mut slices = Vec::with_capacity(bound.len());
732 for row in &bound {
733 let items: Vec<ExprRef> = row
734 .iter()
735 .zip(&types)
736 .map(|(&expr, ty)| self.checked_cast_to(expr, ty, false))
737 .collect::<Result<_>>()?;
738 slices.push(self.plan.add_expr_list(&items));
739 }
740 let rows = self.plan.add_rows(&slices);
741 let fields: Vec<Field> = types
742 .iter()
743 .enumerate()
744 .map(|(at, ty)| Field::new(format!("col{at}"), ty.clone()))
745 .collect();
746 let columns = self.plan.add_fields(&fields);
747 let index = self.fresh_index();
748 let mut node = self.add_node(Node::Values { index, columns, rows });
749 let mut scope = Scope::empty();
750 for (at, field) in fields.iter().enumerate() {
751 scope.push(Visible {
752 table: String::new(),
753 name: field.name.clone(),
754 binding: ColumnBinding::new(index, at as u32),
755 ty: field.ty.clone(),
756 not_null: false,
757 also: None,
758 });
759 }
760 let keys = self.sort_keys(ast, query, &scope, &[])?;
761 if !keys.is_empty() {
762 let keys = self.plan.add_sort_keys(&keys);
763 node = self.add_node(Node::Sort { input: node, keys });
764 }
765 node = self.apply_limit(ast, query, node, &mut scope)?;
766 Ok((node, scope))
767 }
768
769 fn bind_set_op(
770 &mut self,
771 ast: &Ast,
772 query: &ast::Query,
773 operator: Operator,
774 left: ast::QueryRef,
775 right: ast::QueryRef,
776 ) -> Result<(NodeRef, Scope)> {
777 let (left_node, left_scope) = self.bind_query(ast, left)?;
778 let (right_node, right_scope) = self.bind_query(ast, right)?;
779 let merged = if operator.by_name {
780 match_by_name(&left_scope, &right_scope)?
781 } else {
782 match_by_position(&left_scope, &right_scope)?
783 };
784 let left_node = self.conform(left_node, &left_scope, &merged, |column| column.left)?;
785 let right_node = self.conform(right_node, &right_scope, &merged, |column| column.right)?;
786 let index = self.fresh_index();
787 let kind = match operator.op {
788 SetOp::Union => SetOpKind::Union,
789 SetOp::Except => SetOpKind::Except,
790 SetOp::Intersect => SetOpKind::Intersect,
791 };
792 // UNION alone removes duplicates and UNION ALL keeps them, which is the one place the
793 // unwritten quantifier and ALL disagree.
794 let all = operator.quantifier == Quantifier::All;
795 let mut node =
796 self.add_node(Node::SetOp { left: left_node, right: right_node, kind, all, index });
797 let mut scope = Scope::empty();
798 for (at, column) in merged.iter().enumerate() {
799 scope.push(Visible {
800 table: String::new(),
801 name: column.name.clone(),
802 binding: ColumnBinding::new(index, at as u32),
803 ty: column.ty.clone(),
804 // A column of a set operation is nullable whatever the two sides were, because a
805 // column that refuses nulls on one side and takes them on the other takes them.
806 not_null: false,
807 also: None,
808 });
809 }
810 // Above a set operation there is nothing but the output columns, so an ORDER BY term is
811 // either a position, an output name, or an expression over the output, and never needs a
812 // column projected for it that the query did not ask for.
813 let keys = self.sort_keys(ast, query, &scope, &[])?;
814 if !keys.is_empty() {
815 let keys = self.plan.add_sort_keys(&keys);
816 node = self.add_node(Node::Sort { input: node, keys });
817 }
818 node = self.apply_limit(ast, query, node, &mut scope)?;
819 Ok((node, scope))
820 }
821
822 /// Projects one side of a set operation onto the columns the operation comes out with.
823 ///
824 /// `pick` says which column of this side each output column is. It answers nothing for a
825 /// column only the other side wrote, which happens under `BY NAME` and which this side fills
826 /// with a null, since that is the row it would have written if it had written the column.
827 fn conform(
828 &mut self,
829 node: NodeRef,
830 scope: &Scope,
831 merged: &[Merged],
832 pick: impl Fn(&Merged) -> Option<usize>,
833 ) -> Result<NodeRef> {
834 let unchanged = merged.len() == scope.len()
835 && merged
836 .iter()
837 .enumerate()
838 .all(|(at, column)| pick(column) == Some(at) && column.ty == scope.columns[at].ty);
839 if unchanged {
840 return Ok(node);
841 }
842 let index = self.fresh_index();
843 let mut exprs = Vec::with_capacity(merged.len());
844 let mut names = Vec::with_capacity(merged.len());
845 for column in merged {
846 let expr = match pick(column) {
847 Some(at) => {
848 let held = &scope.columns[at];
849 self.plan.add_expr(Expr::Column(held.binding), held.ty.clone())
850 }
851 None => self.plan.add_constant(Value::Null),
852 };
853 exprs.push(self.checked_cast_to(expr, &column.ty, false)?);
854 names.push(self.plan.intern(&column.name));
855 }
856 let exprs = self.plan.add_expr_list(&exprs);
857 let names = self.plan.add_name_list(&names);
858 Ok(self.add_node(Node::Project { input: node, index, exprs, names }))
859 }
860
861 // ----------------------------------------------------------------- select
862
863 fn bind_select(
864 &mut self,
865 ast: &Ast,
866 select: ast::SelectRef,
867 query: &ast::Query,
868 ) -> Result<(NodeRef, Scope)> {
869 let written = ast.select(select);
870 self.want_ascending |= !written.group_by.is_empty() || written.group_by_all;
871 // A window belongs to the block that wrote it, and a block can be bound inside another one
872 // without a subquery in between, so the outer block's runs are put aside for the duration
873 // rather than left where a nested block would append to them.
874 let outer_windows = std::mem::take(&mut self.windows);
875 // Same argument for the queries lifted over this block's grouping. They are recorded while
876 // the select list is being bound and read until the sort keys are done, and a block bound
877 // inside that stretch has its own set, so the outer block's is put aside rather than left
878 // where the inner one would clear it.
879 let outer_joined_above = std::mem::take(&mut self.joined_above);
880 let (mut node, input) = self.bind_from(ast, written.from)?;
881 node = self.attach_scalar_subqueries(node);
882
883 if written.filter != NONE {
884 self.clause = "WHERE clause";
885 let predicate = self.bind_expr(ast, written.filter, &input)?;
886 let predicate = self.as_boolean(predicate, "WHERE")?;
887 node = self.attach_scalar_subqueries(node);
888 node = self.add_node(Node::Filter { input: node, predicate });
889 }
890
891 let targets = ast.target_list(written.targets).to_vec();
892 if targets.is_empty() {
893 return Err(Error::binder("a SELECT needs at least one expression to select"));
894 }
895
896 let group_items = self.group_items(ast, &written, &targets)?;
897 let aggregating = !group_items.is_empty()
898 || written.having != NONE
899 || targets.iter().any(|target| has_aggregate(ast, target.expr));
900 if aggregating {
901 self.clause = "GROUP BY clause";
902 let mut groups = Vec::with_capacity(group_items.len());
903 for item in &group_items {
904 groups.push(self.bind_expr(ast, *item, &input)?);
905 }
906 let index = self.fresh_index();
907 self.aggregation = Some(Aggregation { index, groups, aggregates: Vec::new() });
908 }
909
910 // The queries this block's clauses wrote that are joined in above the grouping rather than
911 // below it. TPC-H q11 is the case in a `HAVING`: `HAVING sum(ps_supplycost * ps_availqty) >
912 // (SELECT sum(...))` compares one group's total against a total over the whole table, and
913 // the second total is one row that has nothing to do with the groups. Joined underneath the
914 // grouping it would be a column of every input row and the grouping rule would ask for it in
915 // the GROUP BY, which is the complaint this used to make.
916 let mut above = Vec::new();
917
918 self.clause = "SELECT clause";
919 let (mut exprs, mut names) = self.bind_targets(ast, &targets, &input, &mut above)?;
920 let visible = exprs.len();
921
922 let mut having = None;
923 if written.having != NONE {
924 self.clause = "HAVING clause";
925 let before = self.scalar_subqueries.len();
926 let predicate = self.bind_expr(ast, written.having, &input)?;
927 self.lift_over_aggregate(before, &mut above, &input)?;
928 let predicate = self.over_aggregate(predicate, &input)?;
929 having = Some(self.as_boolean(predicate, "HAVING")?);
930 }
931
932 // The projection's index has to exist before the sort keys are built, because a key is a
933 // reference to a projected column even when the expression it sorts on is not selected.
934 let project = self.fresh_index();
935 let mut output = Scope::empty();
936 for (at, (expr, name)) in exprs.iter().zip(&names).enumerate() {
937 output.push(Visible {
938 table: String::new(),
939 name: name.clone(),
940 binding: ColumnBinding::new(project, at as u32),
941 ty: self.plan.expr_type(*expr).clone(),
942 not_null: self.passes_through(*expr, &input),
943 also: None,
944 });
945 }
946
947 self.clause = "ORDER BY clause";
948 let mut extra = Vec::new();
949 let keys = self.select_sort_keys(
950 ast, query, &input, &output, project, &mut exprs, &mut names, &mut extra, &mut above,
951 )?;
952 self.joined_above = outer_joined_above;
953 if !extra.is_empty() && written.distinct != Distinct::No {
954 return Err(Error::binder(
955 "For SELECT DISTINCT, ORDER BY expressions must appear in the select list",
956 ));
957 }
958 let on = self.distinct_on(ast, written.distinct, &output)?;
959
960 node = self.attach_scalar_subqueries(node);
961
962 if let Some(aggregation) = self.aggregation.take() {
963 let index = aggregation.index;
964 let groups = self.plan.add_expr_list(&aggregation.groups);
965 let aggregates = self.plan.add_expr_list(&aggregation.aggregates);
966 node = self.add_node(Node::Aggregate { input: node, index, groups, aggregates });
967 }
968 if !above.is_empty() {
969 debug_assert!(self.scalar_subqueries.is_empty(), "a query is waiting to be joined");
970 self.scalar_subqueries = above;
971 node = self.attach_scalar_subqueries(node);
972 }
973 if let Some(predicate) = having {
974 node = self.add_node(Node::Filter { input: node, predicate });
975 }
976
977 // After the grouping and after `HAVING`, which is where the reference binary puts it:
978 // `SELECT j, sum(count(i)) OVER () FROM t GROUP BY j HAVING count(i) > 1` totals only the
979 // groups that survived the filter.
980 for run in std::mem::replace(&mut self.windows, outer_windows) {
981 let partition = self.plan.add_expr_list(&run.partition);
982 let order = self.plan.add_sort_keys(&run.order);
983 let expressions = self.plan.add_expr_list(&run.calls);
984 node = self.add_node(Node::Window {
985 input: node,
986 index: run.index,
987 partition,
988 order,
989 frame: run.frame,
990 expressions,
991 });
992 }
993
994 let interned: Vec<u32> = names.iter().map(|name| self.plan.intern(name)).collect();
995 let exprs_slice = self.plan.add_expr_list(&exprs);
996 let names_slice = self.plan.add_name_list(&interned);
997 node = self.add_node(Node::Project {
998 input: node,
999 index: project,
1000 exprs: exprs_slice,
1001 names: names_slice,
1002 });
1003
1004 if written.distinct != Distinct::No {
1005 let on = self.plan.add_expr_list(&on);
1006 node = self.add_node(Node::Distinct { input: node, on });
1007 }
1008 if !keys.is_empty() {
1009 let keys = self.plan.add_sort_keys(&keys);
1010 node = self.add_node(Node::Sort { input: node, keys });
1011 }
1012 node = self.apply_limit(ast, query, node, &mut output)?;
1013
1014 if extra.is_empty() {
1015 output.columns.truncate(visible);
1016 return Ok((node, output));
1017 }
1018 // An expression sorted on but not selected was carried this far to make the sort possible,
1019 // and now it goes, because the query did not ask for it.
1020 let index = self.fresh_index();
1021 let mut kept = Vec::with_capacity(visible);
1022 let mut kept_names = Vec::with_capacity(visible);
1023 let mut scope = Scope::empty();
1024 for (at, name) in names.iter().enumerate().take(visible) {
1025 let ty = output.columns[at].ty.clone();
1026 // Through the scope rather than through `project`, because a limit that had a query
1027 // joined in under it put a projection of its own over the top and these columns are
1028 // that projection's now.
1029 let binding = output.columns[at].binding;
1030 kept.push(self.plan.add_expr(Expr::Column(binding), ty.clone()));
1031 kept_names.push(self.plan.intern(name));
1032 scope.push(Visible {
1033 table: String::new(),
1034 name: name.clone(),
1035 binding: ColumnBinding::new(index, at as u32),
1036 ty,
1037 not_null: output.columns[at].not_null,
1038 also: None,
1039 });
1040 }
1041 let exprs = self.plan.add_expr_list(&kept);
1042 let names = self.plan.add_name_list(&kept_names);
1043 node = self.add_node(Node::Project { input: node, index, exprs, names });
1044 Ok((node, scope))
1045 }
1046
1047 /// Binds the target list, expanding every star into the columns it stands for.
1048 /// Moves the queries a clause just wrote from under this block's grouping to over it.
1049 ///
1050 /// A query written in a select list, a `HAVING` or an `ORDER BY` is one row that has nothing to
1051 /// do with the groups, so it belongs on top of the grouping and not underneath it. Underneath,
1052 /// its column is a column of every row going into the aggregate, which the grouping rule then
1053 /// asks for in the `GROUP BY`, and the aggregate carries nothing but its groups and its
1054 /// aggregates upward, so the projection could not read the column even if the rule let it
1055 /// through. That is both halves of #1027.
1056 ///
1057 /// A correlated one goes over the grouping too when what it correlates to is a column the block
1058 /// groups by, which is [`Self::lift_correlated`], and stays underneath when it is not. One
1059 /// written inside an aggregate call stays underneath whatever it correlates to, since that is
1060 /// read once per row going into the aggregate and lifting it over would put it where the
1061 /// aggregate that reads it cannot.
1062 ///
1063 /// `before` is what [`Self::scalar_subqueries`] held before the clause was bound, so only the
1064 /// queries that clause wrote are considered.
1065 fn lift_over_aggregate(
1066 &mut self,
1067 before: usize,
1068 above: &mut Vec<PendingSubquery>,
1069 scope: &Scope,
1070 ) -> Result<()> {
1071 if self.aggregation.is_none() {
1072 return Ok(());
1073 }
1074 let mut lifted = Vec::new();
1075 for mut pending in self.scalar_subqueries.split_off(before) {
1076 let stays = pending.inside_aggregate
1077 || (pending.dependent && !self.lift_correlated(&mut pending));
1078 if stays {
1079 self.scalar_subqueries.push(pending);
1080 } else {
1081 self.joined_above.push(pending.index);
1082 lifted.push(pending);
1083 }
1084 }
1085 // A mark join carries its comparison rather than the expression carrying it, and that
1086 // comparison is written over the outer rows, so it needs the same rewrite the expression
1087 // gets. It is done in a second pass so that a comparison reading another query lifted by
1088 // the same clause finds that query's index already recorded.
1089 for pending in &mut lifted {
1090 let conditions = std::mem::take(&mut pending.conditions);
1091 let mut over = Vec::with_capacity(conditions.len());
1092 for condition in conditions {
1093 over.push(self.over_aggregate(condition, scope)?);
1094 }
1095 pending.conditions = over;
1096 }
1097 above.append(&mut lifted);
1098 Ok(())
1099 }
1100
1101 /// Moves one correlated query over this block's grouping, if the grouping lets it.
1102 ///
1103 /// It does when every outer column the query reads is a column this block groups by. That value
1104 /// is the group's own column above the aggregate, the same value read from a different operator,
1105 /// so the query can be joined against the groups instead of against the rows going into them,
1106 /// and what the query answers per group is what it answered per row of a group since every row
1107 /// of a group agreed on it. The rewrite is the references inside the query's body, which were
1108 /// bound against the table underneath and have to read the aggregate's output instead.
1109 ///
1110 /// A correlation on a column that is neither grouped nor aggregated is a different question with
1111 /// a different answer and there is nothing above the grouping that holds it, so that query stays
1112 /// where it is and [`Self::over_aggregate`] reports it as the missing `GROUP BY` it is. That is
1113 /// #1032.
1114 ///
1115 /// The query stays a dependent join either way. What changed is which operator the outer rows
1116 /// come from, not that there are any.
1117 fn lift_correlated(&mut self, pending: &mut PendingSubquery) -> bool {
1118 let Some(index) = self.aggregation.as_ref().map(|aggregation| aggregation.index) else {
1119 return false;
1120 };
1121 let mut moved = Vec::with_capacity(pending.reads.len());
1122 for read in &pending.reads {
1123 let Some(at) = self.group_of(*read) else {
1124 return false;
1125 };
1126 moved.push((*read, ColumnBinding::new(index, at as u32)));
1127 }
1128 let mut rewrites = Vec::new();
1129 self.plan.subtree_columns(pending.node, &mut |reference, binding| {
1130 if let Some(&(_, to)) = moved.iter().find(|(from, _)| *from == binding) {
1131 rewrites.push((reference, to));
1132 }
1133 });
1134 for (reference, to) in rewrites {
1135 self.plan.rebind(reference, to);
1136 }
1137 pending.reads = moved.into_iter().map(|(_, to)| to).collect();
1138 true
1139 }
1140
1141 fn bind_targets(
1142 &mut self,
1143 ast: &Ast,
1144 targets: &[ast::Target],
1145 input: &Scope,
1146 above: &mut Vec<PendingSubquery>,
1147 ) -> Result<(Vec<ExprRef>, Vec<String>)> {
1148 let mut exprs = Vec::with_capacity(targets.len());
1149 let mut names = Vec::with_capacity(targets.len());
1150 for target in targets {
1151 if let ast::Expr::Star { qualifier, replacements } = ast.expr(target.expr) {
1152 let table = ast.name(qualifier).last().map(str::to_string);
1153 let expanded: Vec<Visible> =
1154 input.star(table.as_deref())?.into_iter().cloned().collect();
1155 let replacements = ast.target_list(replacements).to_vec();
1156 let mut used = vec![false; replacements.len()];
1157 for column in expanded {
1158 let found = replacements.iter().zip(&mut used).find(|(replacement, _)| {
1159 same_name(ast.string(replacement.alias), &column.name)
1160 });
1161 // The replacement takes the column's place and its position, and it is named the
1162 // way the replace list spells it rather than the way the table does. That only
1163 // shows when the two differ in case, and `AS EventDate` over a column called
1164 // `eventdate` is exactly the case that shows it.
1165 let before = self.scalar_subqueries.len();
1166 let (expr, name) = match found {
1167 Some((replacement, used)) => {
1168 *used = true;
1169 let expr = self.bind_expr(ast, replacement.expr, input)?;
1170 (expr, ast.string(replacement.alias).to_string())
1171 }
1172 None => (
1173 self.plan.add_expr(Expr::Column(column.binding), column.ty),
1174 column.name,
1175 ),
1176 };
1177 self.lift_over_aggregate(before, above, input)?;
1178 exprs.push(self.over_aggregate(expr, input)?);
1179 names.push(name);
1180 }
1181 // A replace list that named something the star did not stand for is a mistake and
1182 // not a no op, and it is caught here because this is the first point at which the
1183 // set of names the star stands for is known.
1184 if let Some((replacement, _)) =
1185 replacements.iter().zip(&used).find(|(_, used)| !**used)
1186 {
1187 return Err(missing_replacement(ast.string(replacement.alias), input));
1188 }
1189 continue;
1190 }
1191 let before = self.scalar_subqueries.len();
1192 let expr = self.bind_expr(ast, target.expr, input)?;
1193 self.lift_over_aggregate(before, above, input)?;
1194 exprs.push(self.over_aggregate(expr, input)?);
1195 names.push(if target.alias == NONE {
1196 self.output_name(ast, target.expr, input)
1197 } else {
1198 ast.string(target.alias).to_string()
1199 });
1200 }
1201 Ok((exprs, names))
1202 }
1203
1204 /// The name an unaliased target gets.
1205 ///
1206 /// A bare column keeps the spelling the table was created with rather than the spelling the
1207 /// query used, so `SELECT USERID FROM hits` has a column called `UserID`. Identifiers match
1208 /// without regard to case and the catalog is the one that holds the case.
1209 fn output_name(&self, ast: &Ast, target: ast::ExprRef, input: &Scope) -> String {
1210 if let ast::Expr::Column { name } = ast.expr(target) {
1211 let parts: Vec<&str> = ast.name(name).collect();
1212 if let Ok(found) = input.resolve(&parts) {
1213 // A column found by its second name is headed by that name, so `t.range` over
1214 // `range(2) t` is a column called `range` on the pin while `SELECT *` calls it `t`.
1215 let written = parts.last().copied().unwrap_or_default();
1216 if let Some(also) = &found.also {
1217 if !same_name(&found.name, written) && same_name(also, written) {
1218 return also.clone();
1219 }
1220 }
1221 return found.name.clone();
1222 }
1223 }
1224 describe(ast, target, self.semantics)
1225 }
1226
1227 /// The expressions a `GROUP BY` clause names, with positions and output aliases followed.
1228 fn group_items(
1229 &self,
1230 ast: &Ast,
1231 select: &ast::Select,
1232 targets: &[ast::Target],
1233 ) -> Result<Vec<ast::ExprRef>> {
1234 if select.group_by_all {
1235 // GROUP BY ALL means every target that is not itself an aggregate, which is the set
1236 // that would otherwise have to be written out again by hand.
1237 return Ok(targets
1238 .iter()
1239 .filter(|target| !has_aggregate(ast, target.expr))
1240 .map(|target| target.expr)
1241 .collect());
1242 }
1243 let mut items = Vec::new();
1244 for &item in ast.expr_list(select.group_by) {
1245 items.push(self.output_reference(ast, item, targets, "GROUP BY")?.unwrap_or(item));
1246 }
1247 Ok(items)
1248 }
1249
1250 /// The target a `GROUP BY` or `ORDER BY` term names, when it names one by position or alias.
1251 fn output_reference(
1252 &self,
1253 ast: &Ast,
1254 item: ast::ExprRef,
1255 targets: &[ast::Target],
1256 clause: &str,
1257 ) -> Result<Option<ast::ExprRef>> {
1258 match ast.expr(item) {
1259 ast::Expr::Literal { kind: LiteralKind::Number, text } => {
1260 let written = ast.string(text);
1261 let position: usize = written.parse().map_err(|_| {
1262 Error::binder(format!("{clause} term {written} is not a column"))
1263 })?;
1264 if position == 0 || position > targets.len() {
1265 return Err(Error::binder(format!(
1266 "{clause} term out of range - should be between 1 and {}",
1267 targets.len()
1268 )));
1269 }
1270 Ok(Some(targets[position - 1].expr))
1271 }
1272 ast::Expr::Column { name } => {
1273 let parts: Vec<&str> = ast.name(name).collect();
1274 let [written] = parts.as_slice() else { return Ok(None) };
1275 let mut found = None;
1276 for target in targets {
1277 if target.alias != NONE && same_name(ast.string(target.alias), written) {
1278 if found.is_some() {
1279 return Ok(None);
1280 }
1281 found = Some(target.expr);
1282 }
1283 }
1284 Ok(found)
1285 }
1286 _ => Ok(None),
1287 }
1288 }
1289
1290 // -------------------------------------------------------------- modifiers
1291
1292 /// Sort keys for a select, projecting anything sorted on that is not already selected.
1293 #[allow(clippy::too_many_arguments)]
1294 fn select_sort_keys(
1295 &mut self,
1296 ast: &Ast,
1297 query: &ast::Query,
1298 input: &Scope,
1299 output: &Scope,
1300 project: u32,
1301 exprs: &mut Vec<ExprRef>,
1302 names: &mut Vec<String>,
1303 extra: &mut Vec<usize>,
1304 above: &mut Vec<PendingSubquery>,
1305 ) -> Result<Vec<SortKey>> {
1306 if query.order_by_all {
1307 return Ok(self.every_column(output));
1308 }
1309 let items = ast.order_list(query.order_by).to_vec();
1310 let mut keys = Vec::with_capacity(items.len());
1311 for item in items {
1312 self.check_order_literal(ast, item.expr)?;
1313 let position = match self.output_position(ast, item.expr, output)? {
1314 Some(position) => position,
1315 None => {
1316 let before = self.scalar_subqueries.len();
1317 let bound = self.bind_expr(ast, item.expr, input)?;
1318 self.lift_over_aggregate(before, above, input)?;
1319 let bound = self.over_aggregate(bound, input)?;
1320 match exprs.iter().position(|&held| self.same_expr(held, bound)) {
1321 Some(position) => position,
1322 None => {
1323 exprs.push(bound);
1324 names.push(describe(ast, item.expr, self.semantics));
1325 extra.push(exprs.len() - 1);
1326 exprs.len() - 1
1327 }
1328 }
1329 }
1330 };
1331 let ty = self.plan.expr_type(exprs[position]).clone();
1332 let expr = self.column(project, position, ty);
1333 keys.push(self.sort_key(expr, item));
1334 }
1335 Ok(keys)
1336 }
1337
1338 /// Sort keys over an output that has nothing behind it to project, which is a set operation.
1339 fn sort_keys(
1340 &mut self,
1341 ast: &Ast,
1342 query: &ast::Query,
1343 output: &Scope,
1344 targets: &[ast::Target],
1345 ) -> Result<Vec<SortKey>> {
1346 if query.order_by_all {
1347 return Ok(self.every_column(output));
1348 }
1349 let items = ast.order_list(query.order_by).to_vec();
1350 let mut keys = Vec::with_capacity(items.len());
1351 for item in items {
1352 self.check_order_literal(ast, item.expr)?;
1353 let expr = match self.output_position(ast, item.expr, output)? {
1354 Some(position) => {
1355 let column = &output.columns[position];
1356 let (binding, ty) = (column.binding, column.ty.clone());
1357 self.plan.add_expr(Expr::Column(binding), ty)
1358 }
1359 None => {
1360 let _ = targets;
1361 self.bind_expr(ast, item.expr, output)?
1362 }
1363 };
1364 keys.push(self.sort_key(expr, item));
1365 }
1366 Ok(keys)
1367 }
1368
1369 fn every_column(&mut self, output: &Scope) -> Vec<SortKey> {
1370 let columns: Vec<(ColumnBinding, LogicalType)> =
1371 output.columns.iter().map(|column| (column.binding, column.ty.clone())).collect();
1372 columns
1373 .into_iter()
1374 .map(|(binding, ty)| {
1375 let expr = self.plan.add_expr(Expr::Column(binding), ty);
1376 let descending = self.semantics.default_descending();
1377 SortKey { expr, descending, nulls_first: self.semantics.nulls_first(descending) }
1378 })
1379 .collect()
1380 }
1381
1382 /// A sort key with the session defaults filled in.
1383 fn sort_key(&self, expr: ExprRef, item: ast::OrderItem) -> SortKey {
1384 let descending = match item.order {
1385 Order::Unstated => self.semantics.default_descending(),
1386 Order::Ascending => false,
1387 Order::Descending => true,
1388 };
1389 let nulls_first = match item.nulls {
1390 Nulls::First => true,
1391 Nulls::Last => false,
1392 Nulls::Unstated => self.semantics.nulls_first(descending),
1393 };
1394 SortKey { expr, descending, nulls_first }
1395 }
1396
1397 /// Which output column a term names, by position or by name.
1398 fn output_position(
1399 &self,
1400 ast: &Ast,
1401 item: ast::ExprRef,
1402 output: &Scope,
1403 ) -> Result<Option<usize>> {
1404 match ast.expr(item) {
1405 ast::Expr::Literal { kind: LiteralKind::Number, text } => {
1406 let written = ast.string(text);
1407 if written.contains(['.', 'e', 'E']) {
1408 return Ok(None);
1409 }
1410 let position: usize = written.parse().map_err(|_| {
1411 Error::binder(format!("ORDER BY term {written} is not a column"))
1412 })?;
1413 if position == 0 || position > output.len() {
1414 return Err(Error::binder(format!(
1415 "ORDER BY term out of range - should be between 1 and {}",
1416 output.len()
1417 )));
1418 }
1419 Ok(Some(position - 1))
1420 }
1421 ast::Expr::Column { name } => {
1422 let parts: Vec<&str> = ast.name(name).collect();
1423 let [written] = parts.as_slice() else { return Ok(None) };
1424 Ok(output.position_of(None, written))
1425 }
1426 _ => Ok(None),
1427 }
1428 }
1429
1430 /// Refuses a literal sort key unless the session explicitly accepts its no-op behavior.
1431 fn check_order_literal(&self, ast: &Ast, item: ast::ExprRef) -> Result<()> {
1432 if !self.semantics.order_by_non_integer_literal()
1433 && matches!(
1434 ast.expr(item),
1435 ast::Expr::Literal { kind, text }
1436 if kind != LiteralKind::Number
1437 || ast.string(text).contains(['.', 'e', 'E'])
1438 )
1439 {
1440 return Err(Error::binder(
1441 "ORDER BY non-integer literal has no effect.\n* SET order_by_non_integer_literal=true to allow this behavior.",
1442 ));
1443 }
1444 Ok(())
1445 }
1446
1447 /// The expressions a `DISTINCT ON` names, which have to be columns of the output.
1448 fn distinct_on(
1449 &mut self,
1450 ast: &Ast,
1451 distinct: Distinct,
1452 output: &Scope,
1453 ) -> Result<Vec<ExprRef>> {
1454 let Distinct::On(items) = distinct else {
1455 return Ok(Vec::new());
1456 };
1457 let items = ast.expr_list(items).to_vec();
1458 let mut on = Vec::with_capacity(items.len());
1459 for item in items {
1460 let Some(position) = self.output_position(ast, item, output)? else {
1461 return Err(Error::not_implemented(
1462 "DISTINCT ON an expression that is not in the select list",
1463 ));
1464 };
1465 let column = &output.columns[position];
1466 let (binding, ty) = (column.binding, column.ty.clone());
1467 on.push(self.plan.add_expr(Expr::Column(binding), ty));
1468 }
1469 Ok(on)
1470 }
1471
1472 /// The `LIMIT` and the `OFFSET`, over the rows everything else in the query produced.
1473 ///
1474 /// The scope is taken by reference because a limit the binder could not work out reads its
1475 /// number off a query joined in underneath, and that join puts a column in the rows which the
1476 /// query did not ask for. A projection over the limit drops it again, and the scope has to say
1477 /// so, since its bindings are what anything above this reads.
1478 fn apply_limit(
1479 &mut self,
1480 ast: &Ast,
1481 query: &ast::Query,
1482 input: NodeRef,
1483 scope: &mut Scope,
1484 ) -> Result<NodeRef> {
1485 let waiting = self.scalar_subqueries.len();
1486 if query.limit_percent {
1487 let percent = self.share(ast, query.limit)?;
1488 let offset = self.skipped(ast, query.offset)?;
1489 let node = |binder: &mut Self, input| match percent {
1490 Some(percent) => binder.add_node(Node::LimitPercent { input, percent, offset }),
1491 // A null share is no limit at all, the same as a null row count, so what is left
1492 // is whatever the offset asked for.
1493 None => binder.limited(input, Bound::All, offset),
1494 };
1495 return self.over_subqueries(waiting, input, scope, node);
1496 }
1497 let count = self.count_bound(ast, query.limit, "LIMIT")?;
1498 let offset = self.skipped(ast, query.offset)?;
1499 let node = |binder: &mut Self, input| binder.limited(input, count, offset);
1500 self.over_subqueries(waiting, input, scope, node)
1501 }
1502
1503 /// The offset a query wrote, as nought rows skipped when it wrote none.
1504 ///
1505 /// An offset the query left off is nought rows skipped, where a limit it left off is every row
1506 /// emitted, so the two clauses read the same word differently.
1507 fn skipped(&mut self, ast: &Ast, written: ast::ExprRef) -> Result<Bound> {
1508 Ok(match self.count_bound(ast, written, "OFFSET")? {
1509 Bound::All => Bound::Rows(0),
1510 named => named,
1511 })
1512 }
1513
1514 /// Builds a limit node over `input`, joining in whatever queries its bounds turned out to need.
1515 ///
1516 /// A bound the binder could not work out reads its number off a column, and that column comes
1517 /// from a query joined in underneath. The join puts a column in the rows nobody asked for, so a
1518 /// projection over the limit drops it again and the scope is told to read that projection. When
1519 /// no query had to be joined in there is nothing to drop and the limit stands on its own.
1520 fn over_subqueries(
1521 &mut self,
1522 waiting: usize,
1523 input: NodeRef,
1524 scope: &mut Scope,
1525 node: impl FnOnce(&mut Self, NodeRef) -> NodeRef,
1526 ) -> Result<NodeRef> {
1527 let joined = self.scalar_subqueries.split_off(waiting);
1528 if joined.is_empty() {
1529 return Ok(node(self, input));
1530 }
1531 let mut input = input;
1532 for pending in joined {
1533 input = self.attach_subquery(input, pending);
1534 }
1535 let limit = node(self, input);
1536 Ok(self.reproject(limit, scope))
1537 }
1538
1539 /// A row count limit over `input`, or `input` itself when neither half of the clause asks for
1540 /// anything.
1541 fn limited(&mut self, input: NodeRef, count: Bound, offset: Bound) -> NodeRef {
1542 if count == Bound::All && offset == Bound::Rows(0) {
1543 return input;
1544 }
1545 self.add_node(Node::Limit { input, count, offset })
1546 }
1547
1548 /// A projection over `node` handing back exactly the columns `scope` names.
1549 ///
1550 /// The scope's bindings are rewritten to this projection's, because its columns are the ones
1551 /// anything above reads. Only a limit that had a query joined in under it wants this, and only
1552 /// because there is not always a projection above to drop the column that join added.
1553 fn reproject(&mut self, node: NodeRef, scope: &mut Scope) -> NodeRef {
1554 let index = self.fresh_index();
1555 let mut exprs = Vec::with_capacity(scope.columns.len());
1556 let mut names = Vec::with_capacity(scope.columns.len());
1557 for column in &scope.columns {
1558 exprs.push(self.plan.add_expr(Expr::Column(column.binding), column.ty.clone()));
1559 names.push(self.plan.intern(&column.name));
1560 }
1561 for (at, column) in scope.columns.iter_mut().enumerate() {
1562 column.binding = ColumnBinding::new(index, at as u32);
1563 }
1564 let exprs = self.plan.add_expr_list(&exprs);
1565 let names = self.plan.add_name_list(&names);
1566 self.add_node(Node::Project { input: node, index, exprs, names })
1567 }
1568
1569 /// The share of the input a `LIMIT n PERCENT` names.
1570 ///
1571 /// The same evaluation as a row count and a different type at the end of it: the value is cast
1572 /// to `DOUBLE` rather than to `BIGINT`, so `LIMIT '30'%` is thirty percent and `LIMIT true%` is
1573 /// one percent, which is what the pin answers. A null is no limit at all.
1574 ///
1575 /// The range is checked here because the pin checks it here. `LIMIT 101 PERCENT` fails an
1576 /// `EXPLAIN` on the pinned binary, so it is refused while the query is planned and not when it
1577 /// is run, and a `NAN` is outside the range like any other value that is not between nought and
1578 /// a hundred.
1579 ///
1580 /// What the binder cannot work out is a subquery and a call that answers differently every
1581 /// time, the same two things a row count cannot work out, and those become a [`Share::Read`]
1582 /// over the expression. The value is checked where it turns up instead, which is the executor.
1583 /// Only the sign can be written that way, because the grammar refuses `PERCENT` after a closing
1584 /// bracket, but nothing below here depends on which of the two was typed.
1585 fn share(&mut self, ast: &Ast, written: ast::ExprRef) -> Result<Option<Share>> {
1586 if written == NONE {
1587 return Ok(None);
1588 }
1589 self.clause = "LIMIT clause";
1590 let scope = Scope::empty();
1591 let bound = self.bind_expr(ast, written, &scope)?;
1592 let Some(value) = fold::value_of(&self.plan, bound)? else {
1593 return Ok(Some(Share::Read(bound)));
1594 };
1595 if value.is_null() {
1596 return Ok(None);
1597 }
1598 let percent = percentage(&value)?;
1599 if !(0.0..=100.0).contains(&percent) {
1600 return Err(Error::out_of_range(
1601 "Limit percent out of range, should be between 0% and 100%",
1602 ));
1603 }
1604 Ok(Some(Share::Percent(percent)))
1605 }
1606
1607 /// The row count a `LIMIT` or an `OFFSET` names.
1608 ///
1609 /// It does not have to be a literal. Anything whose value is settled before the first row is
1610 /// read will do, so `LIMIT 1 + 1` and `LIMIT CAST(3 AS BIGINT)` are both two, and that is what
1611 /// the pin does with them: its binder evaluates the expression and writes the number down.
1612 ///
1613 /// What is left over is an expression the binder cannot settle, which is a subquery, because it
1614 /// has to run first, and a call that answers differently every time it is made, such as
1615 /// `RANDOM()` or `nextval`. Those become a [`Bound::Read`] holding the expression, and the
1616 /// number comes off the first chunk that reaches the limit. The pin takes both and answers them
1617 /// the same way.
1618 ///
1619 /// The value is cast to `BIGINT` whatever it was written as, which is the whole of the type
1620 /// rule. `LIMIT '3'` is three rows because the string converts, `LIMIT 2.5` is three rows
1621 /// because the conversion rounds, `LIMIT true` is one row, and `LIMIT DATE '2020-01-01'` is the
1622 /// cast refusing a date. Every one of those messages is the cast's own, which is why there is
1623 /// no type check here to write a worse one. A limit that is read while the query runs is cast
1624 /// the same way by the operator that reads it, so the two paths answer alike.
1625 fn count_bound(&mut self, ast: &Ast, written: ast::ExprRef, clause: &str) -> Result<Bound> {
1626 if written == NONE {
1627 return Ok(Bound::All);
1628 }
1629 self.clause = "LIMIT clause";
1630 let scope = Scope::empty();
1631 let bound = self.bind_expr(ast, written, &scope)?;
1632 let Some(value) = fold::value_of(&self.plan, bound)? else {
1633 return Ok(Bound::Read(bound));
1634 };
1635 // A null is no limit at all, the same as leaving the clause off, and the pin agrees:
1636 // `LIMIT NULL` and `LIMIT CAST(NULL AS INTEGER)` both answer every row.
1637 if value.is_null() {
1638 return Ok(Bound::All);
1639 }
1640 row_count(&value, clause).map(Bound::Rows)
1641 }
1642
1643 // ------------------------------------------------------------------- from
1644
1645 fn bind_from(&mut self, ast: &Ast, from: ast::Slice) -> Result<(NodeRef, Scope)> {
1646 let sources = ast.source_list(from).to_vec();
1647 let Some((first, rest)) = sources.split_first() else {
1648 // No FROM clause is one row of no columns, which is what SELECT 1 sits on. Not an
1649 // empty table: an empty table would make SELECT 1 return nothing.
1650 return Ok((self.add_node(Node::Dummy), Scope::empty()));
1651 };
1652 let (mut node, mut scope) = self.bind_source(ast, *first)?;
1653 for source in rest {
1654 let (right, right_scope, correlations) = self.bind_lateral(ast, *source, &scope)?;
1655 node = if correlations.is_empty() {
1656 self.add_node(Node::CrossProduct { left: node, right })
1657 } else {
1658 let conditions = self.plan.add_expr_list(&[]);
1659 self.add_node(Node::DependentJoin {
1660 left: node,
1661 right,
1662 kind: JoinKind::Inner,
1663 conditions,
1664 })
1665 };
1666 scope = scope.concat(right_scope);
1667 }
1668 Ok((node, scope))
1669 }
1670
1671 /// Binds one FROM entry with everything written to its left already visible.
1672 ///
1673 /// That is what LATERAL means, and it is what a comma separated FROM does here whether the word
1674 /// was written or not, because the pinned build resolves `FROM o, (SELECT o.k + 1)` without it.
1675 /// The keyword therefore changes nothing and is accepted rather than acted on.
1676 ///
1677 /// The columns of the left that the entry read come back with it, and an entry that read none
1678 /// is an ordinary product. The rest are somebody else's: a name that resolved past the left
1679 /// neighbours belongs to an enclosing query, so it is handed up to whichever frame is waiting
1680 /// for it rather than counted here, or the subquery this FROM sits in would lose track of its
1681 /// own correlation.
1682 fn bind_lateral(
1683 &mut self,
1684 ast: &Ast,
1685 source: ast::SourceRef,
1686 left: &Scope,
1687 ) -> Result<(NodeRef, Scope, Vec<ColumnBinding>)> {
1688 self.lateral_scopes.push(self.outer_scopes.len());
1689 self.outer_scopes.push(left.clone());
1690 self.correlations.push(Vec::new());
1691 let bound = self.bind_source(ast, source);
1692 let read = self.correlations.pop().expect("correlation frame");
1693 self.outer_scopes.pop();
1694 self.lateral_scopes.pop();
1695 let (node, scope) = bound?;
1696
1697 let mut here = Vec::new();
1698 for binding in read {
1699 if left.columns.iter().any(|column| column.binding == binding) {
1700 here.push(binding);
1701 } else if let Some(enclosing) = self.correlations.last_mut() {
1702 if !enclosing.contains(&binding) {
1703 enclosing.push(binding);
1704 }
1705 }
1706 }
1707 // A table function is allowed to read the left the same as anything else here. There is
1708 // nothing underneath one for the domain to be pushed into, since its arguments are what
1709 // produce its rows, so the unnesting pass turns it into a `LateralFunction` and the call is
1710 // made once per domain value. That is `domain.rs`.
1711 //
1712 // Nothing has to be turned down here for the functions that would not survive it. The only
1713 // table functions taking an argument that is not a name are the series family, which is the
1714 // family that operator answers, and a name that is not a constant is refused where the
1715 // columns are settled, because settling them means opening the file or reading the catalog.
1716 Ok((node, scope, here))
1717 }
1718
1719 fn bind_source(&mut self, ast: &Ast, source: ast::SourceRef) -> Result<(NodeRef, Scope)> {
1720 match ast.source(source) {
1721 ast::Source::Table { name, alias, columns } => {
1722 self.bind_table(ast, name, alias, columns)
1723 }
1724 ast::Source::Function { name, args, alias, columns, pragma } => {
1725 self.bind_table_function(ast, name, args, alias, columns, pragma)
1726 }
1727 ast::Source::Subquery { query, alias, columns } => {
1728 let (node, mut scope) = self.bind_query(ast, query)?;
1729 let label = if alias == NONE {
1730 "unnamed_subquery".to_string()
1731 } else {
1732 ast.string(alias).to_string()
1733 };
1734 scope.relabel(&label);
1735 if !columns.is_empty() {
1736 let names: Vec<&str> = ast.name(columns).collect();
1737 scope.rename(&names, &label)?;
1738 }
1739 Ok((node, scope))
1740 }
1741 ast::Source::Values { rows, alias, columns } => {
1742 let bare = ast::Query::bare(ast::QueryBody::Values(rows));
1743 let (node, mut scope) = self.bind_values(ast, &bare, rows)?;
1744 let label =
1745 if alias == NONE { String::new() } else { ast.string(alias).to_string() };
1746 scope.relabel(&label);
1747 if !columns.is_empty() {
1748 let names: Vec<&str> = ast.name(columns).collect();
1749 scope.rename(&names, &label)?;
1750 }
1751 Ok((node, scope))
1752 }
1753 ast::Source::Cte { cte, alias, columns } => {
1754 self.bind_cte_scan(ast, cte, alias, columns)
1755 }
1756 ast::Source::Join { left, right, kind, natural, on, using } => {
1757 self.bind_join(ast, left, right, kind, natural, on, using)
1758 }
1759 }
1760 }
1761
1762 /// A read of a materialised `WITH`, which is a leaf the same way a table scan is.
1763 ///
1764 /// Which definition it reads was settled by the parser, so there is no name to look up here and
1765 /// no shadowing left to think about. What is looked up is the materialisation that definition
1766 /// turned into, and the search runs backwards because the same definition is bound again for
1767 /// each use of a plain `WITH` it sits inside, and a read means the innermost of those.
1768 fn bind_cte_scan(
1769 &mut self,
1770 ast: &Ast,
1771 written: u32,
1772 alias: ast::StrRef,
1773 columns: ast::Slice,
1774 ) -> Result<(NodeRef, Scope)> {
1775 let Some(held) = self.materialized.iter().rev().find(|held| held.written == written) else {
1776 let name = ast.string(ast.cte(written).name);
1777 return Err(Error::binder(format!("Table with name {name} does not exist!")));
1778 };
1779 let cte = held.cte;
1780 let fields = held.fields.clone();
1781 let text = held.name.clone();
1782 let label = if alias == NONE { text.clone() } else { ast.string(alias).to_string() };
1783 let name = self.plan.intern(&text);
1784 let index = self.fresh_index();
1785 let mut scope = Scope::empty();
1786 for (at, field) in fields.iter().enumerate() {
1787 scope.push(Visible {
1788 table: label.clone(),
1789 name: field.name.clone(),
1790 binding: ColumnBinding::new(index, at as u32),
1791 ty: field.ty.clone(),
1792 not_null: field.not_null,
1793 also: None,
1794 });
1795 }
1796 if !columns.is_empty() {
1797 let names: Vec<&str> = ast.name(columns).collect();
1798 scope.rename(&names, &label)?;
1799 }
1800 let columns = self.plan.add_fields(&fields);
1801 let node = self.add_node(Node::CteScan { index, cte, name, columns });
1802 Ok((node, scope))
1803 }
1804
1805 fn bind_table(
1806 &mut self,
1807 ast: &Ast,
1808 name: ast::Slice,
1809 alias: ast::StrRef,
1810 columns: ast::Slice,
1811 ) -> Result<(NodeRef, Scope)> {
1812 let parts: Vec<&str> = ast.name(name).collect();
1813 let catalog = self.catalog;
1814 // The catalog is asked first and the file is the fallback, which is the order DuckDB uses:
1815 // a table really called `mixed.parquet` wins over a file of that name sitting next to it.
1816 let resolved = match catalog.resolve(&parts) {
1817 Ok(resolved) => resolved,
1818 Err(missing) => {
1819 return self.bind_replacement_scan(ast, &parts, alias, columns, missing);
1820 }
1821 };
1822 if catalog.entry(&resolved)? == Entry::View {
1823 return self.bind_view(ast, &resolved, alias, columns);
1824 }
1825 let label =
1826 if alias == NONE { resolved.table.clone() } else { ast.string(alias).to_string() };
1827 self.bind_catalog_table(ast, &resolved, label, columns)
1828 }
1829
1830 /// A table the catalog holds, under the name `label`, which is where [`Self::bind_table`] ends
1831 /// and where a Parquet file with a native mirror goes instead of to its reader.
1832 fn bind_catalog_table(
1833 &mut self,
1834 ast: &Ast,
1835 resolved: &QualifiedName,
1836 label: String,
1837 columns: ast::Slice,
1838 ) -> Result<(NodeRef, Scope)> {
1839 let table = self.catalog.table(resolved)?;
1840 let fields: Vec<Field> = table.columns().to_vec();
1841 let index = self.fresh_index();
1842 let mut scope = Scope::empty();
1843 for (at, field) in fields.iter().enumerate() {
1844 scope.push(Visible {
1845 table: label.clone(),
1846 name: field.name.clone(),
1847 binding: ColumnBinding::new(index, at as u32),
1848 ty: field.ty.clone(),
1849 not_null: field.not_null,
1850 also: None,
1851 });
1852 }
1853 if !columns.is_empty() {
1854 let names: Vec<&str> = ast.name(columns).collect();
1855 scope.rename(&names, &label)?;
1856 }
1857 let catalog_name = self.plan.intern(&resolved.catalog);
1858 let schema = self.plan.intern(&resolved.schema);
1859 let table_name = self.plan.intern(&resolved.table);
1860 let alias = self.plan.intern(&label);
1861 let columns = self.plan.add_fields(&fields);
1862 // What the store wrote down about itself, against the table index the same way a Parquet
1863 // footer is. A table with nothing to say records nothing and the estimate falls back to the
1864 // constants it used before, which is what every table did until the file had a directory
1865 // worth asking.
1866 if let Some(zones) = table.rows().zones() {
1867 self.plan.set_zones(index, zones);
1868 }
1869 if let Some(frequencies) = table.frequencies() {
1870 self.plan.set_frequencies(index, frequencies);
1871 }
1872 for (column, distinct) in table.distincts() {
1873 self.plan.measure_distinct(index, &column, distinct);
1874 }
1875 if self.want_ascending {
1876 for column in table.ascending() {
1877 self.plan.mark_ascending(index, &column);
1878 }
1879 }
1880 let node = self.add_node(Node::Get {
1881 catalog: catalog_name,
1882 schema,
1883 table: table_name,
1884 alias,
1885 index,
1886 columns,
1887 });
1888 Ok((node, scope))
1889 }
1890
1891 /// A view where a table goes, which is the body bound again right here.
1892 ///
1893 /// Inline and not behind a node. The view is gone by the time the plan exists, so everything
1894 /// downstream sees the query somebody would have written by hand, and the column pruning that
1895 /// makes `SELECT COUNT(*) FROM 'hits.parquet'` read no columns at all keeps working through
1896 /// `FROM hits`. A `Node::View` would be a barrier with nothing on the other side of it.
1897 ///
1898 /// The scope this builds is a subquery's, right down to the name in the error message. duckdb
1899 /// v1.5.1 reports a view whose column list has gone stale as `table "unnamed_subquery" has 1
1900 /// columns available but 2 columns specified`, which is the sentence its subquery alias rule
1901 /// produces, so a view there is a subquery with the view's name written over it afterwards.
1902 fn bind_view(
1903 &mut self,
1904 ast: &Ast,
1905 name: &QualifiedName,
1906 alias: ast::StrRef,
1907 columns: ast::Slice,
1908 ) -> Result<(NodeRef, Scope)> {
1909 let view = self.catalog.view(name)?;
1910 let full = name.to_string();
1911 if self.expanding.contains(&full) {
1912 // Two quotes each side, which is what the binary prints. It quotes the name on the way
1913 // in and then formats the quoted name into a quoted slot, so a view called `a` comes
1914 // back as `""a""`. That is upstream's wart and copying it is the whole job here.
1915 return Err(Error::binder(format!(
1916 "infinite recursion detected: attempting to recursively bind view \"\"{}\"\"",
1917 name.table
1918 )));
1919 }
1920 let body = parse_ast_with_case(view.sql(), self.semantics.identifier_case())?;
1921 let query = match body.statements.as_slice() {
1922 [ast::Statement::Query(query)] => *query,
1923 // Only a query can have got past the binder at creation, so this is a view the catalog
1924 // was handed some other way rather than anything a statement can produce.
1925 _ => return Err(Error::binder(format!("view \"{}\" is not a query", name.table))),
1926 };
1927 self.expanding.push(full);
1928 let bound = self.bind_query(&body, query);
1929 self.expanding.pop();
1930 let (node, mut scope) = bound?;
1931
1932 let aliases: Vec<&str> = view.aliases().iter().map(String::as_str).collect();
1933 if !aliases.is_empty() {
1934 scope.rename(&aliases, "unnamed_subquery")?;
1935 }
1936 // What the catalog tables report as this view's columns, written down here because this is
1937 // the moment they are known. Upstream refreshes the same cache at the same point, which was
1938 // measured: both `duckdb_columns()` and `duckdb_views().column_count` keep reporting the old
1939 // list after an `ALTER TABLE` underneath until something reads the view, and then both move.
1940 // It is written before the label and before the `AS t(a, b)` list below, because those two
1941 // rename the view for one query and not for everyone.
1942 view.remember(scope.fields());
1943 let label = if alias == NONE { name.table.clone() } else { ast.string(alias).to_string() };
1944 scope.relabel(&label);
1945 if !columns.is_empty() {
1946 let names: Vec<&str> = ast.name(columns).collect();
1947 scope.rename(&names, &label)?;
1948 }
1949 Ok((node, scope))
1950 }
1951
1952 /// A function call where a table goes, such as `range(10)`.
1953 ///
1954 /// The arguments are bound against an empty scope. A table function that can see the row on its
1955 /// left is `LATERAL`, and this is not it, so a column name in here is not resolved against
1956 /// whatever happens to be to the left in the `FROM` list. Letting it would mean `FROM t,
1957 /// range(t.n)` quietly binding to something whose meaning depends on the order the sources were
1958 /// written in.
1959 fn bind_table_function(
1960 &mut self,
1961 ast: &Ast,
1962 name: ast::Slice,
1963 args: ast::Slice,
1964 alias: ast::StrRef,
1965 columns: ast::Slice,
1966 pragma: bool,
1967 ) -> Result<(NodeRef, Scope)> {
1968 // The column names written after the alias, kept under a name of their own because the
1969 // match on what the function's columns are below binds `columns` to something else.
1970 let renamed = columns;
1971 let parts: Vec<&str> = ast.name(name).collect();
1972 // A qualified call names a schema, and the two schemas that exist are the ones every
1973 // built-in lives in. Anything else is a name that has to fail rather than fall through to
1974 // the unqualified lookup and be found somewhere it was not asked for.
1975 let function_name = *parts.last().unwrap_or(&"");
1976 if let Some(schema) = parts.iter().rev().nth(1) {
1977 if !schema.eq_ignore_ascii_case("main") && !schema.eq_ignore_ascii_case("system") {
1978 return Err(Error::catalog(format!(
1979 "Table Function with name {} does not exist!",
1980 parts.join(".")
1981 )));
1982 }
1983 }
1984 // The name is looked up before the arguments are bound so that a call of something that is
1985 // not a table function says that, rather than reporting whatever is wrong with the
1986 // arguments of a function that was never going to exist.
1987 let Some(called) = TableFunction::lookup(function_name) else {
1988 if pragma {
1989 // `PRAGMA database_list` is a view upstream and not a function, and the pragma
1990 // namespace holds both, so a name that is not a function gets one more look in the
1991 // catalog before it is turned down. It has to be the no argument form: a view
1992 // takes none, and `pragma_database_list()` with parentheses is a missing function
1993 // on the pin too.
1994 if args.is_empty() && self.catalog.resolve(&parts).is_ok() {
1995 return self.bind_table(ast, name, alias, columns);
1996 }
1997 let spelled = function_name.strip_prefix("pragma_").unwrap_or(function_name);
1998 return Err(Error::catalog(format!(
1999 "Pragma Function with name {spelled} does not exist!"
2000 )));
2001 }
2002 return Err(Error::catalog(format!(
2003 "Table Function with name {function_name} does not exist!"
2004 )));
2005 };
2006 let written = ast.target_list(args).to_vec();
2007 let empty = Scope::empty();
2008 let previous = std::mem::replace(&mut self.clause, "table function arguments");
2009 let mut bound = Vec::new();
2010 let mut written_options = Vec::new();
2011 for argument in written {
2012 let expr = self.bind_expr(ast, argument.expr, &empty)?;
2013 if argument.alias == NONE {
2014 bound.push(expr);
2015 } else {
2016 let name = ast.string(argument.alias).to_string();
2017 let (parameter, value) = self.named_argument(called, &name, expr)?;
2018 written_options.push((parameter, value, expr));
2019 }
2020 }
2021 self.clause = previous;
2022 let options = Options::of(&written_options)?;
2023
2024 // The types are what resolve the call, not the count, because `read_parquet(3)` is a
2025 // different answer from `read_parquet('3')` and only the types tell them apart.
2026 let given: Vec<LogicalType> =
2027 bound.iter().map(|&expr| self.plan.expr_type(expr).clone()).collect();
2028 let resolved = if pragma {
2029 resolve_pragma(function_name, &given)?
2030 } else {
2031 resolve_table(function_name, &given)?
2032 };
2033 let mut cast: Vec<ExprRef> = bound
2034 .iter()
2035 .zip(&resolved.arguments)
2036 .map(|(&expr, ty)| self.checked_cast_to(expr, ty, false))
2037 .collect::<Result<_>>()?;
2038
2039 if resolved.function.answered_when_bound() {
2040 let Columns::Fixed(fields) = resolved.columns else {
2041 return Err(Error::internal("a pragma that resolved to a file"));
2042 };
2043 let [argument] = cast[..] else {
2044 return Err(Error::internal("a pragma that resolved to more than one name"));
2045 };
2046 return self.bind_pragma(ast, resolved.function, &fields, argument, alias, columns);
2047 }
2048 // Filled in by the arm below that has the file names, and left alone by a function whose
2049 // columns are fixed, because none of those reads a file to find out how tall it is.
2050 let mut measured = Stat::Unknown;
2051 let mut counted: Vec<(String, Stat<u64>)> = Vec::new();
2052 let mut bounded: Option<Arc<dyn Zones>> = None;
2053 let fields = match resolved.columns {
2054 Columns::Fixed(fields) => fields,
2055 columns => {
2056 // The one argument is a pattern, and what replaces it is one constant per file it
2057 // matched. The executor is handed names rather than a pattern, so it never walks a
2058 // directory and the answer cannot change between binding a prepared statement and
2059 // running it, which is the same reason the schema is settled here.
2060 let paths = self.file_paths(cast[0], resolved.function.name())?;
2061 let mut mirrorable = None;
2062 if resolved.function == TableFunction::ReadParquet && !options.file_row_number {
2063 if let Some((path, stamp)) = mirror_target(&paths) {
2064 if let Some(name) =
2065 self.catalog.mirror(&path, options.binary_as_string, stamp)
2066 {
2067 let name = name.clone();
2068 let label = if alias == NONE {
2069 resolved.function.name().to_string()
2070 } else {
2071 ast.string(alias).to_string()
2072 };
2073 return self.bind_catalog_table(ast, &name, label, renamed);
2074 }
2075 mirrorable = Some(path);
2076 }
2077 }
2078 let mut fields = match columns {
2079 // Parquet takes the first file's footer as the answer and CSV sniffs all of
2080 // them, which is not a choice made here. See `csv_fields`.
2081 Columns::Csv => csv_fields(&paths, options.given)?,
2082 _ => {
2083 let footers = self.footers(&paths, mirrorable.as_deref())?;
2084 if let Some(path) = mirrorable.as_deref() {
2085 self.want_mirror(path, options.binary_as_string, &footers.rows);
2086 }
2087 measured = footers.rows;
2088 counted = footers.distincts;
2089 bounded = footers.zones;
2090 footers.fields
2091 }
2092 };
2093 if options.all_varchar {
2094 // The sniffer still ran, because the names come out of the same pass over the
2095 // front of the file and only the types are being overruled. The executor reads
2096 // the text as VARCHAR because this is the schema it is told to read into, which
2097 // is the same road a file in a glob takes when the set is wider than the file.
2098 for field in &mut fields {
2099 field.ty = LogicalType::Varchar;
2100 }
2101 }
2102 if options.binary_as_string {
2103 // A byte array column with no annotation on it is a BLOB, and this is the caller
2104 // saying that the file's writer meant text. The reader already holds both in the
2105 // same string column and already validates the bytes, so the whole of the option
2106 // is what the column is called from here on.
2107 for field in &mut fields {
2108 if field.ty == LogicalType::Blob {
2109 field.ty = LogicalType::Varchar;
2110 }
2111 }
2112 }
2113 if options.file_row_number {
2114 // Not a column of the file, so it goes on the end where a projection cannot be
2115 // confused about which one it is, and the executor counts it as the rows come
2116 // out. A file that already has a column of that name is the one case where the
2117 // option cannot be honoured, and saying so is better than handing back two
2118 // columns with the same name and letting a reference to it pick one.
2119 if fields.iter().any(|field| field.name == FILE_ROW_NUMBER) {
2120 return Err(Error::binder(format!(
2121 "Duplicate column name \"{FILE_ROW_NUMBER}\": the file already has a \
2122 column of that name, so file_row_number cannot add one"
2123 )));
2124 }
2125 fields.push(Field::required(FILE_ROW_NUMBER.to_string(), LogicalType::BigInt));
2126 }
2127 cast = paths.iter().map(|path| self.path_constant(path)).collect();
2128 fields
2129 }
2130 };
2131 let label = if alias == NONE {
2132 resolved.function.name().to_string()
2133 } else {
2134 ast.string(alias).to_string()
2135 };
2136 let names: Vec<&str> = ast.name(columns).collect();
2137 self.table_function_source(
2138 resolved.function,
2139 &cast,
2140 &written_options,
2141 Read { fields, rows: measured, distincts: counted, zones: bounded },
2142 &label,
2143 &names,
2144 )
2145 }
2146
2147 /// `pragma_table_info('t')` or `pragma_show('t')`, answered while it is bound.
2148 ///
2149 /// The same trick `DESCRIBE` uses and for the same reason: the columns of a table are settled by
2150 /// the time the name has resolved, so the rows are a constant from there on and this comes out
2151 /// as a `VALUES` rather than as an operator that reads a catalog while the query runs. It also
2152 /// means `SELECT name FROM pragma_table_info('t') WHERE notnull` is an ordinary query over an
2153 /// ordinary relation, which is the whole reason these exist as functions rather than only as
2154 /// statements.
2155 ///
2156 /// The name arrives as a string rather than as something the parser read, so it is split here
2157 /// under the identifier rule and then resolved like any other name. A name that is not there
2158 /// comes back as the catalog's own complaint, which is what the pin answers with too.
2159 fn bind_pragma(
2160 &mut self,
2161 ast: &Ast,
2162 function: TableFunction,
2163 fields: &[Field],
2164 argument: ExprRef,
2165 alias: ast::StrRef,
2166 columns: ast::Slice,
2167 ) -> Result<(NodeRef, Scope)> {
2168 let written = self.pragma_name(argument, function)?;
2169 let parts = identifier_parts(&written);
2170 let spelled: Vec<&str> = parts.iter().map(String::as_str).collect();
2171 let name = self.catalog.resolve(&spelled)?;
2172 let described = self.described(ast, &name)?;
2173 let mut rows = Vec::with_capacity(described.len());
2174 for (at, field) in described.iter().enumerate() {
2175 let items = if matches!(function, TableFunction::PragmaShow) {
2176 self.describing(field)
2177 } else {
2178 self.table_info(at, field)
2179 };
2180 rows.push(self.plan.add_expr_list(&items));
2181 }
2182 let rows = self.plan.add_rows(&rows);
2183 let held = self.plan.add_fields(fields);
2184 let index = self.fresh_index();
2185 let node = self.add_node(Node::Values { index, columns: held, rows });
2186 let label =
2187 if alias == NONE { function.name().to_string() } else { ast.string(alias).to_string() };
2188 let mut scope = Scope::empty();
2189 for (at, field) in fields.iter().enumerate() {
2190 scope.push(Visible {
2191 table: label.clone(),
2192 name: field.name.clone(),
2193 binding: ColumnBinding::new(index, at as u32),
2194 ty: field.ty.clone(),
2195 not_null: false,
2196 also: None,
2197 });
2198 }
2199 if !columns.is_empty() {
2200 let names: Vec<&str> = ast.name(columns).collect();
2201 scope.rename(&names, &label)?;
2202 }
2203 Ok((node, scope))
2204 }
2205
2206 /// The name a pragma was called with, which has to be a constant.
2207 ///
2208 /// A null is a name spelled `NULL` rather than an error about nulls, because the pin turns
2209 /// whatever it was handed into text before it goes looking and then says a table of that name
2210 /// does not exist. Writing `pragma_table_info(NULL)` is a mistake either way and this is the
2211 /// sentence the mistake already has.
2212 ///
2213 /// `pragma_table_info('t' || 'x')` is the pin's `tx` and is turned away here, which is the same
2214 /// missing constant folding [`Binder::named_argument`] writes about and closes the same day.
2215 fn pragma_name(&self, argument: ExprRef, function: TableFunction) -> Result<String> {
2216 let Expr::Constant(reference) = *self.plan.expr(argument) else {
2217 return Err(Error::not_implemented(format!(
2218 "{}() given a name that is not a constant",
2219 function.name()
2220 )));
2221 };
2222 match self.plan.value(reference) {
2223 Value::Varchar(name) => Ok(name.clone()),
2224 Value::Null => Ok("NULL".to_string()),
2225 other => {
2226 Err(Error::internal(format!("a pragma name bound as VARCHAR arrived as {other}")))
2227 }
2228 }
2229 }
2230
2231 /// The columns of whatever a pragma was pointed at.
2232 ///
2233 /// A view is bound here, which is how it comes to have columns at all. Reading a view is what
2234 /// binds it and describing one counts as reading it, so a view the engine ships with reports a
2235 /// column count from this point on, the same as it would after a select. The node that binding
2236 /// produces is thrown away, because the answer is the scope and not the query.
2237 ///
2238 /// Every column of a view is nullable whatever the column underneath was declared as, which is
2239 /// the pin's answer through `pragma_table_info()`, `pragma_show()` and `duckdb_columns()` alike.
2240 /// [`Scope::fields`] drops the flag on its own, so there is nothing to clear here.
2241 fn described(&mut self, ast: &Ast, name: &QualifiedName) -> Result<Vec<Field>> {
2242 if self.catalog.entry(name)? == Entry::Table {
2243 return Ok(self.catalog.table(name)?.columns().to_vec());
2244 }
2245 let (_, scope) = self.bind_view(ast, name, NONE, ast::Slice::default())?;
2246 Ok(scope.fields())
2247 }
2248
2249 /// One row of `pragma_show()`, which is one row of `DESCRIBE` written by the other caller.
2250 fn describing(&mut self, field: &Field) -> Vec<ExprRef> {
2251 let written = [
2252 field.name.clone(),
2253 field.ty.to_string(),
2254 if field.not_null { "NO" } else { "YES" }.to_owned(),
2255 ];
2256 let mut items: Vec<ExprRef> =
2257 written.into_iter().map(|text| self.plan.add_constant(Value::Varchar(text))).collect();
2258 for _ in 0..3 {
2259 let empty = self.plan.add_constant(Value::Null);
2260 items.push(self.cast_to(empty, &LogicalType::Varchar));
2261 }
2262 items
2263 }
2264
2265 /// One row of `pragma_table_info()`, which is SQLite's six columns about the same column.
2266 ///
2267 /// `cid` counts from zero, which is SQLite's numbering and not the one based `ordinal_position`
2268 /// the standard views report. `dflt_value` and `pk` are the two nothings rudb has to report
2269 /// until `CREATE TABLE` takes a `DEFAULT` or a key.
2270 fn table_info(&mut self, at: usize, field: &Field) -> Vec<ExprRef> {
2271 let cid = self.plan.add_constant(Value::Integer(i32::try_from(at).unwrap_or(i32::MAX)));
2272 let name = self.plan.add_constant(Value::Varchar(field.name.clone()));
2273 let ty = self.plan.add_constant(Value::Varchar(field.ty.to_string()));
2274 let not_null = self.plan.add_constant(Value::Boolean(field.not_null));
2275 let default = self.plan.add_constant(Value::Null);
2276 let default = self.cast_to(default, &LogicalType::Varchar);
2277 let key = self.plan.add_constant(Value::Boolean(false));
2278 vec![cid, name, ty, not_null, default, key]
2279 }
2280
2281 /// One named parameter of a table function call, folded into what the call was given.
2282 ///
2283 /// The value has to be a constant of the type the parameter wants. It has to be constant
2284 /// because an option can decide what the columns are and the columns are settled here, and it
2285 /// has to be already of the type because there is no constant folding in front of the binder
2286 /// yet. DuckDB folds first, so `binary_as_string=1` and `binary_as_string='yes'` are both true
2287 /// there and both are turned away here, which is a gap that closes on its own the day the
2288 /// optimizer runs before the plan is finished. `binary_as_string=True` is what the ClickBench
2289 /// entry writes and is what has to work.
2290 ///
2291 /// A name that is not a parameter of this function is the binary's sentence followed by what it
2292 /// could have been. The binary puts the candidates on their own indented lines and this puts
2293 /// them on the same line, because an error is one line here.
2294 fn named_argument(
2295 &mut self,
2296 function: TableFunction,
2297 name: &str,
2298 expr: ExprRef,
2299 ) -> Result<(&'static str, Value)> {
2300 let known = function
2301 .parameters()
2302 .iter()
2303 .find(|(parameter, _)| parameter.eq_ignore_ascii_case(name));
2304 let Some((parameter, wanted)) = known else {
2305 let candidates: Vec<String> = function
2306 .parameters()
2307 .iter()
2308 .map(|(parameter, ty)| format!(" {parameter} {ty}"))
2309 .collect();
2310 return Err(Error::binder(format!(
2311 "Invalid named parameter \"{name}\" for function {}\nCandidates:\n{}\n",
2312 function.name(),
2313 candidates.join("\n")
2314 )));
2315 };
2316 let Expr::Constant(reference) = *self.plan.expr(expr) else {
2317 return Err(Error::not_implemented(format!(
2318 "the named parameter {parameter} with a value that is not a constant"
2319 )));
2320 };
2321 let value = self.plan.value(reference).clone();
2322 if value == Value::Null {
2323 return Err(Error::binder(null_parameter(function, parameter)));
2324 }
2325 let given = self.plan.expr_type(expr).clone();
2326 if given != *wanted {
2327 return Err(Error::not_implemented(format!(
2328 "the named parameter {parameter} given a {given} where a {wanted} was wanted"
2329 )));
2330 }
2331 Ok((parameter, value))
2332 }
2333
2334 /// A file where a table name goes, which is what DuckDB calls a replacement scan.
2335 ///
2336 /// `SELECT * FROM 'hits.parquet'` is how most DuckDB queries in the wild are written, ClickBench
2337 /// among them, so this is not sugar over `read_parquet` so much as the spelling people use. The
2338 /// catalog has already been asked and has already said no, and `missing` is what it said, so a
2339 /// name that is not a file comes back with the catalog's own answer rather than with a complaint
2340 /// about files.
2341 ///
2342 /// Only a single unqualified name is a candidate. A qualified one names a schema and a schema
2343 /// that does not exist is not a path.
2344 fn bind_replacement_scan(
2345 &mut self,
2346 ast: &Ast,
2347 parts: &[&str],
2348 alias: ast::StrRef,
2349 columns: ast::Slice,
2350 missing: Error,
2351 ) -> Result<(NodeRef, Scope)> {
2352 let [path] = parts else { return Err(missing) };
2353 let path = *path;
2354 let extension = path.rsplit_once('.').map(|(_, after)| after).unwrap_or_default();
2355 let Some(function) = Self::reader_for(extension) else {
2356 if is_file(path) {
2357 // A file that is really there and that nothing here can read is a different mistake
2358 // from a name that is not a file, and DuckDB says so with both lines, the second of
2359 // which is the way out. A file with no dot in it lands here too, which is why the
2360 // test is on the extension having a reader rather than on there being an extension.
2361 return Err(Error::binder(format!(
2362 "No extension found that is capable of reading the file \"{path}\"\n* If this \
2363 file is a supported file format you can explicitly use the reader functions, \
2364 such as read_csv, read_json or read_parquet"
2365 )));
2366 }
2367 return Err(missing);
2368 };
2369 // The pattern is expanded before it is known to match anything, so a name that ends in .csv
2370 // and is not there gives the reader's own message rather than the catalog's. That is
2371 // DuckDB's order and it is the helpful one: somebody who wrote a file name wants to hear
2372 // about the file.
2373 let paths = files(path)?;
2374 // The name the columns answer to is the file's stem, so `SELECT mixed.a FROM
2375 // 'data/mixed.parquet'` works. That is DuckDB's choice and it is the useful one, since the
2376 // alternative is a table name with a dot and a slash in it that nothing can write. A pattern
2377 // keeps the whole of what was written instead, which is DuckDB's choice too and was
2378 // measured: there is no stem to take when the name stands for a directory full of files.
2379 let label = if alias == NONE {
2380 if is_pattern(path) {
2381 path.to_string()
2382 } else {
2383 let file = path.rsplit_once('/').map_or(path, |(_, file)| file);
2384 file.rsplit_once('.').map_or(file, |(stem, _)| stem).to_string()
2385 }
2386 } else {
2387 ast.string(alias).to_string()
2388 };
2389 let mut mirrorable = None;
2390 if function == TableFunction::ReadParquet {
2391 if let Some((canonical, stamp)) = mirror_target(&paths) {
2392 if let Some(name) = self.catalog.mirror(&canonical, false, stamp) {
2393 let name = name.clone();
2394 return self.bind_catalog_table(ast, &name, label, columns);
2395 }
2396 mirrorable = Some(canonical);
2397 }
2398 }
2399 let read = match function {
2400 TableFunction::ReadParquet => {
2401 let footers = self.footers(&paths, mirrorable.as_deref())?;
2402 if let Some(canonical) = mirrorable.as_deref() {
2403 self.want_mirror(canonical, false, &footers.rows);
2404 }
2405 Read {
2406 fields: footers.fields,
2407 rows: footers.rows,
2408 distincts: footers.distincts,
2409 zones: footers.zones,
2410 }
2411 }
2412 _ => Read::uncounted(csv_fields(&paths, Given::default())?),
2413 };
2414 let arguments: Vec<ExprRef> = paths.iter().map(|path| self.path_constant(path)).collect();
2415 let names: Vec<&str> = ast.name(columns).collect();
2416 self.table_function_source(function, &arguments, &[], read, &label, &names)
2417 }
2418
2419 /// What the footers of `paths` say, from the outline alone where this bind is outlined and the
2420 /// read could go through a mirror.
2421 ///
2422 /// An outline that does not state a row count is read again in full, because a read that asks
2423 /// for no mirror would leave the plan outlined with nothing telling the caller to bind again.
2424 fn footers(&self, paths: &[String], mirrorable: Option<&str>) -> Result<Footers> {
2425 if let Some(path) = mirrorable.filter(|_| self.outlined) {
2426 let outline = parquet_outline(path)?;
2427 if outline.rows.value().is_some() {
2428 return Ok(outline);
2429 }
2430 }
2431 parquet_footers(paths)
2432 }
2433
2434 /// Says the Parquet file at `path` could have been read through a native mirror, when its
2435 /// footer says how many rows it holds, which is what the database decides whether one would
2436 /// repay itself by.
2437 fn want_mirror(&mut self, path: &str, binary_as_string: bool, rows: &Stat<u64>) {
2438 if let Some(&rows) = rows.value() {
2439 self.plan.want_mirror(path, binary_as_string, rows);
2440 }
2441 }
2442
2443 /// One file name, as a constant expression in the plan.
2444 fn path_constant(&mut self, path: &str) -> ExprRef {
2445 let value = self.plan.add_value(Value::Varchar(path.to_string()));
2446 self.plan.add_expr(Expr::Constant(value), LogicalType::Varchar)
2447 }
2448
2449 /// The table function a file with this extension is read by, and `None` for one nothing reads.
2450 ///
2451 /// Both spellings of a tab separated file go to the CSV reader, which is not a shortcut: the
2452 /// extension picks the reader and the reader sniffs the punctuation, so a `.tsv` file that holds
2453 /// commas is read as commas. That was measured rather than assumed. The comparison ignores case
2454 /// because `UP.CSV` reads in duckdb v1.4.1.
2455 fn reader_for(extension: &str) -> Option<TableFunction> {
2456 if extension.eq_ignore_ascii_case("parquet") {
2457 return Some(TableFunction::ReadParquet);
2458 }
2459 if extension.eq_ignore_ascii_case("csv") || extension.eq_ignore_ascii_case("tsv") {
2460 return Some(TableFunction::ReadCsv);
2461 }
2462 None
2463 }
2464
2465 /// The node and the scope of a table function call whose arguments and columns are settled.
2466 ///
2467 /// The half a written out call shares with a replacement scan, which is everything after the
2468 /// question of what the file is called has been answered one way or the other.
2469 ///
2470 /// `read` is what the caller found out about the files, which comes in here rather than being
2471 /// read here because this function has the names and not the files: a replacement scan has
2472 /// already expanded its pattern and a written out call has already cast its argument, and
2473 /// neither of them wants to do it twice.
2474 fn table_function_source(
2475 &mut self,
2476 function: TableFunction,
2477 args: &[ExprRef],
2478 written: &[(&'static str, Value, ExprRef)],
2479 read: Read,
2480 label: &str,
2481 names: &[&str],
2482 ) -> Result<(NodeRef, Scope)> {
2483 let Read { fields, rows, distincts, zones } = read;
2484 let index = self.fresh_index();
2485 // Against the table index rather than against the node, because a pass is free to move the
2486 // node and none of them can move an index: an index is what a column reference names and
2487 // rewriting one would mean rewriting every expression above it. Nothing is recorded for a
2488 // function nobody measured, since an absent entry already reads back as unknown.
2489 if rows.is_known() {
2490 self.plan.measure(index, rows);
2491 }
2492 for (column, distinct) in distincts {
2493 self.plan.measure_distinct(index, &column, distinct);
2494 }
2495 if let Some(zones) = zones {
2496 self.plan.set_zones(index, zones);
2497 }
2498 let mut scope = Scope::empty();
2499 for (at, field) in fields.iter().enumerate() {
2500 scope.push(Visible {
2501 table: label.to_string(),
2502 name: field.name.clone(),
2503 binding: ColumnBinding::new(index, at as u32),
2504 ty: field.ty.clone(),
2505 // A reader takes what the file has, and no file format this reads says a column
2506 // cannot be null. The reference binary answers YES for every column of a Parquet.
2507 not_null: false,
2508 also: None,
2509 });
2510 }
2511 if !names.is_empty() {
2512 scope.rename(names, label)?;
2513 } else if matches!(function, TableFunction::Range | TableFunction::GenerateSeries) {
2514 // The PostgreSQL naming, which the pin follows for these two and for no reader: the
2515 // alias names the one column, and the column keeps answering to its own name too.
2516 for column in &mut scope.columns {
2517 column.also = Some(std::mem::replace(&mut column.name, label.to_string()));
2518 }
2519 }
2520 let function = self.plan.intern(function.name());
2521 let args = self.plan.add_expr_list(args);
2522 let named: Vec<u32> =
2523 written.iter().map(|(parameter, _, _)| self.plan.intern(parameter)).collect();
2524 let settings: Vec<ExprRef> = written.iter().map(|(_, _, expr)| *expr).collect();
2525 let options = self.plan.add_name_list(&named);
2526 let settings = self.plan.add_expr_list(&settings);
2527 let columns = self.plan.add_fields(&fields);
2528 let node = self.add_node(Node::TableFunction {
2529 index,
2530 function,
2531 args,
2532 options,
2533 settings,
2534 columns,
2535 });
2536 Ok((node, scope))
2537 }
2538
2539 /// Every file a table function's file argument names, in the order they were written.
2540 ///
2541 /// Each pattern has to find at least one file of its own, which is DuckDB's rule and is why
2542 /// this expands one at a time rather than gathering everything and looking at the total. A
2543 /// list keeps its written order and its duplicates, so a file named twice is read twice, which
2544 /// was measured: the sort and the dedup belong to one pattern rather than to the list.
2545 fn file_paths(&self, expr: ExprRef, name: &str) -> Result<Vec<String>> {
2546 let mut paths = Vec::new();
2547 for pattern in self.file_patterns(expr, name)? {
2548 paths.extend(files(&pattern)?);
2549 }
2550 Ok(paths)
2551 }
2552
2553 /// The patterns a table function argument names, which have to be constants.
2554 ///
2555 /// A table function that reads a file is resolved by opening the file, and that happens here
2556 /// rather than when the query runs, because the rest of the statement cannot bind until the
2557 /// column names are known. So the path has to be something this binder can work out without
2558 /// running anything, and a literal is that. DuckDB folds a constant expression first, so
2559 /// `read_parquet('a' || '.parquet')` works there, and folding is M1 work that this will pick up
2560 /// for free once the optimizer runs before the plan is finished rather than after.
2561 ///
2562 /// One string is one pattern and a list is one pattern an item, which is DuckDB's pair of
2563 /// overloads. A null is a different sentence in each of them, both of them measured.
2564 ///
2565 /// The argument is folded rather than required to be a literal. A list is a call to `list_value`
2566 /// as of the work on #467, so requiring a literal here would have turned every `read_parquet`
2567 /// over a list into the message about a name that is not a constant, and the sentence this
2568 /// comment used to carry about folding being picked up for free was the plan for exactly that.
2569 /// What it buys beyond keeping the list working is `read_parquet('a' || '.parquet')`, which the
2570 /// pin answers and which used to be refused here.
2571 fn file_patterns(&self, expr: ExprRef, name: &str) -> Result<Vec<String>> {
2572 let Some(value) = fold::value_of(&self.plan, expr)? else {
2573 return Err(Error::not_implemented(
2574 "a table function file name that is not a constant",
2575 ));
2576 };
2577 match value {
2578 Value::Varchar(path) => Ok(vec![path]),
2579 // DuckDB's own wording, which says list because its other overload takes one.
2580 Value::Null => Err(Error::parser(format!("{name} cannot take NULL list as parameter"))),
2581 // An empty list reaches the reader rather than failing to bind, because `[]` carries an
2582 // element type of the untyped null and a null promotes to VARCHAR, so the call resolves.
2583 // The pin says this, and it says it as an IO error rather than as a binder one, since
2584 // the list was a fine list and the objection is that there is no file in it.
2585 Value::List { values, .. } if values.is_empty() => {
2586 Err(Error::io(format!("\"{name}\" needs at least one file to read")))
2587 }
2588 Value::List { values, .. } => values
2589 .iter()
2590 .map(|value| match value {
2591 Value::Varchar(path) => Ok(path.clone()),
2592 _ => Err(Error::parser(format!(
2593 "{name} reader cannot take NULL input as parameter"
2594 ))),
2595 })
2596 .collect(),
2597 other => {
2598 Err(Error::internal(format!("a file name bound as VARCHAR arrived as {other}")))
2599 }
2600 }
2601 }
2602
2603 /// Which input of a join a query written in its `ON` has to be joined into.
2604 ///
2605 /// A join condition is evaluated by the join, over the rows its two inputs handed it, so a
2606 /// column the condition reads has to be produced by one of those two. A query written in the
2607 /// `ON` produces columns the condition reads, which means the query cannot be joined in above
2608 /// the join the way one written in a `WHERE` or a `SELECT` is. It has to go underneath, into
2609 /// one input or the other.
2610 ///
2611 /// Which input is decided by what the query reads. A query whose body reads the right side can
2612 /// only be evaluated where those rows are, so it goes into the right input, and the same for
2613 /// the left. A query that reads neither could go into either and goes into the left, which is
2614 /// also where an `IN` puts one whose left hand side reads the left and whose body reads
2615 /// nothing.
2616 ///
2617 /// The one that has no answer is a query that reads both sides. There is no single input that
2618 /// produces what it needs, and the shape upstream calls a pair dependent join is what handles
2619 /// it. `None` is that case, and the caller turns it into a refusal rather than a plan.
2620 fn side_of(
2621 &self,
2622 pending: &PendingSubquery,
2623 left_tables: &[u32],
2624 right_tables: &[u32],
2625 ) -> Option<Side> {
2626 let mut needs_left = false;
2627 let mut needs_right = false;
2628 let mut note = |binding: ColumnBinding| {
2629 needs_left |= left_tables.contains(&binding.table);
2630 needs_right |= right_tables.contains(&binding.table);
2631 };
2632 for &binding in &pending.reads {
2633 note(binding);
2634 }
2635 // A mark join carries the comparison rather than the condition carrying it, and that
2636 // comparison is written over the join's own rows. `l.a IN (SELECT ...)` reads the left side
2637 // there and nowhere else, so leaving it out would put the query on whichever side its body
2638 // happened to name and let the comparison ask a join for a column it was not given.
2639 for &condition in &pending.conditions {
2640 self.plan.read_columns(condition, &mut |_, binding| note(binding));
2641 }
2642 match (needs_left, needs_right) {
2643 (true, true) => None,
2644 (_, true) => Some(Side::Right),
2645 _ => Some(Side::Left),
2646 }
2647 }
2648
2649 /// A join whose condition holds a query that reads rows from both of its inputs.
2650 ///
2651 /// This is the one [`Binder::side_of`] has no side for. The query has to be evaluated once per
2652 /// pair of rows, and there is no input that produces a pair, so it cannot go into either input
2653 /// the way the other two cases do. What produces a pair is the join itself, so the join becomes
2654 /// a product, the query is joined into the product's rows the way a query in a `WHERE` is joined
2655 /// into the rows the whole `FROM` produced, and the condition becomes a filter above that.
2656 ///
2657 /// That rewrite is only the same query for an inner join. An inner join keeps the pairs its
2658 /// condition holds and drops the rest, which is what a product and a filter do. Every other kind
2659 /// does something with the pairs it dropped, a left join pads them, a semi join counts them, and
2660 /// a filter above a product has already thrown away which left row a dropped pair came from, so
2661 /// those are refused by name. Upstream plans them as a pair dependent join and rudb does not
2662 /// have one yet, which is what tamnd/rudb#913 stays open for.
2663 ///
2664 /// The product is not the plan that runs. The condition goes back into the join as a condition
2665 /// when filter pushdown looks at it, which is the pass that already turns a filter over an inner
2666 /// join into a join condition, so an equality in the `ON` is still an equality the hash join can
2667 /// build on. What cannot be pushed back down is the part that reads the query's output, and that
2668 /// part could not have been a join condition in the first place.
2669 #[allow(clippy::too_many_arguments)]
2670 fn bind_pair_dependent_join(
2671 &mut self,
2672 kind: ast::JoinKind,
2673 independent: bool,
2674 left: NodeRef,
2675 right: NodeRef,
2676 pair: Vec<PendingSubquery>,
2677 conditions: Vec<ExprRef>,
2678 scope: Scope,
2679 ) -> Result<(NodeRef, Scope)> {
2680 if kind != ast::JoinKind::Inner {
2681 return Err(Error::not_implemented(
2682 "a subquery that reads both sides of that join, written in the condition of a join \
2683 that is not an inner join"
2684 .to_string(),
2685 ));
2686 }
2687 // A lateral right side is already evaluated per left row, so the product this would build is
2688 // not the product the query means.
2689 if !independent {
2690 return Err(Error::not_implemented(
2691 "a subquery that reads both sides of that join, written in the condition of a join \
2692 whose right side is lateral"
2693 .to_string(),
2694 ));
2695 }
2696 let mut node = self.add_node(Node::CrossProduct { left, right });
2697 for pending in pair {
2698 node = self.attach_subquery(node, pending);
2699 }
2700 // `ON` and `USING` cannot both be written, and this is only reached from the `ON` path, so
2701 // the list is the one bound condition. The fold is here so that it stays right if that stops
2702 // being true rather than for a case that exists today.
2703 let mut conditions = conditions.into_iter();
2704 let mut predicate = conditions.next().expect("a join condition was bound");
2705 for next in conditions {
2706 let children = self.plan.add_expr_list(&[predicate, next]);
2707 let conjunction = Expr::Conjunction { op: ConjunctionOp::And, children };
2708 predicate = self.plan.add_expr(conjunction, LogicalType::Boolean);
2709 }
2710 let node = self.add_node(Node::Filter { input: node, predicate });
2711 Ok((node, scope))
2712 }
2713
2714 #[allow(clippy::too_many_arguments)]
2715 fn bind_join(
2716 &mut self,
2717 ast: &Ast,
2718 left: ast::SourceRef,
2719 right: ast::SourceRef,
2720 kind: ast::JoinKind,
2721 natural: bool,
2722 on: ast::ExprRef,
2723 using: ast::Slice,
2724 ) -> Result<(NodeRef, Scope)> {
2725 let (left_node, left_scope) = self.bind_source(ast, left)?;
2726 let (right_node, right_scope, correlated) = self.bind_lateral(ast, right, &left_scope)?;
2727 // A row of the right side exists only for the left row it was evaluated against, so a kind
2728 // that has to produce right rows with no left row has nothing to produce them from. The
2729 // pinned build says this and names only the two kinds that work.
2730 if !correlated.is_empty()
2731 && !matches!(kind, ast::JoinKind::Inner | ast::JoinKind::Cross | ast::JoinKind::Left)
2732 {
2733 return Err(Error::binder(
2734 "The combining JOIN type must be INNER or LEFT for a LATERAL reference",
2735 ));
2736 }
2737 let split = left_scope.len();
2738 // Which table index came from which side, kept before the two scopes become one. A query
2739 // written in the `ON` is joined into one of the inputs rather than above the join, and this
2740 // is what says which. A `USING` drops the right side's copy of a joined-on column out of
2741 // the scope below, and dropping a column does not change the index it came from, so the
2742 // answer this gives is still right afterwards.
2743 let left_tables: Vec<u32> =
2744 left_scope.columns.iter().map(|column| column.binding.table).collect();
2745 let right_tables: Vec<u32> =
2746 right_scope.columns.iter().map(|column| column.binding.table).collect();
2747 let mut scope = left_scope.concat(right_scope);
2748
2749 // NATURAL is USING over whatever both sides happen to call the same thing, which is why it
2750 // is resolved here and never reaches the plan as its own idea.
2751 let merged: Vec<String> = if natural {
2752 let mut names = Vec::new();
2753 for (at, column) in scope.columns.iter().enumerate().take(split) {
2754 if scope.columns[split..].iter().any(|right| same_name(&right.name, &column.name))
2755 && !names.iter().any(|held: &String| same_name(held, &column.name))
2756 {
2757 let _ = at;
2758 names.push(column.name.clone());
2759 }
2760 }
2761 names
2762 } else {
2763 // A name written twice is one column, not two. `USING (id, id)` is legal and means what
2764 // `USING (id)` means, and the reference binary agrees. Taking it twice would build the
2765 // same equality twice and, worse, drop the right side's copy twice, which takes a
2766 // column out of the answer that nobody named and runs off the end of the scope when the
2767 // copy was the last column in it.
2768 let mut names: Vec<String> = Vec::new();
2769 for name in ast.name(using) {
2770 if !names.iter().any(|held| same_name(held, name)) {
2771 names.push(name.to_string());
2772 }
2773 }
2774 names
2775 };
2776
2777 let mut conditions = Vec::new();
2778 let mut dropped = Vec::new();
2779 for name in &merged {
2780 let left_at = scope.columns[..split]
2781 .iter()
2782 .position(|column| same_name(&column.name, name))
2783 .ok_or_else(|| {
2784 Error::binder(format!(
2785 "column \"{name}\" specified in USING clause does not exist in left table"
2786 ))
2787 })?;
2788 let right_at = scope.columns[split..]
2789 .iter()
2790 .position(|column| same_name(&column.name, name))
2791 .map(|at| at + split)
2792 .ok_or_else(|| {
2793 Error::binder(format!(
2794 "column \"{name}\" specified in USING clause does not exist in right table"
2795 ))
2796 })?;
2797 let left_column = &scope.columns[left_at];
2798 let (left_binding, left_type) = (left_column.binding, left_column.ty.clone());
2799 let right_column = &scope.columns[right_at];
2800 let (right_binding, right_type) = (right_column.binding, right_column.ty.clone());
2801 let left_expr = self.plan.add_expr(Expr::Column(left_binding), left_type);
2802 let right_expr = self.plan.add_expr(Expr::Column(right_binding), right_type);
2803 conditions.push(self.compare(rudb_plan::CompareOp::Equal, left_expr, right_expr)?);
2804 dropped.push(right_at);
2805 }
2806 // A joined-on column appears once, so the right side's copy goes. Dropping from the back
2807 // keeps the positions of the ones still to drop correct.
2808 dropped.sort_unstable();
2809 for at in dropped.into_iter().rev() {
2810 scope.remove(at);
2811 }
2812
2813 let mut left_node = left_node;
2814 let mut right_node = right_node;
2815 let mut pair = Vec::new();
2816 if on != NONE {
2817 if !merged.is_empty() {
2818 return Err(Error::binder("a join cannot have both ON and USING"));
2819 }
2820 self.clause = "JOIN condition";
2821 let waiting = self.scalar_subqueries.len();
2822 let predicate = self.bind_expr(ast, on, &scope)?;
2823 conditions.push(self.as_boolean(predicate, "JOIN")?);
2824 for pending in self.scalar_subqueries.split_off(waiting) {
2825 match self.side_of(&pending, &left_tables, &right_tables) {
2826 Some(Side::Right) => right_node = self.attach_subquery(right_node, pending),
2827 Some(Side::Left) => left_node = self.attach_subquery(left_node, pending),
2828 None => pair.push(pending),
2829 }
2830 }
2831 }
2832
2833 if kind == ast::JoinKind::Cross && !conditions.is_empty() {
2834 return Err(Error::binder("a CROSS JOIN cannot have a condition"));
2835 }
2836 if !pair.is_empty() {
2837 return self.bind_pair_dependent_join(
2838 kind,
2839 correlated.is_empty(),
2840 left_node,
2841 right_node,
2842 pair,
2843 conditions,
2844 scope,
2845 );
2846 }
2847 // A product is the join with nothing to join on, and it is not one when the right side has
2848 // to be evaluated per left row, because then there is a dependency to lower even though
2849 // there is no condition to test.
2850 if correlated.is_empty()
2851 && conditions.is_empty()
2852 && matches!(kind, ast::JoinKind::Cross | ast::JoinKind::Inner)
2853 {
2854 let node = self.add_node(Node::CrossProduct { left: left_node, right: right_node });
2855 return Ok((node, scope));
2856 }
2857 // A semi join and an anti join ask a question about the right side rather than producing
2858 // any of it, so what is in scope after one is the left side alone. The condition is bound
2859 // above and is the last thing that can name the right side. Without this, `SELECT *` over
2860 // one expanded to both sides and the projection asked a join whose output is the left side
2861 // for columns it does not have, which came out as an internal error about a column not
2862 // being in the schema. That is tamnd/rudb#847. The reference binary refuses `b.w` here with
2863 // a binder error naming `a` as the only candidate table, which is the same rule said from
2864 // the other end.
2865 if matches!(kind, ast::JoinKind::Semi | ast::JoinKind::Anti) {
2866 scope.truncate(split);
2867 }
2868 let kind = match kind {
2869 ast::JoinKind::Inner | ast::JoinKind::Cross => JoinKind::Inner,
2870 ast::JoinKind::Left => JoinKind::Left,
2871 ast::JoinKind::Right => JoinKind::Right,
2872 ast::JoinKind::Full => JoinKind::Full,
2873 ast::JoinKind::Semi => JoinKind::Semi,
2874 ast::JoinKind::Anti => JoinKind::Anti,
2875 ast::JoinKind::Positional => JoinKind::Positional,
2876 };
2877 let conditions = self.plan.add_expr_list(&conditions);
2878 let node = if correlated.is_empty() {
2879 self.add_node(Node::Join {
2880 left: left_node,
2881 right: right_node,
2882 kind,
2883 conditions,
2884 build: BuildSide::default(),
2885 })
2886 } else {
2887 self.add_node(Node::DependentJoin {
2888 left: left_node,
2889 right: right_node,
2890 kind,
2891 conditions,
2892 })
2893 };
2894 Ok((node, scope))
2895 }
2896
2897 // -------------------------------------------------------------- aggregates
2898
2899 /// Binds a `FILTER (WHERE ...)` predicate, or says there was none.
2900 ///
2901 /// The predicate is a condition over the input rows and not over the answer, so it is bound in
2902 /// the scope the arguments are bound in, and it is cast to `BOOLEAN` the way a `WHERE` is:
2903 /// `FILTER (WHERE i)` over an integer column is a filter on whether the integer is not zero.
2904 fn bind_filter(
2905 &mut self,
2906 ast: &Ast,
2907 filter: ast::ExprRef,
2908 scope: &Scope,
2909 ) -> Result<Option<ExprRef>> {
2910 if filter == NONE {
2911 return Ok(None);
2912 }
2913 let bound = self.bind_expr(ast, filter, scope)?;
2914 Ok(Some(self.checked_cast_to(bound, &LogicalType::Boolean, false)?))
2915 }
2916
2917 /// Binds an aggregate call, records it, and hands back a reference to where its result lands.
2918 ///
2919 /// An aggregate inside a lambda's body is computed over the rows and not over the elements,
2920 /// so its arguments cannot see the lambda's parameters. See `crate::lambda`.
2921 pub(crate) fn bind_aggregate(
2922 &mut self,
2923 ast: &Ast,
2924 name: &str,
2925 args: &[ast::ExprRef],
2926 distinct: bool,
2927 filter: ast::ExprRef,
2928 scope: &Scope,
2929 ) -> Result<ExprRef> {
2930 let frames = std::mem::take(&mut self.lambda_frames);
2931 let bound = self.bind_aggregate_over_rows(ast, name, args, distinct, filter, scope);
2932 self.lambda_frames = frames;
2933 bound
2934 }
2935
2936 fn bind_aggregate_over_rows(
2937 &mut self,
2938 ast: &Ast,
2939 name: &str,
2940 args: &[ast::ExprRef],
2941 distinct: bool,
2942 filter: ast::ExprRef,
2943 scope: &Scope,
2944 ) -> Result<ExprRef> {
2945 if self.in_filter {
2946 return Err(Error::binder("aggregate functions are not allowed in FILTER"));
2947 }
2948 if self.in_aggregate {
2949 return Err(Error::binder(format!(
2950 "aggregate function calls cannot be nested, and {name}() is inside one"
2951 )));
2952 }
2953 if self.aggregation.is_none() {
2954 return Err(Error::binder(format!(
2955 "aggregate function calls cannot be used in the {}",
2956 self.clause
2957 )));
2958 }
2959 // The predicate goes first, which is the order the messages come out in upstream: a call
2960 // whose argument and whose filter both name columns that are not there is refused over the
2961 // filter. It is bound as if it were inside the call, so an aggregate in it is caught, and a
2962 // window in it is refused with the words a window inside an aggregate is refused with.
2963 self.in_aggregate = true;
2964 self.in_filter = true;
2965 let filter = self.bind_filter(ast, filter, scope);
2966 self.in_filter = false;
2967 self.in_aggregate = false;
2968 let filter = filter?;
2969
2970 self.in_aggregate = true;
2971 let mut bound = Vec::with_capacity(args.len());
2972 let mut failure = None;
2973 for &arg in args {
2974 match self.bind_expr(ast, arg, scope) {
2975 Ok(expr) => bound.push(expr),
2976 Err(error) => {
2977 failure = Some(error);
2978 break;
2979 }
2980 }
2981 }
2982 self.in_aggregate = false;
2983 if let Some(error) = failure {
2984 return Err(error);
2985 }
2986
2987 let types: Vec<LogicalType> =
2988 bound.iter().map(|&arg| self.plan.expr_type(arg).clone()).collect();
2989 let resolved = resolve(name, &types)?;
2990 // The separator is read once per group and not once per row, so the pin wants it to be the
2991 // same on every row and says so in these words.
2992 if resolved.name == "string_agg"
2993 && bound.len() == 2
2994 && !matches!(fold::value_of(&self.plan, bound[1]), Ok(Some(_)))
2995 {
2996 return Err(Error::binder(
2997 "The \"separator\" argument in function \"string_agg\" must be a constant expression",
2998 ));
2999 }
3000 let mut cast = Vec::with_capacity(bound.len());
3001 for (arg, wanted) in bound.iter().zip(&resolved.arguments) {
3002 cast.push(self.checked_cast_to(*arg, wanted, false)?);
3003 }
3004 let args = self.plan.add_expr_list(&cast);
3005 let name = self.plan.intern(resolved.name);
3006 let ty = resolved.returns;
3007 let call = self.plan.add_expr(Expr::Aggregate { name, args, distinct, filter }, ty.clone());
3008
3009 // Two identical aggregates are one column of the aggregate's output. `SELECT sum(x),
3010 // sum(x) / count(*)` computes one sum, not two.
3011 let existing = self.aggregation.as_ref().map(|held| held.aggregates.clone());
3012 let existing = existing.unwrap_or_default();
3013 let at = match existing.iter().position(|&held| self.same_expr(held, call)) {
3014 Some(at) => at,
3015 None => {
3016 let aggregation = self.aggregation.as_mut().expect("checked above");
3017 aggregation.aggregates.push(call);
3018 aggregation.aggregates.len() - 1
3019 }
3020 };
3021 let aggregation = self.aggregation.as_ref().expect("checked above");
3022 let (index, groups) = (aggregation.index, aggregation.groups.len());
3023 Ok(self.column(index, groups + at, ty))
3024 }
3025
3026 // ----------------------------------------------------------------- windows
3027
3028 /// Binds a window call, files it under the run it belongs to, and hands back its column.
3029 ///
3030 /// The result is a column of a [`Node::Window`] rather than the call itself, for the reason the
3031 /// aggregate path returns a column too: the operator produces the value and everything above it
3032 /// reads the value, so a target that wraps a window in arithmetic is arithmetic over a column.
3033 ///
3034 /// A window inside a lambda's body is computed over the rows for the reason an aggregate is,
3035 /// so it cannot see the lambda's parameters either.
3036 pub(crate) fn bind_window(
3037 &mut self,
3038 ast: &Ast,
3039 written: &WindowCall<'_>,
3040 scope: &Scope,
3041 ) -> Result<ExprRef> {
3042 let frames = std::mem::take(&mut self.lambda_frames);
3043 let bound = self.bind_window_over_rows(ast, written, scope);
3044 self.lambda_frames = frames;
3045 bound
3046 }
3047
3048 fn bind_window_over_rows(
3049 &mut self,
3050 ast: &Ast,
3051 written: &WindowCall<'_>,
3052 scope: &Scope,
3053 ) -> Result<ExprRef> {
3054 let WindowCall { name, args, distinct, filter, ignore_nulls, spec, .. } = *written;
3055 if self.in_aggregate {
3056 return Err(Error::binder(
3057 "aggregate function calls cannot contain window function calls",
3058 ));
3059 }
3060 if self.in_window {
3061 return Err(Error::binder("window function calls cannot be nested"));
3062 }
3063 // A join condition is part of the `WHERE` clause as far as this one sentence is concerned,
3064 // which is upstream's wording and not a simplification: `ON sum(a.i) OVER () = b.i` is
3065 // refused there with the words a window in a `WHERE` is refused with.
3066 let clause = if self.clause == "JOIN condition" { "WHERE clause" } else { self.clause };
3067 if clause != "SELECT clause" && clause != "ORDER BY clause" {
3068 return Err(Error::binder(format!("{clause} cannot contain window functions!")));
3069 }
3070
3071 // `count(*)` is a different function from `count(x)` here for the reason it is a different
3072 // function in an ordinary call: one counts rows and the other counts the rows where its
3073 // argument is not null. A star is not an expression and nothing below this binds one.
3074 let starred = args.iter().any(|&arg| {
3075 matches!(ast.expr(arg), ast::Expr::Star { qualifier, replacements }
3076 if qualifier.is_empty() && replacements.is_empty())
3077 });
3078 let (name, args): (&str, &[ast::ExprRef]) = if starred {
3079 if !same_name(name, "count") || args.len() != 1 {
3080 return Err(Error::binder(format!("* is not allowed in {name}()")));
3081 }
3082 ("count_star", &[])
3083 } else if same_name(name, "count") && args.is_empty() {
3084 // `count()` with nothing in it is upstream's other spelling of `count(*)`. It counts
3085 // rows the same way and it is not an arity mistake.
3086 ("count_star", &[])
3087 } else {
3088 (name, args)
3089 };
3090
3091 let held = ast.window(spec);
3092 self.in_window = true;
3093 let parts = self.window_parts(ast, written, args, held, scope);
3094 // The predicate goes last here, which is the other way round from an ordinary aggregate and
3095 // is again the order the messages come out in upstream. It is still inside the window, so a
3096 // window in it is a nested window, while an aggregate in it is an ordinary aggregate over
3097 // the same rows and is answered.
3098 let filter = if parts.is_ok() { self.bind_filter(ast, filter, scope) } else { Ok(None) };
3099 self.in_window = false;
3100 let parts = parts?;
3101 let filter = filter?;
3102 // Upstream's rule, in its words. A `RANGE` offset is a distance from the current row's sort
3103 // key, so there has to be exactly one sort key for it to be a distance from.
3104 let offsets = [parts.frame.start, parts.frame.end]
3105 .iter()
3106 .any(|end| matches!(end, WindowBound::Preceding(_) | WindowBound::Following(_)));
3107 if parts.frame.unit == WindowUnit::Range && offsets && parts.order.len() != 1 {
3108 return Err(Error::binder("RANGE frames must have only one ORDER BY expression"));
3109 }
3110
3111 let types: Vec<LogicalType> =
3112 parts.args.iter().map(|&arg| self.plan.expr_type(arg).clone()).collect();
3113 let resolved = window_signature(name, &types)?;
3114 // `fill` reads the sort key rather than the frame, so what it needs from the query is not
3115 // what any other window needs and it is refused on its own terms.
3116 if resolved.name == "fill" {
3117 let keys: Vec<LogicalType> =
3118 parts.order.iter().map(|key| self.plan.expr_type(key.expr).clone()).collect();
3119 refuse_fill(&types[0], &keys, distinct, ignore_nulls)?;
3120 }
3121 // Upstream's sentence, doubled quotes and all. A DISTINCT over an aggregate inside an OVER
3122 // is ordinary and answered, and a DISTINCT over a ranking window is refused there, because
3123 // there is nothing for it to collapse when the call reads no values in the first place.
3124 if distinct && kind_of(resolved.name) == Some(FunctionKind::Window) {
3125 return Err(Error::binder(format!(
3126 "DISTINCT is not implemented for the window function \"\"{name}\"\""
3127 )));
3128 }
3129 // The same sentence for the same reason. A ranking window reads no values, so there is
3130 // nothing for a predicate over the values to keep or drop.
3131 if filter.is_some() && kind_of(resolved.name) == Some(FunctionKind::Window) {
3132 return Err(Error::binder(format!(
3133 "FILTER is not implemented for the window function \"\"{name}\"\""
3134 )));
3135 }
3136 // An `ORDER BY` inside the brackets puts the rows of the frame in a different order for
3137 // this one call to read them in, which is a question every aggregate and the three that
3138 // count through the frame have an answer to. The rest of the window functions read
3139 // something other than the frame, and what the reference binary does with them under an
3140 // order of their own is a different reading again, so they are turned down rather than
3141 // guessed at. The exclusion is refused first and in the reference binary's own sentence,
3142 // because that is the one it reaches for when both apply. Per #1204.
3143 if !parts.inner.is_empty() && kind_of(resolved.name) == Some(FunctionKind::Window) {
3144 let counts = matches!(resolved.name, "first_value" | "last_value" | "nth_value");
3145 if !counts {
3146 if parts.frame.exclude != WindowExclude::NoOthers {
3147 return Err(Error::binder(format!(
3148 "EXCLUDE is not supported for the window function \"\"{}\"\"",
3149 resolved.name
3150 )));
3151 }
3152 return Err(Error::not_implemented(format!(
3153 "ORDER BY inside the arguments of the window function \"{}\"",
3154 resolved.name
3155 )));
3156 }
3157 }
3158 let mut cast = Vec::with_capacity(parts.args.len());
3159 for (arg, wanted) in parts.args.iter().zip(&resolved.arguments) {
3160 cast.push(self.checked_cast_to(*arg, wanted, false)?);
3161 }
3162 let args = self.plan.add_expr_list(&cast);
3163 let order = self.plan.add_sort_keys(&parts.inner);
3164 let name = self.plan.intern(resolved.name);
3165 let ty = resolved.returns;
3166 let call = self.plan.add_expr(
3167 Expr::Window { name, args, distinct, filter, ignore_nulls, order },
3168 ty.clone(),
3169 );
3170
3171 let at = self.window_run(parts.partition, parts.order, parts.frame, call);
3172 let index = self.windows.last().expect("the run was just filed").index;
3173 Ok(self.column(index, at, ty))
3174 }
3175
3176 /// Files a call under the run that matches it, or opens a new run, and says which column it is.
3177 ///
3178 /// The run that matches is only ever the last one, because a query that goes back to an earlier
3179 /// partitioning after using a different one in between wants the operators in the order it wrote
3180 /// them. Merging the two would be a rewrite, and a rewrite over a window is the optimizer's to
3181 /// make once it knows what the sort below each one costs.
3182 fn window_run(
3183 &mut self,
3184 partition: Vec<ExprRef>,
3185 order: Vec<SortKey>,
3186 frame: WindowFrame,
3187 call: ExprRef,
3188 ) -> usize {
3189 let matches = self.windows.last().is_some_and(|run| {
3190 run.frame == frame
3191 && run.partition.len() == partition.len()
3192 && run.order.len() == order.len()
3193 && run.partition.iter().zip(&partition).all(|(&l, &r)| self.same_expr(l, r))
3194 && run.order.iter().zip(&order).all(|(l, r)| {
3195 l.descending == r.descending
3196 && l.nulls_first == r.nulls_first
3197 && self.same_expr(l.expr, r.expr)
3198 })
3199 });
3200 if !matches {
3201 let index = self.fresh_index();
3202 self.windows.push(WindowRun { index, partition, order, frame, calls: Vec::new() });
3203 }
3204 // Two identical calls over one run are one column, the same way two identical aggregates
3205 // over one grouping are. `SELECT sum(i) OVER (), sum(i) OVER () + 1` totals once.
3206 let calls = self.windows.last().expect("a run is open").calls.clone();
3207 if let Some(at) = calls.iter().position(|&held| self.same_expr(held, call)) {
3208 return at;
3209 }
3210 let run = self.windows.last_mut().expect("a run is open");
3211 run.calls.push(call);
3212 run.calls.len() - 1
3213 }
3214
3215 /// Binds the arguments and everything inside the `OVER`, with the aggregate rule applied.
3216 ///
3217 /// The aggregate rule applies to all of it, which is measured rather than assumed: over a
3218 /// grouped block `sum(count(i)) OVER ()` binds and `sum(i) OVER ()` is the ungrouped column
3219 /// complaint, and the same pair of answers comes back for a partition key and for an order key.
3220 fn window_parts(
3221 &mut self,
3222 ast: &Ast,
3223 written: &WindowCall<'_>,
3224 args: &[ast::ExprRef],
3225 held: ast::WindowSpec,
3226 scope: &Scope,
3227 ) -> Result<WindowParts> {
3228 let mut bound = Vec::with_capacity(args.len());
3229 for &arg in args {
3230 let expr = self.bind_expr(ast, arg, scope)?;
3231 bound.push(self.over_aggregate(expr, scope)?);
3232 }
3233 // The keys inside the brackets are bound against the same rows the arguments are, because
3234 // that is what they sort: the call reads its frame in this order, and the frame is made of
3235 // the operator's input rows.
3236 let mut inner = Vec::new();
3237 for item in ast.order_list(written.order).to_vec() {
3238 let expr = self.bind_expr(ast, item.expr, scope)?;
3239 let expr = self.over_aggregate(expr, scope)?;
3240 inner.push(self.sort_key(expr, item));
3241 }
3242 let mut partition = Vec::new();
3243 for &key in ast.expr_list(held.partition) {
3244 let expr = self.bind_expr(ast, key, scope)?;
3245 partition.push(self.over_aggregate(expr, scope)?);
3246 }
3247 let mut order = Vec::new();
3248 for item in ast.order_list(held.order).to_vec() {
3249 let expr = self.bind_expr(ast, item.expr, scope)?;
3250 let expr = self.over_aggregate(expr, scope)?;
3251 order.push(self.sort_key(expr, item));
3252 }
3253 let frame = WindowFrame {
3254 unit: match held.unit {
3255 ast::WindowUnit::Rows => WindowUnit::Rows,
3256 ast::WindowUnit::Range => WindowUnit::Range,
3257 ast::WindowUnit::Groups => WindowUnit::Groups,
3258 },
3259 start: self.window_bound(ast, held.start, scope)?,
3260 end: self.window_bound(ast, held.end, scope)?,
3261 exclude: match held.exclude {
3262 ast::WindowExclude::NoOthers => WindowExclude::NoOthers,
3263 ast::WindowExclude::CurrentRow => WindowExclude::CurrentRow,
3264 ast::WindowExclude::Group => WindowExclude::Group,
3265 ast::WindowExclude::Ties => WindowExclude::Ties,
3266 },
3267 };
3268 Ok(WindowParts { args: bound, partition, order, inner, frame })
3269 }
3270
3271 /// One end of a frame, with its offset bound where it has one.
3272 fn window_bound(
3273 &mut self,
3274 ast: &Ast,
3275 bound: ast::WindowBound,
3276 scope: &Scope,
3277 ) -> Result<WindowBound> {
3278 let offset = |binder: &mut Self, written| {
3279 let expr = binder.bind_expr(ast, written, scope)?;
3280 binder.over_aggregate(expr, scope)
3281 };
3282 Ok(match bound {
3283 ast::WindowBound::UnboundedPreceding => WindowBound::UnboundedPreceding,
3284 ast::WindowBound::CurrentRow => WindowBound::CurrentRow,
3285 ast::WindowBound::UnboundedFollowing => WindowBound::UnboundedFollowing,
3286 ast::WindowBound::Preceding(written) => WindowBound::Preceding(offset(self, written)?),
3287 ast::WindowBound::Following(written) => WindowBound::Following(offset(self, written)?),
3288 })
3289 }
3290
3291 /// Which of this block's groups is exactly that column, if one of them is.
3292 ///
3293 /// Exactly the column and not an expression over it, because the caller is looking for the same
3294 /// value read from the aggregate instead of from the table underneath it, and `GROUP BY k + 1`
3295 /// carries the sum and not the column.
3296 fn group_of(&self, read: ColumnBinding) -> Option<usize> {
3297 self.aggregation.as_ref()?.groups.iter().position(
3298 |group| matches!(*self.plan.expr(*group), Expr::Column(binding) if binding == read),
3299 )
3300 }
3301
3302 /// The outer column a query still waiting under this grouping correlates to and the grouping
3303 /// does not carry upward, which is the column an error should name.
3304 ///
3305 /// `None` when the binding is not one of those queries, which is every ordinary case of a
3306 /// column read without a group.
3307 fn ungrouped_correlation(&self, binding: ColumnBinding) -> Option<ColumnBinding> {
3308 let pending =
3309 self.scalar_subqueries.iter().find(|pending| pending.index == binding.table)?;
3310 pending.reads.iter().copied().find(|read| self.group_of(*read).is_none())
3311 }
3312
3313 /// Whether a column is the result of a window this block is building.
3314 fn is_window_output(&self, binding: ColumnBinding) -> bool {
3315 self.windows.iter().any(|run| run.index == binding.table)
3316 }
3317
3318 /// Whether a column was resolved in an enclosing query rather than in this one.
3319 ///
3320 /// Every such read is written into the frame of the query being bound as it is resolved, and
3321 /// the frame is only handed up once that query's body is done, so while a select list or a
3322 /// `HAVING` is being bound the frame still holds everything this query read from outside it.
3323 fn is_correlation(&self, binding: ColumnBinding) -> bool {
3324 self.correlations.last().is_some_and(|frame| frame.contains(&binding))
3325 }
3326
3327 /// The name a column is written under, for an error message to say which one it means.
3328 ///
3329 /// A column of an enclosing query is not in this query's scope, so the outer scopes are searched
3330 /// as well. Without that the message names no column at all, which is how `column a column must
3331 /// appear in the GROUP BY clause` came to be a sentence this engine printed.
3332 fn name_of(&self, binding: ColumnBinding, scope: &Scope) -> String {
3333 std::iter::once(scope)
3334 .chain(self.outer_scopes.iter().rev())
3335 .flat_map(|visible| visible.columns.iter())
3336 .find(|column| column.binding == binding)
3337 .map_or_else(|| "a column".to_string(), |column| format!("\"{}\"", column.name))
3338 }
3339
3340 /// Rewrites a bound expression into one the aggregate's output can answer.
3341 ///
3342 /// A subexpression that is one of the group expressions becomes a reference to that group. A
3343 /// column that is neither grouped nor inside an aggregate is the error every SQL user has seen,
3344 /// and it is reported here because this is the first point where it is knowable.
3345 pub(crate) fn over_aggregate(&mut self, expr: ExprRef, scope: &Scope) -> Result<ExprRef> {
3346 let Some(aggregation) = self.aggregation.as_ref() else {
3347 return Ok(expr);
3348 };
3349 let index = aggregation.index;
3350 let groups = aggregation.groups.clone();
3351 for (at, group) in groups.iter().enumerate() {
3352 if self.same_expr(expr, *group) {
3353 let ty = self.plan.expr_type(*group).clone();
3354 return Ok(self.column(index, at, ty));
3355 }
3356 }
3357 let ty = self.plan.expr_type(expr).clone();
3358 match self.plan.expr(expr).clone() {
3359 Expr::Column(binding) if binding.table == index => Ok(expr),
3360 // A window result is not a column of the input and the grouping rule has nothing to say
3361 // about it. It reads the aggregate's output rather than the table's, which is why
3362 // `SELECT sum(count(i)) OVER () FROM t GROUP BY j` binds and `sum(i) OVER ()` over the
3363 // same block does not.
3364 Expr::Column(binding) if self.is_window_output(binding) => Ok(expr),
3365 // The same argument for a query joined in above the grouping. `HAVING sum(x) > (SELECT
3366 // ...)` reads one row out of a query that has nothing to do with the groups, and the
3367 // join that produces it sits on top of the `Aggregate`, so what it produces is not one
3368 // of the grouped table's columns either.
3369 Expr::Column(binding) if self.joined_above.contains(&binding.table) => Ok(expr),
3370 // A column of an enclosing query is one value for the whole of this one, because this
3371 // query is evaluated once per outer row. It is a constant here in the sense the grouping
3372 // rule cares about, so it is allowed wherever a grouped column is and needs no group of
3373 // its own. The grouping rule is about columns of this query's own `FROM`, and a name
3374 // that resolved past it is not one of those. That is #995.
3375 Expr::Column(binding) if self.is_correlation(binding) => Ok(expr),
3376 // A query this block wrote that is still waiting to be joined in underneath the
3377 // grouping lands here as well, and the column the complaint should name is the one that
3378 // query correlates to rather than the column the query produces, which belongs to no
3379 // table anybody wrote. An uncorrelated query and a correlated one whose correlation is
3380 // grouped were both moved over the grouping by [`Self::lift_over_aggregate`] and are
3381 // not here, so what is left correlates to something this block neither grouped nor
3382 // aggregated, and that is an ordinary missing GROUP BY however far inside a query it
3383 // was written. That is #1032.
3384 Expr::Column(binding) => {
3385 let read = self.ungrouped_correlation(binding).unwrap_or(binding);
3386 let name = self.name_of(read, scope);
3387 Err(Error::binder(format!(
3388 "column {name} must appear in the GROUP BY clause or must be part of an aggregate function"
3389 )))
3390 }
3391 Expr::Constant(_)
3392 | Expr::Aggregate { .. }
3393 | Expr::Window { .. }
3394 | Expr::LambdaParam(_) => Ok(expr),
3395 // The body is over the elements and the columns it captures, and a captured column is
3396 // held to the grouping rule like any other, which is the pin's error for
3397 // `list_transform(l, lambda x: x * k) ... GROUP BY l`.
3398 Expr::Lambda { table, params, body } => {
3399 let body = self.over_aggregate(body, scope)?;
3400 Ok(self.plan.add_expr(Expr::Lambda { table, params, body }, ty))
3401 }
3402 Expr::Cast { input, try_cast } => {
3403 let input = self.over_aggregate(input, scope)?;
3404 Ok(self.plan.add_expr(Expr::Cast { input, try_cast }, ty))
3405 }
3406 Expr::Compare { op, left, right } => {
3407 let left = self.over_aggregate(left, scope)?;
3408 let right = self.over_aggregate(right, scope)?;
3409 Ok(self.plan.add_expr(Expr::Compare { op, left, right }, ty))
3410 }
3411 Expr::Conjunction { op, children } => {
3412 let written = self.plan.expr_list(children).to_vec();
3413 let mut rewritten = Vec::with_capacity(written.len());
3414 for child in written {
3415 rewritten.push(self.over_aggregate(child, scope)?);
3416 }
3417 let children = self.plan.add_expr_list(&rewritten);
3418 Ok(self.plan.add_expr(Expr::Conjunction { op, children }, ty))
3419 }
3420 Expr::Function { name, args } => {
3421 let written = self.plan.expr_list(args).to_vec();
3422 let mut rewritten = Vec::with_capacity(written.len());
3423 for arg in written {
3424 rewritten.push(self.over_aggregate(arg, scope)?);
3425 }
3426 let args = self.plan.add_expr_list(&rewritten);
3427 Ok(self.plan.add_expr(Expr::Function { name, args }, ty))
3428 }
3429 Expr::Case { arms, otherwise } => {
3430 let written = self.plan.arm_list(arms).to_vec();
3431 let mut rewritten = Vec::with_capacity(written.len());
3432 for arm in written {
3433 let when = self.over_aggregate(arm.when, scope)?;
3434 let then = self.over_aggregate(arm.then, scope)?;
3435 rewritten.push(rudb_plan::Arm { when, then });
3436 }
3437 let otherwise = match otherwise {
3438 Some(expr) => Some(self.over_aggregate(expr, scope)?),
3439 None => None,
3440 };
3441 let arms = self.plan.add_arms(&rewritten);
3442 Ok(self.plan.add_expr(Expr::Case { arms, otherwise }, ty))
3443 }
3444 }
3445 }
3446
3447 /// Whether two bound expressions are the same expression, by shape rather than by reference.
3448 pub(crate) fn same_expr(&self, left: ExprRef, right: ExprRef) -> bool {
3449 same_expr(&self.plan, left, right)
3450 }
3451}
3452
3453/// The named parameters a table function call was written with.
3454///
3455/// A struct rather than the fields loose, because the seventeen DuckDB has on `read_parquet` and the
3456/// thirty on `read_csv` are all going to want somewhere to go, and because a call with none of them
3457/// written should read as the default of this rather than as a bare false somewhere.
3458///
3459/// The CSV half goes on to the reader and is opened with, here and again in the executor. The
3460/// Parquet half is answered here and nothing downstream sees it, which is what `binary_as_string`
3461/// turning a BLOB column into a VARCHAR one is.
3462#[derive(Debug, Default)]
3463struct Options {
3464 /// `binary_as_string`, which says an unannotated byte array column in a Parquet file holds
3465 /// text. The ClickBench file has twenty eight of those and every query reads them as strings.
3466 binary_as_string: bool,
3467 /// `all_varchar`, which reads every column of a CSV file as text rather than sniffing a type.
3468 all_varchar: bool,
3469 /// `file_row_number`, which adds a column holding each row's ordinal inside its own file.
3470 ///
3471 /// The one Parquet option here that the executor has to act on rather than the binder, since
3472 /// the column is not in the file and has to be counted as the rows come out of it.
3473 file_row_number: bool,
3474 /// `delim`, `sep`, `quote`, `escape` and `header`, which are what the sniffer would decide.
3475 given: Given,
3476}
3477
3478impl Options {
3479 /// What these named parameters add up to.
3480 ///
3481 /// Each one was already checked against the function's list, so a name in here is a name that
3482 /// function takes and the value is already the type it wants. What is left is reading them, and
3483 /// the last one written wins, which is DuckDB's answer to `delim='|', delim=','` and was
3484 /// measured rather than assumed.
3485 fn of(written: &[(&'static str, Value, ExprRef)]) -> Result<Self> {
3486 let mut options = Self::default();
3487 for (parameter, value, _) in written {
3488 match (*parameter, value) {
3489 ("binary_as_string", Value::Boolean(on)) => options.binary_as_string = *on,
3490 ("all_varchar", Value::Boolean(on)) => options.all_varchar = *on,
3491 ("file_row_number", Value::Boolean(on)) => options.file_row_number = *on,
3492 _ => {}
3493 }
3494 }
3495 let named: Vec<(&str, Value)> =
3496 written.iter().map(|(parameter, value, _)| (*parameter, value.clone())).collect();
3497 options.given = csv_given(&named)?;
3498 Ok(options)
3499 }
3500}
3501
3502/// The one file a read names, canonical, with what the file system says about it now, or `None`
3503/// for a read of several files or of something that is not a regular file with a UTF-8 name.
3504fn mirror_target(paths: &[String]) -> Option<(String, FileStamp)> {
3505 let [path] = paths else { return None };
3506 let canonical = std::fs::canonicalize(path).ok()?;
3507 let stamp = FileStamp::of(&canonical)?;
3508 Some((canonical.to_str()?.to_string(), stamp))
3509}
3510
3511/// What was written between the two sides of a set operation.
3512#[derive(Clone, Copy)]
3513struct Operator {
3514 /// `UNION`, `EXCEPT` or `INTERSECT`.
3515 op: SetOp,
3516 /// `ALL`, `DISTINCT`, or neither, which means `DISTINCT` everywhere it is allowed.
3517 quantifier: Quantifier,
3518 /// Whether `BY NAME` was written, which only `UNION` takes.
3519 by_name: bool,
3520}
3521
3522/// One column of the result of a set operation, and where each side keeps it.
3523struct Merged {
3524 /// The name it comes out under, which is the left side's when both sides wrote it.
3525 name: String,
3526 /// What it is, after the two sides' types have met.
3527 ty: LogicalType,
3528 /// Which column of the left side it is, absent when only the right side wrote it.
3529 left: Option<usize>,
3530 /// Which column of the right side it is, absent when only the left side wrote it.
3531 right: Option<usize>,
3532}
3533
3534/// Matches the two sides of an ordinary set operation, which is first column to first column.
3535///
3536/// The names are the left side's, so `SELECT a FROM t UNION SELECT b FROM u` comes out as `a`.
3537fn match_by_position(left: &Scope, right: &Scope) -> Result<Vec<Merged>> {
3538 if left.len() != right.len() {
3539 return Err(Error::binder(format!(
3540 "Set operations can only apply to expressions with the same number of result columns, but left side has {} and right side has {}",
3541 left.len(),
3542 right.len()
3543 )));
3544 }
3545 let mut merged = Vec::with_capacity(left.len());
3546 for (at, (held, other)) in left.columns.iter().zip(&right.columns).enumerate() {
3547 merged.push(Merged {
3548 name: held.name.clone(),
3549 ty: meet(&held.ty, &other.ty)?,
3550 left: Some(at),
3551 right: Some(at),
3552 });
3553 }
3554 Ok(merged)
3555}
3556
3557/// Matches the two sides of a `UNION BY NAME`, which is by column name and not by position.
3558///
3559/// The result has the left side's columns in the order the left side wrote them, then the right
3560/// side's columns the left side did not write, in the order the right side wrote them. A column
3561/// only one side wrote is that side's type and the other side fills it with a null, which is why
3562/// nothing here needs the two sides to be the same width. Names match without regard to case, and
3563/// the spelling that comes out is the left side's, both of which follow the rest of the engine.
3564fn match_by_name(left: &Scope, right: &Scope) -> Result<Vec<Merged>> {
3565 named_once(left)?;
3566 named_once(right)?;
3567 let mut merged = Vec::with_capacity(left.len() + right.len());
3568 for (at, held) in left.columns.iter().enumerate() {
3569 let other = right.columns.iter().position(|column| same_name(&column.name, &held.name));
3570 let ty = match other {
3571 Some(other) => meet(&held.ty, &right.columns[other].ty)?,
3572 None => held.ty.clone(),
3573 };
3574 merged.push(Merged { name: held.name.clone(), ty, left: Some(at), right: other });
3575 }
3576 for (at, held) in right.columns.iter().enumerate() {
3577 if left.columns.iter().any(|column| same_name(&column.name, &held.name)) {
3578 continue;
3579 }
3580 merged.push(Merged {
3581 name: held.name.clone(),
3582 ty: held.ty.clone(),
3583 left: None,
3584 right: Some(at),
3585 });
3586 }
3587 Ok(merged)
3588}
3589
3590/// Refuses a side of a `UNION BY NAME` that wrote one name twice.
3591///
3592/// Matching by name needs the name to say which column, and a side that wrote `a` twice has no
3593/// answer to give. An ordinary union does not care, because there the position says which column.
3594/// The doubled quotes around the name are the reference binary's and not a mistake here.
3595fn named_once(scope: &Scope) -> Result<()> {
3596 for (at, held) in scope.columns.iter().enumerate() {
3597 if scope.columns[..at].iter().any(|column| same_name(&column.name, &held.name)) {
3598 return Err(Error::binder(format!(
3599 "UNION (ALL) BY NAME operation doesn't support duplicate names in the SELECT list - the name \"\"{}\"\" occurs multiple times",
3600 held.name
3601 )));
3602 }
3603 }
3604 Ok(())
3605}
3606
3607/// The one type a column of a set operation comes out as, given what each side wrote.
3608fn meet(left: &LogicalType, right: &LogicalType) -> Result<LogicalType> {
3609 left.promote(right).ok_or_else(|| {
3610 Error::binder(format!(
3611 "Cannot combine a column of type {left} with a column of type {right} in a set operation"
3612 ))
3613 })
3614}
3615
3616/// DuckDB's complaint about a named parameter that was given a null, which is a different sentence
3617/// for almost every parameter.
3618///
3619/// Three of them were measured on `v2.0.0-dev84237` and no two agree: `binary_as_string` is the
3620/// first, `all_varchar` is the second and `header` is the third. They read like three people each
3621/// writing the message in front of them, which is what they are, and a harness that compares error
3622/// text compares all of it. Anything not measured gets the first one, which is the most general of
3623/// the three.
3624fn null_parameter(function: TableFunction, parameter: &str) -> String {
3625 match parameter {
3626 "header" => format!("\"{parameter}\" expects a non-null boolean value (e.g. TRUE or 1)"),
3627 "all_varchar" => format!("{} \"{parameter}\" cannot be NULL", function.name()),
3628 _ => format!("Cannot use NULL as argument to \"{parameter}\""),
3629 }
3630}
3631
3632/// The complaint about a `REPLACE` entry that named a column the star did not stand for.
3633///
3634/// It reads like the complaint about any other name that is not there, down to the list of names
3635/// that are, because from the writer's side it is the same mistake.
3636fn missing_replacement(name: &str, input: &Scope) -> Error {
3637 Error::binder(format!(
3638 "Column \"{name}\" in REPLACE list not found in FROM clause{}",
3639 input.candidates()
3640 ))
3641}
3642
3643/// Whether a type is one `fill` can interpolate over, which is the pin's phrase for it.
3644///
3645/// The pin refuses `fill` with `FILL argument must support subtraction` and its sort key with
3646/// `FILL ordering must support subtraction`, and the two lists are not the same list, which is why
3647/// this takes a flag rather than answering one question. Every number is on both, so are `DATE`,
3648/// `TIME` and the two timestamps, and `TIME WITH TIME ZONE` is a sort key there but not an
3649/// argument. `INTERVAL` is on neither, which is worth saying out loud because an interval does
3650/// subtract: the sentence names subtraction and the rule is narrower than the sentence.
3651fn subtractable(ty: &LogicalType, ordering: bool) -> bool {
3652 if ty.is_numeric() {
3653 return true;
3654 }
3655 match ty {
3656 LogicalType::Date
3657 | LogicalType::Time
3658 | LogicalType::Timestamp
3659 | LogicalType::TimestampS
3660 | LogicalType::TimestampMs
3661 | LogicalType::TimestampNs
3662 | LogicalType::TimestampTz => true,
3663 LogicalType::TimeTz => ordering,
3664 _ => false,
3665 }
3666}
3667
3668/// Refuses a `fill` call the way the pin refuses one, in the pin's order.
3669///
3670/// The order was measured and it is not the order the clauses are written in. A `fill` over a
3671/// `VARCHAR` with no `ORDER BY` at all complains about the argument, so the argument is looked at
3672/// before the sort key is counted, and a `fill` with `DISTINCT` and no `ORDER BY` complains about
3673/// the `ORDER BY`, so the count comes before the clauses. `IGNORE NULLS` is refused here rather
3674/// than being answered as a no-op, since there is nothing for it to skip: `fill` is the one window
3675/// whose whole job is the nulls.
3676fn refuse_fill(
3677 argument: &LogicalType,
3678 order: &[LogicalType],
3679 distinct: bool,
3680 ignore_nulls: bool,
3681) -> Result<()> {
3682 if !subtractable(argument, false) {
3683 return Err(Error::binder("FILL argument must support subtraction"));
3684 }
3685 let [key] = order else {
3686 return Err(Error::binder("FILL functions must have only one ORDER BY expression"));
3687 };
3688 if !subtractable(key, true) {
3689 return Err(Error::binder("FILL ordering must support subtraction"));
3690 }
3691 if distinct {
3692 return Err(Error::binder(
3693 "DISTINCT is not implemented for the window function \"\"fill\"\"",
3694 ));
3695 }
3696 if ignore_nulls {
3697 return Err(Error::binder(
3698 "RESPECT/IGNORE NULLS is not supported for the window function \"fill\"",
3699 ));
3700 }
3701 Ok(())
3702}
3703
3704/// Resolves the call written inside an `OVER`.
3705///
3706/// Every aggregate is also a window, which is why this goes through the same signature table the
3707/// aggregate path uses, and the ranking windows go through it too because they are rows in the same
3708/// table. Everything else is one of three refusals, and all three are the reference binary's: a name
3709/// it knows as a scalar and a name it does not know at all each get their own sentence there.
3710fn window_signature(name: &str, types: &[LogicalType]) -> Result<Resolved> {
3711 match kind_of(name) {
3712 Some(FunctionKind::Aggregate | FunctionKind::Window) => resolve(name, types),
3713 Some(FunctionKind::Scalar) => {
3714 Err(Error::catalog(format!("{name} is not an aggregate function")))
3715 }
3716 None => Err(Error::catalog(format!("Aggregate Function with name {name} does not exist!"))),
3717 }
3718}
3719
3720/// Structural equality over two expressions of one plan.
3721fn same_expr(plan: &Plan, left: ExprRef, right: ExprRef) -> bool {
3722 if left == right {
3723 return true;
3724 }
3725 if plan.expr_type(left) != plan.expr_type(right) {
3726 return false;
3727 }
3728 let lists = |left, right| {
3729 let left: &[ExprRef] = plan.expr_list(left);
3730 let right: &[ExprRef] = plan.expr_list(right);
3731 left.len() == right.len()
3732 && left.iter().zip(right).all(|(&left, &right)| same_expr(plan, left, right))
3733 };
3734 match (plan.expr(left), plan.expr(right)) {
3735 (Expr::Column(left), Expr::Column(right)) => left == right,
3736 (Expr::Constant(left), Expr::Constant(right)) => plan.value(*left) == plan.value(*right),
3737 (
3738 Expr::Cast { input: left, try_cast: left_try },
3739 Expr::Cast { input: right, try_cast: right_try },
3740 ) => left_try == right_try && same_expr(plan, *left, *right),
3741 (
3742 Expr::Compare { op: left_op, left: left_a, right: left_b },
3743 Expr::Compare { op: right_op, left: right_a, right: right_b },
3744 ) => {
3745 left_op == right_op
3746 && same_expr(plan, *left_a, *right_a)
3747 && same_expr(plan, *left_b, *right_b)
3748 }
3749 (
3750 Expr::Conjunction { op: left_op, children: left_children },
3751 Expr::Conjunction { op: right_op, children: right_children },
3752 ) => left_op == right_op && lists(*left_children, *right_children),
3753 (
3754 Expr::Function { name: left_name, args: left_args },
3755 Expr::Function { name: right_name, args: right_args },
3756 ) => plan.string(*left_name) == plan.string(*right_name) && lists(*left_args, *right_args),
3757 (
3758 Expr::Aggregate {
3759 name: left_name,
3760 args: left_args,
3761 distinct: left_distinct,
3762 filter: left_filter,
3763 },
3764 Expr::Aggregate {
3765 name: right_name,
3766 args: right_args,
3767 distinct: right_distinct,
3768 filter: right_filter,
3769 },
3770 ) => {
3771 plan.string(*left_name) == plan.string(*right_name)
3772 && left_distinct == right_distinct
3773 && match (left_filter, right_filter) {
3774 (None, None) => true,
3775 (Some(left), Some(right)) => same_expr(plan, *left, *right),
3776 _ => false,
3777 }
3778 && lists(*left_args, *right_args)
3779 }
3780 // The partition, the order and the frame are not compared here and do not need to be. Two
3781 // window calls are only ever asked about when they are already in the same run, which is
3782 // what agreeing on all three means.
3783 (
3784 Expr::Window {
3785 name: left_name,
3786 args: left_args,
3787 distinct: left_distinct,
3788 filter: left_filter,
3789 ignore_nulls: left_nulls,
3790 order: left_order,
3791 },
3792 Expr::Window {
3793 name: right_name,
3794 args: right_args,
3795 distinct: right_distinct,
3796 filter: right_filter,
3797 ignore_nulls: right_nulls,
3798 order: right_order,
3799 },
3800 ) => {
3801 // The keys inside the brackets are compared, unlike the ones in the `OVER`, because two
3802 // calls in the same run can still read their frame in different orders.
3803 let left_keys = plan.sort_key_list(*left_order);
3804 let right_keys = plan.sort_key_list(*right_order);
3805 plan.string(*left_name) == plan.string(*right_name)
3806 && left_distinct == right_distinct
3807 && left_nulls == right_nulls
3808 && left_keys.len() == right_keys.len()
3809 && left_keys.iter().zip(right_keys).all(|(left, right)| {
3810 left.descending == right.descending
3811 && left.nulls_first == right.nulls_first
3812 && same_expr(plan, left.expr, right.expr)
3813 })
3814 && match (left_filter, right_filter) {
3815 (None, None) => true,
3816 (Some(left), Some(right)) => same_expr(plan, *left, *right),
3817 _ => false,
3818 }
3819 && lists(*left_args, *right_args)
3820 }
3821 (
3822 Expr::Case { arms: left_arms, otherwise: left_otherwise },
3823 Expr::Case { arms: right_arms, otherwise: right_otherwise },
3824 ) => {
3825 let left_arms = plan.arm_list(*left_arms);
3826 let right_arms = plan.arm_list(*right_arms);
3827 left_arms.len() == right_arms.len()
3828 && left_arms.iter().zip(right_arms).all(|(left, right)| {
3829 same_expr(plan, left.when, right.when) && same_expr(plan, left.then, right.then)
3830 })
3831 && match (left_otherwise, right_otherwise) {
3832 (None, None) => true,
3833 (Some(left), Some(right)) => same_expr(plan, *left, *right),
3834 _ => false,
3835 }
3836 }
3837 _ => false,
3838 }
3839}