use sqlparser::ast::{
AlterTable as SqlAlterTable, AlterTableOperation, Assignment as SqlAssignment,
AssignmentTarget, CreateTable, CreateTableLikeKind, CreateView as SqlCreateView, Cte as SqlCte,
Delete as SqlDelete, Expr as SqlExpr, FromTable, Function, FunctionArg, FunctionArgExpr,
FunctionArguments, GroupByExpr, GroupByWithModifier, Ident, Insert as SqlInsert,
JoinConstraint, JoinOperator, JsonPathElem, Merge as SqlMerge, MergeAction, MergeInsertKind,
ObjectName, ObjectType, OnConflictAction, OnInsert, OrderBy, OrderByExpr, OrderByKind,
OutputClause, PipeOperator, PivotValueSource, Query, Select, SelectItem, SetExpr, Statement,
TableAlias, TableFactor, TableObject, TableWithJoins, Update as SqlUpdate, UpdateTableFromKind,
Value, Values as SqlValues,
};
use sqlparser::ast::{
AccessExpr, ConnectByKind, Distinct, FunctionArgumentClause, LimitClause, ListAggOnOverflow,
NamedWindowExpr, SelectItemQualifiedWildcardKind, Subscript, TopQuantity,
WildcardAdditionalOptions, WindowFrameBound, WindowSpec, WindowType,
};
use sqlparser::tokenizer::Span;
use super::logical_plan::{
output_slots, slot_count, Aggregate, AlterTable, Assignment, Binding, BoundColumn, Columns,
CreateTableAs, CreateView, Cte, CteRef, Delete, Drop, Expr, Filter, Insert, Join, LogicalPlan,
Merge, MergeClause, NamedExpr, OutputNames, Projection, Scan, SchemaSource, SetOp, Sort,
SubqueryAlias, TableFunction, Update, Values, With,
};
use super::origins::output_operands;
use crate::casing::{CaseRule, IdentifierStyle};
use crate::catalog::{Catalog, CatalogTable};
use crate::diagnostic::{ColumnLevelDiagnostic, ColumnLevelDiagnosticKind};
use crate::reference::{ColumnWrite, ResolutionKind, TableRead, TableReference, TableWrite};
mod context;
mod expr;
mod query;
mod resolve;
mod scope;
mod statement;
use context::*;
use scope::*;
pub(crate) fn build_with_diagnostics(
statement: &Statement,
catalog: Option<&Catalog>,
style: IdentifierStyle,
) -> (LogicalPlan, Vec<ColumnLevelDiagnostic>) {
let mut binder = Binder {
catalog,
style,
diagnostics: Vec::new(),
context: Context::default(),
};
let op = binder.bind_statement(statement);
(op, binder.diagnostics)
}
struct Binder<'a> {
catalog: Option<&'a Catalog>,
style: IdentifierStyle,
diagnostics: Vec<ColumnLevelDiagnostic>,
context: Context,
}
impl<'a> Binder<'a> {
fn in_scope<R>(&mut self, child: Context, f: impl FnOnce(&mut Self) -> R) -> R {
let saved = std::mem::replace(&mut self.context, child);
let r = f(self);
self.context = saved;
r
}
fn in_ctes<R>(&mut self, ctes: Vec<CteDecl>, f: impl FnOnce(&mut Self) -> R) -> R {
let child = self.context.with_ctes(ctes);
self.in_scope(child, f)
}
fn in_outer<R>(&mut self, relations: Vec<Relation>, f: impl FnOnce(&mut Self) -> R) -> R {
let child = self.context.with_outer(relations);
self.in_scope(child, f)
}
fn in_lambda<R>(
&mut self,
relations: Vec<Relation>,
params: impl IntoIterator<Item = Ident>,
f: impl FnOnce(&mut Self) -> R,
) -> R {
let child = self.context.with_outer(relations).with_lambda(params);
self.in_scope(child, f)
}
pub(super) fn table_ref(&mut self, name: &ObjectName) -> Option<TableReference> {
match TableReference::try_from_name(name) {
Ok(table) => Some(table),
Err(_) => {
self.record_unrepresentable_table(name);
None
}
}
}
pub(super) fn record_wildcard_suppressed(&mut self, description: &str, span: Span) {
let span = (span.start.line != 0).then_some(span);
let suffix = span_suffix(span);
self.diagnostics.push(ColumnLevelDiagnostic {
kind: ColumnLevelDiagnosticKind::WildcardSuppressed,
message: format!(
"{description}{suffix} left unexpanded — column lineage will be incomplete for this projection"
),
span,
});
}
pub(super) fn record_unrepresentable_table(&mut self, name: &ObjectName) {
let span = name
.0
.first()
.and_then(|part| part.as_ident())
.map(|ident| ident.span)
.filter(|s| s.start.line != 0);
let suffix = span_suffix(span);
let reason = if name.0.iter().any(|part| part.as_ident().is_none()) {
"is not a plain catalog.schema.name identifier path"
} else {
"has too many qualifiers (max catalog.schema.name)"
};
self.diagnostics.push(ColumnLevelDiagnostic {
kind: ColumnLevelDiagnosticKind::TooManyTableQualifiers,
message: format!("table reference `{name}`{suffix} {reason} — dropped"),
span,
});
}
pub(super) fn record_insert_columns_unresolved(
&mut self,
target: &TableReference,
source_wildcard: bool,
) {
let message = if source_wildcard {
format!(
"column-list-less INSERT into `{target}`: the `SELECT *` source isn't expanded, so its columns can't be paired with the target — column writes / lineage dropped"
)
} else {
format!(
"column-list-less INSERT into `{target}` can't pair source columns to target columns without a catalog — column writes / lineage dropped"
)
};
self.diagnostics.push(ColumnLevelDiagnostic {
kind: ColumnLevelDiagnosticKind::InsertColumnsUnresolved,
message,
span: None,
});
}
pub(super) fn record_unsupported_dml_target(
&mut self,
statement: &str,
target: &dyn std::fmt::Display,
) {
self.diagnostics.push(ColumnLevelDiagnostic {
kind: ColumnLevelDiagnosticKind::UnsupportedStatement,
message: format!(
"{statement} target `{target}` is not a writable base table (a CTE name, derived table, subquery, table function, or join) — the statement can't be analyzed and is dropped"
),
span: None,
});
}
pub(super) fn record_merge_insert_row_unresolved(&mut self, target: &TableReference) {
self.diagnostics.push(ColumnLevelDiagnostic {
kind: ColumnLevelDiagnosticKind::InsertColumnsUnresolved,
message: format!(
"MERGE INSERT ROW into `{target}` inserts the full source row — its column writes / lineage can't be recovered from SQL text and are dropped"
),
span: None,
});
}
pub(super) fn record_insert_columns_arity_mismatch(
&mut self,
target: &TableReference,
target_columns: usize,
source_columns: usize,
) {
self.diagnostics.push(ColumnLevelDiagnostic {
kind: ColumnLevelDiagnosticKind::InsertColumnsArityMismatch,
message: format!(
"INSERT into `{target}` lists {target_columns} target column(s) but the source projects {source_columns} — lineage pairs only the first {min} and drops the rest",
min = target_columns.min(source_columns)
),
span: None,
});
}
pub(super) fn diagnose_insert_arity(
&mut self,
target: &TableReference,
explicit: bool,
target_columns: usize,
source_columns: usize,
) {
let mismatch = if explicit {
source_columns != target_columns
} else {
source_columns > target_columns
};
if mismatch {
self.record_insert_columns_arity_mismatch(target, target_columns, source_columns);
}
}
pub(super) fn flag_anonymous_relation_columns(
&mut self,
target: &TableReference,
explicit: &[Ident],
input: &LogicalPlan,
) {
if !explicit.is_empty() {
return;
}
let operands = output_operands(input);
let Some(operand) = operands.first() else {
return;
};
let anonymous = output_slots(operand.outputs)
.filter(|(name, _)| name.is_none())
.count();
if anonymous == 0 {
return;
}
self.diagnostics.push(ColumnLevelDiagnostic {
kind: ColumnLevelDiagnosticKind::AnonymousColumnsSuppressed,
message: format!(
"`{target}` has {anonymous} unaliased expression column(s) with no name recoverable from the SQL text — dropped from column writes / lineage; alias them to surface"
),
span: None,
});
}
pub(super) fn diagnose_created_columns(
&mut self,
target: &TableReference,
explicit: &[Ident],
input: &LogicalPlan,
source_wildcard: bool,
) {
if explicit.is_empty() {
self.flag_anonymous_relation_columns(target, explicit, input);
return;
}
if source_wildcard {
return;
}
if let Some(operand) = output_operands(input).first() {
let outputs = slot_count(operand.outputs);
if outputs != 0 && outputs != explicit.len() {
self.record_created_columns_arity_mismatch(target, explicit.len(), outputs);
}
}
}
pub(super) fn record_created_columns_arity_mismatch(
&mut self,
target: &TableReference,
target_columns: usize,
source_columns: usize,
) {
self.diagnostics.push(ColumnLevelDiagnostic {
kind: ColumnLevelDiagnosticKind::InsertColumnsArityMismatch,
message: format!(
"created relation `{target}` lists {target_columns} column(s) but the source projects {source_columns} — lineage pairs only the first {min} and drops the rest",
min = target_columns.min(source_columns)
),
span: None,
});
}
}
fn span_suffix(span: Option<Span>) -> String {
match span {
Some(s) => format!(" at L{}:C{}", s.start.line, s.start.column),
None => String::new(),
}
}
fn fill_query_defaults(written: &TableReference, catalog: &Catalog) -> TableReference {
let mut filled = written.clone();
if filled.schema.is_none() {
if let Some(schema) = catalog.default_schema_segment() {
filled.schema = Some(Ident::with_quote('"', schema));
}
}
if filled.catalog.is_none() && filled.schema.is_some() {
if let Some(catalog_segment) = catalog.default_catalog_segment() {
filled.catalog = Some(Ident::with_quote('"', catalog_segment));
}
}
filled
}
fn surface_with_defaults(written: &TableReference, catalog: &Catalog) -> TableReference {
let plain = |value: &str| Ident {
value: value.to_string(),
quote_style: None,
span: Span::empty(),
};
let schema = written
.schema
.clone()
.or_else(|| catalog.default_schema_segment().map(plain));
let catalog_segment = if written.catalog.is_some() {
written.catalog.clone()
} else if schema.is_some() {
catalog.default_catalog_segment().map(plain)
} else {
None
};
TableReference {
catalog: catalog_segment,
schema,
name: written.name.clone(),
}
}
fn catalog_table_matches(query: &TableReference, table: &CatalogTable, fold: CaseRule) -> bool {
if fold.normalize(&query.name) != normalize_catalog(table.name_segment(), fold) {
return false;
}
if let (Some(query_schema), Some(table_schema)) = (&query.schema, table.schema_segment()) {
if fold.normalize(query_schema) != normalize_catalog(table_schema, fold) {
return false;
}
}
match (&query.catalog, table.catalog_segment()) {
(Some(query_catalog), Some(table_catalog)) => {
fold.normalize(query_catalog) == normalize_catalog(table_catalog, fold)
}
_ => true,
}
}
fn normalize_catalog(segment: &str, fold: CaseRule) -> String {
fold.normalize(&Ident::with_quote('"', segment))
}
fn canonical_ref(table: &CatalogTable, written: &TableReference, quote: char) -> TableReference {
let seg = |value: &str, span: Span| Ident {
value: value.to_string(),
quote_style: Some(quote),
span,
};
let span_of = |ident: Option<&Ident>| ident.map_or(Span::empty(), |i| i.span);
TableReference {
catalog: table
.catalog_segment()
.map(|c| seg(c, span_of(written.catalog.as_ref()))),
schema: table
.schema_segment()
.map(|s| seg(s, span_of(written.schema.as_ref()))),
name: seg(table.name_segment(), written.name.span),
}
}
fn base(table: &TableReference, resolution: ResolutionKind) -> Binding {
Binding::Base {
table: table.clone(),
resolution,
}
}
fn downgrade(binding: Binding) -> Binding {
match binding {
Binding::Base { table, .. } => Binding::Base {
table,
resolution: ResolutionKind::Inferred,
},
other => other,
}
}
fn join(left: LogicalPlan, right: LogicalPlan, on: Vec<Expr>) -> LogicalPlan {
LogicalPlan::Join(Join {
left: Box::new(left),
right: Box::new(right),
on,
})
}
fn combine(left: LogicalPlan, right: LogicalPlan) -> LogicalPlan {
if matches!(left, LogicalPlan::Empty) {
right
} else {
join(left, right, Vec::new())
}
}
fn assignment_target_columns(target: &AssignmentTarget) -> Vec<Ident> {
match target {
AssignmentTarget::ColumnName(name) if name.0.len() <= 4 => name
.0
.last()
.and_then(|p| p.as_ident().cloned())
.into_iter()
.collect(),
AssignmentTarget::ColumnName(_) | AssignmentTarget::Tuple(_) => Vec::new(),
}
}
fn alter_table_op_target_columns(op: &AlterTableOperation) -> Vec<Ident> {
match op {
AlterTableOperation::AddColumn { column_def, .. } => vec![column_def.name.clone()],
AlterTableOperation::DropColumn { column_names, .. } => column_names.clone(),
AlterTableOperation::RenameColumn {
old_column_name,
new_column_name,
} => vec![old_column_name.clone(), new_column_name.clone()],
AlterTableOperation::ChangeColumn {
old_name, new_name, ..
} if old_name != new_name => vec![old_name.clone(), new_name.clone()],
AlterTableOperation::ChangeColumn { old_name, .. } => vec![old_name.clone()],
AlterTableOperation::ModifyColumn { col_name, .. } => vec![col_name.clone()],
AlterTableOperation::AlterColumn { column_name, .. } => vec![column_name.clone()],
AlterTableOperation::AddConstraint { .. }
| AlterTableOperation::AddProjection { .. }
| AlterTableOperation::DropProjection { .. }
| AlterTableOperation::MaterializeProjection { .. }
| AlterTableOperation::ClearProjection { .. }
| AlterTableOperation::DisableRowLevelSecurity
| AlterTableOperation::DisableRule { .. }
| AlterTableOperation::DisableTrigger { .. }
| AlterTableOperation::DropConstraint { .. }
| AlterTableOperation::AttachPartition { .. }
| AlterTableOperation::DetachPartition { .. }
| AlterTableOperation::FreezePartition { .. }
| AlterTableOperation::UnfreezePartition { .. }
| AlterTableOperation::DropPrimaryKey { .. }
| AlterTableOperation::DropForeignKey { .. }
| AlterTableOperation::DropIndex { .. }
| AlterTableOperation::EnableAlwaysRule { .. }
| AlterTableOperation::EnableAlwaysTrigger { .. }
| AlterTableOperation::EnableReplicaRule { .. }
| AlterTableOperation::EnableReplicaTrigger { .. }
| AlterTableOperation::EnableRowLevelSecurity
| AlterTableOperation::ForceRowLevelSecurity
| AlterTableOperation::NoForceRowLevelSecurity
| AlterTableOperation::EnableRule { .. }
| AlterTableOperation::EnableTrigger { .. }
| AlterTableOperation::RenamePartitions { .. }
| AlterTableOperation::ReplicaIdentity { .. }
| AlterTableOperation::AddPartitions { .. }
| AlterTableOperation::DropPartitions { .. }
| AlterTableOperation::RenameTable { .. }
| AlterTableOperation::RenameConstraint { .. }
| AlterTableOperation::SwapWith { .. }
| AlterTableOperation::SetTblProperties { .. }
| AlterTableOperation::OwnerTo { .. }
| AlterTableOperation::ClusterBy { .. }
| AlterTableOperation::DropClusteringKey
| AlterTableOperation::AlterSortKey { .. }
| AlterTableOperation::SuspendRecluster
| AlterTableOperation::ResumeRecluster
| AlterTableOperation::Refresh { .. }
| AlterTableOperation::Suspend
| AlterTableOperation::Resume
| AlterTableOperation::Algorithm { .. }
| AlterTableOperation::Lock { .. }
| AlterTableOperation::AutoIncrement { .. }
| AlterTableOperation::ValidateConstraint { .. }
| AlterTableOperation::SetOptionsParens { .. } => Vec::new(),
}
}
fn sort(input: LogicalPlan, keys: Vec<Expr>) -> LogicalPlan {
LogicalPlan::Sort(Sort {
input: Box::new(input),
keys,
})
}
fn alias_column_names(alias: &TableAlias) -> Vec<Ident> {
alias.columns.iter().map(|c| c.name.clone()).collect()
}
fn rename_outputs(op: &mut LogicalPlan, names: &[Ident]) {
if names.is_empty() {
return;
}
match op {
LogicalPlan::Projection(p) => {
let mut names = names.iter();
for ne in p.exprs.iter_mut() {
match &mut ne.names {
OutputNames::Single(slot) => {
if let Some(n) = names.next() {
*slot = Some(n.clone());
}
}
OutputNames::Fan(fan) => {
for slot in fan.iter_mut() {
if let Some(n) = names.next() {
*slot = n.clone();
}
}
}
}
}
}
LogicalPlan::Sort(s) => rename_outputs(&mut s.input, names),
LogicalPlan::Filter(f) => rename_outputs(&mut f.input, names),
LogicalPlan::With(w) => rename_outputs(&mut w.body, names),
LogicalPlan::SetOp(so) => {
rename_outputs(&mut so.left, names);
rename_outputs(&mut so.right, names);
}
_ => {}
}
}
fn inferred_name(expr: &SqlExpr) -> Option<Ident> {
match expr {
SqlExpr::Identifier(id) => Some(id.clone()),
SqlExpr::CompoundIdentifier(parts) => parts.last().cloned(),
_ => None,
}
}
fn ordinal_output_name(expr: &SqlExpr, scope: &Scope) -> Option<Ident> {
let SqlExpr::Value(v) = expr else {
return None;
};
let Value::Number(digits, _) = &v.value else {
return None;
};
let n: usize = digits.parse().ok()?;
scope.query_outputs.get(n.checked_sub(1)?)?.name.clone()
}
fn join_constraint(op: &JoinOperator) -> Option<&JoinConstraint> {
match op {
JoinOperator::Join(c)
| JoinOperator::Inner(c)
| JoinOperator::Left(c)
| JoinOperator::LeftOuter(c)
| JoinOperator::Right(c)
| JoinOperator::RightOuter(c)
| JoinOperator::FullOuter(c)
| JoinOperator::CrossJoin(c)
| JoinOperator::Semi(c)
| JoinOperator::LeftSemi(c)
| JoinOperator::RightSemi(c)
| JoinOperator::Anti(c)
| JoinOperator::LeftAnti(c)
| JoinOperator::RightAnti(c)
| JoinOperator::StraightJoin(c) => Some(c),
JoinOperator::AsOf { constraint, .. } => Some(constraint),
JoinOperator::CrossApply
| JoinOperator::OuterApply
| JoinOperator::ArrayJoin
| JoinOperator::LeftArrayJoin
| JoinOperator::InnerArrayJoin => None,
}
}
fn table_set_expr_ref(table: &sqlparser::ast::Table) -> Option<TableReference> {
let name = table.table_name.as_ref()?;
let mut parts = Vec::new();
if let Some(schema) = &table.schema_name {
parts.push(Ident::new(schema));
}
parts.push(Ident::new(name));
TableReference::try_from_parts(&parts)
}
fn join_on(op: &JoinOperator) -> Option<&SqlExpr> {
match join_constraint(op) {
Some(JoinConstraint::On(expr)) => Some(expr),
_ => None,
}
}
fn join_using(op: &JoinOperator) -> Vec<Ident> {
match join_constraint(op) {
Some(JoinConstraint::Using(names)) => names
.iter()
.filter_map(|n| n.0.last().and_then(|p| p.as_ident().cloned()))
.collect(),
_ => Vec::new(),
}
}
fn join_is_natural(op: &JoinOperator) -> bool {
matches!(join_constraint(op), Some(JoinConstraint::Natural))
}
fn is_array_join(op: &JoinOperator) -> bool {
matches!(
op,
JoinOperator::ArrayJoin | JoinOperator::LeftArrayJoin | JoinOperator::InnerArrayJoin
)
}
#[cfg(test)]
mod tests {
use super::*;
use crate::catalog::{Catalog, CatalogTable};
use sqlparser::dialect::GenericDialect;
use sqlparser::parser::Parser;
fn bind(sql: &str) -> LogicalPlan {
bind_cat(sql, None)
}
fn bind_cat(sql: &str, catalog: Option<&Catalog>) -> LogicalPlan {
let statements = Parser::parse_sql(&GenericDialect {}, sql).unwrap();
let style = IdentifierStyle {
casing: crate::casing::IdentifierCasing::for_dialect(&GenericDialect {}),
quote: crate::casing::canonical_quote(&GenericDialect {}),
};
build_with_diagnostics(&statements[0], catalog, style).0
}
fn only_binding(plan: &LogicalPlan) -> &Binding {
let LogicalPlan::Projection(p) = plan else {
panic!("expected Projection, got {plan:?}")
};
match &p.exprs[..] {
[NamedExpr {
expr: Expr::Column(c),
..
}] => &c.binding,
other => panic!("expected one column expr, got {other:?}"),
}
}
#[test]
fn catalog_free_single_table_is_inferred() {
assert!(matches!(only_binding(&bind("SELECT a FROM t")),
Binding::Base { table, resolution }
if table.name.value == "t" && *resolution == ResolutionKind::Inferred));
}
#[test]
fn catalog_known_hit_is_cataloged_and_canonical() {
let cat =
Catalog::new().table(CatalogTable::new("public", "users").columns(["id", "name"]));
let op = bind_cat("SELECT name FROM users", Some(&cat));
match only_binding(&op) {
Binding::Base { table, resolution } => {
assert_eq!(table.name.value, "users");
assert_eq!(table.schema.as_ref().unwrap().value, "public"); assert_eq!(*resolution, ResolutionKind::Cataloged);
}
other => panic!("expected Base Cataloged, got {other:?}"),
}
}
#[test]
fn catalog_known_miss_is_unresolved() {
let cat =
Catalog::new().table(CatalogTable::new("public", "users").columns(["id", "name"]));
assert!(matches!(
only_binding(&bind_cat("SELECT nonexistent FROM users", Some(&cat))),
Binding::Unresolved
));
}
#[test]
fn cataloged_witness_over_unknown_downgrades_to_inferred() {
let cat = Catalog::new().table(CatalogTable::new("public", "known_t").columns(["a", "b"]));
let op = bind_cat(
"SELECT a FROM known_t JOIN open_t ON known_t.b = open_t.k",
Some(&cat),
);
match only_binding(&op) {
Binding::Base { table, resolution } => {
assert_eq!(table.name.value, "known_t");
assert_eq!(*resolution, ResolutionKind::Inferred); }
other => panic!("expected Base Inferred (downgraded), got {other:?}"),
}
}
#[test]
fn two_known_owners_is_ambiguous() {
let cat = Catalog::new()
.table(CatalogTable::new("public", "t1").columns(["id"]))
.table(CatalogTable::new("public", "t2").columns(["id"]));
assert!(matches!(
only_binding(&bind_cat(
"SELECT id FROM t1 JOIN t2 ON t1.id = t2.id",
Some(&cat)
)),
Binding::Ambiguous
));
}
}