use crate::push_down_filter::replace_cols_by_name;
use crate::{OptimizerConfig, OptimizerRule};
use datafusion_common::{Column, DFSchema, Result, qualified_name};
use datafusion_expr::logical_plan::{Join, JoinType, LogicalPlan, Projection};
use datafusion_expr::{Expr, Filter, Operator};
use crate::optimizer::ApplyOrder;
use datafusion_common::tree_node::Transformed;
use datafusion_expr::expr::{BinaryExpr, Cast, InList, Like, TryCast};
use std::collections::HashMap;
use std::sync::Arc;
#[derive(Default, Debug)]
pub struct EliminateOuterJoin;
impl EliminateOuterJoin {
#[expect(missing_docs)]
pub fn new() -> Self {
Self {}
}
}
impl OptimizerRule for EliminateOuterJoin {
fn name(&self) -> &str {
"eliminate_outer_join"
}
fn apply_order(&self) -> Option<ApplyOrder> {
Some(ApplyOrder::TopDown)
}
fn supports_rewrite(&self) -> bool {
true
}
fn rewrite(
&self,
plan: LogicalPlan,
_config: &dyn OptimizerConfig,
) -> Result<Transformed<LogicalPlan>> {
let LogicalPlan::Filter(filter) = plan else {
return Ok(Transformed::no(plan));
};
let mut rewritten_predicate = filter.predicate.clone();
let mut projections: Vec<Projection> = Vec::new();
let mut cur = Arc::clone(&filter.input);
let new_join = loop {
match cur.as_ref() {
LogicalPlan::Projection(p) => {
rewritten_predicate =
inline_through_projection(rewritten_predicate, p)?;
let next = Arc::clone(&p.input);
projections.push(p.clone());
cur = next;
}
LogicalPlan::Join(join) => {
let Some(new_join) = try_simplify_join(join, &rewritten_predicate)
else {
return Ok(Transformed::no(LogicalPlan::Filter(filter)));
};
break new_join;
}
_ => {
return Ok(Transformed::no(LogicalPlan::Filter(filter)));
}
}
};
let rebuilt_inner = rewrap_projections(new_join, projections);
Filter::try_new(filter.predicate, Arc::new(rebuilt_inner))
.map(|f| Transformed::yes(LogicalPlan::Filter(f)))
}
}
fn try_simplify_join(join: &Join, predicate: &Expr) -> Option<LogicalPlan> {
if !join.join_type.is_outer() {
return None;
}
let null_rejecting_sides = extract_null_rejecting_sides(
predicate,
join.left.schema(),
join.right.schema(),
true,
);
let new_join_type = eliminate_outer(
join.join_type,
null_rejecting_sides.left,
null_rejecting_sides.right,
);
if new_join_type == join.join_type {
return None;
}
Some(LogicalPlan::Join(Join {
left: Arc::clone(&join.left),
right: Arc::clone(&join.right),
join_type: new_join_type,
join_constraint: join.join_constraint,
on: join.on.clone(),
filter: join.filter.clone(),
schema: Arc::clone(&join.schema),
null_equality: join.null_equality,
null_aware: join.null_aware,
}))
}
fn inline_through_projection(predicate: Expr, p: &Projection) -> Result<Expr> {
let mut map: HashMap<String, Expr> = HashMap::new();
for ((qualifier, field), expr) in p.schema.iter().zip(p.expr.iter()) {
map.insert(
qualified_name(qualifier, field.name()),
unalias(expr).clone(),
);
}
replace_cols_by_name(predicate, &map)
}
fn rewrap_projections(
new_inner: LogicalPlan,
projections: Vec<Projection>,
) -> LogicalPlan {
let mut current = new_inner;
for mut p in projections.into_iter().rev() {
p.input = Arc::new(current);
current = LogicalPlan::Projection(p);
}
current
}
fn unalias(expr: &Expr) -> &Expr {
if let Expr::Alias(a) = expr {
unalias(&a.expr)
} else {
expr
}
}
pub fn eliminate_outer(
join_type: JoinType,
left_non_nullable: bool,
right_non_nullable: bool,
) -> JoinType {
match (join_type, left_non_nullable, right_non_nullable) {
(JoinType::Left, _, true) => JoinType::Inner,
(JoinType::Right, true, _) => JoinType::Inner,
(JoinType::Full, true, true) => JoinType::Inner,
(JoinType::Full, true, false) => JoinType::Left,
(JoinType::Full, false, true) => JoinType::Right,
_ => join_type,
}
}
#[derive(Default, Clone, Copy, Debug, PartialEq, Eq)]
struct NullRejectingSides {
left: bool,
right: bool,
}
impl NullRejectingSides {
fn for_column(col: &Column, left_schema: &DFSchema, right_schema: &DFSchema) -> Self {
Self {
left: left_schema.has_column(col),
right: right_schema.has_column(col),
}
}
fn union(self, other: Self) -> Self {
Self {
left: self.left || other.left,
right: self.right || other.right,
}
}
fn intersection(self, other: Self) -> Self {
Self {
left: self.left && other.left,
right: self.right && other.right,
}
}
}
fn extract_null_rejecting_sides(
expr: &Expr,
left_schema: &Arc<DFSchema>,
right_schema: &Arc<DFSchema>,
top_level: bool,
) -> NullRejectingSides {
match expr {
Expr::Column(col) => {
NullRejectingSides::for_column(col, left_schema, right_schema)
}
Expr::BinaryExpr(BinaryExpr { left, op, right }) => match op {
Operator::And | Operator::Or => {
let left_sides = extract_null_rejecting_sides(
left,
left_schema,
right_schema,
top_level,
);
let right_sides = extract_null_rejecting_sides(
right,
left_schema,
right_schema,
top_level,
);
if top_level && *op == Operator::And {
left_sides.union(right_sides)
} else {
left_sides.intersection(right_sides)
}
}
op if op.returns_null_on_null() => {
let left_sides =
extract_null_rejecting_sides(left, left_schema, right_schema, false);
let right_sides =
extract_null_rejecting_sides(right, left_schema, right_schema, false);
left_sides.union(right_sides)
}
_ => NullRejectingSides::default(),
},
Expr::Not(arg) | Expr::Negative(arg) => {
extract_null_rejecting_sides(arg, left_schema, right_schema, false)
}
Expr::IsNotNull(arg)
| Expr::IsTrue(arg)
| Expr::IsFalse(arg)
| Expr::IsNotUnknown(arg) => {
if top_level {
extract_null_rejecting_sides(arg, left_schema, right_schema, false)
} else {
NullRejectingSides::default()
}
}
Expr::Cast(Cast { expr, field: _ })
| Expr::TryCast(TryCast { expr, field: _ }) => {
extract_null_rejecting_sides(expr, left_schema, right_schema, false)
}
Expr::InList(InList { expr, .. }) => {
extract_null_rejecting_sides(expr, left_schema, right_schema, false)
}
Expr::Between(between) => {
extract_null_rejecting_sides(&between.expr, left_schema, right_schema, false)
}
Expr::Like(Like { expr, pattern, .. }) => {
let expr_sides =
extract_null_rejecting_sides(expr, left_schema, right_schema, false);
let pattern_sides =
extract_null_rejecting_sides(pattern, left_schema, right_schema, false);
expr_sides.union(pattern_sides)
}
Expr::ScalarFunction(func) if func.func.is_strict() => func
.args
.iter()
.map(|arg| {
extract_null_rejecting_sides(arg, left_schema, right_schema, false)
})
.fold(NullRejectingSides::default(), NullRejectingSides::union),
_ => NullRejectingSides::default(),
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::OptimizerContext;
use crate::assert_optimized_plan_eq_snapshot;
use crate::test::*;
use arrow::datatypes::DataType;
use datafusion_common::ScalarValue;
use datafusion_expr::{
ColumnarValue,
Operator::{And, Or},
ScalarFunctionArgs, ScalarUDF, ScalarUDFImpl, Signature, Volatility, binary_expr,
cast, col, lit,
logical_plan::builder::LogicalPlanBuilder,
not, try_cast,
};
#[test]
fn null_rejecting_sides_union() {
let left_side = NullRejectingSides {
left: true,
right: false,
};
let right_side = NullRejectingSides {
left: false,
right: true,
};
assert_eq!(
left_side.union(right_side),
NullRejectingSides {
left: true,
right: true,
}
);
}
#[test]
fn null_rejecting_sides_intersection() {
let both_sides = NullRejectingSides {
left: true,
right: true,
};
let right_side = NullRejectingSides {
left: false,
right: true,
};
assert_eq!(
both_sides.intersection(right_side),
NullRejectingSides {
left: false,
right: true,
}
);
}
macro_rules! assert_optimized_plan_equal {
(
$plan:expr,
@ $expected:literal $(,)?
) => {{
let optimizer_ctx = OptimizerContext::new().with_max_passes(1);
let rules: Vec<Arc<dyn crate::OptimizerRule + Send + Sync>> = vec![Arc::new(EliminateOuterJoin::new())];
assert_optimized_plan_eq_snapshot!(
optimizer_ctx,
rules,
$plan,
@ $expected,
)
}};
}
#[derive(Debug, PartialEq, Eq, Hash)]
struct TestUdf {
name: &'static str,
signature: Signature,
strict: bool,
}
impl TestUdf {
fn new(name: &'static str, strict: bool) -> Self {
Self {
name,
signature: Signature::uniform(
1,
vec![DataType::UInt32],
Volatility::Immutable,
),
strict,
}
}
}
impl ScalarUDFImpl for TestUdf {
fn name(&self) -> &str {
self.name
}
fn signature(&self) -> &Signature {
&self.signature
}
fn return_type(&self, _arg_types: &[DataType]) -> Result<DataType> {
Ok(DataType::UInt32)
}
fn is_strict(&self) -> bool {
self.strict
}
fn invoke_with_args(&self, _args: ScalarFunctionArgs) -> Result<ColumnarValue> {
unimplemented!()
}
}
fn strict_udf(arg: Expr) -> Expr {
ScalarUDF::from(TestUdf::new("strict_test", true)).call(vec![arg])
}
fn non_strict_udf(arg: Expr) -> Expr {
ScalarUDF::from(TestUdf::new("non_strict_test", false)).call(vec![arg])
}
#[test]
fn eliminate_left_with_null() -> Result<()> {
let t1 = test_table_scan_with_name("t1")?;
let t2 = test_table_scan_with_name("t2")?;
let plan = LogicalPlanBuilder::from(t1)
.join(
t2,
JoinType::Left,
(vec![Column::from_name("a")], vec![Column::from_name("a")]),
None,
)?
.filter(col("t2.b").is_null())?
.build()?;
assert_optimized_plan_equal!(plan, @r"
Filter: t2.b IS NULL
Left Join: t1.a = t2.a
TableScan: t1
TableScan: t2
")
}
#[test]
fn eliminate_left_with_not_null() -> Result<()> {
let t1 = test_table_scan_with_name("t1")?;
let t2 = test_table_scan_with_name("t2")?;
let plan = LogicalPlanBuilder::from(t1)
.join(
t2,
JoinType::Left,
(vec![Column::from_name("a")], vec![Column::from_name("a")]),
None,
)?
.filter(col("t2.b").is_not_null())?
.build()?;
assert_optimized_plan_equal!(plan, @r"
Filter: t2.b IS NOT NULL
Inner Join: t1.a = t2.a
TableScan: t1
TableScan: t2
")
}
#[test]
fn eliminate_left_with_strict_function() -> Result<()> {
let t1 = test_table_scan_with_name("t1")?;
let t2 = test_table_scan_with_name("t2")?;
let plan = LogicalPlanBuilder::from(t1)
.join(
t2,
JoinType::Left,
(vec![Column::from_name("a")], vec![Column::from_name("a")]),
None,
)?
.filter(strict_udf(col("t2.b")).gt(lit(5u32)))?
.build()?;
assert_optimized_plan_equal!(plan, @r"
Filter: strict_test(t2.b) > UInt32(5)
Inner Join: t1.a = t2.a
TableScan: t1
TableScan: t2
")
}
#[test]
fn no_eliminate_left_with_non_strict_function() -> Result<()> {
let t1 = test_table_scan_with_name("t1")?;
let t2 = test_table_scan_with_name("t2")?;
let plan = LogicalPlanBuilder::from(t1)
.join(
t2,
JoinType::Left,
(vec![Column::from_name("a")], vec![Column::from_name("a")]),
None,
)?
.filter(non_strict_udf(col("t2.b")).gt(lit(5u32)))?
.build()?;
assert_optimized_plan_equal!(plan, @r"
Filter: non_strict_test(t2.b) > UInt32(5)
Left Join: t1.a = t2.a
TableScan: t1
TableScan: t2
")
}
#[test]
fn eliminate_left_with_nested_strict_is_not_null() -> Result<()> {
let t1 = test_table_scan_with_name("t1")?;
let t2 = test_table_scan_with_name("t2")?;
let plan = LogicalPlanBuilder::from(t1)
.join(
t2,
JoinType::Left,
(vec![Column::from_name("a")], vec![Column::from_name("a")]),
None,
)?
.filter(strict_udf(strict_udf(col("t2.b"))).is_not_null())?
.build()?;
assert_optimized_plan_equal!(plan, @r"
Filter: strict_test(strict_test(t2.b)) IS NOT NULL
Inner Join: t1.a = t2.a
TableScan: t1
TableScan: t2
")
}
#[test]
fn no_eliminate_left_with_strict_function_is_null() -> Result<()> {
let t1 = test_table_scan_with_name("t1")?;
let t2 = test_table_scan_with_name("t2")?;
let plan = LogicalPlanBuilder::from(t1)
.join(
t2,
JoinType::Left,
(vec![Column::from_name("a")], vec![Column::from_name("a")]),
None,
)?
.filter(strict_udf(col("t2.b")).is_null())?
.build()?;
assert_optimized_plan_equal!(plan, @r"
Filter: strict_test(t2.b) IS NULL
Left Join: t1.a = t2.a
TableScan: t1
TableScan: t2
")
}
#[test]
fn eliminate_right_with_or() -> Result<()> {
let t1 = test_table_scan_with_name("t1")?;
let t2 = test_table_scan_with_name("t2")?;
let plan = LogicalPlanBuilder::from(t1)
.join(
t2,
JoinType::Right,
(vec![Column::from_name("a")], vec![Column::from_name("a")]),
None,
)?
.filter(binary_expr(
col("t1.b").gt(lit(10u32)),
Or,
col("t1.c").lt(lit(20u32)),
))?
.build()?;
assert_optimized_plan_equal!(plan, @r"
Filter: t1.b > UInt32(10) OR t1.c < UInt32(20)
Inner Join: t1.a = t2.a
TableScan: t1
TableScan: t2
")
}
#[test]
fn eliminate_full_with_and() -> Result<()> {
let t1 = test_table_scan_with_name("t1")?;
let t2 = test_table_scan_with_name("t2")?;
let plan = LogicalPlanBuilder::from(t1)
.join(
t2,
JoinType::Full,
(vec![Column::from_name("a")], vec![Column::from_name("a")]),
None,
)?
.filter(binary_expr(
col("t1.b").gt(lit(10u32)),
And,
col("t2.c").lt(lit(20u32)),
))?
.build()?;
assert_optimized_plan_equal!(plan, @r"
Filter: t1.b > UInt32(10) AND t2.c < UInt32(20)
Inner Join: t1.a = t2.a
TableScan: t1
TableScan: t2
")
}
#[test]
fn eliminate_left_with_in_list() -> Result<()> {
let t1 = test_table_scan_with_name("t1")?;
let t2 = test_table_scan_with_name("t2")?;
let plan = LogicalPlanBuilder::from(t1)
.join(
t2,
JoinType::Left,
(vec![Column::from_name("a")], vec![Column::from_name("a")]),
None,
)?
.filter(col("t2.b").in_list(vec![lit(1u32), lit(2u32), lit(3u32)], false))?
.build()?;
assert_optimized_plan_equal!(plan, @r"
Filter: t2.b IN ([UInt32(1), UInt32(2), UInt32(3)])
Inner Join: t1.a = t2.a
TableScan: t1
TableScan: t2
")
}
#[test]
fn eliminate_left_with_in_list_containing_null() -> Result<()> {
let t1 = test_table_scan_with_name("t1")?;
let t2 = test_table_scan_with_name("t2")?;
let plan = LogicalPlanBuilder::from(t1)
.join(
t2,
JoinType::Left,
(vec![Column::from_name("a")], vec![Column::from_name("a")]),
None,
)?
.filter(
col("t2.b")
.in_list(vec![lit(1u32), lit(ScalarValue::UInt32(None))], false),
)?
.build()?;
assert_optimized_plan_equal!(plan, @r"
Filter: t2.b IN ([UInt32(1), UInt32(NULL)])
Inner Join: t1.a = t2.a
TableScan: t1
TableScan: t2
")
}
#[test]
fn eliminate_left_with_not_in_list() -> Result<()> {
let t1 = test_table_scan_with_name("t1")?;
let t2 = test_table_scan_with_name("t2")?;
let plan = LogicalPlanBuilder::from(t1)
.join(
t2,
JoinType::Left,
(vec![Column::from_name("a")], vec![Column::from_name("a")]),
None,
)?
.filter(col("t2.b").in_list(vec![lit(1u32), lit(2u32)], true))?
.build()?;
assert_optimized_plan_equal!(plan, @r"
Filter: t2.b NOT IN ([UInt32(1), UInt32(2)])
Inner Join: t1.a = t2.a
TableScan: t1
TableScan: t2
")
}
#[test]
fn eliminate_left_with_between() -> Result<()> {
let t1 = test_table_scan_with_name("t1")?;
let t2 = test_table_scan_with_name("t2")?;
let plan = LogicalPlanBuilder::from(t1)
.join(
t2,
JoinType::Left,
(vec![Column::from_name("a")], vec![Column::from_name("a")]),
None,
)?
.filter(col("t2.b").between(lit(1u32), lit(10u32)))?
.build()?;
assert_optimized_plan_equal!(plan, @r"
Filter: t2.b BETWEEN UInt32(1) AND UInt32(10)
Inner Join: t1.a = t2.a
TableScan: t1
TableScan: t2
")
}
#[test]
fn eliminate_right_with_between() -> Result<()> {
let t1 = test_table_scan_with_name("t1")?;
let t2 = test_table_scan_with_name("t2")?;
let plan = LogicalPlanBuilder::from(t1)
.join(
t2,
JoinType::Right,
(vec![Column::from_name("a")], vec![Column::from_name("a")]),
None,
)?
.filter(col("t1.b").between(lit(1u32), lit(10u32)))?
.build()?;
assert_optimized_plan_equal!(plan, @r"
Filter: t1.b BETWEEN UInt32(1) AND UInt32(10)
Inner Join: t1.a = t2.a
TableScan: t1
TableScan: t2
")
}
#[test]
fn eliminate_full_with_between() -> Result<()> {
let t1 = test_table_scan_with_name("t1")?;
let t2 = test_table_scan_with_name("t2")?;
let plan = LogicalPlanBuilder::from(t1)
.join(
t2,
JoinType::Full,
(vec![Column::from_name("a")], vec![Column::from_name("a")]),
None,
)?
.filter(binary_expr(
col("t1.b").between(lit(1u32), lit(10u32)),
And,
col("t2.b").between(lit(5u32), lit(20u32)),
))?
.build()?;
assert_optimized_plan_equal!(plan, @r"
Filter: t1.b BETWEEN UInt32(1) AND UInt32(10) AND t2.b BETWEEN UInt32(5) AND UInt32(20)
Inner Join: t1.a = t2.a
TableScan: t1
TableScan: t2
")
}
#[test]
fn eliminate_full_with_in_list() -> Result<()> {
let t1 = test_table_scan_with_name("t1")?;
let t2 = test_table_scan_with_name("t2")?;
let plan = LogicalPlanBuilder::from(t1)
.join(
t2,
JoinType::Full,
(vec![Column::from_name("a")], vec![Column::from_name("a")]),
None,
)?
.filter(binary_expr(
col("t1.b").in_list(vec![lit(1u32), lit(2u32)], false),
And,
col("t2.b").in_list(vec![lit(3u32), lit(4u32)], false),
))?
.build()?;
assert_optimized_plan_equal!(plan, @r"
Filter: t1.b IN ([UInt32(1), UInt32(2)]) AND t2.b IN ([UInt32(3), UInt32(4)])
Inner Join: t1.a = t2.a
TableScan: t1
TableScan: t2
")
}
#[test]
fn no_eliminate_left_with_in_list_or_is_null() -> Result<()> {
let t1 = test_table_scan_with_name("t1")?;
let t2 = test_table_scan_with_name("t2")?;
let plan = LogicalPlanBuilder::from(t1)
.join(
t2,
JoinType::Left,
(vec![Column::from_name("a")], vec![Column::from_name("a")]),
None,
)?
.filter(binary_expr(
col("t2.b").in_list(vec![lit(1u32), lit(2u32)], false),
Or,
col("t2.b").is_null(),
))?
.build()?;
assert_optimized_plan_equal!(plan, @r"
Filter: t2.b IN ([UInt32(1), UInt32(2)]) OR t2.b IS NULL
Left Join: t1.a = t2.a
TableScan: t1
TableScan: t2
")
}
#[test]
fn eliminate_left_with_like() -> Result<()> {
let t1 = test_table_scan_with_name("t1")?;
let t2 = test_table_scan_with_name("t2")?;
let plan = LogicalPlanBuilder::from(t1)
.join(
t2,
JoinType::Left,
(vec![Column::from_name("a")], vec![Column::from_name("a")]),
None,
)?
.filter(col("t2.b").like(lit("%pattern%")))?
.build()?;
assert_optimized_plan_equal!(plan, @r#"
Filter: t2.b LIKE Utf8("%pattern%")
Inner Join: t1.a = t2.a
TableScan: t1
TableScan: t2
"#)
}
#[test]
fn eliminate_left_with_like_pattern_column() -> Result<()> {
let t1 = test_table_scan_with_name("t1")?;
let t2 = test_table_scan_with_name("t2")?;
let plan = LogicalPlanBuilder::from(t1)
.join(
t2,
JoinType::Left,
(vec![Column::from_name("a")], vec![Column::from_name("a")]),
None,
)?
.filter(lit("x").like(col("t2.b")))?
.build()?;
assert_optimized_plan_equal!(plan, @r#"
Filter: Utf8("x") LIKE t2.b
Inner Join: t1.a = t2.a
TableScan: t1
TableScan: t2
"#)
}
#[test]
fn eliminate_full_with_like_cross_side() -> Result<()> {
let t1 = test_table_scan_with_name("t1")?;
let t2 = test_table_scan_with_name("t2")?;
let plan = LogicalPlanBuilder::from(t1)
.join(
t2,
JoinType::Full,
(vec![Column::from_name("a")], vec![Column::from_name("a")]),
None,
)?
.filter(col("t1.c").like(col("t2.b")))?
.build()?;
assert_optimized_plan_equal!(plan, @r"
Filter: t1.c LIKE t2.b
Inner Join: t1.a = t2.a
TableScan: t1
TableScan: t2
")
}
#[test]
fn eliminate_left_with_is_true() -> Result<()> {
let t1 = test_table_scan_with_name("t1")?;
let t2 = test_table_scan_with_name("t2")?;
let plan = LogicalPlanBuilder::from(t1)
.join(
t2,
JoinType::Left,
(vec![Column::from_name("a")], vec![Column::from_name("a")]),
None,
)?
.filter(col("t2.b").gt(lit(10u32)).is_true())?
.build()?;
assert_optimized_plan_equal!(plan, @r"
Filter: t2.b > UInt32(10) IS TRUE
Inner Join: t1.a = t2.a
TableScan: t1
TableScan: t2
")
}
#[test]
fn eliminate_left_with_is_false() -> Result<()> {
let t1 = test_table_scan_with_name("t1")?;
let t2 = test_table_scan_with_name("t2")?;
let plan = LogicalPlanBuilder::from(t1)
.join(
t2,
JoinType::Left,
(vec![Column::from_name("a")], vec![Column::from_name("a")]),
None,
)?
.filter(col("t2.b").gt(lit(10u32)).is_false())?
.build()?;
assert_optimized_plan_equal!(plan, @r"
Filter: t2.b > UInt32(10) IS FALSE
Inner Join: t1.a = t2.a
TableScan: t1
TableScan: t2
")
}
#[test]
fn eliminate_left_with_is_not_unknown() -> Result<()> {
let t1 = test_table_scan_with_name("t1")?;
let t2 = test_table_scan_with_name("t2")?;
let plan = LogicalPlanBuilder::from(t1)
.join(
t2,
JoinType::Left,
(vec![Column::from_name("a")], vec![Column::from_name("a")]),
None,
)?
.filter(col("t2.b").gt(lit(10u32)).is_not_unknown())?
.build()?;
assert_optimized_plan_equal!(plan, @r"
Filter: t2.b > UInt32(10) IS NOT UNKNOWN
Inner Join: t1.a = t2.a
TableScan: t1
TableScan: t2
")
}
#[test]
fn no_eliminate_left_with_is_not_true() -> Result<()> {
let t1 = test_table_scan_with_name("t1")?;
let t2 = test_table_scan_with_name("t2")?;
let plan = LogicalPlanBuilder::from(t1)
.join(
t2,
JoinType::Left,
(vec![Column::from_name("a")], vec![Column::from_name("a")]),
None,
)?
.filter(col("t2.b").gt(lit(10u32)).is_not_true())?
.build()?;
assert_optimized_plan_equal!(plan, @r"
Filter: t2.b > UInt32(10) IS NOT TRUE
Left Join: t1.a = t2.a
TableScan: t1
TableScan: t2
")
}
#[test]
fn no_eliminate_left_with_is_unknown() -> Result<()> {
let t1 = test_table_scan_with_name("t1")?;
let t2 = test_table_scan_with_name("t2")?;
let plan = LogicalPlanBuilder::from(t1)
.join(
t2,
JoinType::Left,
(vec![Column::from_name("a")], vec![Column::from_name("a")]),
None,
)?
.filter(col("t2.b").gt(lit(10u32)).is_unknown())?
.build()?;
assert_optimized_plan_equal!(plan, @r"
Filter: t2.b > UInt32(10) IS UNKNOWN
Left Join: t1.a = t2.a
TableScan: t1
TableScan: t2
")
}
#[test]
fn no_eliminate_left_with_not_is_true() -> Result<()> {
let t1 = test_table_scan_with_name("t1")?;
let t2 = test_table_scan_with_name("t2")?;
let plan = LogicalPlanBuilder::from(t1)
.join(
t2,
JoinType::Left,
(vec![Column::from_name("a")], vec![Column::from_name("a")]),
None,
)?
.filter(not(col("t2.b").gt(lit(5u32)).is_true()))?
.build()?;
assert_optimized_plan_equal!(plan, @r"
Filter: NOT t2.b > UInt32(5) IS TRUE
Left Join: t1.a = t2.a
TableScan: t1
TableScan: t2
")
}
#[test]
fn no_eliminate_left_with_not_is_false() -> Result<()> {
let t1 = test_table_scan_with_name("t1")?;
let t2 = test_table_scan_with_name("t2")?;
let plan = LogicalPlanBuilder::from(t1)
.join(
t2,
JoinType::Left,
(vec![Column::from_name("a")], vec![Column::from_name("a")]),
None,
)?
.filter(not(col("t2.b").gt(lit(5u32)).is_false()))?
.build()?;
assert_optimized_plan_equal!(plan, @r"
Filter: NOT t2.b > UInt32(5) IS FALSE
Left Join: t1.a = t2.a
TableScan: t1
TableScan: t2
")
}
#[test]
fn no_eliminate_left_with_not_is_not_unknown() -> Result<()> {
let t1 = test_table_scan_with_name("t1")?;
let t2 = test_table_scan_with_name("t2")?;
let plan = LogicalPlanBuilder::from(t1)
.join(
t2,
JoinType::Left,
(vec![Column::from_name("a")], vec![Column::from_name("a")]),
None,
)?
.filter(not(col("t2.b").gt(lit(5u32)).is_not_unknown()))?
.build()?;
assert_optimized_plan_equal!(plan, @r"
Filter: NOT t2.b > UInt32(5) IS NOT UNKNOWN
Left Join: t1.a = t2.a
TableScan: t1
TableScan: t2
")
}
#[test]
fn eliminate_full_with_type_cast() -> Result<()> {
let t1 = test_table_scan_with_name("t1")?;
let t2 = test_table_scan_with_name("t2")?;
let plan = LogicalPlanBuilder::from(t1)
.join(
t2,
JoinType::Full,
(vec![Column::from_name("a")], vec![Column::from_name("a")]),
None,
)?
.filter(binary_expr(
cast(col("t1.b"), DataType::Int64).gt(lit(10u32)),
And,
try_cast(col("t2.c"), DataType::Int64).lt(lit(20u32)),
))?
.build()?;
assert_optimized_plan_equal!(plan, @r"
Filter: CAST(t1.b AS Int64) > UInt32(10) AND TRY_CAST(t2.c AS Int64) < UInt32(20)
Inner Join: t1.a = t2.a
TableScan: t1
TableScan: t2
")
}
#[test]
fn eliminate_full_to_left_with_left_filter() -> Result<()> {
let t1 = test_table_scan_with_name("t1")?;
let t2 = test_table_scan_with_name("t2")?;
let plan = LogicalPlanBuilder::from(t1)
.join(
t2,
JoinType::Full,
(vec![Column::from_name("a")], vec![Column::from_name("a")]),
None,
)?
.filter(col("t1.b").gt(lit(10u32)))?
.build()?;
assert_optimized_plan_equal!(plan, @r"
Filter: t1.b > UInt32(10)
Left Join: t1.a = t2.a
TableScan: t1
TableScan: t2
")
}
#[test]
fn eliminate_full_to_right_with_right_filter() -> Result<()> {
let t1 = test_table_scan_with_name("t1")?;
let t2 = test_table_scan_with_name("t2")?;
let plan = LogicalPlanBuilder::from(t1)
.join(
t2,
JoinType::Full,
(vec![Column::from_name("a")], vec![Column::from_name("a")]),
None,
)?
.filter(col("t2.b").in_list(vec![lit(1u32), lit(2u32)], false))?
.build()?;
assert_optimized_plan_equal!(plan, @r"
Filter: t2.b IN ([UInt32(1), UInt32(2)])
Right Join: t1.a = t2.a
TableScan: t1
TableScan: t2
")
}
#[test]
fn eliminate_full_to_left_with_like() -> Result<()> {
let t1 = test_table_scan_with_name("t1")?;
let t2 = test_table_scan_with_name("t2")?;
let plan = LogicalPlanBuilder::from(t1)
.join(
t2,
JoinType::Full,
(vec![Column::from_name("a")], vec![Column::from_name("a")]),
None,
)?
.filter(col("t1.b").like(lit("%val%")))?
.build()?;
assert_optimized_plan_equal!(plan, @r#"
Filter: t1.b LIKE Utf8("%val%")
Left Join: t1.a = t2.a
TableScan: t1
TableScan: t2
"#)
}
#[test]
fn eliminate_full_to_right_with_is_true() -> Result<()> {
let t1 = test_table_scan_with_name("t1")?;
let t2 = test_table_scan_with_name("t2")?;
let plan = LogicalPlanBuilder::from(t1)
.join(
t2,
JoinType::Full,
(vec![Column::from_name("a")], vec![Column::from_name("a")]),
None,
)?
.filter(col("t2.b").gt(lit(10u32)).is_true())?
.build()?;
assert_optimized_plan_equal!(plan, @r"
Filter: t2.b > UInt32(10) IS TRUE
Right Join: t1.a = t2.a
TableScan: t1
TableScan: t2
")
}
#[test]
fn eliminate_left_with_and_multiple_null_rejecting() -> Result<()> {
let t1 = test_table_scan_with_name("t1")?;
let t2 = test_table_scan_with_name("t2")?;
let plan = LogicalPlanBuilder::from(t1)
.join(
t2,
JoinType::Left,
(vec![Column::from_name("a")], vec![Column::from_name("a")]),
None,
)?
.filter(binary_expr(
col("t2.b").in_list(vec![lit(1u32), lit(2u32)], false),
And,
col("t2.c").between(lit(5u32), lit(20u32)),
))?
.build()?;
assert_optimized_plan_equal!(plan, @r"
Filter: t2.b IN ([UInt32(1), UInt32(2)]) AND t2.c BETWEEN UInt32(5) AND UInt32(20)
Inner Join: t1.a = t2.a
TableScan: t1
TableScan: t2
")
}
#[test]
fn eliminate_left_with_or_same_side() -> Result<()> {
let t1 = test_table_scan_with_name("t1")?;
let t2 = test_table_scan_with_name("t2")?;
let plan = LogicalPlanBuilder::from(t1)
.join(
t2,
JoinType::Left,
(vec![Column::from_name("a")], vec![Column::from_name("a")]),
None,
)?
.filter(binary_expr(
col("t2.b").gt(lit(10u32)),
Or,
col("t2.c").lt(lit(20u32)),
))?
.build()?;
assert_optimized_plan_equal!(plan, @r"
Filter: t2.b > UInt32(10) OR t2.c < UInt32(20)
Inner Join: t1.a = t2.a
TableScan: t1
TableScan: t2
")
}
#[test]
fn no_eliminate_left_with_or_cross_side() -> Result<()> {
let t1 = test_table_scan_with_name("t1")?;
let t2 = test_table_scan_with_name("t2")?;
let plan = LogicalPlanBuilder::from(t1)
.join(
t2,
JoinType::Left,
(vec![Column::from_name("a")], vec![Column::from_name("a")]),
None,
)?
.filter(binary_expr(
col("t1.b").gt(lit(10u32)),
Or,
col("t2.b").lt(lit(20u32)),
))?
.build()?;
assert_optimized_plan_equal!(plan, @r"
Filter: t1.b > UInt32(10) OR t2.b < UInt32(20)
Left Join: t1.a = t2.a
TableScan: t1
TableScan: t2
")
}
#[test]
fn eliminate_full_with_mixed_predicates() -> Result<()> {
let t1 = test_table_scan_with_name("t1")?;
let t2 = test_table_scan_with_name("t2")?;
let plan = LogicalPlanBuilder::from(t1)
.join(
t2,
JoinType::Full,
(vec![Column::from_name("a")], vec![Column::from_name("a")]),
None,
)?
.filter(binary_expr(
col("t1.b").like(lit("%pattern%")),
And,
col("t2.b").between(lit(1u32), lit(10u32)),
))?
.build()?;
assert_optimized_plan_equal!(plan, @r#"
Filter: t1.b LIKE Utf8("%pattern%") AND t2.b BETWEEN UInt32(1) AND UInt32(10)
Inner Join: t1.a = t2.a
TableScan: t1
TableScan: t2
"#)
}
#[test]
fn eliminate_left_with_is_true_and_in_list() -> Result<()> {
let t1 = test_table_scan_with_name("t1")?;
let t2 = test_table_scan_with_name("t2")?;
let plan = LogicalPlanBuilder::from(t1)
.join(
t2,
JoinType::Left,
(vec![Column::from_name("a")], vec![Column::from_name("a")]),
None,
)?
.filter(binary_expr(
col("t2.b").gt(lit(5u32)).is_true(),
And,
col("t2.c").in_list(vec![lit(1u32), lit(2u32)], false),
))?
.build()?;
assert_optimized_plan_equal!(plan, @r"
Filter: t2.b > UInt32(5) IS TRUE AND t2.c IN ([UInt32(1), UInt32(2)])
Inner Join: t1.a = t2.a
TableScan: t1
TableScan: t2
")
}
#[test]
fn eliminate_left_through_projection() -> Result<()> {
let t1 = test_table_scan_with_name("t1")?;
let t2 = test_table_scan_with_name("t2")?;
let plan = LogicalPlanBuilder::from(t1)
.join(
t2,
JoinType::Left,
(vec![Column::from_name("a")], vec![Column::from_name("a")]),
None,
)?
.project(vec![col("t1.a"), col("t2.b").alias("bb")])?
.filter(col("bb").gt(lit(10u32)))?
.build()?;
assert_optimized_plan_equal!(plan, @r"
Filter: bb > UInt32(10)
Projection: t1.a, t2.b AS bb
Inner Join: t1.a = t2.a
TableScan: t1
TableScan: t2
")
}
#[test]
fn no_eliminate_left_through_projection_with_or_cross_side() -> Result<()> {
let t1 = test_table_scan_with_name("t1")?;
let t2 = test_table_scan_with_name("t2")?;
let plan = LogicalPlanBuilder::from(t1)
.join(
t2,
JoinType::Left,
(vec![Column::from_name("a")], vec![Column::from_name("a")]),
None,
)?
.project(vec![col("t1.b").alias("x"), col("t2.b").alias("y")])?
.filter(binary_expr(
col("x").gt(lit(10u32)),
Or,
col("y").lt(lit(20u32)),
))?
.build()?;
assert_optimized_plan_equal!(plan, @r"
Filter: x > UInt32(10) OR y < UInt32(20)
Projection: t1.b AS x, t2.b AS y
Left Join: t1.a = t2.a
TableScan: t1
TableScan: t2
")
}
#[test]
fn no_eliminate_left_through_projection_with_only_left_filter() -> Result<()> {
let t1 = test_table_scan_with_name("t1")?;
let t2 = test_table_scan_with_name("t2")?;
let plan = LogicalPlanBuilder::from(t1)
.join(
t2,
JoinType::Left,
(vec![Column::from_name("a")], vec![Column::from_name("a")]),
None,
)?
.project(vec![col("t1.b").alias("x"), col("t2.b")])?
.filter(col("x").gt(lit(10u32)))?
.build()?;
assert_optimized_plan_equal!(plan, @r"
Filter: x > UInt32(10)
Projection: t1.b AS x, t2.b
Left Join: t1.a = t2.a
TableScan: t1
TableScan: t2
")
}
#[test]
fn eliminate_left_with_arithmetic_predicate() -> Result<()> {
let t1 = test_table_scan_with_name("t1")?;
let t2 = test_table_scan_with_name("t2")?;
let plan = LogicalPlanBuilder::from(t1)
.join(
t2,
JoinType::Left,
(vec![Column::from_name("a")], vec![Column::from_name("a")]),
None,
)?
.filter(
binary_expr(
binary_expr(col("t2.b"), Operator::Multiply, lit(2u32)),
Operator::Plus,
lit(1u32),
)
.gt(lit(10u32)),
)?
.build()?;
assert_optimized_plan_equal!(plan, @r"
Filter: t2.b * UInt32(2) + UInt32(1) > UInt32(10)
Inner Join: t1.a = t2.a
TableScan: t1
TableScan: t2
")
}
#[test]
fn eliminate_left_with_negative_predicate() -> Result<()> {
let t1 = test_table_scan_with_name("t1")?;
let t2 = test_table_scan_with_name("t2")?;
let plan = LogicalPlanBuilder::from(t1)
.join(
t2,
JoinType::Left,
(vec![Column::from_name("a")], vec![Column::from_name("a")]),
None,
)?
.filter(Expr::Negative(Box::new(col("t2.b"))).gt(lit(0u32)))?
.build()?;
assert_optimized_plan_equal!(plan, @r"
Filter: (- t2.b) > UInt32(0)
Inner Join: t1.a = t2.a
TableScan: t1
TableScan: t2
")
}
#[test]
fn no_eliminate_left_with_is_distinct_from() -> Result<()> {
let t1 = test_table_scan_with_name("t1")?;
let t2 = test_table_scan_with_name("t2")?;
let plan = LogicalPlanBuilder::from(t1)
.join(
t2,
JoinType::Left,
(vec![Column::from_name("a")], vec![Column::from_name("a")]),
None,
)?
.filter(binary_expr(
col("t2.b"),
Operator::IsDistinctFrom,
lit(5u32),
))?
.build()?;
assert_optimized_plan_equal!(plan, @r"
Filter: t2.b IS DISTINCT FROM UInt32(5)
Left Join: t1.a = t2.a
TableScan: t1
TableScan: t2
")
}
#[test]
fn no_eliminate_left_with_is_not_distinct_from() -> Result<()> {
let t1 = test_table_scan_with_name("t1")?;
let t2 = test_table_scan_with_name("t2")?;
let plan = LogicalPlanBuilder::from(t1)
.join(
t2,
JoinType::Left,
(vec![Column::from_name("a")], vec![Column::from_name("a")]),
None,
)?
.filter(binary_expr(
col("t2.b"),
Operator::IsNotDistinctFrom,
lit(ScalarValue::UInt32(None)),
))?
.build()?;
assert_optimized_plan_equal!(plan, @r"
Filter: t2.b IS NOT DISTINCT FROM UInt32(NULL)
Left Join: t1.a = t2.a
TableScan: t1
TableScan: t2
")
}
#[test]
fn no_eliminate_through_non_transparent() -> Result<()> {
let t1 = test_table_scan_with_name("t1")?;
let t2 = test_table_scan_with_name("t2")?;
let plan = LogicalPlanBuilder::from(t1)
.join(
t2,
JoinType::Left,
(vec![Column::from_name("a")], vec![Column::from_name("a")]),
None,
)?
.limit(0, Some(5))?
.filter(col("t2.b").gt(lit(10u32)))?
.build()?;
assert_optimized_plan_equal!(plan, @r"
Filter: t2.b > UInt32(10)
Limit: skip=0, fetch=5
Left Join: t1.a = t2.a
TableScan: t1
TableScan: t2
")
}
}