use std::collections::{HashMap, HashSet};
use std::fmt;
use glaredb_error::{DbError, Result};
use super::ReorderableCondition;
use super::edge::{EdgeId, EdgeType, HyperEdges, NeighborEdge};
use super::statistics::propagate_estimated_cardinality;
use super::subgraph::Subgraph;
use crate::expr::Expression;
use crate::expr::conjunction_expr::{ConjunctionExpr, ConjunctionOperator};
use crate::logical::binder::bind_context::BindContext;
use crate::logical::binder::table_list::TableRef;
use crate::logical::logical_filter::LogicalFilter;
use crate::logical::logical_join::{
JoinType,
LogicalArbitraryJoin,
LogicalComparisonJoin,
LogicalCrossJoin,
};
use crate::logical::operator::{LocationRequirement, LogicalNode, LogicalOperator, Node};
use crate::optimizer::filter_pushdown::extracted_filter::ExtractedFilter;
use crate::statistics::assumptions::DEFAULT_SELECTIVITY;
use crate::statistics::value::StatisticsValue;
use crate::util::fmt::displayable::IntoDisplayableSlice;
pub type RelId = usize;
pub type FilterId = usize;
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct UsedFilters {
pub filters: HashSet<FilterId>,
}
impl UsedFilters {
fn unioned(left: &UsedFilters, right: &UsedFilters) -> Self {
UsedFilters {
filters: left
.filters
.iter()
.chain(right.filters.iter())
.copied()
.collect(),
}
}
fn mark_filters_used(&mut self, filters: impl IntoIterator<Item = FilterId>) {
self.filters.extend(filters)
}
}
#[derive(Debug)]
pub struct BaseRelation {
pub rel_id: RelId,
pub operator: LogicalOperator,
pub output_refs: HashSet<TableRef>,
pub cardinality: f64,
}
#[derive(Debug, Clone)]
pub struct JoinNode {
pub set: RelationSet,
pub cost: f64,
pub left: RelationSet,
pub right: RelationSet,
pub subgraph: Subgraph,
pub output_refs: HashSet<TableRef>,
pub edges: HashSet<EdgeId>,
pub left_filters: HashSet<FilterId>,
pub right_filters: HashSet<FilterId>,
pub used: UsedFilters,
}
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
pub struct RelationSet {
pub relation_indices: Vec<usize>,
}
impl RelationSet {
fn new(indices: impl IntoIterator<Item = usize>) -> Self {
let mut indices: Vec<_> = indices.into_iter().collect();
indices.sort_unstable();
RelationSet {
relation_indices: indices,
}
}
fn empty() -> Self {
RelationSet {
relation_indices: Vec::new(),
}
}
fn base(idx: usize) -> Self {
RelationSet {
relation_indices: vec![idx],
}
}
fn is_base(&self) -> bool {
self.relation_indices.len() == 1
}
fn union(left: &RelationSet, right: &RelationSet) -> Self {
let mut indices: Vec<_> = left
.relation_indices
.iter()
.chain(right.relation_indices.iter())
.copied()
.collect();
indices.sort_unstable();
indices.dedup();
RelationSet {
relation_indices: indices,
}
}
fn get_all_neighbor_sets(mut neighbors: Vec<usize>) -> Vec<RelationSet> {
fn add_supersets(current: &[HashSet<usize>], neighbors: &[usize]) -> Vec<HashSet<usize>> {
let mut added = Vec::new();
for neighbor_set in current {
let max = neighbor_set.iter().max().unwrap();
for &neighbor in neighbors {
if *max >= neighbor {
continue;
}
if !neighbor_set.contains(&neighbor) {
let mut new_set = neighbor_set.clone();
new_set.insert(neighbor);
added.push(new_set);
}
}
}
added
}
let mut sets = Vec::new();
neighbors.sort();
let mut added = Vec::new();
for &neighbor in &neighbors {
let mut set = HashSet::new();
set.insert(neighbor);
added.push(set.clone());
sets.push(set);
}
while !added.is_empty() {
added = add_supersets(&added, &neighbors);
for d in &added {
sets.push(d.clone());
}
}
sets.into_iter().map(RelationSet::new).collect()
}
}
impl fmt::Display for RelationSet {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "{}", self.relation_indices.display_with_brackets())
}
}
#[derive(Debug)]
pub struct Graph {
hyper_edges: HyperEdges,
filters: HashMap<FilterId, ExtractedFilter>,
base_relations: HashMap<RelId, BaseRelation>,
best_plans: HashMap<RelationSet, JoinNode>,
pairs_considered: usize,
}
impl Graph {
pub fn try_new(
base_ops: impl IntoIterator<Item = LogicalOperator>,
conditions: impl IntoIterator<Item = ReorderableCondition>,
filters: impl IntoIterator<Item = ExtractedFilter>,
bind_context: &BindContext,
) -> Result<Self> {
let base_ops = base_ops
.into_iter()
.map(|mut op| {
propagate_estimated_cardinality(&mut op)?;
Ok(op)
})
.collect::<Result<Vec<_>>>()?;
let base_relations: HashMap<RelId, BaseRelation> = base_ops
.into_iter()
.enumerate()
.map(|(rel_id, op)| {
let output_refs = op.get_output_table_refs(bind_context).into_iter().collect();
let cardinality = op
.estimated_cardinality()
.value()
.copied()
.unwrap_or(20_000) as f64;
(
rel_id,
BaseRelation {
rel_id,
operator: op,
output_refs,
cardinality,
},
)
})
.collect();
let hyper_edges = HyperEdges::new(conditions, &base_relations)?;
let filters = filters.into_iter().enumerate().collect();
let mut best_plans = HashMap::with_capacity(base_relations.len());
for (&rel_id, base_rel) in &base_relations {
let subgraph = Subgraph {
numerator: base_rel.cardinality,
selectivity_denom: 1.0,
};
let rel_set = RelationSet::base(rel_id);
let node = JoinNode {
set: rel_set.clone(),
cost: 0.0,
left: RelationSet::empty(),
right: RelationSet::empty(),
subgraph,
output_refs: base_rel.output_refs.clone(),
edges: HashSet::new(),
left_filters: HashSet::new(),
right_filters: HashSet::new(),
used: UsedFilters::default(),
};
best_plans.insert(rel_set, node);
}
Ok(Graph {
hyper_edges,
filters,
best_plans,
base_relations,
pairs_considered: 0,
})
}
pub fn try_build(mut self) -> Result<LogicalOperator> {
self.solve()?;
let longest_set = RelationSet::new(0..self.base_relations.len());
if !self.best_plans.contains_key(&longest_set) {
for i in 0..self.base_relations.len() {
for j in 0..self.base_relations.len() {
if i == j {
continue;
}
let left = self.base_relations.get(&i).unwrap();
let right = self.base_relations.get(&j).unwrap();
self.hyper_edges.insert_cross_product(left, right);
}
}
self.solve()?;
}
let longest = self
.best_plans
.remove(&longest_set)
.ok_or_else(|| DbError::new("Missing longest best plan"))?;
let plan = self.build_from_generated(longest)?;
assert!(self.base_relations.is_empty());
assert!(self.hyper_edges.all_non_empty_edges_removed());
let filter_ids: HashSet<_> = self.filters.keys().copied().collect();
let plan = self.apply_filters(plan, &filter_ids)?;
assert!(self.filters.is_empty());
Ok(plan)
}
fn solve(&mut self) -> Result<()> {
for base_idx in (0..self.base_relations.len()).rev() {
let base_rel = RelationSet::base(base_idx);
self.emit_connected_subgraphs(&base_rel)?;
let exclude: HashSet<_> = (0..base_idx).collect();
self.enumerate_connected_subgraphs_rec(&base_rel, &exclude)?;
}
Ok(())
}
fn emit_connected_subgraphs(&mut self, set: &RelationSet) -> Result<()> {
if set.relation_indices.len() == self.base_relations.len() {
return Ok(());
}
let mut exclude: HashSet<_> = (0..set.relation_indices[0]).collect();
for idx in &set.relation_indices {
exclude.insert(*idx);
}
let mut neighbors = self.hyper_edges.find_neighbors(set, &exclude);
neighbors.sort_unstable_by(|a, b| a.cmp(b).reverse());
exclude.extend(&neighbors);
for neighbor in neighbors {
let neighbor_set = RelationSet::base(neighbor);
let edges = self.hyper_edges.find_edges(set, &neighbor_set);
if !edges.is_empty() {
self.emit_pair(set, &neighbor_set, edges)?;
}
self.enumerate_connected_complement_rec(set, &neighbor_set, &exclude)?;
exclude.remove(&neighbor);
}
Ok(())
}
fn enumerate_connected_subgraphs_rec(
&mut self,
set: &RelationSet,
exclude: &HashSet<usize>,
) -> Result<()> {
let neighbors = self.hyper_edges.find_neighbors(set, exclude);
if neighbors.is_empty() {
return Ok(());
}
let neighbor_sets = RelationSet::get_all_neighbor_sets(neighbors.clone());
let mut combined_sets = Vec::with_capacity(neighbor_sets.len());
for neigbor_set in neighbor_sets {
let combined = RelationSet::union(set, &neigbor_set);
if self.best_plans.contains_key(&combined) {
self.emit_connected_subgraphs(&combined)?;
}
combined_sets.push(combined);
}
let mut exclude = exclude.clone();
exclude.extend(neighbors);
for combined in combined_sets {
self.enumerate_connected_subgraphs_rec(&combined, &exclude)?;
}
Ok(())
}
fn enumerate_connected_complement_rec(
&mut self,
left: &RelationSet,
right: &RelationSet,
exclude: &HashSet<usize>,
) -> Result<()> {
let neighbors = self.hyper_edges.find_neighbors(right, exclude);
if neighbors.is_empty() {
return Ok(());
}
let neighbor_sets = RelationSet::get_all_neighbor_sets(neighbors.clone());
let mut combined_sets = Vec::with_capacity(neighbor_sets.len());
for neigbor_set in neighbor_sets {
let combined = RelationSet::union(right, &neigbor_set);
assert!(combined.relation_indices.len() > right.relation_indices.len());
if self.best_plans.contains_key(&combined) {
let edges = self.hyper_edges.find_edges(left, &combined);
if !edges.is_empty() {
self.emit_pair(left, &combined, edges)?;
}
}
combined_sets.push(combined);
}
let mut exclude = exclude.clone();
exclude.extend(neighbors);
for combined in combined_sets {
self.enumerate_connected_complement_rec(left, &combined, &exclude)?;
}
Ok(())
}
fn emit_pair(
&mut self,
left: &RelationSet,
right: &RelationSet,
edges: Vec<NeighborEdge>,
) -> Result<()> {
self.pairs_considered += 1;
let left = self
.best_plans
.get_key_value(left)
.ok_or_else(|| DbError::new("missing best plan for left"))?;
let right = self
.best_plans
.get_key_value(right)
.ok_or_else(|| DbError::new("missing best plan for right"))?;
let new_set = RelationSet::union(left.0, right.0);
let left_filters = self.find_filters(left.1);
let right_filters = self.find_filters(right.1);
let mut subgraph = left.1.subgraph;
let any_semi = edges
.iter()
.any(|edge| matches!(edge.edge_op, EdgeType::Semi));
if !any_semi {
subgraph.update_numerator(&right.1.subgraph);
}
for _ in 0..left_filters.len() + right_filters.len() {
subgraph.numerator *= DEFAULT_SELECTIVITY;
}
let edge = edges
.iter()
.max_by(|a, b| f64::total_cmp(&a.min_ndv, &b.min_ndv));
if let Some(edge) = edge {
subgraph.update_denom(&right.1.subgraph, edge);
}
let cardinality = subgraph.estimated_cardinality();
let cost = cardinality + left.1.cost + right.1.cost;
if let Some(existing) = self.best_plans.get(&new_set) {
if existing.cost < cost {
return Ok(());
}
}
let left_filters: HashSet<_> = left_filters.iter().map(|&(&id, _)| id).collect();
let right_filters: HashSet<_> = right_filters.iter().map(|&(&id, _)| id).collect();
let edges: HashSet<_> = edges.iter().map(|edge| edge.edge_id).collect();
let mut used = UsedFilters::unioned(&left.1.used, &right.1.used);
used.mark_filters_used(left_filters.iter().copied());
used.mark_filters_used(right_filters.iter().copied());
let output_refs: HashSet<_> = left
.1
.output_refs
.iter()
.chain(&right.1.output_refs)
.copied()
.collect();
self.best_plans.insert(
new_set.clone(),
JoinNode {
set: new_set,
cost,
left: left.0.clone(),
right: right.0.clone(),
subgraph,
output_refs,
edges,
left_filters,
right_filters,
used,
},
);
Ok(())
}
fn find_filters(&self, node: &JoinNode) -> Vec<(&FilterId, &ExtractedFilter)> {
self.filters
.iter()
.filter(|(filter_id, filter)| {
if filter.table_refs.is_empty() {
return false;
}
if node.used.filters.contains(filter_id) {
return false;
}
if !filter.table_refs.is_subset(&node.output_refs) {
return false;
}
true
})
.collect()
}
fn apply_filters(
&mut self,
input: LogicalOperator,
filters: &HashSet<FilterId>,
) -> Result<LogicalOperator> {
if filters.is_empty() {
return Ok(input);
}
let mut input_filters = Vec::with_capacity(filters.len());
for filter_id in filters {
let filter = self
.filters
.remove(filter_id)
.ok_or_else(|| DbError::new(format!("Filter previously used: {filter_id}")))?;
input_filters.push(filter.filter);
}
match input {
LogicalOperator::Filter(filter) => {
let filter_expr = Expression::Conjunction(ConjunctionExpr {
op: ConjunctionOperator::And,
expressions: input_filters
.into_iter()
.chain([filter.node.filter])
.collect(),
});
Ok(LogicalOperator::Filter(Node {
node: LogicalFilter {
filter: filter_expr,
},
location: filter.location,
children: filter.children,
estimated_cardinality: StatisticsValue::Unknown,
}))
}
LogicalOperator::ArbitraryJoin(join) if join.node.join_type == JoinType::Inner => {
let condition = Expression::Conjunction(ConjunctionExpr {
op: ConjunctionOperator::And,
expressions: input_filters
.into_iter()
.chain([join.node.condition])
.collect(),
});
Ok(LogicalOperator::ArbitraryJoin(Node {
node: LogicalArbitraryJoin {
join_type: JoinType::Inner,
condition,
},
location: join.location,
children: join.children,
estimated_cardinality: StatisticsValue::Unknown,
}))
}
LogicalOperator::CrossJoin(join) => {
let condition = Expression::Conjunction(ConjunctionExpr {
op: ConjunctionOperator::And,
expressions: input_filters,
});
Ok(LogicalOperator::ArbitraryJoin(Node {
node: LogicalArbitraryJoin {
join_type: JoinType::Inner,
condition,
},
location: join.location,
children: join.children,
estimated_cardinality: StatisticsValue::Unknown,
}))
}
other => {
let filter = Expression::Conjunction(ConjunctionExpr {
op: ConjunctionOperator::And,
expressions: input_filters,
});
Ok(LogicalOperator::Filter(Node {
node: LogicalFilter { filter },
location: LocationRequirement::Any,
children: vec![other],
estimated_cardinality: StatisticsValue::Unknown,
}))
}
}
}
fn build_from_generated(&mut self, mut node: JoinNode) -> Result<LogicalOperator> {
if node.set.is_base() {
assert!(node.edges.is_empty());
let rel = self
.base_relations
.remove(&node.set.relation_indices[0])
.ok_or_else(|| DbError::new("Missing base relation"))?;
return Ok(rel.operator);
}
let mut left_gen = self
.best_plans
.remove(&node.left)
.ok_or_else(|| DbError::new("Missing left input"))?;
let mut right_gen = self
.best_plans
.remove(&node.right)
.ok_or_else(|| DbError::new("Missing right input"))?;
let mut any_semi = false;
for &edge_id in &node.edges {
let edge = self
.hyper_edges
.get_edge(edge_id)
.ok_or_else(|| DbError::new("Missing edge"))?;
let mut node_flipped = false;
if let Some(cond @ ReorderableCondition::Semi { .. }) = &edge.filter {
let [left_refs, _right_refs] = cond.get_left_right_table_refs();
if !left_refs.is_subset(&left_gen.output_refs) {
std::mem::swap(&mut left_gen, &mut right_gen);
node_flipped = true;
}
any_semi = true;
}
if node_flipped {
std::mem::swap(&mut node.left, &mut node.right);
std::mem::swap(&mut node.left_filters, &mut node.right_filters);
}
}
let mut conditions = Vec::with_capacity(node.edges.len());
for &edge_id in &node.edges {
let edge = self
.hyper_edges
.remove_edge(edge_id)
.ok_or_else(|| DbError::new("Edge already used"))?;
let condition = match edge.filter {
Some(filter) => filter,
None => {
continue;
}
};
match condition {
ReorderableCondition::Inner { mut condition } => {
let condition_swap_sides = edge.left_refs.is_subset(&right_gen.output_refs);
if condition_swap_sides {
condition.flip_sides();
}
conditions.push(condition);
}
ReorderableCondition::Semi {
conditions: mut semi_conditions,
} => {
let condition_swap_sides = edge.left_refs.is_subset(&right_gen.output_refs);
if condition_swap_sides {
for condition in &mut semi_conditions {
condition.flip_sides();
}
}
conditions.append(&mut semi_conditions);
}
}
}
let plan_swap_sides = (!any_semi)
&& right_gen.subgraph.estimated_cardinality()
< left_gen.subgraph.estimated_cardinality();
let left = self.build_from_generated(left_gen)?;
let right = self.build_from_generated(right_gen)?;
let left = self.apply_filters(left, &node.left_filters)?;
let right = self.apply_filters(right, &node.right_filters)?;
let [left, right] = if plan_swap_sides {
[right, left]
} else {
[left, right]
};
if plan_swap_sides {
for cond in &mut conditions {
cond.flip_sides();
}
}
if conditions.is_empty() {
Ok(LogicalOperator::CrossJoin(Node {
node: LogicalCrossJoin,
location: LocationRequirement::Any,
children: vec![left, right],
estimated_cardinality: StatisticsValue::Unknown,
}))
} else {
let join_type = if any_semi {
JoinType::LeftSemi
} else {
JoinType::Inner
};
Ok(LogicalOperator::ComparisonJoin(Node {
node: LogicalComparisonJoin {
join_type,
conditions,
},
location: LocationRequirement::Any,
children: vec![left, right],
estimated_cardinality: StatisticsValue::Estimated(
node.subgraph.estimated_cardinality() as usize,
),
}))
}
}
}