datafusion_physical_optimizer/
aggregate_statistics.rs

1// Licensed to the Apache Software Foundation (ASF) under one
2// or more contributor license agreements.  See the NOTICE file
3// distributed with this work for additional information
4// regarding copyright ownership.  The ASF licenses this file
5// to you under the Apache License, Version 2.0 (the
6// "License"); you may not use this file except in compliance
7// with the License.  You may obtain a copy of the License at
8//
9//   http://www.apache.org/licenses/LICENSE-2.0
10//
11// Unless required by applicable law or agreed to in writing,
12// software distributed under the License is distributed on an
13// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14// KIND, either express or implied.  See the License for the
15// specific language governing permissions and limitations
16// under the License.
17
18//! Utilizing exact statistics from sources to avoid scanning data
19use datafusion_common::Result;
20use datafusion_common::config::ConfigOptions;
21use datafusion_common::scalar::ScalarValue;
22use datafusion_common::tree_node::{Transformed, TransformedResult, TreeNode};
23use datafusion_physical_plan::aggregates::{
24    AggregateExec, AggregateInputMode, AggregateMode,
25};
26use datafusion_physical_plan::placeholder_row::PlaceholderRowExec;
27use datafusion_physical_plan::projection::{ProjectionExec, ProjectionExpr};
28use datafusion_physical_plan::statistics::{StatisticsArgs, StatisticsContext};
29use datafusion_physical_plan::udaf::{
30    AggregateFunctionExpr, StatisticsArgs as PlanStatisticsArgs,
31};
32use datafusion_physical_plan::{ExecutionPlan, expressions};
33use std::sync::Arc;
34
35use crate::PhysicalOptimizerRule;
36
37/// Optimizer that uses available statistics for aggregate functions
38#[derive(Default, Debug)]
39pub struct AggregateStatistics {}
40
41impl AggregateStatistics {
42    #[expect(missing_docs)]
43    pub fn new() -> Self {
44        Self {}
45    }
46}
47
48impl PhysicalOptimizerRule for AggregateStatistics {
49    #[cfg_attr(feature = "recursive_protection", recursive::recursive)]
50    #[expect(clippy::allow_attributes)] // See https://github.com/apache/datafusion/issues/18881#issuecomment-3621545670
51    #[allow(clippy::only_used_in_recursion)] // See https://github.com/rust-lang/rust-clippy/issues/14566
52    fn optimize(
53        &self,
54        plan: Arc<dyn ExecutionPlan>,
55        config: &ConfigOptions,
56    ) -> Result<Arc<dyn ExecutionPlan>> {
57        if let Some(partial_agg_exec) = take_optimizable(&plan) {
58            let partial_agg_exec = partial_agg_exec
59                .downcast_ref::<AggregateExec>()
60                .expect("take_optimizable() ensures that this is a AggregateExec");
61            let stats = StatisticsContext::new()
62                .compute(partial_agg_exec.input().as_ref(), &StatisticsArgs::new())?;
63            let mut projections = vec![];
64            for expr in partial_agg_exec.aggr_expr() {
65                let field = expr.field();
66                let args = expr.expressions();
67                let statistics_args = PlanStatisticsArgs {
68                    statistics: &stats,
69                    return_type: field.data_type(),
70                    is_distinct: expr.is_distinct(),
71                    exprs: args.as_slice(),
72                };
73                if let Some((optimizable_statistic, name)) =
74                    take_optimizable_value_from_statistics(&statistics_args, expr)
75                {
76                    projections.push(ProjectionExpr {
77                        expr: expressions::lit(optimizable_statistic),
78                        alias: name.to_owned(),
79                    });
80                } else {
81                    // TODO: we need all aggr_expr to be resolved (cf TODO fullres)
82                    break;
83                }
84            }
85
86            // TODO fullres: use statistics even if not all aggr_expr could be resolved
87            if projections.len() == partial_agg_exec.aggr_expr().len() {
88                // input can be entirely removed
89                Ok(Arc::new(ProjectionExec::try_new(
90                    projections,
91                    Arc::new(PlaceholderRowExec::new(plan.schema())),
92                )?))
93            } else {
94                plan.map_children(|child| {
95                    self.optimize(child, config).map(Transformed::yes)
96                })
97                .data()
98            }
99        } else {
100            plan.map_children(|child| self.optimize(child, config).map(Transformed::yes))
101                .data()
102        }
103    }
104
105    fn name(&self) -> &str {
106        "aggregate_statistics"
107    }
108
109    /// This rule will change the nullable properties of the schema, disable the schema check.
110    fn schema_check(&self) -> bool {
111        false
112    }
113}
114
115/// Returns an `AggregateExec` whose statistics can replace the aggregate with
116/// literal values: either a `Single`/`SinglePartitioned` aggregate, or a
117/// `Final` aggregate wrapping a `Partial`. Must have no GROUP BY and no
118/// filters.
119fn take_optimizable(plan: &Arc<dyn ExecutionPlan>) -> Option<Arc<dyn ExecutionPlan>> {
120    let agg_exec = plan.downcast_ref::<AggregateExec>()?;
121
122    if matches!(
123        agg_exec.mode(),
124        AggregateMode::Single | AggregateMode::SinglePartitioned
125    ) && agg_exec.group_expr().is_empty()
126        && agg_exec.filter_expr().iter().all(|e| e.is_none())
127    {
128        return Some(Arc::clone(plan));
129    }
130
131    if agg_exec.mode().input_mode() == AggregateInputMode::Partial
132        && agg_exec.group_expr().is_empty()
133    {
134        let mut child = Arc::clone(agg_exec.input());
135        loop {
136            if let Some(partial_agg_exec) = child.downcast_ref::<AggregateExec>()
137                && partial_agg_exec.mode().input_mode() == AggregateInputMode::Raw
138                && partial_agg_exec.group_expr().is_empty()
139                && partial_agg_exec.filter_expr().iter().all(|e| e.is_none())
140            {
141                return Some(child);
142            }
143            if let [childrens_child] = child.children().as_slice() {
144                child = Arc::clone(childrens_child);
145            } else {
146                break;
147            }
148        }
149    }
150    None
151}
152
153/// If this agg_expr is a max that is exactly defined in the statistics, return it.
154fn take_optimizable_value_from_statistics(
155    statistics_args: &PlanStatisticsArgs,
156    agg_expr: &AggregateFunctionExpr,
157) -> Option<(ScalarValue, String)> {
158    let value = agg_expr.fun().value_from_stats(statistics_args);
159    value.map(|val| (val, agg_expr.name().to_string()))
160}
161
162// See tests in datafusion/core/tests/physical_optimizer
datafusion_physical_optimizer/aggregate_statistics.rs

datafusion_physical_optimizer/
aggregate_statistics.rs
