datafusion_comet_spark_expr/agg_funcs/
stddev.rs

1// Licensed to the Apache Software Foundation (ASF) under one
2// or more contributor license agreements.  See the NOTICE file
3// distributed with this work for additional information
4// regarding copyright ownership.  The ASF licenses this file
5// to you under the Apache License, Version 2.0 (the
6// "License"); you may not use this file except in compliance
7// with the License.  You may obtain a copy of the License at
8//
9//   http://www.apache.org/licenses/LICENSE-2.0
10//
11// Unless required by applicable law or agreed to in writing,
12// software distributed under the License is distributed on an
13// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14// KIND, either express or implied.  See the License for the
15// specific language governing permissions and limitations
16// under the License.
17
18use std::{any::Any, sync::Arc};
19
20use crate::agg_funcs::variance::VarianceAccumulator;
21use arrow::datatypes::FieldRef;
22use arrow::{
23    array::ArrayRef,
24    datatypes::{DataType, Field},
25};
26use datafusion::common::types::NativeType;
27use datafusion::common::{internal_err, Result, ScalarValue};
28use datafusion::logical_expr::function::{AccumulatorArgs, StateFieldsArgs};
29use datafusion::logical_expr::{Accumulator, AggregateUDFImpl, Coercion, Signature, Volatility};
30use datafusion::logical_expr_common::signature;
31use datafusion::physical_expr::expressions::format_state_name;
32use datafusion::physical_expr::expressions::StatsType;
33
34/// STDDEV and STDDEV_SAMP (standard deviation) aggregate expression
35/// The implementation mostly is the same as the DataFusion's implementation. The reason
36/// we have our own implementation is that DataFusion has UInt64 for state_field `count`,
37/// while Spark has Double for count. Also we have added `null_on_divide_by_zero`
38/// to be consistent with Spark's implementation.
39#[derive(Debug)]
40pub struct Stddev {
41    name: String,
42    signature: Signature,
43    stats_type: StatsType,
44    null_on_divide_by_zero: bool,
45}
46
47impl Stddev {
48    /// Create a new STDDEV aggregate function
49    pub fn new(
50        name: impl Into<String>,
51        data_type: DataType,
52        stats_type: StatsType,
53        null_on_divide_by_zero: bool,
54    ) -> Self {
55        // the result of stddev just support FLOAT64.
56        assert!(matches!(data_type, DataType::Float64));
57        Self {
58            name: name.into(),
59            signature: Signature::coercible(
60                vec![Coercion::new_exact(signature::TypeSignatureClass::Native(
61                    Arc::new(NativeType::Float64),
62                ))],
63                Volatility::Immutable,
64            ),
65            stats_type,
66            null_on_divide_by_zero,
67        }
68    }
69}
70
71impl AggregateUDFImpl for Stddev {
72    /// Return a reference to Any that can be used for downcasting
73    fn as_any(&self) -> &dyn Any {
74        self
75    }
76
77    fn name(&self) -> &str {
78        &self.name
79    }
80
81    fn signature(&self) -> &Signature {
82        &self.signature
83    }
84
85    fn return_type(&self, _arg_types: &[DataType]) -> Result<DataType> {
86        Ok(DataType::Float64)
87    }
88
89    fn accumulator(&self, _acc_args: AccumulatorArgs) -> Result<Box<dyn Accumulator>> {
90        Ok(Box::new(StddevAccumulator::try_new(
91            self.stats_type,
92            self.null_on_divide_by_zero,
93        )?))
94    }
95
96    fn create_sliding_accumulator(
97        &self,
98        _acc_args: AccumulatorArgs,
99    ) -> Result<Box<dyn Accumulator>> {
100        Ok(Box::new(StddevAccumulator::try_new(
101            self.stats_type,
102            self.null_on_divide_by_zero,
103        )?))
104    }
105
106    fn state_fields(&self, _args: StateFieldsArgs) -> Result<Vec<FieldRef>> {
107        Ok(vec![
108            Arc::new(Field::new(
109                format_state_name(&self.name, "count"),
110                DataType::Float64,
111                true,
112            )),
113            Arc::new(Field::new(
114                format_state_name(&self.name, "mean"),
115                DataType::Float64,
116                true,
117            )),
118            Arc::new(Field::new(
119                format_state_name(&self.name, "m2"),
120                DataType::Float64,
121                true,
122            )),
123        ])
124    }
125
126    fn default_value(&self, _data_type: &DataType) -> Result<ScalarValue> {
127        Ok(ScalarValue::Float64(None))
128    }
129}
130
131/// An accumulator to compute the standard deviation
132#[derive(Debug)]
133pub struct StddevAccumulator {
134    variance: VarianceAccumulator,
135}
136
137impl StddevAccumulator {
138    /// Creates a new `StddevAccumulator`
139    pub fn try_new(s_type: StatsType, null_on_divide_by_zero: bool) -> Result<Self> {
140        Ok(Self {
141            variance: VarianceAccumulator::try_new(s_type, null_on_divide_by_zero)?,
142        })
143    }
144
145    pub fn get_m2(&self) -> f64 {
146        self.variance.get_m2()
147    }
148}
149
150impl Accumulator for StddevAccumulator {
151    fn state(&mut self) -> Result<Vec<ScalarValue>> {
152        Ok(vec![
153            ScalarValue::from(self.variance.get_count()),
154            ScalarValue::from(self.variance.get_mean()),
155            ScalarValue::from(self.variance.get_m2()),
156        ])
157    }
158
159    fn update_batch(&mut self, values: &[ArrayRef]) -> Result<()> {
160        self.variance.update_batch(values)
161    }
162
163    fn retract_batch(&mut self, values: &[ArrayRef]) -> Result<()> {
164        self.variance.retract_batch(values)
165    }
166
167    fn merge_batch(&mut self, states: &[ArrayRef]) -> Result<()> {
168        self.variance.merge_batch(states)
169    }
170
171    fn evaluate(&mut self) -> Result<ScalarValue> {
172        let variance = self.variance.evaluate()?;
173        match variance {
174            ScalarValue::Float64(Some(e)) => Ok(ScalarValue::Float64(Some(e.sqrt()))),
175            ScalarValue::Float64(None) => Ok(ScalarValue::Float64(None)),
176            _ => internal_err!("Variance should be f64"),
177        }
178    }
179
180    fn size(&self) -> usize {
181        std::mem::align_of_val(self) - std::mem::align_of_val(&self.variance) + self.variance.size()
182    }
183}
datafusion_comet_spark_expr/agg_funcs/stddev.rs

datafusion_comet_spark_expr/agg_funcs/
stddev.rs
