datarust_profile/profile/
column.rs

1//! Per-column profile records.
2
3use std::collections::HashMap;
4
5use crate::infer;
6use crate::types::ColumnType;
7
8/// Quantile stubs computed for numeric columns: min, Q1, median, Q3, max.
9///
10/// The five values correspond to the probabilities `[0.0, 0.25, 0.5, 0.75, 1.0]`.
11#[derive(Debug, Clone, PartialEq)]
12#[cfg_attr(feature = "serde", derive(serde::Serialize))]
13pub struct FiveNumber {
14    /// Minimum (0% quantile).
15    pub min: f64,
16    /// First quartile (25% quantile).
17    pub q1: f64,
18    /// Median (50% quantile).
19    pub median: f64,
20    /// Third quartile (75% quantile).
21    pub q3: f64,
22    /// Maximum (100% quantile).
23    pub max: f64,
24}
25
26/// An equal-width histogram over the non-missing values of a numeric column.
27///
28/// The bin count follows Sturges' rule (`ceil(log2(n) + 1)`, floored at 1).
29/// `edges.len() == counts.len() + 1`; the final bin is inclusive of its upper
30/// edge so the maximum value is never dropped.
31#[derive(Debug, Clone, PartialEq)]
32#[cfg_attr(feature = "serde", derive(serde::Serialize))]
33pub struct Histogram {
34    /// Bin edges, ascending. Length is `counts.len() + 1` (empty for an empty
35    /// column).
36    pub edges: Vec<f64>,
37    /// Number of values falling into each bin. Length is one less than
38    /// `edges`.
39    pub counts: Vec<usize>,
40}
41
42impl Histogram {
43    /// Returns the number of bins (== `counts.len()`).
44    pub fn nbins(&self) -> usize {
45        self.counts.len()
46    }
47
48    /// Returns the count in the tallest bin, or `0` for an empty histogram.
49    pub fn max_count(&self) -> usize {
50        self.counts.iter().copied().max().unwrap_or(0)
51    }
52}
53
54/// The descriptive statistics computed for a numeric column.
55///
56/// Missing values (`NaN`) are excluded from every statistic and counted
57/// separately in [`ColumnProfile::missing_count`].
58#[derive(Debug, Clone, PartialEq)]
59#[cfg_attr(feature = "serde", derive(serde::Serialize))]
60pub struct NumericStats {
61    /// Arithmetic mean of the non-missing values (`NaN` if the column is empty).
62    pub mean: f64,
63    /// Sample standard deviation (ddof = 1) of the non-missing values.
64    pub std: f64,
65    /// Min/Q1/median/Q3/max of the non-missing values.
66    pub five: FiveNumber,
67    /// Fisher–Pearson sample skewness of the non-missing values (`0.0` when
68    /// undefined, i.e. fewer than three values or zero variance).
69    pub skewness: f64,
70    /// Excess (Fisher) kurtosis of the non-missing values (`0.0` when
71    /// undefined, i.e. fewer than four values or zero variance).
72    pub kurtosis: f64,
73    /// Equal-width histogram (Sturges bins) of the non-missing values.
74    pub histogram: Histogram,
75    /// Number of values outside the Tukey IQR fences (`Q1 − 1.5·IQR`,
76    /// `Q3 + 1.5·IQR`).
77    pub outlier_count: usize,
78    /// Fraction of values outside the IQR fences, in `[0.0, 1.0]`.
79    pub outlier_fraction: f64,
80}
81
82/// The descriptive statistics computed for a categorical column.
83#[derive(Debug, Clone, PartialEq)]
84#[cfg_attr(feature = "serde", derive(serde::Serialize))]
85pub struct CategoricalStats {
86    /// Number of distinct non-missing values (cardinality).
87    pub unique: usize,
88    /// The most frequent non-missing value (ties broken by insertion order).
89    pub top: String,
90    /// Frequency of [`CategoricalStats::top`].
91    pub freq: usize,
92    /// `freq` divided by the number of non-missing cells, in `[0.0, 1.0]`.
93    /// `1.0` means a single value dominates the whole column.
94    pub imbalance_ratio: f64,
95    /// The most frequent values and their counts, descending, capped at a
96    /// small number for display. Useful for frequency bar charts.
97    pub top_values: Vec<(String, usize)>,
98}
99
100/// A full profile for a single column of the dataset.
101#[derive(Debug, Clone, PartialEq)]
102#[cfg_attr(feature = "serde", derive(serde::Serialize))]
103pub struct ColumnProfile {
104    /// Column name. Falls back to `x{j}` when no name was supplied.
105    pub name: String,
106    /// Inferred semantic type.
107    pub column_type: ColumnType,
108    /// Total number of rows (cells) in the column, including missing.
109    pub count: usize,
110    /// Number of missing/empty cells.
111    pub missing_count: usize,
112    /// Fraction of missing cells, in `[0.0, 1.0]`.
113    pub missing_fraction: f64,
114    /// Numeric descriptive stats. `None` for categorical columns.
115    pub numeric: Option<NumericStats>,
116    /// Categorical descriptive stats. `None` for numeric columns.
117    pub categorical: Option<CategoricalStats>,
118}
119
120/// Cap on the length of [`CategoricalStats::top_values`].
121const TOP_VALUES_CAP: usize = 8;
122
123/// Pre-computed mean/std/five-number summary, used by the `_flat` fast path in
124/// [`crate::profile::DatasetProfile::from_matrix`] to avoid recomputing the
125/// most expensive statistics per column.
126///
127/// When `Some`, [`ColumnProfile::from_numeric_with_stats`] trusts these values
128/// instead of re-deriving them; the distributional stats (skewness, kurtosis,
129/// histogram, outliers) are still computed from the raw values.
130pub(crate) struct PrecomputedStats {
131    pub(crate) mean: f64,
132    pub(crate) std: f64,
133    pub(crate) five: FiveNumber,
134}
135
136impl ColumnProfile {
137    /// Builds a profile from a numeric slice, treating `NaN` as missing.
138    pub(crate) fn from_numeric(name: String, values: &[f64]) -> Self {
139        Self::from_numeric_with_stats(name, values, None)
140    }
141
142    /// Builds a profile from a numeric slice, optionally reusing pre-computed
143    /// mean/std/five-number summary.
144    ///
145    /// When `precomputed` is `Some`, the caller (the `_flat` fast path)
146    /// asserts responsibility for matching the values; when `None`, the stats
147    /// are derived here exactly as in [`Self::from_numeric`].
148    pub(crate) fn from_numeric_with_stats(
149        name: String,
150        values: &[f64],
151        precomputed: Option<PrecomputedStats>,
152    ) -> Self {
153        let count = values.len();
154        let present: Vec<f64> = values.iter().copied().filter(|v| v.is_finite()).collect();
155        let missing_count = count - present.len();
156        let missing_fraction = if count == 0 {
157            0.0
158        } else {
159            missing_count as f64 / count as f64
160        };
161
162        let numeric = if present.is_empty() {
163            None
164        } else {
165            // Sort once; `stats::median_sorted` / `quantile` expect sorted input.
166            let mut sorted = present.clone();
167            sorted.sort_by(|a, b| a.total_cmp(b));
168
169            let (mean, std, five) = match precomputed {
170                Some(p) => (p.mean, p.std, p.five),
171                None => {
172                    let m = datarust::stats::mean(&present);
173                    let s = datarust::stats::std(&present, 1);
174                    let f = FiveNumber {
175                        min: datarust::stats::quantile(&sorted, 0.0).unwrap_or(f64::NAN),
176                        q1: datarust::stats::quantile(&sorted, 0.25).unwrap_or(f64::NAN),
177                        median: datarust::stats::median_sorted(&sorted).unwrap_or(f64::NAN),
178                        q3: datarust::stats::quantile(&sorted, 0.75).unwrap_or(f64::NAN),
179                        max: datarust::stats::quantile(&sorted, 1.0).unwrap_or(f64::NAN),
180                    };
181                    (m, s, f)
182                }
183            };
184
185            let skew = super::distribution::skewness(&present, mean, std);
186            let kurt = super::distribution::kurtosis_excess(&present, mean, std);
187            let histogram = super::distribution::histogram(&sorted, five.min, five.max);
188            let (outlier_count, outlier_fraction) =
189                super::distribution::outlier_count(&sorted, five.q1, five.q3);
190
191            Some(NumericStats {
192                mean,
193                std,
194                five,
195                skewness: skew,
196                kurtosis: kurt,
197                histogram,
198                outlier_count,
199                outlier_fraction,
200            })
201        };
202
203        ColumnProfile {
204            name,
205            column_type: ColumnType::Numeric,
206            count,
207            missing_count,
208            missing_fraction,
209            numeric,
210            categorical: None,
211        }
212    }
213
214    /// Builds a profile from raw string cells, inferring the column type.
215    pub(crate) fn from_strings<T: AsRef<str>>(name: String, cells: &[T]) -> Self {
216        let count = cells.len();
217        let missing_count = cells
218            .iter()
219            .filter(|c| infer::is_missing(c.as_ref()))
220            .count();
221        let missing_fraction = if count == 0 {
222            0.0
223        } else {
224            missing_count as f64 / count as f64
225        };
226
227        let column_type = infer::infer_column(cells);
228        match column_type {
229            ColumnType::Numeric => {
230                let values = infer::parse_numeric_column(cells);
231                let mut p = Self::from_numeric(name, &values);
232                // Preserve the string-source count/fraction even when all cells
233                // were missing (from_numeric would otherwise treat them as NaN).
234                p.count = count;
235                p.missing_count = missing_count;
236                p.missing_fraction = missing_fraction;
237                p
238            }
239            ColumnType::Categorical => {
240                let categorical = compute_categorical(cells);
241                ColumnProfile {
242                    name,
243                    column_type,
244                    count,
245                    missing_count,
246                    missing_fraction,
247                    numeric: None,
248                    categorical,
249                }
250            }
251        }
252    }
253}
254
255/// Tallies the non-missing cells of a categorical column.
256fn compute_categorical<T: AsRef<str>>(cells: &[T]) -> Option<CategoricalStats> {
257    let mut counts: HashMap<&str, usize> = HashMap::new();
258    let mut order: Vec<&str> = Vec::new();
259    for cell in cells {
260        let cell = cell.as_ref();
261        if infer::is_missing(cell) {
262            continue;
263        }
264        let trimmed = cell.trim();
265        match counts.get(trimmed) {
266            None => {
267                counts.insert(trimmed, 1);
268                order.push(trimmed);
269            }
270            Some(c) => *counts.get_mut(trimmed).unwrap() = c + 1,
271        }
272    }
273    if order.is_empty() {
274        return None;
275    }
276    let present_total: usize = order.iter().map(|k| counts[*k]).sum();
277
278    // Top values: sort by count descending, tie-break by first-seen order.
279    let mut entries: Vec<(&str, usize)> = order.iter().map(|k| (*k, counts[*k])).collect();
280    entries.sort_by_key(|&(_, c)| std::cmp::Reverse(c));
281    let top_values: Vec<(String, usize)> = entries
282        .into_iter()
283        .take(TOP_VALUES_CAP)
284        .map(|(k, c)| (k.to_string(), c))
285        .collect();
286
287    let (top, freq) = {
288        let first = top_values.first().expect("non-empty");
289        (first.0.clone(), first.1)
290    };
291    let imbalance_ratio = if present_total == 0 {
292        0.0
293    } else {
294        freq as f64 / present_total as f64
295    };
296
297    Some(CategoricalStats {
298        unique: order.len(),
299        top,
300        freq,
301        imbalance_ratio,
302        top_values,
303    })
304}
305
306#[cfg(test)]
307mod tests {
308    use super::*;
309    use crate::types::ColumnType;
310
311    #[test]
312    fn from_numeric_handles_empty() {
313        let p = ColumnProfile::from_numeric("x".to_string(), &[]);
314        assert_eq!(p.name, "x");
315        assert_eq!(p.column_type, ColumnType::Numeric);
316        assert_eq!(p.count, 0);
317        assert_eq!(p.missing_count, 0);
318        assert_eq!(p.missing_fraction, 0.0);
319        assert!(p.numeric.is_none());
320    }
321
322    #[test]
323    fn from_numeric_all_nan() {
324        let p = ColumnProfile::from_numeric("x".to_string(), &[f64::NAN, f64::NAN]);
325        assert_eq!(p.count, 2);
326        assert_eq!(p.missing_count, 2);
327        assert_eq!(p.missing_fraction, 1.0);
328        assert!(p.numeric.is_none());
329    }
330
331    #[test]
332    fn from_numeric_computes_stats() {
333        let p = ColumnProfile::from_numeric("x".to_string(), &[1.0, 2.0, 3.0, 4.0, 5.0]);
334        assert_eq!(p.count, 5);
335        assert_eq!(p.missing_count, 0);
336        let n = p.numeric.as_ref().unwrap();
337        assert!((n.mean - 3.0).abs() < 1e-9);
338        assert!((n.five.min - 1.0).abs() < 1e-9);
339        assert!((n.five.max - 5.0).abs() < 1e-9);
340    }
341
342    #[test]
343    fn from_numeric_with_nan() {
344        let p = ColumnProfile::from_numeric("x".to_string(), &[1.0, f64::NAN, 3.0]);
345        assert_eq!(p.count, 3);
346        assert_eq!(p.missing_count, 1);
347        assert!((p.missing_fraction - 1.0 / 3.0).abs() < 1e-9);
348        let n = p.numeric.as_ref().unwrap();
349        assert!((n.mean - 2.0).abs() < 1e-9);
350    }
351
352    #[test]
353    fn from_strings_numeric() {
354        let cells = vec!["1.0".to_string(), "2.0".to_string(), "3.0".to_string()];
355        let p = ColumnProfile::from_strings("x".to_string(), &cells);
356        assert_eq!(p.column_type, ColumnType::Numeric);
357        assert!(p.numeric.is_some());
358    }
359
360    #[test]
361    fn from_strings_categorical() {
362        let cells = vec!["a".to_string(), "b".to_string(), "a".to_string()];
363        let p = ColumnProfile::from_strings("x".to_string(), &cells);
364        assert_eq!(p.column_type, ColumnType::Categorical);
365        let c = p.categorical.as_ref().unwrap();
366        assert_eq!(c.unique, 2);
367        assert_eq!(c.top, "a");
368        assert_eq!(c.freq, 2);
369    }
370
371    #[test]
372    fn from_strings_with_missing() {
373        let cells = vec!["1.0".to_string(), "NA".to_string(), "3.0".to_string()];
374        let p = ColumnProfile::from_strings("x".to_string(), &cells);
375        assert_eq!(p.column_type, ColumnType::Numeric);
376        assert_eq!(p.missing_count, 1);
377        assert!((p.missing_fraction - 1.0 / 3.0).abs() < 1e-9);
378    }
379
380    #[test]
381    fn from_strings_all_missing_is_categorical() {
382        let cells = vec!["NA".to_string(), "null".to_string(), "".to_string()];
383        let p = ColumnProfile::from_strings("x".to_string(), &cells);
384        assert_eq!(p.column_type, ColumnType::Categorical);
385        assert_eq!(p.missing_count, 3);
386        assert!(p.categorical.is_none());
387    }
388
389    #[test]
390    fn histogram_nbins() {
391        let h = crate::profile::distribution::histogram(&[1.0, 2.0, 3.0], 1.0, 3.0);
392        assert_eq!(h.nbins(), h.counts.len());
393        assert_eq!(h.max_count(), 1);
394    }
395
396    #[test]
397    fn histogram_max_count() {
398        let h = crate::profile::distribution::histogram(&[1.0, 1.0, 2.0, 3.0], 1.0, 3.0);
399        assert_eq!(h.max_count(), 2);
400    }
401
402    #[test]
403    fn histogram_empty_max_count() {
404        let h = crate::profile::distribution::histogram(&[], 0.0, 0.0);
405        assert_eq!(h.nbins(), 0);
406        assert_eq!(h.max_count(), 0);
407    }
408
409    #[test]
410    fn top_values_cap() {
411        // Create many unique categorical values (non-numeric strings)
412        let cells: Vec<String> = (0..20).map(|i| format!("val_{}", i)).collect();
413        let p = ColumnProfile::from_strings("x".to_string(), &cells);
414        let c = p.categorical.as_ref().unwrap();
415        assert_eq!(c.unique, 20);
416        assert!(c.top_values.len() <= 8); // TOP_VALUES_CAP
417    }
418}
datarust_profile/profile/column.rs

datarust_profile/profile/
column.rs
