datarust_profile/profile/
column.rs

1//! Per-column profile records.
2
3use std::collections::HashMap;
4
5use crate::infer;
6use crate::types::ColumnType;
7
8/// Quantile stubs computed for numeric columns: min, Q1, median, Q3, max.
9///
10/// The five values correspond to the probabilities `[0.0, 0.25, 0.5, 0.75, 1.0]`.
11#[derive(Debug, Clone, PartialEq)]
12#[cfg_attr(feature = "serde", derive(serde::Serialize))]
13pub struct FiveNumber {
14    /// Minimum (0% quantile).
15    pub min: f64,
16    /// First quartile (25% quantile).
17    pub q1: f64,
18    /// Median (50% quantile).
19    pub median: f64,
20    /// Third quartile (75% quantile).
21    pub q3: f64,
22    /// Maximum (100% quantile).
23    pub max: f64,
24}
25
26/// An equal-width histogram over the non-missing values of a numeric column.
27///
28/// The bin count follows Sturges' rule (`ceil(log2(n) + 1)`, floored at 1).
29/// `edges.len() == counts.len() + 1`; the final bin is inclusive of its upper
30/// edge so the maximum value is never dropped.
31#[derive(Debug, Clone, PartialEq)]
32#[cfg_attr(feature = "serde", derive(serde::Serialize))]
33pub struct Histogram {
34    /// Bin edges, ascending. Length is `counts.len() + 1` (empty for an empty
35    /// column).
36    pub edges: Vec<f64>,
37    /// Number of values falling into each bin. Length is one less than
38    /// `edges`.
39    pub counts: Vec<usize>,
40}
41
42impl Histogram {
43    /// Returns the number of bins (== `counts.len()`).
44    pub fn nbins(&self) -> usize {
45        self.counts.len()
46    }
47
48    /// Returns the count in the tallest bin, or `0` for an empty histogram.
49    pub fn max_count(&self) -> usize {
50        self.counts.iter().copied().max().unwrap_or(0)
51    }
52}
53
54/// The descriptive statistics computed for a numeric column.
55///
56/// Missing values (`NaN`) are excluded from every statistic and counted
57/// separately in [`ColumnProfile::missing_count`].
58#[derive(Debug, Clone, PartialEq)]
59#[cfg_attr(feature = "serde", derive(serde::Serialize))]
60pub struct NumericStats {
61    /// Arithmetic mean of the non-missing values (`NaN` if the column is empty).
62    pub mean: f64,
63    /// Sample standard deviation (ddof = 1) of the non-missing values.
64    pub std: f64,
65    /// Min/Q1/median/Q3/max of the non-missing values.
66    pub five: FiveNumber,
67    /// Fisher–Pearson sample skewness of the non-missing values (`0.0` when
68    /// undefined, i.e. fewer than three values or zero variance).
69    pub skewness: f64,
70    /// Excess (Fisher) kurtosis of the non-missing values (`0.0` when
71    /// undefined, i.e. fewer than four values or zero variance).
72    pub kurtosis: f64,
73    /// Equal-width histogram (Sturges bins) of the non-missing values.
74    pub histogram: Histogram,
75    /// Number of values outside the Tukey IQR fences (`Q1 − 1.5·IQR`,
76    /// `Q3 + 1.5·IQR`).
77    pub outlier_count: usize,
78    /// Fraction of values outside the IQR fences, in `[0.0, 1.0]`.
79    pub outlier_fraction: f64,
80}
81
82/// The descriptive statistics computed for a categorical column.
83#[derive(Debug, Clone, PartialEq)]
84#[cfg_attr(feature = "serde", derive(serde::Serialize))]
85pub struct CategoricalStats {
86    /// Number of distinct non-missing values (cardinality).
87    pub unique: usize,
88    /// The most frequent non-missing value (ties broken by insertion order).
89    pub top: String,
90    /// Frequency of [`CategoricalStats::top`].
91    pub freq: usize,
92    /// `freq` divided by the number of non-missing cells, in `[0.0, 1.0]`.
93    /// `1.0` means a single value dominates the whole column.
94    pub imbalance_ratio: f64,
95    /// The most frequent values and their counts, descending, capped at a
96    /// small number for display. Useful for frequency bar charts.
97    pub top_values: Vec<(String, usize)>,
98}
99
100/// A full profile for a single column of the dataset.
101#[derive(Debug, Clone, PartialEq)]
102#[cfg_attr(feature = "serde", derive(serde::Serialize))]
103pub struct ColumnProfile {
104    /// Column name. Falls back to `x{j}` when no name was supplied.
105    pub name: String,
106    /// Inferred semantic type.
107    pub column_type: ColumnType,
108    /// Total number of rows (cells) in the column, including missing.
109    pub count: usize,
110    /// Number of missing/empty cells.
111    pub missing_count: usize,
112    /// Fraction of missing cells, in `[0.0, 1.0]`.
113    pub missing_fraction: f64,
114    /// Numeric descriptive stats. `None` for categorical columns.
115    pub numeric: Option<NumericStats>,
116    /// Categorical descriptive stats. `None` for numeric columns.
117    pub categorical: Option<CategoricalStats>,
118}
119
120/// Cap on the length of [`CategoricalStats::top_values`].
121const TOP_VALUES_CAP: usize = 8;
122
123/// Pre-computed mean/std/five-number summary, used by the `_flat` fast path in
124/// [`crate::profile::DatasetProfile::from_matrix`] to avoid recomputing the
125/// most expensive statistics per column.
126///
127/// When `Some`, [`ColumnProfile::from_numeric_with_stats`] trusts these values
128/// instead of re-deriving them; the distributional stats (skewness, kurtosis,
129/// histogram, outliers) are still computed from the raw values.
130pub(crate) struct PrecomputedStats {
131    pub(crate) mean: f64,
132    pub(crate) std: f64,
133    pub(crate) five: FiveNumber,
134}
135
136impl ColumnProfile {
137    /// Builds a profile from a numeric slice, treating `NaN` as missing.
138    pub(crate) fn from_numeric(name: String, values: &[f64]) -> Self {
139        Self::from_numeric_with_stats(name, values, None)
140    }
141
142    /// Builds a profile from a numeric slice, optionally reusing pre-computed
143    /// mean/std/five-number summary.
144    ///
145    /// When `precomputed` is `Some`, the caller (the `_flat` fast path)
146    /// asserts responsibility for matching the values; when `None`, the stats
147    /// are derived here exactly as in [`Self::from_numeric`].
148    pub(crate) fn from_numeric_with_stats(
149        name: String,
150        values: &[f64],
151        precomputed: Option<PrecomputedStats>,
152    ) -> Self {
153        let count = values.len();
154        let present: Vec<f64> = values.iter().copied().filter(|v| v.is_finite()).collect();
155        let missing_count = count - present.len();
156        let missing_fraction = if count == 0 {
157            0.0
158        } else {
159            missing_count as f64 / count as f64
160        };
161
162        let numeric = if present.is_empty() {
163            None
164        } else {
165            // Sort once; `stats::median_sorted` / `quantile` expect sorted input.
166            let mut sorted = present.clone();
167            sorted.sort_by(|a, b| a.total_cmp(b));
168
169            let (mean, std, five) = match precomputed {
170                Some(p) => (p.mean, p.std, p.five),
171                None => {
172                    let m = datarust::stats::mean(&present);
173                    let s = datarust::stats::std(&present, 1);
174                    let f = FiveNumber {
175                        min: datarust::stats::quantile(&sorted, 0.0).unwrap_or(f64::NAN),
176                        q1: datarust::stats::quantile(&sorted, 0.25).unwrap_or(f64::NAN),
177                        median: datarust::stats::median_sorted(&sorted).unwrap_or(f64::NAN),
178                        q3: datarust::stats::quantile(&sorted, 0.75).unwrap_or(f64::NAN),
179                        max: datarust::stats::quantile(&sorted, 1.0).unwrap_or(f64::NAN),
180                    };
181                    (m, s, f)
182                }
183            };
184
185            let skew = super::distribution::skewness(&present, mean, std);
186            let kurt = super::distribution::kurtosis_excess(&present, mean, std);
187            let histogram = super::distribution::histogram(&sorted, five.min, five.max);
188            let (outlier_count, outlier_fraction) =
189                super::distribution::outlier_count(&sorted, five.q1, five.q3);
190
191            Some(NumericStats {
192                mean,
193                std,
194                five,
195                skewness: skew,
196                kurtosis: kurt,
197                histogram,
198                outlier_count,
199                outlier_fraction,
200            })
201        };
202
203        ColumnProfile {
204            name,
205            column_type: ColumnType::Numeric,
206            count,
207            missing_count,
208            missing_fraction,
209            numeric,
210            categorical: None,
211        }
212    }
213
214    /// Builds a profile from raw string cells, inferring the column type.
215    pub(crate) fn from_strings(name: String, cells: &[String]) -> Self {
216        let count = cells.len();
217        let missing_count = cells.iter().filter(|c| infer::is_missing(c)).count();
218        let missing_fraction = if count == 0 {
219            0.0
220        } else {
221            missing_count as f64 / count as f64
222        };
223
224        let column_type = infer::infer_column(cells);
225        match column_type {
226            ColumnType::Numeric => {
227                let values = infer::parse_numeric_column(cells);
228                let mut p = Self::from_numeric(name, &values);
229                // Preserve the string-source count/fraction even when all cells
230                // were missing (from_numeric would otherwise treat them as NaN).
231                p.count = count;
232                p.missing_count = missing_count;
233                p.missing_fraction = missing_fraction;
234                p
235            }
236            ColumnType::Categorical => {
237                let categorical = compute_categorical(cells);
238                ColumnProfile {
239                    name,
240                    column_type,
241                    count,
242                    missing_count,
243                    missing_fraction,
244                    numeric: None,
245                    categorical,
246                }
247            }
248        }
249    }
250}
251
252/// Tallies the non-missing cells of a categorical column.
253fn compute_categorical(cells: &[String]) -> Option<CategoricalStats> {
254    let mut counts: HashMap<&str, usize> = HashMap::new();
255    let mut order: Vec<&str> = Vec::new();
256    for cell in cells {
257        if infer::is_missing(cell) {
258            continue;
259        }
260        let trimmed = cell.trim();
261        match counts.get(trimmed) {
262            None => {
263                counts.insert(trimmed, 1);
264                order.push(trimmed);
265            }
266            Some(c) => *counts.get_mut(trimmed).unwrap() = c + 1,
267        }
268    }
269    if order.is_empty() {
270        return None;
271    }
272    let present_total: usize = order.iter().map(|k| counts[*k]).sum();
273
274    // Top values: sort by count descending, tie-break by first-seen order.
275    let mut entries: Vec<(&str, usize)> = order.iter().map(|k| (*k, counts[*k])).collect();
276    entries.sort_by_key(|&(_, c)| std::cmp::Reverse(c));
277    let top_values: Vec<(String, usize)> = entries
278        .into_iter()
279        .take(TOP_VALUES_CAP)
280        .map(|(k, c)| (k.to_string(), c))
281        .collect();
282
283    let (top, freq) = {
284        let first = top_values.first().expect("non-empty");
285        (first.0.clone(), first.1)
286    };
287    let imbalance_ratio = if present_total == 0 {
288        0.0
289    } else {
290        freq as f64 / present_total as f64
291    };
292
293    Some(CategoricalStats {
294        unique: order.len(),
295        top,
296        freq,
297        imbalance_ratio,
298        top_values,
299    })
300}
datarust_profile/profile/column.rs

datarust_profile/profile/
column.rs
