dataprof_core/
profile.rs

1use std::collections::HashMap;
2
3use serde::{Deserialize, Serialize};
4
5use crate::classification::DataType;
6use crate::pattern::Pattern;
7
8/// Profiling statistics for a single column.
9#[derive(Debug, Clone, Serialize, Deserialize)]
10pub struct ColumnProfile {
11    pub name: String,
12    pub data_type: DataType,
13    pub null_count: usize,
14    pub total_count: usize,
15    pub unique_count: Option<usize>,
16    pub stats: ColumnStats,
17    /// Detected patterns, or `None` when pattern detection did not run.
18    ///
19    /// `None` and `Some(vec![])` are not interchangeable: the former means the
20    /// column was never scanned, the latter that it was scanned and nothing
21    /// matched. Consumers that gate on sensitivity -- redaction, agent-facing
22    /// output -- must treat `None` as "unknown", never as "no sensitive data".
23    pub patterns: Option<Vec<Pattern>>,
24}
25
26/// Quartile statistics for numeric distributions.
27#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
28pub struct Quartiles {
29    pub q1: f64,
30    pub q2: f64,
31    pub q3: f64,
32    pub iqr: f64,
33}
34
35/// A value and its frequency count within a column.
36#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
37pub struct FrequencyItem {
38    pub value: String,
39    pub count: usize,
40    #[serde(serialize_with = "crate::serde_helpers::round_2")]
41    pub percentage: f64,
42}
43
44/// Statistics for numeric (integer or float) columns.
45#[derive(Debug, Clone, Serialize, Deserialize)]
46pub struct NumericStats {
47    #[serde(serialize_with = "crate::serde_helpers::round_2")]
48    pub min: f64,
49    #[serde(serialize_with = "crate::serde_helpers::round_2")]
50    pub max: f64,
51    #[serde(serialize_with = "crate::serde_helpers::round_4")]
52    pub mean: f64,
53    #[serde(serialize_with = "crate::serde_helpers::round_4")]
54    pub std_dev: f64,
55    #[serde(serialize_with = "crate::serde_helpers::round_4")]
56    pub variance: f64,
57    #[serde(
58        skip_serializing_if = "Option::is_none",
59        serialize_with = "crate::serde_helpers::round_2_opt"
60    )]
61    pub median: Option<f64>,
62    #[serde(
63        skip_serializing_if = "Option::is_none",
64        serialize_with = "crate::serde_helpers::quartiles::serialize"
65    )]
66    pub quartiles: Option<Quartiles>,
67    #[serde(
68        skip_serializing_if = "Option::is_none",
69        serialize_with = "crate::serde_helpers::round_2_opt"
70    )]
71    pub mode: Option<f64>,
72    #[serde(
73        skip_serializing_if = "Option::is_none",
74        serialize_with = "crate::serde_helpers::round_2_opt"
75    )]
76    pub coefficient_of_variation: Option<f64>,
77    #[serde(
78        skip_serializing_if = "Option::is_none",
79        serialize_with = "crate::serde_helpers::round_4_opt"
80    )]
81    pub skewness: Option<f64>,
82    #[serde(
83        skip_serializing_if = "Option::is_none",
84        serialize_with = "crate::serde_helpers::round_4_opt"
85    )]
86    pub kurtosis: Option<f64>,
87    #[serde(skip_serializing_if = "Option::is_none")]
88    pub is_approximate: Option<bool>,
89    /// Number of values flagged as IQR-based outliers in this column.
90    ///
91    /// Uses the same Tukey-style detection (Q1 − k·IQR, Q3 + k·IQR with
92    /// k = 1.5 by default) that feeds the global `accuracy.outlier_ratio`.
93    /// `None` when outlier detection didn't run (sample below the configured
94    /// minimum or non-numeric column).
95    #[serde(default, skip_serializing_if = "Option::is_none")]
96    pub outlier_count: Option<usize>,
97}
98
99impl NumericStats {
100    pub fn empty() -> Self {
101        Self {
102            min: 0.0,
103            max: 0.0,
104            mean: 0.0,
105            std_dev: 0.0,
106            variance: 0.0,
107            median: None,
108            quartiles: None,
109            mode: None,
110            coefficient_of_variation: None,
111            skewness: None,
112            kurtosis: None,
113            is_approximate: None,
114            outlier_count: None,
115        }
116    }
117}
118
119/// Statistics for text/string columns.
120#[derive(Debug, Clone, Serialize, Deserialize)]
121pub struct TextStats {
122    pub min_length: usize,
123    pub max_length: usize,
124    #[serde(serialize_with = "crate::serde_helpers::round_2")]
125    pub avg_length: f64,
126    #[serde(skip_serializing_if = "Option::is_none")]
127    pub most_frequent: Option<Vec<FrequencyItem>>,
128    #[serde(skip_serializing_if = "Option::is_none")]
129    pub least_frequent: Option<Vec<FrequencyItem>>,
130}
131
132impl TextStats {
133    pub fn empty() -> Self {
134        Self {
135            min_length: 0,
136            max_length: 0,
137            avg_length: 0.0,
138            most_frequent: None,
139            least_frequent: None,
140        }
141    }
142
143    pub fn from_lengths(min_length: usize, max_length: usize, avg_length: f64) -> Self {
144        Self {
145            min_length: if min_length == usize::MAX {
146                0
147            } else {
148                min_length
149            },
150            max_length,
151            avg_length,
152            most_frequent: None,
153            least_frequent: None,
154        }
155    }
156}
157
158/// Statistics for date/datetime columns.
159#[derive(Debug, Clone, Serialize, Deserialize)]
160pub struct DateTimeStats {
161    pub min_datetime: String,
162    pub max_datetime: String,
163    #[serde(serialize_with = "crate::serde_helpers::round_2")]
164    pub duration_days: f64,
165    pub year_distribution: HashMap<i32, usize>,
166    pub month_distribution: HashMap<u32, usize>,
167    pub day_of_week_distribution: HashMap<String, usize>,
168    #[serde(skip_serializing_if = "Option::is_none")]
169    pub hour_distribution: Option<HashMap<u32, usize>>,
170}
171
172impl DateTimeStats {
173    pub fn empty() -> Self {
174        Self {
175            min_datetime: String::new(),
176            max_datetime: String::new(),
177            duration_days: 0.0,
178            year_distribution: HashMap::new(),
179            month_distribution: HashMap::new(),
180            day_of_week_distribution: HashMap::new(),
181            hour_distribution: None,
182        }
183    }
184}
185
186/// Statistics for boolean columns.
187#[derive(Debug, Clone, Serialize, Deserialize)]
188pub struct BooleanStats {
189    pub true_count: usize,
190    pub false_count: usize,
191    #[serde(serialize_with = "crate::serde_helpers::round_4")]
192    pub true_ratio: f64,
193}
194
195/// Type-specific statistics for a column, determined by the inferred data type.
196#[derive(Debug, Clone, Serialize, Deserialize)]
197pub enum ColumnStats {
198    Numeric(NumericStats),
199    Text(TextStats),
200    DateTime(DateTimeStats),
201    Boolean(BooleanStats),
202    None,
203}
204
205#[cfg(test)]
206mod tests {
207    use super::*;
208
209    #[test]
210    fn test_column_profile_json_roundtrip() {
211        let profile = ColumnProfile {
212            name: "test_col".to_string(),
213            data_type: DataType::Integer,
214            null_count: 2,
215            total_count: 10,
216            unique_count: Some(8),
217            stats: ColumnStats::Numeric(NumericStats {
218                min: 1.0,
219                max: 100.0,
220                mean: 50.5,
221                std_dev: 28.87,
222                variance: 833.25,
223                median: Some(50.0),
224                quartiles: Some(Quartiles {
225                    q1: 25.0,
226                    q2: 50.0,
227                    q3: 75.0,
228                    iqr: 50.0,
229                }),
230                mode: Some(42.0),
231                coefficient_of_variation: Some(57.17),
232                skewness: Some(0.0),
233                kurtosis: Some(-1.2),
234                is_approximate: Some(false),
235                outlier_count: Some(0),
236            }),
237            patterns: Some(vec![]),
238        };
239
240        let json = serde_json::to_string(&profile).unwrap();
241        let deserialized: ColumnProfile = serde_json::from_str(&json).unwrap();
242
243        assert_eq!(deserialized.name, "test_col");
244        assert_eq!(deserialized.data_type, DataType::Integer);
245        assert_eq!(deserialized.total_count, 10);
246        assert_eq!(deserialized.null_count, 2);
247
248        if let ColumnStats::Numeric(n) = &deserialized.stats {
249            assert!((n.min - 1.0).abs() < 0.01);
250            assert!((n.max - 100.0).abs() < 0.01);
251            assert!((n.mean - 50.5).abs() < 0.01);
252            assert!(n.median.is_some());
253            assert!(n.quartiles.is_some());
254        } else {
255            panic!("Expected Numeric stats after roundtrip");
256        }
257    }
258
259    #[test]
260    fn test_text_stats_json_roundtrip() {
261        let profile = ColumnProfile {
262            name: "name".to_string(),
263            data_type: DataType::String,
264            null_count: 0,
265            total_count: 3,
266            unique_count: Some(3),
267            stats: ColumnStats::Text(TextStats {
268                min_length: 3,
269                max_length: 7,
270                avg_length: 5.0,
271                most_frequent: None,
272                least_frequent: None,
273            }),
274            patterns: Some(vec![]),
275        };
276
277        let json = serde_json::to_string(&profile).unwrap();
278        let deserialized: ColumnProfile = serde_json::from_str(&json).unwrap();
279
280        assert_eq!(deserialized.data_type, DataType::String);
281        if let ColumnStats::Text(t) = &deserialized.stats {
282            assert_eq!(t.min_length, 3);
283            assert_eq!(t.max_length, 7);
284        } else {
285            panic!("Expected Text stats after roundtrip");
286        }
287    }
288}
dataprof_core/profile.rs

dataprof_core/
profile.rs
