dataprof_core/
classification.rs

1/// Inferred column data type.
2#[derive(
3    Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize, schemars::JsonSchema,
4)]
5pub enum DataType {
6    /// Text or string values.
7    String,
8    /// Identifier values that should be treated as semantic strings.
9    Identifier,
10    /// Whole numbers in the i64 range.
11    Integer,
12    /// Floating-point numbers.
13    Float,
14    /// Date or datetime values.
15    Date,
16    /// Boolean values.
17    Boolean,
18}
19
20/// Mutually exclusive lexical form of a single non-null value.
21///
22/// The variants partition non-null values — every value belongs to exactly one —
23/// which is what lets the share held by the largest class describe a column
24/// whose inferred [`DataType`] says nothing about the mixture inside it. A
25/// `String` column of names and a `String` column that is 60% numbers are the
26/// same type; they are not the same class distribution.
27#[derive(
28    Debug,
29    Clone,
30    Copy,
31    PartialEq,
32    Eq,
33    Hash,
34    serde::Serialize,
35    serde::Deserialize,
36    schemars::JsonSchema,
37)]
38#[serde(rename_all = "lowercase")]
39pub enum LexicalClass {
40    /// Whole numbers and fractions alike: `["1.5", "2"]` is one numeric column,
41    /// not a two-class mixture.
42    Numeric,
43    /// Any date or datetime form the profiler recognizes.
44    Date,
45    /// Strict boolean tokens.
46    Boolean,
47    /// Everything else.
48    Text,
49}
50
51impl std::fmt::Display for LexicalClass {
52    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
53        match self {
54            Self::Numeric => write!(f, "numeric"),
55            Self::Date => write!(f, "date"),
56            Self::Boolean => write!(f, "boolean"),
57            Self::Text => write!(f, "text"),
58        }
59    }
60}
61
62/// How a column's non-null values distribute across [`LexicalClass`]es.
63///
64/// Raw counts rather than a share, so that a reader can state the mixture it
65/// actually found ("60% numeric, 40% text") instead of the dominant class and
66/// an unnamed remainder, and so a serialized report carries no rounded
67/// derivative of a number it does not also carry exactly.
68///
69/// All four counts are always present. Every field zero means the column was
70/// classified and had no non-null values to classify — "analyzed, found
71/// nothing", which is not the same as an absent `type_homogeneity`.
72#[derive(
73    Debug,
74    Clone,
75    Copy,
76    Default,
77    PartialEq,
78    Eq,
79    serde::Serialize,
80    serde::Deserialize,
81    schemars::JsonSchema,
82)]
83pub struct TypeHomogeneity {
84    pub numeric: usize,
85    pub date: usize,
86    pub boolean: usize,
87    pub text: usize,
88}
89
90impl TypeHomogeneity {
91    /// Counts in declaration order, which is also the order ties resolve in.
92    fn counts(&self) -> [(LexicalClass, usize); 4] {
93        [
94            (LexicalClass::Numeric, self.numeric),
95            (LexicalClass::Date, self.date),
96            (LexicalClass::Boolean, self.boolean),
97            (LexicalClass::Text, self.text),
98        ]
99    }
100
101    /// Add one value of `class`.
102    pub fn record(&mut self, class: LexicalClass) {
103        match class {
104            LexicalClass::Numeric => self.numeric += 1,
105            LexicalClass::Date => self.date += 1,
106            LexicalClass::Boolean => self.boolean += 1,
107            LexicalClass::Text => self.text += 1,
108        }
109    }
110
111    /// Non-null values classified — the denominator of every share below.
112    pub fn classified_count(&self) -> usize {
113        self.numeric + self.date + self.boolean + self.text
114    }
115
116    /// The class holding the most values and its count, or `None` when nothing
117    /// was classified.
118    ///
119    /// A tie is held by the earliest-declared class. Which class wins does not
120    /// change the reported share — both hold the same count — it only keeps the
121    /// answer stable across runs.
122    pub fn dominant(&self) -> Option<(LexicalClass, usize)> {
123        self.counts()
124            .into_iter()
125            .filter(|(_, count)| *count > 0)
126            .fold(None, |best, candidate| match best {
127                Some((_, best_count)) if best_count >= candidate.1 => best,
128                _ => Some(candidate),
129            })
130    }
131
132    /// Share of classified values held by [`Self::dominant`], in `0.0..=1.0`.
133    /// `None` when nothing was classified, where the ratio is undefined.
134    pub fn dominant_share(&self) -> Option<f64> {
135        let (_, count) = self.dominant()?;
136        Some(count as f64 / self.classified_count() as f64)
137    }
138
139    /// Every class that holds at least one value, largest first, with its share
140    /// of the classified values. Empty when nothing was classified.
141    ///
142    /// Ties keep declaration order, so the sequence is stable across runs.
143    pub fn mixture(&self) -> Vec<(LexicalClass, usize, f64)> {
144        let total = self.classified_count();
145        if total == 0 {
146            return Vec::new();
147        }
148        let mut present: Vec<(LexicalClass, usize)> = self
149            .counts()
150            .into_iter()
151            .filter(|(_, count)| *count > 0)
152            .collect();
153        // Stable sort on the descending count keeps declaration order for ties.
154        present.sort_by_key(|(_, count)| std::cmp::Reverse(*count));
155        present
156            .into_iter()
157            .map(|(class, count)| (class, count, count as f64 / total as f64))
158            .collect()
159    }
160}
161
162/// Semantic category for a detected pattern.
163#[derive(
164    Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize, schemars::JsonSchema,
165)]
166#[serde(rename_all = "snake_case")]
167pub enum PatternCategory {
168    /// Email addresses, phone numbers.
169    Contact,
170    /// UUIDs, fiscal codes, tax IDs.
171    Identifier,
172    /// IPv4, IPv6, MAC addresses, URLs.
173    Network,
174    /// Coordinates and postal codes.
175    Geographic,
176    /// IBANs, credit cards, SWIFT/BIC.
177    Financial,
178    /// Unix or Windows file paths.
179    FilePath,
180    /// Uncategorized patterns.
181    Other,
182}
183
184impl std::fmt::Display for PatternCategory {
185    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
186        match self {
187            Self::Contact => write!(f, "contact"),
188            Self::Identifier => write!(f, "identifier"),
189            Self::Network => write!(f, "network"),
190            Self::Geographic => write!(f, "geographic"),
191            Self::Financial => write!(f, "financial"),
192            Self::FilePath => write!(f, "file_path"),
193            Self::Other => write!(f, "other"),
194        }
195    }
196}
197
198#[cfg(test)]
199mod type_homogeneity_tests {
200    use super::*;
201
202    fn homogeneity(numeric: usize, date: usize, boolean: usize, text: usize) -> TypeHomogeneity {
203        TypeHomogeneity {
204            numeric,
205            date,
206            boolean,
207            text,
208        }
209    }
210
211    #[test]
212    fn recording_values_counts_them_by_class() {
213        let mut counts = TypeHomogeneity::default();
214        counts.record(LexicalClass::Numeric);
215        counts.record(LexicalClass::Text);
216        counts.record(LexicalClass::Numeric);
217
218        assert_eq!(counts, homogeneity(2, 0, 0, 1));
219        assert_eq!(counts.classified_count(), 3);
220        assert_eq!(counts.dominant(), Some((LexicalClass::Numeric, 2)));
221    }
222
223    #[test]
224    fn nothing_classified_has_no_dominant_class_and_no_share() {
225        // The all-zero value is "classified, nothing to classify" — every
226        // derived answer is absent rather than a number invented from a zero
227        // denominator.
228        let empty = TypeHomogeneity::default();
229
230        assert_eq!(empty.classified_count(), 0);
231        assert_eq!(empty.dominant(), None);
232        assert_eq!(empty.dominant_share(), None);
233        assert!(empty.mixture().is_empty());
234    }
235
236    #[test]
237    fn a_tie_is_held_by_the_earliest_declared_class() {
238        // Which class wins a tie does not change the share — both hold the same
239        // count — but it must be the same class on every run, and the same one
240        // the consistency dimension scores against.
241        let tied = homogeneity(50, 50, 0, 0);
242
243        assert_eq!(tied.dominant(), Some((LexicalClass::Numeric, 50)));
244        assert_eq!(tied.dominant_share(), Some(0.5));
245        assert_eq!(
246            tied.mixture(),
247            vec![
248                (LexicalClass::Numeric, 50, 0.5),
249                (LexicalClass::Date, 50, 0.5)
250            ]
251        );
252    }
253
254    #[test]
255    fn mixture_reports_every_present_class_largest_first() {
256        let mixed = homogeneity(60, 10, 0, 30);
257
258        assert_eq!(mixed.dominant_share(), Some(0.6));
259        assert_eq!(
260            mixed.mixture(),
261            vec![
262                (LexicalClass::Numeric, 60, 0.6),
263                (LexicalClass::Text, 30, 0.3),
264                (LexicalClass::Date, 10, 0.1),
265            ],
266            "an absent class must not appear with a 0% share"
267        );
268    }
269
270    #[test]
271    fn counts_survive_a_json_round_trip() {
272        let counts = homogeneity(600, 0, 0, 400);
273        let json = serde_json::to_string(&counts).expect("counts should serialize");
274
275        assert_eq!(json, r#"{"numeric":600,"date":0,"boolean":0,"text":400}"#);
276        assert_eq!(
277            serde_json::from_str::<TypeHomogeneity>(&json).expect("counts should deserialize"),
278            counts
279        );
280    }
281
282    #[test]
283    fn lexical_classes_serialize_as_the_names_reports_use() {
284        for (class, name) in [
285            (LexicalClass::Numeric, "numeric"),
286            (LexicalClass::Date, "date"),
287            (LexicalClass::Boolean, "boolean"),
288            (LexicalClass::Text, "text"),
289        ] {
290            assert_eq!(class.to_string(), name);
291            assert_eq!(
292                serde_json::to_string(&class).expect("class should serialize"),
293                format!("\"{name}\"")
294            );
295        }
296    }
297}
dataprof_core/classification.rs

dataprof_core/
classification.rs
