dataprof_core/
classification.rs

1/// Inferred column data type.
2#[derive(
3    Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize, schemars::JsonSchema,
4)]
5pub enum DataType {
6    /// Text or string values.
7    String,
8    /// Identifier values that should be treated as semantic strings.
9    Identifier,
10    /// Whole numbers in the i64 range.
11    Integer,
12    /// Floating-point numbers.
13    Float,
14    /// Date or datetime values.
15    Date,
16    /// Boolean values.
17    Boolean,
18    /// Container values: structs, lists and maps.
19    ///
20    /// Only the counts are analyzed. Every other measurement of a container
21    /// would describe how it was serialised rather than the data, and the
22    /// serialisation differs by input path, so those fields are `None` (#637).
23    Nested,
24}
25
26/// Mutually exclusive lexical form of a single non-null value.
27///
28/// The variants partition non-null values — every value belongs to exactly one —
29/// which is what lets the share held by the largest class describe a column
30/// whose inferred [`DataType`] says nothing about the mixture inside it. A
31/// `String` column of names and a `String` column that is 60% numbers are the
32/// same type; they are not the same class distribution.
33#[derive(
34    Debug,
35    Clone,
36    Copy,
37    PartialEq,
38    Eq,
39    Hash,
40    serde::Serialize,
41    serde::Deserialize,
42    schemars::JsonSchema,
43)]
44#[serde(rename_all = "lowercase")]
45pub enum LexicalClass {
46    /// Whole numbers and fractions alike: `["1.5", "2"]` is one numeric column,
47    /// not a two-class mixture.
48    Numeric,
49    /// Any date or datetime form the profiler recognizes.
50    Date,
51    /// Strict boolean tokens.
52    Boolean,
53    /// Everything else.
54    Text,
55}
56
57impl std::fmt::Display for LexicalClass {
58    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
59        match self {
60            Self::Numeric => write!(f, "numeric"),
61            Self::Date => write!(f, "date"),
62            Self::Boolean => write!(f, "boolean"),
63            Self::Text => write!(f, "text"),
64        }
65    }
66}
67
68/// How a column's non-null values distribute across [`LexicalClass`]es.
69///
70/// Raw counts rather than a share, so that a reader can state the mixture it
71/// actually found ("60% numeric, 40% text") instead of the dominant class and
72/// an unnamed remainder, and so a serialized report carries no rounded
73/// derivative of a number it does not also carry exactly.
74///
75/// All four counts are always present. Every field zero means the column was
76/// classified and had no non-null values to classify — "analyzed, found
77/// nothing", which is not the same as an absent `type_homogeneity`.
78#[derive(
79    Debug,
80    Clone,
81    Copy,
82    Default,
83    PartialEq,
84    Eq,
85    serde::Serialize,
86    serde::Deserialize,
87    schemars::JsonSchema,
88)]
89pub struct TypeHomogeneity {
90    pub numeric: usize,
91    pub date: usize,
92    pub boolean: usize,
93    pub text: usize,
94}
95
96impl TypeHomogeneity {
97    /// Counts in declaration order, which is also the order ties resolve in.
98    fn counts(&self) -> [(LexicalClass, usize); 4] {
99        [
100            (LexicalClass::Numeric, self.numeric),
101            (LexicalClass::Date, self.date),
102            (LexicalClass::Boolean, self.boolean),
103            (LexicalClass::Text, self.text),
104        ]
105    }
106
107    /// Add one value of `class`.
108    pub fn record(&mut self, class: LexicalClass) {
109        match class {
110            LexicalClass::Numeric => self.numeric += 1,
111            LexicalClass::Date => self.date += 1,
112            LexicalClass::Boolean => self.boolean += 1,
113            LexicalClass::Text => self.text += 1,
114        }
115    }
116
117    /// Non-null values classified — the denominator of every share below.
118    pub fn classified_count(&self) -> usize {
119        self.numeric + self.date + self.boolean + self.text
120    }
121
122    /// The class holding the most values and its count, or `None` when nothing
123    /// was classified.
124    ///
125    /// A tie is held by the earliest-declared class. Which class wins does not
126    /// change the reported share — both hold the same count — it only keeps the
127    /// answer stable across runs.
128    pub fn dominant(&self) -> Option<(LexicalClass, usize)> {
129        self.counts()
130            .into_iter()
131            .filter(|(_, count)| *count > 0)
132            .fold(None, |best, candidate| match best {
133                Some((_, best_count)) if best_count >= candidate.1 => best,
134                _ => Some(candidate),
135            })
136    }
137
138    /// Share of classified values held by [`Self::dominant`], in `0.0..=1.0`.
139    /// `None` when nothing was classified, where the ratio is undefined.
140    pub fn dominant_share(&self) -> Option<f64> {
141        let (_, count) = self.dominant()?;
142        Some(count as f64 / self.classified_count() as f64)
143    }
144
145    /// Every class that holds at least one value, largest first, with its share
146    /// of the classified values. Empty when nothing was classified.
147    ///
148    /// Ties keep declaration order, so the sequence is stable across runs.
149    pub fn mixture(&self) -> Vec<(LexicalClass, usize, f64)> {
150        let total = self.classified_count();
151        if total == 0 {
152            return Vec::new();
153        }
154        let mut present: Vec<(LexicalClass, usize)> = self
155            .counts()
156            .into_iter()
157            .filter(|(_, count)| *count > 0)
158            .collect();
159        // Stable sort on the descending count keeps declaration order for ties.
160        present.sort_by_key(|(_, count)| std::cmp::Reverse(*count));
161        present
162            .into_iter()
163            .map(|(class, count)| (class, count, count as f64 / total as f64))
164            .collect()
165    }
166}
167
168/// Semantic category for a detected pattern.
169#[derive(
170    Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize, schemars::JsonSchema,
171)]
172#[serde(rename_all = "snake_case")]
173pub enum PatternCategory {
174    /// Email addresses, phone numbers.
175    Contact,
176    /// UUIDs, fiscal codes, tax IDs.
177    Identifier,
178    /// IPv4, IPv6, MAC addresses, URLs.
179    Network,
180    /// Coordinates and postal codes.
181    Geographic,
182    /// IBANs, credit cards, SWIFT/BIC.
183    Financial,
184    /// Unix or Windows file paths.
185    FilePath,
186    /// Uncategorized patterns.
187    Other,
188}
189
190impl std::fmt::Display for PatternCategory {
191    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
192        match self {
193            Self::Contact => write!(f, "contact"),
194            Self::Identifier => write!(f, "identifier"),
195            Self::Network => write!(f, "network"),
196            Self::Geographic => write!(f, "geographic"),
197            Self::Financial => write!(f, "financial"),
198            Self::FilePath => write!(f, "file_path"),
199            Self::Other => write!(f, "other"),
200        }
201    }
202}
203
204#[cfg(test)]
205mod type_homogeneity_tests {
206    use super::*;
207
208    fn homogeneity(numeric: usize, date: usize, boolean: usize, text: usize) -> TypeHomogeneity {
209        TypeHomogeneity {
210            numeric,
211            date,
212            boolean,
213            text,
214        }
215    }
216
217    #[test]
218    fn recording_values_counts_them_by_class() {
219        let mut counts = TypeHomogeneity::default();
220        counts.record(LexicalClass::Numeric);
221        counts.record(LexicalClass::Text);
222        counts.record(LexicalClass::Numeric);
223
224        assert_eq!(counts, homogeneity(2, 0, 0, 1));
225        assert_eq!(counts.classified_count(), 3);
226        assert_eq!(counts.dominant(), Some((LexicalClass::Numeric, 2)));
227    }
228
229    #[test]
230    fn nothing_classified_has_no_dominant_class_and_no_share() {
231        // The all-zero value is "classified, nothing to classify" — every
232        // derived answer is absent rather than a number invented from a zero
233        // denominator.
234        let empty = TypeHomogeneity::default();
235
236        assert_eq!(empty.classified_count(), 0);
237        assert_eq!(empty.dominant(), None);
238        assert_eq!(empty.dominant_share(), None);
239        assert!(empty.mixture().is_empty());
240    }
241
242    #[test]
243    fn a_tie_is_held_by_the_earliest_declared_class() {
244        // Which class wins a tie does not change the share — both hold the same
245        // count — but it must be the same class on every run, and the same one
246        // the consistency dimension scores against.
247        let tied = homogeneity(50, 50, 0, 0);
248
249        assert_eq!(tied.dominant(), Some((LexicalClass::Numeric, 50)));
250        assert_eq!(tied.dominant_share(), Some(0.5));
251        assert_eq!(
252            tied.mixture(),
253            vec![
254                (LexicalClass::Numeric, 50, 0.5),
255                (LexicalClass::Date, 50, 0.5)
256            ]
257        );
258    }
259
260    #[test]
261    fn mixture_reports_every_present_class_largest_first() {
262        let mixed = homogeneity(60, 10, 0, 30);
263
264        assert_eq!(mixed.dominant_share(), Some(0.6));
265        assert_eq!(
266            mixed.mixture(),
267            vec![
268                (LexicalClass::Numeric, 60, 0.6),
269                (LexicalClass::Text, 30, 0.3),
270                (LexicalClass::Date, 10, 0.1),
271            ],
272            "an absent class must not appear with a 0% share"
273        );
274    }
275
276    #[test]
277    fn counts_survive_a_json_round_trip() {
278        let counts = homogeneity(600, 0, 0, 400);
279        let json = serde_json::to_string(&counts).expect("counts should serialize");
280
281        assert_eq!(json, r#"{"numeric":600,"date":0,"boolean":0,"text":400}"#);
282        assert_eq!(
283            serde_json::from_str::<TypeHomogeneity>(&json).expect("counts should deserialize"),
284            counts
285        );
286    }
287
288    #[test]
289    fn lexical_classes_serialize_as_the_names_reports_use() {
290        for (class, name) in [
291            (LexicalClass::Numeric, "numeric"),
292            (LexicalClass::Date, "date"),
293            (LexicalClass::Boolean, "boolean"),
294            (LexicalClass::Text, "text"),
295        ] {
296            assert_eq!(class.to_string(), name);
297            assert_eq!(
298                serde_json::to_string(&class).expect("class should serialize"),
299                format!("\"{name}\"")
300            );
301        }
302    }
303}
dataprof_core/classification.rs

dataprof_core/
classification.rs
