Skip to main content

dataprof_core/
quality.rs

1/// ISO 25012 quality dimensions that can be selectively requested.
2#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]
3#[serde(rename_all = "lowercase")]
4pub enum QualityDimension {
5    Completeness,
6    Consistency,
7    Uniqueness,
8    Accuracy,
9    Timeliness,
10    Validity,
11    Precision,
12}
13
14impl QualityDimension {
15    /// All currently implemented dimensions.
16    pub fn all() -> Vec<Self> {
17        vec![
18            Self::Completeness,
19            Self::Consistency,
20            Self::Uniqueness,
21            Self::Accuracy,
22            Self::Timeliness,
23            Self::Validity,
24            Self::Precision,
25        ]
26    }
27}
28
29impl std::str::FromStr for QualityDimension {
30    type Err = String;
31
32    fn from_str(s: &str) -> Result<Self, Self::Err> {
33        match s.to_lowercase().as_str() {
34            "completeness" => Ok(Self::Completeness),
35            "consistency" => Ok(Self::Consistency),
36            "uniqueness" => Ok(Self::Uniqueness),
37            "accuracy" => Ok(Self::Accuracy),
38            "timeliness" => Ok(Self::Timeliness),
39            "validity" => Ok(Self::Validity),
40            "precision" => Ok(Self::Precision),
41            _ => Err(format!("Unknown quality dimension: {s}")),
42        }
43    }
44}
45
46impl std::fmt::Display for QualityDimension {
47    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
48        match self {
49            Self::Completeness => write!(f, "completeness"),
50            Self::Consistency => write!(f, "consistency"),
51            Self::Uniqueness => write!(f, "uniqueness"),
52            Self::Accuracy => write!(f, "accuracy"),
53            Self::Timeliness => write!(f, "timeliness"),
54            Self::Validity => write!(f, "validity"),
55            Self::Precision => write!(f, "precision"),
56        }
57    }
58}
59
60/// What the range and sign rules of the accuracy score count, as recorded in
61/// a report's metric semantics.
62///
63/// Only the current definition has a value. Through 0.12 every rule failure
64/// counted, a value breaking two rules twice, and the failures were divided
65/// by the numeric values of every column, so clean columns diluted a broken
66/// one; those releases recorded no definition.
67#[derive(
68    Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize, schemars::JsonSchema,
69)]
70#[serde(rename_all = "snake_case")]
71pub enum RangeRuleCount {
72    /// Values breaking at least one rule, each counted once, over the values
73    /// of the columns a rule applies to.
74    ViolatingValues,
75}
76
77/// What `key_uniqueness` divides the key column's distinct values by, as
78/// recorded in a report's metric semantics.
79///
80/// Only the current definition has a value. Through 0.12 it divided by every
81/// row, nulls included, so a nullable key lost uniqueness for each null the
82/// completeness score already counted; those releases recorded no
83/// definition.
84#[derive(
85    Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize, schemars::JsonSchema,
86)]
87#[serde(rename_all = "snake_case")]
88pub enum KeyUniquenessUnits {
89    /// The key column's non-null values.
90    NonNullValues,
91}
92
93/// What `outlier_ratio` is measured over, as recorded in a report's metric
94/// semantics.
95///
96/// Only the current definition has a value. Through 0.12 a source in which no
97/// column reached `outlier_min_samples` reported `outlier_ratio` 0, and its
98/// numeric values alone made the accuracy dimension assessed; those releases
99/// recorded no definition.
100#[derive(
101    Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize, schemars::JsonSchema,
102)]
103#[serde(rename_all = "snake_case")]
104pub enum OutlierUnits {
105    /// The values of the columns that reached `outlier_min_samples`. With
106    /// none, the component is absent rather than 0.
107    TestedValues,
108}
109
110/// Which column `key_uniqueness` may measure, as recorded in a report's
111/// metric semantics.
112///
113/// Only the current definition has a value. Through 0.12, with no column
114/// declared, the first column whose name had an `id`, `key` or `pk` word was
115/// the key, so a foreign key listed first was scored as the table key and
116/// column order moved the score; those releases recorded no definition.
117#[derive(
118    Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize, schemars::JsonSchema,
119)]
120#[serde(rename_all = "snake_case")]
121pub enum KeyColumnSource {
122    /// The first `identifier_columns` entry the source has. With none
123    /// declared there is no key component.
124    Declared,
125}
126
127/// How the accuracy score compares a value with its outlier fences and its
128/// range and sign bounds, as recorded in a report's metric semantics.
129///
130/// Only the current definition has a value. Through 0.13 both comparisons
131/// went through the value's nearest `f64`, so a 64-bit integer past 2^53, a
132/// decimal held exactly, or a literal one digit past a bound could land on
133/// the other side of a fence or bound; those releases recorded no definition.
134#[derive(
135    Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize, schemars::JsonSchema,
136)]
137#[serde(rename_all = "snake_case")]
138pub enum AccuracyComparison {
139    /// Each value as written: exact Tukey fences, and bounds compared by the
140    /// value's digits where its nearest `f64` equals the bound. A literal
141    /// that overflows `f64` is a numeric value past every bound.
142    ExactValues,
143}
144
145/// How a JSON number whose value its nearest `f64` does not hold is read, as
146/// recorded in a report's metric semantics.
147///
148/// Only the current definition has a value. Through 0.13 such a number, like
149/// `1000000000000000.01` or `1e-400`, was read as its nearest `f64`, so
150/// distinct literals could count as one value and lose their decimal places;
151/// those releases recorded no definition.
152#[derive(
153    Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize, schemars::JsonSchema,
154)]
155#[serde(rename_all = "snake_case")]
156pub enum JsonNumberReading {
157    /// Its literal digits, as a CSV cell with the same text is read. Covers a
158    /// field of a JSON or JSONL record on every route and a database JSON
159    /// cell; a number nested in an array or object keeps its `f64`.
160    LiteralDigits,
161}
162
163/// How a quality score computed over a retained sample estimates the score
164/// over every scanned value, as recorded in a report's metric semantics.
165///
166/// Only the current definition has a value. Through 0.13 a sampled score
167/// pooled every column's reservoir, so a column weighed as many values as its
168/// reservoir kept, at most 10,000, and the score could fall outside its own
169/// `score_bounds`; those releases recorded no definition.
170#[derive(
171    Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize, schemars::JsonSchema,
172)]
173#[serde(rename_all = "snake_case")]
174pub enum SampledScoreEstimator {
175    /// Each column's sampled failure and checked counts are scaled to its
176    /// non-null values in the source before the shares are formed (#1293).
177    /// A source whose samples hold every value scores as before.
178    PopulationWeighted,
179}
180
181/// Which values a date column's statistics are computed over, as recorded
182/// in a report's metric semantics.
183///
184/// Only the current definition has a value. Through 0.13 the engines that
185/// retain a bounded sample of each column computed `min_datetime`,
186/// `max_datetime`, `duration_days`, the year, month, weekday and hour
187/// distributions and `slash_date_order` from that sample, so on a source
188/// larger than the sample they described the sample; those releases recorded
189/// no definition.
190#[derive(
191    Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize, schemars::JsonSchema,
192)]
193#[serde(rename_all = "snake_case")]
194pub enum DateStatistics {
195    /// Every non-null value the scan read, on every engine (#1297). The
196    /// `largest_gap` added in 0.14 (#1300) is computed over them too.
197    EveryValue,
198}
199
200/// Which columns the outlier test behind `outlier_ratio` runs on, as
201/// recorded in a report's metric semantics.
202///
203/// Only the current definition has a value. Through 0.13 a column whose first
204/// and third quartiles were one value was tested too: its IQR was 0, so
205/// every value off that quartile was an outlier, and a column of 75% zeros
206/// and 25% fives reported a quarter of its values as outliers. Those releases
207/// recorded no definition.
208#[derive(
209    Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize, schemars::JsonSchema,
210)]
211#[serde(rename_all = "snake_case")]
212pub enum OutlierSpread {
213    /// Columns whose first and third quartiles differ (#1304). A column whose
214    /// quartiles coincide is not tested, as one below `outlier_min_samples`
215    /// is not, and adds nothing to `outlier_values_checked`.
216    NonzeroIqr,
217}
218
219/// Which text forms read as dates, as recorded in a report's metric
220/// semantics. Type inference, the consistency check and the timeliness
221/// counts read them.
222///
223/// Only the current definition has a value. Through 0.13, and on 0.14
224/// release candidates before #1340, the ISO datetime with a space for the `T`
225/// was a date only in its plain whole-second form: with a fraction or an
226/// offset (`2026-06-01 12:00:00+00:00`, what Python's `str()` writes for an
227/// aware datetime) it was text. Those reports recorded no definition.
228#[derive(
229    Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize, schemars::JsonSchema,
230)]
231#[serde(rename_all = "snake_case")]
232pub enum DateForms {
233    /// The spaced ISO datetime reads as a date with a fraction and an offset,
234    /// as the `T` form does (#1335).
235    SpacedIsoDatetimes,
236}
237
238/// High-level categories of analysis that can be selectively enabled.
239#[derive(
240    Debug,
241    Clone,
242    Copy,
243    PartialEq,
244    Eq,
245    Hash,
246    serde::Serialize,
247    serde::Deserialize,
248    schemars::JsonSchema,
249)]
250#[serde(rename_all = "lowercase")]
251pub enum MetricPack {
252    /// Column names, data types, null counts — always included.
253    Schema,
254    /// Numeric stats (min/max/mean/median/std_dev/quartiles), text lengths.
255    Statistics,
256    /// Regex pattern detection (email, phone, UUID, etc.).
257    Patterns,
258    /// ISO 25012 quality dimensions (completeness, consistency, etc.).
259    Quality,
260}
261
262impl MetricPack {
263    /// All available metric packs.
264    pub fn all() -> Vec<Self> {
265        vec![
266            Self::Schema,
267            Self::Statistics,
268            Self::Patterns,
269            Self::Quality,
270        ]
271    }
272
273    /// Whether statistics should be computed given the selected packs.
274    pub fn include_statistics(packs: Option<&[Self]>) -> bool {
275        match packs {
276            None => true,
277            Some(p) => p.contains(&Self::Statistics),
278        }
279    }
280
281    /// Whether pattern detection should run given the selected packs.
282    pub fn include_patterns(packs: Option<&[Self]>) -> bool {
283        match packs {
284            None => true,
285            Some(p) => p.contains(&Self::Patterns),
286        }
287    }
288
289    /// Whether quality metrics should be computed given the selected packs.
290    pub fn include_quality(packs: Option<&[Self]>) -> bool {
291        match packs {
292            None => true,
293            Some(p) => p.contains(&Self::Quality),
294        }
295    }
296
297    /// Resolve the packs to compute, folding in an explicit quality-dimension
298    /// selection.
299    ///
300    /// Requesting an empty set of dimensions (`Some(&[])`) says "assess no
301    /// dimension", which is the same statement as not selecting the quality
302    /// pack: nothing is analyzed, so the report carries no quality. Without
303    /// this, an empty selection produces a quality object whose dimensions are
304    /// all absent — an artifact that reads like a measured result.
305    ///
306    /// This is deliberately about what was *requested*. A caller that asks for
307    /// dimensions and gets none back (an empty dataset, no date column) was
308    /// still analyzed, and keeps a quality object reporting exactly that.
309    pub fn resolve_with_dimensions(
310        packs: Option<&[Self]>,
311        dimensions: Option<&[QualityDimension]>,
312    ) -> Option<Vec<Self>> {
313        if !dimensions.is_some_and(<[QualityDimension]>::is_empty) {
314            return packs.map(<[Self]>::to_vec);
315        }
316        // `None` packs means "all packs"; make that explicit before removing
317        // Quality, so the result still selects everything else.
318        let selected = packs.map_or_else(Self::all, <[Self]>::to_vec);
319        Some(
320            selected
321                .into_iter()
322                .filter(|pack| *pack != Self::Quality)
323                .collect(),
324        )
325    }
326}
327
328impl std::str::FromStr for MetricPack {
329    type Err = String;
330
331    fn from_str(s: &str) -> Result<Self, Self::Err> {
332        match s.to_lowercase().as_str() {
333            "schema" => Ok(Self::Schema),
334            "statistics" => Ok(Self::Statistics),
335            "patterns" => Ok(Self::Patterns),
336            "quality" => Ok(Self::Quality),
337            _ => Err(format!(
338                "Unknown metric pack: {s}. Valid packs: schema, statistics, patterns, quality"
339            )),
340        }
341    }
342}
343
344impl std::fmt::Display for MetricPack {
345    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
346        match self {
347            Self::Schema => write!(f, "schema"),
348            Self::Statistics => write!(f, "statistics"),
349            Self::Patterns => write!(f, "patterns"),
350            Self::Quality => write!(f, "quality"),
351        }
352    }
353}
354
355#[cfg(test)]
356mod tests {
357    use super::*;
358
359    #[test]
360    fn test_metric_pack_include_helpers_none_means_all() {
361        assert!(MetricPack::include_statistics(None));
362        assert!(MetricPack::include_patterns(None));
363        assert!(MetricPack::include_quality(None));
364    }
365
366    #[test]
367    fn test_resolve_with_dimensions_empty_selection_drops_quality() {
368        // Requesting no dimension is the same statement as not asking for
369        // quality at all, so the pack must fall out and the report carry no
370        // quality object.
371        let resolved = MetricPack::resolve_with_dimensions(None, Some(&[])).unwrap();
372        assert!(!resolved.contains(&MetricPack::Quality));
373        assert!(!MetricPack::include_quality(Some(&resolved)));
374        // The other packs survive: only quality was deselected.
375        assert!(resolved.contains(&MetricPack::Schema));
376        assert!(resolved.contains(&MetricPack::Statistics));
377        assert!(resolved.contains(&MetricPack::Patterns));
378    }
379
380    #[test]
381    fn test_resolve_with_dimensions_empty_selection_keeps_explicit_packs() {
382        let packs = vec![MetricPack::Schema, MetricPack::Quality];
383        let resolved = MetricPack::resolve_with_dimensions(Some(&packs), Some(&[])).unwrap();
384        assert_eq!(resolved, vec![MetricPack::Schema]);
385    }
386
387    #[test]
388    fn test_resolve_with_dimensions_non_empty_selection_is_untouched() {
389        // A real selection says which dimensions to assess, not whether to.
390        let dims = vec![QualityDimension::Completeness];
391        assert_eq!(MetricPack::resolve_with_dimensions(None, Some(&dims)), None);
392        assert!(MetricPack::include_quality(None));
393
394        let packs = vec![MetricPack::Quality];
395        let resolved = MetricPack::resolve_with_dimensions(Some(&packs), Some(&dims)).unwrap();
396        assert!(MetricPack::include_quality(Some(&resolved)));
397    }
398
399    #[test]
400    fn test_resolve_with_dimensions_absent_selection_is_untouched() {
401        // `None` means "all dimensions", which is not the same as "none".
402        assert_eq!(MetricPack::resolve_with_dimensions(None, None), None);
403        let packs = vec![MetricPack::Quality];
404        assert_eq!(
405            MetricPack::resolve_with_dimensions(Some(&packs), None),
406            Some(packs)
407        );
408    }
409
410    #[test]
411    fn test_metric_pack_include_helpers_selective() {
412        let packs = vec![MetricPack::Schema, MetricPack::Quality];
413        assert!(!MetricPack::include_statistics(Some(&packs)));
414        assert!(!MetricPack::include_patterns(Some(&packs)));
415        assert!(MetricPack::include_quality(Some(&packs)));
416    }
417
418    #[test]
419    fn test_metric_pack_from_str() {
420        assert_eq!(
421            "statistics".parse::<MetricPack>().unwrap(),
422            MetricPack::Statistics
423        );
424        assert_eq!(
425            "QUALITY".parse::<MetricPack>().unwrap(),
426            MetricPack::Quality
427        );
428        assert!("invalid".parse::<MetricPack>().is_err());
429    }
430}