dataprof-core 0.14.1

Shared core types for dataprof
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
/// ISO 25012 quality dimensions that can be selectively requested.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]
#[serde(rename_all = "lowercase")]
pub enum QualityDimension {
    Completeness,
    Consistency,
    Uniqueness,
    Accuracy,
    Timeliness,
    Validity,
    Precision,
}

impl QualityDimension {
    /// All currently implemented dimensions.
    pub fn all() -> Vec<Self> {
        vec![
            Self::Completeness,
            Self::Consistency,
            Self::Uniqueness,
            Self::Accuracy,
            Self::Timeliness,
            Self::Validity,
            Self::Precision,
        ]
    }
}

impl std::str::FromStr for QualityDimension {
    type Err = String;

    fn from_str(s: &str) -> Result<Self, Self::Err> {
        match s.to_lowercase().as_str() {
            "completeness" => Ok(Self::Completeness),
            "consistency" => Ok(Self::Consistency),
            "uniqueness" => Ok(Self::Uniqueness),
            "accuracy" => Ok(Self::Accuracy),
            "timeliness" => Ok(Self::Timeliness),
            "validity" => Ok(Self::Validity),
            "precision" => Ok(Self::Precision),
            _ => Err(format!("Unknown quality dimension: {s}")),
        }
    }
}

impl std::fmt::Display for QualityDimension {
    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        match self {
            Self::Completeness => write!(f, "completeness"),
            Self::Consistency => write!(f, "consistency"),
            Self::Uniqueness => write!(f, "uniqueness"),
            Self::Accuracy => write!(f, "accuracy"),
            Self::Timeliness => write!(f, "timeliness"),
            Self::Validity => write!(f, "validity"),
            Self::Precision => write!(f, "precision"),
        }
    }
}

/// What the range and sign rules of the accuracy score count, as recorded in
/// a report's metric semantics.
///
/// Only the current definition has a value. Through 0.12 every rule failure
/// counted, a value breaking two rules twice, and the failures were divided
/// by the numeric values of every column, so clean columns diluted a broken
/// one; those releases recorded no definition.
#[derive(
    Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize, schemars::JsonSchema,
)]
#[serde(rename_all = "snake_case")]
pub enum RangeRuleCount {
    /// Values breaking at least one rule, each counted once, over the values
    /// of the columns a rule applies to.
    ViolatingValues,
}

/// What `key_uniqueness` divides the key column's distinct values by, as
/// recorded in a report's metric semantics.
///
/// Only the current definition has a value. Through 0.12 it divided by every
/// row, nulls included, so a nullable key lost uniqueness for each null the
/// completeness score already counted; those releases recorded no
/// definition.
#[derive(
    Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize, schemars::JsonSchema,
)]
#[serde(rename_all = "snake_case")]
pub enum KeyUniquenessUnits {
    /// The key column's non-null values.
    NonNullValues,
}

/// What `outlier_ratio` is measured over, as recorded in a report's metric
/// semantics.
///
/// Only the current definition has a value. Through 0.12 a source in which no
/// column reached `outlier_min_samples` reported `outlier_ratio` 0, and its
/// numeric values alone made the accuracy dimension assessed; those releases
/// recorded no definition.
#[derive(
    Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize, schemars::JsonSchema,
)]
#[serde(rename_all = "snake_case")]
pub enum OutlierUnits {
    /// The values of the columns that reached `outlier_min_samples`. With
    /// none, the component is absent rather than 0.
    TestedValues,
}

/// Which column `key_uniqueness` may measure, as recorded in a report's
/// metric semantics.
///
/// Only the current definition has a value. Through 0.12, with no column
/// declared, the first column whose name had an `id`, `key` or `pk` word was
/// the key, so a foreign key listed first was scored as the table key and
/// column order moved the score; those releases recorded no definition.
#[derive(
    Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize, schemars::JsonSchema,
)]
#[serde(rename_all = "snake_case")]
pub enum KeyColumnSource {
    /// The first `identifier_columns` entry the source has. With none
    /// declared there is no key component.
    Declared,
}

/// How the accuracy score compares a value with its outlier fences and its
/// range and sign bounds, as recorded in a report's metric semantics.
///
/// Only the current definition has a value. Through 0.13 both comparisons
/// went through the value's nearest `f64`, so a 64-bit integer past 2^53, a
/// decimal held exactly, or a literal one digit past a bound could land on
/// the other side of a fence or bound; those releases recorded no definition.
#[derive(
    Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize, schemars::JsonSchema,
)]
#[serde(rename_all = "snake_case")]
pub enum AccuracyComparison {
    /// Each value as written: exact Tukey fences, and bounds compared by the
    /// value's digits where its nearest `f64` equals the bound. A literal
    /// that overflows `f64` is a numeric value past every bound.
    ExactValues,
}

/// How a JSON number whose value its nearest `f64` does not hold is read, as
/// recorded in a report's metric semantics.
///
/// Only the current definition has a value. Through 0.13 such a number, like
/// `1000000000000000.01` or `1e-400`, was read as its nearest `f64`, so
/// distinct literals could count as one value and lose their decimal places;
/// those releases recorded no definition.
#[derive(
    Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize, schemars::JsonSchema,
)]
#[serde(rename_all = "snake_case")]
pub enum JsonNumberReading {
    /// Its literal digits, as a CSV cell with the same text is read. Covers a
    /// field of a JSON or JSONL record on every route and a database JSON
    /// cell; a number nested in an array or object keeps its `f64`.
    LiteralDigits,
}

/// How a quality score computed over a retained sample estimates the score
/// over every scanned value, as recorded in a report's metric semantics.
///
/// Only the current definition has a value. Through 0.13 a sampled score
/// pooled every column's reservoir, so a column weighed as many values as its
/// reservoir kept, at most 10,000, and the score could fall outside its own
/// `score_bounds`; those releases recorded no definition.
#[derive(
    Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize, schemars::JsonSchema,
)]
#[serde(rename_all = "snake_case")]
pub enum SampledScoreEstimator {
    /// Each column's sampled failure and checked counts are scaled to its
    /// non-null values in the source before the shares are formed (#1293).
    /// A source whose samples hold every value scores as before.
    PopulationWeighted,
}

/// Which values a date column's statistics are computed over, as recorded
/// in a report's metric semantics.
///
/// Only the current definition has a value. Through 0.13 the engines that
/// retain a bounded sample of each column computed `min_datetime`,
/// `max_datetime`, `duration_days`, the year, month, weekday and hour
/// distributions and `slash_date_order` from that sample, so on a source
/// larger than the sample they described the sample; those releases recorded
/// no definition.
#[derive(
    Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize, schemars::JsonSchema,
)]
#[serde(rename_all = "snake_case")]
pub enum DateStatistics {
    /// Every non-null value the scan read, on every engine (#1297). The
    /// `largest_gap` added in 0.14 (#1300) is computed over them too.
    EveryValue,
}

/// Which columns the outlier test behind `outlier_ratio` runs on, as
/// recorded in a report's metric semantics.
///
/// Only the current definition has a value. Through 0.13 a column whose first
/// and third quartiles were one value was tested too: its IQR was 0, so
/// every value off that quartile was an outlier, and a column of 75% zeros
/// and 25% fives reported a quarter of its values as outliers. Those releases
/// recorded no definition.
#[derive(
    Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize, schemars::JsonSchema,
)]
#[serde(rename_all = "snake_case")]
pub enum OutlierSpread {
    /// Columns whose first and third quartiles differ (#1304). A column whose
    /// quartiles coincide is not tested, as one below `outlier_min_samples`
    /// is not, and adds nothing to `outlier_values_checked`.
    NonzeroIqr,
}

/// Which text forms read as dates, as recorded in a report's metric
/// semantics. Type inference, the consistency check and the timeliness
/// counts read them.
///
/// Only the current definition has a value. Through 0.13, and on 0.14
/// release candidates before #1340, the ISO datetime with a space for the `T`
/// was a date only in its plain whole-second form: with a fraction or an
/// offset (`2026-06-01 12:00:00+00:00`, what Python's `str()` writes for an
/// aware datetime) it was text. Those reports recorded no definition.
#[derive(
    Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize, schemars::JsonSchema,
)]
#[serde(rename_all = "snake_case")]
pub enum DateForms {
    /// The spaced ISO datetime reads as a date with a fraction and an offset,
    /// as the `T` form does (#1335).
    SpacedIsoDatetimes,
}

/// High-level categories of analysis that can be selectively enabled.
#[derive(
    Debug,
    Clone,
    Copy,
    PartialEq,
    Eq,
    Hash,
    serde::Serialize,
    serde::Deserialize,
    schemars::JsonSchema,
)]
#[serde(rename_all = "lowercase")]
pub enum MetricPack {
    /// Column names, data types, null counts — always included.
    Schema,
    /// Numeric stats (min/max/mean/median/std_dev/quartiles), text lengths.
    Statistics,
    /// Regex pattern detection (email, phone, UUID, etc.).
    Patterns,
    /// ISO 25012 quality dimensions (completeness, consistency, etc.).
    Quality,
}

impl MetricPack {
    /// All available metric packs.
    pub fn all() -> Vec<Self> {
        vec![
            Self::Schema,
            Self::Statistics,
            Self::Patterns,
            Self::Quality,
        ]
    }

    /// Whether statistics should be computed given the selected packs.
    pub fn include_statistics(packs: Option<&[Self]>) -> bool {
        match packs {
            None => true,
            Some(p) => p.contains(&Self::Statistics),
        }
    }

    /// Whether pattern detection should run given the selected packs.
    pub fn include_patterns(packs: Option<&[Self]>) -> bool {
        match packs {
            None => true,
            Some(p) => p.contains(&Self::Patterns),
        }
    }

    /// Whether quality metrics should be computed given the selected packs.
    pub fn include_quality(packs: Option<&[Self]>) -> bool {
        match packs {
            None => true,
            Some(p) => p.contains(&Self::Quality),
        }
    }

    /// Resolve the packs to compute, folding in an explicit quality-dimension
    /// selection.
    ///
    /// Requesting an empty set of dimensions (`Some(&[])`) says "assess no
    /// dimension", which is the same statement as not selecting the quality
    /// pack: nothing is analyzed, so the report carries no quality. Without
    /// this, an empty selection produces a quality object whose dimensions are
    /// all absent — an artifact that reads like a measured result.
    ///
    /// This is deliberately about what was *requested*. A caller that asks for
    /// dimensions and gets none back (an empty dataset, no date column) was
    /// still analyzed, and keeps a quality object reporting exactly that.
    pub fn resolve_with_dimensions(
        packs: Option<&[Self]>,
        dimensions: Option<&[QualityDimension]>,
    ) -> Option<Vec<Self>> {
        if !dimensions.is_some_and(<[QualityDimension]>::is_empty) {
            return packs.map(<[Self]>::to_vec);
        }
        // `None` packs means "all packs"; make that explicit before removing
        // Quality, so the result still selects everything else.
        let selected = packs.map_or_else(Self::all, <[Self]>::to_vec);
        Some(
            selected
                .into_iter()
                .filter(|pack| *pack != Self::Quality)
                .collect(),
        )
    }
}

impl std::str::FromStr for MetricPack {
    type Err = String;

    fn from_str(s: &str) -> Result<Self, Self::Err> {
        match s.to_lowercase().as_str() {
            "schema" => Ok(Self::Schema),
            "statistics" => Ok(Self::Statistics),
            "patterns" => Ok(Self::Patterns),
            "quality" => Ok(Self::Quality),
            _ => Err(format!(
                "Unknown metric pack: {s}. Valid packs: schema, statistics, patterns, quality"
            )),
        }
    }
}

impl std::fmt::Display for MetricPack {
    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        match self {
            Self::Schema => write!(f, "schema"),
            Self::Statistics => write!(f, "statistics"),
            Self::Patterns => write!(f, "patterns"),
            Self::Quality => write!(f, "quality"),
        }
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn test_metric_pack_include_helpers_none_means_all() {
        assert!(MetricPack::include_statistics(None));
        assert!(MetricPack::include_patterns(None));
        assert!(MetricPack::include_quality(None));
    }

    #[test]
    fn test_resolve_with_dimensions_empty_selection_drops_quality() {
        // Requesting no dimension is the same statement as not asking for
        // quality at all, so the pack must fall out and the report carry no
        // quality object.
        let resolved = MetricPack::resolve_with_dimensions(None, Some(&[])).unwrap();
        assert!(!resolved.contains(&MetricPack::Quality));
        assert!(!MetricPack::include_quality(Some(&resolved)));
        // The other packs survive: only quality was deselected.
        assert!(resolved.contains(&MetricPack::Schema));
        assert!(resolved.contains(&MetricPack::Statistics));
        assert!(resolved.contains(&MetricPack::Patterns));
    }

    #[test]
    fn test_resolve_with_dimensions_empty_selection_keeps_explicit_packs() {
        let packs = vec![MetricPack::Schema, MetricPack::Quality];
        let resolved = MetricPack::resolve_with_dimensions(Some(&packs), Some(&[])).unwrap();
        assert_eq!(resolved, vec![MetricPack::Schema]);
    }

    #[test]
    fn test_resolve_with_dimensions_non_empty_selection_is_untouched() {
        // A real selection says which dimensions to assess, not whether to.
        let dims = vec![QualityDimension::Completeness];
        assert_eq!(MetricPack::resolve_with_dimensions(None, Some(&dims)), None);
        assert!(MetricPack::include_quality(None));

        let packs = vec![MetricPack::Quality];
        let resolved = MetricPack::resolve_with_dimensions(Some(&packs), Some(&dims)).unwrap();
        assert!(MetricPack::include_quality(Some(&resolved)));
    }

    #[test]
    fn test_resolve_with_dimensions_absent_selection_is_untouched() {
        // `None` means "all dimensions", which is not the same as "none".
        assert_eq!(MetricPack::resolve_with_dimensions(None, None), None);
        let packs = vec![MetricPack::Quality];
        assert_eq!(
            MetricPack::resolve_with_dimensions(Some(&packs), None),
            Some(packs)
        );
    }

    #[test]
    fn test_metric_pack_include_helpers_selective() {
        let packs = vec![MetricPack::Schema, MetricPack::Quality];
        assert!(!MetricPack::include_statistics(Some(&packs)));
        assert!(!MetricPack::include_patterns(Some(&packs)));
        assert!(MetricPack::include_quality(Some(&packs)));
    }

    #[test]
    fn test_metric_pack_from_str() {
        assert_eq!(
            "statistics".parse::<MetricPack>().unwrap(),
            MetricPack::Statistics
        );
        assert_eq!(
            "QUALITY".parse::<MetricPack>().unwrap(),
            MetricPack::Quality
        );
        assert!("invalid".parse::<MetricPack>().is_err());
    }
}