dataprof_core/quality.rs
1/// ISO 25012 quality dimensions that can be selectively requested.
2#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]
3#[serde(rename_all = "lowercase")]
4pub enum QualityDimension {
5 Completeness,
6 Consistency,
7 Uniqueness,
8 Accuracy,
9 Timeliness,
10 Validity,
11 Precision,
12}
13
14impl QualityDimension {
15 /// All currently implemented dimensions.
16 pub fn all() -> Vec<Self> {
17 vec![
18 Self::Completeness,
19 Self::Consistency,
20 Self::Uniqueness,
21 Self::Accuracy,
22 Self::Timeliness,
23 Self::Validity,
24 Self::Precision,
25 ]
26 }
27}
28
29impl std::str::FromStr for QualityDimension {
30 type Err = String;
31
32 fn from_str(s: &str) -> Result<Self, Self::Err> {
33 match s.to_lowercase().as_str() {
34 "completeness" => Ok(Self::Completeness),
35 "consistency" => Ok(Self::Consistency),
36 "uniqueness" => Ok(Self::Uniqueness),
37 "accuracy" => Ok(Self::Accuracy),
38 "timeliness" => Ok(Self::Timeliness),
39 "validity" => Ok(Self::Validity),
40 "precision" => Ok(Self::Precision),
41 _ => Err(format!("Unknown quality dimension: {s}")),
42 }
43 }
44}
45
46impl std::fmt::Display for QualityDimension {
47 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
48 match self {
49 Self::Completeness => write!(f, "completeness"),
50 Self::Consistency => write!(f, "consistency"),
51 Self::Uniqueness => write!(f, "uniqueness"),
52 Self::Accuracy => write!(f, "accuracy"),
53 Self::Timeliness => write!(f, "timeliness"),
54 Self::Validity => write!(f, "validity"),
55 Self::Precision => write!(f, "precision"),
56 }
57 }
58}
59
60/// What the range and sign rules of the accuracy score count, as recorded in
61/// a report's metric semantics.
62///
63/// Only the current definition has a value. Through 0.12 every rule failure
64/// counted, a value breaking two rules twice, and the failures were divided
65/// by the numeric values of every column, so clean columns diluted a broken
66/// one; those releases recorded no definition.
67#[derive(
68 Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize, schemars::JsonSchema,
69)]
70#[serde(rename_all = "snake_case")]
71pub enum RangeRuleCount {
72 /// Values breaking at least one rule, each counted once, over the values
73 /// of the columns a rule applies to.
74 ViolatingValues,
75}
76
77/// What `key_uniqueness` divides the key column's distinct values by, as
78/// recorded in a report's metric semantics.
79///
80/// Only the current definition has a value. Through 0.12 it divided by every
81/// row, nulls included, so a nullable key lost uniqueness for each null the
82/// completeness score already counted; those releases recorded no
83/// definition.
84#[derive(
85 Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize, schemars::JsonSchema,
86)]
87#[serde(rename_all = "snake_case")]
88pub enum KeyUniquenessUnits {
89 /// The key column's non-null values.
90 NonNullValues,
91}
92
93/// What `outlier_ratio` is measured over, as recorded in a report's metric
94/// semantics.
95///
96/// Only the current definition has a value. Through 0.12 a source in which no
97/// column reached `outlier_min_samples` reported `outlier_ratio` 0, and its
98/// numeric values alone made the accuracy dimension assessed; those releases
99/// recorded no definition.
100#[derive(
101 Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize, schemars::JsonSchema,
102)]
103#[serde(rename_all = "snake_case")]
104pub enum OutlierUnits {
105 /// The values of the columns that reached `outlier_min_samples`. With
106 /// none, the component is absent rather than 0.
107 TestedValues,
108}
109
110/// Which column `key_uniqueness` may measure, as recorded in a report's
111/// metric semantics.
112///
113/// Only the current definition has a value. Through 0.12, with no column
114/// declared, the first column whose name had an `id`, `key` or `pk` word was
115/// the key, so a foreign key listed first was scored as the table key and
116/// column order moved the score; those releases recorded no definition.
117#[derive(
118 Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize, schemars::JsonSchema,
119)]
120#[serde(rename_all = "snake_case")]
121pub enum KeyColumnSource {
122 /// The first `identifier_columns` entry the source has. With none
123 /// declared there is no key component.
124 Declared,
125}
126
127/// How the accuracy score compares a value with its outlier fences and its
128/// range and sign bounds, as recorded in a report's metric semantics.
129///
130/// Only the current definition has a value. Through 0.13 both comparisons
131/// went through the value's nearest `f64`, so a 64-bit integer past 2^53, a
132/// decimal held exactly, or a literal one digit past a bound could land on
133/// the other side of a fence or bound; those releases recorded no definition.
134#[derive(
135 Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize, schemars::JsonSchema,
136)]
137#[serde(rename_all = "snake_case")]
138pub enum AccuracyComparison {
139 /// Each value as written: exact Tukey fences, and bounds compared by the
140 /// value's digits where its nearest `f64` equals the bound. A literal
141 /// that overflows `f64` is a numeric value past every bound.
142 ExactValues,
143}
144
145/// How a JSON number whose value its nearest `f64` does not hold is read, as
146/// recorded in a report's metric semantics.
147///
148/// Only the current definition has a value. Through 0.13 such a number, like
149/// `1000000000000000.01` or `1e-400`, was read as its nearest `f64`, so
150/// distinct literals could count as one value and lose their decimal places;
151/// those releases recorded no definition.
152#[derive(
153 Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize, schemars::JsonSchema,
154)]
155#[serde(rename_all = "snake_case")]
156pub enum JsonNumberReading {
157 /// Its literal digits, as a CSV cell with the same text is read. Covers a
158 /// field of a JSON or JSONL record on every route and a database JSON
159 /// cell; a number nested in an array or object keeps its `f64`.
160 LiteralDigits,
161}
162
163/// How a quality score computed over a retained sample estimates the score
164/// over every scanned value, as recorded in a report's metric semantics.
165///
166/// Only the current definition has a value. Through 0.13 a sampled score
167/// pooled every column's reservoir, so a column weighed as many values as its
168/// reservoir kept, at most 10,000, and the score could fall outside its own
169/// `score_bounds`; those releases recorded no definition.
170#[derive(
171 Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize, schemars::JsonSchema,
172)]
173#[serde(rename_all = "snake_case")]
174pub enum SampledScoreEstimator {
175 /// Each column's sampled failure and checked counts are scaled to its
176 /// non-null values in the source before the shares are formed (#1293).
177 /// A source whose samples hold every value scores as before.
178 PopulationWeighted,
179}
180
181/// Which values a date column's statistics are computed over, as recorded
182/// in a report's metric semantics.
183///
184/// Only the current definition has a value. Through 0.13 the engines that
185/// retain a bounded sample of each column computed `min_datetime`,
186/// `max_datetime`, `duration_days`, the year, month, weekday and hour
187/// distributions and `slash_date_order` from that sample, so on a source
188/// larger than the sample they described the sample; those releases recorded
189/// no definition.
190#[derive(
191 Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize, schemars::JsonSchema,
192)]
193#[serde(rename_all = "snake_case")]
194pub enum DateStatistics {
195 /// Every non-null value the scan read, on every engine (#1297). The
196 /// `largest_gap` added in 0.14 (#1300) is computed over them too.
197 EveryValue,
198}
199
200/// Which columns the outlier test behind `outlier_ratio` runs on, as
201/// recorded in a report's metric semantics.
202///
203/// Only the current definition has a value. Through 0.13 a column whose first
204/// and third quartiles were one value was tested too: its IQR was 0, so
205/// every value off that quartile was an outlier, and a column of 75% zeros
206/// and 25% fives reported a quarter of its values as outliers. Those releases
207/// recorded no definition.
208#[derive(
209 Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize, schemars::JsonSchema,
210)]
211#[serde(rename_all = "snake_case")]
212pub enum OutlierSpread {
213 /// Columns whose first and third quartiles differ (#1304). A column whose
214 /// quartiles coincide is not tested, as one below `outlier_min_samples`
215 /// is not, and adds nothing to `outlier_values_checked`.
216 NonzeroIqr,
217}
218
219/// Which text forms read as dates, as recorded in a report's metric
220/// semantics. Type inference, the consistency check and the timeliness
221/// counts read them.
222///
223/// Only the current definition has a value. Through 0.13, and on 0.14
224/// release candidates before #1340, the ISO datetime with a space for the `T`
225/// was a date only in its plain whole-second form: with a fraction or an
226/// offset (`2026-06-01 12:00:00+00:00`, what Python's `str()` writes for an
227/// aware datetime) it was text. Those reports recorded no definition.
228#[derive(
229 Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize, schemars::JsonSchema,
230)]
231#[serde(rename_all = "snake_case")]
232pub enum DateForms {
233 /// The spaced ISO datetime reads as a date with a fraction and an offset,
234 /// as the `T` form does (#1335).
235 SpacedIsoDatetimes,
236}
237
238/// High-level categories of analysis that can be selectively enabled.
239#[derive(
240 Debug,
241 Clone,
242 Copy,
243 PartialEq,
244 Eq,
245 Hash,
246 serde::Serialize,
247 serde::Deserialize,
248 schemars::JsonSchema,
249)]
250#[serde(rename_all = "lowercase")]
251pub enum MetricPack {
252 /// Column names, data types, null counts — always included.
253 Schema,
254 /// Numeric stats (min/max/mean/median/std_dev/quartiles), text lengths.
255 Statistics,
256 /// Regex pattern detection (email, phone, UUID, etc.).
257 Patterns,
258 /// ISO 25012 quality dimensions (completeness, consistency, etc.).
259 Quality,
260}
261
262impl MetricPack {
263 /// All available metric packs.
264 pub fn all() -> Vec<Self> {
265 vec![
266 Self::Schema,
267 Self::Statistics,
268 Self::Patterns,
269 Self::Quality,
270 ]
271 }
272
273 /// Whether statistics should be computed given the selected packs.
274 pub fn include_statistics(packs: Option<&[Self]>) -> bool {
275 match packs {
276 None => true,
277 Some(p) => p.contains(&Self::Statistics),
278 }
279 }
280
281 /// Whether pattern detection should run given the selected packs.
282 pub fn include_patterns(packs: Option<&[Self]>) -> bool {
283 match packs {
284 None => true,
285 Some(p) => p.contains(&Self::Patterns),
286 }
287 }
288
289 /// Whether quality metrics should be computed given the selected packs.
290 pub fn include_quality(packs: Option<&[Self]>) -> bool {
291 match packs {
292 None => true,
293 Some(p) => p.contains(&Self::Quality),
294 }
295 }
296
297 /// Resolve the packs to compute, folding in an explicit quality-dimension
298 /// selection.
299 ///
300 /// Requesting an empty set of dimensions (`Some(&[])`) says "assess no
301 /// dimension", which is the same statement as not selecting the quality
302 /// pack: nothing is analyzed, so the report carries no quality. Without
303 /// this, an empty selection produces a quality object whose dimensions are
304 /// all absent — an artifact that reads like a measured result.
305 ///
306 /// This is deliberately about what was *requested*. A caller that asks for
307 /// dimensions and gets none back (an empty dataset, no date column) was
308 /// still analyzed, and keeps a quality object reporting exactly that.
309 pub fn resolve_with_dimensions(
310 packs: Option<&[Self]>,
311 dimensions: Option<&[QualityDimension]>,
312 ) -> Option<Vec<Self>> {
313 if !dimensions.is_some_and(<[QualityDimension]>::is_empty) {
314 return packs.map(<[Self]>::to_vec);
315 }
316 // `None` packs means "all packs"; make that explicit before removing
317 // Quality, so the result still selects everything else.
318 let selected = packs.map_or_else(Self::all, <[Self]>::to_vec);
319 Some(
320 selected
321 .into_iter()
322 .filter(|pack| *pack != Self::Quality)
323 .collect(),
324 )
325 }
326}
327
328impl std::str::FromStr for MetricPack {
329 type Err = String;
330
331 fn from_str(s: &str) -> Result<Self, Self::Err> {
332 match s.to_lowercase().as_str() {
333 "schema" => Ok(Self::Schema),
334 "statistics" => Ok(Self::Statistics),
335 "patterns" => Ok(Self::Patterns),
336 "quality" => Ok(Self::Quality),
337 _ => Err(format!(
338 "Unknown metric pack: {s}. Valid packs: schema, statistics, patterns, quality"
339 )),
340 }
341 }
342}
343
344impl std::fmt::Display for MetricPack {
345 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
346 match self {
347 Self::Schema => write!(f, "schema"),
348 Self::Statistics => write!(f, "statistics"),
349 Self::Patterns => write!(f, "patterns"),
350 Self::Quality => write!(f, "quality"),
351 }
352 }
353}
354
355#[cfg(test)]
356mod tests {
357 use super::*;
358
359 #[test]
360 fn test_metric_pack_include_helpers_none_means_all() {
361 assert!(MetricPack::include_statistics(None));
362 assert!(MetricPack::include_patterns(None));
363 assert!(MetricPack::include_quality(None));
364 }
365
366 #[test]
367 fn test_resolve_with_dimensions_empty_selection_drops_quality() {
368 // Requesting no dimension is the same statement as not asking for
369 // quality at all, so the pack must fall out and the report carry no
370 // quality object.
371 let resolved = MetricPack::resolve_with_dimensions(None, Some(&[])).unwrap();
372 assert!(!resolved.contains(&MetricPack::Quality));
373 assert!(!MetricPack::include_quality(Some(&resolved)));
374 // The other packs survive: only quality was deselected.
375 assert!(resolved.contains(&MetricPack::Schema));
376 assert!(resolved.contains(&MetricPack::Statistics));
377 assert!(resolved.contains(&MetricPack::Patterns));
378 }
379
380 #[test]
381 fn test_resolve_with_dimensions_empty_selection_keeps_explicit_packs() {
382 let packs = vec![MetricPack::Schema, MetricPack::Quality];
383 let resolved = MetricPack::resolve_with_dimensions(Some(&packs), Some(&[])).unwrap();
384 assert_eq!(resolved, vec![MetricPack::Schema]);
385 }
386
387 #[test]
388 fn test_resolve_with_dimensions_non_empty_selection_is_untouched() {
389 // A real selection says which dimensions to assess, not whether to.
390 let dims = vec![QualityDimension::Completeness];
391 assert_eq!(MetricPack::resolve_with_dimensions(None, Some(&dims)), None);
392 assert!(MetricPack::include_quality(None));
393
394 let packs = vec![MetricPack::Quality];
395 let resolved = MetricPack::resolve_with_dimensions(Some(&packs), Some(&dims)).unwrap();
396 assert!(MetricPack::include_quality(Some(&resolved)));
397 }
398
399 #[test]
400 fn test_resolve_with_dimensions_absent_selection_is_untouched() {
401 // `None` means "all dimensions", which is not the same as "none".
402 assert_eq!(MetricPack::resolve_with_dimensions(None, None), None);
403 let packs = vec![MetricPack::Quality];
404 assert_eq!(
405 MetricPack::resolve_with_dimensions(Some(&packs), None),
406 Some(packs)
407 );
408 }
409
410 #[test]
411 fn test_metric_pack_include_helpers_selective() {
412 let packs = vec![MetricPack::Schema, MetricPack::Quality];
413 assert!(!MetricPack::include_statistics(Some(&packs)));
414 assert!(!MetricPack::include_patterns(Some(&packs)));
415 assert!(MetricPack::include_quality(Some(&packs)));
416 }
417
418 #[test]
419 fn test_metric_pack_from_str() {
420 assert_eq!(
421 "statistics".parse::<MetricPack>().unwrap(),
422 MetricPack::Statistics
423 );
424 assert_eq!(
425 "QUALITY".parse::<MetricPack>().unwrap(),
426 MetricPack::Quality
427 );
428 assert!("invalid".parse::<MetricPack>().is_err());
429 }
430}