datarust_profile/quality/checks.rs
1//! Data-quality checks derived from a [`DatasetProfile`].
2//!
3//! Each [`QualityIssue`] is a single human-readable finding with a severity.
4//! The thresholds are conservative defaults; callers may filter the returned
5//! list as desired.
6
7use crate::profile::DatasetProfile;
8use crate::types::{ColumnType, Severity};
9
10/// The category of a [`QualityIssue`].
11#[derive(Debug, Clone, Copy, PartialEq, Eq)]
12#[cfg_attr(feature = "serde", derive(serde::Serialize))]
13pub enum QualityKind {
14 /// A column has a missing fraction at or above the threshold.
15 HighMissing,
16 /// A numeric column has (near-)zero variance — it carries no signal.
17 ConstantColumn,
18 /// A categorical column has cardinality equal to the row count (likely an
19 /// identifier rather than a feature).
20 NearUnique,
21 /// The dataset contains exact-duplicate rows.
22 DuplicateRows,
23 /// A numeric column has values outside the Tukey IQR fences.
24 Outliers,
25 /// A categorical column is dominated by a single value.
26 Imbalance,
27}
28
29/// A single data-quality finding.
30#[derive(Debug, Clone, PartialEq)]
31#[cfg_attr(feature = "serde", derive(serde::Serialize))]
32pub struct QualityIssue {
33 /// What kind of issue this is.
34 pub kind: QualityKind,
35 /// How serious the issue is.
36 pub severity: Severity,
37 /// Which column the issue concerns, or `None` for dataset-wide findings.
38 pub column: Option<String>,
39 /// Human-readable description, suitable for direct display in a report.
40 pub message: String,
41}
42
43/// Thresholds controlling when each check fires.
44///
45/// All fields are intentionally `pub` so callers can tune them before running
46/// [`run_checks`].
47#[derive(Debug, Clone, Copy, PartialEq)]
48#[cfg_attr(feature = "serde", derive(serde::Serialize))]
49pub struct Thresholds {
50 /// Missing fraction at or above which [`QualityKind::HighMissing`] fires.
51 pub missing_fraction: f64,
52 /// Variance at or below which [`QualityKind::ConstantColumn`] fires.
53 pub near_zero_variance: f64,
54 /// `unique / n_rows` at or above which [`QualityKind::NearUnique`] fires.
55 pub near_unique_ratio: f64,
56 /// Outlier fraction at or above which [`QualityKind::Outliers`] fires.
57 pub outlier_fraction: f64,
58 /// Imbalance ratio (`freq / present`) at or above which
59 /// [`QualityKind::Imbalance`] fires.
60 pub imbalance_ratio: f64,
61}
62
63impl Default for Thresholds {
64 fn default() -> Self {
65 Thresholds {
66 missing_fraction: 0.5,
67 near_zero_variance: 1e-12,
68 near_unique_ratio: 0.98,
69 outlier_fraction: 0.05,
70 imbalance_ratio: 0.95,
71 }
72 }
73}
74
75/// Runs all data-quality checks against `profile` using `thresholds`.
76pub fn run_checks(profile: &DatasetProfile, thresholds: &Thresholds) -> Vec<QualityIssue> {
77 let mut issues = Vec::new();
78
79 for col in &profile.columns {
80 if col.missing_fraction >= thresholds.missing_fraction && col.count > 0 {
81 issues.push(QualityIssue {
82 kind: QualityKind::HighMissing,
83 severity: if col.missing_fraction >= 0.9 {
84 Severity::Critical
85 } else {
86 Severity::Warning
87 },
88 column: Some(col.name.clone()),
89 message: format!(
90 "{}: {:.1}% of values are missing",
91 col.name,
92 col.missing_fraction * 100.0
93 ),
94 });
95 }
96
97 match col.column_type {
98 ColumnType::Numeric => {
99 if let Some(n) = &col.numeric {
100 let var = n.std * n.std;
101 if var <= thresholds.near_zero_variance {
102 issues.push(QualityIssue {
103 kind: QualityKind::ConstantColumn,
104 severity: Severity::Warning,
105 column: Some(col.name.clone()),
106 message: format!(
107 "{}: near-zero variance ({:.3e}); column is effectively constant",
108 col.name, var
109 ),
110 });
111 }
112 if n.outlier_count > 0 && n.outlier_fraction >= thresholds.outlier_fraction {
113 issues.push(QualityIssue {
114 kind: QualityKind::Outliers,
115 severity: if n.outlier_fraction >= 0.2 {
116 Severity::Warning
117 } else {
118 Severity::Info
119 },
120 column: Some(col.name.clone()),
121 message: format!(
122 "{}: {} outliers ({:.1}%) beyond IQR fences",
123 col.name,
124 n.outlier_count,
125 n.outlier_fraction * 100.0
126 ),
127 });
128 }
129 }
130 }
131 ColumnType::Categorical => {
132 if let Some(c) = &col.categorical {
133 if col.count > 0 {
134 let ratio = c.unique as f64 / col.count as f64;
135 if ratio >= thresholds.near_unique_ratio {
136 issues.push(QualityIssue {
137 kind: QualityKind::NearUnique,
138 severity: Severity::Info,
139 column: Some(col.name.clone()),
140 message: format!(
141 "{}: {} unique values across {} rows (ratio {:.2}); likely an identifier",
142 col.name, c.unique, col.count, ratio
143 ),
144 });
145 }
146 if c.imbalance_ratio >= thresholds.imbalance_ratio {
147 issues.push(QualityIssue {
148 kind: QualityKind::Imbalance,
149 severity: Severity::Critical,
150 column: Some(col.name.clone()),
151 message: format!(
152 "{}: top value '{}' covers {:.1}% of rows",
153 col.name,
154 c.top,
155 c.imbalance_ratio * 100.0
156 ),
157 });
158 }
159 }
160 }
161 }
162 }
163 }
164
165 if profile.duplicate_rows > 0 {
166 issues.push(QualityIssue {
167 kind: QualityKind::DuplicateRows,
168 severity: if profile.duplicate_fraction >= 0.1 {
169 Severity::Warning
170 } else {
171 Severity::Info
172 },
173 column: None,
174 message: format!(
175 "{} of {} rows are exact duplicates ({:.2}%)",
176 profile.duplicate_rows,
177 profile.n_rows,
178 profile.duplicate_fraction * 100.0
179 ),
180 });
181 }
182
183 issues
184}