datarust_profile/quality/
checks.rs

1//! Data-quality checks derived from a [`DatasetProfile`].
2//!
3//! Each [`QualityIssue`] is a single human-readable finding with a severity.
4//! The thresholds are conservative defaults; callers may filter the returned
5//! list as desired.
6
7use crate::profile::DatasetProfile;
8use crate::types::{ColumnType, Severity};
9
10/// The category of a [`QualityIssue`].
11#[derive(Debug, Clone, Copy, PartialEq, Eq)]
12#[cfg_attr(feature = "serde", derive(serde::Serialize))]
13pub enum QualityKind {
14    /// A column has a missing fraction at or above the threshold.
15    HighMissing,
16    /// A numeric column has (near-)zero variance — it carries no signal.
17    ConstantColumn,
18    /// A categorical column has cardinality equal to the row count (likely an
19    /// identifier rather than a feature).
20    NearUnique,
21    /// The dataset contains exact-duplicate rows.
22    DuplicateRows,
23    /// A numeric column has values outside the Tukey IQR fences.
24    Outliers,
25    /// A categorical column is dominated by a single value.
26    Imbalance,
27    /// A pair of numeric columns is highly correlated (`|r| >= threshold`).
28    HighCorrelation,
29    /// A feature column is suspiciously highly correlated with the target column.
30    TargetLeakage,
31}
32
33/// A single data-quality finding.
34#[derive(Debug, Clone, PartialEq)]
35#[cfg_attr(feature = "serde", derive(serde::Serialize))]
36pub struct QualityIssue {
37    /// What kind of issue this is.
38    pub kind: QualityKind,
39    /// How serious the issue is.
40    pub severity: Severity,
41    /// Which column the issue concerns, or `None` for dataset-wide findings.
42    pub column: Option<String>,
43    /// Human-readable description, suitable for direct display in a report.
44    pub message: String,
45}
46
47/// Thresholds controlling when each check fires.
48///
49/// All fields are intentionally `pub` so callers can tune them before running
50/// [`run_checks`].
51#[derive(Debug, Clone, Copy, PartialEq)]
52#[cfg_attr(feature = "serde", derive(serde::Serialize))]
53pub struct Thresholds {
54    /// Missing fraction at or above which [`QualityKind::HighMissing`] fires.
55    pub missing_fraction: f64,
56    /// Variance at or below which [`QualityKind::ConstantColumn`] fires.
57    pub near_zero_variance: f64,
58    /// `unique / n_rows` at or above which [`QualityKind::NearUnique`] fires.
59    pub near_unique_ratio: f64,
60    /// Outlier fraction at or above which [`QualityKind::Outliers`] fires.
61    pub outlier_fraction: f64,
62    /// Imbalance ratio (`freq / present`) at or above which
63    /// [`QualityKind::Imbalance`] fires.
64    pub imbalance_ratio: f64,
65    /// Correlation magnitude `|r|` at or above which [`QualityKind::HighCorrelation`] fires.
66    pub high_correlation: f64,
67    /// Correlation magnitude `|r|` or Cramér's V at or above which [`QualityKind::TargetLeakage`] fires.
68    pub target_leakage: f64,
69}
70
71impl Default for Thresholds {
72    fn default() -> Self {
73        Thresholds {
74            missing_fraction: 0.5,
75            near_zero_variance: 1e-12,
76            near_unique_ratio: 0.98,
77            outlier_fraction: 0.05,
78            imbalance_ratio: 0.95,
79            high_correlation: 0.95,
80            target_leakage: 0.90,
81        }
82    }
83}
84
85/// Runs all data-quality checks against `profile` using `thresholds`.
86pub fn run_checks(profile: &DatasetProfile, thresholds: &Thresholds) -> Vec<QualityIssue> {
87    let mut issues = Vec::new();
88
89    for col in &profile.columns {
90        if col.missing_fraction >= thresholds.missing_fraction && col.count > 0 {
91            issues.push(QualityIssue {
92                kind: QualityKind::HighMissing,
93                severity: if col.missing_fraction >= 0.9 {
94                    Severity::Critical
95                } else {
96                    Severity::Warning
97                },
98                column: Some(col.name.clone()),
99                message: format!(
100                    "{}: {:.1}% of values are missing",
101                    col.name,
102                    col.missing_fraction * 100.0
103                ),
104            });
105        }
106
107        match col.column_type {
108            ColumnType::Numeric => {
109                if let Some(n) = &col.numeric {
110                    let var = n.std * n.std;
111                    if var <= thresholds.near_zero_variance {
112                        issues.push(QualityIssue {
113                            kind: QualityKind::ConstantColumn,
114                            severity: Severity::Warning,
115                            column: Some(col.name.clone()),
116                            message: format!(
117                                "{}: near-zero variance ({:.3e}); column is effectively constant",
118                                col.name, var
119                            ),
120                        });
121                    }
122                    if n.outlier_count > 0 && n.outlier_fraction >= thresholds.outlier_fraction {
123                        issues.push(QualityIssue {
124                            kind: QualityKind::Outliers,
125                            severity: if n.outlier_fraction >= 0.2 {
126                                Severity::Warning
127                            } else {
128                                Severity::Info
129                            },
130                            column: Some(col.name.clone()),
131                            message: format!(
132                                "{}: {} outliers ({:.1}%) beyond IQR fences",
133                                col.name,
134                                n.outlier_count,
135                                n.outlier_fraction * 100.0
136                            ),
137                        });
138                    }
139                }
140            }
141            ColumnType::Categorical => {
142                if let Some(c) = &col.categorical {
143                    if col.count > 0 {
144                        let ratio = c.unique as f64 / col.count as f64;
145                        if ratio >= thresholds.near_unique_ratio {
146                            issues.push(QualityIssue {
147                                kind: QualityKind::NearUnique,
148                                severity: Severity::Info,
149                                column: Some(col.name.clone()),
150                                message: format!(
151                                    "{}: {} unique values across {} rows (ratio {:.2}); likely an identifier",
152                                    col.name, c.unique, col.count, ratio
153                                ),
154                            });
155                        }
156                        if c.imbalance_ratio >= thresholds.imbalance_ratio {
157                            issues.push(QualityIssue {
158                                kind: QualityKind::Imbalance,
159                                severity: Severity::Critical,
160                                column: Some(col.name.clone()),
161                                message: format!(
162                                    "{}: top value '{}' covers {:.1}% of rows",
163                                    col.name,
164                                    c.top,
165                                    c.imbalance_ratio * 100.0
166                                ),
167                            });
168                        }
169                    }
170                }
171            }
172        }
173    }
174
175    if profile.duplicate_rows > 0 {
176        issues.push(QualityIssue {
177            kind: QualityKind::DuplicateRows,
178            severity: if profile.duplicate_fraction >= 0.1 {
179                Severity::Warning
180            } else {
181                Severity::Info
182            },
183            column: None,
184            message: format!(
185                "{} of {} rows are exact duplicates ({:.2}%)",
186                profile.duplicate_rows,
187                profile.n_rows,
188                profile.duplicate_fraction * 100.0
189            ),
190        });
191    }
192
193    // Check relationship findings (HighCorrelation & TargetLeakage)
194    if let Some(rels) = &profile.relationships {
195        // Pearson high correlation
196        if let Some(pearson) = &rels.pearson {
197            let p = pearson.labels.len();
198            for i in 0..p {
199                for j in (i + 1)..p {
200                    let r = pearson.values[i][j];
201                    let abs_r = r.abs();
202
203                    // Check high correlation between feature pairs
204                    if abs_r >= thresholds.high_correlation {
205                        issues.push(QualityIssue {
206                            kind: QualityKind::HighCorrelation,
207                            severity: Severity::Warning,
208                            column: Some(pearson.labels[i].clone()),
209                            message: format!(
210                                "High Pearson correlation between '{}' and '{}' (r = {:.3})",
211                                pearson.labels[i], pearson.labels[j], r
212                            ),
213                        });
214                    }
215
216                    // Check target leakage if target_column matches either column
217                    if let Some(target) = &profile.target_column {
218                        let is_i_target = &pearson.labels[i] == target;
219                        let is_j_target = &pearson.labels[j] == target;
220                        if (is_i_target || is_j_target)
221                            && !(is_i_target && is_j_target)
222                            && abs_r >= thresholds.target_leakage
223                        {
224                            let feature = if is_i_target {
225                                &pearson.labels[j]
226                            } else {
227                                &pearson.labels[i]
228                            };
229                            issues.push(QualityIssue {
230                                kind: QualityKind::TargetLeakage,
231                                severity: Severity::Critical,
232                                column: Some(feature.clone()),
233                                message: format!(
234                                    "Suspected target leakage: feature '{}' has strong correlation with target '{}' (r = {:.3})",
235                                    feature, target, r
236                                ),
237                            });
238                        }
239                    }
240                }
241            }
242        }
243
244        // Cramér's V target leakage & high correlation
245        if let Some(cramers) = &rels.cramers_v {
246            let p = cramers.labels.len();
247            for i in 0..p {
248                for j in (i + 1)..p {
249                    let v = cramers.values[i][j];
250
251                    if let Some(target) = &profile.target_column {
252                        let is_i_target = &cramers.labels[i] == target;
253                        let is_j_target = &cramers.labels[j] == target;
254                        if (is_i_target || is_j_target)
255                            && !(is_i_target && is_j_target)
256                            && v >= thresholds.target_leakage
257                        {
258                            let feature = if is_i_target {
259                                &cramers.labels[j]
260                            } else {
261                                &cramers.labels[i]
262                            };
263                            issues.push(QualityIssue {
264                                kind: QualityKind::TargetLeakage,
265                                severity: Severity::Critical,
266                                column: Some(feature.clone()),
267                                message: format!(
268                                    "Suspected target leakage: categorical feature '{}' has high Cramér's V with target '{}' (V = {:.3})",
269                                    feature, target, v
270                                ),
271                            });
272                        }
273                    }
274                }
275            }
276        }
277
278        // Point-biserial target leakage
279        if let Some(target) = &profile.target_column {
280            for pb in &rels.point_biserial {
281                let abs_r = pb.correlation.abs();
282                if abs_r >= thresholds.target_leakage {
283                    let is_cat_target = &pb.categorical == target;
284                    let is_num_target = &pb.numeric == target;
285                    if (is_cat_target || is_num_target) && !(is_cat_target && is_num_target) {
286                        let feature = if is_cat_target {
287                            &pb.numeric
288                        } else {
289                            &pb.categorical
290                        };
291                        issues.push(QualityIssue {
292                            kind: QualityKind::TargetLeakage,
293                            severity: Severity::Critical,
294                            column: Some(feature.clone()),
295                            message: format!(
296                                "Suspected target leakage: feature '{}' has high point-biserial correlation with target '{}' (r = {:.3})",
297                                feature, target, pb.correlation
298                            ),
299                        });
300                    }
301                }
302            }
303        }
304    }
305
306    issues
307}
datarust_profile/quality/checks.rs

datarust_profile/quality/
checks.rs
