datarust_profile/quality/
checks.rs

1//! Data-quality checks derived from a [`DatasetProfile`].
2//!
3//! Each [`QualityIssue`] is a single human-readable finding with a severity.
4//! The thresholds are conservative defaults; callers may filter the returned
5//! list as desired.
6
7use crate::profile::DatasetProfile;
8use crate::types::{ColumnType, Severity};
9
10/// The category of a [`QualityIssue`].
11#[derive(Debug, Clone, Copy, PartialEq, Eq)]
12#[cfg_attr(feature = "serde", derive(serde::Serialize))]
13pub enum QualityKind {
14    /// A column has a missing fraction at or above the threshold.
15    HighMissing,
16    /// A numeric column has (near-)zero variance — it carries no signal.
17    ConstantColumn,
18    /// A categorical column has cardinality equal to the row count (likely an
19    /// identifier rather than a feature).
20    NearUnique,
21    /// The dataset contains exact-duplicate rows.
22    DuplicateRows,
23    /// A numeric column has values outside the Tukey IQR fences.
24    Outliers,
25    /// A categorical column is dominated by a single value.
26    Imbalance,
27    /// A pair of numeric columns is highly correlated (`|r| >= threshold`).
28    HighCorrelation,
29    /// A feature column is suspiciously highly correlated with the target column.
30    TargetLeakage,
31}
32
33/// A single data-quality finding.
34#[derive(Debug, Clone, PartialEq)]
35#[cfg_attr(feature = "serde", derive(serde::Serialize))]
36pub struct QualityIssue {
37    /// What kind of issue this is.
38    pub kind: QualityKind,
39    /// How serious the issue is.
40    pub severity: Severity,
41    /// Which column the issue concerns, or `None` for dataset-wide findings.
42    pub column: Option<String>,
43    /// Human-readable description, suitable for direct display in a report.
44    pub message: String,
45}
46
47/// Thresholds controlling when each check fires.
48///
49/// All fields are intentionally `pub` so callers can tune them before running
50/// [`run_checks`].
51#[derive(Debug, Clone, Copy, PartialEq)]
52#[cfg_attr(feature = "serde", derive(serde::Serialize))]
53pub struct Thresholds {
54    /// Missing fraction at or above which [`QualityKind::HighMissing`] fires.
55    pub missing_fraction: f64,
56    /// Variance at or below which [`QualityKind::ConstantColumn`] fires.
57    pub near_zero_variance: f64,
58    /// `unique / n_rows` at or above which [`QualityKind::NearUnique`] fires.
59    pub near_unique_ratio: f64,
60    /// Outlier fraction at or above which [`QualityKind::Outliers`] fires.
61    pub outlier_fraction: f64,
62    /// Imbalance ratio (`freq / present`) at or above which
63    /// [`QualityKind::Imbalance`] fires.
64    pub imbalance_ratio: f64,
65    /// Correlation magnitude `|r|` at or above which [`QualityKind::HighCorrelation`] fires.
66    pub high_correlation: f64,
67    /// Correlation magnitude `|r|` or Cramér's V at or above which [`QualityKind::TargetLeakage`] fires.
68    pub target_leakage: f64,
69}
70
71impl Default for Thresholds {
72    fn default() -> Self {
73        Thresholds {
74            missing_fraction: 0.5,
75            near_zero_variance: 1e-12,
76            near_unique_ratio: 0.98,
77            outlier_fraction: 0.05,
78            imbalance_ratio: 0.95,
79            high_correlation: 0.95,
80            target_leakage: 0.90,
81        }
82    }
83}
84
85/// Runs all data-quality checks against `profile` using `thresholds`.
86pub fn run_checks(profile: &DatasetProfile, thresholds: &Thresholds) -> Vec<QualityIssue> {
87    let mut issues = Vec::new();
88
89    for col in &profile.columns {
90        if col.missing_fraction >= thresholds.missing_fraction && col.count > 0 {
91            issues.push(QualityIssue {
92                kind: QualityKind::HighMissing,
93                severity: if col.missing_fraction >= 0.9 {
94                    Severity::Critical
95                } else {
96                    Severity::Warning
97                },
98                column: Some(col.name.clone()),
99                message: format!(
100                    "{}: {:.1}% of values are missing",
101                    col.name,
102                    col.missing_fraction * 100.0
103                ),
104            });
105        }
106
107        match col.column_type {
108            ColumnType::Numeric => {
109                if let Some(n) = &col.numeric {
110                    let var = n.std * n.std;
111                    if var <= thresholds.near_zero_variance {
112                        issues.push(QualityIssue {
113                            kind: QualityKind::ConstantColumn,
114                            severity: Severity::Warning,
115                            column: Some(col.name.clone()),
116                            message: format!(
117                                "{}: near-zero variance ({:.3e}); column is effectively constant",
118                                col.name, var
119                            ),
120                        });
121                    }
122                    if n.outlier_count > 0 && n.outlier_fraction >= thresholds.outlier_fraction {
123                        issues.push(QualityIssue {
124                            kind: QualityKind::Outliers,
125                            severity: if n.outlier_fraction >= 0.2 {
126                                Severity::Warning
127                            } else {
128                                Severity::Info
129                            },
130                            column: Some(col.name.clone()),
131                            message: format!(
132                                "{}: {} outliers ({:.1}%) beyond IQR fences",
133                                col.name,
134                                n.outlier_count,
135                                n.outlier_fraction * 100.0
136                            ),
137                        });
138                    }
139                }
140            }
141            ColumnType::Categorical => {
142                if let Some(c) = &col.categorical {
143                    if col.count > 0 {
144                        let ratio = c.unique as f64 / col.count as f64;
145                        if ratio >= thresholds.near_unique_ratio {
146                            issues.push(QualityIssue {
147                                kind: QualityKind::NearUnique,
148                                severity: Severity::Info,
149                                column: Some(col.name.clone()),
150                                message: format!(
151                                    "{}: {} unique values across {} rows (ratio {:.2}); likely an identifier",
152                                    col.name, c.unique, col.count, ratio
153                                ),
154                            });
155                        }
156                        if c.imbalance_ratio >= thresholds.imbalance_ratio {
157                            issues.push(QualityIssue {
158                                kind: QualityKind::Imbalance,
159                                severity: Severity::Critical,
160                                column: Some(col.name.clone()),
161                                message: format!(
162                                    "{}: top value '{}' covers {:.1}% of rows",
163                                    col.name,
164                                    c.top,
165                                    c.imbalance_ratio * 100.0
166                                ),
167                            });
168                        }
169                    }
170                }
171            }
172        }
173    }
174
175    if profile.duplicate_rows > 0 {
176        issues.push(QualityIssue {
177            kind: QualityKind::DuplicateRows,
178            severity: if profile.duplicate_fraction >= 0.1 {
179                Severity::Warning
180            } else {
181                Severity::Info
182            },
183            column: None,
184            message: format!(
185                "{} of {} rows are exact duplicates ({:.2}%)",
186                profile.duplicate_rows,
187                profile.n_rows,
188                profile.duplicate_fraction * 100.0
189            ),
190        });
191    }
192
193    // Check relationship findings (HighCorrelation & TargetLeakage)
194    if let Some(rels) = &profile.relationships {
195        // Pearson high correlation
196        if let Some(pearson) = &rels.pearson {
197            let p = pearson.labels.len();
198            for i in 0..p {
199                for j in (i + 1)..p {
200                    let r = pearson.values[i][j];
201                    let abs_r = r.abs();
202
203                    // Check high correlation between feature pairs
204                    if abs_r >= thresholds.high_correlation {
205                        issues.push(QualityIssue {
206                            kind: QualityKind::HighCorrelation,
207                            severity: Severity::Warning,
208                            column: Some(pearson.labels[i].clone()),
209                            message: format!(
210                                "High Pearson correlation between '{}' and '{}' (r = {:.3})",
211                                pearson.labels[i], pearson.labels[j], r
212                            ),
213                        });
214                    }
215
216                    // Check target leakage if target_column matches either column
217                    if let Some(target) = &profile.target_column {
218                        let is_i_target = &pearson.labels[i] == target;
219                        let is_j_target = &pearson.labels[j] == target;
220                        if (is_i_target || is_j_target)
221                            && !(is_i_target && is_j_target)
222                            && abs_r >= thresholds.target_leakage
223                        {
224                            let feature = if is_i_target {
225                                &pearson.labels[j]
226                            } else {
227                                &pearson.labels[i]
228                            };
229                            issues.push(QualityIssue {
230                                kind: QualityKind::TargetLeakage,
231                                severity: Severity::Critical,
232                                column: Some(feature.clone()),
233                                message: format!(
234                                    "Suspected target leakage: feature '{}' has strong correlation with target '{}' (r = {:.3})",
235                                    feature, target, r
236                                ),
237                            });
238                        }
239                    }
240                }
241            }
242        }
243
244        // Cramér's V target leakage & high correlation
245        if let Some(cramers) = &rels.cramers_v {
246            let p = cramers.labels.len();
247            for i in 0..p {
248                for j in (i + 1)..p {
249                    let v = cramers.values[i][j];
250
251                    if let Some(target) = &profile.target_column {
252                        let is_i_target = &cramers.labels[i] == target;
253                        let is_j_target = &cramers.labels[j] == target;
254                        if (is_i_target || is_j_target)
255                            && !(is_i_target && is_j_target)
256                            && v >= thresholds.target_leakage
257                        {
258                            let feature = if is_i_target {
259                                &cramers.labels[j]
260                            } else {
261                                &cramers.labels[i]
262                            };
263                            issues.push(QualityIssue {
264                                kind: QualityKind::TargetLeakage,
265                                severity: Severity::Critical,
266                                column: Some(feature.clone()),
267                                message: format!(
268                                    "Suspected target leakage: categorical feature '{}' has high Cramér's V with target '{}' (V = {:.3})",
269                                    feature, target, v
270                                ),
271                            });
272                        }
273                    }
274                }
275            }
276        }
277
278        // Point-biserial target leakage
279        if let Some(target) = &profile.target_column {
280            for pb in &rels.point_biserial {
281                let abs_r = pb.correlation.abs();
282                if abs_r >= thresholds.target_leakage {
283                    let is_cat_target = &pb.categorical == target;
284                    let is_num_target = &pb.numeric == target;
285                    if (is_cat_target || is_num_target) && !(is_cat_target && is_num_target) {
286                        let feature = if is_cat_target {
287                            &pb.numeric
288                        } else {
289                            &pb.categorical
290                        };
291                        issues.push(QualityIssue {
292                            kind: QualityKind::TargetLeakage,
293                            severity: Severity::Critical,
294                            column: Some(feature.clone()),
295                            message: format!(
296                                "Suspected target leakage: feature '{}' has high point-biserial correlation with target '{}' (r = {:.3})",
297                                feature, target, pb.correlation
298                            ),
299                        });
300                    }
301                }
302            }
303        }
304    }
305
306    issues
307}
308
309#[cfg(test)]
310mod tests {
311    use super::*;
312    use crate::profile::{ColumnProfile, DatasetProfile, FiveNumber, Histogram, NumericStats};
313    use crate::types::{ColumnType, Severity};
314
315    fn make_numeric_profile(
316        name: &str,
317        mean: f64,
318        std: f64,
319        missing_fraction: f64,
320    ) -> ColumnProfile {
321        ColumnProfile {
322            name: name.to_string(),
323            column_type: ColumnType::Numeric,
324            count: 100,
325            missing_count: (missing_fraction * 100.0) as usize,
326            missing_fraction,
327            numeric: Some(NumericStats {
328                mean,
329                std,
330                five: FiveNumber {
331                    min: mean - 2.0 * std,
332                    q1: mean - 0.67 * std,
333                    median: mean,
334                    q3: mean + 0.67 * std,
335                    max: mean + 2.0 * std,
336                },
337                skewness: 0.0,
338                kurtosis: 0.0,
339                histogram: Histogram {
340                    edges: vec![],
341                    counts: vec![],
342                },
343                outlier_count: 0,
344                outlier_fraction: 0.0,
345            }),
346            categorical: None,
347        }
348    }
349
350    fn make_categorical_profile(
351        name: &str,
352        unique: usize,
353        imbalance_ratio: f64,
354        missing_fraction: f64,
355    ) -> ColumnProfile {
356        ColumnProfile {
357            name: name.to_string(),
358            column_type: ColumnType::Categorical,
359            count: 100,
360            missing_count: (missing_fraction * 100.0) as usize,
361            missing_fraction,
362            numeric: None,
363            categorical: Some(crate::profile::CategoricalStats {
364                unique,
365                top: "dominant".to_string(),
366                freq: (imbalance_ratio * 100.0) as usize,
367                imbalance_ratio,
368                top_values: vec![],
369            }),
370        }
371    }
372
373    #[test]
374    fn run_checks_high_missing_warning() {
375        let col = make_numeric_profile("high_miss", 0.0, 1.0, 0.6);
376        let profile = DatasetProfile {
377            n_rows: 100,
378            n_columns: 1,
379            memory_bytes: 800,
380            duplicate_rows: 0,
381            duplicate_fraction: 0.0,
382            target_column: None,
383            columns: vec![col],
384            relationships: None,
385        };
386        let issues = run_checks(&profile, &Thresholds::default());
387        assert!(issues
388            .iter()
389            .any(|i| i.kind == QualityKind::HighMissing && i.severity == Severity::Warning));
390    }
391
392    #[test]
393    fn run_checks_high_missing_critical() {
394        let col = make_numeric_profile("crit_miss", 0.0, 1.0, 0.95);
395        let profile = DatasetProfile {
396            n_rows: 100,
397            n_columns: 1,
398            memory_bytes: 800,
399            duplicate_rows: 0,
400            duplicate_fraction: 0.0,
401            target_column: None,
402            columns: vec![col],
403            relationships: None,
404        };
405        let issues = run_checks(&profile, &Thresholds::default());
406        assert!(issues
407            .iter()
408            .any(|i| i.kind == QualityKind::HighMissing && i.severity == Severity::Critical));
409    }
410
411    #[test]
412    fn run_checks_constant_column() {
413        let col = make_numeric_profile("const", 5.0, 1e-13, 0.0);
414        let profile = DatasetProfile {
415            n_rows: 100,
416            n_columns: 1,
417            memory_bytes: 800,
418            duplicate_rows: 0,
419            duplicate_fraction: 0.0,
420            target_column: None,
421            columns: vec![col],
422            relationships: None,
423        };
424        let issues = run_checks(&profile, &Thresholds::default());
425        assert!(issues.iter().any(|i| i.kind == QualityKind::ConstantColumn));
426    }
427
428    #[test]
429    fn run_checks_outliers_detected() {
430        let mut col = make_numeric_profile("out", 0.0, 1.0, 0.0);
431        col.numeric.as_mut().unwrap().outlier_count = 10;
432        col.numeric.as_mut().unwrap().outlier_fraction = 0.1;
433        let profile = DatasetProfile {
434            n_rows: 100,
435            n_columns: 1,
436            memory_bytes: 800,
437            duplicate_rows: 0,
438            duplicate_fraction: 0.0,
439            target_column: None,
440            columns: vec![col],
441            relationships: None,
442        };
443        let issues = run_checks(&profile, &Thresholds::default());
444        assert!(issues.iter().any(|i| i.kind == QualityKind::Outliers));
445    }
446
447    #[test]
448    fn run_checks_categorical_imbalance() {
449        let col = make_categorical_profile("imb", 2, 0.96, 0.0);
450        let profile = DatasetProfile {
451            n_rows: 100,
452            n_columns: 1,
453            memory_bytes: 800,
454            duplicate_rows: 0,
455            duplicate_fraction: 0.0,
456            target_column: None,
457            columns: vec![col],
458            relationships: None,
459        };
460        let issues = run_checks(&profile, &Thresholds::default());
461        assert!(issues
462            .iter()
463            .any(|i| i.kind == QualityKind::Imbalance && i.severity == Severity::Critical));
464    }
465
466    #[test]
467    fn run_checks_near_unique() {
468        let col = make_categorical_profile("uid", 99, 0.01, 0.0);
469        let profile = DatasetProfile {
470            n_rows: 100,
471            n_columns: 1,
472            memory_bytes: 800,
473            duplicate_rows: 0,
474            duplicate_fraction: 0.0,
475            target_column: None,
476            columns: vec![col],
477            relationships: None,
478        };
479        let issues = run_checks(&profile, &Thresholds::default());
480        assert!(issues.iter().any(|i| i.kind == QualityKind::NearUnique));
481    }
482
483    #[test]
484    fn run_checks_duplicate_rows() {
485        let col = make_numeric_profile("x", 0.0, 1.0, 0.0);
486        let profile = DatasetProfile {
487            n_rows: 100,
488            n_columns: 1,
489            memory_bytes: 800,
490            duplicate_rows: 10,
491            duplicate_fraction: 0.1,
492            target_column: None,
493            columns: vec![col],
494            relationships: None,
495        };
496        let issues = run_checks(&profile, &Thresholds::default());
497        assert!(issues
498            .iter()
499            .any(|i| i.kind == QualityKind::DuplicateRows && i.severity == Severity::Warning));
500    }
501
502    #[test]
503    fn run_checks_high_correlation() {
504        use crate::profile::relationships::{CorrelationMatrix, Relationships};
505        let col1 = make_numeric_profile("a", 0.0, 1.0, 0.0);
506        let col2 = make_numeric_profile("b", 0.0, 1.0, 0.0);
507        let pearson = CorrelationMatrix {
508            labels: vec!["a".to_string(), "b".to_string()],
509            values: vec![vec![1.0, 0.99], vec![0.99, 1.0]],
510        };
511        let profile = DatasetProfile {
512            n_rows: 100,
513            n_columns: 2,
514            memory_bytes: 1600,
515            duplicate_rows: 0,
516            duplicate_fraction: 0.0,
517            target_column: None,
518            columns: vec![col1, col2],
519            relationships: Some(Relationships {
520                pearson: Some(pearson),
521                cramers_v: None,
522                point_biserial: vec![],
523            }),
524        };
525        let issues = run_checks(&profile, &Thresholds::default());
526        assert!(issues
527            .iter()
528            .any(|i| i.kind == QualityKind::HighCorrelation));
529    }
530
531    #[test]
532    fn run_checks_target_leakage_pearson() {
533        use crate::profile::relationships::{CorrelationMatrix, Relationships};
534        let col1 = make_numeric_profile("feature", 0.0, 1.0, 0.0);
535        let col2 = make_numeric_profile("target", 0.0, 1.0, 0.0);
536        let pearson = CorrelationMatrix {
537            labels: vec!["feature".to_string(), "target".to_string()],
538            values: vec![vec![1.0, 0.95], vec![0.95, 1.0]],
539        };
540        let profile = DatasetProfile {
541            n_rows: 100,
542            n_columns: 2,
543            memory_bytes: 1600,
544            duplicate_rows: 0,
545            duplicate_fraction: 0.0,
546            target_column: Some("target".to_string()),
547            columns: vec![col1, col2],
548            relationships: Some(Relationships {
549                pearson: Some(pearson),
550                cramers_v: None,
551                point_biserial: vec![],
552            }),
553        };
554        let issues = run_checks(&profile, &Thresholds::default());
555        assert!(issues
556            .iter()
557            .any(|i| i.kind == QualityKind::TargetLeakage && i.severity == Severity::Critical));
558    }
559
560    #[test]
561    fn run_checks_custom_thresholds() {
562        let col = make_numeric_profile("x", 0.0, 1.0, 0.3);
563        let profile = DatasetProfile {
564            n_rows: 100,
565            n_columns: 1,
566            memory_bytes: 800,
567            duplicate_rows: 0,
568            duplicate_fraction: 0.0,
569            target_column: None,
570            columns: vec![col],
571            relationships: None,
572        };
573        // Default threshold is 0.5, so 0.3 shouldn't trigger
574        let issues = run_checks(&profile, &Thresholds::default());
575        assert!(!issues.iter().any(|i| i.kind == QualityKind::HighMissing));
576        // But with custom threshold of 0.25 it should
577        let t = Thresholds {
578            missing_fraction: 0.25,
579            ..Default::default()
580        };
581        let issues = run_checks(&profile, &t);
582        assert!(issues.iter().any(|i| i.kind == QualityKind::HighMissing));
583    }
584
585    #[test]
586    fn quality_issue_serialization() {
587        let issue = QualityIssue {
588            kind: QualityKind::HighMissing,
589            severity: Severity::Warning,
590            column: Some("test".to_string()),
591            message: "test message".to_string(),
592        };
593        // Just verify it can be constructed
594        assert_eq!(issue.column, Some("test".to_string()));
595        assert_eq!(issue.severity, Severity::Warning);
596    }
597
598    #[test]
599    fn thresholds_default_values() {
600        let t = Thresholds::default();
601        assert_eq!(t.missing_fraction, 0.5);
602        assert_eq!(t.near_zero_variance, 1e-12);
603        assert_eq!(t.near_unique_ratio, 0.98);
604        assert_eq!(t.outlier_fraction, 0.05);
605        assert_eq!(t.imbalance_ratio, 0.95);
606        assert_eq!(t.high_correlation, 0.95);
607        assert_eq!(t.target_leakage, 0.90);
608    }
609}
datarust_profile/quality/checks.rs

datarust_profile/quality/
checks.rs
