use std::collections::BTreeMap;
use std::fmt;
use dataprof_core::serde_helpers::rounded_2;
use dataprof_core::{
DataSource, DataType, DelimiterDetection, FileFormat, KeyColumnSource, PatternCategory,
TypeHomogeneity,
};
use crate::profile_report::ProfileReport;
use crate::quality_gate::{
Evidence, EvidenceGap, Provenance, quality_status_name, sampled_component_evidence,
};
pub const DEFAULT_NULL_HEAVY_PERCENTAGE: f64 = 20.0;
pub const DEFAULT_MIXED_TYPES_PERCENTAGE: f64 = 5.0;
const MIN_PATTERN_CONFIDENCE: f64 = 0.5;
const PERSONAL_IDENTIFIER_PATTERNS: [&str; 2] = ["Codice Fiscale (IT)", "SSN (US)"];
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, serde::Serialize)]
#[serde(rename_all = "snake_case")]
#[non_exhaustive]
pub enum Severity {
Warning,
Info,
}
impl fmt::Display for Severity {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::Warning => write!(f, "warning"),
Self::Info => write!(f, "info"),
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, serde::Serialize)]
#[serde(rename_all = "snake_case")]
#[non_exhaustive]
pub enum FindingCode {
AllNull,
ConstantColumn,
DelimiterAmbiguous,
DuplicateRows,
FutureDates,
HeaderNotSplit,
KeyDuplicates,
LocaleNumbers,
MixedTypes,
NullHeavy,
PartialScan,
RaggedRows,
RecordsSkipped,
SensitivePattern,
TemporalOrderViolations,
UnterminatedQuote,
}
impl FindingCode {
pub const ALL: [Self; 16] = [
Self::AllNull,
Self::ConstantColumn,
Self::DelimiterAmbiguous,
Self::DuplicateRows,
Self::FutureDates,
Self::HeaderNotSplit,
Self::KeyDuplicates,
Self::LocaleNumbers,
Self::MixedTypes,
Self::NullHeavy,
Self::PartialScan,
Self::RaggedRows,
Self::RecordsSkipped,
Self::SensitivePattern,
Self::TemporalOrderViolations,
Self::UnterminatedQuote,
];
pub fn severity(self) -> Severity {
match self {
Self::AllNull
| Self::DelimiterAmbiguous
| Self::DuplicateRows
| Self::FutureDates
| Self::HeaderNotSplit
| Self::KeyDuplicates
| Self::LocaleNumbers
| Self::MixedTypes
| Self::NullHeavy
| Self::RaggedRows
| Self::RecordsSkipped
| Self::TemporalOrderViolations
| Self::UnterminatedQuote => Severity::Warning,
Self::ConstantColumn | Self::PartialScan | Self::SensitivePattern => Severity::Info,
}
}
fn summary(self) -> &'static str {
match self {
Self::AllNull => "every value in this column is null",
Self::ConstantColumn => "every non-null value in this column is the same",
Self::DelimiterAmbiguous => {
"another delimiter reads the sampled CSV records as well as the one detection \
chose, so the columns may be split on the wrong one; set csv_delimiter to choose"
}
Self::DuplicateRows => "the source contains exact duplicate rows",
Self::FutureDates => "some date values lie in the future",
Self::HeaderNotSplit => {
"no delimiter detection tries (comma, semicolon, tab, pipe) splits the CSV \
header, so the source was read as one column, but another character splits the \
header and every sampled record alike; set csv_delimiter to read it as a table"
}
Self::KeyDuplicates => {
"some non-null values of the declared key column occur more than once"
}
Self::LocaleNumbers => {
"at least half of the text values in this column are numbers written with a \
decimal comma or digit grouping, so no numeric statistic includes them"
}
Self::MixedTypes => "this column's values are split across lexical types",
Self::NullHeavy => "this column's null share is at or above the threshold",
Self::PartialScan => {
"only part of the source was read; findings describe the rows that were"
}
Self::RaggedRows => {
"some rows had a different field count from the header and were recovered"
}
Self::RecordsSkipped => "errors were counted while reading the source",
Self::SensitivePattern => {
"values in this column match a pattern for personal or financial data"
}
Self::TemporalOrderViolations => "some start dates fall after their paired end dates",
Self::UnterminatedQuote => {
"the source ends inside a quoted field, so its last record may hold several rows"
}
}
}
}
impl fmt::Display for FindingCode {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
let name = match self {
Self::AllNull => "all_null",
Self::ConstantColumn => "constant_column",
Self::DelimiterAmbiguous => "delimiter_ambiguous",
Self::DuplicateRows => "duplicate_rows",
Self::FutureDates => "future_dates",
Self::HeaderNotSplit => "header_not_split",
Self::KeyDuplicates => "key_duplicates",
Self::LocaleNumbers => "locale_numbers",
Self::MixedTypes => "mixed_types",
Self::NullHeavy => "null_heavy",
Self::PartialScan => "partial_scan",
Self::RaggedRows => "ragged_rows",
Self::RecordsSkipped => "records_skipped",
Self::SensitivePattern => "sensitive_pattern",
Self::TemporalOrderViolations => "temporal_order_violations",
Self::UnterminatedQuote => "unterminated_quote",
};
write!(f, "{name}")
}
}
impl std::str::FromStr for FindingCode {
type Err = String;
fn from_str(s: &str) -> Result<Self, Self::Err> {
Self::ALL
.into_iter()
.find(|code| code.to_string() == s)
.ok_or_else(|| {
let valid: Vec<String> = Self::ALL.iter().map(ToString::to_string).collect();
format!(
"Unknown finding code: {s}. Valid codes: {}",
valid.join(", ")
)
})
}
}
#[derive(Debug, Clone, PartialEq, serde::Serialize)]
#[serde(untagged)]
#[non_exhaustive]
pub enum EvidenceValue {
Count(usize),
Percentage(#[serde(serialize_with = "dataprof_core::serde_helpers::round_2")] f64),
Text(String),
Flag(bool),
}
#[derive(Debug, Clone, PartialEq, serde::Serialize)]
#[non_exhaustive]
pub struct Finding {
pub code: FindingCode,
pub severity: Severity,
#[serde(skip_serializing_if = "Option::is_none")]
pub column: Option<String>,
pub evidence: BTreeMap<String, EvidenceValue>,
pub summary: String,
}
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize)]
#[serde(tag = "reason", rename_all = "snake_case")]
#[non_exhaustive]
pub enum NotEvaluatedReason {
#[non_exhaustive]
QualityUnavailable {
quality_status: String,
},
NotAssessed,
Estimated,
Sampled,
Unrecorded,
NotComputed,
NoValues,
}
impl NotEvaluatedReason {
fn rank(&self) -> u8 {
match self {
Self::QualityUnavailable { .. } => 0,
Self::NotAssessed => 1,
Self::Estimated => 2,
Self::Sampled => 3,
Self::Unrecorded => 4,
Self::NotComputed => 5,
Self::NoValues => 6,
}
}
}
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize)]
#[non_exhaustive]
pub struct UnevaluatedRule {
pub code: FindingCode,
#[serde(flatten)]
pub reason: NotEvaluatedReason,
#[serde(skip_serializing_if = "Vec::is_empty")]
pub columns: Vec<String>,
}
#[derive(Debug, Clone, PartialEq, serde::Serialize)]
#[non_exhaustive]
pub struct FindingsResult {
pub findings: Vec<Finding>,
pub not_evaluated: Vec<UnevaluatedRule>,
}
#[derive(Debug, Clone, PartialEq, serde::Deserialize)]
#[non_exhaustive]
pub struct FindingsInput {
rows_processed: usize,
source_exhausted: Option<bool>,
truncated: bool,
sampling_applied: Option<bool>,
ragged_row_count: Option<usize>,
unterminated_quote: Option<bool>,
#[serde(default)]
csv_source: Option<bool>,
error_count: usize,
#[serde(default)]
delimiter_detection: Option<DelimiterDetection>,
#[serde(default)]
csv_source_unrecorded: bool,
#[serde(default)]
key_column_source: Option<KeyColumnSource>,
quality_status: String,
quality: Option<QualityFacts>,
columns: Vec<ColumnFacts>,
}
#[derive(Debug, Clone, PartialEq, serde::Deserialize)]
struct QualityFacts {
uniqueness: Option<UniquenessFacts>,
timeliness: Option<TimelinessFacts>,
sampled_dimensions: Option<Vec<String>>,
unlisted_components: Vec<String>,
}
#[derive(Debug, Clone, PartialEq, serde::Deserialize)]
struct UniquenessFacts {
#[serde(default)]
rows_checked: usize,
#[serde(default)]
duplicate_rows: usize,
#[serde(default)]
duplicate_rows_approximate: bool,
#[serde(default)]
key_column: Option<String>,
}
#[derive(Debug, Clone, PartialEq, serde::Deserialize)]
struct TimelinessFacts {
#[serde(default)]
date_values_checked: usize,
#[serde(default)]
future_dates_count: usize,
#[serde(default)]
temporal_pairs_checked: usize,
#[serde(default)]
temporal_violations: usize,
}
#[derive(Debug, Clone, PartialEq, serde::Deserialize)]
struct ColumnFacts {
name: String,
identifier: bool,
total_count: usize,
null_count: usize,
unique_count: Option<usize>,
#[serde(default)]
unique_count_is_approximate: Option<bool>,
type_homogeneity: Option<TypeHomogeneity>,
locale_number_count: Option<usize>,
patterns: Option<Vec<PatternFacts>>,
}
#[derive(Debug, Clone, PartialEq, serde::Deserialize)]
struct PatternFacts {
name: String,
category: String,
confidence: f64,
match_percentage: f64,
}
impl FindingsInput {
pub fn from_json(text: &str) -> serde_json::Result<Self> {
serde_json::from_str(text)
}
pub(crate) fn reads_a_partial_sample(&self, code: FindingCode) -> bool {
let exempts_identifiers = match code {
FindingCode::MixedTypes | FindingCode::LocaleNumbers => true,
FindingCode::SensitivePattern => false,
_ => return false,
};
self.columns
.iter()
.filter(|column| !(exempts_identifiers && column.identifier))
.any(|column| {
let non_null = column.total_count.saturating_sub(column.null_count);
column
.type_homogeneity
.as_ref()
.is_some_and(|forms| forms.classified_count() < non_null)
})
}
}
impl From<&ProfileReport> for FindingsInput {
fn from(report: &ProfileReport) -> Self {
let execution = &report.execution;
Self {
rows_processed: execution.rows_processed,
source_exhausted: Some(execution.source_exhausted),
truncated: execution.truncation_reason.is_some(),
sampling_applied: Some(execution.sampling_applied),
ragged_row_count: (report.schema_version != 0).then_some(execution.ragged_row_count),
unterminated_quote: execution.unterminated_quote,
csv_source: csv_source(&report.data_source),
error_count: execution.error_count,
delimiter_detection: report
.data_source
.csv_metadata()
.and_then(|metadata| metadata.delimiter_detection.clone()),
csv_source_unrecorded: csv_source_unrecorded(&report.data_source),
key_column_source: report
.metric_semantics
.as_ref()
.and_then(|semantics| semantics.key_column_source),
quality_status: quality_status_name(&report.quality_status).to_string(),
quality: report.quality.as_ref().map(|quality| QualityFacts {
uniqueness: quality
.metrics
.uniqueness
.as_ref()
.map(|uniqueness| UniquenessFacts {
rows_checked: uniqueness.rows_checked,
duplicate_rows: uniqueness.duplicate_rows,
duplicate_rows_approximate: uniqueness.duplicate_rows_approximate,
key_column: uniqueness.key_column.clone(),
}),
timeliness: quality
.metrics
.timeliness
.as_ref()
.map(|timeliness| TimelinessFacts {
date_values_checked: timeliness.date_values_checked,
future_dates_count: timeliness.future_dates_count,
temporal_pairs_checked: timeliness.temporal_pairs_checked,
temporal_violations: timeliness.temporal_violations,
}),
sampled_dimensions: quality.sampled_dimensions(),
unlisted_components: quality.unlisted_components(),
}),
columns: report
.column_profiles
.iter()
.map(|column| ColumnFacts {
name: column.name.clone(),
identifier: column.data_type == DataType::Identifier,
total_count: column.total_count,
null_count: column.null_count,
unique_count: column.unique_count,
unique_count_is_approximate: column.unique_count_is_approximate,
type_homogeneity: column.type_homogeneity,
locale_number_count: column.locale_number_count,
patterns: column.patterns.as_ref().map(|patterns| {
patterns
.iter()
.map(|pattern| PatternFacts {
name: pattern.name.clone(),
category: pattern.category.to_string(),
confidence: pattern.confidence,
match_percentage: pattern.match_percentage,
})
.collect()
}),
})
.collect(),
}
}
}
#[derive(Debug, Clone, PartialEq)]
#[non_exhaustive]
pub enum FindingPolicyError {
#[non_exhaustive]
ThresholdOutOfRange {
setting: &'static str,
value: f64,
},
}
impl fmt::Display for FindingPolicyError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::ThresholdOutOfRange { setting, value } => write!(
f,
"{setting} must be a percentage above 0 and at most 100 at two decimal \
places, got {value}"
),
}
}
}
impl std::error::Error for FindingPolicyError {}
#[derive(Debug, Clone, PartialEq)]
#[non_exhaustive]
pub struct FindingPolicy {
null_heavy_percentage: f64,
mixed_types_percentage: f64,
}
impl Default for FindingPolicy {
fn default() -> Self {
Self {
null_heavy_percentage: DEFAULT_NULL_HEAVY_PERCENTAGE,
mixed_types_percentage: DEFAULT_MIXED_TYPES_PERCENTAGE,
}
}
}
impl FindingPolicy {
pub fn new() -> Self {
Self::default()
}
pub fn null_heavy_percentage(mut self, percentage: f64) -> Self {
self.null_heavy_percentage = percentage;
self
}
pub fn mixed_types_percentage(mut self, percentage: f64) -> Self {
self.mixed_types_percentage = percentage;
self
}
pub fn validate(&self) -> Result<(), FindingPolicyError> {
for (setting, value) in [
("null_heavy_percentage", self.null_heavy_percentage),
("mixed_types_percentage", self.mixed_types_percentage),
] {
let applied = rounded_2(value);
if !(value.is_finite() && applied > 0.0 && applied <= 100.0) {
return Err(FindingPolicyError::ThresholdOutOfRange { setting, value });
}
}
Ok(())
}
pub fn evaluate(&self, report: &ProfileReport) -> Result<FindingsResult, FindingPolicyError> {
self.evaluate_input(&FindingsInput::from(report))
}
pub fn evaluate_input(
&self,
input: &FindingsInput,
) -> Result<FindingsResult, FindingPolicyError> {
self.validate()?;
Ok(self.collect(input))
}
fn null_heavy_threshold(&self) -> f64 {
rounded_2(self.null_heavy_percentage)
}
fn mixed_types_threshold(&self) -> f64 {
rounded_2(self.mixed_types_percentage)
}
pub(crate) fn collect(&self, input: &FindingsInput) -> FindingsResult {
let mut collector = Collector::default();
scan_findings(input, &mut collector);
quality_findings(input, &mut collector);
for (index, column) in input.columns.iter().enumerate() {
self.column_findings(index, column, &mut collector);
}
collector.finish()
}
fn column_findings(&self, index: usize, column: &ColumnFacts, out: &mut Collector) {
let name = column.name.as_str();
let non_null = column.total_count.saturating_sub(column.null_count);
if column.total_count == 0 {
out.skip(FindingCode::AllNull, NotEvaluatedReason::NoValues, name);
out.skip(FindingCode::NullHeavy, NotEvaluatedReason::NoValues, name);
} else if non_null == 0 {
out.column(
FindingCode::AllNull,
index,
name,
[
("null_count", EvidenceValue::Count(column.null_count)),
("total_count", EvidenceValue::Count(column.total_count)),
],
);
} else {
let null_percentage =
rounded_2(column.null_count as f64 / column.total_count as f64 * 100.0);
if null_percentage >= self.null_heavy_threshold() {
out.column(
FindingCode::NullHeavy,
index,
name,
[
("null_percentage", percentage(null_percentage)),
("threshold", percentage(self.null_heavy_threshold())),
],
);
}
}
match column.unique_count {
None => out.skip(
FindingCode::ConstantColumn,
NotEvaluatedReason::NotComputed,
name,
),
Some(_) if non_null == 0 => out.skip(
FindingCode::ConstantColumn,
NotEvaluatedReason::NoValues,
name,
),
Some(1) if non_null > 1 => out.column(
FindingCode::ConstantColumn,
index,
name,
[
("unique_count", EvidenceValue::Count(1)),
("non_null_count", EvidenceValue::Count(non_null)),
],
),
Some(_) => {}
}
locale_numbers(index, column, out);
self.mixed_types(index, column, non_null, out);
sensitive_patterns(index, column, non_null, out);
}
fn mixed_types(
&self,
index: usize,
column: &ColumnFacts,
non_null: usize,
out: &mut Collector,
) {
let name = column.name.as_str();
if column.identifier {
return;
}
let Some(homogeneity) = column.type_homogeneity.as_ref() else {
out.skip(
FindingCode::MixedTypes,
NotEvaluatedReason::NotComputed,
name,
);
return;
};
let classified = homogeneity.classified_count();
let Some((dominant, dominant_count)) = homogeneity.dominant() else {
out.skip(FindingCode::MixedTypes, NotEvaluatedReason::NoValues, name);
return;
};
let outside = classified - dominant_count;
if outside == 0 {
return;
}
let outside_percentage = rounded_2(outside as f64 / classified as f64 * 100.0);
if outside_percentage < self.mixed_types_threshold() {
return;
}
out.column(
FindingCode::MixedTypes,
index,
name,
[
("dominant_type", EvidenceValue::Text(dominant.to_string())),
(
"dominant_percentage",
percentage(dominant_count as f64 / classified as f64 * 100.0),
),
("threshold", percentage(self.mixed_types_threshold())),
("classified_count", EvidenceValue::Count(classified)),
("non_null_count", EvidenceValue::Count(non_null)),
],
);
}
}
fn locale_numbers(index: usize, column: &ColumnFacts, out: &mut Collector) {
let name = column.name.as_str();
if column.identifier {
return;
}
let Some(homogeneity) = column.type_homogeneity.as_ref() else {
out.skip(
FindingCode::LocaleNumbers,
NotEvaluatedReason::NotComputed,
name,
);
return;
};
let Some(locale_numbers) = column.locale_number_count else {
out.skip(
FindingCode::LocaleNumbers,
NotEvaluatedReason::Unrecorded,
name,
);
return;
};
let classified = homogeneity.classified_count();
if classified == 0 {
out.skip(
FindingCode::LocaleNumbers,
NotEvaluatedReason::NoValues,
name,
);
return;
}
let text = homogeneity.text;
if locale_numbers == 0 || locale_numbers * 2 < text {
return;
}
out.column(
FindingCode::LocaleNumbers,
index,
name,
[
("locale_number_count", EvidenceValue::Count(locale_numbers)),
("text_count", EvidenceValue::Count(text)),
("classified_count", EvidenceValue::Count(classified)),
(
"non_null_count",
EvidenceValue::Count(column.total_count.saturating_sub(column.null_count)),
),
],
);
}
impl ProfileReport {
pub fn findings(&self) -> FindingsResult {
FindingPolicy::default().collect(&FindingsInput::from(self))
}
}
fn scan_findings(input: &FindingsInput, out: &mut Collector) {
let partial = if input.source_exhausted == Some(false) || input.truncated {
Some("truncated")
} else if input.sampling_applied == Some(true) {
Some("sampled")
} else {
None
};
match partial {
Some(reason) => out.report(
FindingCode::PartialScan,
[
("reason", EvidenceValue::Text(reason.to_string())),
("rows_processed", EvidenceValue::Count(input.rows_processed)),
],
),
None if input.source_exhausted.is_none() || input.sampling_applied.is_none() => {
out.skip_report(FindingCode::PartialScan, NotEvaluatedReason::Unrecorded);
}
None => {}
}
match input.ragged_row_count {
None => out.skip_report(FindingCode::RaggedRows, NotEvaluatedReason::Unrecorded),
Some(0) => {}
Some(ragged_row_count) => out.report(
FindingCode::RaggedRows,
[
("ragged_row_count", EvidenceValue::Count(ragged_row_count)),
("rows_processed", EvidenceValue::Count(input.rows_processed)),
],
),
}
match (input.unterminated_quote, input.csv_source) {
(Some(true), _) => out.report(
FindingCode::UnterminatedQuote,
[("rows_processed", EvidenceValue::Count(input.rows_processed))],
),
(Some(false), _) => {}
(None, Some(false)) => {}
(None, _) => out.skip_report(
FindingCode::UnterminatedQuote,
NotEvaluatedReason::Unrecorded,
),
}
if input.error_count > 0 {
out.report(
FindingCode::RecordsSkipped,
[("error_count", EvidenceValue::Count(input.error_count))],
);
}
if input.csv_source_unrecorded {
out.skip_report(
FindingCode::DelimiterAmbiguous,
NotEvaluatedReason::Unrecorded,
);
out.skip_report(FindingCode::HeaderNotSplit, NotEvaluatedReason::Unrecorded);
}
if let Some(detection) = &input.delimiter_detection {
if let Some(runner_up) = &detection.runner_up {
out.report(
FindingCode::DelimiterAmbiguous,
[
(
"delimiter",
EvidenceValue::Text(detection.delimiter.clone()),
),
("runner_up", EvidenceValue::Text(runner_up.clone())),
],
);
}
if let Some(unlisted) = &detection.unlisted_delimiter {
out.report(
FindingCode::HeaderNotSplit,
[("unlisted_delimiter", EvidenceValue::Text(unlisted.clone()))],
);
}
}
}
fn csv_source(source: &DataSource) -> Option<bool> {
match source {
DataSource::Stream { format: None, .. } => None,
_ => Some(matches!(source.format(), Some(FileFormat::Csv))),
}
}
fn csv_source_unrecorded(source: &DataSource) -> bool {
match source {
DataSource::File { format, .. }
| DataSource::Bytes { format, .. }
| DataSource::Stream {
format: Some(format),
..
} => matches!(format, FileFormat::Csv) && source.csv_metadata().is_none(),
_ => false,
}
}
fn quality_findings(input: &FindingsInput, out: &mut Collector) {
const QUALITY_RULES: [FindingCode; 4] = [
FindingCode::DuplicateRows,
FindingCode::FutureDates,
FindingCode::KeyDuplicates,
FindingCode::TemporalOrderViolations,
];
let Some(quality) = input.quality.as_ref() else {
for code in QUALITY_RULES {
out.skip_report(
code,
NotEvaluatedReason::QualityUnavailable {
quality_status: input.quality_status.clone(),
},
);
}
return;
};
match quality
.uniqueness
.as_ref()
.filter(|uniqueness| uniqueness.rows_checked > 0)
{
None => out.skip_report(FindingCode::DuplicateRows, NotEvaluatedReason::NotAssessed),
Some(uniqueness) if uniqueness.duplicate_rows_approximate => {
out.skip_report(FindingCode::DuplicateRows, NotEvaluatedReason::Estimated)
}
Some(uniqueness) if uniqueness.duplicate_rows > 0 => out.report(
FindingCode::DuplicateRows,
[
(
"duplicate_rows",
EvidenceValue::Count(uniqueness.duplicate_rows),
),
(
"rows_checked",
EvidenceValue::Count(uniqueness.rows_checked),
),
],
),
Some(_) => skip_unless_complete(out, FindingCode::DuplicateRows, quality, "duplicate_rows"),
}
key_duplicates(input, quality, out);
let timeliness = quality.timeliness.as_ref();
match timeliness.filter(|timeliness| timeliness.date_values_checked > 0) {
None => out.skip_report(FindingCode::FutureDates, NotEvaluatedReason::NotAssessed),
Some(timeliness) if timeliness.future_dates_count > 0 => out.report(
FindingCode::FutureDates,
[
(
"future_dates_count",
EvidenceValue::Count(timeliness.future_dates_count),
),
(
"date_values_checked",
EvidenceValue::Count(timeliness.date_values_checked),
),
],
),
Some(_) => skip_unless_complete(out, FindingCode::FutureDates, quality, "timeliness"),
}
match timeliness.filter(|timeliness| timeliness.temporal_pairs_checked > 0) {
None => out.skip_report(
FindingCode::TemporalOrderViolations,
NotEvaluatedReason::NotAssessed,
),
Some(timeliness) if timeliness.temporal_violations > 0 => out.report(
FindingCode::TemporalOrderViolations,
[
(
"temporal_violations",
EvidenceValue::Count(timeliness.temporal_violations),
),
(
"temporal_pairs_checked",
EvidenceValue::Count(timeliness.temporal_pairs_checked),
),
],
),
Some(_) => skip_unless_complete(
out,
FindingCode::TemporalOrderViolations,
quality,
"timeliness",
),
}
}
fn key_duplicates(input: &FindingsInput, quality: &QualityFacts, out: &mut Collector) {
const CODE: FindingCode = FindingCode::KeyDuplicates;
let Some(key) = quality
.uniqueness
.as_ref()
.and_then(|uniqueness| uniqueness.key_column.as_deref())
else {
out.skip_report(CODE, NotEvaluatedReason::NotAssessed);
return;
};
let Some(source) = input.key_column_source else {
out.skip(CODE, NotEvaluatedReason::Unrecorded, key);
return;
};
let Some((index, column)) = input
.columns
.iter()
.enumerate()
.find(|(_, column)| column.name == key)
else {
out.skip(CODE, NotEvaluatedReason::NotComputed, key);
return;
};
let Some(unique_count) = column.unique_count else {
out.skip(CODE, NotEvaluatedReason::NotComputed, key);
return;
};
let values = column.total_count.saturating_sub(column.null_count);
if values == 0 {
out.skip(CODE, NotEvaluatedReason::NoValues, key);
return;
}
let coverage = sampled_component_evidence(
quality.sampled_dimensions.as_deref(),
&quality.unlisted_components,
&[],
false,
Provenance::Component("key_uniqueness"),
);
let estimated = column.unique_count_is_approximate == Some(true)
|| matches!(
coverage,
Evidence::Incomplete {
reason: EvidenceGap::QualitySampled
}
);
if estimated {
out.skip(CODE, NotEvaluatedReason::Estimated, key);
return;
}
let exact =
column.unique_count_is_approximate == Some(false) || matches!(coverage, Evidence::Complete);
if !exact {
out.skip(CODE, NotEvaluatedReason::Unrecorded, key);
return;
}
let repeated = values.saturating_sub(unique_count);
if repeated == 0 {
return;
}
let source = match source {
KeyColumnSource::Declared => "declared",
};
out.column(
CODE,
index,
key,
[
("key_values_checked", EvidenceValue::Count(values)),
("repeated_values", EvidenceValue::Count(repeated)),
("key_column_source", EvidenceValue::Text(source.to_string())),
],
);
}
fn skip_unless_complete(
out: &mut Collector,
code: FindingCode,
quality: &QualityFacts,
component: &'static str,
) {
let coverage = sampled_component_evidence(
quality.sampled_dimensions.as_deref(),
&quality.unlisted_components,
&[],
false,
Provenance::Component(component),
);
match coverage {
Evidence::Complete => {}
Evidence::Incomplete {
reason: EvidenceGap::QualitySampled,
} => out.skip_report(code, NotEvaluatedReason::Sampled),
Evidence::Incomplete { .. } => out.skip_report(code, NotEvaluatedReason::Unrecorded),
}
}
fn is_sensitive(category: &str, name: &str) -> bool {
[PatternCategory::Contact, PatternCategory::Financial]
.iter()
.any(|sensitive| sensitive.to_string() == category)
|| PERSONAL_IDENTIFIER_PATTERNS.contains(&name)
}
fn sensitive_patterns(index: usize, column: &ColumnFacts, non_null: usize, out: &mut Collector) {
let Some(patterns) = column.patterns.as_ref() else {
out.skip(
FindingCode::SensitivePattern,
NotEvaluatedReason::NotComputed,
&column.name,
);
return;
};
if non_null == 0 {
out.skip(
FindingCode::SensitivePattern,
NotEvaluatedReason::NoValues,
&column.name,
);
return;
}
for pattern in patterns {
if pattern.confidence < MIN_PATTERN_CONFIDENCE
|| !is_sensitive(&pattern.category, &pattern.name)
{
continue;
}
out.push(
Finding {
code: FindingCode::SensitivePattern,
severity: FindingCode::SensitivePattern.severity(),
column: Some(column.name.clone()),
evidence: evidence([
("pattern", EvidenceValue::Text(pattern.name.clone())),
("category", EvidenceValue::Text(pattern.category.clone())),
("match_percentage", percentage(pattern.match_percentage)),
]),
summary: FindingCode::SensitivePattern.summary().to_string(),
},
Some(index),
pattern.name.clone(),
);
}
}
fn percentage(value: f64) -> EvidenceValue {
EvidenceValue::Percentage(rounded_2(value))
}
fn evidence<const N: usize>(
entries: [(&'static str, EvidenceValue); N],
) -> BTreeMap<String, EvidenceValue> {
entries
.into_iter()
.map(|(name, value)| (name.to_string(), value))
.collect()
}
struct Ranked {
finding: Finding,
column_index: Option<usize>,
detail: String,
}
#[derive(Default)]
struct Collector {
findings: Vec<Ranked>,
not_evaluated: Vec<UnevaluatedRule>,
}
impl Collector {
fn push(&mut self, finding: Finding, column_index: Option<usize>, detail: String) {
self.findings.push(Ranked {
finding,
column_index,
detail,
});
}
fn report<const N: usize>(
&mut self,
code: FindingCode,
entries: [(&'static str, EvidenceValue); N],
) {
self.push(
Finding {
code,
severity: code.severity(),
column: None,
evidence: evidence(entries),
summary: code.summary().to_string(),
},
None,
String::new(),
);
}
fn column<const N: usize>(
&mut self,
code: FindingCode,
index: usize,
name: &str,
entries: [(&'static str, EvidenceValue); N],
) {
self.push(
Finding {
code,
severity: code.severity(),
column: Some(name.to_string()),
evidence: evidence(entries),
summary: code.summary().to_string(),
},
Some(index),
String::new(),
);
}
fn skip_report(&mut self, code: FindingCode, reason: NotEvaluatedReason) {
self.not_evaluated.push(UnevaluatedRule {
code,
reason,
columns: Vec::new(),
});
}
fn skip(&mut self, code: FindingCode, reason: NotEvaluatedReason, column: &str) {
match self
.not_evaluated
.iter_mut()
.find(|entry| entry.code == code && entry.reason == reason)
{
Some(entry) => entry.columns.push(column.to_string()),
None => self.not_evaluated.push(UnevaluatedRule {
code,
reason,
columns: vec![column.to_string()],
}),
}
}
fn finish(mut self) -> FindingsResult {
self.findings.sort_by(|left, right| {
(left.finding.severity, left.finding.code)
.cmp(&(right.finding.severity, right.finding.code))
.then_with(|| left.column_index.cmp(&right.column_index))
.then_with(|| left.detail.cmp(&right.detail))
});
self.not_evaluated.sort_by(|left, right| {
(left.code, left.reason.rank()).cmp(&(right.code, right.reason.rank()))
});
FindingsResult {
findings: self
.findings
.into_iter()
.map(|ranked| ranked.finding)
.collect(),
not_evaluated: self.not_evaluated,
}
}
}
#[cfg(test)]
mod tests {
use dataprof_core::{
ColumnProfile, ColumnStats, CsvSourceMetadata, DataSource, ExecutionMetadata, FileFormat,
Pattern, TruncationReason, TypeHomogeneity,
};
use dataprof_metrics::{
MetricConfidence, QualityAssessment, QualityMetrics, TimelinessMetrics, UniquenessMetrics,
};
use super::*;
fn source() -> DataSource {
DataSource::File {
path: "drop.csv".to_string(),
format: FileFormat::Csv,
size_bytes: 256,
modified_at: None,
parquet_metadata: None,
csv_metadata: None,
}
}
fn column(name: &str, total: usize, nulls: usize) -> ColumnProfile {
ColumnProfile {
name: name.to_string(),
data_type: DataType::String,
null_count: nulls,
total_count: total,
unique_count: None,
unique_count_is_approximate: None,
unique_count_lower_bound: None,
invalid_count: None,
type_homogeneity: None,
locale_number_count: None,
native_type: None,
stats: ColumnStats::None,
patterns: None,
}
}
fn pattern(name: &str, category: PatternCategory, confidence: f64) -> Pattern {
Pattern {
name: name.to_string(),
regex: String::new(),
match_count: 10,
match_percentage: 100.0,
category,
confidence,
}
}
fn report(columns: Vec<ColumnProfile>, execution: ExecutionMetadata) -> ProfileReport {
ProfileReport::new(source(), columns, execution, None)
}
fn codes(result: &FindingsResult) -> Vec<(FindingCode, Option<&str>)> {
result
.findings
.iter()
.map(|finding| (finding.code, finding.column.as_deref()))
.collect()
}
#[test]
fn an_estimated_duplicate_count_witnesses_nothing() {
let metrics = QualityMetrics {
uniqueness: Some(UniquenessMetrics {
duplicate_rows: 1_200,
key_uniqueness: 100.0,
high_cardinality_warning: false,
rows_checked: 5_000_000,
key_column: None,
key_values_checked: None,
duplicate_rows_approximate: true,
duplicate_rows_exact_limit: None,
}),
..QualityMetrics::default()
};
let report = ProfileReport::new(
source(),
vec![],
ExecutionMetadata::new(5_000_000, 0, 10),
Some(QualityAssessment::exact(metrics)),
);
let result = report.findings();
assert!(
!codes(&result).contains(&(FindingCode::DuplicateRows, None)),
"an HLL estimate reported as a duplicate: {:?}",
result.findings
);
assert!(result.not_evaluated.contains(&UnevaluatedRule {
code: FindingCode::DuplicateRows,
reason: NotEvaluatedReason::Estimated,
columns: vec![],
}));
}
#[test]
fn the_threshold_is_compared_at_the_serialized_precision() {
let report = report(
vec![column("near", 25_000, 4_999)],
ExecutionMetadata::new(25_000, 1, 10),
);
let finding = report
.findings()
.findings
.into_iter()
.find(|finding| finding.code == FindingCode::NullHeavy)
.expect("reported at the rounded value");
assert_eq!(
finding.evidence["null_percentage"],
EvidenceValue::Percentage(20.0)
);
}
#[test]
fn only_confident_personal_or_financial_patterns_are_sensitive() {
let mut profile = column("mixed_ids", 10, 0);
profile.patterns = Some(vec![
pattern("UUID", PatternCategory::Identifier, 0.9),
pattern("SSN (US)", PatternCategory::Identifier, 0.9),
pattern("IBAN", PatternCategory::Financial, 0.4),
pattern("Email", PatternCategory::Contact, 0.8),
pattern("IPv4", PatternCategory::Network, 0.9),
]);
let result = report(vec![profile], ExecutionMetadata::new(10, 1, 10)).findings();
let reported: Vec<_> = result
.findings
.iter()
.filter(|finding| finding.code == FindingCode::SensitivePattern)
.map(|finding| finding.evidence["pattern"].clone())
.collect();
assert_eq!(
reported,
vec![
EvidenceValue::Text("Email".to_string()),
EvidenceValue::Text("SSN (US)".to_string()),
]
);
}
#[test]
fn evidence_holds_the_value_it_serializes() {
let mut profile = column("email", 3, 0);
profile.type_homogeneity = Some(TypeHomogeneity {
numeric: 1,
date: 0,
boolean: 0,
text: 2,
});
let mut email = pattern("Email", PatternCategory::Contact, 0.8);
email.match_percentage = 200.0 / 3.0;
profile.patterns = Some(vec![email]);
let result = report(vec![profile], ExecutionMetadata::new(3, 1, 10)).findings();
let serialized: FindingsResult = {
let value = serde_json::to_value(&result).expect("serializes");
let mut rebuilt = result.clone();
for (finding, document) in rebuilt
.findings
.iter_mut()
.zip(value["findings"].as_array().unwrap())
{
for (name, evidence) in finding.evidence.iter_mut() {
if let EvidenceValue::Percentage(_) = evidence {
*evidence =
EvidenceValue::Percentage(document["evidence"][name].as_f64().unwrap());
}
}
}
rebuilt
};
assert_eq!(result, serialized);
}
fn quality_report(future_dates: usize, confidence: MetricConfidence) -> ProfileReport {
let metrics = QualityMetrics {
uniqueness: Some(UniquenessMetrics {
duplicate_rows: 0,
key_uniqueness: 100.0,
high_cardinality_warning: false,
rows_checked: 60_000,
key_column: None,
key_values_checked: None,
duplicate_rows_approximate: false,
duplicate_rows_exact_limit: None,
}),
timeliness: Some(TimelinessMetrics {
future_dates_count: future_dates,
stale_data_ratio: 0.0,
temporal_violations: 0,
invalid_date_values: 0,
date_values_checked: 10_000,
temporal_pairs_checked: 10_000,
clock: None,
}),
..QualityMetrics::default()
};
ProfileReport::new(
source(),
vec![],
ExecutionMetadata::new(60_000, 0, 10),
Some(QualityAssessment::new(metrics, confidence)),
)
}
fn skipped(result: &FindingsResult, code: FindingCode) -> Option<&NotEvaluatedReason> {
result
.not_evaluated
.iter()
.find(|entry| entry.code == code)
.map(|entry| &entry.reason)
}
#[test]
fn a_zero_count_from_the_quality_sample_rules_nothing_out() {
let sampled = || MetricConfidence::Mixed {
exact_dimensions: vec!["duplicate_rows".to_string()],
sampled_dimensions: vec!["timeliness".to_string()],
sample_size: 10_000,
};
let clean = quality_report(0, sampled()).findings();
assert_eq!(
skipped(&clean, FindingCode::FutureDates),
Some(&NotEvaluatedReason::Sampled)
);
assert_eq!(
skipped(&clean, FindingCode::TemporalOrderViolations),
Some(&NotEvaluatedReason::Sampled)
);
assert_eq!(skipped(&clean, FindingCode::DuplicateRows), None);
let witnessed = quality_report(3, sampled()).findings();
assert!(codes(&witnessed).contains(&(FindingCode::FutureDates, None)));
assert_eq!(skipped(&witnessed, FindingCode::FutureDates), None);
}
#[test]
fn unrecorded_coverage_does_not_read_as_clean() {
let result = quality_report(0, MetricConfidence::Unrecorded).findings();
for code in [
FindingCode::DuplicateRows,
FindingCode::FutureDates,
FindingCode::TemporalOrderViolations,
] {
assert_eq!(
skipped(&result, code),
Some(&NotEvaluatedReason::Unrecorded),
"{code}"
);
}
}
#[test]
fn a_report_from_before_ragged_counting_does_not_read_as_clean() {
let mut legacy = report(vec![], ExecutionMetadata::new(5, 0, 10));
legacy.schema_version = 0;
assert_eq!(
skipped(&legacy.findings(), FindingCode::RaggedRows),
Some(&NotEvaluatedReason::Unrecorded)
);
let current = report(vec![], ExecutionMetadata::new(5, 0, 10)).findings();
assert_eq!(skipped(¤t, FindingCode::RaggedRows), None);
}
#[test]
fn a_csv_report_without_source_metadata_does_not_read_as_a_clear_delimiter() {
let rules = [FindingCode::DelimiterAmbiguous, FindingCode::HeaderNotSplit];
let with_source = |data_source: DataSource| {
ProfileReport::new(data_source, vec![], ExecutionMetadata::new(5, 0, 10), None)
.findings()
};
let mut wrong = Vec::new();
let legacy = with_source(source());
for code in rules {
if skipped(&legacy, code) != Some(&NotEvaluatedReason::Unrecorded) {
wrong.push(format!("legacy csv: {code} not listed as unrecorded"));
}
}
let json = DataSource::File {
path: "drop.json".to_string(),
format: FileFormat::Json,
size_bytes: 256,
modified_at: None,
parquet_metadata: None,
csv_metadata: None,
};
let recorded = [
(
"clear detection",
source().with_csv_metadata(CsvSourceMetadata::detected(DelimiterDetection::new(
b',', 3, None,
))),
),
(
"configured",
source().with_csv_metadata(CsvSourceMetadata::configured(b';')),
),
("json", json),
];
for (case, data_source) in recorded {
let result = with_source(data_source);
for code in rules {
if skipped(&result, code).is_some()
|| result.findings.iter().any(|finding| finding.code == code)
{
wrong.push(format!("{case}: {code} skipped or reported"));
}
}
}
assert!(wrong.is_empty(), "{}", wrong.join("\n"));
}
fn classified(name: &str, numeric: usize, text: usize, locale: Option<usize>) -> ColumnProfile {
let mut profile = column(name, numeric + text, 0);
profile.type_homogeneity = Some(TypeHomogeneity {
numeric,
date: 0,
boolean: 0,
text,
});
profile.locale_number_count = locale;
profile
}
fn locale_number_columns(result: &FindingsResult) -> Vec<&str> {
result
.findings
.iter()
.filter(|finding| finding.code == FindingCode::LocaleNumbers)
.filter_map(|finding| finding.column.as_deref())
.collect()
}
#[test]
fn locale_numbers_are_reported_from_half_of_the_text_values() {
let result = report(
vec![
classified("half", 0, 4, Some(2)),
classified("under_half", 0, 5, Some(2)),
classified("float_with_commas", 9, 1, Some(1)),
classified("clean", 0, 4, Some(0)),
],
ExecutionMetadata::new(10, 0, 10),
)
.findings();
assert_eq!(
locale_number_columns(&result),
["half", "float_with_commas"]
);
let half = &result.findings[0];
assert_eq!(half.code, FindingCode::LocaleNumbers);
assert_eq!(
half.evidence,
BTreeMap::from([
("classified_count".to_string(), EvidenceValue::Count(4)),
("locale_number_count".to_string(), EvidenceValue::Count(2)),
("non_null_count".to_string(), EvidenceValue::Count(4)),
("text_count".to_string(), EvidenceValue::Count(4)),
])
);
assert_eq!(skipped(&result, FindingCode::LocaleNumbers), None);
}
#[test]
fn an_identifier_column_is_exempt_from_locale_numbers() {
let mut codes = classified("codes", 0, 4, Some(4));
codes.data_type = DataType::Identifier;
let result = report(vec![codes], ExecutionMetadata::new(4, 0, 10)).findings();
assert!(locale_number_columns(&result).is_empty());
assert_eq!(skipped(&result, FindingCode::LocaleNumbers), None);
}
#[test]
fn a_report_from_before_locale_number_counting_does_not_read_as_clean() {
let result = report(
vec![classified("legacy", 0, 4, None)],
ExecutionMetadata::new(4, 0, 10),
)
.findings();
assert_eq!(
skipped(&result, FindingCode::LocaleNumbers),
Some(&NotEvaluatedReason::Unrecorded)
);
let result = report(
vec![column("unclassified", 4, 0)],
ExecutionMetadata::new(4, 0, 10),
)
.findings();
assert_eq!(
skipped(&result, FindingCode::LocaleNumbers),
Some(&NotEvaluatedReason::NotComputed)
);
}
#[test]
fn thresholds_apply_at_the_precision_they_are_reported_at() {
let result = FindingPolicy::new()
.null_heavy_percentage(20.004)
.evaluate(&report(
vec![column("fifth", 5, 1)],
ExecutionMetadata::new(5, 1, 10),
))
.expect("in range");
let finding = result
.findings
.iter()
.find(|finding| finding.code == FindingCode::NullHeavy)
.expect("reported at the applied threshold");
assert_eq!(
finding.evidence["threshold"],
EvidenceValue::Percentage(20.0)
);
for value in [1e-9, 0.004] {
assert!(
FindingPolicy::new()
.null_heavy_percentage(value)
.validate()
.is_err(),
"{value} was accepted"
);
}
assert!(
FindingPolicy::new()
.null_heavy_percentage(0.005)
.validate()
.is_ok()
);
}
#[test]
fn a_nested_column_is_assessed_by_its_counts_only() {
let mut nested = column("payload", 4, 4);
nested.data_type = DataType::Nested;
let result = report(vec![nested], ExecutionMetadata::new(4, 1, 10)).findings();
assert_eq!(
codes(&result),
vec![(FindingCode::AllNull, Some("payload"))]
);
for code in [
FindingCode::ConstantColumn,
FindingCode::MixedTypes,
FindingCode::SensitivePattern,
] {
assert!(
result.not_evaluated.contains(&UnevaluatedRule {
code,
reason: NotEvaluatedReason::NotComputed,
columns: vec!["payload".to_string()],
}),
"{code} was not listed: {:?}",
result.not_evaluated
);
}
}
#[test]
fn a_sampled_scan_names_its_reason_and_truncation_wins() {
let sampled = report(
vec![],
ExecutionMetadata::new(100, 0, 10).with_sampling(0.1),
);
let both = report(
vec![],
ExecutionMetadata::new(100, 0, 10)
.with_sampling(0.1)
.with_truncation(TruncationReason::MaxRows(100)),
);
for (report, expected) in [(sampled, "sampled"), (both, "truncated")] {
let finding = report
.findings()
.findings
.into_iter()
.find(|finding| finding.code == FindingCode::PartialScan)
.expect("a partial scan is disclosed");
assert_eq!(
finding.evidence["reason"],
EvidenceValue::Text(expected.to_string())
);
}
}
fn key_report(
unique: usize,
approximate: Option<bool>,
confidence: MetricConfidence,
) -> ProfileReport {
let values = 10;
let mut key = column("order_id", values + 1, 1);
key.data_type = DataType::Identifier;
key.unique_count = Some(unique);
key.unique_count_is_approximate = approximate;
let metrics = QualityMetrics {
uniqueness: Some(UniquenessMetrics {
duplicate_rows: 0,
key_uniqueness: unique as f64 / values as f64 * 100.0,
high_cardinality_warning: false,
rows_checked: values + 1,
key_column: Some("order_id".to_string()),
key_values_checked: Some(values),
duplicate_rows_approximate: false,
duplicate_rows_exact_limit: None,
}),
..QualityMetrics::default()
};
ProfileReport::new(
source(),
vec![column("amount", values + 1, 0), key],
ExecutionMetadata::new(values + 1, 0, 10),
Some(QualityAssessment::new(metrics, confidence)),
)
}
fn key_sampled() -> MetricConfidence {
MetricConfidence::Mixed {
exact_dimensions: vec!["duplicate_rows".to_string()],
sampled_dimensions: vec!["key_uniqueness".to_string()],
sample_size: 10,
}
}
fn key_outcome(result: &FindingsResult) -> String {
let found: Vec<_> = result
.findings
.iter()
.filter(|finding| finding.code == FindingCode::KeyDuplicates)
.collect();
match (
found.as_slice(),
skipped(result, FindingCode::KeyDuplicates),
) {
([finding], None) => format!(
"found on {:?}: {}",
finding.column,
serde_json::to_string(&finding.evidence).unwrap()
),
([], Some(reason)) => format!("skipped: {}", serde_json::to_string(reason).unwrap()),
([], None) => "clean".to_string(),
_ => format!("inconsistent: {found:?}"),
}
}
#[test]
fn a_repeated_declared_key_is_reported_only_from_an_exact_count() {
let found = r#"found on Some("order_id"): {"key_column_source":"declared","key_values_checked":10,"repeated_values":2}"#;
let skip = |reason: &str| format!(r#"skipped: {{"reason":"{reason}"}}"#);
let mut legacy_key = key_report(8, Some(false), MetricConfidence::Exact);
legacy_key
.metric_semantics
.as_mut()
.expect("a current report records its semantics")
.key_column_source = None;
let mut no_key = key_report(8, Some(false), MetricConfidence::Exact);
no_key
.quality
.as_mut()
.and_then(|quality| quality.metrics.uniqueness.as_mut())
.expect("uniqueness assessed")
.key_column = None;
let mut all_null_key = key_report(0, Some(false), MetricConfidence::Exact);
all_null_key.column_profiles[1].null_count = 11;
let cases = [
(
"exact count with repeats",
key_report(8, Some(false), MetricConfidence::Exact),
found.to_string(),
),
(
"exact count, distinct, one null",
key_report(10, Some(false), MetricConfidence::Exact),
"clean".to_string(),
),
(
"estimated distinct count",
key_report(8, Some(true), MetricConfidence::Exact),
skip("estimated"),
),
(
"key component filed as sampled",
key_report(8, Some(false), key_sampled()),
skip("estimated"),
),
(
"exactness recorded on the column only",
key_report(8, Some(false), MetricConfidence::Unrecorded),
found.to_string(),
),
(
"exactness recorded in the coverage only",
key_report(8, None, MetricConfidence::Exact),
found.to_string(),
),
(
"exactness recorded nowhere",
key_report(8, None, MetricConfidence::Unrecorded),
skip("unrecorded"),
),
("key chosen before 0.13", legacy_key, skip("unrecorded")),
("no key measured", no_key, skip("not_assessed")),
("key without values", all_null_key, skip("no_values")),
];
let mut wrong = Vec::new();
for (case, report, expected) in cases {
let outcome = key_outcome(&report.findings());
if outcome != expected {
wrong.push(format!("{case}: {outcome}, expected {expected}"));
}
}
assert!(wrong.is_empty(), "{}", wrong.join("\n"));
}
#[test]
fn warnings_sort_before_info_then_by_code_then_column_position() {
let mut constant = column("z_constant", 5, 0);
constant.unique_count = Some(1);
let mut mixed = column("a_mixed", 5, 0);
mixed.type_homogeneity = Some(TypeHomogeneity {
numeric: 3,
date: 0,
boolean: 0,
text: 2,
});
let result = report(
vec![
constant,
column("sparse", 5, 4),
mixed,
column("empty", 5, 5),
],
ExecutionMetadata::new(5, 4, 10).with_ragged_row_count(1),
)
.findings();
assert_eq!(
codes(&result),
vec![
(FindingCode::AllNull, Some("empty")),
(FindingCode::MixedTypes, Some("a_mixed")),
(FindingCode::NullHeavy, Some("sparse")),
(FindingCode::RaggedRows, None),
(FindingCode::ConstantColumn, Some("z_constant")),
]
);
assert!(
result
.findings
.iter()
.all(|finding| finding.severity == finding.code.severity())
);
}
}