1use std::collections::BTreeMap;
36use std::fmt;
37
38use dataprof_core::serde_helpers::rounded_2;
39use dataprof_core::{
40 DataSource, DataType, DelimiterDetection, FileFormat, KeyColumnSource, PatternCategory,
41 TypeHomogeneity,
42};
43
44use crate::profile_report::ProfileReport;
45use crate::quality_gate::{
46 Evidence, EvidenceGap, Provenance, quality_status_name, sampled_component_evidence,
47};
48
49pub const DEFAULT_NULL_HEAVY_PERCENTAGE: f64 = 20.0;
52
53pub const DEFAULT_MIXED_TYPES_PERCENTAGE: f64 = 5.0;
56
57const MIN_PATTERN_CONFIDENCE: f64 = 0.5;
60
61const PERSONAL_IDENTIFIER_PATTERNS: [&str; 2] = ["Codice Fiscale (IT)", "SSN (US)"];
64
65#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, serde::Serialize)]
69#[serde(rename_all = "snake_case")]
70#[non_exhaustive]
71pub enum Severity {
72 Warning,
74 Info,
76}
77
78impl fmt::Display for Severity {
79 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
80 match self {
81 Self::Warning => write!(f, "warning"),
82 Self::Info => write!(f, "info"),
83 }
84 }
85}
86
87#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, serde::Serialize)]
93#[serde(rename_all = "snake_case")]
94#[non_exhaustive]
95pub enum FindingCode {
96 AllNull,
98 ConstantColumn,
100 DelimiterAmbiguous,
103 DuplicateRows,
105 FutureDates,
107 HeaderNotSplit,
111 KeyDuplicates,
113 LocaleNumbers,
117 MixedTypes,
119 NullHeavy,
121 PartialScan,
123 RaggedRows,
125 RecordsSkipped,
127 SensitivePattern,
129 TemporalOrderViolations,
131 UnterminatedQuote,
133}
134
135impl FindingCode {
136 pub const ALL: [Self; 16] = [
140 Self::AllNull,
141 Self::ConstantColumn,
142 Self::DelimiterAmbiguous,
143 Self::DuplicateRows,
144 Self::FutureDates,
145 Self::HeaderNotSplit,
146 Self::KeyDuplicates,
147 Self::LocaleNumbers,
148 Self::MixedTypes,
149 Self::NullHeavy,
150 Self::PartialScan,
151 Self::RaggedRows,
152 Self::RecordsSkipped,
153 Self::SensitivePattern,
154 Self::TemporalOrderViolations,
155 Self::UnterminatedQuote,
156 ];
157
158 pub fn severity(self) -> Severity {
160 match self {
161 Self::AllNull
162 | Self::DelimiterAmbiguous
163 | Self::DuplicateRows
164 | Self::FutureDates
165 | Self::HeaderNotSplit
166 | Self::KeyDuplicates
167 | Self::LocaleNumbers
168 | Self::MixedTypes
169 | Self::NullHeavy
170 | Self::RaggedRows
171 | Self::RecordsSkipped
172 | Self::TemporalOrderViolations
173 | Self::UnterminatedQuote => Severity::Warning,
174 Self::ConstantColumn | Self::PartialScan | Self::SensitivePattern => Severity::Info,
175 }
176 }
177
178 fn summary(self) -> &'static str {
183 match self {
184 Self::AllNull => "every value in this column is null",
185 Self::ConstantColumn => "every non-null value in this column is the same",
186 Self::DelimiterAmbiguous => {
187 "another delimiter reads the sampled CSV records as well as the one detection \
188 chose, so the columns may be split on the wrong one; set csv_delimiter to choose"
189 }
190 Self::DuplicateRows => "the source contains exact duplicate rows",
191 Self::FutureDates => "some date values lie in the future",
192 Self::HeaderNotSplit => {
193 "no delimiter detection tries (comma, semicolon, tab, pipe) splits the CSV \
194 header, so the source was read as one column, but another character splits the \
195 header and every sampled record alike; set csv_delimiter to read it as a table"
196 }
197 Self::KeyDuplicates => {
198 "some non-null values of the declared key column occur more than once"
199 }
200 Self::LocaleNumbers => {
201 "at least half of the text values in this column are numbers written with a \
202 decimal comma or digit grouping, so no numeric statistic includes them"
203 }
204 Self::MixedTypes => "this column's values are split across lexical types",
205 Self::NullHeavy => "this column's null share is at or above the threshold",
206 Self::PartialScan => {
207 "only part of the source was read; findings describe the rows that were"
208 }
209 Self::RaggedRows => {
210 "some rows had a different field count from the header and were recovered"
211 }
212 Self::RecordsSkipped => "errors were counted while reading the source",
213 Self::SensitivePattern => {
214 "values in this column match a pattern for personal or financial data"
215 }
216 Self::TemporalOrderViolations => "some start dates fall after their paired end dates",
217 Self::UnterminatedQuote => {
218 "the source ends inside a quoted field, so its last record may hold several rows"
219 }
220 }
221 }
222}
223
224impl fmt::Display for FindingCode {
225 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
226 let name = match self {
227 Self::AllNull => "all_null",
228 Self::ConstantColumn => "constant_column",
229 Self::DelimiterAmbiguous => "delimiter_ambiguous",
230 Self::DuplicateRows => "duplicate_rows",
231 Self::FutureDates => "future_dates",
232 Self::HeaderNotSplit => "header_not_split",
233 Self::KeyDuplicates => "key_duplicates",
234 Self::LocaleNumbers => "locale_numbers",
235 Self::MixedTypes => "mixed_types",
236 Self::NullHeavy => "null_heavy",
237 Self::PartialScan => "partial_scan",
238 Self::RaggedRows => "ragged_rows",
239 Self::RecordsSkipped => "records_skipped",
240 Self::SensitivePattern => "sensitive_pattern",
241 Self::TemporalOrderViolations => "temporal_order_violations",
242 Self::UnterminatedQuote => "unterminated_quote",
243 };
244 write!(f, "{name}")
245 }
246}
247
248impl std::str::FromStr for FindingCode {
249 type Err = String;
250
251 fn from_str(s: &str) -> Result<Self, Self::Err> {
253 Self::ALL
254 .into_iter()
255 .find(|code| code.to_string() == s)
256 .ok_or_else(|| {
257 let valid: Vec<String> = Self::ALL.iter().map(ToString::to_string).collect();
258 format!(
259 "Unknown finding code: {s}. Valid codes: {}",
260 valid.join(", ")
261 )
262 })
263 }
264}
265
266#[derive(Debug, Clone, PartialEq, serde::Serialize)]
271#[serde(untagged)]
272#[non_exhaustive]
273pub enum EvidenceValue {
274 Count(usize),
276 Percentage(#[serde(serialize_with = "dataprof_core::serde_helpers::round_2")] f64),
278 Text(String),
280 Flag(bool),
282}
283
284#[derive(Debug, Clone, PartialEq, serde::Serialize)]
286#[non_exhaustive]
287pub struct Finding {
288 pub code: FindingCode,
290 pub severity: Severity,
292 #[serde(skip_serializing_if = "Option::is_none")]
294 pub column: Option<String>,
295 pub evidence: BTreeMap<String, EvidenceValue>,
298 pub summary: String,
300}
301
302#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize)]
304#[serde(tag = "reason", rename_all = "snake_case")]
305#[non_exhaustive]
306pub enum NotEvaluatedReason {
307 #[non_exhaustive]
309 QualityUnavailable {
310 quality_status: String,
312 },
313 NotAssessed,
316 Estimated,
320 Sampled,
324 Unrecorded,
331 NotComputed,
334 NoValues,
336}
337
338impl NotEvaluatedReason {
339 fn rank(&self) -> u8 {
341 match self {
342 Self::QualityUnavailable { .. } => 0,
343 Self::NotAssessed => 1,
344 Self::Estimated => 2,
345 Self::Sampled => 3,
346 Self::Unrecorded => 4,
347 Self::NotComputed => 5,
348 Self::NoValues => 6,
349 }
350 }
351}
352
353#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize)]
355#[non_exhaustive]
356pub struct UnevaluatedRule {
357 pub code: FindingCode,
359 #[serde(flatten)]
361 pub reason: NotEvaluatedReason,
362 #[serde(skip_serializing_if = "Vec::is_empty")]
365 pub columns: Vec<String>,
366}
367
368#[derive(Debug, Clone, PartialEq, serde::Serialize)]
370#[non_exhaustive]
371pub struct FindingsResult {
372 pub findings: Vec<Finding>,
376 pub not_evaluated: Vec<UnevaluatedRule>,
379}
380
381#[derive(Debug, Clone, PartialEq, serde::Deserialize)]
394#[non_exhaustive]
395pub struct FindingsInput {
396 rows_processed: usize,
397 source_exhausted: Option<bool>,
399 truncated: bool,
401 sampling_applied: Option<bool>,
403 ragged_row_count: Option<usize>,
406 unterminated_quote: Option<bool>,
410 #[serde(default)]
415 csv_source: Option<bool>,
416 error_count: usize,
417 #[serde(default)]
420 delimiter_detection: Option<DelimiterDetection>,
421 #[serde(default)]
425 csv_source_unrecorded: bool,
426 #[serde(default)]
430 key_column_source: Option<KeyColumnSource>,
431 quality_status: String,
433 quality: Option<QualityFacts>,
434 columns: Vec<ColumnFacts>,
435}
436
437#[derive(Debug, Clone, PartialEq, serde::Deserialize)]
438struct QualityFacts {
439 uniqueness: Option<UniquenessFacts>,
440 timeliness: Option<TimelinessFacts>,
441 sampled_dimensions: Option<Vec<String>>,
444 unlisted_components: Vec<String>,
447}
448
449#[derive(Debug, Clone, PartialEq, serde::Deserialize)]
450struct UniquenessFacts {
451 #[serde(default)]
452 rows_checked: usize,
453 #[serde(default)]
454 duplicate_rows: usize,
455 #[serde(default)]
456 duplicate_rows_approximate: bool,
457 #[serde(default)]
459 key_column: Option<String>,
460}
461
462#[derive(Debug, Clone, PartialEq, serde::Deserialize)]
463struct TimelinessFacts {
464 #[serde(default)]
465 date_values_checked: usize,
466 #[serde(default)]
467 future_dates_count: usize,
468 #[serde(default)]
469 temporal_pairs_checked: usize,
470 #[serde(default)]
471 temporal_violations: usize,
472}
473
474#[derive(Debug, Clone, PartialEq, serde::Deserialize)]
475struct ColumnFacts {
476 name: String,
477 identifier: bool,
479 total_count: usize,
480 null_count: usize,
481 unique_count: Option<usize>,
482 #[serde(default)]
484 unique_count_is_approximate: Option<bool>,
485 type_homogeneity: Option<TypeHomogeneity>,
486 locale_number_count: Option<usize>,
487 patterns: Option<Vec<PatternFacts>>,
488}
489
490#[derive(Debug, Clone, PartialEq, serde::Deserialize)]
491struct PatternFacts {
492 name: String,
493 category: String,
496 confidence: f64,
497 match_percentage: f64,
498}
499
500impl FindingsInput {
501 pub fn from_json(text: &str) -> serde_json::Result<Self> {
504 serde_json::from_str(text)
505 }
506
507 pub(crate) fn reads_a_partial_sample(&self, code: FindingCode) -> bool {
518 let exempts_identifiers = match code {
519 FindingCode::MixedTypes | FindingCode::LocaleNumbers => true,
520 FindingCode::SensitivePattern => false,
521 _ => return false,
522 };
523 self.columns
524 .iter()
525 .filter(|column| !(exempts_identifiers && column.identifier))
526 .any(|column| {
527 let non_null = column.total_count.saturating_sub(column.null_count);
528 column
529 .type_homogeneity
530 .as_ref()
531 .is_some_and(|forms| forms.classified_count() < non_null)
532 })
533 }
534}
535
536impl From<&ProfileReport> for FindingsInput {
537 fn from(report: &ProfileReport) -> Self {
538 let execution = &report.execution;
539 Self {
540 rows_processed: execution.rows_processed,
541 source_exhausted: Some(execution.source_exhausted),
542 truncated: execution.truncation_reason.is_some(),
543 sampling_applied: Some(execution.sampling_applied),
544 ragged_row_count: (report.schema_version != 0).then_some(execution.ragged_row_count),
547 unterminated_quote: execution.unterminated_quote,
548 csv_source: csv_source(&report.data_source),
549 error_count: execution.error_count,
550 delimiter_detection: report
551 .data_source
552 .csv_metadata()
553 .and_then(|metadata| metadata.delimiter_detection.clone()),
554 csv_source_unrecorded: csv_source_unrecorded(&report.data_source),
555 key_column_source: report
556 .metric_semantics
557 .as_ref()
558 .and_then(|semantics| semantics.key_column_source),
559 quality_status: quality_status_name(&report.quality_status).to_string(),
560 quality: report.quality.as_ref().map(|quality| QualityFacts {
561 uniqueness: quality
562 .metrics
563 .uniqueness
564 .as_ref()
565 .map(|uniqueness| UniquenessFacts {
566 rows_checked: uniqueness.rows_checked,
567 duplicate_rows: uniqueness.duplicate_rows,
568 duplicate_rows_approximate: uniqueness.duplicate_rows_approximate,
569 key_column: uniqueness.key_column.clone(),
570 }),
571 timeliness: quality
572 .metrics
573 .timeliness
574 .as_ref()
575 .map(|timeliness| TimelinessFacts {
576 date_values_checked: timeliness.date_values_checked,
577 future_dates_count: timeliness.future_dates_count,
578 temporal_pairs_checked: timeliness.temporal_pairs_checked,
579 temporal_violations: timeliness.temporal_violations,
580 }),
581 sampled_dimensions: quality.sampled_dimensions(),
582 unlisted_components: quality.unlisted_components(),
583 }),
584 columns: report
585 .column_profiles
586 .iter()
587 .map(|column| ColumnFacts {
588 name: column.name.clone(),
589 identifier: column.data_type == DataType::Identifier,
590 total_count: column.total_count,
591 null_count: column.null_count,
592 unique_count: column.unique_count,
593 unique_count_is_approximate: column.unique_count_is_approximate,
594 type_homogeneity: column.type_homogeneity,
595 locale_number_count: column.locale_number_count,
596 patterns: column.patterns.as_ref().map(|patterns| {
597 patterns
598 .iter()
599 .map(|pattern| PatternFacts {
600 name: pattern.name.clone(),
601 category: pattern.category.to_string(),
602 confidence: pattern.confidence,
603 match_percentage: pattern.match_percentage,
604 })
605 .collect()
606 }),
607 })
608 .collect(),
609 }
610 }
611}
612
613#[derive(Debug, Clone, PartialEq)]
615#[non_exhaustive]
616pub enum FindingPolicyError {
617 #[non_exhaustive]
621 ThresholdOutOfRange {
622 setting: &'static str,
624 value: f64,
626 },
627}
628
629impl fmt::Display for FindingPolicyError {
630 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
631 match self {
632 Self::ThresholdOutOfRange { setting, value } => write!(
633 f,
634 "{setting} must be a percentage above 0 and at most 100 at two decimal \
635 places, got {value}"
636 ),
637 }
638 }
639}
640
641impl std::error::Error for FindingPolicyError {}
642
643#[derive(Debug, Clone, PartialEq)]
649#[non_exhaustive]
650pub struct FindingPolicy {
651 null_heavy_percentage: f64,
652 mixed_types_percentage: f64,
653}
654
655impl Default for FindingPolicy {
656 fn default() -> Self {
657 Self {
658 null_heavy_percentage: DEFAULT_NULL_HEAVY_PERCENTAGE,
659 mixed_types_percentage: DEFAULT_MIXED_TYPES_PERCENTAGE,
660 }
661 }
662}
663
664impl FindingPolicy {
665 pub fn new() -> Self {
667 Self::default()
668 }
669
670 pub fn null_heavy_percentage(mut self, percentage: f64) -> Self {
674 self.null_heavy_percentage = percentage;
675 self
676 }
677
678 pub fn mixed_types_percentage(mut self, percentage: f64) -> Self {
682 self.mixed_types_percentage = percentage;
683 self
684 }
685
686 pub fn validate(&self) -> Result<(), FindingPolicyError> {
688 for (setting, value) in [
689 ("null_heavy_percentage", self.null_heavy_percentage),
690 ("mixed_types_percentage", self.mixed_types_percentage),
691 ] {
692 let applied = rounded_2(value);
693 if !(value.is_finite() && applied > 0.0 && applied <= 100.0) {
694 return Err(FindingPolicyError::ThresholdOutOfRange { setting, value });
695 }
696 }
697 Ok(())
698 }
699
700 pub fn evaluate(&self, report: &ProfileReport) -> Result<FindingsResult, FindingPolicyError> {
704 self.evaluate_input(&FindingsInput::from(report))
705 }
706
707 pub fn evaluate_input(
710 &self,
711 input: &FindingsInput,
712 ) -> Result<FindingsResult, FindingPolicyError> {
713 self.validate()?;
714 Ok(self.collect(input))
715 }
716
717 fn null_heavy_threshold(&self) -> f64 {
720 rounded_2(self.null_heavy_percentage)
721 }
722
723 fn mixed_types_threshold(&self) -> f64 {
724 rounded_2(self.mixed_types_percentage)
725 }
726
727 pub(crate) fn collect(&self, input: &FindingsInput) -> FindingsResult {
728 let mut collector = Collector::default();
729 scan_findings(input, &mut collector);
730 quality_findings(input, &mut collector);
731 for (index, column) in input.columns.iter().enumerate() {
732 self.column_findings(index, column, &mut collector);
733 }
734 collector.finish()
735 }
736
737 fn column_findings(&self, index: usize, column: &ColumnFacts, out: &mut Collector) {
738 let name = column.name.as_str();
739 let non_null = column.total_count.saturating_sub(column.null_count);
740
741 if column.total_count == 0 {
742 out.skip(FindingCode::AllNull, NotEvaluatedReason::NoValues, name);
743 out.skip(FindingCode::NullHeavy, NotEvaluatedReason::NoValues, name);
744 } else if non_null == 0 {
745 out.column(
746 FindingCode::AllNull,
747 index,
748 name,
749 [
750 ("null_count", EvidenceValue::Count(column.null_count)),
751 ("total_count", EvidenceValue::Count(column.total_count)),
752 ],
753 );
754 } else {
755 let null_percentage =
758 rounded_2(column.null_count as f64 / column.total_count as f64 * 100.0);
759 if null_percentage >= self.null_heavy_threshold() {
760 out.column(
761 FindingCode::NullHeavy,
762 index,
763 name,
764 [
765 ("null_percentage", percentage(null_percentage)),
766 ("threshold", percentage(self.null_heavy_threshold())),
767 ],
768 );
769 }
770 }
771
772 match column.unique_count {
773 None => out.skip(
774 FindingCode::ConstantColumn,
775 NotEvaluatedReason::NotComputed,
776 name,
777 ),
778 Some(_) if non_null == 0 => out.skip(
779 FindingCode::ConstantColumn,
780 NotEvaluatedReason::NoValues,
781 name,
782 ),
783 Some(1) if non_null > 1 => out.column(
785 FindingCode::ConstantColumn,
786 index,
787 name,
788 [
789 ("unique_count", EvidenceValue::Count(1)),
790 ("non_null_count", EvidenceValue::Count(non_null)),
791 ],
792 ),
793 Some(_) => {}
794 }
795
796 locale_numbers(index, column, out);
797 self.mixed_types(index, column, non_null, out);
798 sensitive_patterns(index, column, non_null, out);
799 }
800
801 fn mixed_types(
802 &self,
803 index: usize,
804 column: &ColumnFacts,
805 non_null: usize,
806 out: &mut Collector,
807 ) {
808 let name = column.name.as_str();
809 if column.identifier {
812 return;
813 }
814 let Some(homogeneity) = column.type_homogeneity.as_ref() else {
815 out.skip(
816 FindingCode::MixedTypes,
817 NotEvaluatedReason::NotComputed,
818 name,
819 );
820 return;
821 };
822 let classified = homogeneity.classified_count();
823 let Some((dominant, dominant_count)) = homogeneity.dominant() else {
824 out.skip(FindingCode::MixedTypes, NotEvaluatedReason::NoValues, name);
825 return;
826 };
827 let outside = classified - dominant_count;
828 if outside == 0 {
829 return;
830 }
831 let outside_percentage = rounded_2(outside as f64 / classified as f64 * 100.0);
832 if outside_percentage < self.mixed_types_threshold() {
833 return;
834 }
835 out.column(
836 FindingCode::MixedTypes,
837 index,
838 name,
839 [
840 ("dominant_type", EvidenceValue::Text(dominant.to_string())),
841 (
842 "dominant_percentage",
843 percentage(dominant_count as f64 / classified as f64 * 100.0),
844 ),
845 ("threshold", percentage(self.mixed_types_threshold())),
846 ("classified_count", EvidenceValue::Count(classified)),
850 ("non_null_count", EvidenceValue::Count(non_null)),
851 ],
852 );
853 }
854}
855
856fn locale_numbers(index: usize, column: &ColumnFacts, out: &mut Collector) {
865 let name = column.name.as_str();
866 if column.identifier {
869 return;
870 }
871 let Some(homogeneity) = column.type_homogeneity.as_ref() else {
872 out.skip(
873 FindingCode::LocaleNumbers,
874 NotEvaluatedReason::NotComputed,
875 name,
876 );
877 return;
878 };
879 let Some(locale_numbers) = column.locale_number_count else {
881 out.skip(
882 FindingCode::LocaleNumbers,
883 NotEvaluatedReason::Unrecorded,
884 name,
885 );
886 return;
887 };
888 let classified = homogeneity.classified_count();
889 if classified == 0 {
890 out.skip(
891 FindingCode::LocaleNumbers,
892 NotEvaluatedReason::NoValues,
893 name,
894 );
895 return;
896 }
897 let text = homogeneity.text;
898 if locale_numbers == 0 || locale_numbers * 2 < text {
899 return;
900 }
901 out.column(
902 FindingCode::LocaleNumbers,
903 index,
904 name,
905 [
906 ("locale_number_count", EvidenceValue::Count(locale_numbers)),
907 ("text_count", EvidenceValue::Count(text)),
908 ("classified_count", EvidenceValue::Count(classified)),
911 (
912 "non_null_count",
913 EvidenceValue::Count(column.total_count.saturating_sub(column.null_count)),
914 ),
915 ],
916 );
917}
918
919impl ProfileReport {
920 pub fn findings(&self) -> FindingsResult {
925 FindingPolicy::default().collect(&FindingsInput::from(self))
926 }
927}
928
929fn scan_findings(input: &FindingsInput, out: &mut Collector) {
930 let partial = if input.source_exhausted == Some(false) || input.truncated {
933 Some("truncated")
934 } else if input.sampling_applied == Some(true) {
935 Some("sampled")
936 } else {
937 None
938 };
939 match partial {
940 Some(reason) => out.report(
941 FindingCode::PartialScan,
942 [
943 ("reason", EvidenceValue::Text(reason.to_string())),
944 ("rows_processed", EvidenceValue::Count(input.rows_processed)),
945 ],
946 ),
947 None if input.source_exhausted.is_none() || input.sampling_applied.is_none() => {
948 out.skip_report(FindingCode::PartialScan, NotEvaluatedReason::Unrecorded);
949 }
950 None => {}
951 }
952 match input.ragged_row_count {
953 None => out.skip_report(FindingCode::RaggedRows, NotEvaluatedReason::Unrecorded),
954 Some(0) => {}
955 Some(ragged_row_count) => out.report(
956 FindingCode::RaggedRows,
957 [
958 ("ragged_row_count", EvidenceValue::Count(ragged_row_count)),
959 ("rows_processed", EvidenceValue::Count(input.rows_processed)),
960 ],
961 ),
962 }
963 match (input.unterminated_quote, input.csv_source) {
964 (Some(true), _) => out.report(
965 FindingCode::UnterminatedQuote,
966 [("rows_processed", EvidenceValue::Count(input.rows_processed))],
967 ),
968 (Some(false), _) => {}
969 (None, Some(false)) => {}
972 (None, _) => out.skip_report(
973 FindingCode::UnterminatedQuote,
974 NotEvaluatedReason::Unrecorded,
975 ),
976 }
977 if input.error_count > 0 {
978 out.report(
979 FindingCode::RecordsSkipped,
980 [("error_count", EvidenceValue::Count(input.error_count))],
981 );
982 }
983 if input.csv_source_unrecorded {
984 out.skip_report(
985 FindingCode::DelimiterAmbiguous,
986 NotEvaluatedReason::Unrecorded,
987 );
988 out.skip_report(FindingCode::HeaderNotSplit, NotEvaluatedReason::Unrecorded);
989 }
990 if let Some(detection) = &input.delimiter_detection {
993 if let Some(runner_up) = &detection.runner_up {
994 out.report(
995 FindingCode::DelimiterAmbiguous,
996 [
997 (
998 "delimiter",
999 EvidenceValue::Text(detection.delimiter.clone()),
1000 ),
1001 ("runner_up", EvidenceValue::Text(runner_up.clone())),
1002 ],
1003 );
1004 }
1005 if let Some(unlisted) = &detection.unlisted_delimiter {
1009 out.report(
1010 FindingCode::HeaderNotSplit,
1011 [("unlisted_delimiter", EvidenceValue::Text(unlisted.clone()))],
1012 );
1013 }
1014 }
1015}
1016
1017fn csv_source(source: &DataSource) -> Option<bool> {
1021 match source {
1022 DataSource::Stream { format: None, .. } => None,
1023 _ => Some(matches!(source.format(), Some(FileFormat::Csv))),
1024 }
1025}
1026
1027fn csv_source_unrecorded(source: &DataSource) -> bool {
1031 match source {
1032 DataSource::File { format, .. }
1033 | DataSource::Bytes { format, .. }
1034 | DataSource::Stream {
1035 format: Some(format),
1036 ..
1037 } => matches!(format, FileFormat::Csv) && source.csv_metadata().is_none(),
1038 _ => false,
1039 }
1040}
1041
1042fn quality_findings(input: &FindingsInput, out: &mut Collector) {
1043 const QUALITY_RULES: [FindingCode; 4] = [
1044 FindingCode::DuplicateRows,
1045 FindingCode::FutureDates,
1046 FindingCode::KeyDuplicates,
1047 FindingCode::TemporalOrderViolations,
1048 ];
1049 let Some(quality) = input.quality.as_ref() else {
1050 for code in QUALITY_RULES {
1051 out.skip_report(
1052 code,
1053 NotEvaluatedReason::QualityUnavailable {
1054 quality_status: input.quality_status.clone(),
1055 },
1056 );
1057 }
1058 return;
1059 };
1060
1061 match quality
1062 .uniqueness
1063 .as_ref()
1064 .filter(|uniqueness| uniqueness.rows_checked > 0)
1065 {
1066 None => out.skip_report(FindingCode::DuplicateRows, NotEvaluatedReason::NotAssessed),
1067 Some(uniqueness) if uniqueness.duplicate_rows_approximate => {
1068 out.skip_report(FindingCode::DuplicateRows, NotEvaluatedReason::Estimated)
1069 }
1070 Some(uniqueness) if uniqueness.duplicate_rows > 0 => out.report(
1071 FindingCode::DuplicateRows,
1072 [
1073 (
1074 "duplicate_rows",
1075 EvidenceValue::Count(uniqueness.duplicate_rows),
1076 ),
1077 (
1078 "rows_checked",
1079 EvidenceValue::Count(uniqueness.rows_checked),
1080 ),
1081 ],
1082 ),
1083 Some(_) => skip_unless_complete(out, FindingCode::DuplicateRows, quality, "duplicate_rows"),
1084 }
1085
1086 key_duplicates(input, quality, out);
1087
1088 let timeliness = quality.timeliness.as_ref();
1089 match timeliness.filter(|timeliness| timeliness.date_values_checked > 0) {
1090 None => out.skip_report(FindingCode::FutureDates, NotEvaluatedReason::NotAssessed),
1091 Some(timeliness) if timeliness.future_dates_count > 0 => out.report(
1092 FindingCode::FutureDates,
1093 [
1094 (
1095 "future_dates_count",
1096 EvidenceValue::Count(timeliness.future_dates_count),
1097 ),
1098 (
1099 "date_values_checked",
1100 EvidenceValue::Count(timeliness.date_values_checked),
1101 ),
1102 ],
1103 ),
1104 Some(_) => skip_unless_complete(out, FindingCode::FutureDates, quality, "timeliness"),
1105 }
1106 match timeliness.filter(|timeliness| timeliness.temporal_pairs_checked > 0) {
1107 None => out.skip_report(
1108 FindingCode::TemporalOrderViolations,
1109 NotEvaluatedReason::NotAssessed,
1110 ),
1111 Some(timeliness) if timeliness.temporal_violations > 0 => out.report(
1112 FindingCode::TemporalOrderViolations,
1113 [
1114 (
1115 "temporal_violations",
1116 EvidenceValue::Count(timeliness.temporal_violations),
1117 ),
1118 (
1119 "temporal_pairs_checked",
1120 EvidenceValue::Count(timeliness.temporal_pairs_checked),
1121 ),
1122 ],
1123 ),
1124 Some(_) => skip_unless_complete(
1125 out,
1126 FindingCode::TemporalOrderViolations,
1127 quality,
1128 "timeliness",
1129 ),
1130 }
1131}
1132
1133fn key_duplicates(input: &FindingsInput, quality: &QualityFacts, out: &mut Collector) {
1145 const CODE: FindingCode = FindingCode::KeyDuplicates;
1146 let Some(key) = quality
1147 .uniqueness
1148 .as_ref()
1149 .and_then(|uniqueness| uniqueness.key_column.as_deref())
1150 else {
1151 out.skip_report(CODE, NotEvaluatedReason::NotAssessed);
1152 return;
1153 };
1154 let Some(source) = input.key_column_source else {
1157 out.skip(CODE, NotEvaluatedReason::Unrecorded, key);
1158 return;
1159 };
1160 let Some((index, column)) = input
1161 .columns
1162 .iter()
1163 .enumerate()
1164 .find(|(_, column)| column.name == key)
1165 else {
1166 out.skip(CODE, NotEvaluatedReason::NotComputed, key);
1167 return;
1168 };
1169 let Some(unique_count) = column.unique_count else {
1170 out.skip(CODE, NotEvaluatedReason::NotComputed, key);
1171 return;
1172 };
1173 let values = column.total_count.saturating_sub(column.null_count);
1174 if values == 0 {
1175 out.skip(CODE, NotEvaluatedReason::NoValues, key);
1176 return;
1177 }
1178 let coverage = sampled_component_evidence(
1179 quality.sampled_dimensions.as_deref(),
1180 &quality.unlisted_components,
1181 &[],
1182 false,
1183 Provenance::Component("key_uniqueness"),
1184 );
1185 let estimated = column.unique_count_is_approximate == Some(true)
1188 || matches!(
1189 coverage,
1190 Evidence::Incomplete {
1191 reason: EvidenceGap::QualitySampled
1192 }
1193 );
1194 if estimated {
1195 out.skip(CODE, NotEvaluatedReason::Estimated, key);
1196 return;
1197 }
1198 let exact =
1199 column.unique_count_is_approximate == Some(false) || matches!(coverage, Evidence::Complete);
1200 if !exact {
1201 out.skip(CODE, NotEvaluatedReason::Unrecorded, key);
1202 return;
1203 }
1204 let repeated = values.saturating_sub(unique_count);
1205 if repeated == 0 {
1206 return;
1207 }
1208 let source = match source {
1209 KeyColumnSource::Declared => "declared",
1210 };
1211 out.column(
1212 CODE,
1213 index,
1214 key,
1215 [
1216 ("key_values_checked", EvidenceValue::Count(values)),
1217 ("repeated_values", EvidenceValue::Count(repeated)),
1218 ("key_column_source", EvidenceValue::Text(source.to_string())),
1219 ],
1220 );
1221}
1222
1223fn skip_unless_complete(
1232 out: &mut Collector,
1233 code: FindingCode,
1234 quality: &QualityFacts,
1235 component: &'static str,
1236) {
1237 let coverage = sampled_component_evidence(
1240 quality.sampled_dimensions.as_deref(),
1241 &quality.unlisted_components,
1242 &[],
1243 false,
1244 Provenance::Component(component),
1245 );
1246 match coverage {
1247 Evidence::Complete => {}
1248 Evidence::Incomplete {
1249 reason: EvidenceGap::QualitySampled,
1250 } => out.skip_report(code, NotEvaluatedReason::Sampled),
1251 Evidence::Incomplete { .. } => out.skip_report(code, NotEvaluatedReason::Unrecorded),
1252 }
1253}
1254
1255fn is_sensitive(category: &str, name: &str) -> bool {
1257 [PatternCategory::Contact, PatternCategory::Financial]
1258 .iter()
1259 .any(|sensitive| sensitive.to_string() == category)
1260 || PERSONAL_IDENTIFIER_PATTERNS.contains(&name)
1261}
1262
1263fn sensitive_patterns(index: usize, column: &ColumnFacts, non_null: usize, out: &mut Collector) {
1264 let Some(patterns) = column.patterns.as_ref() else {
1265 out.skip(
1266 FindingCode::SensitivePattern,
1267 NotEvaluatedReason::NotComputed,
1268 &column.name,
1269 );
1270 return;
1271 };
1272 if non_null == 0 {
1273 out.skip(
1275 FindingCode::SensitivePattern,
1276 NotEvaluatedReason::NoValues,
1277 &column.name,
1278 );
1279 return;
1280 }
1281 for pattern in patterns {
1282 if pattern.confidence < MIN_PATTERN_CONFIDENCE
1283 || !is_sensitive(&pattern.category, &pattern.name)
1284 {
1285 continue;
1286 }
1287 out.push(
1288 Finding {
1289 code: FindingCode::SensitivePattern,
1290 severity: FindingCode::SensitivePattern.severity(),
1291 column: Some(column.name.clone()),
1292 evidence: evidence([
1293 ("pattern", EvidenceValue::Text(pattern.name.clone())),
1294 ("category", EvidenceValue::Text(pattern.category.clone())),
1295 ("match_percentage", percentage(pattern.match_percentage)),
1296 ]),
1297 summary: FindingCode::SensitivePattern.summary().to_string(),
1298 },
1299 Some(index),
1300 pattern.name.clone(),
1301 );
1302 }
1303}
1304
1305fn percentage(value: f64) -> EvidenceValue {
1308 EvidenceValue::Percentage(rounded_2(value))
1309}
1310
1311fn evidence<const N: usize>(
1312 entries: [(&'static str, EvidenceValue); N],
1313) -> BTreeMap<String, EvidenceValue> {
1314 entries
1315 .into_iter()
1316 .map(|(name, value)| (name.to_string(), value))
1317 .collect()
1318}
1319
1320struct Ranked {
1322 finding: Finding,
1323 column_index: Option<usize>,
1325 detail: String,
1327}
1328
1329#[derive(Default)]
1330struct Collector {
1331 findings: Vec<Ranked>,
1332 not_evaluated: Vec<UnevaluatedRule>,
1333}
1334
1335impl Collector {
1336 fn push(&mut self, finding: Finding, column_index: Option<usize>, detail: String) {
1337 self.findings.push(Ranked {
1338 finding,
1339 column_index,
1340 detail,
1341 });
1342 }
1343
1344 fn report<const N: usize>(
1345 &mut self,
1346 code: FindingCode,
1347 entries: [(&'static str, EvidenceValue); N],
1348 ) {
1349 self.push(
1350 Finding {
1351 code,
1352 severity: code.severity(),
1353 column: None,
1354 evidence: evidence(entries),
1355 summary: code.summary().to_string(),
1356 },
1357 None,
1358 String::new(),
1359 );
1360 }
1361
1362 fn column<const N: usize>(
1363 &mut self,
1364 code: FindingCode,
1365 index: usize,
1366 name: &str,
1367 entries: [(&'static str, EvidenceValue); N],
1368 ) {
1369 self.push(
1370 Finding {
1371 code,
1372 severity: code.severity(),
1373 column: Some(name.to_string()),
1374 evidence: evidence(entries),
1375 summary: code.summary().to_string(),
1376 },
1377 Some(index),
1378 String::new(),
1379 );
1380 }
1381
1382 fn skip_report(&mut self, code: FindingCode, reason: NotEvaluatedReason) {
1383 self.not_evaluated.push(UnevaluatedRule {
1384 code,
1385 reason,
1386 columns: Vec::new(),
1387 });
1388 }
1389
1390 fn skip(&mut self, code: FindingCode, reason: NotEvaluatedReason, column: &str) {
1393 match self
1394 .not_evaluated
1395 .iter_mut()
1396 .find(|entry| entry.code == code && entry.reason == reason)
1397 {
1398 Some(entry) => entry.columns.push(column.to_string()),
1399 None => self.not_evaluated.push(UnevaluatedRule {
1400 code,
1401 reason,
1402 columns: vec![column.to_string()],
1403 }),
1404 }
1405 }
1406
1407 fn finish(mut self) -> FindingsResult {
1408 self.findings.sort_by(|left, right| {
1409 (left.finding.severity, left.finding.code)
1410 .cmp(&(right.finding.severity, right.finding.code))
1411 .then_with(|| left.column_index.cmp(&right.column_index))
1412 .then_with(|| left.detail.cmp(&right.detail))
1413 });
1414 self.not_evaluated.sort_by(|left, right| {
1416 (left.code, left.reason.rank()).cmp(&(right.code, right.reason.rank()))
1417 });
1418 FindingsResult {
1419 findings: self
1420 .findings
1421 .into_iter()
1422 .map(|ranked| ranked.finding)
1423 .collect(),
1424 not_evaluated: self.not_evaluated,
1425 }
1426 }
1427}
1428
1429#[cfg(test)]
1430mod tests {
1431 use dataprof_core::{
1432 ColumnProfile, ColumnStats, CsvSourceMetadata, DataSource, ExecutionMetadata, FileFormat,
1433 Pattern, TruncationReason, TypeHomogeneity,
1434 };
1435 use dataprof_metrics::{
1436 MetricConfidence, QualityAssessment, QualityMetrics, TimelinessMetrics, UniquenessMetrics,
1437 };
1438
1439 use super::*;
1440
1441 fn source() -> DataSource {
1442 DataSource::File {
1443 path: "drop.csv".to_string(),
1444 format: FileFormat::Csv,
1445 size_bytes: 256,
1446 modified_at: None,
1447 parquet_metadata: None,
1448 csv_metadata: None,
1449 }
1450 }
1451
1452 fn column(name: &str, total: usize, nulls: usize) -> ColumnProfile {
1453 ColumnProfile {
1454 name: name.to_string(),
1455 data_type: DataType::String,
1456 null_count: nulls,
1457 total_count: total,
1458 unique_count: None,
1459 unique_count_is_approximate: None,
1460 unique_count_lower_bound: None,
1461 invalid_count: None,
1462 type_homogeneity: None,
1463 locale_number_count: None,
1464 native_type: None,
1465 stats: ColumnStats::None,
1466 patterns: None,
1467 }
1468 }
1469
1470 fn pattern(name: &str, category: PatternCategory, confidence: f64) -> Pattern {
1471 Pattern {
1472 name: name.to_string(),
1473 regex: String::new(),
1474 match_count: 10,
1475 match_percentage: 100.0,
1476 category,
1477 confidence,
1478 }
1479 }
1480
1481 fn report(columns: Vec<ColumnProfile>, execution: ExecutionMetadata) -> ProfileReport {
1482 ProfileReport::new(source(), columns, execution, None)
1483 }
1484
1485 fn codes(result: &FindingsResult) -> Vec<(FindingCode, Option<&str>)> {
1486 result
1487 .findings
1488 .iter()
1489 .map(|finding| (finding.code, finding.column.as_deref()))
1490 .collect()
1491 }
1492
1493 #[test]
1494 fn an_estimated_duplicate_count_witnesses_nothing() {
1495 let metrics = QualityMetrics {
1496 uniqueness: Some(UniquenessMetrics {
1497 duplicate_rows: 1_200,
1498 key_uniqueness: 100.0,
1499 high_cardinality_warning: false,
1500 rows_checked: 5_000_000,
1501 key_column: None,
1502 key_values_checked: None,
1503 duplicate_rows_approximate: true,
1504 duplicate_rows_exact_limit: None,
1505 }),
1506 ..QualityMetrics::default()
1507 };
1508 let report = ProfileReport::new(
1509 source(),
1510 vec![],
1511 ExecutionMetadata::new(5_000_000, 0, 10),
1512 Some(QualityAssessment::exact(metrics)),
1513 );
1514
1515 let result = report.findings();
1516
1517 assert!(
1518 !codes(&result).contains(&(FindingCode::DuplicateRows, None)),
1519 "an HLL estimate reported as a duplicate: {:?}",
1520 result.findings
1521 );
1522 assert!(result.not_evaluated.contains(&UnevaluatedRule {
1523 code: FindingCode::DuplicateRows,
1524 reason: NotEvaluatedReason::Estimated,
1525 columns: vec![],
1526 }));
1527 }
1528
1529 #[test]
1530 fn the_threshold_is_compared_at_the_serialized_precision() {
1531 let report = report(
1535 vec![column("near", 25_000, 4_999)],
1536 ExecutionMetadata::new(25_000, 1, 10),
1537 );
1538
1539 let finding = report
1540 .findings()
1541 .findings
1542 .into_iter()
1543 .find(|finding| finding.code == FindingCode::NullHeavy)
1544 .expect("reported at the rounded value");
1545 assert_eq!(
1546 finding.evidence["null_percentage"],
1547 EvidenceValue::Percentage(20.0)
1548 );
1549 }
1550
1551 #[test]
1552 fn only_confident_personal_or_financial_patterns_are_sensitive() {
1553 let mut profile = column("mixed_ids", 10, 0);
1554 profile.patterns = Some(vec![
1555 pattern("UUID", PatternCategory::Identifier, 0.9),
1556 pattern("SSN (US)", PatternCategory::Identifier, 0.9),
1557 pattern("IBAN", PatternCategory::Financial, 0.4),
1558 pattern("Email", PatternCategory::Contact, 0.8),
1559 pattern("IPv4", PatternCategory::Network, 0.9),
1560 ]);
1561 let result = report(vec![profile], ExecutionMetadata::new(10, 1, 10)).findings();
1562
1563 let reported: Vec<_> = result
1564 .findings
1565 .iter()
1566 .filter(|finding| finding.code == FindingCode::SensitivePattern)
1567 .map(|finding| finding.evidence["pattern"].clone())
1568 .collect();
1569 assert_eq!(
1571 reported,
1572 vec![
1573 EvidenceValue::Text("Email".to_string()),
1574 EvidenceValue::Text("SSN (US)".to_string()),
1575 ]
1576 );
1577 }
1578
1579 #[test]
1580 fn evidence_holds_the_value_it_serializes() {
1581 let mut profile = column("email", 3, 0);
1585 profile.type_homogeneity = Some(TypeHomogeneity {
1586 numeric: 1,
1587 date: 0,
1588 boolean: 0,
1589 text: 2,
1590 });
1591 let mut email = pattern("Email", PatternCategory::Contact, 0.8);
1592 email.match_percentage = 200.0 / 3.0;
1593 profile.patterns = Some(vec![email]);
1594 let result = report(vec![profile], ExecutionMetadata::new(3, 1, 10)).findings();
1595
1596 let serialized: FindingsResult = {
1597 let value = serde_json::to_value(&result).expect("serializes");
1598 let mut rebuilt = result.clone();
1599 for (finding, document) in rebuilt
1600 .findings
1601 .iter_mut()
1602 .zip(value["findings"].as_array().unwrap())
1603 {
1604 for (name, evidence) in finding.evidence.iter_mut() {
1605 if let EvidenceValue::Percentage(_) = evidence {
1606 *evidence =
1607 EvidenceValue::Percentage(document["evidence"][name].as_f64().unwrap());
1608 }
1609 }
1610 }
1611 rebuilt
1612 };
1613 assert_eq!(result, serialized);
1614 }
1615
1616 fn quality_report(future_dates: usize, confidence: MetricConfidence) -> ProfileReport {
1617 let metrics = QualityMetrics {
1618 uniqueness: Some(UniquenessMetrics {
1619 duplicate_rows: 0,
1620 key_uniqueness: 100.0,
1621 high_cardinality_warning: false,
1622 rows_checked: 60_000,
1623 key_column: None,
1624 key_values_checked: None,
1625 duplicate_rows_approximate: false,
1626 duplicate_rows_exact_limit: None,
1627 }),
1628 timeliness: Some(TimelinessMetrics {
1629 future_dates_count: future_dates,
1630 stale_data_ratio: 0.0,
1631 temporal_violations: 0,
1632 invalid_date_values: 0,
1633 date_values_checked: 10_000,
1634 temporal_pairs_checked: 10_000,
1635 clock: None,
1636 }),
1637 ..QualityMetrics::default()
1638 };
1639 ProfileReport::new(
1640 source(),
1641 vec![],
1642 ExecutionMetadata::new(60_000, 0, 10),
1643 Some(QualityAssessment::new(metrics, confidence)),
1644 )
1645 }
1646
1647 fn skipped(result: &FindingsResult, code: FindingCode) -> Option<&NotEvaluatedReason> {
1648 result
1649 .not_evaluated
1650 .iter()
1651 .find(|entry| entry.code == code)
1652 .map(|entry| &entry.reason)
1653 }
1654
1655 #[test]
1656 fn a_zero_count_from_the_quality_sample_rules_nothing_out() {
1657 let sampled = || MetricConfidence::Mixed {
1660 exact_dimensions: vec!["duplicate_rows".to_string()],
1661 sampled_dimensions: vec!["timeliness".to_string()],
1662 sample_size: 10_000,
1663 };
1664
1665 let clean = quality_report(0, sampled()).findings();
1666 assert_eq!(
1667 skipped(&clean, FindingCode::FutureDates),
1668 Some(&NotEvaluatedReason::Sampled)
1669 );
1670 assert_eq!(
1671 skipped(&clean, FindingCode::TemporalOrderViolations),
1672 Some(&NotEvaluatedReason::Sampled)
1673 );
1674 assert_eq!(skipped(&clean, FindingCode::DuplicateRows), None);
1676
1677 let witnessed = quality_report(3, sampled()).findings();
1679 assert!(codes(&witnessed).contains(&(FindingCode::FutureDates, None)));
1680 assert_eq!(skipped(&witnessed, FindingCode::FutureDates), None);
1681 }
1682
1683 #[test]
1684 fn unrecorded_coverage_does_not_read_as_clean() {
1685 let result = quality_report(0, MetricConfidence::Unrecorded).findings();
1686 for code in [
1687 FindingCode::DuplicateRows,
1688 FindingCode::FutureDates,
1689 FindingCode::TemporalOrderViolations,
1690 ] {
1691 assert_eq!(
1692 skipped(&result, code),
1693 Some(&NotEvaluatedReason::Unrecorded),
1694 "{code}"
1695 );
1696 }
1697 }
1698
1699 #[test]
1700 fn a_report_from_before_ragged_counting_does_not_read_as_clean() {
1701 let mut legacy = report(vec![], ExecutionMetadata::new(5, 0, 10));
1702 legacy.schema_version = 0;
1703 assert_eq!(
1704 skipped(&legacy.findings(), FindingCode::RaggedRows),
1705 Some(&NotEvaluatedReason::Unrecorded)
1706 );
1707
1708 let current = report(vec![], ExecutionMetadata::new(5, 0, 10)).findings();
1709 assert_eq!(skipped(¤t, FindingCode::RaggedRows), None);
1710 }
1711
1712 #[test]
1717 fn a_csv_report_without_source_metadata_does_not_read_as_a_clear_delimiter() {
1718 let rules = [FindingCode::DelimiterAmbiguous, FindingCode::HeaderNotSplit];
1719 let with_source = |data_source: DataSource| {
1720 ProfileReport::new(data_source, vec![], ExecutionMetadata::new(5, 0, 10), None)
1721 .findings()
1722 };
1723 let mut wrong = Vec::new();
1724
1725 let legacy = with_source(source());
1726 for code in rules {
1727 if skipped(&legacy, code) != Some(&NotEvaluatedReason::Unrecorded) {
1728 wrong.push(format!("legacy csv: {code} not listed as unrecorded"));
1729 }
1730 }
1731
1732 let json = DataSource::File {
1733 path: "drop.json".to_string(),
1734 format: FileFormat::Json,
1735 size_bytes: 256,
1736 modified_at: None,
1737 parquet_metadata: None,
1738 csv_metadata: None,
1739 };
1740 let recorded = [
1741 (
1742 "clear detection",
1743 source().with_csv_metadata(CsvSourceMetadata::detected(DelimiterDetection::new(
1744 b',', 3, None,
1745 ))),
1746 ),
1747 (
1748 "configured",
1749 source().with_csv_metadata(CsvSourceMetadata::configured(b';')),
1750 ),
1751 ("json", json),
1752 ];
1753 for (case, data_source) in recorded {
1754 let result = with_source(data_source);
1755 for code in rules {
1756 if skipped(&result, code).is_some()
1757 || result.findings.iter().any(|finding| finding.code == code)
1758 {
1759 wrong.push(format!("{case}: {code} skipped or reported"));
1760 }
1761 }
1762 }
1763 assert!(wrong.is_empty(), "{}", wrong.join("\n"));
1764 }
1765
1766 fn classified(name: &str, numeric: usize, text: usize, locale: Option<usize>) -> ColumnProfile {
1769 let mut profile = column(name, numeric + text, 0);
1770 profile.type_homogeneity = Some(TypeHomogeneity {
1771 numeric,
1772 date: 0,
1773 boolean: 0,
1774 text,
1775 });
1776 profile.locale_number_count = locale;
1777 profile
1778 }
1779
1780 fn locale_number_columns(result: &FindingsResult) -> Vec<&str> {
1781 result
1782 .findings
1783 .iter()
1784 .filter(|finding| finding.code == FindingCode::LocaleNumbers)
1785 .filter_map(|finding| finding.column.as_deref())
1786 .collect()
1787 }
1788
1789 #[test]
1790 fn locale_numbers_are_reported_from_half_of_the_text_values() {
1791 let result = report(
1792 vec![
1793 classified("half", 0, 4, Some(2)),
1794 classified("under_half", 0, 5, Some(2)),
1795 classified("float_with_commas", 9, 1, Some(1)),
1797 classified("clean", 0, 4, Some(0)),
1798 ],
1799 ExecutionMetadata::new(10, 0, 10),
1800 )
1801 .findings();
1802
1803 assert_eq!(
1804 locale_number_columns(&result),
1805 ["half", "float_with_commas"]
1806 );
1807 let half = &result.findings[0];
1808 assert_eq!(half.code, FindingCode::LocaleNumbers);
1809 assert_eq!(
1810 half.evidence,
1811 BTreeMap::from([
1812 ("classified_count".to_string(), EvidenceValue::Count(4)),
1813 ("locale_number_count".to_string(), EvidenceValue::Count(2)),
1814 ("non_null_count".to_string(), EvidenceValue::Count(4)),
1815 ("text_count".to_string(), EvidenceValue::Count(4)),
1816 ])
1817 );
1818 assert_eq!(skipped(&result, FindingCode::LocaleNumbers), None);
1819 }
1820
1821 #[test]
1822 fn an_identifier_column_is_exempt_from_locale_numbers() {
1823 let mut codes = classified("codes", 0, 4, Some(4));
1824 codes.data_type = DataType::Identifier;
1825
1826 let result = report(vec![codes], ExecutionMetadata::new(4, 0, 10)).findings();
1827
1828 assert!(locale_number_columns(&result).is_empty());
1829 assert_eq!(skipped(&result, FindingCode::LocaleNumbers), None);
1830 }
1831
1832 #[test]
1833 fn a_report_from_before_locale_number_counting_does_not_read_as_clean() {
1834 let result = report(
1836 vec![classified("legacy", 0, 4, None)],
1837 ExecutionMetadata::new(4, 0, 10),
1838 )
1839 .findings();
1840 assert_eq!(
1841 skipped(&result, FindingCode::LocaleNumbers),
1842 Some(&NotEvaluatedReason::Unrecorded)
1843 );
1844
1845 let result = report(
1847 vec![column("unclassified", 4, 0)],
1848 ExecutionMetadata::new(4, 0, 10),
1849 )
1850 .findings();
1851 assert_eq!(
1852 skipped(&result, FindingCode::LocaleNumbers),
1853 Some(&NotEvaluatedReason::NotComputed)
1854 );
1855 }
1856
1857 #[test]
1858 fn thresholds_apply_at_the_precision_they_are_reported_at() {
1859 let result = FindingPolicy::new()
1862 .null_heavy_percentage(20.004)
1863 .evaluate(&report(
1864 vec![column("fifth", 5, 1)],
1865 ExecutionMetadata::new(5, 1, 10),
1866 ))
1867 .expect("in range");
1868 let finding = result
1869 .findings
1870 .iter()
1871 .find(|finding| finding.code == FindingCode::NullHeavy)
1872 .expect("reported at the applied threshold");
1873 assert_eq!(
1874 finding.evidence["threshold"],
1875 EvidenceValue::Percentage(20.0)
1876 );
1877
1878 for value in [1e-9, 0.004] {
1880 assert!(
1881 FindingPolicy::new()
1882 .null_heavy_percentage(value)
1883 .validate()
1884 .is_err(),
1885 "{value} was accepted"
1886 );
1887 }
1888 assert!(
1889 FindingPolicy::new()
1890 .null_heavy_percentage(0.005)
1891 .validate()
1892 .is_ok()
1893 );
1894 }
1895
1896 #[test]
1897 fn a_nested_column_is_assessed_by_its_counts_only() {
1898 let mut nested = column("payload", 4, 4);
1901 nested.data_type = DataType::Nested;
1902 let result = report(vec![nested], ExecutionMetadata::new(4, 1, 10)).findings();
1903
1904 assert_eq!(
1905 codes(&result),
1906 vec![(FindingCode::AllNull, Some("payload"))]
1907 );
1908 for code in [
1909 FindingCode::ConstantColumn,
1910 FindingCode::MixedTypes,
1911 FindingCode::SensitivePattern,
1912 ] {
1913 assert!(
1914 result.not_evaluated.contains(&UnevaluatedRule {
1915 code,
1916 reason: NotEvaluatedReason::NotComputed,
1917 columns: vec!["payload".to_string()],
1918 }),
1919 "{code} was not listed: {:?}",
1920 result.not_evaluated
1921 );
1922 }
1923 }
1924
1925 #[test]
1926 fn a_sampled_scan_names_its_reason_and_truncation_wins() {
1927 let sampled = report(
1928 vec![],
1929 ExecutionMetadata::new(100, 0, 10).with_sampling(0.1),
1930 );
1931 let both = report(
1932 vec![],
1933 ExecutionMetadata::new(100, 0, 10)
1934 .with_sampling(0.1)
1935 .with_truncation(TruncationReason::MaxRows(100)),
1936 );
1937
1938 for (report, expected) in [(sampled, "sampled"), (both, "truncated")] {
1939 let finding = report
1940 .findings()
1941 .findings
1942 .into_iter()
1943 .find(|finding| finding.code == FindingCode::PartialScan)
1944 .expect("a partial scan is disclosed");
1945 assert_eq!(
1946 finding.evidence["reason"],
1947 EvidenceValue::Text(expected.to_string())
1948 );
1949 }
1950 }
1951
1952 fn key_report(
1955 unique: usize,
1956 approximate: Option<bool>,
1957 confidence: MetricConfidence,
1958 ) -> ProfileReport {
1959 let values = 10;
1960 let mut key = column("order_id", values + 1, 1);
1961 key.data_type = DataType::Identifier;
1962 key.unique_count = Some(unique);
1963 key.unique_count_is_approximate = approximate;
1964 let metrics = QualityMetrics {
1965 uniqueness: Some(UniquenessMetrics {
1966 duplicate_rows: 0,
1967 key_uniqueness: unique as f64 / values as f64 * 100.0,
1968 high_cardinality_warning: false,
1969 rows_checked: values + 1,
1970 key_column: Some("order_id".to_string()),
1971 key_values_checked: Some(values),
1972 duplicate_rows_approximate: false,
1973 duplicate_rows_exact_limit: None,
1974 }),
1975 ..QualityMetrics::default()
1976 };
1977 ProfileReport::new(
1978 source(),
1979 vec![column("amount", values + 1, 0), key],
1980 ExecutionMetadata::new(values + 1, 0, 10),
1981 Some(QualityAssessment::new(metrics, confidence)),
1982 )
1983 }
1984
1985 fn key_sampled() -> MetricConfidence {
1986 MetricConfidence::Mixed {
1987 exact_dimensions: vec!["duplicate_rows".to_string()],
1988 sampled_dimensions: vec!["key_uniqueness".to_string()],
1989 sample_size: 10,
1990 }
1991 }
1992
1993 fn key_outcome(result: &FindingsResult) -> String {
1995 let found: Vec<_> = result
1996 .findings
1997 .iter()
1998 .filter(|finding| finding.code == FindingCode::KeyDuplicates)
1999 .collect();
2000 match (
2001 found.as_slice(),
2002 skipped(result, FindingCode::KeyDuplicates),
2003 ) {
2004 ([finding], None) => format!(
2005 "found on {:?}: {}",
2006 finding.column,
2007 serde_json::to_string(&finding.evidence).unwrap()
2008 ),
2009 ([], Some(reason)) => format!("skipped: {}", serde_json::to_string(reason).unwrap()),
2010 ([], None) => "clean".to_string(),
2011 _ => format!("inconsistent: {found:?}"),
2012 }
2013 }
2014
2015 #[test]
2016 fn a_repeated_declared_key_is_reported_only_from_an_exact_count() {
2017 let found = r#"found on Some("order_id"): {"key_column_source":"declared","key_values_checked":10,"repeated_values":2}"#;
2018 let skip = |reason: &str| format!(r#"skipped: {{"reason":"{reason}"}}"#);
2019 let mut legacy_key = key_report(8, Some(false), MetricConfidence::Exact);
2020 legacy_key
2021 .metric_semantics
2022 .as_mut()
2023 .expect("a current report records its semantics")
2024 .key_column_source = None;
2025 let mut no_key = key_report(8, Some(false), MetricConfidence::Exact);
2026 no_key
2027 .quality
2028 .as_mut()
2029 .and_then(|quality| quality.metrics.uniqueness.as_mut())
2030 .expect("uniqueness assessed")
2031 .key_column = None;
2032 let mut all_null_key = key_report(0, Some(false), MetricConfidence::Exact);
2033 all_null_key.column_profiles[1].null_count = 11;
2034
2035 let cases = [
2036 (
2037 "exact count with repeats",
2038 key_report(8, Some(false), MetricConfidence::Exact),
2039 found.to_string(),
2040 ),
2041 (
2042 "exact count, distinct, one null",
2044 key_report(10, Some(false), MetricConfidence::Exact),
2045 "clean".to_string(),
2046 ),
2047 (
2048 "estimated distinct count",
2049 key_report(8, Some(true), MetricConfidence::Exact),
2050 skip("estimated"),
2051 ),
2052 (
2053 "key component filed as sampled",
2054 key_report(8, Some(false), key_sampled()),
2055 skip("estimated"),
2056 ),
2057 (
2058 "exactness recorded on the column only",
2059 key_report(8, Some(false), MetricConfidence::Unrecorded),
2060 found.to_string(),
2061 ),
2062 (
2063 "exactness recorded in the coverage only",
2064 key_report(8, None, MetricConfidence::Exact),
2065 found.to_string(),
2066 ),
2067 (
2068 "exactness recorded nowhere",
2069 key_report(8, None, MetricConfidence::Unrecorded),
2070 skip("unrecorded"),
2071 ),
2072 ("key chosen before 0.13", legacy_key, skip("unrecorded")),
2073 ("no key measured", no_key, skip("not_assessed")),
2074 ("key without values", all_null_key, skip("no_values")),
2075 ];
2076 let mut wrong = Vec::new();
2077 for (case, report, expected) in cases {
2078 let outcome = key_outcome(&report.findings());
2079 if outcome != expected {
2080 wrong.push(format!("{case}: {outcome}, expected {expected}"));
2081 }
2082 }
2083 assert!(wrong.is_empty(), "{}", wrong.join("\n"));
2084 }
2085
2086 #[test]
2087 fn warnings_sort_before_info_then_by_code_then_column_position() {
2088 let mut constant = column("z_constant", 5, 0);
2089 constant.unique_count = Some(1);
2090 let mut mixed = column("a_mixed", 5, 0);
2091 mixed.type_homogeneity = Some(TypeHomogeneity {
2092 numeric: 3,
2093 date: 0,
2094 boolean: 0,
2095 text: 2,
2096 });
2097 let result = report(
2098 vec![
2099 constant,
2100 column("sparse", 5, 4),
2101 mixed,
2102 column("empty", 5, 5),
2103 ],
2104 ExecutionMetadata::new(5, 4, 10).with_ragged_row_count(1),
2105 )
2106 .findings();
2107
2108 assert_eq!(
2109 codes(&result),
2110 vec![
2111 (FindingCode::AllNull, Some("empty")),
2112 (FindingCode::MixedTypes, Some("a_mixed")),
2113 (FindingCode::NullHeavy, Some("sparse")),
2114 (FindingCode::RaggedRows, None),
2115 (FindingCode::ConstantColumn, Some("z_constant")),
2116 ]
2117 );
2118 assert!(
2119 result
2120 .findings
2121 .iter()
2122 .all(|finding| finding.severity == finding.code.severity())
2123 );
2124 }
2125}