Skip to main content

dataprof_runtime/
findings.rs

1//! Structured, prioritized findings over a [`ProfileReport`] (#375).
2//!
3//! A report carries every metric, and says nothing about which of them
4//! deserves attention. Answering that by hand means inventing thresholds at
5//! every call site. [`FindingPolicy::evaluate`] (or [`ProfileReport::findings`]
6//! with the default thresholds) turns the metrics a report already holds into
7//! a short, deterministic list of [`Finding`]s.
8//!
9//! Findings are interpretation, not data cleaning: each one names what was
10//! observed and the evidence behind it, and none carries a raw cell value.
11//!
12//! # Computed, not stored
13//!
14//! Findings are derived from the report on demand and are not part of the
15//! serialized report. A report read back from its document yields the same
16//! findings as the report that wrote it, and changing a threshold does not
17//! require re-profiling or rewriting stored reports.
18//!
19//! # Absence is not a clean result
20//!
21//! A rule whose input the report does not carry, such as patterns that were
22//! never detected or a quality assessment that was not requested, produces no
23//! finding, and is listed in [`FindingsResult::not_evaluated`] with the reason.
24//! An empty `findings` list means "looked, found nothing" only for the rules
25//! that are not listed there.
26//!
27//! # Scope
28//!
29//! Findings describe the rows the report read. When the scan stopped early or
30//! sampled, a [`FindingCode::PartialScan`] finding says so; the other findings
31//! are not withheld, because what was observed is still true of those rows.
32//! Use a [`QualityPolicy`](crate::QualityPolicy) for pass/fail decisions about
33//! the whole source.
34
35use std::collections::BTreeMap;
36use std::fmt;
37
38use dataprof_core::serde_helpers::rounded_2;
39use dataprof_core::{
40    DataSource, DataType, DelimiterDetection, FileFormat, KeyColumnSource, PatternCategory,
41    TypeHomogeneity,
42};
43
44use crate::profile_report::ProfileReport;
45use crate::quality_gate::{
46    Evidence, EvidenceGap, Provenance, quality_status_name, sampled_component_evidence,
47};
48
49/// Default for [`FindingPolicy::null_heavy_percentage`]. Matches the
50/// `null-heavy` flag in the Python binding's `to_llm_context()`.
51pub const DEFAULT_NULL_HEAVY_PERCENTAGE: f64 = 20.0;
52
53/// Default for [`FindingPolicy::mixed_types_percentage`]. Matches the
54/// `mixed types` flag in the Python binding's `to_llm_context()`.
55pub const DEFAULT_MIXED_TYPES_PERCENTAGE: f64 = 5.0;
56
57/// Pattern confidence a detection needs before a finding reports it: the
58/// threshold validity scoring and the report summaries already use.
59const MIN_PATTERN_CONFIDENCE: f64 = 0.5;
60
61/// Identifier patterns that name a person. The rest of the identifier
62/// category (UUIDs, product codes, VAT numbers) identifies things.
63const PERSONAL_IDENTIFIER_PATTERNS: [&str; 2] = ["Codice Fiscale (IT)", "SSN (US)"];
64
65/// How much attention a finding asks for.
66///
67/// Declared order is priority order: warnings sort first.
68#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, serde::Serialize)]
69#[serde(rename_all = "snake_case")]
70#[non_exhaustive]
71pub enum Severity {
72    /// Likely a defect in the data or in how it was read.
73    Warning,
74    /// Worth knowing when reading the report; often intended.
75    Info,
76}
77
78impl fmt::Display for Severity {
79    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
80        match self {
81            Self::Warning => write!(f, "warning"),
82            Self::Info => write!(f, "info"),
83        }
84    }
85}
86
87/// Stable identifier for the rule a finding came from.
88///
89/// Part of the contract: a consumer matches on these, so they do not change
90/// when the prose does. Declared in alphabetical order, which is the order
91/// findings of equal severity sort in.
92#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, serde::Serialize)]
93#[serde(rename_all = "snake_case")]
94#[non_exhaustive]
95pub enum FindingCode {
96    /// Every value of a column is null.
97    AllNull,
98    /// Every non-null value of a column is the same one.
99    ConstantColumn,
100    /// Delimiter detection found a second candidate that read the sampled CSV
101    /// records as well as the one it chose.
102    DelimiterAmbiguous,
103    /// The source holds exact duplicate rows.
104    DuplicateRows,
105    /// Date values lie after the time the report was produced.
106    FutureDates,
107    /// No delimiter detection tries splits the CSV header, so the source was
108    /// read as one column, while a character outside them splits the header
109    /// and every sampled record alike.
110    HeaderNotSplit,
111    /// The declared key column holds a non-null value more than once.
112    KeyDuplicates,
113    /// At least half of a column's text values are numbers written with a
114    /// decimal comma or digit-group separators, which no numeric statistic
115    /// includes.
116    LocaleNumbers,
117    /// A column's values split across lexical types.
118    MixedTypes,
119    /// A column's null share is at or above the policy threshold.
120    NullHeavy,
121    /// The scan stopped early or sampled rows.
122    PartialScan,
123    /// Rows had a different field count from the header and were recovered.
124    RaggedRows,
125    /// Errors were counted while reading the source.
126    RecordsSkipped,
127    /// A column matches a pattern for personal or financial data.
128    SensitivePattern,
129    /// Start dates fall after their paired end dates.
130    TemporalOrderViolations,
131    /// A CSV source ended inside a quoted field.
132    UnterminatedQuote,
133}
134
135impl FindingCode {
136    /// Every code, in declared order. A new code is added here too: parsing
137    /// a code name, and so `QualityPolicy::forbid_findings` from a binding,
138    /// reads this list.
139    pub const ALL: [Self; 16] = [
140        Self::AllNull,
141        Self::ConstantColumn,
142        Self::DelimiterAmbiguous,
143        Self::DuplicateRows,
144        Self::FutureDates,
145        Self::HeaderNotSplit,
146        Self::KeyDuplicates,
147        Self::LocaleNumbers,
148        Self::MixedTypes,
149        Self::NullHeavy,
150        Self::PartialScan,
151        Self::RaggedRows,
152        Self::RecordsSkipped,
153        Self::SensitivePattern,
154        Self::TemporalOrderViolations,
155        Self::UnterminatedQuote,
156    ];
157
158    /// The severity every finding with this code carries.
159    pub fn severity(self) -> Severity {
160        match self {
161            Self::AllNull
162            | Self::DelimiterAmbiguous
163            | Self::DuplicateRows
164            | Self::FutureDates
165            | Self::HeaderNotSplit
166            | Self::KeyDuplicates
167            | Self::LocaleNumbers
168            | Self::MixedTypes
169            | Self::NullHeavy
170            | Self::RaggedRows
171            | Self::RecordsSkipped
172            | Self::TemporalOrderViolations
173            | Self::UnterminatedQuote => Severity::Warning,
174            Self::ConstantColumn | Self::PartialScan | Self::SensitivePattern => Severity::Info,
175        }
176    }
177
178    /// A fixed sentence naming what the rule observed.
179    ///
180    /// Deliberately free of numbers, as the gate's messages are: the evidence
181    /// carries those, at the precision it serializes with.
182    fn summary(self) -> &'static str {
183        match self {
184            Self::AllNull => "every value in this column is null",
185            Self::ConstantColumn => "every non-null value in this column is the same",
186            Self::DelimiterAmbiguous => {
187                "another delimiter reads the sampled CSV records as well as the one detection \
188                 chose, so the columns may be split on the wrong one; set csv_delimiter to choose"
189            }
190            Self::DuplicateRows => "the source contains exact duplicate rows",
191            Self::FutureDates => "some date values lie in the future",
192            Self::HeaderNotSplit => {
193                "no delimiter detection tries (comma, semicolon, tab, pipe) splits the CSV \
194                 header, so the source was read as one column, but another character splits the \
195                 header and every sampled record alike; set csv_delimiter to read it as a table"
196            }
197            Self::KeyDuplicates => {
198                "some non-null values of the declared key column occur more than once"
199            }
200            Self::LocaleNumbers => {
201                "at least half of the text values in this column are numbers written with a \
202                 decimal comma or digit grouping, so no numeric statistic includes them"
203            }
204            Self::MixedTypes => "this column's values are split across lexical types",
205            Self::NullHeavy => "this column's null share is at or above the threshold",
206            Self::PartialScan => {
207                "only part of the source was read; findings describe the rows that were"
208            }
209            Self::RaggedRows => {
210                "some rows had a different field count from the header and were recovered"
211            }
212            Self::RecordsSkipped => "errors were counted while reading the source",
213            Self::SensitivePattern => {
214                "values in this column match a pattern for personal or financial data"
215            }
216            Self::TemporalOrderViolations => "some start dates fall after their paired end dates",
217            Self::UnterminatedQuote => {
218                "the source ends inside a quoted field, so its last record may hold several rows"
219            }
220        }
221    }
222}
223
224impl fmt::Display for FindingCode {
225    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
226        let name = match self {
227            Self::AllNull => "all_null",
228            Self::ConstantColumn => "constant_column",
229            Self::DelimiterAmbiguous => "delimiter_ambiguous",
230            Self::DuplicateRows => "duplicate_rows",
231            Self::FutureDates => "future_dates",
232            Self::HeaderNotSplit => "header_not_split",
233            Self::KeyDuplicates => "key_duplicates",
234            Self::LocaleNumbers => "locale_numbers",
235            Self::MixedTypes => "mixed_types",
236            Self::NullHeavy => "null_heavy",
237            Self::PartialScan => "partial_scan",
238            Self::RaggedRows => "ragged_rows",
239            Self::RecordsSkipped => "records_skipped",
240            Self::SensitivePattern => "sensitive_pattern",
241            Self::TemporalOrderViolations => "temporal_order_violations",
242            Self::UnterminatedQuote => "unterminated_quote",
243        };
244        write!(f, "{name}")
245    }
246}
247
248impl std::str::FromStr for FindingCode {
249    type Err = String;
250
251    /// Parse a code from its serialized name, such as `locale_numbers`.
252    fn from_str(s: &str) -> Result<Self, Self::Err> {
253        Self::ALL
254            .into_iter()
255            .find(|code| code.to_string() == s)
256            .ok_or_else(|| {
257                let valid: Vec<String> = Self::ALL.iter().map(ToString::to_string).collect();
258                format!(
259                    "Unknown finding code: {s}. Valid codes: {}",
260                    valid.join(", ")
261                )
262            })
263    }
264}
265
266/// One value a finding read from the report, or a threshold it applied.
267///
268/// Percentages follow the report's 2dp serialization convention; counts are
269/// whole numbers and are not widened into rounded floats.
270#[derive(Debug, Clone, PartialEq, serde::Serialize)]
271#[serde(untagged)]
272#[non_exhaustive]
273pub enum EvidenceValue {
274    /// A count of rows or values.
275    Count(usize),
276    /// A 0..=100 percentage.
277    Percentage(#[serde(serialize_with = "dataprof_core::serde_helpers::round_2")] f64),
278    /// A name from the report: a lexical type, a pattern, a reason.
279    Text(String),
280    /// A yes/no property of the evidence.
281    Flag(bool),
282}
283
284/// Something in the report that deserves attention.
285#[derive(Debug, Clone, PartialEq, serde::Serialize)]
286#[non_exhaustive]
287pub struct Finding {
288    /// Which rule produced it.
289    pub code: FindingCode,
290    /// How much attention it asks for. Always `code.severity()`.
291    pub severity: Severity,
292    /// The column it concerns, absent for a finding about the whole report.
293    #[serde(skip_serializing_if = "Option::is_none")]
294    pub column: Option<String>,
295    /// The metric values and thresholds that caused it, by name. Never a raw
296    /// cell value.
297    pub evidence: BTreeMap<String, EvidenceValue>,
298    /// A fixed sentence naming what was observed.
299    pub summary: String,
300}
301
302/// Why a rule produced no finding without having looked.
303#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize)]
304#[serde(tag = "reason", rename_all = "snake_case")]
305#[non_exhaustive]
306pub enum NotEvaluatedReason {
307    /// The report carries no quality assessment, for this recorded reason.
308    #[non_exhaustive]
309    QualityUnavailable {
310        /// The report's `quality_status` state, verbatim.
311        quality_status: String,
312    },
313    /// Quality was computed, but the dimension the rule reads had nothing to
314    /// assess in this run.
315    NotAssessed,
316    /// The number the rule reads is an estimate, which witnesses nothing: an
317    /// estimated duplicate or distinct count is not evidence that a duplicate
318    /// exists.
319    Estimated,
320    /// The count the rule reads is zero, but it came from the retained quality
321    /// sample rather than every scanned row, so it rules nothing out for the
322    /// rows the sample left behind. A nonzero count is still reported.
323    Sampled,
324    /// The report does not record what the rule needs to tell a zero from an
325    /// unmeasured value: a document written before dataprof recorded it
326    /// (ragged rows before 0.10, quality sample coverage before 0.12, how the
327    /// key column was chosen before 0.13), a flat summary that omits it, or a
328    /// CSV scan that stopped before the end of its source, where the quote
329    /// state at the end was never checked.
330    Unrecorded,
331    /// The metric the rule reads was not computed for these columns: the
332    /// metric pack was not selected, or the column type does not carry it.
333    NotComputed,
334    /// These columns had no values for the rule to look at.
335    NoValues,
336}
337
338impl NotEvaluatedReason {
339    /// Position in the declared order, which is the order entries sort in.
340    fn rank(&self) -> u8 {
341        match self {
342            Self::QualityUnavailable { .. } => 0,
343            Self::NotAssessed => 1,
344            Self::Estimated => 2,
345            Self::Sampled => 3,
346            Self::Unrecorded => 4,
347            Self::NotComputed => 5,
348            Self::NoValues => 6,
349        }
350    }
351}
352
353/// A rule that could not be evaluated, and why.
354#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize)]
355#[non_exhaustive]
356pub struct UnevaluatedRule {
357    /// The rule.
358    pub code: FindingCode,
359    /// Why it was not evaluated.
360    #[serde(flatten)]
361    pub reason: NotEvaluatedReason,
362    /// For a column rule, the columns it could not look at, in report order.
363    /// Empty for a rule about the whole report.
364    #[serde(skip_serializing_if = "Vec::is_empty")]
365    pub columns: Vec<String>,
366}
367
368/// What [`FindingPolicy::evaluate`] found.
369#[derive(Debug, Clone, PartialEq, serde::Serialize)]
370#[non_exhaustive]
371pub struct FindingsResult {
372    /// Every finding, most severe first. Within a severity: by code, then
373    /// report-level before column-level, then by column position in the
374    /// report, then by pattern name.
375    pub findings: Vec<Finding>,
376    /// Every rule that could not look, by code then reason. A rule listed
377    /// here for some columns was still evaluated for the others.
378    pub not_evaluated: Vec<UnevaluatedRule>,
379}
380
381/// The facts of a report the findings rules read, and nothing else (#909).
382///
383/// [`FindingPolicy::evaluate`] builds one from a [`ProfileReport`]. A report a
384/// binding loaded from a flat summary has no `ProfileReport` behind it: the
385/// binding writes these facts from the summary as JSON and reads them with
386/// [`FindingsInput::from_json`], so both kinds of report go through one set
387/// of rules. A count the summary does not carry reads as zero, as the rules
388/// read an absent count.
389///
390/// A fact about how the source was read is `None` when the report does not
391/// record it (#1306). The rule that reads it then lists itself as not
392/// evaluated, so an unrecorded fact never reads as a clean one.
393#[derive(Debug, Clone, PartialEq, serde::Deserialize)]
394#[non_exhaustive]
395pub struct FindingsInput {
396    rows_processed: usize,
397    /// `None` only for a flat summary that omits it.
398    source_exhausted: Option<bool>,
399    /// Whether the report records a truncation reason.
400    truncated: bool,
401    /// `None` only for a flat summary that omits it.
402    sampling_applied: Option<bool>,
403    /// `None` for a document written before 0.10, which did not count ragged
404    /// rows and reads back with a zero that was never measured.
405    ragged_row_count: Option<usize>,
406    /// `None` where the quote state was not checked: a source that is not
407    /// CSV, a scan that stopped before the end of its source, or a report
408    /// written before 0.12; or where a flat summary omits it.
409    unterminated_quote: Option<bool>,
410    /// Whether the source was read as CSV, the only format with quoted
411    /// fields. `None` when the report does not record the source format, so
412    /// an unchecked quote state cannot be told apart from one that does not
413    /// apply.
414    #[serde(default)]
415    csv_source: Option<bool>,
416    error_count: usize,
417    /// What delimiter detection saw, when the report records that it ran.
418    /// A flat summary records no source metadata, so it reads as absent.
419    #[serde(default)]
420    delimiter_detection: Option<DelimiterDetection>,
421    /// A CSV file or bytes source whose report predates the CSV source
422    /// metadata (#1100), so the delimiter rules cannot tell a clear choice
423    /// from an unrecorded one.
424    #[serde(default)]
425    csv_source_unrecorded: bool,
426    /// How the report chose the column `key_uniqueness` measured. Absent for
427    /// a report written before 0.13, whose key may be a guess from column
428    /// names (#1065).
429    #[serde(default)]
430    key_column_source: Option<KeyColumnSource>,
431    /// The report's `quality_status` state, read when `quality` is absent.
432    quality_status: String,
433    quality: Option<QualityFacts>,
434    columns: Vec<ColumnFacts>,
435}
436
437#[derive(Debug, Clone, PartialEq, serde::Deserialize)]
438struct QualityFacts {
439    uniqueness: Option<UniquenessFacts>,
440    timeliness: Option<TimelinessFacts>,
441    /// The sampled component labels, `None` when the report did not record
442    /// them.
443    sampled_dimensions: Option<Vec<String>>,
444    /// Components with signal that neither recorded list names. Empty for a
445    /// flat summary written before 0.14, which records no exact list (#1027).
446    unlisted_components: Vec<String>,
447}
448
449#[derive(Debug, Clone, PartialEq, serde::Deserialize)]
450struct UniquenessFacts {
451    #[serde(default)]
452    rows_checked: usize,
453    #[serde(default)]
454    duplicate_rows: usize,
455    #[serde(default)]
456    duplicate_rows_approximate: bool,
457    /// The column `key_uniqueness` measured, absent when no key was measured.
458    #[serde(default)]
459    key_column: Option<String>,
460}
461
462#[derive(Debug, Clone, PartialEq, serde::Deserialize)]
463struct TimelinessFacts {
464    #[serde(default)]
465    date_values_checked: usize,
466    #[serde(default)]
467    future_dates_count: usize,
468    #[serde(default)]
469    temporal_pairs_checked: usize,
470    #[serde(default)]
471    temporal_violations: usize,
472}
473
474#[derive(Debug, Clone, PartialEq, serde::Deserialize)]
475struct ColumnFacts {
476    name: String,
477    /// Typed as an identifier, which `mixed_types` and `locale_numbers` exempt.
478    identifier: bool,
479    total_count: usize,
480    null_count: usize,
481    unique_count: Option<usize>,
482    /// Whether `unique_count` is an estimate; absent when not recorded.
483    #[serde(default)]
484    unique_count_is_approximate: Option<bool>,
485    type_homogeneity: Option<TypeHomogeneity>,
486    locale_number_count: Option<usize>,
487    patterns: Option<Vec<PatternFacts>>,
488}
489
490#[derive(Debug, Clone, PartialEq, serde::Deserialize)]
491struct PatternFacts {
492    name: String,
493    /// The category as the report names it; a summary from a later release
494    /// may name one this build does not know.
495    category: String,
496    confidence: f64,
497    match_percentage: f64,
498}
499
500impl FindingsInput {
501    /// Read the facts a binding wrote for a report it holds no
502    /// [`ProfileReport`] for.
503    pub fn from_json(text: &str) -> serde_json::Result<Self> {
504        serde_json::from_str(text)
505    }
506
507    /// Whether the rule for `code` read some column's retained value sample
508    /// rather than all of its values.
509    ///
510    /// `mixed_types`, `locale_numbers` and `sensitive_pattern` read a
511    /// column's lexical forms, locale numbers and patterns, which the
512    /// profiler takes from the same retained sample. A classified count short
513    /// of the column's non-null count says the sample did not hold every
514    /// value, even on a scan that read the whole source. Columns a rule
515    /// exempts (identifiers, for the first two) are not counted. Every other
516    /// code reads exact counters, or records its sampling itself.
517    pub(crate) fn reads_a_partial_sample(&self, code: FindingCode) -> bool {
518        let exempts_identifiers = match code {
519            FindingCode::MixedTypes | FindingCode::LocaleNumbers => true,
520            FindingCode::SensitivePattern => false,
521            _ => return false,
522        };
523        self.columns
524            .iter()
525            .filter(|column| !(exempts_identifiers && column.identifier))
526            .any(|column| {
527                let non_null = column.total_count.saturating_sub(column.null_count);
528                column
529                    .type_homogeneity
530                    .as_ref()
531                    .is_some_and(|forms| forms.classified_count() < non_null)
532            })
533    }
534}
535
536impl From<&ProfileReport> for FindingsInput {
537    fn from(report: &ProfileReport) -> Self {
538        let execution = &report.execution;
539        Self {
540            rows_processed: execution.rows_processed,
541            source_exhausted: Some(execution.source_exhausted),
542            truncated: execution.truncation_reason.is_some(),
543            sampling_applied: Some(execution.sampling_applied),
544            // Ragged rows were first counted by the release that introduced
545            // schema versioning (0.10, #452).
546            ragged_row_count: (report.schema_version != 0).then_some(execution.ragged_row_count),
547            unterminated_quote: execution.unterminated_quote,
548            csv_source: csv_source(&report.data_source),
549            error_count: execution.error_count,
550            delimiter_detection: report
551                .data_source
552                .csv_metadata()
553                .and_then(|metadata| metadata.delimiter_detection.clone()),
554            csv_source_unrecorded: csv_source_unrecorded(&report.data_source),
555            key_column_source: report
556                .metric_semantics
557                .as_ref()
558                .and_then(|semantics| semantics.key_column_source),
559            quality_status: quality_status_name(&report.quality_status).to_string(),
560            quality: report.quality.as_ref().map(|quality| QualityFacts {
561                uniqueness: quality
562                    .metrics
563                    .uniqueness
564                    .as_ref()
565                    .map(|uniqueness| UniquenessFacts {
566                        rows_checked: uniqueness.rows_checked,
567                        duplicate_rows: uniqueness.duplicate_rows,
568                        duplicate_rows_approximate: uniqueness.duplicate_rows_approximate,
569                        key_column: uniqueness.key_column.clone(),
570                    }),
571                timeliness: quality
572                    .metrics
573                    .timeliness
574                    .as_ref()
575                    .map(|timeliness| TimelinessFacts {
576                        date_values_checked: timeliness.date_values_checked,
577                        future_dates_count: timeliness.future_dates_count,
578                        temporal_pairs_checked: timeliness.temporal_pairs_checked,
579                        temporal_violations: timeliness.temporal_violations,
580                    }),
581                sampled_dimensions: quality.sampled_dimensions(),
582                unlisted_components: quality.unlisted_components(),
583            }),
584            columns: report
585                .column_profiles
586                .iter()
587                .map(|column| ColumnFacts {
588                    name: column.name.clone(),
589                    identifier: column.data_type == DataType::Identifier,
590                    total_count: column.total_count,
591                    null_count: column.null_count,
592                    unique_count: column.unique_count,
593                    unique_count_is_approximate: column.unique_count_is_approximate,
594                    type_homogeneity: column.type_homogeneity,
595                    locale_number_count: column.locale_number_count,
596                    patterns: column.patterns.as_ref().map(|patterns| {
597                        patterns
598                            .iter()
599                            .map(|pattern| PatternFacts {
600                                name: pattern.name.clone(),
601                                category: pattern.category.to_string(),
602                                confidence: pattern.confidence,
603                                match_percentage: pattern.match_percentage,
604                            })
605                            .collect()
606                    }),
607                })
608                .collect(),
609        }
610    }
611}
612
613/// A finding threshold that cannot be applied as written.
614#[derive(Debug, Clone, PartialEq)]
615#[non_exhaustive]
616pub enum FindingPolicyError {
617    /// A threshold fell outside `(0, 100]` at the 2dp precision it is applied
618    /// at, or was not a finite number. Percentages are on the report's 0..100
619    /// scale, not 0..1 ratios, and a zero threshold would report every column.
620    #[non_exhaustive]
621    ThresholdOutOfRange {
622        /// The setting that carried it.
623        setting: &'static str,
624        /// The value as supplied.
625        value: f64,
626    },
627}
628
629impl fmt::Display for FindingPolicyError {
630    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
631        match self {
632            Self::ThresholdOutOfRange { setting, value } => write!(
633                f,
634                "{setting} must be a percentage above 0 and at most 100 at two decimal \
635                 places, got {value}"
636            ),
637        }
638    }
639}
640
641impl std::error::Error for FindingPolicyError {}
642
643/// The thresholds findings apply.
644///
645/// Separate from the profiler's configuration on purpose: findings interpret
646/// a finished report, including one read back from disk that no profiler
647/// configuration accompanies.
648#[derive(Debug, Clone, PartialEq)]
649#[non_exhaustive]
650pub struct FindingPolicy {
651    null_heavy_percentage: f64,
652    mixed_types_percentage: f64,
653}
654
655impl Default for FindingPolicy {
656    fn default() -> Self {
657        Self {
658            null_heavy_percentage: DEFAULT_NULL_HEAVY_PERCENTAGE,
659            mixed_types_percentage: DEFAULT_MIXED_TYPES_PERCENTAGE,
660        }
661    }
662}
663
664impl FindingPolicy {
665    /// The default thresholds.
666    pub fn new() -> Self {
667        Self::default()
668    }
669
670    /// Report a column as `null_heavy` when its null percentage, at the
671    /// report's 2dp precision, is at least `percentage`. The threshold is
672    /// applied at 2dp too, so the evidence states exactly what was compared.
673    pub fn null_heavy_percentage(mut self, percentage: f64) -> Self {
674        self.null_heavy_percentage = percentage;
675        self
676    }
677
678    /// Report a column as `mixed_types` when the values outside its dominant
679    /// lexical type make up at least `percentage` of those classified, both
680    /// at 2dp.
681    pub fn mixed_types_percentage(mut self, percentage: f64) -> Self {
682        self.mixed_types_percentage = percentage;
683        self
684    }
685
686    /// Check that every threshold can be applied, without needing a report.
687    pub fn validate(&self) -> Result<(), FindingPolicyError> {
688        for (setting, value) in [
689            ("null_heavy_percentage", self.null_heavy_percentage),
690            ("mixed_types_percentage", self.mixed_types_percentage),
691        ] {
692            let applied = rounded_2(value);
693            if !(value.is_finite() && applied > 0.0 && applied <= 100.0) {
694                return Err(FindingPolicyError::ThresholdOutOfRange { setting, value });
695            }
696        }
697        Ok(())
698    }
699
700    /// Derive the findings a report supports.
701    ///
702    /// The report is not modified and nothing is printed.
703    pub fn evaluate(&self, report: &ProfileReport) -> Result<FindingsResult, FindingPolicyError> {
704        self.evaluate_input(&FindingsInput::from(report))
705    }
706
707    /// [`evaluate`](Self::evaluate) over the facts of a report, for a report
708    /// a binding holds no [`ProfileReport`] for.
709    pub fn evaluate_input(
710        &self,
711        input: &FindingsInput,
712    ) -> Result<FindingsResult, FindingPolicyError> {
713        self.validate()?;
714        Ok(self.collect(input))
715    }
716
717    /// The thresholds as applied: at the precision a percentage is compared
718    /// and reported at.
719    fn null_heavy_threshold(&self) -> f64 {
720        rounded_2(self.null_heavy_percentage)
721    }
722
723    fn mixed_types_threshold(&self) -> f64 {
724        rounded_2(self.mixed_types_percentage)
725    }
726
727    pub(crate) fn collect(&self, input: &FindingsInput) -> FindingsResult {
728        let mut collector = Collector::default();
729        scan_findings(input, &mut collector);
730        quality_findings(input, &mut collector);
731        for (index, column) in input.columns.iter().enumerate() {
732            self.column_findings(index, column, &mut collector);
733        }
734        collector.finish()
735    }
736
737    fn column_findings(&self, index: usize, column: &ColumnFacts, out: &mut Collector) {
738        let name = column.name.as_str();
739        let non_null = column.total_count.saturating_sub(column.null_count);
740
741        if column.total_count == 0 {
742            out.skip(FindingCode::AllNull, NotEvaluatedReason::NoValues, name);
743            out.skip(FindingCode::NullHeavy, NotEvaluatedReason::NoValues, name);
744        } else if non_null == 0 {
745            out.column(
746                FindingCode::AllNull,
747                index,
748                name,
749                [
750                    ("null_count", EvidenceValue::Count(column.null_count)),
751                    ("total_count", EvidenceValue::Count(column.total_count)),
752                ],
753            );
754        } else {
755            // Compared at the serialized precision, so a report read back from
756            // its document lands on the same side of the threshold.
757            let null_percentage =
758                rounded_2(column.null_count as f64 / column.total_count as f64 * 100.0);
759            if null_percentage >= self.null_heavy_threshold() {
760                out.column(
761                    FindingCode::NullHeavy,
762                    index,
763                    name,
764                    [
765                        ("null_percentage", percentage(null_percentage)),
766                        ("threshold", percentage(self.null_heavy_threshold())),
767                    ],
768                );
769            }
770        }
771
772        match column.unique_count {
773            None => out.skip(
774                FindingCode::ConstantColumn,
775                NotEvaluatedReason::NotComputed,
776                name,
777            ),
778            Some(_) if non_null == 0 => out.skip(
779                FindingCode::ConstantColumn,
780                NotEvaluatedReason::NoValues,
781                name,
782            ),
783            // One value seen once is not a constant, just a single value.
784            Some(1) if non_null > 1 => out.column(
785                FindingCode::ConstantColumn,
786                index,
787                name,
788                [
789                    ("unique_count", EvidenceValue::Count(1)),
790                    ("non_null_count", EvidenceValue::Count(non_null)),
791                ],
792            ),
793            Some(_) => {}
794        }
795
796        locale_numbers(index, column, out);
797        self.mixed_types(index, column, non_null, out);
798        sensitive_patterns(index, column, non_null, out);
799    }
800
801    fn mixed_types(
802        &self,
803        index: usize,
804        column: &ColumnFacts,
805        non_null: usize,
806        out: &mut Collector,
807    ) {
808        let name = column.name.as_str();
809        // Mixing forms ("A1", "123") is what an identifier scheme does, not a
810        // defect: the same exemption the consistency dimension makes.
811        if column.identifier {
812            return;
813        }
814        let Some(homogeneity) = column.type_homogeneity.as_ref() else {
815            out.skip(
816                FindingCode::MixedTypes,
817                NotEvaluatedReason::NotComputed,
818                name,
819            );
820            return;
821        };
822        let classified = homogeneity.classified_count();
823        let Some((dominant, dominant_count)) = homogeneity.dominant() else {
824            out.skip(FindingCode::MixedTypes, NotEvaluatedReason::NoValues, name);
825            return;
826        };
827        let outside = classified - dominant_count;
828        if outside == 0 {
829            return;
830        }
831        let outside_percentage = rounded_2(outside as f64 / classified as f64 * 100.0);
832        if outside_percentage < self.mixed_types_threshold() {
833            return;
834        }
835        out.column(
836            FindingCode::MixedTypes,
837            index,
838            name,
839            [
840                ("dominant_type", EvidenceValue::Text(dominant.to_string())),
841                (
842                    "dominant_percentage",
843                    percentage(dominant_count as f64 / classified as f64 * 100.0),
844                ),
845                ("threshold", percentage(self.mixed_types_threshold())),
846                // Shares are counted over the values the profiler retained; a
847                // classified count short of the non-null count says they were
848                // sampled.
849                ("classified_count", EvidenceValue::Count(classified)),
850                ("non_null_count", EvidenceValue::Count(non_null)),
851            ],
852        );
853    }
854}
855
856/// Report a column at least half of whose text values are numbers written in
857/// a locale format (#433).
858///
859/// Measured against the text values rather than the whole column, so it
860/// covers both ways these numbers go missing: a column of `1.234,56` typed
861/// `string` with no numeric statistics, and a numeric column whose
862/// decimal-comma values were counted as invalid. At least half, so that a free
863/// text column holding a stray `1,5` is not reported.
864fn locale_numbers(index: usize, column: &ColumnFacts, out: &mut Collector) {
865    let name = column.name.as_str();
866    // The exemption `mixed_types` makes: `1.234.567` in an identifier
867    // scheme is a code, not an amount.
868    if column.identifier {
869        return;
870    }
871    let Some(homogeneity) = column.type_homogeneity.as_ref() else {
872        out.skip(
873            FindingCode::LocaleNumbers,
874            NotEvaluatedReason::NotComputed,
875            name,
876        );
877        return;
878    };
879    // Classified, but by a release that did not count these yet.
880    let Some(locale_numbers) = column.locale_number_count else {
881        out.skip(
882            FindingCode::LocaleNumbers,
883            NotEvaluatedReason::Unrecorded,
884            name,
885        );
886        return;
887    };
888    let classified = homogeneity.classified_count();
889    if classified == 0 {
890        out.skip(
891            FindingCode::LocaleNumbers,
892            NotEvaluatedReason::NoValues,
893            name,
894        );
895        return;
896    }
897    let text = homogeneity.text;
898    if locale_numbers == 0 || locale_numbers * 2 < text {
899        return;
900    }
901    out.column(
902        FindingCode::LocaleNumbers,
903        index,
904        name,
905        [
906            ("locale_number_count", EvidenceValue::Count(locale_numbers)),
907            ("text_count", EvidenceValue::Count(text)),
908            // Counted over the values the profiler retained, as in
909            // `mixed_types`.
910            ("classified_count", EvidenceValue::Count(classified)),
911            (
912                "non_null_count",
913                EvidenceValue::Count(column.total_count.saturating_sub(column.null_count)),
914            ),
915        ],
916    );
917}
918
919impl ProfileReport {
920    /// The findings this report supports under the default thresholds.
921    ///
922    /// Shorthand for `FindingPolicy::default().evaluate(self)`, which cannot
923    /// fail because the defaults are in range.
924    pub fn findings(&self) -> FindingsResult {
925        FindingPolicy::default().collect(&FindingsInput::from(self))
926    }
927}
928
929fn scan_findings(input: &FindingsInput, out: &mut Collector) {
930    // A recorded stop or sample witnesses a partial scan even when the other
931    // fact is unrecorded; only "read everything" needs both.
932    let partial = if input.source_exhausted == Some(false) || input.truncated {
933        Some("truncated")
934    } else if input.sampling_applied == Some(true) {
935        Some("sampled")
936    } else {
937        None
938    };
939    match partial {
940        Some(reason) => out.report(
941            FindingCode::PartialScan,
942            [
943                ("reason", EvidenceValue::Text(reason.to_string())),
944                ("rows_processed", EvidenceValue::Count(input.rows_processed)),
945            ],
946        ),
947        None if input.source_exhausted.is_none() || input.sampling_applied.is_none() => {
948            out.skip_report(FindingCode::PartialScan, NotEvaluatedReason::Unrecorded);
949        }
950        None => {}
951    }
952    match input.ragged_row_count {
953        None => out.skip_report(FindingCode::RaggedRows, NotEvaluatedReason::Unrecorded),
954        Some(0) => {}
955        Some(ragged_row_count) => out.report(
956            FindingCode::RaggedRows,
957            [
958                ("ragged_row_count", EvidenceValue::Count(ragged_row_count)),
959                ("rows_processed", EvidenceValue::Count(input.rows_processed)),
960            ],
961        ),
962    }
963    match (input.unterminated_quote, input.csv_source) {
964        (Some(true), _) => out.report(
965            FindingCode::UnterminatedQuote,
966            [("rows_processed", EvidenceValue::Count(input.rows_processed))],
967        ),
968        (Some(false), _) => {}
969        // Not applicable: a source without quoted fields ends inside none.
970        // Decided from the recorded format, never from the absence.
971        (None, Some(false)) => {}
972        (None, _) => out.skip_report(
973            FindingCode::UnterminatedQuote,
974            NotEvaluatedReason::Unrecorded,
975        ),
976    }
977    if input.error_count > 0 {
978        out.report(
979            FindingCode::RecordsSkipped,
980            [("error_count", EvidenceValue::Count(input.error_count))],
981        );
982    }
983    if input.csv_source_unrecorded {
984        out.skip_report(
985            FindingCode::DelimiterAmbiguous,
986            NotEvaluatedReason::Unrecorded,
987        );
988        out.skip_report(FindingCode::HeaderNotSplit, NotEvaluatedReason::Unrecorded);
989    }
990    // A delimiter the caller set records no detection, so neither rule asks
991    // about a choice nobody made.
992    if let Some(detection) = &input.delimiter_detection {
993        if let Some(runner_up) = &detection.runner_up {
994            out.report(
995                FindingCode::DelimiterAmbiguous,
996                [
997                    (
998                        "delimiter",
999                        EvidenceValue::Text(detection.delimiter.clone()),
1000                    ),
1001                    ("runner_up", EvidenceValue::Text(runner_up.clone())),
1002                ],
1003            );
1004        }
1005        // A one-column source is often intended, so an unsplit header alone
1006        // is no finding: only a character outside the candidates that splits
1007        // the header and every sampled record alike makes the reading doubtful.
1008        if let Some(unlisted) = &detection.unlisted_delimiter {
1009            out.report(
1010                FindingCode::HeaderNotSplit,
1011                [("unlisted_delimiter", EvidenceValue::Text(unlisted.clone()))],
1012            );
1013        }
1014    }
1015}
1016
1017/// Whether `source` was read as CSV, `None` for a stream that records no
1018/// format (written before 0.14, #1042). A query or dataframe source has no
1019/// text to quote.
1020fn csv_source(source: &DataSource) -> Option<bool> {
1021    match source {
1022        DataSource::Stream { format: None, .. } => None,
1023        _ => Some(matches!(source.format(), Some(FileFormat::Csv))),
1024    }
1025}
1026
1027/// Whether `source` is a CSV source that records no CSV source metadata: a
1028/// report written before 0.14 recorded it. A stream written before 0.14
1029/// records no format either (#1042), so it cannot be told apart.
1030fn csv_source_unrecorded(source: &DataSource) -> bool {
1031    match source {
1032        DataSource::File { format, .. }
1033        | DataSource::Bytes { format, .. }
1034        | DataSource::Stream {
1035            format: Some(format),
1036            ..
1037        } => matches!(format, FileFormat::Csv) && source.csv_metadata().is_none(),
1038        _ => false,
1039    }
1040}
1041
1042fn quality_findings(input: &FindingsInput, out: &mut Collector) {
1043    const QUALITY_RULES: [FindingCode; 4] = [
1044        FindingCode::DuplicateRows,
1045        FindingCode::FutureDates,
1046        FindingCode::KeyDuplicates,
1047        FindingCode::TemporalOrderViolations,
1048    ];
1049    let Some(quality) = input.quality.as_ref() else {
1050        for code in QUALITY_RULES {
1051            out.skip_report(
1052                code,
1053                NotEvaluatedReason::QualityUnavailable {
1054                    quality_status: input.quality_status.clone(),
1055                },
1056            );
1057        }
1058        return;
1059    };
1060
1061    match quality
1062        .uniqueness
1063        .as_ref()
1064        .filter(|uniqueness| uniqueness.rows_checked > 0)
1065    {
1066        None => out.skip_report(FindingCode::DuplicateRows, NotEvaluatedReason::NotAssessed),
1067        Some(uniqueness) if uniqueness.duplicate_rows_approximate => {
1068            out.skip_report(FindingCode::DuplicateRows, NotEvaluatedReason::Estimated)
1069        }
1070        Some(uniqueness) if uniqueness.duplicate_rows > 0 => out.report(
1071            FindingCode::DuplicateRows,
1072            [
1073                (
1074                    "duplicate_rows",
1075                    EvidenceValue::Count(uniqueness.duplicate_rows),
1076                ),
1077                (
1078                    "rows_checked",
1079                    EvidenceValue::Count(uniqueness.rows_checked),
1080                ),
1081            ],
1082        ),
1083        Some(_) => skip_unless_complete(out, FindingCode::DuplicateRows, quality, "duplicate_rows"),
1084    }
1085
1086    key_duplicates(input, quality, out);
1087
1088    let timeliness = quality.timeliness.as_ref();
1089    match timeliness.filter(|timeliness| timeliness.date_values_checked > 0) {
1090        None => out.skip_report(FindingCode::FutureDates, NotEvaluatedReason::NotAssessed),
1091        Some(timeliness) if timeliness.future_dates_count > 0 => out.report(
1092            FindingCode::FutureDates,
1093            [
1094                (
1095                    "future_dates_count",
1096                    EvidenceValue::Count(timeliness.future_dates_count),
1097                ),
1098                (
1099                    "date_values_checked",
1100                    EvidenceValue::Count(timeliness.date_values_checked),
1101                ),
1102            ],
1103        ),
1104        Some(_) => skip_unless_complete(out, FindingCode::FutureDates, quality, "timeliness"),
1105    }
1106    match timeliness.filter(|timeliness| timeliness.temporal_pairs_checked > 0) {
1107        None => out.skip_report(
1108            FindingCode::TemporalOrderViolations,
1109            NotEvaluatedReason::NotAssessed,
1110        ),
1111        Some(timeliness) if timeliness.temporal_violations > 0 => out.report(
1112            FindingCode::TemporalOrderViolations,
1113            [
1114                (
1115                    "temporal_violations",
1116                    EvidenceValue::Count(timeliness.temporal_violations),
1117                ),
1118                (
1119                    "temporal_pairs_checked",
1120                    EvidenceValue::Count(timeliness.temporal_pairs_checked),
1121                ),
1122            ],
1123        ),
1124        Some(_) => skip_unless_complete(
1125            out,
1126            FindingCode::TemporalOrderViolations,
1127            quality,
1128            "timeliness",
1129        ),
1130    }
1131}
1132
1133/// Report repeated values in the declared key column (#1068).
1134///
1135/// Reads the key the uniqueness dimension measured, which is only ever a
1136/// declared one (#1065), and the key column's own distinct count over its
1137/// non-null values: a null key is completeness's finding, not a repeat. Whole
1138/// duplicate rows are `duplicate_rows`; this finds the same key on rows that
1139/// differ elsewhere, which no row comparison sees.
1140///
1141/// The rule needs an exact distinct count. An estimate witnesses no repeat,
1142/// and a report that does not record whether its count was exact, or how its
1143/// key was chosen, is listed rather than judged.
1144fn key_duplicates(input: &FindingsInput, quality: &QualityFacts, out: &mut Collector) {
1145    const CODE: FindingCode = FindingCode::KeyDuplicates;
1146    let Some(key) = quality
1147        .uniqueness
1148        .as_ref()
1149        .and_then(|uniqueness| uniqueness.key_column.as_deref())
1150    else {
1151        out.skip_report(CODE, NotEvaluatedReason::NotAssessed);
1152        return;
1153    };
1154    // Before 0.13 an undeclared key was the first id-like column name, so
1155    // the column may be a foreign key that repeats by design.
1156    let Some(source) = input.key_column_source else {
1157        out.skip(CODE, NotEvaluatedReason::Unrecorded, key);
1158        return;
1159    };
1160    let Some((index, column)) = input
1161        .columns
1162        .iter()
1163        .enumerate()
1164        .find(|(_, column)| column.name == key)
1165    else {
1166        out.skip(CODE, NotEvaluatedReason::NotComputed, key);
1167        return;
1168    };
1169    let Some(unique_count) = column.unique_count else {
1170        out.skip(CODE, NotEvaluatedReason::NotComputed, key);
1171        return;
1172    };
1173    let values = column.total_count.saturating_sub(column.null_count);
1174    if values == 0 {
1175        out.skip(CODE, NotEvaluatedReason::NoValues, key);
1176        return;
1177    }
1178    let coverage = sampled_component_evidence(
1179        quality.sampled_dimensions.as_deref(),
1180        &quality.unlisted_components,
1181        &[],
1182        false,
1183        Provenance::Component("key_uniqueness"),
1184    );
1185    // The key component is filed as sampled exactly when the key's distinct
1186    // count is an estimate.
1187    let estimated = column.unique_count_is_approximate == Some(true)
1188        || matches!(
1189            coverage,
1190            Evidence::Incomplete {
1191                reason: EvidenceGap::QualitySampled
1192            }
1193        );
1194    if estimated {
1195        out.skip(CODE, NotEvaluatedReason::Estimated, key);
1196        return;
1197    }
1198    let exact =
1199        column.unique_count_is_approximate == Some(false) || matches!(coverage, Evidence::Complete);
1200    if !exact {
1201        out.skip(CODE, NotEvaluatedReason::Unrecorded, key);
1202        return;
1203    }
1204    let repeated = values.saturating_sub(unique_count);
1205    if repeated == 0 {
1206        return;
1207    }
1208    let source = match source {
1209        KeyColumnSource::Declared => "declared",
1210    };
1211    out.column(
1212        CODE,
1213        index,
1214        key,
1215        [
1216            ("key_values_checked", EvidenceValue::Count(values)),
1217            ("repeated_values", EvidenceValue::Count(repeated)),
1218            ("key_column_source", EvidenceValue::Text(source.to_string())),
1219        ],
1220    );
1221}
1222
1223/// A zero count is a clean result only when it covers every scanned row.
1224///
1225/// A count from the retained quality sample, or from a report that does not
1226/// record where its counts came from, rules nothing out for the rows it did
1227/// not see, so the rule is listed rather than read as clean. A nonzero count
1228/// never reaches here: rows already witnessed stay witnessed. Coverage is
1229/// resolved by the quality gate's rule, so a finding and a check never
1230/// disagree about one component.
1231fn skip_unless_complete(
1232    out: &mut Collector,
1233    code: FindingCode,
1234    quality: &QualityFacts,
1235    component: &'static str,
1236) {
1237    // A component's coverage does not depend on the dimensions the overall
1238    // score weighs, so none are passed.
1239    let coverage = sampled_component_evidence(
1240        quality.sampled_dimensions.as_deref(),
1241        &quality.unlisted_components,
1242        &[],
1243        false,
1244        Provenance::Component(component),
1245    );
1246    match coverage {
1247        Evidence::Complete => {}
1248        Evidence::Incomplete {
1249            reason: EvidenceGap::QualitySampled,
1250        } => out.skip_report(code, NotEvaluatedReason::Sampled),
1251        Evidence::Incomplete { .. } => out.skip_report(code, NotEvaluatedReason::Unrecorded),
1252    }
1253}
1254
1255/// Whether a detected pattern names personal or financial data.
1256fn is_sensitive(category: &str, name: &str) -> bool {
1257    [PatternCategory::Contact, PatternCategory::Financial]
1258        .iter()
1259        .any(|sensitive| sensitive.to_string() == category)
1260        || PERSONAL_IDENTIFIER_PATTERNS.contains(&name)
1261}
1262
1263fn sensitive_patterns(index: usize, column: &ColumnFacts, non_null: usize, out: &mut Collector) {
1264    let Some(patterns) = column.patterns.as_ref() else {
1265        out.skip(
1266            FindingCode::SensitivePattern,
1267            NotEvaluatedReason::NotComputed,
1268            &column.name,
1269        );
1270        return;
1271    };
1272    if non_null == 0 {
1273        // Detection ran over nothing, which is no evidence either way.
1274        out.skip(
1275            FindingCode::SensitivePattern,
1276            NotEvaluatedReason::NoValues,
1277            &column.name,
1278        );
1279        return;
1280    }
1281    for pattern in patterns {
1282        if pattern.confidence < MIN_PATTERN_CONFIDENCE
1283            || !is_sensitive(&pattern.category, &pattern.name)
1284        {
1285            continue;
1286        }
1287        out.push(
1288            Finding {
1289                code: FindingCode::SensitivePattern,
1290                severity: FindingCode::SensitivePattern.severity(),
1291                column: Some(column.name.clone()),
1292                evidence: evidence([
1293                    ("pattern", EvidenceValue::Text(pattern.name.clone())),
1294                    ("category", EvidenceValue::Text(pattern.category.clone())),
1295                    ("match_percentage", percentage(pattern.match_percentage)),
1296                ]),
1297                summary: FindingCode::SensitivePattern.summary().to_string(),
1298            },
1299            Some(index),
1300            pattern.name.clone(),
1301        );
1302    }
1303}
1304
1305/// Percentage evidence at the precision it serializes with, so a finding
1306/// compares equal to one rebuilt from its own document.
1307fn percentage(value: f64) -> EvidenceValue {
1308    EvidenceValue::Percentage(rounded_2(value))
1309}
1310
1311fn evidence<const N: usize>(
1312    entries: [(&'static str, EvidenceValue); N],
1313) -> BTreeMap<String, EvidenceValue> {
1314    entries
1315        .into_iter()
1316        .map(|(name, value)| (name.to_string(), value))
1317        .collect()
1318}
1319
1320/// A finding with the keys it sorts by.
1321struct Ranked {
1322    finding: Finding,
1323    /// `None` for a report-level finding, which sorts before column findings.
1324    column_index: Option<usize>,
1325    /// Breaks ties between findings of one code on one column.
1326    detail: String,
1327}
1328
1329#[derive(Default)]
1330struct Collector {
1331    findings: Vec<Ranked>,
1332    not_evaluated: Vec<UnevaluatedRule>,
1333}
1334
1335impl Collector {
1336    fn push(&mut self, finding: Finding, column_index: Option<usize>, detail: String) {
1337        self.findings.push(Ranked {
1338            finding,
1339            column_index,
1340            detail,
1341        });
1342    }
1343
1344    fn report<const N: usize>(
1345        &mut self,
1346        code: FindingCode,
1347        entries: [(&'static str, EvidenceValue); N],
1348    ) {
1349        self.push(
1350            Finding {
1351                code,
1352                severity: code.severity(),
1353                column: None,
1354                evidence: evidence(entries),
1355                summary: code.summary().to_string(),
1356            },
1357            None,
1358            String::new(),
1359        );
1360    }
1361
1362    fn column<const N: usize>(
1363        &mut self,
1364        code: FindingCode,
1365        index: usize,
1366        name: &str,
1367        entries: [(&'static str, EvidenceValue); N],
1368    ) {
1369        self.push(
1370            Finding {
1371                code,
1372                severity: code.severity(),
1373                column: Some(name.to_string()),
1374                evidence: evidence(entries),
1375                summary: code.summary().to_string(),
1376            },
1377            Some(index),
1378            String::new(),
1379        );
1380    }
1381
1382    fn skip_report(&mut self, code: FindingCode, reason: NotEvaluatedReason) {
1383        self.not_evaluated.push(UnevaluatedRule {
1384            code,
1385            reason,
1386            columns: Vec::new(),
1387        });
1388    }
1389
1390    /// Record a column a rule could not look at, joining the entry for the
1391    /// same rule and reason so each pair is listed once.
1392    fn skip(&mut self, code: FindingCode, reason: NotEvaluatedReason, column: &str) {
1393        match self
1394            .not_evaluated
1395            .iter_mut()
1396            .find(|entry| entry.code == code && entry.reason == reason)
1397        {
1398            Some(entry) => entry.columns.push(column.to_string()),
1399            None => self.not_evaluated.push(UnevaluatedRule {
1400                code,
1401                reason,
1402                columns: vec![column.to_string()],
1403            }),
1404        }
1405    }
1406
1407    fn finish(mut self) -> FindingsResult {
1408        self.findings.sort_by(|left, right| {
1409            (left.finding.severity, left.finding.code)
1410                .cmp(&(right.finding.severity, right.finding.code))
1411                .then_with(|| left.column_index.cmp(&right.column_index))
1412                .then_with(|| left.detail.cmp(&right.detail))
1413        });
1414        // Stable, so the columns inside one entry keep report order.
1415        self.not_evaluated.sort_by(|left, right| {
1416            (left.code, left.reason.rank()).cmp(&(right.code, right.reason.rank()))
1417        });
1418        FindingsResult {
1419            findings: self
1420                .findings
1421                .into_iter()
1422                .map(|ranked| ranked.finding)
1423                .collect(),
1424            not_evaluated: self.not_evaluated,
1425        }
1426    }
1427}
1428
1429#[cfg(test)]
1430mod tests {
1431    use dataprof_core::{
1432        ColumnProfile, ColumnStats, CsvSourceMetadata, DataSource, ExecutionMetadata, FileFormat,
1433        Pattern, TruncationReason, TypeHomogeneity,
1434    };
1435    use dataprof_metrics::{
1436        MetricConfidence, QualityAssessment, QualityMetrics, TimelinessMetrics, UniquenessMetrics,
1437    };
1438
1439    use super::*;
1440
1441    fn source() -> DataSource {
1442        DataSource::File {
1443            path: "drop.csv".to_string(),
1444            format: FileFormat::Csv,
1445            size_bytes: 256,
1446            modified_at: None,
1447            parquet_metadata: None,
1448            csv_metadata: None,
1449        }
1450    }
1451
1452    fn column(name: &str, total: usize, nulls: usize) -> ColumnProfile {
1453        ColumnProfile {
1454            name: name.to_string(),
1455            data_type: DataType::String,
1456            null_count: nulls,
1457            total_count: total,
1458            unique_count: None,
1459            unique_count_is_approximate: None,
1460            unique_count_lower_bound: None,
1461            invalid_count: None,
1462            type_homogeneity: None,
1463            locale_number_count: None,
1464            native_type: None,
1465            stats: ColumnStats::None,
1466            patterns: None,
1467        }
1468    }
1469
1470    fn pattern(name: &str, category: PatternCategory, confidence: f64) -> Pattern {
1471        Pattern {
1472            name: name.to_string(),
1473            regex: String::new(),
1474            match_count: 10,
1475            match_percentage: 100.0,
1476            category,
1477            confidence,
1478        }
1479    }
1480
1481    fn report(columns: Vec<ColumnProfile>, execution: ExecutionMetadata) -> ProfileReport {
1482        ProfileReport::new(source(), columns, execution, None)
1483    }
1484
1485    fn codes(result: &FindingsResult) -> Vec<(FindingCode, Option<&str>)> {
1486        result
1487            .findings
1488            .iter()
1489            .map(|finding| (finding.code, finding.column.as_deref()))
1490            .collect()
1491    }
1492
1493    #[test]
1494    fn an_estimated_duplicate_count_witnesses_nothing() {
1495        let metrics = QualityMetrics {
1496            uniqueness: Some(UniquenessMetrics {
1497                duplicate_rows: 1_200,
1498                key_uniqueness: 100.0,
1499                high_cardinality_warning: false,
1500                rows_checked: 5_000_000,
1501                key_column: None,
1502                key_values_checked: None,
1503                duplicate_rows_approximate: true,
1504                duplicate_rows_exact_limit: None,
1505            }),
1506            ..QualityMetrics::default()
1507        };
1508        let report = ProfileReport::new(
1509            source(),
1510            vec![],
1511            ExecutionMetadata::new(5_000_000, 0, 10),
1512            Some(QualityAssessment::exact(metrics)),
1513        );
1514
1515        let result = report.findings();
1516
1517        assert!(
1518            !codes(&result).contains(&(FindingCode::DuplicateRows, None)),
1519            "an HLL estimate reported as a duplicate: {:?}",
1520            result.findings
1521        );
1522        assert!(result.not_evaluated.contains(&UnevaluatedRule {
1523            code: FindingCode::DuplicateRows,
1524            reason: NotEvaluatedReason::Estimated,
1525            columns: vec![],
1526        }));
1527    }
1528
1529    #[test]
1530    fn the_threshold_is_compared_at_the_serialized_precision() {
1531        // 4,999 of 25,000 is 19.996%, which the report serializes as 20.0. A
1532        // full-precision comparison would miss it here and report it once the
1533        // report was read back from its document.
1534        let report = report(
1535            vec![column("near", 25_000, 4_999)],
1536            ExecutionMetadata::new(25_000, 1, 10),
1537        );
1538
1539        let finding = report
1540            .findings()
1541            .findings
1542            .into_iter()
1543            .find(|finding| finding.code == FindingCode::NullHeavy)
1544            .expect("reported at the rounded value");
1545        assert_eq!(
1546            finding.evidence["null_percentage"],
1547            EvidenceValue::Percentage(20.0)
1548        );
1549    }
1550
1551    #[test]
1552    fn only_confident_personal_or_financial_patterns_are_sensitive() {
1553        let mut profile = column("mixed_ids", 10, 0);
1554        profile.patterns = Some(vec![
1555            pattern("UUID", PatternCategory::Identifier, 0.9),
1556            pattern("SSN (US)", PatternCategory::Identifier, 0.9),
1557            pattern("IBAN", PatternCategory::Financial, 0.4),
1558            pattern("Email", PatternCategory::Contact, 0.8),
1559            pattern("IPv4", PatternCategory::Network, 0.9),
1560        ]);
1561        let result = report(vec![profile], ExecutionMetadata::new(10, 1, 10)).findings();
1562
1563        let reported: Vec<_> = result
1564            .findings
1565            .iter()
1566            .filter(|finding| finding.code == FindingCode::SensitivePattern)
1567            .map(|finding| finding.evidence["pattern"].clone())
1568            .collect();
1569        // Ordered by pattern name within the column, not by detection order.
1570        assert_eq!(
1571            reported,
1572            vec![
1573                EvidenceValue::Text("Email".to_string()),
1574                EvidenceValue::Text("SSN (US)".to_string()),
1575            ]
1576        );
1577    }
1578
1579    #[test]
1580    fn evidence_holds_the_value_it_serializes() {
1581        // A percentage kept at full precision in memory and rounded only on
1582        // the way out would make two findings that serialize identically
1583        // compare unequal, depending on who wrote the document they came from.
1584        let mut profile = column("email", 3, 0);
1585        profile.type_homogeneity = Some(TypeHomogeneity {
1586            numeric: 1,
1587            date: 0,
1588            boolean: 0,
1589            text: 2,
1590        });
1591        let mut email = pattern("Email", PatternCategory::Contact, 0.8);
1592        email.match_percentage = 200.0 / 3.0;
1593        profile.patterns = Some(vec![email]);
1594        let result = report(vec![profile], ExecutionMetadata::new(3, 1, 10)).findings();
1595
1596        let serialized: FindingsResult = {
1597            let value = serde_json::to_value(&result).expect("serializes");
1598            let mut rebuilt = result.clone();
1599            for (finding, document) in rebuilt
1600                .findings
1601                .iter_mut()
1602                .zip(value["findings"].as_array().unwrap())
1603            {
1604                for (name, evidence) in finding.evidence.iter_mut() {
1605                    if let EvidenceValue::Percentage(_) = evidence {
1606                        *evidence =
1607                            EvidenceValue::Percentage(document["evidence"][name].as_f64().unwrap());
1608                    }
1609                }
1610            }
1611            rebuilt
1612        };
1613        assert_eq!(result, serialized);
1614    }
1615
1616    fn quality_report(future_dates: usize, confidence: MetricConfidence) -> ProfileReport {
1617        let metrics = QualityMetrics {
1618            uniqueness: Some(UniquenessMetrics {
1619                duplicate_rows: 0,
1620                key_uniqueness: 100.0,
1621                high_cardinality_warning: false,
1622                rows_checked: 60_000,
1623                key_column: None,
1624                key_values_checked: None,
1625                duplicate_rows_approximate: false,
1626                duplicate_rows_exact_limit: None,
1627            }),
1628            timeliness: Some(TimelinessMetrics {
1629                future_dates_count: future_dates,
1630                stale_data_ratio: 0.0,
1631                temporal_violations: 0,
1632                invalid_date_values: 0,
1633                date_values_checked: 10_000,
1634                temporal_pairs_checked: 10_000,
1635                clock: None,
1636            }),
1637            ..QualityMetrics::default()
1638        };
1639        ProfileReport::new(
1640            source(),
1641            vec![],
1642            ExecutionMetadata::new(60_000, 0, 10),
1643            Some(QualityAssessment::new(metrics, confidence)),
1644        )
1645    }
1646
1647    fn skipped(result: &FindingsResult, code: FindingCode) -> Option<&NotEvaluatedReason> {
1648        result
1649            .not_evaluated
1650            .iter()
1651            .find(|entry| entry.code == code)
1652            .map(|entry| &entry.reason)
1653    }
1654
1655    #[test]
1656    fn a_zero_count_from_the_quality_sample_rules_nothing_out() {
1657        // The ordinary large-file shape: every row scanned, timeliness computed
1658        // over the retained reservoir, duplicates over the full stream.
1659        let sampled = || MetricConfidence::Mixed {
1660            exact_dimensions: vec!["duplicate_rows".to_string()],
1661            sampled_dimensions: vec!["timeliness".to_string()],
1662            sample_size: 10_000,
1663        };
1664
1665        let clean = quality_report(0, sampled()).findings();
1666        assert_eq!(
1667            skipped(&clean, FindingCode::FutureDates),
1668            Some(&NotEvaluatedReason::Sampled)
1669        );
1670        assert_eq!(
1671            skipped(&clean, FindingCode::TemporalOrderViolations),
1672            Some(&NotEvaluatedReason::Sampled)
1673        );
1674        // Counted over every row, so its zero is a clean result.
1675        assert_eq!(skipped(&clean, FindingCode::DuplicateRows), None);
1676
1677        // A witnessed future date stays witnessed, sample or not.
1678        let witnessed = quality_report(3, sampled()).findings();
1679        assert!(codes(&witnessed).contains(&(FindingCode::FutureDates, None)));
1680        assert_eq!(skipped(&witnessed, FindingCode::FutureDates), None);
1681    }
1682
1683    #[test]
1684    fn unrecorded_coverage_does_not_read_as_clean() {
1685        let result = quality_report(0, MetricConfidence::Unrecorded).findings();
1686        for code in [
1687            FindingCode::DuplicateRows,
1688            FindingCode::FutureDates,
1689            FindingCode::TemporalOrderViolations,
1690        ] {
1691            assert_eq!(
1692                skipped(&result, code),
1693                Some(&NotEvaluatedReason::Unrecorded),
1694                "{code}"
1695            );
1696        }
1697    }
1698
1699    #[test]
1700    fn a_report_from_before_ragged_counting_does_not_read_as_clean() {
1701        let mut legacy = report(vec![], ExecutionMetadata::new(5, 0, 10));
1702        legacy.schema_version = 0;
1703        assert_eq!(
1704            skipped(&legacy.findings(), FindingCode::RaggedRows),
1705            Some(&NotEvaluatedReason::Unrecorded)
1706        );
1707
1708        let current = report(vec![], ExecutionMetadata::new(5, 0, 10)).findings();
1709        assert_eq!(skipped(&current, FindingCode::RaggedRows), None);
1710    }
1711
1712    /// A CSV report from before the CSV source metadata cannot say whether
1713    /// detection was clear, so both delimiter rules are listed as unrecorded
1714    /// rather than passed. A recorded clear detection, a configured delimiter
1715    /// and a non-CSV source list neither (#1100).
1716    #[test]
1717    fn a_csv_report_without_source_metadata_does_not_read_as_a_clear_delimiter() {
1718        let rules = [FindingCode::DelimiterAmbiguous, FindingCode::HeaderNotSplit];
1719        let with_source = |data_source: DataSource| {
1720            ProfileReport::new(data_source, vec![], ExecutionMetadata::new(5, 0, 10), None)
1721                .findings()
1722        };
1723        let mut wrong = Vec::new();
1724
1725        let legacy = with_source(source());
1726        for code in rules {
1727            if skipped(&legacy, code) != Some(&NotEvaluatedReason::Unrecorded) {
1728                wrong.push(format!("legacy csv: {code} not listed as unrecorded"));
1729            }
1730        }
1731
1732        let json = DataSource::File {
1733            path: "drop.json".to_string(),
1734            format: FileFormat::Json,
1735            size_bytes: 256,
1736            modified_at: None,
1737            parquet_metadata: None,
1738            csv_metadata: None,
1739        };
1740        let recorded = [
1741            (
1742                "clear detection",
1743                source().with_csv_metadata(CsvSourceMetadata::detected(DelimiterDetection::new(
1744                    b',', 3, None,
1745                ))),
1746            ),
1747            (
1748                "configured",
1749                source().with_csv_metadata(CsvSourceMetadata::configured(b';')),
1750            ),
1751            ("json", json),
1752        ];
1753        for (case, data_source) in recorded {
1754            let result = with_source(data_source);
1755            for code in rules {
1756                if skipped(&result, code).is_some()
1757                    || result.findings.iter().any(|finding| finding.code == code)
1758                {
1759                    wrong.push(format!("{case}: {code} skipped or reported"));
1760                }
1761            }
1762        }
1763        assert!(wrong.is_empty(), "{}", wrong.join("\n"));
1764    }
1765
1766    /// A classified column of `numeric` numbers and `text` text values,
1767    /// `locale` of them locale-formatted numbers.
1768    fn classified(name: &str, numeric: usize, text: usize, locale: Option<usize>) -> ColumnProfile {
1769        let mut profile = column(name, numeric + text, 0);
1770        profile.type_homogeneity = Some(TypeHomogeneity {
1771            numeric,
1772            date: 0,
1773            boolean: 0,
1774            text,
1775        });
1776        profile.locale_number_count = locale;
1777        profile
1778    }
1779
1780    fn locale_number_columns(result: &FindingsResult) -> Vec<&str> {
1781        result
1782            .findings
1783            .iter()
1784            .filter(|finding| finding.code == FindingCode::LocaleNumbers)
1785            .filter_map(|finding| finding.column.as_deref())
1786            .collect()
1787    }
1788
1789    #[test]
1790    fn locale_numbers_are_reported_from_half_of_the_text_values() {
1791        let result = report(
1792            vec![
1793                classified("half", 0, 4, Some(2)),
1794                classified("under_half", 0, 5, Some(2)),
1795                // A float column whose decimal-comma values were invalid.
1796                classified("float_with_commas", 9, 1, Some(1)),
1797                classified("clean", 0, 4, Some(0)),
1798            ],
1799            ExecutionMetadata::new(10, 0, 10),
1800        )
1801        .findings();
1802
1803        assert_eq!(
1804            locale_number_columns(&result),
1805            ["half", "float_with_commas"]
1806        );
1807        let half = &result.findings[0];
1808        assert_eq!(half.code, FindingCode::LocaleNumbers);
1809        assert_eq!(
1810            half.evidence,
1811            BTreeMap::from([
1812                ("classified_count".to_string(), EvidenceValue::Count(4)),
1813                ("locale_number_count".to_string(), EvidenceValue::Count(2)),
1814                ("non_null_count".to_string(), EvidenceValue::Count(4)),
1815                ("text_count".to_string(), EvidenceValue::Count(4)),
1816            ])
1817        );
1818        assert_eq!(skipped(&result, FindingCode::LocaleNumbers), None);
1819    }
1820
1821    #[test]
1822    fn an_identifier_column_is_exempt_from_locale_numbers() {
1823        let mut codes = classified("codes", 0, 4, Some(4));
1824        codes.data_type = DataType::Identifier;
1825
1826        let result = report(vec![codes], ExecutionMetadata::new(4, 0, 10)).findings();
1827
1828        assert!(locale_number_columns(&result).is_empty());
1829        assert_eq!(skipped(&result, FindingCode::LocaleNumbers), None);
1830    }
1831
1832    #[test]
1833    fn a_report_from_before_locale_number_counting_does_not_read_as_clean() {
1834        // Classified by a release that did not count locale numbers yet.
1835        let result = report(
1836            vec![classified("legacy", 0, 4, None)],
1837            ExecutionMetadata::new(4, 0, 10),
1838        )
1839        .findings();
1840        assert_eq!(
1841            skipped(&result, FindingCode::LocaleNumbers),
1842            Some(&NotEvaluatedReason::Unrecorded)
1843        );
1844
1845        // Not classified at all: nothing was computed to count from.
1846        let result = report(
1847            vec![column("unclassified", 4, 0)],
1848            ExecutionMetadata::new(4, 0, 10),
1849        )
1850        .findings();
1851        assert_eq!(
1852            skipped(&result, FindingCode::LocaleNumbers),
1853            Some(&NotEvaluatedReason::NotComputed)
1854        );
1855    }
1856
1857    #[test]
1858    fn thresholds_apply_at_the_precision_they_are_reported_at() {
1859        // 20.004 is applied as 20.00, so a 20.00% column is reported and the
1860        // evidence names the threshold that was compared.
1861        let result = FindingPolicy::new()
1862            .null_heavy_percentage(20.004)
1863            .evaluate(&report(
1864                vec![column("fifth", 5, 1)],
1865                ExecutionMetadata::new(5, 1, 10),
1866            ))
1867            .expect("in range");
1868        let finding = result
1869            .findings
1870            .iter()
1871            .find(|finding| finding.code == FindingCode::NullHeavy)
1872            .expect("reported at the applied threshold");
1873        assert_eq!(
1874            finding.evidence["threshold"],
1875            EvidenceValue::Percentage(20.0)
1876        );
1877
1878        // A threshold that rounds to zero would report every column.
1879        for value in [1e-9, 0.004] {
1880            assert!(
1881                FindingPolicy::new()
1882                    .null_heavy_percentage(value)
1883                    .validate()
1884                    .is_err(),
1885                "{value} was accepted"
1886            );
1887        }
1888        assert!(
1889            FindingPolicy::new()
1890                .null_heavy_percentage(0.005)
1891                .validate()
1892                .is_ok()
1893        );
1894    }
1895
1896    #[test]
1897    fn a_nested_column_is_assessed_by_its_counts_only() {
1898        // Containers carry counts and nothing else (#637); the value-level
1899        // rules say they did not look rather than reporting a clean column.
1900        let mut nested = column("payload", 4, 4);
1901        nested.data_type = DataType::Nested;
1902        let result = report(vec![nested], ExecutionMetadata::new(4, 1, 10)).findings();
1903
1904        assert_eq!(
1905            codes(&result),
1906            vec![(FindingCode::AllNull, Some("payload"))]
1907        );
1908        for code in [
1909            FindingCode::ConstantColumn,
1910            FindingCode::MixedTypes,
1911            FindingCode::SensitivePattern,
1912        ] {
1913            assert!(
1914                result.not_evaluated.contains(&UnevaluatedRule {
1915                    code,
1916                    reason: NotEvaluatedReason::NotComputed,
1917                    columns: vec!["payload".to_string()],
1918                }),
1919                "{code} was not listed: {:?}",
1920                result.not_evaluated
1921            );
1922        }
1923    }
1924
1925    #[test]
1926    fn a_sampled_scan_names_its_reason_and_truncation_wins() {
1927        let sampled = report(
1928            vec![],
1929            ExecutionMetadata::new(100, 0, 10).with_sampling(0.1),
1930        );
1931        let both = report(
1932            vec![],
1933            ExecutionMetadata::new(100, 0, 10)
1934                .with_sampling(0.1)
1935                .with_truncation(TruncationReason::MaxRows(100)),
1936        );
1937
1938        for (report, expected) in [(sampled, "sampled"), (both, "truncated")] {
1939            let finding = report
1940                .findings()
1941                .findings
1942                .into_iter()
1943                .find(|finding| finding.code == FindingCode::PartialScan)
1944                .expect("a partial scan is disclosed");
1945            assert_eq!(
1946                finding.evidence["reason"],
1947                EvidenceValue::Text(expected.to_string())
1948            );
1949        }
1950    }
1951
1952    /// A report whose declared key `order_id` holds ten non-null values and
1953    /// one null, `unique` of them distinct.
1954    fn key_report(
1955        unique: usize,
1956        approximate: Option<bool>,
1957        confidence: MetricConfidence,
1958    ) -> ProfileReport {
1959        let values = 10;
1960        let mut key = column("order_id", values + 1, 1);
1961        key.data_type = DataType::Identifier;
1962        key.unique_count = Some(unique);
1963        key.unique_count_is_approximate = approximate;
1964        let metrics = QualityMetrics {
1965            uniqueness: Some(UniquenessMetrics {
1966                duplicate_rows: 0,
1967                key_uniqueness: unique as f64 / values as f64 * 100.0,
1968                high_cardinality_warning: false,
1969                rows_checked: values + 1,
1970                key_column: Some("order_id".to_string()),
1971                key_values_checked: Some(values),
1972                duplicate_rows_approximate: false,
1973                duplicate_rows_exact_limit: None,
1974            }),
1975            ..QualityMetrics::default()
1976        };
1977        ProfileReport::new(
1978            source(),
1979            vec![column("amount", values + 1, 0), key],
1980            ExecutionMetadata::new(values + 1, 0, 10),
1981            Some(QualityAssessment::new(metrics, confidence)),
1982        )
1983    }
1984
1985    fn key_sampled() -> MetricConfidence {
1986        MetricConfidence::Mixed {
1987            exact_dimensions: vec!["duplicate_rows".to_string()],
1988            sampled_dimensions: vec!["key_uniqueness".to_string()],
1989            sample_size: 10,
1990        }
1991    }
1992
1993    /// What the key rule did: its finding's evidence, or its skip reason.
1994    fn key_outcome(result: &FindingsResult) -> String {
1995        let found: Vec<_> = result
1996            .findings
1997            .iter()
1998            .filter(|finding| finding.code == FindingCode::KeyDuplicates)
1999            .collect();
2000        match (
2001            found.as_slice(),
2002            skipped(result, FindingCode::KeyDuplicates),
2003        ) {
2004            ([finding], None) => format!(
2005                "found on {:?}: {}",
2006                finding.column,
2007                serde_json::to_string(&finding.evidence).unwrap()
2008            ),
2009            ([], Some(reason)) => format!("skipped: {}", serde_json::to_string(reason).unwrap()),
2010            ([], None) => "clean".to_string(),
2011            _ => format!("inconsistent: {found:?}"),
2012        }
2013    }
2014
2015    #[test]
2016    fn a_repeated_declared_key_is_reported_only_from_an_exact_count() {
2017        let found = r#"found on Some("order_id"): {"key_column_source":"declared","key_values_checked":10,"repeated_values":2}"#;
2018        let skip = |reason: &str| format!(r#"skipped: {{"reason":"{reason}"}}"#);
2019        let mut legacy_key = key_report(8, Some(false), MetricConfidence::Exact);
2020        legacy_key
2021            .metric_semantics
2022            .as_mut()
2023            .expect("a current report records its semantics")
2024            .key_column_source = None;
2025        let mut no_key = key_report(8, Some(false), MetricConfidence::Exact);
2026        no_key
2027            .quality
2028            .as_mut()
2029            .and_then(|quality| quality.metrics.uniqueness.as_mut())
2030            .expect("uniqueness assessed")
2031            .key_column = None;
2032        let mut all_null_key = key_report(0, Some(false), MetricConfidence::Exact);
2033        all_null_key.column_profiles[1].null_count = 11;
2034
2035        let cases = [
2036            (
2037                "exact count with repeats",
2038                key_report(8, Some(false), MetricConfidence::Exact),
2039                found.to_string(),
2040            ),
2041            (
2042                // One null and ten distinct values: a null is not a repeat.
2043                "exact count, distinct, one null",
2044                key_report(10, Some(false), MetricConfidence::Exact),
2045                "clean".to_string(),
2046            ),
2047            (
2048                "estimated distinct count",
2049                key_report(8, Some(true), MetricConfidence::Exact),
2050                skip("estimated"),
2051            ),
2052            (
2053                "key component filed as sampled",
2054                key_report(8, Some(false), key_sampled()),
2055                skip("estimated"),
2056            ),
2057            (
2058                "exactness recorded on the column only",
2059                key_report(8, Some(false), MetricConfidence::Unrecorded),
2060                found.to_string(),
2061            ),
2062            (
2063                "exactness recorded in the coverage only",
2064                key_report(8, None, MetricConfidence::Exact),
2065                found.to_string(),
2066            ),
2067            (
2068                "exactness recorded nowhere",
2069                key_report(8, None, MetricConfidence::Unrecorded),
2070                skip("unrecorded"),
2071            ),
2072            ("key chosen before 0.13", legacy_key, skip("unrecorded")),
2073            ("no key measured", no_key, skip("not_assessed")),
2074            ("key without values", all_null_key, skip("no_values")),
2075        ];
2076        let mut wrong = Vec::new();
2077        for (case, report, expected) in cases {
2078            let outcome = key_outcome(&report.findings());
2079            if outcome != expected {
2080                wrong.push(format!("{case}: {outcome}, expected {expected}"));
2081            }
2082        }
2083        assert!(wrong.is_empty(), "{}", wrong.join("\n"));
2084    }
2085
2086    #[test]
2087    fn warnings_sort_before_info_then_by_code_then_column_position() {
2088        let mut constant = column("z_constant", 5, 0);
2089        constant.unique_count = Some(1);
2090        let mut mixed = column("a_mixed", 5, 0);
2091        mixed.type_homogeneity = Some(TypeHomogeneity {
2092            numeric: 3,
2093            date: 0,
2094            boolean: 0,
2095            text: 2,
2096        });
2097        let result = report(
2098            vec![
2099                constant,
2100                column("sparse", 5, 4),
2101                mixed,
2102                column("empty", 5, 5),
2103            ],
2104            ExecutionMetadata::new(5, 4, 10).with_ragged_row_count(1),
2105        )
2106        .findings();
2107
2108        assert_eq!(
2109            codes(&result),
2110            vec![
2111                (FindingCode::AllNull, Some("empty")),
2112                (FindingCode::MixedTypes, Some("a_mixed")),
2113                (FindingCode::NullHeavy, Some("sparse")),
2114                (FindingCode::RaggedRows, None),
2115                (FindingCode::ConstantColumn, Some("z_constant")),
2116            ]
2117        );
2118        assert!(
2119            result
2120                .findings
2121                .iter()
2122                .all(|finding| finding.severity == finding.code.severity())
2123        );
2124    }
2125}