Skip to main content

dataprof_runtime/
report_delta.rs

1//! What changed between two reports.
2//!
3//! A pipeline that keeps a baseline report asks the same questions of every
4//! new one: did quality move, did a column appear or vanish, did a column's
5//! null rate change. [`ReportDelta`] answers them in one typed value, so the
6//! answer does not depend on which language computed it, and so a policy over
7//! drift can be written against a contract rather than a dictionary.
8//!
9//! # The contract
10//!
11//! The two reports are `a` and `b`, compared in that order. The serialized
12//! shape is the one `ProfileReport.compare()` returns in Python, and it is
13//! stable: later releases may add keys, never remove or rename one.
14//!
15//! - **Signs.** Every change is `b - a`. A positive `abs` on a score is an
16//!   improvement, because a score is higher when quality is better. A positive
17//!   `null_pct_delta` is a degradation, because it is more nulls.
18//! - **Precision.** Each side's number is taken at its serialized precision
19//!   (2 decimals on a 0-100 scale), and the change is rounded to it too, so a
20//!   report and its saved document compare alike.
21//! - **No invented numbers.** A number one side does not carry, an
22//!   unassessed dimension, an empty column or an empty report, leaves the
23//!   change `None`. `rel_pct` is `None` as well when `a` is zero, where a
24//!   relative change is undefined.
25//! - **Provenance.** Each number carries the [`Evidence`] behind it on both
26//!   sides, so an exact baseline compared with a sampled or truncated run says
27//!   so instead of reading a sampling difference as a change in the data.
28//! - **Definitions.** `metric_semantics.comparable` is `Some(true)` only when
29//!   both reports record every definition this build knows, with the same
30//!   values. A difference in a measurement whose definition changed between
31//!   releases is not a difference in the data.
32//! - **Configuration.** `configuration` lists each recorded setting that moves
33//!   a score on unchanged data (the score weights, the semantic hints, the
34//!   pattern locale, the timeliness clock, the column selection, the metric
35//!   packs, the declared temporal pairs) on both
36//!   sides, with `same`: `Some(true)` when both recorded it and agree,
37//!   `Some(false)` when they differ, `None` when either did not record it.
38//!   Configuration is not comparability: a score that moved under a different
39//!   setting may not reflect a change in the data. Each policy reads only the
40//!   settings it depends on.
41//! - **Builds.** `reports` names each side's `generator`, the build that
42//!   wrote it, and its `source_identity`, the caller's version of the source,
43//!   `None` where the side did not record one. `same_build` says whether one
44//!   build wrote both: a difference between two builds can come from a
45//!   definition that changed between them, not from the data.
46//!
47//! Nothing here decides whether a change is acceptable; that is a policy's
48//! job. A window-over-window comparison reuses the same vocabulary: `a` and
49//! `b` name the two sides without saying which came first.
50//!
51//! # Example
52//!
53//! ```no_run
54//! use dataprof_runtime::{ProfileReport, ReportDelta};
55//!
56//! fn drift(baseline: &ProfileReport, current: &ProfileReport) {
57//!     let delta = ReportDelta::between(baseline, current);
58//!     if let Some(change) = delta.quality_score.abs {
59//!         println!("quality moved {change:+.2} points");
60//!     }
61//!     for name in &delta.schema.removed {
62//!         println!("column {name} is gone");
63//!     }
64//! }
65//! ```
66
67use std::collections::{BTreeMap, HashSet};
68
69use dataprof_core::serde_helpers::rounded_2;
70use dataprof_core::{
71    ColumnStats, DataSource, DataType, FileFormat, MetricPack, NativeType, QualityDimension,
72    QualityScoreWeights, Quartiles, SemanticHintBinding, SemanticHintKind,
73};
74use dataprof_metrics::{
75    AccuracyMetrics, CompletenessMetrics, ConsistencyMetrics, PrecisionMetrics, QualityAssessment,
76    QualityMetrics, ScoreBounds, TimelinessClock, TimelinessMetrics, UniquenessMetrics,
77    ValidityMetrics,
78};
79use serde::ser::{SerializeMap, Serializer};
80
81use crate::profile_report::{
82    AnalysisSettings, ColumnSelection, MetricSemantics, PatternLocale, ProfileReport,
83    ReportGenerator, SourceIdentity, TemporalPair,
84};
85use crate::quality_gate::{
86    Evidence, EvidenceGap, Provenance, overall_left_open, sampled_component_evidence,
87    scan_coverage, scan_evidence,
88};
89
90/// The same fact about each side of a comparison.
91#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize)]
92#[non_exhaustive]
93pub struct Sides<T> {
94    /// The first report.
95    pub a: T,
96    /// The second report.
97    pub b: T,
98}
99
100/// Which report a side is, and how much of its source it read.
101#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize)]
102#[non_exhaustive]
103pub struct ReportReference {
104    /// The report's identifier. `None` for a flat summary, which does not
105    /// record one.
106    pub id: Option<String>,
107    /// When the report was generated. `None` for a flat summary.
108    pub timestamp: Option<String>,
109    /// The source identifier: a path, a query, or an in-memory label.
110    pub source: String,
111    /// Rows the scan read. `None` for a flat summary that did not record it.
112    pub rows_processed: Option<usize>,
113    /// Whether the scan covered the whole source.
114    pub evidence: Evidence,
115    /// The build that wrote the report, `None` when the report did not
116    /// record one (before 0.13). Provenance, not a measurement: it never
117    /// moves a number in the comparison.
118    pub generator: Option<ReportGenerator>,
119    /// The caller's version of the source the report describes, `None` when
120    /// none was recorded. Provenance, like `generator`.
121    pub source_identity: Option<SourceIdentity>,
122}
123
124/// The two reports a comparison read, and whether one build wrote both.
125///
126/// Serialized as `{"a": ..., "b": ..., "same_build": ...}`.
127#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize)]
128#[non_exhaustive]
129pub struct ReportPair {
130    /// The first report.
131    pub a: ReportReference,
132    /// The second report.
133    pub b: ReportReference,
134    /// Whether the same build wrote both reports, read in order and stopping
135    /// at the first answer: `None` when either side recorded no build;
136    /// `Some(false)` when the versions differ; `None` when either side
137    /// recorded no commit; `Some(false)` when the commits differ; `None` when
138    /// either side recorded no dirty flag; then `Some(true)` when both are
139    /// clean, `Some(false)` when one is dirty, and `None` when both are dirty
140    /// builds of one commit, whose uncommitted changes nothing records. A
141    /// known difference gives `Some(false)` even when a later field is
142    /// missing.
143    pub same_build: Option<bool>,
144}
145
146impl ReportPair {
147    fn new(a: ReportReference, b: ReportReference) -> Self {
148        let same_build = same_build(a.generator.as_ref(), b.generator.as_ref());
149        Self { a, b, same_build }
150    }
151}
152
153/// Whether two recorded builds are the same build; see
154/// [`ReportPair::same_build`] for the rule.
155fn same_build(a: Option<&ReportGenerator>, b: Option<&ReportGenerator>) -> Option<bool> {
156    let (a, b) = (a?, b?);
157    if a.version != b.version {
158        return Some(false);
159    }
160    let (commit_a, commit_b) = (a.commit.as_ref()?, b.commit.as_ref()?);
161    if commit_a != commit_b {
162        return Some(false);
163    }
164    match (a.dirty?, b.dirty?) {
165        (false, false) => Some(true),
166        (true, true) => None,
167        _ => Some(false),
168    }
169}
170
171/// One score on both sides and how it moved.
172#[derive(Debug, Clone, PartialEq, serde::Serialize)]
173#[non_exhaustive]
174pub struct ScoreChange {
175    /// The score in `a`, or `None` when `a` did not assess it.
176    pub a: Option<f64>,
177    /// The score in `b`, or `None` when `b` did not assess it.
178    pub b: Option<f64>,
179    /// `b - a` in score points, when both sides have a score.
180    pub abs: Option<f64>,
181    /// `b - a` as a percentage of `a`, when both have a score and `a` is not
182    /// zero.
183    pub rel_pct: Option<f64>,
184    /// The evidence behind each side's score: the scan's coverage, and
185    /// whether the score was computed from a retained sample of the rows.
186    pub evidence: Sides<Evidence>,
187}
188
189impl ScoreChange {
190    fn new(a: Option<f64>, b: Option<f64>, evidence: Sides<Evidence>) -> Self {
191        let (abs, rel_pct) = match (a, b) {
192            (Some(a), Some(b)) => (
193                Some(rounded_2(b - a)),
194                (a != 0.0).then(|| rounded_2((b - a) / a.abs() * 100.0)),
195            ),
196            _ => (None, None),
197        };
198        Self {
199            a,
200            b,
201            abs,
202            rel_pct,
203            evidence,
204        }
205    }
206}
207
208/// One quality dimension's change. Serialized as an entry of the
209/// `dimensions` object, keyed by the dimension's name.
210#[derive(Debug, Clone, PartialEq)]
211#[non_exhaustive]
212pub struct DimensionChange {
213    /// The dimension.
214    pub dimension: QualityDimension,
215    /// Its score on both sides.
216    pub score: ScoreChange,
217}
218
219/// One column's change. Serialized as an entry of the `columns` object, keyed
220/// by the column's name. Each side's number covers what that side's scan
221/// read; see [`ReportReference::evidence`].
222#[derive(Debug, Clone, PartialEq, serde::Serialize)]
223#[non_exhaustive]
224pub struct ColumnChange {
225    /// The column's name.
226    #[serde(skip)]
227    pub name: String,
228    /// Null percentage in `a`, or `None` when `a` has no such column or the
229    /// column holds no values.
230    pub null_pct_a: Option<f64>,
231    /// Null percentage in `b`, under the same rules.
232    pub null_pct_b: Option<f64>,
233    /// `null_pct_b - null_pct_a`, when both are present. Positive means more
234    /// nulls.
235    pub null_pct_delta: Option<f64>,
236    /// The type `a` inferred, by its lowercase name (`"integer"`), or `None`
237    /// when `a` has no such column, read no value for it that is not null,
238    /// or did not record its counts or a type this build knows. A column
239    /// whose values are all null establishes no type, so two names that
240    /// differ here are the change the quality gate reports as
241    /// `column_type_changed`.
242    #[serde(serialize_with = "serialize_type_name")]
243    pub data_type_a: Option<DataType>,
244    /// The type `b` inferred, under the same rules.
245    #[serde(serialize_with = "serialize_type_name")]
246    pub data_type_b: Option<DataType>,
247    /// The type `a`'s source declared for the column (#906), or `None` when
248    /// `a` has no such column or recorded no declaration. A declaration does
249    /// not depend on the values, so it is kept for an all-null column too.
250    pub native_type_a: Option<NativeType>,
251    /// The type `b`'s source declared, under the same rules.
252    pub native_type_b: Option<NativeType>,
253}
254
255fn serialize_type_name<S: Serializer>(
256    data_type: &Option<DataType>,
257    serializer: S,
258) -> Result<S::Ok, S::Error> {
259    serde::Serialize::serialize(&data_type.as_ref().map(DataType::name), serializer)
260}
261
262/// Each side's recorded measurement definitions, and whether they agree.
263#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize)]
264#[non_exhaustive]
265pub struct SemanticsComparison {
266    /// What `a` recorded, or `None` when it predates the record.
267    pub a: Option<MetricSemantics>,
268    /// What `b` recorded.
269    pub b: Option<MetricSemantics>,
270    /// `Some(true)` when both record every definition this build knows and
271    /// they match; `Some(false)` when they record different ones; `None`
272    /// when either side leaves a definition unrecorded, which includes every
273    /// report written before 0.12.
274    pub comparable: Option<bool>,
275}
276
277/// One recorded setting on each side, and whether the two agree.
278#[derive(Debug, Clone, PartialEq, serde::Serialize)]
279#[non_exhaustive]
280pub struct SettingComparison<T> {
281    /// What `a` recorded, or `None` when it did not record the setting.
282    pub a: Option<T>,
283    /// What `b` recorded.
284    pub b: Option<T>,
285    /// `Some(true)` when both recorded the setting and it is the same,
286    /// `Some(false)` when it differs, `None` when either side did not
287    /// record it.
288    pub same: Option<bool>,
289}
290
291impl<T: PartialEq> SettingComparison<T> {
292    fn new(a: Option<T>, b: Option<T>) -> Self {
293        Self::new_by(a, b, |left, right| left == right)
294    }
295}
296
297impl<T> SettingComparison<T> {
298    fn new_by(a: Option<T>, b: Option<T>, same: impl Fn(&T, &T) -> bool) -> Self {
299        let same = match (&a, &b) {
300            (Some(left), Some(right)) => Some(same(left, right)),
301            _ => None,
302        };
303        Self { a, b, same }
304    }
305}
306
307/// Whether two selections profile the same columns. The order a `columns=`
308/// list gives orders the report's columns and moves no score, so it is not
309/// compared; `all` and a list are different settings even when the list
310/// names every column, since only `all` follows the source's columns.
311fn same_selection(a: &ColumnSelection, b: &ColumnSelection) -> bool {
312    match (a, b) {
313        (ColumnSelection::Listed { names: left }, ColumnSelection::Listed { names: right }) => {
314            let left: HashSet<&str> = left.iter().map(String::as_str).collect();
315            let right: HashSet<&str> = right.iter().map(String::as_str).collect();
316            left == right
317        }
318        _ => a == b,
319    }
320}
321
322/// Whether two records declare the same pairs. A writer records them sorted
323/// and once each; a document edited by hand may not, and the calculator
324/// compares each pair once in any order, so neither order nor repetition is
325/// compared. Direction is: a reversed pair counts the opposite rows.
326fn same_pairs(a: &[TemporalPair], b: &[TemporalPair]) -> bool {
327    let left: HashSet<&TemporalPair> = a.iter().collect();
328    let right: HashSet<&TemporalPair> = b.iter().collect();
329    left == right
330}
331
332/// The semantic hints a report recorded, by the configuration field that
333/// carries each, read from its hint bindings.
334///
335/// Each list is sorted and keeps the repetitions a report recorded. A report
336/// written since #1004 records a repeated column once, but one written before
337/// it records the column twice and scanned it twice, which moved the
338/// timeliness score, so such a report and one listing the column once are not
339/// the same configuration.
340#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize)]
341#[non_exhaustive]
342pub struct RecordedHints {
343    /// Columns named by `positive_columns`, sorted.
344    pub positive_columns: Vec<String>,
345    /// Columns named by `identifier_columns`, sorted.
346    pub identifier_columns: Vec<String>,
347    /// Columns named by `temporal_columns`, sorted.
348    pub temporal_columns: Vec<String>,
349    /// Columns named by `temporal_pairs`, sorted. The bindings record which
350    /// columns the pairs named; which column pairs with which is recorded
351    /// from 0.14 in `analysis_settings` (see
352    /// [`ConfigurationComparison::temporal_pairs`]).
353    pub temporal_pair_columns: Vec<String>,
354}
355
356impl RecordedHints {
357    fn from_bindings(bindings: &[SemanticHintBinding]) -> Self {
358        let columns = |kind: SemanticHintKind| {
359            let mut names: Vec<String> = bindings
360                .iter()
361                .filter(|binding| binding.kind == kind)
362                .map(|binding| binding.column.clone())
363                .collect();
364            names.sort();
365            names
366        };
367        Self {
368            positive_columns: columns(SemanticHintKind::Positive),
369            identifier_columns: columns(SemanticHintKind::Identifier),
370            temporal_columns: columns(SemanticHintKind::Temporal),
371            temporal_pair_columns: columns(SemanticHintKind::TemporalPair),
372        }
373    }
374}
375
376/// The recorded settings that move a score on unchanged data, on each side.
377#[derive(Debug, Clone, PartialEq, serde::Serialize)]
378#[non_exhaustive]
379pub struct ConfigurationComparison {
380    /// The weights behind the overall score. `None` on a side with no quality
381    /// assessment, or on a flat summary written before 0.12, which could drop
382    /// custom weights.
383    pub score_weights: SettingComparison<QualityScoreWeights>,
384    /// The semantic hints. `None` on a side written before 0.10, which did
385    /// not record them.
386    pub semantic_hints: SettingComparison<RecordedHints>,
387    /// The locale detected patterns were ranked with, which the validity
388    /// score reads. `None` on a side written before dataprof recorded it.
389    pub pattern_locale: SettingComparison<PatternLocale>,
390    /// The date the timeliness score was measured against, how it was
391    /// chosen, and the age past which a value is stale (#903). `None` on a
392    /// side with no timeliness assessment, or written before 0.14. Two
393    /// rolling clocks read on different days differ here, as their scores
394    /// were measured against different dates.
395    pub timeliness_clock: SettingComparison<TimelinessClock>,
396    /// Which columns each side profiled: `all`, or the `columns=` list as
397    /// given. A dataset score covers only the selected columns. `same`
398    /// compares the columns selected, not the order a list gives them in.
399    /// `None` on a side written before 0.13.
400    pub columns: SettingComparison<ColumnSelection>,
401    /// The metric packs each side computed, `schema` always among them, in
402    /// a fixed order. Without `patterns` validity is not assessed and the
403    /// overall score averages the other dimensions. `None` on a side written
404    /// before 0.14.
405    pub metric_packs: SettingComparison<Vec<MetricPack>>,
406    /// The start/end column pairs each side declared through
407    /// `temporal_pairs`, which timeliness compares. `same` compares the
408    /// pairs, direction included, not the order or repetition of a list.
409    /// `None` on a side written before 0.14, which recorded only the
410    /// columns the pairs named (see [`RecordedHints::temporal_pair_columns`]).
411    pub temporal_pairs: SettingComparison<Vec<TemporalPair>>,
412}
413
414/// Which columns each side has.
415#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize)]
416#[non_exhaustive]
417pub struct SchemaChange {
418    /// In `b` only, in `b`'s order.
419    pub added: Vec<String>,
420    /// In `a` only, in `a`'s order.
421    pub removed: Vec<String>,
422    /// In both, in `a`'s order.
423    pub common: Vec<String>,
424}
425
426/// What changed between two reports. See the [module docs](self) for the
427/// contract.
428#[derive(Debug, Clone, PartialEq, serde::Serialize)]
429#[non_exhaustive]
430pub struct ReportDelta {
431    /// The overall quality score.
432    pub quality_score: ScoreChange,
433    /// Every quality dimension this build knows, in [`QualityDimension::all`]
434    /// order, assessed or not.
435    #[serde(serialize_with = "serialize_dimensions")]
436    pub dimensions: Vec<DimensionChange>,
437    /// The union of both sides' columns: `a`'s in its order, then the ones
438    /// only `b` has, in `b`'s order.
439    #[serde(serialize_with = "serialize_columns")]
440    pub columns: Vec<ColumnChange>,
441    /// Whether the two reports measured the same way.
442    pub metric_semantics: SemanticsComparison,
443    /// The settings each report was profiled with that move its scores.
444    pub configuration: ConfigurationComparison,
445    /// Columns added, removed and shared.
446    pub schema: SchemaChange,
447    /// Which two reports were compared, and whether one build wrote both.
448    pub reports: ReportPair,
449}
450
451impl ReportDelta {
452    /// Compare two reports, `a` first.
453    pub fn between(a: &ProfileReport, b: &ProfileReport) -> Self {
454        Self::between_sides(&ReportSide::from_report(a), &ReportSide::from_report(b))
455    }
456
457    /// Compare two sides, either of which may come from a flat summary.
458    pub fn between_sides(a: &ReportSide, b: &ReportSide) -> Self {
459        let pair = |a: Evidence, b: Evidence| Sides { a, b };
460        let quality_score = ScoreChange::new(
461            a.quality_score,
462            b.quality_score,
463            pair(a.overall_evidence, b.overall_evidence),
464        );
465        let dimensions = a
466            .dimensions
467            .iter()
468            .zip(&b.dimensions)
469            .map(|(left, right)| {
470                debug_assert_eq!(left.dimension, right.dimension);
471                DimensionChange {
472                    dimension: left.dimension,
473                    score: ScoreChange::new(
474                        left.score,
475                        right.score,
476                        pair(left.evidence, right.evidence),
477                    ),
478                }
479            })
480            .collect();
481
482        let a_names: HashSet<&str> = a
483            .columns
484            .iter()
485            .map(|column| column.name.as_str())
486            .collect();
487        let b_names: HashSet<&str> = b
488            .columns
489            .iter()
490            .map(|column| column.name.as_str())
491            .collect();
492        let a_by_name: BTreeMap<&str, &ColumnSide> = a
493            .columns
494            .iter()
495            .map(|column| (column.name.as_str(), column))
496            .collect();
497        let b_by_name: BTreeMap<&str, &ColumnSide> = b
498            .columns
499            .iter()
500            .map(|column| (column.name.as_str(), column))
501            .collect();
502        let added: Vec<String> = b
503            .columns
504            .iter()
505            .filter(|column| !a_names.contains(column.name.as_str()))
506            .map(|column| column.name.clone())
507            .collect();
508        let columns = a
509            .columns
510            .iter()
511            .map(|column| &column.name)
512            .chain(&added)
513            .map(|name| {
514                let side_a = a_by_name.get(name.as_str()).copied();
515                let side_b = b_by_name.get(name.as_str()).copied();
516                let null_pct_a = side_a.and_then(|column| column.null_pct);
517                let null_pct_b = side_b.and_then(|column| column.null_pct);
518                ColumnChange {
519                    name: name.clone(),
520                    null_pct_a,
521                    null_pct_b,
522                    null_pct_delta: null_pct_a.zip(null_pct_b).map(|(a, b)| rounded_2(b - a)),
523                    data_type_a: side_a.and_then(ColumnSide::established_type),
524                    data_type_b: side_b.and_then(ColumnSide::established_type),
525                    native_type_a: side_a.and_then(|column| column.native_type.clone()),
526                    native_type_b: side_b.and_then(|column| column.native_type.clone()),
527                }
528            })
529            .collect();
530        let (removed, common) = a
531            .columns
532            .iter()
533            .map(|column| column.name.clone())
534            .partition(|name| !b_names.contains(name.as_str()));
535
536        let comparable = match (&a.metric_semantics, &b.metric_semantics) {
537            (Some(left), Some(right)) if left.is_fully_recorded() && right.is_fully_recorded() => {
538                Some(left == right)
539            }
540            _ => None,
541        };
542
543        Self {
544            quality_score,
545            dimensions,
546            columns,
547            metric_semantics: SemanticsComparison {
548                a: a.metric_semantics.clone(),
549                b: b.metric_semantics.clone(),
550                comparable,
551            },
552            configuration: ConfigurationComparison {
553                score_weights: SettingComparison::new(a.score_weights, b.score_weights),
554                semantic_hints: SettingComparison::new(
555                    a.semantic_hints.clone(),
556                    b.semantic_hints.clone(),
557                ),
558                pattern_locale: SettingComparison::new(a.pattern_locale, b.pattern_locale),
559                timeliness_clock: SettingComparison::new(a.timeliness_clock, b.timeliness_clock),
560                columns: SettingComparison::new_by(
561                    a.column_selection.clone(),
562                    b.column_selection.clone(),
563                    same_selection,
564                ),
565                metric_packs: SettingComparison::new(
566                    a.metric_packs.clone(),
567                    b.metric_packs.clone(),
568                ),
569                temporal_pairs: SettingComparison::new_by(
570                    a.temporal_pairs.clone(),
571                    b.temporal_pairs.clone(),
572                    |left, right| same_pairs(left, right),
573                ),
574            },
575            schema: SchemaChange {
576                added,
577                removed,
578                common,
579            },
580            reports: ReportPair::new(a.reference.clone(), b.reference.clone()),
581        }
582    }
583}
584
585fn serialize_dimensions<S: Serializer>(
586    dimensions: &[DimensionChange],
587    serializer: S,
588) -> Result<S::Ok, S::Error> {
589    let mut map = serializer.serialize_map(Some(dimensions.len()))?;
590    for change in dimensions {
591        map.serialize_entry(&change.dimension.to_string(), &change.score)?;
592    }
593    map.end()
594}
595
596fn serialize_columns<S: Serializer>(
597    columns: &[ColumnChange],
598    serializer: S,
599) -> Result<S::Ok, S::Error> {
600    let mut map = serializer.serialize_map(Some(columns.len()))?;
601    for change in columns {
602        map.serialize_entry(&change.name, change)?;
603    }
604    map.end()
605}
606
607/// What a comparison reads from one report.
608///
609/// Built from a [`ProfileReport`], or from a flat summary (the document
610/// [`ProfileReport::summary_json`] writes, and the one Python's `to_dict()`
611/// and pre-0.12 `save()` produced). A flat summary never becomes a
612/// `ProfileReport`, because it cannot acquire provenance it never recorded,
613/// but it carries everything a comparison reads, and what it did not record
614/// is read as unrecorded.
615#[derive(Debug, Clone, PartialEq)]
616#[non_exhaustive]
617pub struct ReportSide {
618    reference: ReportReference,
619    quality_score: Option<f64>,
620    overall_evidence: Evidence,
621    /// [`DimensionSide::sample_evidence`] for the overall score.
622    overall_sample_evidence: Evidence,
623    dimensions: Vec<DimensionSide>,
624    /// The recorded intervals for the scores over every scanned row, `None`
625    /// when the side recorded none: nothing was sampled, or it predates them.
626    score_bounds: Option<ScoreBounds>,
627    columns: Vec<ColumnSide>,
628    metric_semantics: Option<MetricSemantics>,
629    score_weights: Option<QualityScoreWeights>,
630    semantic_hints: Option<RecordedHints>,
631    /// The `identifier_columns` hint in the order it was given, which
632    /// [`RecordedHints`] sorts away. The first one present is the key the
633    /// uniqueness score reads, so the quality gate compares this order.
634    identifier_order: Option<Vec<String>>,
635    pattern_locale: Option<PatternLocale>,
636    timeliness_clock: Option<TimelinessClock>,
637    /// Which columns the side profiled, `None` when it did not record it.
638    column_selection: Option<ColumnSelection>,
639    /// The metric packs the side computed, `None` when it did not record
640    /// them.
641    metric_packs: Option<Vec<MetricPack>>,
642    /// The temporal pairs the side declared, `None` when it did not record
643    /// them.
644    temporal_pairs: Option<Vec<TemporalPair>>,
645    /// Whether the source declares its columns up front, so the column set
646    /// does not depend on how many rows were read.
647    schema_declared: bool,
648    /// Whether the source may hold a JSON number. True when the side did not
649    /// record its kind and format.
650    json_numbers: bool,
651    /// Components carrying signal that neither recorded coverage list names,
652    /// so their coverage is unrecorded (#749). Empty for a flat summary that
653    /// records no exact list (#1027).
654    unlisted_components: Vec<String>,
655}
656
657/// The identifier hint's columns in binding order, the order they were given.
658fn identifier_order(bindings: &[SemanticHintBinding]) -> Vec<String> {
659    bindings
660        .iter()
661        .filter(|binding| binding.kind == SemanticHintKind::Identifier)
662        .map(|binding| binding.column.clone())
663        .collect()
664}
665
666#[derive(Debug, Clone, PartialEq)]
667struct DimensionSide {
668    dimension: QualityDimension,
669    score: Option<f64>,
670    evidence: Evidence,
671    /// Whether the score covers every value the scan read, apart from how
672    /// much of the source the scan read (#1317).
673    sample_evidence: Evidence,
674}
675
676/// What a comparison reads about one column.
677#[derive(Debug, Clone, PartialEq)]
678pub(crate) struct ColumnSide {
679    pub(crate) name: String,
680    /// The null percentage at its serialized precision, `None` when the
681    /// column holds no values.
682    pub(crate) null_pct: Option<f64>,
683    /// The inferred type, `None` on a flat summary that did not record one
684    /// this build knows.
685    pub(crate) data_type: Option<DataType>,
686    /// `(total_count, null_count)`, `None` on a flat summary that did not
687    /// record both. Exact where the rounded percentage is not: 99.999% null
688    /// rounds to 100.00 and still holds a value.
689    pub(crate) counts: Option<(usize, usize)>,
690    /// The type the source declared, `None` when it recorded none.
691    pub(crate) native_type: Option<NativeType>,
692    /// Distinct values among the non-null values read, `None` when the side
693    /// did not count them or did not record the count.
694    pub(crate) unique_count: Option<usize>,
695    /// Whether `unique_count` is an estimate, `None` when the side did not
696    /// record it. Only `Some(false)` makes the count exact.
697    pub(crate) unique_count_is_approximate: Option<bool>,
698    /// The column's exact recorded quartiles differ. False when they
699    /// coincide, are estimates, or were not recorded.
700    pub(crate) quartiles_differ: bool,
701    /// The longest value of a text column, `None` when the side recorded no
702    /// text lengths for it.
703    pub(crate) max_text_length: Option<usize>,
704}
705
706/// The fewest characters a spaced ISO datetime has when `date_forms` reads it
707/// as a date and earlier releases did not: the plain `2026-06-01 12:00:00`
708/// (19) was a date in both, and the new forms add a fraction (`.5`) or an
709/// offset (`Z`, `+00:00`).
710const SPACED_DATETIME_MIN_LENGTH: usize = 20;
711
712impl ColumnSide {
713    /// The inferred type, when the side read a value for the column that is
714    /// not null. The quality gate holds the same condition before it calls a
715    /// type a change.
716    fn established_type(&self) -> Option<DataType> {
717        self.populated()
718            .filter(|&populated| populated)
719            .and(self.data_type.clone())
720    }
721
722    /// At least one value read for the column is not null. `None` when the
723    /// counts were not recorded.
724    pub(crate) fn populated(&self) -> Option<bool> {
725        self.counts.map(|(total, nulls)| nulls < total)
726    }
727
728    /// Whether the outlier test may have run on this column with its first
729    /// and third quartiles one value, which `outlier_spread` decides (#1304).
730    /// Only a numeric column with values is tested, and the exact quartiles
731    /// of its values settle the rest: the Type 7 quartiles coincide exactly
732    /// when the two values the test draws them from do. An unrecorded type,
733    /// count or quartile may.
734    fn may_have_coinciding_quartiles(&self) -> bool {
735        let numeric = self
736            .data_type
737            .as_ref()
738            .is_none_or(|data_type| matches!(data_type, DataType::Integer | DataType::Float));
739        numeric && self.populated() != Some(false) && !self.quartiles_differ
740    }
741
742    /// Whether the column may hold a spaced ISO datetime with a fraction or
743    /// an offset, which `date_forms` reads as a date and earlier releases as
744    /// text (#1340). Such a value moves a score only in a column the checks
745    /// read as text or as dates. A text column whose longest value is
746    /// shorter than any such datetime holds none; its lengths cover every
747    /// value read, and so every value a score reads. A date column records
748    /// no lengths. An unrecorded type or length may.
749    fn may_hold_spaced_datetime(&self) -> bool {
750        if self.populated() == Some(false) {
751            return false;
752        }
753        match self.data_type {
754            None | Some(DataType::Date) => true,
755            Some(DataType::String | DataType::Identifier) => self
756                .max_text_length
757                .is_none_or(|length| length >= SPACED_DATETIME_MIN_LENGTH),
758            Some(DataType::Integer | DataType::Float | DataType::Boolean | DataType::Nested) => {
759                false
760            }
761        }
762    }
763
764    /// Values were read for the column and every one is null. `None` when
765    /// the counts were not recorded.
766    pub(crate) fn all_null(&self) -> Option<bool> {
767        self.counts
768            .map(|(total, nulls)| total > 0 && nulls == total)
769    }
770}
771
772/// One entry per column name, in order, holding the first profile of a
773/// repeated name. Most input paths refuse duplicate names, but a report
774/// rebuilt from a document can carry them, and the quality gate reads the
775/// first profile too, so the two never disagree about which one a name means.
776fn first_of_each_name(columns: impl Iterator<Item = ColumnSide>) -> Vec<ColumnSide> {
777    let mut seen = HashSet::new();
778    columns
779        .filter(|column| seen.insert(column.name.clone()))
780        .collect()
781}
782
783/// Whether exact quartiles were recorded and differ. Quartiles that differ
784/// still differ at the serialized precision or coincide there, and equal ones
785/// stay equal, so a document read back never claims a spread it lacks.
786fn exact_quartiles_differ(quartiles: Option<&Quartiles>, approximate: Option<bool>) -> bool {
787    approximate == Some(false) && quartiles.is_some_and(|quartiles| quartiles.q1 != quartiles.q3)
788}
789
790/// Whether a source of this kind may hold a JSON number, which
791/// `json_number_reading` decides (#1243): a field of a JSON or JSONL record,
792/// or a database's JSON cell (#1168). A format this build does not know, or
793/// a stream that records none, may.
794fn may_read_json_numbers(source: &DataSource) -> bool {
795    match source {
796        DataSource::File { format, .. } | DataSource::Bytes { format, .. } => {
797            format_may_hold_json_numbers(format)
798        }
799        DataSource::Stream { format, .. } => {
800            format.as_ref().is_none_or(format_may_hold_json_numbers)
801        }
802        DataSource::Query { .. } => true,
803        DataSource::DataFrame { .. } => false,
804    }
805}
806
807/// Whether a source in this format may hold a JSON number.
808fn format_may_hold_json_numbers(format: &FileFormat) -> bool {
809    !matches!(
810        format,
811        FileFormat::Csv | FileFormat::Parquet | FileFormat::Arrow
812    )
813}
814
815/// A number at its serialized precision; a non-finite one serializes as
816/// null, so it is no number at all.
817fn at_precision(value: Option<f64>) -> Option<f64> {
818    value.filter(|value| value.is_finite()).map(rounded_2)
819}
820
821impl ReportSide {
822    /// Read a profiled or loaded report.
823    pub fn from_report(report: &ProfileReport) -> Self {
824        let scan = scan_evidence(&report.execution);
825        let quality = report.quality.as_ref();
826        let scores = quality.map(|assessment| assessment.scores());
827        let sampled = quality.map(|assessment| assessment.sampled_dimensions());
828        let assessed = quality
829            .map(|assessment| assessment.metrics.assessed_dimensions())
830            .unwrap_or_default();
831        let unlisted = quality
832            .map(|assessment| assessment.unlisted_components())
833            .unwrap_or_default();
834        let overall_open = quality.is_some_and(|assessment| {
835            overall_left_open(
836                assessment.recorded_score_bounds(),
837                |dimension| assessment.metrics.dimension_score(dimension).is_none(),
838                Some(&assessment.metrics.score_weights),
839            )
840        });
841        // No assessment means no score for evidence to describe; the scan's
842        // coverage is all there is to say.
843        let sample_evidence = |provenance| match &sampled {
844            Some(sampled) => sampled_component_evidence(
845                sampled.as_deref(),
846                &unlisted,
847                &assessed,
848                overall_open,
849                provenance,
850            ),
851            None => Evidence::Complete,
852        };
853        let evidence = |provenance| scan.and(sample_evidence(provenance));
854        let dimensions = QualityDimension::all()
855            .into_iter()
856            .map(|dimension| DimensionSide {
857                dimension,
858                score: at_precision(
859                    scores
860                        .as_ref()
861                        .and_then(|scores| scores.dimension_scores.get(&dimension.to_string()))
862                        .copied()
863                        .flatten(),
864                ),
865                evidence: evidence(Provenance::Dimension(dimension)),
866                sample_evidence: sample_evidence(Provenance::Dimension(dimension)),
867            })
868            .collect();
869        Self {
870            reference: ReportReference {
871                id: Some(report.id.clone()),
872                timestamp: Some(report.timestamp.clone()),
873                source: report.source_identifier(),
874                rows_processed: Some(report.execution.rows_processed),
875                evidence: scan,
876                generator: report.generator.clone(),
877                source_identity: report.source_identity.clone(),
878            },
879            quality_score: at_precision(scores.as_ref().and_then(|scores| scores.overall_score)),
880            overall_evidence: evidence(Provenance::Overall),
881            overall_sample_evidence: sample_evidence(Provenance::Overall),
882            dimensions,
883            score_bounds: quality.and_then(QualityAssessment::score_bounds).cloned(),
884            columns: first_of_each_name(report.column_profiles.iter().map(|column| {
885                let pct = (column.total_count > 0)
886                    .then(|| column.null_count as f64 / column.total_count as f64 * 100.0);
887                ColumnSide {
888                    name: column.name.clone(),
889                    null_pct: at_precision(pct),
890                    data_type: Some(column.data_type.clone()),
891                    counts: Some((column.total_count, column.null_count)),
892                    native_type: column.native_type.clone(),
893                    unique_count: column.unique_count,
894                    unique_count_is_approximate: column.unique_count_is_approximate,
895                    quartiles_differ: match &column.stats {
896                        ColumnStats::Numeric(stats) => {
897                            exact_quartiles_differ(stats.quartiles.as_ref(), stats.is_approximate)
898                        }
899                        _ => false,
900                    },
901                    max_text_length: match &column.stats {
902                        ColumnStats::Text(stats) => Some(stats.max_length),
903                        _ => None,
904                    },
905                }
906            })),
907            metric_semantics: report.metric_semantics.clone(),
908            // The weights behind the score this build reports: recorded, or
909            // the defaults, which did not change from 0.10 (when weights
910            // became configurable) to 0.12, and which a document written
911            // before 0.10 has its scores recomputed with.
912            score_weights: quality.map(|assessment| assessment.metrics.score_weights),
913            // Bindings are recorded from 0.10 (schema version 1) on, and an
914            // empty list there means no hints were given.
915            semantic_hints: (report.schema_version >= 1)
916                .then(|| RecordedHints::from_bindings(&report.semantic_hint_bindings)),
917            identifier_order: (report.schema_version >= 1)
918                .then(|| identifier_order(&report.semantic_hint_bindings)),
919            pattern_locale: report
920                .analysis_settings
921                .as_ref()
922                .map(|settings| settings.pattern_locale),
923            // Only an assessed dimension is written, so a clock behind no
924            // timeliness score is not part of the report a summary reads.
925            timeliness_clock: quality
926                .and_then(|assessment| assessment.metrics.timeliness.as_ref())
927                .filter(|timeliness| timeliness.is_assessed())
928                .and_then(|timeliness| timeliness.clock),
929            column_selection: report
930                .analysis_settings
931                .as_ref()
932                .map(|settings| settings.columns.clone()),
933            metric_packs: report
934                .analysis_settings
935                .as_ref()
936                .and_then(|settings| settings.metric_packs.clone()),
937            temporal_pairs: report
938                .analysis_settings
939                .as_ref()
940                .and_then(|settings| settings.temporal_pairs.clone()),
941            schema_declared: declares_schema(&report.data_source),
942            json_numbers: may_read_json_numbers(&report.data_source),
943            unlisted_components: unlisted,
944        }
945    }
946
947    /// Components carrying signal that neither of the side's recorded
948    /// coverage lists names. Their coverage, and that of any score weighing
949    /// them, is unrecorded. Empty for a flat summary written before 0.14,
950    /// which records no exact list.
951    pub fn unlisted_components(&self) -> &[String] {
952        &self.unlisted_components
953    }
954
955    /// [`Self::unlisted_components`] for the `quality` object of a flat
956    /// summary alone, for a reader that has no whole summary to read, such
957    /// as findings over a loaded one. Fails when the object holds a field of
958    /// the wrong type.
959    pub fn summary_quality_unlisted_components(quality: &str) -> serde_json::Result<Vec<String>> {
960        let quality: SummaryQuality = serde_json::from_str(quality)?;
961        Ok(quality.unlisted_components())
962    }
963
964    /// The recorded intervals for the side's scores over every scanned row,
965    /// or `None` when it recorded none.
966    pub(crate) fn score_bounds(&self) -> Option<&ScoreBounds> {
967        self.score_bounds.as_ref()
968    }
969
970    /// The identifier hint in the order it was given, or `None` when the side
971    /// did not record its hints.
972    pub(crate) fn identifier_order(&self) -> Option<&[String]> {
973        self.identifier_order.as_deref()
974    }
975
976    /// Whether a score, the overall one for `None`, covers every value the
977    /// scan read. Unlike the score's evidence, a scan that stopped early
978    /// leaves it complete.
979    pub(crate) fn sample_evidence(&self, dimension: Option<QualityDimension>) -> Evidence {
980        match dimension {
981            None => self.overall_sample_evidence,
982            Some(dimension) => self
983                .dimensions
984                .iter()
985                .find(|side| side.dimension == dimension)
986                .map_or(
987                    Evidence::Incomplete {
988                        reason: EvidenceGap::CoverageUnrecorded,
989                    },
990                    |side| side.sample_evidence,
991                ),
992        }
993    }
994
995    /// Whether the side's source may hold a JSON number, so that
996    /// `json_number_reading` may move its scores.
997    pub(crate) fn may_read_json_numbers(&self) -> bool {
998        self.json_numbers
999    }
1000
1001    /// Whether the side's outlier test may have run on a column whose
1002    /// quartiles coincide, so that `outlier_spread` may move its accuracy
1003    /// score. The recorded quartiles describe every value the scan read and
1004    /// the test reads the quality sample, so they speak for the test only
1005    /// where that sample held every value read. How much of the source the
1006    /// scan read does not matter: both describe the same values.
1007    pub(crate) fn may_test_coinciding_quartiles(&self) -> bool {
1008        let every_value = self
1009            .sample_evidence(Some(QualityDimension::Accuracy))
1010            .is_complete();
1011        !every_value
1012            || self
1013                .columns
1014                .iter()
1015                .any(ColumnSide::may_have_coinciding_quartiles)
1016    }
1017
1018    /// Whether a column the side scored may hold a spaced ISO datetime with
1019    /// a fraction or an offset, so that `date_forms` may move its consistency
1020    /// and timeliness scores (#1340). Timeliness parses a column named by
1021    /// `temporal_columns` or `temporal_pairs` whatever its type, so a
1022    /// populated one may, a numeric one included; a side that recorded no
1023    /// hints may have named any column.
1024    pub(crate) fn may_hold_spaced_datetimes(&self) -> bool {
1025        let temporal = self.semantic_hints.as_ref().map(|hints| {
1026            hints
1027                .temporal_columns
1028                .iter()
1029                .chain(&hints.temporal_pair_columns)
1030                .collect::<HashSet<_>>()
1031        });
1032        self.columns.iter().any(|column| {
1033            column.may_hold_spaced_datetime()
1034                || (column.populated() != Some(false)
1035                    && temporal
1036                        .as_ref()
1037                        .is_none_or(|names| names.contains(&column.name)))
1038        })
1039    }
1040
1041    /// Which columns the side profiled, or `None` when it did not record it.
1042    pub(crate) fn column_selection(&self) -> Option<&ColumnSelection> {
1043        self.column_selection.as_ref()
1044    }
1045
1046    /// Whether the side's column set covers its whole source. A source that
1047    /// declares its columns up front (a CSV header, a Parquet schema, a
1048    /// DataFrame, a query result) lists them however few rows were read;
1049    /// otherwise the columns are the keys of the records read, and only a scan
1050    /// of every record covers the source.
1051    pub(crate) fn schema_evidence(&self) -> Evidence {
1052        if self.schema_declared {
1053            Evidence::Complete
1054        } else {
1055            self.reference.evidence
1056        }
1057    }
1058
1059    /// The side's columns, one per name, in its order.
1060    pub(crate) fn columns(&self) -> &[ColumnSide] {
1061        &self.columns
1062    }
1063
1064    /// Read a flat summary document.
1065    ///
1066    /// A field an older release did not write reads as unrecorded: scan
1067    /// coverage it does not state is `coverage_unrecorded`, never complete.
1068    /// Fails when the document lacks `source` or `columns`, or holds a
1069    /// field of the wrong type.
1070    pub fn from_summary_json(text: &str) -> serde_json::Result<Self> {
1071        let summary: Summary = serde_json::from_str(text)?;
1072        let execution = &summary.execution;
1073        let known_gap = scan_coverage(
1074            execution.source_exhausted == Some(false) || execution.truncation_reason.is_some(),
1075            execution.sampling_applied == Some(true),
1076            execution.error_count.is_some_and(|count| count > 0),
1077        );
1078        let recorded = execution.source_exhausted.is_some()
1079            && execution.sampling_applied.is_some()
1080            && execution.error_count.is_some();
1081        // A recorded gap is named; otherwise an unrecorded fact is a gap of
1082        // its own, because unknown coverage is not full coverage.
1083        let scan = match known_gap {
1084            Evidence::Complete if !recorded => Evidence::Incomplete {
1085                reason: EvidenceGap::CoverageUnrecorded,
1086            },
1087            known => known,
1088        };
1089        let quality = summary.quality.as_ref();
1090        // The dimensions the overall score weighs. An overall score always
1091        // weighs at least one, so an empty list beside a score was not
1092        // recorded: a missing list reads as empty here, as it does in the
1093        // Python loader.
1094        let assessed: Option<Vec<QualityDimension>> = quality.and_then(|quality| {
1095            let parsed: Vec<QualityDimension> = quality
1096                .assessed_dimensions
1097                .iter()
1098                // A dimension a later release added is dropped here: when it
1099                // was sampled, its component label is unknown and
1100                // `sampled_component_evidence` leaves the overall score
1101                // unrecorded; when it was exact it changes no coverage.
1102                .filter_map(|name| name.parse().ok())
1103                .collect();
1104            (!(parsed.is_empty() && quality.overall_score.is_some())).then_some(parsed)
1105        });
1106        let unlisted = quality
1107            .map(SummaryQuality::unlisted_components)
1108            .unwrap_or_default();
1109        let sample_evidence = |provenance| {
1110            let Some(quality) = quality else {
1111                return Evidence::Complete;
1112            };
1113            let sampled = quality.sampled_dimensions.as_deref();
1114            match (provenance, &assessed) {
1115                // Without the weights, a sampled component may or may not
1116                // reach the overall score; only "nothing was sampled" settles it.
1117                (Provenance::Overall, None)
1118                    if sampled.is_some_and(|s| s.is_empty()) && unlisted.is_empty() =>
1119                {
1120                    Evidence::Complete
1121                }
1122                (Provenance::Overall, None) => Evidence::Incomplete {
1123                    reason: EvidenceGap::CoverageUnrecorded,
1124                },
1125                (_, assessed) => sampled_component_evidence(
1126                    sampled,
1127                    &unlisted,
1128                    assessed.as_deref().unwrap_or_default(),
1129                    overall_left_open(
1130                        quality.score_bounds.as_ref(),
1131                        |dimension| {
1132                            quality
1133                                .dimension_scores
1134                                .get(&dimension.to_string())
1135                                .is_none_or(Option::is_none)
1136                        },
1137                        quality.score_weights.as_ref(),
1138                    ),
1139                    provenance,
1140                ),
1141            }
1142        };
1143        let evidence = |provenance| scan.and(sample_evidence(provenance));
1144        let dimensions = QualityDimension::all()
1145            .into_iter()
1146            .map(|dimension| DimensionSide {
1147                dimension,
1148                score: at_precision(
1149                    quality
1150                        .and_then(|quality| quality.dimension_scores.get(&dimension.to_string()))
1151                        .copied()
1152                        .flatten(),
1153                ),
1154                evidence: evidence(Provenance::Dimension(dimension)),
1155                sample_evidence: sample_evidence(Provenance::Dimension(dimension)),
1156            })
1157            .collect();
1158        Ok(Self {
1159            reference: ReportReference {
1160                id: None,
1161                timestamp: None,
1162                source: summary.source,
1163                rows_processed: execution.rows_processed,
1164                evidence: scan,
1165                generator: summary.generator,
1166                source_identity: summary.source_identity,
1167            },
1168            quality_score: at_precision(quality.and_then(|quality| quality.overall_score)),
1169            overall_evidence: evidence(Provenance::Overall),
1170            overall_sample_evidence: sample_evidence(Provenance::Overall),
1171            dimensions,
1172            score_bounds: quality.and_then(|quality| quality.score_bounds.clone()),
1173            columns: first_of_each_name(summary.columns.into_iter().map(|column| ColumnSide {
1174                name: column.name,
1175                null_pct: at_precision(column.null_percentage),
1176                // A name this build does not know comes from a later release and
1177                // reads as unrecorded, since this build has no type to name it with.
1178                data_type: column.data_type.as_deref().and_then(DataType::from_name),
1179                counts: column.total_count.zip(column.null_count),
1180                native_type: column.native_type,
1181                unique_count: column.unique_count,
1182                unique_count_is_approximate: column.unique_count_is_approximate,
1183                quartiles_differ: column.stats.as_ref().is_some_and(summary_quartiles_differ),
1184                max_text_length: column.stats.as_ref().and_then(summary_max_text_length),
1185            })),
1186            // A 0.12 summary, the first to record `metric_semantics`, omitted
1187            // only default weights; earlier ones could drop custom weights too.
1188            score_weights: quality.and_then(|quality| {
1189                quality.score_weights.or_else(|| {
1190                    summary
1191                        .metric_semantics
1192                        .is_some()
1193                        .then(QualityScoreWeights::default)
1194                })
1195            }),
1196            semantic_hints: summary
1197                .schema_version
1198                .is_some_and(|version| version >= 1)
1199                .then(|| RecordedHints::from_bindings(&summary.semantic_hint_bindings)),
1200            identifier_order: summary
1201                .schema_version
1202                .is_some_and(|version| version >= 1)
1203                .then(|| identifier_order(&summary.semantic_hint_bindings)),
1204            metric_semantics: summary.metric_semantics,
1205            pattern_locale: summary
1206                .analysis_settings
1207                .as_ref()
1208                .map(|settings| settings.pattern_locale),
1209            timeliness_clock: quality
1210                .and_then(|quality| quality.timeliness.as_ref())
1211                .and_then(|timeliness| timeliness.clock),
1212            metric_packs: summary
1213                .analysis_settings
1214                .as_ref()
1215                .and_then(|settings| settings.metric_packs.clone()),
1216            temporal_pairs: summary
1217                .analysis_settings
1218                .as_ref()
1219                .and_then(|settings| settings.temporal_pairs.clone()),
1220            column_selection: summary.analysis_settings.map(|settings| settings.columns),
1221            // The rule the canonical reader applies, over the kind and the
1222            // format the summary recorded. A summary written before 0.14 has
1223            // no format, so only the kinds that always declare their columns
1224            // count there (#1028).
1225            schema_declared: match summary.source_type {
1226                Some(SummarySourceType::Dataframe | SummarySourceType::Query) => true,
1227                Some(SummarySourceType::File | SummarySourceType::Bytes) => summary
1228                    .source_format
1229                    .as_ref()
1230                    .is_some_and(file_format_declares_schema),
1231                Some(SummarySourceType::Stream) => summary
1232                    .source_format
1233                    .as_ref()
1234                    .is_some_and(stream_format_declares_schema),
1235                Some(SummarySourceType::Other) | None => false,
1236            },
1237            // The kind and format the summary recorded. One written before
1238            // 0.14 has no format, so only a DataFrame settles it there.
1239            json_numbers: match summary.source_type {
1240                Some(SummarySourceType::Dataframe) => false,
1241                Some(
1242                    SummarySourceType::File | SummarySourceType::Bytes | SummarySourceType::Stream,
1243                ) => summary
1244                    .source_format
1245                    .as_ref()
1246                    .is_none_or(format_may_hold_json_numbers),
1247                Some(SummarySourceType::Query | SummarySourceType::Other) | None => true,
1248            },
1249            unlisted_components: unlisted.clone(),
1250        })
1251    }
1252}
1253
1254/// The field's absence represents a legacy summary; JSON null is malformed.
1255/// Include its name because serde_json's default type error omits the key.
1256fn summary_metric_semantics<'de, D>(deserializer: D) -> Result<Option<MetricSemantics>, D::Error>
1257where
1258    D: serde::Deserializer<'de>,
1259{
1260    crate::profile_report::present_value(deserializer).map_err(|error| {
1261        <D::Error as serde::de::Error>::custom(format!("metric_semantics: {error}"))
1262    })
1263}
1264
1265fn summary_schema_version<'de, D>(deserializer: D) -> Result<Option<u32>, D::Error>
1266where
1267    D: serde::Deserializer<'de>,
1268{
1269    crate::profile_report::present_value(deserializer)
1270        .map_err(|error| <D::Error as serde::de::Error>::custom(format!("schema_version: {error}")))
1271}
1272
1273/// An absent value belongs to a legacy summary; an explicit null does not.
1274fn summary_analysis_settings<'de, D>(deserializer: D) -> Result<Option<AnalysisSettings>, D::Error>
1275where
1276    D: serde::Deserializer<'de>,
1277{
1278    crate::profile_report::present_value(deserializer).map_err(|error| {
1279        <D::Error as serde::de::Error>::custom(format!("analysis_settings: {error}"))
1280    })
1281}
1282
1283/// Only an absent key may use the summary version's legacy weight semantics.
1284fn summary_score_weights<'de, D>(deserializer: D) -> Result<Option<QualityScoreWeights>, D::Error>
1285where
1286    D: serde::Deserializer<'de>,
1287{
1288    crate::profile_report::present_value(deserializer).map_err(|error| {
1289        <D::Error as serde::de::Error>::custom(format!("quality.score_weights: {error}"))
1290    })
1291}
1292
1293/// The fields of a flat summary a comparison reads. Everything else in the
1294/// document is ignored.
1295#[derive(serde::Deserialize)]
1296struct Summary {
1297    source: String,
1298    /// Absent from a summary that did not record it, which declares nothing.
1299    #[serde(default)]
1300    source_type: Option<SummarySourceType>,
1301    /// Absent before 0.14, and for a source that records no format. A format
1302    /// this build does not know reads as `Unknown`, which declares nothing.
1303    #[serde(default)]
1304    source_format: Option<FileFormat>,
1305    execution: SummaryExecution,
1306    columns: Vec<SummaryColumn>,
1307    quality: Option<SummaryQuality>,
1308    #[serde(default, deserialize_with = "summary_metric_semantics")]
1309    metric_semantics: Option<MetricSemantics>,
1310    /// Absent before 0.10, whose summaries did not record hints either.
1311    #[serde(default, deserialize_with = "summary_schema_version")]
1312    schema_version: Option<u32>,
1313    #[serde(default)]
1314    semantic_hint_bindings: Vec<SemanticHintBinding>,
1315    /// Absent before 0.13. An explicit null is malformed, as in the
1316    /// canonical reader and Python's `from_dict()`.
1317    #[serde(default, deserialize_with = "summary_analysis_settings")]
1318    analysis_settings: Option<AnalysisSettings>,
1319    /// Absent before 0.14, or when the build recorded none. An explicit null
1320    /// is malformed, as in the canonical reader.
1321    #[serde(default, deserialize_with = "summary_generator")]
1322    generator: Option<ReportGenerator>,
1323    /// Absent when the caller supplied no version. An explicit null is
1324    /// malformed, as in the canonical reader.
1325    #[serde(default, deserialize_with = "summary_source_identity")]
1326    source_identity: Option<SourceIdentity>,
1327}
1328
1329fn summary_generator<'de, D>(deserializer: D) -> Result<Option<ReportGenerator>, D::Error>
1330where
1331    D: serde::Deserializer<'de>,
1332{
1333    crate::profile_report::present_value(deserializer)
1334        .map_err(|error| <D::Error as serde::de::Error>::custom(format!("generator: {error}")))
1335}
1336
1337fn summary_source_identity<'de, D>(deserializer: D) -> Result<Option<SourceIdentity>, D::Error>
1338where
1339    D: serde::Deserializer<'de>,
1340{
1341    crate::profile_report::present_value(deserializer).map_err(|error| {
1342        <D::Error as serde::de::Error>::custom(format!("source_identity: {error}"))
1343    })
1344}
1345
1346/// A flat summary's `source_type`. A kind a later release adds declares
1347/// nothing this build can rely on.
1348#[derive(serde::Deserialize)]
1349#[serde(rename_all = "lowercase")]
1350enum SummarySourceType {
1351    File,
1352    Bytes,
1353    Stream,
1354    Dataframe,
1355    Query,
1356    #[serde(other)]
1357    Other,
1358}
1359
1360/// Whether a source declares its columns up front: a CSV header, a Parquet
1361/// footer or an Arrow stream schema lists every column before the first row. A
1362/// JSON or JSONL source has the keys of the records read, and a format this
1363/// build does not know, or a stream that records none, declares nothing it can
1364/// rely on.
1365fn declares_schema(source: &DataSource) -> bool {
1366    match source {
1367        DataSource::File { format, .. } | DataSource::Bytes { format, .. } => {
1368            file_format_declares_schema(format)
1369        }
1370        DataSource::Stream {
1371            format: Some(format),
1372            ..
1373        } => stream_format_declares_schema(format),
1374        DataSource::DataFrame { .. } | DataSource::Query { .. } => true,
1375        DataSource::Stream { format: None, .. } => false,
1376    }
1377}
1378
1379/// Whether a file or bytes source in this format lists every column up front.
1380/// Only the Arrow C stream route records `Arrow`, so a file or bytes source
1381/// claiming it stays conservative.
1382fn file_format_declares_schema(format: &FileFormat) -> bool {
1383    matches!(format, FileFormat::Csv | FileFormat::Parquet)
1384}
1385
1386/// Whether a stream in this format lists every column up front.
1387fn stream_format_declares_schema(format: &FileFormat) -> bool {
1388    matches!(
1389        format,
1390        FileFormat::Csv | FileFormat::Parquet | FileFormat::Arrow
1391    )
1392}
1393
1394#[derive(serde::Deserialize)]
1395struct SummaryExecution {
1396    rows_processed: Option<usize>,
1397    source_exhausted: Option<bool>,
1398    truncation_reason: Option<String>,
1399    sampling_applied: Option<bool>,
1400    error_count: Option<usize>,
1401}
1402
1403#[derive(serde::Deserialize)]
1404struct SummaryColumn {
1405    name: String,
1406    null_percentage: Option<f64>,
1407    #[serde(default)]
1408    data_type: Option<String>,
1409    #[serde(default)]
1410    total_count: Option<usize>,
1411    #[serde(default)]
1412    null_count: Option<usize>,
1413    /// Absent before 0.14, and for a column whose source declared no type.
1414    #[serde(default)]
1415    native_type: Option<NativeType>,
1416    #[serde(default)]
1417    unique_count: Option<usize>,
1418    /// Absent, or null, where the writer did not record whether
1419    /// `unique_count` is exact; that reads as unrecorded, never as exact.
1420    #[serde(default)]
1421    unique_count_is_approximate: Option<bool>,
1422    /// Read only for its quartiles, through [`summary_quartiles_differ`]:
1423    /// statistics no comparison reads do not make a summary unreadable.
1424    #[serde(default)]
1425    stats: Option<serde_json::Value>,
1426}
1427
1428/// [`exact_quartiles_differ`] over a summary column's `stats`. A shape this
1429/// build cannot read records no quartiles.
1430fn summary_quartiles_differ(stats: &serde_json::Value) -> bool {
1431    let quartile = |name: &str| stats.get("quartiles")?.get(name)?.as_f64();
1432    stats
1433        .get("is_approximate")
1434        .and_then(serde_json::Value::as_bool)
1435        == Some(false)
1436        && quartile("q1")
1437            .zip(quartile("q3"))
1438            .is_some_and(|(q1, q3)| q1 != q3)
1439}
1440
1441/// A summary column's recorded `max_length`, `None` when its `stats` hold no
1442/// text lengths this build can read.
1443fn summary_max_text_length(stats: &serde_json::Value) -> Option<usize> {
1444    stats
1445        .get("max_length")?
1446        .as_u64()
1447        .and_then(|length| usize::try_from(length).ok())
1448}
1449
1450#[derive(serde::Deserialize)]
1451struct SummaryQuality {
1452    overall_score: Option<f64>,
1453    #[serde(default)]
1454    dimension_scores: BTreeMap<String, Option<f64>>,
1455    #[serde(default)]
1456    assessed_dimensions: Vec<String>,
1457    /// Absent in a summary written before 0.14, which reads as before: a
1458    /// component not listed as sampled is exact (#1027).
1459    #[serde(default)]
1460    exact_dimensions: Option<Vec<String>>,
1461    /// Absent in a summary written before dataprof recorded it, which reads
1462    /// as unrecorded coverage, never as "nothing was sampled".
1463    sampled_dimensions: Option<Vec<String>>,
1464    // Which dimensions carry signal, read as the full document reads them:
1465    // a dimension's block is present, and the uniqueness facts that decide
1466    // its two components. The values themselves are not read here.
1467    #[serde(default)]
1468    completeness: Option<serde::de::IgnoredAny>,
1469    #[serde(default)]
1470    consistency: Option<serde::de::IgnoredAny>,
1471    #[serde(default)]
1472    uniqueness: Option<SummaryUniqueness>,
1473    #[serde(default)]
1474    accuracy: Option<serde::de::IgnoredAny>,
1475    #[serde(default)]
1476    timeliness: Option<SummaryTimeliness>,
1477    #[serde(default)]
1478    validity: Option<serde::de::IgnoredAny>,
1479    #[serde(default)]
1480    precision: Option<serde::de::IgnoredAny>,
1481    /// Written whatever the weights since 0.13; a 0.12 summary omitted only
1482    /// the defaults, and earlier ones did not record them. Explicit null is
1483    /// malformed, as in Python's `from_dict()`.
1484    #[serde(default, deserialize_with = "summary_score_weights")]
1485    score_weights: Option<QualityScoreWeights>,
1486    /// Written when a score was computed over a retained sample (0.12+).
1487    /// Bounds outside 0..=100, reversed, or at an impossible confidence
1488    /// level are malformed, as in the full document's reader.
1489    #[serde(default, deserialize_with = "summary_score_bounds")]
1490    score_bounds: Option<ScoreBounds>,
1491}
1492
1493/// Recorded score bounds, refused when they are not valid bounds.
1494fn summary_score_bounds<'de, D>(deserializer: D) -> Result<Option<ScoreBounds>, D::Error>
1495where
1496    D: serde::Deserializer<'de>,
1497{
1498    let bounds: Option<ScoreBounds> = serde::Deserialize::deserialize(deserializer)?;
1499    if bounds.as_ref().is_some_and(|bounds| !bounds.is_valid()) {
1500        return Err(<D::Error as serde::de::Error>::custom(
1501            "quality.score_bounds: intervals must lie in 0..=100 with lower <= upper, at a \
1502             confidence level between 0 and 1",
1503        ));
1504    }
1505    Ok(bounds)
1506}
1507
1508/// The timeliness facts a comparison reads: the clock its score used.
1509#[derive(serde::Deserialize)]
1510struct SummaryTimeliness {
1511    /// Absent before 0.14. Explicit null is malformed.
1512    #[serde(default, deserialize_with = "summary_timeliness_clock")]
1513    clock: Option<TimelinessClock>,
1514}
1515
1516/// Only an absent key means the summary did not record its clock.
1517fn summary_timeliness_clock<'de, D>(deserializer: D) -> Result<Option<TimelinessClock>, D::Error>
1518where
1519    D: serde::Deserializer<'de>,
1520{
1521    crate::profile_report::present_value(deserializer).map_err(|error| {
1522        <D::Error as serde::de::Error>::custom(format!("quality.timeliness.clock: {error}"))
1523    })
1524}
1525
1526/// The uniqueness facts that decide which of its components carry signal,
1527/// typed as [`UniquenessMetrics`] types them.
1528#[derive(serde::Deserialize)]
1529struct SummaryUniqueness {
1530    #[serde(default)]
1531    rows_checked: usize,
1532    #[serde(default)]
1533    key_column: Option<String>,
1534    #[serde(default)]
1535    key_values_checked: Option<usize>,
1536}
1537
1538impl SummaryQuality {
1539    /// The full document's rule, once the summary records both lists: a
1540    /// component carrying signal that neither names is unrecorded. A summary
1541    /// written before 0.14 has no exact list, so nothing is unlisted and "not
1542    /// sampled" still reads as exact there (#1027).
1543    fn unlisted_components(&self) -> Vec<String> {
1544        match (&self.exact_dimensions, &self.sampled_dimensions) {
1545            (Some(exact), Some(sampled)) => {
1546                self.signal_metrics().unlisted_components(exact, sampled)
1547            }
1548            _ => Vec::new(),
1549        }
1550    }
1551
1552    /// Metrics holding only what [`QualityMetrics::signal_components`] reads:
1553    /// which dimensions are present and the uniqueness facts. The values are
1554    /// defaults and must not be read for anything else.
1555    fn signal_metrics(&self) -> QualityMetrics {
1556        QualityMetrics {
1557            completeness: self
1558                .completeness
1559                .as_ref()
1560                .map(|_| CompletenessMetrics::default()),
1561            consistency: self
1562                .consistency
1563                .as_ref()
1564                .map(|_| ConsistencyMetrics::default()),
1565            uniqueness: self
1566                .uniqueness
1567                .as_ref()
1568                .map(|uniqueness| UniquenessMetrics {
1569                    rows_checked: uniqueness.rows_checked,
1570                    key_column: uniqueness.key_column.clone(),
1571                    key_values_checked: uniqueness.key_values_checked,
1572                    ..UniquenessMetrics::default()
1573                }),
1574            accuracy: self.accuracy.as_ref().map(|_| AccuracyMetrics::default()),
1575            timeliness: self
1576                .timeliness
1577                .as_ref()
1578                .map(|_| TimelinessMetrics::default()),
1579            validity: self.validity.as_ref().map(|_| ValidityMetrics::default()),
1580            precision: self.precision.as_ref().map(|_| PrecisionMetrics::default()),
1581            ..QualityMetrics::default()
1582        }
1583    }
1584}
1585
1586#[cfg(test)]
1587mod tests {
1588    use super::*;
1589
1590    fn complete() -> Sides<Evidence> {
1591        Sides {
1592            a: Evidence::Complete,
1593            b: Evidence::Complete,
1594        }
1595    }
1596
1597    #[test]
1598    fn a_zero_baseline_has_an_absolute_change_and_no_relative_one() {
1599        let change = ScoreChange::new(Some(0.0), Some(50.0), complete());
1600        assert_eq!(change.abs, Some(50.0));
1601        assert_eq!(change.rel_pct, None);
1602    }
1603
1604    #[test]
1605    fn a_change_is_rounded_to_the_serialized_precision() {
1606        // 80.1 - 80.0 is 0.0999... in binary; the contract states 0.1. The
1607        // relative change divides that same unrounded difference, as the
1608        // Python implementation that shipped did: 0.1249..., so 0.12.
1609        let change = ScoreChange::new(Some(80.0), Some(80.1), complete());
1610        assert_eq!(change.abs, Some(0.1));
1611        assert_eq!(change.rel_pct, Some(0.12));
1612    }
1613
1614    #[test]
1615    fn a_missing_side_leaves_both_changes_absent() {
1616        for (a, b) in [(None, Some(1.0)), (Some(1.0), None), (None, None)] {
1617            let change = ScoreChange::new(a, b, complete());
1618            assert_eq!((change.abs, change.rel_pct), (None, None));
1619        }
1620    }
1621
1622    #[test]
1623    fn a_non_finite_input_is_no_number() {
1624        assert_eq!(at_precision(Some(f64::NAN)), None);
1625        assert_eq!(at_precision(Some(f64::INFINITY)), None);
1626        assert_eq!(at_precision(Some(12.345)), Some(12.35));
1627    }
1628}