1use std::collections::{BTreeMap, HashSet};
68
69use dataprof_core::serde_helpers::rounded_2;
70use dataprof_core::{
71 ColumnStats, DataSource, DataType, FileFormat, MetricPack, NativeType, QualityDimension,
72 QualityScoreWeights, Quartiles, SemanticHintBinding, SemanticHintKind,
73};
74use dataprof_metrics::{
75 AccuracyMetrics, CompletenessMetrics, ConsistencyMetrics, PrecisionMetrics, QualityAssessment,
76 QualityMetrics, ScoreBounds, TimelinessClock, TimelinessMetrics, UniquenessMetrics,
77 ValidityMetrics,
78};
79use serde::ser::{SerializeMap, Serializer};
80
81use crate::profile_report::{
82 AnalysisSettings, ColumnSelection, MetricSemantics, PatternLocale, ProfileReport,
83 ReportGenerator, SourceIdentity, TemporalPair,
84};
85use crate::quality_gate::{
86 Evidence, EvidenceGap, Provenance, overall_left_open, sampled_component_evidence,
87 scan_coverage, scan_evidence,
88};
89
90#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize)]
92#[non_exhaustive]
93pub struct Sides<T> {
94 pub a: T,
96 pub b: T,
98}
99
100#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize)]
102#[non_exhaustive]
103pub struct ReportReference {
104 pub id: Option<String>,
107 pub timestamp: Option<String>,
109 pub source: String,
111 pub rows_processed: Option<usize>,
113 pub evidence: Evidence,
115 pub generator: Option<ReportGenerator>,
119 pub source_identity: Option<SourceIdentity>,
122}
123
124#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize)]
128#[non_exhaustive]
129pub struct ReportPair {
130 pub a: ReportReference,
132 pub b: ReportReference,
134 pub same_build: Option<bool>,
144}
145
146impl ReportPair {
147 fn new(a: ReportReference, b: ReportReference) -> Self {
148 let same_build = same_build(a.generator.as_ref(), b.generator.as_ref());
149 Self { a, b, same_build }
150 }
151}
152
153fn same_build(a: Option<&ReportGenerator>, b: Option<&ReportGenerator>) -> Option<bool> {
156 let (a, b) = (a?, b?);
157 if a.version != b.version {
158 return Some(false);
159 }
160 let (commit_a, commit_b) = (a.commit.as_ref()?, b.commit.as_ref()?);
161 if commit_a != commit_b {
162 return Some(false);
163 }
164 match (a.dirty?, b.dirty?) {
165 (false, false) => Some(true),
166 (true, true) => None,
167 _ => Some(false),
168 }
169}
170
171#[derive(Debug, Clone, PartialEq, serde::Serialize)]
173#[non_exhaustive]
174pub struct ScoreChange {
175 pub a: Option<f64>,
177 pub b: Option<f64>,
179 pub abs: Option<f64>,
181 pub rel_pct: Option<f64>,
184 pub evidence: Sides<Evidence>,
187}
188
189impl ScoreChange {
190 fn new(a: Option<f64>, b: Option<f64>, evidence: Sides<Evidence>) -> Self {
191 let (abs, rel_pct) = match (a, b) {
192 (Some(a), Some(b)) => (
193 Some(rounded_2(b - a)),
194 (a != 0.0).then(|| rounded_2((b - a) / a.abs() * 100.0)),
195 ),
196 _ => (None, None),
197 };
198 Self {
199 a,
200 b,
201 abs,
202 rel_pct,
203 evidence,
204 }
205 }
206}
207
208#[derive(Debug, Clone, PartialEq)]
211#[non_exhaustive]
212pub struct DimensionChange {
213 pub dimension: QualityDimension,
215 pub score: ScoreChange,
217}
218
219#[derive(Debug, Clone, PartialEq, serde::Serialize)]
223#[non_exhaustive]
224pub struct ColumnChange {
225 #[serde(skip)]
227 pub name: String,
228 pub null_pct_a: Option<f64>,
231 pub null_pct_b: Option<f64>,
233 pub null_pct_delta: Option<f64>,
236 #[serde(serialize_with = "serialize_type_name")]
243 pub data_type_a: Option<DataType>,
244 #[serde(serialize_with = "serialize_type_name")]
246 pub data_type_b: Option<DataType>,
247 pub native_type_a: Option<NativeType>,
251 pub native_type_b: Option<NativeType>,
253}
254
255fn serialize_type_name<S: Serializer>(
256 data_type: &Option<DataType>,
257 serializer: S,
258) -> Result<S::Ok, S::Error> {
259 serde::Serialize::serialize(&data_type.as_ref().map(DataType::name), serializer)
260}
261
262#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize)]
264#[non_exhaustive]
265pub struct SemanticsComparison {
266 pub a: Option<MetricSemantics>,
268 pub b: Option<MetricSemantics>,
270 pub comparable: Option<bool>,
275}
276
277#[derive(Debug, Clone, PartialEq, serde::Serialize)]
279#[non_exhaustive]
280pub struct SettingComparison<T> {
281 pub a: Option<T>,
283 pub b: Option<T>,
285 pub same: Option<bool>,
289}
290
291impl<T: PartialEq> SettingComparison<T> {
292 fn new(a: Option<T>, b: Option<T>) -> Self {
293 Self::new_by(a, b, |left, right| left == right)
294 }
295}
296
297impl<T> SettingComparison<T> {
298 fn new_by(a: Option<T>, b: Option<T>, same: impl Fn(&T, &T) -> bool) -> Self {
299 let same = match (&a, &b) {
300 (Some(left), Some(right)) => Some(same(left, right)),
301 _ => None,
302 };
303 Self { a, b, same }
304 }
305}
306
307fn same_selection(a: &ColumnSelection, b: &ColumnSelection) -> bool {
312 match (a, b) {
313 (ColumnSelection::Listed { names: left }, ColumnSelection::Listed { names: right }) => {
314 let left: HashSet<&str> = left.iter().map(String::as_str).collect();
315 let right: HashSet<&str> = right.iter().map(String::as_str).collect();
316 left == right
317 }
318 _ => a == b,
319 }
320}
321
322fn same_pairs(a: &[TemporalPair], b: &[TemporalPair]) -> bool {
327 let left: HashSet<&TemporalPair> = a.iter().collect();
328 let right: HashSet<&TemporalPair> = b.iter().collect();
329 left == right
330}
331
332#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize)]
341#[non_exhaustive]
342pub struct RecordedHints {
343 pub positive_columns: Vec<String>,
345 pub identifier_columns: Vec<String>,
347 pub temporal_columns: Vec<String>,
349 pub temporal_pair_columns: Vec<String>,
354}
355
356impl RecordedHints {
357 fn from_bindings(bindings: &[SemanticHintBinding]) -> Self {
358 let columns = |kind: SemanticHintKind| {
359 let mut names: Vec<String> = bindings
360 .iter()
361 .filter(|binding| binding.kind == kind)
362 .map(|binding| binding.column.clone())
363 .collect();
364 names.sort();
365 names
366 };
367 Self {
368 positive_columns: columns(SemanticHintKind::Positive),
369 identifier_columns: columns(SemanticHintKind::Identifier),
370 temporal_columns: columns(SemanticHintKind::Temporal),
371 temporal_pair_columns: columns(SemanticHintKind::TemporalPair),
372 }
373 }
374}
375
376#[derive(Debug, Clone, PartialEq, serde::Serialize)]
378#[non_exhaustive]
379pub struct ConfigurationComparison {
380 pub score_weights: SettingComparison<QualityScoreWeights>,
384 pub semantic_hints: SettingComparison<RecordedHints>,
387 pub pattern_locale: SettingComparison<PatternLocale>,
390 pub timeliness_clock: SettingComparison<TimelinessClock>,
396 pub columns: SettingComparison<ColumnSelection>,
401 pub metric_packs: SettingComparison<Vec<MetricPack>>,
406 pub temporal_pairs: SettingComparison<Vec<TemporalPair>>,
412}
413
414#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize)]
416#[non_exhaustive]
417pub struct SchemaChange {
418 pub added: Vec<String>,
420 pub removed: Vec<String>,
422 pub common: Vec<String>,
424}
425
426#[derive(Debug, Clone, PartialEq, serde::Serialize)]
429#[non_exhaustive]
430pub struct ReportDelta {
431 pub quality_score: ScoreChange,
433 #[serde(serialize_with = "serialize_dimensions")]
436 pub dimensions: Vec<DimensionChange>,
437 #[serde(serialize_with = "serialize_columns")]
440 pub columns: Vec<ColumnChange>,
441 pub metric_semantics: SemanticsComparison,
443 pub configuration: ConfigurationComparison,
445 pub schema: SchemaChange,
447 pub reports: ReportPair,
449}
450
451impl ReportDelta {
452 pub fn between(a: &ProfileReport, b: &ProfileReport) -> Self {
454 Self::between_sides(&ReportSide::from_report(a), &ReportSide::from_report(b))
455 }
456
457 pub fn between_sides(a: &ReportSide, b: &ReportSide) -> Self {
459 let pair = |a: Evidence, b: Evidence| Sides { a, b };
460 let quality_score = ScoreChange::new(
461 a.quality_score,
462 b.quality_score,
463 pair(a.overall_evidence, b.overall_evidence),
464 );
465 let dimensions = a
466 .dimensions
467 .iter()
468 .zip(&b.dimensions)
469 .map(|(left, right)| {
470 debug_assert_eq!(left.dimension, right.dimension);
471 DimensionChange {
472 dimension: left.dimension,
473 score: ScoreChange::new(
474 left.score,
475 right.score,
476 pair(left.evidence, right.evidence),
477 ),
478 }
479 })
480 .collect();
481
482 let a_names: HashSet<&str> = a
483 .columns
484 .iter()
485 .map(|column| column.name.as_str())
486 .collect();
487 let b_names: HashSet<&str> = b
488 .columns
489 .iter()
490 .map(|column| column.name.as_str())
491 .collect();
492 let a_by_name: BTreeMap<&str, &ColumnSide> = a
493 .columns
494 .iter()
495 .map(|column| (column.name.as_str(), column))
496 .collect();
497 let b_by_name: BTreeMap<&str, &ColumnSide> = b
498 .columns
499 .iter()
500 .map(|column| (column.name.as_str(), column))
501 .collect();
502 let added: Vec<String> = b
503 .columns
504 .iter()
505 .filter(|column| !a_names.contains(column.name.as_str()))
506 .map(|column| column.name.clone())
507 .collect();
508 let columns = a
509 .columns
510 .iter()
511 .map(|column| &column.name)
512 .chain(&added)
513 .map(|name| {
514 let side_a = a_by_name.get(name.as_str()).copied();
515 let side_b = b_by_name.get(name.as_str()).copied();
516 let null_pct_a = side_a.and_then(|column| column.null_pct);
517 let null_pct_b = side_b.and_then(|column| column.null_pct);
518 ColumnChange {
519 name: name.clone(),
520 null_pct_a,
521 null_pct_b,
522 null_pct_delta: null_pct_a.zip(null_pct_b).map(|(a, b)| rounded_2(b - a)),
523 data_type_a: side_a.and_then(ColumnSide::established_type),
524 data_type_b: side_b.and_then(ColumnSide::established_type),
525 native_type_a: side_a.and_then(|column| column.native_type.clone()),
526 native_type_b: side_b.and_then(|column| column.native_type.clone()),
527 }
528 })
529 .collect();
530 let (removed, common) = a
531 .columns
532 .iter()
533 .map(|column| column.name.clone())
534 .partition(|name| !b_names.contains(name.as_str()));
535
536 let comparable = match (&a.metric_semantics, &b.metric_semantics) {
537 (Some(left), Some(right)) if left.is_fully_recorded() && right.is_fully_recorded() => {
538 Some(left == right)
539 }
540 _ => None,
541 };
542
543 Self {
544 quality_score,
545 dimensions,
546 columns,
547 metric_semantics: SemanticsComparison {
548 a: a.metric_semantics.clone(),
549 b: b.metric_semantics.clone(),
550 comparable,
551 },
552 configuration: ConfigurationComparison {
553 score_weights: SettingComparison::new(a.score_weights, b.score_weights),
554 semantic_hints: SettingComparison::new(
555 a.semantic_hints.clone(),
556 b.semantic_hints.clone(),
557 ),
558 pattern_locale: SettingComparison::new(a.pattern_locale, b.pattern_locale),
559 timeliness_clock: SettingComparison::new(a.timeliness_clock, b.timeliness_clock),
560 columns: SettingComparison::new_by(
561 a.column_selection.clone(),
562 b.column_selection.clone(),
563 same_selection,
564 ),
565 metric_packs: SettingComparison::new(
566 a.metric_packs.clone(),
567 b.metric_packs.clone(),
568 ),
569 temporal_pairs: SettingComparison::new_by(
570 a.temporal_pairs.clone(),
571 b.temporal_pairs.clone(),
572 |left, right| same_pairs(left, right),
573 ),
574 },
575 schema: SchemaChange {
576 added,
577 removed,
578 common,
579 },
580 reports: ReportPair::new(a.reference.clone(), b.reference.clone()),
581 }
582 }
583}
584
585fn serialize_dimensions<S: Serializer>(
586 dimensions: &[DimensionChange],
587 serializer: S,
588) -> Result<S::Ok, S::Error> {
589 let mut map = serializer.serialize_map(Some(dimensions.len()))?;
590 for change in dimensions {
591 map.serialize_entry(&change.dimension.to_string(), &change.score)?;
592 }
593 map.end()
594}
595
596fn serialize_columns<S: Serializer>(
597 columns: &[ColumnChange],
598 serializer: S,
599) -> Result<S::Ok, S::Error> {
600 let mut map = serializer.serialize_map(Some(columns.len()))?;
601 for change in columns {
602 map.serialize_entry(&change.name, change)?;
603 }
604 map.end()
605}
606
607#[derive(Debug, Clone, PartialEq)]
616#[non_exhaustive]
617pub struct ReportSide {
618 reference: ReportReference,
619 quality_score: Option<f64>,
620 overall_evidence: Evidence,
621 overall_sample_evidence: Evidence,
623 dimensions: Vec<DimensionSide>,
624 score_bounds: Option<ScoreBounds>,
627 columns: Vec<ColumnSide>,
628 metric_semantics: Option<MetricSemantics>,
629 score_weights: Option<QualityScoreWeights>,
630 semantic_hints: Option<RecordedHints>,
631 identifier_order: Option<Vec<String>>,
635 pattern_locale: Option<PatternLocale>,
636 timeliness_clock: Option<TimelinessClock>,
637 column_selection: Option<ColumnSelection>,
639 metric_packs: Option<Vec<MetricPack>>,
642 temporal_pairs: Option<Vec<TemporalPair>>,
645 schema_declared: bool,
648 json_numbers: bool,
651 unlisted_components: Vec<String>,
655}
656
657fn identifier_order(bindings: &[SemanticHintBinding]) -> Vec<String> {
659 bindings
660 .iter()
661 .filter(|binding| binding.kind == SemanticHintKind::Identifier)
662 .map(|binding| binding.column.clone())
663 .collect()
664}
665
666#[derive(Debug, Clone, PartialEq)]
667struct DimensionSide {
668 dimension: QualityDimension,
669 score: Option<f64>,
670 evidence: Evidence,
671 sample_evidence: Evidence,
674}
675
676#[derive(Debug, Clone, PartialEq)]
678pub(crate) struct ColumnSide {
679 pub(crate) name: String,
680 pub(crate) null_pct: Option<f64>,
683 pub(crate) data_type: Option<DataType>,
686 pub(crate) counts: Option<(usize, usize)>,
690 pub(crate) native_type: Option<NativeType>,
692 pub(crate) unique_count: Option<usize>,
695 pub(crate) unique_count_is_approximate: Option<bool>,
698 pub(crate) quartiles_differ: bool,
701 pub(crate) max_text_length: Option<usize>,
704}
705
706const SPACED_DATETIME_MIN_LENGTH: usize = 20;
711
712impl ColumnSide {
713 fn established_type(&self) -> Option<DataType> {
717 self.populated()
718 .filter(|&populated| populated)
719 .and(self.data_type.clone())
720 }
721
722 pub(crate) fn populated(&self) -> Option<bool> {
725 self.counts.map(|(total, nulls)| nulls < total)
726 }
727
728 fn may_have_coinciding_quartiles(&self) -> bool {
735 let numeric = self
736 .data_type
737 .as_ref()
738 .is_none_or(|data_type| matches!(data_type, DataType::Integer | DataType::Float));
739 numeric && self.populated() != Some(false) && !self.quartiles_differ
740 }
741
742 fn may_hold_spaced_datetime(&self) -> bool {
750 if self.populated() == Some(false) {
751 return false;
752 }
753 match self.data_type {
754 None | Some(DataType::Date) => true,
755 Some(DataType::String | DataType::Identifier) => self
756 .max_text_length
757 .is_none_or(|length| length >= SPACED_DATETIME_MIN_LENGTH),
758 Some(DataType::Integer | DataType::Float | DataType::Boolean | DataType::Nested) => {
759 false
760 }
761 }
762 }
763
764 pub(crate) fn all_null(&self) -> Option<bool> {
767 self.counts
768 .map(|(total, nulls)| total > 0 && nulls == total)
769 }
770}
771
772fn first_of_each_name(columns: impl Iterator<Item = ColumnSide>) -> Vec<ColumnSide> {
777 let mut seen = HashSet::new();
778 columns
779 .filter(|column| seen.insert(column.name.clone()))
780 .collect()
781}
782
783fn exact_quartiles_differ(quartiles: Option<&Quartiles>, approximate: Option<bool>) -> bool {
787 approximate == Some(false) && quartiles.is_some_and(|quartiles| quartiles.q1 != quartiles.q3)
788}
789
790fn may_read_json_numbers(source: &DataSource) -> bool {
795 match source {
796 DataSource::File { format, .. } | DataSource::Bytes { format, .. } => {
797 format_may_hold_json_numbers(format)
798 }
799 DataSource::Stream { format, .. } => {
800 format.as_ref().is_none_or(format_may_hold_json_numbers)
801 }
802 DataSource::Query { .. } => true,
803 DataSource::DataFrame { .. } => false,
804 }
805}
806
807fn format_may_hold_json_numbers(format: &FileFormat) -> bool {
809 !matches!(
810 format,
811 FileFormat::Csv | FileFormat::Parquet | FileFormat::Arrow
812 )
813}
814
815fn at_precision(value: Option<f64>) -> Option<f64> {
818 value.filter(|value| value.is_finite()).map(rounded_2)
819}
820
821impl ReportSide {
822 pub fn from_report(report: &ProfileReport) -> Self {
824 let scan = scan_evidence(&report.execution);
825 let quality = report.quality.as_ref();
826 let scores = quality.map(|assessment| assessment.scores());
827 let sampled = quality.map(|assessment| assessment.sampled_dimensions());
828 let assessed = quality
829 .map(|assessment| assessment.metrics.assessed_dimensions())
830 .unwrap_or_default();
831 let unlisted = quality
832 .map(|assessment| assessment.unlisted_components())
833 .unwrap_or_default();
834 let overall_open = quality.is_some_and(|assessment| {
835 overall_left_open(
836 assessment.recorded_score_bounds(),
837 |dimension| assessment.metrics.dimension_score(dimension).is_none(),
838 Some(&assessment.metrics.score_weights),
839 )
840 });
841 let sample_evidence = |provenance| match &sampled {
844 Some(sampled) => sampled_component_evidence(
845 sampled.as_deref(),
846 &unlisted,
847 &assessed,
848 overall_open,
849 provenance,
850 ),
851 None => Evidence::Complete,
852 };
853 let evidence = |provenance| scan.and(sample_evidence(provenance));
854 let dimensions = QualityDimension::all()
855 .into_iter()
856 .map(|dimension| DimensionSide {
857 dimension,
858 score: at_precision(
859 scores
860 .as_ref()
861 .and_then(|scores| scores.dimension_scores.get(&dimension.to_string()))
862 .copied()
863 .flatten(),
864 ),
865 evidence: evidence(Provenance::Dimension(dimension)),
866 sample_evidence: sample_evidence(Provenance::Dimension(dimension)),
867 })
868 .collect();
869 Self {
870 reference: ReportReference {
871 id: Some(report.id.clone()),
872 timestamp: Some(report.timestamp.clone()),
873 source: report.source_identifier(),
874 rows_processed: Some(report.execution.rows_processed),
875 evidence: scan,
876 generator: report.generator.clone(),
877 source_identity: report.source_identity.clone(),
878 },
879 quality_score: at_precision(scores.as_ref().and_then(|scores| scores.overall_score)),
880 overall_evidence: evidence(Provenance::Overall),
881 overall_sample_evidence: sample_evidence(Provenance::Overall),
882 dimensions,
883 score_bounds: quality.and_then(QualityAssessment::score_bounds).cloned(),
884 columns: first_of_each_name(report.column_profiles.iter().map(|column| {
885 let pct = (column.total_count > 0)
886 .then(|| column.null_count as f64 / column.total_count as f64 * 100.0);
887 ColumnSide {
888 name: column.name.clone(),
889 null_pct: at_precision(pct),
890 data_type: Some(column.data_type.clone()),
891 counts: Some((column.total_count, column.null_count)),
892 native_type: column.native_type.clone(),
893 unique_count: column.unique_count,
894 unique_count_is_approximate: column.unique_count_is_approximate,
895 quartiles_differ: match &column.stats {
896 ColumnStats::Numeric(stats) => {
897 exact_quartiles_differ(stats.quartiles.as_ref(), stats.is_approximate)
898 }
899 _ => false,
900 },
901 max_text_length: match &column.stats {
902 ColumnStats::Text(stats) => Some(stats.max_length),
903 _ => None,
904 },
905 }
906 })),
907 metric_semantics: report.metric_semantics.clone(),
908 score_weights: quality.map(|assessment| assessment.metrics.score_weights),
913 semantic_hints: (report.schema_version >= 1)
916 .then(|| RecordedHints::from_bindings(&report.semantic_hint_bindings)),
917 identifier_order: (report.schema_version >= 1)
918 .then(|| identifier_order(&report.semantic_hint_bindings)),
919 pattern_locale: report
920 .analysis_settings
921 .as_ref()
922 .map(|settings| settings.pattern_locale),
923 timeliness_clock: quality
926 .and_then(|assessment| assessment.metrics.timeliness.as_ref())
927 .filter(|timeliness| timeliness.is_assessed())
928 .and_then(|timeliness| timeliness.clock),
929 column_selection: report
930 .analysis_settings
931 .as_ref()
932 .map(|settings| settings.columns.clone()),
933 metric_packs: report
934 .analysis_settings
935 .as_ref()
936 .and_then(|settings| settings.metric_packs.clone()),
937 temporal_pairs: report
938 .analysis_settings
939 .as_ref()
940 .and_then(|settings| settings.temporal_pairs.clone()),
941 schema_declared: declares_schema(&report.data_source),
942 json_numbers: may_read_json_numbers(&report.data_source),
943 unlisted_components: unlisted,
944 }
945 }
946
947 pub fn unlisted_components(&self) -> &[String] {
952 &self.unlisted_components
953 }
954
955 pub fn summary_quality_unlisted_components(quality: &str) -> serde_json::Result<Vec<String>> {
960 let quality: SummaryQuality = serde_json::from_str(quality)?;
961 Ok(quality.unlisted_components())
962 }
963
964 pub(crate) fn score_bounds(&self) -> Option<&ScoreBounds> {
967 self.score_bounds.as_ref()
968 }
969
970 pub(crate) fn identifier_order(&self) -> Option<&[String]> {
973 self.identifier_order.as_deref()
974 }
975
976 pub(crate) fn sample_evidence(&self, dimension: Option<QualityDimension>) -> Evidence {
980 match dimension {
981 None => self.overall_sample_evidence,
982 Some(dimension) => self
983 .dimensions
984 .iter()
985 .find(|side| side.dimension == dimension)
986 .map_or(
987 Evidence::Incomplete {
988 reason: EvidenceGap::CoverageUnrecorded,
989 },
990 |side| side.sample_evidence,
991 ),
992 }
993 }
994
995 pub(crate) fn may_read_json_numbers(&self) -> bool {
998 self.json_numbers
999 }
1000
1001 pub(crate) fn may_test_coinciding_quartiles(&self) -> bool {
1008 let every_value = self
1009 .sample_evidence(Some(QualityDimension::Accuracy))
1010 .is_complete();
1011 !every_value
1012 || self
1013 .columns
1014 .iter()
1015 .any(ColumnSide::may_have_coinciding_quartiles)
1016 }
1017
1018 pub(crate) fn may_hold_spaced_datetimes(&self) -> bool {
1025 let temporal = self.semantic_hints.as_ref().map(|hints| {
1026 hints
1027 .temporal_columns
1028 .iter()
1029 .chain(&hints.temporal_pair_columns)
1030 .collect::<HashSet<_>>()
1031 });
1032 self.columns.iter().any(|column| {
1033 column.may_hold_spaced_datetime()
1034 || (column.populated() != Some(false)
1035 && temporal
1036 .as_ref()
1037 .is_none_or(|names| names.contains(&column.name)))
1038 })
1039 }
1040
1041 pub(crate) fn column_selection(&self) -> Option<&ColumnSelection> {
1043 self.column_selection.as_ref()
1044 }
1045
1046 pub(crate) fn schema_evidence(&self) -> Evidence {
1052 if self.schema_declared {
1053 Evidence::Complete
1054 } else {
1055 self.reference.evidence
1056 }
1057 }
1058
1059 pub(crate) fn columns(&self) -> &[ColumnSide] {
1061 &self.columns
1062 }
1063
1064 pub fn from_summary_json(text: &str) -> serde_json::Result<Self> {
1071 let summary: Summary = serde_json::from_str(text)?;
1072 let execution = &summary.execution;
1073 let known_gap = scan_coverage(
1074 execution.source_exhausted == Some(false) || execution.truncation_reason.is_some(),
1075 execution.sampling_applied == Some(true),
1076 execution.error_count.is_some_and(|count| count > 0),
1077 );
1078 let recorded = execution.source_exhausted.is_some()
1079 && execution.sampling_applied.is_some()
1080 && execution.error_count.is_some();
1081 let scan = match known_gap {
1084 Evidence::Complete if !recorded => Evidence::Incomplete {
1085 reason: EvidenceGap::CoverageUnrecorded,
1086 },
1087 known => known,
1088 };
1089 let quality = summary.quality.as_ref();
1090 let assessed: Option<Vec<QualityDimension>> = quality.and_then(|quality| {
1095 let parsed: Vec<QualityDimension> = quality
1096 .assessed_dimensions
1097 .iter()
1098 .filter_map(|name| name.parse().ok())
1103 .collect();
1104 (!(parsed.is_empty() && quality.overall_score.is_some())).then_some(parsed)
1105 });
1106 let unlisted = quality
1107 .map(SummaryQuality::unlisted_components)
1108 .unwrap_or_default();
1109 let sample_evidence = |provenance| {
1110 let Some(quality) = quality else {
1111 return Evidence::Complete;
1112 };
1113 let sampled = quality.sampled_dimensions.as_deref();
1114 match (provenance, &assessed) {
1115 (Provenance::Overall, None)
1118 if sampled.is_some_and(|s| s.is_empty()) && unlisted.is_empty() =>
1119 {
1120 Evidence::Complete
1121 }
1122 (Provenance::Overall, None) => Evidence::Incomplete {
1123 reason: EvidenceGap::CoverageUnrecorded,
1124 },
1125 (_, assessed) => sampled_component_evidence(
1126 sampled,
1127 &unlisted,
1128 assessed.as_deref().unwrap_or_default(),
1129 overall_left_open(
1130 quality.score_bounds.as_ref(),
1131 |dimension| {
1132 quality
1133 .dimension_scores
1134 .get(&dimension.to_string())
1135 .is_none_or(Option::is_none)
1136 },
1137 quality.score_weights.as_ref(),
1138 ),
1139 provenance,
1140 ),
1141 }
1142 };
1143 let evidence = |provenance| scan.and(sample_evidence(provenance));
1144 let dimensions = QualityDimension::all()
1145 .into_iter()
1146 .map(|dimension| DimensionSide {
1147 dimension,
1148 score: at_precision(
1149 quality
1150 .and_then(|quality| quality.dimension_scores.get(&dimension.to_string()))
1151 .copied()
1152 .flatten(),
1153 ),
1154 evidence: evidence(Provenance::Dimension(dimension)),
1155 sample_evidence: sample_evidence(Provenance::Dimension(dimension)),
1156 })
1157 .collect();
1158 Ok(Self {
1159 reference: ReportReference {
1160 id: None,
1161 timestamp: None,
1162 source: summary.source,
1163 rows_processed: execution.rows_processed,
1164 evidence: scan,
1165 generator: summary.generator,
1166 source_identity: summary.source_identity,
1167 },
1168 quality_score: at_precision(quality.and_then(|quality| quality.overall_score)),
1169 overall_evidence: evidence(Provenance::Overall),
1170 overall_sample_evidence: sample_evidence(Provenance::Overall),
1171 dimensions,
1172 score_bounds: quality.and_then(|quality| quality.score_bounds.clone()),
1173 columns: first_of_each_name(summary.columns.into_iter().map(|column| ColumnSide {
1174 name: column.name,
1175 null_pct: at_precision(column.null_percentage),
1176 data_type: column.data_type.as_deref().and_then(DataType::from_name),
1179 counts: column.total_count.zip(column.null_count),
1180 native_type: column.native_type,
1181 unique_count: column.unique_count,
1182 unique_count_is_approximate: column.unique_count_is_approximate,
1183 quartiles_differ: column.stats.as_ref().is_some_and(summary_quartiles_differ),
1184 max_text_length: column.stats.as_ref().and_then(summary_max_text_length),
1185 })),
1186 score_weights: quality.and_then(|quality| {
1189 quality.score_weights.or_else(|| {
1190 summary
1191 .metric_semantics
1192 .is_some()
1193 .then(QualityScoreWeights::default)
1194 })
1195 }),
1196 semantic_hints: summary
1197 .schema_version
1198 .is_some_and(|version| version >= 1)
1199 .then(|| RecordedHints::from_bindings(&summary.semantic_hint_bindings)),
1200 identifier_order: summary
1201 .schema_version
1202 .is_some_and(|version| version >= 1)
1203 .then(|| identifier_order(&summary.semantic_hint_bindings)),
1204 metric_semantics: summary.metric_semantics,
1205 pattern_locale: summary
1206 .analysis_settings
1207 .as_ref()
1208 .map(|settings| settings.pattern_locale),
1209 timeliness_clock: quality
1210 .and_then(|quality| quality.timeliness.as_ref())
1211 .and_then(|timeliness| timeliness.clock),
1212 metric_packs: summary
1213 .analysis_settings
1214 .as_ref()
1215 .and_then(|settings| settings.metric_packs.clone()),
1216 temporal_pairs: summary
1217 .analysis_settings
1218 .as_ref()
1219 .and_then(|settings| settings.temporal_pairs.clone()),
1220 column_selection: summary.analysis_settings.map(|settings| settings.columns),
1221 schema_declared: match summary.source_type {
1226 Some(SummarySourceType::Dataframe | SummarySourceType::Query) => true,
1227 Some(SummarySourceType::File | SummarySourceType::Bytes) => summary
1228 .source_format
1229 .as_ref()
1230 .is_some_and(file_format_declares_schema),
1231 Some(SummarySourceType::Stream) => summary
1232 .source_format
1233 .as_ref()
1234 .is_some_and(stream_format_declares_schema),
1235 Some(SummarySourceType::Other) | None => false,
1236 },
1237 json_numbers: match summary.source_type {
1240 Some(SummarySourceType::Dataframe) => false,
1241 Some(
1242 SummarySourceType::File | SummarySourceType::Bytes | SummarySourceType::Stream,
1243 ) => summary
1244 .source_format
1245 .as_ref()
1246 .is_none_or(format_may_hold_json_numbers),
1247 Some(SummarySourceType::Query | SummarySourceType::Other) | None => true,
1248 },
1249 unlisted_components: unlisted.clone(),
1250 })
1251 }
1252}
1253
1254fn summary_metric_semantics<'de, D>(deserializer: D) -> Result<Option<MetricSemantics>, D::Error>
1257where
1258 D: serde::Deserializer<'de>,
1259{
1260 crate::profile_report::present_value(deserializer).map_err(|error| {
1261 <D::Error as serde::de::Error>::custom(format!("metric_semantics: {error}"))
1262 })
1263}
1264
1265fn summary_schema_version<'de, D>(deserializer: D) -> Result<Option<u32>, D::Error>
1266where
1267 D: serde::Deserializer<'de>,
1268{
1269 crate::profile_report::present_value(deserializer)
1270 .map_err(|error| <D::Error as serde::de::Error>::custom(format!("schema_version: {error}")))
1271}
1272
1273fn summary_analysis_settings<'de, D>(deserializer: D) -> Result<Option<AnalysisSettings>, D::Error>
1275where
1276 D: serde::Deserializer<'de>,
1277{
1278 crate::profile_report::present_value(deserializer).map_err(|error| {
1279 <D::Error as serde::de::Error>::custom(format!("analysis_settings: {error}"))
1280 })
1281}
1282
1283fn summary_score_weights<'de, D>(deserializer: D) -> Result<Option<QualityScoreWeights>, D::Error>
1285where
1286 D: serde::Deserializer<'de>,
1287{
1288 crate::profile_report::present_value(deserializer).map_err(|error| {
1289 <D::Error as serde::de::Error>::custom(format!("quality.score_weights: {error}"))
1290 })
1291}
1292
1293#[derive(serde::Deserialize)]
1296struct Summary {
1297 source: String,
1298 #[serde(default)]
1300 source_type: Option<SummarySourceType>,
1301 #[serde(default)]
1304 source_format: Option<FileFormat>,
1305 execution: SummaryExecution,
1306 columns: Vec<SummaryColumn>,
1307 quality: Option<SummaryQuality>,
1308 #[serde(default, deserialize_with = "summary_metric_semantics")]
1309 metric_semantics: Option<MetricSemantics>,
1310 #[serde(default, deserialize_with = "summary_schema_version")]
1312 schema_version: Option<u32>,
1313 #[serde(default)]
1314 semantic_hint_bindings: Vec<SemanticHintBinding>,
1315 #[serde(default, deserialize_with = "summary_analysis_settings")]
1318 analysis_settings: Option<AnalysisSettings>,
1319 #[serde(default, deserialize_with = "summary_generator")]
1322 generator: Option<ReportGenerator>,
1323 #[serde(default, deserialize_with = "summary_source_identity")]
1326 source_identity: Option<SourceIdentity>,
1327}
1328
1329fn summary_generator<'de, D>(deserializer: D) -> Result<Option<ReportGenerator>, D::Error>
1330where
1331 D: serde::Deserializer<'de>,
1332{
1333 crate::profile_report::present_value(deserializer)
1334 .map_err(|error| <D::Error as serde::de::Error>::custom(format!("generator: {error}")))
1335}
1336
1337fn summary_source_identity<'de, D>(deserializer: D) -> Result<Option<SourceIdentity>, D::Error>
1338where
1339 D: serde::Deserializer<'de>,
1340{
1341 crate::profile_report::present_value(deserializer).map_err(|error| {
1342 <D::Error as serde::de::Error>::custom(format!("source_identity: {error}"))
1343 })
1344}
1345
1346#[derive(serde::Deserialize)]
1349#[serde(rename_all = "lowercase")]
1350enum SummarySourceType {
1351 File,
1352 Bytes,
1353 Stream,
1354 Dataframe,
1355 Query,
1356 #[serde(other)]
1357 Other,
1358}
1359
1360fn declares_schema(source: &DataSource) -> bool {
1366 match source {
1367 DataSource::File { format, .. } | DataSource::Bytes { format, .. } => {
1368 file_format_declares_schema(format)
1369 }
1370 DataSource::Stream {
1371 format: Some(format),
1372 ..
1373 } => stream_format_declares_schema(format),
1374 DataSource::DataFrame { .. } | DataSource::Query { .. } => true,
1375 DataSource::Stream { format: None, .. } => false,
1376 }
1377}
1378
1379fn file_format_declares_schema(format: &FileFormat) -> bool {
1383 matches!(format, FileFormat::Csv | FileFormat::Parquet)
1384}
1385
1386fn stream_format_declares_schema(format: &FileFormat) -> bool {
1388 matches!(
1389 format,
1390 FileFormat::Csv | FileFormat::Parquet | FileFormat::Arrow
1391 )
1392}
1393
1394#[derive(serde::Deserialize)]
1395struct SummaryExecution {
1396 rows_processed: Option<usize>,
1397 source_exhausted: Option<bool>,
1398 truncation_reason: Option<String>,
1399 sampling_applied: Option<bool>,
1400 error_count: Option<usize>,
1401}
1402
1403#[derive(serde::Deserialize)]
1404struct SummaryColumn {
1405 name: String,
1406 null_percentage: Option<f64>,
1407 #[serde(default)]
1408 data_type: Option<String>,
1409 #[serde(default)]
1410 total_count: Option<usize>,
1411 #[serde(default)]
1412 null_count: Option<usize>,
1413 #[serde(default)]
1415 native_type: Option<NativeType>,
1416 #[serde(default)]
1417 unique_count: Option<usize>,
1418 #[serde(default)]
1421 unique_count_is_approximate: Option<bool>,
1422 #[serde(default)]
1425 stats: Option<serde_json::Value>,
1426}
1427
1428fn summary_quartiles_differ(stats: &serde_json::Value) -> bool {
1431 let quartile = |name: &str| stats.get("quartiles")?.get(name)?.as_f64();
1432 stats
1433 .get("is_approximate")
1434 .and_then(serde_json::Value::as_bool)
1435 == Some(false)
1436 && quartile("q1")
1437 .zip(quartile("q3"))
1438 .is_some_and(|(q1, q3)| q1 != q3)
1439}
1440
1441fn summary_max_text_length(stats: &serde_json::Value) -> Option<usize> {
1444 stats
1445 .get("max_length")?
1446 .as_u64()
1447 .and_then(|length| usize::try_from(length).ok())
1448}
1449
1450#[derive(serde::Deserialize)]
1451struct SummaryQuality {
1452 overall_score: Option<f64>,
1453 #[serde(default)]
1454 dimension_scores: BTreeMap<String, Option<f64>>,
1455 #[serde(default)]
1456 assessed_dimensions: Vec<String>,
1457 #[serde(default)]
1460 exact_dimensions: Option<Vec<String>>,
1461 sampled_dimensions: Option<Vec<String>>,
1464 #[serde(default)]
1468 completeness: Option<serde::de::IgnoredAny>,
1469 #[serde(default)]
1470 consistency: Option<serde::de::IgnoredAny>,
1471 #[serde(default)]
1472 uniqueness: Option<SummaryUniqueness>,
1473 #[serde(default)]
1474 accuracy: Option<serde::de::IgnoredAny>,
1475 #[serde(default)]
1476 timeliness: Option<SummaryTimeliness>,
1477 #[serde(default)]
1478 validity: Option<serde::de::IgnoredAny>,
1479 #[serde(default)]
1480 precision: Option<serde::de::IgnoredAny>,
1481 #[serde(default, deserialize_with = "summary_score_weights")]
1485 score_weights: Option<QualityScoreWeights>,
1486 #[serde(default, deserialize_with = "summary_score_bounds")]
1490 score_bounds: Option<ScoreBounds>,
1491}
1492
1493fn summary_score_bounds<'de, D>(deserializer: D) -> Result<Option<ScoreBounds>, D::Error>
1495where
1496 D: serde::Deserializer<'de>,
1497{
1498 let bounds: Option<ScoreBounds> = serde::Deserialize::deserialize(deserializer)?;
1499 if bounds.as_ref().is_some_and(|bounds| !bounds.is_valid()) {
1500 return Err(<D::Error as serde::de::Error>::custom(
1501 "quality.score_bounds: intervals must lie in 0..=100 with lower <= upper, at a \
1502 confidence level between 0 and 1",
1503 ));
1504 }
1505 Ok(bounds)
1506}
1507
1508#[derive(serde::Deserialize)]
1510struct SummaryTimeliness {
1511 #[serde(default, deserialize_with = "summary_timeliness_clock")]
1513 clock: Option<TimelinessClock>,
1514}
1515
1516fn summary_timeliness_clock<'de, D>(deserializer: D) -> Result<Option<TimelinessClock>, D::Error>
1518where
1519 D: serde::Deserializer<'de>,
1520{
1521 crate::profile_report::present_value(deserializer).map_err(|error| {
1522 <D::Error as serde::de::Error>::custom(format!("quality.timeliness.clock: {error}"))
1523 })
1524}
1525
1526#[derive(serde::Deserialize)]
1529struct SummaryUniqueness {
1530 #[serde(default)]
1531 rows_checked: usize,
1532 #[serde(default)]
1533 key_column: Option<String>,
1534 #[serde(default)]
1535 key_values_checked: Option<usize>,
1536}
1537
1538impl SummaryQuality {
1539 fn unlisted_components(&self) -> Vec<String> {
1544 match (&self.exact_dimensions, &self.sampled_dimensions) {
1545 (Some(exact), Some(sampled)) => {
1546 self.signal_metrics().unlisted_components(exact, sampled)
1547 }
1548 _ => Vec::new(),
1549 }
1550 }
1551
1552 fn signal_metrics(&self) -> QualityMetrics {
1556 QualityMetrics {
1557 completeness: self
1558 .completeness
1559 .as_ref()
1560 .map(|_| CompletenessMetrics::default()),
1561 consistency: self
1562 .consistency
1563 .as_ref()
1564 .map(|_| ConsistencyMetrics::default()),
1565 uniqueness: self
1566 .uniqueness
1567 .as_ref()
1568 .map(|uniqueness| UniquenessMetrics {
1569 rows_checked: uniqueness.rows_checked,
1570 key_column: uniqueness.key_column.clone(),
1571 key_values_checked: uniqueness.key_values_checked,
1572 ..UniquenessMetrics::default()
1573 }),
1574 accuracy: self.accuracy.as_ref().map(|_| AccuracyMetrics::default()),
1575 timeliness: self
1576 .timeliness
1577 .as_ref()
1578 .map(|_| TimelinessMetrics::default()),
1579 validity: self.validity.as_ref().map(|_| ValidityMetrics::default()),
1580 precision: self.precision.as_ref().map(|_| PrecisionMetrics::default()),
1581 ..QualityMetrics::default()
1582 }
1583 }
1584}
1585
1586#[cfg(test)]
1587mod tests {
1588 use super::*;
1589
1590 fn complete() -> Sides<Evidence> {
1591 Sides {
1592 a: Evidence::Complete,
1593 b: Evidence::Complete,
1594 }
1595 }
1596
1597 #[test]
1598 fn a_zero_baseline_has_an_absolute_change_and_no_relative_one() {
1599 let change = ScoreChange::new(Some(0.0), Some(50.0), complete());
1600 assert_eq!(change.abs, Some(50.0));
1601 assert_eq!(change.rel_pct, None);
1602 }
1603
1604 #[test]
1605 fn a_change_is_rounded_to_the_serialized_precision() {
1606 let change = ScoreChange::new(Some(80.0), Some(80.1), complete());
1610 assert_eq!(change.abs, Some(0.1));
1611 assert_eq!(change.rel_pct, Some(0.12));
1612 }
1613
1614 #[test]
1615 fn a_missing_side_leaves_both_changes_absent() {
1616 for (a, b) in [(None, Some(1.0)), (Some(1.0), None), (None, None)] {
1617 let change = ScoreChange::new(a, b, complete());
1618 assert_eq!((change.abs, change.rel_pct), (None, None));
1619 }
1620 }
1621
1622 #[test]
1623 fn a_non_finite_input_is_no_number() {
1624 assert_eq!(at_precision(Some(f64::NAN)), None);
1625 assert_eq!(at_precision(Some(f64::INFINITY)), None);
1626 assert_eq!(at_precision(Some(12.345)), Some(12.35));
1627 }
1628}