use std::collections::{BTreeMap, HashSet};
use dataprof_core::serde_helpers::rounded_2;
use dataprof_core::{
ColumnStats, DataSource, DataType, FileFormat, MetricPack, NativeType, QualityDimension,
QualityScoreWeights, Quartiles, SemanticHintBinding, SemanticHintKind,
};
use dataprof_metrics::{
AccuracyMetrics, CompletenessMetrics, ConsistencyMetrics, PrecisionMetrics, QualityAssessment,
QualityMetrics, ScoreBounds, TimelinessClock, TimelinessMetrics, UniquenessMetrics,
ValidityMetrics,
};
use serde::ser::{SerializeMap, Serializer};
use crate::profile_report::{
AnalysisSettings, ColumnSelection, MetricSemantics, PatternLocale, ProfileReport,
ReportGenerator, SourceIdentity, TemporalPair,
};
use crate::quality_gate::{
Evidence, EvidenceGap, Provenance, overall_left_open, sampled_component_evidence,
scan_coverage, scan_evidence,
};
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize)]
#[non_exhaustive]
pub struct Sides<T> {
pub a: T,
pub b: T,
}
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize)]
#[non_exhaustive]
pub struct ReportReference {
pub id: Option<String>,
pub timestamp: Option<String>,
pub source: String,
pub rows_processed: Option<usize>,
pub evidence: Evidence,
pub generator: Option<ReportGenerator>,
pub source_identity: Option<SourceIdentity>,
}
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize)]
#[non_exhaustive]
pub struct ReportPair {
pub a: ReportReference,
pub b: ReportReference,
pub same_build: Option<bool>,
}
impl ReportPair {
fn new(a: ReportReference, b: ReportReference) -> Self {
let same_build = same_build(a.generator.as_ref(), b.generator.as_ref());
Self { a, b, same_build }
}
}
fn same_build(a: Option<&ReportGenerator>, b: Option<&ReportGenerator>) -> Option<bool> {
let (a, b) = (a?, b?);
if a.version != b.version {
return Some(false);
}
let (commit_a, commit_b) = (a.commit.as_ref()?, b.commit.as_ref()?);
if commit_a != commit_b {
return Some(false);
}
match (a.dirty?, b.dirty?) {
(false, false) => Some(true),
(true, true) => None,
_ => Some(false),
}
}
#[derive(Debug, Clone, PartialEq, serde::Serialize)]
#[non_exhaustive]
pub struct ScoreChange {
pub a: Option<f64>,
pub b: Option<f64>,
pub abs: Option<f64>,
pub rel_pct: Option<f64>,
pub evidence: Sides<Evidence>,
}
impl ScoreChange {
fn new(a: Option<f64>, b: Option<f64>, evidence: Sides<Evidence>) -> Self {
let (abs, rel_pct) = match (a, b) {
(Some(a), Some(b)) => (
Some(rounded_2(b - a)),
(a != 0.0).then(|| rounded_2((b - a) / a.abs() * 100.0)),
),
_ => (None, None),
};
Self {
a,
b,
abs,
rel_pct,
evidence,
}
}
}
#[derive(Debug, Clone, PartialEq)]
#[non_exhaustive]
pub struct DimensionChange {
pub dimension: QualityDimension,
pub score: ScoreChange,
}
#[derive(Debug, Clone, PartialEq, serde::Serialize)]
#[non_exhaustive]
pub struct ColumnChange {
#[serde(skip)]
pub name: String,
pub null_pct_a: Option<f64>,
pub null_pct_b: Option<f64>,
pub null_pct_delta: Option<f64>,
#[serde(serialize_with = "serialize_type_name")]
pub data_type_a: Option<DataType>,
#[serde(serialize_with = "serialize_type_name")]
pub data_type_b: Option<DataType>,
pub native_type_a: Option<NativeType>,
pub native_type_b: Option<NativeType>,
}
fn serialize_type_name<S: Serializer>(
data_type: &Option<DataType>,
serializer: S,
) -> Result<S::Ok, S::Error> {
serde::Serialize::serialize(&data_type.as_ref().map(DataType::name), serializer)
}
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize)]
#[non_exhaustive]
pub struct SemanticsComparison {
pub a: Option<MetricSemantics>,
pub b: Option<MetricSemantics>,
pub comparable: Option<bool>,
}
#[derive(Debug, Clone, PartialEq, serde::Serialize)]
#[non_exhaustive]
pub struct SettingComparison<T> {
pub a: Option<T>,
pub b: Option<T>,
pub same: Option<bool>,
}
impl<T: PartialEq> SettingComparison<T> {
fn new(a: Option<T>, b: Option<T>) -> Self {
Self::new_by(a, b, |left, right| left == right)
}
}
impl<T> SettingComparison<T> {
fn new_by(a: Option<T>, b: Option<T>, same: impl Fn(&T, &T) -> bool) -> Self {
let same = match (&a, &b) {
(Some(left), Some(right)) => Some(same(left, right)),
_ => None,
};
Self { a, b, same }
}
}
fn same_selection(a: &ColumnSelection, b: &ColumnSelection) -> bool {
match (a, b) {
(ColumnSelection::Listed { names: left }, ColumnSelection::Listed { names: right }) => {
let left: HashSet<&str> = left.iter().map(String::as_str).collect();
let right: HashSet<&str> = right.iter().map(String::as_str).collect();
left == right
}
_ => a == b,
}
}
fn same_pairs(a: &[TemporalPair], b: &[TemporalPair]) -> bool {
let left: HashSet<&TemporalPair> = a.iter().collect();
let right: HashSet<&TemporalPair> = b.iter().collect();
left == right
}
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize)]
#[non_exhaustive]
pub struct RecordedHints {
pub positive_columns: Vec<String>,
pub identifier_columns: Vec<String>,
pub temporal_columns: Vec<String>,
pub temporal_pair_columns: Vec<String>,
}
impl RecordedHints {
fn from_bindings(bindings: &[SemanticHintBinding]) -> Self {
let columns = |kind: SemanticHintKind| {
let mut names: Vec<String> = bindings
.iter()
.filter(|binding| binding.kind == kind)
.map(|binding| binding.column.clone())
.collect();
names.sort();
names
};
Self {
positive_columns: columns(SemanticHintKind::Positive),
identifier_columns: columns(SemanticHintKind::Identifier),
temporal_columns: columns(SemanticHintKind::Temporal),
temporal_pair_columns: columns(SemanticHintKind::TemporalPair),
}
}
}
#[derive(Debug, Clone, PartialEq, serde::Serialize)]
#[non_exhaustive]
pub struct ConfigurationComparison {
pub score_weights: SettingComparison<QualityScoreWeights>,
pub semantic_hints: SettingComparison<RecordedHints>,
pub pattern_locale: SettingComparison<PatternLocale>,
pub timeliness_clock: SettingComparison<TimelinessClock>,
pub columns: SettingComparison<ColumnSelection>,
pub metric_packs: SettingComparison<Vec<MetricPack>>,
pub temporal_pairs: SettingComparison<Vec<TemporalPair>>,
}
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize)]
#[non_exhaustive]
pub struct SchemaChange {
pub added: Vec<String>,
pub removed: Vec<String>,
pub common: Vec<String>,
}
#[derive(Debug, Clone, PartialEq, serde::Serialize)]
#[non_exhaustive]
pub struct ReportDelta {
pub quality_score: ScoreChange,
#[serde(serialize_with = "serialize_dimensions")]
pub dimensions: Vec<DimensionChange>,
#[serde(serialize_with = "serialize_columns")]
pub columns: Vec<ColumnChange>,
pub metric_semantics: SemanticsComparison,
pub configuration: ConfigurationComparison,
pub schema: SchemaChange,
pub reports: ReportPair,
}
impl ReportDelta {
pub fn between(a: &ProfileReport, b: &ProfileReport) -> Self {
Self::between_sides(&ReportSide::from_report(a), &ReportSide::from_report(b))
}
pub fn between_sides(a: &ReportSide, b: &ReportSide) -> Self {
let pair = |a: Evidence, b: Evidence| Sides { a, b };
let quality_score = ScoreChange::new(
a.quality_score,
b.quality_score,
pair(a.overall_evidence, b.overall_evidence),
);
let dimensions = a
.dimensions
.iter()
.zip(&b.dimensions)
.map(|(left, right)| {
debug_assert_eq!(left.dimension, right.dimension);
DimensionChange {
dimension: left.dimension,
score: ScoreChange::new(
left.score,
right.score,
pair(left.evidence, right.evidence),
),
}
})
.collect();
let a_names: HashSet<&str> = a
.columns
.iter()
.map(|column| column.name.as_str())
.collect();
let b_names: HashSet<&str> = b
.columns
.iter()
.map(|column| column.name.as_str())
.collect();
let a_by_name: BTreeMap<&str, &ColumnSide> = a
.columns
.iter()
.map(|column| (column.name.as_str(), column))
.collect();
let b_by_name: BTreeMap<&str, &ColumnSide> = b
.columns
.iter()
.map(|column| (column.name.as_str(), column))
.collect();
let added: Vec<String> = b
.columns
.iter()
.filter(|column| !a_names.contains(column.name.as_str()))
.map(|column| column.name.clone())
.collect();
let columns = a
.columns
.iter()
.map(|column| &column.name)
.chain(&added)
.map(|name| {
let side_a = a_by_name.get(name.as_str()).copied();
let side_b = b_by_name.get(name.as_str()).copied();
let null_pct_a = side_a.and_then(|column| column.null_pct);
let null_pct_b = side_b.and_then(|column| column.null_pct);
ColumnChange {
name: name.clone(),
null_pct_a,
null_pct_b,
null_pct_delta: null_pct_a.zip(null_pct_b).map(|(a, b)| rounded_2(b - a)),
data_type_a: side_a.and_then(ColumnSide::established_type),
data_type_b: side_b.and_then(ColumnSide::established_type),
native_type_a: side_a.and_then(|column| column.native_type.clone()),
native_type_b: side_b.and_then(|column| column.native_type.clone()),
}
})
.collect();
let (removed, common) = a
.columns
.iter()
.map(|column| column.name.clone())
.partition(|name| !b_names.contains(name.as_str()));
let comparable = match (&a.metric_semantics, &b.metric_semantics) {
(Some(left), Some(right)) if left.is_fully_recorded() && right.is_fully_recorded() => {
Some(left == right)
}
_ => None,
};
Self {
quality_score,
dimensions,
columns,
metric_semantics: SemanticsComparison {
a: a.metric_semantics.clone(),
b: b.metric_semantics.clone(),
comparable,
},
configuration: ConfigurationComparison {
score_weights: SettingComparison::new(a.score_weights, b.score_weights),
semantic_hints: SettingComparison::new(
a.semantic_hints.clone(),
b.semantic_hints.clone(),
),
pattern_locale: SettingComparison::new(a.pattern_locale, b.pattern_locale),
timeliness_clock: SettingComparison::new(a.timeliness_clock, b.timeliness_clock),
columns: SettingComparison::new_by(
a.column_selection.clone(),
b.column_selection.clone(),
same_selection,
),
metric_packs: SettingComparison::new(
a.metric_packs.clone(),
b.metric_packs.clone(),
),
temporal_pairs: SettingComparison::new_by(
a.temporal_pairs.clone(),
b.temporal_pairs.clone(),
|left, right| same_pairs(left, right),
),
},
schema: SchemaChange {
added,
removed,
common,
},
reports: ReportPair::new(a.reference.clone(), b.reference.clone()),
}
}
}
fn serialize_dimensions<S: Serializer>(
dimensions: &[DimensionChange],
serializer: S,
) -> Result<S::Ok, S::Error> {
let mut map = serializer.serialize_map(Some(dimensions.len()))?;
for change in dimensions {
map.serialize_entry(&change.dimension.to_string(), &change.score)?;
}
map.end()
}
fn serialize_columns<S: Serializer>(
columns: &[ColumnChange],
serializer: S,
) -> Result<S::Ok, S::Error> {
let mut map = serializer.serialize_map(Some(columns.len()))?;
for change in columns {
map.serialize_entry(&change.name, change)?;
}
map.end()
}
#[derive(Debug, Clone, PartialEq)]
#[non_exhaustive]
pub struct ReportSide {
reference: ReportReference,
quality_score: Option<f64>,
overall_evidence: Evidence,
overall_sample_evidence: Evidence,
dimensions: Vec<DimensionSide>,
score_bounds: Option<ScoreBounds>,
columns: Vec<ColumnSide>,
metric_semantics: Option<MetricSemantics>,
score_weights: Option<QualityScoreWeights>,
semantic_hints: Option<RecordedHints>,
identifier_order: Option<Vec<String>>,
pattern_locale: Option<PatternLocale>,
timeliness_clock: Option<TimelinessClock>,
column_selection: Option<ColumnSelection>,
metric_packs: Option<Vec<MetricPack>>,
temporal_pairs: Option<Vec<TemporalPair>>,
schema_declared: bool,
json_numbers: bool,
unlisted_components: Vec<String>,
}
fn identifier_order(bindings: &[SemanticHintBinding]) -> Vec<String> {
bindings
.iter()
.filter(|binding| binding.kind == SemanticHintKind::Identifier)
.map(|binding| binding.column.clone())
.collect()
}
#[derive(Debug, Clone, PartialEq)]
struct DimensionSide {
dimension: QualityDimension,
score: Option<f64>,
evidence: Evidence,
sample_evidence: Evidence,
}
#[derive(Debug, Clone, PartialEq)]
pub(crate) struct ColumnSide {
pub(crate) name: String,
pub(crate) null_pct: Option<f64>,
pub(crate) data_type: Option<DataType>,
pub(crate) counts: Option<(usize, usize)>,
pub(crate) native_type: Option<NativeType>,
pub(crate) unique_count: Option<usize>,
pub(crate) unique_count_is_approximate: Option<bool>,
pub(crate) quartiles_differ: bool,
pub(crate) max_text_length: Option<usize>,
}
const SPACED_DATETIME_MIN_LENGTH: usize = 20;
impl ColumnSide {
fn established_type(&self) -> Option<DataType> {
self.populated()
.filter(|&populated| populated)
.and(self.data_type.clone())
}
pub(crate) fn populated(&self) -> Option<bool> {
self.counts.map(|(total, nulls)| nulls < total)
}
fn may_have_coinciding_quartiles(&self) -> bool {
let numeric = self
.data_type
.as_ref()
.is_none_or(|data_type| matches!(data_type, DataType::Integer | DataType::Float));
numeric && self.populated() != Some(false) && !self.quartiles_differ
}
fn may_hold_spaced_datetime(&self) -> bool {
if self.populated() == Some(false) {
return false;
}
match self.data_type {
None | Some(DataType::Date) => true,
Some(DataType::String | DataType::Identifier) => self
.max_text_length
.is_none_or(|length| length >= SPACED_DATETIME_MIN_LENGTH),
Some(DataType::Integer | DataType::Float | DataType::Boolean | DataType::Nested) => {
false
}
}
}
pub(crate) fn all_null(&self) -> Option<bool> {
self.counts
.map(|(total, nulls)| total > 0 && nulls == total)
}
}
fn first_of_each_name(columns: impl Iterator<Item = ColumnSide>) -> Vec<ColumnSide> {
let mut seen = HashSet::new();
columns
.filter(|column| seen.insert(column.name.clone()))
.collect()
}
fn exact_quartiles_differ(quartiles: Option<&Quartiles>, approximate: Option<bool>) -> bool {
approximate == Some(false) && quartiles.is_some_and(|quartiles| quartiles.q1 != quartiles.q3)
}
fn may_read_json_numbers(source: &DataSource) -> bool {
match source {
DataSource::File { format, .. } | DataSource::Bytes { format, .. } => {
format_may_hold_json_numbers(format)
}
DataSource::Stream { format, .. } => {
format.as_ref().is_none_or(format_may_hold_json_numbers)
}
DataSource::Query { .. } => true,
DataSource::DataFrame { .. } => false,
}
}
fn format_may_hold_json_numbers(format: &FileFormat) -> bool {
!matches!(
format,
FileFormat::Csv | FileFormat::Parquet | FileFormat::Arrow
)
}
fn at_precision(value: Option<f64>) -> Option<f64> {
value.filter(|value| value.is_finite()).map(rounded_2)
}
impl ReportSide {
pub fn from_report(report: &ProfileReport) -> Self {
let scan = scan_evidence(&report.execution);
let quality = report.quality.as_ref();
let scores = quality.map(|assessment| assessment.scores());
let sampled = quality.map(|assessment| assessment.sampled_dimensions());
let assessed = quality
.map(|assessment| assessment.metrics.assessed_dimensions())
.unwrap_or_default();
let unlisted = quality
.map(|assessment| assessment.unlisted_components())
.unwrap_or_default();
let overall_open = quality.is_some_and(|assessment| {
overall_left_open(
assessment.recorded_score_bounds(),
|dimension| assessment.metrics.dimension_score(dimension).is_none(),
Some(&assessment.metrics.score_weights),
)
});
let sample_evidence = |provenance| match &sampled {
Some(sampled) => sampled_component_evidence(
sampled.as_deref(),
&unlisted,
&assessed,
overall_open,
provenance,
),
None => Evidence::Complete,
};
let evidence = |provenance| scan.and(sample_evidence(provenance));
let dimensions = QualityDimension::all()
.into_iter()
.map(|dimension| DimensionSide {
dimension,
score: at_precision(
scores
.as_ref()
.and_then(|scores| scores.dimension_scores.get(&dimension.to_string()))
.copied()
.flatten(),
),
evidence: evidence(Provenance::Dimension(dimension)),
sample_evidence: sample_evidence(Provenance::Dimension(dimension)),
})
.collect();
Self {
reference: ReportReference {
id: Some(report.id.clone()),
timestamp: Some(report.timestamp.clone()),
source: report.source_identifier(),
rows_processed: Some(report.execution.rows_processed),
evidence: scan,
generator: report.generator.clone(),
source_identity: report.source_identity.clone(),
},
quality_score: at_precision(scores.as_ref().and_then(|scores| scores.overall_score)),
overall_evidence: evidence(Provenance::Overall),
overall_sample_evidence: sample_evidence(Provenance::Overall),
dimensions,
score_bounds: quality.and_then(QualityAssessment::score_bounds).cloned(),
columns: first_of_each_name(report.column_profiles.iter().map(|column| {
let pct = (column.total_count > 0)
.then(|| column.null_count as f64 / column.total_count as f64 * 100.0);
ColumnSide {
name: column.name.clone(),
null_pct: at_precision(pct),
data_type: Some(column.data_type.clone()),
counts: Some((column.total_count, column.null_count)),
native_type: column.native_type.clone(),
unique_count: column.unique_count,
unique_count_is_approximate: column.unique_count_is_approximate,
quartiles_differ: match &column.stats {
ColumnStats::Numeric(stats) => {
exact_quartiles_differ(stats.quartiles.as_ref(), stats.is_approximate)
}
_ => false,
},
max_text_length: match &column.stats {
ColumnStats::Text(stats) => Some(stats.max_length),
_ => None,
},
}
})),
metric_semantics: report.metric_semantics.clone(),
score_weights: quality.map(|assessment| assessment.metrics.score_weights),
semantic_hints: (report.schema_version >= 1)
.then(|| RecordedHints::from_bindings(&report.semantic_hint_bindings)),
identifier_order: (report.schema_version >= 1)
.then(|| identifier_order(&report.semantic_hint_bindings)),
pattern_locale: report
.analysis_settings
.as_ref()
.map(|settings| settings.pattern_locale),
timeliness_clock: quality
.and_then(|assessment| assessment.metrics.timeliness.as_ref())
.filter(|timeliness| timeliness.is_assessed())
.and_then(|timeliness| timeliness.clock),
column_selection: report
.analysis_settings
.as_ref()
.map(|settings| settings.columns.clone()),
metric_packs: report
.analysis_settings
.as_ref()
.and_then(|settings| settings.metric_packs.clone()),
temporal_pairs: report
.analysis_settings
.as_ref()
.and_then(|settings| settings.temporal_pairs.clone()),
schema_declared: declares_schema(&report.data_source),
json_numbers: may_read_json_numbers(&report.data_source),
unlisted_components: unlisted,
}
}
pub fn unlisted_components(&self) -> &[String] {
&self.unlisted_components
}
pub fn summary_quality_unlisted_components(quality: &str) -> serde_json::Result<Vec<String>> {
let quality: SummaryQuality = serde_json::from_str(quality)?;
Ok(quality.unlisted_components())
}
pub(crate) fn score_bounds(&self) -> Option<&ScoreBounds> {
self.score_bounds.as_ref()
}
pub(crate) fn identifier_order(&self) -> Option<&[String]> {
self.identifier_order.as_deref()
}
pub(crate) fn sample_evidence(&self, dimension: Option<QualityDimension>) -> Evidence {
match dimension {
None => self.overall_sample_evidence,
Some(dimension) => self
.dimensions
.iter()
.find(|side| side.dimension == dimension)
.map_or(
Evidence::Incomplete {
reason: EvidenceGap::CoverageUnrecorded,
},
|side| side.sample_evidence,
),
}
}
pub(crate) fn may_read_json_numbers(&self) -> bool {
self.json_numbers
}
pub(crate) fn may_test_coinciding_quartiles(&self) -> bool {
let every_value = self
.sample_evidence(Some(QualityDimension::Accuracy))
.is_complete();
!every_value
|| self
.columns
.iter()
.any(ColumnSide::may_have_coinciding_quartiles)
}
pub(crate) fn may_hold_spaced_datetimes(&self) -> bool {
let temporal = self.semantic_hints.as_ref().map(|hints| {
hints
.temporal_columns
.iter()
.chain(&hints.temporal_pair_columns)
.collect::<HashSet<_>>()
});
self.columns.iter().any(|column| {
column.may_hold_spaced_datetime()
|| (column.populated() != Some(false)
&& temporal
.as_ref()
.is_none_or(|names| names.contains(&column.name)))
})
}
pub(crate) fn column_selection(&self) -> Option<&ColumnSelection> {
self.column_selection.as_ref()
}
pub(crate) fn schema_evidence(&self) -> Evidence {
if self.schema_declared {
Evidence::Complete
} else {
self.reference.evidence
}
}
pub(crate) fn columns(&self) -> &[ColumnSide] {
&self.columns
}
pub fn from_summary_json(text: &str) -> serde_json::Result<Self> {
let summary: Summary = serde_json::from_str(text)?;
let execution = &summary.execution;
let known_gap = scan_coverage(
execution.source_exhausted == Some(false) || execution.truncation_reason.is_some(),
execution.sampling_applied == Some(true),
execution.error_count.is_some_and(|count| count > 0),
);
let recorded = execution.source_exhausted.is_some()
&& execution.sampling_applied.is_some()
&& execution.error_count.is_some();
let scan = match known_gap {
Evidence::Complete if !recorded => Evidence::Incomplete {
reason: EvidenceGap::CoverageUnrecorded,
},
known => known,
};
let quality = summary.quality.as_ref();
let assessed: Option<Vec<QualityDimension>> = quality.and_then(|quality| {
let parsed: Vec<QualityDimension> = quality
.assessed_dimensions
.iter()
.filter_map(|name| name.parse().ok())
.collect();
(!(parsed.is_empty() && quality.overall_score.is_some())).then_some(parsed)
});
let unlisted = quality
.map(SummaryQuality::unlisted_components)
.unwrap_or_default();
let sample_evidence = |provenance| {
let Some(quality) = quality else {
return Evidence::Complete;
};
let sampled = quality.sampled_dimensions.as_deref();
match (provenance, &assessed) {
(Provenance::Overall, None)
if sampled.is_some_and(|s| s.is_empty()) && unlisted.is_empty() =>
{
Evidence::Complete
}
(Provenance::Overall, None) => Evidence::Incomplete {
reason: EvidenceGap::CoverageUnrecorded,
},
(_, assessed) => sampled_component_evidence(
sampled,
&unlisted,
assessed.as_deref().unwrap_or_default(),
overall_left_open(
quality.score_bounds.as_ref(),
|dimension| {
quality
.dimension_scores
.get(&dimension.to_string())
.is_none_or(Option::is_none)
},
quality.score_weights.as_ref(),
),
provenance,
),
}
};
let evidence = |provenance| scan.and(sample_evidence(provenance));
let dimensions = QualityDimension::all()
.into_iter()
.map(|dimension| DimensionSide {
dimension,
score: at_precision(
quality
.and_then(|quality| quality.dimension_scores.get(&dimension.to_string()))
.copied()
.flatten(),
),
evidence: evidence(Provenance::Dimension(dimension)),
sample_evidence: sample_evidence(Provenance::Dimension(dimension)),
})
.collect();
Ok(Self {
reference: ReportReference {
id: None,
timestamp: None,
source: summary.source,
rows_processed: execution.rows_processed,
evidence: scan,
generator: summary.generator,
source_identity: summary.source_identity,
},
quality_score: at_precision(quality.and_then(|quality| quality.overall_score)),
overall_evidence: evidence(Provenance::Overall),
overall_sample_evidence: sample_evidence(Provenance::Overall),
dimensions,
score_bounds: quality.and_then(|quality| quality.score_bounds.clone()),
columns: first_of_each_name(summary.columns.into_iter().map(|column| ColumnSide {
name: column.name,
null_pct: at_precision(column.null_percentage),
data_type: column.data_type.as_deref().and_then(DataType::from_name),
counts: column.total_count.zip(column.null_count),
native_type: column.native_type,
unique_count: column.unique_count,
unique_count_is_approximate: column.unique_count_is_approximate,
quartiles_differ: column.stats.as_ref().is_some_and(summary_quartiles_differ),
max_text_length: column.stats.as_ref().and_then(summary_max_text_length),
})),
score_weights: quality.and_then(|quality| {
quality.score_weights.or_else(|| {
summary
.metric_semantics
.is_some()
.then(QualityScoreWeights::default)
})
}),
semantic_hints: summary
.schema_version
.is_some_and(|version| version >= 1)
.then(|| RecordedHints::from_bindings(&summary.semantic_hint_bindings)),
identifier_order: summary
.schema_version
.is_some_and(|version| version >= 1)
.then(|| identifier_order(&summary.semantic_hint_bindings)),
metric_semantics: summary.metric_semantics,
pattern_locale: summary
.analysis_settings
.as_ref()
.map(|settings| settings.pattern_locale),
timeliness_clock: quality
.and_then(|quality| quality.timeliness.as_ref())
.and_then(|timeliness| timeliness.clock),
metric_packs: summary
.analysis_settings
.as_ref()
.and_then(|settings| settings.metric_packs.clone()),
temporal_pairs: summary
.analysis_settings
.as_ref()
.and_then(|settings| settings.temporal_pairs.clone()),
column_selection: summary.analysis_settings.map(|settings| settings.columns),
schema_declared: match summary.source_type {
Some(SummarySourceType::Dataframe | SummarySourceType::Query) => true,
Some(SummarySourceType::File | SummarySourceType::Bytes) => summary
.source_format
.as_ref()
.is_some_and(file_format_declares_schema),
Some(SummarySourceType::Stream) => summary
.source_format
.as_ref()
.is_some_and(stream_format_declares_schema),
Some(SummarySourceType::Other) | None => false,
},
json_numbers: match summary.source_type {
Some(SummarySourceType::Dataframe) => false,
Some(
SummarySourceType::File | SummarySourceType::Bytes | SummarySourceType::Stream,
) => summary
.source_format
.as_ref()
.is_none_or(format_may_hold_json_numbers),
Some(SummarySourceType::Query | SummarySourceType::Other) | None => true,
},
unlisted_components: unlisted.clone(),
})
}
}
fn summary_metric_semantics<'de, D>(deserializer: D) -> Result<Option<MetricSemantics>, D::Error>
where
D: serde::Deserializer<'de>,
{
crate::profile_report::present_value(deserializer).map_err(|error| {
<D::Error as serde::de::Error>::custom(format!("metric_semantics: {error}"))
})
}
fn summary_schema_version<'de, D>(deserializer: D) -> Result<Option<u32>, D::Error>
where
D: serde::Deserializer<'de>,
{
crate::profile_report::present_value(deserializer)
.map_err(|error| <D::Error as serde::de::Error>::custom(format!("schema_version: {error}")))
}
fn summary_analysis_settings<'de, D>(deserializer: D) -> Result<Option<AnalysisSettings>, D::Error>
where
D: serde::Deserializer<'de>,
{
crate::profile_report::present_value(deserializer).map_err(|error| {
<D::Error as serde::de::Error>::custom(format!("analysis_settings: {error}"))
})
}
fn summary_score_weights<'de, D>(deserializer: D) -> Result<Option<QualityScoreWeights>, D::Error>
where
D: serde::Deserializer<'de>,
{
crate::profile_report::present_value(deserializer).map_err(|error| {
<D::Error as serde::de::Error>::custom(format!("quality.score_weights: {error}"))
})
}
#[derive(serde::Deserialize)]
struct Summary {
source: String,
#[serde(default)]
source_type: Option<SummarySourceType>,
#[serde(default)]
source_format: Option<FileFormat>,
execution: SummaryExecution,
columns: Vec<SummaryColumn>,
quality: Option<SummaryQuality>,
#[serde(default, deserialize_with = "summary_metric_semantics")]
metric_semantics: Option<MetricSemantics>,
#[serde(default, deserialize_with = "summary_schema_version")]
schema_version: Option<u32>,
#[serde(default)]
semantic_hint_bindings: Vec<SemanticHintBinding>,
#[serde(default, deserialize_with = "summary_analysis_settings")]
analysis_settings: Option<AnalysisSettings>,
#[serde(default, deserialize_with = "summary_generator")]
generator: Option<ReportGenerator>,
#[serde(default, deserialize_with = "summary_source_identity")]
source_identity: Option<SourceIdentity>,
}
fn summary_generator<'de, D>(deserializer: D) -> Result<Option<ReportGenerator>, D::Error>
where
D: serde::Deserializer<'de>,
{
crate::profile_report::present_value(deserializer)
.map_err(|error| <D::Error as serde::de::Error>::custom(format!("generator: {error}")))
}
fn summary_source_identity<'de, D>(deserializer: D) -> Result<Option<SourceIdentity>, D::Error>
where
D: serde::Deserializer<'de>,
{
crate::profile_report::present_value(deserializer).map_err(|error| {
<D::Error as serde::de::Error>::custom(format!("source_identity: {error}"))
})
}
#[derive(serde::Deserialize)]
#[serde(rename_all = "lowercase")]
enum SummarySourceType {
File,
Bytes,
Stream,
Dataframe,
Query,
#[serde(other)]
Other,
}
fn declares_schema(source: &DataSource) -> bool {
match source {
DataSource::File { format, .. } | DataSource::Bytes { format, .. } => {
file_format_declares_schema(format)
}
DataSource::Stream {
format: Some(format),
..
} => stream_format_declares_schema(format),
DataSource::DataFrame { .. } | DataSource::Query { .. } => true,
DataSource::Stream { format: None, .. } => false,
}
}
fn file_format_declares_schema(format: &FileFormat) -> bool {
matches!(format, FileFormat::Csv | FileFormat::Parquet)
}
fn stream_format_declares_schema(format: &FileFormat) -> bool {
matches!(
format,
FileFormat::Csv | FileFormat::Parquet | FileFormat::Arrow
)
}
#[derive(serde::Deserialize)]
struct SummaryExecution {
rows_processed: Option<usize>,
source_exhausted: Option<bool>,
truncation_reason: Option<String>,
sampling_applied: Option<bool>,
error_count: Option<usize>,
}
#[derive(serde::Deserialize)]
struct SummaryColumn {
name: String,
null_percentage: Option<f64>,
#[serde(default)]
data_type: Option<String>,
#[serde(default)]
total_count: Option<usize>,
#[serde(default)]
null_count: Option<usize>,
#[serde(default)]
native_type: Option<NativeType>,
#[serde(default)]
unique_count: Option<usize>,
#[serde(default)]
unique_count_is_approximate: Option<bool>,
#[serde(default)]
stats: Option<serde_json::Value>,
}
fn summary_quartiles_differ(stats: &serde_json::Value) -> bool {
let quartile = |name: &str| stats.get("quartiles")?.get(name)?.as_f64();
stats
.get("is_approximate")
.and_then(serde_json::Value::as_bool)
== Some(false)
&& quartile("q1")
.zip(quartile("q3"))
.is_some_and(|(q1, q3)| q1 != q3)
}
fn summary_max_text_length(stats: &serde_json::Value) -> Option<usize> {
stats
.get("max_length")?
.as_u64()
.and_then(|length| usize::try_from(length).ok())
}
#[derive(serde::Deserialize)]
struct SummaryQuality {
overall_score: Option<f64>,
#[serde(default)]
dimension_scores: BTreeMap<String, Option<f64>>,
#[serde(default)]
assessed_dimensions: Vec<String>,
#[serde(default)]
exact_dimensions: Option<Vec<String>>,
sampled_dimensions: Option<Vec<String>>,
#[serde(default)]
completeness: Option<serde::de::IgnoredAny>,
#[serde(default)]
consistency: Option<serde::de::IgnoredAny>,
#[serde(default)]
uniqueness: Option<SummaryUniqueness>,
#[serde(default)]
accuracy: Option<serde::de::IgnoredAny>,
#[serde(default)]
timeliness: Option<SummaryTimeliness>,
#[serde(default)]
validity: Option<serde::de::IgnoredAny>,
#[serde(default)]
precision: Option<serde::de::IgnoredAny>,
#[serde(default, deserialize_with = "summary_score_weights")]
score_weights: Option<QualityScoreWeights>,
#[serde(default, deserialize_with = "summary_score_bounds")]
score_bounds: Option<ScoreBounds>,
}
fn summary_score_bounds<'de, D>(deserializer: D) -> Result<Option<ScoreBounds>, D::Error>
where
D: serde::Deserializer<'de>,
{
let bounds: Option<ScoreBounds> = serde::Deserialize::deserialize(deserializer)?;
if bounds.as_ref().is_some_and(|bounds| !bounds.is_valid()) {
return Err(<D::Error as serde::de::Error>::custom(
"quality.score_bounds: intervals must lie in 0..=100 with lower <= upper, at a \
confidence level between 0 and 1",
));
}
Ok(bounds)
}
#[derive(serde::Deserialize)]
struct SummaryTimeliness {
#[serde(default, deserialize_with = "summary_timeliness_clock")]
clock: Option<TimelinessClock>,
}
fn summary_timeliness_clock<'de, D>(deserializer: D) -> Result<Option<TimelinessClock>, D::Error>
where
D: serde::Deserializer<'de>,
{
crate::profile_report::present_value(deserializer).map_err(|error| {
<D::Error as serde::de::Error>::custom(format!("quality.timeliness.clock: {error}"))
})
}
#[derive(serde::Deserialize)]
struct SummaryUniqueness {
#[serde(default)]
rows_checked: usize,
#[serde(default)]
key_column: Option<String>,
#[serde(default)]
key_values_checked: Option<usize>,
}
impl SummaryQuality {
fn unlisted_components(&self) -> Vec<String> {
match (&self.exact_dimensions, &self.sampled_dimensions) {
(Some(exact), Some(sampled)) => {
self.signal_metrics().unlisted_components(exact, sampled)
}
_ => Vec::new(),
}
}
fn signal_metrics(&self) -> QualityMetrics {
QualityMetrics {
completeness: self
.completeness
.as_ref()
.map(|_| CompletenessMetrics::default()),
consistency: self
.consistency
.as_ref()
.map(|_| ConsistencyMetrics::default()),
uniqueness: self
.uniqueness
.as_ref()
.map(|uniqueness| UniquenessMetrics {
rows_checked: uniqueness.rows_checked,
key_column: uniqueness.key_column.clone(),
key_values_checked: uniqueness.key_values_checked,
..UniquenessMetrics::default()
}),
accuracy: self.accuracy.as_ref().map(|_| AccuracyMetrics::default()),
timeliness: self
.timeliness
.as_ref()
.map(|_| TimelinessMetrics::default()),
validity: self.validity.as_ref().map(|_| ValidityMetrics::default()),
precision: self.precision.as_ref().map(|_| PrecisionMetrics::default()),
..QualityMetrics::default()
}
}
}
#[cfg(test)]
mod tests {
use super::*;
fn complete() -> Sides<Evidence> {
Sides {
a: Evidence::Complete,
b: Evidence::Complete,
}
}
#[test]
fn a_zero_baseline_has_an_absolute_change_and_no_relative_one() {
let change = ScoreChange::new(Some(0.0), Some(50.0), complete());
assert_eq!(change.abs, Some(50.0));
assert_eq!(change.rel_pct, None);
}
#[test]
fn a_change_is_rounded_to_the_serialized_precision() {
let change = ScoreChange::new(Some(80.0), Some(80.1), complete());
assert_eq!(change.abs, Some(0.1));
assert_eq!(change.rel_pct, Some(0.12));
}
#[test]
fn a_missing_side_leaves_both_changes_absent() {
for (a, b) in [(None, Some(1.0)), (Some(1.0), None), (None, None)] {
let change = ScoreChange::new(a, b, complete());
assert_eq!((change.abs, change.rel_pct), (None, None));
}
}
#[test]
fn a_non_finite_input_is_no_number() {
assert_eq!(at_precision(Some(f64::NAN)), None);
assert_eq!(at_precision(Some(f64::INFINITY)), None);
assert_eq!(at_precision(Some(12.345)), Some(12.35));
}
}