1use std::sync::OnceLock;
2
3use dataprof_core::{
4 AccuracyComparison, AnalysisOptions, ColumnProfile, CsvSourceMetadata, DataSource, DateForms,
5 DateStatistics, ExecutionMetadata, FileFormat, JsonNumberReading, KeyColumnSource,
6 KeyUniquenessUnits, Locale, MetricPack, NullTokenSet, OutlierSpread, OutlierUnits,
7 PatternMatchCount, QualityScoreWeights, RangeRuleCount, SampledScoreEstimator,
8 SemanticHintBinding, ShapeMomentEstimator, StatisticRounding, TextLengthUnit,
9};
10use dataprof_metrics::{
11 AccuracyMetrics, CompletenessMetrics, ConsistencyMetrics, MetricConfidence, PrecisionMetrics,
12 QualityAssessment, QualityMetrics, TimelinessMetrics, ValidityMetrics,
13};
14
15pub const REPORT_SCHEMA_VERSION: u32 = 1;
30
31#[derive(
39 Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize, schemars::JsonSchema,
40)]
41#[serde(tag = "state", rename_all = "snake_case")]
42pub enum QualityAnalysisStatus {
43 Computed,
45 NotRequested,
47 NoData,
50 WithheldByProjection,
56 Failed {
59 error: String,
61 },
62 Unrecorded,
66}
67
68#[derive(
81 Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize, schemars::JsonSchema,
82)]
83#[non_exhaustive]
84pub struct MetricSemantics {
85 #[serde(
90 default,
91 skip_serializing_if = "Option::is_none",
92 deserialize_with = "present_value"
93 )]
94 #[schemars(with = "TextLengthUnit")]
95 pub text_length_unit: Option<TextLengthUnit>,
96 #[serde(
102 default,
103 skip_serializing_if = "Option::is_none",
104 deserialize_with = "present_value"
105 )]
106 #[schemars(with = "NullTokenSet")]
107 pub null_tokens: Option<NullTokenSet>,
108 #[serde(
113 default,
114 skip_serializing_if = "Option::is_none",
115 deserialize_with = "present_value"
116 )]
117 #[schemars(with = "PatternMatchCount")]
118 pub pattern_match_count: Option<PatternMatchCount>,
119 #[serde(
124 default,
125 skip_serializing_if = "Option::is_none",
126 deserialize_with = "present_value"
127 )]
128 #[schemars(with = "ShapeMomentEstimator")]
129 pub shape_moments: Option<ShapeMomentEstimator>,
130 #[serde(
136 default,
137 skip_serializing_if = "Option::is_none",
138 deserialize_with = "present_value"
139 )]
140 #[schemars(with = "StatisticRounding")]
141 pub statistic_rounding: Option<StatisticRounding>,
142 #[serde(
147 default,
148 skip_serializing_if = "Option::is_none",
149 deserialize_with = "present_value"
150 )]
151 #[schemars(with = "RangeRuleCount")]
152 pub range_rule_count: Option<RangeRuleCount>,
153 #[serde(
158 default,
159 skip_serializing_if = "Option::is_none",
160 deserialize_with = "present_value"
161 )]
162 #[schemars(with = "KeyUniquenessUnits")]
163 pub key_uniqueness_units: Option<KeyUniquenessUnits>,
164 #[serde(
169 default,
170 skip_serializing_if = "Option::is_none",
171 deserialize_with = "present_value"
172 )]
173 #[schemars(with = "OutlierUnits")]
174 pub outlier_units: Option<OutlierUnits>,
175 #[serde(
180 default,
181 skip_serializing_if = "Option::is_none",
182 deserialize_with = "present_value"
183 )]
184 #[schemars(with = "KeyColumnSource")]
185 pub key_column_source: Option<KeyColumnSource>,
186 #[serde(
192 default,
193 skip_serializing_if = "Option::is_none",
194 deserialize_with = "present_value"
195 )]
196 #[schemars(with = "AccuracyComparison")]
197 pub accuracy_comparison: Option<AccuracyComparison>,
198 #[serde(
204 default,
205 skip_serializing_if = "Option::is_none",
206 deserialize_with = "present_value"
207 )]
208 #[schemars(with = "JsonNumberReading")]
209 pub json_number_reading: Option<JsonNumberReading>,
210 #[serde(
216 default,
217 skip_serializing_if = "Option::is_none",
218 deserialize_with = "present_value"
219 )]
220 #[schemars(with = "SampledScoreEstimator")]
221 pub sampled_score_estimator: Option<SampledScoreEstimator>,
222 #[serde(
230 default,
231 skip_serializing_if = "Option::is_none",
232 deserialize_with = "present_value"
233 )]
234 #[schemars(with = "DateStatistics")]
235 pub date_statistics: Option<DateStatistics>,
236 #[serde(
241 default,
242 skip_serializing_if = "Option::is_none",
243 deserialize_with = "present_value"
244 )]
245 #[schemars(with = "OutlierSpread")]
246 pub outlier_spread: Option<OutlierSpread>,
247 #[serde(
252 default,
253 skip_serializing_if = "Option::is_none",
254 deserialize_with = "present_value"
255 )]
256 #[schemars(with = "DateForms")]
257 pub date_forms: Option<DateForms>,
258}
259
260pub(crate) fn present_value<'de, D, T>(deserializer: D) -> Result<Option<T>, D::Error>
263where
264 D: serde::Deserializer<'de>,
265 T: serde::Deserialize<'de>,
266{
267 T::deserialize(deserializer).map(Some)
268}
269
270impl MetricSemantics {
271 pub fn current() -> Self {
273 Self {
274 text_length_unit: Some(TextLengthUnit::UnicodeScalar),
275 null_tokens: Some(NullTokenSet::CommonMarkers),
276 pattern_match_count: Some(PatternMatchCount::Validated),
277 shape_moments: Some(ShapeMomentEstimator::AdjustedSample),
278 statistic_rounding: Some(StatisticRounding::FourDecimalsTwelveSignificant),
279 range_rule_count: Some(RangeRuleCount::ViolatingValues),
280 key_uniqueness_units: Some(KeyUniquenessUnits::NonNullValues),
281 outlier_units: Some(OutlierUnits::TestedValues),
282 key_column_source: Some(KeyColumnSource::Declared),
283 accuracy_comparison: Some(AccuracyComparison::ExactValues),
284 json_number_reading: Some(JsonNumberReading::LiteralDigits),
285 sampled_score_estimator: Some(SampledScoreEstimator::PopulationWeighted),
286 date_statistics: Some(DateStatistics::EveryValue),
287 outlier_spread: Some(OutlierSpread::NonzeroIqr),
288 date_forms: Some(DateForms::SpacedIsoDatetimes),
289 }
290 }
291
292 pub(crate) fn is_fully_recorded(&self) -> bool {
297 let Self {
298 text_length_unit,
299 null_tokens,
300 pattern_match_count,
301 shape_moments,
302 statistic_rounding,
303 range_rule_count,
304 key_uniqueness_units,
305 outlier_units,
306 key_column_source,
307 accuracy_comparison,
308 json_number_reading,
309 sampled_score_estimator,
310 date_statistics,
311 outlier_spread,
312 date_forms,
313 } = self;
314 text_length_unit.is_some()
315 && null_tokens.is_some()
316 && pattern_match_count.is_some()
317 && shape_moments.is_some()
318 && statistic_rounding.is_some()
319 && range_rule_count.is_some()
320 && key_uniqueness_units.is_some()
321 && outlier_units.is_some()
322 && key_column_source.is_some()
323 && accuracy_comparison.is_some()
324 && json_number_reading.is_some()
325 && sampled_score_estimator.is_some()
326 && date_statistics.is_some()
327 && outlier_spread.is_some()
328 && date_forms.is_some()
329 }
330}
331
332#[derive(
335 Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize, schemars::JsonSchema,
336)]
337#[non_exhaustive]
338pub enum PatternLocale {
339 #[serde(rename = "none")]
341 Unset,
342 #[serde(rename = "CA")]
344 Ca,
345 #[serde(rename = "DE")]
347 De,
348 #[serde(rename = "FR")]
350 Fr,
351 #[serde(rename = "GB")]
353 Gb,
354 #[serde(rename = "IT")]
356 It,
357 #[serde(rename = "US")]
359 Us,
360}
361
362impl From<Option<Locale>> for PatternLocale {
363 fn from(locale: Option<Locale>) -> Self {
364 match locale {
365 None => Self::Unset,
366 Some(Locale::Ca) => Self::Ca,
367 Some(Locale::De) => Self::De,
368 Some(Locale::Fr) => Self::Fr,
369 Some(Locale::Gb) => Self::Gb,
370 Some(Locale::It) => Self::It,
371 Some(Locale::Us) => Self::Us,
372 }
373 }
374}
375
376impl PatternLocale {
377 pub fn locale(self) -> Option<Locale> {
379 match self {
380 Self::Unset => None,
381 Self::Ca => Some(Locale::Ca),
382 Self::De => Some(Locale::De),
383 Self::Fr => Some(Locale::Fr),
384 Self::Gb => Some(Locale::Gb),
385 Self::It => Some(Locale::It),
386 Self::Us => Some(Locale::Us),
387 }
388 }
389}
390
391#[derive(
398 Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize, schemars::JsonSchema,
399)]
400#[serde(tag = "selection", rename_all = "snake_case")]
401#[non_exhaustive]
402pub enum ColumnSelection {
403 All,
405 #[non_exhaustive]
407 Listed {
408 names: Vec<String>,
410 },
411}
412
413impl ColumnSelection {
414 pub fn from_options(options: &AnalysisOptions) -> Self {
416 match options.columns() {
417 None => Self::All,
418 Some(names) => Self::Listed {
419 names: names.to_vec(),
420 },
421 }
422 }
423
424 pub fn includes(&self, column: &str) -> bool {
427 match self {
428 Self::All => true,
429 Self::Listed { names } => names.iter().any(|name| name == column),
430 }
431 }
432}
433
434#[derive(
443 Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize, schemars::JsonSchema,
444)]
445#[non_exhaustive]
446pub struct AnalysisSettings {
447 pub pattern_locale: PatternLocale,
450 pub columns: ColumnSelection,
454 #[serde(
461 default,
462 skip_serializing_if = "Option::is_none",
463 deserialize_with = "present_value"
464 )]
465 #[schemars(with = "Vec<MetricPack>")]
466 pub metric_packs: Option<Vec<MetricPack>>,
467 #[serde(
476 default,
477 skip_serializing_if = "Option::is_none",
478 deserialize_with = "present_value"
479 )]
480 #[schemars(with = "Vec<TemporalPair>")]
481 pub temporal_pairs: Option<Vec<TemporalPair>>,
482}
483
484#[derive(
490 Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, serde::Serialize, schemars::JsonSchema,
491)]
492#[non_exhaustive]
493pub struct TemporalPair {
494 pub start: String,
496 pub end: String,
498}
499
500impl<'de> serde::Deserialize<'de> for TemporalPair {
501 fn deserialize<D: serde::Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
502 struct PairVisitor;
503
504 impl<'de> serde::de::Visitor<'de> for PairVisitor {
505 type Value = TemporalPair;
506
507 fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result {
508 formatter.write_str(r#"a temporal pair {"start": ..., "end": ...}"#)
509 }
510
511 fn visit_map<A: serde::de::MapAccess<'de>>(
512 self,
513 mut map: A,
514 ) -> Result<TemporalPair, A::Error> {
515 use serde::de::Error;
516 let (mut start, mut end) = (None, None);
517 while let Some(key) = map.next_key::<String>()? {
518 match key.as_str() {
519 "start" if start.is_some() => {
520 return Err(A::Error::duplicate_field("start"));
521 }
522 "end" if end.is_some() => return Err(A::Error::duplicate_field("end")),
523 "start" => start = Some(map.next_value::<String>()?),
524 "end" => end = Some(map.next_value::<String>()?),
525 _ => {
527 map.next_value::<serde::de::IgnoredAny>()?;
528 }
529 }
530 }
531 Ok(TemporalPair {
532 start: start.ok_or_else(|| A::Error::missing_field("start"))?,
533 end: end.ok_or_else(|| A::Error::missing_field("end"))?,
534 })
535 }
536 }
537
538 deserializer.deserialize_map(PairVisitor)
539 }
540}
541
542impl TemporalPair {
543 pub fn new(start: impl Into<String>, end: impl Into<String>) -> Self {
545 Self {
546 start: start.into(),
547 end: end.into(),
548 }
549 }
550
551 pub fn canonical(declared: &[(String, String)]) -> Vec<Self> {
554 let mut pairs: Vec<Self> = declared
555 .iter()
556 .map(|(start, end)| Self::new(start.clone(), end.clone()))
557 .collect();
558 pairs.sort();
559 pairs.dedup();
560 pairs
561 }
562}
563
564impl AnalysisSettings {
565 pub fn from_options(options: &AnalysisOptions) -> Self {
567 Self {
568 pattern_locale: options.locale().into(),
569 columns: ColumnSelection::from_options(options),
570 metric_packs: Some(options.selected_metric_packs()),
571 temporal_pairs: Some(TemporalPair::canonical(
572 &options.semantic_hints().temporal_pairs,
573 )),
574 }
575 }
576}
577
578#[derive(
589 Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize, schemars::JsonSchema,
590)]
591#[non_exhaustive]
592pub struct ReportGenerator {
593 pub version: String,
595 #[serde(
597 default,
598 skip_serializing_if = "Option::is_none",
599 deserialize_with = "present_value"
600 )]
601 #[schemars(with = "String")]
602 pub commit: Option<String>,
603 #[serde(
606 default,
607 skip_serializing_if = "Option::is_none",
608 deserialize_with = "present_value"
609 )]
610 #[schemars(with = "bool")]
611 pub dirty: Option<bool>,
612}
613
614static BUILD_IDENTITY: OnceLock<(Option<&'static str>, Option<bool>)> = OnceLock::new();
616
617#[doc(hidden)]
625pub fn record_build_identity(commit: Option<&'static str>, dirty: Option<bool>) {
626 let _ = BUILD_IDENTITY.set((commit, dirty));
627}
628
629impl ReportGenerator {
630 pub fn current() -> Self {
632 let (commit, dirty) = BUILD_IDENTITY.get().copied().unwrap_or((None, None));
633 Self {
634 version: env!("CARGO_PKG_VERSION").to_string(),
635 commit: commit.map(str::to_string),
636 dirty,
637 }
638 }
639}
640
641#[derive(
654 Debug, Clone, Default, PartialEq, Eq, serde::Serialize, serde::Deserialize, schemars::JsonSchema,
655)]
656#[non_exhaustive]
657pub struct SourceIdentity {
658 #[serde(
662 default,
663 skip_serializing_if = "Option::is_none",
664 deserialize_with = "present_value"
665 )]
666 #[schemars(with = "String")]
667 pub version: Option<String>,
668}
669
670impl SourceIdentity {
671 pub fn with_version(version: impl Into<String>) -> Self {
673 Self {
674 version: Some(version.into()),
675 }
676 }
677}
678
679#[derive(Debug, Clone, serde::Serialize, schemars::JsonSchema)]
686pub struct ProfileReport {
687 #[schemars(schema_with = "schema_version_schema")]
692 pub schema_version: u32,
693 pub id: String,
695 pub timestamp: String,
697 pub data_source: DataSource,
699 pub column_profiles: Vec<ColumnProfile>,
701 pub execution: ExecutionMetadata,
703 #[serde(skip_serializing_if = "Option::is_none")]
705 pub quality: Option<QualityAssessment>,
706 pub quality_status: QualityAnalysisStatus,
721 #[serde(default, skip_serializing_if = "Vec::is_empty")]
728 pub semantic_hint_bindings: Vec<SemanticHintBinding>,
729 #[serde(skip_serializing_if = "Option::is_none")]
736 #[schemars(with = "MetricSemantics")]
737 pub metric_semantics: Option<MetricSemantics>,
738 #[serde(skip_serializing_if = "Option::is_none")]
745 #[schemars(with = "AnalysisSettings")]
746 pub analysis_settings: Option<AnalysisSettings>,
747 #[serde(skip_serializing_if = "Option::is_none")]
754 #[schemars(with = "ReportGenerator")]
755 pub generator: Option<ReportGenerator>,
756 #[serde(skip_serializing_if = "Option::is_none")]
762 #[schemars(with = "SourceIdentity")]
763 pub source_identity: Option<SourceIdentity>,
764}
765
766fn schema_version_schema(_: &mut schemars::SchemaGenerator) -> schemars::Schema {
767 schemars::json_schema!({
768 "type": "integer",
769 "const": REPORT_SCHEMA_VERSION,
770 "minimum": 0
771 })
772}
773
774#[allow(dead_code)]
782#[derive(serde::Serialize, schemars::JsonSchema)]
783#[serde(untagged)]
784#[schemars(title = "ProfileReport")]
785enum SerializedProfileReport {
786 Rust(Box<ProfileReport>),
787 Python(Box<PythonProfileReportDocument>),
788}
789
790#[derive(serde::Serialize, serde::Deserialize, schemars::JsonSchema)]
791struct PythonProfileReportDocument {
792 #[schemars(schema_with = "schema_version_schema")]
793 schema_version: u32,
794 source: String,
795 source_type: PythonSourceType,
796 #[serde(default, skip_serializing_if = "Option::is_none")]
800 #[schemars(with = "FileFormat")]
801 source_format: Option<FileFormat>,
802 execution: PythonExecutionDocument,
803 columns: Vec<PythonColumnDocument>,
804 quality: Option<PythonQualityDocument>,
805 quality_status: QualityAnalysisStatus,
806 #[serde(skip_serializing_if = "Vec::is_empty")]
807 semantic_hint_bindings: Vec<SemanticHintBinding>,
808 #[serde(skip_serializing_if = "Option::is_none")]
809 #[schemars(with = "MetricSemantics")]
810 metric_semantics: Option<MetricSemantics>,
811 #[serde(skip_serializing_if = "Option::is_none")]
812 #[schemars(with = "AnalysisSettings")]
813 analysis_settings: Option<AnalysisSettings>,
814 #[serde(skip_serializing_if = "Option::is_none")]
815 #[schemars(with = "ReportGenerator")]
816 generator: Option<ReportGenerator>,
817 #[serde(skip_serializing_if = "Option::is_none")]
818 #[schemars(with = "SourceIdentity")]
819 source_identity: Option<SourceIdentity>,
820 #[serde(skip_serializing_if = "Option::is_none")]
821 #[schemars(with = "CsvSourceMetadata")]
822 csv_metadata: Option<CsvSourceMetadata>,
823}
824
825#[derive(serde::Serialize, serde::Deserialize, schemars::JsonSchema)]
826#[serde(rename_all = "lowercase")]
827enum PythonSourceType {
828 File,
829 Query,
830 Dataframe,
831 Stream,
832 Bytes,
833}
834
835#[derive(serde::Serialize, serde::Deserialize, schemars::JsonSchema)]
836struct PythonExecutionDocument {
837 engine: Option<String>,
838 #[serde(skip_serializing_if = "Option::is_none")]
839 recovery_events: Option<Vec<dataprof_core::execution::RecoveryEvent>>,
840 rows_processed: usize,
841 columns_detected: usize,
842 scan_time_ms: u128,
843 source_exhausted: bool,
844 truncation_reason: Option<String>,
845 bytes_consumed: Option<u64>,
846 #[serde(serialize_with = "dataprof_core::serde_helpers::round_4_opt")]
847 throughput_rows_sec: Option<f64>,
848 #[serde(serialize_with = "dataprof_core::serde_helpers::round_2_opt")]
849 memory_peak_mb: Option<f64>,
850 error_count: usize,
851 ragged_row_count: usize,
852 #[serde(default, skip_serializing_if = "Option::is_none")]
853 unterminated_quote: Option<bool>,
854 sampling_applied: bool,
855 #[serde(serialize_with = "dataprof_core::serde_helpers::round_4_opt")]
856 sampling_ratio: Option<f64>,
857 #[serde(skip_serializing_if = "Option::is_none")]
858 sampled_row_ranges: Option<Vec<[u64; 2]>>,
859}
860
861#[derive(serde::Serialize, serde::Deserialize, schemars::JsonSchema)]
862struct PythonColumnDocument {
863 name: String,
864 data_type: PythonDataType,
865 total_count: usize,
866 null_count: usize,
867 #[serde(serialize_with = "dataprof_core::serde_helpers::round_2_opt")]
868 null_percentage: Option<f64>,
869 unique_count: Option<usize>,
870 unique_count_is_approximate: Option<bool>,
871 #[serde(serialize_with = "dataprof_core::serde_helpers::round_4_opt")]
872 uniqueness_ratio: Option<f64>,
873 #[serde(skip_serializing_if = "Option::is_none")]
874 invalid_count: Option<usize>,
875 #[serde(skip_serializing_if = "Option::is_none")]
881 type_homogeneity: Option<dataprof_core::TypeHomogeneity>,
882 #[serde(skip_serializing_if = "Option::is_none")]
883 locale_number_count: Option<usize>,
884 #[serde(skip_serializing_if = "Option::is_none")]
885 native_type: Option<dataprof_core::NativeType>,
886 #[serde(skip_serializing_if = "Option::is_none")]
887 stats: Option<PythonColumnStatsDocument>,
888 #[serde(skip_serializing_if = "Option::is_none")]
889 patterns: Option<Vec<PythonPatternDocument>>,
890}
891
892#[derive(serde::Serialize, serde::Deserialize, schemars::JsonSchema)]
893#[serde(rename_all = "lowercase")]
894enum PythonDataType {
895 String,
896 Identifier,
897 Integer,
898 Float,
899 Date,
900 Boolean,
901 Nested,
902}
903
904#[derive(serde::Serialize, serde::Deserialize, schemars::JsonSchema)]
905struct PythonColumnStatsDocument {
906 #[serde(skip_serializing_if = "Option::is_none")]
907 min: Option<f64>,
908 #[serde(skip_serializing_if = "Option::is_none")]
909 max: Option<f64>,
910 #[serde(skip_serializing_if = "Option::is_none")]
911 mean: Option<f64>,
912 #[serde(
915 default,
916 deserialize_with = "deserialize_present_nullable_stat",
917 skip_serializing_if = "Option::is_none"
918 )]
919 #[schemars(with = "Option<f64>")]
920 std_dev: Option<Option<f64>>,
921 #[serde(
922 default,
923 deserialize_with = "deserialize_present_nullable_stat",
924 skip_serializing_if = "Option::is_none"
925 )]
926 #[schemars(with = "Option<f64>")]
927 variance: Option<Option<f64>>,
928 #[serde(skip_serializing_if = "Option::is_none")]
929 median: Option<f64>,
930 #[serde(skip_serializing_if = "Option::is_none")]
931 mode: Option<f64>,
932 #[serde(skip_serializing_if = "Option::is_none")]
933 skewness: Option<f64>,
934 #[serde(skip_serializing_if = "Option::is_none")]
935 kurtosis: Option<f64>,
936 #[serde(skip_serializing_if = "Option::is_none")]
937 coefficient_of_variation: Option<f64>,
938 #[serde(skip_serializing_if = "Option::is_none")]
939 quartiles: Option<dataprof_core::Quartiles>,
940 #[serde(skip_serializing_if = "Option::is_none")]
941 is_approximate: Option<bool>,
942 #[serde(skip_serializing_if = "Option::is_none")]
943 outlier_count: Option<usize>,
944 #[serde(skip_serializing_if = "Option::is_none")]
945 exact: Option<dataprof_core::ExactNumericStats>,
946 #[serde(skip_serializing_if = "Option::is_none")]
951 min_length: Option<usize>,
952 #[serde(skip_serializing_if = "Option::is_none")]
954 max_length: Option<usize>,
955 #[serde(skip_serializing_if = "Option::is_none")]
957 avg_length: Option<f64>,
958 #[serde(skip_serializing_if = "Option::is_none")]
959 true_count: Option<usize>,
960 #[serde(skip_serializing_if = "Option::is_none")]
961 false_count: Option<usize>,
962 #[serde(skip_serializing_if = "Option::is_none")]
963 true_ratio: Option<f64>,
964}
965
966fn deserialize_present_nullable_stat<'de, D>(
967 deserializer: D,
968) -> Result<Option<Option<f64>>, D::Error>
969where
970 D: serde::Deserializer<'de>,
971{
972 serde::Deserialize::deserialize(deserializer).map(Some)
973}
974
975#[derive(serde::Serialize, serde::Deserialize, schemars::JsonSchema)]
976struct PythonPatternDocument {
977 name: String,
978 regex: String,
979 match_count: usize,
980 #[serde(serialize_with = "dataprof_core::serde_helpers::round_2")]
981 match_percentage: f64,
982 category: dataprof_core::PatternCategory,
983 #[serde(serialize_with = "dataprof_core::serde_helpers::round_4")]
984 confidence: f64,
985}
986
987#[derive(serde::Serialize, serde::Deserialize, schemars::JsonSchema)]
988struct PythonQualityDocument {
989 overall_score: Option<f64>,
992 assessed_dimensions: Vec<PythonQualityDimension>,
993 dimension_scores: std::collections::BTreeMap<PythonQualityDimension, Option<f64>>,
994 low_sample_warning: bool,
995 #[serde(default, skip_serializing_if = "Option::is_none")]
1001 exact_dimensions: Option<Vec<String>>,
1002 #[serde(skip_serializing_if = "Option::is_none")]
1008 sampled_dimensions: Option<Vec<String>>,
1009 #[serde(default, skip_serializing_if = "Option::is_none")]
1013 score_bounds: Option<dataprof_metrics::ScoreBounds>,
1014 #[serde(
1020 default,
1021 skip_serializing_if = "Option::is_none",
1022 deserialize_with = "present_value"
1023 )]
1024 #[schemars(with = "Vec<dataprof_metrics::ColumnWithoutValueChecks>")]
1025 columns_without_value_checks: Option<Vec<dataprof_metrics::ColumnWithoutValueChecks>>,
1026 #[serde(default)]
1032 score_weights: QualityScoreWeights,
1033 #[serde(skip_serializing_if = "Option::is_none")]
1034 completeness: Option<CompletenessMetrics>,
1035 #[serde(skip_serializing_if = "Option::is_none")]
1036 consistency: Option<ConsistencyMetrics>,
1037 #[serde(skip_serializing_if = "Option::is_none")]
1038 uniqueness: Option<PythonUniquenessDocument>,
1039 #[serde(skip_serializing_if = "Option::is_none")]
1040 accuracy: Option<AccuracyMetrics>,
1041 #[serde(skip_serializing_if = "Option::is_none")]
1042 timeliness: Option<TimelinessMetrics>,
1043 #[serde(skip_serializing_if = "Option::is_none")]
1044 validity: Option<ValidityMetrics>,
1045 #[serde(skip_serializing_if = "Option::is_none")]
1046 precision: Option<PrecisionMetrics>,
1047}
1048
1049#[derive(
1050 Debug,
1051 Clone,
1052 PartialEq,
1053 Eq,
1054 PartialOrd,
1055 Ord,
1056 Hash,
1057 serde::Serialize,
1058 serde::Deserialize,
1059 schemars::JsonSchema,
1060)]
1061#[serde(rename_all = "lowercase")]
1062enum PythonQualityDimension {
1063 Completeness,
1064 Consistency,
1065 Uniqueness,
1066 Accuracy,
1067 Timeliness,
1068 Validity,
1069 Precision,
1070}
1071
1072#[derive(serde::Serialize, serde::Deserialize, schemars::JsonSchema)]
1073struct PythonUniquenessDocument {
1074 duplicate_rows: usize,
1075 key_uniqueness: f64,
1076 high_cardinality_warning: bool,
1077 rows_checked: usize,
1078 key_column: Option<String>,
1079 #[serde(default, skip_serializing_if = "Option::is_none")]
1080 key_values_checked: Option<usize>,
1081 duplicate_rows_approximate: bool,
1082 #[serde(default, skip_serializing_if = "Option::is_none")]
1083 duplicate_rows_exact_limit: Option<usize>,
1084}
1085
1086impl PythonProfileReportDocument {
1087 fn from_report(report: &ProfileReport) -> serde_json::Result<Self> {
1088 use dataprof_core::{ColumnStats, DataType, TruncationReason};
1089
1090 let columns = report
1093 .column_profiles
1094 .iter()
1095 .map(|column| {
1096 let stats = match &column.stats {
1097 ColumnStats::Numeric(stats) => Some(serde_json::to_value(stats)?),
1098 ColumnStats::Text(stats) if column.total_count > 0 => {
1099 Some(serde_json::to_value(stats)?)
1100 }
1101 ColumnStats::Boolean(stats) => Some(serde_json::to_value(stats)?),
1102 _ => None,
1103 }
1104 .map(serde_json::from_value)
1105 .transpose()?;
1106 Ok(PythonColumnDocument {
1107 name: column.name.clone(),
1108 data_type: match column.data_type {
1109 DataType::String => PythonDataType::String,
1110 DataType::Identifier => PythonDataType::Identifier,
1111 DataType::Integer => PythonDataType::Integer,
1112 DataType::Float => PythonDataType::Float,
1113 DataType::Date => PythonDataType::Date,
1114 DataType::Boolean => PythonDataType::Boolean,
1115 DataType::Nested => PythonDataType::Nested,
1116 },
1117 total_count: column.total_count,
1118 null_count: column.null_count,
1119 null_percentage: (column.total_count > 0)
1120 .then(|| column.null_count as f64 / column.total_count as f64 * 100.0),
1121 unique_count: column.unique_count,
1122 unique_count_is_approximate: column.unique_count_is_approximate,
1123 uniqueness_ratio: column
1124 .unique_count
1125 .filter(|_| column.total_count > 0)
1126 .map(|count| count as f64 / column.total_count as f64),
1127 invalid_count: column.invalid_count,
1128 type_homogeneity: column.type_homogeneity,
1129 locale_number_count: column.locale_number_count,
1130 native_type: column.native_type.clone(),
1131 stats,
1132 patterns: column.patterns.as_ref().map(|patterns| {
1133 patterns
1134 .iter()
1135 .map(|pattern| PythonPatternDocument {
1136 name: pattern.name.clone(),
1137 regex: pattern.regex.clone(),
1138 match_count: pattern.match_count,
1139 match_percentage: pattern.match_percentage,
1140 category: pattern.category.clone(),
1141 confidence: pattern.confidence,
1142 })
1143 .collect()
1144 }),
1145 })
1146 })
1147 .collect::<serde_json::Result<Vec<_>>>()?;
1148
1149 let mut execution = serde_json::to_value(&report.execution)?;
1150 execution["truncation_reason"] =
1151 serde_json::to_value(report.execution.truncation_reason.as_ref().map(|reason| {
1152 match reason {
1153 TruncationReason::MaxRows(n) => format!("max_rows({n})"),
1154 TruncationReason::MaxBytes(n) => format!("max_bytes({n})"),
1155 TruncationReason::MemoryPressure => "memory_pressure".to_string(),
1156 TruncationReason::StopCondition(s) => format!("stop_condition({s})"),
1157 TruncationReason::StreamClosed => "stream_closed".to_string(),
1158 TruncationReason::Timeout => "timeout".to_string(),
1159 }
1160 }))?;
1161
1162 let quality = report
1163 .quality
1164 .as_ref()
1165 .map(|assessment| {
1166 let metrics = &assessment.metrics;
1167 let mut document = serde_json::to_value(metrics)?;
1168 let scores = serde_json::to_value(assessment.scores())?;
1169 document["overall_score"] = scores["overall_score"].clone();
1170 let known: Vec<String> = dataprof_core::QualityDimension::all()
1174 .into_iter()
1175 .map(|dimension| dimension.to_string())
1176 .collect();
1177 document["dimension_scores"] = scores["dimension_scores"]
1178 .as_object()
1179 .expect("dimension scores serialize as an object")
1180 .iter()
1181 .filter(|(name, _)| known.contains(name))
1182 .map(|(name, score)| (name.clone(), score.clone()))
1183 .collect();
1184 document["assessed_dimensions"] =
1185 serde_json::to_value(metrics.assessed_dimensions())?;
1186 document["low_sample_warning"] =
1187 serde_json::Value::Bool(metrics.low_sample_warning);
1188 if let Some(exact) = assessment.exact_dimensions() {
1189 document["exact_dimensions"] = serde_json::to_value(exact)?;
1190 }
1191 if let Some(sampled) = assessment.sampled_dimensions() {
1192 document["sampled_dimensions"] = serde_json::to_value(sampled)?;
1193 }
1194 if let Some(bounds) = assessment.score_bounds() {
1195 document["score_bounds"] = serde_json::to_value(bounds)?;
1196 }
1197 if let Some(columns) = assessment.columns_without_value_checks() {
1198 document["columns_without_value_checks"] = serde_json::to_value(columns)?;
1199 }
1200 if let Some(uniqueness) = document.get_mut("uniqueness") {
1202 uniqueness["duplicate_rows_approximate"] = serde_json::Value::Bool(
1203 metrics
1204 .uniqueness
1205 .as_ref()
1206 .expect("serialized uniqueness is present")
1207 .duplicate_rows_approximate,
1208 );
1209 }
1210 serde_json::from_value(document)
1211 })
1212 .transpose()?;
1213
1214 Ok(Self {
1215 schema_version: report.schema_version,
1216 source: report.data_source.identifier(),
1217 source_type: match report.data_source {
1218 DataSource::File { .. } => PythonSourceType::File,
1219 DataSource::Query { .. } => PythonSourceType::Query,
1220 DataSource::DataFrame { .. } => PythonSourceType::Dataframe,
1221 DataSource::Stream { .. } => PythonSourceType::Stream,
1222 DataSource::Bytes { .. } => PythonSourceType::Bytes,
1223 },
1224 source_format: report.data_source.format().cloned(),
1225 execution: serde_json::from_value(execution)?,
1226 columns,
1227 quality,
1228 quality_status: report.quality_status.clone(),
1229 semantic_hint_bindings: report.semantic_hint_bindings.clone(),
1230 metric_semantics: report.metric_semantics.clone(),
1231 analysis_settings: report.analysis_settings.clone(),
1232 generator: report.generator.clone(),
1233 source_identity: report.source_identity.clone(),
1234 csv_metadata: report.data_source.csv_metadata().cloned(),
1235 })
1236 }
1237}
1238
1239#[doc(hidden)]
1245pub fn profile_report_schema_document() -> serde_json::Value {
1246 let settings = schemars::generate::SchemaSettings::draft2020_12().for_serialize();
1247 let schema = settings
1248 .into_generator()
1249 .into_root_schema_for::<SerializedProfileReport>();
1250 let mut document =
1251 serde_json::to_value(schema).expect("a Schemars schema must serialize to a JSON document");
1252 let object = document
1253 .as_object_mut()
1254 .expect("a root Schemars schema must be a JSON object");
1255 object.insert(
1256 "$id".to_string(),
1257 serde_json::Value::String(format!(
1258 "https://andreabozzo.github.io/dataprof/schema/profile-report.v{REPORT_SCHEMA_VERSION}.schema.json"
1259 )),
1260 );
1261 make_compatibility_defaults_optional(&mut document);
1262 allow_additive_properties(&mut document);
1263 canonicalize_key_order(&mut document);
1264 document
1265}
1266
1267fn canonicalize_key_order(value: &mut serde_json::Value) {
1276 match value {
1277 serde_json::Value::Object(object) => {
1278 for child in object.values_mut() {
1279 canonicalize_key_order(child);
1280 }
1281 let mut sorted: Vec<(String, serde_json::Value)> =
1282 std::mem::take(object).into_iter().collect();
1283 sorted.sort_by(|(left, _), (right, _)| left.cmp(right));
1284 object.extend(sorted);
1285 }
1286 serde_json::Value::Array(items) => {
1287 for item in items {
1288 canonicalize_key_order(item);
1289 }
1290 }
1291 _ => {}
1292 }
1293}
1294
1295fn make_compatibility_defaults_optional(document: &mut serde_json::Value) {
1302 for (pointer, field) in [
1303 ("/$defs/ExecutionMetadata/required", "ragged_row_count"),
1304 ("/$defs/ProfileReport/required", "quality_status"),
1305 (
1306 "/$defs/PythonProfileReportDocument/required",
1307 "quality_status",
1308 ),
1309 (
1310 "/$defs/PythonQualityDocument/required",
1311 "sampled_dimensions",
1312 ),
1313 ("/$defs/QualityAssessment/required", "scores"),
1314 ("/$defs/QualityMetrics/required", "score_weights"),
1316 ("/$defs/PythonQualityDocument/required", "score_weights"),
1317 ("/$defs/QualityAssessment/required", "score_bounds"),
1319 (
1321 "/$defs/QualityAssessment/required",
1322 "columns_without_value_checks",
1323 ),
1324 ("/$defs/AccuracyMetrics/required", "rule_values_checked"),
1326 ("/$defs/AccuracyMetrics/required", "rule_violating_values"),
1327 ] {
1328 if let Some(required) = document
1329 .pointer_mut(pointer)
1330 .and_then(serde_json::Value::as_array_mut)
1331 {
1332 required.retain(|declared| declared.as_str() != Some(field));
1333 }
1334 }
1335 if let Some(definitions) = document
1337 .pointer_mut("/$defs")
1338 .and_then(serde_json::Value::as_object_mut)
1339 {
1340 for definition in definitions.values_mut() {
1341 if let Some(object) = definition.as_object_mut()
1342 && object
1343 .get("required")
1344 .and_then(serde_json::Value::as_array)
1345 .is_some_and(Vec::is_empty)
1346 {
1347 object.remove("required");
1348 }
1349 }
1350 }
1351}
1352
1353fn allow_additive_properties(value: &mut serde_json::Value) {
1354 match value {
1355 serde_json::Value::Object(object) => {
1356 if object.get("additionalProperties") == Some(&serde_json::Value::Bool(false)) {
1357 object.remove("additionalProperties");
1358 }
1359 for child in object.values_mut() {
1360 allow_additive_properties(child);
1361 }
1362 }
1363 serde_json::Value::Array(items) => {
1364 for item in items {
1365 allow_additive_properties(item);
1366 }
1367 }
1368 _ => {}
1369 }
1370}
1371
1372impl ProfileReport {
1373 pub fn to_json(&self) -> serde_json::Result<String> {
1376 let mut document = serde_json::to_value(self)?;
1377 canonicalize_key_order(&mut document);
1378 serde_json::to_string_pretty(&document)
1379 }
1380
1381 #[doc(hidden)]
1384 pub fn summary_json(&self) -> serde_json::Result<String> {
1385 let summary = PythonProfileReportDocument::from_report(self)?;
1386 serde_json::to_string(&summary)
1387 }
1388
1389 pub fn new(
1391 data_source: DataSource,
1392 column_profiles: Vec<ColumnProfile>,
1393 execution: ExecutionMetadata,
1394 quality: Option<QualityAssessment>,
1395 ) -> Self {
1396 Self {
1397 schema_version: REPORT_SCHEMA_VERSION,
1398 id: uuid::Uuid::new_v4().to_string(),
1399 timestamp: chrono::Utc::now().to_rfc3339(),
1400 data_source,
1401 column_profiles,
1402 execution,
1403 quality_status: match quality {
1404 Some(_) => QualityAnalysisStatus::Computed,
1405 None => QualityAnalysisStatus::NotRequested,
1409 },
1410 quality,
1411 semantic_hint_bindings: Vec::new(),
1412 metric_semantics: Some(MetricSemantics::current()),
1413 analysis_settings: None,
1416 generator: Some(ReportGenerator::current()),
1417 source_identity: None,
1418 }
1419 }
1420
1421 pub(crate) fn with_analysis_settings(mut self, settings: Option<AnalysisSettings>) -> Self {
1423 self.analysis_settings = settings;
1424 self
1425 }
1426
1427 pub(crate) fn with_quality_status(mut self, status: QualityAnalysisStatus) -> Self {
1437 self.quality_status = status;
1438 self
1439 }
1440
1441 pub fn with_semantic_hint_bindings(mut self, bindings: Vec<SemanticHintBinding>) -> Self {
1443 self.semantic_hint_bindings = bindings;
1444 self
1445 }
1446
1447 pub fn with_id(mut self, id: impl Into<String>) -> Self {
1449 self.id = id.into();
1450 self
1451 }
1452
1453 pub fn with_timestamp(mut self, timestamp: impl Into<String>) -> Self {
1455 self.timestamp = timestamp.into();
1456 self
1457 }
1458
1459 pub fn quality_score(&self) -> Option<f64> {
1464 self.quality.as_ref().and_then(|q| q.score())
1465 }
1466
1467 pub fn source_identifier(&self) -> String {
1469 self.data_source.identifier()
1470 }
1471
1472 fn check_quality_pairing(self) -> Result<Self, String> {
1483 let assessed = self.quality.is_some();
1484 let claims_computed = matches!(self.quality_status, QualityAnalysisStatus::Computed);
1485 if claims_computed == assessed {
1486 return Ok(self);
1487 }
1488 Err(if assessed {
1489 format!(
1490 "report carries a quality assessment but reports quality_status \
1491 {:?}; only `computed` may accompany an assessment",
1492 self.quality_status
1493 )
1494 } else {
1495 "report reports quality_status `computed` but carries no quality assessment".to_string()
1496 })
1497 }
1498}
1499
1500#[derive(serde::Deserialize)]
1505struct ProfileReportFields {
1506 #[serde(default)]
1507 schema_version: u32,
1508 id: String,
1509 timestamp: String,
1510 data_source: DataSource,
1511 column_profiles: Vec<ColumnProfile>,
1512 #[serde(alias = "scan_info")]
1513 execution: ExecutionMetadata,
1514 #[serde(
1515 alias = "data_quality_metrics",
1516 default,
1517 deserialize_with = "deserialize_quality_compat"
1518 )]
1519 quality: Option<QualityAssessment>,
1520 #[serde(default)]
1521 quality_status: Option<QualityAnalysisStatus>,
1522 #[serde(default)]
1523 semantic_hint_bindings: Vec<SemanticHintBinding>,
1524 #[serde(default)]
1525 metric_semantics: Option<MetricSemantics>,
1526 #[serde(default)]
1527 analysis_settings: Option<AnalysisSettings>,
1528 #[serde(default)]
1529 generator: Option<ReportGenerator>,
1530 #[serde(default)]
1531 source_identity: Option<SourceIdentity>,
1532}
1533
1534impl From<ProfileReportFields> for ProfileReport {
1535 fn from(fields: ProfileReportFields) -> Self {
1536 Self {
1537 schema_version: fields.schema_version,
1538 id: fields.id,
1539 timestamp: fields.timestamp,
1540 data_source: fields.data_source,
1541 column_profiles: fields.column_profiles,
1542 execution: fields.execution,
1543 quality_status: fields.quality_status.unwrap_or({
1546 if fields.quality.is_some() {
1547 QualityAnalysisStatus::Computed
1548 } else {
1549 QualityAnalysisStatus::Unrecorded
1550 }
1551 }),
1552 quality: fields.quality,
1553 semantic_hint_bindings: fields.semantic_hint_bindings,
1554 metric_semantics: fields.metric_semantics,
1555 analysis_settings: fields.analysis_settings,
1556 generator: fields.generator,
1557 source_identity: fields.source_identity,
1558 }
1559 }
1560}
1561
1562impl<'de> serde::Deserialize<'de> for ProfileReport {
1568 fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
1569 where
1570 D: serde::Deserializer<'de>,
1571 {
1572 use serde::de::Error;
1573
1574 let value = serde_json::Value::deserialize(deserializer)?;
1575 match value.get("schema_version") {
1576 None => {}
1578 Some(serde_json::Value::Number(n)) if n.as_u64().is_some() => {
1579 let version = n.as_u64().unwrap_or_default();
1580 if version > u64::from(REPORT_SCHEMA_VERSION) {
1581 return Err(D::Error::custom(format!(
1582 "report schema version {version} is newer than the latest supported \
1583 version {REPORT_SCHEMA_VERSION}; upgrade dataprof to read this report"
1584 )));
1585 }
1586 }
1587 Some(other) => {
1589 return Err(D::Error::custom(format!(
1590 "report schema_version must be a non-negative integer, got {other}"
1591 )));
1592 }
1593 }
1594 if value.get("quality_status") == Some(&serde_json::Value::Null) {
1599 return Err(D::Error::custom(
1600 "report quality_status must be an object; an explicit null is malformed, \
1601 not a document written before the field existed",
1602 ));
1603 }
1604 for key in [
1607 "metric_semantics",
1608 "analysis_settings",
1609 "generator",
1610 "source_identity",
1611 ] {
1612 if value.get(key) == Some(&serde_json::Value::Null) {
1613 return Err(D::Error::custom(format!(
1614 "report {key} must be an object; an explicit null is malformed, \
1615 not a document written before the field existed"
1616 )));
1617 }
1618 }
1619 let report = ProfileReportFields::deserialize(value)
1620 .map(ProfileReport::from)
1621 .map_err(D::Error::custom)?;
1622 report.check_quality_pairing().map_err(D::Error::custom)
1623 }
1624}
1625
1626fn deserialize_quality_compat<'de, D>(
1629 deserializer: D,
1630) -> Result<Option<QualityAssessment>, D::Error>
1631where
1632 D: serde::Deserializer<'de>,
1633{
1634 use serde::Deserialize;
1635
1636 let value: Option<serde_json::Value> = Option::deserialize(deserializer)?;
1637 match value {
1638 None => Ok(None),
1639 Some(mut v) => {
1640 if v.get("metrics").is_some() || v.get("confidence").is_some() {
1644 if let Some(object) = v.as_object_mut() {
1645 object
1648 .entry("confidence")
1649 .or_insert_with(|| serde_json::json!("Unrecorded"));
1650 }
1651 let assessment: QualityAssessment =
1652 serde_json::from_value(v).map_err(serde::de::Error::custom)?;
1653 Ok(Some(assessment))
1654 } else {
1655 let metrics: QualityMetrics =
1656 serde_json::from_value(v).map_err(serde::de::Error::custom)?;
1657 Ok(Some(QualityAssessment::new(
1662 metrics,
1663 MetricConfidence::Unrecorded,
1664 )))
1665 }
1666 }
1667 }
1668}
1669
1670#[cfg(test)]
1671mod tests {
1672 use super::*;
1673 use dataprof_core::FileFormat;
1674 use dataprof_metrics::MetricConfidence;
1675 use serde_json::json;
1676
1677 fn report_without_quality() -> ProfileReport {
1678 ProfileReport::new(
1679 DataSource::File {
1680 path: "test.csv".to_string(),
1681 format: FileFormat::Csv,
1682 size_bytes: 1024,
1683 modified_at: None,
1684 parquet_metadata: None,
1685 csv_metadata: None,
1686 },
1687 vec![],
1688 ExecutionMetadata::new(100, 5, 50),
1689 None,
1690 )
1691 }
1692
1693 #[test]
1697 fn every_status_has_a_stable_wire_name() {
1698 let cases = [
1699 (QualityAnalysisStatus::Computed, "computed"),
1700 (QualityAnalysisStatus::NotRequested, "not_requested"),
1701 (QualityAnalysisStatus::NoData, "no_data"),
1702 (
1703 QualityAnalysisStatus::WithheldByProjection,
1704 "withheld_by_projection",
1705 ),
1706 (
1707 QualityAnalysisStatus::Failed {
1708 error: "boom".to_string(),
1709 },
1710 "failed",
1711 ),
1712 (QualityAnalysisStatus::Unrecorded, "unrecorded"),
1713 ];
1714
1715 for (status, name) in cases {
1716 let value = serde_json::to_value(&status).unwrap();
1717 assert_eq!(value.get("state"), Some(&json!(name)), "{status:?}");
1718 let restored: QualityAnalysisStatus = serde_json::from_value(value).unwrap();
1719 assert_eq!(restored, status);
1720 }
1721 }
1722
1723 #[test]
1727 fn contradictory_status_and_assessment_fail_to_decode() {
1728 let with_assessment =
1729 report_without_quality().with_quality_status(QualityAnalysisStatus::Failed {
1730 error: "boom".to_string(),
1731 });
1732 let mut document = serde_json::to_value(&with_assessment).unwrap();
1733 document.as_object_mut().unwrap().insert(
1734 "quality".to_string(),
1735 serde_json::to_value(QualityAssessment::exact(QualityMetrics::empty())).unwrap(),
1736 );
1737 let error = serde_json::from_value::<ProfileReport>(document)
1738 .expect_err("an assessment under a `failed` status is not a readable report");
1739 assert!(
1740 error.to_string().contains("only `computed`"),
1741 "unhelpful error: {error}"
1742 );
1743
1744 let mut document = serde_json::to_value(
1745 report_without_quality().with_quality_status(QualityAnalysisStatus::Computed),
1746 )
1747 .unwrap();
1748 document.as_object_mut().unwrap().remove("quality");
1749 let error = serde_json::from_value::<ProfileReport>(document)
1750 .expect_err("`computed` without an assessment is not a readable report");
1751 assert!(
1752 error.to_string().contains("carries no quality assessment"),
1753 "unhelpful error: {error}"
1754 );
1755 }
1756
1757 #[test]
1761 fn an_explicit_null_status_is_malformed_not_legacy() {
1762 let mut document = serde_json::to_value(report_without_quality()).unwrap();
1763 document
1764 .as_object_mut()
1765 .unwrap()
1766 .insert("quality_status".to_string(), serde_json::Value::Null);
1767
1768 let error = serde_json::from_value::<ProfileReport>(document)
1769 .expect_err("an explicit null is not a readable status");
1770 assert!(
1771 error.to_string().contains("explicit null is malformed"),
1772 "unhelpful error: {error}"
1773 );
1774 }
1775
1776 fn summary_value(report: &ProfileReport) -> serde_json::Value {
1777 serde_json::from_str(&report.summary_json().unwrap()).unwrap()
1778 }
1779
1780 #[test]
1784 fn metric_semantics_are_recorded_and_absence_survives_a_roundtrip() {
1785 let report = report_without_quality();
1786 assert_eq!(report.metric_semantics, Some(MetricSemantics::current()));
1787 let mut document = serde_json::to_value(&report).unwrap();
1788 let recorded = json!({
1789 "text_length_unit": "unicode_scalar",
1790 "null_tokens": "common_markers",
1791 "pattern_match_count": "validated",
1792 "shape_moments": "adjusted_sample",
1793 "statistic_rounding": "four_decimals_twelve_significant",
1794 "range_rule_count": "violating_values",
1795 "key_uniqueness_units": "non_null_values",
1796 "outlier_units": "tested_values",
1797 "key_column_source": "declared",
1798 "accuracy_comparison": "exact_values",
1799 "json_number_reading": "literal_digits",
1800 "sampled_score_estimator": "population_weighted",
1801 "date_statistics": "every_value",
1802 "outlier_spread": "nonzero_iqr",
1803 "date_forms": "spaced_iso_datetimes",
1804 });
1805 assert_eq!(document["metric_semantics"], recorded);
1806 assert_eq!(summary_value(&report)["metric_semantics"], recorded);
1807
1808 document.as_object_mut().unwrap().remove("metric_semantics");
1809 let legacy: ProfileReport = serde_json::from_value(document).unwrap();
1810 assert_eq!(legacy.metric_semantics, None);
1811 assert!(
1812 serde_json::to_value(&legacy)
1813 .unwrap()
1814 .get("metric_semantics")
1815 .is_none()
1816 );
1817 assert!(summary_value(&legacy).get("metric_semantics").is_none());
1818 }
1819
1820 #[test]
1821 fn malformed_metric_semantics_fail_to_decode() {
1822 for value in [
1823 serde_json::Value::Null,
1824 json!("unicode_scalar"),
1825 json!({"text_length_unit": "utf8_byte"}),
1826 json!({"text_length_unit": null}),
1827 json!({"null_tokens": "empty_null_nan"}),
1828 json!({"null_tokens": null}),
1829 json!({"pattern_match_count": "regex"}),
1830 json!({"pattern_match_count": null}),
1831 json!({"shape_moments": "population"}),
1832 json!({"shape_moments": null}),
1833 json!({"range_rule_count": "rule_failures"}),
1834 json!({"range_rule_count": null}),
1835 json!({"key_uniqueness_units": "rows"}),
1836 json!({"key_uniqueness_units": null}),
1837 json!({"outlier_units": "numeric_values"}),
1838 json!({"outlier_units": null}),
1839 json!({"key_column_source": "name_heuristic"}),
1840 json!({"key_column_source": null}),
1841 json!({"accuracy_comparison": "nearest_f64"}),
1842 json!({"accuracy_comparison": null}),
1843 json!({"json_number_reading": "nearest_f64"}),
1844 json!({"json_number_reading": null}),
1845 json!({"outlier_spread": "every_column"}),
1846 json!({"outlier_spread": null}),
1847 json!({"date_forms": "plain_spaced_datetimes"}),
1848 json!({"date_forms": null}),
1849 ] {
1850 let mut document = serde_json::to_value(report_without_quality()).unwrap();
1851 document["metric_semantics"] = value.clone();
1852 assert!(
1853 serde_json::from_value::<ProfileReport>(document).is_err(),
1854 "{value} decoded"
1855 );
1856 }
1857 }
1858
1859 #[test]
1861 fn unknown_metric_definitions_are_ignored() {
1862 let mut document = serde_json::to_value(report_without_quality()).unwrap();
1863 document["metric_semantics"]["grapheme_policy"] = json!("extended");
1864 let restored: ProfileReport = serde_json::from_value(document).unwrap();
1865 assert_eq!(restored.metric_semantics, Some(MetricSemantics::current()));
1866 }
1867
1868 #[test]
1869 fn quality_status_survives_a_json_roundtrip() {
1870 let report = report_without_quality().with_quality_status(QualityAnalysisStatus::Failed {
1871 error: "Metrics calculation failed: no data columns found".to_string(),
1872 });
1873
1874 let json = serde_json::to_string(&report).unwrap();
1875 let restored: ProfileReport = serde_json::from_str(&json).unwrap();
1876
1877 assert_eq!(restored.quality_status, report.quality_status);
1878 assert!(restored.quality.is_none());
1879 }
1880
1881 #[test]
1884 fn serialized_failure_and_skip_differ() {
1885 let failed = serde_json::to_value(report_without_quality().with_quality_status(
1886 QualityAnalysisStatus::Failed {
1887 error: "boom".to_string(),
1888 },
1889 ))
1890 .unwrap();
1891 let skipped = serde_json::to_value(
1892 report_without_quality().with_quality_status(QualityAnalysisStatus::NotRequested),
1893 )
1894 .unwrap();
1895
1896 assert_eq!(
1897 failed.get("quality_status"),
1898 Some(&json!({"state": "failed", "error": "boom"}))
1899 );
1900 assert_eq!(
1901 skipped.get("quality_status"),
1902 Some(&json!({"state": "not_requested"}))
1903 );
1904 assert!(failed.get("quality").is_none());
1905 assert!(skipped.get("quality").is_none());
1906 }
1907
1908 #[test]
1911 fn documents_without_the_field_read_back_honestly() {
1912 let mut document = serde_json::to_value(report_without_quality()).unwrap();
1913 document.as_object_mut().unwrap().remove("quality_status");
1914
1915 let restored: ProfileReport = serde_json::from_value(document.clone()).unwrap();
1916 assert_eq!(restored.quality_status, QualityAnalysisStatus::Unrecorded);
1917
1918 document.as_object_mut().unwrap().insert(
1919 "quality".to_string(),
1920 serde_json::to_value(QualityAssessment::exact(QualityMetrics::empty())).unwrap(),
1921 );
1922 let restored: ProfileReport = serde_json::from_value(document).unwrap();
1923 assert_eq!(restored.quality_status, QualityAnalysisStatus::Computed);
1924 }
1925
1926 #[test]
1927 fn test_profile_report_json_roundtrip() {
1928 let report = ProfileReport::new(
1929 DataSource::File {
1930 path: "test.csv".to_string(),
1931 format: FileFormat::Csv,
1932 size_bytes: 1024,
1933 modified_at: None,
1934 parquet_metadata: None,
1935 csv_metadata: None,
1936 },
1937 vec![],
1938 ExecutionMetadata::new(100, 5, 50),
1939 Some(QualityAssessment::exact(QualityMetrics::empty())),
1940 );
1941
1942 let json = serde_json::to_string(&report).unwrap();
1943 let deserialized: ProfileReport = serde_json::from_str(&json).unwrap();
1944
1945 assert_eq!(deserialized.id, report.id);
1946 assert_eq!(deserialized.timestamp, report.timestamp);
1947 assert_eq!(deserialized.source_identifier(), "test.csv");
1948 assert_eq!(deserialized.execution.rows_processed, 100);
1949 assert!(deserialized.quality.is_some());
1950 assert_eq!(deserialized.schema_version, REPORT_SCHEMA_VERSION);
1951 }
1952
1953 #[test]
1956 fn summary_names_its_score_weights_including_defaults() {
1957 let summary_quality = |weights| {
1958 let mut metrics = QualityMetrics::empty();
1959 metrics.score_weights = weights;
1960 let report = ProfileReport::new(
1961 DataSource::File {
1962 path: "test.csv".to_string(),
1963 format: FileFormat::Csv,
1964 size_bytes: 1024,
1965 modified_at: None,
1966 parquet_metadata: None,
1967 csv_metadata: None,
1968 },
1969 vec![],
1970 ExecutionMetadata::new(100, 5, 50),
1971 Some(QualityAssessment::exact(metrics)),
1972 );
1973 let summary: serde_json::Value =
1974 serde_json::from_str(&report.summary_json().unwrap()).unwrap();
1975 summary["quality"].clone()
1976 };
1977
1978 let custom = QualityScoreWeights {
1979 completeness: 1.0,
1980 consistency: 0.0,
1981 uniqueness: 0.0,
1982 accuracy: 0.0,
1983 timeliness: 0.0,
1984 validity: 0.0,
1985 precision: 3.0,
1986 };
1987 assert_eq!(
1988 summary_quality(custom)["score_weights"],
1989 serde_json::to_value(custom).unwrap()
1990 );
1991 assert_eq!(
1994 summary_quality(QualityScoreWeights::default())["score_weights"],
1995 serde_json::to_value(QualityScoreWeights::default()).unwrap()
1996 );
1997 }
1998
1999 #[test]
2000 fn test_serialized_report_carries_schema_version() {
2001 let report = ProfileReport::new(
2002 DataSource::File {
2003 path: "test.csv".to_string(),
2004 format: FileFormat::Csv,
2005 size_bytes: 1024,
2006 modified_at: None,
2007 parquet_metadata: None,
2008 csv_metadata: None,
2009 },
2010 vec![],
2011 ExecutionMetadata::new(100, 5, 50),
2012 None,
2013 );
2014
2015 let value = serde_json::to_value(&report).unwrap();
2016 assert_eq!(
2017 value.get("schema_version").and_then(|v| v.as_u64()),
2018 Some(u64::from(REPORT_SCHEMA_VERSION))
2019 );
2020 }
2021
2022 #[test]
2023 fn test_profile_report_without_quality() {
2024 let report = ProfileReport::new(
2025 DataSource::File {
2026 path: "test.csv".to_string(),
2027 format: FileFormat::Csv,
2028 size_bytes: 1024,
2029 modified_at: None,
2030 parquet_metadata: None,
2031 csv_metadata: None,
2032 },
2033 vec![],
2034 ExecutionMetadata::new(100, 5, 50),
2035 None,
2036 );
2037
2038 let json = serde_json::to_string(&report).unwrap();
2039 let deserialized: ProfileReport = serde_json::from_str(&json).unwrap();
2040
2041 assert!(deserialized.quality.is_none());
2042 assert_eq!(deserialized.execution.rows_processed, 100);
2043 }
2044
2045 #[test]
2046 fn test_profile_report_deserializes_legacy_quality_metrics() {
2047 let json = json!({
2048 "id": "legacy-report",
2049 "timestamp": "2026-05-22T10:00:00Z",
2050 "data_source": {
2051 "type": "file",
2052 "path": "test.csv",
2053 "format": "csv",
2054 "size_bytes": 42
2055 },
2056 "column_profiles": [],
2057 "scan_info": {
2058 "rows_processed": 10,
2059 "columns_detected": 2,
2060 "scan_time_ms": 5,
2061 "error_count": 0,
2062 "source_exhausted": true,
2063 "sampling_applied": false
2064 },
2065 "data_quality_metrics": {
2066 "completeness": {
2067 "missing_values_ratio": 0.0,
2068 "complete_records_ratio": 100.0,
2069 "null_columns": []
2070 }
2071 }
2072 });
2073
2074 let report: ProfileReport = serde_json::from_value(json).unwrap();
2075
2076 assert_eq!(report.id, "legacy-report");
2077 assert_eq!(report.schema_version, 0);
2080 assert_eq!(report.execution.rows_processed, 10);
2081 assert!(report.quality_score().is_none());
2084 let quality = report
2085 .quality
2086 .expect("expected legacy quality to deserialize");
2087 assert!(matches!(quality.confidence, MetricConfidence::NotAssessed));
2090 let completeness = quality
2091 .metrics
2092 .completeness
2093 .as_ref()
2094 .expect("legacy completeness facts should deserialize");
2095 assert!((completeness.complete_records_ratio - 100.0).abs() < 0.01);
2096 assert!(quality.metrics.assessed_dimensions().is_empty());
2097 }
2098
2099 #[test]
2103 fn a_canonical_quality_without_confidence_keeps_its_metrics() {
2104 let mut document = current_document();
2105 document["quality"] = json!({
2106 "metrics": {
2107 "completeness": {
2108 "missing_values_ratio": 0.0,
2109 "complete_records_ratio": 100.0,
2110 "null_columns": []
2111 }
2112 }
2113 });
2114
2115 let report: ProfileReport = serde_json::from_value(document).unwrap();
2116
2117 let quality = report.quality.expect("the assessment survives");
2118 let completeness = quality
2119 .metrics
2120 .completeness
2121 .as_ref()
2122 .expect("the metrics survive");
2123 assert!((completeness.complete_records_ratio - 100.0).abs() < 0.01);
2124 assert!(!matches!(quality.confidence, MetricConfidence::Exact));
2125 }
2126
2127 #[test]
2130 fn a_canonical_quality_without_metrics_is_malformed() {
2131 let mut document = current_document();
2132 document["quality"] = json!({ "confidence": "Exact" });
2133
2134 assert!(serde_json::from_value::<ProfileReport>(document).is_err());
2135 }
2136
2137 fn current_document() -> serde_json::Value {
2138 json!({
2139 "schema_version": REPORT_SCHEMA_VERSION,
2140 "id": "current-report",
2141 "timestamp": "2026-07-16T10:00:00Z",
2142 "data_source": {
2143 "type": "file",
2144 "path": "test.csv",
2145 "format": "csv",
2146 "size_bytes": 42
2147 },
2148 "column_profiles": [],
2149 "execution": {
2150 "rows_processed": 10,
2151 "columns_detected": 2,
2152 "scan_time_ms": 5,
2153 "error_count": 0,
2154 "source_exhausted": true,
2155 "sampling_applied": false
2156 }
2157 })
2158 }
2159
2160 #[test]
2161 fn test_additive_fields_from_newer_writer_are_ignored() {
2162 let mut json = current_document();
2165 json["a_future_additive_field"] = json!({"anything": true});
2166 json["column_profiles"] = json!([]);
2167
2168 let report: ProfileReport = serde_json::from_value(json).unwrap();
2169 assert_eq!(report.schema_version, REPORT_SCHEMA_VERSION);
2170 assert_eq!(report.id, "current-report");
2171 }
2172
2173 #[test]
2174 fn test_unsupported_future_schema_version_fails_explicitly() {
2175 let mut json = current_document();
2176 json["schema_version"] = json!(REPORT_SCHEMA_VERSION + 1);
2177
2178 let err = serde_json::from_value::<ProfileReport>(json).unwrap_err();
2179 let msg = err.to_string();
2180 assert!(
2181 msg.contains("schema version") && msg.contains("upgrade dataprof"),
2182 "expected an actionable schema-version error, got: {msg}"
2183 );
2184 }
2185
2186 #[test]
2187 fn test_version_error_wins_over_structural_errors() {
2188 let json_text = format!(
2192 r#"{{"column_profiles": "not-an-array", "schema_version": {}}}"#,
2193 REPORT_SCHEMA_VERSION + 1
2194 );
2195 let err = serde_json::from_str::<ProfileReport>(&json_text).unwrap_err();
2196 let msg = err.to_string();
2197 assert!(
2198 msg.contains("upgrade dataprof"),
2199 "expected the schema-version error to win, got: {msg}"
2200 );
2201 }
2202
2203 #[test]
2204 fn test_null_schema_version_is_malformed_not_legacy() {
2205 let mut json = current_document();
2206 json["schema_version"] = json!(null);
2207
2208 let err = serde_json::from_value::<ProfileReport>(json).unwrap_err();
2209 let msg = err.to_string();
2210 assert!(
2211 msg.contains("schema_version must be a non-negative integer"),
2212 "expected explicit rejection of null schema_version, got: {msg}"
2213 );
2214 }
2215
2216 #[test]
2217 fn test_non_integer_schema_version_is_rejected() {
2218 for bad in [json!("1"), json!(1.5), json!(-1), json!(true)] {
2219 let mut json = current_document();
2220 json["schema_version"] = bad.clone();
2221 let err = serde_json::from_value::<ProfileReport>(json).unwrap_err();
2222 assert!(
2223 err.to_string().contains("non-negative integer"),
2224 "expected rejection of {bad}, got: {err}"
2225 );
2226 }
2227 }
2228}