Skip to main content

dataprof_runtime/
profile_report.rs

1use std::sync::OnceLock;
2
3use dataprof_core::{
4    AccuracyComparison, AnalysisOptions, ColumnProfile, CsvSourceMetadata, DataSource, DateForms,
5    DateStatistics, ExecutionMetadata, FileFormat, JsonNumberReading, KeyColumnSource,
6    KeyUniquenessUnits, Locale, MetricPack, NullTokenSet, OutlierSpread, OutlierUnits,
7    PatternMatchCount, QualityScoreWeights, RangeRuleCount, SampledScoreEstimator,
8    SemanticHintBinding, ShapeMomentEstimator, StatisticRounding, TextLengthUnit,
9};
10use dataprof_metrics::{
11    AccuracyMetrics, CompletenessMetrics, ConsistencyMetrics, MetricConfidence, PrecisionMetrics,
12    QualityAssessment, QualityMetrics, TimelinessMetrics, ValidityMetrics,
13};
14
15/// Version of the serialized `ProfileReport` schema written by this build.
16///
17/// This is intentionally independent of the package version: the document
18/// format only changes when the schema itself changes, not on every release.
19///
20/// Compatibility policy for readers:
21/// - Documents without a `schema_version` field are legacy pre-0.10 reports
22///   and deserialize with `schema_version == 0`.
23/// - Unknown *additive* fields written by a newer dataprof are ignored, so a
24///   reader accepts any document whose `schema_version` is at most this
25///   constant.
26/// - A document with a `schema_version` greater than this constant fails to
27///   deserialize with an explicit error instead of being partially decoded
28///   into a plausible-but-wrong report.
29pub const REPORT_SCHEMA_VERSION: u32 = 1;
30
31/// Why a report does or does not carry a quality assessment.
32///
33/// `quality` alone cannot answer that. A run that never asked for quality
34/// metrics and a run whose quality computation failed both leave it `None`,
35/// so absence itself became the plausible value that hid the failure. This
36/// names the difference: a consumer deciding on a report can tell "you did
37/// not ask for this" from "this broke" from "there was nothing to measure".
38#[derive(
39    Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize, schemars::JsonSchema,
40)]
41#[serde(tag = "state", rename_all = "snake_case")]
42pub enum QualityAnalysisStatus {
43    /// Computed: `quality` carries the assessment.
44    Computed,
45    /// Not requested for this run — the quality pack was deselected.
46    NotRequested,
47    /// Requested, but no quality sample was supplied to the assembler.
48    /// An empty sample is analyzed and reports `Computed`, even without columns.
49    NoData,
50    /// Requested, but every requested dimension measures whole rows and the
51    /// run profiled a subset of columns. Completeness and uniqueness mean
52    /// something else after projection, and the report cannot label only
53    /// their row-level fields as projected, so they are withheld rather than
54    /// published under full-row names.
55    WithheldByProjection,
56    /// Requested and attempted; the computation failed. `quality` is absent
57    /// because the computation broke, not because nothing was asked for.
58    Failed {
59        /// The error the metrics calculator reported.
60        error: String,
61    },
62    /// Written by a release that did not record this. Only reachable by
63    /// deserializing a document from before the field existed that carries no
64    /// quality assessment; a stored assessment is read back as `Computed`.
65    Unrecorded,
66}
67
68/// How the measurements in a report were defined.
69///
70/// `schema_version` answers whether a document validates. It cannot answer
71/// whether two valid documents measured the same way: text lengths counted
72/// UTF-8 bytes through 0.11 and count Unicode scalar values since, with no
73/// change to the document's shape. This records each intentionally changed
74/// definition by name, so a comparison can check exactly the ones both
75/// reports declare.
76///
77/// Every field is optional. A field missing from a stored report means the
78/// release that wrote it did not record that definition: unknown, never the
79/// current one. Later releases add fields, which older readers ignore.
80#[derive(
81    Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize, schemars::JsonSchema,
82)]
83#[non_exhaustive]
84pub struct MetricSemantics {
85    /// Unit of `min_length`, `max_length` and `avg_length` on text columns.
86    ///
87    /// Absent means not recorded; an explicit null is malformed, so neither
88    /// the reader nor the schema accepts one.
89    #[serde(
90        default,
91        skip_serializing_if = "Option::is_none",
92        deserialize_with = "present_value"
93    )]
94    #[schemars(with = "TextLengthUnit")]
95    pub text_length_unit: Option<TextLengthUnit>,
96    /// The text values counted as null, which every `null_count` and every
97    /// completeness figure depend on.
98    ///
99    /// Absent means not recorded: the release that wrote the report counted
100    /// only empty, `null` and `nan`, or recorded nothing at all.
101    #[serde(
102        default,
103        skip_serializing_if = "Option::is_none",
104        deserialize_with = "present_value"
105    )]
106    #[schemars(with = "NullTokenSet")]
107    pub null_tokens: Option<NullTokenSet>,
108    /// What every pattern's `match_count` and `match_percentage` count.
109    ///
110    /// Absent means not recorded: through 0.12 they counted every regular
111    /// expression match, validated or not.
112    #[serde(
113        default,
114        skip_serializing_if = "Option::is_none",
115        deserialize_with = "present_value"
116    )]
117    #[schemars(with = "PatternMatchCount")]
118    pub pattern_match_count: Option<PatternMatchCount>,
119    /// The estimator behind every column's `skewness` and `kurtosis`.
120    ///
121    /// Absent means not recorded: through 0.12 both used a form that matches
122    /// neither the population nor the adjusted estimators.
123    #[serde(
124        default,
125        skip_serializing_if = "Option::is_none",
126        deserialize_with = "present_value"
127    )]
128    #[schemars(with = "ShapeMomentEstimator")]
129    pub shape_moments: Option<ShapeMomentEstimator>,
130    /// How every column's `mean`, `std_dev`, `variance`, `skewness` and
131    /// `kurtosis` are rounded when serialized.
132    ///
133    /// Absent means not recorded: through 0.13 they kept 4 decimal places at
134    /// any magnitude, where accumulation order could change the last digits.
135    #[serde(
136        default,
137        skip_serializing_if = "Option::is_none",
138        deserialize_with = "present_value"
139    )]
140    #[schemars(with = "StatisticRounding")]
141    pub statistic_rounding: Option<StatisticRounding>,
142    /// What the range and sign rules behind the accuracy score count.
143    ///
144    /// Absent means not recorded: through 0.12 every rule failure counted,
145    /// over the numeric values of every column.
146    #[serde(
147        default,
148        skip_serializing_if = "Option::is_none",
149        deserialize_with = "present_value"
150    )]
151    #[schemars(with = "RangeRuleCount")]
152    pub range_rule_count: Option<RangeRuleCount>,
153    /// What `key_uniqueness` divides the key's distinct values by.
154    ///
155    /// Absent means not recorded: through 0.12 it divided by every row,
156    /// nulls included.
157    #[serde(
158        default,
159        skip_serializing_if = "Option::is_none",
160        deserialize_with = "present_value"
161    )]
162    #[schemars(with = "KeyUniquenessUnits")]
163    pub key_uniqueness_units: Option<KeyUniquenessUnits>,
164    /// What `outlier_ratio` is measured over.
165    ///
166    /// Absent means not recorded: through 0.12 an untested source read 0 and
167    /// still assessed accuracy.
168    #[serde(
169        default,
170        skip_serializing_if = "Option::is_none",
171        deserialize_with = "present_value"
172    )]
173    #[schemars(with = "OutlierUnits")]
174    pub outlier_units: Option<OutlierUnits>,
175    /// Which column `key_uniqueness` may measure.
176    ///
177    /// Absent means not recorded: through 0.12 the first column with an
178    /// id-like name was the key when none was declared.
179    #[serde(
180        default,
181        skip_serializing_if = "Option::is_none",
182        deserialize_with = "present_value"
183    )]
184    #[schemars(with = "KeyColumnSource")]
185    pub key_column_source: Option<KeyColumnSource>,
186    /// How the accuracy score compares a value with its outlier fences and
187    /// its range and sign bounds.
188    ///
189    /// Absent means not recorded: through 0.13 both comparisons went through
190    /// the value's nearest `f64`.
191    #[serde(
192        default,
193        skip_serializing_if = "Option::is_none",
194        deserialize_with = "present_value"
195    )]
196    #[schemars(with = "AccuracyComparison")]
197    pub accuracy_comparison: Option<AccuracyComparison>,
198    /// How a JSON number whose value its nearest `f64` does not hold is read.
199    ///
200    /// Absent means not recorded: through 0.13 such a number was read as its
201    /// nearest `f64`, which moved the consistency, precision, uniqueness,
202    /// accuracy and overall scores of the same bytes.
203    #[serde(
204        default,
205        skip_serializing_if = "Option::is_none",
206        deserialize_with = "present_value"
207    )]
208    #[schemars(with = "JsonNumberReading")]
209    pub json_number_reading: Option<JsonNumberReading>,
210    /// How a score computed over a retained sample estimates the score over
211    /// every scanned value.
212    ///
213    /// Absent means not recorded: through 0.13 a sampled score pooled the
214    /// column samples, so it could fall outside its own `score_bounds`.
215    #[serde(
216        default,
217        skip_serializing_if = "Option::is_none",
218        deserialize_with = "present_value"
219    )]
220    #[schemars(with = "SampledScoreEstimator")]
221    pub sampled_score_estimator: Option<SampledScoreEstimator>,
222    /// Which values a date column's `min_datetime`, `max_datetime`,
223    /// `duration_days`, distributions, `slash_date_order` and `largest_gap`
224    /// are computed over.
225    ///
226    /// Absent means not recorded: through 0.13 an engine that retained a
227    /// bounded sample computed them from it, so on a source larger than the
228    /// sample a date outside it was missing.
229    #[serde(
230        default,
231        skip_serializing_if = "Option::is_none",
232        deserialize_with = "present_value"
233    )]
234    #[schemars(with = "DateStatistics")]
235    pub date_statistics: Option<DateStatistics>,
236    /// Which columns the outlier test behind `outlier_ratio` runs on.
237    ///
238    /// Absent means not recorded: through 0.13 a column whose quartiles
239    /// coincided was tested, and every value off them was an outlier.
240    #[serde(
241        default,
242        skip_serializing_if = "Option::is_none",
243        deserialize_with = "present_value"
244    )]
245    #[schemars(with = "OutlierSpread")]
246    pub outlier_spread: Option<OutlierSpread>,
247    /// Which text forms read as dates.
248    ///
249    /// Absent means not recorded: through 0.13 the spaced ISO datetime with a
250    /// fraction or an offset (`2026-06-01 12:00:00+00:00`) was text.
251    #[serde(
252        default,
253        skip_serializing_if = "Option::is_none",
254        deserialize_with = "present_value"
255    )]
256    #[schemars(with = "DateForms")]
257    pub date_forms: Option<DateForms>,
258}
259
260/// Decode a field that may be absent (serde's `default` covers that) but,
261/// when present, must carry a value: `Option<T>` alone reads `null` as `None`.
262pub(crate) fn present_value<'de, D, T>(deserializer: D) -> Result<Option<T>, D::Error>
263where
264    D: serde::Deserializer<'de>,
265    T: serde::Deserialize<'de>,
266{
267    T::deserialize(deserializer).map(Some)
268}
269
270impl MetricSemantics {
271    /// The definitions this build measures with.
272    pub fn current() -> Self {
273        Self {
274            text_length_unit: Some(TextLengthUnit::UnicodeScalar),
275            null_tokens: Some(NullTokenSet::CommonMarkers),
276            pattern_match_count: Some(PatternMatchCount::Validated),
277            shape_moments: Some(ShapeMomentEstimator::AdjustedSample),
278            statistic_rounding: Some(StatisticRounding::FourDecimalsTwelveSignificant),
279            range_rule_count: Some(RangeRuleCount::ViolatingValues),
280            key_uniqueness_units: Some(KeyUniquenessUnits::NonNullValues),
281            outlier_units: Some(OutlierUnits::TestedValues),
282            key_column_source: Some(KeyColumnSource::Declared),
283            accuracy_comparison: Some(AccuracyComparison::ExactValues),
284            json_number_reading: Some(JsonNumberReading::LiteralDigits),
285            sampled_score_estimator: Some(SampledScoreEstimator::PopulationWeighted),
286            date_statistics: Some(DateStatistics::EveryValue),
287            outlier_spread: Some(OutlierSpread::NonzeroIqr),
288            date_forms: Some(DateForms::SpacedIsoDatetimes),
289        }
290    }
291
292    /// Whether every definition this build knows is recorded. A comparison
293    /// can only vouch for definitions both reports declare, so an unrecorded
294    /// one leaves it unknown. Destructured, so a new field fails to compile
295    /// here until it is counted.
296    pub(crate) fn is_fully_recorded(&self) -> bool {
297        let Self {
298            text_length_unit,
299            null_tokens,
300            pattern_match_count,
301            shape_moments,
302            statistic_rounding,
303            range_rule_count,
304            key_uniqueness_units,
305            outlier_units,
306            key_column_source,
307            accuracy_comparison,
308            json_number_reading,
309            sampled_score_estimator,
310            date_statistics,
311            outlier_spread,
312            date_forms,
313        } = self;
314        text_length_unit.is_some()
315            && null_tokens.is_some()
316            && pattern_match_count.is_some()
317            && shape_moments.is_some()
318            && statistic_rounding.is_some()
319            && range_rule_count.is_some()
320            && key_uniqueness_units.is_some()
321            && outlier_units.is_some()
322            && key_column_source.is_some()
323            && accuracy_comparison.is_some()
324            && json_number_reading.is_some()
325            && sampled_score_estimator.is_some()
326            && date_statistics.is_some()
327            && outlier_spread.is_some()
328            && date_forms.is_some()
329    }
330}
331
332/// The locale a report's detected patterns were ranked with: `none`, or the
333/// ISO 3166-1 alpha-2 code of a [`Locale`] the pattern catalogue supports.
334#[derive(
335    Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize, schemars::JsonSchema,
336)]
337#[non_exhaustive]
338pub enum PatternLocale {
339    /// No locale: patterns were ranked without a locale preference.
340    #[serde(rename = "none")]
341    Unset,
342    /// Canada.
343    #[serde(rename = "CA")]
344    Ca,
345    /// Germany.
346    #[serde(rename = "DE")]
347    De,
348    /// France.
349    #[serde(rename = "FR")]
350    Fr,
351    /// United Kingdom.
352    #[serde(rename = "GB")]
353    Gb,
354    /// Italy.
355    #[serde(rename = "IT")]
356    It,
357    /// United States.
358    #[serde(rename = "US")]
359    Us,
360}
361
362impl From<Option<Locale>> for PatternLocale {
363    fn from(locale: Option<Locale>) -> Self {
364        match locale {
365            None => Self::Unset,
366            Some(Locale::Ca) => Self::Ca,
367            Some(Locale::De) => Self::De,
368            Some(Locale::Fr) => Self::Fr,
369            Some(Locale::Gb) => Self::Gb,
370            Some(Locale::It) => Self::It,
371            Some(Locale::Us) => Self::Us,
372        }
373    }
374}
375
376impl PatternLocale {
377    /// The locale, or `None` when patterns were ranked without one.
378    pub fn locale(self) -> Option<Locale> {
379        match self {
380            Self::Unset => None,
381            Self::Ca => Some(Locale::Ca),
382            Self::De => Some(Locale::De),
383            Self::Fr => Some(Locale::Fr),
384            Self::Gb => Some(Locale::Gb),
385            Self::It => Some(Locale::It),
386            Self::Us => Some(Locale::Us),
387        }
388    }
389}
390
391/// Which of the source's columns a run profiled.
392///
393/// A projection that names a column the source lacks is refused before
394/// profiling, so a report with [`ColumnSelection::All`] that has no profile
395/// for a column shows the source has no such column, and one with
396/// [`ColumnSelection::Listed`] shows only that the caller did not select it.
397#[derive(
398    Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize, schemars::JsonSchema,
399)]
400#[serde(tag = "selection", rename_all = "snake_case")]
401#[non_exhaustive]
402pub enum ColumnSelection {
403    /// Every column of the source.
404    All,
405    /// Only the named columns.
406    #[non_exhaustive]
407    Listed {
408        /// The names as the caller gave them.
409        names: Vec<String>,
410    },
411}
412
413impl ColumnSelection {
414    /// The selection `options` profile with.
415    pub fn from_options(options: &AnalysisOptions) -> Self {
416        match options.columns() {
417            None => Self::All,
418            Some(names) => Self::Listed {
419                names: names.to_vec(),
420            },
421        }
422    }
423
424    /// Whether the selection includes `column`, which a source that has the
425    /// column profiles.
426    pub fn includes(&self, column: &str) -> bool {
427        match self {
428            Self::All => true,
429            Self::Listed { names } => names.iter().any(|name| name == column),
430        }
431    }
432}
433
434/// The analysis settings a report was profiled with that change what its
435/// scores measure on unchanged data and are recorded nowhere else in it.
436///
437/// [`MetricSemantics`] records how a release defines its measurements; this
438/// records what the caller chose. The score weights live in the quality
439/// assessment and the semantic hints in the hint bindings, so they are not
440/// repeated here. Later releases add fields, which older readers ignore, and a
441/// field missing from a stored report means the release did not record it.
442#[derive(
443    Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize, schemars::JsonSchema,
444)]
445#[non_exhaustive]
446pub struct AnalysisSettings {
447    /// The locale detected patterns were ranked with, which the validity
448    /// score reads.
449    pub pattern_locale: PatternLocale,
450    /// The columns the run profiled. A dataset-level score has the column
451    /// set as its subject, and a report without a column's profile needs this
452    /// to tell a column the source lacks from one that was not selected.
453    pub columns: ColumnSelection,
454    /// The metric packs the run computed, `schema` always among them, in a
455    /// fixed order: `schema`, `statistics`, `patterns`, `quality`. Without
456    /// `patterns` no column has detected patterns, so validity is not
457    /// assessed and the overall score averages the other dimensions; an
458    /// empty `quality_dimensions` selection drops `quality`. Absent from a
459    /// report written before 0.14, which did not record them.
460    #[serde(
461        default,
462        skip_serializing_if = "Option::is_none",
463        deserialize_with = "present_value"
464    )]
465    #[schemars(with = "Vec<MetricPack>")]
466    pub metric_packs: Option<Vec<MetricPack>>,
467    /// The start/end column pairs `temporal_pairs` declared, each once,
468    /// sorted by start then end. Timeliness counts a row whose start date
469    /// falls after its end date in a declared pair, so two runs that pair
470    /// the same columns differently score timeliness differently. The
471    /// calculator compares each pair once whatever order the list gives, so
472    /// the record keeps direction and drops order and repetition. Empty when
473    /// none were declared; absent from a report written before 0.14, which
474    /// recorded only the columns the pairs named.
475    #[serde(
476        default,
477        skip_serializing_if = "Option::is_none",
478        deserialize_with = "present_value"
479    )]
480    #[schemars(with = "Vec<TemporalPair>")]
481    pub temporal_pairs: Option<Vec<TemporalPair>>,
482}
483
484/// A start/end column pair declared through `temporal_pairs`.
485///
486/// Read only from its object form, `{"start": ..., "end": ...}`: a derived
487/// reader would also take a two-element list, which the schema and the
488/// Python reader reject.
489#[derive(
490    Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, serde::Serialize, schemars::JsonSchema,
491)]
492#[non_exhaustive]
493pub struct TemporalPair {
494    /// The column whose date must not fall after `end`'s.
495    pub start: String,
496    /// The column whose date must not fall before `start`'s.
497    pub end: String,
498}
499
500impl<'de> serde::Deserialize<'de> for TemporalPair {
501    fn deserialize<D: serde::Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
502        struct PairVisitor;
503
504        impl<'de> serde::de::Visitor<'de> for PairVisitor {
505            type Value = TemporalPair;
506
507            fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result {
508                formatter.write_str(r#"a temporal pair {"start": ..., "end": ...}"#)
509            }
510
511            fn visit_map<A: serde::de::MapAccess<'de>>(
512                self,
513                mut map: A,
514            ) -> Result<TemporalPair, A::Error> {
515                use serde::de::Error;
516                let (mut start, mut end) = (None, None);
517                while let Some(key) = map.next_key::<String>()? {
518                    match key.as_str() {
519                        "start" if start.is_some() => {
520                            return Err(A::Error::duplicate_field("start"));
521                        }
522                        "end" if end.is_some() => return Err(A::Error::duplicate_field("end")),
523                        "start" => start = Some(map.next_value::<String>()?),
524                        "end" => end = Some(map.next_value::<String>()?),
525                        // A later writer may add keys, which this reader ignores.
526                        _ => {
527                            map.next_value::<serde::de::IgnoredAny>()?;
528                        }
529                    }
530                }
531                Ok(TemporalPair {
532                    start: start.ok_or_else(|| A::Error::missing_field("start"))?,
533                    end: end.ok_or_else(|| A::Error::missing_field("end"))?,
534                })
535            }
536        }
537
538        deserializer.deserialize_map(PairVisitor)
539    }
540}
541
542impl TemporalPair {
543    /// The pair comparing `start` with `end`.
544    pub fn new(start: impl Into<String>, end: impl Into<String>) -> Self {
545        Self {
546            start: start.into(),
547            end: end.into(),
548        }
549    }
550
551    /// The pairs `declared` names, each once, sorted: what the calculator
552    /// compares, whatever order or repetition the list was given in.
553    pub fn canonical(declared: &[(String, String)]) -> Vec<Self> {
554        let mut pairs: Vec<Self> = declared
555            .iter()
556            .map(|(start, end)| Self::new(start.clone(), end.clone()))
557            .collect();
558        pairs.sort();
559        pairs.dedup();
560        pairs
561    }
562}
563
564impl AnalysisSettings {
565    /// The settings `options` profile with.
566    pub fn from_options(options: &AnalysisOptions) -> Self {
567        Self {
568            pattern_locale: options.locale().into(),
569            columns: ColumnSelection::from_options(options),
570            metric_packs: Some(options.selected_metric_packs()),
571            temporal_pairs: Some(TemporalPair::canonical(
572                &options.semantic_hints().temporal_pairs,
573            )),
574        }
575    }
576}
577
578/// The dataprof build that produced a report.
579///
580/// `version` alone cannot tell a development build from the release it
581/// follows: both carry the workspace version until the next release bumps it.
582/// The commit and the dirty flag tell them apart where the build knows them.
583///
584/// Absent `commit` and `dirty` mean the build did not record them: a crate
585/// built from a package rather than a git checkout, or a build without a
586/// usable `git`. An explicit null is malformed, so neither the reader nor the
587/// schema accepts one.
588#[derive(
589    Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize, schemars::JsonSchema,
590)]
591#[non_exhaustive]
592pub struct ReportGenerator {
593    /// The dataprof package version, such as `"0.13.0"`.
594    pub version: String,
595    /// The full git commit hash the build was compiled from.
596    #[serde(
597        default,
598        skip_serializing_if = "Option::is_none",
599        deserialize_with = "present_value"
600    )]
601    #[schemars(with = "String")]
602    pub commit: Option<String>,
603    /// Whether the compiled sources (`Cargo.toml`, `Cargo.lock` and `crates/`)
604    /// had uncommitted changes when the build ran.
605    #[serde(
606        default,
607        skip_serializing_if = "Option::is_none",
608        deserialize_with = "present_value"
609    )]
610    #[schemars(with = "bool")]
611    pub dirty: Option<bool>,
612}
613
614/// The commit and dirty flag the embedding build recorded, if any.
615static BUILD_IDENTITY: OnceLock<(Option<&'static str>, Option<bool>)> = OnceLock::new();
616
617/// Record the build identity of the binary embedding this crate.
618///
619/// A build script here would rerun on every source change and rebuild every
620/// crate that depends on this one, so the leaf crate that owns the build
621/// (the Python extension) reads git and records the result once, at load.
622/// Only the first call takes effect; reports built before it, and in builds
623/// that never call it, record the version alone.
624#[doc(hidden)]
625pub fn record_build_identity(commit: Option<&'static str>, dirty: Option<bool>) {
626    let _ = BUILD_IDENTITY.set((commit, dirty));
627}
628
629impl ReportGenerator {
630    /// The build running now.
631    pub fn current() -> Self {
632        let (commit, dirty) = BUILD_IDENTITY.get().copied().unwrap_or((None, None));
633        Self {
634            version: env!("CARGO_PKG_VERSION").to_string(),
635            commit: commit.map(str::to_string),
636            dirty,
637        }
638    }
639}
640
641/// Which data a report describes, as far as the caller or the read can say.
642///
643/// [`DataSource`] says where the data was read from; a path or a stream name
644/// is a location, and the data behind it can change. This records an identity
645/// of the data itself, so a pipeline can check that what it promotes is what
646/// was profiled. Each member names its kind; a later release may add others,
647/// such as a digest of the bytes read, and a reader ignores members it does
648/// not know.
649///
650/// It is provenance, not a metric: two input paths over the same logical data
651/// carry different identities, so it stays outside the serialized-value
652/// parity comparison.
653#[derive(
654    Debug, Clone, Default, PartialEq, Eq, serde::Serialize, serde::Deserialize, schemars::JsonSchema,
655)]
656#[non_exhaustive]
657pub struct SourceIdentity {
658    /// A version of the source supplied by the caller and recorded verbatim:
659    /// a table snapshot ID, a commit version, a load or batch ID. dataprof
660    /// neither interprets nor verifies it.
661    #[serde(
662        default,
663        skip_serializing_if = "Option::is_none",
664        deserialize_with = "present_value"
665    )]
666    #[schemars(with = "String")]
667    pub version: Option<String>,
668}
669
670impl SourceIdentity {
671    /// An identity holding a caller-supplied version.
672    pub fn with_version(version: impl Into<String>) -> Self {
673        Self {
674            version: Some(version.into()),
675        }
676    }
677}
678
679/// Complete profiling report for a data source.
680///
681/// Contains column-level statistics, execution metadata, and an optional
682/// Quality assessment informed by ISO 8000/25012 concepts. This is the primary output of all
683/// profiling operations (`Profiler::analyze_file`, `Profiler::analyze_source`,
684/// `Profiler::profile_stream`, etc.).
685#[derive(Debug, Clone, serde::Serialize, schemars::JsonSchema)]
686pub struct ProfileReport {
687    /// Version of the serialized report schema (see [`REPORT_SCHEMA_VERSION`]).
688    ///
689    /// `0` means the document predates schema versioning (a 0.9-era report).
690    /// Deserialization rejects versions newer than [`REPORT_SCHEMA_VERSION`].
691    #[schemars(schema_with = "schema_version_schema")]
692    pub schema_version: u32,
693    /// Unique identifier for this report (UUID v4)
694    pub id: String,
695    /// Timestamp when the report was generated (ISO 8601 / RFC 3339)
696    pub timestamp: String,
697    /// Data source metadata (file, query, etc.)
698    pub data_source: DataSource,
699    /// Column-level profiling results
700    pub column_profiles: Vec<ColumnProfile>,
701    /// Execution metadata (timing, rows processed, truncation info, etc.)
702    pub execution: ExecutionMetadata,
703    /// Data quality assessment (optional — partial analysis may skip quality)
704    #[serde(skip_serializing_if = "Option::is_none")]
705    pub quality: Option<QualityAssessment>,
706    /// What happened to the quality computation. Always present: it is the
707    /// reason `quality` is or is not there, and a report must never leave a
708    /// failed computation to a log line. Additive field — documents written
709    /// before it deserialize as [`QualityAnalysisStatus::Unrecorded`], or as
710    /// `Computed` when they carry an assessment.
711    ///
712    /// This and `quality` are two halves of one fact:
713    /// [`Computed`](QualityAnalysisStatus::Computed) accompanies an assessment
714    /// and every other status accompanies its absence. Construction goes
715    /// through [`ReportAssembler`](crate::ReportAssembler), which derives one
716    /// from the other, and deserialization rejects a document that pairs them
717    /// any other way. Assigning either field directly can still break the
718    /// pair, and such a report serializes into a document this crate will
719    /// refuse to read back.
720    pub quality_status: QualityAnalysisStatus,
721    /// Per-column evidence of how each semantic hint bound to the data.
722    ///
723    /// Empty when no hints were supplied. Recorded for provenance: a hint proven
724    /// inert over the full data is rejected before a report is returned, so a
725    /// successful report only carries bindings that matched something or whose
726    /// evidence was sampled. Additive field — older readers ignore it.
727    #[serde(default, skip_serializing_if = "Vec::is_empty")]
728    pub semantic_hint_bindings: Vec<SemanticHintBinding>,
729    /// How this report's measurements were defined; see [`MetricSemantics`].
730    ///
731    /// Every report this build produces records [`MetricSemantics::current`].
732    /// `None` is a document written before dataprof recorded it, and its
733    /// definitions are unknown. Loading and saving keep the value the
734    /// document had rather than the reader's. Additive field.
735    #[serde(skip_serializing_if = "Option::is_none")]
736    #[schemars(with = "MetricSemantics")]
737    pub metric_semantics: Option<MetricSemantics>,
738    /// The analysis settings it was profiled with; see [`AnalysisSettings`].
739    ///
740    /// Every report a profiling run assembles records them. `None` is a
741    /// document written before dataprof recorded them, or a report built
742    /// directly with [`ProfileReport::new`]: the settings are unknown, not the
743    /// defaults. Additive field.
744    #[serde(skip_serializing_if = "Option::is_none")]
745    #[schemars(with = "AnalysisSettings")]
746    pub analysis_settings: Option<AnalysisSettings>,
747    /// The dataprof build that produced the report; see [`ReportGenerator`].
748    ///
749    /// Every report this build produces records [`ReportGenerator::current`].
750    /// `None` is a document written before dataprof recorded it. Loading and
751    /// saving keep the value the document had rather than the reader's.
752    /// Additive field.
753    #[serde(skip_serializing_if = "Option::is_none")]
754    #[schemars(with = "ReportGenerator")]
755    pub generator: Option<ReportGenerator>,
756    /// Which data the report describes; see [`SourceIdentity`].
757    ///
758    /// `None` when nothing was recorded: no profiling path sets it on its
759    /// own, so a caller that knows the version it profiled sets it. Loading
760    /// and saving keep it. Additive field.
761    #[serde(skip_serializing_if = "Option::is_none")]
762    #[schemars(with = "SourceIdentity")]
763    pub source_identity: Option<SourceIdentity>,
764}
765
766fn schema_version_schema(_: &mut schemars::SchemaGenerator) -> schemars::Schema {
767    schemars::json_schema!({
768        "type": "integer",
769        "const": REPORT_SCHEMA_VERSION,
770        "minimum": 0
771    })
772}
773
774/// The canonical persisted report and the legacy Python compatibility summary.
775///
776/// New JSON saves in both languages use `ProfileReport`. Python's `to_dict()`
777/// remains a convenience projection, produced by `PythonProfileReportDocument`
778/// below. Existing flat documents cannot acquire the source/quality provenance
779/// they never recorded, so the v1 union remains an intentional compatibility
780/// surface for loading and resaving those documents.
781#[allow(dead_code)]
782#[derive(serde::Serialize, schemars::JsonSchema)]
783#[serde(untagged)]
784#[schemars(title = "ProfileReport")]
785enum SerializedProfileReport {
786    Rust(Box<ProfileReport>),
787    Python(Box<PythonProfileReportDocument>),
788}
789
790#[derive(serde::Serialize, serde::Deserialize, schemars::JsonSchema)]
791struct PythonProfileReportDocument {
792    #[schemars(schema_with = "schema_version_schema")]
793    schema_version: u32,
794    source: String,
795    source_type: PythonSourceType,
796    /// The format a file, bytes or stream source was parsed as (#1028).
797    /// Absent for a query or dataframe source, for a stream that recorded
798    /// none, and in summaries written before 0.14.
799    #[serde(default, skip_serializing_if = "Option::is_none")]
800    #[schemars(with = "FileFormat")]
801    source_format: Option<FileFormat>,
802    execution: PythonExecutionDocument,
803    columns: Vec<PythonColumnDocument>,
804    quality: Option<PythonQualityDocument>,
805    quality_status: QualityAnalysisStatus,
806    #[serde(skip_serializing_if = "Vec::is_empty")]
807    semantic_hint_bindings: Vec<SemanticHintBinding>,
808    #[serde(skip_serializing_if = "Option::is_none")]
809    #[schemars(with = "MetricSemantics")]
810    metric_semantics: Option<MetricSemantics>,
811    #[serde(skip_serializing_if = "Option::is_none")]
812    #[schemars(with = "AnalysisSettings")]
813    analysis_settings: Option<AnalysisSettings>,
814    #[serde(skip_serializing_if = "Option::is_none")]
815    #[schemars(with = "ReportGenerator")]
816    generator: Option<ReportGenerator>,
817    #[serde(skip_serializing_if = "Option::is_none")]
818    #[schemars(with = "SourceIdentity")]
819    source_identity: Option<SourceIdentity>,
820    #[serde(skip_serializing_if = "Option::is_none")]
821    #[schemars(with = "CsvSourceMetadata")]
822    csv_metadata: Option<CsvSourceMetadata>,
823}
824
825#[derive(serde::Serialize, serde::Deserialize, schemars::JsonSchema)]
826#[serde(rename_all = "lowercase")]
827enum PythonSourceType {
828    File,
829    Query,
830    Dataframe,
831    Stream,
832    Bytes,
833}
834
835#[derive(serde::Serialize, serde::Deserialize, schemars::JsonSchema)]
836struct PythonExecutionDocument {
837    engine: Option<String>,
838    #[serde(skip_serializing_if = "Option::is_none")]
839    recovery_events: Option<Vec<dataprof_core::execution::RecoveryEvent>>,
840    rows_processed: usize,
841    columns_detected: usize,
842    scan_time_ms: u128,
843    source_exhausted: bool,
844    truncation_reason: Option<String>,
845    bytes_consumed: Option<u64>,
846    #[serde(serialize_with = "dataprof_core::serde_helpers::round_4_opt")]
847    throughput_rows_sec: Option<f64>,
848    #[serde(serialize_with = "dataprof_core::serde_helpers::round_2_opt")]
849    memory_peak_mb: Option<f64>,
850    error_count: usize,
851    ragged_row_count: usize,
852    #[serde(default, skip_serializing_if = "Option::is_none")]
853    unterminated_quote: Option<bool>,
854    sampling_applied: bool,
855    #[serde(serialize_with = "dataprof_core::serde_helpers::round_4_opt")]
856    sampling_ratio: Option<f64>,
857    #[serde(skip_serializing_if = "Option::is_none")]
858    sampled_row_ranges: Option<Vec<[u64; 2]>>,
859}
860
861#[derive(serde::Serialize, serde::Deserialize, schemars::JsonSchema)]
862struct PythonColumnDocument {
863    name: String,
864    data_type: PythonDataType,
865    total_count: usize,
866    null_count: usize,
867    #[serde(serialize_with = "dataprof_core::serde_helpers::round_2_opt")]
868    null_percentage: Option<f64>,
869    unique_count: Option<usize>,
870    unique_count_is_approximate: Option<bool>,
871    #[serde(serialize_with = "dataprof_core::serde_helpers::round_4_opt")]
872    uniqueness_ratio: Option<f64>,
873    #[serde(skip_serializing_if = "Option::is_none")]
874    invalid_count: Option<usize>,
875    /// Counts per lexical class. The Python dialect writes them as a plain
876    /// mapping, but it is the same four-key object the Rust dialect writes, so
877    /// it is described by the same definition: a reader that accepted a partial
878    /// mapping here would validate a document the Python loader then discards
879    /// as incomplete rather than inventing the missing counts.
880    #[serde(skip_serializing_if = "Option::is_none")]
881    type_homogeneity: Option<dataprof_core::TypeHomogeneity>,
882    #[serde(skip_serializing_if = "Option::is_none")]
883    locale_number_count: Option<usize>,
884    #[serde(skip_serializing_if = "Option::is_none")]
885    native_type: Option<dataprof_core::NativeType>,
886    #[serde(skip_serializing_if = "Option::is_none")]
887    stats: Option<PythonColumnStatsDocument>,
888    #[serde(skip_serializing_if = "Option::is_none")]
889    patterns: Option<Vec<PythonPatternDocument>>,
890}
891
892#[derive(serde::Serialize, serde::Deserialize, schemars::JsonSchema)]
893#[serde(rename_all = "lowercase")]
894enum PythonDataType {
895    String,
896    Identifier,
897    Integer,
898    Float,
899    Date,
900    Boolean,
901    Nested,
902}
903
904#[derive(serde::Serialize, serde::Deserialize, schemars::JsonSchema)]
905struct PythonColumnStatsDocument {
906    #[serde(skip_serializing_if = "Option::is_none")]
907    min: Option<f64>,
908    #[serde(skip_serializing_if = "Option::is_none")]
909    max: Option<f64>,
910    #[serde(skip_serializing_if = "Option::is_none")]
911    mean: Option<f64>,
912    // Outer None means this statistic does not apply to the column. An inner
913    // None is a numeric spread that overflowed: keep its explicit JSON null.
914    #[serde(
915        default,
916        deserialize_with = "deserialize_present_nullable_stat",
917        skip_serializing_if = "Option::is_none"
918    )]
919    #[schemars(with = "Option<f64>")]
920    std_dev: Option<Option<f64>>,
921    #[serde(
922        default,
923        deserialize_with = "deserialize_present_nullable_stat",
924        skip_serializing_if = "Option::is_none"
925    )]
926    #[schemars(with = "Option<f64>")]
927    variance: Option<Option<f64>>,
928    #[serde(skip_serializing_if = "Option::is_none")]
929    median: Option<f64>,
930    #[serde(skip_serializing_if = "Option::is_none")]
931    mode: Option<f64>,
932    #[serde(skip_serializing_if = "Option::is_none")]
933    skewness: Option<f64>,
934    #[serde(skip_serializing_if = "Option::is_none")]
935    kurtosis: Option<f64>,
936    #[serde(skip_serializing_if = "Option::is_none")]
937    coefficient_of_variation: Option<f64>,
938    #[serde(skip_serializing_if = "Option::is_none")]
939    quartiles: Option<dataprof_core::Quartiles>,
940    #[serde(skip_serializing_if = "Option::is_none")]
941    is_approximate: Option<bool>,
942    #[serde(skip_serializing_if = "Option::is_none")]
943    outlier_count: Option<usize>,
944    #[serde(skip_serializing_if = "Option::is_none")]
945    exact: Option<dataprof_core::ExactNumericStats>,
946    /// Shortest value, in Unicode scalar values. Not UTF-8 bytes and not
947    /// grapheme clusters: ASCII text is unaffected by that distinction, while a
948    /// combining sequence counts each scalar, so the decomposed and precomposed
949    /// spellings of the same word report different lengths.
950    #[serde(skip_serializing_if = "Option::is_none")]
951    min_length: Option<usize>,
952    /// Longest value, in Unicode scalar values. See `min_length`.
953    #[serde(skip_serializing_if = "Option::is_none")]
954    max_length: Option<usize>,
955    /// Mean length, in Unicode scalar values. See `min_length`.
956    #[serde(skip_serializing_if = "Option::is_none")]
957    avg_length: Option<f64>,
958    #[serde(skip_serializing_if = "Option::is_none")]
959    true_count: Option<usize>,
960    #[serde(skip_serializing_if = "Option::is_none")]
961    false_count: Option<usize>,
962    #[serde(skip_serializing_if = "Option::is_none")]
963    true_ratio: Option<f64>,
964}
965
966fn deserialize_present_nullable_stat<'de, D>(
967    deserializer: D,
968) -> Result<Option<Option<f64>>, D::Error>
969where
970    D: serde::Deserializer<'de>,
971{
972    serde::Deserialize::deserialize(deserializer).map(Some)
973}
974
975#[derive(serde::Serialize, serde::Deserialize, schemars::JsonSchema)]
976struct PythonPatternDocument {
977    name: String,
978    regex: String,
979    match_count: usize,
980    #[serde(serialize_with = "dataprof_core::serde_helpers::round_2")]
981    match_percentage: f64,
982    category: dataprof_core::PatternCategory,
983    #[serde(serialize_with = "dataprof_core::serde_helpers::round_4")]
984    confidence: f64,
985}
986
987#[derive(serde::Serialize, serde::Deserialize, schemars::JsonSchema)]
988struct PythonQualityDocument {
989    /// Null when no dimension was assessable; `assessed_dimensions` is then
990    /// empty and every entry in `dimension_scores` is null.
991    overall_score: Option<f64>,
992    assessed_dimensions: Vec<PythonQualityDimension>,
993    dimension_scores: std::collections::BTreeMap<PythonQualityDimension, Option<f64>>,
994    low_sample_warning: bool,
995    /// Metric components computed from every scanned row, in the labels of
996    /// `sampled_dimensions`. With both lists recorded, a component carrying
997    /// signal that neither names has unrecorded coverage, as on the full
998    /// document (#1027). Absent in summaries written before 0.14, which read
999    /// as before: a component not listed as sampled is exact.
1000    #[serde(default, skip_serializing_if = "Option::is_none")]
1001    exact_dimensions: Option<Vec<String>>,
1002    /// Metric components computed from a retained quality sample rather than
1003    /// from every scanned row. Additive field: a document written before it
1004    /// existed omits it, and its absence is preserved on reload because
1005    /// unknown coverage is not the same statement as "nothing was sampled".
1006    /// Uniqueness appears here as `key_uniqueness` and `duplicate_rows`.
1007    #[serde(skip_serializing_if = "Option::is_none")]
1008    sampled_dimensions: Option<Vec<String>>,
1009    /// Where the scores over every scanned row lie, when some were computed over a
1010    /// retained sample. Additive field, absent when every score is exact and
1011    /// on documents written before it existed.
1012    #[serde(default, skip_serializing_if = "Option::is_none")]
1013    score_bounds: Option<dataprof_metrics::ScoreBounds>,
1014    /// Columns whose values no value-level check read, each with the reason,
1015    /// so a reader of `overall_score` sees which columns those checks never
1016    /// read (#1134).
1017    /// Empty when every column was checked. Absent in summaries written
1018    /// before 0.14, which did not record it.
1019    #[serde(
1020        default,
1021        skip_serializing_if = "Option::is_none",
1022        deserialize_with = "present_value"
1023    )]
1024    #[schemars(with = "Vec<dataprof_metrics::ColumnWithoutValueChecks>")]
1025    columns_without_value_checks: Option<Vec<dataprof_metrics::ColumnWithoutValueChecks>>,
1026    /// Weights behind `overall_score`, always written as the canonical
1027    /// document writes them, default weights included (#749). Summaries
1028    /// written before 0.13 omitted default weights, and before the field
1029    /// existed dropped custom ones too, so its absence there does not show
1030    /// which weights applied.
1031    #[serde(default)]
1032    score_weights: QualityScoreWeights,
1033    #[serde(skip_serializing_if = "Option::is_none")]
1034    completeness: Option<CompletenessMetrics>,
1035    #[serde(skip_serializing_if = "Option::is_none")]
1036    consistency: Option<ConsistencyMetrics>,
1037    #[serde(skip_serializing_if = "Option::is_none")]
1038    uniqueness: Option<PythonUniquenessDocument>,
1039    #[serde(skip_serializing_if = "Option::is_none")]
1040    accuracy: Option<AccuracyMetrics>,
1041    #[serde(skip_serializing_if = "Option::is_none")]
1042    timeliness: Option<TimelinessMetrics>,
1043    #[serde(skip_serializing_if = "Option::is_none")]
1044    validity: Option<ValidityMetrics>,
1045    #[serde(skip_serializing_if = "Option::is_none")]
1046    precision: Option<PrecisionMetrics>,
1047}
1048
1049#[derive(
1050    Debug,
1051    Clone,
1052    PartialEq,
1053    Eq,
1054    PartialOrd,
1055    Ord,
1056    Hash,
1057    serde::Serialize,
1058    serde::Deserialize,
1059    schemars::JsonSchema,
1060)]
1061#[serde(rename_all = "lowercase")]
1062enum PythonQualityDimension {
1063    Completeness,
1064    Consistency,
1065    Uniqueness,
1066    Accuracy,
1067    Timeliness,
1068    Validity,
1069    Precision,
1070}
1071
1072#[derive(serde::Serialize, serde::Deserialize, schemars::JsonSchema)]
1073struct PythonUniquenessDocument {
1074    duplicate_rows: usize,
1075    key_uniqueness: f64,
1076    high_cardinality_warning: bool,
1077    rows_checked: usize,
1078    key_column: Option<String>,
1079    #[serde(default, skip_serializing_if = "Option::is_none")]
1080    key_values_checked: Option<usize>,
1081    duplicate_rows_approximate: bool,
1082    #[serde(default, skip_serializing_if = "Option::is_none")]
1083    duplicate_rows_exact_limit: Option<usize>,
1084}
1085
1086impl PythonProfileReportDocument {
1087    fn from_report(report: &ProfileReport) -> serde_json::Result<Self> {
1088        use dataprof_core::{ColumnStats, DataType, TruncationReason};
1089
1090        // Reuse the serialized metric blocks: their Serde implementations own
1091        // rounding and omission, rather than repeating those rules in Python.
1092        let columns = report
1093            .column_profiles
1094            .iter()
1095            .map(|column| {
1096                let stats = match &column.stats {
1097                    ColumnStats::Numeric(stats) => Some(serde_json::to_value(stats)?),
1098                    ColumnStats::Text(stats) if column.total_count > 0 => {
1099                        Some(serde_json::to_value(stats)?)
1100                    }
1101                    ColumnStats::Boolean(stats) => Some(serde_json::to_value(stats)?),
1102                    _ => None,
1103                }
1104                .map(serde_json::from_value)
1105                .transpose()?;
1106                Ok(PythonColumnDocument {
1107                    name: column.name.clone(),
1108                    data_type: match column.data_type {
1109                        DataType::String => PythonDataType::String,
1110                        DataType::Identifier => PythonDataType::Identifier,
1111                        DataType::Integer => PythonDataType::Integer,
1112                        DataType::Float => PythonDataType::Float,
1113                        DataType::Date => PythonDataType::Date,
1114                        DataType::Boolean => PythonDataType::Boolean,
1115                        DataType::Nested => PythonDataType::Nested,
1116                    },
1117                    total_count: column.total_count,
1118                    null_count: column.null_count,
1119                    null_percentage: (column.total_count > 0)
1120                        .then(|| column.null_count as f64 / column.total_count as f64 * 100.0),
1121                    unique_count: column.unique_count,
1122                    unique_count_is_approximate: column.unique_count_is_approximate,
1123                    uniqueness_ratio: column
1124                        .unique_count
1125                        .filter(|_| column.total_count > 0)
1126                        .map(|count| count as f64 / column.total_count as f64),
1127                    invalid_count: column.invalid_count,
1128                    type_homogeneity: column.type_homogeneity,
1129                    locale_number_count: column.locale_number_count,
1130                    native_type: column.native_type.clone(),
1131                    stats,
1132                    patterns: column.patterns.as_ref().map(|patterns| {
1133                        patterns
1134                            .iter()
1135                            .map(|pattern| PythonPatternDocument {
1136                                name: pattern.name.clone(),
1137                                regex: pattern.regex.clone(),
1138                                match_count: pattern.match_count,
1139                                match_percentage: pattern.match_percentage,
1140                                category: pattern.category.clone(),
1141                                confidence: pattern.confidence,
1142                            })
1143                            .collect()
1144                    }),
1145                })
1146            })
1147            .collect::<serde_json::Result<Vec<_>>>()?;
1148
1149        let mut execution = serde_json::to_value(&report.execution)?;
1150        execution["truncation_reason"] =
1151            serde_json::to_value(report.execution.truncation_reason.as_ref().map(|reason| {
1152                match reason {
1153                    TruncationReason::MaxRows(n) => format!("max_rows({n})"),
1154                    TruncationReason::MaxBytes(n) => format!("max_bytes({n})"),
1155                    TruncationReason::MemoryPressure => "memory_pressure".to_string(),
1156                    TruncationReason::StopCondition(s) => format!("stop_condition({s})"),
1157                    TruncationReason::StreamClosed => "stream_closed".to_string(),
1158                    TruncationReason::Timeout => "timeout".to_string(),
1159                }
1160            }))?;
1161
1162        let quality = report
1163            .quality
1164            .as_ref()
1165            .map(|assessment| {
1166                let metrics = &assessment.metrics;
1167                let mut document = serde_json::to_value(metrics)?;
1168                let scores = serde_json::to_value(assessment.scores())?;
1169                document["overall_score"] = scores["overall_score"].clone();
1170                // A document from a later v1 release may retain scores for a
1171                // dimension this build does not know. The canonical document
1172                // keeps them; the summary lists only the dimensions it types.
1173                let known: Vec<String> = dataprof_core::QualityDimension::all()
1174                    .into_iter()
1175                    .map(|dimension| dimension.to_string())
1176                    .collect();
1177                document["dimension_scores"] = scores["dimension_scores"]
1178                    .as_object()
1179                    .expect("dimension scores serialize as an object")
1180                    .iter()
1181                    .filter(|(name, _)| known.contains(name))
1182                    .map(|(name, score)| (name.clone(), score.clone()))
1183                    .collect();
1184                document["assessed_dimensions"] =
1185                    serde_json::to_value(metrics.assessed_dimensions())?;
1186                document["low_sample_warning"] =
1187                    serde_json::Value::Bool(metrics.low_sample_warning);
1188                if let Some(exact) = assessment.exact_dimensions() {
1189                    document["exact_dimensions"] = serde_json::to_value(exact)?;
1190                }
1191                if let Some(sampled) = assessment.sampled_dimensions() {
1192                    document["sampled_dimensions"] = serde_json::to_value(sampled)?;
1193                }
1194                if let Some(bounds) = assessment.score_bounds() {
1195                    document["score_bounds"] = serde_json::to_value(bounds)?;
1196                }
1197                if let Some(columns) = assessment.columns_without_value_checks() {
1198                    document["columns_without_value_checks"] = serde_json::to_value(columns)?;
1199                }
1200                // The summary has always included this boolean, even when false.
1201                if let Some(uniqueness) = document.get_mut("uniqueness") {
1202                    uniqueness["duplicate_rows_approximate"] = serde_json::Value::Bool(
1203                        metrics
1204                            .uniqueness
1205                            .as_ref()
1206                            .expect("serialized uniqueness is present")
1207                            .duplicate_rows_approximate,
1208                    );
1209                }
1210                serde_json::from_value(document)
1211            })
1212            .transpose()?;
1213
1214        Ok(Self {
1215            schema_version: report.schema_version,
1216            source: report.data_source.identifier(),
1217            source_type: match report.data_source {
1218                DataSource::File { .. } => PythonSourceType::File,
1219                DataSource::Query { .. } => PythonSourceType::Query,
1220                DataSource::DataFrame { .. } => PythonSourceType::Dataframe,
1221                DataSource::Stream { .. } => PythonSourceType::Stream,
1222                DataSource::Bytes { .. } => PythonSourceType::Bytes,
1223            },
1224            source_format: report.data_source.format().cloned(),
1225            execution: serde_json::from_value(execution)?,
1226            columns,
1227            quality,
1228            quality_status: report.quality_status.clone(),
1229            semantic_hint_bindings: report.semantic_hint_bindings.clone(),
1230            metric_semantics: report.metric_semantics.clone(),
1231            analysis_settings: report.analysis_settings.clone(),
1232            generator: report.generator.clone(),
1233            source_identity: report.source_identity.clone(),
1234            csv_metadata: report.data_source.csv_metadata().cloned(),
1235        })
1236    }
1237}
1238
1239/// Generate the JSON Schema 2020-12 document for the current serialized report.
1240///
1241/// This is primarily used by the repository's schema generator and drift tests.
1242/// The committed, versioned document under `docs/schema/` is the public
1243/// interoperability contract.
1244#[doc(hidden)]
1245pub fn profile_report_schema_document() -> serde_json::Value {
1246    let settings = schemars::generate::SchemaSettings::draft2020_12().for_serialize();
1247    let schema = settings
1248        .into_generator()
1249        .into_root_schema_for::<SerializedProfileReport>();
1250    let mut document =
1251        serde_json::to_value(schema).expect("a Schemars schema must serialize to a JSON document");
1252    let object = document
1253        .as_object_mut()
1254        .expect("a root Schemars schema must be a JSON object");
1255    object.insert(
1256        "$id".to_string(),
1257        serde_json::Value::String(format!(
1258            "https://andreabozzo.github.io/dataprof/schema/profile-report.v{REPORT_SCHEMA_VERSION}.schema.json"
1259        )),
1260    );
1261    make_compatibility_defaults_optional(&mut document);
1262    allow_additive_properties(&mut document);
1263    canonicalize_key_order(&mut document);
1264    document
1265}
1266
1267/// Sort every object's keys so the committed schema has one canonical byte
1268/// layout.
1269///
1270/// Key order carries no meaning in JSON Schema, but the committed document is a
1271/// reviewed artifact: it must not churn because `serde_json` switched its map
1272/// backing (`preserve_order`) or because Schemars changed the order in which it
1273/// builds a schema. Arrays such as `required` and `oneOf` keep their order,
1274/// which *is* meaningful.
1275fn canonicalize_key_order(value: &mut serde_json::Value) {
1276    match value {
1277        serde_json::Value::Object(object) => {
1278            for child in object.values_mut() {
1279                canonicalize_key_order(child);
1280            }
1281            let mut sorted: Vec<(String, serde_json::Value)> =
1282                std::mem::take(object).into_iter().collect();
1283            sorted.sort_by(|(left, _), (right, _)| left.cmp(right));
1284            object.extend(sorted);
1285        }
1286        serde_json::Value::Array(items) => {
1287            for item in items {
1288                canonicalize_key_order(item);
1289            }
1290        }
1291        _ => {}
1292    }
1293}
1294
1295/// Drop fields that this build always writes but a reader supplies a default
1296/// for, from the schema's `required` lists.
1297///
1298/// A document written under an earlier v1 build does not carry them. Declaring
1299/// them required would fail validation for documents the readers accept, which
1300/// is the opposite of what an additive widening means.
1301fn make_compatibility_defaults_optional(document: &mut serde_json::Value) {
1302    for (pointer, field) in [
1303        ("/$defs/ExecutionMetadata/required", "ragged_row_count"),
1304        ("/$defs/ProfileReport/required", "quality_status"),
1305        (
1306            "/$defs/PythonProfileReportDocument/required",
1307            "quality_status",
1308        ),
1309        (
1310            "/$defs/PythonQualityDocument/required",
1311            "sampled_dimensions",
1312        ),
1313        ("/$defs/QualityAssessment/required", "scores"),
1314        // Written always since 0.13 (#749), omitted at the defaults before.
1315        ("/$defs/QualityMetrics/required", "score_weights"),
1316        ("/$defs/PythonQualityDocument/required", "score_weights"),
1317        // Written only when some score was sampled.
1318        ("/$defs/QualityAssessment/required", "score_bounds"),
1319        // Written always since 0.14 (#1134), absent before.
1320        (
1321            "/$defs/QualityAssessment/required",
1322            "columns_without_value_checks",
1323        ),
1324        // Written always since 0.13 (#1005), absent before.
1325        ("/$defs/AccuracyMetrics/required", "rule_values_checked"),
1326        ("/$defs/AccuracyMetrics/required", "rule_violating_values"),
1327    ] {
1328        if let Some(required) = document
1329            .pointer_mut(pointer)
1330            .and_then(serde_json::Value::as_array_mut)
1331        {
1332            required.retain(|declared| declared.as_str() != Some(field));
1333        }
1334    }
1335    // A definition left with nothing required carries no `required` list.
1336    if let Some(definitions) = document
1337        .pointer_mut("/$defs")
1338        .and_then(serde_json::Value::as_object_mut)
1339    {
1340        for definition in definitions.values_mut() {
1341            if let Some(object) = definition.as_object_mut()
1342                && object
1343                    .get("required")
1344                    .and_then(serde_json::Value::as_array)
1345                    .is_some_and(Vec::is_empty)
1346            {
1347                object.remove("required");
1348            }
1349        }
1350    }
1351}
1352
1353fn allow_additive_properties(value: &mut serde_json::Value) {
1354    match value {
1355        serde_json::Value::Object(object) => {
1356            if object.get("additionalProperties") == Some(&serde_json::Value::Bool(false)) {
1357                object.remove("additionalProperties");
1358            }
1359            for child in object.values_mut() {
1360                allow_additive_properties(child);
1361            }
1362        }
1363        serde_json::Value::Array(items) => {
1364            for item in items {
1365                allow_additive_properties(item);
1366            }
1367        }
1368        _ => {}
1369    }
1370}
1371
1372impl ProfileReport {
1373    /// Serialize the canonical persisted document with deterministic object-key
1374    /// order. Array order (columns, patterns, recovery events) is preserved.
1375    pub fn to_json(&self) -> serde_json::Result<String> {
1376        let mut document = serde_json::to_value(self)?;
1377        canonicalize_key_order(&mut document);
1378        serde_json::to_string_pretty(&document)
1379    }
1380
1381    /// Serialize Python's historical summary projection. It deliberately omits
1382    /// some runtime metadata; use [`Self::to_json`] for lossless persistence.
1383    #[doc(hidden)]
1384    pub fn summary_json(&self) -> serde_json::Result<String> {
1385        let summary = PythonProfileReportDocument::from_report(self)?;
1386        serde_json::to_string(&summary)
1387    }
1388
1389    /// Create a new ProfileReport with auto-generated id and timestamp
1390    pub fn new(
1391        data_source: DataSource,
1392        column_profiles: Vec<ColumnProfile>,
1393        execution: ExecutionMetadata,
1394        quality: Option<QualityAssessment>,
1395    ) -> Self {
1396        Self {
1397            schema_version: REPORT_SCHEMA_VERSION,
1398            id: uuid::Uuid::new_v4().to_string(),
1399            timestamp: chrono::Utc::now().to_rfc3339(),
1400            data_source,
1401            column_profiles,
1402            execution,
1403            quality_status: match quality {
1404                Some(_) => QualityAnalysisStatus::Computed,
1405                // A caller constructing a report directly did not compute
1406                // quality; `ReportAssembler` overrides this with the reason it
1407                // actually observed.
1408                None => QualityAnalysisStatus::NotRequested,
1409            },
1410            quality,
1411            semantic_hint_bindings: Vec::new(),
1412            metric_semantics: Some(MetricSemantics::current()),
1413            // Only the assembler knows the options a report was profiled
1414            // with; it records them through `with_analysis_settings`.
1415            analysis_settings: None,
1416            generator: Some(ReportGenerator::current()),
1417            source_identity: None,
1418        }
1419    }
1420
1421    /// Record the analysis settings the report was profiled with.
1422    pub(crate) fn with_analysis_settings(mut self, settings: Option<AnalysisSettings>) -> Self {
1423        self.analysis_settings = settings;
1424        self
1425    }
1426
1427    /// Record what happened to the quality computation.
1428    ///
1429    /// Deliberately not public. `quality_status` and `quality` are two halves
1430    /// of one fact, and [`check_quality_pairing`](Self::check_quality_pairing)
1431    /// refuses to decode a document that pairs them wrongly — so a public
1432    /// setter would let a caller build a report that serializes and then
1433    /// cannot be read back by this same crate. Every path assigns the status
1434    /// through [`ReportAssembler`](crate::ReportAssembler), which derives it
1435    /// from the computation that produced the assessment.
1436    pub(crate) fn with_quality_status(mut self, status: QualityAnalysisStatus) -> Self {
1437        self.quality_status = status;
1438        self
1439    }
1440
1441    /// Attach per-column semantic-hint binding evidence.
1442    pub fn with_semantic_hint_bindings(mut self, bindings: Vec<SemanticHintBinding>) -> Self {
1443        self.semantic_hint_bindings = bindings;
1444        self
1445    }
1446
1447    /// Override the auto-generated ID (useful for deterministic caching/testing)
1448    pub fn with_id(mut self, id: impl Into<String>) -> Self {
1449        self.id = id.into();
1450        self
1451    }
1452
1453    /// Override the auto-generated timestamp
1454    pub fn with_timestamp(mut self, timestamp: impl Into<String>) -> Self {
1455        self.timestamp = timestamp.into();
1456        self
1457    }
1458
1459    /// Calculate the overall quality score (weighted across the assessed
1460    /// dimensions). Returns `None` if quality metrics were not computed, or
1461    /// if no dimension had anything to assess (e.g. an empty dataset) —
1462    /// absence of evidence is neither a perfect score nor a zero.
1463    pub fn quality_score(&self) -> Option<f64> {
1464        self.quality.as_ref().and_then(|q| q.score())
1465    }
1466
1467    /// Get the data source identifier (for backwards compatibility)
1468    pub fn source_identifier(&self) -> String {
1469        self.data_source.identifier()
1470    }
1471
1472    /// Reject a document whose `quality_status` contradicts its `quality`.
1473    ///
1474    /// `Computed` and an assessment imply each other: no writer can produce one
1475    /// without the other, because [`ReportAssembler`](crate::ReportAssembler)
1476    /// derives the status from the computation that produced the assessment. A
1477    /// document that pairs them any other way is internally inconsistent, and
1478    /// loading it would hand a consumer a report that states a verdict its own
1479    /// contents contradict — a gate could act on an assessment the report says
1480    /// failed. Failing the decode is the answer the rest of this crate gives to
1481    /// a malformed document, rather than repairing it into something plausible.
1482    fn check_quality_pairing(self) -> Result<Self, String> {
1483        let assessed = self.quality.is_some();
1484        let claims_computed = matches!(self.quality_status, QualityAnalysisStatus::Computed);
1485        if claims_computed == assessed {
1486            return Ok(self);
1487        }
1488        Err(if assessed {
1489            format!(
1490                "report carries a quality assessment but reports quality_status \
1491                 {:?}; only `computed` may accompany an assessment",
1492                self.quality_status
1493            )
1494        } else {
1495            "report reports quality_status `computed` but carries no quality assessment".to_string()
1496        })
1497    }
1498}
1499
1500/// Mirror of [`ProfileReport`] carrying the field-level deserialization
1501/// rules (legacy aliases, quality compat). Kept private: the public entry
1502/// point is the manual [`serde::Deserialize`] impl below, which gates on
1503/// `schema_version` before any of these fields decode.
1504#[derive(serde::Deserialize)]
1505struct ProfileReportFields {
1506    #[serde(default)]
1507    schema_version: u32,
1508    id: String,
1509    timestamp: String,
1510    data_source: DataSource,
1511    column_profiles: Vec<ColumnProfile>,
1512    #[serde(alias = "scan_info")]
1513    execution: ExecutionMetadata,
1514    #[serde(
1515        alias = "data_quality_metrics",
1516        default,
1517        deserialize_with = "deserialize_quality_compat"
1518    )]
1519    quality: Option<QualityAssessment>,
1520    #[serde(default)]
1521    quality_status: Option<QualityAnalysisStatus>,
1522    #[serde(default)]
1523    semantic_hint_bindings: Vec<SemanticHintBinding>,
1524    #[serde(default)]
1525    metric_semantics: Option<MetricSemantics>,
1526    #[serde(default)]
1527    analysis_settings: Option<AnalysisSettings>,
1528    #[serde(default)]
1529    generator: Option<ReportGenerator>,
1530    #[serde(default)]
1531    source_identity: Option<SourceIdentity>,
1532}
1533
1534impl From<ProfileReportFields> for ProfileReport {
1535    fn from(fields: ProfileReportFields) -> Self {
1536        Self {
1537            schema_version: fields.schema_version,
1538            id: fields.id,
1539            timestamp: fields.timestamp,
1540            data_source: fields.data_source,
1541            column_profiles: fields.column_profiles,
1542            execution: fields.execution,
1543            // A document written before the field: a stored assessment proves
1544            // the computation ran, and nothing else about it is knowable.
1545            quality_status: fields.quality_status.unwrap_or({
1546                if fields.quality.is_some() {
1547                    QualityAnalysisStatus::Computed
1548                } else {
1549                    QualityAnalysisStatus::Unrecorded
1550                }
1551            }),
1552            quality: fields.quality,
1553            semantic_hint_bindings: fields.semantic_hint_bindings,
1554            metric_semantics: fields.metric_semantics,
1555            analysis_settings: fields.analysis_settings,
1556            generator: fields.generator,
1557            source_identity: fields.source_identity,
1558        }
1559    }
1560}
1561
1562/// Manual deserialization so the `schema_version` gate runs before any other
1563/// field decodes. A derived deserializer visits fields in document order, so
1564/// an unsupported future document that also changed structure could fail with
1565/// a confusing structural error instead of the actionable version error; by
1566/// buffering the document first, the version check always wins.
1567impl<'de> serde::Deserialize<'de> for ProfileReport {
1568    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
1569    where
1570        D: serde::Deserializer<'de>,
1571    {
1572        use serde::de::Error;
1573
1574        let value = serde_json::Value::deserialize(deserializer)?;
1575        match value.get("schema_version") {
1576            // Absent field: legacy pre-0.10 document, defaults to version 0.
1577            None => {}
1578            Some(serde_json::Value::Number(n)) if n.as_u64().is_some() => {
1579                let version = n.as_u64().unwrap_or_default();
1580                if version > u64::from(REPORT_SCHEMA_VERSION) {
1581                    return Err(D::Error::custom(format!(
1582                        "report schema version {version} is newer than the latest supported \
1583                         version {REPORT_SCHEMA_VERSION}; upgrade dataprof to read this report"
1584                    )));
1585                }
1586            }
1587            // An explicit null or non-integer is malformed, not legacy.
1588            Some(other) => {
1589                return Err(D::Error::custom(format!(
1590                    "report schema_version must be a non-negative integer, got {other}"
1591                )));
1592            }
1593        }
1594        // `quality_status` deserializes through `Option`, which cannot tell an
1595        // absent field from an explicit null. Only the absent one is legacy;
1596        // a null is a malformed current document, and the fallback would
1597        // silently repair it into a plausible reason it never recorded.
1598        if value.get("quality_status") == Some(&serde_json::Value::Null) {
1599            return Err(D::Error::custom(
1600                "report quality_status must be an object; an explicit null is malformed, \
1601                 not a document written before the field existed",
1602            ));
1603        }
1604        // Same rule: absent is "not recorded", which reads back as unknown;
1605        // a null would be a current writer's malformed output.
1606        for key in [
1607            "metric_semantics",
1608            "analysis_settings",
1609            "generator",
1610            "source_identity",
1611        ] {
1612            if value.get(key) == Some(&serde_json::Value::Null) {
1613                return Err(D::Error::custom(format!(
1614                    "report {key} must be an object; an explicit null is malformed, \
1615                     not a document written before the field existed"
1616                )));
1617            }
1618        }
1619        let report = ProfileReportFields::deserialize(value)
1620            .map(ProfileReport::from)
1621            .map_err(D::Error::custom)?;
1622        report.check_quality_pairing().map_err(D::Error::custom)
1623    }
1624}
1625
1626/// Custom deserializer that handles both legacy `DataQualityMetrics` (flat)
1627/// and new `QualityAssessment` (wrapped with confidence) JSON formats.
1628fn deserialize_quality_compat<'de, D>(
1629    deserializer: D,
1630) -> Result<Option<QualityAssessment>, D::Error>
1631where
1632    D: serde::Deserializer<'de>,
1633{
1634    use serde::Deserialize;
1635
1636    let value: Option<serde_json::Value> = Option::deserialize(deserializer)?;
1637    match value {
1638        None => Ok(None),
1639        Some(mut v) => {
1640            // Either wrapper key marks the canonical shape. Read as flat
1641            // metrics, a canonical object matches no field and decodes as an
1642            // empty assessment, dropping every metric without an error.
1643            if v.get("metrics").is_some() || v.get("confidence").is_some() {
1644                if let Some(object) = v.as_object_mut() {
1645                    // Metrics whose provenance was lost are kept, and the
1646                    // provenance stays unknown, as for a flat legacy object.
1647                    object
1648                        .entry("confidence")
1649                        .or_insert_with(|| serde_json::json!("Unrecorded"));
1650                }
1651                let assessment: QualityAssessment =
1652                    serde_json::from_value(v).map_err(serde::de::Error::custom)?;
1653                Ok(Some(assessment))
1654            } else {
1655                let metrics: QualityMetrics =
1656                    serde_json::from_value(v).map_err(serde::de::Error::custom)?;
1657                // A flat legacy quality object says nothing about how its
1658                // numbers were obtained. Recording `Exact` here would invent
1659                // that, and a gate asked about the whole source would then
1660                // rest a verdict on numbers that may have come from a sample.
1661                Ok(Some(QualityAssessment::new(
1662                    metrics,
1663                    MetricConfidence::Unrecorded,
1664                )))
1665            }
1666        }
1667    }
1668}
1669
1670#[cfg(test)]
1671mod tests {
1672    use super::*;
1673    use dataprof_core::FileFormat;
1674    use dataprof_metrics::MetricConfidence;
1675    use serde_json::json;
1676
1677    fn report_without_quality() -> ProfileReport {
1678        ProfileReport::new(
1679            DataSource::File {
1680                path: "test.csv".to_string(),
1681                format: FileFormat::Csv,
1682                size_bytes: 1024,
1683                modified_at: None,
1684                parquet_metadata: None,
1685                csv_metadata: None,
1686            },
1687            vec![],
1688            ExecutionMetadata::new(100, 5, 50),
1689            None,
1690        )
1691    }
1692
1693    /// The `state` strings are the wire vocabulary: consumers branch on them,
1694    /// the Python binding repeats them by hand, and the committed schema
1695    /// enumerates them. Pin them here so a variant rename has to be deliberate.
1696    #[test]
1697    fn every_status_has_a_stable_wire_name() {
1698        let cases = [
1699            (QualityAnalysisStatus::Computed, "computed"),
1700            (QualityAnalysisStatus::NotRequested, "not_requested"),
1701            (QualityAnalysisStatus::NoData, "no_data"),
1702            (
1703                QualityAnalysisStatus::WithheldByProjection,
1704                "withheld_by_projection",
1705            ),
1706            (
1707                QualityAnalysisStatus::Failed {
1708                    error: "boom".to_string(),
1709                },
1710                "failed",
1711            ),
1712            (QualityAnalysisStatus::Unrecorded, "unrecorded"),
1713        ];
1714
1715        for (status, name) in cases {
1716            let value = serde_json::to_value(&status).unwrap();
1717            assert_eq!(value.get("state"), Some(&json!(name)), "{status:?}");
1718            let restored: QualityAnalysisStatus = serde_json::from_value(value).unwrap();
1719            assert_eq!(restored, status);
1720        }
1721    }
1722
1723    /// A document may not claim one thing in `quality_status` and another in
1724    /// `quality`. Loading one would let a consumer act on an assessment the
1725    /// report says never completed.
1726    #[test]
1727    fn contradictory_status_and_assessment_fail_to_decode() {
1728        let with_assessment =
1729            report_without_quality().with_quality_status(QualityAnalysisStatus::Failed {
1730                error: "boom".to_string(),
1731            });
1732        let mut document = serde_json::to_value(&with_assessment).unwrap();
1733        document.as_object_mut().unwrap().insert(
1734            "quality".to_string(),
1735            serde_json::to_value(QualityAssessment::exact(QualityMetrics::empty())).unwrap(),
1736        );
1737        let error = serde_json::from_value::<ProfileReport>(document)
1738            .expect_err("an assessment under a `failed` status is not a readable report");
1739        assert!(
1740            error.to_string().contains("only `computed`"),
1741            "unhelpful error: {error}"
1742        );
1743
1744        let mut document = serde_json::to_value(
1745            report_without_quality().with_quality_status(QualityAnalysisStatus::Computed),
1746        )
1747        .unwrap();
1748        document.as_object_mut().unwrap().remove("quality");
1749        let error = serde_json::from_value::<ProfileReport>(document)
1750            .expect_err("`computed` without an assessment is not a readable report");
1751        assert!(
1752            error.to_string().contains("carries no quality assessment"),
1753            "unhelpful error: {error}"
1754        );
1755    }
1756
1757    /// An absent field is a document written before it existed. An explicit
1758    /// null is a malformed current one, and the legacy fallback would repair it
1759    /// into a reason the writer never recorded.
1760    #[test]
1761    fn an_explicit_null_status_is_malformed_not_legacy() {
1762        let mut document = serde_json::to_value(report_without_quality()).unwrap();
1763        document
1764            .as_object_mut()
1765            .unwrap()
1766            .insert("quality_status".to_string(), serde_json::Value::Null);
1767
1768        let error = serde_json::from_value::<ProfileReport>(document)
1769            .expect_err("an explicit null is not a readable status");
1770        assert!(
1771            error.to_string().contains("explicit null is malformed"),
1772            "unhelpful error: {error}"
1773        );
1774    }
1775
1776    fn summary_value(report: &ProfileReport) -> serde_json::Value {
1777        serde_json::from_str(&report.summary_json().unwrap()).unwrap()
1778    }
1779
1780    /// A report built by this crate records how it measured, in both dialects.
1781    /// A document written before the field reads back as unknown, and saving
1782    /// it again must not stamp it with the reader's definitions.
1783    #[test]
1784    fn metric_semantics_are_recorded_and_absence_survives_a_roundtrip() {
1785        let report = report_without_quality();
1786        assert_eq!(report.metric_semantics, Some(MetricSemantics::current()));
1787        let mut document = serde_json::to_value(&report).unwrap();
1788        let recorded = json!({
1789            "text_length_unit": "unicode_scalar",
1790            "null_tokens": "common_markers",
1791            "pattern_match_count": "validated",
1792            "shape_moments": "adjusted_sample",
1793            "statistic_rounding": "four_decimals_twelve_significant",
1794            "range_rule_count": "violating_values",
1795            "key_uniqueness_units": "non_null_values",
1796            "outlier_units": "tested_values",
1797            "key_column_source": "declared",
1798            "accuracy_comparison": "exact_values",
1799            "json_number_reading": "literal_digits",
1800            "sampled_score_estimator": "population_weighted",
1801            "date_statistics": "every_value",
1802            "outlier_spread": "nonzero_iqr",
1803            "date_forms": "spaced_iso_datetimes",
1804        });
1805        assert_eq!(document["metric_semantics"], recorded);
1806        assert_eq!(summary_value(&report)["metric_semantics"], recorded);
1807
1808        document.as_object_mut().unwrap().remove("metric_semantics");
1809        let legacy: ProfileReport = serde_json::from_value(document).unwrap();
1810        assert_eq!(legacy.metric_semantics, None);
1811        assert!(
1812            serde_json::to_value(&legacy)
1813                .unwrap()
1814                .get("metric_semantics")
1815                .is_none()
1816        );
1817        assert!(summary_value(&legacy).get("metric_semantics").is_none());
1818    }
1819
1820    #[test]
1821    fn malformed_metric_semantics_fail_to_decode() {
1822        for value in [
1823            serde_json::Value::Null,
1824            json!("unicode_scalar"),
1825            json!({"text_length_unit": "utf8_byte"}),
1826            json!({"text_length_unit": null}),
1827            json!({"null_tokens": "empty_null_nan"}),
1828            json!({"null_tokens": null}),
1829            json!({"pattern_match_count": "regex"}),
1830            json!({"pattern_match_count": null}),
1831            json!({"shape_moments": "population"}),
1832            json!({"shape_moments": null}),
1833            json!({"range_rule_count": "rule_failures"}),
1834            json!({"range_rule_count": null}),
1835            json!({"key_uniqueness_units": "rows"}),
1836            json!({"key_uniqueness_units": null}),
1837            json!({"outlier_units": "numeric_values"}),
1838            json!({"outlier_units": null}),
1839            json!({"key_column_source": "name_heuristic"}),
1840            json!({"key_column_source": null}),
1841            json!({"accuracy_comparison": "nearest_f64"}),
1842            json!({"accuracy_comparison": null}),
1843            json!({"json_number_reading": "nearest_f64"}),
1844            json!({"json_number_reading": null}),
1845            json!({"outlier_spread": "every_column"}),
1846            json!({"outlier_spread": null}),
1847            json!({"date_forms": "plain_spaced_datetimes"}),
1848            json!({"date_forms": null}),
1849        ] {
1850            let mut document = serde_json::to_value(report_without_quality()).unwrap();
1851            document["metric_semantics"] = value.clone();
1852            assert!(
1853                serde_json::from_value::<ProfileReport>(document).is_err(),
1854                "{value} decoded"
1855            );
1856        }
1857    }
1858
1859    /// Later releases add definitions; an older reader keeps the ones it knows.
1860    #[test]
1861    fn unknown_metric_definitions_are_ignored() {
1862        let mut document = serde_json::to_value(report_without_quality()).unwrap();
1863        document["metric_semantics"]["grapheme_policy"] = json!("extended");
1864        let restored: ProfileReport = serde_json::from_value(document).unwrap();
1865        assert_eq!(restored.metric_semantics, Some(MetricSemantics::current()));
1866    }
1867
1868    #[test]
1869    fn quality_status_survives_a_json_roundtrip() {
1870        let report = report_without_quality().with_quality_status(QualityAnalysisStatus::Failed {
1871            error: "Metrics calculation failed: no data columns found".to_string(),
1872        });
1873
1874        let json = serde_json::to_string(&report).unwrap();
1875        let restored: ProfileReport = serde_json::from_str(&json).unwrap();
1876
1877        assert_eq!(restored.quality_status, report.quality_status);
1878        assert!(restored.quality.is_none());
1879    }
1880
1881    /// The serialized document is the contract, so the two states have to be
1882    /// distinguishable there and not only in the Rust value.
1883    #[test]
1884    fn serialized_failure_and_skip_differ() {
1885        let failed = serde_json::to_value(report_without_quality().with_quality_status(
1886            QualityAnalysisStatus::Failed {
1887                error: "boom".to_string(),
1888            },
1889        ))
1890        .unwrap();
1891        let skipped = serde_json::to_value(
1892            report_without_quality().with_quality_status(QualityAnalysisStatus::NotRequested),
1893        )
1894        .unwrap();
1895
1896        assert_eq!(
1897            failed.get("quality_status"),
1898            Some(&json!({"state": "failed", "error": "boom"}))
1899        );
1900        assert_eq!(
1901            skipped.get("quality_status"),
1902            Some(&json!({"state": "not_requested"}))
1903        );
1904        assert!(failed.get("quality").is_none());
1905        assert!(skipped.get("quality").is_none());
1906    }
1907
1908    /// A document written before the field existed: an assessment proves the
1909    /// computation ran, and its absence proves nothing.
1910    #[test]
1911    fn documents_without_the_field_read_back_honestly() {
1912        let mut document = serde_json::to_value(report_without_quality()).unwrap();
1913        document.as_object_mut().unwrap().remove("quality_status");
1914
1915        let restored: ProfileReport = serde_json::from_value(document.clone()).unwrap();
1916        assert_eq!(restored.quality_status, QualityAnalysisStatus::Unrecorded);
1917
1918        document.as_object_mut().unwrap().insert(
1919            "quality".to_string(),
1920            serde_json::to_value(QualityAssessment::exact(QualityMetrics::empty())).unwrap(),
1921        );
1922        let restored: ProfileReport = serde_json::from_value(document).unwrap();
1923        assert_eq!(restored.quality_status, QualityAnalysisStatus::Computed);
1924    }
1925
1926    #[test]
1927    fn test_profile_report_json_roundtrip() {
1928        let report = ProfileReport::new(
1929            DataSource::File {
1930                path: "test.csv".to_string(),
1931                format: FileFormat::Csv,
1932                size_bytes: 1024,
1933                modified_at: None,
1934                parquet_metadata: None,
1935                csv_metadata: None,
1936            },
1937            vec![],
1938            ExecutionMetadata::new(100, 5, 50),
1939            Some(QualityAssessment::exact(QualityMetrics::empty())),
1940        );
1941
1942        let json = serde_json::to_string(&report).unwrap();
1943        let deserialized: ProfileReport = serde_json::from_str(&json).unwrap();
1944
1945        assert_eq!(deserialized.id, report.id);
1946        assert_eq!(deserialized.timestamp, report.timestamp);
1947        assert_eq!(deserialized.source_identifier(), "test.csv");
1948        assert_eq!(deserialized.execution.rows_processed, 100);
1949        assert!(deserialized.quality.is_some());
1950        assert_eq!(deserialized.schema_version, REPORT_SCHEMA_VERSION);
1951    }
1952
1953    /// The weights determine the aggregate score, so a summary that drops
1954    /// custom ones reloads with a score its weights no longer produce.
1955    #[test]
1956    fn summary_names_its_score_weights_including_defaults() {
1957        let summary_quality = |weights| {
1958            let mut metrics = QualityMetrics::empty();
1959            metrics.score_weights = weights;
1960            let report = ProfileReport::new(
1961                DataSource::File {
1962                    path: "test.csv".to_string(),
1963                    format: FileFormat::Csv,
1964                    size_bytes: 1024,
1965                    modified_at: None,
1966                    parquet_metadata: None,
1967                    csv_metadata: None,
1968                },
1969                vec![],
1970                ExecutionMetadata::new(100, 5, 50),
1971                Some(QualityAssessment::exact(metrics)),
1972            );
1973            let summary: serde_json::Value =
1974                serde_json::from_str(&report.summary_json().unwrap()).unwrap();
1975            summary["quality"].clone()
1976        };
1977
1978        let custom = QualityScoreWeights {
1979            completeness: 1.0,
1980            consistency: 0.0,
1981            uniqueness: 0.0,
1982            accuracy: 0.0,
1983            timeliness: 0.0,
1984            validity: 0.0,
1985            precision: 3.0,
1986        };
1987        assert_eq!(
1988            summary_quality(custom)["score_weights"],
1989            serde_json::to_value(custom).unwrap()
1990        );
1991        // Defaults too: a stored score has to name the weights behind it in
1992        // case a later release changes the defaults (#749).
1993        assert_eq!(
1994            summary_quality(QualityScoreWeights::default())["score_weights"],
1995            serde_json::to_value(QualityScoreWeights::default()).unwrap()
1996        );
1997    }
1998
1999    #[test]
2000    fn test_serialized_report_carries_schema_version() {
2001        let report = ProfileReport::new(
2002            DataSource::File {
2003                path: "test.csv".to_string(),
2004                format: FileFormat::Csv,
2005                size_bytes: 1024,
2006                modified_at: None,
2007                parquet_metadata: None,
2008                csv_metadata: None,
2009            },
2010            vec![],
2011            ExecutionMetadata::new(100, 5, 50),
2012            None,
2013        );
2014
2015        let value = serde_json::to_value(&report).unwrap();
2016        assert_eq!(
2017            value.get("schema_version").and_then(|v| v.as_u64()),
2018            Some(u64::from(REPORT_SCHEMA_VERSION))
2019        );
2020    }
2021
2022    #[test]
2023    fn test_profile_report_without_quality() {
2024        let report = ProfileReport::new(
2025            DataSource::File {
2026                path: "test.csv".to_string(),
2027                format: FileFormat::Csv,
2028                size_bytes: 1024,
2029                modified_at: None,
2030                parquet_metadata: None,
2031                csv_metadata: None,
2032            },
2033            vec![],
2034            ExecutionMetadata::new(100, 5, 50),
2035            None,
2036        );
2037
2038        let json = serde_json::to_string(&report).unwrap();
2039        let deserialized: ProfileReport = serde_json::from_str(&json).unwrap();
2040
2041        assert!(deserialized.quality.is_none());
2042        assert_eq!(deserialized.execution.rows_processed, 100);
2043    }
2044
2045    #[test]
2046    fn test_profile_report_deserializes_legacy_quality_metrics() {
2047        let json = json!({
2048            "id": "legacy-report",
2049            "timestamp": "2026-05-22T10:00:00Z",
2050            "data_source": {
2051                "type": "file",
2052                "path": "test.csv",
2053                "format": "csv",
2054                "size_bytes": 42
2055            },
2056            "column_profiles": [],
2057            "scan_info": {
2058                "rows_processed": 10,
2059                "columns_detected": 2,
2060                "scan_time_ms": 5,
2061                "error_count": 0,
2062                "source_exhausted": true,
2063                "sampling_applied": false
2064            },
2065            "data_quality_metrics": {
2066                "completeness": {
2067                    "missing_values_ratio": 0.0,
2068                    "complete_records_ratio": 100.0,
2069                    "null_columns": []
2070                }
2071            }
2072        });
2073
2074        let report: ProfileReport = serde_json::from_value(json).unwrap();
2075
2076        assert_eq!(report.id, "legacy-report");
2077        // A document without a schema_version field is a legacy pre-0.10
2078        // report; it must load and be identifiable as such.
2079        assert_eq!(report.schema_version, 0);
2080        assert_eq!(report.execution.rows_processed, 10);
2081        // Legacy metrics predate the assessability denominators: the facts
2082        // stay readable, but no score is fabricated from them.
2083        assert!(report.quality_score().is_none());
2084        let quality = report
2085            .quality
2086            .expect("expected legacy quality to deserialize");
2087        // Nothing was assessable, so the assessment says so rather than
2088        // claiming exactness about a score it never produced (#571).
2089        assert!(matches!(quality.confidence, MetricConfidence::NotAssessed));
2090        let completeness = quality
2091            .metrics
2092            .completeness
2093            .as_ref()
2094            .expect("legacy completeness facts should deserialize");
2095        assert!((completeness.complete_records_ratio - 100.0).abs() < 0.01);
2096        assert!(quality.metrics.assessed_dimensions().is_empty());
2097    }
2098
2099    /// A canonical quality object that lost its `confidence` still holds its
2100    /// metrics. The flat legacy reader dropped every one of them without an
2101    /// error; they are kept, and the missing provenance stays unknown.
2102    #[test]
2103    fn a_canonical_quality_without_confidence_keeps_its_metrics() {
2104        let mut document = current_document();
2105        document["quality"] = json!({
2106            "metrics": {
2107                "completeness": {
2108                    "missing_values_ratio": 0.0,
2109                    "complete_records_ratio": 100.0,
2110                    "null_columns": []
2111                }
2112            }
2113        });
2114
2115        let report: ProfileReport = serde_json::from_value(document).unwrap();
2116
2117        let quality = report.quality.expect("the assessment survives");
2118        let completeness = quality
2119            .metrics
2120            .completeness
2121            .as_ref()
2122            .expect("the metrics survive");
2123        assert!((completeness.complete_records_ratio - 100.0).abs() < 0.01);
2124        assert!(!matches!(quality.confidence, MetricConfidence::Exact));
2125    }
2126
2127    /// A `confidence` with no `metrics` is a malformed canonical object, not
2128    /// an empty legacy one.
2129    #[test]
2130    fn a_canonical_quality_without_metrics_is_malformed() {
2131        let mut document = current_document();
2132        document["quality"] = json!({ "confidence": "Exact" });
2133
2134        assert!(serde_json::from_value::<ProfileReport>(document).is_err());
2135    }
2136
2137    fn current_document() -> serde_json::Value {
2138        json!({
2139            "schema_version": REPORT_SCHEMA_VERSION,
2140            "id": "current-report",
2141            "timestamp": "2026-07-16T10:00:00Z",
2142            "data_source": {
2143                "type": "file",
2144                "path": "test.csv",
2145                "format": "csv",
2146                "size_bytes": 42
2147            },
2148            "column_profiles": [],
2149            "execution": {
2150                "rows_processed": 10,
2151                "columns_detected": 2,
2152                "scan_time_ms": 5,
2153                "error_count": 0,
2154                "source_exhausted": true,
2155                "sampling_applied": false
2156            }
2157        })
2158    }
2159
2160    #[test]
2161    fn test_additive_fields_from_newer_writer_are_ignored() {
2162        // A newer dataprof may add fields within the same schema version;
2163        // a compatible reader must not break on them.
2164        let mut json = current_document();
2165        json["a_future_additive_field"] = json!({"anything": true});
2166        json["column_profiles"] = json!([]);
2167
2168        let report: ProfileReport = serde_json::from_value(json).unwrap();
2169        assert_eq!(report.schema_version, REPORT_SCHEMA_VERSION);
2170        assert_eq!(report.id, "current-report");
2171    }
2172
2173    #[test]
2174    fn test_unsupported_future_schema_version_fails_explicitly() {
2175        let mut json = current_document();
2176        json["schema_version"] = json!(REPORT_SCHEMA_VERSION + 1);
2177
2178        let err = serde_json::from_value::<ProfileReport>(json).unwrap_err();
2179        let msg = err.to_string();
2180        assert!(
2181            msg.contains("schema version") && msg.contains("upgrade dataprof"),
2182            "expected an actionable schema-version error, got: {msg}"
2183        );
2184    }
2185
2186    #[test]
2187    fn test_version_error_wins_over_structural_errors() {
2188        // A future document that also broke structure, with schema_version
2189        // appearing after the broken field, must still fail with the version
2190        // error — not a confusing type error from the field encountered first.
2191        let json_text = format!(
2192            r#"{{"column_profiles": "not-an-array", "schema_version": {}}}"#,
2193            REPORT_SCHEMA_VERSION + 1
2194        );
2195        let err = serde_json::from_str::<ProfileReport>(&json_text).unwrap_err();
2196        let msg = err.to_string();
2197        assert!(
2198            msg.contains("upgrade dataprof"),
2199            "expected the schema-version error to win, got: {msg}"
2200        );
2201    }
2202
2203    #[test]
2204    fn test_null_schema_version_is_malformed_not_legacy() {
2205        let mut json = current_document();
2206        json["schema_version"] = json!(null);
2207
2208        let err = serde_json::from_value::<ProfileReport>(json).unwrap_err();
2209        let msg = err.to_string();
2210        assert!(
2211            msg.contains("schema_version must be a non-negative integer"),
2212            "expected explicit rejection of null schema_version, got: {msg}"
2213        );
2214    }
2215
2216    #[test]
2217    fn test_non_integer_schema_version_is_rejected() {
2218        for bad in [json!("1"), json!(1.5), json!(-1), json!(true)] {
2219            let mut json = current_document();
2220            json["schema_version"] = bad.clone();
2221            let err = serde_json::from_value::<ProfileReport>(json).unwrap_err();
2222            assert!(
2223                err.to_string().contains("non-negative integer"),
2224                "expected rejection of {bad}, got: {err}"
2225            );
2226        }
2227    }
2228}