1use dataprof_core::{ColumnProfile, DataSource, ExecutionMetadata, SemanticHintBinding};
2use dataprof_metrics::{
3 AccuracyMetrics, CompletenessMetrics, ConsistencyMetrics, PrecisionMetrics, QualityAssessment,
4 QualityMetrics, TimelinessMetrics, ValidityMetrics,
5};
6
7pub const REPORT_SCHEMA_VERSION: u32 = 1;
22
23#[derive(Debug, Clone, serde::Serialize, schemars::JsonSchema)]
30pub struct ProfileReport {
31 #[schemars(schema_with = "schema_version_schema")]
36 pub schema_version: u32,
37 pub id: String,
39 pub timestamp: String,
41 pub data_source: DataSource,
43 pub column_profiles: Vec<ColumnProfile>,
45 pub execution: ExecutionMetadata,
47 #[serde(skip_serializing_if = "Option::is_none")]
49 pub quality: Option<QualityAssessment>,
50 #[serde(default, skip_serializing_if = "Vec::is_empty")]
57 pub semantic_hint_bindings: Vec<SemanticHintBinding>,
58}
59
60fn schema_version_schema(_: &mut schemars::SchemaGenerator) -> schemars::Schema {
61 schemars::json_schema!({
62 "type": "integer",
63 "const": REPORT_SCHEMA_VERSION,
64 "minimum": 0
65 })
66}
67
68#[allow(dead_code)]
75#[derive(serde::Serialize, schemars::JsonSchema)]
76#[serde(untagged)]
77#[schemars(title = "ProfileReport")]
78enum SerializedProfileReport {
79 Rust(Box<ProfileReport>),
80 Python(Box<PythonProfileReportDocument>),
81}
82
83#[allow(dead_code)]
84#[derive(serde::Serialize, schemars::JsonSchema)]
85struct PythonProfileReportDocument {
86 #[schemars(schema_with = "schema_version_schema")]
87 schema_version: u32,
88 source: String,
89 source_type: PythonSourceType,
90 execution: PythonExecutionDocument,
91 columns: Vec<PythonColumnDocument>,
92 quality: Option<PythonQualityDocument>,
93 #[serde(skip_serializing_if = "Vec::is_empty")]
94 semantic_hint_bindings: Vec<SemanticHintBinding>,
95}
96
97#[allow(dead_code)]
98#[derive(serde::Serialize, schemars::JsonSchema)]
99#[serde(rename_all = "lowercase")]
100enum PythonSourceType {
101 File,
102 Query,
103 Dataframe,
104 Stream,
105 Bytes,
106}
107
108#[allow(dead_code)]
109#[derive(serde::Serialize, schemars::JsonSchema)]
110struct PythonExecutionDocument {
111 engine: Option<String>,
112 rows_processed: usize,
113 columns_detected: usize,
114 scan_time_ms: u128,
115 source_exhausted: bool,
116 truncation_reason: Option<String>,
117 bytes_consumed: Option<u64>,
118 throughput_rows_sec: Option<f64>,
119 memory_peak_mb: Option<f64>,
120 error_count: usize,
121 ragged_row_count: usize,
122 sampling_applied: bool,
123 sampling_ratio: Option<f64>,
124}
125
126#[allow(dead_code)]
127#[derive(serde::Serialize, schemars::JsonSchema)]
128struct PythonColumnDocument {
129 name: String,
130 data_type: PythonDataType,
131 total_count: usize,
132 null_count: usize,
133 null_percentage: Option<f64>,
134 unique_count: Option<usize>,
135 unique_count_is_approximate: Option<bool>,
136 uniqueness_ratio: Option<f64>,
137 #[serde(skip_serializing_if = "Option::is_none")]
138 invalid_count: Option<usize>,
139 #[serde(skip_serializing_if = "Option::is_none")]
145 type_homogeneity: Option<dataprof_core::TypeHomogeneity>,
146 #[serde(skip_serializing_if = "Option::is_none")]
147 stats: Option<PythonColumnStatsDocument>,
148 #[serde(skip_serializing_if = "Option::is_none")]
149 patterns: Option<Vec<PythonPatternDocument>>,
150}
151
152#[allow(dead_code)]
153#[derive(serde::Serialize, schemars::JsonSchema)]
154#[serde(rename_all = "lowercase")]
155enum PythonDataType {
156 String,
157 Identifier,
158 Integer,
159 Float,
160 Date,
161 Boolean,
162}
163
164#[allow(dead_code)]
165#[derive(serde::Serialize, schemars::JsonSchema)]
166struct PythonColumnStatsDocument {
167 #[serde(skip_serializing_if = "Option::is_none")]
168 min: Option<f64>,
169 #[serde(skip_serializing_if = "Option::is_none")]
170 max: Option<f64>,
171 #[serde(skip_serializing_if = "Option::is_none")]
172 mean: Option<f64>,
173 #[serde(skip_serializing_if = "Option::is_none")]
174 std_dev: Option<f64>,
175 #[serde(skip_serializing_if = "Option::is_none")]
176 variance: Option<f64>,
177 #[serde(skip_serializing_if = "Option::is_none")]
178 median: Option<f64>,
179 #[serde(skip_serializing_if = "Option::is_none")]
180 mode: Option<f64>,
181 #[serde(skip_serializing_if = "Option::is_none")]
182 skewness: Option<f64>,
183 #[serde(skip_serializing_if = "Option::is_none")]
184 kurtosis: Option<f64>,
185 #[serde(skip_serializing_if = "Option::is_none")]
186 coefficient_of_variation: Option<f64>,
187 #[serde(skip_serializing_if = "Option::is_none")]
188 quartiles: Option<dataprof_core::Quartiles>,
189 #[serde(skip_serializing_if = "Option::is_none")]
190 is_approximate: Option<bool>,
191 #[serde(skip_serializing_if = "Option::is_none")]
192 outlier_count: Option<usize>,
193 #[serde(skip_serializing_if = "Option::is_none")]
194 min_length: Option<usize>,
195 #[serde(skip_serializing_if = "Option::is_none")]
196 max_length: Option<usize>,
197 #[serde(skip_serializing_if = "Option::is_none")]
198 avg_length: Option<f64>,
199 #[serde(skip_serializing_if = "Option::is_none")]
200 true_count: Option<usize>,
201 #[serde(skip_serializing_if = "Option::is_none")]
202 false_count: Option<usize>,
203 #[serde(skip_serializing_if = "Option::is_none")]
204 true_ratio: Option<f64>,
205}
206
207#[allow(dead_code)]
208#[derive(serde::Serialize, schemars::JsonSchema)]
209struct PythonPatternDocument {
210 name: String,
211 regex: String,
212 match_count: usize,
213 match_percentage: f64,
214 category: dataprof_core::PatternCategory,
215 confidence: f64,
216}
217
218#[allow(dead_code)]
219#[derive(serde::Serialize, schemars::JsonSchema)]
220struct PythonQualityDocument {
221 overall_score: f64,
222 assessed_dimensions: Vec<PythonQualityDimension>,
223 dimension_scores: std::collections::BTreeMap<PythonQualityDimension, Option<f64>>,
224 low_sample_warning: bool,
225 #[serde(skip_serializing_if = "Option::is_none")]
226 completeness: Option<CompletenessMetrics>,
227 #[serde(skip_serializing_if = "Option::is_none")]
228 consistency: Option<ConsistencyMetrics>,
229 #[serde(skip_serializing_if = "Option::is_none")]
230 uniqueness: Option<PythonUniquenessDocument>,
231 #[serde(skip_serializing_if = "Option::is_none")]
232 accuracy: Option<AccuracyMetrics>,
233 #[serde(skip_serializing_if = "Option::is_none")]
234 timeliness: Option<TimelinessMetrics>,
235 #[serde(skip_serializing_if = "Option::is_none")]
236 validity: Option<ValidityMetrics>,
237 #[serde(skip_serializing_if = "Option::is_none")]
238 precision: Option<PrecisionMetrics>,
239}
240
241#[allow(dead_code)]
242#[derive(
243 Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, serde::Serialize, schemars::JsonSchema,
244)]
245#[serde(rename_all = "lowercase")]
246enum PythonQualityDimension {
247 Completeness,
248 Consistency,
249 Uniqueness,
250 Accuracy,
251 Timeliness,
252 Validity,
253 Precision,
254}
255
256#[allow(dead_code)]
257#[derive(serde::Serialize, schemars::JsonSchema)]
258struct PythonUniquenessDocument {
259 duplicate_rows: usize,
260 key_uniqueness: f64,
261 high_cardinality_warning: bool,
262 rows_checked: usize,
263 key_column: Option<String>,
264 duplicate_rows_approximate: bool,
265}
266
267#[doc(hidden)]
273pub fn profile_report_schema_document() -> serde_json::Value {
274 let settings = schemars::generate::SchemaSettings::draft2020_12().for_serialize();
275 let schema = settings
276 .into_generator()
277 .into_root_schema_for::<SerializedProfileReport>();
278 let mut document =
279 serde_json::to_value(schema).expect("a Schemars schema must serialize to a JSON document");
280 let object = document
281 .as_object_mut()
282 .expect("a root Schemars schema must be a JSON object");
283 object.insert(
284 "$id".to_string(),
285 serde_json::Value::String(format!(
286 "https://andreabozzo.github.io/dataprof/schema/profile-report.v{REPORT_SCHEMA_VERSION}.schema.json"
287 )),
288 );
289 make_compatibility_defaults_optional(&mut document);
290 allow_additive_properties(&mut document);
291 canonicalize_key_order(&mut document);
292 document
293}
294
295fn canonicalize_key_order(value: &mut serde_json::Value) {
304 match value {
305 serde_json::Value::Object(object) => {
306 for child in object.values_mut() {
307 canonicalize_key_order(child);
308 }
309 let mut sorted: Vec<(String, serde_json::Value)> =
310 std::mem::take(object).into_iter().collect();
311 sorted.sort_by(|(left, _), (right, _)| left.cmp(right));
312 object.extend(sorted);
313 }
314 serde_json::Value::Array(items) => {
315 for item in items {
316 canonicalize_key_order(item);
317 }
318 }
319 _ => {}
320 }
321}
322
323fn make_compatibility_defaults_optional(document: &mut serde_json::Value) {
324 let Some(required) = document
325 .pointer_mut("/$defs/ExecutionMetadata/required")
326 .and_then(serde_json::Value::as_array_mut)
327 else {
328 return;
329 };
330 required.retain(|field| field.as_str() != Some("ragged_row_count"));
331}
332
333fn allow_additive_properties(value: &mut serde_json::Value) {
334 match value {
335 serde_json::Value::Object(object) => {
336 if object.get("additionalProperties") == Some(&serde_json::Value::Bool(false)) {
337 object.remove("additionalProperties");
338 }
339 for child in object.values_mut() {
340 allow_additive_properties(child);
341 }
342 }
343 serde_json::Value::Array(items) => {
344 for item in items {
345 allow_additive_properties(item);
346 }
347 }
348 _ => {}
349 }
350}
351
352impl ProfileReport {
353 pub fn new(
355 data_source: DataSource,
356 column_profiles: Vec<ColumnProfile>,
357 execution: ExecutionMetadata,
358 quality: Option<QualityAssessment>,
359 ) -> Self {
360 Self {
361 schema_version: REPORT_SCHEMA_VERSION,
362 id: uuid::Uuid::new_v4().to_string(),
363 timestamp: chrono::Utc::now().to_rfc3339(),
364 data_source,
365 column_profiles,
366 execution,
367 quality,
368 semantic_hint_bindings: Vec::new(),
369 }
370 }
371
372 pub fn with_semantic_hint_bindings(mut self, bindings: Vec<SemanticHintBinding>) -> Self {
374 self.semantic_hint_bindings = bindings;
375 self
376 }
377
378 pub fn with_id(mut self, id: impl Into<String>) -> Self {
380 self.id = id.into();
381 self
382 }
383
384 pub fn with_timestamp(mut self, timestamp: impl Into<String>) -> Self {
386 self.timestamp = timestamp.into();
387 self
388 }
389
390 pub fn quality_score(&self) -> Option<f64> {
395 self.quality
396 .as_ref()
397 .filter(|q| !q.metrics.assessed_dimensions().is_empty())
398 .map(|q| q.score())
399 }
400
401 pub fn source_identifier(&self) -> String {
403 self.data_source.identifier()
404 }
405}
406
407#[derive(serde::Deserialize)]
412struct ProfileReportFields {
413 #[serde(default)]
414 schema_version: u32,
415 id: String,
416 timestamp: String,
417 data_source: DataSource,
418 column_profiles: Vec<ColumnProfile>,
419 #[serde(alias = "scan_info")]
420 execution: ExecutionMetadata,
421 #[serde(
422 alias = "data_quality_metrics",
423 default,
424 deserialize_with = "deserialize_quality_compat"
425 )]
426 quality: Option<QualityAssessment>,
427 #[serde(default)]
428 semantic_hint_bindings: Vec<SemanticHintBinding>,
429}
430
431impl From<ProfileReportFields> for ProfileReport {
432 fn from(fields: ProfileReportFields) -> Self {
433 Self {
434 schema_version: fields.schema_version,
435 id: fields.id,
436 timestamp: fields.timestamp,
437 data_source: fields.data_source,
438 column_profiles: fields.column_profiles,
439 execution: fields.execution,
440 quality: fields.quality,
441 semantic_hint_bindings: fields.semantic_hint_bindings,
442 }
443 }
444}
445
446impl<'de> serde::Deserialize<'de> for ProfileReport {
452 fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
453 where
454 D: serde::Deserializer<'de>,
455 {
456 use serde::de::Error;
457
458 let value = serde_json::Value::deserialize(deserializer)?;
459 match value.get("schema_version") {
460 None => {}
462 Some(serde_json::Value::Number(n)) if n.as_u64().is_some() => {
463 let version = n.as_u64().unwrap_or_default();
464 if version > u64::from(REPORT_SCHEMA_VERSION) {
465 return Err(D::Error::custom(format!(
466 "report schema version {version} is newer than the latest supported \
467 version {REPORT_SCHEMA_VERSION}; upgrade dataprof to read this report"
468 )));
469 }
470 }
471 Some(other) => {
473 return Err(D::Error::custom(format!(
474 "report schema_version must be a non-negative integer, got {other}"
475 )));
476 }
477 }
478 ProfileReportFields::deserialize(value)
479 .map(ProfileReport::from)
480 .map_err(D::Error::custom)
481 }
482}
483
484fn deserialize_quality_compat<'de, D>(
487 deserializer: D,
488) -> Result<Option<QualityAssessment>, D::Error>
489where
490 D: serde::Deserializer<'de>,
491{
492 use serde::Deserialize;
493
494 let value: Option<serde_json::Value> = Option::deserialize(deserializer)?;
495 match value {
496 None => Ok(None),
497 Some(v) => {
498 if v.get("metrics").is_some() && v.get("confidence").is_some() {
499 let assessment: QualityAssessment =
500 serde_json::from_value(v).map_err(serde::de::Error::custom)?;
501 Ok(Some(assessment))
502 } else {
503 let metrics: QualityMetrics =
504 serde_json::from_value(v).map_err(serde::de::Error::custom)?;
505 Ok(Some(QualityAssessment::exact(metrics)))
506 }
507 }
508 }
509}
510
511#[cfg(test)]
512mod tests {
513 use super::*;
514 use dataprof_core::FileFormat;
515 use dataprof_metrics::MetricConfidence;
516 use serde_json::json;
517
518 #[test]
519 fn test_profile_report_json_roundtrip() {
520 let report = ProfileReport::new(
521 DataSource::File {
522 path: "test.csv".to_string(),
523 format: FileFormat::Csv,
524 size_bytes: 1024,
525 modified_at: None,
526 parquet_metadata: None,
527 },
528 vec![],
529 ExecutionMetadata::new(100, 5, 50),
530 Some(QualityAssessment::exact(QualityMetrics::empty())),
531 );
532
533 let json = serde_json::to_string(&report).unwrap();
534 let deserialized: ProfileReport = serde_json::from_str(&json).unwrap();
535
536 assert_eq!(deserialized.id, report.id);
537 assert_eq!(deserialized.timestamp, report.timestamp);
538 assert_eq!(deserialized.source_identifier(), "test.csv");
539 assert_eq!(deserialized.execution.rows_processed, 100);
540 assert!(deserialized.quality.is_some());
541 assert_eq!(deserialized.schema_version, REPORT_SCHEMA_VERSION);
542 }
543
544 #[test]
545 fn test_serialized_report_carries_schema_version() {
546 let report = ProfileReport::new(
547 DataSource::File {
548 path: "test.csv".to_string(),
549 format: FileFormat::Csv,
550 size_bytes: 1024,
551 modified_at: None,
552 parquet_metadata: None,
553 },
554 vec![],
555 ExecutionMetadata::new(100, 5, 50),
556 None,
557 );
558
559 let value = serde_json::to_value(&report).unwrap();
560 assert_eq!(
561 value.get("schema_version").and_then(|v| v.as_u64()),
562 Some(u64::from(REPORT_SCHEMA_VERSION))
563 );
564 }
565
566 #[test]
567 fn test_profile_report_without_quality() {
568 let report = ProfileReport::new(
569 DataSource::File {
570 path: "test.csv".to_string(),
571 format: FileFormat::Csv,
572 size_bytes: 1024,
573 modified_at: None,
574 parquet_metadata: None,
575 },
576 vec![],
577 ExecutionMetadata::new(100, 5, 50),
578 None,
579 );
580
581 let json = serde_json::to_string(&report).unwrap();
582 let deserialized: ProfileReport = serde_json::from_str(&json).unwrap();
583
584 assert!(deserialized.quality.is_none());
585 assert_eq!(deserialized.execution.rows_processed, 100);
586 }
587
588 #[test]
589 fn test_profile_report_deserializes_legacy_quality_metrics() {
590 let json = json!({
591 "id": "legacy-report",
592 "timestamp": "2026-05-22T10:00:00Z",
593 "data_source": {
594 "type": "file",
595 "path": "test.csv",
596 "format": "csv",
597 "size_bytes": 42
598 },
599 "column_profiles": [],
600 "scan_info": {
601 "rows_processed": 10,
602 "columns_detected": 2,
603 "scan_time_ms": 5,
604 "error_count": 0,
605 "source_exhausted": true,
606 "sampling_applied": false
607 },
608 "data_quality_metrics": {
609 "completeness": {
610 "missing_values_ratio": 0.0,
611 "complete_records_ratio": 100.0,
612 "null_columns": []
613 }
614 }
615 });
616
617 let report: ProfileReport = serde_json::from_value(json).unwrap();
618
619 assert_eq!(report.id, "legacy-report");
620 assert_eq!(report.schema_version, 0);
623 assert_eq!(report.execution.rows_processed, 10);
624 assert!(report.quality_score().is_none());
627 let quality = report
628 .quality
629 .expect("expected legacy quality to deserialize");
630 assert!(matches!(quality.confidence, MetricConfidence::Exact));
631 let completeness = quality
632 .metrics
633 .completeness
634 .as_ref()
635 .expect("legacy completeness facts should deserialize");
636 assert!((completeness.complete_records_ratio - 100.0).abs() < 0.01);
637 assert!(quality.metrics.assessed_dimensions().is_empty());
638 }
639
640 fn current_document() -> serde_json::Value {
641 json!({
642 "schema_version": REPORT_SCHEMA_VERSION,
643 "id": "current-report",
644 "timestamp": "2026-07-16T10:00:00Z",
645 "data_source": {
646 "type": "file",
647 "path": "test.csv",
648 "format": "csv",
649 "size_bytes": 42
650 },
651 "column_profiles": [],
652 "execution": {
653 "rows_processed": 10,
654 "columns_detected": 2,
655 "scan_time_ms": 5,
656 "error_count": 0,
657 "source_exhausted": true,
658 "sampling_applied": false
659 }
660 })
661 }
662
663 #[test]
664 fn test_additive_fields_from_newer_writer_are_ignored() {
665 let mut json = current_document();
668 json["a_future_additive_field"] = json!({"anything": true});
669 json["column_profiles"] = json!([]);
670
671 let report: ProfileReport = serde_json::from_value(json).unwrap();
672 assert_eq!(report.schema_version, REPORT_SCHEMA_VERSION);
673 assert_eq!(report.id, "current-report");
674 }
675
676 #[test]
677 fn test_unsupported_future_schema_version_fails_explicitly() {
678 let mut json = current_document();
679 json["schema_version"] = json!(REPORT_SCHEMA_VERSION + 1);
680
681 let err = serde_json::from_value::<ProfileReport>(json).unwrap_err();
682 let msg = err.to_string();
683 assert!(
684 msg.contains("schema version") && msg.contains("upgrade dataprof"),
685 "expected an actionable schema-version error, got: {msg}"
686 );
687 }
688
689 #[test]
690 fn test_version_error_wins_over_structural_errors() {
691 let json_text = format!(
695 r#"{{"column_profiles": "not-an-array", "schema_version": {}}}"#,
696 REPORT_SCHEMA_VERSION + 1
697 );
698 let err = serde_json::from_str::<ProfileReport>(&json_text).unwrap_err();
699 let msg = err.to_string();
700 assert!(
701 msg.contains("upgrade dataprof"),
702 "expected the schema-version error to win, got: {msg}"
703 );
704 }
705
706 #[test]
707 fn test_null_schema_version_is_malformed_not_legacy() {
708 let mut json = current_document();
709 json["schema_version"] = json!(null);
710
711 let err = serde_json::from_value::<ProfileReport>(json).unwrap_err();
712 let msg = err.to_string();
713 assert!(
714 msg.contains("schema_version must be a non-negative integer"),
715 "expected explicit rejection of null schema_version, got: {msg}"
716 );
717 }
718
719 #[test]
720 fn test_non_integer_schema_version_is_rejected() {
721 for bad in [json!("1"), json!(1.5), json!(-1), json!(true)] {
722 let mut json = current_document();
723 json["schema_version"] = bad.clone();
724 let err = serde_json::from_value::<ProfileReport>(json).unwrap_err();
725 assert!(
726 err.to_string().contains("non-negative integer"),
727 "expected rejection of {bad}, got: {err}"
728 );
729 }
730 }
731}