use dataprof_core::{ColumnProfile, DataSource, ExecutionMetadata, SemanticHintBinding};
use dataprof_metrics::{
AccuracyMetrics, CompletenessMetrics, ConsistencyMetrics, PrecisionMetrics, QualityAssessment,
QualityMetrics, TimelinessMetrics, ValidityMetrics,
};
pub const REPORT_SCHEMA_VERSION: u32 = 1;
#[derive(Debug, Clone, serde::Serialize, schemars::JsonSchema)]
pub struct ProfileReport {
#[schemars(schema_with = "schema_version_schema")]
pub schema_version: u32,
pub id: String,
pub timestamp: String,
pub data_source: DataSource,
pub column_profiles: Vec<ColumnProfile>,
pub execution: ExecutionMetadata,
#[serde(skip_serializing_if = "Option::is_none")]
pub quality: Option<QualityAssessment>,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub semantic_hint_bindings: Vec<SemanticHintBinding>,
}
fn schema_version_schema(_: &mut schemars::SchemaGenerator) -> schemars::Schema {
schemars::json_schema!({
"type": "integer",
"const": REPORT_SCHEMA_VERSION,
"minimum": 0
})
}
#[allow(dead_code)]
#[derive(serde::Serialize, schemars::JsonSchema)]
#[serde(untagged)]
#[schemars(title = "ProfileReport")]
enum SerializedProfileReport {
Rust(Box<ProfileReport>),
Python(Box<PythonProfileReportDocument>),
}
#[allow(dead_code)]
#[derive(serde::Serialize, schemars::JsonSchema)]
struct PythonProfileReportDocument {
#[schemars(schema_with = "schema_version_schema")]
schema_version: u32,
source: String,
source_type: PythonSourceType,
execution: PythonExecutionDocument,
columns: Vec<PythonColumnDocument>,
quality: Option<PythonQualityDocument>,
#[serde(skip_serializing_if = "Vec::is_empty")]
semantic_hint_bindings: Vec<SemanticHintBinding>,
}
#[allow(dead_code)]
#[derive(serde::Serialize, schemars::JsonSchema)]
#[serde(rename_all = "lowercase")]
enum PythonSourceType {
File,
Query,
Dataframe,
Stream,
Bytes,
}
#[allow(dead_code)]
#[derive(serde::Serialize, schemars::JsonSchema)]
struct PythonExecutionDocument {
engine: Option<String>,
rows_processed: usize,
columns_detected: usize,
scan_time_ms: u128,
source_exhausted: bool,
truncation_reason: Option<String>,
bytes_consumed: Option<u64>,
throughput_rows_sec: Option<f64>,
memory_peak_mb: Option<f64>,
error_count: usize,
ragged_row_count: usize,
sampling_applied: bool,
sampling_ratio: Option<f64>,
}
#[allow(dead_code)]
#[derive(serde::Serialize, schemars::JsonSchema)]
struct PythonColumnDocument {
name: String,
data_type: PythonDataType,
total_count: usize,
null_count: usize,
null_percentage: Option<f64>,
unique_count: Option<usize>,
unique_count_is_approximate: Option<bool>,
uniqueness_ratio: Option<f64>,
#[serde(skip_serializing_if = "Option::is_none")]
invalid_count: Option<usize>,
#[serde(skip_serializing_if = "Option::is_none")]
type_homogeneity: Option<dataprof_core::TypeHomogeneity>,
#[serde(skip_serializing_if = "Option::is_none")]
stats: Option<PythonColumnStatsDocument>,
#[serde(skip_serializing_if = "Option::is_none")]
patterns: Option<Vec<PythonPatternDocument>>,
}
#[allow(dead_code)]
#[derive(serde::Serialize, schemars::JsonSchema)]
#[serde(rename_all = "lowercase")]
enum PythonDataType {
String,
Identifier,
Integer,
Float,
Date,
Boolean,
}
#[allow(dead_code)]
#[derive(serde::Serialize, schemars::JsonSchema)]
struct PythonColumnStatsDocument {
#[serde(skip_serializing_if = "Option::is_none")]
min: Option<f64>,
#[serde(skip_serializing_if = "Option::is_none")]
max: Option<f64>,
#[serde(skip_serializing_if = "Option::is_none")]
mean: Option<f64>,
#[serde(skip_serializing_if = "Option::is_none")]
std_dev: Option<f64>,
#[serde(skip_serializing_if = "Option::is_none")]
variance: Option<f64>,
#[serde(skip_serializing_if = "Option::is_none")]
median: Option<f64>,
#[serde(skip_serializing_if = "Option::is_none")]
mode: Option<f64>,
#[serde(skip_serializing_if = "Option::is_none")]
skewness: Option<f64>,
#[serde(skip_serializing_if = "Option::is_none")]
kurtosis: Option<f64>,
#[serde(skip_serializing_if = "Option::is_none")]
coefficient_of_variation: Option<f64>,
#[serde(skip_serializing_if = "Option::is_none")]
quartiles: Option<dataprof_core::Quartiles>,
#[serde(skip_serializing_if = "Option::is_none")]
is_approximate: Option<bool>,
#[serde(skip_serializing_if = "Option::is_none")]
outlier_count: Option<usize>,
#[serde(skip_serializing_if = "Option::is_none")]
min_length: Option<usize>,
#[serde(skip_serializing_if = "Option::is_none")]
max_length: Option<usize>,
#[serde(skip_serializing_if = "Option::is_none")]
avg_length: Option<f64>,
#[serde(skip_serializing_if = "Option::is_none")]
true_count: Option<usize>,
#[serde(skip_serializing_if = "Option::is_none")]
false_count: Option<usize>,
#[serde(skip_serializing_if = "Option::is_none")]
true_ratio: Option<f64>,
}
#[allow(dead_code)]
#[derive(serde::Serialize, schemars::JsonSchema)]
struct PythonPatternDocument {
name: String,
regex: String,
match_count: usize,
match_percentage: f64,
category: dataprof_core::PatternCategory,
confidence: f64,
}
#[allow(dead_code)]
#[derive(serde::Serialize, schemars::JsonSchema)]
struct PythonQualityDocument {
overall_score: f64,
assessed_dimensions: Vec<PythonQualityDimension>,
dimension_scores: std::collections::BTreeMap<PythonQualityDimension, Option<f64>>,
low_sample_warning: bool,
#[serde(skip_serializing_if = "Option::is_none")]
completeness: Option<CompletenessMetrics>,
#[serde(skip_serializing_if = "Option::is_none")]
consistency: Option<ConsistencyMetrics>,
#[serde(skip_serializing_if = "Option::is_none")]
uniqueness: Option<PythonUniquenessDocument>,
#[serde(skip_serializing_if = "Option::is_none")]
accuracy: Option<AccuracyMetrics>,
#[serde(skip_serializing_if = "Option::is_none")]
timeliness: Option<TimelinessMetrics>,
#[serde(skip_serializing_if = "Option::is_none")]
validity: Option<ValidityMetrics>,
#[serde(skip_serializing_if = "Option::is_none")]
precision: Option<PrecisionMetrics>,
}
#[allow(dead_code)]
#[derive(
Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, serde::Serialize, schemars::JsonSchema,
)]
#[serde(rename_all = "lowercase")]
enum PythonQualityDimension {
Completeness,
Consistency,
Uniqueness,
Accuracy,
Timeliness,
Validity,
Precision,
}
#[allow(dead_code)]
#[derive(serde::Serialize, schemars::JsonSchema)]
struct PythonUniquenessDocument {
duplicate_rows: usize,
key_uniqueness: f64,
high_cardinality_warning: bool,
rows_checked: usize,
key_column: Option<String>,
duplicate_rows_approximate: bool,
}
#[doc(hidden)]
pub fn profile_report_schema_document() -> serde_json::Value {
let settings = schemars::generate::SchemaSettings::draft2020_12().for_serialize();
let schema = settings
.into_generator()
.into_root_schema_for::<SerializedProfileReport>();
let mut document =
serde_json::to_value(schema).expect("a Schemars schema must serialize to a JSON document");
let object = document
.as_object_mut()
.expect("a root Schemars schema must be a JSON object");
object.insert(
"$id".to_string(),
serde_json::Value::String(format!(
"https://andreabozzo.github.io/dataprof/schema/profile-report.v{REPORT_SCHEMA_VERSION}.schema.json"
)),
);
make_compatibility_defaults_optional(&mut document);
allow_additive_properties(&mut document);
canonicalize_key_order(&mut document);
document
}
fn canonicalize_key_order(value: &mut serde_json::Value) {
match value {
serde_json::Value::Object(object) => {
for child in object.values_mut() {
canonicalize_key_order(child);
}
let mut sorted: Vec<(String, serde_json::Value)> =
std::mem::take(object).into_iter().collect();
sorted.sort_by(|(left, _), (right, _)| left.cmp(right));
object.extend(sorted);
}
serde_json::Value::Array(items) => {
for item in items {
canonicalize_key_order(item);
}
}
_ => {}
}
}
fn make_compatibility_defaults_optional(document: &mut serde_json::Value) {
let Some(required) = document
.pointer_mut("/$defs/ExecutionMetadata/required")
.and_then(serde_json::Value::as_array_mut)
else {
return;
};
required.retain(|field| field.as_str() != Some("ragged_row_count"));
}
fn allow_additive_properties(value: &mut serde_json::Value) {
match value {
serde_json::Value::Object(object) => {
if object.get("additionalProperties") == Some(&serde_json::Value::Bool(false)) {
object.remove("additionalProperties");
}
for child in object.values_mut() {
allow_additive_properties(child);
}
}
serde_json::Value::Array(items) => {
for item in items {
allow_additive_properties(item);
}
}
_ => {}
}
}
impl ProfileReport {
pub fn new(
data_source: DataSource,
column_profiles: Vec<ColumnProfile>,
execution: ExecutionMetadata,
quality: Option<QualityAssessment>,
) -> Self {
Self {
schema_version: REPORT_SCHEMA_VERSION,
id: uuid::Uuid::new_v4().to_string(),
timestamp: chrono::Utc::now().to_rfc3339(),
data_source,
column_profiles,
execution,
quality,
semantic_hint_bindings: Vec::new(),
}
}
pub fn with_semantic_hint_bindings(mut self, bindings: Vec<SemanticHintBinding>) -> Self {
self.semantic_hint_bindings = bindings;
self
}
pub fn with_id(mut self, id: impl Into<String>) -> Self {
self.id = id.into();
self
}
pub fn with_timestamp(mut self, timestamp: impl Into<String>) -> Self {
self.timestamp = timestamp.into();
self
}
pub fn quality_score(&self) -> Option<f64> {
self.quality
.as_ref()
.filter(|q| !q.metrics.assessed_dimensions().is_empty())
.map(|q| q.score())
}
pub fn source_identifier(&self) -> String {
self.data_source.identifier()
}
}
#[derive(serde::Deserialize)]
struct ProfileReportFields {
#[serde(default)]
schema_version: u32,
id: String,
timestamp: String,
data_source: DataSource,
column_profiles: Vec<ColumnProfile>,
#[serde(alias = "scan_info")]
execution: ExecutionMetadata,
#[serde(
alias = "data_quality_metrics",
default,
deserialize_with = "deserialize_quality_compat"
)]
quality: Option<QualityAssessment>,
#[serde(default)]
semantic_hint_bindings: Vec<SemanticHintBinding>,
}
impl From<ProfileReportFields> for ProfileReport {
fn from(fields: ProfileReportFields) -> Self {
Self {
schema_version: fields.schema_version,
id: fields.id,
timestamp: fields.timestamp,
data_source: fields.data_source,
column_profiles: fields.column_profiles,
execution: fields.execution,
quality: fields.quality,
semantic_hint_bindings: fields.semantic_hint_bindings,
}
}
}
impl<'de> serde::Deserialize<'de> for ProfileReport {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: serde::Deserializer<'de>,
{
use serde::de::Error;
let value = serde_json::Value::deserialize(deserializer)?;
match value.get("schema_version") {
None => {}
Some(serde_json::Value::Number(n)) if n.as_u64().is_some() => {
let version = n.as_u64().unwrap_or_default();
if version > u64::from(REPORT_SCHEMA_VERSION) {
return Err(D::Error::custom(format!(
"report schema version {version} is newer than the latest supported \
version {REPORT_SCHEMA_VERSION}; upgrade dataprof to read this report"
)));
}
}
Some(other) => {
return Err(D::Error::custom(format!(
"report schema_version must be a non-negative integer, got {other}"
)));
}
}
ProfileReportFields::deserialize(value)
.map(ProfileReport::from)
.map_err(D::Error::custom)
}
}
fn deserialize_quality_compat<'de, D>(
deserializer: D,
) -> Result<Option<QualityAssessment>, D::Error>
where
D: serde::Deserializer<'de>,
{
use serde::Deserialize;
let value: Option<serde_json::Value> = Option::deserialize(deserializer)?;
match value {
None => Ok(None),
Some(v) => {
if v.get("metrics").is_some() && v.get("confidence").is_some() {
let assessment: QualityAssessment =
serde_json::from_value(v).map_err(serde::de::Error::custom)?;
Ok(Some(assessment))
} else {
let metrics: QualityMetrics =
serde_json::from_value(v).map_err(serde::de::Error::custom)?;
Ok(Some(QualityAssessment::exact(metrics)))
}
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use dataprof_core::FileFormat;
use dataprof_metrics::MetricConfidence;
use serde_json::json;
#[test]
fn test_profile_report_json_roundtrip() {
let report = ProfileReport::new(
DataSource::File {
path: "test.csv".to_string(),
format: FileFormat::Csv,
size_bytes: 1024,
modified_at: None,
parquet_metadata: None,
},
vec![],
ExecutionMetadata::new(100, 5, 50),
Some(QualityAssessment::exact(QualityMetrics::empty())),
);
let json = serde_json::to_string(&report).unwrap();
let deserialized: ProfileReport = serde_json::from_str(&json).unwrap();
assert_eq!(deserialized.id, report.id);
assert_eq!(deserialized.timestamp, report.timestamp);
assert_eq!(deserialized.source_identifier(), "test.csv");
assert_eq!(deserialized.execution.rows_processed, 100);
assert!(deserialized.quality.is_some());
assert_eq!(deserialized.schema_version, REPORT_SCHEMA_VERSION);
}
#[test]
fn test_serialized_report_carries_schema_version() {
let report = ProfileReport::new(
DataSource::File {
path: "test.csv".to_string(),
format: FileFormat::Csv,
size_bytes: 1024,
modified_at: None,
parquet_metadata: None,
},
vec![],
ExecutionMetadata::new(100, 5, 50),
None,
);
let value = serde_json::to_value(&report).unwrap();
assert_eq!(
value.get("schema_version").and_then(|v| v.as_u64()),
Some(u64::from(REPORT_SCHEMA_VERSION))
);
}
#[test]
fn test_profile_report_without_quality() {
let report = ProfileReport::new(
DataSource::File {
path: "test.csv".to_string(),
format: FileFormat::Csv,
size_bytes: 1024,
modified_at: None,
parquet_metadata: None,
},
vec![],
ExecutionMetadata::new(100, 5, 50),
None,
);
let json = serde_json::to_string(&report).unwrap();
let deserialized: ProfileReport = serde_json::from_str(&json).unwrap();
assert!(deserialized.quality.is_none());
assert_eq!(deserialized.execution.rows_processed, 100);
}
#[test]
fn test_profile_report_deserializes_legacy_quality_metrics() {
let json = json!({
"id": "legacy-report",
"timestamp": "2026-05-22T10:00:00Z",
"data_source": {
"type": "file",
"path": "test.csv",
"format": "csv",
"size_bytes": 42
},
"column_profiles": [],
"scan_info": {
"rows_processed": 10,
"columns_detected": 2,
"scan_time_ms": 5,
"error_count": 0,
"source_exhausted": true,
"sampling_applied": false
},
"data_quality_metrics": {
"completeness": {
"missing_values_ratio": 0.0,
"complete_records_ratio": 100.0,
"null_columns": []
}
}
});
let report: ProfileReport = serde_json::from_value(json).unwrap();
assert_eq!(report.id, "legacy-report");
assert_eq!(report.schema_version, 0);
assert_eq!(report.execution.rows_processed, 10);
assert!(report.quality_score().is_none());
let quality = report
.quality
.expect("expected legacy quality to deserialize");
assert!(matches!(quality.confidence, MetricConfidence::Exact));
let completeness = quality
.metrics
.completeness
.as_ref()
.expect("legacy completeness facts should deserialize");
assert!((completeness.complete_records_ratio - 100.0).abs() < 0.01);
assert!(quality.metrics.assessed_dimensions().is_empty());
}
fn current_document() -> serde_json::Value {
json!({
"schema_version": REPORT_SCHEMA_VERSION,
"id": "current-report",
"timestamp": "2026-07-16T10:00:00Z",
"data_source": {
"type": "file",
"path": "test.csv",
"format": "csv",
"size_bytes": 42
},
"column_profiles": [],
"execution": {
"rows_processed": 10,
"columns_detected": 2,
"scan_time_ms": 5,
"error_count": 0,
"source_exhausted": true,
"sampling_applied": false
}
})
}
#[test]
fn test_additive_fields_from_newer_writer_are_ignored() {
let mut json = current_document();
json["a_future_additive_field"] = json!({"anything": true});
json["column_profiles"] = json!([]);
let report: ProfileReport = serde_json::from_value(json).unwrap();
assert_eq!(report.schema_version, REPORT_SCHEMA_VERSION);
assert_eq!(report.id, "current-report");
}
#[test]
fn test_unsupported_future_schema_version_fails_explicitly() {
let mut json = current_document();
json["schema_version"] = json!(REPORT_SCHEMA_VERSION + 1);
let err = serde_json::from_value::<ProfileReport>(json).unwrap_err();
let msg = err.to_string();
assert!(
msg.contains("schema version") && msg.contains("upgrade dataprof"),
"expected an actionable schema-version error, got: {msg}"
);
}
#[test]
fn test_version_error_wins_over_structural_errors() {
let json_text = format!(
r#"{{"column_profiles": "not-an-array", "schema_version": {}}}"#,
REPORT_SCHEMA_VERSION + 1
);
let err = serde_json::from_str::<ProfileReport>(&json_text).unwrap_err();
let msg = err.to_string();
assert!(
msg.contains("upgrade dataprof"),
"expected the schema-version error to win, got: {msg}"
);
}
#[test]
fn test_null_schema_version_is_malformed_not_legacy() {
let mut json = current_document();
json["schema_version"] = json!(null);
let err = serde_json::from_value::<ProfileReport>(json).unwrap_err();
let msg = err.to_string();
assert!(
msg.contains("schema_version must be a non-negative integer"),
"expected explicit rejection of null schema_version, got: {msg}"
);
}
#[test]
fn test_non_integer_schema_version_is_rejected() {
for bad in [json!("1"), json!(1.5), json!(-1), json!(true)] {
let mut json = current_document();
json["schema_version"] = bad.clone();
let err = serde_json::from_value::<ProfileReport>(json).unwrap_err();
assert!(
err.to_string().contains("non-negative integer"),
"expected rejection of {bad}, got: {err}"
);
}
}
}