use std::collections::HashMap;
use serde::{Deserialize, Serialize};
use crate::classification::{DataType, TypeHomogeneity};
use crate::pattern::Pattern;
#[derive(Debug, Clone, Serialize, Deserialize, schemars::JsonSchema)]
pub struct ColumnProfile {
pub name: String,
pub data_type: DataType,
pub null_count: usize,
pub total_count: usize,
pub unique_count: Option<usize>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub unique_count_is_approximate: Option<bool>,
#[serde(skip)]
#[schemars(skip)]
pub unique_count_lower_bound: Option<usize>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub invalid_count: Option<usize>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub type_homogeneity: Option<TypeHomogeneity>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub locale_number_count: Option<usize>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub native_type: Option<NativeType>,
pub stats: ColumnStats,
pub patterns: Option<Vec<Pattern>>,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)]
#[non_exhaustive]
pub struct NativeType {
pub system: String,
pub name: String,
}
impl NativeType {
pub const ARROW: &'static str = "arrow";
pub const PANDAS: &'static str = "pandas";
pub const POSTGRESQL: &'static str = "postgresql";
pub const MYSQL: &'static str = "mysql";
pub fn new(system: impl Into<String>, name: impl Into<String>) -> Self {
Self {
system: system.into(),
name: name.into(),
}
}
pub fn declares_decimal(&self) -> bool {
let arrow_decimal = |name: &str| {
["decimal32(", "decimal64(", "decimal128(", "decimal256("]
.iter()
.any(|prefix| name.starts_with(prefix))
};
match self.system.as_str() {
Self::ARROW => arrow_decimal(&self.name),
Self::PANDAS => self
.name
.strip_suffix("[pyarrow]")
.is_some_and(arrow_decimal),
Self::POSTGRESQL => self.name == "NUMERIC",
Self::MYSQL => self.name == "DECIMAL",
_ => false,
}
}
pub fn declares_date(&self) -> bool {
let arrow_date = |name: &str| {
name == "date32[day]" || name == "date64[ms]" || name.starts_with("timestamp[")
};
match self.system.as_str() {
Self::ARROW => arrow_date(&self.name),
Self::PANDAS => {
self.name.starts_with("datetime64[")
|| self.name.strip_suffix("[pyarrow]").is_some_and(arrow_date)
}
Self::POSTGRESQL => matches!(self.name.as_str(), "DATE" | "TIMESTAMP" | "TIMESTAMPTZ"),
Self::MYSQL => matches!(self.name.as_str(), "DATE" | "DATETIME" | "TIMESTAMP"),
_ => false,
}
}
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, schemars::JsonSchema)]
pub struct Quartiles {
pub q1: f64,
pub q2: f64,
pub q3: f64,
#[serde(deserialize_with = "crate::serde_helpers::required_nullable")]
pub iqr: Option<f64>,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)]
pub struct ExactNumericStats {
pub min: String,
pub max: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub median: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub q1: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub q2: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub q3: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub mode: Option<String>,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, schemars::JsonSchema)]
pub struct FrequencyItem {
pub value: String,
pub count: usize,
#[serde(serialize_with = "crate::serde_helpers::round_2")]
pub percentage: f64,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)]
#[serde(rename_all = "snake_case")]
pub enum StatisticRounding {
FourDecimalsTwelveSignificant,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)]
#[serde(rename_all = "snake_case")]
pub enum ShapeMomentEstimator {
AdjustedSample,
}
#[derive(Debug, Clone, Serialize, Deserialize, schemars::JsonSchema)]
pub struct NumericStats {
#[serde(serialize_with = "crate::serde_helpers::round_4")]
pub min: f64,
#[serde(serialize_with = "crate::serde_helpers::round_4")]
pub max: f64,
#[serde(serialize_with = "crate::serde_helpers::round_statistic")]
pub mean: f64,
#[serde(
serialize_with = "crate::serde_helpers::round_statistic_opt",
deserialize_with = "crate::serde_helpers::required_nullable"
)]
pub std_dev: Option<f64>,
#[serde(
serialize_with = "crate::serde_helpers::round_statistic_opt",
deserialize_with = "crate::serde_helpers::required_nullable"
)]
pub variance: Option<f64>,
#[serde(
skip_serializing_if = "Option::is_none",
serialize_with = "crate::serde_helpers::round_4_opt"
)]
pub median: Option<f64>,
#[serde(
skip_serializing_if = "Option::is_none",
serialize_with = "crate::serde_helpers::quartiles::serialize"
)]
pub quartiles: Option<Quartiles>,
#[serde(
skip_serializing_if = "Option::is_none",
serialize_with = "crate::serde_helpers::round_4_opt"
)]
pub mode: Option<f64>,
#[serde(
skip_serializing_if = "Option::is_none",
serialize_with = "crate::serde_helpers::round_2_opt"
)]
pub coefficient_of_variation: Option<f64>,
#[serde(
skip_serializing_if = "Option::is_none",
serialize_with = "crate::serde_helpers::round_statistic_opt"
)]
pub skewness: Option<f64>,
#[serde(
skip_serializing_if = "Option::is_none",
serialize_with = "crate::serde_helpers::round_statistic_opt"
)]
pub kurtosis: Option<f64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub is_approximate: Option<bool>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub outlier_count: Option<usize>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub exact: Option<ExactNumericStats>,
}
impl NumericStats {
pub fn empty() -> Self {
Self {
min: 0.0,
max: 0.0,
mean: 0.0,
std_dev: Some(0.0),
variance: Some(0.0),
median: None,
quartiles: None,
mode: None,
coefficient_of_variation: None,
skewness: None,
kurtosis: None,
is_approximate: None,
outlier_count: None,
exact: None,
}
}
}
#[derive(Debug, Clone, Serialize, Deserialize, schemars::JsonSchema)]
pub struct TextStats {
pub min_length: usize,
pub max_length: usize,
#[serde(serialize_with = "crate::serde_helpers::round_4")]
pub avg_length: f64,
#[serde(skip_serializing_if = "Option::is_none")]
pub most_frequent: Option<Vec<FrequencyItem>>,
#[serde(skip_serializing_if = "Option::is_none")]
pub least_frequent: Option<Vec<FrequencyItem>>,
}
impl TextStats {
pub fn empty() -> Self {
Self {
min_length: 0,
max_length: 0,
avg_length: 0.0,
most_frequent: None,
least_frequent: None,
}
}
pub fn from_lengths(min_length: usize, max_length: usize, avg_length: f64) -> Self {
Self {
min_length: if min_length == usize::MAX {
0
} else {
min_length
},
max_length,
avg_length,
most_frequent: None,
least_frequent: None,
}
}
}
#[derive(Debug, Clone, Serialize, Deserialize, schemars::JsonSchema)]
pub struct DateTimeStats {
pub min_datetime: String,
pub max_datetime: String,
#[serde(serialize_with = "crate::serde_helpers::round_2")]
pub duration_days: f64,
pub year_distribution: HashMap<i32, usize>,
pub month_distribution: HashMap<u32, usize>,
pub day_of_week_distribution: HashMap<String, usize>,
#[serde(skip_serializing_if = "Option::is_none")]
pub hour_distribution: Option<HashMap<u32, usize>>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub slash_date_order: Option<SlashDateOrder>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub largest_gap: Option<DateGap>,
}
pub const DATE_GAP_YEAR_LIMIT: usize = 10_000;
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)]
#[non_exhaustive]
pub struct DateGap {
pub days: i64,
pub from: String,
pub to: String,
}
impl DateGap {
pub fn new(from: chrono::NaiveDate, to: chrono::NaiveDate) -> Self {
Self {
days: (to - from).num_days(),
from: from.format("%Y-%m-%d").to_string(),
to: to.format("%Y-%m-%d").to_string(),
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)]
#[serde(rename_all = "snake_case")]
pub enum SlashDateOrder {
DayFirst,
MonthFirst,
AssumedDayFirst,
Mixed,
}
impl DateTimeStats {
pub fn empty() -> Self {
Self {
min_datetime: String::new(),
max_datetime: String::new(),
duration_days: 0.0,
year_distribution: HashMap::new(),
month_distribution: HashMap::new(),
day_of_week_distribution: HashMap::new(),
hour_distribution: None,
slash_date_order: None,
largest_gap: None,
}
}
}
#[derive(Debug, Clone, Serialize, Deserialize, schemars::JsonSchema)]
pub struct BooleanStats {
pub true_count: usize,
pub false_count: usize,
#[serde(serialize_with = "crate::serde_helpers::round_4")]
pub true_ratio: f64,
}
#[derive(Debug, Clone, Serialize, Deserialize, schemars::JsonSchema)]
pub enum ColumnStats {
Numeric(NumericStats),
Text(TextStats),
DateTime(DateTimeStats),
Boolean(BooleanStats),
None,
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn declares_decimal_names_each_systems_decimal() {
let cases = [
(NativeType::ARROW, "decimal32(9, 2)", true),
(NativeType::ARROW, "decimal64(18, 2)", true),
(NativeType::ARROW, "decimal128(12, 2)", true),
(NativeType::ARROW, "decimal256(40, -2)", true),
(NativeType::PANDAS, "decimal128(12, 2)[pyarrow]", true),
(NativeType::POSTGRESQL, "NUMERIC", true),
(NativeType::MYSQL, "DECIMAL", true),
(NativeType::ARROW, "double", false),
(NativeType::ARROW, "list<item: decimal128(12, 2)>", false),
(NativeType::ARROW, "extension<example.money>", false),
(NativeType::PANDAS, "object", false),
(NativeType::PANDAS, "decimal128(12, 2)", false),
(NativeType::PANDAS, "Float64", false),
(NativeType::POSTGRESQL, "NUMERIC[]", false),
(NativeType::POSTGRESQL, "FLOAT8", false),
(NativeType::MYSQL, "DOUBLE", false),
("duckdb", "DECIMAL", false),
];
let wrong: Vec<String> = cases
.iter()
.filter(|(system, name, want)| {
NativeType::new(*system, *name).declares_decimal() != *want
})
.map(|(system, name, want)| format!("{system} {name}: expected {want}"))
.collect();
assert!(wrong.is_empty(), "{}", wrong.join("\n"));
}
#[test]
fn declares_date_names_each_systems_date() {
let cases = [
(NativeType::ARROW, "date32[day]", true),
(NativeType::ARROW, "date64[ms]", true),
(NativeType::ARROW, "timestamp[us]", true),
(NativeType::ARROW, "timestamp[ns, tz=UTC]", true),
(NativeType::PANDAS, "datetime64[ns]", true),
(NativeType::PANDAS, "datetime64[us, Europe/Rome]", true),
(NativeType::PANDAS, "timestamp[us, tz=UTC][pyarrow]", true),
(NativeType::PANDAS, "date32[day][pyarrow]", true),
(NativeType::POSTGRESQL, "DATE", true),
(NativeType::POSTGRESQL, "TIMESTAMP", true),
(NativeType::POSTGRESQL, "TIMESTAMPTZ", true),
(NativeType::MYSQL, "DATE", true),
(NativeType::MYSQL, "DATETIME", true),
(NativeType::MYSQL, "TIMESTAMP", true),
(NativeType::ARROW, "time64[us]", false),
(NativeType::ARROW, "string", false),
(NativeType::ARROW, "large_string", false),
(NativeType::ARROW, "list<item: date32[day]>", false),
(NativeType::ARROW, "extension<example.day>", false),
(NativeType::PANDAS, "object", false),
(NativeType::PANDAS, "string", false),
(NativeType::PANDAS, "date32[day]", false),
(NativeType::POSTGRESQL, "TEXT", false),
(NativeType::POSTGRESQL, "TIME", false),
(NativeType::POSTGRESQL, "DATE[]", false),
(NativeType::MYSQL, "VARCHAR", false),
("duckdb", "DATE", false),
];
let wrong: Vec<String> = cases
.iter()
.filter(|(system, name, want)| NativeType::new(*system, *name).declares_date() != *want)
.map(|(system, name, want)| format!("{system} {name}: expected {want}"))
.collect();
assert!(wrong.is_empty(), "{}", wrong.join("\n"));
}
#[test]
fn nullable_spread_fields_are_still_required() {
let stats = NumericStats {
std_dev: None,
variance: None,
quartiles: Some(Quartiles {
q1: 1.0,
q2: 2.0,
q3: 3.0,
iqr: None,
}),
..NumericStats::empty()
};
let document = serde_json::to_value(&stats).unwrap();
let restored: NumericStats = serde_json::from_value(document.clone()).unwrap();
assert_eq!(serde_json::to_value(restored).unwrap(), document);
for (parent, key) in [
(None, "std_dev"),
(None, "variance"),
(Some("quartiles"), "iqr"),
] {
let mut malformed = document.clone();
let object = match parent {
Some(parent) => &mut malformed[parent],
None => &mut malformed,
};
object.as_object_mut().unwrap().remove(key).unwrap();
let error = serde_json::from_value::<NumericStats>(malformed).unwrap_err();
assert!(
error
.to_string()
.contains(&format!("missing field `{key}`")),
"{key}: {error}"
);
}
}
#[test]
fn test_column_profile_json_roundtrip() {
let profile = ColumnProfile {
name: "test_col".to_string(),
data_type: DataType::Integer,
null_count: 2,
total_count: 10,
unique_count: Some(8),
unique_count_is_approximate: Some(false),
unique_count_lower_bound: None,
invalid_count: Some(0),
type_homogeneity: None,
locale_number_count: None,
native_type: None,
stats: ColumnStats::Numeric(NumericStats {
min: 1.0,
max: 100.0,
mean: 50.5,
std_dev: Some(28.87),
variance: Some(833.25),
median: Some(50.0),
quartiles: Some(Quartiles {
q1: 25.0,
q2: 50.0,
q3: 75.0,
iqr: Some(50.0),
}),
mode: Some(42.0),
coefficient_of_variation: Some(57.17),
skewness: Some(0.0),
kurtosis: Some(-1.2),
is_approximate: Some(false),
outlier_count: Some(0),
exact: None,
}),
patterns: Some(vec![]),
};
let json = serde_json::to_string(&profile).unwrap();
let deserialized: ColumnProfile = serde_json::from_str(&json).unwrap();
assert_eq!(deserialized.name, "test_col");
assert_eq!(deserialized.data_type, DataType::Integer);
assert_eq!(deserialized.total_count, 10);
assert_eq!(deserialized.null_count, 2);
assert_eq!(deserialized.unique_count_is_approximate, Some(false));
if let ColumnStats::Numeric(n) = &deserialized.stats {
assert!((n.min - 1.0).abs() < 0.01);
assert!((n.max - 100.0).abs() < 0.01);
assert!((n.mean - 50.5).abs() < 0.01);
assert!(n.median.is_some());
assert!(n.quartiles.is_some());
} else {
panic!("Expected Numeric stats after roundtrip");
}
}
#[test]
fn test_text_stats_json_roundtrip() {
let profile = ColumnProfile {
name: "name".to_string(),
data_type: DataType::String,
null_count: 0,
total_count: 3,
unique_count: Some(3),
unique_count_is_approximate: Some(false),
unique_count_lower_bound: None,
invalid_count: None,
type_homogeneity: None,
locale_number_count: None,
native_type: None,
stats: ColumnStats::Text(TextStats {
min_length: 3,
max_length: 7,
avg_length: 5.0,
most_frequent: None,
least_frequent: None,
}),
patterns: Some(vec![]),
};
let json = serde_json::to_string(&profile).unwrap();
let deserialized: ColumnProfile = serde_json::from_str(&json).unwrap();
assert_eq!(deserialized.data_type, DataType::String);
if let ColumnStats::Text(t) = &deserialized.stats {
assert_eq!(t.min_length, 3);
assert_eq!(t.max_length, 7);
} else {
panic!("Expected Text stats after roundtrip");
}
}
}