Skip to main content

dataprof_core/
profile.rs

1use std::collections::HashMap;
2
3use serde::{Deserialize, Serialize};
4
5use crate::classification::{DataType, TypeHomogeneity};
6use crate::pattern::Pattern;
7
8/// Profiling statistics for a single column.
9#[derive(Debug, Clone, Serialize, Deserialize, schemars::JsonSchema)]
10pub struct ColumnProfile {
11    pub name: String,
12    pub data_type: DataType,
13    pub null_count: usize,
14    pub total_count: usize,
15    pub unique_count: Option<usize>,
16    /// Whether `unique_count` is an approximate (HyperLogLog) estimate rather
17    /// than an exact distinct count.
18    ///
19    /// `None` when `unique_count` is `None` (never computed); `Some(false)` for
20    /// an exact count; `Some(true)` once the cardinality estimator has spilled
21    /// to its HLL sketch (~1% relative error). Consumers running key or
22    /// high-cardinality/uniqueness gates must treat `Some(true)` as "do not rely
23    /// on this as an exact integer" -- an exact-looking count with no provenance
24    /// is unsafe for those checks.
25    #[serde(default, skip_serializing_if = "Option::is_none")]
26    pub unique_count_is_approximate: Option<bool>,
27    /// A distinct count the column is certain to reach when `unique_count` is
28    /// an estimate: the distinct values the exact set held when it was
29    /// dropped. `None` when no floor was recorded, which includes every exact
30    /// count (the count is its own floor) and every report read back from a
31    /// document.
32    ///
33    /// Not serialized: it exists so quality bounds computed during the run can
34    /// bound a key column's uniqueness with certainty (#819). A saved report
35    /// keeps the bounds themselves.
36    #[serde(skip)]
37    #[schemars(skip)]
38    pub unique_count_lower_bound: Option<usize>,
39    /// Non-null values that failed the column type's raw validity predicate:
40    /// non-finite or malformed numbers on numeric columns, and values that do
41    /// not parse directly as calendar dates on date columns. The date predicate
42    /// intentionally does not trim surrounding whitespace; descriptive date
43    /// statistics may normalize whitespace independently.
44    ///
45    /// For numeric columns, `mean`/`std_dev` cover
46    /// `total_count - null_count - invalid_count` values. For date columns,
47    /// this count audits the strict raw-value quality predicate. `None` means
48    /// the check did not run (another column type, or statistics skipped) —
49    /// never "no invalid values", which is `Some(0)`.
50    #[serde(default, skip_serializing_if = "Option::is_none")]
51    pub invalid_count: Option<usize>,
52    /// How the column's non-null values distribute across lexical classes.
53    ///
54    /// `data_type` cannot answer "did this column have a dominant form?": a
55    /// column of names and a column that is 60% numbers are both `String`, and
56    /// `invalid_count` is absent on string columns by contract. This carries the
57    /// evidence, so a consumer can tell a textual column from one that defeated
58    /// type inference.
59    ///
60    /// Counted over the values the profiler retained — the engine's bounded
61    /// reservoir sample on a large source, the whole column on a small or
62    /// in-memory one. `classified_count()` against `total_count - null_count`
63    /// is what says which happened; treat the shares as sampled whenever it is
64    /// short.
65    ///
66    /// `None` means the classification did not run, never "one uniform class":
67    /// a column that was classified and had nothing to classify (all-null, or
68    /// zero rows) is `Some` with every count zero.
69    #[serde(default, skip_serializing_if = "Option::is_none")]
70    pub type_homogeneity: Option<TypeHomogeneity>,
71    /// Of the values `type_homogeneity` counted as `text`, how many are
72    /// numbers written with a decimal comma or with digit-group separators:
73    /// `10,50`, `1.234,56`, `1,234.56`, `1'234.56`, `1 234,56` (#433).
74    ///
75    /// dataprof does not parse these as numbers, so they are in no numeric
76    /// statistic: a column of them is typed `string`, and a numeric column
77    /// counts them in `invalid_count`. The count does not say which
78    /// convention they follow; `1,234` is a thousand in one and a fraction in
79    /// another.
80    ///
81    /// Counted over the same values as `type_homogeneity`, and present
82    /// whenever it is. `None` means the count did not run, including a report
83    /// written before dataprof recorded it; `Some(0)` means none were found.
84    #[serde(default, skip_serializing_if = "Option::is_none")]
85    pub locale_number_count: Option<usize>,
86    /// The type the source declared for the column, recorded as provenance
87    /// (#906). `None` when the source declares none, which is every text
88    /// input (CSV, JSON, JSONL), Python objects and SQLite, and on a report
89    /// written before dataprof recorded it. See [`NativeType`] for the
90    /// sources that declare one.
91    ///
92    /// `data_type` is what dataprof profiled the column as, and several
93    /// declared types share one: an Arrow `decimal128(12, 2)` profiles as
94    /// `Float` and a `timestamp[us, tz=UTC]` as `Date`. This keeps the
95    /// declaration itself, so a reader can tell that a column kept its type.
96    ///
97    /// Provenance, not a metric: two input paths over the same logical data
98    /// declare different types, or none, so it stays outside the
99    /// serialized-value parity comparison.
100    #[serde(default, skip_serializing_if = "Option::is_none")]
101    pub native_type: Option<NativeType>,
102    pub stats: ColumnStats,
103    /// Detected patterns, or `None` when pattern detection did not run.
104    ///
105    /// `None` and `Some(vec![])` are not interchangeable: the former means the
106    /// column was never scanned, the latter that it was scanned and nothing
107    /// matched. Consumers that gate on sensitivity -- redaction, agent-facing
108    /// output -- must treat `None` as "unknown", never as "no sensitive data".
109    pub patterns: Option<Vec<Pattern>>,
110}
111
112/// A column type as the source declared it (#906).
113///
114/// `system` names the type system the declaration belongs to, and `name`
115/// spells the type the way that system does:
116///
117/// - `arrow`: Arrow tables, record batches and streams, polars frames, ADBC
118///   results and Parquet files. The name is the one pyarrow prints for the
119///   type, such as `decimal128(12, 2)`, `timestamp[us, tz=UTC]` or
120///   `dictionary<values=string, indices=int32, ordered=0>`. An extension type
121///   is named as the extension (`extension<arrow.uuid>`), not its storage
122///   type, and without the parameters pyarrow adds to a parameterized one.
123///   A Parquet column
124///   carries the Arrow type its reader produced from the file's schema, which
125///   keeps the Arrow schema the writer stored with the file.
126/// - `pandas`: pandas DataFrames. The name is the column's dtype as pandas
127///   prints it (`int64`, `object`, `datetime64[ns, UTC]`, `Int64`). An
128///   `object` column declares no type for its values; pyarrow infers one from
129///   them, which is why the pandas dtype is recorded rather than the Arrow
130///   type the profiler read.
131/// - `postgresql` and `mysql`: the database connectors. The name is the
132///   server's type name as the driver reports it (`NUMERIC`, `INT4`,
133///   `TIMESTAMPTZ`, `DECIMAL`, `BIGINT UNSIGNED`).
134///
135/// Both are strings so that a report from a later release, which may record
136/// a system this build does not know, still loads. Build one with
137/// [`NativeType::new`]: the struct is `non_exhaustive`, as the rest of the
138/// report model is to become before 1.0 (#1162).
139#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)]
140#[non_exhaustive]
141pub struct NativeType {
142    pub system: String,
143    pub name: String,
144}
145
146impl NativeType {
147    /// The type system of Arrow and the readers built on it.
148    pub const ARROW: &'static str = "arrow";
149    /// pandas dtypes.
150    pub const PANDAS: &'static str = "pandas";
151    /// PostgreSQL type names.
152    pub const POSTGRESQL: &'static str = "postgresql";
153    /// MySQL type names.
154    pub const MYSQL: &'static str = "mysql";
155
156    pub fn new(system: impl Into<String>, name: impl Into<String>) -> Self {
157        Self {
158            system: system.into(),
159            name: name.into(),
160        }
161    }
162
163    /// Whether this declares a decimal, a type whose values no float wrote,
164    /// so they are read as decimals and kept exact (#1268): an Arrow
165    /// `decimal32` to `decimal256`, a pandas column of one
166    /// (`decimal128(12, 2)[pyarrow]`), PostgreSQL `NUMERIC` and MySQL
167    /// `DECIMAL`.
168    ///
169    /// A pandas `object` column declares no type, though pyarrow may read its
170    /// `Decimal` values into a decimal; nor does an Arrow extension type over
171    /// decimal storage, which is named as the extension.
172    pub fn declares_decimal(&self) -> bool {
173        let arrow_decimal = |name: &str| {
174            ["decimal32(", "decimal64(", "decimal128(", "decimal256("]
175                .iter()
176                .any(|prefix| name.starts_with(prefix))
177        };
178        match self.system.as_str() {
179            Self::ARROW => arrow_decimal(&self.name),
180            Self::PANDAS => self
181                .name
182                .strip_suffix("[pyarrow]")
183                .is_some_and(arrow_decimal),
184            Self::POSTGRESQL => self.name == "NUMERIC",
185            Self::MYSQL => self.name == "DECIMAL",
186            _ => false,
187        }
188    }
189
190    /// Whether this declares a date or a timestamp, a type every value of
191    /// the column holds whichever part of it was read (#1333): an Arrow
192    /// `date32`, `date64` or `timestamp`, a pandas `datetime64` column or an
193    /// Arrow-backed one of those, PostgreSQL `DATE`, `TIMESTAMP` and
194    /// `TIMESTAMPTZ`, and MySQL `DATE`, `DATETIME` and `TIMESTAMP`.
195    ///
196    /// A text column holding dates declares none, so the values a partial
197    /// read missed may still type it as text; so does a pandas `object`
198    /// column of `datetime` values, and an extension type over date storage.
199    pub fn declares_date(&self) -> bool {
200        let arrow_date = |name: &str| {
201            name == "date32[day]" || name == "date64[ms]" || name.starts_with("timestamp[")
202        };
203        match self.system.as_str() {
204            Self::ARROW => arrow_date(&self.name),
205            Self::PANDAS => {
206                self.name.starts_with("datetime64[")
207                    || self.name.strip_suffix("[pyarrow]").is_some_and(arrow_date)
208            }
209            Self::POSTGRESQL => matches!(self.name.as_str(), "DATE" | "TIMESTAMP" | "TIMESTAMPTZ"),
210            Self::MYSQL => matches!(self.name.as_str(), "DATE" | "DATETIME" | "TIMESTAMP"),
211            _ => false,
212        }
213    }
214}
215
216/// Quartile statistics for numeric distributions.
217#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, schemars::JsonSchema)]
218pub struct Quartiles {
219    pub q1: f64,
220    pub q2: f64,
221    pub q3: f64,
222    /// `q3 - q1`, or `None` when that difference exceeds the finite `f64`
223    /// range; the quartiles themselves are still reported. Equal quartiles
224    /// give `Some(0.0)`, so `None` never means "no spread".
225    #[serde(deserialize_with = "crate::serde_helpers::required_nullable")]
226    pub iqr: Option<f64>,
227}
228
229/// The order statistics of an integer or decimal column, exactly, as
230/// canonical decimal strings: no exponent, no trailing zeros in a fraction,
231/// `-` for a negative value (#1038). An integer past 2^53 or a decimal with
232/// more digits than `f64` keeps is rounded in the `f64` fields beside these.
233///
234/// Present on every integer column and every decimal-typed column, absent on
235/// a float column. Each field covers what the `f64` field of the same name
236/// covers: `min` and `max` every value, the others the same sample, and each
237/// is present exactly when that field is.
238#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)]
239pub struct ExactNumericStats {
240    pub min: String,
241    pub max: String,
242    #[serde(default, skip_serializing_if = "Option::is_none")]
243    pub median: Option<String>,
244    #[serde(default, skip_serializing_if = "Option::is_none")]
245    pub q1: Option<String>,
246    #[serde(default, skip_serializing_if = "Option::is_none")]
247    pub q2: Option<String>,
248    #[serde(default, skip_serializing_if = "Option::is_none")]
249    pub q3: Option<String>,
250    #[serde(default, skip_serializing_if = "Option::is_none")]
251    pub mode: Option<String>,
252}
253
254/// A value and its frequency count within a column.
255#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, schemars::JsonSchema)]
256pub struct FrequencyItem {
257    pub value: String,
258    pub count: usize,
259    #[serde(serialize_with = "crate::serde_helpers::round_2")]
260    pub percentage: f64,
261}
262
263/// How a report's moment statistics (`mean`, `std_dev`, `variance`,
264/// `skewness`, `kurtosis`) are rounded when serialized, as recorded in its
265/// metric semantics.
266///
267/// Only the current rule has a value. Through 0.13 they kept 4 decimal places
268/// at any magnitude, and those releases recorded no definition.
269#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)]
270#[serde(rename_all = "snake_case")]
271pub enum StatisticRounding {
272    /// 4 decimal places and at most 12 significant digits, so a value past
273    /// 1e8 drops the digits where accumulation order drifts (#1178).
274    FourDecimalsTwelveSignificant,
275}
276
277/// The estimator behind `skewness` and `kurtosis`, as recorded in a report's
278/// metric semantics.
279///
280/// Only the current estimator has a value. Through 0.12 both divided
281/// population moments by the sample standard deviation, a form matching
282/// neither the population nor the adjusted estimators, and those releases
283/// recorded no definition.
284#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)]
285#[serde(rename_all = "snake_case")]
286pub enum ShapeMomentEstimator {
287    /// Adjusted Fisher-Pearson sample skewness from three values and adjusted
288    /// Fisher sample excess kurtosis from four, as pandas computes them.
289    AdjustedSample,
290}
291
292/// Statistics for numeric (integer or float) columns.
293#[derive(Debug, Clone, Serialize, Deserialize, schemars::JsonSchema)]
294pub struct NumericStats {
295    // min/max/median/mode are data values, not summary percentages: they are
296    // rounded at the statistics precision so a column whose values carry more
297    // than two decimals is not reported with a min it never contained.
298    #[serde(serialize_with = "crate::serde_helpers::round_4")]
299    pub min: f64,
300    #[serde(serialize_with = "crate::serde_helpers::round_4")]
301    pub max: f64,
302    #[serde(serialize_with = "crate::serde_helpers::round_statistic")]
303    pub mean: f64,
304    /// `None` when the spread of the values exceeds the finite `f64` range.
305    /// A constant column is `Some(0.0)`, so `None` never means "no spread".
306    #[serde(
307        serialize_with = "crate::serde_helpers::round_statistic_opt",
308        deserialize_with = "crate::serde_helpers::required_nullable"
309    )]
310    pub std_dev: Option<f64>,
311    /// `None` when the spread of the values exceeds the finite `f64` range.
312    /// A constant column is `Some(0.0)`, so `None` never means "no spread".
313    #[serde(
314        serialize_with = "crate::serde_helpers::round_statistic_opt",
315        deserialize_with = "crate::serde_helpers::required_nullable"
316    )]
317    pub variance: Option<f64>,
318    #[serde(
319        skip_serializing_if = "Option::is_none",
320        serialize_with = "crate::serde_helpers::round_4_opt"
321    )]
322    pub median: Option<f64>,
323    #[serde(
324        skip_serializing_if = "Option::is_none",
325        serialize_with = "crate::serde_helpers::quartiles::serialize"
326    )]
327    pub quartiles: Option<Quartiles>,
328    #[serde(
329        skip_serializing_if = "Option::is_none",
330        serialize_with = "crate::serde_helpers::round_4_opt"
331    )]
332    pub mode: Option<f64>,
333    #[serde(
334        skip_serializing_if = "Option::is_none",
335        serialize_with = "crate::serde_helpers::round_2_opt"
336    )]
337    pub coefficient_of_variation: Option<f64>,
338    #[serde(
339        skip_serializing_if = "Option::is_none",
340        serialize_with = "crate::serde_helpers::round_statistic_opt"
341    )]
342    pub skewness: Option<f64>,
343    #[serde(
344        skip_serializing_if = "Option::is_none",
345        serialize_with = "crate::serde_helpers::round_statistic_opt"
346    )]
347    pub kurtosis: Option<f64>,
348    #[serde(skip_serializing_if = "Option::is_none")]
349    pub is_approximate: Option<bool>,
350    /// Number of values flagged as IQR-based outliers in this column.
351    ///
352    /// Uses the same Tukey-style detection (Q1 − k·IQR, Q3 + k·IQR with
353    /// k = 1.5 by default) that feeds the global `accuracy.outlier_ratio`.
354    /// `None` when outlier detection didn't run (sample below the configured
355    /// minimum or non-numeric column).
356    #[serde(default, skip_serializing_if = "Option::is_none")]
357    pub outlier_count: Option<usize>,
358    /// The order statistics exactly, on an integer or decimal column; `None`
359    /// on a float column, whose values are not held exactly. See
360    /// [`ExactNumericStats`].
361    #[serde(default, skip_serializing_if = "Option::is_none")]
362    pub exact: Option<ExactNumericStats>,
363}
364
365impl NumericStats {
366    pub fn empty() -> Self {
367        Self {
368            min: 0.0,
369            max: 0.0,
370            mean: 0.0,
371            std_dev: Some(0.0),
372            variance: Some(0.0),
373            median: None,
374            quartiles: None,
375            mode: None,
376            coefficient_of_variation: None,
377            skewness: None,
378            kurtosis: None,
379            is_approximate: None,
380            outlier_count: None,
381            exact: None,
382        }
383    }
384}
385
386/// Statistics for text/string columns.
387///
388/// Every length here is a count of Unicode scalar values, not UTF-8 bytes and
389/// not grapheme clusters. ASCII text is unaffected by that distinction; a
390/// combining sequence counts each scalar, so the decomposed and precomposed
391/// spellings of the same word report different lengths. The `text_units` module
392/// carries the reasoning.
393#[derive(Debug, Clone, Serialize, Deserialize, schemars::JsonSchema)]
394pub struct TextStats {
395    /// Shortest value, in Unicode scalar values.
396    pub min_length: usize,
397    /// Longest value, in Unicode scalar values.
398    pub max_length: usize,
399    /// Mean length in Unicode scalar values. A mean, so it rounds at the
400    /// statistics precision rather than the percentage one.
401    #[serde(serialize_with = "crate::serde_helpers::round_4")]
402    pub avg_length: f64,
403    #[serde(skip_serializing_if = "Option::is_none")]
404    pub most_frequent: Option<Vec<FrequencyItem>>,
405    #[serde(skip_serializing_if = "Option::is_none")]
406    pub least_frequent: Option<Vec<FrequencyItem>>,
407}
408
409impl TextStats {
410    pub fn empty() -> Self {
411        Self {
412            min_length: 0,
413            max_length: 0,
414            avg_length: 0.0,
415            most_frequent: None,
416            least_frequent: None,
417        }
418    }
419
420    pub fn from_lengths(min_length: usize, max_length: usize, avg_length: f64) -> Self {
421        Self {
422            min_length: if min_length == usize::MAX {
423                0
424            } else {
425                min_length
426            },
427            max_length,
428            avg_length,
429            most_frequent: None,
430            least_frequent: None,
431        }
432    }
433}
434
435/// Statistics for date/datetime columns.
436#[derive(Debug, Clone, Serialize, Deserialize, schemars::JsonSchema)]
437pub struct DateTimeStats {
438    pub min_datetime: String,
439    pub max_datetime: String,
440    #[serde(serialize_with = "crate::serde_helpers::round_2")]
441    pub duration_days: f64,
442    pub year_distribution: HashMap<i32, usize>,
443    pub month_distribution: HashMap<u32, usize>,
444    pub day_of_week_distribution: HashMap<String, usize>,
445    #[serde(skip_serializing_if = "Option::is_none")]
446    pub hour_distribution: Option<HashMap<u32, usize>>,
447    /// How the column's `NN/NN/YYYY` values were read (#811).
448    ///
449    /// Decided once per column from its own values, so `01/02/2024` is read
450    /// the same way as the `12/31/2024` beside it. `None` when the column
451    /// holds no slash dates, or the report predates the field.
452    #[serde(default, skip_serializing_if = "Option::is_none")]
453    pub slash_date_order: Option<SlashDateOrder>,
454    /// The longest stretch between two consecutive distinct days of the
455    /// column (#1300), over every value read.
456    ///
457    /// A daily file that lost a period from the middle of its range keeps its
458    /// newest date and most of its rows, and shows here: the gap grows from 1
459    /// to the length of the hole. `None` when no value parsed as a date, the
460    /// column's dates span more than 10,000 distinct years (the profiler
461    /// stops tracking days past that), or the report predates the field.
462    #[serde(default, skip_serializing_if = "Option::is_none")]
463    pub largest_gap: Option<DateGap>,
464}
465
466/// How many distinct years a date column may span before the profiler stops
467/// tracking its distinct days, and reports no [`DateTimeStats::largest_gap`].
468///
469/// The days are kept as one 48-byte bitmap per year, about 60 bytes with its
470/// key and tree overhead, so one set at the limit holds about 0.6 MB. The
471/// limit applies to each set: a column keeps one for the values whose reading
472/// is settled and one for each reading of its ambiguous slash dates, so a
473/// column holding both at the limit retains about 1.8 MB, and finishing its
474/// statistics copies one set more while it merges them.
475pub const DATE_GAP_YEAR_LIMIT: usize = 10_000;
476
477/// Two consecutive distinct days of a date column, and the days between
478/// them.
479#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)]
480#[non_exhaustive]
481pub struct DateGap {
482    /// Days from `from` to `to`: `1` for consecutive days, `0` for a column
483    /// holding a single distinct day.
484    pub days: i64,
485    /// The last day before the gap, `YYYY-MM-DD`.
486    pub from: String,
487    /// The first day after the gap, `YYYY-MM-DD`.
488    pub to: String,
489}
490
491impl DateGap {
492    /// The gap from `from` to `to`, both written `YYYY-MM-DD`.
493    pub fn new(from: chrono::NaiveDate, to: chrono::NaiveDate) -> Self {
494        Self {
495            days: (to - from).num_days(),
496            from: from.format("%Y-%m-%d").to_string(),
497            to: to.format("%Y-%m-%d").to_string(),
498        }
499    }
500}
501
502/// The day/month order a column's slash dates were read in.
503#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)]
504#[serde(rename_all = "snake_case")]
505pub enum SlashDateOrder {
506    /// Some values only parse day-first (`31/12/2024`) and none only
507    /// month-first.
508    DayFirst,
509    /// Some values only parse month-first (`12/31/2024`) and none only
510    /// day-first.
511    MonthFirst,
512    /// Every value parses both ways, so nothing in the column decides it; the
513    /// values were read day-first.
514    AssumedDayFirst,
515    /// Values that only parse day-first and values that only parse
516    /// month-first are both present. The column contradicts itself: each value
517    /// was read in the order it parses, day-first where both do, and the dates
518    /// cannot all be right.
519    Mixed,
520}
521
522impl DateTimeStats {
523    pub fn empty() -> Self {
524        Self {
525            min_datetime: String::new(),
526            max_datetime: String::new(),
527            duration_days: 0.0,
528            year_distribution: HashMap::new(),
529            month_distribution: HashMap::new(),
530            day_of_week_distribution: HashMap::new(),
531            hour_distribution: None,
532            slash_date_order: None,
533            largest_gap: None,
534        }
535    }
536}
537
538/// Statistics for boolean columns.
539#[derive(Debug, Clone, Serialize, Deserialize, schemars::JsonSchema)]
540pub struct BooleanStats {
541    pub true_count: usize,
542    pub false_count: usize,
543    #[serde(serialize_with = "crate::serde_helpers::round_4")]
544    pub true_ratio: f64,
545}
546
547/// Type-specific statistics for a column, determined by the inferred data type.
548#[derive(Debug, Clone, Serialize, Deserialize, schemars::JsonSchema)]
549pub enum ColumnStats {
550    Numeric(NumericStats),
551    Text(TextStats),
552    DateTime(DateTimeStats),
553    Boolean(BooleanStats),
554    None,
555}
556
557#[cfg(test)]
558mod tests {
559    use super::*;
560
561    /// Every declaration that names a decimal, and the neighbours that do
562    /// not: a float, an array of decimals, an extension over one, a pandas
563    /// `object` column, and a system this build does not know (#1268).
564    #[test]
565    fn declares_decimal_names_each_systems_decimal() {
566        let cases = [
567            (NativeType::ARROW, "decimal32(9, 2)", true),
568            (NativeType::ARROW, "decimal64(18, 2)", true),
569            (NativeType::ARROW, "decimal128(12, 2)", true),
570            (NativeType::ARROW, "decimal256(40, -2)", true),
571            (NativeType::PANDAS, "decimal128(12, 2)[pyarrow]", true),
572            (NativeType::POSTGRESQL, "NUMERIC", true),
573            (NativeType::MYSQL, "DECIMAL", true),
574            (NativeType::ARROW, "double", false),
575            (NativeType::ARROW, "list<item: decimal128(12, 2)>", false),
576            (NativeType::ARROW, "extension<example.money>", false),
577            (NativeType::PANDAS, "object", false),
578            (NativeType::PANDAS, "decimal128(12, 2)", false),
579            (NativeType::PANDAS, "Float64", false),
580            (NativeType::POSTGRESQL, "NUMERIC[]", false),
581            (NativeType::POSTGRESQL, "FLOAT8", false),
582            (NativeType::MYSQL, "DOUBLE", false),
583            ("duckdb", "DECIMAL", false),
584        ];
585        let wrong: Vec<String> = cases
586            .iter()
587            .filter(|(system, name, want)| {
588                NativeType::new(*system, *name).declares_decimal() != *want
589            })
590            .map(|(system, name, want)| format!("{system} {name}: expected {want}"))
591            .collect();
592        assert!(wrong.is_empty(), "{}", wrong.join("\n"));
593    }
594
595    /// Every declaration that names a date or timestamp, and the neighbours
596    /// that do not: a time of day, text, a list of dates, an extension over
597    /// one, a pandas `object` column and a system this build does not know
598    /// (#1333).
599    #[test]
600    fn declares_date_names_each_systems_date() {
601        let cases = [
602            (NativeType::ARROW, "date32[day]", true),
603            (NativeType::ARROW, "date64[ms]", true),
604            (NativeType::ARROW, "timestamp[us]", true),
605            (NativeType::ARROW, "timestamp[ns, tz=UTC]", true),
606            (NativeType::PANDAS, "datetime64[ns]", true),
607            (NativeType::PANDAS, "datetime64[us, Europe/Rome]", true),
608            (NativeType::PANDAS, "timestamp[us, tz=UTC][pyarrow]", true),
609            (NativeType::PANDAS, "date32[day][pyarrow]", true),
610            (NativeType::POSTGRESQL, "DATE", true),
611            (NativeType::POSTGRESQL, "TIMESTAMP", true),
612            (NativeType::POSTGRESQL, "TIMESTAMPTZ", true),
613            (NativeType::MYSQL, "DATE", true),
614            (NativeType::MYSQL, "DATETIME", true),
615            (NativeType::MYSQL, "TIMESTAMP", true),
616            (NativeType::ARROW, "time64[us]", false),
617            (NativeType::ARROW, "string", false),
618            (NativeType::ARROW, "large_string", false),
619            (NativeType::ARROW, "list<item: date32[day]>", false),
620            (NativeType::ARROW, "extension<example.day>", false),
621            (NativeType::PANDAS, "object", false),
622            (NativeType::PANDAS, "string", false),
623            (NativeType::PANDAS, "date32[day]", false),
624            (NativeType::POSTGRESQL, "TEXT", false),
625            (NativeType::POSTGRESQL, "TIME", false),
626            (NativeType::POSTGRESQL, "DATE[]", false),
627            (NativeType::MYSQL, "VARCHAR", false),
628            ("duckdb", "DATE", false),
629        ];
630        let wrong: Vec<String> = cases
631            .iter()
632            .filter(|(system, name, want)| NativeType::new(*system, *name).declares_date() != *want)
633            .map(|(system, name, want)| format!("{system} {name}: expected {want}"))
634            .collect();
635        assert!(wrong.is_empty(), "{}", wrong.join("\n"));
636    }
637
638    /// `std_dev`, `variance` and `quartiles.iqr` are nullable but required: a
639    /// `null` is a spread that overflowed `f64`, a missing key a malformed
640    /// document. Serde reads a missing `Option` as `None`, which conflates them.
641    #[test]
642    fn nullable_spread_fields_are_still_required() {
643        let stats = NumericStats {
644            std_dev: None,
645            variance: None,
646            quartiles: Some(Quartiles {
647                q1: 1.0,
648                q2: 2.0,
649                q3: 3.0,
650                iqr: None,
651            }),
652            ..NumericStats::empty()
653        };
654        let document = serde_json::to_value(&stats).unwrap();
655        let restored: NumericStats = serde_json::from_value(document.clone()).unwrap();
656        assert_eq!(serde_json::to_value(restored).unwrap(), document);
657
658        for (parent, key) in [
659            (None, "std_dev"),
660            (None, "variance"),
661            (Some("quartiles"), "iqr"),
662        ] {
663            let mut malformed = document.clone();
664            let object = match parent {
665                Some(parent) => &mut malformed[parent],
666                None => &mut malformed,
667            };
668            object.as_object_mut().unwrap().remove(key).unwrap();
669            let error = serde_json::from_value::<NumericStats>(malformed).unwrap_err();
670            assert!(
671                error
672                    .to_string()
673                    .contains(&format!("missing field `{key}`")),
674                "{key}: {error}"
675            );
676        }
677    }
678
679    #[test]
680    fn test_column_profile_json_roundtrip() {
681        let profile = ColumnProfile {
682            name: "test_col".to_string(),
683            data_type: DataType::Integer,
684            null_count: 2,
685            total_count: 10,
686            unique_count: Some(8),
687            unique_count_is_approximate: Some(false),
688            unique_count_lower_bound: None,
689            invalid_count: Some(0),
690            type_homogeneity: None,
691            locale_number_count: None,
692            native_type: None,
693            stats: ColumnStats::Numeric(NumericStats {
694                min: 1.0,
695                max: 100.0,
696                mean: 50.5,
697                std_dev: Some(28.87),
698                variance: Some(833.25),
699                median: Some(50.0),
700                quartiles: Some(Quartiles {
701                    q1: 25.0,
702                    q2: 50.0,
703                    q3: 75.0,
704                    iqr: Some(50.0),
705                }),
706                mode: Some(42.0),
707                coefficient_of_variation: Some(57.17),
708                skewness: Some(0.0),
709                kurtosis: Some(-1.2),
710                is_approximate: Some(false),
711                outlier_count: Some(0),
712                exact: None,
713            }),
714            patterns: Some(vec![]),
715        };
716
717        let json = serde_json::to_string(&profile).unwrap();
718        let deserialized: ColumnProfile = serde_json::from_str(&json).unwrap();
719
720        assert_eq!(deserialized.name, "test_col");
721        assert_eq!(deserialized.data_type, DataType::Integer);
722        assert_eq!(deserialized.total_count, 10);
723        assert_eq!(deserialized.null_count, 2);
724        assert_eq!(deserialized.unique_count_is_approximate, Some(false));
725
726        if let ColumnStats::Numeric(n) = &deserialized.stats {
727            assert!((n.min - 1.0).abs() < 0.01);
728            assert!((n.max - 100.0).abs() < 0.01);
729            assert!((n.mean - 50.5).abs() < 0.01);
730            assert!(n.median.is_some());
731            assert!(n.quartiles.is_some());
732        } else {
733            panic!("Expected Numeric stats after roundtrip");
734        }
735    }
736
737    #[test]
738    fn test_text_stats_json_roundtrip() {
739        let profile = ColumnProfile {
740            name: "name".to_string(),
741            data_type: DataType::String,
742            null_count: 0,
743            total_count: 3,
744            unique_count: Some(3),
745            unique_count_is_approximate: Some(false),
746            unique_count_lower_bound: None,
747            invalid_count: None,
748            type_homogeneity: None,
749            locale_number_count: None,
750            native_type: None,
751            stats: ColumnStats::Text(TextStats {
752                min_length: 3,
753                max_length: 7,
754                avg_length: 5.0,
755                most_frequent: None,
756                least_frequent: None,
757            }),
758            patterns: Some(vec![]),
759        };
760
761        let json = serde_json::to_string(&profile).unwrap();
762        let deserialized: ColumnProfile = serde_json::from_str(&json).unwrap();
763
764        assert_eq!(deserialized.data_type, DataType::String);
765        if let ColumnStats::Text(t) = &deserialized.stats {
766            assert_eq!(t.min_length, 3);
767            assert_eq!(t.max_length, 7);
768        } else {
769            panic!("Expected Text stats after roundtrip");
770        }
771    }
772}