dataprof_core/profile.rs
1use std::collections::HashMap;
2
3use serde::{Deserialize, Serialize};
4
5use crate::classification::{DataType, TypeHomogeneity};
6use crate::pattern::Pattern;
7
8/// Profiling statistics for a single column.
9#[derive(Debug, Clone, Serialize, Deserialize, schemars::JsonSchema)]
10pub struct ColumnProfile {
11 pub name: String,
12 pub data_type: DataType,
13 pub null_count: usize,
14 pub total_count: usize,
15 pub unique_count: Option<usize>,
16 /// Whether `unique_count` is an approximate (HyperLogLog) estimate rather
17 /// than an exact distinct count.
18 ///
19 /// `None` when `unique_count` is `None` (never computed); `Some(false)` for
20 /// an exact count; `Some(true)` once the cardinality estimator has spilled
21 /// to its HLL sketch (~1% relative error). Consumers running key or
22 /// high-cardinality/uniqueness gates must treat `Some(true)` as "do not rely
23 /// on this as an exact integer" -- an exact-looking count with no provenance
24 /// is unsafe for those checks.
25 #[serde(default, skip_serializing_if = "Option::is_none")]
26 pub unique_count_is_approximate: Option<bool>,
27 /// A distinct count the column is certain to reach when `unique_count` is
28 /// an estimate: the distinct values the exact set held when it was
29 /// dropped. `None` when no floor was recorded, which includes every exact
30 /// count (the count is its own floor) and every report read back from a
31 /// document.
32 ///
33 /// Not serialized: it exists so quality bounds computed during the run can
34 /// bound a key column's uniqueness with certainty (#819). A saved report
35 /// keeps the bounds themselves.
36 #[serde(skip)]
37 #[schemars(skip)]
38 pub unique_count_lower_bound: Option<usize>,
39 /// Non-null values that failed the column type's raw validity predicate:
40 /// non-finite or malformed numbers on numeric columns, and values that do
41 /// not parse directly as calendar dates on date columns. The date predicate
42 /// intentionally does not trim surrounding whitespace; descriptive date
43 /// statistics may normalize whitespace independently.
44 ///
45 /// For numeric columns, `mean`/`std_dev` cover
46 /// `total_count - null_count - invalid_count` values. For date columns,
47 /// this count audits the strict raw-value quality predicate. `None` means
48 /// the check did not run (another column type, or statistics skipped) —
49 /// never "no invalid values", which is `Some(0)`.
50 #[serde(default, skip_serializing_if = "Option::is_none")]
51 pub invalid_count: Option<usize>,
52 /// How the column's non-null values distribute across lexical classes.
53 ///
54 /// `data_type` cannot answer "did this column have a dominant form?": a
55 /// column of names and a column that is 60% numbers are both `String`, and
56 /// `invalid_count` is absent on string columns by contract. This carries the
57 /// evidence, so a consumer can tell a textual column from one that defeated
58 /// type inference.
59 ///
60 /// Counted over the values the profiler retained — the engine's bounded
61 /// reservoir sample on a large source, the whole column on a small or
62 /// in-memory one. `classified_count()` against `total_count - null_count`
63 /// is what says which happened; treat the shares as sampled whenever it is
64 /// short.
65 ///
66 /// `None` means the classification did not run, never "one uniform class":
67 /// a column that was classified and had nothing to classify (all-null, or
68 /// zero rows) is `Some` with every count zero.
69 #[serde(default, skip_serializing_if = "Option::is_none")]
70 pub type_homogeneity: Option<TypeHomogeneity>,
71 /// Of the values `type_homogeneity` counted as `text`, how many are
72 /// numbers written with a decimal comma or with digit-group separators:
73 /// `10,50`, `1.234,56`, `1,234.56`, `1'234.56`, `1 234,56` (#433).
74 ///
75 /// dataprof does not parse these as numbers, so they are in no numeric
76 /// statistic: a column of them is typed `string`, and a numeric column
77 /// counts them in `invalid_count`. The count does not say which
78 /// convention they follow; `1,234` is a thousand in one and a fraction in
79 /// another.
80 ///
81 /// Counted over the same values as `type_homogeneity`, and present
82 /// whenever it is. `None` means the count did not run, including a report
83 /// written before dataprof recorded it; `Some(0)` means none were found.
84 #[serde(default, skip_serializing_if = "Option::is_none")]
85 pub locale_number_count: Option<usize>,
86 /// The type the source declared for the column, recorded as provenance
87 /// (#906). `None` when the source declares none, which is every text
88 /// input (CSV, JSON, JSONL), Python objects and SQLite, and on a report
89 /// written before dataprof recorded it. See [`NativeType`] for the
90 /// sources that declare one.
91 ///
92 /// `data_type` is what dataprof profiled the column as, and several
93 /// declared types share one: an Arrow `decimal128(12, 2)` profiles as
94 /// `Float` and a `timestamp[us, tz=UTC]` as `Date`. This keeps the
95 /// declaration itself, so a reader can tell that a column kept its type.
96 ///
97 /// Provenance, not a metric: two input paths over the same logical data
98 /// declare different types, or none, so it stays outside the
99 /// serialized-value parity comparison.
100 #[serde(default, skip_serializing_if = "Option::is_none")]
101 pub native_type: Option<NativeType>,
102 pub stats: ColumnStats,
103 /// Detected patterns, or `None` when pattern detection did not run.
104 ///
105 /// `None` and `Some(vec![])` are not interchangeable: the former means the
106 /// column was never scanned, the latter that it was scanned and nothing
107 /// matched. Consumers that gate on sensitivity -- redaction, agent-facing
108 /// output -- must treat `None` as "unknown", never as "no sensitive data".
109 pub patterns: Option<Vec<Pattern>>,
110}
111
112/// A column type as the source declared it (#906).
113///
114/// `system` names the type system the declaration belongs to, and `name`
115/// spells the type the way that system does:
116///
117/// - `arrow`: Arrow tables, record batches and streams, polars frames, ADBC
118/// results and Parquet files. The name is the one pyarrow prints for the
119/// type, such as `decimal128(12, 2)`, `timestamp[us, tz=UTC]` or
120/// `dictionary<values=string, indices=int32, ordered=0>`. An extension type
121/// is named as the extension (`extension<arrow.uuid>`), not its storage
122/// type, and without the parameters pyarrow adds to a parameterized one.
123/// A Parquet column
124/// carries the Arrow type its reader produced from the file's schema, which
125/// keeps the Arrow schema the writer stored with the file.
126/// - `pandas`: pandas DataFrames. The name is the column's dtype as pandas
127/// prints it (`int64`, `object`, `datetime64[ns, UTC]`, `Int64`). An
128/// `object` column declares no type for its values; pyarrow infers one from
129/// them, which is why the pandas dtype is recorded rather than the Arrow
130/// type the profiler read.
131/// - `postgresql` and `mysql`: the database connectors. The name is the
132/// server's type name as the driver reports it (`NUMERIC`, `INT4`,
133/// `TIMESTAMPTZ`, `DECIMAL`, `BIGINT UNSIGNED`).
134///
135/// Both are strings so that a report from a later release, which may record
136/// a system this build does not know, still loads. Build one with
137/// [`NativeType::new`]: the struct is `non_exhaustive`, as the rest of the
138/// report model is to become before 1.0 (#1162).
139#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)]
140#[non_exhaustive]
141pub struct NativeType {
142 pub system: String,
143 pub name: String,
144}
145
146impl NativeType {
147 /// The type system of Arrow and the readers built on it.
148 pub const ARROW: &'static str = "arrow";
149 /// pandas dtypes.
150 pub const PANDAS: &'static str = "pandas";
151 /// PostgreSQL type names.
152 pub const POSTGRESQL: &'static str = "postgresql";
153 /// MySQL type names.
154 pub const MYSQL: &'static str = "mysql";
155
156 pub fn new(system: impl Into<String>, name: impl Into<String>) -> Self {
157 Self {
158 system: system.into(),
159 name: name.into(),
160 }
161 }
162
163 /// Whether this declares a decimal, a type whose values no float wrote,
164 /// so they are read as decimals and kept exact (#1268): an Arrow
165 /// `decimal32` to `decimal256`, a pandas column of one
166 /// (`decimal128(12, 2)[pyarrow]`), PostgreSQL `NUMERIC` and MySQL
167 /// `DECIMAL`.
168 ///
169 /// A pandas `object` column declares no type, though pyarrow may read its
170 /// `Decimal` values into a decimal; nor does an Arrow extension type over
171 /// decimal storage, which is named as the extension.
172 pub fn declares_decimal(&self) -> bool {
173 let arrow_decimal = |name: &str| {
174 ["decimal32(", "decimal64(", "decimal128(", "decimal256("]
175 .iter()
176 .any(|prefix| name.starts_with(prefix))
177 };
178 match self.system.as_str() {
179 Self::ARROW => arrow_decimal(&self.name),
180 Self::PANDAS => self
181 .name
182 .strip_suffix("[pyarrow]")
183 .is_some_and(arrow_decimal),
184 Self::POSTGRESQL => self.name == "NUMERIC",
185 Self::MYSQL => self.name == "DECIMAL",
186 _ => false,
187 }
188 }
189
190 /// Whether this declares a date or a timestamp, a type every value of
191 /// the column holds whichever part of it was read (#1333): an Arrow
192 /// `date32`, `date64` or `timestamp`, a pandas `datetime64` column or an
193 /// Arrow-backed one of those, PostgreSQL `DATE`, `TIMESTAMP` and
194 /// `TIMESTAMPTZ`, and MySQL `DATE`, `DATETIME` and `TIMESTAMP`.
195 ///
196 /// A text column holding dates declares none, so the values a partial
197 /// read missed may still type it as text; so does a pandas `object`
198 /// column of `datetime` values, and an extension type over date storage.
199 pub fn declares_date(&self) -> bool {
200 let arrow_date = |name: &str| {
201 name == "date32[day]" || name == "date64[ms]" || name.starts_with("timestamp[")
202 };
203 match self.system.as_str() {
204 Self::ARROW => arrow_date(&self.name),
205 Self::PANDAS => {
206 self.name.starts_with("datetime64[")
207 || self.name.strip_suffix("[pyarrow]").is_some_and(arrow_date)
208 }
209 Self::POSTGRESQL => matches!(self.name.as_str(), "DATE" | "TIMESTAMP" | "TIMESTAMPTZ"),
210 Self::MYSQL => matches!(self.name.as_str(), "DATE" | "DATETIME" | "TIMESTAMP"),
211 _ => false,
212 }
213 }
214}
215
216/// Quartile statistics for numeric distributions.
217#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, schemars::JsonSchema)]
218pub struct Quartiles {
219 pub q1: f64,
220 pub q2: f64,
221 pub q3: f64,
222 /// `q3 - q1`, or `None` when that difference exceeds the finite `f64`
223 /// range; the quartiles themselves are still reported. Equal quartiles
224 /// give `Some(0.0)`, so `None` never means "no spread".
225 #[serde(deserialize_with = "crate::serde_helpers::required_nullable")]
226 pub iqr: Option<f64>,
227}
228
229/// The order statistics of an integer or decimal column, exactly, as
230/// canonical decimal strings: no exponent, no trailing zeros in a fraction,
231/// `-` for a negative value (#1038). An integer past 2^53 or a decimal with
232/// more digits than `f64` keeps is rounded in the `f64` fields beside these.
233///
234/// Present on every integer column and every decimal-typed column, absent on
235/// a float column. Each field covers what the `f64` field of the same name
236/// covers: `min` and `max` every value, the others the same sample, and each
237/// is present exactly when that field is.
238#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)]
239pub struct ExactNumericStats {
240 pub min: String,
241 pub max: String,
242 #[serde(default, skip_serializing_if = "Option::is_none")]
243 pub median: Option<String>,
244 #[serde(default, skip_serializing_if = "Option::is_none")]
245 pub q1: Option<String>,
246 #[serde(default, skip_serializing_if = "Option::is_none")]
247 pub q2: Option<String>,
248 #[serde(default, skip_serializing_if = "Option::is_none")]
249 pub q3: Option<String>,
250 #[serde(default, skip_serializing_if = "Option::is_none")]
251 pub mode: Option<String>,
252}
253
254/// A value and its frequency count within a column.
255#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, schemars::JsonSchema)]
256pub struct FrequencyItem {
257 pub value: String,
258 pub count: usize,
259 #[serde(serialize_with = "crate::serde_helpers::round_2")]
260 pub percentage: f64,
261}
262
263/// How a report's moment statistics (`mean`, `std_dev`, `variance`,
264/// `skewness`, `kurtosis`) are rounded when serialized, as recorded in its
265/// metric semantics.
266///
267/// Only the current rule has a value. Through 0.13 they kept 4 decimal places
268/// at any magnitude, and those releases recorded no definition.
269#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)]
270#[serde(rename_all = "snake_case")]
271pub enum StatisticRounding {
272 /// 4 decimal places and at most 12 significant digits, so a value past
273 /// 1e8 drops the digits where accumulation order drifts (#1178).
274 FourDecimalsTwelveSignificant,
275}
276
277/// The estimator behind `skewness` and `kurtosis`, as recorded in a report's
278/// metric semantics.
279///
280/// Only the current estimator has a value. Through 0.12 both divided
281/// population moments by the sample standard deviation, a form matching
282/// neither the population nor the adjusted estimators, and those releases
283/// recorded no definition.
284#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)]
285#[serde(rename_all = "snake_case")]
286pub enum ShapeMomentEstimator {
287 /// Adjusted Fisher-Pearson sample skewness from three values and adjusted
288 /// Fisher sample excess kurtosis from four, as pandas computes them.
289 AdjustedSample,
290}
291
292/// Statistics for numeric (integer or float) columns.
293#[derive(Debug, Clone, Serialize, Deserialize, schemars::JsonSchema)]
294pub struct NumericStats {
295 // min/max/median/mode are data values, not summary percentages: they are
296 // rounded at the statistics precision so a column whose values carry more
297 // than two decimals is not reported with a min it never contained.
298 #[serde(serialize_with = "crate::serde_helpers::round_4")]
299 pub min: f64,
300 #[serde(serialize_with = "crate::serde_helpers::round_4")]
301 pub max: f64,
302 #[serde(serialize_with = "crate::serde_helpers::round_statistic")]
303 pub mean: f64,
304 /// `None` when the spread of the values exceeds the finite `f64` range.
305 /// A constant column is `Some(0.0)`, so `None` never means "no spread".
306 #[serde(
307 serialize_with = "crate::serde_helpers::round_statistic_opt",
308 deserialize_with = "crate::serde_helpers::required_nullable"
309 )]
310 pub std_dev: Option<f64>,
311 /// `None` when the spread of the values exceeds the finite `f64` range.
312 /// A constant column is `Some(0.0)`, so `None` never means "no spread".
313 #[serde(
314 serialize_with = "crate::serde_helpers::round_statistic_opt",
315 deserialize_with = "crate::serde_helpers::required_nullable"
316 )]
317 pub variance: Option<f64>,
318 #[serde(
319 skip_serializing_if = "Option::is_none",
320 serialize_with = "crate::serde_helpers::round_4_opt"
321 )]
322 pub median: Option<f64>,
323 #[serde(
324 skip_serializing_if = "Option::is_none",
325 serialize_with = "crate::serde_helpers::quartiles::serialize"
326 )]
327 pub quartiles: Option<Quartiles>,
328 #[serde(
329 skip_serializing_if = "Option::is_none",
330 serialize_with = "crate::serde_helpers::round_4_opt"
331 )]
332 pub mode: Option<f64>,
333 #[serde(
334 skip_serializing_if = "Option::is_none",
335 serialize_with = "crate::serde_helpers::round_2_opt"
336 )]
337 pub coefficient_of_variation: Option<f64>,
338 #[serde(
339 skip_serializing_if = "Option::is_none",
340 serialize_with = "crate::serde_helpers::round_statistic_opt"
341 )]
342 pub skewness: Option<f64>,
343 #[serde(
344 skip_serializing_if = "Option::is_none",
345 serialize_with = "crate::serde_helpers::round_statistic_opt"
346 )]
347 pub kurtosis: Option<f64>,
348 #[serde(skip_serializing_if = "Option::is_none")]
349 pub is_approximate: Option<bool>,
350 /// Number of values flagged as IQR-based outliers in this column.
351 ///
352 /// Uses the same Tukey-style detection (Q1 − k·IQR, Q3 + k·IQR with
353 /// k = 1.5 by default) that feeds the global `accuracy.outlier_ratio`.
354 /// `None` when outlier detection didn't run (sample below the configured
355 /// minimum or non-numeric column).
356 #[serde(default, skip_serializing_if = "Option::is_none")]
357 pub outlier_count: Option<usize>,
358 /// The order statistics exactly, on an integer or decimal column; `None`
359 /// on a float column, whose values are not held exactly. See
360 /// [`ExactNumericStats`].
361 #[serde(default, skip_serializing_if = "Option::is_none")]
362 pub exact: Option<ExactNumericStats>,
363}
364
365impl NumericStats {
366 pub fn empty() -> Self {
367 Self {
368 min: 0.0,
369 max: 0.0,
370 mean: 0.0,
371 std_dev: Some(0.0),
372 variance: Some(0.0),
373 median: None,
374 quartiles: None,
375 mode: None,
376 coefficient_of_variation: None,
377 skewness: None,
378 kurtosis: None,
379 is_approximate: None,
380 outlier_count: None,
381 exact: None,
382 }
383 }
384}
385
386/// Statistics for text/string columns.
387///
388/// Every length here is a count of Unicode scalar values, not UTF-8 bytes and
389/// not grapheme clusters. ASCII text is unaffected by that distinction; a
390/// combining sequence counts each scalar, so the decomposed and precomposed
391/// spellings of the same word report different lengths. The `text_units` module
392/// carries the reasoning.
393#[derive(Debug, Clone, Serialize, Deserialize, schemars::JsonSchema)]
394pub struct TextStats {
395 /// Shortest value, in Unicode scalar values.
396 pub min_length: usize,
397 /// Longest value, in Unicode scalar values.
398 pub max_length: usize,
399 /// Mean length in Unicode scalar values. A mean, so it rounds at the
400 /// statistics precision rather than the percentage one.
401 #[serde(serialize_with = "crate::serde_helpers::round_4")]
402 pub avg_length: f64,
403 #[serde(skip_serializing_if = "Option::is_none")]
404 pub most_frequent: Option<Vec<FrequencyItem>>,
405 #[serde(skip_serializing_if = "Option::is_none")]
406 pub least_frequent: Option<Vec<FrequencyItem>>,
407}
408
409impl TextStats {
410 pub fn empty() -> Self {
411 Self {
412 min_length: 0,
413 max_length: 0,
414 avg_length: 0.0,
415 most_frequent: None,
416 least_frequent: None,
417 }
418 }
419
420 pub fn from_lengths(min_length: usize, max_length: usize, avg_length: f64) -> Self {
421 Self {
422 min_length: if min_length == usize::MAX {
423 0
424 } else {
425 min_length
426 },
427 max_length,
428 avg_length,
429 most_frequent: None,
430 least_frequent: None,
431 }
432 }
433}
434
435/// Statistics for date/datetime columns.
436#[derive(Debug, Clone, Serialize, Deserialize, schemars::JsonSchema)]
437pub struct DateTimeStats {
438 pub min_datetime: String,
439 pub max_datetime: String,
440 #[serde(serialize_with = "crate::serde_helpers::round_2")]
441 pub duration_days: f64,
442 pub year_distribution: HashMap<i32, usize>,
443 pub month_distribution: HashMap<u32, usize>,
444 pub day_of_week_distribution: HashMap<String, usize>,
445 #[serde(skip_serializing_if = "Option::is_none")]
446 pub hour_distribution: Option<HashMap<u32, usize>>,
447 /// How the column's `NN/NN/YYYY` values were read (#811).
448 ///
449 /// Decided once per column from its own values, so `01/02/2024` is read
450 /// the same way as the `12/31/2024` beside it. `None` when the column
451 /// holds no slash dates, or the report predates the field.
452 #[serde(default, skip_serializing_if = "Option::is_none")]
453 pub slash_date_order: Option<SlashDateOrder>,
454 /// The longest stretch between two consecutive distinct days of the
455 /// column (#1300), over every value read.
456 ///
457 /// A daily file that lost a period from the middle of its range keeps its
458 /// newest date and most of its rows, and shows here: the gap grows from 1
459 /// to the length of the hole. `None` when no value parsed as a date, the
460 /// column's dates span more than 10,000 distinct years (the profiler
461 /// stops tracking days past that), or the report predates the field.
462 #[serde(default, skip_serializing_if = "Option::is_none")]
463 pub largest_gap: Option<DateGap>,
464}
465
466/// How many distinct years a date column may span before the profiler stops
467/// tracking its distinct days, and reports no [`DateTimeStats::largest_gap`].
468///
469/// The days are kept as one 48-byte bitmap per year, about 60 bytes with its
470/// key and tree overhead, so one set at the limit holds about 0.6 MB. The
471/// limit applies to each set: a column keeps one for the values whose reading
472/// is settled and one for each reading of its ambiguous slash dates, so a
473/// column holding both at the limit retains about 1.8 MB, and finishing its
474/// statistics copies one set more while it merges them.
475pub const DATE_GAP_YEAR_LIMIT: usize = 10_000;
476
477/// Two consecutive distinct days of a date column, and the days between
478/// them.
479#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)]
480#[non_exhaustive]
481pub struct DateGap {
482 /// Days from `from` to `to`: `1` for consecutive days, `0` for a column
483 /// holding a single distinct day.
484 pub days: i64,
485 /// The last day before the gap, `YYYY-MM-DD`.
486 pub from: String,
487 /// The first day after the gap, `YYYY-MM-DD`.
488 pub to: String,
489}
490
491impl DateGap {
492 /// The gap from `from` to `to`, both written `YYYY-MM-DD`.
493 pub fn new(from: chrono::NaiveDate, to: chrono::NaiveDate) -> Self {
494 Self {
495 days: (to - from).num_days(),
496 from: from.format("%Y-%m-%d").to_string(),
497 to: to.format("%Y-%m-%d").to_string(),
498 }
499 }
500}
501
502/// The day/month order a column's slash dates were read in.
503#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)]
504#[serde(rename_all = "snake_case")]
505pub enum SlashDateOrder {
506 /// Some values only parse day-first (`31/12/2024`) and none only
507 /// month-first.
508 DayFirst,
509 /// Some values only parse month-first (`12/31/2024`) and none only
510 /// day-first.
511 MonthFirst,
512 /// Every value parses both ways, so nothing in the column decides it; the
513 /// values were read day-first.
514 AssumedDayFirst,
515 /// Values that only parse day-first and values that only parse
516 /// month-first are both present. The column contradicts itself: each value
517 /// was read in the order it parses, day-first where both do, and the dates
518 /// cannot all be right.
519 Mixed,
520}
521
522impl DateTimeStats {
523 pub fn empty() -> Self {
524 Self {
525 min_datetime: String::new(),
526 max_datetime: String::new(),
527 duration_days: 0.0,
528 year_distribution: HashMap::new(),
529 month_distribution: HashMap::new(),
530 day_of_week_distribution: HashMap::new(),
531 hour_distribution: None,
532 slash_date_order: None,
533 largest_gap: None,
534 }
535 }
536}
537
538/// Statistics for boolean columns.
539#[derive(Debug, Clone, Serialize, Deserialize, schemars::JsonSchema)]
540pub struct BooleanStats {
541 pub true_count: usize,
542 pub false_count: usize,
543 #[serde(serialize_with = "crate::serde_helpers::round_4")]
544 pub true_ratio: f64,
545}
546
547/// Type-specific statistics for a column, determined by the inferred data type.
548#[derive(Debug, Clone, Serialize, Deserialize, schemars::JsonSchema)]
549pub enum ColumnStats {
550 Numeric(NumericStats),
551 Text(TextStats),
552 DateTime(DateTimeStats),
553 Boolean(BooleanStats),
554 None,
555}
556
557#[cfg(test)]
558mod tests {
559 use super::*;
560
561 /// Every declaration that names a decimal, and the neighbours that do
562 /// not: a float, an array of decimals, an extension over one, a pandas
563 /// `object` column, and a system this build does not know (#1268).
564 #[test]
565 fn declares_decimal_names_each_systems_decimal() {
566 let cases = [
567 (NativeType::ARROW, "decimal32(9, 2)", true),
568 (NativeType::ARROW, "decimal64(18, 2)", true),
569 (NativeType::ARROW, "decimal128(12, 2)", true),
570 (NativeType::ARROW, "decimal256(40, -2)", true),
571 (NativeType::PANDAS, "decimal128(12, 2)[pyarrow]", true),
572 (NativeType::POSTGRESQL, "NUMERIC", true),
573 (NativeType::MYSQL, "DECIMAL", true),
574 (NativeType::ARROW, "double", false),
575 (NativeType::ARROW, "list<item: decimal128(12, 2)>", false),
576 (NativeType::ARROW, "extension<example.money>", false),
577 (NativeType::PANDAS, "object", false),
578 (NativeType::PANDAS, "decimal128(12, 2)", false),
579 (NativeType::PANDAS, "Float64", false),
580 (NativeType::POSTGRESQL, "NUMERIC[]", false),
581 (NativeType::POSTGRESQL, "FLOAT8", false),
582 (NativeType::MYSQL, "DOUBLE", false),
583 ("duckdb", "DECIMAL", false),
584 ];
585 let wrong: Vec<String> = cases
586 .iter()
587 .filter(|(system, name, want)| {
588 NativeType::new(*system, *name).declares_decimal() != *want
589 })
590 .map(|(system, name, want)| format!("{system} {name}: expected {want}"))
591 .collect();
592 assert!(wrong.is_empty(), "{}", wrong.join("\n"));
593 }
594
595 /// Every declaration that names a date or timestamp, and the neighbours
596 /// that do not: a time of day, text, a list of dates, an extension over
597 /// one, a pandas `object` column and a system this build does not know
598 /// (#1333).
599 #[test]
600 fn declares_date_names_each_systems_date() {
601 let cases = [
602 (NativeType::ARROW, "date32[day]", true),
603 (NativeType::ARROW, "date64[ms]", true),
604 (NativeType::ARROW, "timestamp[us]", true),
605 (NativeType::ARROW, "timestamp[ns, tz=UTC]", true),
606 (NativeType::PANDAS, "datetime64[ns]", true),
607 (NativeType::PANDAS, "datetime64[us, Europe/Rome]", true),
608 (NativeType::PANDAS, "timestamp[us, tz=UTC][pyarrow]", true),
609 (NativeType::PANDAS, "date32[day][pyarrow]", true),
610 (NativeType::POSTGRESQL, "DATE", true),
611 (NativeType::POSTGRESQL, "TIMESTAMP", true),
612 (NativeType::POSTGRESQL, "TIMESTAMPTZ", true),
613 (NativeType::MYSQL, "DATE", true),
614 (NativeType::MYSQL, "DATETIME", true),
615 (NativeType::MYSQL, "TIMESTAMP", true),
616 (NativeType::ARROW, "time64[us]", false),
617 (NativeType::ARROW, "string", false),
618 (NativeType::ARROW, "large_string", false),
619 (NativeType::ARROW, "list<item: date32[day]>", false),
620 (NativeType::ARROW, "extension<example.day>", false),
621 (NativeType::PANDAS, "object", false),
622 (NativeType::PANDAS, "string", false),
623 (NativeType::PANDAS, "date32[day]", false),
624 (NativeType::POSTGRESQL, "TEXT", false),
625 (NativeType::POSTGRESQL, "TIME", false),
626 (NativeType::POSTGRESQL, "DATE[]", false),
627 (NativeType::MYSQL, "VARCHAR", false),
628 ("duckdb", "DATE", false),
629 ];
630 let wrong: Vec<String> = cases
631 .iter()
632 .filter(|(system, name, want)| NativeType::new(*system, *name).declares_date() != *want)
633 .map(|(system, name, want)| format!("{system} {name}: expected {want}"))
634 .collect();
635 assert!(wrong.is_empty(), "{}", wrong.join("\n"));
636 }
637
638 /// `std_dev`, `variance` and `quartiles.iqr` are nullable but required: a
639 /// `null` is a spread that overflowed `f64`, a missing key a malformed
640 /// document. Serde reads a missing `Option` as `None`, which conflates them.
641 #[test]
642 fn nullable_spread_fields_are_still_required() {
643 let stats = NumericStats {
644 std_dev: None,
645 variance: None,
646 quartiles: Some(Quartiles {
647 q1: 1.0,
648 q2: 2.0,
649 q3: 3.0,
650 iqr: None,
651 }),
652 ..NumericStats::empty()
653 };
654 let document = serde_json::to_value(&stats).unwrap();
655 let restored: NumericStats = serde_json::from_value(document.clone()).unwrap();
656 assert_eq!(serde_json::to_value(restored).unwrap(), document);
657
658 for (parent, key) in [
659 (None, "std_dev"),
660 (None, "variance"),
661 (Some("quartiles"), "iqr"),
662 ] {
663 let mut malformed = document.clone();
664 let object = match parent {
665 Some(parent) => &mut malformed[parent],
666 None => &mut malformed,
667 };
668 object.as_object_mut().unwrap().remove(key).unwrap();
669 let error = serde_json::from_value::<NumericStats>(malformed).unwrap_err();
670 assert!(
671 error
672 .to_string()
673 .contains(&format!("missing field `{key}`")),
674 "{key}: {error}"
675 );
676 }
677 }
678
679 #[test]
680 fn test_column_profile_json_roundtrip() {
681 let profile = ColumnProfile {
682 name: "test_col".to_string(),
683 data_type: DataType::Integer,
684 null_count: 2,
685 total_count: 10,
686 unique_count: Some(8),
687 unique_count_is_approximate: Some(false),
688 unique_count_lower_bound: None,
689 invalid_count: Some(0),
690 type_homogeneity: None,
691 locale_number_count: None,
692 native_type: None,
693 stats: ColumnStats::Numeric(NumericStats {
694 min: 1.0,
695 max: 100.0,
696 mean: 50.5,
697 std_dev: Some(28.87),
698 variance: Some(833.25),
699 median: Some(50.0),
700 quartiles: Some(Quartiles {
701 q1: 25.0,
702 q2: 50.0,
703 q3: 75.0,
704 iqr: Some(50.0),
705 }),
706 mode: Some(42.0),
707 coefficient_of_variation: Some(57.17),
708 skewness: Some(0.0),
709 kurtosis: Some(-1.2),
710 is_approximate: Some(false),
711 outlier_count: Some(0),
712 exact: None,
713 }),
714 patterns: Some(vec![]),
715 };
716
717 let json = serde_json::to_string(&profile).unwrap();
718 let deserialized: ColumnProfile = serde_json::from_str(&json).unwrap();
719
720 assert_eq!(deserialized.name, "test_col");
721 assert_eq!(deserialized.data_type, DataType::Integer);
722 assert_eq!(deserialized.total_count, 10);
723 assert_eq!(deserialized.null_count, 2);
724 assert_eq!(deserialized.unique_count_is_approximate, Some(false));
725
726 if let ColumnStats::Numeric(n) = &deserialized.stats {
727 assert!((n.min - 1.0).abs() < 0.01);
728 assert!((n.max - 100.0).abs() < 0.01);
729 assert!((n.mean - 50.5).abs() < 0.01);
730 assert!(n.median.is_some());
731 assert!(n.quartiles.is_some());
732 } else {
733 panic!("Expected Numeric stats after roundtrip");
734 }
735 }
736
737 #[test]
738 fn test_text_stats_json_roundtrip() {
739 let profile = ColumnProfile {
740 name: "name".to_string(),
741 data_type: DataType::String,
742 null_count: 0,
743 total_count: 3,
744 unique_count: Some(3),
745 unique_count_is_approximate: Some(false),
746 unique_count_lower_bound: None,
747 invalid_count: None,
748 type_homogeneity: None,
749 locale_number_count: None,
750 native_type: None,
751 stats: ColumnStats::Text(TextStats {
752 min_length: 3,
753 max_length: 7,
754 avg_length: 5.0,
755 most_frequent: None,
756 least_frequent: None,
757 }),
758 patterns: Some(vec![]),
759 };
760
761 let json = serde_json::to_string(&profile).unwrap();
762 let deserialized: ColumnProfile = serde_json::from_str(&json).unwrap();
763
764 assert_eq!(deserialized.data_type, DataType::String);
765 if let ColumnStats::Text(t) = &deserialized.stats {
766 assert_eq!(t.min_length, 3);
767 assert_eq!(t.max_length, 7);
768 } else {
769 panic!("Expected Text stats after roundtrip");
770 }
771 }
772}