Skip to main content

dataprof_runtime/
profile_builder.rs

1//! Shared conversion from [`StreamingColumnCollection`] / [`StreamingStatistics`]
2//! into [`ColumnProfile`] and quality-check sample maps.
3//!
4//! All engines that need to produce a [`ColumnProfile`] should call
5//! [`build_column_profile`] instead of constructing one manually.
6//! This ensures consistent stats calculation and pattern detection.
7
8use std::collections::HashMap;
9
10use dataprof_core::{
11    BooleanStats, ColumnProfile, ColumnStats, DataType, DateTimeStats, Locale, SemanticHints,
12    TextStats,
13};
14use dataprof_metrics::{
15    DateTimeAccumulator, ParsedNumber,
16    analysis::column::stats_over_values,
17    analysis::inference::{
18        classify_lexical_forms, count_locale_numbers, holds_digit_codes, is_inferred_date_token,
19        is_integer_token, is_null_like_token, parse_strict_boolean_token,
20    },
21    calculate_datetime_stats, calculate_text_stats, detect_patterns,
22    stats::numeric::{
23        calculate_coefficient_of_variation, compute_decimal_stats_with_parsed_count,
24        compute_integer_stats_with_parsed_count, compute_numeric_stats_with_parsed_count,
25    },
26};
27
28use crate::streaming_stats::{StreamingColumnCollection, StreamingStatistics};
29
30/// Inputs that every engine can provide for centralized profile construction.
31pub struct ColumnProfileInput<'a> {
32    pub name: String,
33    pub data_type: DataType,
34    pub total_count: usize,
35    pub null_count: usize,
36    pub unique_count: Option<usize>,
37    /// Whether `unique_count` is an approximate (HLL) estimate. `None` mirrors
38    /// `unique_count: None`; `Some(false)` for exact counts; `Some(true)` once
39    /// the engine's cardinality estimator has spilled to its HLL sketch.
40    pub unique_count_is_approximate: Option<bool>,
41    /// See [`ColumnProfile::unique_count_lower_bound`]: the estimator's
42    /// [`distinct_lower_bound`](dataprof_metrics::CardinalityEstimator::distinct_lower_bound),
43    /// or `None` when the caller has no estimator behind the count.
44    pub unique_count_lower_bound: Option<usize>,
45    pub sample_values: &'a [String],
46    /// Pre-computed text lengths for engines that track them incrementally.
47    /// When `Some`, text stats are built from these instead of re-scanning samples.
48    pub text_lengths: Option<TextLengths>,
49    /// Pre-computed boolean counts (true_count, false_count) for boolean columns.
50    pub boolean_counts: Option<(usize, usize)>,
51    /// When true, skip statistics computation (produce `ColumnStats::None`).
52    pub skip_statistics: bool,
53    /// When true, skip pattern detection (produce `patterns: None`).
54    pub skip_patterns: bool,
55    /// Optional locale for pattern detection (e.g. "IT", "US").
56    pub locale: Option<Locale>,
57    /// Exact aggregates over every numeric value the engine streamed.
58    ///
59    /// When present, these override the sample-derived `min`, `max`, `mean`,
60    /// `std_dev`, and `variance` on numeric columns, so those fields stay
61    /// exact even when `sample_values` no longer covers the full stream.
62    /// `None` means `sample_values` *is* the full data (in-memory sources)
63    /// or the engine has no exact accumulators for the column.
64    pub exact_numeric: Option<ExactNumericAggregates>,
65    /// Number of values over the full stream accepted by the temporal parser.
66    /// `None` means `sample_values` is the complete in-memory column.
67    pub exact_date_matches: Option<usize>,
68    /// Date statistics over every value the engine streamed (#1297).
69    ///
70    /// When present, a date column's statistics come from here rather than
71    /// from `sample_values`, so its extremes, distributions and slash order
72    /// cover the full stream. `None` means `sample_values` is the complete
73    /// in-memory column.
74    pub exact_dates: Option<&'a DateTimeAccumulator>,
75    /// The samples render a column typed as a decimal (an Arrow decimal of
76    /// any width, 32 to 256 bits), which no float wrote, so they are read with
77    /// `ParsedNumber::parse_decimal` and kept exact from fewer digits than
78    /// text is (#1149).
79    pub decimal_values: bool,
80}
81
82/// Exact streaming aggregates for a numeric column: O(1)-memory statistics that an
83/// engine computed over the entire stream, as opposed to the bounded
84/// `sample_values` it retained.
85#[derive(Debug, Clone, Copy, PartialEq)]
86pub struct ExactNumericAggregates {
87    pub min: f64,
88    pub max: f64,
89    pub mean: f64,
90    /// Standard deviation with the unbiased (n-1) denominator.
91    pub std_dev: f64,
92    /// Variance with the unbiased (n-1) denominator.
93    pub variance: f64,
94    /// Number of parsed numeric values covered by these aggregates.
95    pub count: usize,
96    /// The smallest and largest value as given, which `min` and `max` round
97    /// past what `f64` holds: the exact extremes of an integer or decimal
98    /// column (#1038).
99    pub min_value: Option<ParsedNumber>,
100    pub max_value: Option<ParsedNumber>,
101}
102
103/// Pre-computed text length stats from streaming/columnar engines.
104pub struct TextLengths {
105    pub min_length: usize,
106    pub max_length: usize,
107    pub avg_length: f64,
108}
109
110/// Build a [`ColumnProfile`] from engine-agnostic inputs.
111///
112/// This is the single canonical construction path. Engines provide
113/// pre-inferred `DataType`, counters, sample values, and optionally
114/// pre-computed text lengths; this function handles stats calculation
115/// and pattern detection.
116pub fn build_column_profile(input: ColumnProfileInput<'_>) -> ColumnProfile {
117    if input.data_type == DataType::Nested {
118        return nested_column_profile(input.name, input.total_count, input.null_count);
119    }
120
121    let mut invalid_count = None;
122    let stats = if input.skip_statistics {
123        // Timeliness reads a date column's slash order from its date
124        // statistics (#1305), so a run without the statistics pack still
125        // carries them into the quality pass. `ReportAssembler::build` drops
126        // them from the report.
127        match (&input.data_type, input.exact_dates) {
128            (DataType::Date, Some(dates)) => ColumnStats::DateTime(dates.finish()),
129            _ => ColumnStats::None,
130        }
131    } else {
132        match input.data_type {
133            DataType::Integer | DataType::Float => {
134                let (mut numeric, sampled_numeric) = if input.decimal_values {
135                    compute_decimal_stats_with_parsed_count(input.sample_values)
136                } else if input.data_type == DataType::Integer {
137                    compute_integer_stats_with_parsed_count(input.sample_values)
138                } else {
139                    compute_numeric_stats_with_parsed_count(input.sample_values)
140                };
141                // Values the statistics actually cover. With exact stream
142                // aggregates that is the engine's full parsed count; without
143                // them the sample *is* the full data (in-memory sources).
144                let parsed_total = match &input.exact_numeric {
145                    Some(exact) => exact.count,
146                    None => sampled_numeric,
147                };
148                invalid_count = Some(
149                    input
150                        .total_count
151                        .saturating_sub(input.null_count)
152                        .saturating_sub(parsed_total),
153                );
154                if let Some(exact) = &input.exact_numeric {
155                    numeric.min = exact.min;
156                    numeric.max = exact.max;
157                    // The exact extremes cover every value, as `min` and
158                    // `max` do; without them the column has no exact form.
159                    if let Some(order) = numeric.exact.as_mut() {
160                        match (
161                            exact.min_value.and_then(ParsedNumber::exact_string),
162                            exact.max_value.and_then(ParsedNumber::exact_string),
163                        ) {
164                            (Some(min), Some(max)) => {
165                                order.min = min;
166                                order.max = max;
167                            }
168                            _ => numeric.exact = None,
169                        }
170                    }
171                    numeric.mean = exact.mean;
172                    numeric.std_dev = exact.std_dev.is_finite().then_some(exact.std_dev);
173                    numeric.variance = exact.variance.is_finite().then_some(exact.variance);
174                    numeric.coefficient_of_variation = numeric
175                        .std_dev
176                        .and_then(|value| calculate_coefficient_of_variation(value, exact.mean));
177                    if numeric.std_dev.is_none() {
178                        numeric.skewness = None;
179                        numeric.kurtosis = None;
180                    }
181                    // Order statistics (median, quartiles, mode, skewness,
182                    // kurtosis, outliers) still come from the retained sample;
183                    // disclose that whenever the sample no longer covers every
184                    // numeric value the exact aggregates saw.
185                    if exact.count > sampled_numeric {
186                        numeric.is_approximate = Some(true);
187                    }
188                }
189                if parsed_total == 0 {
190                    ColumnStats::None
191                } else {
192                    ColumnStats::Numeric(numeric)
193                }
194            }
195            DataType::Date => {
196                let parsed_dates = input.exact_date_matches.unwrap_or_else(|| {
197                    input
198                        .sample_values
199                        .iter()
200                        .filter(|value| {
201                            dataprof_metrics::value_matches_hint(
202                                value,
203                                dataprof_core::SemanticHintKind::Temporal,
204                            )
205                        })
206                        .count()
207                });
208                invalid_count = Some(
209                    input
210                        .total_count
211                        .saturating_sub(input.null_count)
212                        .saturating_sub(parsed_dates),
213                );
214                if let Some(dates) = input.exact_dates {
215                    ColumnStats::DateTime(dates.finish())
216                } else if !input.sample_values.is_empty() {
217                    calculate_datetime_stats(input.sample_values)
218                } else if let Some(tl) = &input.text_lengths {
219                    ColumnStats::Text(TextStats::from_lengths(
220                        tl.min_length,
221                        tl.max_length,
222                        tl.avg_length,
223                    ))
224                } else {
225                    ColumnStats::DateTime(DateTimeStats::empty())
226                }
227            }
228            DataType::Boolean => {
229                let (true_count, false_count) = input.boolean_counts.unwrap_or_else(|| {
230                    let tc = input
231                        .sample_values
232                        .iter()
233                        .filter(|v| parse_strict_boolean_token(v.trim()) == Some(true))
234                        .count();
235                    let fc = input
236                        .sample_values
237                        .iter()
238                        .filter(|v| parse_strict_boolean_token(v.trim()) == Some(false))
239                        .count();
240                    (tc, fc)
241                });
242                let total = true_count + false_count;
243                if total > 0 {
244                    ColumnStats::Boolean(BooleanStats {
245                        true_count,
246                        false_count,
247                        true_ratio: true_count as f64 / total as f64,
248                    })
249                } else {
250                    ColumnStats::None
251                }
252            }
253            DataType::String | DataType::Identifier => {
254                if let Some(tl) = &input.text_lengths {
255                    ColumnStats::Text(TextStats::from_lengths(
256                        tl.min_length,
257                        tl.max_length,
258                        tl.avg_length,
259                    ))
260                } else {
261                    calculate_text_stats(input.sample_values)
262                }
263            }
264            DataType::Nested => unreachable!("nested columns return above"),
265        }
266    };
267
268    // A column with no non-null value has counts, but no statistics: every
269    // aggregate over zero values is undefined, and the repo's convention for
270    // undefined is absent, not zero. Without this guard, a column with no rows
271    // reported a numeric min, max, mean, std_dev and variance of 0.0 and a
272    // true_ratio of 0.0, and a text column of nulls reported lengths of 0
273    // (#983): plausible numbers describing values that do not exist, which is
274    // the failure class AGENTS.md singles out. 0.0 and 0 are ordinary values
275    // for a real column, so nothing about that output looked wrong.
276    //
277    // The counts above are untouched. "0 of 0 values were invalid" is a fact
278    // about a column that was analyzed, not a statistic over nothing.
279    let stats = stats_over_values(stats, input.total_count, input.null_count);
280
281    let patterns = if input.skip_patterns {
282        None
283    } else {
284        Some(detect_patterns(input.sample_values, input.locale))
285    };
286
287    ColumnProfile {
288        name: input.name,
289        data_type: input.data_type,
290        null_count: input.null_count,
291        total_count: input.total_count,
292        unique_count: input.unique_count,
293        unique_count_is_approximate: input.unique_count_is_approximate,
294        unique_count_lower_bound: input.unique_count_lower_bound,
295        invalid_count,
296        // Classified over the retained sample, so the counts are bounded by the
297        // engine's reservoir on a large source. `classified_count()` against
298        // `total_count - null_count` is what tells a reader which happened.
299        // Not gated by `skip_statistics`: which forms a column holds is
300        // schema-level evidence like the inferred type, not a statistic.
301        type_homogeneity: Some(classify_lexical_forms(input.sample_values)),
302        // Over the same retained sample, so it stays a subset of `text` above.
303        locale_number_count: Some(count_locale_numbers(input.sample_values)),
304        native_type: None,
305        stats,
306        patterns,
307    }
308}
309
310/// Build the profile of a container column: structs, lists and maps.
311///
312/// It reports its counts and nothing else. Its lengths, distinct values,
313/// patterns and lexical forms would all be measured on a serialisation, and
314/// every input path serialises containers differently (#637). Absent is "not
315/// analyzed", which is what they are.
316pub fn nested_column_profile(name: String, total_count: usize, null_count: usize) -> ColumnProfile {
317    ColumnProfile {
318        name,
319        data_type: DataType::Nested,
320        null_count,
321        total_count,
322        unique_count: None,
323        unique_count_is_approximate: None,
324        unique_count_lower_bound: None,
325        invalid_count: None,
326        type_homogeneity: None,
327        locale_number_count: None,
328        native_type: None,
329        stats: ColumnStats::None,
330        patterns: None,
331    }
332}
333
334/// Report every column whose non-null values were all containers as
335/// [`DataType::Nested`], counts only (#637).
336///
337/// For text-based inputs that know which of their values were containers, as
338/// the JSON readers do. `container_counts` holds, per column name, how many of
339/// the analyzed values were an object or an array.
340///
341/// This matches what a typed source reports for the same records: Parquet and
342/// Arrow carry a struct or list type, and profile such a column the same way.
343/// A column that mixes containers with scalars has no typed twin, and stays
344/// the text column its values make it.
345///
346/// The comparison runs against the profile's own counts, so a container is
347/// weighed against exactly the values the null rule left standing. A container
348/// always renders with a bracket, which the null rule never matches.
349pub fn mark_container_columns(
350    profiles: Vec<ColumnProfile>,
351    container_counts: &HashMap<String, usize>,
352) -> Vec<ColumnProfile> {
353    profiles
354        .into_iter()
355        .map(|profile| {
356            let containers = container_counts.get(&profile.name).copied().unwrap_or(0);
357            let non_null = profile.total_count - profile.null_count;
358            if containers > 0 && containers == non_null {
359                // The declaration is provenance, not a measurement over the
360                // values, so it outlives the statistics this drops (#906).
361                let mut nested =
362                    nested_column_profile(profile.name, profile.total_count, profile.null_count);
363                nested.native_type = profile.native_type;
364                nested
365            } else {
366                profile
367            }
368        })
369        .collect()
370}
371
372/// Convert all columns in a [`StreamingColumnCollection`] into [`ColumnProfile`]s.
373pub fn profiles_from_streaming(
374    column_stats: &StreamingColumnCollection,
375    skip_statistics: bool,
376    skip_patterns: bool,
377    locale: Option<Locale>,
378) -> Vec<ColumnProfile> {
379    profiles_from_streaming_with_hints(
380        column_stats,
381        skip_statistics,
382        skip_patterns,
383        locale,
384        &SemanticHints::default(),
385    )
386}
387
388/// Convert all columns into [`ColumnProfile`]s while applying semantic hints.
389pub fn profiles_from_streaming_with_hints(
390    column_stats: &StreamingColumnCollection,
391    skip_statistics: bool,
392    skip_patterns: bool,
393    locale: Option<Locale>,
394    semantic_hints: &SemanticHints,
395) -> Vec<ColumnProfile> {
396    let mut profiles = Vec::new();
397
398    for column_name in column_stats.column_names() {
399        if let Some(stats) = column_stats.get_column_stats(&column_name) {
400            let profile = profile_from_stats_with_hints(
401                &column_name,
402                stats,
403                skip_statistics,
404                skip_patterns,
405                locale,
406                semantic_hints,
407            );
408            profiles.push(profile);
409        }
410    }
411
412    profiles
413}
414
415/// Convert a single column's [`StreamingStatistics`] into a [`ColumnProfile`].
416pub fn profile_from_stats(
417    name: &str,
418    stats: &StreamingStatistics,
419    skip_statistics: bool,
420    skip_patterns: bool,
421    locale: Option<Locale>,
422) -> ColumnProfile {
423    profile_from_stats_with_hints(
424        name,
425        stats,
426        skip_statistics,
427        skip_patterns,
428        locale,
429        &SemanticHints::default(),
430    )
431}
432
433/// Convert a single column into a [`ColumnProfile`] while applying semantic hints.
434pub fn profile_from_stats_with_hints(
435    name: &str,
436    stats: &StreamingStatistics,
437    skip_statistics: bool,
438    skip_patterns: bool,
439    locale: Option<Locale>,
440    semantic_hints: &SemanticHints,
441) -> ColumnProfile {
442    let data_type = if semantic_hints.is_identifier_column(name) {
443        DataType::Identifier
444    } else {
445        infer_data_type_streaming(stats)
446    };
447    let text_stats = stats.text_length_stats();
448
449    build_column_profile(ColumnProfileInput {
450        name: name.to_string(),
451        data_type,
452        total_count: stats.count,
453        null_count: stats.null_count,
454        unique_count: Some(stats.unique_count()),
455        unique_count_is_approximate: Some(stats.unique_count_is_approximate()),
456        unique_count_lower_bound: Some(stats.unique_count_lower_bound()),
457        sample_values: stats.sample_values(),
458        text_lengths: Some(TextLengths {
459            min_length: text_stats.min_length,
460            max_length: text_stats.max_length,
461            avg_length: text_stats.avg_length,
462        }),
463        boolean_counts: None,
464        skip_statistics,
465        skip_patterns,
466        locale,
467        exact_numeric: stats.exact_numeric_aggregates(),
468        exact_date_matches: Some(stats.date_match_count()),
469        exact_dates: Some(stats.date_statistics()),
470        decimal_values: false,
471    })
472}
473
474/// Infer [`DataType`] from [`StreamingStatistics`] sample values.
475pub fn infer_data_type_streaming(stats: &StreamingStatistics) -> DataType {
476    let sample_values = stats.sample_values();
477    let non_empty: Vec<&String> = sample_values
478        .iter()
479        .filter(|s| !is_null_like_token(s.trim()))
480        .collect();
481
482    if !non_empty.is_empty() {
483        // The same question `infer_type` asks, so codes are text on every engine.
484        let digit_codes = holds_digit_codes(non_empty.iter().map(|s| s.trim()));
485
486        let all_integers = non_empty.iter().all(|s| is_integer_token(s.trim()));
487        if !digit_codes && all_integers {
488            return DataType::Integer;
489        }
490
491        // Numeric lexical forms remain numeric even if none are finite enough
492        // to enter the streaming aggregates. `build_column_profile` reports
493        // those unusable values through `invalid_count`.
494        let numeric_count = non_empty
495            .iter()
496            .filter(|s| s.trim().parse::<f64>().is_ok())
497            .count();
498        if !digit_codes && numeric_count as f64 / non_empty.len() as f64 > 0.8 {
499            return DataType::Float;
500        }
501
502        // `.trim()`, like every sibling predicate in this function and like
503        // `infer_type`. Without it a value that arrived padded — CSV parsing
504        // does not trim by default — was a date to the buffered engines and
505        // text to this one, which is the same cross-engine split the shared
506        // grammar exists to close. Over every value, as `infer_type` counts and
507        // as the shares above do: a column whose first 100 values were dates
508        // was a date here and text everywhere else (#915).
509        let date_like_count = non_empty
510            .iter()
511            .filter(|s| is_inferred_date_token(s.trim()))
512            .count();
513
514        if date_like_count as f64 / non_empty.len() as f64 > 0.7 {
515            return DataType::Date;
516        }
517
518        let bool_count = non_empty
519            .iter()
520            .filter(|s| parse_strict_boolean_token(s.trim()).is_some())
521            .count();
522
523        if bool_count as f64 / non_empty.len() as f64 >= 0.9 {
524            return DataType::Boolean;
525        }
526    }
527
528    DataType::String
529}
530
531/// Build a sample `HashMap` from a [`StreamingColumnCollection`] suitable for
532/// `QualityMetrics::calculate_from_data()`.
533pub fn quality_check_samples(
534    column_stats: &StreamingColumnCollection,
535) -> HashMap<String, Vec<String>> {
536    let mut samples = HashMap::new();
537
538    for column_name in column_stats.column_names() {
539        if let Some(stats) = column_stats.get_column_stats(&column_name) {
540            let sample_values: Vec<String> = stats.sample_values().to_vec();
541            samples.insert(column_name, sample_values);
542        }
543    }
544
545    samples
546}
547
548#[cfg(test)]
549mod tests {
550    use super::*;
551    use crate::streaming_stats::StreamingColumnCollection;
552
553    #[test]
554    fn test_profiles_from_streaming() {
555        let mut collection = StreamingColumnCollection::new();
556        let headers = vec!["name".to_string(), "age".to_string()];
557
558        collection.process_record(&headers, vec!["Alice".to_string(), "30".to_string()]);
559        collection.process_record(&headers, vec!["Bob".to_string(), "25".to_string()]);
560        collection.process_record(&headers, vec!["Charlie".to_string(), "35".to_string()]);
561
562        let profiles = profiles_from_streaming(&collection, false, false, None);
563        assert_eq!(profiles.len(), 2);
564
565        let age = profiles.iter().find(|p| p.name == "age").unwrap();
566        assert_eq!(age.data_type, DataType::Integer);
567        assert_eq!(age.total_count, 3);
568    }
569
570    #[test]
571    fn test_all_non_finite_tokens_keep_streaming_float_type() {
572        let mut collection = StreamingColumnCollection::new();
573        let headers = vec!["value".to_string()];
574
575        collection.process_record(&headers, vec!["Infinity".to_string()]);
576        collection.process_record(&headers, vec!["-inf".to_string()]);
577
578        let profiles = profiles_from_streaming(&collection, false, false, None);
579        let value = profiles
580            .iter()
581            .find(|profile| profile.name == "value")
582            .expect("value profile");
583
584        assert_eq!(value.data_type, DataType::Float);
585        assert_eq!(value.invalid_count, Some(2));
586        assert!(matches!(value.stats, ColumnStats::None));
587    }
588
589    #[test]
590    fn type_homogeneity_is_classified_from_the_retained_sample() {
591        // The streaming path only holds its reservoir, so the counts describe
592        // the sample. `classified_count()` short of the non-null total is the
593        // signal that the shares are sampled — nothing else discloses it.
594        let samples = ["1", "2", "junk", ""].map(String::from).to_vec();
595        let profile = build_column_profile(ColumnProfileInput {
596            name: "v".to_string(),
597            data_type: DataType::String,
598            total_count: 4_000,
599            null_count: 1_000,
600            unique_count: Some(3),
601            unique_count_is_approximate: Some(false),
602            unique_count_lower_bound: None,
603            sample_values: &samples,
604            text_lengths: None,
605            boolean_counts: None,
606            // Not gated by the analysis selection: a column's lexical makeup is
607            // schema-level evidence, like its inferred type.
608            skip_statistics: true,
609            skip_patterns: true,
610            exact_numeric: None,
611            exact_date_matches: None,
612            exact_dates: None,
613            decimal_values: false,
614            locale: None,
615        });
616
617        let counts = profile
618            .type_homogeneity
619            .expect("classification runs on every column");
620        assert_eq!(counts.numeric, 2);
621        assert_eq!(counts.text, 1);
622        assert_eq!(counts.classified_count(), 3, "null-like values are skipped");
623        assert!(
624            counts.classified_count() < profile.total_count - profile.null_count,
625            "a sample-derived count must read as short of the column"
626        );
627    }
628
629    #[test]
630    fn test_quality_check_samples() {
631        let mut collection = StreamingColumnCollection::new();
632        let headers = vec!["col".to_string()];
633
634        collection.process_record(&headers, vec!["val1".to_string()]);
635        collection.process_record(&headers, vec!["val2".to_string()]);
636
637        let samples = quality_check_samples(&collection);
638        assert!(samples.contains_key("col"));
639        assert_eq!(samples["col"].len(), 2);
640    }
641
642    #[test]
643    fn typed_columns_without_parsed_values_have_no_statistics() {
644        for data_type in [DataType::Integer, DataType::Float, DataType::Boolean] {
645            for values in [["", ""], ["junk", "Infinity"]] {
646                let samples = values.map(String::from);
647                let null_count = if values[0].is_empty() { 2 } else { 0 };
648                let profile = build_column_profile(ColumnProfileInput {
649                    name: "v".to_string(),
650                    data_type: data_type.clone(),
651                    total_count: 2,
652                    null_count,
653                    unique_count: None,
654                    unique_count_is_approximate: None,
655                    unique_count_lower_bound: None,
656                    sample_values: &samples,
657                    text_lengths: None,
658                    boolean_counts: None,
659                    skip_statistics: false,
660                    skip_patterns: false,
661                    locale: None,
662                    exact_numeric: None,
663                    exact_date_matches: None,
664                    exact_dates: None,
665                    decimal_values: false,
666                });
667
668                assert!(matches!(profile.stats, ColumnStats::None));
669                assert_eq!(profile.data_type, data_type);
670                assert_eq!(profile.total_count, 2);
671                assert_eq!(profile.null_count, null_count);
672                if matches!(data_type, DataType::Integer | DataType::Float) {
673                    assert_eq!(profile.invalid_count, Some(2 - null_count));
674                }
675                assert!(profile.patterns.is_some());
676            }
677        }
678    }
679
680    #[test]
681    fn exact_zero_statistics_survive_a_sample_without_parsed_values() {
682        // A bounded sample can miss every usable value. The full-stream
683        // count, not the sample count or the aggregate's value, governs absence.
684        for data_type in [DataType::Integer, DataType::Boolean] {
685            let samples = [String::new()];
686            let profile = build_column_profile(ColumnProfileInput {
687                name: "v".to_string(),
688                data_type: data_type.clone(),
689                total_count: 3,
690                null_count: 1,
691                unique_count: None,
692                unique_count_is_approximate: None,
693                unique_count_lower_bound: None,
694                sample_values: &samples,
695                text_lengths: None,
696                boolean_counts: Some((0, 2)),
697                skip_statistics: false,
698                skip_patterns: true,
699                locale: None,
700                exact_numeric: Some(ExactNumericAggregates {
701                    min: 0.0,
702                    max: 0.0,
703                    mean: 0.0,
704                    std_dev: 0.0,
705                    variance: 0.0,
706                    count: 2,
707                    min_value: None,
708                    max_value: None,
709                }),
710                exact_date_matches: None,
711                exact_dates: None,
712                decimal_values: false,
713            });
714
715            match profile.stats {
716                ColumnStats::Numeric(stats) => {
717                    assert_eq!(stats.min, 0.0);
718                    assert_eq!(stats.mean, 0.0);
719                    assert_eq!(stats.is_approximate, Some(true));
720                    assert_eq!(profile.invalid_count, Some(0));
721                }
722                ColumnStats::Boolean(stats) => {
723                    assert_eq!(stats.true_ratio, 0.0);
724                    assert_eq!(stats.false_count, 2);
725                }
726                other => panic!("expected measured zero statistics, got {other:?}"),
727            }
728        }
729    }
730
731    #[test]
732    fn an_overflowing_exact_spread_clears_the_sample_shape() {
733        // The bounded sample missed the extreme values, so on its own it has a
734        // finite spread; the full stream does not (#784).
735        let samples: Vec<String> = ["5", "0.5", "5", "7", "3"]
736            .iter()
737            .map(|value| value.to_string())
738            .collect();
739        let profile = build_column_profile(ColumnProfileInput {
740            name: "x".to_string(),
741            data_type: DataType::Float,
742            total_count: 20_000,
743            null_count: 0,
744            unique_count: None,
745            unique_count_is_approximate: None,
746            unique_count_lower_bound: None,
747            sample_values: &samples,
748            text_lengths: None,
749            boolean_counts: None,
750            skip_statistics: false,
751            skip_patterns: true,
752            locale: None,
753            exact_numeric: Some(ExactNumericAggregates {
754                min: -1e308,
755                max: 1e308,
756                mean: 2.9167,
757                std_dev: f64::NAN,
758                variance: f64::NAN,
759                count: 20_000,
760                min_value: None,
761                max_value: None,
762            }),
763            exact_date_matches: None,
764            exact_dates: None,
765            decimal_values: false,
766        });
767
768        let ColumnStats::Numeric(stats) = profile.stats else {
769            panic!("expected numeric statistics, got {:?}", profile.stats);
770        };
771        assert_eq!(stats.variance, None);
772        assert_eq!(stats.std_dev, None);
773        assert_eq!(stats.coefficient_of_variation, None);
774        assert_eq!(stats.skewness, None);
775        assert_eq!(stats.kurtosis, None);
776        // Order statistics still describe the retained sample.
777        assert_eq!(stats.median, Some(5.0));
778        assert_eq!(stats.is_approximate, Some(true));
779    }
780
781    #[test]
782    fn test_boolean_stats_with_counts() {
783        let samples = vec!["True".to_string(), "False".to_string(), "True".to_string()];
784        let profile = build_column_profile(ColumnProfileInput {
785            name: "flag".to_string(),
786            data_type: DataType::Boolean,
787            total_count: 3,
788            null_count: 0,
789            unique_count: Some(2),
790            unique_count_is_approximate: Some(false),
791            unique_count_lower_bound: None,
792            sample_values: &samples,
793            text_lengths: None,
794            boolean_counts: Some((2, 1)),
795            skip_statistics: false,
796            skip_patterns: false,
797            exact_numeric: None,
798            exact_date_matches: None,
799            exact_dates: None,
800            decimal_values: false,
801            locale: None,
802        });
803
804        match &profile.stats {
805            ColumnStats::Boolean(b) => {
806                assert_eq!(b.true_count, 2);
807                assert_eq!(b.false_count, 1);
808                assert!((b.true_ratio - 2.0 / 3.0).abs() < 0.001);
809            }
810            other => panic!("expected Boolean stats, got {:?}", other),
811        }
812    }
813
814    #[test]
815    fn test_boolean_stats_fallback_case_insensitive() {
816        let samples = vec![
817            "true".to_string(),
818            "FALSE".to_string(),
819            " True ".to_string(),
820        ];
821        let profile = build_column_profile(ColumnProfileInput {
822            name: "flag".to_string(),
823            data_type: DataType::Boolean,
824            total_count: 3,
825            null_count: 0,
826            unique_count: Some(2),
827            unique_count_is_approximate: Some(false),
828            unique_count_lower_bound: None,
829            sample_values: &samples,
830            text_lengths: None,
831            boolean_counts: None,
832            skip_statistics: false,
833            skip_patterns: false,
834            exact_numeric: None,
835            exact_date_matches: None,
836            exact_dates: None,
837            decimal_values: false,
838            locale: None,
839        });
840
841        match &profile.stats {
842            ColumnStats::Boolean(b) => {
843                assert_eq!(b.true_count, 2);
844                assert_eq!(b.false_count, 1);
845                assert!((b.true_ratio - 2.0 / 3.0).abs() < 0.001);
846            }
847            other => panic!("expected Boolean stats, got {:?}", other),
848        }
849    }
850
851    #[test]
852    fn test_skip_statistics() {
853        let samples = vec!["10".to_string(), "20".to_string(), "30".to_string()];
854        let profile = build_column_profile(ColumnProfileInput {
855            name: "num".to_string(),
856            data_type: DataType::Integer,
857            total_count: 3,
858            null_count: 0,
859            unique_count: Some(3),
860            unique_count_is_approximate: Some(false),
861            unique_count_lower_bound: None,
862            sample_values: &samples,
863            text_lengths: None,
864            boolean_counts: None,
865            skip_statistics: true,
866            skip_patterns: false,
867            exact_numeric: None,
868            exact_date_matches: None,
869            exact_dates: None,
870            decimal_values: false,
871            locale: None,
872        });
873
874        assert!(matches!(profile.stats, ColumnStats::None));
875        assert_eq!(profile.data_type, DataType::Integer);
876    }
877
878    #[test]
879    fn test_skip_patterns() {
880        let samples = vec!["hello".to_string(), "world".to_string()];
881        let profile = build_column_profile(ColumnProfileInput {
882            name: "text".to_string(),
883            data_type: DataType::String,
884            total_count: 2,
885            null_count: 0,
886            unique_count: Some(2),
887            unique_count_is_approximate: Some(false),
888            unique_count_lower_bound: None,
889            sample_values: &samples,
890            text_lengths: None,
891            boolean_counts: None,
892            skip_statistics: false,
893            skip_patterns: true,
894            exact_numeric: None,
895            exact_date_matches: None,
896            exact_dates: None,
897            decimal_values: false,
898            locale: None,
899        });
900
901        // skip_patterns must be distinguishable from "scanned, nothing matched",
902        // otherwise downstream redaction gates cannot fail closed.
903        assert!(profile.patterns.is_none());
904        assert!(matches!(profile.stats, ColumnStats::Text(_)));
905    }
906
907    #[test]
908    fn test_scanned_without_matches_is_not_none() {
909        // The counterpart to `test_skip_patterns`: a column that *was* scanned
910        // and matched nothing yields `Some([])`, which downstream consumers may
911        // safely read as "no sensitive data here".
912        let samples = vec!["hello".to_string(), "world".to_string()];
913        let profile = build_column_profile(ColumnProfileInput {
914            name: "text".to_string(),
915            data_type: DataType::String,
916            total_count: 2,
917            null_count: 0,
918            unique_count: Some(2),
919            unique_count_is_approximate: Some(false),
920            unique_count_lower_bound: None,
921            sample_values: &samples,
922            text_lengths: None,
923            boolean_counts: None,
924            skip_statistics: false,
925            skip_patterns: false,
926            exact_numeric: None,
927            exact_date_matches: None,
928            exact_dates: None,
929            decimal_values: false,
930            locale: None,
931        });
932
933        assert!(profile.patterns.is_some_and(|p| p.is_empty()));
934    }
935
936    #[test]
937    fn test_all_packs_default() {
938        let samples = vec!["42".to_string(), "99".to_string()];
939        let profile = build_column_profile(ColumnProfileInput {
940            name: "val".to_string(),
941            data_type: DataType::Integer,
942            total_count: 2,
943            null_count: 0,
944            unique_count: Some(2),
945            unique_count_is_approximate: Some(false),
946            unique_count_lower_bound: None,
947            sample_values: &samples,
948            text_lengths: None,
949            boolean_counts: None,
950            skip_statistics: false,
951            skip_patterns: false,
952            exact_numeric: None,
953            exact_date_matches: None,
954            exact_dates: None,
955            decimal_values: false,
956            locale: None,
957        });
958
959        assert!(matches!(profile.stats, ColumnStats::Numeric(_)));
960        assert_eq!(profile.data_type, DataType::Integer);
961    }
962}