1use std::collections::HashMap;
9
10use dataprof_core::{
11 BooleanStats, ColumnProfile, ColumnStats, DataType, DateTimeStats, Locale, SemanticHints,
12 TextStats,
13};
14use dataprof_metrics::{
15 DateTimeAccumulator, ParsedNumber,
16 analysis::column::stats_over_values,
17 analysis::inference::{
18 classify_lexical_forms, count_locale_numbers, holds_digit_codes, is_inferred_date_token,
19 is_integer_token, is_null_like_token, parse_strict_boolean_token,
20 },
21 calculate_datetime_stats, calculate_text_stats, detect_patterns,
22 stats::numeric::{
23 calculate_coefficient_of_variation, compute_decimal_stats_with_parsed_count,
24 compute_integer_stats_with_parsed_count, compute_numeric_stats_with_parsed_count,
25 },
26};
27
28use crate::streaming_stats::{StreamingColumnCollection, StreamingStatistics};
29
30pub struct ColumnProfileInput<'a> {
32 pub name: String,
33 pub data_type: DataType,
34 pub total_count: usize,
35 pub null_count: usize,
36 pub unique_count: Option<usize>,
37 pub unique_count_is_approximate: Option<bool>,
41 pub unique_count_lower_bound: Option<usize>,
45 pub sample_values: &'a [String],
46 pub text_lengths: Option<TextLengths>,
49 pub boolean_counts: Option<(usize, usize)>,
51 pub skip_statistics: bool,
53 pub skip_patterns: bool,
55 pub locale: Option<Locale>,
57 pub exact_numeric: Option<ExactNumericAggregates>,
65 pub exact_date_matches: Option<usize>,
68 pub exact_dates: Option<&'a DateTimeAccumulator>,
75 pub decimal_values: bool,
80}
81
82#[derive(Debug, Clone, Copy, PartialEq)]
86pub struct ExactNumericAggregates {
87 pub min: f64,
88 pub max: f64,
89 pub mean: f64,
90 pub std_dev: f64,
92 pub variance: f64,
94 pub count: usize,
96 pub min_value: Option<ParsedNumber>,
100 pub max_value: Option<ParsedNumber>,
101}
102
103pub struct TextLengths {
105 pub min_length: usize,
106 pub max_length: usize,
107 pub avg_length: f64,
108}
109
110pub fn build_column_profile(input: ColumnProfileInput<'_>) -> ColumnProfile {
117 if input.data_type == DataType::Nested {
118 return nested_column_profile(input.name, input.total_count, input.null_count);
119 }
120
121 let mut invalid_count = None;
122 let stats = if input.skip_statistics {
123 match (&input.data_type, input.exact_dates) {
128 (DataType::Date, Some(dates)) => ColumnStats::DateTime(dates.finish()),
129 _ => ColumnStats::None,
130 }
131 } else {
132 match input.data_type {
133 DataType::Integer | DataType::Float => {
134 let (mut numeric, sampled_numeric) = if input.decimal_values {
135 compute_decimal_stats_with_parsed_count(input.sample_values)
136 } else if input.data_type == DataType::Integer {
137 compute_integer_stats_with_parsed_count(input.sample_values)
138 } else {
139 compute_numeric_stats_with_parsed_count(input.sample_values)
140 };
141 let parsed_total = match &input.exact_numeric {
145 Some(exact) => exact.count,
146 None => sampled_numeric,
147 };
148 invalid_count = Some(
149 input
150 .total_count
151 .saturating_sub(input.null_count)
152 .saturating_sub(parsed_total),
153 );
154 if let Some(exact) = &input.exact_numeric {
155 numeric.min = exact.min;
156 numeric.max = exact.max;
157 if let Some(order) = numeric.exact.as_mut() {
160 match (
161 exact.min_value.and_then(ParsedNumber::exact_string),
162 exact.max_value.and_then(ParsedNumber::exact_string),
163 ) {
164 (Some(min), Some(max)) => {
165 order.min = min;
166 order.max = max;
167 }
168 _ => numeric.exact = None,
169 }
170 }
171 numeric.mean = exact.mean;
172 numeric.std_dev = exact.std_dev.is_finite().then_some(exact.std_dev);
173 numeric.variance = exact.variance.is_finite().then_some(exact.variance);
174 numeric.coefficient_of_variation = numeric
175 .std_dev
176 .and_then(|value| calculate_coefficient_of_variation(value, exact.mean));
177 if numeric.std_dev.is_none() {
178 numeric.skewness = None;
179 numeric.kurtosis = None;
180 }
181 if exact.count > sampled_numeric {
186 numeric.is_approximate = Some(true);
187 }
188 }
189 if parsed_total == 0 {
190 ColumnStats::None
191 } else {
192 ColumnStats::Numeric(numeric)
193 }
194 }
195 DataType::Date => {
196 let parsed_dates = input.exact_date_matches.unwrap_or_else(|| {
197 input
198 .sample_values
199 .iter()
200 .filter(|value| {
201 dataprof_metrics::value_matches_hint(
202 value,
203 dataprof_core::SemanticHintKind::Temporal,
204 )
205 })
206 .count()
207 });
208 invalid_count = Some(
209 input
210 .total_count
211 .saturating_sub(input.null_count)
212 .saturating_sub(parsed_dates),
213 );
214 if let Some(dates) = input.exact_dates {
215 ColumnStats::DateTime(dates.finish())
216 } else if !input.sample_values.is_empty() {
217 calculate_datetime_stats(input.sample_values)
218 } else if let Some(tl) = &input.text_lengths {
219 ColumnStats::Text(TextStats::from_lengths(
220 tl.min_length,
221 tl.max_length,
222 tl.avg_length,
223 ))
224 } else {
225 ColumnStats::DateTime(DateTimeStats::empty())
226 }
227 }
228 DataType::Boolean => {
229 let (true_count, false_count) = input.boolean_counts.unwrap_or_else(|| {
230 let tc = input
231 .sample_values
232 .iter()
233 .filter(|v| parse_strict_boolean_token(v.trim()) == Some(true))
234 .count();
235 let fc = input
236 .sample_values
237 .iter()
238 .filter(|v| parse_strict_boolean_token(v.trim()) == Some(false))
239 .count();
240 (tc, fc)
241 });
242 let total = true_count + false_count;
243 if total > 0 {
244 ColumnStats::Boolean(BooleanStats {
245 true_count,
246 false_count,
247 true_ratio: true_count as f64 / total as f64,
248 })
249 } else {
250 ColumnStats::None
251 }
252 }
253 DataType::String | DataType::Identifier => {
254 if let Some(tl) = &input.text_lengths {
255 ColumnStats::Text(TextStats::from_lengths(
256 tl.min_length,
257 tl.max_length,
258 tl.avg_length,
259 ))
260 } else {
261 calculate_text_stats(input.sample_values)
262 }
263 }
264 DataType::Nested => unreachable!("nested columns return above"),
265 }
266 };
267
268 let stats = stats_over_values(stats, input.total_count, input.null_count);
280
281 let patterns = if input.skip_patterns {
282 None
283 } else {
284 Some(detect_patterns(input.sample_values, input.locale))
285 };
286
287 ColumnProfile {
288 name: input.name,
289 data_type: input.data_type,
290 null_count: input.null_count,
291 total_count: input.total_count,
292 unique_count: input.unique_count,
293 unique_count_is_approximate: input.unique_count_is_approximate,
294 unique_count_lower_bound: input.unique_count_lower_bound,
295 invalid_count,
296 type_homogeneity: Some(classify_lexical_forms(input.sample_values)),
302 locale_number_count: Some(count_locale_numbers(input.sample_values)),
304 native_type: None,
305 stats,
306 patterns,
307 }
308}
309
310pub fn nested_column_profile(name: String, total_count: usize, null_count: usize) -> ColumnProfile {
317 ColumnProfile {
318 name,
319 data_type: DataType::Nested,
320 null_count,
321 total_count,
322 unique_count: None,
323 unique_count_is_approximate: None,
324 unique_count_lower_bound: None,
325 invalid_count: None,
326 type_homogeneity: None,
327 locale_number_count: None,
328 native_type: None,
329 stats: ColumnStats::None,
330 patterns: None,
331 }
332}
333
334pub fn mark_container_columns(
350 profiles: Vec<ColumnProfile>,
351 container_counts: &HashMap<String, usize>,
352) -> Vec<ColumnProfile> {
353 profiles
354 .into_iter()
355 .map(|profile| {
356 let containers = container_counts.get(&profile.name).copied().unwrap_or(0);
357 let non_null = profile.total_count - profile.null_count;
358 if containers > 0 && containers == non_null {
359 let mut nested =
362 nested_column_profile(profile.name, profile.total_count, profile.null_count);
363 nested.native_type = profile.native_type;
364 nested
365 } else {
366 profile
367 }
368 })
369 .collect()
370}
371
372pub fn profiles_from_streaming(
374 column_stats: &StreamingColumnCollection,
375 skip_statistics: bool,
376 skip_patterns: bool,
377 locale: Option<Locale>,
378) -> Vec<ColumnProfile> {
379 profiles_from_streaming_with_hints(
380 column_stats,
381 skip_statistics,
382 skip_patterns,
383 locale,
384 &SemanticHints::default(),
385 )
386}
387
388pub fn profiles_from_streaming_with_hints(
390 column_stats: &StreamingColumnCollection,
391 skip_statistics: bool,
392 skip_patterns: bool,
393 locale: Option<Locale>,
394 semantic_hints: &SemanticHints,
395) -> Vec<ColumnProfile> {
396 let mut profiles = Vec::new();
397
398 for column_name in column_stats.column_names() {
399 if let Some(stats) = column_stats.get_column_stats(&column_name) {
400 let profile = profile_from_stats_with_hints(
401 &column_name,
402 stats,
403 skip_statistics,
404 skip_patterns,
405 locale,
406 semantic_hints,
407 );
408 profiles.push(profile);
409 }
410 }
411
412 profiles
413}
414
415pub fn profile_from_stats(
417 name: &str,
418 stats: &StreamingStatistics,
419 skip_statistics: bool,
420 skip_patterns: bool,
421 locale: Option<Locale>,
422) -> ColumnProfile {
423 profile_from_stats_with_hints(
424 name,
425 stats,
426 skip_statistics,
427 skip_patterns,
428 locale,
429 &SemanticHints::default(),
430 )
431}
432
433pub fn profile_from_stats_with_hints(
435 name: &str,
436 stats: &StreamingStatistics,
437 skip_statistics: bool,
438 skip_patterns: bool,
439 locale: Option<Locale>,
440 semantic_hints: &SemanticHints,
441) -> ColumnProfile {
442 let data_type = if semantic_hints.is_identifier_column(name) {
443 DataType::Identifier
444 } else {
445 infer_data_type_streaming(stats)
446 };
447 let text_stats = stats.text_length_stats();
448
449 build_column_profile(ColumnProfileInput {
450 name: name.to_string(),
451 data_type,
452 total_count: stats.count,
453 null_count: stats.null_count,
454 unique_count: Some(stats.unique_count()),
455 unique_count_is_approximate: Some(stats.unique_count_is_approximate()),
456 unique_count_lower_bound: Some(stats.unique_count_lower_bound()),
457 sample_values: stats.sample_values(),
458 text_lengths: Some(TextLengths {
459 min_length: text_stats.min_length,
460 max_length: text_stats.max_length,
461 avg_length: text_stats.avg_length,
462 }),
463 boolean_counts: None,
464 skip_statistics,
465 skip_patterns,
466 locale,
467 exact_numeric: stats.exact_numeric_aggregates(),
468 exact_date_matches: Some(stats.date_match_count()),
469 exact_dates: Some(stats.date_statistics()),
470 decimal_values: false,
471 })
472}
473
474pub fn infer_data_type_streaming(stats: &StreamingStatistics) -> DataType {
476 let sample_values = stats.sample_values();
477 let non_empty: Vec<&String> = sample_values
478 .iter()
479 .filter(|s| !is_null_like_token(s.trim()))
480 .collect();
481
482 if !non_empty.is_empty() {
483 let digit_codes = holds_digit_codes(non_empty.iter().map(|s| s.trim()));
485
486 let all_integers = non_empty.iter().all(|s| is_integer_token(s.trim()));
487 if !digit_codes && all_integers {
488 return DataType::Integer;
489 }
490
491 let numeric_count = non_empty
495 .iter()
496 .filter(|s| s.trim().parse::<f64>().is_ok())
497 .count();
498 if !digit_codes && numeric_count as f64 / non_empty.len() as f64 > 0.8 {
499 return DataType::Float;
500 }
501
502 let date_like_count = non_empty
510 .iter()
511 .filter(|s| is_inferred_date_token(s.trim()))
512 .count();
513
514 if date_like_count as f64 / non_empty.len() as f64 > 0.7 {
515 return DataType::Date;
516 }
517
518 let bool_count = non_empty
519 .iter()
520 .filter(|s| parse_strict_boolean_token(s.trim()).is_some())
521 .count();
522
523 if bool_count as f64 / non_empty.len() as f64 >= 0.9 {
524 return DataType::Boolean;
525 }
526 }
527
528 DataType::String
529}
530
531pub fn quality_check_samples(
534 column_stats: &StreamingColumnCollection,
535) -> HashMap<String, Vec<String>> {
536 let mut samples = HashMap::new();
537
538 for column_name in column_stats.column_names() {
539 if let Some(stats) = column_stats.get_column_stats(&column_name) {
540 let sample_values: Vec<String> = stats.sample_values().to_vec();
541 samples.insert(column_name, sample_values);
542 }
543 }
544
545 samples
546}
547
548#[cfg(test)]
549mod tests {
550 use super::*;
551 use crate::streaming_stats::StreamingColumnCollection;
552
553 #[test]
554 fn test_profiles_from_streaming() {
555 let mut collection = StreamingColumnCollection::new();
556 let headers = vec!["name".to_string(), "age".to_string()];
557
558 collection.process_record(&headers, vec!["Alice".to_string(), "30".to_string()]);
559 collection.process_record(&headers, vec!["Bob".to_string(), "25".to_string()]);
560 collection.process_record(&headers, vec!["Charlie".to_string(), "35".to_string()]);
561
562 let profiles = profiles_from_streaming(&collection, false, false, None);
563 assert_eq!(profiles.len(), 2);
564
565 let age = profiles.iter().find(|p| p.name == "age").unwrap();
566 assert_eq!(age.data_type, DataType::Integer);
567 assert_eq!(age.total_count, 3);
568 }
569
570 #[test]
571 fn test_all_non_finite_tokens_keep_streaming_float_type() {
572 let mut collection = StreamingColumnCollection::new();
573 let headers = vec!["value".to_string()];
574
575 collection.process_record(&headers, vec!["Infinity".to_string()]);
576 collection.process_record(&headers, vec!["-inf".to_string()]);
577
578 let profiles = profiles_from_streaming(&collection, false, false, None);
579 let value = profiles
580 .iter()
581 .find(|profile| profile.name == "value")
582 .expect("value profile");
583
584 assert_eq!(value.data_type, DataType::Float);
585 assert_eq!(value.invalid_count, Some(2));
586 assert!(matches!(value.stats, ColumnStats::None));
587 }
588
589 #[test]
590 fn type_homogeneity_is_classified_from_the_retained_sample() {
591 let samples = ["1", "2", "junk", ""].map(String::from).to_vec();
595 let profile = build_column_profile(ColumnProfileInput {
596 name: "v".to_string(),
597 data_type: DataType::String,
598 total_count: 4_000,
599 null_count: 1_000,
600 unique_count: Some(3),
601 unique_count_is_approximate: Some(false),
602 unique_count_lower_bound: None,
603 sample_values: &samples,
604 text_lengths: None,
605 boolean_counts: None,
606 skip_statistics: true,
609 skip_patterns: true,
610 exact_numeric: None,
611 exact_date_matches: None,
612 exact_dates: None,
613 decimal_values: false,
614 locale: None,
615 });
616
617 let counts = profile
618 .type_homogeneity
619 .expect("classification runs on every column");
620 assert_eq!(counts.numeric, 2);
621 assert_eq!(counts.text, 1);
622 assert_eq!(counts.classified_count(), 3, "null-like values are skipped");
623 assert!(
624 counts.classified_count() < profile.total_count - profile.null_count,
625 "a sample-derived count must read as short of the column"
626 );
627 }
628
629 #[test]
630 fn test_quality_check_samples() {
631 let mut collection = StreamingColumnCollection::new();
632 let headers = vec!["col".to_string()];
633
634 collection.process_record(&headers, vec!["val1".to_string()]);
635 collection.process_record(&headers, vec!["val2".to_string()]);
636
637 let samples = quality_check_samples(&collection);
638 assert!(samples.contains_key("col"));
639 assert_eq!(samples["col"].len(), 2);
640 }
641
642 #[test]
643 fn typed_columns_without_parsed_values_have_no_statistics() {
644 for data_type in [DataType::Integer, DataType::Float, DataType::Boolean] {
645 for values in [["", ""], ["junk", "Infinity"]] {
646 let samples = values.map(String::from);
647 let null_count = if values[0].is_empty() { 2 } else { 0 };
648 let profile = build_column_profile(ColumnProfileInput {
649 name: "v".to_string(),
650 data_type: data_type.clone(),
651 total_count: 2,
652 null_count,
653 unique_count: None,
654 unique_count_is_approximate: None,
655 unique_count_lower_bound: None,
656 sample_values: &samples,
657 text_lengths: None,
658 boolean_counts: None,
659 skip_statistics: false,
660 skip_patterns: false,
661 locale: None,
662 exact_numeric: None,
663 exact_date_matches: None,
664 exact_dates: None,
665 decimal_values: false,
666 });
667
668 assert!(matches!(profile.stats, ColumnStats::None));
669 assert_eq!(profile.data_type, data_type);
670 assert_eq!(profile.total_count, 2);
671 assert_eq!(profile.null_count, null_count);
672 if matches!(data_type, DataType::Integer | DataType::Float) {
673 assert_eq!(profile.invalid_count, Some(2 - null_count));
674 }
675 assert!(profile.patterns.is_some());
676 }
677 }
678 }
679
680 #[test]
681 fn exact_zero_statistics_survive_a_sample_without_parsed_values() {
682 for data_type in [DataType::Integer, DataType::Boolean] {
685 let samples = [String::new()];
686 let profile = build_column_profile(ColumnProfileInput {
687 name: "v".to_string(),
688 data_type: data_type.clone(),
689 total_count: 3,
690 null_count: 1,
691 unique_count: None,
692 unique_count_is_approximate: None,
693 unique_count_lower_bound: None,
694 sample_values: &samples,
695 text_lengths: None,
696 boolean_counts: Some((0, 2)),
697 skip_statistics: false,
698 skip_patterns: true,
699 locale: None,
700 exact_numeric: Some(ExactNumericAggregates {
701 min: 0.0,
702 max: 0.0,
703 mean: 0.0,
704 std_dev: 0.0,
705 variance: 0.0,
706 count: 2,
707 min_value: None,
708 max_value: None,
709 }),
710 exact_date_matches: None,
711 exact_dates: None,
712 decimal_values: false,
713 });
714
715 match profile.stats {
716 ColumnStats::Numeric(stats) => {
717 assert_eq!(stats.min, 0.0);
718 assert_eq!(stats.mean, 0.0);
719 assert_eq!(stats.is_approximate, Some(true));
720 assert_eq!(profile.invalid_count, Some(0));
721 }
722 ColumnStats::Boolean(stats) => {
723 assert_eq!(stats.true_ratio, 0.0);
724 assert_eq!(stats.false_count, 2);
725 }
726 other => panic!("expected measured zero statistics, got {other:?}"),
727 }
728 }
729 }
730
731 #[test]
732 fn an_overflowing_exact_spread_clears_the_sample_shape() {
733 let samples: Vec<String> = ["5", "0.5", "5", "7", "3"]
736 .iter()
737 .map(|value| value.to_string())
738 .collect();
739 let profile = build_column_profile(ColumnProfileInput {
740 name: "x".to_string(),
741 data_type: DataType::Float,
742 total_count: 20_000,
743 null_count: 0,
744 unique_count: None,
745 unique_count_is_approximate: None,
746 unique_count_lower_bound: None,
747 sample_values: &samples,
748 text_lengths: None,
749 boolean_counts: None,
750 skip_statistics: false,
751 skip_patterns: true,
752 locale: None,
753 exact_numeric: Some(ExactNumericAggregates {
754 min: -1e308,
755 max: 1e308,
756 mean: 2.9167,
757 std_dev: f64::NAN,
758 variance: f64::NAN,
759 count: 20_000,
760 min_value: None,
761 max_value: None,
762 }),
763 exact_date_matches: None,
764 exact_dates: None,
765 decimal_values: false,
766 });
767
768 let ColumnStats::Numeric(stats) = profile.stats else {
769 panic!("expected numeric statistics, got {:?}", profile.stats);
770 };
771 assert_eq!(stats.variance, None);
772 assert_eq!(stats.std_dev, None);
773 assert_eq!(stats.coefficient_of_variation, None);
774 assert_eq!(stats.skewness, None);
775 assert_eq!(stats.kurtosis, None);
776 assert_eq!(stats.median, Some(5.0));
778 assert_eq!(stats.is_approximate, Some(true));
779 }
780
781 #[test]
782 fn test_boolean_stats_with_counts() {
783 let samples = vec!["True".to_string(), "False".to_string(), "True".to_string()];
784 let profile = build_column_profile(ColumnProfileInput {
785 name: "flag".to_string(),
786 data_type: DataType::Boolean,
787 total_count: 3,
788 null_count: 0,
789 unique_count: Some(2),
790 unique_count_is_approximate: Some(false),
791 unique_count_lower_bound: None,
792 sample_values: &samples,
793 text_lengths: None,
794 boolean_counts: Some((2, 1)),
795 skip_statistics: false,
796 skip_patterns: false,
797 exact_numeric: None,
798 exact_date_matches: None,
799 exact_dates: None,
800 decimal_values: false,
801 locale: None,
802 });
803
804 match &profile.stats {
805 ColumnStats::Boolean(b) => {
806 assert_eq!(b.true_count, 2);
807 assert_eq!(b.false_count, 1);
808 assert!((b.true_ratio - 2.0 / 3.0).abs() < 0.001);
809 }
810 other => panic!("expected Boolean stats, got {:?}", other),
811 }
812 }
813
814 #[test]
815 fn test_boolean_stats_fallback_case_insensitive() {
816 let samples = vec![
817 "true".to_string(),
818 "FALSE".to_string(),
819 " True ".to_string(),
820 ];
821 let profile = build_column_profile(ColumnProfileInput {
822 name: "flag".to_string(),
823 data_type: DataType::Boolean,
824 total_count: 3,
825 null_count: 0,
826 unique_count: Some(2),
827 unique_count_is_approximate: Some(false),
828 unique_count_lower_bound: None,
829 sample_values: &samples,
830 text_lengths: None,
831 boolean_counts: None,
832 skip_statistics: false,
833 skip_patterns: false,
834 exact_numeric: None,
835 exact_date_matches: None,
836 exact_dates: None,
837 decimal_values: false,
838 locale: None,
839 });
840
841 match &profile.stats {
842 ColumnStats::Boolean(b) => {
843 assert_eq!(b.true_count, 2);
844 assert_eq!(b.false_count, 1);
845 assert!((b.true_ratio - 2.0 / 3.0).abs() < 0.001);
846 }
847 other => panic!("expected Boolean stats, got {:?}", other),
848 }
849 }
850
851 #[test]
852 fn test_skip_statistics() {
853 let samples = vec!["10".to_string(), "20".to_string(), "30".to_string()];
854 let profile = build_column_profile(ColumnProfileInput {
855 name: "num".to_string(),
856 data_type: DataType::Integer,
857 total_count: 3,
858 null_count: 0,
859 unique_count: Some(3),
860 unique_count_is_approximate: Some(false),
861 unique_count_lower_bound: None,
862 sample_values: &samples,
863 text_lengths: None,
864 boolean_counts: None,
865 skip_statistics: true,
866 skip_patterns: false,
867 exact_numeric: None,
868 exact_date_matches: None,
869 exact_dates: None,
870 decimal_values: false,
871 locale: None,
872 });
873
874 assert!(matches!(profile.stats, ColumnStats::None));
875 assert_eq!(profile.data_type, DataType::Integer);
876 }
877
878 #[test]
879 fn test_skip_patterns() {
880 let samples = vec!["hello".to_string(), "world".to_string()];
881 let profile = build_column_profile(ColumnProfileInput {
882 name: "text".to_string(),
883 data_type: DataType::String,
884 total_count: 2,
885 null_count: 0,
886 unique_count: Some(2),
887 unique_count_is_approximate: Some(false),
888 unique_count_lower_bound: None,
889 sample_values: &samples,
890 text_lengths: None,
891 boolean_counts: None,
892 skip_statistics: false,
893 skip_patterns: true,
894 exact_numeric: None,
895 exact_date_matches: None,
896 exact_dates: None,
897 decimal_values: false,
898 locale: None,
899 });
900
901 assert!(profile.patterns.is_none());
904 assert!(matches!(profile.stats, ColumnStats::Text(_)));
905 }
906
907 #[test]
908 fn test_scanned_without_matches_is_not_none() {
909 let samples = vec!["hello".to_string(), "world".to_string()];
913 let profile = build_column_profile(ColumnProfileInput {
914 name: "text".to_string(),
915 data_type: DataType::String,
916 total_count: 2,
917 null_count: 0,
918 unique_count: Some(2),
919 unique_count_is_approximate: Some(false),
920 unique_count_lower_bound: None,
921 sample_values: &samples,
922 text_lengths: None,
923 boolean_counts: None,
924 skip_statistics: false,
925 skip_patterns: false,
926 exact_numeric: None,
927 exact_date_matches: None,
928 exact_dates: None,
929 decimal_values: false,
930 locale: None,
931 });
932
933 assert!(profile.patterns.is_some_and(|p| p.is_empty()));
934 }
935
936 #[test]
937 fn test_all_packs_default() {
938 let samples = vec!["42".to_string(), "99".to_string()];
939 let profile = build_column_profile(ColumnProfileInput {
940 name: "val".to_string(),
941 data_type: DataType::Integer,
942 total_count: 2,
943 null_count: 0,
944 unique_count: Some(2),
945 unique_count_is_approximate: Some(false),
946 unique_count_lower_bound: None,
947 sample_values: &samples,
948 text_lengths: None,
949 boolean_counts: None,
950 skip_statistics: false,
951 skip_patterns: false,
952 exact_numeric: None,
953 exact_date_matches: None,
954 exact_dates: None,
955 decimal_values: false,
956 locale: None,
957 });
958
959 assert!(matches!(profile.stats, ColumnStats::Numeric(_)));
960 assert_eq!(profile.data_type, DataType::Integer);
961 }
962}