1use csv::ReaderBuilder;
2use std::io::Read;
3use std::path::Path;
4
5use dataprof_core::{
6 AnalysisOptions, ColumnProfile, CsvSourceMetadata, DataProfilerError, DataSource,
7 DelimiterDetection, ExecutionMetadata, FileFormat, TruncationReason,
8};
9use dataprof_runtime::{
10 ProfileReport, ReportAssembler, StreamingColumnCollection, profile_builder,
11};
12
13mod encoding;
14mod memmap_reader;
15mod quote_state;
16#[doc(hidden)]
17pub mod row_count;
18
19pub use encoding::{EncodingDiagnostic, diagnose_non_utf8};
20pub use memmap_reader::MemoryMappedCsvReader;
21pub use quote_state::{QuoteOutcome, QuoteTrackingReader, unterminated_quote_error};
22
23#[derive(Debug, Clone)]
28pub struct CsvParserConfig {
29 pub flexible: bool,
31 pub delimiter: Option<u8>,
33 pub quote_char: u8,
35 pub trim_whitespace: bool,
37 pub has_header: bool,
40 pub max_rows: Option<usize>,
42}
43
44impl Default for CsvParserConfig {
45 fn default() -> Self {
46 Self {
47 flexible: true,
48 delimiter: None,
49 quote_char: b'"',
50 trim_whitespace: false,
51 has_header: true,
52 max_rows: None,
53 }
54 }
55}
56
57impl CsvParserConfig {
58 pub fn strict() -> Self {
60 Self {
61 flexible: false,
62 ..Default::default()
63 }
64 }
65
66 pub fn with_delimiter(mut self, delimiter: u8) -> Self {
68 self.delimiter = Some(delimiter);
69 self
70 }
71
72 pub fn has_header(mut self, yes: bool) -> Self {
74 self.has_header = yes;
75 self
76 }
77
78 pub fn max_rows(mut self, max: Option<usize>) -> Self {
80 self.max_rows = max;
81 self
82 }
83}
84
85fn count_fields(line: &[u8], delimiter: u8, quote: u8) -> (usize, usize) {
98 let decodes_empty = |field: &[u8]| field.is_empty() || field == [quote, quote];
99 let mut fields = 1;
100 let mut trailing_empty = 0;
101 let mut field_start = 0;
102 let mut in_quotes = false;
103 let mut bytes = line.iter().copied().enumerate().peekable();
104 while let Some((index, byte)) = bytes.next() {
105 if byte == quote && in_quotes {
106 if bytes.peek().map(|&(_, next)| next) == Some(quote) {
107 bytes.next();
108 } else {
109 in_quotes = false;
110 }
111 } else if byte == quote && index == field_start {
112 in_quotes = true;
113 } else if byte == delimiter && !in_quotes {
114 trailing_empty = if decodes_empty(&line[field_start..index]) {
115 trailing_empty + 1
116 } else {
117 0
118 };
119 fields += 1;
120 field_start = index + 1;
121 }
122 }
123 trailing_empty = if decodes_empty(&line[field_start..]) {
124 trailing_empty + 1
125 } else {
126 0
127 };
128 (fields, trailing_empty)
129}
130
131fn agrees_with_header(fields: usize, trailing_empty: usize, header_fields: usize) -> bool {
135 fields == header_fields || (fields > header_fields && fields - trailing_empty <= header_fields)
136}
137
138fn split_sample_records(bytes: &[u8], delimiter: u8, quote: u8) -> (Vec<&[u8]>, bool) {
156 let mut records = Vec::new();
157 let mut in_quotes = false;
158 let mut start = 0usize;
159 let mut i = 0usize;
160 while i < bytes.len() {
161 match bytes[i] {
162 byte if byte == quote && in_quotes => {
163 if bytes.get(i + 1) == Some("e) {
164 i += 1; } else {
166 in_quotes = false;
167 }
168 }
169 byte if byte == quote
172 && (i == start || bytes[i - 1] == delimiter)
173 && !(i == start && matches!(byte, b'\r' | b'\n')) =>
174 {
175 in_quotes = true;
176 }
177 b'\n' if !in_quotes => {
178 let end = if i > start && bytes[i - 1] == b'\r' {
179 i - 1
180 } else {
181 i
182 };
183 records.push(&bytes[start..end]);
184 start = i + 1;
185 }
186 b'\r' if !in_quotes && bytes.get(i + 1) != Some(&b'\n') => {
187 records.push(&bytes[start..i]);
188 start = i + 1;
189 }
190 _ => {}
191 }
192 i += 1;
193 }
194 let last_unterminated = start < bytes.len();
195 if last_unterminated {
196 let mut end = bytes.len();
197 while end > start && bytes[end - 1] == b'\r' {
198 end -= 1;
199 }
200 records.push(&bytes[start..end]);
201 }
202 (records, last_unterminated)
203}
204
205const CANDIDATES: [u8; 4] = *b",;\t|";
207
208const DETECTION_SAMPLE_BYTES: usize = 4096;
210
211const MAX_HEADER_SAMPLE_BYTES: usize = 1 << 20;
213
214fn head_bytes(head: &[u8]) -> &[u8] {
220 head.strip_prefix(b"\xEF\xBB\xBF").unwrap_or(head)
221}
222
223fn record_is_blank(record: &[u8]) -> bool {
225 String::from_utf8_lossy(record).trim().is_empty()
226}
227
228fn header_is_complete(head: &[u8], quote: u8) -> bool {
232 let head = head_bytes(head);
233 CANDIDATES.iter().all(|&delimiter| {
234 let (records, last_unterminated) = split_sample_records(head, delimiter, quote);
235 records
236 .iter()
237 .position(|record| !record_is_blank(record))
238 .is_some_and(|index| index + 1 < records.len() || !last_unterminated)
239 })
240}
241
242fn read_detection_head<R: Read>(reader: &mut R, quote: u8) -> std::io::Result<(Vec<u8>, bool)> {
251 let mut head = Vec::new();
252 let mut target = DETECTION_SAMPLE_BYTES;
253 loop {
254 let wanted = (target + 1 - head.len()) as u64;
255 reader.by_ref().take(wanted).read_to_end(&mut head)?;
256 let cut = head.len() > target;
257 if !cut || target == MAX_HEADER_SAMPLE_BYTES || header_is_complete(&head, quote) {
258 return Ok((head, cut));
259 }
260 target = (target * 2).min(MAX_HEADER_SAMPLE_BYTES);
261 }
262}
263
264const HEADER_TOO_LONG: &str = "The first 1 MiB of the CSV source holds no complete header record, so its delimiter could not be detected.";
265const HEADER_TOO_LONG_HINT: &str = "Set the delimiter explicitly (csv_delimiter on Profiler or dataprof.profile) to read this file.";
266
267pub fn detect_delimiter_with_head<R: Read>(
275 reader: &mut R,
276) -> Result<(u8, Vec<u8>), DataProfilerError> {
277 let (detection, head) = detect_delimiter_detailed_with_head(reader)?;
278 Ok((detection_byte(&detection), head))
279}
280
281pub fn detect_delimiter_detailed_with_head<R: Read>(
286 reader: &mut R,
287) -> Result<(DelimiterDetection, Vec<u8>), DataProfilerError> {
288 detect_delimiter_with_quote(reader, b'"')
289}
290
291fn detect_delimiter_with_quote<R: Read>(
293 reader: &mut R,
294 quote: u8,
295) -> Result<(DelimiterDetection, Vec<u8>), DataProfilerError> {
296 let (head, cut) = read_detection_head(reader, quote)?;
297 match score_head(&head, cut, quote) {
298 Some(detection) => Ok((detection, head)),
299 None => Err(DataProfilerError::csv_structure(
300 HEADER_TOO_LONG,
301 HEADER_TOO_LONG_HINT,
302 )),
303 }
304}
305
306pub fn detect_delimiter_detailed<R: Read>(mut reader: R) -> std::io::Result<DelimiterDetection> {
309 let (head, cut) = read_detection_head(&mut reader, b'"')?;
310 score_head(&head, cut, b'"').ok_or_else(|| {
311 std::io::Error::new(
312 std::io::ErrorKind::InvalidData,
313 format!("{HEADER_TOO_LONG} {HEADER_TOO_LONG_HINT}"),
314 )
315 })
316}
317
318pub fn detection_byte(detection: &DelimiterDetection) -> u8 {
321 detection.delimiter.as_bytes()[0]
322}
323
324pub fn detect_delimiter<R: Read>(reader: R) -> std::io::Result<u8> {
347 detect_delimiter_detailed(reader).map(|detection| detection_byte(&detection))
348}
349
350fn score_head(head: &[u8], cut: bool, quote: u8) -> Option<DelimiterDetection> {
358 let head = head_bytes(head);
359 let mut scores: Vec<((bool, usize, usize, usize), u8)> = Vec::new();
365 let mut sampled: Vec<Vec<&[u8]>> = Vec::new();
367
368 for delimiter in CANDIDATES {
369 let (mut records, last_unterminated) = split_sample_records(head, delimiter, quote);
370 if cut && last_unterminated {
373 records.pop();
374 }
375 let scored: Vec<&[u8]> = records
379 .into_iter()
380 .filter(|record| !record_is_blank(record))
381 .take(5)
382 .collect();
383 let Some((header, rows)) = scored.split_first() else {
384 if cut {
388 continue;
389 }
390 return Some(DelimiterDetection::new(b',', 0, None));
391 };
392
393 let (header_fields, _) = count_fields(header, delimiter, quote);
394 let (mut agreeing, mut exact) = (1, 1);
395 for record in rows {
396 let (fields, trailing_empty) = count_fields(record, delimiter, quote);
397 if agrees_with_header(fields, trailing_empty, header_fields) {
398 agreeing += 1;
399 }
400 if fields == header_fields {
401 exact += 1;
402 }
403 }
404
405 scores.push((
406 (header_fields > 1, agreeing, exact, header_fields),
407 delimiter,
408 ));
409 sampled.push(scored);
410 }
411
412 let pick = |candidates: &mut dyn Iterator<Item = ((bool, usize, usize, usize), u8)>| {
414 candidates.fold(
415 None,
416 |best: Option<((bool, usize, usize, usize), u8)>, entry| match best {
417 Some(held) if entry.0 <= held.0 => Some(held),
418 _ => Some(entry),
419 },
420 )
421 };
422 let (score, delimiter) = pick(&mut scores.iter().copied())?;
423 let (splits, agreeing, _, header_fields) = score;
424 let runner_up = pick(
425 &mut scores.iter().copied().filter(|&(other, other_delimiter)| {
426 splits && other_delimiter != delimiter && other.0 && other.1 == agreeing
427 }),
428 )
429 .map(|(_, other_delimiter)| other_delimiter);
430 let unlisted = if splits {
431 None
432 } else {
433 scores
434 .iter()
435 .position(|&(_, candidate)| candidate == delimiter)
436 .and_then(|index| unlisted_delimiter(&sampled[index], quote))
437 };
438 Some(
439 DelimiterDetection::new(delimiter, header_fields, runner_up)
440 .with_unlisted_delimiter(unlisted),
441 )
442}
443
444const UNLISTED_DELIMITERS: &[u8] = b"!\"#$%&'()*+/:<=>?@[\\]^`{}~";
449
450fn unlisted_delimiter(records: &[&[u8]], quote: u8) -> Option<u8> {
458 let (header, rows) = records.split_first()?;
459 if rows.is_empty() {
460 return None;
461 }
462 UNLISTED_DELIMITERS.iter().copied().find(|&byte| {
463 if byte == quote {
464 return false;
465 }
466 let occurrences = |record: &[u8]| count_fields(record, byte, quote).0 - 1;
467 let in_header = occurrences(header);
468 in_header > 0 && rows.iter().all(|row| occurrences(row) == in_header)
469 })
470}
471
472pub fn detect_delimiter_from_path(path: &Path) -> std::io::Result<u8> {
474 let file = std::fs::File::open(path)?;
475 detect_delimiter(std::io::BufReader::new(file))
476}
477
478pub fn analyze_csv_from_reader<R: Read>(
486 reader: R,
487 config: &CsvParserConfig,
488) -> Result<
489 (
490 Vec<ColumnProfile>,
491 StreamingColumnCollection,
492 usize,
493 Vec<String>,
494 ),
495 DataProfilerError,
496> {
497 let scan = scan_csv(reader, config)?;
498 let profiles = profile_builder::profiles_from_streaming(&scan.column_stats, false, false, None);
499
500 Ok((
501 profiles,
502 scan.column_stats,
503 scan.rows_read,
504 scan.header_names,
505 ))
506}
507
508struct CsvScan {
510 column_stats: StreamingColumnCollection,
511 header_names: Vec<String>,
512 rows_read: usize,
513 ragged_rows: usize,
516 truncation: Option<TruncationReason>,
518 csv_metadata: CsvSourceMetadata,
520}
521
522fn scan_csv<R: Read>(reader: R, config: &CsvParserConfig) -> Result<CsvScan, DataProfilerError> {
523 let mut csv_builder = ReaderBuilder::new();
524 csv_builder.has_headers(config.has_header);
525 csv_builder.flexible(config.flexible);
526 csv_builder.quote(config.quote_char);
527 if config.trim_whitespace {
528 csv_builder.trim(csv::Trim::All);
529 }
530 let (boxed_reader, csv_metadata, delimiter): (Box<dyn std::io::Read + '_>, _, _) = match config
533 .delimiter
534 {
535 Some(delimiter) => (
536 Box::new(reader),
537 CsvSourceMetadata::configured(delimiter),
538 delimiter,
539 ),
540 None => {
541 let mut reader = reader;
542 let (detection, head) = detect_delimiter_with_quote(&mut reader, config.quote_char)?;
543 let delimiter = detection_byte(&detection);
544 (
545 Box::new(std::io::Cursor::new(head).chain(reader)),
546 CsvSourceMetadata::detected(detection),
547 delimiter,
548 )
549 }
550 };
551 csv_builder.delimiter(delimiter);
552
553 let mut csv_reader = csv_builder.from_reader(boxed_reader);
554
555 let header_names: Vec<String> = if config.has_header {
556 let headers = csv_reader.headers()?;
557 let names: Vec<String> = headers.iter().map(|header| header.to_string()).collect();
558 dataprof_core::validate_unique_column_names(&names, "CSV header")?;
561 names
562 } else {
563 let headers = csv_reader.headers()?;
564 (0..headers.len())
565 .map(|index| format!("column_{index}"))
566 .collect()
567 };
568
569 let mut column_stats = StreamingColumnCollection::new();
570 column_stats.init_columns(&header_names);
572 let mut rows_read = 0;
573 let mut ragged_rows = 0;
574 let mut truncation = None;
575
576 for result in csv_reader.records() {
577 if let Some(max_rows) = config.max_rows
578 && rows_read >= max_rows
579 {
580 truncation = Some(TruncationReason::MaxRows(max_rows as u64));
583 break;
584 }
585
586 let record = result?;
587 let mut values: Vec<String> = record.iter().map(|value| value.to_string()).collect();
588
589 let header_len = header_names.len();
590 if values.len() != header_len {
591 ragged_rows += 1;
592 }
593 if values.len() < header_len {
594 values.resize(header_len, String::new());
595 } else if values.len() > header_len {
596 values.truncate(header_len);
597 }
598
599 column_stats.process_record(&header_names, values);
600 rows_read += 1;
601 }
602
603 Ok(CsvScan {
604 column_stats,
605 header_names,
606 rows_read,
607 ragged_rows,
608 truncation,
609 csv_metadata,
610 })
611}
612
613pub fn analyze_csv_file(
620 file_path: &Path,
621 config: &CsvParserConfig,
622) -> Result<ProfileReport, DataProfilerError> {
623 let metadata = std::fs::metadata(file_path).map_err(|error| map_io_error(file_path, error))?;
624 dataprof_core::binary_input::reject_binary_input(file_path)?;
625 let start = std::time::Instant::now();
626
627 let file = std::fs::File::open(file_path).map_err(|error| map_io_error(file_path, error))?;
628 let buf_reader = std::io::BufReader::new(file);
629
630 let scan = scan_csv(buf_reader, config)?;
631 let column_stats = &scan.column_stats;
632 let column_profiles =
633 profile_builder::profiles_from_streaming(column_stats, false, false, None);
634
635 let file_source = DataSource::File {
636 path: file_path.display().to_string(),
637 format: FileFormat::Csv,
638 size_bytes: metadata.len(),
639 modified_at: None,
640 parquet_metadata: None,
641 csv_metadata: Some(scan.csv_metadata.clone()),
642 };
643
644 let sample_columns = profile_builder::quality_check_samples(column_stats);
645 let scan_time_ms = start.elapsed().as_millis();
646 let num_columns = column_profiles.len();
647
648 let mut execution = ExecutionMetadata::new(scan.rows_read, num_columns, scan_time_ms)
649 .with_engine("csv")
650 .with_ragged_row_count(scan.ragged_rows);
651 if let Some(reason) = scan.truncation {
652 execution = execution.with_truncation(reason);
653 }
654
655 Ok(ReportAssembler::new(file_source, execution)
658 .with_analysis_options(&AnalysisOptions::default())
659 .columns(column_profiles)
660 .with_quality_data(sample_columns)
661 .with_row_duplicates(column_stats.row_duplicate_summary())
662 .with_row_completeness(column_stats.row_completeness_summary())
663 .build())
664}
665
666fn map_io_error(file_path: &Path, error: std::io::Error) -> DataProfilerError {
667 if error.kind() == std::io::ErrorKind::NotFound {
668 DataProfilerError::FileNotFound {
669 path: file_path.display().to_string(),
670 }
671 } else {
672 DataProfilerError::from(error)
673 }
674}
675
676#[cfg(test)]
677mod tests {
678 use super::*;
679 use dataprof_core::DataType;
680 use std::io::{Cursor, Write};
681 use tempfile::NamedTempFile;
682
683 fn write_csv(content: &str) -> NamedTempFile {
684 let mut file = NamedTempFile::new().unwrap();
685 write!(file, "{}", content).unwrap();
686 file.flush().unwrap();
687 file
688 }
689
690 #[test]
691 fn test_analyze_csv_from_reader_basic() {
692 let data = b"name,age\nAlice,25\nBob,30\n";
693 let cursor = Cursor::new(data.as_ref());
694 let config = CsvParserConfig::default();
695
696 let (profiles, _stats, rows, _) = analyze_csv_from_reader(cursor, &config).unwrap();
697 assert_eq!(rows, 2);
698 assert_eq!(profiles.len(), 2);
699
700 let age = profiles
701 .iter()
702 .find(|profile| profile.name == "age")
703 .unwrap();
704 assert_eq!(age.total_count, 2);
705 assert_eq!(age.data_type, DataType::Integer);
706 }
707
708 #[test]
709 fn test_analyze_csv_from_reader_with_max_rows() {
710 let data = b"val\n1\n2\n3\n4\n5\n";
711 let cursor = Cursor::new(data.as_ref());
712 let config = CsvParserConfig::default().max_rows(Some(3));
713
714 let (_profiles, _stats, rows, _) = analyze_csv_from_reader(cursor, &config).unwrap();
715 assert_eq!(rows, 3);
716 }
717
718 #[test]
719 fn test_analyze_csv_from_reader_strict_rejects_ragged() {
720 let data = b"a,b\n1,2\n3,4,5\n";
721 let cursor = Cursor::new(data.as_ref());
722 let config = CsvParserConfig::strict();
723
724 assert!(analyze_csv_from_reader(cursor, &config).is_err());
725 }
726
727 #[test]
728 fn test_analyze_csv_from_reader_flexible_handles_ragged() {
729 let data = b"a,b\n1,2\n3,4,5\n6\n";
730 let cursor = Cursor::new(data.as_ref());
731 let config = CsvParserConfig::default();
732
733 let (profiles, _stats, rows, _) = analyze_csv_from_reader(cursor, &config).unwrap();
734 assert_eq!(rows, 3);
735 assert!(!profiles.is_empty());
736 }
737
738 #[test]
739 fn test_analyze_csv_from_reader_custom_delimiter() {
740 let data = b"a;b\n1;2\n3;4\n";
741 let cursor = Cursor::new(data.as_ref());
742 let config = CsvParserConfig::default().with_delimiter(b';');
743
744 let (profiles, _stats, rows, _) = analyze_csv_from_reader(cursor, &config).unwrap();
745 assert_eq!(rows, 2);
746 assert_eq!(profiles.len(), 2);
747 }
748
749 #[test]
750 fn test_analyze_csv_file_returns_quality_report() {
751 let csv = write_csv("x,y\n1,a\n2,b\n3,c\n");
752 let config = CsvParserConfig::default();
753 let report = analyze_csv_file(csv.path(), &config).unwrap();
754
755 assert_eq!(report.column_profiles.len(), 2);
756 assert_eq!(report.execution.rows_processed, 3);
757 assert!(report.quality_score().unwrap() >= 0.0);
758 assert!(report.quality_score().unwrap() <= 100.0);
759 }
760
761 #[test]
762 fn test_csv_auto_detects_delimiter_from_reader() {
763 let data = b"name;age\nAlice;25\nBob;30\n";
764 let cursor = Cursor::new(data.as_ref());
765 let config = CsvParserConfig::default();
766
767 let (profiles, _stats, rows, _) = analyze_csv_from_reader(cursor, &config).unwrap();
768 assert_eq!(rows, 2);
769 assert_eq!(profiles.len(), 2);
770
771 let age = profiles
772 .iter()
773 .find(|profile| profile.name == "age")
774 .unwrap();
775 assert_eq!(age.total_count, 2);
776 }
777
778 #[test]
779 fn test_analyze_csv_basic() {
780 let csv = write_csv("name,age\nAlice,25\nBob,30\n");
781 let config = CsvParserConfig::default();
782 let report = analyze_csv_file(csv.path(), &config).unwrap();
783 let profiles = &report.column_profiles;
784 assert_eq!(profiles.len(), 2);
785
786 let names: Vec<&str> = profiles
787 .iter()
788 .map(|profile| profile.name.as_str())
789 .collect();
790 assert!(names.contains(&"name"));
791 assert!(names.contains(&"age"));
792
793 let age = profiles
794 .iter()
795 .find(|profile| profile.name == "age")
796 .unwrap();
797 assert_eq!(age.total_count, 2);
798 assert_eq!(age.null_count, 0);
799 }
800
801 #[test]
802 fn test_analyze_csv_with_nulls() {
803 let csv = write_csv("name,age,email\nAlice,25,a@b.com\nBob,,\nCharlie,30,c@d.com\n");
804 let config = CsvParserConfig::default();
805 let report = analyze_csv_file(csv.path(), &config).unwrap();
806 let profiles = &report.column_profiles;
807
808 let age = profiles
809 .iter()
810 .find(|profile| profile.name == "age")
811 .unwrap();
812 assert_eq!(age.total_count, 3);
813 assert_eq!(age.null_count, 1);
814
815 let email = profiles
816 .iter()
817 .find(|profile| profile.name == "email")
818 .unwrap();
819 assert_eq!(email.null_count, 1);
820 }
821
822 #[test]
823 fn test_analyze_csv_file_returns_quality_report_legacy() {
824 let csv = write_csv("x,y\n1,a\n2,b\n3,c\n");
825 let config = CsvParserConfig::default();
826 let report = analyze_csv_file(csv.path(), &config).unwrap();
827
828 assert_eq!(report.column_profiles.len(), 2);
829 assert_eq!(report.execution.rows_processed, 3);
830 assert_eq!(report.execution.columns_detected, 2);
831 assert!(report.quality_score().unwrap() >= 0.0);
832 assert!(report.quality_score().unwrap() <= 100.0);
833 }
834
835 #[test]
836 fn test_analyze_csv_file_header_only_preserves_declared_columns() {
837 let csv = write_csv("name,age\n");
838 let config = CsvParserConfig::default();
839 let report = analyze_csv_file(csv.path(), &config).unwrap();
840
841 assert_eq!(
842 report
843 .column_profiles
844 .iter()
845 .map(|column| (column.name.as_str(), column.data_type.clone()))
846 .collect::<Vec<_>>(),
847 vec![("name", DataType::String), ("age", DataType::String)]
848 );
849 assert_eq!(report.execution.rows_processed, 0);
850 assert!(report.quality.is_some());
851 }
852
853 #[test]
854 fn test_analyze_csv_file_empty_file_has_no_columns() {
855 let csv = write_csv("");
856 let config = CsvParserConfig::default();
857 let report = analyze_csv_file(csv.path(), &config).unwrap();
858
859 assert!(report.column_profiles.is_empty());
860 assert_eq!(report.execution.rows_processed, 0);
861 assert_eq!(report.execution.columns_detected, 0);
862 assert_eq!(
863 report.quality_status,
864 dataprof_runtime::QualityAnalysisStatus::Computed
865 );
866 assert!(report.quality.is_some());
867 assert!(report.quality_score().is_none());
868 }
869
870 #[test]
871 fn test_analyze_csv_with_max_rows_small_file() {
872 let csv = write_csv("val\n1\n2\n3\n4\n5\n");
873 let config = CsvParserConfig::default();
874 let report = analyze_csv_file(csv.path(), &config).unwrap();
875
876 assert_eq!(report.execution.rows_processed, 5);
877 assert_eq!(report.execution.columns_detected, 1);
878 assert!((report.execution.sampling_ratio.unwrap_or(1.0) - 1.0).abs() < 0.01);
879 }
880
881 #[test]
882 fn a_row_cap_that_leaves_rows_unread_is_a_truncation() {
883 let csv = write_csv("val\n1\n2\n3\n4\n5\n");
884 let config = CsvParserConfig::default().max_rows(Some(2));
885 let report = analyze_csv_file(csv.path(), &config).unwrap();
886
887 assert_eq!(report.execution.rows_processed, 2);
888 assert!(!report.execution.source_exhausted);
889 assert_eq!(
890 report.execution.truncation_reason,
891 Some(TruncationReason::MaxRows(2))
892 );
893 }
894
895 #[test]
896 fn a_row_cap_at_the_last_row_is_a_complete_scan() {
897 let csv = write_csv("val\n1\n2\n");
898 let config = CsvParserConfig::default().max_rows(Some(2));
899 let report = analyze_csv_file(csv.path(), &config).unwrap();
900
901 assert_eq!(report.execution.rows_processed, 2);
902 assert!(report.execution.source_exhausted);
903 assert_eq!(report.execution.truncation_reason, None);
904 }
905
906 #[test]
907 fn short_and_long_rows_are_counted_as_ragged() {
908 let csv = write_csv("a,b,c\n1,2,3\n4,5\n6,7,8,9\n10,11,12\n");
909 let report = analyze_csv_file(csv.path(), &CsvParserConfig::default()).unwrap();
910
911 assert_eq!(report.execution.rows_processed, 4);
912 assert_eq!(report.execution.ragged_row_count, 2);
913 }
914
915 #[test]
916 fn a_ragged_row_past_the_row_cap_is_not_counted() {
917 let csv = write_csv("a,b,c\n1,2,3\n4,5\n");
918 let config = CsvParserConfig::default().max_rows(Some(1));
919 let report = analyze_csv_file(csv.path(), &config).unwrap();
920
921 assert_eq!(report.execution.ragged_row_count, 0);
922 assert_eq!(
923 report.execution.truncation_reason,
924 Some(TruncationReason::MaxRows(1))
925 );
926 }
927
928 #[test]
929 fn test_analyze_csv_numeric_types_detected() {
930 let csv = write_csv("int_col,float_col,str_col\n1,1.5,hello\n2,2.5,world\n3,3.5,foo\n");
931 let config = CsvParserConfig::default();
932 let report = analyze_csv_file(csv.path(), &config).unwrap();
933 let profiles = &report.column_profiles;
934
935 let int_col = profiles
936 .iter()
937 .find(|profile| profile.name == "int_col")
938 .unwrap();
939 assert!(matches!(int_col.data_type, DataType::Integer));
940
941 let float_col = profiles
942 .iter()
943 .find(|profile| profile.name == "float_col")
944 .unwrap();
945 assert!(matches!(float_col.data_type, DataType::Float));
946
947 let str_col = profiles
948 .iter()
949 .find(|profile| profile.name == "str_col")
950 .unwrap();
951 assert!(matches!(str_col.data_type, DataType::String));
952 }
953
954 #[test]
955 fn test_analyze_csv_strict_rejects_ragged_rows() {
956 let csv = write_csv("a,b\n1,2\n3,4,5\n");
957 let config = CsvParserConfig::strict();
958 assert!(analyze_csv_file(csv.path(), &config).is_err());
959 }
960
961 #[test]
962 fn test_analyze_csv_file_handles_ragged_rows() {
963 let csv = write_csv("a,b\n1,2\n3,4,5\n6\n");
964 let config = CsvParserConfig::default();
965 let report = analyze_csv_file(csv.path(), &config).unwrap();
966 assert!(!report.column_profiles.is_empty());
967 }
968
969 #[test]
970 fn test_analyze_csv_with_quality_metrics() {
971 let csv = write_csv(
972 "name,age\n\
973 Alice,25\nBob,\nCharlie,30\nDave,\nEve,28\n\
974 Frank,\nGrace,35\nHeidi,40\nIvan,\nJudy,22\n",
975 );
976 let config = CsvParserConfig::default();
977 let report = analyze_csv_file(csv.path(), &config).unwrap();
978
979 let age = report
980 .column_profiles
981 .iter()
982 .find(|profile| profile.name == "age")
983 .unwrap();
984 assert_eq!(age.null_count, 4);
985 assert_eq!(age.total_count, 10);
986 assert!(report.quality_score().unwrap() >= 0.0);
987 }
988
989 #[test]
993 fn detection_records_a_tie_and_an_unlisted_delimiter_only_when_present() {
994 type Case<'a> = (
996 &'a str,
997 &'a [u8],
998 &'a str,
999 usize,
1000 Option<&'a str>,
1001 Option<&'a str>,
1002 );
1003 let cases: [Case; 9] = [
1004 (
1005 "tie",
1006 b"id;amount,EUR\n1;10,50;\n2;20,75;\n",
1007 ",",
1008 2,
1009 Some(";"),
1010 None,
1011 ),
1012 ("clear comma", b"a,b,c\n1,2,3\n4,5,6\n", ",", 3, None, None),
1013 (
1014 "clear semicolon",
1015 b"a;b\n1,5;2\n3,5;4\n",
1016 ";",
1017 2,
1018 None,
1019 None,
1020 ),
1021 (
1022 "unlisted caret",
1023 b"id^amount\n1^10\n2^20\n",
1024 ",",
1025 1,
1026 None,
1027 Some("^"),
1028 ),
1029 ("one column", b"value\n1\n2\n", ",", 1, None, None),
1030 (
1031 "dot inside values",
1032 b"name.first\njohn.doe\njane.roe\n",
1033 ",",
1034 1,
1035 None,
1036 None,
1037 ),
1038 ("header lacks it", b"ratio\n1/2\n3/4\n", ",", 1, None, None),
1039 ("a record lacks it", b"a:b\n1:2\n3\n", ",", 1, None, None),
1040 ("header only", b"a^b\n", ",", 1, None, None),
1041 ];
1042 let mut wrong = Vec::new();
1043 for (case, data, delimiter, header_fields, runner_up, unlisted) in cases {
1044 let detection = detect_delimiter_detailed(Cursor::new(data)).unwrap();
1045 let seen = (
1046 detection.delimiter.as_str(),
1047 detection.header_fields,
1048 detection.runner_up.as_deref(),
1049 detection.unlisted_delimiter.as_deref(),
1050 );
1051 if seen != (delimiter, header_fields, runner_up, unlisted) {
1052 wrong.push(format!("{case}: {seen:?}"));
1053 }
1054 }
1055 assert!(wrong.is_empty(), "{}", wrong.join("\n"));
1056 }
1057
1058 #[test]
1059 fn test_detect_delimiter_comma() {
1060 let data = b"name,age,city\nAlice,25,NYC\nBob,30,London\n";
1061 assert_eq!(detect_delimiter(Cursor::new(data.as_ref())).unwrap(), b',');
1062 }
1063
1064 #[test]
1065 fn test_detect_delimiter_semicolon() {
1066 let data = b"name;age;city\nAlice;25;NYC\nBob;30;London\n";
1067 assert_eq!(detect_delimiter(Cursor::new(data.as_ref())).unwrap(), b';');
1068 }
1069
1070 #[test]
1071 fn test_detect_delimiter_pipe() {
1072 let data = b"name|age|city\nAlice|25|NYC\nBob|30|London\n";
1073 assert_eq!(detect_delimiter(Cursor::new(data.as_ref())).unwrap(), b'|');
1074 }
1075
1076 #[test]
1077 fn test_detect_delimiter_tab() {
1078 let data = b"name\tage\tcity\nAlice\t25\tNYC\nBob\t30\tLondon\n";
1079 assert_eq!(detect_delimiter(Cursor::new(data.as_ref())).unwrap(), b'\t');
1080 }
1081
1082 #[test]
1083 fn test_detect_delimiter_from_path_semicolon() {
1084 let csv = write_csv("id;name;salary\n1;Alice;50000\n2;Bob;60000\n");
1085 assert_eq!(detect_delimiter_from_path(csv.path()).unwrap(), b';');
1086 }
1087
1088 #[test]
1089 fn test_detect_delimiter_blank_records_do_not_outvote_header() {
1090 let data = b"name,age\n\n\n";
1095 assert_eq!(detect_delimiter(Cursor::new(data.as_ref())).unwrap(), b',');
1096
1097 let semicolon = b"name;age\n\n";
1098 assert_eq!(
1099 detect_delimiter(Cursor::new(semicolon.as_ref())).unwrap(),
1100 b';'
1101 );
1102 }
1103
1104 #[test]
1105 fn test_detect_delimiter_quoted_fields() {
1106 let data = b"name;desc;val\n\"hello;world\";foo;1\nbar;baz;2\n";
1107 assert_eq!(detect_delimiter(Cursor::new(data.as_ref())).unwrap(), b';');
1108 }
1109
1110 #[test]
1111 fn test_detect_delimiter_quoted_embedded_newline() {
1112 let data = b"id,comment\n1,\"line one\nline two\"\n2,\"normal\"\n3,\"has \"\"quotes\"\" inside\"\n";
1117 assert_eq!(detect_delimiter(Cursor::new(data.as_ref())).unwrap(), b',');
1118 }
1119
1120 #[test]
1121 fn test_detect_delimiter_absent_candidate_never_beats_real_one() {
1122 let data = b"a,b,c\n1,2,3\nragged line without delimiters\n4,5,6\n";
1125 assert_eq!(detect_delimiter(Cursor::new(data.as_ref())).unwrap(), b',');
1126 }
1127
1128 #[test]
1129 fn test_detect_delimiter_single_column_falls_back_to_comma() {
1130 let data = b"value\n1\n2\n3\n";
1131 assert_eq!(detect_delimiter(Cursor::new(data.as_ref())).unwrap(), b',');
1132 }
1133
1134 #[test]
1135 fn test_detect_delimiter_two_column_semicolon_with_commas_in_text() {
1136 let data = b"id;note\n1;hello, world\n2;foo, bar, baz\n3;plain\n";
1138 assert_eq!(detect_delimiter(Cursor::new(data.as_ref())).unwrap(), b';');
1139 }
1140
1141 #[test]
1142 fn test_detect_delimiter_never_splits_rows_the_header_does_not() {
1143 let data = b"key\n123|456\n123|789\n";
1146 assert_eq!(detect_delimiter(Cursor::new(data.as_ref())).unwrap(), b',');
1147 }
1148
1149 #[test]
1150 fn test_detect_delimiter_header_split_beats_uniform_rows() {
1151 let data = b"a,b\n1,2,\n3,4,\n";
1154 assert_eq!(detect_delimiter(Cursor::new(data.as_ref())).unwrap(), b',');
1155 }
1156
1157 #[test]
1158 fn test_detect_delimiter_trailing_empty_fields_still_agree() {
1159 let data = b"id;amount,EUR\n1;10,50;\n2;20;\n";
1163 assert_eq!(detect_delimiter(Cursor::new(data.as_ref())).unwrap(), b';');
1164 }
1165
1166 #[test]
1167 fn test_detect_delimiter_quoted_empty_trailing_field_still_agrees() {
1168 let data = b"id;amount,EUR\n1;10,50;\"\"\n2;20;\"\"\n";
1171 assert_eq!(detect_delimiter(Cursor::new(data.as_ref())).unwrap(), b';');
1172 }
1173
1174 #[test]
1175 fn test_detect_delimiter_exact_width_beats_cut_trailing_fields() {
1176 let data = b"id,part;value\nx;y,z,\nu;v,w,\n";
1180 assert_eq!(detect_delimiter(Cursor::new(data.as_ref())).unwrap(), b';');
1181 }
1182
1183 #[test]
1184 fn test_detect_delimiter_skips_a_utf8_bom() {
1185 let data = b"\xEF\xBB\xBF\nid;value\n1;2\n3;4\n";
1188 assert_eq!(detect_delimiter(Cursor::new(data.as_ref())).unwrap(), b';');
1189 let no_blank = b"\xEF\xBB\xBFid;value\n1;2\n";
1190 assert_eq!(
1191 detect_delimiter(Cursor::new(no_blank.as_ref())).unwrap(),
1192 b';'
1193 );
1194 }
1195
1196 #[test]
1197 fn test_detect_delimiter_lone_cr_ends_a_record() {
1198 let data = b"key\r123|456\r123|789\r";
1201 assert_eq!(detect_delimiter(Cursor::new(data.as_ref())).unwrap(), b',');
1202 let semicolon = b"id;value\r1;10\r2;20\r";
1203 assert_eq!(
1204 detect_delimiter(Cursor::new(semicolon.as_ref())).unwrap(),
1205 b';'
1206 );
1207 }
1208
1209 #[test]
1210 fn test_count_fields_reports_trailing_empty_fields() {
1211 assert_eq!(count_fields(b"1;10,50;", b';', b'"'), (3, 1));
1212 assert_eq!(count_fields(b"a;;", b';', b'"'), (3, 2));
1213 assert_eq!(count_fields(b"a;\"\"", b';', b'"'), (2, 1));
1214 assert_eq!(count_fields(b"a;\"\"\"\"", b';', b'"'), (2, 0));
1215 assert_eq!(count_fields(b"a; ", b';', b'"'), (2, 0));
1216 assert_eq!(count_fields(b"a;b", b';', b'"'), (2, 0));
1217 assert_eq!(count_fields(b"\"x;y\";", b';', b'"'), (2, 1));
1218 }
1219
1220 #[test]
1221 fn test_split_sample_records_quote_aware() {
1222 let (records, last_unterminated) =
1223 split_sample_records(b"a,b\r\n1,\"x\ny\"\n2,z\n", b',', b'"');
1224 assert_eq!(records, ["a,b", "1,\"x\ny\"", "2,z"].map(str::as_bytes));
1225 assert!(!last_unterminated);
1226
1227 let (records, last_unterminated) =
1228 split_sample_records(b"a,b\n1,\"cut mid-fie", b',', b'"');
1229 assert_eq!(records, ["a,b", "1,\"cut mid-fie"].map(str::as_bytes));
1230 assert!(last_unterminated);
1231
1232 let (records, last_unterminated) =
1233 split_sample_records(b"a,b\r1,\"x\ry\"\r2,z\r", b',', b'"');
1234 assert_eq!(records, ["a,b", "1,\"x\ry\"", "2,z"].map(str::as_bytes));
1235 assert!(!last_unterminated);
1236
1237 let (records, _) = split_sample_records(b"a\rb\r\nc\n", b',', b'"');
1238 assert_eq!(records, ["a", "b", "c"].map(str::as_bytes));
1239 }
1240
1241 #[test]
1242 fn test_detect_delimiter_full_sample_ending_on_record_boundary() {
1243 let mut data = Vec::from(&b"a;b;c\n"[..]);
1246 while data.len() < 4090 {
1247 data.extend_from_slice(b"1;2;3\n");
1248 }
1249 while data.len() < 4095 {
1250 data.push(b'#');
1251 }
1252 data.push(b'\n');
1253 assert_eq!(data.len(), 4096);
1254 assert_eq!(
1255 detect_delimiter(Cursor::new(data.as_slice())).unwrap(),
1256 b';'
1257 );
1258 }
1259
1260 #[test]
1261 fn test_detect_delimiter_reads_past_the_sample_to_end_the_header() {
1262 let issue = format!("\"{}\";value\n1;10\n2;20\n", "x".repeat(4094));
1265 assert_eq!(
1266 detect_delimiter(Cursor::new(issue.as_bytes())).unwrap(),
1267 b';'
1268 );
1269
1270 let mut exact = "a".repeat(4093).into_bytes();
1273 exact.extend_from_slice(b";b\n1;2\n");
1274 let mut reader = Cursor::new(exact.as_slice());
1275 let (delimiter, head) = detect_delimiter_with_head(&mut reader).unwrap();
1276 assert_eq!((delimiter, head.len()), (b';', 4097));
1277 }
1278
1279 #[test]
1280 fn test_detect_delimiter_with_head_returns_every_byte_it_read() {
1281 let data = format!("{};value\n1;2\n", "a".repeat(20_000));
1282 let mut reader = Cursor::new(data.as_bytes());
1283 let (delimiter, head) = detect_delimiter_with_head(&mut reader).unwrap();
1284 assert_eq!(delimiter, b';');
1285 assert_eq!(head.len(), 32_768.min(data.len()));
1286 let mut rest = Vec::new();
1287 reader.read_to_end(&mut rest).unwrap();
1288 assert_eq!([head, rest].concat(), data.as_bytes());
1289 }
1290
1291 #[test]
1292 fn test_detect_delimiter_refuses_a_header_past_the_bound() {
1293 let data = format!("{};value\n1;2\n", "h".repeat(MAX_HEADER_SAMPLE_BYTES));
1294 let error = detect_delimiter(Cursor::new(data.as_bytes())).unwrap_err();
1295 assert_eq!(error.kind(), std::io::ErrorKind::InvalidData);
1296 assert!(error.to_string().contains("csv_delimiter"), "{error}");
1297 let error = detect_delimiter_with_head(&mut Cursor::new(data.as_bytes())).unwrap_err();
1298 assert!(matches!(error, DataProfilerError::CsvParsingError { .. }));
1299
1300 let short = "h".repeat(MAX_HEADER_SAMPLE_BYTES);
1302 assert_eq!(
1303 detect_delimiter(Cursor::new(short.as_bytes())).unwrap(),
1304 b','
1305 );
1306 }
1307
1308 #[test]
1309 fn test_detect_delimiter_quote_rule_is_per_candidate() {
1310 for (header, row, expected) in [
1314 (&b"id,height;\"in\n"[..], &b"1,5\n"[..], b','),
1315 (&b"id;a;b,\"c\n"[..], &b"1;2;3\n"[..], b';'),
1316 ] {
1317 let mut data = header.to_vec();
1318 while data.len() <= MAX_HEADER_SAMPLE_BYTES {
1319 data.extend_from_slice(row);
1320 }
1321 assert_eq!(
1322 detect_delimiter(Cursor::new(data.as_slice())).unwrap(),
1323 expected,
1324 "{}",
1325 String::from_utf8_lossy(header)
1326 );
1327 }
1328 }
1329
1330 #[test]
1331 fn test_detect_delimiter_refuses_a_head_of_blank_records() {
1332 let mut data = vec![b'\n'; MAX_HEADER_SAMPLE_BYTES + 1];
1335 data.extend_from_slice(b"id;value\n1;2\n");
1336 assert!(detect_delimiter(Cursor::new(data.as_slice())).is_err());
1337
1338 let blank = vec![b'\n'; 10];
1340 assert_eq!(
1341 detect_delimiter(Cursor::new(blank.as_slice())).unwrap(),
1342 b','
1343 );
1344 }
1345
1346 #[test]
1347 fn test_detect_delimiter_mid_field_quote_is_literal() {
1348 let mut data = b"id;height\"in;note\n".to_vec();
1352 while data.len() <= MAX_HEADER_SAMPLE_BYTES {
1353 data.extend_from_slice(b"1;5;x\n");
1354 }
1355 assert_eq!(
1356 detect_delimiter(Cursor::new(data.as_slice())).unwrap(),
1357 b';'
1358 );
1359
1360 let (records, last_unterminated) =
1361 split_sample_records(b"a;b\"c;d\n1;\"x\ny\";2\n", b';', b'"');
1362 assert_eq!(records, ["a;b\"c;d", "1;\"x\ny\";2"].map(str::as_bytes));
1363 assert!(!last_unterminated);
1364
1365 assert_eq!(count_fields(b"id\"x;value", b';', b'"'), (2, 0));
1368 assert_eq!(count_fields(b"\"a;\"\"b\";c", b';', b'"'), (2, 0));
1369 assert_eq!(
1370 detect_delimiter(Cursor::new(&b"id\"x;value\n1;2\n3;4\n"[..])).unwrap(),
1371 b';'
1372 );
1373 }
1374}