1use csv::ReaderBuilder;
2use std::io::Read;
3use std::path::Path;
4
5use dataprof_core::{
6 AnalysisOptions, ColumnProfile, DataProfilerError, DataSource, ExecutionMetadata, FileFormat,
7 TruncationReason,
8};
9use dataprof_runtime::{
10 ProfileReport, ReportAssembler, StreamingColumnCollection, profile_builder,
11};
12
13mod encoding;
14mod memmap_reader;
15mod quote_state;
16#[doc(hidden)]
17pub mod row_count;
18
19pub use encoding::{EncodingDiagnostic, diagnose_non_utf8};
20pub use memmap_reader::MemoryMappedCsvReader;
21pub use quote_state::{QuoteOutcome, QuoteTrackingReader, unterminated_quote_error};
22
23#[derive(Debug, Clone)]
28pub struct CsvParserConfig {
29 pub flexible: bool,
31 pub delimiter: Option<u8>,
33 pub quote_char: u8,
35 pub trim_whitespace: bool,
37 pub has_header: bool,
40 pub max_rows: Option<usize>,
42}
43
44impl Default for CsvParserConfig {
45 fn default() -> Self {
46 Self {
47 flexible: true,
48 delimiter: None,
49 quote_char: b'"',
50 trim_whitespace: false,
51 has_header: true,
52 max_rows: None,
53 }
54 }
55}
56
57impl CsvParserConfig {
58 pub fn strict() -> Self {
60 Self {
61 flexible: false,
62 ..Default::default()
63 }
64 }
65
66 pub fn with_delimiter(mut self, delimiter: u8) -> Self {
68 self.delimiter = Some(delimiter);
69 self
70 }
71
72 pub fn has_header(mut self, yes: bool) -> Self {
74 self.has_header = yes;
75 self
76 }
77
78 pub fn max_rows(mut self, max: Option<usize>) -> Self {
80 self.max_rows = max;
81 self
82 }
83}
84
85fn count_fields(line: &str, delimiter: char) -> (usize, usize) {
94 let decodes_empty = |field: &str| field.is_empty() || field == "\"\"";
95 let mut fields = 1;
96 let mut trailing_empty = 0;
97 let mut field_start = 0;
98 let mut in_quotes = false;
99 let mut chars = line.char_indices().peekable();
100 while let Some((index, ch)) = chars.next() {
101 if ch == '"' {
102 if chars.peek().map(|&(_, next)| next) == Some('"') {
103 chars.next();
104 } else {
105 in_quotes = !in_quotes;
106 }
107 } else if ch == delimiter && !in_quotes {
108 trailing_empty = if decodes_empty(&line[field_start..index]) {
109 trailing_empty + 1
110 } else {
111 0
112 };
113 fields += 1;
114 field_start = index + ch.len_utf8();
115 }
116 }
117 trailing_empty = if decodes_empty(&line[field_start..]) {
118 trailing_empty + 1
119 } else {
120 0
121 };
122 (fields, trailing_empty)
123}
124
125fn agrees_with_header(fields: usize, trailing_empty: usize, header_fields: usize) -> bool {
129 fields == header_fields || (fields > header_fields && fields - trailing_empty <= header_fields)
130}
131
132fn split_sample_records(text: &str) -> (Vec<&str>, bool) {
143 let bytes = text.as_bytes();
144 let mut records = Vec::new();
145 let mut in_quotes = false;
146 let mut start = 0usize;
147 let mut i = 0usize;
148 while i < bytes.len() {
149 match bytes[i] {
150 b'"' => {
151 if in_quotes && bytes.get(i + 1) == Some(&b'"') {
152 i += 1; } else {
154 in_quotes = !in_quotes;
155 }
156 }
157 b'\n' if !in_quotes => {
158 let end = if i > start && bytes[i - 1] == b'\r' {
159 i - 1
160 } else {
161 i
162 };
163 records.push(&text[start..end]);
164 start = i + 1;
165 }
166 b'\r' if !in_quotes && bytes.get(i + 1) != Some(&b'\n') => {
167 records.push(&text[start..i]);
168 start = i + 1;
169 }
170 _ => {}
171 }
172 i += 1;
173 }
174 let last_unterminated = start < text.len();
175 if last_unterminated {
176 records.push(text[start..].trim_end_matches('\r'));
177 }
178 (records, last_unterminated)
179}
180
181pub fn detect_delimiter<R: Read>(reader: R) -> std::io::Result<u8> {
200 const SAMPLE_BYTES: u64 = 4096;
201 let mut preamble = Vec::new();
202 let mut take = reader.take(SAMPLE_BYTES);
203 take.read_to_end(&mut preamble)?;
204 let truncated = preamble.len() as u64 == SAMPLE_BYTES;
205
206 let sample = preamble.strip_prefix(b"\xEF\xBB\xBF").unwrap_or(&preamble);
210 let preamble_str = String::from_utf8_lossy(sample);
211 let (mut records, last_unterminated) = split_sample_records(&preamble_str);
212 if truncated && last_unterminated && records.len() > 1 {
215 records.pop();
216 }
217
218 let scored: Vec<&str> = records
222 .into_iter()
223 .filter(|record| !record.trim().is_empty())
224 .take(5)
225 .collect();
226 let Some((header, rows)) = scored.split_first() else {
227 return Ok(b',');
228 };
229
230 let delimiters = *b",;\t|";
231 let mut best_delimiter = b',';
232 let mut best_score = (false, 0usize, 0usize, 0usize);
238
239 for &delimiter in &delimiters {
240 let (header_fields, _) = count_fields(header, delimiter as char);
241 let (mut agreeing, mut exact) = (1, 1);
242 for record in rows {
243 let (fields, trailing_empty) = count_fields(record, delimiter as char);
244 if agrees_with_header(fields, trailing_empty, header_fields) {
245 agreeing += 1;
246 }
247 if fields == header_fields {
248 exact += 1;
249 }
250 }
251
252 let score = (header_fields > 1, agreeing, exact, header_fields);
253 if score > best_score {
254 best_score = score;
255 best_delimiter = delimiter;
256 }
257 }
258
259 Ok(best_delimiter)
260}
261
262pub fn detect_delimiter_from_path(path: &Path) -> std::io::Result<u8> {
264 let file = std::fs::File::open(path)?;
265 detect_delimiter(std::io::BufReader::new(file))
266}
267
268pub fn analyze_csv_from_reader<R: Read>(
276 reader: R,
277 config: &CsvParserConfig,
278) -> Result<
279 (
280 Vec<ColumnProfile>,
281 StreamingColumnCollection,
282 usize,
283 Vec<String>,
284 ),
285 DataProfilerError,
286> {
287 let scan = scan_csv(reader, config)?;
288 let profiles = profile_builder::profiles_from_streaming(&scan.column_stats, false, false, None);
289
290 Ok((
291 profiles,
292 scan.column_stats,
293 scan.rows_read,
294 scan.header_names,
295 ))
296}
297
298struct CsvScan {
300 column_stats: StreamingColumnCollection,
301 header_names: Vec<String>,
302 rows_read: usize,
303 ragged_rows: usize,
306 truncation: Option<TruncationReason>,
308}
309
310fn scan_csv<R: Read>(reader: R, config: &CsvParserConfig) -> Result<CsvScan, DataProfilerError> {
311 let mut csv_builder = ReaderBuilder::new();
312 csv_builder.has_headers(config.has_header);
313 csv_builder.flexible(config.flexible);
314 csv_builder.quote(config.quote_char);
315 if config.trim_whitespace {
316 csv_builder.trim(csv::Trim::All);
317 }
318 let mut actual_delimiter = config.delimiter;
319
320 let boxed_reader: Box<dyn std::io::Read + '_> = if config.delimiter.is_none() {
321 let mut preamble = Vec::new();
322 let mut take = reader.take(4096);
323 take.read_to_end(&mut preamble)?;
324
325 let detected = detect_delimiter(std::io::Cursor::new(&preamble)).unwrap_or(b',');
326 actual_delimiter = Some(detected);
327
328 Box::new(std::io::Cursor::new(preamble).chain(take.into_inner()))
329 } else {
330 Box::new(reader)
331 };
332
333 if let Some(delimiter) = actual_delimiter {
334 csv_builder.delimiter(delimiter);
335 }
336
337 let mut csv_reader = csv_builder.from_reader(boxed_reader);
338
339 let header_names: Vec<String> = if config.has_header {
340 let headers = csv_reader.headers()?;
341 let names: Vec<String> = headers.iter().map(|header| header.to_string()).collect();
342 dataprof_core::validate_unique_column_names(&names, "CSV header")?;
345 names
346 } else {
347 let headers = csv_reader.headers()?;
348 (0..headers.len())
349 .map(|index| format!("column_{index}"))
350 .collect()
351 };
352
353 let mut column_stats = StreamingColumnCollection::new();
354 column_stats.init_columns(&header_names);
356 let mut rows_read = 0;
357 let mut ragged_rows = 0;
358 let mut truncation = None;
359
360 for result in csv_reader.records() {
361 if let Some(max_rows) = config.max_rows
362 && rows_read >= max_rows
363 {
364 truncation = Some(TruncationReason::MaxRows(max_rows as u64));
367 break;
368 }
369
370 let record = result?;
371 let mut values: Vec<String> = record.iter().map(|value| value.to_string()).collect();
372
373 let header_len = header_names.len();
374 if values.len() != header_len {
375 ragged_rows += 1;
376 }
377 if values.len() < header_len {
378 values.resize(header_len, String::new());
379 } else if values.len() > header_len {
380 values.truncate(header_len);
381 }
382
383 column_stats.process_record(&header_names, values);
384 rows_read += 1;
385 }
386
387 Ok(CsvScan {
388 column_stats,
389 header_names,
390 rows_read,
391 ragged_rows,
392 truncation,
393 })
394}
395
396pub fn analyze_csv_file(
403 file_path: &Path,
404 config: &CsvParserConfig,
405) -> Result<ProfileReport, DataProfilerError> {
406 let metadata = std::fs::metadata(file_path).map_err(|error| map_io_error(file_path, error))?;
407 dataprof_core::binary_input::reject_binary_input(file_path)?;
408 let start = std::time::Instant::now();
409
410 let file = std::fs::File::open(file_path).map_err(|error| map_io_error(file_path, error))?;
411 let buf_reader = std::io::BufReader::new(file);
412
413 let scan = scan_csv(buf_reader, config)?;
414 let column_stats = &scan.column_stats;
415 let column_profiles =
416 profile_builder::profiles_from_streaming(column_stats, false, false, None);
417
418 let file_source = DataSource::File {
419 path: file_path.display().to_string(),
420 format: FileFormat::Csv,
421 size_bytes: metadata.len(),
422 modified_at: None,
423 parquet_metadata: None,
424 };
425
426 let sample_columns = profile_builder::quality_check_samples(column_stats);
427 let scan_time_ms = start.elapsed().as_millis();
428 let num_columns = column_profiles.len();
429
430 let mut execution = ExecutionMetadata::new(scan.rows_read, num_columns, scan_time_ms)
431 .with_engine("csv")
432 .with_ragged_row_count(scan.ragged_rows);
433 if let Some(reason) = scan.truncation {
434 execution = execution.with_truncation(reason);
435 }
436
437 Ok(ReportAssembler::new(file_source, execution)
440 .with_analysis_options(&AnalysisOptions::default())
441 .columns(column_profiles)
442 .with_quality_data(sample_columns)
443 .with_row_duplicates(column_stats.row_duplicate_summary())
444 .with_row_completeness(column_stats.row_completeness_summary())
445 .build())
446}
447
448fn map_io_error(file_path: &Path, error: std::io::Error) -> DataProfilerError {
449 if error.kind() == std::io::ErrorKind::NotFound {
450 DataProfilerError::FileNotFound {
451 path: file_path.display().to_string(),
452 }
453 } else {
454 DataProfilerError::from(error)
455 }
456}
457
458#[cfg(test)]
459mod tests {
460 use super::*;
461 use dataprof_core::DataType;
462 use std::io::{Cursor, Write};
463 use tempfile::NamedTempFile;
464
465 fn write_csv(content: &str) -> NamedTempFile {
466 let mut file = NamedTempFile::new().unwrap();
467 write!(file, "{}", content).unwrap();
468 file.flush().unwrap();
469 file
470 }
471
472 #[test]
473 fn test_analyze_csv_from_reader_basic() {
474 let data = b"name,age\nAlice,25\nBob,30\n";
475 let cursor = Cursor::new(data.as_ref());
476 let config = CsvParserConfig::default();
477
478 let (profiles, _stats, rows, _) = analyze_csv_from_reader(cursor, &config).unwrap();
479 assert_eq!(rows, 2);
480 assert_eq!(profiles.len(), 2);
481
482 let age = profiles
483 .iter()
484 .find(|profile| profile.name == "age")
485 .unwrap();
486 assert_eq!(age.total_count, 2);
487 assert_eq!(age.data_type, DataType::Integer);
488 }
489
490 #[test]
491 fn test_analyze_csv_from_reader_with_max_rows() {
492 let data = b"val\n1\n2\n3\n4\n5\n";
493 let cursor = Cursor::new(data.as_ref());
494 let config = CsvParserConfig::default().max_rows(Some(3));
495
496 let (_profiles, _stats, rows, _) = analyze_csv_from_reader(cursor, &config).unwrap();
497 assert_eq!(rows, 3);
498 }
499
500 #[test]
501 fn test_analyze_csv_from_reader_strict_rejects_ragged() {
502 let data = b"a,b\n1,2\n3,4,5\n";
503 let cursor = Cursor::new(data.as_ref());
504 let config = CsvParserConfig::strict();
505
506 assert!(analyze_csv_from_reader(cursor, &config).is_err());
507 }
508
509 #[test]
510 fn test_analyze_csv_from_reader_flexible_handles_ragged() {
511 let data = b"a,b\n1,2\n3,4,5\n6\n";
512 let cursor = Cursor::new(data.as_ref());
513 let config = CsvParserConfig::default();
514
515 let (profiles, _stats, rows, _) = analyze_csv_from_reader(cursor, &config).unwrap();
516 assert_eq!(rows, 3);
517 assert!(!profiles.is_empty());
518 }
519
520 #[test]
521 fn test_analyze_csv_from_reader_custom_delimiter() {
522 let data = b"a;b\n1;2\n3;4\n";
523 let cursor = Cursor::new(data.as_ref());
524 let config = CsvParserConfig::default().with_delimiter(b';');
525
526 let (profiles, _stats, rows, _) = analyze_csv_from_reader(cursor, &config).unwrap();
527 assert_eq!(rows, 2);
528 assert_eq!(profiles.len(), 2);
529 }
530
531 #[test]
532 fn test_analyze_csv_file_returns_quality_report() {
533 let csv = write_csv("x,y\n1,a\n2,b\n3,c\n");
534 let config = CsvParserConfig::default();
535 let report = analyze_csv_file(csv.path(), &config).unwrap();
536
537 assert_eq!(report.column_profiles.len(), 2);
538 assert_eq!(report.execution.rows_processed, 3);
539 assert!(report.quality_score().unwrap() >= 0.0);
540 assert!(report.quality_score().unwrap() <= 100.0);
541 }
542
543 #[test]
544 fn test_csv_auto_detects_delimiter_from_reader() {
545 let data = b"name;age\nAlice;25\nBob;30\n";
546 let cursor = Cursor::new(data.as_ref());
547 let config = CsvParserConfig::default();
548
549 let (profiles, _stats, rows, _) = analyze_csv_from_reader(cursor, &config).unwrap();
550 assert_eq!(rows, 2);
551 assert_eq!(profiles.len(), 2);
552
553 let age = profiles
554 .iter()
555 .find(|profile| profile.name == "age")
556 .unwrap();
557 assert_eq!(age.total_count, 2);
558 }
559
560 #[test]
561 fn test_analyze_csv_basic() {
562 let csv = write_csv("name,age\nAlice,25\nBob,30\n");
563 let config = CsvParserConfig::default();
564 let report = analyze_csv_file(csv.path(), &config).unwrap();
565 let profiles = &report.column_profiles;
566 assert_eq!(profiles.len(), 2);
567
568 let names: Vec<&str> = profiles
569 .iter()
570 .map(|profile| profile.name.as_str())
571 .collect();
572 assert!(names.contains(&"name"));
573 assert!(names.contains(&"age"));
574
575 let age = profiles
576 .iter()
577 .find(|profile| profile.name == "age")
578 .unwrap();
579 assert_eq!(age.total_count, 2);
580 assert_eq!(age.null_count, 0);
581 }
582
583 #[test]
584 fn test_analyze_csv_with_nulls() {
585 let csv = write_csv("name,age,email\nAlice,25,a@b.com\nBob,,\nCharlie,30,c@d.com\n");
586 let config = CsvParserConfig::default();
587 let report = analyze_csv_file(csv.path(), &config).unwrap();
588 let profiles = &report.column_profiles;
589
590 let age = profiles
591 .iter()
592 .find(|profile| profile.name == "age")
593 .unwrap();
594 assert_eq!(age.total_count, 3);
595 assert_eq!(age.null_count, 1);
596
597 let email = profiles
598 .iter()
599 .find(|profile| profile.name == "email")
600 .unwrap();
601 assert_eq!(email.null_count, 1);
602 }
603
604 #[test]
605 fn test_analyze_csv_file_returns_quality_report_legacy() {
606 let csv = write_csv("x,y\n1,a\n2,b\n3,c\n");
607 let config = CsvParserConfig::default();
608 let report = analyze_csv_file(csv.path(), &config).unwrap();
609
610 assert_eq!(report.column_profiles.len(), 2);
611 assert_eq!(report.execution.rows_processed, 3);
612 assert_eq!(report.execution.columns_detected, 2);
613 assert!(report.quality_score().unwrap() >= 0.0);
614 assert!(report.quality_score().unwrap() <= 100.0);
615 }
616
617 #[test]
618 fn test_analyze_csv_file_header_only_preserves_declared_columns() {
619 let csv = write_csv("name,age\n");
620 let config = CsvParserConfig::default();
621 let report = analyze_csv_file(csv.path(), &config).unwrap();
622
623 assert_eq!(
624 report
625 .column_profiles
626 .iter()
627 .map(|column| (column.name.as_str(), column.data_type.clone()))
628 .collect::<Vec<_>>(),
629 vec![("name", DataType::String), ("age", DataType::String)]
630 );
631 assert_eq!(report.execution.rows_processed, 0);
632 assert!(report.quality.is_some());
633 }
634
635 #[test]
636 fn test_analyze_csv_file_empty_file_has_no_columns() {
637 let csv = write_csv("");
638 let config = CsvParserConfig::default();
639 let report = analyze_csv_file(csv.path(), &config).unwrap();
640
641 assert!(report.column_profiles.is_empty());
642 assert_eq!(report.execution.rows_processed, 0);
643 assert_eq!(report.execution.columns_detected, 0);
644 assert_eq!(
645 report.quality_status,
646 dataprof_runtime::QualityAnalysisStatus::Computed
647 );
648 assert!(report.quality.is_some());
649 assert!(report.quality_score().is_none());
650 }
651
652 #[test]
653 fn test_analyze_csv_with_max_rows_small_file() {
654 let csv = write_csv("val\n1\n2\n3\n4\n5\n");
655 let config = CsvParserConfig::default();
656 let report = analyze_csv_file(csv.path(), &config).unwrap();
657
658 assert_eq!(report.execution.rows_processed, 5);
659 assert_eq!(report.execution.columns_detected, 1);
660 assert!((report.execution.sampling_ratio.unwrap_or(1.0) - 1.0).abs() < 0.01);
661 }
662
663 #[test]
664 fn a_row_cap_that_leaves_rows_unread_is_a_truncation() {
665 let csv = write_csv("val\n1\n2\n3\n4\n5\n");
666 let config = CsvParserConfig::default().max_rows(Some(2));
667 let report = analyze_csv_file(csv.path(), &config).unwrap();
668
669 assert_eq!(report.execution.rows_processed, 2);
670 assert!(!report.execution.source_exhausted);
671 assert_eq!(
672 report.execution.truncation_reason,
673 Some(TruncationReason::MaxRows(2))
674 );
675 }
676
677 #[test]
678 fn a_row_cap_at_the_last_row_is_a_complete_scan() {
679 let csv = write_csv("val\n1\n2\n");
680 let config = CsvParserConfig::default().max_rows(Some(2));
681 let report = analyze_csv_file(csv.path(), &config).unwrap();
682
683 assert_eq!(report.execution.rows_processed, 2);
684 assert!(report.execution.source_exhausted);
685 assert_eq!(report.execution.truncation_reason, None);
686 }
687
688 #[test]
689 fn short_and_long_rows_are_counted_as_ragged() {
690 let csv = write_csv("a,b,c\n1,2,3\n4,5\n6,7,8,9\n10,11,12\n");
691 let report = analyze_csv_file(csv.path(), &CsvParserConfig::default()).unwrap();
692
693 assert_eq!(report.execution.rows_processed, 4);
694 assert_eq!(report.execution.ragged_row_count, 2);
695 }
696
697 #[test]
698 fn a_ragged_row_past_the_row_cap_is_not_counted() {
699 let csv = write_csv("a,b,c\n1,2,3\n4,5\n");
700 let config = CsvParserConfig::default().max_rows(Some(1));
701 let report = analyze_csv_file(csv.path(), &config).unwrap();
702
703 assert_eq!(report.execution.ragged_row_count, 0);
704 assert_eq!(
705 report.execution.truncation_reason,
706 Some(TruncationReason::MaxRows(1))
707 );
708 }
709
710 #[test]
711 fn test_analyze_csv_numeric_types_detected() {
712 let csv = write_csv("int_col,float_col,str_col\n1,1.5,hello\n2,2.5,world\n3,3.5,foo\n");
713 let config = CsvParserConfig::default();
714 let report = analyze_csv_file(csv.path(), &config).unwrap();
715 let profiles = &report.column_profiles;
716
717 let int_col = profiles
718 .iter()
719 .find(|profile| profile.name == "int_col")
720 .unwrap();
721 assert!(matches!(int_col.data_type, DataType::Integer));
722
723 let float_col = profiles
724 .iter()
725 .find(|profile| profile.name == "float_col")
726 .unwrap();
727 assert!(matches!(float_col.data_type, DataType::Float));
728
729 let str_col = profiles
730 .iter()
731 .find(|profile| profile.name == "str_col")
732 .unwrap();
733 assert!(matches!(str_col.data_type, DataType::String));
734 }
735
736 #[test]
737 fn test_analyze_csv_strict_rejects_ragged_rows() {
738 let csv = write_csv("a,b\n1,2\n3,4,5\n");
739 let config = CsvParserConfig::strict();
740 assert!(analyze_csv_file(csv.path(), &config).is_err());
741 }
742
743 #[test]
744 fn test_analyze_csv_file_handles_ragged_rows() {
745 let csv = write_csv("a,b\n1,2\n3,4,5\n6\n");
746 let config = CsvParserConfig::default();
747 let report = analyze_csv_file(csv.path(), &config).unwrap();
748 assert!(!report.column_profiles.is_empty());
749 }
750
751 #[test]
752 fn test_analyze_csv_with_quality_metrics() {
753 let csv = write_csv(
754 "name,age\n\
755 Alice,25\nBob,\nCharlie,30\nDave,\nEve,28\n\
756 Frank,\nGrace,35\nHeidi,40\nIvan,\nJudy,22\n",
757 );
758 let config = CsvParserConfig::default();
759 let report = analyze_csv_file(csv.path(), &config).unwrap();
760
761 let age = report
762 .column_profiles
763 .iter()
764 .find(|profile| profile.name == "age")
765 .unwrap();
766 assert_eq!(age.null_count, 4);
767 assert_eq!(age.total_count, 10);
768 assert!(report.quality_score().unwrap() >= 0.0);
769 }
770
771 #[test]
772 fn test_detect_delimiter_comma() {
773 let data = b"name,age,city\nAlice,25,NYC\nBob,30,London\n";
774 assert_eq!(detect_delimiter(Cursor::new(data.as_ref())).unwrap(), b',');
775 }
776
777 #[test]
778 fn test_detect_delimiter_semicolon() {
779 let data = b"name;age;city\nAlice;25;NYC\nBob;30;London\n";
780 assert_eq!(detect_delimiter(Cursor::new(data.as_ref())).unwrap(), b';');
781 }
782
783 #[test]
784 fn test_detect_delimiter_pipe() {
785 let data = b"name|age|city\nAlice|25|NYC\nBob|30|London\n";
786 assert_eq!(detect_delimiter(Cursor::new(data.as_ref())).unwrap(), b'|');
787 }
788
789 #[test]
790 fn test_detect_delimiter_tab() {
791 let data = b"name\tage\tcity\nAlice\t25\tNYC\nBob\t30\tLondon\n";
792 assert_eq!(detect_delimiter(Cursor::new(data.as_ref())).unwrap(), b'\t');
793 }
794
795 #[test]
796 fn test_detect_delimiter_from_path_semicolon() {
797 let csv = write_csv("id;name;salary\n1;Alice;50000\n2;Bob;60000\n");
798 assert_eq!(detect_delimiter_from_path(csv.path()).unwrap(), b';');
799 }
800
801 #[test]
802 fn test_detect_delimiter_blank_records_do_not_outvote_header() {
803 let data = b"name,age\n\n\n";
808 assert_eq!(detect_delimiter(Cursor::new(data.as_ref())).unwrap(), b',');
809
810 let semicolon = b"name;age\n\n";
811 assert_eq!(
812 detect_delimiter(Cursor::new(semicolon.as_ref())).unwrap(),
813 b';'
814 );
815 }
816
817 #[test]
818 fn test_detect_delimiter_quoted_fields() {
819 let data = b"name;desc;val\n\"hello;world\";foo;1\nbar;baz;2\n";
820 assert_eq!(detect_delimiter(Cursor::new(data.as_ref())).unwrap(), b';');
821 }
822
823 #[test]
824 fn test_detect_delimiter_quoted_embedded_newline() {
825 let data = b"id,comment\n1,\"line one\nline two\"\n2,\"normal\"\n3,\"has \"\"quotes\"\" inside\"\n";
830 assert_eq!(detect_delimiter(Cursor::new(data.as_ref())).unwrap(), b',');
831 }
832
833 #[test]
834 fn test_detect_delimiter_absent_candidate_never_beats_real_one() {
835 let data = b"a,b,c\n1,2,3\nragged line without delimiters\n4,5,6\n";
838 assert_eq!(detect_delimiter(Cursor::new(data.as_ref())).unwrap(), b',');
839 }
840
841 #[test]
842 fn test_detect_delimiter_single_column_falls_back_to_comma() {
843 let data = b"value\n1\n2\n3\n";
844 assert_eq!(detect_delimiter(Cursor::new(data.as_ref())).unwrap(), b',');
845 }
846
847 #[test]
848 fn test_detect_delimiter_two_column_semicolon_with_commas_in_text() {
849 let data = b"id;note\n1;hello, world\n2;foo, bar, baz\n3;plain\n";
851 assert_eq!(detect_delimiter(Cursor::new(data.as_ref())).unwrap(), b';');
852 }
853
854 #[test]
855 fn test_detect_delimiter_never_splits_rows_the_header_does_not() {
856 let data = b"key\n123|456\n123|789\n";
859 assert_eq!(detect_delimiter(Cursor::new(data.as_ref())).unwrap(), b',');
860 }
861
862 #[test]
863 fn test_detect_delimiter_header_split_beats_uniform_rows() {
864 let data = b"a,b\n1,2,\n3,4,\n";
867 assert_eq!(detect_delimiter(Cursor::new(data.as_ref())).unwrap(), b',');
868 }
869
870 #[test]
871 fn test_detect_delimiter_trailing_empty_fields_still_agree() {
872 let data = b"id;amount,EUR\n1;10,50;\n2;20;\n";
876 assert_eq!(detect_delimiter(Cursor::new(data.as_ref())).unwrap(), b';');
877 }
878
879 #[test]
880 fn test_detect_delimiter_quoted_empty_trailing_field_still_agrees() {
881 let data = b"id;amount,EUR\n1;10,50;\"\"\n2;20;\"\"\n";
884 assert_eq!(detect_delimiter(Cursor::new(data.as_ref())).unwrap(), b';');
885 }
886
887 #[test]
888 fn test_detect_delimiter_exact_width_beats_cut_trailing_fields() {
889 let data = b"id,part;value\nx;y,z,\nu;v,w,\n";
893 assert_eq!(detect_delimiter(Cursor::new(data.as_ref())).unwrap(), b';');
894 }
895
896 #[test]
897 fn test_detect_delimiter_skips_a_utf8_bom() {
898 let data = b"\xEF\xBB\xBF\nid;value\n1;2\n3;4\n";
901 assert_eq!(detect_delimiter(Cursor::new(data.as_ref())).unwrap(), b';');
902 let no_blank = b"\xEF\xBB\xBFid;value\n1;2\n";
903 assert_eq!(
904 detect_delimiter(Cursor::new(no_blank.as_ref())).unwrap(),
905 b';'
906 );
907 }
908
909 #[test]
910 fn test_detect_delimiter_lone_cr_ends_a_record() {
911 let data = b"key\r123|456\r123|789\r";
914 assert_eq!(detect_delimiter(Cursor::new(data.as_ref())).unwrap(), b',');
915 let semicolon = b"id;value\r1;10\r2;20\r";
916 assert_eq!(
917 detect_delimiter(Cursor::new(semicolon.as_ref())).unwrap(),
918 b';'
919 );
920 }
921
922 #[test]
923 fn test_count_fields_reports_trailing_empty_fields() {
924 assert_eq!(count_fields("1;10,50;", ';'), (3, 1));
925 assert_eq!(count_fields("a;;", ';'), (3, 2));
926 assert_eq!(count_fields("a;\"\"", ';'), (2, 1));
927 assert_eq!(count_fields("a;\"\"\"\"", ';'), (2, 0));
928 assert_eq!(count_fields("a; ", ';'), (2, 0));
929 assert_eq!(count_fields("a;b", ';'), (2, 0));
930 assert_eq!(count_fields("\"x;y\";", ';'), (2, 1));
931 }
932
933 #[test]
934 fn test_split_sample_records_quote_aware() {
935 let (records, last_unterminated) = split_sample_records("a,b\r\n1,\"x\ny\"\n2,z\n");
936 assert_eq!(records, vec!["a,b", "1,\"x\ny\"", "2,z"]);
937 assert!(!last_unterminated);
938
939 let (records, last_unterminated) = split_sample_records("a,b\n1,\"cut mid-fie");
940 assert_eq!(records, vec!["a,b", "1,\"cut mid-fie"]);
941 assert!(last_unterminated);
942
943 let (records, last_unterminated) = split_sample_records("a,b\r1,\"x\ry\"\r2,z\r");
944 assert_eq!(records, vec!["a,b", "1,\"x\ry\"", "2,z"]);
945 assert!(!last_unterminated);
946
947 let (records, _) = split_sample_records("a\rb\r\nc\n");
948 assert_eq!(records, vec!["a", "b", "c"]);
949 }
950
951 #[test]
952 fn test_detect_delimiter_full_sample_ending_on_record_boundary() {
953 let mut data = Vec::from(&b"a;b;c\n"[..]);
956 while data.len() < 4090 {
957 data.extend_from_slice(b"1;2;3\n");
958 }
959 while data.len() < 4095 {
960 data.push(b'#');
961 }
962 data.push(b'\n');
963 assert_eq!(data.len(), 4096);
964 assert_eq!(
965 detect_delimiter(Cursor::new(data.as_slice())).unwrap(),
966 b';'
967 );
968 }
969}