1use csv::ReaderBuilder;
2use std::io::Read;
3use std::path::Path;
4
5use dataprof_core::{
6 ColumnProfile, DataProfilerError, DataSource, ExecutionMetadata, FileFormat, QualityDimension,
7 SemanticHints,
8};
9use dataprof_runtime::{
10 ProfileReport, ReportAssembler, StreamingColumnCollection, profile_builder,
11};
12
13mod memmap_reader;
14mod quote_state;
15mod robust_csv;
16
17pub use memmap_reader::MemoryMappedCsvReader;
18pub use quote_state::{QuoteOutcome, QuoteTrackingReader, unterminated_quote_error};
19pub use robust_csv::{
20 CsvDiagnostics, CsvParseOutput, EncodingDiagnostic, RobustCsvParser, RobustParseResult,
21 diagnose_non_utf8,
22};
23
24#[derive(Debug, Clone)]
29pub struct CsvParserConfig {
30 pub flexible: bool,
32 pub delimiter: Option<u8>,
34 pub quote_char: u8,
36 pub trim_whitespace: bool,
38 pub has_header: bool,
41 pub max_rows: Option<usize>,
43}
44
45impl Default for CsvParserConfig {
46 fn default() -> Self {
47 Self {
48 flexible: true,
49 delimiter: None,
50 quote_char: b'"',
51 trim_whitespace: false,
52 has_header: true,
53 max_rows: None,
54 }
55 }
56}
57
58impl CsvParserConfig {
59 pub fn strict() -> Self {
61 Self {
62 flexible: false,
63 ..Default::default()
64 }
65 }
66
67 pub fn with_delimiter(mut self, delimiter: u8) -> Self {
69 self.delimiter = Some(delimiter);
70 self
71 }
72
73 pub fn has_header(mut self, yes: bool) -> Self {
75 self.has_header = yes;
76 self
77 }
78
79 pub fn max_rows(mut self, max: Option<usize>) -> Self {
81 self.max_rows = max;
82 self
83 }
84}
85
86pub(crate) fn count_fields(line: &str, delimiter: char) -> usize {
88 let mut count = 1;
89 let mut in_quotes = false;
90 let mut chars = line.chars().peekable();
91 while let Some(ch) = chars.next() {
92 if ch == '"' {
93 if chars.peek() == Some(&'"') {
94 chars.next();
95 } else {
96 in_quotes = !in_quotes;
97 }
98 } else if ch == delimiter && !in_quotes {
99 count += 1;
100 }
101 }
102 count
103}
104
105fn split_sample_records(text: &str) -> (Vec<&str>, bool) {
113 let bytes = text.as_bytes();
114 let mut records = Vec::new();
115 let mut in_quotes = false;
116 let mut start = 0usize;
117 let mut i = 0usize;
118 while i < bytes.len() {
119 match bytes[i] {
120 b'"' => {
121 if in_quotes && bytes.get(i + 1) == Some(&b'"') {
122 i += 1; } else {
124 in_quotes = !in_quotes;
125 }
126 }
127 b'\n' if !in_quotes => {
128 let end = if i > start && bytes[i - 1] == b'\r' {
129 i - 1
130 } else {
131 i
132 };
133 records.push(&text[start..end]);
134 start = i + 1;
135 }
136 _ => {}
137 }
138 i += 1;
139 }
140 let last_unterminated = start < text.len();
141 if last_unterminated {
142 records.push(text[start..].trim_end_matches('\r'));
143 }
144 (records, last_unterminated)
145}
146
147pub fn detect_delimiter<R: Read>(reader: R) -> std::io::Result<u8> {
158 const SAMPLE_BYTES: u64 = 4096;
159 let mut preamble = Vec::new();
160 let mut take = reader.take(SAMPLE_BYTES);
161 take.read_to_end(&mut preamble)?;
162 let truncated = preamble.len() as u64 == SAMPLE_BYTES;
163
164 let preamble_str = String::from_utf8_lossy(&preamble);
165 let (mut records, last_unterminated) = split_sample_records(&preamble_str);
166 if truncated && last_unterminated && records.len() > 1 {
169 records.pop();
170 }
171
172 let delimiters = *b",;\t|";
173 let mut best_delimiter = b',';
174 let mut best_score = (false, 0usize, 0usize);
177
178 for &delimiter in &delimiters {
179 let mut counts = std::collections::HashMap::new();
180 let scored = records
184 .iter()
185 .filter(|record| !record.trim().is_empty())
186 .take(5);
187 for record in scored {
188 let fields = count_fields(record, delimiter as char);
189 *counts.entry(fields).or_insert(0usize) += 1;
190 }
191
192 let (modal_fields, consistency) = counts
195 .into_iter()
196 .max_by_key(|&(fields, count)| (count, fields))
197 .unwrap_or((0, 0));
198
199 let score = (modal_fields > 1, consistency, modal_fields);
200 if score > best_score {
201 best_score = score;
202 best_delimiter = delimiter;
203 }
204 }
205
206 Ok(best_delimiter)
207}
208
209pub fn detect_delimiter_from_path(path: &Path) -> std::io::Result<u8> {
211 let file = std::fs::File::open(path)?;
212 detect_delimiter(std::io::BufReader::new(file))
213}
214
215pub fn analyze_csv_from_reader<R: Read>(
223 reader: R,
224 config: &CsvParserConfig,
225) -> Result<
226 (
227 Vec<ColumnProfile>,
228 StreamingColumnCollection,
229 usize,
230 Vec<String>,
231 ),
232 DataProfilerError,
233> {
234 analyze_csv_from_reader_with_hints(reader, config, &SemanticHints::default())
235}
236
237pub fn analyze_csv_from_reader_with_hints<R: Read>(
239 reader: R,
240 config: &CsvParserConfig,
241 semantic_hints: &SemanticHints,
242) -> Result<
243 (
244 Vec<ColumnProfile>,
245 StreamingColumnCollection,
246 usize,
247 Vec<String>,
248 ),
249 DataProfilerError,
250> {
251 let mut csv_builder = ReaderBuilder::new();
252 csv_builder.has_headers(config.has_header);
253 csv_builder.flexible(config.flexible);
254 csv_builder.quote(config.quote_char);
255 if config.trim_whitespace {
256 csv_builder.trim(csv::Trim::All);
257 }
258 let mut actual_delimiter = config.delimiter;
259
260 let boxed_reader: Box<dyn std::io::Read + '_> = if config.delimiter.is_none() {
261 let mut preamble = Vec::new();
262 let mut take = reader.take(4096);
263 take.read_to_end(&mut preamble)?;
264
265 let detected = detect_delimiter(std::io::Cursor::new(&preamble)).unwrap_or(b',');
266 actual_delimiter = Some(detected);
267
268 Box::new(std::io::Cursor::new(preamble).chain(take.into_inner()))
269 } else {
270 Box::new(reader)
271 };
272
273 if let Some(delimiter) = actual_delimiter {
274 csv_builder.delimiter(delimiter);
275 }
276
277 let mut csv_reader = csv_builder.from_reader(boxed_reader);
278
279 let header_names: Vec<String> = if config.has_header {
280 let headers = csv_reader.headers()?;
281 let names: Vec<String> = headers.iter().map(|header| header.to_string()).collect();
282 dataprof_core::validate_unique_column_names(&names, "CSV header")?;
285 names
286 } else {
287 let headers = csv_reader.headers()?;
288 (0..headers.len())
289 .map(|index| format!("column_{index}"))
290 .collect()
291 };
292
293 let mut column_stats = StreamingColumnCollection::new().with_semantic_hints(semantic_hints);
294 column_stats.init_columns(&header_names);
296 let mut rows_read = 0;
297
298 for result in csv_reader.records() {
299 if let Some(max_rows) = config.max_rows
300 && rows_read >= max_rows
301 {
302 break;
303 }
304
305 let record = result?;
306 let mut values: Vec<String> = record.iter().map(|value| value.to_string()).collect();
307
308 let header_len = header_names.len();
309 if values.len() < header_len {
310 values.resize(header_len, String::new());
311 } else if values.len() > header_len {
312 values.truncate(header_len);
313 }
314
315 column_stats.process_record(&header_names, values);
316 rows_read += 1;
317 }
318
319 let profiles = profile_builder::profiles_from_streaming_with_hints(
320 &column_stats,
321 false,
322 false,
323 None,
324 semantic_hints,
325 );
326
327 Ok((profiles, column_stats, rows_read, header_names))
328}
329
330pub fn analyze_csv_file(
335 file_path: &Path,
336 config: &CsvParserConfig,
337) -> Result<ProfileReport, DataProfilerError> {
338 analyze_csv_file_with_dimensions(file_path, config, None)
339}
340
341pub fn analyze_csv_file_with_dimensions(
343 file_path: &Path,
344 config: &CsvParserConfig,
345 quality_dimensions: Option<&[QualityDimension]>,
346) -> Result<ProfileReport, DataProfilerError> {
347 analyze_csv_file_with_dimensions_and_hints(
348 file_path,
349 config,
350 quality_dimensions,
351 &SemanticHints::default(),
352 )
353}
354
355pub fn analyze_csv_file_with_dimensions_and_hints(
356 file_path: &Path,
357 config: &CsvParserConfig,
358 quality_dimensions: Option<&[QualityDimension]>,
359 semantic_hints: &SemanticHints,
360) -> Result<ProfileReport, DataProfilerError> {
361 let metadata = std::fs::metadata(file_path).map_err(|error| map_io_error(file_path, error))?;
362 let start = std::time::Instant::now();
363
364 let file = std::fs::File::open(file_path).map_err(|error| map_io_error(file_path, error))?;
365 let buf_reader = std::io::BufReader::new(file);
366
367 let (column_profiles, column_stats, rows_read, _header_names) =
368 analyze_csv_from_reader_with_hints(buf_reader, config, semantic_hints)?;
369
370 let file_source = DataSource::File {
371 path: file_path.display().to_string(),
372 format: FileFormat::Csv,
373 size_bytes: metadata.len(),
374 modified_at: None,
375 parquet_metadata: None,
376 };
377
378 let sample_columns = profile_builder::quality_check_samples(&column_stats);
379 let scan_time_ms = start.elapsed().as_millis();
380 let num_columns = column_profiles.len();
381
382 let mut assembler = ReportAssembler::new(
383 file_source,
384 ExecutionMetadata::new(rows_read, num_columns, scan_time_ms).with_engine("csv"),
385 )
386 .columns(column_profiles)
387 .with_quality_data(sample_columns)
388 .with_row_duplicates(column_stats.row_duplicate_summary())
389 .with_row_completeness(column_stats.row_completeness_summary())
390 .with_exact_value_hint_bindings(column_stats.semantic_hint_bindings())
391 .with_semantic_hints(semantic_hints.clone());
392 if let Some(dimensions) = quality_dimensions {
393 assembler = assembler.with_requested_dimensions(dimensions.to_vec());
394 }
395 Ok(assembler.build())
396}
397
398fn map_io_error(file_path: &Path, error: std::io::Error) -> DataProfilerError {
399 if error.kind() == std::io::ErrorKind::NotFound {
400 DataProfilerError::FileNotFound {
401 path: file_path.display().to_string(),
402 }
403 } else {
404 DataProfilerError::from(error)
405 }
406}
407
408#[cfg(test)]
409mod tests {
410 use super::*;
411 use dataprof_core::DataType;
412 use std::io::{Cursor, Write};
413 use tempfile::NamedTempFile;
414
415 fn write_csv(content: &str) -> NamedTempFile {
416 let mut file = NamedTempFile::new().unwrap();
417 write!(file, "{}", content).unwrap();
418 file.flush().unwrap();
419 file
420 }
421
422 #[test]
423 fn test_analyze_csv_from_reader_basic() {
424 let data = b"name,age\nAlice,25\nBob,30\n";
425 let cursor = Cursor::new(data.as_ref());
426 let config = CsvParserConfig::default();
427
428 let (profiles, _stats, rows, _) = analyze_csv_from_reader(cursor, &config).unwrap();
429 assert_eq!(rows, 2);
430 assert_eq!(profiles.len(), 2);
431
432 let age = profiles
433 .iter()
434 .find(|profile| profile.name == "age")
435 .unwrap();
436 assert_eq!(age.total_count, 2);
437 assert_eq!(age.data_type, DataType::Integer);
438 }
439
440 #[test]
441 fn test_analyze_csv_from_reader_with_max_rows() {
442 let data = b"val\n1\n2\n3\n4\n5\n";
443 let cursor = Cursor::new(data.as_ref());
444 let config = CsvParserConfig::default().max_rows(Some(3));
445
446 let (_profiles, _stats, rows, _) = analyze_csv_from_reader(cursor, &config).unwrap();
447 assert_eq!(rows, 3);
448 }
449
450 #[test]
451 fn test_analyze_csv_from_reader_strict_rejects_ragged() {
452 let data = b"a,b\n1,2\n3,4,5\n";
453 let cursor = Cursor::new(data.as_ref());
454 let config = CsvParserConfig::strict();
455
456 assert!(analyze_csv_from_reader(cursor, &config).is_err());
457 }
458
459 #[test]
460 fn test_analyze_csv_from_reader_flexible_handles_ragged() {
461 let data = b"a,b\n1,2\n3,4,5\n6\n";
462 let cursor = Cursor::new(data.as_ref());
463 let config = CsvParserConfig::default();
464
465 let (profiles, _stats, rows, _) = analyze_csv_from_reader(cursor, &config).unwrap();
466 assert_eq!(rows, 3);
467 assert!(!profiles.is_empty());
468 }
469
470 #[test]
471 fn test_analyze_csv_from_reader_custom_delimiter() {
472 let data = b"a;b\n1;2\n3;4\n";
473 let cursor = Cursor::new(data.as_ref());
474 let config = CsvParserConfig::default().with_delimiter(b';');
475
476 let (profiles, _stats, rows, _) = analyze_csv_from_reader(cursor, &config).unwrap();
477 assert_eq!(rows, 2);
478 assert_eq!(profiles.len(), 2);
479 }
480
481 #[test]
482 fn test_analyze_csv_file_returns_quality_report() {
483 let csv = write_csv("x,y\n1,a\n2,b\n3,c\n");
484 let config = CsvParserConfig::default();
485 let report = analyze_csv_file(csv.path(), &config).unwrap();
486
487 assert_eq!(report.column_profiles.len(), 2);
488 assert_eq!(report.execution.rows_processed, 3);
489 assert!(report.quality_score().unwrap() >= 0.0);
490 assert!(report.quality_score().unwrap() <= 100.0);
491 }
492
493 #[test]
494 fn test_csv_auto_detects_delimiter_from_reader() {
495 let data = b"name;age\nAlice;25\nBob;30\n";
496 let cursor = Cursor::new(data.as_ref());
497 let config = CsvParserConfig::default();
498
499 let (profiles, _stats, rows, _) = analyze_csv_from_reader(cursor, &config).unwrap();
500 assert_eq!(rows, 2);
501 assert_eq!(profiles.len(), 2);
502
503 let age = profiles
504 .iter()
505 .find(|profile| profile.name == "age")
506 .unwrap();
507 assert_eq!(age.total_count, 2);
508 }
509
510 #[test]
511 fn test_analyze_csv_basic() {
512 let csv = write_csv("name,age\nAlice,25\nBob,30\n");
513 let config = CsvParserConfig::default();
514 let report = analyze_csv_file(csv.path(), &config).unwrap();
515 let profiles = &report.column_profiles;
516 assert_eq!(profiles.len(), 2);
517
518 let names: Vec<&str> = profiles
519 .iter()
520 .map(|profile| profile.name.as_str())
521 .collect();
522 assert!(names.contains(&"name"));
523 assert!(names.contains(&"age"));
524
525 let age = profiles
526 .iter()
527 .find(|profile| profile.name == "age")
528 .unwrap();
529 assert_eq!(age.total_count, 2);
530 assert_eq!(age.null_count, 0);
531 }
532
533 #[test]
534 fn test_analyze_csv_with_nulls() {
535 let csv = write_csv("name,age,email\nAlice,25,a@b.com\nBob,,\nCharlie,30,c@d.com\n");
536 let config = CsvParserConfig::default();
537 let report = analyze_csv_file(csv.path(), &config).unwrap();
538 let profiles = &report.column_profiles;
539
540 let age = profiles
541 .iter()
542 .find(|profile| profile.name == "age")
543 .unwrap();
544 assert_eq!(age.total_count, 3);
545 assert_eq!(age.null_count, 1);
546
547 let email = profiles
548 .iter()
549 .find(|profile| profile.name == "email")
550 .unwrap();
551 assert_eq!(email.null_count, 1);
552 }
553
554 #[test]
555 fn test_analyze_csv_file_returns_quality_report_legacy() {
556 let csv = write_csv("x,y\n1,a\n2,b\n3,c\n");
557 let config = CsvParserConfig::default();
558 let report = analyze_csv_file(csv.path(), &config).unwrap();
559
560 assert_eq!(report.column_profiles.len(), 2);
561 assert_eq!(report.execution.rows_processed, 3);
562 assert_eq!(report.execution.columns_detected, 2);
563 assert!(report.quality_score().unwrap() >= 0.0);
564 assert!(report.quality_score().unwrap() <= 100.0);
565 }
566
567 #[test]
568 fn test_analyze_csv_file_header_only_preserves_declared_columns() {
569 let csv = write_csv("name,age\n");
570 let config = CsvParserConfig::default();
571 let report = analyze_csv_file(csv.path(), &config).unwrap();
572
573 assert_eq!(
574 report
575 .column_profiles
576 .iter()
577 .map(|column| (column.name.as_str(), column.data_type.clone()))
578 .collect::<Vec<_>>(),
579 vec![("name", DataType::String), ("age", DataType::String)]
580 );
581 assert_eq!(report.execution.rows_processed, 0);
582 assert!(report.quality.is_some());
583 }
584
585 #[test]
586 fn test_analyze_csv_file_empty_file_has_no_columns() {
587 let csv = write_csv("");
588 let config = CsvParserConfig::default();
589 let report = analyze_csv_file(csv.path(), &config).unwrap();
590
591 assert!(report.column_profiles.is_empty());
592 assert_eq!(report.execution.rows_processed, 0);
593 assert_eq!(report.execution.columns_detected, 0);
594 assert_eq!(
595 report.quality_status,
596 dataprof_runtime::QualityAnalysisStatus::Computed
597 );
598 assert!(report.quality.is_some());
599 assert!(report.quality_score().is_none());
600 }
601
602 #[test]
603 fn test_analyze_csv_with_max_rows_small_file() {
604 let csv = write_csv("val\n1\n2\n3\n4\n5\n");
605 let config = CsvParserConfig::default();
606 let report = analyze_csv_file(csv.path(), &config).unwrap();
607
608 assert_eq!(report.execution.rows_processed, 5);
609 assert_eq!(report.execution.columns_detected, 1);
610 assert!((report.execution.sampling_ratio.unwrap_or(1.0) - 1.0).abs() < 0.01);
611 }
612
613 #[test]
614 fn test_analyze_csv_numeric_types_detected() {
615 let csv = write_csv("int_col,float_col,str_col\n1,1.5,hello\n2,2.5,world\n3,3.5,foo\n");
616 let config = CsvParserConfig::default();
617 let report = analyze_csv_file(csv.path(), &config).unwrap();
618 let profiles = &report.column_profiles;
619
620 let int_col = profiles
621 .iter()
622 .find(|profile| profile.name == "int_col")
623 .unwrap();
624 assert!(matches!(int_col.data_type, DataType::Integer));
625
626 let float_col = profiles
627 .iter()
628 .find(|profile| profile.name == "float_col")
629 .unwrap();
630 assert!(matches!(float_col.data_type, DataType::Float));
631
632 let str_col = profiles
633 .iter()
634 .find(|profile| profile.name == "str_col")
635 .unwrap();
636 assert!(matches!(str_col.data_type, DataType::String));
637 }
638
639 #[test]
640 fn test_analyze_csv_strict_rejects_ragged_rows() {
641 let csv = write_csv("a,b\n1,2\n3,4,5\n");
642 let config = CsvParserConfig::strict();
643 assert!(analyze_csv_file(csv.path(), &config).is_err());
644 }
645
646 #[test]
647 fn test_analyze_csv_file_handles_ragged_rows() {
648 let csv = write_csv("a,b\n1,2\n3,4,5\n6\n");
649 let config = CsvParserConfig::default();
650 let report = analyze_csv_file(csv.path(), &config).unwrap();
651 assert!(!report.column_profiles.is_empty());
652 }
653
654 #[test]
655 fn test_analyze_csv_with_quality_metrics() {
656 let csv = write_csv(
657 "name,age\n\
658 Alice,25\nBob,\nCharlie,30\nDave,\nEve,28\n\
659 Frank,\nGrace,35\nHeidi,40\nIvan,\nJudy,22\n",
660 );
661 let config = CsvParserConfig::default();
662 let report = analyze_csv_file(csv.path(), &config).unwrap();
663
664 let age = report
665 .column_profiles
666 .iter()
667 .find(|profile| profile.name == "age")
668 .unwrap();
669 assert_eq!(age.null_count, 4);
670 assert_eq!(age.total_count, 10);
671 assert!(report.quality_score().unwrap() >= 0.0);
672 }
673
674 #[test]
675 fn test_detect_delimiter_comma() {
676 let data = b"name,age,city\nAlice,25,NYC\nBob,30,London\n";
677 assert_eq!(detect_delimiter(Cursor::new(data.as_ref())).unwrap(), b',');
678 }
679
680 #[test]
681 fn test_detect_delimiter_semicolon() {
682 let data = b"name;age;city\nAlice;25;NYC\nBob;30;London\n";
683 assert_eq!(detect_delimiter(Cursor::new(data.as_ref())).unwrap(), b';');
684 }
685
686 #[test]
687 fn test_detect_delimiter_pipe() {
688 let data = b"name|age|city\nAlice|25|NYC\nBob|30|London\n";
689 assert_eq!(detect_delimiter(Cursor::new(data.as_ref())).unwrap(), b'|');
690 }
691
692 #[test]
693 fn test_detect_delimiter_tab() {
694 let data = b"name\tage\tcity\nAlice\t25\tNYC\nBob\t30\tLondon\n";
695 assert_eq!(detect_delimiter(Cursor::new(data.as_ref())).unwrap(), b'\t');
696 }
697
698 #[test]
699 fn test_detect_delimiter_from_path_semicolon() {
700 let csv = write_csv("id;name;salary\n1;Alice;50000\n2;Bob;60000\n");
701 assert_eq!(detect_delimiter_from_path(csv.path()).unwrap(), b';');
702 }
703
704 #[test]
705 fn test_detect_delimiter_blank_records_do_not_outvote_header() {
706 let data = b"name,age\n\n\n";
711 assert_eq!(detect_delimiter(Cursor::new(data.as_ref())).unwrap(), b',');
712
713 let semicolon = b"name;age\n\n";
714 assert_eq!(
715 detect_delimiter(Cursor::new(semicolon.as_ref())).unwrap(),
716 b';'
717 );
718 }
719
720 #[test]
721 fn test_detect_delimiter_quoted_fields() {
722 let data = b"name;desc;val\n\"hello;world\";foo;1\nbar;baz;2\n";
723 assert_eq!(detect_delimiter(Cursor::new(data.as_ref())).unwrap(), b';');
724 }
725
726 #[test]
727 fn test_detect_delimiter_quoted_embedded_newline() {
728 let data = b"id,comment\n1,\"line one\nline two\"\n2,\"normal\"\n3,\"has \"\"quotes\"\" inside\"\n";
733 assert_eq!(detect_delimiter(Cursor::new(data.as_ref())).unwrap(), b',');
734 }
735
736 #[test]
737 fn test_detect_delimiter_absent_candidate_never_beats_real_one() {
738 let data = b"a,b,c\n1,2,3\nragged line without delimiters\n4,5,6\n";
741 assert_eq!(detect_delimiter(Cursor::new(data.as_ref())).unwrap(), b',');
742 }
743
744 #[test]
745 fn test_detect_delimiter_single_column_falls_back_to_comma() {
746 let data = b"value\n1\n2\n3\n";
747 assert_eq!(detect_delimiter(Cursor::new(data.as_ref())).unwrap(), b',');
748 }
749
750 #[test]
751 fn test_detect_delimiter_two_column_semicolon_with_commas_in_text() {
752 let data = b"id;note\n1;hello, world\n2;foo, bar, baz\n3;plain\n";
754 assert_eq!(detect_delimiter(Cursor::new(data.as_ref())).unwrap(), b';');
755 }
756
757 #[test]
758 fn test_split_sample_records_quote_aware() {
759 let (records, last_unterminated) = split_sample_records("a,b\r\n1,\"x\ny\"\n2,z\n");
760 assert_eq!(records, vec!["a,b", "1,\"x\ny\"", "2,z"]);
761 assert!(!last_unterminated);
762
763 let (records, last_unterminated) = split_sample_records("a,b\n1,\"cut mid-fie");
764 assert_eq!(records, vec!["a,b", "1,\"cut mid-fie"]);
765 assert!(last_unterminated);
766 }
767
768 #[test]
769 fn test_detect_delimiter_full_sample_ending_on_record_boundary() {
770 let mut data = Vec::from(&b"a;b;c\n"[..]);
773 while data.len() < 4090 {
774 data.extend_from_slice(b"1;2;3\n");
775 }
776 while data.len() < 4095 {
777 data.push(b'#');
778 }
779 data.push(b'\n');
780 assert_eq!(data.len(), 4096);
781 assert_eq!(
782 detect_delimiter(Cursor::new(data.as_slice())).unwrap(),
783 b';'
784 );
785 }
786}