1use csv::ReaderBuilder;
2use std::io::Read;
3use std::path::Path;
4
5use dataprof_core::{
6 ColumnProfile, DataProfilerError, DataSource, ExecutionMetadata, FileFormat, QualityDimension,
7 SemanticHints,
8};
9use dataprof_runtime::{
10 ProfileReport, ReportAssembler, StreamingColumnCollection, profile_builder,
11};
12
13mod memmap_reader;
14mod robust_csv;
15
16pub use memmap_reader::MemoryMappedCsvReader;
17pub use robust_csv::{
18 CsvDiagnostics, EncodingDiagnostic, RobustCsvParser, RobustParseResult, diagnose_non_utf8,
19};
20
21#[derive(Debug, Clone)]
26pub struct CsvParserConfig {
27 pub flexible: bool,
29 pub delimiter: Option<u8>,
31 pub quote_char: u8,
33 pub trim_whitespace: bool,
35 pub has_header: bool,
38 pub max_rows: Option<usize>,
40}
41
42impl Default for CsvParserConfig {
43 fn default() -> Self {
44 Self {
45 flexible: true,
46 delimiter: None,
47 quote_char: b'"',
48 trim_whitespace: false,
49 has_header: true,
50 max_rows: None,
51 }
52 }
53}
54
55impl CsvParserConfig {
56 pub fn strict() -> Self {
58 Self {
59 flexible: false,
60 ..Default::default()
61 }
62 }
63
64 pub fn with_delimiter(mut self, delimiter: u8) -> Self {
66 self.delimiter = Some(delimiter);
67 self
68 }
69
70 pub fn has_header(mut self, yes: bool) -> Self {
72 self.has_header = yes;
73 self
74 }
75
76 pub fn max_rows(mut self, max: Option<usize>) -> Self {
78 self.max_rows = max;
79 self
80 }
81}
82
83pub(crate) fn count_fields(line: &str, delimiter: char) -> usize {
85 let mut count = 1;
86 let mut in_quotes = false;
87 let mut chars = line.chars().peekable();
88 while let Some(ch) = chars.next() {
89 if ch == '"' {
90 if chars.peek() == Some(&'"') {
91 chars.next();
92 } else {
93 in_quotes = !in_quotes;
94 }
95 } else if ch == delimiter && !in_quotes {
96 count += 1;
97 }
98 }
99 count
100}
101
102fn split_sample_records(text: &str) -> (Vec<&str>, bool) {
110 let bytes = text.as_bytes();
111 let mut records = Vec::new();
112 let mut in_quotes = false;
113 let mut start = 0usize;
114 let mut i = 0usize;
115 while i < bytes.len() {
116 match bytes[i] {
117 b'"' => {
118 if in_quotes && bytes.get(i + 1) == Some(&b'"') {
119 i += 1; } else {
121 in_quotes = !in_quotes;
122 }
123 }
124 b'\n' if !in_quotes => {
125 let end = if i > start && bytes[i - 1] == b'\r' {
126 i - 1
127 } else {
128 i
129 };
130 records.push(&text[start..end]);
131 start = i + 1;
132 }
133 _ => {}
134 }
135 i += 1;
136 }
137 let last_unterminated = start < text.len();
138 if last_unterminated {
139 records.push(text[start..].trim_end_matches('\r'));
140 }
141 (records, last_unterminated)
142}
143
144pub fn detect_delimiter<R: Read>(reader: R) -> std::io::Result<u8> {
155 const SAMPLE_BYTES: u64 = 4096;
156 let mut preamble = Vec::new();
157 let mut take = reader.take(SAMPLE_BYTES);
158 take.read_to_end(&mut preamble)?;
159 let truncated = preamble.len() as u64 == SAMPLE_BYTES;
160
161 let preamble_str = String::from_utf8_lossy(&preamble);
162 let (mut records, last_unterminated) = split_sample_records(&preamble_str);
163 if truncated && last_unterminated && records.len() > 1 {
166 records.pop();
167 }
168
169 let delimiters = *b",;\t|";
170 let mut best_delimiter = b',';
171 let mut best_score = (false, 0usize, 0usize);
174
175 for &delimiter in &delimiters {
176 let mut counts = std::collections::HashMap::new();
177 let scored = records
181 .iter()
182 .filter(|record| !record.trim().is_empty())
183 .take(5);
184 for record in scored {
185 let fields = count_fields(record, delimiter as char);
186 *counts.entry(fields).or_insert(0usize) += 1;
187 }
188
189 let (modal_fields, consistency) = counts
192 .into_iter()
193 .max_by_key(|&(fields, count)| (count, fields))
194 .unwrap_or((0, 0));
195
196 let score = (modal_fields > 1, consistency, modal_fields);
197 if score > best_score {
198 best_score = score;
199 best_delimiter = delimiter;
200 }
201 }
202
203 Ok(best_delimiter)
204}
205
206pub fn detect_delimiter_from_path(path: &Path) -> std::io::Result<u8> {
208 let file = std::fs::File::open(path)?;
209 detect_delimiter(std::io::BufReader::new(file))
210}
211
212pub fn analyze_csv_from_reader<R: Read>(
220 reader: R,
221 config: &CsvParserConfig,
222) -> Result<
223 (
224 Vec<ColumnProfile>,
225 StreamingColumnCollection,
226 usize,
227 Vec<String>,
228 ),
229 DataProfilerError,
230> {
231 analyze_csv_from_reader_with_hints(reader, config, &SemanticHints::default())
232}
233
234pub fn analyze_csv_from_reader_with_hints<R: Read>(
236 reader: R,
237 config: &CsvParserConfig,
238 semantic_hints: &SemanticHints,
239) -> Result<
240 (
241 Vec<ColumnProfile>,
242 StreamingColumnCollection,
243 usize,
244 Vec<String>,
245 ),
246 DataProfilerError,
247> {
248 let mut csv_builder = ReaderBuilder::new();
249 csv_builder.has_headers(config.has_header);
250 csv_builder.flexible(config.flexible);
251 csv_builder.quote(config.quote_char);
252 if config.trim_whitespace {
253 csv_builder.trim(csv::Trim::All);
254 }
255 let mut actual_delimiter = config.delimiter;
256
257 let boxed_reader: Box<dyn std::io::Read + '_> = if config.delimiter.is_none() {
258 let mut preamble = Vec::new();
259 let mut take = reader.take(4096);
260 take.read_to_end(&mut preamble)?;
261
262 let detected = detect_delimiter(std::io::Cursor::new(&preamble)).unwrap_or(b',');
263 actual_delimiter = Some(detected);
264
265 Box::new(std::io::Cursor::new(preamble).chain(take.into_inner()))
266 } else {
267 Box::new(reader)
268 };
269
270 if let Some(delimiter) = actual_delimiter {
271 csv_builder.delimiter(delimiter);
272 }
273
274 let mut csv_reader = csv_builder.from_reader(boxed_reader);
275
276 let header_names: Vec<String> = if config.has_header {
277 let headers = csv_reader.headers()?;
278 let names: Vec<String> = headers.iter().map(|header| header.to_string()).collect();
279 dataprof_core::validate_unique_column_names(&names, "CSV header")?;
282 names
283 } else {
284 let headers = csv_reader.headers()?;
285 (0..headers.len())
286 .map(|index| format!("column_{index}"))
287 .collect()
288 };
289
290 let mut column_stats = StreamingColumnCollection::new().with_semantic_hints(semantic_hints);
291 column_stats.init_columns(&header_names);
293 let mut rows_read = 0;
294
295 for result in csv_reader.records() {
296 if let Some(max_rows) = config.max_rows
297 && rows_read >= max_rows
298 {
299 break;
300 }
301
302 let record = result?;
303 let mut values: Vec<String> = record.iter().map(|value| value.to_string()).collect();
304
305 let header_len = header_names.len();
306 if values.len() < header_len {
307 values.resize(header_len, String::new());
308 } else if values.len() > header_len {
309 values.truncate(header_len);
310 }
311
312 column_stats.process_record(&header_names, values);
313 rows_read += 1;
314 }
315
316 let profiles = profile_builder::profiles_from_streaming_with_hints(
317 &column_stats,
318 false,
319 false,
320 None,
321 semantic_hints,
322 );
323
324 Ok((profiles, column_stats, rows_read, header_names))
325}
326
327pub fn analyze_csv_file(
332 file_path: &Path,
333 config: &CsvParserConfig,
334) -> Result<ProfileReport, DataProfilerError> {
335 analyze_csv_file_with_dimensions(file_path, config, None)
336}
337
338pub fn analyze_csv_file_with_dimensions(
340 file_path: &Path,
341 config: &CsvParserConfig,
342 quality_dimensions: Option<&[QualityDimension]>,
343) -> Result<ProfileReport, DataProfilerError> {
344 analyze_csv_file_with_dimensions_and_hints(
345 file_path,
346 config,
347 quality_dimensions,
348 &SemanticHints::default(),
349 )
350}
351
352pub fn analyze_csv_file_with_dimensions_and_hints(
353 file_path: &Path,
354 config: &CsvParserConfig,
355 quality_dimensions: Option<&[QualityDimension]>,
356 semantic_hints: &SemanticHints,
357) -> Result<ProfileReport, DataProfilerError> {
358 let metadata = std::fs::metadata(file_path).map_err(|error| map_io_error(file_path, error))?;
359 let start = std::time::Instant::now();
360
361 let file = std::fs::File::open(file_path).map_err(|error| map_io_error(file_path, error))?;
362 let buf_reader = std::io::BufReader::new(file);
363
364 let (column_profiles, column_stats, rows_read, _header_names) =
365 analyze_csv_from_reader_with_hints(buf_reader, config, semantic_hints)?;
366
367 let file_source = DataSource::File {
368 path: file_path.display().to_string(),
369 format: FileFormat::Csv,
370 size_bytes: metadata.len(),
371 modified_at: None,
372 parquet_metadata: None,
373 };
374
375 if column_profiles.is_empty() && rows_read == 0 {
378 return Ok(ReportAssembler::new(
379 file_source,
380 ExecutionMetadata::new(0, 0, start.elapsed().as_millis()).with_engine("csv"),
381 )
382 .skip_quality()
383 .build());
384 }
385
386 let sample_columns = profile_builder::quality_check_samples(&column_stats);
387 let scan_time_ms = start.elapsed().as_millis();
388 let num_columns = column_profiles.len();
389
390 let mut assembler = ReportAssembler::new(
391 file_source,
392 ExecutionMetadata::new(rows_read, num_columns, scan_time_ms).with_engine("csv"),
393 )
394 .columns(column_profiles)
395 .with_quality_data(sample_columns)
396 .with_row_duplicates(column_stats.row_duplicate_summary())
397 .with_row_completeness(column_stats.row_completeness_summary())
398 .with_exact_value_hint_bindings(column_stats.semantic_hint_bindings())
399 .with_semantic_hints(semantic_hints.clone());
400 if let Some(dimensions) = quality_dimensions {
401 assembler = assembler.with_requested_dimensions(dimensions.to_vec());
402 }
403 Ok(assembler.build())
404}
405
406fn map_io_error(file_path: &Path, error: std::io::Error) -> DataProfilerError {
407 if error.kind() == std::io::ErrorKind::NotFound {
408 DataProfilerError::FileNotFound {
409 path: file_path.display().to_string(),
410 }
411 } else {
412 DataProfilerError::from(error)
413 }
414}
415
416#[cfg(test)]
417mod tests {
418 use super::*;
419 use dataprof_core::DataType;
420 use std::io::{Cursor, Write};
421 use tempfile::NamedTempFile;
422
423 fn write_csv(content: &str) -> NamedTempFile {
424 let mut file = NamedTempFile::new().unwrap();
425 write!(file, "{}", content).unwrap();
426 file.flush().unwrap();
427 file
428 }
429
430 #[test]
431 fn test_analyze_csv_from_reader_basic() {
432 let data = b"name,age\nAlice,25\nBob,30\n";
433 let cursor = Cursor::new(data.as_ref());
434 let config = CsvParserConfig::default();
435
436 let (profiles, _stats, rows, _) = analyze_csv_from_reader(cursor, &config).unwrap();
437 assert_eq!(rows, 2);
438 assert_eq!(profiles.len(), 2);
439
440 let age = profiles
441 .iter()
442 .find(|profile| profile.name == "age")
443 .unwrap();
444 assert_eq!(age.total_count, 2);
445 assert_eq!(age.data_type, DataType::Integer);
446 }
447
448 #[test]
449 fn test_analyze_csv_from_reader_with_max_rows() {
450 let data = b"val\n1\n2\n3\n4\n5\n";
451 let cursor = Cursor::new(data.as_ref());
452 let config = CsvParserConfig::default().max_rows(Some(3));
453
454 let (_profiles, _stats, rows, _) = analyze_csv_from_reader(cursor, &config).unwrap();
455 assert_eq!(rows, 3);
456 }
457
458 #[test]
459 fn test_analyze_csv_from_reader_strict_rejects_ragged() {
460 let data = b"a,b\n1,2\n3,4,5\n";
461 let cursor = Cursor::new(data.as_ref());
462 let config = CsvParserConfig::strict();
463
464 assert!(analyze_csv_from_reader(cursor, &config).is_err());
465 }
466
467 #[test]
468 fn test_analyze_csv_from_reader_flexible_handles_ragged() {
469 let data = b"a,b\n1,2\n3,4,5\n6\n";
470 let cursor = Cursor::new(data.as_ref());
471 let config = CsvParserConfig::default();
472
473 let (profiles, _stats, rows, _) = analyze_csv_from_reader(cursor, &config).unwrap();
474 assert_eq!(rows, 3);
475 assert!(!profiles.is_empty());
476 }
477
478 #[test]
479 fn test_analyze_csv_from_reader_custom_delimiter() {
480 let data = b"a;b\n1;2\n3;4\n";
481 let cursor = Cursor::new(data.as_ref());
482 let config = CsvParserConfig::default().with_delimiter(b';');
483
484 let (profiles, _stats, rows, _) = analyze_csv_from_reader(cursor, &config).unwrap();
485 assert_eq!(rows, 2);
486 assert_eq!(profiles.len(), 2);
487 }
488
489 #[test]
490 fn test_analyze_csv_file_returns_quality_report() {
491 let csv = write_csv("x,y\n1,a\n2,b\n3,c\n");
492 let config = CsvParserConfig::default();
493 let report = analyze_csv_file(csv.path(), &config).unwrap();
494
495 assert_eq!(report.column_profiles.len(), 2);
496 assert_eq!(report.execution.rows_processed, 3);
497 assert!(report.quality_score().unwrap() >= 0.0);
498 assert!(report.quality_score().unwrap() <= 100.0);
499 }
500
501 #[test]
502 fn test_csv_auto_detects_delimiter_from_reader() {
503 let data = b"name;age\nAlice;25\nBob;30\n";
504 let cursor = Cursor::new(data.as_ref());
505 let config = CsvParserConfig::default();
506
507 let (profiles, _stats, rows, _) = analyze_csv_from_reader(cursor, &config).unwrap();
508 assert_eq!(rows, 2);
509 assert_eq!(profiles.len(), 2);
510
511 let age = profiles
512 .iter()
513 .find(|profile| profile.name == "age")
514 .unwrap();
515 assert_eq!(age.total_count, 2);
516 }
517
518 #[test]
519 fn test_analyze_csv_basic() {
520 let csv = write_csv("name,age\nAlice,25\nBob,30\n");
521 let config = CsvParserConfig::default();
522 let report = analyze_csv_file(csv.path(), &config).unwrap();
523 let profiles = &report.column_profiles;
524 assert_eq!(profiles.len(), 2);
525
526 let names: Vec<&str> = profiles
527 .iter()
528 .map(|profile| profile.name.as_str())
529 .collect();
530 assert!(names.contains(&"name"));
531 assert!(names.contains(&"age"));
532
533 let age = profiles
534 .iter()
535 .find(|profile| profile.name == "age")
536 .unwrap();
537 assert_eq!(age.total_count, 2);
538 assert_eq!(age.null_count, 0);
539 }
540
541 #[test]
542 fn test_analyze_csv_with_nulls() {
543 let csv = write_csv("name,age,email\nAlice,25,a@b.com\nBob,,\nCharlie,30,c@d.com\n");
544 let config = CsvParserConfig::default();
545 let report = analyze_csv_file(csv.path(), &config).unwrap();
546 let profiles = &report.column_profiles;
547
548 let age = profiles
549 .iter()
550 .find(|profile| profile.name == "age")
551 .unwrap();
552 assert_eq!(age.total_count, 3);
553 assert_eq!(age.null_count, 1);
554
555 let email = profiles
556 .iter()
557 .find(|profile| profile.name == "email")
558 .unwrap();
559 assert_eq!(email.null_count, 1);
560 }
561
562 #[test]
563 fn test_analyze_csv_file_returns_quality_report_legacy() {
564 let csv = write_csv("x,y\n1,a\n2,b\n3,c\n");
565 let config = CsvParserConfig::default();
566 let report = analyze_csv_file(csv.path(), &config).unwrap();
567
568 assert_eq!(report.column_profiles.len(), 2);
569 assert_eq!(report.execution.rows_processed, 3);
570 assert_eq!(report.execution.columns_detected, 2);
571 assert!(report.quality_score().unwrap() >= 0.0);
572 assert!(report.quality_score().unwrap() <= 100.0);
573 }
574
575 #[test]
576 fn test_analyze_csv_file_header_only_preserves_declared_columns() {
577 let csv = write_csv("name,age\n");
578 let config = CsvParserConfig::default();
579 let report = analyze_csv_file(csv.path(), &config).unwrap();
580
581 assert_eq!(
582 report
583 .column_profiles
584 .iter()
585 .map(|column| (column.name.as_str(), column.data_type.clone()))
586 .collect::<Vec<_>>(),
587 vec![("name", DataType::String), ("age", DataType::String)]
588 );
589 assert_eq!(report.execution.rows_processed, 0);
590 assert!(report.quality.is_some());
591 }
592
593 #[test]
594 fn test_analyze_csv_file_empty_file_has_no_columns() {
595 let csv = write_csv("");
596 let config = CsvParserConfig::default();
597 let report = analyze_csv_file(csv.path(), &config).unwrap();
598
599 assert!(report.column_profiles.is_empty());
600 assert_eq!(report.execution.rows_processed, 0);
601 assert_eq!(report.execution.columns_detected, 0);
602 assert!(report.quality.is_none());
606 }
607
608 #[test]
609 fn test_analyze_csv_with_max_rows_small_file() {
610 let csv = write_csv("val\n1\n2\n3\n4\n5\n");
611 let config = CsvParserConfig::default();
612 let report = analyze_csv_file(csv.path(), &config).unwrap();
613
614 assert_eq!(report.execution.rows_processed, 5);
615 assert_eq!(report.execution.columns_detected, 1);
616 assert!((report.execution.sampling_ratio.unwrap_or(1.0) - 1.0).abs() < 0.01);
617 }
618
619 #[test]
620 fn test_analyze_csv_numeric_types_detected() {
621 let csv = write_csv("int_col,float_col,str_col\n1,1.5,hello\n2,2.5,world\n3,3.5,foo\n");
622 let config = CsvParserConfig::default();
623 let report = analyze_csv_file(csv.path(), &config).unwrap();
624 let profiles = &report.column_profiles;
625
626 let int_col = profiles
627 .iter()
628 .find(|profile| profile.name == "int_col")
629 .unwrap();
630 assert!(matches!(int_col.data_type, DataType::Integer));
631
632 let float_col = profiles
633 .iter()
634 .find(|profile| profile.name == "float_col")
635 .unwrap();
636 assert!(matches!(float_col.data_type, DataType::Float));
637
638 let str_col = profiles
639 .iter()
640 .find(|profile| profile.name == "str_col")
641 .unwrap();
642 assert!(matches!(str_col.data_type, DataType::String));
643 }
644
645 #[test]
646 fn test_analyze_csv_strict_rejects_ragged_rows() {
647 let csv = write_csv("a,b\n1,2\n3,4,5\n");
648 let config = CsvParserConfig::strict();
649 assert!(analyze_csv_file(csv.path(), &config).is_err());
650 }
651
652 #[test]
653 fn test_analyze_csv_file_handles_ragged_rows() {
654 let csv = write_csv("a,b\n1,2\n3,4,5\n6\n");
655 let config = CsvParserConfig::default();
656 let report = analyze_csv_file(csv.path(), &config).unwrap();
657 assert!(!report.column_profiles.is_empty());
658 }
659
660 #[test]
661 fn test_analyze_csv_with_quality_metrics() {
662 let csv = write_csv(
663 "name,age\n\
664 Alice,25\nBob,\nCharlie,30\nDave,\nEve,28\n\
665 Frank,\nGrace,35\nHeidi,40\nIvan,\nJudy,22\n",
666 );
667 let config = CsvParserConfig::default();
668 let report = analyze_csv_file(csv.path(), &config).unwrap();
669
670 let age = report
671 .column_profiles
672 .iter()
673 .find(|profile| profile.name == "age")
674 .unwrap();
675 assert_eq!(age.null_count, 4);
676 assert_eq!(age.total_count, 10);
677 assert!(report.quality_score().unwrap() >= 0.0);
678 }
679
680 #[test]
681 fn test_detect_delimiter_comma() {
682 let data = b"name,age,city\nAlice,25,NYC\nBob,30,London\n";
683 assert_eq!(detect_delimiter(Cursor::new(data.as_ref())).unwrap(), b',');
684 }
685
686 #[test]
687 fn test_detect_delimiter_semicolon() {
688 let data = b"name;age;city\nAlice;25;NYC\nBob;30;London\n";
689 assert_eq!(detect_delimiter(Cursor::new(data.as_ref())).unwrap(), b';');
690 }
691
692 #[test]
693 fn test_detect_delimiter_pipe() {
694 let data = b"name|age|city\nAlice|25|NYC\nBob|30|London\n";
695 assert_eq!(detect_delimiter(Cursor::new(data.as_ref())).unwrap(), b'|');
696 }
697
698 #[test]
699 fn test_detect_delimiter_tab() {
700 let data = b"name\tage\tcity\nAlice\t25\tNYC\nBob\t30\tLondon\n";
701 assert_eq!(detect_delimiter(Cursor::new(data.as_ref())).unwrap(), b'\t');
702 }
703
704 #[test]
705 fn test_detect_delimiter_from_path_semicolon() {
706 let csv = write_csv("id;name;salary\n1;Alice;50000\n2;Bob;60000\n");
707 assert_eq!(detect_delimiter_from_path(csv.path()).unwrap(), b';');
708 }
709
710 #[test]
711 fn test_detect_delimiter_blank_records_do_not_outvote_header() {
712 let data = b"name,age\n\n\n";
717 assert_eq!(detect_delimiter(Cursor::new(data.as_ref())).unwrap(), b',');
718
719 let semicolon = b"name;age\n\n";
720 assert_eq!(
721 detect_delimiter(Cursor::new(semicolon.as_ref())).unwrap(),
722 b';'
723 );
724 }
725
726 #[test]
727 fn test_detect_delimiter_quoted_fields() {
728 let data = b"name;desc;val\n\"hello;world\";foo;1\nbar;baz;2\n";
729 assert_eq!(detect_delimiter(Cursor::new(data.as_ref())).unwrap(), b';');
730 }
731
732 #[test]
733 fn test_detect_delimiter_quoted_embedded_newline() {
734 let data = b"id,comment\n1,\"line one\nline two\"\n2,\"normal\"\n3,\"has \"\"quotes\"\" inside\"\n";
739 assert_eq!(detect_delimiter(Cursor::new(data.as_ref())).unwrap(), b',');
740 }
741
742 #[test]
743 fn test_detect_delimiter_absent_candidate_never_beats_real_one() {
744 let data = b"a,b,c\n1,2,3\nragged line without delimiters\n4,5,6\n";
747 assert_eq!(detect_delimiter(Cursor::new(data.as_ref())).unwrap(), b',');
748 }
749
750 #[test]
751 fn test_detect_delimiter_single_column_falls_back_to_comma() {
752 let data = b"value\n1\n2\n3\n";
753 assert_eq!(detect_delimiter(Cursor::new(data.as_ref())).unwrap(), b',');
754 }
755
756 #[test]
757 fn test_detect_delimiter_two_column_semicolon_with_commas_in_text() {
758 let data = b"id;note\n1;hello, world\n2;foo, bar, baz\n3;plain\n";
760 assert_eq!(detect_delimiter(Cursor::new(data.as_ref())).unwrap(), b';');
761 }
762
763 #[test]
764 fn test_split_sample_records_quote_aware() {
765 let (records, last_unterminated) = split_sample_records("a,b\r\n1,\"x\ny\"\n2,z\n");
766 assert_eq!(records, vec!["a,b", "1,\"x\ny\"", "2,z"]);
767 assert!(!last_unterminated);
768
769 let (records, last_unterminated) = split_sample_records("a,b\n1,\"cut mid-fie");
770 assert_eq!(records, vec!["a,b", "1,\"cut mid-fie"]);
771 assert!(last_unterminated);
772 }
773
774 #[test]
775 fn test_detect_delimiter_full_sample_ending_on_record_boundary() {
776 let mut data = Vec::from(&b"a;b;c\n"[..]);
779 while data.len() < 4090 {
780 data.extend_from_slice(b"1;2;3\n");
781 }
782 while data.len() < 4095 {
783 data.push(b'#');
784 }
785 data.push(b'\n');
786 assert_eq!(data.len(), 4096);
787 assert_eq!(
788 detect_delimiter(Cursor::new(data.as_slice())).unwrap(),
789 b';'
790 );
791 }
792}