1use anyhow::{Context, Result};
2use csv::{ReaderBuilder, Trim};
3use std::io::{BufRead, BufReader, Cursor};
4use std::path::Path;
5
6use dataprof_core::errors::AutoRecoveryManager;
7use dataprof_core::execution::{RecoveryEvent, RecoveryKind};
8use dataprof_core::{DataProfilerError, RecoveryStrategy, RetryConfig};
9
10pub type RobustParseResult = (Option<Vec<String>>, Vec<Vec<String>>);
12
13#[derive(Debug)]
18pub struct CsvParseOutput {
19 pub headers: Vec<String>,
20 pub records: Vec<Vec<String>>,
21 pub recovery_events: Vec<RecoveryEvent>,
22}
23
24#[derive(Debug, Clone, PartialEq, Eq)]
26pub struct EncodingDiagnostic {
27 pub guess: &'static str,
29 pub first_invalid_byte: Option<usize>,
33}
34
35impl EncodingDiagnostic {
36 pub fn detail(&self) -> String {
38 match self.first_invalid_byte {
41 Some(offset) => format!(
42 "looks like {}; first invalid UTF-8 byte at offset {offset}",
43 self.guess
44 ),
45 None => format!(
46 "looks like {}; contains bytes that are not valid UTF-8",
47 self.guess
48 ),
49 }
50 }
51}
52
53const ENCODING_SNIFF_LIMIT: usize = 1 << 20; pub fn diagnose_non_utf8(path: &Path) -> Option<EncodingDiagnostic> {
65 use std::io::Read;
66
67 let file = std::fs::File::open(path).ok()?;
71 let mut sample = Vec::new();
72 file.take(ENCODING_SNIFF_LIMIT as u64)
75 .read_to_end(&mut sample)
76 .ok()?;
77
78 if sample.starts_with(&[0xFF, 0xFE]) {
79 return Some(EncodingDiagnostic {
80 guess: "utf-16le",
81 first_invalid_byte: Some(0),
82 });
83 }
84 if sample.starts_with(&[0xFE, 0xFF]) {
85 return Some(EncodingDiagnostic {
86 guess: "utf-16be",
87 first_invalid_byte: Some(0),
88 });
89 }
90
91 let first_invalid_byte = match std::str::from_utf8(&sample) {
94 Ok(_) => return None,
95 Err(err) => Some(err.valid_up_to()),
96 };
97
98 let has_c1_controls = sample.iter().any(|&b| (0x80..=0x9F).contains(&b));
99 let guess = if has_c1_controls {
100 "windows-1252"
101 } else {
102 "latin-1"
103 };
104
105 Some(EncodingDiagnostic {
106 guess,
107 first_invalid_byte,
108 })
109}
110
111pub struct RobustCsvParser {
113 pub flexible: bool,
114 pub quote_char: Option<u8>,
115 pub delimiter: Option<u8>,
116 pub allow_variable_columns: bool,
117 pub trim_whitespace: bool,
118 pub auto_recovery: bool,
119 pub retry_config: RetryConfig,
120 pub verbosity: u8,
121}
122
123impl Default for RobustCsvParser {
124 fn default() -> Self {
125 Self {
126 flexible: true,
127 quote_char: Some(b'"'),
128 delimiter: None,
129 allow_variable_columns: true,
130 trim_whitespace: true,
131 auto_recovery: true,
132 retry_config: RetryConfig::default(),
133 verbosity: 1,
134 }
135 }
136}
137
138impl RobustCsvParser {
139 pub fn new() -> Self {
140 Self::default()
141 }
142
143 pub fn flexible(mut self, flexible: bool) -> Self {
144 self.flexible = flexible;
145 self
146 }
147
148 pub fn allow_variable_columns(mut self, allow: bool) -> Self {
149 self.allow_variable_columns = allow;
150 self
151 }
152
153 pub fn verbosity(mut self, level: u8) -> Self {
156 self.verbosity = level;
157 self
158 }
159
160 pub fn with_auto_recovery(mut self, enabled: bool) -> Self {
162 self.auto_recovery = enabled;
163 self
164 }
165
166 pub fn with_retry_config(mut self, config: RetryConfig) -> Self {
168 self.retry_config = config;
169 self
170 }
171
172 pub fn detect_encoding(&self, file_path: &Path) -> Result<&'static str> {
174 let mut file = std::fs::File::open(file_path)?;
175 let mut buffer = [0; 4096];
176 let bytes_read = std::io::Read::read(&mut file, &mut buffer)?;
177
178 let sample = &buffer[..bytes_read];
179
180 if sample.starts_with(&[0xEF, 0xBB, 0xBF]) {
181 return Ok("utf-8");
182 }
183 if sample.starts_with(&[0xFF, 0xFE]) {
184 return Ok("utf-16le");
185 }
186 if sample.starts_with(&[0xFE, 0xFF]) {
187 return Ok("utf-16be");
188 }
189
190 if String::from_utf8(sample.to_vec()).is_ok() {
191 return Ok("utf-8");
192 }
193
194 let latin1_indicators = [
195 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8A, 0x8B, 0x8C, 0x8D,
196 0x8E, 0x8F,
197 ];
198 let has_latin1_chars = sample.iter().any(|&byte| latin1_indicators.contains(&byte));
199
200 if has_latin1_chars {
201 Ok("latin1")
202 } else {
203 Ok("utf-8")
204 }
205 }
206
207 pub fn parse_csv_with_recovery(&self, file_path: &Path) -> Result<CsvParseOutput> {
209 if !self.auto_recovery {
210 return self.parse_csv(file_path);
211 }
212
213 let mut recovery_manager = AutoRecoveryManager::new(self.retry_config.clone());
214 let mut recovery_events = Vec::new();
215
216 match self.parse_csv_recording(file_path, &mut recovery_events) {
217 Ok((headers, records)) => Ok(CsvParseOutput {
218 headers,
219 records,
220 recovery_events,
221 }),
222 Err(initial_error) => {
223 log::warn!(
224 "Initial CSV parsing failed: {}. Attempting auto-recovery...",
225 initial_error
226 );
227
228 let initial_error_string = format!("{initial_error:#}");
229 let dp_error = if let Ok(dp_err) = initial_error.downcast::<DataProfilerError>() {
230 dp_err
231 } else {
232 DataProfilerError::csv_parsing(
233 &initial_error_string,
234 Some(file_path.to_string_lossy().as_ref()),
235 )
236 };
237
238 if matches!(dp_error, DataProfilerError::DuplicateColumnName { .. }) {
242 return Err(dp_error.into());
243 }
244
245 let (headers, records) = recovery_manager
246 .attempt_recovery(dp_error, |strategy| {
247 self.try_recovery_strategy(file_path, strategy)
248 })?;
249 let mut attempted = recovery_events
250 .last()
251 .map(|event| event.retry.clone())
252 .unwrap_or_else(|| "configured_parse".to_string());
253 let mut error = initial_error_string;
254 for attempt in recovery_manager.get_recovery_log() {
255 let retry = match &attempt.strategy {
258 RecoveryStrategy::EncodingConversion { .. }
259 | RecoveryStrategy::FlexibleParsing => "flexible".to_string(),
260 strategy => format!("{strategy:?}"),
261 };
262 recovery_events.push(RecoveryEvent {
263 kind: RecoveryKind::CsvAutoRecovery,
264 attempted,
265 retry: retry.clone(),
266 error,
267 });
268 attempted = retry;
269 error = match &attempt.error_message {
270 Some(message) => message.clone(),
271 None => break, };
273 }
274 Ok(CsvParseOutput {
275 headers,
276 records,
277 recovery_events,
278 })
279 }
280 }
281 }
282
283 fn try_recovery_strategy(
284 &self,
285 file_path: &Path,
286 strategy: RecoveryStrategy,
287 ) -> Result<(Vec<String>, Vec<Vec<String>>), DataProfilerError> {
288 match strategy {
289 RecoveryStrategy::DelimiterDetection { delimiter } => {
290 log::debug!("Trying delimiter: '{}'", delimiter);
291 self.parse_with_override(file_path, Some(delimiter as u8), None, None)
292 .map_err(|error| {
293 DataProfilerError::csv_parsing(
294 &error.to_string(),
295 Some(file_path.to_string_lossy().as_ref()),
296 )
297 })
298 }
299 RecoveryStrategy::EncodingConversion { from: _, to: _ } => {
300 log::debug!(
301 "Attempting encoding conversion (placeholder - full implementation needed)"
302 );
303 self.parse_with_override(file_path, None, Some(true), Some(true))
304 .map_err(|error| {
305 DataProfilerError::csv_parsing(
306 &error.to_string(),
307 Some(file_path.to_string_lossy().as_ref()),
308 )
309 })
310 }
311 RecoveryStrategy::FlexibleParsing => {
312 log::debug!("Enabling flexible parsing");
313 self.parse_with_override(file_path, None, Some(true), Some(true))
314 .map_err(|error| {
315 DataProfilerError::csv_parsing(
316 &error.to_string(),
317 Some(file_path.to_string_lossy().as_ref()),
318 )
319 })
320 }
321 _ => Err(DataProfilerError::csv_parsing(
322 "Unsupported recovery strategy for CSV parsing",
323 Some(file_path.to_string_lossy().as_ref()),
324 )),
325 }
326 }
327
328 fn parse_with_override(
329 &self,
330 file_path: &Path,
331 delimiter_override: Option<u8>,
332 flexible_override: Option<bool>,
333 allow_variable_override: Option<bool>,
334 ) -> Result<(Vec<String>, Vec<Vec<String>>)> {
335 let delimiter = delimiter_override
336 .or(self.delimiter)
337 .unwrap_or_else(|| self.detect_delimiter(file_path).unwrap_or(b','));
338
339 let flexible = flexible_override.unwrap_or(self.flexible);
340 let allow_variable = allow_variable_override.unwrap_or(self.allow_variable_columns);
341
342 self.parse_with_config(file_path, delimiter, flexible, allow_variable)
343 }
344
345 fn parse_with_config(
346 &self,
347 file_path: &Path,
348 delimiter: u8,
349 flexible: bool,
350 _allow_variable: bool,
351 ) -> Result<(Vec<String>, Vec<Vec<String>>)> {
352 let mut reader = ReaderBuilder::new()
353 .delimiter(delimiter)
354 .has_headers(true)
355 .flexible(flexible)
356 .quote(self.quote_char.unwrap_or(b'"'))
357 .trim(if self.trim_whitespace {
358 Trim::All
359 } else {
360 Trim::None
361 })
362 .from_path(file_path)?;
363
364 let headers: Vec<String> = reader
365 .headers()?
366 .iter()
367 .map(|value| value.to_string())
368 .collect();
369 let expected_field_count = headers.len();
370 let mut records = Vec::new();
371
372 if flexible {
373 for result in reader.records() {
374 match result {
375 Ok(record) => {
376 let mut row: Vec<String> =
377 record.iter().map(|value| value.to_string()).collect();
378 if row.len() < expected_field_count {
379 row.resize(expected_field_count, String::new());
380 } else if row.len() > expected_field_count {
381 row.truncate(expected_field_count);
382 }
383 records.push(row);
384 }
385 Err(_) => continue,
386 }
387 }
388 } else {
389 for result in reader.records() {
390 let record = result?;
391 let row: Vec<String> = record.iter().map(|value| value.to_string()).collect();
392 records.push(row);
393 }
394 }
395
396 Ok((headers, records))
397 }
398
399 pub fn detect_delimiter(&self, file_path: &Path) -> Result<u8> {
404 Ok(crate::detect_delimiter_from_path(file_path)?)
405 }
406
407 pub fn parse_csv(&self, file_path: &Path) -> Result<CsvParseOutput> {
409 let mut recovery_events = Vec::new();
410 let (headers, records) = self.parse_csv_recording(file_path, &mut recovery_events)?;
411 Ok(CsvParseOutput {
412 headers,
413 records,
414 recovery_events,
415 })
416 }
417
418 fn parse_csv_recording(
419 &self,
420 file_path: &Path,
421 recovery_events: &mut Vec<RecoveryEvent>,
422 ) -> Result<(Vec<String>, Vec<Vec<String>>)> {
423 let delimiter = if let Some(delimiter) = self.delimiter {
424 delimiter
425 } else {
426 self.detect_delimiter(file_path)
427 .context("Failed to detect delimiter")?
428 };
429
430 let result = match self.try_strict_parsing(file_path, delimiter) {
431 Ok(result) => result,
432 Err(error) => {
433 recovery_events.push(RecoveryEvent {
434 kind: RecoveryKind::CsvAutoRecovery,
435 attempted: "strict".to_string(),
436 retry: "flexible".to_string(),
437 error: error.to_string(),
438 });
439 if self.verbosity >= 2 {
440 log::info!("Using flexible CSV parsing (strict mode failed: {})", error);
441 }
442 self.try_flexible_parsing(file_path, delimiter)
443 .context("Both strict and flexible CSV parsing failed")?
444 }
445 };
446
447 dataprof_core::validate_unique_column_names(&result.0, "CSV header")?;
450 Ok(result)
451 }
452
453 fn try_strict_parsing(
454 &self,
455 file_path: &Path,
456 delimiter: u8,
457 ) -> Result<(Vec<String>, Vec<Vec<String>>)> {
458 let mut reader = ReaderBuilder::new()
459 .delimiter(delimiter)
460 .has_headers(true)
461 .flexible(false)
462 .quote(self.quote_char.unwrap_or(b'"'))
463 .trim(if self.trim_whitespace {
464 Trim::All
465 } else {
466 Trim::None
467 })
468 .from_path(file_path)?;
469
470 let headers: Vec<String> = reader
471 .headers()?
472 .iter()
473 .map(|value| value.to_string())
474 .collect();
475 let mut records = Vec::new();
476
477 for result in reader.records() {
478 let record = result?;
479 let row: Vec<String> = record.iter().map(|value| value.to_string()).collect();
480 records.push(row);
481 }
482
483 Ok((headers, records))
484 }
485
486 fn try_flexible_parsing(
487 &self,
488 file_path: &Path,
489 delimiter: u8,
490 ) -> Result<(Vec<String>, Vec<Vec<String>>)> {
491 let mut reader = ReaderBuilder::new()
492 .delimiter(delimiter)
493 .has_headers(true)
494 .flexible(true)
495 .quote(self.quote_char.unwrap_or(b'"'))
496 .trim(if self.trim_whitespace {
497 Trim::All
498 } else {
499 Trim::None
500 })
501 .from_path(file_path)?;
502
503 let headers: Vec<String> = reader
504 .headers()?
505 .iter()
506 .map(|value| value.to_string())
507 .collect();
508 let expected_field_count = headers.len();
509 let mut records = Vec::new();
510 let mut error_count = 0;
511 let max_errors = 100;
512
513 for (row_index, result) in reader.records().enumerate() {
514 match result {
515 Ok(record) => {
516 let mut row: Vec<String> =
517 record.iter().map(|value| value.to_string()).collect();
518
519 if self.allow_variable_columns {
520 if row.len() < expected_field_count {
521 row.resize(expected_field_count, String::new());
522 } else if row.len() > expected_field_count {
523 if row.len() == expected_field_count + 1
524 && let (Some(second_last), Some(last)) = (
525 row.get(expected_field_count - 1),
526 row.get(expected_field_count),
527 )
528 {
529 let combined =
530 format!("{}{}{}", second_last, delimiter as char, last);
531 row[expected_field_count - 1] = combined;
532 }
533 row.truncate(expected_field_count);
534 }
535 }
536
537 records.push(row);
538 }
539 Err(error) => {
540 error_count += 1;
541
542 let file_path_str = file_path.to_string_lossy();
543 let enhanced_error = DataProfilerError::csv_parsing(
544 &error.to_string(),
545 Some(file_path_str.as_ref()),
546 );
547 log::warn!("Row {}: {}", row_index + 2, enhanced_error);
548
549 if error_count > max_errors {
550 return Err(DataProfilerError::csv_parsing(
551 &format!("Too many parsing errors ({})", error_count),
552 Some(file_path_str.as_ref()),
553 )
554 .into());
555 }
556 }
557 }
558 }
559
560 if error_count > 0 {
561 log::warn!(
562 "Successfully parsed {} rows with {} errors skipped.",
563 records.len(),
564 error_count
565 );
566 }
567
568 Ok((headers, records))
569 }
570
571 pub fn parse_csv_chunk(
573 &self,
574 data: &str,
575 delimiter: u8,
576 has_headers: bool,
577 ) -> Result<RobustParseResult> {
578 let mut reader = ReaderBuilder::new()
579 .delimiter(delimiter)
580 .has_headers(has_headers)
581 .flexible(self.flexible)
582 .quote(self.quote_char.unwrap_or(b'"'))
583 .trim(if self.trim_whitespace {
584 Trim::All
585 } else {
586 Trim::None
587 })
588 .from_reader(Cursor::new(data));
589
590 let headers: Option<Vec<String>> = if has_headers {
591 let names: Vec<String> = reader
592 .headers()?
593 .iter()
594 .map(|value| value.to_string())
595 .collect();
596 dataprof_core::validate_unique_column_names(&names, "CSV header")?;
597 Some(names)
598 } else {
599 None
600 };
601
602 let mut records = Vec::new();
603 let mut error_count = 0;
604
605 for (row_index, result) in reader.records().enumerate() {
606 match result {
607 Ok(record) => {
608 let row: Vec<String> = record.iter().map(|value| value.to_string()).collect();
609 records.push(row);
610 }
611 Err(error) => {
612 error_count += 1;
613 if error_count <= 10 {
614 log::warn!("Chunk row {}: {}", row_index + 1, error);
615 }
616 }
617 }
618 }
619
620 Ok((headers, records))
621 }
622
623 pub fn validate_csv(&self, file_path: &Path) -> Result<CsvDiagnostics> {
625 let file = std::fs::File::open(file_path)?;
626 let reader = BufReader::new(file);
627 let mut diagnostics = CsvDiagnostics::default();
628
629 let mut field_counts = std::collections::HashMap::new();
630
631 for line_result in reader.lines() {
632 let line = line_result?;
633
634 if line.trim().is_empty() {
635 diagnostics.empty_lines += 1;
636 continue;
637 }
638
639 if line.contains('\0') {
640 diagnostics.null_bytes += 1;
641 }
642
643 let quote_count = line.chars().filter(|&ch| ch == '"').count();
644 if quote_count % 2 != 0 {
645 diagnostics.unmatched_quotes += 1;
646 }
647
648 let delimiter = self.delimiter.unwrap_or(b',') as char;
649 let field_count = crate::count_fields(&line, delimiter);
650 *field_counts.entry(field_count).or_insert(0) += 1;
651
652 diagnostics.total_lines += 1;
653 }
654
655 if let Some((&most_common_fields, &count)) =
656 field_counts.iter().max_by_key(|(_, count)| *count)
657 {
658 diagnostics.most_common_field_count = most_common_fields;
659 diagnostics.consistent_field_rows = count;
660 }
661
662 diagnostics.field_count_variations = field_counts.len();
663 diagnostics.inconsistent_field_rows =
664 diagnostics.total_lines - diagnostics.consistent_field_rows;
665
666 Ok(diagnostics)
667 }
668}
669
670#[derive(Debug, Default)]
671pub struct CsvDiagnostics {
672 pub total_lines: usize,
673 pub empty_lines: usize,
674 pub most_common_field_count: usize,
675 pub consistent_field_rows: usize,
676 pub inconsistent_field_rows: usize,
677 pub field_count_variations: usize,
678 pub unmatched_quotes: usize,
679 pub null_bytes: usize,
680}
681
682impl CsvDiagnostics {
683 pub fn print_summary(&self) {
684 println!("CSV File Diagnostics:");
685 println!(" Total lines: {}", self.total_lines);
686 println!(" Empty lines: {}", self.empty_lines);
687 println!(
688 " Most common field count: {}",
689 self.most_common_field_count
690 );
691 println!(
692 " Consistent rows: {} ({:.1}%)",
693 self.consistent_field_rows,
694 self.consistent_field_rows as f64 / self.total_lines as f64 * 100.0
695 );
696
697 if self.inconsistent_field_rows > 0 {
698 println!(
699 " WARNING: Inconsistent rows: {} ({:.1}%)",
700 self.inconsistent_field_rows,
701 self.inconsistent_field_rows as f64 / self.total_lines as f64 * 100.0
702 );
703 }
704
705 if self.field_count_variations > 1 {
706 println!(
707 " WARNING: Different field counts detected: {}",
708 self.field_count_variations
709 );
710 }
711
712 if self.unmatched_quotes > 0 {
713 println!(
714 " WARNING: Lines with unmatched quotes: {}",
715 self.unmatched_quotes
716 );
717 }
718
719 if self.null_bytes > 0 {
720 println!(" WARNING: Lines with null bytes: {}", self.null_bytes);
721 }
722 }
723}
724
725#[cfg(test)]
726mod tests {
727 use super::*;
728 use std::io::Write;
729 use tempfile::NamedTempFile;
730
731 fn write_bytes(bytes: &[u8]) -> NamedTempFile {
732 let mut f = NamedTempFile::new().unwrap();
733 f.write_all(bytes).unwrap();
734 f.flush().unwrap();
735 f
736 }
737
738 #[test]
739 fn diagnose_non_utf8_flags_latin1() {
740 let f = write_bytes(b"name\ncaf\xE9\n");
744 let d = diagnose_non_utf8(f.path()).expect("latin-1 should be flagged");
745 assert_eq!(d.guess, "latin-1");
746 assert_eq!(d.first_invalid_byte, Some(8));
747 }
748
749 #[test]
750 fn diagnose_non_utf8_flags_windows1252() {
751 let f = write_bytes(b"quote\n\x93hi\x94\n");
753 let d = diagnose_non_utf8(f.path()).expect("windows-1252 should be flagged");
754 assert_eq!(d.guess, "windows-1252");
755 assert_eq!(d.first_invalid_byte, Some(6));
756 }
757
758 #[test]
759 fn diagnose_non_utf8_flags_utf16_bom() {
760 let f = write_bytes(&[0xFF, 0xFE, b'a', 0x00]);
761 let d = diagnose_non_utf8(f.path()).expect("utf-16 BOM should be flagged");
762 assert_eq!(d.guess, "utf-16le");
763 }
764
765 #[test]
766 fn diagnose_non_utf8_ignores_clean_utf8() {
767 let f = write_bytes("name\ncafé\n".as_bytes());
768 assert!(diagnose_non_utf8(f.path()).is_none());
769 }
770
771 #[test]
772 fn encoding_diagnostic_detail_never_leaks_bytes() {
773 let d = EncodingDiagnostic {
774 guess: "latin-1",
775 first_invalid_byte: Some(8),
776 };
777 let detail = d.detail();
778 assert!(detail.contains("latin-1"));
779 assert!(detail.contains("offset 8"));
780 }
781
782 #[test]
783 fn test_robust_csv_parser_basic() -> Result<()> {
784 let mut temp_file = NamedTempFile::new()?;
785 writeln!(temp_file, "name,age,city")?;
786 writeln!(temp_file, "Alice,25,New York")?;
787 writeln!(temp_file, "Bob,30,London")?;
788 temp_file.flush()?;
789
790 let parser = RobustCsvParser::new();
791 let CsvParseOutput {
792 headers,
793 records,
794 recovery_events,
795 } = parser.parse_csv(temp_file.path())?;
796 assert!(recovery_events.is_empty());
797
798 assert_eq!(headers, vec!["name", "age", "city"]);
799 assert_eq!(records.len(), 2);
800 assert_eq!(records[0], vec!["Alice", "25", "New York"]);
801
802 Ok(())
803 }
804
805 #[test]
806 fn test_flexible_field_counts() -> Result<()> {
807 let mut temp_file = NamedTempFile::new()?;
808 writeln!(temp_file, "name,age,city")?;
809 writeln!(temp_file, "Alice,25,New York")?;
810 writeln!(temp_file, "Bob,30")?;
811 writeln!(temp_file, "Charlie,35,London,UK")?;
812 temp_file.flush()?;
813
814 let mut parser = RobustCsvParser::new();
815 parser.delimiter = Some(b',');
816 let parser = parser.allow_variable_columns(true);
817 let CsvParseOutput {
818 headers,
819 records,
820 recovery_events,
821 } = parser.parse_csv(temp_file.path())?;
822 assert_eq!(recovery_events.len(), 1);
823 assert_eq!(recovery_events[0].kind, RecoveryKind::CsvAutoRecovery);
824 assert_eq!(recovery_events[0].attempted, "strict");
825 assert_eq!(recovery_events[0].retry, "flexible");
826 assert!(recovery_events[0].error.contains("fields"));
827
828 assert_eq!(headers, vec!["name", "age", "city"]);
829 assert_eq!(records.len(), 3);
830 assert_eq!(records[1], vec!["Bob", "30", ""]);
831 assert_eq!(records[2], vec!["Charlie", "35", "London,UK"]);
832
833 Ok(())
834 }
835
836 #[test]
837 fn test_delimiter_detection() -> Result<()> {
838 let mut temp_file = NamedTempFile::new()?;
839 writeln!(temp_file, "name;age;city")?;
840 writeln!(temp_file, "Alice;25;New York")?;
841 writeln!(temp_file, "Bob;30;London")?;
842 temp_file.flush()?;
843
844 let parser = RobustCsvParser::new();
845 let delimiter = parser.detect_delimiter(temp_file.path())?;
846 assert!(delimiter == b';' || delimiter == b',');
847
848 Ok(())
849 }
850
851 #[test]
852 fn auto_recovery_records_initial_failure_and_every_retry() -> Result<()> {
853 let mut file = NamedTempFile::new()?;
854 writeln!(file, "name,value")?;
855 for _ in 0..101 {
858 file.write_all(b"bad,\xff\n")?;
859 }
860 writeln!(file, "good,42")?;
861 file.flush()?;
862
863 let mut parser = RobustCsvParser::new();
864 parser.delimiter = Some(b',');
865 parser.flexible = false;
868 parser.retry_config.max_attempts = 5;
869 let output = parser.parse_csv_with_recovery(file.path())?;
870 assert_eq!(output.records, vec![vec!["good", "42"]]);
871 assert_eq!(output.recovery_events.len(), 6);
872 assert_eq!(output.recovery_events[0].attempted, "strict");
873 assert_eq!(output.recovery_events[0].retry, "flexible");
874 assert_eq!(output.recovery_events[1].attempted, "flexible");
875 assert!(
876 output.recovery_events[1]
877 .error
878 .contains("Both strict and flexible")
879 );
880 for pair in output.recovery_events.windows(2) {
881 assert_eq!(pair[0].retry, pair[1].attempted);
882 }
883 assert_eq!(output.recovery_events.last().unwrap().retry, "flexible");
884 assert!(
885 output
886 .recovery_events
887 .iter()
888 .all(|event| !event.retry.contains("EncodingConversion"))
889 );
890 assert!(
891 output
892 .recovery_events
893 .iter()
894 .all(|event| !event.error.is_empty())
895 );
896 assert!(
897 parser
898 .with_auto_recovery(false)
899 .parse_csv_with_recovery(file.path())
900 .is_err()
901 );
902 Ok(())
903 }
904
905 #[test]
906 fn test_csv_diagnostics() -> Result<()> {
907 let mut temp_file = NamedTempFile::new()?;
908 writeln!(temp_file, "name,age,city")?;
909 writeln!(temp_file, "Alice,25,New York")?;
910 writeln!(temp_file, "Bob,30")?;
911 writeln!(temp_file)?;
912 writeln!(temp_file, "Charlie,35,London")?;
913 temp_file.flush()?;
914
915 let parser = RobustCsvParser::new();
916 let diagnostics = parser.validate_csv(temp_file.path())?;
917
918 assert_eq!(diagnostics.total_lines, 4);
919 assert_eq!(diagnostics.empty_lines, 1);
920 assert_eq!(diagnostics.most_common_field_count, 3);
921 assert_eq!(diagnostics.inconsistent_field_rows, 1);
922
923 Ok(())
924 }
925}