1use anyhow::{Context, Result};
2use csv::{ReaderBuilder, Trim};
3use std::io::{BufRead, BufReader, Cursor};
4use std::path::Path;
5
6use dataprof_core::errors::AutoRecoveryManager;
7use dataprof_core::{DataProfilerError, RecoveryStrategy, RetryConfig};
8
9pub type RobustParseResult = (Option<Vec<String>>, Vec<Vec<String>>);
11
12#[derive(Debug, Clone, PartialEq, Eq)]
14pub struct EncodingDiagnostic {
15 pub guess: &'static str,
17 pub first_invalid_byte: Option<usize>,
21}
22
23impl EncodingDiagnostic {
24 pub fn detail(&self) -> String {
26 match self.first_invalid_byte {
29 Some(offset) => format!(
30 "looks like {}; first invalid UTF-8 byte at offset {offset}",
31 self.guess
32 ),
33 None => format!(
34 "looks like {}; contains bytes that are not valid UTF-8",
35 self.guess
36 ),
37 }
38 }
39}
40
41const ENCODING_SNIFF_LIMIT: usize = 1 << 20; pub fn diagnose_non_utf8(path: &Path) -> Option<EncodingDiagnostic> {
53 use std::io::Read;
54
55 let file = std::fs::File::open(path).ok()?;
59 let mut sample = Vec::new();
60 file.take(ENCODING_SNIFF_LIMIT as u64)
63 .read_to_end(&mut sample)
64 .ok()?;
65
66 if sample.starts_with(&[0xFF, 0xFE]) {
67 return Some(EncodingDiagnostic {
68 guess: "utf-16le",
69 first_invalid_byte: Some(0),
70 });
71 }
72 if sample.starts_with(&[0xFE, 0xFF]) {
73 return Some(EncodingDiagnostic {
74 guess: "utf-16be",
75 first_invalid_byte: Some(0),
76 });
77 }
78
79 let first_invalid_byte = match std::str::from_utf8(&sample) {
82 Ok(_) => return None,
83 Err(err) => Some(err.valid_up_to()),
84 };
85
86 let has_c1_controls = sample.iter().any(|&b| (0x80..=0x9F).contains(&b));
87 let guess = if has_c1_controls {
88 "windows-1252"
89 } else {
90 "latin-1"
91 };
92
93 Some(EncodingDiagnostic {
94 guess,
95 first_invalid_byte,
96 })
97}
98
99pub struct RobustCsvParser {
101 pub flexible: bool,
102 pub quote_char: Option<u8>,
103 pub delimiter: Option<u8>,
104 pub allow_variable_columns: bool,
105 pub trim_whitespace: bool,
106 pub auto_recovery: bool,
107 pub retry_config: RetryConfig,
108 pub verbosity: u8,
109}
110
111impl Default for RobustCsvParser {
112 fn default() -> Self {
113 Self {
114 flexible: true,
115 quote_char: Some(b'"'),
116 delimiter: None,
117 allow_variable_columns: true,
118 trim_whitespace: true,
119 auto_recovery: true,
120 retry_config: RetryConfig::default(),
121 verbosity: 1,
122 }
123 }
124}
125
126impl RobustCsvParser {
127 pub fn new() -> Self {
128 Self::default()
129 }
130
131 pub fn flexible(mut self, flexible: bool) -> Self {
132 self.flexible = flexible;
133 self
134 }
135
136 pub fn allow_variable_columns(mut self, allow: bool) -> Self {
137 self.allow_variable_columns = allow;
138 self
139 }
140
141 pub fn verbosity(mut self, level: u8) -> Self {
144 self.verbosity = level;
145 self
146 }
147
148 pub fn with_auto_recovery(mut self, enabled: bool) -> Self {
150 self.auto_recovery = enabled;
151 self
152 }
153
154 pub fn with_retry_config(mut self, config: RetryConfig) -> Self {
156 self.retry_config = config;
157 self
158 }
159
160 pub fn detect_encoding(&self, file_path: &Path) -> Result<&'static str> {
162 let mut file = std::fs::File::open(file_path)?;
163 let mut buffer = [0; 4096];
164 let bytes_read = std::io::Read::read(&mut file, &mut buffer)?;
165
166 let sample = &buffer[..bytes_read];
167
168 if sample.starts_with(&[0xEF, 0xBB, 0xBF]) {
169 return Ok("utf-8");
170 }
171 if sample.starts_with(&[0xFF, 0xFE]) {
172 return Ok("utf-16le");
173 }
174 if sample.starts_with(&[0xFE, 0xFF]) {
175 return Ok("utf-16be");
176 }
177
178 if String::from_utf8(sample.to_vec()).is_ok() {
179 return Ok("utf-8");
180 }
181
182 let latin1_indicators = [
183 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8A, 0x8B, 0x8C, 0x8D,
184 0x8E, 0x8F,
185 ];
186 let has_latin1_chars = sample.iter().any(|&byte| latin1_indicators.contains(&byte));
187
188 if has_latin1_chars {
189 Ok("latin1")
190 } else {
191 Ok("utf-8")
192 }
193 }
194
195 pub fn parse_csv_with_recovery(
197 &self,
198 file_path: &Path,
199 ) -> Result<(Vec<String>, Vec<Vec<String>>)> {
200 if !self.auto_recovery {
201 return self.parse_csv(file_path);
202 }
203
204 let mut recovery_manager = AutoRecoveryManager::new(self.retry_config.clone());
205
206 match self.parse_csv(file_path) {
207 Ok(result) => Ok(result),
208 Err(initial_error) => {
209 log::warn!(
210 "Initial CSV parsing failed: {}. Attempting auto-recovery...",
211 initial_error
212 );
213
214 let initial_error_string = initial_error.to_string();
215 let dp_error = if let Ok(dp_err) = initial_error.downcast::<DataProfilerError>() {
216 dp_err
217 } else {
218 DataProfilerError::csv_parsing(
219 &initial_error_string,
220 Some(file_path.to_string_lossy().as_ref()),
221 )
222 };
223
224 if matches!(dp_error, DataProfilerError::DuplicateColumnName { .. }) {
228 return Err(dp_error.into());
229 }
230
231 recovery_manager
232 .attempt_recovery(&dp_error, |strategy| {
233 self.try_recovery_strategy(file_path, strategy)
234 })
235 .map_err(|error| error.into())
236 }
237 }
238 }
239
240 fn try_recovery_strategy(
241 &self,
242 file_path: &Path,
243 strategy: RecoveryStrategy,
244 ) -> Result<(Vec<String>, Vec<Vec<String>>), DataProfilerError> {
245 match strategy {
246 RecoveryStrategy::DelimiterDetection { delimiter } => {
247 log::debug!("Trying delimiter: '{}'", delimiter);
248 self.parse_with_override(file_path, Some(delimiter as u8), None, None)
249 .map_err(|error| {
250 DataProfilerError::csv_parsing(
251 &error.to_string(),
252 Some(file_path.to_string_lossy().as_ref()),
253 )
254 })
255 }
256 RecoveryStrategy::EncodingConversion { from: _, to: _ } => {
257 log::debug!(
258 "Attempting encoding conversion (placeholder - full implementation needed)"
259 );
260 self.parse_with_override(file_path, None, Some(true), Some(true))
261 .map_err(|error| {
262 DataProfilerError::csv_parsing(
263 &error.to_string(),
264 Some(file_path.to_string_lossy().as_ref()),
265 )
266 })
267 }
268 RecoveryStrategy::FlexibleParsing => {
269 log::debug!("Enabling flexible parsing");
270 self.parse_with_override(file_path, None, Some(true), Some(true))
271 .map_err(|error| {
272 DataProfilerError::csv_parsing(
273 &error.to_string(),
274 Some(file_path.to_string_lossy().as_ref()),
275 )
276 })
277 }
278 _ => Err(DataProfilerError::csv_parsing(
279 "Unsupported recovery strategy for CSV parsing",
280 Some(file_path.to_string_lossy().as_ref()),
281 )),
282 }
283 }
284
285 fn parse_with_override(
286 &self,
287 file_path: &Path,
288 delimiter_override: Option<u8>,
289 flexible_override: Option<bool>,
290 allow_variable_override: Option<bool>,
291 ) -> Result<(Vec<String>, Vec<Vec<String>>)> {
292 let delimiter = delimiter_override
293 .or(self.delimiter)
294 .unwrap_or_else(|| self.detect_delimiter(file_path).unwrap_or(b','));
295
296 let flexible = flexible_override.unwrap_or(self.flexible);
297 let allow_variable = allow_variable_override.unwrap_or(self.allow_variable_columns);
298
299 self.parse_with_config(file_path, delimiter, flexible, allow_variable)
300 }
301
302 fn parse_with_config(
303 &self,
304 file_path: &Path,
305 delimiter: u8,
306 flexible: bool,
307 _allow_variable: bool,
308 ) -> Result<(Vec<String>, Vec<Vec<String>>)> {
309 let mut reader = ReaderBuilder::new()
310 .delimiter(delimiter)
311 .has_headers(true)
312 .flexible(flexible)
313 .quote(self.quote_char.unwrap_or(b'"'))
314 .trim(if self.trim_whitespace {
315 Trim::All
316 } else {
317 Trim::None
318 })
319 .from_path(file_path)?;
320
321 let headers: Vec<String> = reader
322 .headers()?
323 .iter()
324 .map(|value| value.to_string())
325 .collect();
326 let expected_field_count = headers.len();
327 let mut records = Vec::new();
328
329 if flexible {
330 for result in reader.records() {
331 match result {
332 Ok(record) => {
333 let mut row: Vec<String> =
334 record.iter().map(|value| value.to_string()).collect();
335 if row.len() < expected_field_count {
336 row.resize(expected_field_count, String::new());
337 } else if row.len() > expected_field_count {
338 row.truncate(expected_field_count);
339 }
340 records.push(row);
341 }
342 Err(_) => continue,
343 }
344 }
345 } else {
346 for result in reader.records() {
347 let record = result?;
348 let row: Vec<String> = record.iter().map(|value| value.to_string()).collect();
349 records.push(row);
350 }
351 }
352
353 Ok((headers, records))
354 }
355
356 pub fn detect_delimiter(&self, file_path: &Path) -> Result<u8> {
361 Ok(crate::detect_delimiter_from_path(file_path)?)
362 }
363
364 pub fn parse_csv(&self, file_path: &Path) -> Result<(Vec<String>, Vec<Vec<String>>)> {
366 let delimiter = if let Some(delimiter) = self.delimiter {
367 delimiter
368 } else {
369 self.detect_delimiter(file_path)
370 .context("Failed to detect delimiter")?
371 };
372
373 let result = match self.try_strict_parsing(file_path, delimiter) {
374 Ok(result) => result,
375 Err(error) => {
376 if self.verbosity >= 2 {
377 log::info!("Using flexible CSV parsing (strict mode failed: {})", error);
378 }
379 self.try_flexible_parsing(file_path, delimiter)
380 .context("Both strict and flexible CSV parsing failed")?
381 }
382 };
383
384 dataprof_core::validate_unique_column_names(&result.0, "CSV header")?;
387 Ok(result)
388 }
389
390 fn try_strict_parsing(
391 &self,
392 file_path: &Path,
393 delimiter: u8,
394 ) -> Result<(Vec<String>, Vec<Vec<String>>)> {
395 let mut reader = ReaderBuilder::new()
396 .delimiter(delimiter)
397 .has_headers(true)
398 .flexible(false)
399 .quote(self.quote_char.unwrap_or(b'"'))
400 .trim(if self.trim_whitespace {
401 Trim::All
402 } else {
403 Trim::None
404 })
405 .from_path(file_path)?;
406
407 let headers: Vec<String> = reader
408 .headers()?
409 .iter()
410 .map(|value| value.to_string())
411 .collect();
412 let mut records = Vec::new();
413
414 for result in reader.records() {
415 let record = result?;
416 let row: Vec<String> = record.iter().map(|value| value.to_string()).collect();
417 records.push(row);
418 }
419
420 Ok((headers, records))
421 }
422
423 fn try_flexible_parsing(
424 &self,
425 file_path: &Path,
426 delimiter: u8,
427 ) -> Result<(Vec<String>, Vec<Vec<String>>)> {
428 let mut reader = ReaderBuilder::new()
429 .delimiter(delimiter)
430 .has_headers(true)
431 .flexible(true)
432 .quote(self.quote_char.unwrap_or(b'"'))
433 .trim(if self.trim_whitespace {
434 Trim::All
435 } else {
436 Trim::None
437 })
438 .from_path(file_path)?;
439
440 let headers: Vec<String> = reader
441 .headers()?
442 .iter()
443 .map(|value| value.to_string())
444 .collect();
445 let expected_field_count = headers.len();
446 let mut records = Vec::new();
447 let mut error_count = 0;
448 let max_errors = 100;
449
450 for (row_index, result) in reader.records().enumerate() {
451 match result {
452 Ok(record) => {
453 let mut row: Vec<String> =
454 record.iter().map(|value| value.to_string()).collect();
455
456 if self.allow_variable_columns {
457 if row.len() < expected_field_count {
458 row.resize(expected_field_count, String::new());
459 } else if row.len() > expected_field_count {
460 if row.len() == expected_field_count + 1
461 && let (Some(second_last), Some(last)) = (
462 row.get(expected_field_count - 1),
463 row.get(expected_field_count),
464 )
465 {
466 let combined =
467 format!("{}{}{}", second_last, delimiter as char, last);
468 row[expected_field_count - 1] = combined;
469 }
470 row.truncate(expected_field_count);
471 }
472 }
473
474 records.push(row);
475 }
476 Err(error) => {
477 error_count += 1;
478
479 let file_path_str = file_path.to_string_lossy();
480 let enhanced_error = DataProfilerError::csv_parsing(
481 &error.to_string(),
482 Some(file_path_str.as_ref()),
483 );
484 log::warn!("Row {}: {}", row_index + 2, enhanced_error);
485
486 if error_count > max_errors {
487 return Err(DataProfilerError::csv_parsing(
488 &format!("Too many parsing errors ({})", error_count),
489 Some(file_path_str.as_ref()),
490 )
491 .into());
492 }
493 }
494 }
495 }
496
497 if error_count > 0 {
498 log::warn!(
499 "Successfully parsed {} rows with {} errors skipped.",
500 records.len(),
501 error_count
502 );
503 }
504
505 Ok((headers, records))
506 }
507
508 pub fn parse_csv_chunk(
510 &self,
511 data: &str,
512 delimiter: u8,
513 has_headers: bool,
514 ) -> Result<RobustParseResult> {
515 let mut reader = ReaderBuilder::new()
516 .delimiter(delimiter)
517 .has_headers(has_headers)
518 .flexible(self.flexible)
519 .quote(self.quote_char.unwrap_or(b'"'))
520 .trim(if self.trim_whitespace {
521 Trim::All
522 } else {
523 Trim::None
524 })
525 .from_reader(Cursor::new(data));
526
527 let headers: Option<Vec<String>> = if has_headers {
528 let names: Vec<String> = reader
529 .headers()?
530 .iter()
531 .map(|value| value.to_string())
532 .collect();
533 dataprof_core::validate_unique_column_names(&names, "CSV header")?;
534 Some(names)
535 } else {
536 None
537 };
538
539 let mut records = Vec::new();
540 let mut error_count = 0;
541
542 for (row_index, result) in reader.records().enumerate() {
543 match result {
544 Ok(record) => {
545 let row: Vec<String> = record.iter().map(|value| value.to_string()).collect();
546 records.push(row);
547 }
548 Err(error) => {
549 error_count += 1;
550 if error_count <= 10 {
551 log::warn!("Chunk row {}: {}", row_index + 1, error);
552 }
553 }
554 }
555 }
556
557 Ok((headers, records))
558 }
559
560 pub fn validate_csv(&self, file_path: &Path) -> Result<CsvDiagnostics> {
562 let file = std::fs::File::open(file_path)?;
563 let reader = BufReader::new(file);
564 let mut diagnostics = CsvDiagnostics::default();
565
566 let mut field_counts = std::collections::HashMap::new();
567
568 for line_result in reader.lines() {
569 let line = line_result?;
570
571 if line.trim().is_empty() {
572 diagnostics.empty_lines += 1;
573 continue;
574 }
575
576 if line.contains('\0') {
577 diagnostics.null_bytes += 1;
578 }
579
580 let quote_count = line.chars().filter(|&ch| ch == '"').count();
581 if quote_count % 2 != 0 {
582 diagnostics.unmatched_quotes += 1;
583 }
584
585 let delimiter = self.delimiter.unwrap_or(b',') as char;
586 let field_count = crate::count_fields(&line, delimiter);
587 *field_counts.entry(field_count).or_insert(0) += 1;
588
589 diagnostics.total_lines += 1;
590 }
591
592 if let Some((&most_common_fields, &count)) =
593 field_counts.iter().max_by_key(|(_, count)| *count)
594 {
595 diagnostics.most_common_field_count = most_common_fields;
596 diagnostics.consistent_field_rows = count;
597 }
598
599 diagnostics.field_count_variations = field_counts.len();
600 diagnostics.inconsistent_field_rows =
601 diagnostics.total_lines - diagnostics.consistent_field_rows;
602
603 Ok(diagnostics)
604 }
605}
606
607#[derive(Debug, Default)]
608pub struct CsvDiagnostics {
609 pub total_lines: usize,
610 pub empty_lines: usize,
611 pub most_common_field_count: usize,
612 pub consistent_field_rows: usize,
613 pub inconsistent_field_rows: usize,
614 pub field_count_variations: usize,
615 pub unmatched_quotes: usize,
616 pub null_bytes: usize,
617}
618
619impl CsvDiagnostics {
620 pub fn print_summary(&self) {
621 println!("CSV File Diagnostics:");
622 println!(" Total lines: {}", self.total_lines);
623 println!(" Empty lines: {}", self.empty_lines);
624 println!(
625 " Most common field count: {}",
626 self.most_common_field_count
627 );
628 println!(
629 " Consistent rows: {} ({:.1}%)",
630 self.consistent_field_rows,
631 self.consistent_field_rows as f64 / self.total_lines as f64 * 100.0
632 );
633
634 if self.inconsistent_field_rows > 0 {
635 println!(
636 " WARNING: Inconsistent rows: {} ({:.1}%)",
637 self.inconsistent_field_rows,
638 self.inconsistent_field_rows as f64 / self.total_lines as f64 * 100.0
639 );
640 }
641
642 if self.field_count_variations > 1 {
643 println!(
644 " WARNING: Different field counts detected: {}",
645 self.field_count_variations
646 );
647 }
648
649 if self.unmatched_quotes > 0 {
650 println!(
651 " WARNING: Lines with unmatched quotes: {}",
652 self.unmatched_quotes
653 );
654 }
655
656 if self.null_bytes > 0 {
657 println!(" WARNING: Lines with null bytes: {}", self.null_bytes);
658 }
659 }
660}
661
662#[cfg(test)]
663mod tests {
664 use super::*;
665 use std::io::Write;
666 use tempfile::NamedTempFile;
667
668 fn write_bytes(bytes: &[u8]) -> NamedTempFile {
669 let mut f = NamedTempFile::new().unwrap();
670 f.write_all(bytes).unwrap();
671 f.flush().unwrap();
672 f
673 }
674
675 #[test]
676 fn diagnose_non_utf8_flags_latin1() {
677 let f = write_bytes(b"name\ncaf\xE9\n");
681 let d = diagnose_non_utf8(f.path()).expect("latin-1 should be flagged");
682 assert_eq!(d.guess, "latin-1");
683 assert_eq!(d.first_invalid_byte, Some(8));
684 }
685
686 #[test]
687 fn diagnose_non_utf8_flags_windows1252() {
688 let f = write_bytes(b"quote\n\x93hi\x94\n");
690 let d = diagnose_non_utf8(f.path()).expect("windows-1252 should be flagged");
691 assert_eq!(d.guess, "windows-1252");
692 assert_eq!(d.first_invalid_byte, Some(6));
693 }
694
695 #[test]
696 fn diagnose_non_utf8_flags_utf16_bom() {
697 let f = write_bytes(&[0xFF, 0xFE, b'a', 0x00]);
698 let d = diagnose_non_utf8(f.path()).expect("utf-16 BOM should be flagged");
699 assert_eq!(d.guess, "utf-16le");
700 }
701
702 #[test]
703 fn diagnose_non_utf8_ignores_clean_utf8() {
704 let f = write_bytes("name\ncafé\n".as_bytes());
705 assert!(diagnose_non_utf8(f.path()).is_none());
706 }
707
708 #[test]
709 fn encoding_diagnostic_detail_never_leaks_bytes() {
710 let d = EncodingDiagnostic {
711 guess: "latin-1",
712 first_invalid_byte: Some(8),
713 };
714 let detail = d.detail();
715 assert!(detail.contains("latin-1"));
716 assert!(detail.contains("offset 8"));
717 }
718
719 #[test]
720 fn test_robust_csv_parser_basic() -> Result<()> {
721 let mut temp_file = NamedTempFile::new()?;
722 writeln!(temp_file, "name,age,city")?;
723 writeln!(temp_file, "Alice,25,New York")?;
724 writeln!(temp_file, "Bob,30,London")?;
725 temp_file.flush()?;
726
727 let parser = RobustCsvParser::new();
728 let (headers, records) = parser.parse_csv(temp_file.path())?;
729
730 assert_eq!(headers, vec!["name", "age", "city"]);
731 assert_eq!(records.len(), 2);
732 assert_eq!(records[0], vec!["Alice", "25", "New York"]);
733
734 Ok(())
735 }
736
737 #[test]
738 fn test_flexible_field_counts() -> Result<()> {
739 let mut temp_file = NamedTempFile::new()?;
740 writeln!(temp_file, "name,age,city")?;
741 writeln!(temp_file, "Alice,25,New York")?;
742 writeln!(temp_file, "Bob,30")?;
743 writeln!(temp_file, "Charlie,35,London,UK")?;
744 temp_file.flush()?;
745
746 let mut parser = RobustCsvParser::new();
747 parser.delimiter = Some(b',');
748 let parser = parser.allow_variable_columns(true);
749 let (headers, records) = parser.parse_csv(temp_file.path())?;
750
751 assert_eq!(headers, vec!["name", "age", "city"]);
752 assert_eq!(records.len(), 3);
753 assert_eq!(records[1], vec!["Bob", "30", ""]);
754 assert_eq!(records[2], vec!["Charlie", "35", "London,UK"]);
755
756 Ok(())
757 }
758
759 #[test]
760 fn test_delimiter_detection() -> Result<()> {
761 let mut temp_file = NamedTempFile::new()?;
762 writeln!(temp_file, "name;age;city")?;
763 writeln!(temp_file, "Alice;25;New York")?;
764 writeln!(temp_file, "Bob;30;London")?;
765 temp_file.flush()?;
766
767 let parser = RobustCsvParser::new();
768 let delimiter = parser.detect_delimiter(temp_file.path())?;
769 assert!(delimiter == b';' || delimiter == b',');
770
771 Ok(())
772 }
773
774 #[test]
775 fn test_csv_diagnostics() -> Result<()> {
776 let mut temp_file = NamedTempFile::new()?;
777 writeln!(temp_file, "name,age,city")?;
778 writeln!(temp_file, "Alice,25,New York")?;
779 writeln!(temp_file, "Bob,30")?;
780 writeln!(temp_file)?;
781 writeln!(temp_file, "Charlie,35,London")?;
782 temp_file.flush()?;
783
784 let parser = RobustCsvParser::new();
785 let diagnostics = parser.validate_csv(temp_file.path())?;
786
787 assert_eq!(diagnostics.total_lines, 4);
788 assert_eq!(diagnostics.empty_lines, 1);
789 assert_eq!(diagnostics.most_common_field_count, 3);
790 assert_eq!(diagnostics.inconsistent_field_rows, 1);
791
792 Ok(())
793 }
794}