1use anyhow::{Context, Result};
2use csv::{ReaderBuilder, Trim};
3use std::io::{BufRead, BufReader, Cursor};
4use std::path::Path;
5
6use dataprof_core::errors::AutoRecoveryManager;
7use dataprof_core::{DataProfilerError, RecoveryStrategy, RetryConfig};
8
9pub type RobustParseResult = (Option<Vec<String>>, Vec<Vec<String>>);
11
12pub struct RobustCsvParser {
14 pub flexible: bool,
15 pub quote_char: Option<u8>,
16 pub delimiter: Option<u8>,
17 pub allow_variable_columns: bool,
18 pub trim_whitespace: bool,
19 pub auto_recovery: bool,
20 pub retry_config: RetryConfig,
21 pub verbosity: u8,
22}
23
24impl Default for RobustCsvParser {
25 fn default() -> Self {
26 Self {
27 flexible: true,
28 quote_char: Some(b'"'),
29 delimiter: None,
30 allow_variable_columns: true,
31 trim_whitespace: true,
32 auto_recovery: true,
33 retry_config: RetryConfig::default(),
34 verbosity: 1,
35 }
36 }
37}
38
39impl RobustCsvParser {
40 pub fn new() -> Self {
41 Self::default()
42 }
43
44 pub fn flexible(mut self, flexible: bool) -> Self {
45 self.flexible = flexible;
46 self
47 }
48
49 pub fn allow_variable_columns(mut self, allow: bool) -> Self {
50 self.allow_variable_columns = allow;
51 self
52 }
53
54 pub fn verbosity(mut self, level: u8) -> Self {
57 self.verbosity = level;
58 self
59 }
60
61 pub fn with_auto_recovery(mut self, enabled: bool) -> Self {
63 self.auto_recovery = enabled;
64 self
65 }
66
67 pub fn with_retry_config(mut self, config: RetryConfig) -> Self {
69 self.retry_config = config;
70 self
71 }
72
73 pub fn detect_encoding(&self, file_path: &Path) -> Result<&'static str> {
75 let mut file = std::fs::File::open(file_path)?;
76 let mut buffer = [0; 4096];
77 let bytes_read = std::io::Read::read(&mut file, &mut buffer)?;
78
79 let sample = &buffer[..bytes_read];
80
81 if sample.starts_with(&[0xEF, 0xBB, 0xBF]) {
82 return Ok("utf-8");
83 }
84 if sample.starts_with(&[0xFF, 0xFE]) {
85 return Ok("utf-16le");
86 }
87 if sample.starts_with(&[0xFE, 0xFF]) {
88 return Ok("utf-16be");
89 }
90
91 if String::from_utf8(sample.to_vec()).is_ok() {
92 return Ok("utf-8");
93 }
94
95 let latin1_indicators = [
96 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8A, 0x8B, 0x8C, 0x8D,
97 0x8E, 0x8F,
98 ];
99 let has_latin1_chars = sample.iter().any(|&byte| latin1_indicators.contains(&byte));
100
101 if has_latin1_chars {
102 Ok("latin1")
103 } else {
104 Ok("utf-8")
105 }
106 }
107
108 pub fn parse_csv_with_recovery(
110 &self,
111 file_path: &Path,
112 ) -> Result<(Vec<String>, Vec<Vec<String>>)> {
113 if !self.auto_recovery {
114 return self.parse_csv(file_path);
115 }
116
117 let mut recovery_manager = AutoRecoveryManager::new(self.retry_config.clone());
118
119 match self.parse_csv(file_path) {
120 Ok(result) => Ok(result),
121 Err(initial_error) => {
122 log::warn!(
123 "Initial CSV parsing failed: {}. Attempting auto-recovery...",
124 initial_error
125 );
126
127 let initial_error_string = initial_error.to_string();
128 let dp_error = if let Ok(dp_err) = initial_error.downcast::<DataProfilerError>() {
129 dp_err
130 } else {
131 DataProfilerError::csv_parsing(
132 &initial_error_string,
133 &file_path.to_string_lossy(),
134 )
135 };
136
137 recovery_manager
138 .attempt_recovery(&dp_error, |strategy| {
139 self.try_recovery_strategy(file_path, strategy)
140 })
141 .map_err(|error| error.into())
142 }
143 }
144 }
145
146 fn try_recovery_strategy(
147 &self,
148 file_path: &Path,
149 strategy: RecoveryStrategy,
150 ) -> Result<(Vec<String>, Vec<Vec<String>>), DataProfilerError> {
151 match strategy {
152 RecoveryStrategy::DelimiterDetection { delimiter } => {
153 log::debug!("Trying delimiter: '{}'", delimiter);
154 self.parse_with_override(file_path, Some(delimiter as u8), None, None)
155 .map_err(|error| {
156 DataProfilerError::csv_parsing(
157 &error.to_string(),
158 &file_path.to_string_lossy(),
159 )
160 })
161 }
162 RecoveryStrategy::EncodingConversion { from: _, to: _ } => {
163 log::debug!(
164 "Attempting encoding conversion (placeholder - full implementation needed)"
165 );
166 self.parse_with_override(file_path, None, Some(true), Some(true))
167 .map_err(|error| {
168 DataProfilerError::csv_parsing(
169 &error.to_string(),
170 &file_path.to_string_lossy(),
171 )
172 })
173 }
174 RecoveryStrategy::FlexibleParsing => {
175 log::debug!("Enabling flexible parsing");
176 self.parse_with_override(file_path, None, Some(true), Some(true))
177 .map_err(|error| {
178 DataProfilerError::csv_parsing(
179 &error.to_string(),
180 &file_path.to_string_lossy(),
181 )
182 })
183 }
184 _ => Err(DataProfilerError::csv_parsing(
185 "Unsupported recovery strategy for CSV parsing",
186 &file_path.to_string_lossy(),
187 )),
188 }
189 }
190
191 fn parse_with_override(
192 &self,
193 file_path: &Path,
194 delimiter_override: Option<u8>,
195 flexible_override: Option<bool>,
196 allow_variable_override: Option<bool>,
197 ) -> Result<(Vec<String>, Vec<Vec<String>>)> {
198 let delimiter = delimiter_override
199 .or(self.delimiter)
200 .unwrap_or_else(|| self.detect_delimiter(file_path).unwrap_or(b','));
201
202 let flexible = flexible_override.unwrap_or(self.flexible);
203 let allow_variable = allow_variable_override.unwrap_or(self.allow_variable_columns);
204
205 self.parse_with_config(file_path, delimiter, flexible, allow_variable)
206 }
207
208 fn parse_with_config(
209 &self,
210 file_path: &Path,
211 delimiter: u8,
212 flexible: bool,
213 _allow_variable: bool,
214 ) -> Result<(Vec<String>, Vec<Vec<String>>)> {
215 let mut reader = ReaderBuilder::new()
216 .delimiter(delimiter)
217 .has_headers(true)
218 .flexible(flexible)
219 .quote(self.quote_char.unwrap_or(b'"'))
220 .trim(if self.trim_whitespace {
221 Trim::All
222 } else {
223 Trim::None
224 })
225 .from_path(file_path)?;
226
227 let headers: Vec<String> = reader
228 .headers()?
229 .iter()
230 .map(|value| value.to_string())
231 .collect();
232 let expected_field_count = headers.len();
233 let mut records = Vec::new();
234
235 if flexible {
236 for result in reader.records() {
237 match result {
238 Ok(record) => {
239 let mut row: Vec<String> =
240 record.iter().map(|value| value.to_string()).collect();
241 if row.len() < expected_field_count {
242 row.resize(expected_field_count, String::new());
243 } else if row.len() > expected_field_count {
244 row.truncate(expected_field_count);
245 }
246 records.push(row);
247 }
248 Err(_) => continue,
249 }
250 }
251 } else {
252 for result in reader.records() {
253 let record = result?;
254 let row: Vec<String> = record.iter().map(|value| value.to_string()).collect();
255 records.push(row);
256 }
257 }
258
259 Ok((headers, records))
260 }
261
262 pub fn detect_delimiter(&self, file_path: &Path) -> Result<u8> {
264 let file = std::fs::File::open(file_path)?;
265 let reader = BufReader::new(file);
266
267 let mut lines = Vec::new();
268 for (index, line) in reader.lines().enumerate() {
269 if index >= 5 {
270 break;
271 }
272 lines.push(line?);
273 }
274
275 let delimiters = *b",;\t|";
276 let mut best_delimiter = b',';
277 let mut max_consistency = 0;
278 let mut max_field_count = 0;
279
280 for &delimiter in &delimiters {
281 let consistency = self.measure_delimiter_consistency(&lines, delimiter);
282 let avg_field_count = self.average_field_count(&lines, delimiter);
283
284 let should_update = if avg_field_count > 1 && max_field_count <= 1 {
285 true
286 } else if avg_field_count <= 1 && max_field_count > 1 {
287 false
288 } else {
289 consistency > max_consistency
290 || (consistency == max_consistency && avg_field_count > max_field_count)
291 };
292
293 if should_update {
294 max_consistency = consistency;
295 max_field_count = avg_field_count;
296 best_delimiter = delimiter;
297 }
298 }
299
300 Ok(best_delimiter)
301 }
302
303 fn measure_delimiter_consistency(&self, lines: &[String], delimiter: u8) -> usize {
304 if lines.is_empty() {
305 return 0;
306 }
307
308 let delimiter_char = delimiter as char;
309 let field_counts: Vec<usize> = lines
310 .iter()
311 .map(|line| self.count_fields_simple(line, delimiter_char))
312 .collect();
313
314 let mut counts = std::collections::HashMap::new();
315 for &count in &field_counts {
316 *counts.entry(count).or_insert(0) += 1;
317 }
318
319 counts.values().max().copied().unwrap_or(0)
320 }
321
322 fn average_field_count(&self, lines: &[String], delimiter: u8) -> usize {
323 if lines.is_empty() {
324 return 0;
325 }
326
327 let delimiter_char = delimiter as char;
328 let total_fields: usize = lines
329 .iter()
330 .map(|line| self.count_fields_simple(line, delimiter_char))
331 .sum();
332
333 total_fields / lines.len()
334 }
335
336 fn count_fields_simple(&self, line: &str, delimiter: char) -> usize {
337 let mut field_count = 1;
338 let mut in_quotes = false;
339 let mut chars = line.chars().peekable();
340
341 while let Some(ch) = chars.next() {
342 match ch {
343 '"' => {
344 if chars.peek() == Some(&'"') {
345 chars.next();
346 } else {
347 in_quotes = !in_quotes;
348 }
349 }
350 current if current == delimiter && !in_quotes => {
351 field_count += 1;
352 }
353 _ => {}
354 }
355 }
356
357 field_count
358 }
359
360 pub fn parse_csv(&self, file_path: &Path) -> Result<(Vec<String>, Vec<Vec<String>>)> {
362 let delimiter = if let Some(delimiter) = self.delimiter {
363 delimiter
364 } else {
365 self.detect_delimiter(file_path)
366 .context("Failed to detect delimiter")?
367 };
368
369 match self.try_strict_parsing(file_path, delimiter) {
370 Ok(result) => return Ok(result),
371 Err(error) => {
372 if self.verbosity >= 2 {
373 log::info!("Using flexible CSV parsing (strict mode failed: {})", error);
374 }
375 }
376 }
377
378 self.try_flexible_parsing(file_path, delimiter)
379 .context("Both strict and flexible CSV parsing failed")
380 }
381
382 fn try_strict_parsing(
383 &self,
384 file_path: &Path,
385 delimiter: u8,
386 ) -> Result<(Vec<String>, Vec<Vec<String>>)> {
387 let mut reader = ReaderBuilder::new()
388 .delimiter(delimiter)
389 .has_headers(true)
390 .flexible(false)
391 .quote(self.quote_char.unwrap_or(b'"'))
392 .trim(if self.trim_whitespace {
393 Trim::All
394 } else {
395 Trim::None
396 })
397 .from_path(file_path)?;
398
399 let headers: Vec<String> = reader
400 .headers()?
401 .iter()
402 .map(|value| value.to_string())
403 .collect();
404 let mut records = Vec::new();
405
406 for result in reader.records() {
407 let record = result?;
408 let row: Vec<String> = record.iter().map(|value| value.to_string()).collect();
409 records.push(row);
410 }
411
412 Ok((headers, records))
413 }
414
415 fn try_flexible_parsing(
416 &self,
417 file_path: &Path,
418 delimiter: u8,
419 ) -> Result<(Vec<String>, Vec<Vec<String>>)> {
420 let mut reader = ReaderBuilder::new()
421 .delimiter(delimiter)
422 .has_headers(true)
423 .flexible(true)
424 .quote(self.quote_char.unwrap_or(b'"'))
425 .trim(if self.trim_whitespace {
426 Trim::All
427 } else {
428 Trim::None
429 })
430 .from_path(file_path)?;
431
432 let headers: Vec<String> = reader
433 .headers()?
434 .iter()
435 .map(|value| value.to_string())
436 .collect();
437 let expected_field_count = headers.len();
438 let mut records = Vec::new();
439 let mut error_count = 0;
440 let max_errors = 100;
441
442 for (row_index, result) in reader.records().enumerate() {
443 match result {
444 Ok(record) => {
445 let mut row: Vec<String> =
446 record.iter().map(|value| value.to_string()).collect();
447
448 if self.allow_variable_columns {
449 if row.len() < expected_field_count {
450 row.resize(expected_field_count, String::new());
451 } else if row.len() > expected_field_count {
452 if row.len() == expected_field_count + 1
453 && let (Some(second_last), Some(last)) = (
454 row.get(expected_field_count - 1),
455 row.get(expected_field_count),
456 )
457 {
458 let combined =
459 format!("{}{}{}", second_last, delimiter as char, last);
460 row[expected_field_count - 1] = combined;
461 }
462 row.truncate(expected_field_count);
463 }
464 }
465
466 records.push(row);
467 }
468 Err(error) => {
469 error_count += 1;
470
471 let enhanced_error =
472 DataProfilerError::csv_parsing(&error.to_string(), "current file");
473 log::warn!("Row {}: {}", row_index + 2, enhanced_error);
474
475 if error_count > max_errors {
476 return Err(DataProfilerError::csv_parsing(
477 &format!("Too many parsing errors ({})", error_count),
478 "current file",
479 )
480 .into());
481 }
482 }
483 }
484 }
485
486 if error_count > 0 {
487 log::warn!(
488 "Successfully parsed {} rows with {} errors skipped.",
489 records.len(),
490 error_count
491 );
492 }
493
494 Ok((headers, records))
495 }
496
497 pub fn parse_csv_chunk(
499 &self,
500 data: &str,
501 delimiter: u8,
502 has_headers: bool,
503 ) -> Result<RobustParseResult> {
504 let mut reader = ReaderBuilder::new()
505 .delimiter(delimiter)
506 .has_headers(has_headers)
507 .flexible(self.flexible)
508 .quote(self.quote_char.unwrap_or(b'"'))
509 .trim(if self.trim_whitespace {
510 Trim::All
511 } else {
512 Trim::None
513 })
514 .from_reader(Cursor::new(data));
515
516 let headers = if has_headers {
517 Some(
518 reader
519 .headers()?
520 .iter()
521 .map(|value| value.to_string())
522 .collect(),
523 )
524 } else {
525 None
526 };
527
528 let mut records = Vec::new();
529 let mut error_count = 0;
530
531 for (row_index, result) in reader.records().enumerate() {
532 match result {
533 Ok(record) => {
534 let row: Vec<String> = record.iter().map(|value| value.to_string()).collect();
535 records.push(row);
536 }
537 Err(error) => {
538 error_count += 1;
539 if error_count <= 10 {
540 log::warn!("Chunk row {}: {}", row_index + 1, error);
541 }
542 }
543 }
544 }
545
546 Ok((headers, records))
547 }
548
549 pub fn validate_csv(&self, file_path: &Path) -> Result<CsvDiagnostics> {
551 let file = std::fs::File::open(file_path)?;
552 let reader = BufReader::new(file);
553 let mut diagnostics = CsvDiagnostics::default();
554
555 let mut field_counts = std::collections::HashMap::new();
556
557 for line_result in reader.lines() {
558 let line = line_result?;
559
560 if line.trim().is_empty() {
561 diagnostics.empty_lines += 1;
562 continue;
563 }
564
565 if line.contains('\0') {
566 diagnostics.null_bytes += 1;
567 }
568
569 let quote_count = line.chars().filter(|&ch| ch == '"').count();
570 if quote_count % 2 != 0 {
571 diagnostics.unmatched_quotes += 1;
572 }
573
574 let delimiter = self.delimiter.unwrap_or(b',') as char;
575 let field_count = self.count_fields_simple(&line, delimiter);
576 *field_counts.entry(field_count).or_insert(0) += 1;
577
578 diagnostics.total_lines += 1;
579 }
580
581 if let Some((&most_common_fields, &count)) =
582 field_counts.iter().max_by_key(|(_, count)| *count)
583 {
584 diagnostics.most_common_field_count = most_common_fields;
585 diagnostics.consistent_field_rows = count;
586 }
587
588 diagnostics.field_count_variations = field_counts.len();
589 diagnostics.inconsistent_field_rows =
590 diagnostics.total_lines - diagnostics.consistent_field_rows;
591
592 Ok(diagnostics)
593 }
594}
595
596#[derive(Debug, Default)]
597pub struct CsvDiagnostics {
598 pub total_lines: usize,
599 pub empty_lines: usize,
600 pub most_common_field_count: usize,
601 pub consistent_field_rows: usize,
602 pub inconsistent_field_rows: usize,
603 pub field_count_variations: usize,
604 pub unmatched_quotes: usize,
605 pub null_bytes: usize,
606}
607
608impl CsvDiagnostics {
609 pub fn print_summary(&self) {
610 println!("CSV File Diagnostics:");
611 println!(" Total lines: {}", self.total_lines);
612 println!(" Empty lines: {}", self.empty_lines);
613 println!(
614 " Most common field count: {}",
615 self.most_common_field_count
616 );
617 println!(
618 " Consistent rows: {} ({:.1}%)",
619 self.consistent_field_rows,
620 self.consistent_field_rows as f64 / self.total_lines as f64 * 100.0
621 );
622
623 if self.inconsistent_field_rows > 0 {
624 println!(
625 " WARNING: Inconsistent rows: {} ({:.1}%)",
626 self.inconsistent_field_rows,
627 self.inconsistent_field_rows as f64 / self.total_lines as f64 * 100.0
628 );
629 }
630
631 if self.field_count_variations > 1 {
632 println!(
633 " WARNING: Different field counts detected: {}",
634 self.field_count_variations
635 );
636 }
637
638 if self.unmatched_quotes > 0 {
639 println!(
640 " WARNING: Lines with unmatched quotes: {}",
641 self.unmatched_quotes
642 );
643 }
644
645 if self.null_bytes > 0 {
646 println!(" WARNING: Lines with null bytes: {}", self.null_bytes);
647 }
648 }
649}
650
651#[cfg(test)]
652mod tests {
653 use super::*;
654 use std::io::Write;
655 use tempfile::NamedTempFile;
656
657 #[test]
658 fn test_robust_csv_parser_basic() -> Result<()> {
659 let mut temp_file = NamedTempFile::new()?;
660 writeln!(temp_file, "name,age,city")?;
661 writeln!(temp_file, "Alice,25,New York")?;
662 writeln!(temp_file, "Bob,30,London")?;
663 temp_file.flush()?;
664
665 let parser = RobustCsvParser::new();
666 let (headers, records) = parser.parse_csv(temp_file.path())?;
667
668 assert_eq!(headers, vec!["name", "age", "city"]);
669 assert_eq!(records.len(), 2);
670 assert_eq!(records[0], vec!["Alice", "25", "New York"]);
671
672 Ok(())
673 }
674
675 #[test]
676 fn test_flexible_field_counts() -> Result<()> {
677 let mut temp_file = NamedTempFile::new()?;
678 writeln!(temp_file, "name,age,city")?;
679 writeln!(temp_file, "Alice,25,New York")?;
680 writeln!(temp_file, "Bob,30")?;
681 writeln!(temp_file, "Charlie,35,London,UK")?;
682 temp_file.flush()?;
683
684 let mut parser = RobustCsvParser::new();
685 parser.delimiter = Some(b',');
686 let parser = parser.allow_variable_columns(true);
687 let (headers, records) = parser.parse_csv(temp_file.path())?;
688
689 assert_eq!(headers, vec!["name", "age", "city"]);
690 assert_eq!(records.len(), 3);
691 assert_eq!(records[1], vec!["Bob", "30", ""]);
692 assert_eq!(records[2], vec!["Charlie", "35", "London,UK"]);
693
694 Ok(())
695 }
696
697 #[test]
698 fn test_delimiter_detection() -> Result<()> {
699 let mut temp_file = NamedTempFile::new()?;
700 writeln!(temp_file, "name;age;city")?;
701 writeln!(temp_file, "Alice;25;New York")?;
702 writeln!(temp_file, "Bob;30;London")?;
703 temp_file.flush()?;
704
705 let parser = RobustCsvParser::new();
706 let delimiter = parser.detect_delimiter(temp_file.path())?;
707 assert!(delimiter == b';' || delimiter == b',');
708
709 Ok(())
710 }
711
712 #[test]
713 fn test_csv_diagnostics() -> Result<()> {
714 let mut temp_file = NamedTempFile::new()?;
715 writeln!(temp_file, "name,age,city")?;
716 writeln!(temp_file, "Alice,25,New York")?;
717 writeln!(temp_file, "Bob,30")?;
718 writeln!(temp_file)?;
719 writeln!(temp_file, "Charlie,35,London")?;
720 temp_file.flush()?;
721
722 let parser = RobustCsvParser::new();
723 let diagnostics = parser.validate_csv(temp_file.path())?;
724
725 assert_eq!(diagnostics.total_lines, 4);
726 assert_eq!(diagnostics.empty_lines, 1);
727 assert_eq!(diagnostics.most_common_field_count, 3);
728 assert_eq!(diagnostics.inconsistent_field_rows, 1);
729
730 Ok(())
731 }
732}