Skip to main content

dataprof_csv/
robust_csv.rs

1use anyhow::{Context, Result};
2use csv::{ReaderBuilder, Trim};
3use std::io::{BufRead, BufReader, Cursor};
4use std::path::Path;
5
6use dataprof_core::errors::AutoRecoveryManager;
7use dataprof_core::{DataProfilerError, RecoveryStrategy, RetryConfig};
8
9/// Result type for robust CSV parsing operations.
10pub type RobustParseResult = (Option<Vec<String>>, Vec<Vec<String>>);
11
12/// Robust CSV parser that handles edge cases and malformed data.
13pub struct RobustCsvParser {
14    pub flexible: bool,
15    pub quote_char: Option<u8>,
16    pub delimiter: Option<u8>,
17    pub allow_variable_columns: bool,
18    pub trim_whitespace: bool,
19    pub auto_recovery: bool,
20    pub retry_config: RetryConfig,
21    pub verbosity: u8,
22}
23
24impl Default for RobustCsvParser {
25    fn default() -> Self {
26        Self {
27            flexible: true,
28            quote_char: Some(b'"'),
29            delimiter: None,
30            allow_variable_columns: true,
31            trim_whitespace: true,
32            auto_recovery: true,
33            retry_config: RetryConfig::default(),
34            verbosity: 1,
35        }
36    }
37}
38
39impl RobustCsvParser {
40    pub fn new() -> Self {
41        Self::default()
42    }
43
44    pub fn flexible(mut self, flexible: bool) -> Self {
45        self.flexible = flexible;
46        self
47    }
48
49    pub fn allow_variable_columns(mut self, allow: bool) -> Self {
50        self.allow_variable_columns = allow;
51        self
52    }
53
54    /// Set verbosity level for logging.
55    /// 0=quiet, 1=normal, 2=verbose, 3=debug
56    pub fn verbosity(mut self, level: u8) -> Self {
57        self.verbosity = level;
58        self
59    }
60
61    /// Enable or disable auto-recovery features.
62    pub fn with_auto_recovery(mut self, enabled: bool) -> Self {
63        self.auto_recovery = enabled;
64        self
65    }
66
67    /// Set custom retry configuration.
68    pub fn with_retry_config(mut self, config: RetryConfig) -> Self {
69        self.retry_config = config;
70        self
71    }
72
73    /// Detect encoding by analyzing file bytes.
74    pub fn detect_encoding(&self, file_path: &Path) -> Result<&'static str> {
75        let mut file = std::fs::File::open(file_path)?;
76        let mut buffer = [0; 4096];
77        let bytes_read = std::io::Read::read(&mut file, &mut buffer)?;
78
79        let sample = &buffer[..bytes_read];
80
81        if sample.starts_with(&[0xEF, 0xBB, 0xBF]) {
82            return Ok("utf-8");
83        }
84        if sample.starts_with(&[0xFF, 0xFE]) {
85            return Ok("utf-16le");
86        }
87        if sample.starts_with(&[0xFE, 0xFF]) {
88            return Ok("utf-16be");
89        }
90
91        if String::from_utf8(sample.to_vec()).is_ok() {
92            return Ok("utf-8");
93        }
94
95        let latin1_indicators = [
96            0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8A, 0x8B, 0x8C, 0x8D,
97            0x8E, 0x8F,
98        ];
99        let has_latin1_chars = sample.iter().any(|&byte| latin1_indicators.contains(&byte));
100
101        if has_latin1_chars {
102            Ok("latin1")
103        } else {
104            Ok("utf-8")
105        }
106    }
107
108    /// Enhanced parsing with auto-recovery.
109    pub fn parse_csv_with_recovery(
110        &self,
111        file_path: &Path,
112    ) -> Result<(Vec<String>, Vec<Vec<String>>)> {
113        if !self.auto_recovery {
114            return self.parse_csv(file_path);
115        }
116
117        let mut recovery_manager = AutoRecoveryManager::new(self.retry_config.clone());
118
119        match self.parse_csv(file_path) {
120            Ok(result) => Ok(result),
121            Err(initial_error) => {
122                log::warn!(
123                    "Initial CSV parsing failed: {}. Attempting auto-recovery...",
124                    initial_error
125                );
126
127                let initial_error_string = initial_error.to_string();
128                let dp_error = if let Ok(dp_err) = initial_error.downcast::<DataProfilerError>() {
129                    dp_err
130                } else {
131                    DataProfilerError::csv_parsing(
132                        &initial_error_string,
133                        &file_path.to_string_lossy(),
134                    )
135                };
136
137                recovery_manager
138                    .attempt_recovery(&dp_error, |strategy| {
139                        self.try_recovery_strategy(file_path, strategy)
140                    })
141                    .map_err(|error| error.into())
142            }
143        }
144    }
145
146    fn try_recovery_strategy(
147        &self,
148        file_path: &Path,
149        strategy: RecoveryStrategy,
150    ) -> Result<(Vec<String>, Vec<Vec<String>>), DataProfilerError> {
151        match strategy {
152            RecoveryStrategy::DelimiterDetection { delimiter } => {
153                log::debug!("Trying delimiter: '{}'", delimiter);
154                self.parse_with_override(file_path, Some(delimiter as u8), None, None)
155                    .map_err(|error| {
156                        DataProfilerError::csv_parsing(
157                            &error.to_string(),
158                            &file_path.to_string_lossy(),
159                        )
160                    })
161            }
162            RecoveryStrategy::EncodingConversion { from: _, to: _ } => {
163                log::debug!(
164                    "Attempting encoding conversion (placeholder - full implementation needed)"
165                );
166                self.parse_with_override(file_path, None, Some(true), Some(true))
167                    .map_err(|error| {
168                        DataProfilerError::csv_parsing(
169                            &error.to_string(),
170                            &file_path.to_string_lossy(),
171                        )
172                    })
173            }
174            RecoveryStrategy::FlexibleParsing => {
175                log::debug!("Enabling flexible parsing");
176                self.parse_with_override(file_path, None, Some(true), Some(true))
177                    .map_err(|error| {
178                        DataProfilerError::csv_parsing(
179                            &error.to_string(),
180                            &file_path.to_string_lossy(),
181                        )
182                    })
183            }
184            _ => Err(DataProfilerError::csv_parsing(
185                "Unsupported recovery strategy for CSV parsing",
186                &file_path.to_string_lossy(),
187            )),
188        }
189    }
190
191    fn parse_with_override(
192        &self,
193        file_path: &Path,
194        delimiter_override: Option<u8>,
195        flexible_override: Option<bool>,
196        allow_variable_override: Option<bool>,
197    ) -> Result<(Vec<String>, Vec<Vec<String>>)> {
198        let delimiter = delimiter_override
199            .or(self.delimiter)
200            .unwrap_or_else(|| self.detect_delimiter(file_path).unwrap_or(b','));
201
202        let flexible = flexible_override.unwrap_or(self.flexible);
203        let allow_variable = allow_variable_override.unwrap_or(self.allow_variable_columns);
204
205        self.parse_with_config(file_path, delimiter, flexible, allow_variable)
206    }
207
208    fn parse_with_config(
209        &self,
210        file_path: &Path,
211        delimiter: u8,
212        flexible: bool,
213        _allow_variable: bool,
214    ) -> Result<(Vec<String>, Vec<Vec<String>>)> {
215        let mut reader = ReaderBuilder::new()
216            .delimiter(delimiter)
217            .has_headers(true)
218            .flexible(flexible)
219            .quote(self.quote_char.unwrap_or(b'"'))
220            .trim(if self.trim_whitespace {
221                Trim::All
222            } else {
223                Trim::None
224            })
225            .from_path(file_path)?;
226
227        let headers: Vec<String> = reader
228            .headers()?
229            .iter()
230            .map(|value| value.to_string())
231            .collect();
232        let expected_field_count = headers.len();
233        let mut records = Vec::new();
234
235        if flexible {
236            for result in reader.records() {
237                match result {
238                    Ok(record) => {
239                        let mut row: Vec<String> =
240                            record.iter().map(|value| value.to_string()).collect();
241                        if row.len() < expected_field_count {
242                            row.resize(expected_field_count, String::new());
243                        } else if row.len() > expected_field_count {
244                            row.truncate(expected_field_count);
245                        }
246                        records.push(row);
247                    }
248                    Err(_) => continue,
249                }
250            }
251        } else {
252            for result in reader.records() {
253                let record = result?;
254                let row: Vec<String> = record.iter().map(|value| value.to_string()).collect();
255                records.push(row);
256            }
257        }
258
259        Ok((headers, records))
260    }
261
262    /// Detect delimiter by sampling the file.
263    pub fn detect_delimiter(&self, file_path: &Path) -> Result<u8> {
264        let file = std::fs::File::open(file_path)?;
265        let reader = BufReader::new(file);
266
267        let mut lines = Vec::new();
268        for (index, line) in reader.lines().enumerate() {
269            if index >= 5 {
270                break;
271            }
272            lines.push(line?);
273        }
274
275        let delimiters = [b',', b';', b'\t', b'|'];
276        let mut best_delimiter = b',';
277        let mut max_consistency = 0;
278        let mut max_field_count = 0;
279
280        for &delimiter in &delimiters {
281            let consistency = self.measure_delimiter_consistency(&lines, delimiter);
282            let avg_field_count = self.average_field_count(&lines, delimiter);
283
284            let should_update = if avg_field_count > 1 && max_field_count <= 1 {
285                true
286            } else if avg_field_count <= 1 && max_field_count > 1 {
287                false
288            } else {
289                consistency > max_consistency
290                    || (consistency == max_consistency && avg_field_count > max_field_count)
291            };
292
293            if should_update {
294                max_consistency = consistency;
295                max_field_count = avg_field_count;
296                best_delimiter = delimiter;
297            }
298        }
299
300        Ok(best_delimiter)
301    }
302
303    fn measure_delimiter_consistency(&self, lines: &[String], delimiter: u8) -> usize {
304        if lines.is_empty() {
305            return 0;
306        }
307
308        let delimiter_char = delimiter as char;
309        let field_counts: Vec<usize> = lines
310            .iter()
311            .map(|line| self.count_fields_simple(line, delimiter_char))
312            .collect();
313
314        let mut counts = std::collections::HashMap::new();
315        for &count in &field_counts {
316            *counts.entry(count).or_insert(0) += 1;
317        }
318
319        counts.values().max().copied().unwrap_or(0)
320    }
321
322    fn average_field_count(&self, lines: &[String], delimiter: u8) -> usize {
323        if lines.is_empty() {
324            return 0;
325        }
326
327        let delimiter_char = delimiter as char;
328        let total_fields: usize = lines
329            .iter()
330            .map(|line| self.count_fields_simple(line, delimiter_char))
331            .sum();
332
333        total_fields / lines.len()
334    }
335
336    fn count_fields_simple(&self, line: &str, delimiter: char) -> usize {
337        let mut field_count = 1;
338        let mut in_quotes = false;
339        let mut chars = line.chars().peekable();
340
341        while let Some(ch) = chars.next() {
342            match ch {
343                '"' => {
344                    if chars.peek() == Some(&'"') {
345                        chars.next();
346                    } else {
347                        in_quotes = !in_quotes;
348                    }
349                }
350                current if current == delimiter && !in_quotes => {
351                    field_count += 1;
352                }
353                _ => {}
354            }
355        }
356
357        field_count
358    }
359
360    /// Parse CSV with robust error handling.
361    pub fn parse_csv(&self, file_path: &Path) -> Result<(Vec<String>, Vec<Vec<String>>)> {
362        let delimiter = if let Some(delimiter) = self.delimiter {
363            delimiter
364        } else {
365            self.detect_delimiter(file_path)
366                .context("Failed to detect delimiter")?
367        };
368
369        match self.try_strict_parsing(file_path, delimiter) {
370            Ok(result) => return Ok(result),
371            Err(error) => {
372                if self.verbosity >= 2 {
373                    log::info!("Using flexible CSV parsing (strict mode failed: {})", error);
374                }
375            }
376        }
377
378        self.try_flexible_parsing(file_path, delimiter)
379            .context("Both strict and flexible CSV parsing failed")
380    }
381
382    fn try_strict_parsing(
383        &self,
384        file_path: &Path,
385        delimiter: u8,
386    ) -> Result<(Vec<String>, Vec<Vec<String>>)> {
387        let mut reader = ReaderBuilder::new()
388            .delimiter(delimiter)
389            .has_headers(true)
390            .flexible(false)
391            .quote(self.quote_char.unwrap_or(b'"'))
392            .trim(if self.trim_whitespace {
393                Trim::All
394            } else {
395                Trim::None
396            })
397            .from_path(file_path)?;
398
399        let headers: Vec<String> = reader
400            .headers()?
401            .iter()
402            .map(|value| value.to_string())
403            .collect();
404        let mut records = Vec::new();
405
406        for result in reader.records() {
407            let record = result?;
408            let row: Vec<String> = record.iter().map(|value| value.to_string()).collect();
409            records.push(row);
410        }
411
412        Ok((headers, records))
413    }
414
415    fn try_flexible_parsing(
416        &self,
417        file_path: &Path,
418        delimiter: u8,
419    ) -> Result<(Vec<String>, Vec<Vec<String>>)> {
420        let mut reader = ReaderBuilder::new()
421            .delimiter(delimiter)
422            .has_headers(true)
423            .flexible(true)
424            .quote(self.quote_char.unwrap_or(b'"'))
425            .trim(if self.trim_whitespace {
426                Trim::All
427            } else {
428                Trim::None
429            })
430            .from_path(file_path)?;
431
432        let headers: Vec<String> = reader
433            .headers()?
434            .iter()
435            .map(|value| value.to_string())
436            .collect();
437        let expected_field_count = headers.len();
438        let mut records = Vec::new();
439        let mut error_count = 0;
440        let max_errors = 100;
441
442        for (row_index, result) in reader.records().enumerate() {
443            match result {
444                Ok(record) => {
445                    let mut row: Vec<String> =
446                        record.iter().map(|value| value.to_string()).collect();
447
448                    if self.allow_variable_columns {
449                        if row.len() < expected_field_count {
450                            row.resize(expected_field_count, String::new());
451                        } else if row.len() > expected_field_count {
452                            if row.len() == expected_field_count + 1
453                                && let (Some(second_last), Some(last)) = (
454                                    row.get(expected_field_count - 1),
455                                    row.get(expected_field_count),
456                                )
457                            {
458                                let combined =
459                                    format!("{}{}{}", second_last, delimiter as char, last);
460                                row[expected_field_count - 1] = combined;
461                            }
462                            row.truncate(expected_field_count);
463                        }
464                    }
465
466                    records.push(row);
467                }
468                Err(error) => {
469                    error_count += 1;
470
471                    let enhanced_error =
472                        DataProfilerError::csv_parsing(&error.to_string(), "current file");
473                    log::warn!("Row {}: {}", row_index + 2, enhanced_error);
474
475                    if error_count > max_errors {
476                        return Err(DataProfilerError::csv_parsing(
477                            &format!("Too many parsing errors ({})", error_count),
478                            "current file",
479                        )
480                        .into());
481                    }
482                }
483            }
484        }
485
486        if error_count > 0 {
487            log::warn!(
488                "Successfully parsed {} rows with {} errors skipped.",
489                records.len(),
490                error_count
491            );
492        }
493
494        Ok((headers, records))
495    }
496
497    /// Parse a specific chunk of CSV data.
498    pub fn parse_csv_chunk(
499        &self,
500        data: &str,
501        delimiter: u8,
502        has_headers: bool,
503    ) -> Result<RobustParseResult> {
504        let mut reader = ReaderBuilder::new()
505            .delimiter(delimiter)
506            .has_headers(has_headers)
507            .flexible(self.flexible)
508            .quote(self.quote_char.unwrap_or(b'"'))
509            .trim(if self.trim_whitespace {
510                Trim::All
511            } else {
512                Trim::None
513            })
514            .from_reader(Cursor::new(data));
515
516        let headers = if has_headers {
517            Some(
518                reader
519                    .headers()?
520                    .iter()
521                    .map(|value| value.to_string())
522                    .collect(),
523            )
524        } else {
525            None
526        };
527
528        let mut records = Vec::new();
529        let mut error_count = 0;
530
531        for (row_index, result) in reader.records().enumerate() {
532            match result {
533                Ok(record) => {
534                    let row: Vec<String> = record.iter().map(|value| value.to_string()).collect();
535                    records.push(row);
536                }
537                Err(error) => {
538                    error_count += 1;
539                    if error_count <= 10 {
540                        log::warn!("Chunk row {}: {}", row_index + 1, error);
541                    }
542                }
543            }
544        }
545
546        Ok((headers, records))
547    }
548
549    /// Validate CSV structure and provide diagnostic information.
550    pub fn validate_csv(&self, file_path: &Path) -> Result<CsvDiagnostics> {
551        let file = std::fs::File::open(file_path)?;
552        let reader = BufReader::new(file);
553        let mut diagnostics = CsvDiagnostics::default();
554
555        let mut field_counts = std::collections::HashMap::new();
556
557        for line_result in reader.lines() {
558            let line = line_result?;
559
560            if line.trim().is_empty() {
561                diagnostics.empty_lines += 1;
562                continue;
563            }
564
565            if line.contains('\0') {
566                diagnostics.null_bytes += 1;
567            }
568
569            let quote_count = line.chars().filter(|&ch| ch == '"').count();
570            if quote_count % 2 != 0 {
571                diagnostics.unmatched_quotes += 1;
572            }
573
574            let delimiter = self.delimiter.unwrap_or(b',') as char;
575            let field_count = self.count_fields_simple(&line, delimiter);
576            *field_counts.entry(field_count).or_insert(0) += 1;
577
578            diagnostics.total_lines += 1;
579        }
580
581        if let Some((&most_common_fields, &count)) =
582            field_counts.iter().max_by_key(|(_, count)| *count)
583        {
584            diagnostics.most_common_field_count = most_common_fields;
585            diagnostics.consistent_field_rows = count;
586        }
587
588        diagnostics.field_count_variations = field_counts.len();
589        diagnostics.inconsistent_field_rows =
590            diagnostics.total_lines - diagnostics.consistent_field_rows;
591
592        Ok(diagnostics)
593    }
594}
595
596#[derive(Debug, Default)]
597pub struct CsvDiagnostics {
598    pub total_lines: usize,
599    pub empty_lines: usize,
600    pub most_common_field_count: usize,
601    pub consistent_field_rows: usize,
602    pub inconsistent_field_rows: usize,
603    pub field_count_variations: usize,
604    pub unmatched_quotes: usize,
605    pub null_bytes: usize,
606}
607
608impl CsvDiagnostics {
609    pub fn print_summary(&self) {
610        println!("CSV File Diagnostics:");
611        println!("  Total lines: {}", self.total_lines);
612        println!("  Empty lines: {}", self.empty_lines);
613        println!(
614            "  Most common field count: {}",
615            self.most_common_field_count
616        );
617        println!(
618            "  Consistent rows: {} ({:.1}%)",
619            self.consistent_field_rows,
620            self.consistent_field_rows as f64 / self.total_lines as f64 * 100.0
621        );
622
623        if self.inconsistent_field_rows > 0 {
624            println!(
625                "  WARNING: Inconsistent rows: {} ({:.1}%)",
626                self.inconsistent_field_rows,
627                self.inconsistent_field_rows as f64 / self.total_lines as f64 * 100.0
628            );
629        }
630
631        if self.field_count_variations > 1 {
632            println!(
633                "  WARNING: Different field counts detected: {}",
634                self.field_count_variations
635            );
636        }
637
638        if self.unmatched_quotes > 0 {
639            println!(
640                "  WARNING: Lines with unmatched quotes: {}",
641                self.unmatched_quotes
642            );
643        }
644
645        if self.null_bytes > 0 {
646            println!("  WARNING: Lines with null bytes: {}", self.null_bytes);
647        }
648    }
649}
650
651#[cfg(test)]
652mod tests {
653    use super::*;
654    use std::io::Write;
655    use tempfile::NamedTempFile;
656
657    #[test]
658    fn test_robust_csv_parser_basic() -> Result<()> {
659        let mut temp_file = NamedTempFile::new()?;
660        writeln!(temp_file, "name,age,city")?;
661        writeln!(temp_file, "Alice,25,New York")?;
662        writeln!(temp_file, "Bob,30,London")?;
663        temp_file.flush()?;
664
665        let parser = RobustCsvParser::new();
666        let (headers, records) = parser.parse_csv(temp_file.path())?;
667
668        assert_eq!(headers, vec!["name", "age", "city"]);
669        assert_eq!(records.len(), 2);
670        assert_eq!(records[0], vec!["Alice", "25", "New York"]);
671
672        Ok(())
673    }
674
675    #[test]
676    fn test_flexible_field_counts() -> Result<()> {
677        let mut temp_file = NamedTempFile::new()?;
678        writeln!(temp_file, "name,age,city")?;
679        writeln!(temp_file, "Alice,25,New York")?;
680        writeln!(temp_file, "Bob,30")?;
681        writeln!(temp_file, "Charlie,35,London,UK")?;
682        temp_file.flush()?;
683
684        let mut parser = RobustCsvParser::new();
685        parser.delimiter = Some(b',');
686        let parser = parser.allow_variable_columns(true);
687        let (headers, records) = parser.parse_csv(temp_file.path())?;
688
689        assert_eq!(headers, vec!["name", "age", "city"]);
690        assert_eq!(records.len(), 3);
691        assert_eq!(records[1], vec!["Bob", "30", ""]);
692        assert_eq!(records[2], vec!["Charlie", "35", "London,UK"]);
693
694        Ok(())
695    }
696
697    #[test]
698    fn test_delimiter_detection() -> Result<()> {
699        let mut temp_file = NamedTempFile::new()?;
700        writeln!(temp_file, "name;age;city")?;
701        writeln!(temp_file, "Alice;25;New York")?;
702        writeln!(temp_file, "Bob;30;London")?;
703        temp_file.flush()?;
704
705        let parser = RobustCsvParser::new();
706        let delimiter = parser.detect_delimiter(temp_file.path())?;
707        assert!(delimiter == b';' || delimiter == b',');
708
709        Ok(())
710    }
711
712    #[test]
713    fn test_csv_diagnostics() -> Result<()> {
714        let mut temp_file = NamedTempFile::new()?;
715        writeln!(temp_file, "name,age,city")?;
716        writeln!(temp_file, "Alice,25,New York")?;
717        writeln!(temp_file, "Bob,30")?;
718        writeln!(temp_file)?;
719        writeln!(temp_file, "Charlie,35,London")?;
720        temp_file.flush()?;
721
722        let parser = RobustCsvParser::new();
723        let diagnostics = parser.validate_csv(temp_file.path())?;
724
725        assert_eq!(diagnostics.total_lines, 4);
726        assert_eq!(diagnostics.empty_lines, 1);
727        assert_eq!(diagnostics.most_common_field_count, 3);
728        assert_eq!(diagnostics.inconsistent_field_rows, 1);
729
730        Ok(())
731    }
732}