Skip to main content

dataprof_csv/
robust_csv.rs

1use anyhow::{Context, Result};
2use csv::{ReaderBuilder, Trim};
3use std::io::{BufRead, BufReader, Cursor};
4use std::path::Path;
5
6use dataprof_core::errors::AutoRecoveryManager;
7use dataprof_core::{DataProfilerError, RecoveryStrategy, RetryConfig};
8
9/// Result type for robust CSV parsing operations.
10pub type RobustParseResult = (Option<Vec<String>>, Vec<Vec<String>>);
11
12/// What [`diagnose_non_utf8`] found about a file that failed UTF-8 decoding.
13#[derive(Debug, Clone, PartialEq, Eq)]
14pub struct EncodingDiagnostic {
15    /// Best-effort encoding guess (e.g. `"latin-1"`, `"windows-1252"`, `"utf-16le"`).
16    pub guess: &'static str,
17    /// Byte offset of the first invalid UTF-8 sequence, when found within the
18    /// sampled prefix. `None` means the sampled prefix was valid UTF-8 and the
19    /// offending byte lies further into the file.
20    pub first_invalid_byte: Option<usize>,
21}
22
23impl EncodingDiagnostic {
24    /// Human-readable detail for an error message, without echoing file contents.
25    pub fn detail(&self) -> String {
26        // Plain ASCII only: this string ends up in an encoding-error message that
27        // may be printed to a non-UTF-8 terminal, so it must never add mojibake.
28        match self.first_invalid_byte {
29            Some(offset) => format!(
30                "looks like {}; first invalid UTF-8 byte at offset {offset}",
31                self.guess
32            ),
33            None => format!(
34                "looks like {}; contains bytes that are not valid UTF-8",
35                self.guess
36            ),
37        }
38    }
39}
40
41/// How many leading bytes [`diagnose_non_utf8`] inspects. Large enough to locate
42/// the offending byte in the common case, bounded so the error path never reads
43/// an entire large file.
44const ENCODING_SNIFF_LIMIT: usize = 1 << 20; // 1 MiB
45
46/// Inspect a file that failed UTF-8 decoding and guess its encoding.
47///
48/// Returns `None` when the sampled prefix is valid UTF-8 (so a failure that is
49/// *not* an encoding problem is never mislabeled as one). BOM-prefixed UTF-16 is
50/// reported directly; otherwise a byte in `0x80..=0x9F` points at windows-1252
51/// and anything else at latin-1 — a best-effort guess, not a definitive verdict.
52pub fn diagnose_non_utf8(path: &Path) -> Option<EncodingDiagnostic> {
53    use std::io::Read;
54
55    // decode-audit: unknown — this runs only to enrich an error the caller
56    // already has. If the file cannot be reopened/read for sniffing, returning
57    // None keeps that original engine error, so the failure is surfaced, not lost.
58    let file = std::fs::File::open(path).ok()?;
59    let mut sample = Vec::new();
60    // decode-audit: unknown — as above; a read failure abandons the guess and
61    // preserves the caller's original error rather than swallowing it.
62    file.take(ENCODING_SNIFF_LIMIT as u64)
63        .read_to_end(&mut sample)
64        .ok()?;
65
66    if sample.starts_with(&[0xFF, 0xFE]) {
67        return Some(EncodingDiagnostic {
68            guess: "utf-16le",
69            first_invalid_byte: Some(0),
70        });
71    }
72    if sample.starts_with(&[0xFE, 0xFF]) {
73        return Some(EncodingDiagnostic {
74            guess: "utf-16be",
75            first_invalid_byte: Some(0),
76        });
77    }
78
79    // A valid-UTF-8 prefix means the fault, if any, is beyond the sample — do not
80    // claim an encoding problem we cannot see.
81    let first_invalid_byte = match std::str::from_utf8(&sample) {
82        Ok(_) => return None,
83        Err(err) => Some(err.valid_up_to()),
84    };
85
86    let has_c1_controls = sample.iter().any(|&b| (0x80..=0x9F).contains(&b));
87    let guess = if has_c1_controls {
88        "windows-1252"
89    } else {
90        "latin-1"
91    };
92
93    Some(EncodingDiagnostic {
94        guess,
95        first_invalid_byte,
96    })
97}
98
99/// Robust CSV parser that handles edge cases and malformed data.
100pub struct RobustCsvParser {
101    pub flexible: bool,
102    pub quote_char: Option<u8>,
103    pub delimiter: Option<u8>,
104    pub allow_variable_columns: bool,
105    pub trim_whitespace: bool,
106    pub auto_recovery: bool,
107    pub retry_config: RetryConfig,
108    pub verbosity: u8,
109}
110
111impl Default for RobustCsvParser {
112    fn default() -> Self {
113        Self {
114            flexible: true,
115            quote_char: Some(b'"'),
116            delimiter: None,
117            allow_variable_columns: true,
118            trim_whitespace: true,
119            auto_recovery: true,
120            retry_config: RetryConfig::default(),
121            verbosity: 1,
122        }
123    }
124}
125
126impl RobustCsvParser {
127    pub fn new() -> Self {
128        Self::default()
129    }
130
131    pub fn flexible(mut self, flexible: bool) -> Self {
132        self.flexible = flexible;
133        self
134    }
135
136    pub fn allow_variable_columns(mut self, allow: bool) -> Self {
137        self.allow_variable_columns = allow;
138        self
139    }
140
141    /// Set verbosity level for logging.
142    /// 0=quiet, 1=normal, 2=verbose, 3=debug
143    pub fn verbosity(mut self, level: u8) -> Self {
144        self.verbosity = level;
145        self
146    }
147
148    /// Enable or disable auto-recovery features.
149    pub fn with_auto_recovery(mut self, enabled: bool) -> Self {
150        self.auto_recovery = enabled;
151        self
152    }
153
154    /// Set custom retry configuration.
155    pub fn with_retry_config(mut self, config: RetryConfig) -> Self {
156        self.retry_config = config;
157        self
158    }
159
160    /// Detect encoding by analyzing file bytes.
161    pub fn detect_encoding(&self, file_path: &Path) -> Result<&'static str> {
162        let mut file = std::fs::File::open(file_path)?;
163        let mut buffer = [0; 4096];
164        let bytes_read = std::io::Read::read(&mut file, &mut buffer)?;
165
166        let sample = &buffer[..bytes_read];
167
168        if sample.starts_with(&[0xEF, 0xBB, 0xBF]) {
169            return Ok("utf-8");
170        }
171        if sample.starts_with(&[0xFF, 0xFE]) {
172            return Ok("utf-16le");
173        }
174        if sample.starts_with(&[0xFE, 0xFF]) {
175            return Ok("utf-16be");
176        }
177
178        if String::from_utf8(sample.to_vec()).is_ok() {
179            return Ok("utf-8");
180        }
181
182        let latin1_indicators = [
183            0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8A, 0x8B, 0x8C, 0x8D,
184            0x8E, 0x8F,
185        ];
186        let has_latin1_chars = sample.iter().any(|&byte| latin1_indicators.contains(&byte));
187
188        if has_latin1_chars {
189            Ok("latin1")
190        } else {
191            Ok("utf-8")
192        }
193    }
194
195    /// Enhanced parsing with auto-recovery.
196    pub fn parse_csv_with_recovery(
197        &self,
198        file_path: &Path,
199    ) -> Result<(Vec<String>, Vec<Vec<String>>)> {
200        if !self.auto_recovery {
201            return self.parse_csv(file_path);
202        }
203
204        let mut recovery_manager = AutoRecoveryManager::new(self.retry_config.clone());
205
206        match self.parse_csv(file_path) {
207            Ok(result) => Ok(result),
208            Err(initial_error) => {
209                log::warn!(
210                    "Initial CSV parsing failed: {}. Attempting auto-recovery...",
211                    initial_error
212                );
213
214                let initial_error_string = initial_error.to_string();
215                let dp_error = if let Ok(dp_err) = initial_error.downcast::<DataProfilerError>() {
216                    dp_err
217                } else {
218                    DataProfilerError::csv_parsing(
219                        &initial_error_string,
220                        Some(file_path.to_string_lossy().as_ref()),
221                    )
222                };
223
224                // Duplicate headers are deterministic across strategies; recovery
225                // cannot resolve them, so surface the clear error instead of
226                // burying it under "auto-recovery failed".
227                if matches!(dp_error, DataProfilerError::DuplicateColumnName { .. }) {
228                    return Err(dp_error.into());
229                }
230
231                recovery_manager
232                    .attempt_recovery(dp_error, |strategy| {
233                        self.try_recovery_strategy(file_path, strategy)
234                    })
235                    .map_err(|error| error.into())
236            }
237        }
238    }
239
240    fn try_recovery_strategy(
241        &self,
242        file_path: &Path,
243        strategy: RecoveryStrategy,
244    ) -> Result<(Vec<String>, Vec<Vec<String>>), DataProfilerError> {
245        match strategy {
246            RecoveryStrategy::DelimiterDetection { delimiter } => {
247                log::debug!("Trying delimiter: '{}'", delimiter);
248                self.parse_with_override(file_path, Some(delimiter as u8), None, None)
249                    .map_err(|error| {
250                        DataProfilerError::csv_parsing(
251                            &error.to_string(),
252                            Some(file_path.to_string_lossy().as_ref()),
253                        )
254                    })
255            }
256            RecoveryStrategy::EncodingConversion { from: _, to: _ } => {
257                log::debug!(
258                    "Attempting encoding conversion (placeholder - full implementation needed)"
259                );
260                self.parse_with_override(file_path, None, Some(true), Some(true))
261                    .map_err(|error| {
262                        DataProfilerError::csv_parsing(
263                            &error.to_string(),
264                            Some(file_path.to_string_lossy().as_ref()),
265                        )
266                    })
267            }
268            RecoveryStrategy::FlexibleParsing => {
269                log::debug!("Enabling flexible parsing");
270                self.parse_with_override(file_path, None, Some(true), Some(true))
271                    .map_err(|error| {
272                        DataProfilerError::csv_parsing(
273                            &error.to_string(),
274                            Some(file_path.to_string_lossy().as_ref()),
275                        )
276                    })
277            }
278            _ => Err(DataProfilerError::csv_parsing(
279                "Unsupported recovery strategy for CSV parsing",
280                Some(file_path.to_string_lossy().as_ref()),
281            )),
282        }
283    }
284
285    fn parse_with_override(
286        &self,
287        file_path: &Path,
288        delimiter_override: Option<u8>,
289        flexible_override: Option<bool>,
290        allow_variable_override: Option<bool>,
291    ) -> Result<(Vec<String>, Vec<Vec<String>>)> {
292        let delimiter = delimiter_override
293            .or(self.delimiter)
294            .unwrap_or_else(|| self.detect_delimiter(file_path).unwrap_or(b','));
295
296        let flexible = flexible_override.unwrap_or(self.flexible);
297        let allow_variable = allow_variable_override.unwrap_or(self.allow_variable_columns);
298
299        self.parse_with_config(file_path, delimiter, flexible, allow_variable)
300    }
301
302    fn parse_with_config(
303        &self,
304        file_path: &Path,
305        delimiter: u8,
306        flexible: bool,
307        _allow_variable: bool,
308    ) -> Result<(Vec<String>, Vec<Vec<String>>)> {
309        let mut reader = ReaderBuilder::new()
310            .delimiter(delimiter)
311            .has_headers(true)
312            .flexible(flexible)
313            .quote(self.quote_char.unwrap_or(b'"'))
314            .trim(if self.trim_whitespace {
315                Trim::All
316            } else {
317                Trim::None
318            })
319            .from_path(file_path)?;
320
321        let headers: Vec<String> = reader
322            .headers()?
323            .iter()
324            .map(|value| value.to_string())
325            .collect();
326        let expected_field_count = headers.len();
327        let mut records = Vec::new();
328
329        if flexible {
330            for result in reader.records() {
331                match result {
332                    Ok(record) => {
333                        let mut row: Vec<String> =
334                            record.iter().map(|value| value.to_string()).collect();
335                        if row.len() < expected_field_count {
336                            row.resize(expected_field_count, String::new());
337                        } else if row.len() > expected_field_count {
338                            row.truncate(expected_field_count);
339                        }
340                        records.push(row);
341                    }
342                    Err(_) => continue,
343                }
344            }
345        } else {
346            for result in reader.records() {
347                let record = result?;
348                let row: Vec<String> = record.iter().map(|value| value.to_string()).collect();
349                records.push(row);
350            }
351        }
352
353        Ok((headers, records))
354    }
355
356    /// Detect delimiter by sampling the file.
357    ///
358    /// Delegates to the shared quote-aware sniffer so both parsers agree on
359    /// the delimiter for the same file.
360    pub fn detect_delimiter(&self, file_path: &Path) -> Result<u8> {
361        Ok(crate::detect_delimiter_from_path(file_path)?)
362    }
363
364    /// Parse CSV with robust error handling.
365    pub fn parse_csv(&self, file_path: &Path) -> Result<(Vec<String>, Vec<Vec<String>>)> {
366        let delimiter = if let Some(delimiter) = self.delimiter {
367            delimiter
368        } else {
369            self.detect_delimiter(file_path)
370                .context("Failed to detect delimiter")?
371        };
372
373        let result = match self.try_strict_parsing(file_path, delimiter) {
374            Ok(result) => result,
375            Err(error) => {
376                if self.verbosity >= 2 {
377                    log::info!("Using flexible CSV parsing (strict mode failed: {})", error);
378                }
379                self.try_flexible_parsing(file_path, delimiter)
380                    .context("Both strict and flexible CSV parsing failed")?
381            }
382        };
383
384        // Duplicate headers are a property of the file, identical across parse
385        // strategies, so reject once here rather than in each leaf parser.
386        dataprof_core::validate_unique_column_names(&result.0, "CSV header")?;
387        Ok(result)
388    }
389
390    fn try_strict_parsing(
391        &self,
392        file_path: &Path,
393        delimiter: u8,
394    ) -> Result<(Vec<String>, Vec<Vec<String>>)> {
395        let mut reader = ReaderBuilder::new()
396            .delimiter(delimiter)
397            .has_headers(true)
398            .flexible(false)
399            .quote(self.quote_char.unwrap_or(b'"'))
400            .trim(if self.trim_whitespace {
401                Trim::All
402            } else {
403                Trim::None
404            })
405            .from_path(file_path)?;
406
407        let headers: Vec<String> = reader
408            .headers()?
409            .iter()
410            .map(|value| value.to_string())
411            .collect();
412        let mut records = Vec::new();
413
414        for result in reader.records() {
415            let record = result?;
416            let row: Vec<String> = record.iter().map(|value| value.to_string()).collect();
417            records.push(row);
418        }
419
420        Ok((headers, records))
421    }
422
423    fn try_flexible_parsing(
424        &self,
425        file_path: &Path,
426        delimiter: u8,
427    ) -> Result<(Vec<String>, Vec<Vec<String>>)> {
428        let mut reader = ReaderBuilder::new()
429            .delimiter(delimiter)
430            .has_headers(true)
431            .flexible(true)
432            .quote(self.quote_char.unwrap_or(b'"'))
433            .trim(if self.trim_whitespace {
434                Trim::All
435            } else {
436                Trim::None
437            })
438            .from_path(file_path)?;
439
440        let headers: Vec<String> = reader
441            .headers()?
442            .iter()
443            .map(|value| value.to_string())
444            .collect();
445        let expected_field_count = headers.len();
446        let mut records = Vec::new();
447        let mut error_count = 0;
448        let max_errors = 100;
449
450        for (row_index, result) in reader.records().enumerate() {
451            match result {
452                Ok(record) => {
453                    let mut row: Vec<String> =
454                        record.iter().map(|value| value.to_string()).collect();
455
456                    if self.allow_variable_columns {
457                        if row.len() < expected_field_count {
458                            row.resize(expected_field_count, String::new());
459                        } else if row.len() > expected_field_count {
460                            if row.len() == expected_field_count + 1
461                                && let (Some(second_last), Some(last)) = (
462                                    row.get(expected_field_count - 1),
463                                    row.get(expected_field_count),
464                                )
465                            {
466                                let combined =
467                                    format!("{}{}{}", second_last, delimiter as char, last);
468                                row[expected_field_count - 1] = combined;
469                            }
470                            row.truncate(expected_field_count);
471                        }
472                    }
473
474                    records.push(row);
475                }
476                Err(error) => {
477                    error_count += 1;
478
479                    let file_path_str = file_path.to_string_lossy();
480                    let enhanced_error = DataProfilerError::csv_parsing(
481                        &error.to_string(),
482                        Some(file_path_str.as_ref()),
483                    );
484                    log::warn!("Row {}: {}", row_index + 2, enhanced_error);
485
486                    if error_count > max_errors {
487                        return Err(DataProfilerError::csv_parsing(
488                            &format!("Too many parsing errors ({})", error_count),
489                            Some(file_path_str.as_ref()),
490                        )
491                        .into());
492                    }
493                }
494            }
495        }
496
497        if error_count > 0 {
498            log::warn!(
499                "Successfully parsed {} rows with {} errors skipped.",
500                records.len(),
501                error_count
502            );
503        }
504
505        Ok((headers, records))
506    }
507
508    /// Parse a specific chunk of CSV data.
509    pub fn parse_csv_chunk(
510        &self,
511        data: &str,
512        delimiter: u8,
513        has_headers: bool,
514    ) -> Result<RobustParseResult> {
515        let mut reader = ReaderBuilder::new()
516            .delimiter(delimiter)
517            .has_headers(has_headers)
518            .flexible(self.flexible)
519            .quote(self.quote_char.unwrap_or(b'"'))
520            .trim(if self.trim_whitespace {
521                Trim::All
522            } else {
523                Trim::None
524            })
525            .from_reader(Cursor::new(data));
526
527        let headers: Option<Vec<String>> = if has_headers {
528            let names: Vec<String> = reader
529                .headers()?
530                .iter()
531                .map(|value| value.to_string())
532                .collect();
533            dataprof_core::validate_unique_column_names(&names, "CSV header")?;
534            Some(names)
535        } else {
536            None
537        };
538
539        let mut records = Vec::new();
540        let mut error_count = 0;
541
542        for (row_index, result) in reader.records().enumerate() {
543            match result {
544                Ok(record) => {
545                    let row: Vec<String> = record.iter().map(|value| value.to_string()).collect();
546                    records.push(row);
547                }
548                Err(error) => {
549                    error_count += 1;
550                    if error_count <= 10 {
551                        log::warn!("Chunk row {}: {}", row_index + 1, error);
552                    }
553                }
554            }
555        }
556
557        Ok((headers, records))
558    }
559
560    /// Validate CSV structure and provide diagnostic information.
561    pub fn validate_csv(&self, file_path: &Path) -> Result<CsvDiagnostics> {
562        let file = std::fs::File::open(file_path)?;
563        let reader = BufReader::new(file);
564        let mut diagnostics = CsvDiagnostics::default();
565
566        let mut field_counts = std::collections::HashMap::new();
567
568        for line_result in reader.lines() {
569            let line = line_result?;
570
571            if line.trim().is_empty() {
572                diagnostics.empty_lines += 1;
573                continue;
574            }
575
576            if line.contains('\0') {
577                diagnostics.null_bytes += 1;
578            }
579
580            let quote_count = line.chars().filter(|&ch| ch == '"').count();
581            if quote_count % 2 != 0 {
582                diagnostics.unmatched_quotes += 1;
583            }
584
585            let delimiter = self.delimiter.unwrap_or(b',') as char;
586            let field_count = crate::count_fields(&line, delimiter);
587            *field_counts.entry(field_count).or_insert(0) += 1;
588
589            diagnostics.total_lines += 1;
590        }
591
592        if let Some((&most_common_fields, &count)) =
593            field_counts.iter().max_by_key(|(_, count)| *count)
594        {
595            diagnostics.most_common_field_count = most_common_fields;
596            diagnostics.consistent_field_rows = count;
597        }
598
599        diagnostics.field_count_variations = field_counts.len();
600        diagnostics.inconsistent_field_rows =
601            diagnostics.total_lines - diagnostics.consistent_field_rows;
602
603        Ok(diagnostics)
604    }
605}
606
607#[derive(Debug, Default)]
608pub struct CsvDiagnostics {
609    pub total_lines: usize,
610    pub empty_lines: usize,
611    pub most_common_field_count: usize,
612    pub consistent_field_rows: usize,
613    pub inconsistent_field_rows: usize,
614    pub field_count_variations: usize,
615    pub unmatched_quotes: usize,
616    pub null_bytes: usize,
617}
618
619impl CsvDiagnostics {
620    pub fn print_summary(&self) {
621        println!("CSV File Diagnostics:");
622        println!("  Total lines: {}", self.total_lines);
623        println!("  Empty lines: {}", self.empty_lines);
624        println!(
625            "  Most common field count: {}",
626            self.most_common_field_count
627        );
628        println!(
629            "  Consistent rows: {} ({:.1}%)",
630            self.consistent_field_rows,
631            self.consistent_field_rows as f64 / self.total_lines as f64 * 100.0
632        );
633
634        if self.inconsistent_field_rows > 0 {
635            println!(
636                "  WARNING: Inconsistent rows: {} ({:.1}%)",
637                self.inconsistent_field_rows,
638                self.inconsistent_field_rows as f64 / self.total_lines as f64 * 100.0
639            );
640        }
641
642        if self.field_count_variations > 1 {
643            println!(
644                "  WARNING: Different field counts detected: {}",
645                self.field_count_variations
646            );
647        }
648
649        if self.unmatched_quotes > 0 {
650            println!(
651                "  WARNING: Lines with unmatched quotes: {}",
652                self.unmatched_quotes
653            );
654        }
655
656        if self.null_bytes > 0 {
657            println!("  WARNING: Lines with null bytes: {}", self.null_bytes);
658        }
659    }
660}
661
662#[cfg(test)]
663mod tests {
664    use super::*;
665    use std::io::Write;
666    use tempfile::NamedTempFile;
667
668    fn write_bytes(bytes: &[u8]) -> NamedTempFile {
669        let mut f = NamedTempFile::new().unwrap();
670        f.write_all(bytes).unwrap();
671        f.flush().unwrap();
672        f
673    }
674
675    #[test]
676    fn diagnose_non_utf8_flags_latin1() {
677        // "café" in latin-1: 0xE9 (é) is a UTF-8 leading byte for a 3-byte
678        // sequence, but here it is not followed by continuation bytes, so the
679        // stream is invalid UTF-8 at that position.
680        let f = write_bytes(b"name\ncaf\xE9\n");
681        let d = diagnose_non_utf8(f.path()).expect("latin-1 should be flagged");
682        assert_eq!(d.guess, "latin-1");
683        assert_eq!(d.first_invalid_byte, Some(8));
684    }
685
686    #[test]
687    fn diagnose_non_utf8_flags_windows1252() {
688        // 0x93/0x94 are smart quotes in windows-1252 (C1 range), invalid UTF-8.
689        let f = write_bytes(b"quote\n\x93hi\x94\n");
690        let d = diagnose_non_utf8(f.path()).expect("windows-1252 should be flagged");
691        assert_eq!(d.guess, "windows-1252");
692        assert_eq!(d.first_invalid_byte, Some(6));
693    }
694
695    #[test]
696    fn diagnose_non_utf8_flags_utf16_bom() {
697        let f = write_bytes(&[0xFF, 0xFE, b'a', 0x00]);
698        let d = diagnose_non_utf8(f.path()).expect("utf-16 BOM should be flagged");
699        assert_eq!(d.guess, "utf-16le");
700    }
701
702    #[test]
703    fn diagnose_non_utf8_ignores_clean_utf8() {
704        let f = write_bytes("name\ncafé\n".as_bytes());
705        assert!(diagnose_non_utf8(f.path()).is_none());
706    }
707
708    #[test]
709    fn encoding_diagnostic_detail_never_leaks_bytes() {
710        let d = EncodingDiagnostic {
711            guess: "latin-1",
712            first_invalid_byte: Some(8),
713        };
714        let detail = d.detail();
715        assert!(detail.contains("latin-1"));
716        assert!(detail.contains("offset 8"));
717    }
718
719    #[test]
720    fn test_robust_csv_parser_basic() -> Result<()> {
721        let mut temp_file = NamedTempFile::new()?;
722        writeln!(temp_file, "name,age,city")?;
723        writeln!(temp_file, "Alice,25,New York")?;
724        writeln!(temp_file, "Bob,30,London")?;
725        temp_file.flush()?;
726
727        let parser = RobustCsvParser::new();
728        let (headers, records) = parser.parse_csv(temp_file.path())?;
729
730        assert_eq!(headers, vec!["name", "age", "city"]);
731        assert_eq!(records.len(), 2);
732        assert_eq!(records[0], vec!["Alice", "25", "New York"]);
733
734        Ok(())
735    }
736
737    #[test]
738    fn test_flexible_field_counts() -> Result<()> {
739        let mut temp_file = NamedTempFile::new()?;
740        writeln!(temp_file, "name,age,city")?;
741        writeln!(temp_file, "Alice,25,New York")?;
742        writeln!(temp_file, "Bob,30")?;
743        writeln!(temp_file, "Charlie,35,London,UK")?;
744        temp_file.flush()?;
745
746        let mut parser = RobustCsvParser::new();
747        parser.delimiter = Some(b',');
748        let parser = parser.allow_variable_columns(true);
749        let (headers, records) = parser.parse_csv(temp_file.path())?;
750
751        assert_eq!(headers, vec!["name", "age", "city"]);
752        assert_eq!(records.len(), 3);
753        assert_eq!(records[1], vec!["Bob", "30", ""]);
754        assert_eq!(records[2], vec!["Charlie", "35", "London,UK"]);
755
756        Ok(())
757    }
758
759    #[test]
760    fn test_delimiter_detection() -> Result<()> {
761        let mut temp_file = NamedTempFile::new()?;
762        writeln!(temp_file, "name;age;city")?;
763        writeln!(temp_file, "Alice;25;New York")?;
764        writeln!(temp_file, "Bob;30;London")?;
765        temp_file.flush()?;
766
767        let parser = RobustCsvParser::new();
768        let delimiter = parser.detect_delimiter(temp_file.path())?;
769        assert!(delimiter == b';' || delimiter == b',');
770
771        Ok(())
772    }
773
774    #[test]
775    fn test_csv_diagnostics() -> Result<()> {
776        let mut temp_file = NamedTempFile::new()?;
777        writeln!(temp_file, "name,age,city")?;
778        writeln!(temp_file, "Alice,25,New York")?;
779        writeln!(temp_file, "Bob,30")?;
780        writeln!(temp_file)?;
781        writeln!(temp_file, "Charlie,35,London")?;
782        temp_file.flush()?;
783
784        let parser = RobustCsvParser::new();
785        let diagnostics = parser.validate_csv(temp_file.path())?;
786
787        assert_eq!(diagnostics.total_lines, 4);
788        assert_eq!(diagnostics.empty_lines, 1);
789        assert_eq!(diagnostics.most_common_field_count, 3);
790        assert_eq!(diagnostics.inconsistent_field_rows, 1);
791
792        Ok(())
793    }
794}