Skip to main content

dataprof_csv/
robust_csv.rs

1use anyhow::{Context, Result};
2use csv::{ReaderBuilder, Trim};
3use std::io::{BufRead, BufReader, Cursor};
4use std::path::Path;
5
6use dataprof_core::errors::AutoRecoveryManager;
7use dataprof_core::execution::{RecoveryEvent, RecoveryKind};
8use dataprof_core::{DataProfilerError, RecoveryStrategy, RetryConfig};
9
10/// Result type for robust CSV parsing operations.
11pub type RobustParseResult = (Option<Vec<String>>, Vec<Vec<String>>);
12
13/// Parsed CSV data and the recovery history needed to interpret it.
14///
15/// Unlike row-level diagnostics, these events describe whole parse attempts.
16/// Callers assembling a report can retain them in ExecutionMetadata.recovery_events.
17#[derive(Debug)]
18pub struct CsvParseOutput {
19    pub headers: Vec<String>,
20    pub records: Vec<Vec<String>>,
21    pub recovery_events: Vec<RecoveryEvent>,
22}
23
24/// What [`diagnose_non_utf8`] found about a file that failed UTF-8 decoding.
25#[derive(Debug, Clone, PartialEq, Eq)]
26pub struct EncodingDiagnostic {
27    /// Best-effort encoding guess (e.g. `"latin-1"`, `"windows-1252"`, `"utf-16le"`).
28    pub guess: &'static str,
29    /// Byte offset of the first invalid UTF-8 sequence, when found within the
30    /// sampled prefix. `None` means the sampled prefix was valid UTF-8 and the
31    /// offending byte lies further into the file.
32    pub first_invalid_byte: Option<usize>,
33}
34
35impl EncodingDiagnostic {
36    /// Human-readable detail for an error message, without echoing file contents.
37    pub fn detail(&self) -> String {
38        // Plain ASCII only: this string ends up in an encoding-error message that
39        // may be printed to a non-UTF-8 terminal, so it must never add mojibake.
40        match self.first_invalid_byte {
41            Some(offset) => format!(
42                "looks like {}; first invalid UTF-8 byte at offset {offset}",
43                self.guess
44            ),
45            None => format!(
46                "looks like {}; contains bytes that are not valid UTF-8",
47                self.guess
48            ),
49        }
50    }
51}
52
53/// How many leading bytes [`diagnose_non_utf8`] inspects. Large enough to locate
54/// the offending byte in the common case, bounded so the error path never reads
55/// an entire large file.
56const ENCODING_SNIFF_LIMIT: usize = 1 << 20; // 1 MiB
57
58/// Inspect a file that failed UTF-8 decoding and guess its encoding.
59///
60/// Returns `None` when the sampled prefix is valid UTF-8 (so a failure that is
61/// *not* an encoding problem is never mislabeled as one). BOM-prefixed UTF-16 is
62/// reported directly; otherwise a byte in `0x80..=0x9F` points at windows-1252
63/// and anything else at latin-1 — a best-effort guess, not a definitive verdict.
64pub fn diagnose_non_utf8(path: &Path) -> Option<EncodingDiagnostic> {
65    use std::io::Read;
66
67    // decode-audit: unknown — this runs only to enrich an error the caller
68    // already has. If the file cannot be reopened/read for sniffing, returning
69    // None keeps that original engine error, so the failure is surfaced, not lost.
70    let file = std::fs::File::open(path).ok()?;
71    let mut sample = Vec::new();
72    // decode-audit: unknown — as above; a read failure abandons the guess and
73    // preserves the caller's original error rather than swallowing it.
74    file.take(ENCODING_SNIFF_LIMIT as u64)
75        .read_to_end(&mut sample)
76        .ok()?;
77
78    if sample.starts_with(&[0xFF, 0xFE]) {
79        return Some(EncodingDiagnostic {
80            guess: "utf-16le",
81            first_invalid_byte: Some(0),
82        });
83    }
84    if sample.starts_with(&[0xFE, 0xFF]) {
85        return Some(EncodingDiagnostic {
86            guess: "utf-16be",
87            first_invalid_byte: Some(0),
88        });
89    }
90
91    // A valid-UTF-8 prefix means the fault, if any, is beyond the sample — do not
92    // claim an encoding problem we cannot see.
93    let first_invalid_byte = match std::str::from_utf8(&sample) {
94        Ok(_) => return None,
95        Err(err) => Some(err.valid_up_to()),
96    };
97
98    let has_c1_controls = sample.iter().any(|&b| (0x80..=0x9F).contains(&b));
99    let guess = if has_c1_controls {
100        "windows-1252"
101    } else {
102        "latin-1"
103    };
104
105    Some(EncodingDiagnostic {
106        guess,
107        first_invalid_byte,
108    })
109}
110
111/// Robust CSV parser that handles edge cases and malformed data.
112pub struct RobustCsvParser {
113    pub flexible: bool,
114    pub quote_char: Option<u8>,
115    pub delimiter: Option<u8>,
116    pub allow_variable_columns: bool,
117    pub trim_whitespace: bool,
118    pub auto_recovery: bool,
119    pub retry_config: RetryConfig,
120    pub verbosity: u8,
121}
122
123impl Default for RobustCsvParser {
124    fn default() -> Self {
125        Self {
126            flexible: true,
127            quote_char: Some(b'"'),
128            delimiter: None,
129            allow_variable_columns: true,
130            trim_whitespace: true,
131            auto_recovery: true,
132            retry_config: RetryConfig::default(),
133            verbosity: 1,
134        }
135    }
136}
137
138impl RobustCsvParser {
139    pub fn new() -> Self {
140        Self::default()
141    }
142
143    pub fn flexible(mut self, flexible: bool) -> Self {
144        self.flexible = flexible;
145        self
146    }
147
148    pub fn allow_variable_columns(mut self, allow: bool) -> Self {
149        self.allow_variable_columns = allow;
150        self
151    }
152
153    /// Set verbosity level for logging.
154    /// 0=quiet, 1=normal, 2=verbose, 3=debug
155    pub fn verbosity(mut self, level: u8) -> Self {
156        self.verbosity = level;
157        self
158    }
159
160    /// Enable or disable auto-recovery features.
161    pub fn with_auto_recovery(mut self, enabled: bool) -> Self {
162        self.auto_recovery = enabled;
163        self
164    }
165
166    /// Set custom retry configuration.
167    pub fn with_retry_config(mut self, config: RetryConfig) -> Self {
168        self.retry_config = config;
169        self
170    }
171
172    /// Detect encoding by analyzing file bytes.
173    pub fn detect_encoding(&self, file_path: &Path) -> Result<&'static str> {
174        let mut file = std::fs::File::open(file_path)?;
175        let mut buffer = [0; 4096];
176        let bytes_read = std::io::Read::read(&mut file, &mut buffer)?;
177
178        let sample = &buffer[..bytes_read];
179
180        if sample.starts_with(&[0xEF, 0xBB, 0xBF]) {
181            return Ok("utf-8");
182        }
183        if sample.starts_with(&[0xFF, 0xFE]) {
184            return Ok("utf-16le");
185        }
186        if sample.starts_with(&[0xFE, 0xFF]) {
187            return Ok("utf-16be");
188        }
189
190        if String::from_utf8(sample.to_vec()).is_ok() {
191            return Ok("utf-8");
192        }
193
194        let latin1_indicators = [
195            0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8A, 0x8B, 0x8C, 0x8D,
196            0x8E, 0x8F,
197        ];
198        let has_latin1_chars = sample.iter().any(|&byte| latin1_indicators.contains(&byte));
199
200        if has_latin1_chars {
201            Ok("latin1")
202        } else {
203            Ok("utf-8")
204        }
205    }
206
207    /// Enhanced parsing with auto-recovery.
208    pub fn parse_csv_with_recovery(&self, file_path: &Path) -> Result<CsvParseOutput> {
209        if !self.auto_recovery {
210            return self.parse_csv(file_path);
211        }
212
213        let mut recovery_manager = AutoRecoveryManager::new(self.retry_config.clone());
214        let mut recovery_events = Vec::new();
215
216        match self.parse_csv_recording(file_path, &mut recovery_events) {
217            Ok((headers, records)) => Ok(CsvParseOutput {
218                headers,
219                records,
220                recovery_events,
221            }),
222            Err(initial_error) => {
223                log::warn!(
224                    "Initial CSV parsing failed: {}. Attempting auto-recovery...",
225                    initial_error
226                );
227
228                let initial_error_string = format!("{initial_error:#}");
229                let dp_error = if let Ok(dp_err) = initial_error.downcast::<DataProfilerError>() {
230                    dp_err
231                } else {
232                    DataProfilerError::csv_parsing(
233                        &initial_error_string,
234                        Some(file_path.to_string_lossy().as_ref()),
235                    )
236                };
237
238                // Duplicate headers are deterministic across strategies; recovery
239                // cannot resolve them, so surface the clear error instead of
240                // burying it under "auto-recovery failed".
241                if matches!(dp_error, DataProfilerError::DuplicateColumnName { .. }) {
242                    return Err(dp_error.into());
243                }
244
245                let (headers, records) = recovery_manager
246                    .attempt_recovery(dp_error, |strategy| {
247                        self.try_recovery_strategy(file_path, strategy)
248                    })?;
249                let mut attempted = recovery_events
250                    .last()
251                    .map(|event| event.retry.clone())
252                    .unwrap_or_else(|| "configured_parse".to_string());
253                let mut error = initial_error_string;
254                for attempt in recovery_manager.get_recovery_log() {
255                    // The encoding strategy is currently only a flexible
256                    // parsing retry. Do not claim that bytes were transcoded.
257                    let retry = match &attempt.strategy {
258                        RecoveryStrategy::EncodingConversion { .. }
259                        | RecoveryStrategy::FlexibleParsing => "flexible".to_string(),
260                        strategy => format!("{strategy:?}"),
261                    };
262                    recovery_events.push(RecoveryEvent {
263                        kind: RecoveryKind::CsvAutoRecovery,
264                        attempted,
265                        retry: retry.clone(),
266                        error,
267                    });
268                    attempted = retry;
269                    error = match &attempt.error_message {
270                        Some(message) => message.clone(),
271                        None => break, // The final retry succeeded.
272                    };
273                }
274                Ok(CsvParseOutput {
275                    headers,
276                    records,
277                    recovery_events,
278                })
279            }
280        }
281    }
282
283    fn try_recovery_strategy(
284        &self,
285        file_path: &Path,
286        strategy: RecoveryStrategy,
287    ) -> Result<(Vec<String>, Vec<Vec<String>>), DataProfilerError> {
288        match strategy {
289            RecoveryStrategy::DelimiterDetection { delimiter } => {
290                log::debug!("Trying delimiter: '{}'", delimiter);
291                self.parse_with_override(file_path, Some(delimiter as u8), None, None)
292                    .map_err(|error| {
293                        DataProfilerError::csv_parsing(
294                            &error.to_string(),
295                            Some(file_path.to_string_lossy().as_ref()),
296                        )
297                    })
298            }
299            RecoveryStrategy::EncodingConversion { from: _, to: _ } => {
300                log::debug!(
301                    "Attempting encoding conversion (placeholder - full implementation needed)"
302                );
303                self.parse_with_override(file_path, None, Some(true), Some(true))
304                    .map_err(|error| {
305                        DataProfilerError::csv_parsing(
306                            &error.to_string(),
307                            Some(file_path.to_string_lossy().as_ref()),
308                        )
309                    })
310            }
311            RecoveryStrategy::FlexibleParsing => {
312                log::debug!("Enabling flexible parsing");
313                self.parse_with_override(file_path, None, Some(true), Some(true))
314                    .map_err(|error| {
315                        DataProfilerError::csv_parsing(
316                            &error.to_string(),
317                            Some(file_path.to_string_lossy().as_ref()),
318                        )
319                    })
320            }
321            _ => Err(DataProfilerError::csv_parsing(
322                "Unsupported recovery strategy for CSV parsing",
323                Some(file_path.to_string_lossy().as_ref()),
324            )),
325        }
326    }
327
328    fn parse_with_override(
329        &self,
330        file_path: &Path,
331        delimiter_override: Option<u8>,
332        flexible_override: Option<bool>,
333        allow_variable_override: Option<bool>,
334    ) -> Result<(Vec<String>, Vec<Vec<String>>)> {
335        let delimiter = delimiter_override
336            .or(self.delimiter)
337            .unwrap_or_else(|| self.detect_delimiter(file_path).unwrap_or(b','));
338
339        let flexible = flexible_override.unwrap_or(self.flexible);
340        let allow_variable = allow_variable_override.unwrap_or(self.allow_variable_columns);
341
342        self.parse_with_config(file_path, delimiter, flexible, allow_variable)
343    }
344
345    fn parse_with_config(
346        &self,
347        file_path: &Path,
348        delimiter: u8,
349        flexible: bool,
350        _allow_variable: bool,
351    ) -> Result<(Vec<String>, Vec<Vec<String>>)> {
352        let mut reader = ReaderBuilder::new()
353            .delimiter(delimiter)
354            .has_headers(true)
355            .flexible(flexible)
356            .quote(self.quote_char.unwrap_or(b'"'))
357            .trim(if self.trim_whitespace {
358                Trim::All
359            } else {
360                Trim::None
361            })
362            .from_path(file_path)?;
363
364        let headers: Vec<String> = reader
365            .headers()?
366            .iter()
367            .map(|value| value.to_string())
368            .collect();
369        let expected_field_count = headers.len();
370        let mut records = Vec::new();
371
372        if flexible {
373            for result in reader.records() {
374                match result {
375                    Ok(record) => {
376                        let mut row: Vec<String> =
377                            record.iter().map(|value| value.to_string()).collect();
378                        if row.len() < expected_field_count {
379                            row.resize(expected_field_count, String::new());
380                        } else if row.len() > expected_field_count {
381                            row.truncate(expected_field_count);
382                        }
383                        records.push(row);
384                    }
385                    Err(_) => continue,
386                }
387            }
388        } else {
389            for result in reader.records() {
390                let record = result?;
391                let row: Vec<String> = record.iter().map(|value| value.to_string()).collect();
392                records.push(row);
393            }
394        }
395
396        Ok((headers, records))
397    }
398
399    /// Detect delimiter by sampling the file.
400    ///
401    /// Delegates to the shared quote-aware sniffer so both parsers agree on
402    /// the delimiter for the same file.
403    pub fn detect_delimiter(&self, file_path: &Path) -> Result<u8> {
404        Ok(crate::detect_delimiter_from_path(file_path)?)
405    }
406
407    /// Parse CSV with robust error handling.
408    pub fn parse_csv(&self, file_path: &Path) -> Result<CsvParseOutput> {
409        let mut recovery_events = Vec::new();
410        let (headers, records) = self.parse_csv_recording(file_path, &mut recovery_events)?;
411        Ok(CsvParseOutput {
412            headers,
413            records,
414            recovery_events,
415        })
416    }
417
418    fn parse_csv_recording(
419        &self,
420        file_path: &Path,
421        recovery_events: &mut Vec<RecoveryEvent>,
422    ) -> Result<(Vec<String>, Vec<Vec<String>>)> {
423        let delimiter = if let Some(delimiter) = self.delimiter {
424            delimiter
425        } else {
426            self.detect_delimiter(file_path)
427                .context("Failed to detect delimiter")?
428        };
429
430        let result = match self.try_strict_parsing(file_path, delimiter) {
431            Ok(result) => result,
432            Err(error) => {
433                recovery_events.push(RecoveryEvent {
434                    kind: RecoveryKind::CsvAutoRecovery,
435                    attempted: "strict".to_string(),
436                    retry: "flexible".to_string(),
437                    error: error.to_string(),
438                });
439                if self.verbosity >= 2 {
440                    log::info!("Using flexible CSV parsing (strict mode failed: {})", error);
441                }
442                self.try_flexible_parsing(file_path, delimiter)
443                    .context("Both strict and flexible CSV parsing failed")?
444            }
445        };
446
447        // Duplicate headers are a property of the file, identical across parse
448        // strategies, so reject once here rather than in each leaf parser.
449        dataprof_core::validate_unique_column_names(&result.0, "CSV header")?;
450        Ok(result)
451    }
452
453    fn try_strict_parsing(
454        &self,
455        file_path: &Path,
456        delimiter: u8,
457    ) -> Result<(Vec<String>, Vec<Vec<String>>)> {
458        let mut reader = ReaderBuilder::new()
459            .delimiter(delimiter)
460            .has_headers(true)
461            .flexible(false)
462            .quote(self.quote_char.unwrap_or(b'"'))
463            .trim(if self.trim_whitespace {
464                Trim::All
465            } else {
466                Trim::None
467            })
468            .from_path(file_path)?;
469
470        let headers: Vec<String> = reader
471            .headers()?
472            .iter()
473            .map(|value| value.to_string())
474            .collect();
475        let mut records = Vec::new();
476
477        for result in reader.records() {
478            let record = result?;
479            let row: Vec<String> = record.iter().map(|value| value.to_string()).collect();
480            records.push(row);
481        }
482
483        Ok((headers, records))
484    }
485
486    fn try_flexible_parsing(
487        &self,
488        file_path: &Path,
489        delimiter: u8,
490    ) -> Result<(Vec<String>, Vec<Vec<String>>)> {
491        let mut reader = ReaderBuilder::new()
492            .delimiter(delimiter)
493            .has_headers(true)
494            .flexible(true)
495            .quote(self.quote_char.unwrap_or(b'"'))
496            .trim(if self.trim_whitespace {
497                Trim::All
498            } else {
499                Trim::None
500            })
501            .from_path(file_path)?;
502
503        let headers: Vec<String> = reader
504            .headers()?
505            .iter()
506            .map(|value| value.to_string())
507            .collect();
508        let expected_field_count = headers.len();
509        let mut records = Vec::new();
510        let mut error_count = 0;
511        let max_errors = 100;
512
513        for (row_index, result) in reader.records().enumerate() {
514            match result {
515                Ok(record) => {
516                    let mut row: Vec<String> =
517                        record.iter().map(|value| value.to_string()).collect();
518
519                    if self.allow_variable_columns {
520                        if row.len() < expected_field_count {
521                            row.resize(expected_field_count, String::new());
522                        } else if row.len() > expected_field_count {
523                            if row.len() == expected_field_count + 1
524                                && let (Some(second_last), Some(last)) = (
525                                    row.get(expected_field_count - 1),
526                                    row.get(expected_field_count),
527                                )
528                            {
529                                let combined =
530                                    format!("{}{}{}", second_last, delimiter as char, last);
531                                row[expected_field_count - 1] = combined;
532                            }
533                            row.truncate(expected_field_count);
534                        }
535                    }
536
537                    records.push(row);
538                }
539                Err(error) => {
540                    error_count += 1;
541
542                    let file_path_str = file_path.to_string_lossy();
543                    let enhanced_error = DataProfilerError::csv_parsing(
544                        &error.to_string(),
545                        Some(file_path_str.as_ref()),
546                    );
547                    log::warn!("Row {}: {}", row_index + 2, enhanced_error);
548
549                    if error_count > max_errors {
550                        return Err(DataProfilerError::csv_parsing(
551                            &format!("Too many parsing errors ({})", error_count),
552                            Some(file_path_str.as_ref()),
553                        )
554                        .into());
555                    }
556                }
557            }
558        }
559
560        if error_count > 0 {
561            log::warn!(
562                "Successfully parsed {} rows with {} errors skipped.",
563                records.len(),
564                error_count
565            );
566        }
567
568        Ok((headers, records))
569    }
570
571    /// Parse a specific chunk of CSV data.
572    pub fn parse_csv_chunk(
573        &self,
574        data: &str,
575        delimiter: u8,
576        has_headers: bool,
577    ) -> Result<RobustParseResult> {
578        let mut reader = ReaderBuilder::new()
579            .delimiter(delimiter)
580            .has_headers(has_headers)
581            .flexible(self.flexible)
582            .quote(self.quote_char.unwrap_or(b'"'))
583            .trim(if self.trim_whitespace {
584                Trim::All
585            } else {
586                Trim::None
587            })
588            .from_reader(Cursor::new(data));
589
590        let headers: Option<Vec<String>> = if has_headers {
591            let names: Vec<String> = reader
592                .headers()?
593                .iter()
594                .map(|value| value.to_string())
595                .collect();
596            dataprof_core::validate_unique_column_names(&names, "CSV header")?;
597            Some(names)
598        } else {
599            None
600        };
601
602        let mut records = Vec::new();
603        let mut error_count = 0;
604
605        for (row_index, result) in reader.records().enumerate() {
606            match result {
607                Ok(record) => {
608                    let row: Vec<String> = record.iter().map(|value| value.to_string()).collect();
609                    records.push(row);
610                }
611                Err(error) => {
612                    error_count += 1;
613                    if error_count <= 10 {
614                        log::warn!("Chunk row {}: {}", row_index + 1, error);
615                    }
616                }
617            }
618        }
619
620        Ok((headers, records))
621    }
622
623    /// Validate CSV structure and provide diagnostic information.
624    pub fn validate_csv(&self, file_path: &Path) -> Result<CsvDiagnostics> {
625        let file = std::fs::File::open(file_path)?;
626        let reader = BufReader::new(file);
627        let mut diagnostics = CsvDiagnostics::default();
628
629        let mut field_counts = std::collections::HashMap::new();
630
631        for line_result in reader.lines() {
632            let line = line_result?;
633
634            if line.trim().is_empty() {
635                diagnostics.empty_lines += 1;
636                continue;
637            }
638
639            if line.contains('\0') {
640                diagnostics.null_bytes += 1;
641            }
642
643            let quote_count = line.chars().filter(|&ch| ch == '"').count();
644            if quote_count % 2 != 0 {
645                diagnostics.unmatched_quotes += 1;
646            }
647
648            let delimiter = self.delimiter.unwrap_or(b',') as char;
649            let field_count = crate::count_fields(&line, delimiter);
650            *field_counts.entry(field_count).or_insert(0) += 1;
651
652            diagnostics.total_lines += 1;
653        }
654
655        if let Some((&most_common_fields, &count)) =
656            field_counts.iter().max_by_key(|(_, count)| *count)
657        {
658            diagnostics.most_common_field_count = most_common_fields;
659            diagnostics.consistent_field_rows = count;
660        }
661
662        diagnostics.field_count_variations = field_counts.len();
663        diagnostics.inconsistent_field_rows =
664            diagnostics.total_lines - diagnostics.consistent_field_rows;
665
666        Ok(diagnostics)
667    }
668}
669
670#[derive(Debug, Default)]
671pub struct CsvDiagnostics {
672    pub total_lines: usize,
673    pub empty_lines: usize,
674    pub most_common_field_count: usize,
675    pub consistent_field_rows: usize,
676    pub inconsistent_field_rows: usize,
677    pub field_count_variations: usize,
678    pub unmatched_quotes: usize,
679    pub null_bytes: usize,
680}
681
682impl CsvDiagnostics {
683    pub fn print_summary(&self) {
684        println!("CSV File Diagnostics:");
685        println!("  Total lines: {}", self.total_lines);
686        println!("  Empty lines: {}", self.empty_lines);
687        println!(
688            "  Most common field count: {}",
689            self.most_common_field_count
690        );
691        println!(
692            "  Consistent rows: {} ({:.1}%)",
693            self.consistent_field_rows,
694            self.consistent_field_rows as f64 / self.total_lines as f64 * 100.0
695        );
696
697        if self.inconsistent_field_rows > 0 {
698            println!(
699                "  WARNING: Inconsistent rows: {} ({:.1}%)",
700                self.inconsistent_field_rows,
701                self.inconsistent_field_rows as f64 / self.total_lines as f64 * 100.0
702            );
703        }
704
705        if self.field_count_variations > 1 {
706            println!(
707                "  WARNING: Different field counts detected: {}",
708                self.field_count_variations
709            );
710        }
711
712        if self.unmatched_quotes > 0 {
713            println!(
714                "  WARNING: Lines with unmatched quotes: {}",
715                self.unmatched_quotes
716            );
717        }
718
719        if self.null_bytes > 0 {
720            println!("  WARNING: Lines with null bytes: {}", self.null_bytes);
721        }
722    }
723}
724
725#[cfg(test)]
726mod tests {
727    use super::*;
728    use std::io::Write;
729    use tempfile::NamedTempFile;
730
731    fn write_bytes(bytes: &[u8]) -> NamedTempFile {
732        let mut f = NamedTempFile::new().unwrap();
733        f.write_all(bytes).unwrap();
734        f.flush().unwrap();
735        f
736    }
737
738    #[test]
739    fn diagnose_non_utf8_flags_latin1() {
740        // "café" in latin-1: 0xE9 (é) is a UTF-8 leading byte for a 3-byte
741        // sequence, but here it is not followed by continuation bytes, so the
742        // stream is invalid UTF-8 at that position.
743        let f = write_bytes(b"name\ncaf\xE9\n");
744        let d = diagnose_non_utf8(f.path()).expect("latin-1 should be flagged");
745        assert_eq!(d.guess, "latin-1");
746        assert_eq!(d.first_invalid_byte, Some(8));
747    }
748
749    #[test]
750    fn diagnose_non_utf8_flags_windows1252() {
751        // 0x93/0x94 are smart quotes in windows-1252 (C1 range), invalid UTF-8.
752        let f = write_bytes(b"quote\n\x93hi\x94\n");
753        let d = diagnose_non_utf8(f.path()).expect("windows-1252 should be flagged");
754        assert_eq!(d.guess, "windows-1252");
755        assert_eq!(d.first_invalid_byte, Some(6));
756    }
757
758    #[test]
759    fn diagnose_non_utf8_flags_utf16_bom() {
760        let f = write_bytes(&[0xFF, 0xFE, b'a', 0x00]);
761        let d = diagnose_non_utf8(f.path()).expect("utf-16 BOM should be flagged");
762        assert_eq!(d.guess, "utf-16le");
763    }
764
765    #[test]
766    fn diagnose_non_utf8_ignores_clean_utf8() {
767        let f = write_bytes("name\ncafé\n".as_bytes());
768        assert!(diagnose_non_utf8(f.path()).is_none());
769    }
770
771    #[test]
772    fn encoding_diagnostic_detail_never_leaks_bytes() {
773        let d = EncodingDiagnostic {
774            guess: "latin-1",
775            first_invalid_byte: Some(8),
776        };
777        let detail = d.detail();
778        assert!(detail.contains("latin-1"));
779        assert!(detail.contains("offset 8"));
780    }
781
782    #[test]
783    fn test_robust_csv_parser_basic() -> Result<()> {
784        let mut temp_file = NamedTempFile::new()?;
785        writeln!(temp_file, "name,age,city")?;
786        writeln!(temp_file, "Alice,25,New York")?;
787        writeln!(temp_file, "Bob,30,London")?;
788        temp_file.flush()?;
789
790        let parser = RobustCsvParser::new();
791        let CsvParseOutput {
792            headers,
793            records,
794            recovery_events,
795        } = parser.parse_csv(temp_file.path())?;
796        assert!(recovery_events.is_empty());
797
798        assert_eq!(headers, vec!["name", "age", "city"]);
799        assert_eq!(records.len(), 2);
800        assert_eq!(records[0], vec!["Alice", "25", "New York"]);
801
802        Ok(())
803    }
804
805    #[test]
806    fn test_flexible_field_counts() -> Result<()> {
807        let mut temp_file = NamedTempFile::new()?;
808        writeln!(temp_file, "name,age,city")?;
809        writeln!(temp_file, "Alice,25,New York")?;
810        writeln!(temp_file, "Bob,30")?;
811        writeln!(temp_file, "Charlie,35,London,UK")?;
812        temp_file.flush()?;
813
814        let mut parser = RobustCsvParser::new();
815        parser.delimiter = Some(b',');
816        let parser = parser.allow_variable_columns(true);
817        let CsvParseOutput {
818            headers,
819            records,
820            recovery_events,
821        } = parser.parse_csv(temp_file.path())?;
822        assert_eq!(recovery_events.len(), 1);
823        assert_eq!(recovery_events[0].kind, RecoveryKind::CsvAutoRecovery);
824        assert_eq!(recovery_events[0].attempted, "strict");
825        assert_eq!(recovery_events[0].retry, "flexible");
826        assert!(recovery_events[0].error.contains("fields"));
827
828        assert_eq!(headers, vec!["name", "age", "city"]);
829        assert_eq!(records.len(), 3);
830        assert_eq!(records[1], vec!["Bob", "30", ""]);
831        assert_eq!(records[2], vec!["Charlie", "35", "London,UK"]);
832
833        Ok(())
834    }
835
836    #[test]
837    fn test_delimiter_detection() -> Result<()> {
838        let mut temp_file = NamedTempFile::new()?;
839        writeln!(temp_file, "name;age;city")?;
840        writeln!(temp_file, "Alice;25;New York")?;
841        writeln!(temp_file, "Bob;30;London")?;
842        temp_file.flush()?;
843
844        let parser = RobustCsvParser::new();
845        let delimiter = parser.detect_delimiter(temp_file.path())?;
846        assert!(delimiter == b';' || delimiter == b',');
847
848        Ok(())
849    }
850
851    #[test]
852    fn auto_recovery_records_initial_failure_and_every_retry() -> Result<()> {
853        let mut file = NamedTempFile::new()?;
854        writeln!(file, "name,value")?;
855        // Enough invalid UTF-8 records to exhaust the ordinary flexible
856        // parser's error budget and reach the outer recovery manager.
857        for _ in 0..101 {
858            file.write_all(b"bad,\xff\n")?;
859        }
860        writeln!(file, "good,42")?;
861        file.flush()?;
862
863        let mut parser = RobustCsvParser::new();
864        parser.delimiter = Some(b',');
865        // Delimiter retries retain strict parsing and fail. The encoding
866        // strategy currently retries flexible parsing and succeeds.
867        parser.flexible = false;
868        parser.retry_config.max_attempts = 5;
869        let output = parser.parse_csv_with_recovery(file.path())?;
870        assert_eq!(output.records, vec![vec!["good", "42"]]);
871        assert_eq!(output.recovery_events.len(), 6);
872        assert_eq!(output.recovery_events[0].attempted, "strict");
873        assert_eq!(output.recovery_events[0].retry, "flexible");
874        assert_eq!(output.recovery_events[1].attempted, "flexible");
875        assert!(
876            output.recovery_events[1]
877                .error
878                .contains("Both strict and flexible")
879        );
880        for pair in output.recovery_events.windows(2) {
881            assert_eq!(pair[0].retry, pair[1].attempted);
882        }
883        assert_eq!(output.recovery_events.last().unwrap().retry, "flexible");
884        assert!(
885            output
886                .recovery_events
887                .iter()
888                .all(|event| !event.retry.contains("EncodingConversion"))
889        );
890        assert!(
891            output
892                .recovery_events
893                .iter()
894                .all(|event| !event.error.is_empty())
895        );
896        assert!(
897            parser
898                .with_auto_recovery(false)
899                .parse_csv_with_recovery(file.path())
900                .is_err()
901        );
902        Ok(())
903    }
904
905    #[test]
906    fn test_csv_diagnostics() -> Result<()> {
907        let mut temp_file = NamedTempFile::new()?;
908        writeln!(temp_file, "name,age,city")?;
909        writeln!(temp_file, "Alice,25,New York")?;
910        writeln!(temp_file, "Bob,30")?;
911        writeln!(temp_file)?;
912        writeln!(temp_file, "Charlie,35,London")?;
913        temp_file.flush()?;
914
915        let parser = RobustCsvParser::new();
916        let diagnostics = parser.validate_csv(temp_file.path())?;
917
918        assert_eq!(diagnostics.total_lines, 4);
919        assert_eq!(diagnostics.empty_lines, 1);
920        assert_eq!(diagnostics.most_common_field_count, 3);
921        assert_eq!(diagnostics.inconsistent_field_rows, 1);
922
923        Ok(())
924    }
925}