Skip to main content

dataprof_csv/
lib.rs

1use csv::ReaderBuilder;
2use std::io::Read;
3use std::path::Path;
4
5use dataprof_core::{
6    AnalysisOptions, ColumnProfile, DataProfilerError, DataSource, ExecutionMetadata, FileFormat,
7    TruncationReason,
8};
9use dataprof_runtime::{
10    ProfileReport, ReportAssembler, StreamingColumnCollection, profile_builder,
11};
12
13mod encoding;
14mod memmap_reader;
15mod quote_state;
16#[doc(hidden)]
17pub mod row_count;
18
19pub use encoding::{EncodingDiagnostic, diagnose_non_utf8};
20pub use memmap_reader::MemoryMappedCsvReader;
21pub use quote_state::{QuoteOutcome, QuoteTrackingReader, unterminated_quote_error};
22
23/// Configuration for CSV parsing and analysis.
24///
25/// Replaces the 7 separate `analyze_csv_*` functions with a single config struct.
26/// Supports both file-based and reader-based (streaming) entry points.
27#[derive(Debug, Clone)]
28pub struct CsvParserConfig {
29    /// Allow ragged rows (flexible field counts).
30    pub flexible: bool,
31    /// Custom delimiter (None = comma).
32    pub delimiter: Option<u8>,
33    /// Quote character.
34    pub quote_char: u8,
35    /// Trim whitespace from fields.
36    pub trim_whitespace: bool,
37    /// Whether the first row contains column headers.
38    /// When `false`, columns are named `column_0`, `column_1`, etc.
39    pub has_header: bool,
40    /// Maximum rows to process (None = all rows).
41    pub max_rows: Option<usize>,
42}
43
44impl Default for CsvParserConfig {
45    fn default() -> Self {
46        Self {
47            flexible: true,
48            delimiter: None,
49            quote_char: b'"',
50            trim_whitespace: false,
51            has_header: true,
52            max_rows: None,
53        }
54    }
55}
56
57impl CsvParserConfig {
58    /// Create a strict config that rejects ragged rows.
59    pub fn strict() -> Self {
60        Self {
61            flexible: false,
62            ..Default::default()
63        }
64    }
65
66    /// Set the delimiter.
67    pub fn with_delimiter(mut self, delimiter: u8) -> Self {
68        self.delimiter = Some(delimiter);
69        self
70    }
71
72    /// Set whether the first row contains column headers.
73    pub fn has_header(mut self, yes: bool) -> Self {
74        self.has_header = yes;
75        self
76    }
77
78    /// Set maximum number of rows to evaluate.
79    pub fn max_rows(mut self, max: Option<usize>) -> Self {
80        self.max_rows = max;
81        self
82    }
83}
84
85/// Count the fields in a record for a given delimiter, respecting quoted
86/// sections, and how many of them at the end are empty.
87///
88/// Returns `(fields, trailing_empty)`. A field is empty when it decodes to
89/// nothing, as the readers decode it: nothing stands between its delimiters,
90/// or it is a quoted empty string `""`. Judging the raw text instead counted
91/// `""` as content, and a row ending in `;""` disagreed with its header
92/// (#1102).
93fn count_fields(line: &str, delimiter: char) -> (usize, usize) {
94    let decodes_empty = |field: &str| field.is_empty() || field == "\"\"";
95    let mut fields = 1;
96    let mut trailing_empty = 0;
97    let mut field_start = 0;
98    let mut in_quotes = false;
99    let mut chars = line.char_indices().peekable();
100    while let Some((index, ch)) = chars.next() {
101        if ch == '"' {
102            if chars.peek().map(|&(_, next)| next) == Some('"') {
103                chars.next();
104            } else {
105                in_quotes = !in_quotes;
106            }
107        } else if ch == delimiter && !in_quotes {
108            trailing_empty = if decodes_empty(&line[field_start..index]) {
109                trailing_empty + 1
110            } else {
111                0
112            };
113            fields += 1;
114            field_start = index + ch.len_utf8();
115        }
116    }
117    trailing_empty = if decodes_empty(&line[field_start..]) {
118        trailing_empty + 1
119    } else {
120        0
121    };
122    (fields, trailing_empty)
123}
124
125/// Whether a record reads at the header's width without losing or inventing a
126/// value: it has exactly that many fields, or more whose extras are all empty,
127/// so cutting it to the header drops nothing (#1096).
128fn agrees_with_header(fields: usize, trailing_empty: usize, header_fields: usize) -> bool {
129    fields == header_fields || (fields > header_fields && fields - trailing_empty <= header_fields)
130}
131
132/// Split raw CSV sample text into logical records, honoring quoted sections
133/// so an embedded newline inside quotes does not start a new record.
134///
135/// A record ends at LF, CRLF or a lone CR outside quotes, as in the readers.
136/// Splitting on LF alone read a CR-only file as one header record (#1097).
137///
138/// The returned flag is true when the final record was not terminated by a
139/// newline outside quotes — i.e. the text ends mid-record (or exactly at a
140/// record's end with no trailing newline, which a caller sampling a fixed
141/// byte budget cannot distinguish from a cut).
142fn split_sample_records(text: &str) -> (Vec<&str>, bool) {
143    let bytes = text.as_bytes();
144    let mut records = Vec::new();
145    let mut in_quotes = false;
146    let mut start = 0usize;
147    let mut i = 0usize;
148    while i < bytes.len() {
149        match bytes[i] {
150            b'"' => {
151                if in_quotes && bytes.get(i + 1) == Some(&b'"') {
152                    i += 1; // escaped quote inside a quoted section
153                } else {
154                    in_quotes = !in_quotes;
155                }
156            }
157            b'\n' if !in_quotes => {
158                let end = if i > start && bytes[i - 1] == b'\r' {
159                    i - 1
160                } else {
161                    i
162                };
163                records.push(&text[start..end]);
164                start = i + 1;
165            }
166            b'\r' if !in_quotes && bytes.get(i + 1) != Some(&b'\n') => {
167                records.push(&text[start..i]);
168                start = i + 1;
169            }
170            _ => {}
171        }
172        i += 1;
173    }
174    let last_unterminated = start < text.len();
175    if last_unterminated {
176        records.push(text[start..].trim_end_matches('\r'));
177    }
178    (records, last_unterminated)
179}
180
181/// Detect the most likely CSV delimiter by sampling up to 4 KB of data.
182///
183/// The sample is split into logical records with quote handling, so quoted
184/// fields containing embedded newlines or delimiter characters do not skew
185/// detection. Tests comma, semicolon, tab, and pipe over the first 5 non-blank
186/// records, measured against the first one, the header: it sets the column
187/// count, and the readers cut or pad a wider or narrower record to it, or
188/// reject it under strict parsing. A delimiter that splits the header always
189/// beats one that does not; then more records agreeing with the header win,
190/// then more records at exactly the header's width, then a wider header, and
191/// comma wins a tie. A record agrees when it reads at the header's width
192/// without losing a value: the same field count, or extra fields that are all
193/// empty, as an exporter's trailing delimiter leaves.
194///
195/// Judging by the data rows alone let a delimiter that never occurs in the
196/// header win, and the readers then cut every row back to its first field: a
197/// one-column file of `123|456` keys was profiled as `123` with fabricated
198/// duplicates (#1092).
199pub fn detect_delimiter<R: Read>(reader: R) -> std::io::Result<u8> {
200    const SAMPLE_BYTES: u64 = 4096;
201    let mut preamble = Vec::new();
202    let mut take = reader.take(SAMPLE_BYTES);
203    take.read_to_end(&mut preamble)?;
204    let truncated = preamble.len() as u64 == SAMPLE_BYTES;
205
206    // A UTF-8 BOM marks the encoding, as in the readers, and is no part of
207    // the first record: left in, a BOM followed by a blank line was scored as
208    // a one-field header and outvoted the real one.
209    let sample = preamble.strip_prefix(b"\xEF\xBB\xBF").unwrap_or(&preamble);
210    let preamble_str = String::from_utf8_lossy(sample);
211    let (mut records, last_unterminated) = split_sample_records(&preamble_str);
212    // A full sample buffer that ends mid-record leaves a fragment; never score
213    // it. A sample ending on a clean newline keeps all its records.
214    if truncated && last_unterminated && records.len() > 1 {
215        records.pop();
216    }
217
218    // Blank records carry no delimiter signal: a header followed by trailing
219    // blank lines must not outvote the true delimiter with single-field
220    // "records" (gh #559).
221    let scored: Vec<&str> = records
222        .into_iter()
223        .filter(|record| !record.trim().is_empty())
224        .take(5)
225        .collect();
226    let Some((header, rows)) = scored.split_first() else {
227        return Ok(b',');
228    };
229
230    let delimiters = *b",;\t|";
231    let mut best_delimiter = b',';
232    // (splits the header into >1 field, records agreeing with the header,
233    //  records at exactly the header's width, header field count) — compared
234    //  lexicographically, first wins ties. Exact width only breaks a tie: a
235    //  candidate that reads every record as it stands beats one that needs
236    //  trailing empty fields cut to agree.
237    let mut best_score = (false, 0usize, 0usize, 0usize);
238
239    for &delimiter in &delimiters {
240        let (header_fields, _) = count_fields(header, delimiter as char);
241        let (mut agreeing, mut exact) = (1, 1);
242        for record in rows {
243            let (fields, trailing_empty) = count_fields(record, delimiter as char);
244            if agrees_with_header(fields, trailing_empty, header_fields) {
245                agreeing += 1;
246            }
247            if fields == header_fields {
248                exact += 1;
249            }
250        }
251
252        let score = (header_fields > 1, agreeing, exact, header_fields);
253        if score > best_score {
254            best_score = score;
255            best_delimiter = delimiter;
256        }
257    }
258
259    Ok(best_delimiter)
260}
261
262/// Detect the most likely CSV delimiter from a file path.
263pub fn detect_delimiter_from_path(path: &Path) -> std::io::Result<u8> {
264    let file = std::fs::File::open(path)?;
265    detect_delimiter(std::io::BufReader::new(file))
266}
267
268/// Analyze CSV data from any `Read` source using streaming statistics.
269///
270/// This is the primary generic entry point for both file-based and stream-based
271/// CSV profiling. Data is processed incrementally through [`StreamingColumnCollection`]
272/// with bounded memory, rather than accumulating all rows in a `HashMap`.
273///
274/// Returns `(column_profiles, streaming_stats, rows_read)`.
275pub fn analyze_csv_from_reader<R: Read>(
276    reader: R,
277    config: &CsvParserConfig,
278) -> Result<
279    (
280        Vec<ColumnProfile>,
281        StreamingColumnCollection,
282        usize,
283        Vec<String>,
284    ),
285    DataProfilerError,
286> {
287    let scan = scan_csv(reader, config)?;
288    let profiles = profile_builder::profiles_from_streaming(&scan.column_stats, false, false, None);
289
290    Ok((
291        profiles,
292        scan.column_stats,
293        scan.rows_read,
294        scan.header_names,
295    ))
296}
297
298/// What one pass over CSV input read, and how much of the input that was.
299struct CsvScan {
300    column_stats: StreamingColumnCollection,
301    header_names: Vec<String>,
302    rows_read: usize,
303    /// Records whose field count differs from the header's, padded or cut to
304    /// the header width before profiling.
305    ragged_rows: usize,
306    /// Set when `max_rows` stopped the pass with records left in the input.
307    truncation: Option<TruncationReason>,
308}
309
310fn scan_csv<R: Read>(reader: R, config: &CsvParserConfig) -> Result<CsvScan, DataProfilerError> {
311    let mut csv_builder = ReaderBuilder::new();
312    csv_builder.has_headers(config.has_header);
313    csv_builder.flexible(config.flexible);
314    csv_builder.quote(config.quote_char);
315    if config.trim_whitespace {
316        csv_builder.trim(csv::Trim::All);
317    }
318    let mut actual_delimiter = config.delimiter;
319
320    let boxed_reader: Box<dyn std::io::Read + '_> = if config.delimiter.is_none() {
321        let mut preamble = Vec::new();
322        let mut take = reader.take(4096);
323        take.read_to_end(&mut preamble)?;
324
325        let detected = detect_delimiter(std::io::Cursor::new(&preamble)).unwrap_or(b',');
326        actual_delimiter = Some(detected);
327
328        Box::new(std::io::Cursor::new(preamble).chain(take.into_inner()))
329    } else {
330        Box::new(reader)
331    };
332
333    if let Some(delimiter) = actual_delimiter {
334        csv_builder.delimiter(delimiter);
335    }
336
337    let mut csv_reader = csv_builder.from_reader(boxed_reader);
338
339    let header_names: Vec<String> = if config.has_header {
340        let headers = csv_reader.headers()?;
341        let names: Vec<String> = headers.iter().map(|header| header.to_string()).collect();
342        // Reject duplicate headers before profiling: process_record keys on name,
343        // so a repeat would merge two source columns into one inflated profile.
344        dataprof_core::validate_unique_column_names(&names, "CSV header")?;
345        names
346    } else {
347        let headers = csv_reader.headers()?;
348        (0..headers.len())
349            .map(|index| format!("column_{index}"))
350            .collect()
351    };
352
353    let mut column_stats = StreamingColumnCollection::new();
354    // A header declares the columns even when no data records follow it.
355    column_stats.init_columns(&header_names);
356    let mut rows_read = 0;
357    let mut ragged_rows = 0;
358    let mut truncation = None;
359
360    for result in csv_reader.records() {
361        if let Some(max_rows) = config.max_rows
362            && rows_read >= max_rows
363        {
364            // A record past the cap means the cap, not the end of the input,
365            // stopped the pass. That record is neither profiled nor judged.
366            truncation = Some(TruncationReason::MaxRows(max_rows as u64));
367            break;
368        }
369
370        let record = result?;
371        let mut values: Vec<String> = record.iter().map(|value| value.to_string()).collect();
372
373        let header_len = header_names.len();
374        if values.len() != header_len {
375            ragged_rows += 1;
376        }
377        if values.len() < header_len {
378            values.resize(header_len, String::new());
379        } else if values.len() > header_len {
380            values.truncate(header_len);
381        }
382
383        column_stats.process_record(&header_names, values);
384        rows_read += 1;
385    }
386
387    Ok(CsvScan {
388        column_stats,
389        header_names,
390        rows_read,
391        ragged_rows,
392        truncation,
393    })
394}
395
396/// Analyze a CSV file, returning a full [`ProfileReport`].
397///
398/// Opens the file, reads it as [`analyze_csv_from_reader`] does, and wraps the
399/// result with file metadata, ISO 8000/25012 data quality metrics, and the
400/// scan's own record: how many rows were ragged, and whether `max_rows` left
401/// rows unread.
402pub fn analyze_csv_file(
403    file_path: &Path,
404    config: &CsvParserConfig,
405) -> Result<ProfileReport, DataProfilerError> {
406    let metadata = std::fs::metadata(file_path).map_err(|error| map_io_error(file_path, error))?;
407    dataprof_core::binary_input::reject_binary_input(file_path)?;
408    let start = std::time::Instant::now();
409
410    let file = std::fs::File::open(file_path).map_err(|error| map_io_error(file_path, error))?;
411    let buf_reader = std::io::BufReader::new(file);
412
413    let scan = scan_csv(buf_reader, config)?;
414    let column_stats = &scan.column_stats;
415    let column_profiles =
416        profile_builder::profiles_from_streaming(column_stats, false, false, None);
417
418    let file_source = DataSource::File {
419        path: file_path.display().to_string(),
420        format: FileFormat::Csv,
421        size_bytes: metadata.len(),
422        modified_at: None,
423        parquet_metadata: None,
424    };
425
426    let sample_columns = profile_builder::quality_check_samples(column_stats);
427    let scan_time_ms = start.elapsed().as_millis();
428    let num_columns = column_profiles.len();
429
430    let mut execution = ExecutionMetadata::new(scan.rows_read, num_columns, scan_time_ms)
431        .with_engine("csv")
432        .with_ragged_row_count(scan.ragged_rows);
433    if let Some(reason) = scan.truncation {
434        execution = execution.with_truncation(reason);
435    }
436
437    // This path takes no analysis options: it profiles with the defaults,
438    // so it records them rather than leaving its settings unknown.
439    Ok(ReportAssembler::new(file_source, execution)
440        .with_analysis_options(&AnalysisOptions::default())
441        .columns(column_profiles)
442        .with_quality_data(sample_columns)
443        .with_row_duplicates(column_stats.row_duplicate_summary())
444        .with_row_completeness(column_stats.row_completeness_summary())
445        .build())
446}
447
448fn map_io_error(file_path: &Path, error: std::io::Error) -> DataProfilerError {
449    if error.kind() == std::io::ErrorKind::NotFound {
450        DataProfilerError::FileNotFound {
451            path: file_path.display().to_string(),
452        }
453    } else {
454        DataProfilerError::from(error)
455    }
456}
457
458#[cfg(test)]
459mod tests {
460    use super::*;
461    use dataprof_core::DataType;
462    use std::io::{Cursor, Write};
463    use tempfile::NamedTempFile;
464
465    fn write_csv(content: &str) -> NamedTempFile {
466        let mut file = NamedTempFile::new().unwrap();
467        write!(file, "{}", content).unwrap();
468        file.flush().unwrap();
469        file
470    }
471
472    #[test]
473    fn test_analyze_csv_from_reader_basic() {
474        let data = b"name,age\nAlice,25\nBob,30\n";
475        let cursor = Cursor::new(data.as_ref());
476        let config = CsvParserConfig::default();
477
478        let (profiles, _stats, rows, _) = analyze_csv_from_reader(cursor, &config).unwrap();
479        assert_eq!(rows, 2);
480        assert_eq!(profiles.len(), 2);
481
482        let age = profiles
483            .iter()
484            .find(|profile| profile.name == "age")
485            .unwrap();
486        assert_eq!(age.total_count, 2);
487        assert_eq!(age.data_type, DataType::Integer);
488    }
489
490    #[test]
491    fn test_analyze_csv_from_reader_with_max_rows() {
492        let data = b"val\n1\n2\n3\n4\n5\n";
493        let cursor = Cursor::new(data.as_ref());
494        let config = CsvParserConfig::default().max_rows(Some(3));
495
496        let (_profiles, _stats, rows, _) = analyze_csv_from_reader(cursor, &config).unwrap();
497        assert_eq!(rows, 3);
498    }
499
500    #[test]
501    fn test_analyze_csv_from_reader_strict_rejects_ragged() {
502        let data = b"a,b\n1,2\n3,4,5\n";
503        let cursor = Cursor::new(data.as_ref());
504        let config = CsvParserConfig::strict();
505
506        assert!(analyze_csv_from_reader(cursor, &config).is_err());
507    }
508
509    #[test]
510    fn test_analyze_csv_from_reader_flexible_handles_ragged() {
511        let data = b"a,b\n1,2\n3,4,5\n6\n";
512        let cursor = Cursor::new(data.as_ref());
513        let config = CsvParserConfig::default();
514
515        let (profiles, _stats, rows, _) = analyze_csv_from_reader(cursor, &config).unwrap();
516        assert_eq!(rows, 3);
517        assert!(!profiles.is_empty());
518    }
519
520    #[test]
521    fn test_analyze_csv_from_reader_custom_delimiter() {
522        let data = b"a;b\n1;2\n3;4\n";
523        let cursor = Cursor::new(data.as_ref());
524        let config = CsvParserConfig::default().with_delimiter(b';');
525
526        let (profiles, _stats, rows, _) = analyze_csv_from_reader(cursor, &config).unwrap();
527        assert_eq!(rows, 2);
528        assert_eq!(profiles.len(), 2);
529    }
530
531    #[test]
532    fn test_analyze_csv_file_returns_quality_report() {
533        let csv = write_csv("x,y\n1,a\n2,b\n3,c\n");
534        let config = CsvParserConfig::default();
535        let report = analyze_csv_file(csv.path(), &config).unwrap();
536
537        assert_eq!(report.column_profiles.len(), 2);
538        assert_eq!(report.execution.rows_processed, 3);
539        assert!(report.quality_score().unwrap() >= 0.0);
540        assert!(report.quality_score().unwrap() <= 100.0);
541    }
542
543    #[test]
544    fn test_csv_auto_detects_delimiter_from_reader() {
545        let data = b"name;age\nAlice;25\nBob;30\n";
546        let cursor = Cursor::new(data.as_ref());
547        let config = CsvParserConfig::default();
548
549        let (profiles, _stats, rows, _) = analyze_csv_from_reader(cursor, &config).unwrap();
550        assert_eq!(rows, 2);
551        assert_eq!(profiles.len(), 2);
552
553        let age = profiles
554            .iter()
555            .find(|profile| profile.name == "age")
556            .unwrap();
557        assert_eq!(age.total_count, 2);
558    }
559
560    #[test]
561    fn test_analyze_csv_basic() {
562        let csv = write_csv("name,age\nAlice,25\nBob,30\n");
563        let config = CsvParserConfig::default();
564        let report = analyze_csv_file(csv.path(), &config).unwrap();
565        let profiles = &report.column_profiles;
566        assert_eq!(profiles.len(), 2);
567
568        let names: Vec<&str> = profiles
569            .iter()
570            .map(|profile| profile.name.as_str())
571            .collect();
572        assert!(names.contains(&"name"));
573        assert!(names.contains(&"age"));
574
575        let age = profiles
576            .iter()
577            .find(|profile| profile.name == "age")
578            .unwrap();
579        assert_eq!(age.total_count, 2);
580        assert_eq!(age.null_count, 0);
581    }
582
583    #[test]
584    fn test_analyze_csv_with_nulls() {
585        let csv = write_csv("name,age,email\nAlice,25,a@b.com\nBob,,\nCharlie,30,c@d.com\n");
586        let config = CsvParserConfig::default();
587        let report = analyze_csv_file(csv.path(), &config).unwrap();
588        let profiles = &report.column_profiles;
589
590        let age = profiles
591            .iter()
592            .find(|profile| profile.name == "age")
593            .unwrap();
594        assert_eq!(age.total_count, 3);
595        assert_eq!(age.null_count, 1);
596
597        let email = profiles
598            .iter()
599            .find(|profile| profile.name == "email")
600            .unwrap();
601        assert_eq!(email.null_count, 1);
602    }
603
604    #[test]
605    fn test_analyze_csv_file_returns_quality_report_legacy() {
606        let csv = write_csv("x,y\n1,a\n2,b\n3,c\n");
607        let config = CsvParserConfig::default();
608        let report = analyze_csv_file(csv.path(), &config).unwrap();
609
610        assert_eq!(report.column_profiles.len(), 2);
611        assert_eq!(report.execution.rows_processed, 3);
612        assert_eq!(report.execution.columns_detected, 2);
613        assert!(report.quality_score().unwrap() >= 0.0);
614        assert!(report.quality_score().unwrap() <= 100.0);
615    }
616
617    #[test]
618    fn test_analyze_csv_file_header_only_preserves_declared_columns() {
619        let csv = write_csv("name,age\n");
620        let config = CsvParserConfig::default();
621        let report = analyze_csv_file(csv.path(), &config).unwrap();
622
623        assert_eq!(
624            report
625                .column_profiles
626                .iter()
627                .map(|column| (column.name.as_str(), column.data_type.clone()))
628                .collect::<Vec<_>>(),
629            vec![("name", DataType::String), ("age", DataType::String)]
630        );
631        assert_eq!(report.execution.rows_processed, 0);
632        assert!(report.quality.is_some());
633    }
634
635    #[test]
636    fn test_analyze_csv_file_empty_file_has_no_columns() {
637        let csv = write_csv("");
638        let config = CsvParserConfig::default();
639        let report = analyze_csv_file(csv.path(), &config).unwrap();
640
641        assert!(report.column_profiles.is_empty());
642        assert_eq!(report.execution.rows_processed, 0);
643        assert_eq!(report.execution.columns_detected, 0);
644        assert_eq!(
645            report.quality_status,
646            dataprof_runtime::QualityAnalysisStatus::Computed
647        );
648        assert!(report.quality.is_some());
649        assert!(report.quality_score().is_none());
650    }
651
652    #[test]
653    fn test_analyze_csv_with_max_rows_small_file() {
654        let csv = write_csv("val\n1\n2\n3\n4\n5\n");
655        let config = CsvParserConfig::default();
656        let report = analyze_csv_file(csv.path(), &config).unwrap();
657
658        assert_eq!(report.execution.rows_processed, 5);
659        assert_eq!(report.execution.columns_detected, 1);
660        assert!((report.execution.sampling_ratio.unwrap_or(1.0) - 1.0).abs() < 0.01);
661    }
662
663    #[test]
664    fn a_row_cap_that_leaves_rows_unread_is_a_truncation() {
665        let csv = write_csv("val\n1\n2\n3\n4\n5\n");
666        let config = CsvParserConfig::default().max_rows(Some(2));
667        let report = analyze_csv_file(csv.path(), &config).unwrap();
668
669        assert_eq!(report.execution.rows_processed, 2);
670        assert!(!report.execution.source_exhausted);
671        assert_eq!(
672            report.execution.truncation_reason,
673            Some(TruncationReason::MaxRows(2))
674        );
675    }
676
677    #[test]
678    fn a_row_cap_at_the_last_row_is_a_complete_scan() {
679        let csv = write_csv("val\n1\n2\n");
680        let config = CsvParserConfig::default().max_rows(Some(2));
681        let report = analyze_csv_file(csv.path(), &config).unwrap();
682
683        assert_eq!(report.execution.rows_processed, 2);
684        assert!(report.execution.source_exhausted);
685        assert_eq!(report.execution.truncation_reason, None);
686    }
687
688    #[test]
689    fn short_and_long_rows_are_counted_as_ragged() {
690        let csv = write_csv("a,b,c\n1,2,3\n4,5\n6,7,8,9\n10,11,12\n");
691        let report = analyze_csv_file(csv.path(), &CsvParserConfig::default()).unwrap();
692
693        assert_eq!(report.execution.rows_processed, 4);
694        assert_eq!(report.execution.ragged_row_count, 2);
695    }
696
697    #[test]
698    fn a_ragged_row_past_the_row_cap_is_not_counted() {
699        let csv = write_csv("a,b,c\n1,2,3\n4,5\n");
700        let config = CsvParserConfig::default().max_rows(Some(1));
701        let report = analyze_csv_file(csv.path(), &config).unwrap();
702
703        assert_eq!(report.execution.ragged_row_count, 0);
704        assert_eq!(
705            report.execution.truncation_reason,
706            Some(TruncationReason::MaxRows(1))
707        );
708    }
709
710    #[test]
711    fn test_analyze_csv_numeric_types_detected() {
712        let csv = write_csv("int_col,float_col,str_col\n1,1.5,hello\n2,2.5,world\n3,3.5,foo\n");
713        let config = CsvParserConfig::default();
714        let report = analyze_csv_file(csv.path(), &config).unwrap();
715        let profiles = &report.column_profiles;
716
717        let int_col = profiles
718            .iter()
719            .find(|profile| profile.name == "int_col")
720            .unwrap();
721        assert!(matches!(int_col.data_type, DataType::Integer));
722
723        let float_col = profiles
724            .iter()
725            .find(|profile| profile.name == "float_col")
726            .unwrap();
727        assert!(matches!(float_col.data_type, DataType::Float));
728
729        let str_col = profiles
730            .iter()
731            .find(|profile| profile.name == "str_col")
732            .unwrap();
733        assert!(matches!(str_col.data_type, DataType::String));
734    }
735
736    #[test]
737    fn test_analyze_csv_strict_rejects_ragged_rows() {
738        let csv = write_csv("a,b\n1,2\n3,4,5\n");
739        let config = CsvParserConfig::strict();
740        assert!(analyze_csv_file(csv.path(), &config).is_err());
741    }
742
743    #[test]
744    fn test_analyze_csv_file_handles_ragged_rows() {
745        let csv = write_csv("a,b\n1,2\n3,4,5\n6\n");
746        let config = CsvParserConfig::default();
747        let report = analyze_csv_file(csv.path(), &config).unwrap();
748        assert!(!report.column_profiles.is_empty());
749    }
750
751    #[test]
752    fn test_analyze_csv_with_quality_metrics() {
753        let csv = write_csv(
754            "name,age\n\
755             Alice,25\nBob,\nCharlie,30\nDave,\nEve,28\n\
756             Frank,\nGrace,35\nHeidi,40\nIvan,\nJudy,22\n",
757        );
758        let config = CsvParserConfig::default();
759        let report = analyze_csv_file(csv.path(), &config).unwrap();
760
761        let age = report
762            .column_profiles
763            .iter()
764            .find(|profile| profile.name == "age")
765            .unwrap();
766        assert_eq!(age.null_count, 4);
767        assert_eq!(age.total_count, 10);
768        assert!(report.quality_score().unwrap() >= 0.0);
769    }
770
771    #[test]
772    fn test_detect_delimiter_comma() {
773        let data = b"name,age,city\nAlice,25,NYC\nBob,30,London\n";
774        assert_eq!(detect_delimiter(Cursor::new(data.as_ref())).unwrap(), b',');
775    }
776
777    #[test]
778    fn test_detect_delimiter_semicolon() {
779        let data = b"name;age;city\nAlice;25;NYC\nBob;30;London\n";
780        assert_eq!(detect_delimiter(Cursor::new(data.as_ref())).unwrap(), b';');
781    }
782
783    #[test]
784    fn test_detect_delimiter_pipe() {
785        let data = b"name|age|city\nAlice|25|NYC\nBob|30|London\n";
786        assert_eq!(detect_delimiter(Cursor::new(data.as_ref())).unwrap(), b'|');
787    }
788
789    #[test]
790    fn test_detect_delimiter_tab() {
791        let data = b"name\tage\tcity\nAlice\t25\tNYC\nBob\t30\tLondon\n";
792        assert_eq!(detect_delimiter(Cursor::new(data.as_ref())).unwrap(), b'\t');
793    }
794
795    #[test]
796    fn test_detect_delimiter_from_path_semicolon() {
797        let csv = write_csv("id;name;salary\n1;Alice;50000\n2;Bob;60000\n");
798        assert_eq!(detect_delimiter_from_path(csv.path()).unwrap(), b';');
799    }
800
801    #[test]
802    fn test_detect_delimiter_blank_records_do_not_outvote_header() {
803        // A header-only file with trailing blank lines: the blank records are
804        // single-field for every candidate, so counting them made `;` win on
805        // perfect consistency and collapse the whole header into one column
806        // (gh #559).
807        let data = b"name,age\n\n\n";
808        assert_eq!(detect_delimiter(Cursor::new(data.as_ref())).unwrap(), b',');
809
810        let semicolon = b"name;age\n\n";
811        assert_eq!(
812            detect_delimiter(Cursor::new(semicolon.as_ref())).unwrap(),
813            b';'
814        );
815    }
816
817    #[test]
818    fn test_detect_delimiter_quoted_fields() {
819        let data = b"name;desc;val\n\"hello;world\";foo;1\nbar;baz;2\n";
820        assert_eq!(detect_delimiter(Cursor::new(data.as_ref())).unwrap(), b';');
821    }
822
823    #[test]
824    fn test_detect_delimiter_quoted_embedded_newline() {
825        // A quoted cell containing a newline must not break record splitting:
826        // before the quote-aware sniffer this file was detected as semicolon
827        // (zero occurrences, "perfectly consistent" single-field records) and
828        // profiled as a single column named "id,comment".
829        let data = b"id,comment\n1,\"line one\nline two\"\n2,\"normal\"\n3,\"has \"\"quotes\"\" inside\"\n";
830        assert_eq!(detect_delimiter(Cursor::new(data.as_ref())).unwrap(), b',');
831    }
832
833    #[test]
834    fn test_detect_delimiter_absent_candidate_never_beats_real_one() {
835        // One ragged record makes comma imperfect (consistency 3/4); an absent
836        // delimiter is uniformly single-field (4/4) and must still lose.
837        let data = b"a,b,c\n1,2,3\nragged line without delimiters\n4,5,6\n";
838        assert_eq!(detect_delimiter(Cursor::new(data.as_ref())).unwrap(), b',');
839    }
840
841    #[test]
842    fn test_detect_delimiter_single_column_falls_back_to_comma() {
843        let data = b"value\n1\n2\n3\n";
844        assert_eq!(detect_delimiter(Cursor::new(data.as_ref())).unwrap(), b',');
845    }
846
847    #[test]
848    fn test_detect_delimiter_two_column_semicolon_with_commas_in_text() {
849        // Free text containing commas must not outvote the true delimiter.
850        let data = b"id;note\n1;hello, world\n2;foo, bar, baz\n3;plain\n";
851        assert_eq!(detect_delimiter(Cursor::new(data.as_ref())).unwrap(), b';');
852    }
853
854    #[test]
855    fn test_detect_delimiter_never_splits_rows_the_header_does_not() {
856        // Pipe splits both data rows and not the header; the readers cut rows
857        // to the header's width, so choosing it kept only `123` (gh #1092).
858        let data = b"key\n123|456\n123|789\n";
859        assert_eq!(detect_delimiter(Cursor::new(data.as_ref())).unwrap(), b',');
860    }
861
862    #[test]
863    fn test_detect_delimiter_header_split_beats_uniform_rows() {
864        // A trailing delimiter on every data row: comma splits the header,
865        // which no other candidate does.
866        let data = b"a,b\n1,2,\n3,4,\n";
867        assert_eq!(detect_delimiter(Cursor::new(data.as_ref())).unwrap(), b',');
868    }
869
870    #[test]
871    fn test_detect_delimiter_trailing_empty_fields_still_agree() {
872        // Semicolon rows carry a trailing empty field, comma occurs inside the
873        // header name and a decimal. Counting the empty extras against
874        // semicolon let comma win and invent a null column (gh #1096).
875        let data = b"id;amount,EUR\n1;10,50;\n2;20;\n";
876        assert_eq!(detect_delimiter(Cursor::new(data.as_ref())).unwrap(), b';');
877    }
878
879    #[test]
880    fn test_detect_delimiter_quoted_empty_trailing_field_still_agrees() {
881        // `""` decodes to an empty field in every reader; judged on its raw
882        // text it counted as content and comma won again (gh #1102).
883        let data = b"id;amount,EUR\n1;10,50;\"\"\n2;20;\"\"\n";
884        assert_eq!(detect_delimiter(Cursor::new(data.as_ref())).unwrap(), b';');
885    }
886
887    #[test]
888    fn test_detect_delimiter_exact_width_beats_cut_trailing_fields() {
889        // Both candidates split the header and keep every value; semicolon
890        // reads each row as it stands, comma only after cutting an empty
891        // trailing field. Comma won the tie and renamed the columns.
892        let data = b"id,part;value\nx;y,z,\nu;v,w,\n";
893        assert_eq!(detect_delimiter(Cursor::new(data.as_ref())).unwrap(), b';');
894    }
895
896    #[test]
897    fn test_detect_delimiter_skips_a_utf8_bom() {
898        // A BOM then a blank line: the BOM alone was scored as a one-field
899        // header, and comma won over the real semicolon header.
900        let data = b"\xEF\xBB\xBF\nid;value\n1;2\n3;4\n";
901        assert_eq!(detect_delimiter(Cursor::new(data.as_ref())).unwrap(), b';');
902        let no_blank = b"\xEF\xBB\xBFid;value\n1;2\n";
903        assert_eq!(
904            detect_delimiter(Cursor::new(no_blank.as_ref())).unwrap(),
905            b';'
906        );
907    }
908
909    #[test]
910    fn test_detect_delimiter_lone_cr_ends_a_record() {
911        // CR-only line endings: read as one header record, pipe split it and
912        // won, and the keys were cut to `123` (gh #1097).
913        let data = b"key\r123|456\r123|789\r";
914        assert_eq!(detect_delimiter(Cursor::new(data.as_ref())).unwrap(), b',');
915        let semicolon = b"id;value\r1;10\r2;20\r";
916        assert_eq!(
917            detect_delimiter(Cursor::new(semicolon.as_ref())).unwrap(),
918            b';'
919        );
920    }
921
922    #[test]
923    fn test_count_fields_reports_trailing_empty_fields() {
924        assert_eq!(count_fields("1;10,50;", ';'), (3, 1));
925        assert_eq!(count_fields("a;;", ';'), (3, 2));
926        assert_eq!(count_fields("a;\"\"", ';'), (2, 1));
927        assert_eq!(count_fields("a;\"\"\"\"", ';'), (2, 0));
928        assert_eq!(count_fields("a; ", ';'), (2, 0));
929        assert_eq!(count_fields("a;b", ';'), (2, 0));
930        assert_eq!(count_fields("\"x;y\";", ';'), (2, 1));
931    }
932
933    #[test]
934    fn test_split_sample_records_quote_aware() {
935        let (records, last_unterminated) = split_sample_records("a,b\r\n1,\"x\ny\"\n2,z\n");
936        assert_eq!(records, vec!["a,b", "1,\"x\ny\"", "2,z"]);
937        assert!(!last_unterminated);
938
939        let (records, last_unterminated) = split_sample_records("a,b\n1,\"cut mid-fie");
940        assert_eq!(records, vec!["a,b", "1,\"cut mid-fie"]);
941        assert!(last_unterminated);
942
943        let (records, last_unterminated) = split_sample_records("a,b\r1,\"x\ry\"\r2,z\r");
944        assert_eq!(records, vec!["a,b", "1,\"x\ry\"", "2,z"]);
945        assert!(!last_unterminated);
946
947        let (records, _) = split_sample_records("a\rb\r\nc\n");
948        assert_eq!(records, vec!["a", "b", "c"]);
949    }
950
951    #[test]
952    fn test_detect_delimiter_full_sample_ending_on_record_boundary() {
953        // Exactly 4096 bytes ending with a clean newline: the final record is
954        // complete and must be scored, not dropped as a fragment.
955        let mut data = Vec::from(&b"a;b;c\n"[..]);
956        while data.len() < 4090 {
957            data.extend_from_slice(b"1;2;3\n");
958        }
959        while data.len() < 4095 {
960            data.push(b'#');
961        }
962        data.push(b'\n');
963        assert_eq!(data.len(), 4096);
964        assert_eq!(
965            detect_delimiter(Cursor::new(data.as_slice())).unwrap(),
966            b';'
967        );
968    }
969}