Skip to main content

dataprof_csv/
lib.rs

1use csv::ReaderBuilder;
2use std::io::Read;
3use std::path::Path;
4
5use dataprof_core::{
6    ColumnProfile, DataProfilerError, DataSource, ExecutionMetadata, FileFormat, QualityDimension,
7    SemanticHints,
8};
9use dataprof_runtime::{
10    ProfileReport, ReportAssembler, StreamingColumnCollection, profile_builder,
11};
12
13mod memmap_reader;
14mod robust_csv;
15
16pub use memmap_reader::MemoryMappedCsvReader;
17pub use robust_csv::{
18    CsvDiagnostics, EncodingDiagnostic, RobustCsvParser, RobustParseResult, diagnose_non_utf8,
19};
20
21/// Configuration for CSV parsing and analysis.
22///
23/// Replaces the 7 separate `analyze_csv_*` functions with a single config struct.
24/// Supports both file-based and reader-based (streaming) entry points.
25#[derive(Debug, Clone)]
26pub struct CsvParserConfig {
27    /// Allow ragged rows (flexible field counts).
28    pub flexible: bool,
29    /// Custom delimiter (None = comma).
30    pub delimiter: Option<u8>,
31    /// Quote character.
32    pub quote_char: u8,
33    /// Trim whitespace from fields.
34    pub trim_whitespace: bool,
35    /// Whether the first row contains column headers.
36    /// When `false`, columns are named `column_0`, `column_1`, etc.
37    pub has_header: bool,
38    /// Maximum rows to process (None = all rows).
39    pub max_rows: Option<usize>,
40}
41
42impl Default for CsvParserConfig {
43    fn default() -> Self {
44        Self {
45            flexible: true,
46            delimiter: None,
47            quote_char: b'"',
48            trim_whitespace: false,
49            has_header: true,
50            max_rows: None,
51        }
52    }
53}
54
55impl CsvParserConfig {
56    /// Create a strict config that rejects ragged rows.
57    pub fn strict() -> Self {
58        Self {
59            flexible: false,
60            ..Default::default()
61        }
62    }
63
64    /// Set the delimiter.
65    pub fn with_delimiter(mut self, delimiter: u8) -> Self {
66        self.delimiter = Some(delimiter);
67        self
68    }
69
70    /// Set whether the first row contains column headers.
71    pub fn has_header(mut self, yes: bool) -> Self {
72        self.has_header = yes;
73        self
74    }
75
76    /// Set maximum number of rows to evaluate.
77    pub fn max_rows(mut self, max: Option<usize>) -> Self {
78        self.max_rows = max;
79        self
80    }
81}
82
83/// Count fields in a line for a given delimiter, respecting quoted sections.
84pub(crate) fn count_fields(line: &str, delimiter: char) -> usize {
85    let mut count = 1;
86    let mut in_quotes = false;
87    let mut chars = line.chars().peekable();
88    while let Some(ch) = chars.next() {
89        if ch == '"' {
90            if chars.peek() == Some(&'"') {
91                chars.next();
92            } else {
93                in_quotes = !in_quotes;
94            }
95        } else if ch == delimiter && !in_quotes {
96            count += 1;
97        }
98    }
99    count
100}
101
102/// Split raw CSV sample text into logical records, honoring quoted sections
103/// so an embedded newline inside quotes does not start a new record.
104///
105/// The returned flag is true when the final record was not terminated by a
106/// newline outside quotes — i.e. the text ends mid-record (or exactly at a
107/// record's end with no trailing newline, which a caller sampling a fixed
108/// byte budget cannot distinguish from a cut).
109fn split_sample_records(text: &str) -> (Vec<&str>, bool) {
110    let bytes = text.as_bytes();
111    let mut records = Vec::new();
112    let mut in_quotes = false;
113    let mut start = 0usize;
114    let mut i = 0usize;
115    while i < bytes.len() {
116        match bytes[i] {
117            b'"' => {
118                if in_quotes && bytes.get(i + 1) == Some(&b'"') {
119                    i += 1; // escaped quote inside a quoted section
120                } else {
121                    in_quotes = !in_quotes;
122                }
123            }
124            b'\n' if !in_quotes => {
125                let end = if i > start && bytes[i - 1] == b'\r' {
126                    i - 1
127                } else {
128                    i
129                };
130                records.push(&text[start..end]);
131                start = i + 1;
132            }
133            _ => {}
134        }
135        i += 1;
136    }
137    let last_unterminated = start < text.len();
138    if last_unterminated {
139        records.push(text[start..].trim_end_matches('\r'));
140    }
141    (records, last_unterminated)
142}
143
144/// Detect the most likely CSV delimiter by sampling up to 4 KB of data.
145///
146/// The sample is split into logical records with quote handling, so quoted
147/// fields containing embedded newlines or delimiter characters do not skew
148/// detection. Tests comma, semicolon, tab, and pipe over the first 5 non-blank
149/// records: a delimiter whose modal field count is greater than one always
150/// beats one that only ever yields single-field records, then higher record
151/// agreement and higher field counts win.
152///
153/// Falls back to comma if no candidate splits the sample.
154pub fn detect_delimiter<R: Read>(reader: R) -> std::io::Result<u8> {
155    const SAMPLE_BYTES: u64 = 4096;
156    let mut preamble = Vec::new();
157    let mut take = reader.take(SAMPLE_BYTES);
158    take.read_to_end(&mut preamble)?;
159    let truncated = preamble.len() as u64 == SAMPLE_BYTES;
160
161    let preamble_str = String::from_utf8_lossy(&preamble);
162    let (mut records, last_unterminated) = split_sample_records(&preamble_str);
163    // A full sample buffer that ends mid-record leaves a fragment; never score
164    // it. A sample ending on a clean newline keeps all its records.
165    if truncated && last_unterminated && records.len() > 1 {
166        records.pop();
167    }
168
169    let delimiters = *b",;\t|";
170    let mut best_delimiter = b',';
171    // (splits records into >1 field, records agreeing on the modal count,
172    //  modal field count) — compared lexicographically, first wins ties.
173    let mut best_score = (false, 0usize, 0usize);
174
175    for &delimiter in &delimiters {
176        let mut counts = std::collections::HashMap::new();
177        // Blank records carry no delimiter signal: a header followed by
178        // trailing blank lines must not outvote the true delimiter with
179        // single-field "records" (gh #559).
180        let scored = records
181            .iter()
182            .filter(|record| !record.trim().is_empty())
183            .take(5);
184        for record in scored {
185            let fields = count_fields(record, delimiter as char);
186            *counts.entry(fields).or_insert(0usize) += 1;
187        }
188
189        // decode-audit: no-data — an empty sample scores 0 for this delimiter
190        // candidate; nothing is fabricated.
191        let (modal_fields, consistency) = counts
192            .into_iter()
193            .max_by_key(|&(fields, count)| (count, fields))
194            .unwrap_or((0, 0));
195
196        let score = (modal_fields > 1, consistency, modal_fields);
197        if score > best_score {
198            best_score = score;
199            best_delimiter = delimiter;
200        }
201    }
202
203    Ok(best_delimiter)
204}
205
206/// Detect the most likely CSV delimiter from a file path.
207pub fn detect_delimiter_from_path(path: &Path) -> std::io::Result<u8> {
208    let file = std::fs::File::open(path)?;
209    detect_delimiter(std::io::BufReader::new(file))
210}
211
212/// Analyze CSV data from any `Read` source using streaming statistics.
213///
214/// This is the primary generic entry point for both file-based and stream-based
215/// CSV profiling. Data is processed incrementally through [`StreamingColumnCollection`]
216/// with bounded memory, rather than accumulating all rows in a `HashMap`.
217///
218/// Returns `(column_profiles, streaming_stats, rows_read)`.
219pub fn analyze_csv_from_reader<R: Read>(
220    reader: R,
221    config: &CsvParserConfig,
222) -> Result<
223    (
224        Vec<ColumnProfile>,
225        StreamingColumnCollection,
226        usize,
227        Vec<String>,
228    ),
229    DataProfilerError,
230> {
231    analyze_csv_from_reader_with_hints(reader, config, &SemanticHints::default())
232}
233
234/// Analyze CSV data from any `Read` source while applying semantic hints.
235pub fn analyze_csv_from_reader_with_hints<R: Read>(
236    reader: R,
237    config: &CsvParserConfig,
238    semantic_hints: &SemanticHints,
239) -> Result<
240    (
241        Vec<ColumnProfile>,
242        StreamingColumnCollection,
243        usize,
244        Vec<String>,
245    ),
246    DataProfilerError,
247> {
248    let mut csv_builder = ReaderBuilder::new();
249    csv_builder.has_headers(config.has_header);
250    csv_builder.flexible(config.flexible);
251    csv_builder.quote(config.quote_char);
252    if config.trim_whitespace {
253        csv_builder.trim(csv::Trim::All);
254    }
255    let mut actual_delimiter = config.delimiter;
256
257    let boxed_reader: Box<dyn std::io::Read + '_> = if config.delimiter.is_none() {
258        let mut preamble = Vec::new();
259        let mut take = reader.take(4096);
260        take.read_to_end(&mut preamble)?;
261
262        let detected = detect_delimiter(std::io::Cursor::new(&preamble)).unwrap_or(b',');
263        actual_delimiter = Some(detected);
264
265        Box::new(std::io::Cursor::new(preamble).chain(take.into_inner()))
266    } else {
267        Box::new(reader)
268    };
269
270    if let Some(delimiter) = actual_delimiter {
271        csv_builder.delimiter(delimiter);
272    }
273
274    let mut csv_reader = csv_builder.from_reader(boxed_reader);
275
276    let header_names: Vec<String> = if config.has_header {
277        let headers = csv_reader.headers()?;
278        let names: Vec<String> = headers.iter().map(|header| header.to_string()).collect();
279        // Reject duplicate headers before profiling: process_record keys on name,
280        // so a repeat would merge two source columns into one inflated profile.
281        dataprof_core::validate_unique_column_names(&names, "CSV header")?;
282        names
283    } else {
284        let headers = csv_reader.headers()?;
285        (0..headers.len())
286            .map(|index| format!("column_{index}"))
287            .collect()
288    };
289
290    let mut column_stats = StreamingColumnCollection::new().with_semantic_hints(semantic_hints);
291    // A header declares the columns even when no data records follow it.
292    column_stats.init_columns(&header_names);
293    let mut rows_read = 0;
294
295    for result in csv_reader.records() {
296        if let Some(max_rows) = config.max_rows
297            && rows_read >= max_rows
298        {
299            break;
300        }
301
302        let record = result?;
303        let mut values: Vec<String> = record.iter().map(|value| value.to_string()).collect();
304
305        let header_len = header_names.len();
306        if values.len() < header_len {
307            values.resize(header_len, String::new());
308        } else if values.len() > header_len {
309            values.truncate(header_len);
310        }
311
312        column_stats.process_record(&header_names, values);
313        rows_read += 1;
314    }
315
316    let profiles = profile_builder::profiles_from_streaming_with_hints(
317        &column_stats,
318        false,
319        false,
320        None,
321        semantic_hints,
322    );
323
324    Ok((profiles, column_stats, rows_read, header_names))
325}
326
327/// Analyze a CSV file, returning a full [`ProfileReport`].
328///
329/// Opens the file, delegates to [`analyze_csv_from_reader`], and wraps the
330/// result with file metadata and ISO 8000/25012 data quality metrics.
331pub fn analyze_csv_file(
332    file_path: &Path,
333    config: &CsvParserConfig,
334) -> Result<ProfileReport, DataProfilerError> {
335    analyze_csv_file_with_dimensions(file_path, config, None)
336}
337
338/// Like [`analyze_csv_file`] but only computes the requested quality dimensions.
339pub fn analyze_csv_file_with_dimensions(
340    file_path: &Path,
341    config: &CsvParserConfig,
342    quality_dimensions: Option<&[QualityDimension]>,
343) -> Result<ProfileReport, DataProfilerError> {
344    analyze_csv_file_with_dimensions_and_hints(
345        file_path,
346        config,
347        quality_dimensions,
348        &SemanticHints::default(),
349    )
350}
351
352pub fn analyze_csv_file_with_dimensions_and_hints(
353    file_path: &Path,
354    config: &CsvParserConfig,
355    quality_dimensions: Option<&[QualityDimension]>,
356    semantic_hints: &SemanticHints,
357) -> Result<ProfileReport, DataProfilerError> {
358    let metadata = std::fs::metadata(file_path).map_err(|error| map_io_error(file_path, error))?;
359    let start = std::time::Instant::now();
360
361    let file = std::fs::File::open(file_path).map_err(|error| map_io_error(file_path, error))?;
362    let buf_reader = std::io::BufReader::new(file);
363
364    let (column_profiles, column_stats, rows_read, _header_names) =
365        analyze_csv_from_reader_with_hints(buf_reader, config, semantic_hints)?;
366
367    let file_source = DataSource::File {
368        path: file_path.display().to_string(),
369        format: FileFormat::Csv,
370        size_bytes: metadata.len(),
371        modified_at: None,
372        parquet_metadata: None,
373    };
374
375    // Every header name is pre-registered above, so no profiles means no header
376    // declared a column: the source is empty, not merely rowless.
377    if column_profiles.is_empty() && rows_read == 0 {
378        return Ok(ReportAssembler::new(
379            file_source,
380            ExecutionMetadata::new(0, 0, start.elapsed().as_millis()).with_engine("csv"),
381        )
382        .skip_quality()
383        .build());
384    }
385
386    let sample_columns = profile_builder::quality_check_samples(&column_stats);
387    let scan_time_ms = start.elapsed().as_millis();
388    let num_columns = column_profiles.len();
389
390    let mut assembler = ReportAssembler::new(
391        file_source,
392        ExecutionMetadata::new(rows_read, num_columns, scan_time_ms).with_engine("csv"),
393    )
394    .columns(column_profiles)
395    .with_quality_data(sample_columns)
396    .with_row_duplicates(column_stats.row_duplicate_summary())
397    .with_row_completeness(column_stats.row_completeness_summary())
398    .with_exact_value_hint_bindings(column_stats.semantic_hint_bindings())
399    .with_semantic_hints(semantic_hints.clone());
400    if let Some(dimensions) = quality_dimensions {
401        assembler = assembler.with_requested_dimensions(dimensions.to_vec());
402    }
403    Ok(assembler.build())
404}
405
406fn map_io_error(file_path: &Path, error: std::io::Error) -> DataProfilerError {
407    if error.kind() == std::io::ErrorKind::NotFound {
408        DataProfilerError::FileNotFound {
409            path: file_path.display().to_string(),
410        }
411    } else {
412        DataProfilerError::from(error)
413    }
414}
415
416#[cfg(test)]
417mod tests {
418    use super::*;
419    use dataprof_core::DataType;
420    use std::io::{Cursor, Write};
421    use tempfile::NamedTempFile;
422
423    fn write_csv(content: &str) -> NamedTempFile {
424        let mut file = NamedTempFile::new().unwrap();
425        write!(file, "{}", content).unwrap();
426        file.flush().unwrap();
427        file
428    }
429
430    #[test]
431    fn test_analyze_csv_from_reader_basic() {
432        let data = b"name,age\nAlice,25\nBob,30\n";
433        let cursor = Cursor::new(data.as_ref());
434        let config = CsvParserConfig::default();
435
436        let (profiles, _stats, rows, _) = analyze_csv_from_reader(cursor, &config).unwrap();
437        assert_eq!(rows, 2);
438        assert_eq!(profiles.len(), 2);
439
440        let age = profiles
441            .iter()
442            .find(|profile| profile.name == "age")
443            .unwrap();
444        assert_eq!(age.total_count, 2);
445        assert_eq!(age.data_type, DataType::Integer);
446    }
447
448    #[test]
449    fn test_analyze_csv_from_reader_with_max_rows() {
450        let data = b"val\n1\n2\n3\n4\n5\n";
451        let cursor = Cursor::new(data.as_ref());
452        let config = CsvParserConfig::default().max_rows(Some(3));
453
454        let (_profiles, _stats, rows, _) = analyze_csv_from_reader(cursor, &config).unwrap();
455        assert_eq!(rows, 3);
456    }
457
458    #[test]
459    fn test_analyze_csv_from_reader_strict_rejects_ragged() {
460        let data = b"a,b\n1,2\n3,4,5\n";
461        let cursor = Cursor::new(data.as_ref());
462        let config = CsvParserConfig::strict();
463
464        assert!(analyze_csv_from_reader(cursor, &config).is_err());
465    }
466
467    #[test]
468    fn test_analyze_csv_from_reader_flexible_handles_ragged() {
469        let data = b"a,b\n1,2\n3,4,5\n6\n";
470        let cursor = Cursor::new(data.as_ref());
471        let config = CsvParserConfig::default();
472
473        let (profiles, _stats, rows, _) = analyze_csv_from_reader(cursor, &config).unwrap();
474        assert_eq!(rows, 3);
475        assert!(!profiles.is_empty());
476    }
477
478    #[test]
479    fn test_analyze_csv_from_reader_custom_delimiter() {
480        let data = b"a;b\n1;2\n3;4\n";
481        let cursor = Cursor::new(data.as_ref());
482        let config = CsvParserConfig::default().with_delimiter(b';');
483
484        let (profiles, _stats, rows, _) = analyze_csv_from_reader(cursor, &config).unwrap();
485        assert_eq!(rows, 2);
486        assert_eq!(profiles.len(), 2);
487    }
488
489    #[test]
490    fn test_analyze_csv_file_returns_quality_report() {
491        let csv = write_csv("x,y\n1,a\n2,b\n3,c\n");
492        let config = CsvParserConfig::default();
493        let report = analyze_csv_file(csv.path(), &config).unwrap();
494
495        assert_eq!(report.column_profiles.len(), 2);
496        assert_eq!(report.execution.rows_processed, 3);
497        assert!(report.quality_score().unwrap() >= 0.0);
498        assert!(report.quality_score().unwrap() <= 100.0);
499    }
500
501    #[test]
502    fn test_csv_auto_detects_delimiter_from_reader() {
503        let data = b"name;age\nAlice;25\nBob;30\n";
504        let cursor = Cursor::new(data.as_ref());
505        let config = CsvParserConfig::default();
506
507        let (profiles, _stats, rows, _) = analyze_csv_from_reader(cursor, &config).unwrap();
508        assert_eq!(rows, 2);
509        assert_eq!(profiles.len(), 2);
510
511        let age = profiles
512            .iter()
513            .find(|profile| profile.name == "age")
514            .unwrap();
515        assert_eq!(age.total_count, 2);
516    }
517
518    #[test]
519    fn test_analyze_csv_basic() {
520        let csv = write_csv("name,age\nAlice,25\nBob,30\n");
521        let config = CsvParserConfig::default();
522        let report = analyze_csv_file(csv.path(), &config).unwrap();
523        let profiles = &report.column_profiles;
524        assert_eq!(profiles.len(), 2);
525
526        let names: Vec<&str> = profiles
527            .iter()
528            .map(|profile| profile.name.as_str())
529            .collect();
530        assert!(names.contains(&"name"));
531        assert!(names.contains(&"age"));
532
533        let age = profiles
534            .iter()
535            .find(|profile| profile.name == "age")
536            .unwrap();
537        assert_eq!(age.total_count, 2);
538        assert_eq!(age.null_count, 0);
539    }
540
541    #[test]
542    fn test_analyze_csv_with_nulls() {
543        let csv = write_csv("name,age,email\nAlice,25,a@b.com\nBob,,\nCharlie,30,c@d.com\n");
544        let config = CsvParserConfig::default();
545        let report = analyze_csv_file(csv.path(), &config).unwrap();
546        let profiles = &report.column_profiles;
547
548        let age = profiles
549            .iter()
550            .find(|profile| profile.name == "age")
551            .unwrap();
552        assert_eq!(age.total_count, 3);
553        assert_eq!(age.null_count, 1);
554
555        let email = profiles
556            .iter()
557            .find(|profile| profile.name == "email")
558            .unwrap();
559        assert_eq!(email.null_count, 1);
560    }
561
562    #[test]
563    fn test_analyze_csv_file_returns_quality_report_legacy() {
564        let csv = write_csv("x,y\n1,a\n2,b\n3,c\n");
565        let config = CsvParserConfig::default();
566        let report = analyze_csv_file(csv.path(), &config).unwrap();
567
568        assert_eq!(report.column_profiles.len(), 2);
569        assert_eq!(report.execution.rows_processed, 3);
570        assert_eq!(report.execution.columns_detected, 2);
571        assert!(report.quality_score().unwrap() >= 0.0);
572        assert!(report.quality_score().unwrap() <= 100.0);
573    }
574
575    #[test]
576    fn test_analyze_csv_file_header_only_preserves_declared_columns() {
577        let csv = write_csv("name,age\n");
578        let config = CsvParserConfig::default();
579        let report = analyze_csv_file(csv.path(), &config).unwrap();
580
581        assert_eq!(
582            report
583                .column_profiles
584                .iter()
585                .map(|column| (column.name.as_str(), column.data_type.clone()))
586                .collect::<Vec<_>>(),
587            vec![("name", DataType::String), ("age", DataType::String)]
588        );
589        assert_eq!(report.execution.rows_processed, 0);
590        assert!(report.quality.is_some());
591    }
592
593    #[test]
594    fn test_analyze_csv_file_empty_file_has_no_columns() {
595        let csv = write_csv("");
596        let config = CsvParserConfig::default();
597        let report = analyze_csv_file(csv.path(), &config).unwrap();
598
599        assert!(report.column_profiles.is_empty());
600        assert_eq!(report.execution.rows_processed, 0);
601        assert_eq!(report.execution.columns_detected, 0);
602        // Records this path's behaviour, not a settled contract: the engines
603        // answer the same empty file with a quality block scoring 0.0/Exact
604        // over nothing at all. See #573.
605        assert!(report.quality.is_none());
606    }
607
608    #[test]
609    fn test_analyze_csv_with_max_rows_small_file() {
610        let csv = write_csv("val\n1\n2\n3\n4\n5\n");
611        let config = CsvParserConfig::default();
612        let report = analyze_csv_file(csv.path(), &config).unwrap();
613
614        assert_eq!(report.execution.rows_processed, 5);
615        assert_eq!(report.execution.columns_detected, 1);
616        assert!((report.execution.sampling_ratio.unwrap_or(1.0) - 1.0).abs() < 0.01);
617    }
618
619    #[test]
620    fn test_analyze_csv_numeric_types_detected() {
621        let csv = write_csv("int_col,float_col,str_col\n1,1.5,hello\n2,2.5,world\n3,3.5,foo\n");
622        let config = CsvParserConfig::default();
623        let report = analyze_csv_file(csv.path(), &config).unwrap();
624        let profiles = &report.column_profiles;
625
626        let int_col = profiles
627            .iter()
628            .find(|profile| profile.name == "int_col")
629            .unwrap();
630        assert!(matches!(int_col.data_type, DataType::Integer));
631
632        let float_col = profiles
633            .iter()
634            .find(|profile| profile.name == "float_col")
635            .unwrap();
636        assert!(matches!(float_col.data_type, DataType::Float));
637
638        let str_col = profiles
639            .iter()
640            .find(|profile| profile.name == "str_col")
641            .unwrap();
642        assert!(matches!(str_col.data_type, DataType::String));
643    }
644
645    #[test]
646    fn test_analyze_csv_strict_rejects_ragged_rows() {
647        let csv = write_csv("a,b\n1,2\n3,4,5\n");
648        let config = CsvParserConfig::strict();
649        assert!(analyze_csv_file(csv.path(), &config).is_err());
650    }
651
652    #[test]
653    fn test_analyze_csv_file_handles_ragged_rows() {
654        let csv = write_csv("a,b\n1,2\n3,4,5\n6\n");
655        let config = CsvParserConfig::default();
656        let report = analyze_csv_file(csv.path(), &config).unwrap();
657        assert!(!report.column_profiles.is_empty());
658    }
659
660    #[test]
661    fn test_analyze_csv_with_quality_metrics() {
662        let csv = write_csv(
663            "name,age\n\
664             Alice,25\nBob,\nCharlie,30\nDave,\nEve,28\n\
665             Frank,\nGrace,35\nHeidi,40\nIvan,\nJudy,22\n",
666        );
667        let config = CsvParserConfig::default();
668        let report = analyze_csv_file(csv.path(), &config).unwrap();
669
670        let age = report
671            .column_profiles
672            .iter()
673            .find(|profile| profile.name == "age")
674            .unwrap();
675        assert_eq!(age.null_count, 4);
676        assert_eq!(age.total_count, 10);
677        assert!(report.quality_score().unwrap() >= 0.0);
678    }
679
680    #[test]
681    fn test_detect_delimiter_comma() {
682        let data = b"name,age,city\nAlice,25,NYC\nBob,30,London\n";
683        assert_eq!(detect_delimiter(Cursor::new(data.as_ref())).unwrap(), b',');
684    }
685
686    #[test]
687    fn test_detect_delimiter_semicolon() {
688        let data = b"name;age;city\nAlice;25;NYC\nBob;30;London\n";
689        assert_eq!(detect_delimiter(Cursor::new(data.as_ref())).unwrap(), b';');
690    }
691
692    #[test]
693    fn test_detect_delimiter_pipe() {
694        let data = b"name|age|city\nAlice|25|NYC\nBob|30|London\n";
695        assert_eq!(detect_delimiter(Cursor::new(data.as_ref())).unwrap(), b'|');
696    }
697
698    #[test]
699    fn test_detect_delimiter_tab() {
700        let data = b"name\tage\tcity\nAlice\t25\tNYC\nBob\t30\tLondon\n";
701        assert_eq!(detect_delimiter(Cursor::new(data.as_ref())).unwrap(), b'\t');
702    }
703
704    #[test]
705    fn test_detect_delimiter_from_path_semicolon() {
706        let csv = write_csv("id;name;salary\n1;Alice;50000\n2;Bob;60000\n");
707        assert_eq!(detect_delimiter_from_path(csv.path()).unwrap(), b';');
708    }
709
710    #[test]
711    fn test_detect_delimiter_blank_records_do_not_outvote_header() {
712        // A header-only file with trailing blank lines: the blank records are
713        // single-field for every candidate, so counting them made `;` win on
714        // perfect consistency and collapse the whole header into one column
715        // (gh #559).
716        let data = b"name,age\n\n\n";
717        assert_eq!(detect_delimiter(Cursor::new(data.as_ref())).unwrap(), b',');
718
719        let semicolon = b"name;age\n\n";
720        assert_eq!(
721            detect_delimiter(Cursor::new(semicolon.as_ref())).unwrap(),
722            b';'
723        );
724    }
725
726    #[test]
727    fn test_detect_delimiter_quoted_fields() {
728        let data = b"name;desc;val\n\"hello;world\";foo;1\nbar;baz;2\n";
729        assert_eq!(detect_delimiter(Cursor::new(data.as_ref())).unwrap(), b';');
730    }
731
732    #[test]
733    fn test_detect_delimiter_quoted_embedded_newline() {
734        // A quoted cell containing a newline must not break record splitting:
735        // before the quote-aware sniffer this file was detected as semicolon
736        // (zero occurrences, "perfectly consistent" single-field records) and
737        // profiled as a single column named "id,comment".
738        let data = b"id,comment\n1,\"line one\nline two\"\n2,\"normal\"\n3,\"has \"\"quotes\"\" inside\"\n";
739        assert_eq!(detect_delimiter(Cursor::new(data.as_ref())).unwrap(), b',');
740    }
741
742    #[test]
743    fn test_detect_delimiter_absent_candidate_never_beats_real_one() {
744        // One ragged record makes comma imperfect (consistency 3/4); an absent
745        // delimiter is uniformly single-field (4/4) and must still lose.
746        let data = b"a,b,c\n1,2,3\nragged line without delimiters\n4,5,6\n";
747        assert_eq!(detect_delimiter(Cursor::new(data.as_ref())).unwrap(), b',');
748    }
749
750    #[test]
751    fn test_detect_delimiter_single_column_falls_back_to_comma() {
752        let data = b"value\n1\n2\n3\n";
753        assert_eq!(detect_delimiter(Cursor::new(data.as_ref())).unwrap(), b',');
754    }
755
756    #[test]
757    fn test_detect_delimiter_two_column_semicolon_with_commas_in_text() {
758        // Free text containing commas must not outvote the true delimiter.
759        let data = b"id;note\n1;hello, world\n2;foo, bar, baz\n3;plain\n";
760        assert_eq!(detect_delimiter(Cursor::new(data.as_ref())).unwrap(), b';');
761    }
762
763    #[test]
764    fn test_split_sample_records_quote_aware() {
765        let (records, last_unterminated) = split_sample_records("a,b\r\n1,\"x\ny\"\n2,z\n");
766        assert_eq!(records, vec!["a,b", "1,\"x\ny\"", "2,z"]);
767        assert!(!last_unterminated);
768
769        let (records, last_unterminated) = split_sample_records("a,b\n1,\"cut mid-fie");
770        assert_eq!(records, vec!["a,b", "1,\"cut mid-fie"]);
771        assert!(last_unterminated);
772    }
773
774    #[test]
775    fn test_detect_delimiter_full_sample_ending_on_record_boundary() {
776        // Exactly 4096 bytes ending with a clean newline: the final record is
777        // complete and must be scored, not dropped as a fragment.
778        let mut data = Vec::from(&b"a;b;c\n"[..]);
779        while data.len() < 4090 {
780            data.extend_from_slice(b"1;2;3\n");
781        }
782        while data.len() < 4095 {
783            data.push(b'#');
784        }
785        data.push(b'\n');
786        assert_eq!(data.len(), 4096);
787        assert_eq!(
788            detect_delimiter(Cursor::new(data.as_slice())).unwrap(),
789            b';'
790        );
791    }
792}