Skip to main content

dataprof_csv/
lib.rs

1use csv::ReaderBuilder;
2use std::io::Read;
3use std::path::Path;
4
5use dataprof_core::{
6    ColumnProfile, DataProfilerError, DataSource, ExecutionMetadata, FileFormat, QualityDimension,
7    SemanticHints,
8};
9use dataprof_runtime::{
10    ProfileReport, ReportAssembler, StreamingColumnCollection, profile_builder,
11};
12
13mod memmap_reader;
14mod quote_state;
15mod robust_csv;
16
17pub use memmap_reader::MemoryMappedCsvReader;
18pub use quote_state::{QuoteOutcome, QuoteTrackingReader, unterminated_quote_error};
19pub use robust_csv::{
20    CsvDiagnostics, CsvParseOutput, EncodingDiagnostic, RobustCsvParser, RobustParseResult,
21    diagnose_non_utf8,
22};
23
24/// Configuration for CSV parsing and analysis.
25///
26/// Replaces the 7 separate `analyze_csv_*` functions with a single config struct.
27/// Supports both file-based and reader-based (streaming) entry points.
28#[derive(Debug, Clone)]
29pub struct CsvParserConfig {
30    /// Allow ragged rows (flexible field counts).
31    pub flexible: bool,
32    /// Custom delimiter (None = comma).
33    pub delimiter: Option<u8>,
34    /// Quote character.
35    pub quote_char: u8,
36    /// Trim whitespace from fields.
37    pub trim_whitespace: bool,
38    /// Whether the first row contains column headers.
39    /// When `false`, columns are named `column_0`, `column_1`, etc.
40    pub has_header: bool,
41    /// Maximum rows to process (None = all rows).
42    pub max_rows: Option<usize>,
43}
44
45impl Default for CsvParserConfig {
46    fn default() -> Self {
47        Self {
48            flexible: true,
49            delimiter: None,
50            quote_char: b'"',
51            trim_whitespace: false,
52            has_header: true,
53            max_rows: None,
54        }
55    }
56}
57
58impl CsvParserConfig {
59    /// Create a strict config that rejects ragged rows.
60    pub fn strict() -> Self {
61        Self {
62            flexible: false,
63            ..Default::default()
64        }
65    }
66
67    /// Set the delimiter.
68    pub fn with_delimiter(mut self, delimiter: u8) -> Self {
69        self.delimiter = Some(delimiter);
70        self
71    }
72
73    /// Set whether the first row contains column headers.
74    pub fn has_header(mut self, yes: bool) -> Self {
75        self.has_header = yes;
76        self
77    }
78
79    /// Set maximum number of rows to evaluate.
80    pub fn max_rows(mut self, max: Option<usize>) -> Self {
81        self.max_rows = max;
82        self
83    }
84}
85
86/// Count fields in a line for a given delimiter, respecting quoted sections.
87pub(crate) fn count_fields(line: &str, delimiter: char) -> usize {
88    let mut count = 1;
89    let mut in_quotes = false;
90    let mut chars = line.chars().peekable();
91    while let Some(ch) = chars.next() {
92        if ch == '"' {
93            if chars.peek() == Some(&'"') {
94                chars.next();
95            } else {
96                in_quotes = !in_quotes;
97            }
98        } else if ch == delimiter && !in_quotes {
99            count += 1;
100        }
101    }
102    count
103}
104
105/// Split raw CSV sample text into logical records, honoring quoted sections
106/// so an embedded newline inside quotes does not start a new record.
107///
108/// The returned flag is true when the final record was not terminated by a
109/// newline outside quotes — i.e. the text ends mid-record (or exactly at a
110/// record's end with no trailing newline, which a caller sampling a fixed
111/// byte budget cannot distinguish from a cut).
112fn split_sample_records(text: &str) -> (Vec<&str>, bool) {
113    let bytes = text.as_bytes();
114    let mut records = Vec::new();
115    let mut in_quotes = false;
116    let mut start = 0usize;
117    let mut i = 0usize;
118    while i < bytes.len() {
119        match bytes[i] {
120            b'"' => {
121                if in_quotes && bytes.get(i + 1) == Some(&b'"') {
122                    i += 1; // escaped quote inside a quoted section
123                } else {
124                    in_quotes = !in_quotes;
125                }
126            }
127            b'\n' if !in_quotes => {
128                let end = if i > start && bytes[i - 1] == b'\r' {
129                    i - 1
130                } else {
131                    i
132                };
133                records.push(&text[start..end]);
134                start = i + 1;
135            }
136            _ => {}
137        }
138        i += 1;
139    }
140    let last_unterminated = start < text.len();
141    if last_unterminated {
142        records.push(text[start..].trim_end_matches('\r'));
143    }
144    (records, last_unterminated)
145}
146
147/// Detect the most likely CSV delimiter by sampling up to 4 KB of data.
148///
149/// The sample is split into logical records with quote handling, so quoted
150/// fields containing embedded newlines or delimiter characters do not skew
151/// detection. Tests comma, semicolon, tab, and pipe over the first 5 non-blank
152/// records: a delimiter whose modal field count is greater than one always
153/// beats one that only ever yields single-field records, then higher record
154/// agreement and higher field counts win.
155///
156/// Falls back to comma if no candidate splits the sample.
157pub fn detect_delimiter<R: Read>(reader: R) -> std::io::Result<u8> {
158    const SAMPLE_BYTES: u64 = 4096;
159    let mut preamble = Vec::new();
160    let mut take = reader.take(SAMPLE_BYTES);
161    take.read_to_end(&mut preamble)?;
162    let truncated = preamble.len() as u64 == SAMPLE_BYTES;
163
164    let preamble_str = String::from_utf8_lossy(&preamble);
165    let (mut records, last_unterminated) = split_sample_records(&preamble_str);
166    // A full sample buffer that ends mid-record leaves a fragment; never score
167    // it. A sample ending on a clean newline keeps all its records.
168    if truncated && last_unterminated && records.len() > 1 {
169        records.pop();
170    }
171
172    let delimiters = *b",;\t|";
173    let mut best_delimiter = b',';
174    // (splits records into >1 field, records agreeing on the modal count,
175    //  modal field count) — compared lexicographically, first wins ties.
176    let mut best_score = (false, 0usize, 0usize);
177
178    for &delimiter in &delimiters {
179        let mut counts = std::collections::HashMap::new();
180        // Blank records carry no delimiter signal: a header followed by
181        // trailing blank lines must not outvote the true delimiter with
182        // single-field "records" (gh #559).
183        let scored = records
184            .iter()
185            .filter(|record| !record.trim().is_empty())
186            .take(5);
187        for record in scored {
188            let fields = count_fields(record, delimiter as char);
189            *counts.entry(fields).or_insert(0usize) += 1;
190        }
191
192        // decode-audit: no-data — an empty sample scores 0 for this delimiter
193        // candidate; nothing is fabricated.
194        let (modal_fields, consistency) = counts
195            .into_iter()
196            .max_by_key(|&(fields, count)| (count, fields))
197            .unwrap_or((0, 0));
198
199        let score = (modal_fields > 1, consistency, modal_fields);
200        if score > best_score {
201            best_score = score;
202            best_delimiter = delimiter;
203        }
204    }
205
206    Ok(best_delimiter)
207}
208
209/// Detect the most likely CSV delimiter from a file path.
210pub fn detect_delimiter_from_path(path: &Path) -> std::io::Result<u8> {
211    let file = std::fs::File::open(path)?;
212    detect_delimiter(std::io::BufReader::new(file))
213}
214
215/// Analyze CSV data from any `Read` source using streaming statistics.
216///
217/// This is the primary generic entry point for both file-based and stream-based
218/// CSV profiling. Data is processed incrementally through [`StreamingColumnCollection`]
219/// with bounded memory, rather than accumulating all rows in a `HashMap`.
220///
221/// Returns `(column_profiles, streaming_stats, rows_read)`.
222pub fn analyze_csv_from_reader<R: Read>(
223    reader: R,
224    config: &CsvParserConfig,
225) -> Result<
226    (
227        Vec<ColumnProfile>,
228        StreamingColumnCollection,
229        usize,
230        Vec<String>,
231    ),
232    DataProfilerError,
233> {
234    analyze_csv_from_reader_with_hints(reader, config, &SemanticHints::default())
235}
236
237/// Analyze CSV data from any `Read` source while applying semantic hints.
238pub fn analyze_csv_from_reader_with_hints<R: Read>(
239    reader: R,
240    config: &CsvParserConfig,
241    semantic_hints: &SemanticHints,
242) -> Result<
243    (
244        Vec<ColumnProfile>,
245        StreamingColumnCollection,
246        usize,
247        Vec<String>,
248    ),
249    DataProfilerError,
250> {
251    let mut csv_builder = ReaderBuilder::new();
252    csv_builder.has_headers(config.has_header);
253    csv_builder.flexible(config.flexible);
254    csv_builder.quote(config.quote_char);
255    if config.trim_whitespace {
256        csv_builder.trim(csv::Trim::All);
257    }
258    let mut actual_delimiter = config.delimiter;
259
260    let boxed_reader: Box<dyn std::io::Read + '_> = if config.delimiter.is_none() {
261        let mut preamble = Vec::new();
262        let mut take = reader.take(4096);
263        take.read_to_end(&mut preamble)?;
264
265        let detected = detect_delimiter(std::io::Cursor::new(&preamble)).unwrap_or(b',');
266        actual_delimiter = Some(detected);
267
268        Box::new(std::io::Cursor::new(preamble).chain(take.into_inner()))
269    } else {
270        Box::new(reader)
271    };
272
273    if let Some(delimiter) = actual_delimiter {
274        csv_builder.delimiter(delimiter);
275    }
276
277    let mut csv_reader = csv_builder.from_reader(boxed_reader);
278
279    let header_names: Vec<String> = if config.has_header {
280        let headers = csv_reader.headers()?;
281        let names: Vec<String> = headers.iter().map(|header| header.to_string()).collect();
282        // Reject duplicate headers before profiling: process_record keys on name,
283        // so a repeat would merge two source columns into one inflated profile.
284        dataprof_core::validate_unique_column_names(&names, "CSV header")?;
285        names
286    } else {
287        let headers = csv_reader.headers()?;
288        (0..headers.len())
289            .map(|index| format!("column_{index}"))
290            .collect()
291    };
292
293    let mut column_stats = StreamingColumnCollection::new().with_semantic_hints(semantic_hints);
294    // A header declares the columns even when no data records follow it.
295    column_stats.init_columns(&header_names);
296    let mut rows_read = 0;
297
298    for result in csv_reader.records() {
299        if let Some(max_rows) = config.max_rows
300            && rows_read >= max_rows
301        {
302            break;
303        }
304
305        let record = result?;
306        let mut values: Vec<String> = record.iter().map(|value| value.to_string()).collect();
307
308        let header_len = header_names.len();
309        if values.len() < header_len {
310            values.resize(header_len, String::new());
311        } else if values.len() > header_len {
312            values.truncate(header_len);
313        }
314
315        column_stats.process_record(&header_names, values);
316        rows_read += 1;
317    }
318
319    let profiles = profile_builder::profiles_from_streaming_with_hints(
320        &column_stats,
321        false,
322        false,
323        None,
324        semantic_hints,
325    );
326
327    Ok((profiles, column_stats, rows_read, header_names))
328}
329
330/// Analyze a CSV file, returning a full [`ProfileReport`].
331///
332/// Opens the file, delegates to [`analyze_csv_from_reader`], and wraps the
333/// result with file metadata and ISO 8000/25012 data quality metrics.
334pub fn analyze_csv_file(
335    file_path: &Path,
336    config: &CsvParserConfig,
337) -> Result<ProfileReport, DataProfilerError> {
338    analyze_csv_file_with_dimensions(file_path, config, None)
339}
340
341/// Like [`analyze_csv_file`] but only computes the requested quality dimensions.
342pub fn analyze_csv_file_with_dimensions(
343    file_path: &Path,
344    config: &CsvParserConfig,
345    quality_dimensions: Option<&[QualityDimension]>,
346) -> Result<ProfileReport, DataProfilerError> {
347    analyze_csv_file_with_dimensions_and_hints(
348        file_path,
349        config,
350        quality_dimensions,
351        &SemanticHints::default(),
352    )
353}
354
355pub fn analyze_csv_file_with_dimensions_and_hints(
356    file_path: &Path,
357    config: &CsvParserConfig,
358    quality_dimensions: Option<&[QualityDimension]>,
359    semantic_hints: &SemanticHints,
360) -> Result<ProfileReport, DataProfilerError> {
361    let metadata = std::fs::metadata(file_path).map_err(|error| map_io_error(file_path, error))?;
362    let start = std::time::Instant::now();
363
364    let file = std::fs::File::open(file_path).map_err(|error| map_io_error(file_path, error))?;
365    let buf_reader = std::io::BufReader::new(file);
366
367    let (column_profiles, column_stats, rows_read, _header_names) =
368        analyze_csv_from_reader_with_hints(buf_reader, config, semantic_hints)?;
369
370    let file_source = DataSource::File {
371        path: file_path.display().to_string(),
372        format: FileFormat::Csv,
373        size_bytes: metadata.len(),
374        modified_at: None,
375        parquet_metadata: None,
376    };
377
378    let sample_columns = profile_builder::quality_check_samples(&column_stats);
379    let scan_time_ms = start.elapsed().as_millis();
380    let num_columns = column_profiles.len();
381
382    let mut assembler = ReportAssembler::new(
383        file_source,
384        ExecutionMetadata::new(rows_read, num_columns, scan_time_ms).with_engine("csv"),
385    )
386    .columns(column_profiles)
387    .with_quality_data(sample_columns)
388    .with_row_duplicates(column_stats.row_duplicate_summary())
389    .with_row_completeness(column_stats.row_completeness_summary())
390    .with_exact_value_hint_bindings(column_stats.semantic_hint_bindings())
391    .with_semantic_hints(semantic_hints.clone());
392    if let Some(dimensions) = quality_dimensions {
393        assembler = assembler.with_requested_dimensions(dimensions.to_vec());
394    }
395    Ok(assembler.build())
396}
397
398fn map_io_error(file_path: &Path, error: std::io::Error) -> DataProfilerError {
399    if error.kind() == std::io::ErrorKind::NotFound {
400        DataProfilerError::FileNotFound {
401            path: file_path.display().to_string(),
402        }
403    } else {
404        DataProfilerError::from(error)
405    }
406}
407
408#[cfg(test)]
409mod tests {
410    use super::*;
411    use dataprof_core::DataType;
412    use std::io::{Cursor, Write};
413    use tempfile::NamedTempFile;
414
415    fn write_csv(content: &str) -> NamedTempFile {
416        let mut file = NamedTempFile::new().unwrap();
417        write!(file, "{}", content).unwrap();
418        file.flush().unwrap();
419        file
420    }
421
422    #[test]
423    fn test_analyze_csv_from_reader_basic() {
424        let data = b"name,age\nAlice,25\nBob,30\n";
425        let cursor = Cursor::new(data.as_ref());
426        let config = CsvParserConfig::default();
427
428        let (profiles, _stats, rows, _) = analyze_csv_from_reader(cursor, &config).unwrap();
429        assert_eq!(rows, 2);
430        assert_eq!(profiles.len(), 2);
431
432        let age = profiles
433            .iter()
434            .find(|profile| profile.name == "age")
435            .unwrap();
436        assert_eq!(age.total_count, 2);
437        assert_eq!(age.data_type, DataType::Integer);
438    }
439
440    #[test]
441    fn test_analyze_csv_from_reader_with_max_rows() {
442        let data = b"val\n1\n2\n3\n4\n5\n";
443        let cursor = Cursor::new(data.as_ref());
444        let config = CsvParserConfig::default().max_rows(Some(3));
445
446        let (_profiles, _stats, rows, _) = analyze_csv_from_reader(cursor, &config).unwrap();
447        assert_eq!(rows, 3);
448    }
449
450    #[test]
451    fn test_analyze_csv_from_reader_strict_rejects_ragged() {
452        let data = b"a,b\n1,2\n3,4,5\n";
453        let cursor = Cursor::new(data.as_ref());
454        let config = CsvParserConfig::strict();
455
456        assert!(analyze_csv_from_reader(cursor, &config).is_err());
457    }
458
459    #[test]
460    fn test_analyze_csv_from_reader_flexible_handles_ragged() {
461        let data = b"a,b\n1,2\n3,4,5\n6\n";
462        let cursor = Cursor::new(data.as_ref());
463        let config = CsvParserConfig::default();
464
465        let (profiles, _stats, rows, _) = analyze_csv_from_reader(cursor, &config).unwrap();
466        assert_eq!(rows, 3);
467        assert!(!profiles.is_empty());
468    }
469
470    #[test]
471    fn test_analyze_csv_from_reader_custom_delimiter() {
472        let data = b"a;b\n1;2\n3;4\n";
473        let cursor = Cursor::new(data.as_ref());
474        let config = CsvParserConfig::default().with_delimiter(b';');
475
476        let (profiles, _stats, rows, _) = analyze_csv_from_reader(cursor, &config).unwrap();
477        assert_eq!(rows, 2);
478        assert_eq!(profiles.len(), 2);
479    }
480
481    #[test]
482    fn test_analyze_csv_file_returns_quality_report() {
483        let csv = write_csv("x,y\n1,a\n2,b\n3,c\n");
484        let config = CsvParserConfig::default();
485        let report = analyze_csv_file(csv.path(), &config).unwrap();
486
487        assert_eq!(report.column_profiles.len(), 2);
488        assert_eq!(report.execution.rows_processed, 3);
489        assert!(report.quality_score().unwrap() >= 0.0);
490        assert!(report.quality_score().unwrap() <= 100.0);
491    }
492
493    #[test]
494    fn test_csv_auto_detects_delimiter_from_reader() {
495        let data = b"name;age\nAlice;25\nBob;30\n";
496        let cursor = Cursor::new(data.as_ref());
497        let config = CsvParserConfig::default();
498
499        let (profiles, _stats, rows, _) = analyze_csv_from_reader(cursor, &config).unwrap();
500        assert_eq!(rows, 2);
501        assert_eq!(profiles.len(), 2);
502
503        let age = profiles
504            .iter()
505            .find(|profile| profile.name == "age")
506            .unwrap();
507        assert_eq!(age.total_count, 2);
508    }
509
510    #[test]
511    fn test_analyze_csv_basic() {
512        let csv = write_csv("name,age\nAlice,25\nBob,30\n");
513        let config = CsvParserConfig::default();
514        let report = analyze_csv_file(csv.path(), &config).unwrap();
515        let profiles = &report.column_profiles;
516        assert_eq!(profiles.len(), 2);
517
518        let names: Vec<&str> = profiles
519            .iter()
520            .map(|profile| profile.name.as_str())
521            .collect();
522        assert!(names.contains(&"name"));
523        assert!(names.contains(&"age"));
524
525        let age = profiles
526            .iter()
527            .find(|profile| profile.name == "age")
528            .unwrap();
529        assert_eq!(age.total_count, 2);
530        assert_eq!(age.null_count, 0);
531    }
532
533    #[test]
534    fn test_analyze_csv_with_nulls() {
535        let csv = write_csv("name,age,email\nAlice,25,a@b.com\nBob,,\nCharlie,30,c@d.com\n");
536        let config = CsvParserConfig::default();
537        let report = analyze_csv_file(csv.path(), &config).unwrap();
538        let profiles = &report.column_profiles;
539
540        let age = profiles
541            .iter()
542            .find(|profile| profile.name == "age")
543            .unwrap();
544        assert_eq!(age.total_count, 3);
545        assert_eq!(age.null_count, 1);
546
547        let email = profiles
548            .iter()
549            .find(|profile| profile.name == "email")
550            .unwrap();
551        assert_eq!(email.null_count, 1);
552    }
553
554    #[test]
555    fn test_analyze_csv_file_returns_quality_report_legacy() {
556        let csv = write_csv("x,y\n1,a\n2,b\n3,c\n");
557        let config = CsvParserConfig::default();
558        let report = analyze_csv_file(csv.path(), &config).unwrap();
559
560        assert_eq!(report.column_profiles.len(), 2);
561        assert_eq!(report.execution.rows_processed, 3);
562        assert_eq!(report.execution.columns_detected, 2);
563        assert!(report.quality_score().unwrap() >= 0.0);
564        assert!(report.quality_score().unwrap() <= 100.0);
565    }
566
567    #[test]
568    fn test_analyze_csv_file_header_only_preserves_declared_columns() {
569        let csv = write_csv("name,age\n");
570        let config = CsvParserConfig::default();
571        let report = analyze_csv_file(csv.path(), &config).unwrap();
572
573        assert_eq!(
574            report
575                .column_profiles
576                .iter()
577                .map(|column| (column.name.as_str(), column.data_type.clone()))
578                .collect::<Vec<_>>(),
579            vec![("name", DataType::String), ("age", DataType::String)]
580        );
581        assert_eq!(report.execution.rows_processed, 0);
582        assert!(report.quality.is_some());
583    }
584
585    #[test]
586    fn test_analyze_csv_file_empty_file_has_no_columns() {
587        let csv = write_csv("");
588        let config = CsvParserConfig::default();
589        let report = analyze_csv_file(csv.path(), &config).unwrap();
590
591        assert!(report.column_profiles.is_empty());
592        assert_eq!(report.execution.rows_processed, 0);
593        assert_eq!(report.execution.columns_detected, 0);
594        assert_eq!(
595            report.quality_status,
596            dataprof_runtime::QualityAnalysisStatus::Computed
597        );
598        assert!(report.quality.is_some());
599        assert!(report.quality_score().is_none());
600    }
601
602    #[test]
603    fn test_analyze_csv_with_max_rows_small_file() {
604        let csv = write_csv("val\n1\n2\n3\n4\n5\n");
605        let config = CsvParserConfig::default();
606        let report = analyze_csv_file(csv.path(), &config).unwrap();
607
608        assert_eq!(report.execution.rows_processed, 5);
609        assert_eq!(report.execution.columns_detected, 1);
610        assert!((report.execution.sampling_ratio.unwrap_or(1.0) - 1.0).abs() < 0.01);
611    }
612
613    #[test]
614    fn test_analyze_csv_numeric_types_detected() {
615        let csv = write_csv("int_col,float_col,str_col\n1,1.5,hello\n2,2.5,world\n3,3.5,foo\n");
616        let config = CsvParserConfig::default();
617        let report = analyze_csv_file(csv.path(), &config).unwrap();
618        let profiles = &report.column_profiles;
619
620        let int_col = profiles
621            .iter()
622            .find(|profile| profile.name == "int_col")
623            .unwrap();
624        assert!(matches!(int_col.data_type, DataType::Integer));
625
626        let float_col = profiles
627            .iter()
628            .find(|profile| profile.name == "float_col")
629            .unwrap();
630        assert!(matches!(float_col.data_type, DataType::Float));
631
632        let str_col = profiles
633            .iter()
634            .find(|profile| profile.name == "str_col")
635            .unwrap();
636        assert!(matches!(str_col.data_type, DataType::String));
637    }
638
639    #[test]
640    fn test_analyze_csv_strict_rejects_ragged_rows() {
641        let csv = write_csv("a,b\n1,2\n3,4,5\n");
642        let config = CsvParserConfig::strict();
643        assert!(analyze_csv_file(csv.path(), &config).is_err());
644    }
645
646    #[test]
647    fn test_analyze_csv_file_handles_ragged_rows() {
648        let csv = write_csv("a,b\n1,2\n3,4,5\n6\n");
649        let config = CsvParserConfig::default();
650        let report = analyze_csv_file(csv.path(), &config).unwrap();
651        assert!(!report.column_profiles.is_empty());
652    }
653
654    #[test]
655    fn test_analyze_csv_with_quality_metrics() {
656        let csv = write_csv(
657            "name,age\n\
658             Alice,25\nBob,\nCharlie,30\nDave,\nEve,28\n\
659             Frank,\nGrace,35\nHeidi,40\nIvan,\nJudy,22\n",
660        );
661        let config = CsvParserConfig::default();
662        let report = analyze_csv_file(csv.path(), &config).unwrap();
663
664        let age = report
665            .column_profiles
666            .iter()
667            .find(|profile| profile.name == "age")
668            .unwrap();
669        assert_eq!(age.null_count, 4);
670        assert_eq!(age.total_count, 10);
671        assert!(report.quality_score().unwrap() >= 0.0);
672    }
673
674    #[test]
675    fn test_detect_delimiter_comma() {
676        let data = b"name,age,city\nAlice,25,NYC\nBob,30,London\n";
677        assert_eq!(detect_delimiter(Cursor::new(data.as_ref())).unwrap(), b',');
678    }
679
680    #[test]
681    fn test_detect_delimiter_semicolon() {
682        let data = b"name;age;city\nAlice;25;NYC\nBob;30;London\n";
683        assert_eq!(detect_delimiter(Cursor::new(data.as_ref())).unwrap(), b';');
684    }
685
686    #[test]
687    fn test_detect_delimiter_pipe() {
688        let data = b"name|age|city\nAlice|25|NYC\nBob|30|London\n";
689        assert_eq!(detect_delimiter(Cursor::new(data.as_ref())).unwrap(), b'|');
690    }
691
692    #[test]
693    fn test_detect_delimiter_tab() {
694        let data = b"name\tage\tcity\nAlice\t25\tNYC\nBob\t30\tLondon\n";
695        assert_eq!(detect_delimiter(Cursor::new(data.as_ref())).unwrap(), b'\t');
696    }
697
698    #[test]
699    fn test_detect_delimiter_from_path_semicolon() {
700        let csv = write_csv("id;name;salary\n1;Alice;50000\n2;Bob;60000\n");
701        assert_eq!(detect_delimiter_from_path(csv.path()).unwrap(), b';');
702    }
703
704    #[test]
705    fn test_detect_delimiter_blank_records_do_not_outvote_header() {
706        // A header-only file with trailing blank lines: the blank records are
707        // single-field for every candidate, so counting them made `;` win on
708        // perfect consistency and collapse the whole header into one column
709        // (gh #559).
710        let data = b"name,age\n\n\n";
711        assert_eq!(detect_delimiter(Cursor::new(data.as_ref())).unwrap(), b',');
712
713        let semicolon = b"name;age\n\n";
714        assert_eq!(
715            detect_delimiter(Cursor::new(semicolon.as_ref())).unwrap(),
716            b';'
717        );
718    }
719
720    #[test]
721    fn test_detect_delimiter_quoted_fields() {
722        let data = b"name;desc;val\n\"hello;world\";foo;1\nbar;baz;2\n";
723        assert_eq!(detect_delimiter(Cursor::new(data.as_ref())).unwrap(), b';');
724    }
725
726    #[test]
727    fn test_detect_delimiter_quoted_embedded_newline() {
728        // A quoted cell containing a newline must not break record splitting:
729        // before the quote-aware sniffer this file was detected as semicolon
730        // (zero occurrences, "perfectly consistent" single-field records) and
731        // profiled as a single column named "id,comment".
732        let data = b"id,comment\n1,\"line one\nline two\"\n2,\"normal\"\n3,\"has \"\"quotes\"\" inside\"\n";
733        assert_eq!(detect_delimiter(Cursor::new(data.as_ref())).unwrap(), b',');
734    }
735
736    #[test]
737    fn test_detect_delimiter_absent_candidate_never_beats_real_one() {
738        // One ragged record makes comma imperfect (consistency 3/4); an absent
739        // delimiter is uniformly single-field (4/4) and must still lose.
740        let data = b"a,b,c\n1,2,3\nragged line without delimiters\n4,5,6\n";
741        assert_eq!(detect_delimiter(Cursor::new(data.as_ref())).unwrap(), b',');
742    }
743
744    #[test]
745    fn test_detect_delimiter_single_column_falls_back_to_comma() {
746        let data = b"value\n1\n2\n3\n";
747        assert_eq!(detect_delimiter(Cursor::new(data.as_ref())).unwrap(), b',');
748    }
749
750    #[test]
751    fn test_detect_delimiter_two_column_semicolon_with_commas_in_text() {
752        // Free text containing commas must not outvote the true delimiter.
753        let data = b"id;note\n1;hello, world\n2;foo, bar, baz\n3;plain\n";
754        assert_eq!(detect_delimiter(Cursor::new(data.as_ref())).unwrap(), b';');
755    }
756
757    #[test]
758    fn test_split_sample_records_quote_aware() {
759        let (records, last_unterminated) = split_sample_records("a,b\r\n1,\"x\ny\"\n2,z\n");
760        assert_eq!(records, vec!["a,b", "1,\"x\ny\"", "2,z"]);
761        assert!(!last_unterminated);
762
763        let (records, last_unterminated) = split_sample_records("a,b\n1,\"cut mid-fie");
764        assert_eq!(records, vec!["a,b", "1,\"cut mid-fie"]);
765        assert!(last_unterminated);
766    }
767
768    #[test]
769    fn test_detect_delimiter_full_sample_ending_on_record_boundary() {
770        // Exactly 4096 bytes ending with a clean newline: the final record is
771        // complete and must be scored, not dropped as a fragment.
772        let mut data = Vec::from(&b"a;b;c\n"[..]);
773        while data.len() < 4090 {
774            data.extend_from_slice(b"1;2;3\n");
775        }
776        while data.len() < 4095 {
777            data.push(b'#');
778        }
779        data.push(b'\n');
780        assert_eq!(data.len(), 4096);
781        assert_eq!(
782            detect_delimiter(Cursor::new(data.as_slice())).unwrap(),
783            b';'
784        );
785    }
786}