Skip to main content

dataprof_csv/
lib.rs

1use csv::ReaderBuilder;
2use std::io::Read;
3use std::path::Path;
4
5use dataprof_core::{
6    ColumnProfile, DataProfilerError, DataSource, ExecutionMetadata, FileFormat, QualityDimension,
7    SemanticHints,
8};
9use dataprof_runtime::{
10    ProfileReport, ReportAssembler, StreamingColumnCollection, profile_builder,
11};
12
13mod memmap_reader;
14mod robust_csv;
15
16pub use memmap_reader::MemoryMappedCsvReader;
17pub use robust_csv::{
18    CsvDiagnostics, EncodingDiagnostic, RobustCsvParser, RobustParseResult, diagnose_non_utf8,
19};
20
21/// Configuration for CSV parsing and analysis.
22///
23/// Replaces the 7 separate `analyze_csv_*` functions with a single config struct.
24/// Supports both file-based and reader-based (streaming) entry points.
25#[derive(Debug, Clone)]
26pub struct CsvParserConfig {
27    /// Allow ragged rows (flexible field counts).
28    pub flexible: bool,
29    /// Custom delimiter (None = comma).
30    pub delimiter: Option<u8>,
31    /// Quote character.
32    pub quote_char: u8,
33    /// Trim whitespace from fields.
34    pub trim_whitespace: bool,
35    /// Whether the first row contains column headers.
36    /// When `false`, columns are named `column_0`, `column_1`, etc.
37    pub has_header: bool,
38    /// Maximum rows to process (None = all rows).
39    pub max_rows: Option<usize>,
40}
41
42impl Default for CsvParserConfig {
43    fn default() -> Self {
44        Self {
45            flexible: true,
46            delimiter: None,
47            quote_char: b'"',
48            trim_whitespace: false,
49            has_header: true,
50            max_rows: None,
51        }
52    }
53}
54
55impl CsvParserConfig {
56    /// Create a strict config that rejects ragged rows.
57    pub fn strict() -> Self {
58        Self {
59            flexible: false,
60            ..Default::default()
61        }
62    }
63
64    /// Set the delimiter.
65    pub fn with_delimiter(mut self, delimiter: u8) -> Self {
66        self.delimiter = Some(delimiter);
67        self
68    }
69
70    /// Set whether the first row contains column headers.
71    pub fn has_header(mut self, yes: bool) -> Self {
72        self.has_header = yes;
73        self
74    }
75
76    /// Set maximum number of rows to evaluate.
77    pub fn max_rows(mut self, max: Option<usize>) -> Self {
78        self.max_rows = max;
79        self
80    }
81}
82
83/// Count fields in a line for a given delimiter, respecting quoted sections.
84pub(crate) fn count_fields(line: &str, delimiter: char) -> usize {
85    let mut count = 1;
86    let mut in_quotes = false;
87    let mut chars = line.chars().peekable();
88    while let Some(ch) = chars.next() {
89        if ch == '"' {
90            if chars.peek() == Some(&'"') {
91                chars.next();
92            } else {
93                in_quotes = !in_quotes;
94            }
95        } else if ch == delimiter && !in_quotes {
96            count += 1;
97        }
98    }
99    count
100}
101
102/// Split raw CSV sample text into logical records, honoring quoted sections
103/// so an embedded newline inside quotes does not start a new record.
104///
105/// The returned flag is true when the final record was not terminated by a
106/// newline outside quotes — i.e. the text ends mid-record (or exactly at a
107/// record's end with no trailing newline, which a caller sampling a fixed
108/// byte budget cannot distinguish from a cut).
109fn split_sample_records(text: &str) -> (Vec<&str>, bool) {
110    let bytes = text.as_bytes();
111    let mut records = Vec::new();
112    let mut in_quotes = false;
113    let mut start = 0usize;
114    let mut i = 0usize;
115    while i < bytes.len() {
116        match bytes[i] {
117            b'"' => {
118                if in_quotes && bytes.get(i + 1) == Some(&b'"') {
119                    i += 1; // escaped quote inside a quoted section
120                } else {
121                    in_quotes = !in_quotes;
122                }
123            }
124            b'\n' if !in_quotes => {
125                let end = if i > start && bytes[i - 1] == b'\r' {
126                    i - 1
127                } else {
128                    i
129                };
130                records.push(&text[start..end]);
131                start = i + 1;
132            }
133            _ => {}
134        }
135        i += 1;
136    }
137    let last_unterminated = start < text.len();
138    if last_unterminated {
139        records.push(text[start..].trim_end_matches('\r'));
140    }
141    (records, last_unterminated)
142}
143
144/// Detect the most likely CSV delimiter by sampling up to 4 KB of data.
145///
146/// The sample is split into logical records with quote handling, so quoted
147/// fields containing embedded newlines or delimiter characters do not skew
148/// detection. Tests comma, semicolon, tab, and pipe over the first 5 records:
149/// a delimiter whose modal field count is greater than one always beats one
150/// that only ever yields single-field records, then higher record agreement
151/// and higher field counts win.
152///
153/// Falls back to comma if no candidate splits the sample.
154pub fn detect_delimiter<R: Read>(reader: R) -> std::io::Result<u8> {
155    const SAMPLE_BYTES: u64 = 4096;
156    let mut preamble = Vec::new();
157    let mut take = reader.take(SAMPLE_BYTES);
158    take.read_to_end(&mut preamble)?;
159    let truncated = preamble.len() as u64 == SAMPLE_BYTES;
160
161    let preamble_str = String::from_utf8_lossy(&preamble);
162    let (mut records, last_unterminated) = split_sample_records(&preamble_str);
163    // A full sample buffer that ends mid-record leaves a fragment; never score
164    // it. A sample ending on a clean newline keeps all its records.
165    if truncated && last_unterminated && records.len() > 1 {
166        records.pop();
167    }
168
169    let delimiters = *b",;\t|";
170    let mut best_delimiter = b',';
171    // (splits records into >1 field, records agreeing on the modal count,
172    //  modal field count) — compared lexicographically, first wins ties.
173    let mut best_score = (false, 0usize, 0usize);
174
175    for &delimiter in &delimiters {
176        let mut counts = std::collections::HashMap::new();
177        for record in records.iter().take(5) {
178            let fields = count_fields(record, delimiter as char);
179            *counts.entry(fields).or_insert(0usize) += 1;
180        }
181
182        // decode-audit: no-data — an empty sample scores 0 for this delimiter
183        // candidate; nothing is fabricated.
184        let (modal_fields, consistency) = counts
185            .into_iter()
186            .max_by_key(|&(fields, count)| (count, fields))
187            .unwrap_or((0, 0));
188
189        let score = (modal_fields > 1, consistency, modal_fields);
190        if score > best_score {
191            best_score = score;
192            best_delimiter = delimiter;
193        }
194    }
195
196    Ok(best_delimiter)
197}
198
199/// Detect the most likely CSV delimiter from a file path.
200pub fn detect_delimiter_from_path(path: &Path) -> std::io::Result<u8> {
201    let file = std::fs::File::open(path)?;
202    detect_delimiter(std::io::BufReader::new(file))
203}
204
205/// Analyze CSV data from any `Read` source using streaming statistics.
206///
207/// This is the primary generic entry point for both file-based and stream-based
208/// CSV profiling. Data is processed incrementally through [`StreamingColumnCollection`]
209/// with bounded memory, rather than accumulating all rows in a `HashMap`.
210///
211/// Returns `(column_profiles, streaming_stats, rows_read)`.
212pub fn analyze_csv_from_reader<R: Read>(
213    reader: R,
214    config: &CsvParserConfig,
215) -> Result<
216    (
217        Vec<ColumnProfile>,
218        StreamingColumnCollection,
219        usize,
220        Vec<String>,
221    ),
222    DataProfilerError,
223> {
224    analyze_csv_from_reader_with_hints(reader, config, &SemanticHints::default())
225}
226
227/// Analyze CSV data from any `Read` source while applying semantic hints.
228pub fn analyze_csv_from_reader_with_hints<R: Read>(
229    reader: R,
230    config: &CsvParserConfig,
231    semantic_hints: &SemanticHints,
232) -> Result<
233    (
234        Vec<ColumnProfile>,
235        StreamingColumnCollection,
236        usize,
237        Vec<String>,
238    ),
239    DataProfilerError,
240> {
241    let mut csv_builder = ReaderBuilder::new();
242    csv_builder.has_headers(config.has_header);
243    csv_builder.flexible(config.flexible);
244    csv_builder.quote(config.quote_char);
245    if config.trim_whitespace {
246        csv_builder.trim(csv::Trim::All);
247    }
248    let mut actual_delimiter = config.delimiter;
249
250    let boxed_reader: Box<dyn std::io::Read + '_> = if config.delimiter.is_none() {
251        let mut preamble = Vec::new();
252        let mut take = reader.take(4096);
253        take.read_to_end(&mut preamble)?;
254
255        let detected = detect_delimiter(std::io::Cursor::new(&preamble)).unwrap_or(b',');
256        actual_delimiter = Some(detected);
257
258        Box::new(std::io::Cursor::new(preamble).chain(take.into_inner()))
259    } else {
260        Box::new(reader)
261    };
262
263    if let Some(delimiter) = actual_delimiter {
264        csv_builder.delimiter(delimiter);
265    }
266
267    let mut csv_reader = csv_builder.from_reader(boxed_reader);
268
269    let header_names: Vec<String> = if config.has_header {
270        let headers = csv_reader.headers()?;
271        let names: Vec<String> = headers.iter().map(|header| header.to_string()).collect();
272        // Reject duplicate headers before profiling: process_record keys on name,
273        // so a repeat would merge two source columns into one inflated profile.
274        dataprof_core::validate_unique_column_names(&names, "CSV header")?;
275        names
276    } else {
277        let headers = csv_reader.headers()?;
278        (0..headers.len())
279            .map(|index| format!("column_{index}"))
280            .collect()
281    };
282
283    let mut column_stats = StreamingColumnCollection::new().with_semantic_hints(semantic_hints);
284    let mut rows_read = 0;
285
286    for result in csv_reader.records() {
287        if let Some(max_rows) = config.max_rows
288            && rows_read >= max_rows
289        {
290            break;
291        }
292
293        let record = result?;
294        let mut values: Vec<String> = record.iter().map(|value| value.to_string()).collect();
295
296        let header_len = header_names.len();
297        if values.len() < header_len {
298            values.resize(header_len, String::new());
299        } else if values.len() > header_len {
300            values.truncate(header_len);
301        }
302
303        column_stats.process_record(&header_names, values);
304        rows_read += 1;
305    }
306
307    let profiles = profile_builder::profiles_from_streaming_with_hints(
308        &column_stats,
309        false,
310        false,
311        None,
312        semantic_hints,
313    );
314
315    Ok((profiles, column_stats, rows_read, header_names))
316}
317
318/// Analyze a CSV file, returning a full [`ProfileReport`].
319///
320/// Opens the file, delegates to [`analyze_csv_from_reader`], and wraps the
321/// result with file metadata and ISO 8000/25012 data quality metrics.
322pub fn analyze_csv_file(
323    file_path: &Path,
324    config: &CsvParserConfig,
325) -> Result<ProfileReport, DataProfilerError> {
326    analyze_csv_file_with_dimensions(file_path, config, None)
327}
328
329/// Like [`analyze_csv_file`] but only computes the requested quality dimensions.
330pub fn analyze_csv_file_with_dimensions(
331    file_path: &Path,
332    config: &CsvParserConfig,
333    quality_dimensions: Option<&[QualityDimension]>,
334) -> Result<ProfileReport, DataProfilerError> {
335    analyze_csv_file_with_dimensions_and_hints(
336        file_path,
337        config,
338        quality_dimensions,
339        &SemanticHints::default(),
340    )
341}
342
343pub fn analyze_csv_file_with_dimensions_and_hints(
344    file_path: &Path,
345    config: &CsvParserConfig,
346    quality_dimensions: Option<&[QualityDimension]>,
347    semantic_hints: &SemanticHints,
348) -> Result<ProfileReport, DataProfilerError> {
349    let metadata = std::fs::metadata(file_path).map_err(|error| map_io_error(file_path, error))?;
350    let start = std::time::Instant::now();
351
352    let file = std::fs::File::open(file_path).map_err(|error| map_io_error(file_path, error))?;
353    let buf_reader = std::io::BufReader::new(file);
354
355    let (column_profiles, column_stats, rows_read, header_names) =
356        analyze_csv_from_reader_with_hints(buf_reader, config, semantic_hints)?;
357
358    let file_source = DataSource::File {
359        path: file_path.display().to_string(),
360        format: FileFormat::Csv,
361        size_bytes: metadata.len(),
362        modified_at: None,
363        parquet_metadata: None,
364    };
365
366    if column_profiles.is_empty() && rows_read == 0 {
367        return Ok(ReportAssembler::new(
368            file_source,
369            ExecutionMetadata::new(0, header_names.len(), start.elapsed().as_millis())
370                .with_engine("csv"),
371        )
372        .skip_quality()
373        .build());
374    }
375
376    let sample_columns = profile_builder::quality_check_samples(&column_stats);
377    let scan_time_ms = start.elapsed().as_millis();
378    let num_columns = column_profiles.len();
379
380    let mut assembler = ReportAssembler::new(
381        file_source,
382        ExecutionMetadata::new(rows_read, num_columns, scan_time_ms).with_engine("csv"),
383    )
384    .columns(column_profiles)
385    .with_quality_data(sample_columns)
386    .with_row_duplicates(column_stats.row_duplicate_summary())
387    .with_exact_value_hint_bindings(column_stats.semantic_hint_bindings())
388    .with_semantic_hints(semantic_hints.clone());
389    if let Some(dimensions) = quality_dimensions {
390        assembler = assembler.with_requested_dimensions(dimensions.to_vec());
391    }
392    Ok(assembler.build())
393}
394
395fn map_io_error(file_path: &Path, error: std::io::Error) -> DataProfilerError {
396    if error.kind() == std::io::ErrorKind::NotFound {
397        DataProfilerError::FileNotFound {
398            path: file_path.display().to_string(),
399        }
400    } else {
401        DataProfilerError::from(error)
402    }
403}
404
405#[cfg(test)]
406mod tests {
407    use super::*;
408    use dataprof_core::DataType;
409    use std::io::{Cursor, Write};
410    use tempfile::NamedTempFile;
411
412    fn write_csv(content: &str) -> NamedTempFile {
413        let mut file = NamedTempFile::new().unwrap();
414        write!(file, "{}", content).unwrap();
415        file.flush().unwrap();
416        file
417    }
418
419    #[test]
420    fn test_analyze_csv_from_reader_basic() {
421        let data = b"name,age\nAlice,25\nBob,30\n";
422        let cursor = Cursor::new(data.as_ref());
423        let config = CsvParserConfig::default();
424
425        let (profiles, _stats, rows, _) = analyze_csv_from_reader(cursor, &config).unwrap();
426        assert_eq!(rows, 2);
427        assert_eq!(profiles.len(), 2);
428
429        let age = profiles
430            .iter()
431            .find(|profile| profile.name == "age")
432            .unwrap();
433        assert_eq!(age.total_count, 2);
434        assert_eq!(age.data_type, DataType::Integer);
435    }
436
437    #[test]
438    fn test_analyze_csv_from_reader_with_max_rows() {
439        let data = b"val\n1\n2\n3\n4\n5\n";
440        let cursor = Cursor::new(data.as_ref());
441        let config = CsvParserConfig::default().max_rows(Some(3));
442
443        let (_profiles, _stats, rows, _) = analyze_csv_from_reader(cursor, &config).unwrap();
444        assert_eq!(rows, 3);
445    }
446
447    #[test]
448    fn test_analyze_csv_from_reader_strict_rejects_ragged() {
449        let data = b"a,b\n1,2\n3,4,5\n";
450        let cursor = Cursor::new(data.as_ref());
451        let config = CsvParserConfig::strict();
452
453        assert!(analyze_csv_from_reader(cursor, &config).is_err());
454    }
455
456    #[test]
457    fn test_analyze_csv_from_reader_flexible_handles_ragged() {
458        let data = b"a,b\n1,2\n3,4,5\n6\n";
459        let cursor = Cursor::new(data.as_ref());
460        let config = CsvParserConfig::default();
461
462        let (profiles, _stats, rows, _) = analyze_csv_from_reader(cursor, &config).unwrap();
463        assert_eq!(rows, 3);
464        assert!(!profiles.is_empty());
465    }
466
467    #[test]
468    fn test_analyze_csv_from_reader_custom_delimiter() {
469        let data = b"a;b\n1;2\n3;4\n";
470        let cursor = Cursor::new(data.as_ref());
471        let config = CsvParserConfig::default().with_delimiter(b';');
472
473        let (profiles, _stats, rows, _) = analyze_csv_from_reader(cursor, &config).unwrap();
474        assert_eq!(rows, 2);
475        assert_eq!(profiles.len(), 2);
476    }
477
478    #[test]
479    fn test_analyze_csv_file_returns_quality_report() {
480        let csv = write_csv("x,y\n1,a\n2,b\n3,c\n");
481        let config = CsvParserConfig::default();
482        let report = analyze_csv_file(csv.path(), &config).unwrap();
483
484        assert_eq!(report.column_profiles.len(), 2);
485        assert_eq!(report.execution.rows_processed, 3);
486        assert!(report.quality_score().unwrap() >= 0.0);
487        assert!(report.quality_score().unwrap() <= 100.0);
488    }
489
490    #[test]
491    fn test_csv_auto_detects_delimiter_from_reader() {
492        let data = b"name;age\nAlice;25\nBob;30\n";
493        let cursor = Cursor::new(data.as_ref());
494        let config = CsvParserConfig::default();
495
496        let (profiles, _stats, rows, _) = analyze_csv_from_reader(cursor, &config).unwrap();
497        assert_eq!(rows, 2);
498        assert_eq!(profiles.len(), 2);
499
500        let age = profiles
501            .iter()
502            .find(|profile| profile.name == "age")
503            .unwrap();
504        assert_eq!(age.total_count, 2);
505    }
506
507    #[test]
508    fn test_analyze_csv_basic() {
509        let csv = write_csv("name,age\nAlice,25\nBob,30\n");
510        let config = CsvParserConfig::default();
511        let report = analyze_csv_file(csv.path(), &config).unwrap();
512        let profiles = &report.column_profiles;
513        assert_eq!(profiles.len(), 2);
514
515        let names: Vec<&str> = profiles
516            .iter()
517            .map(|profile| profile.name.as_str())
518            .collect();
519        assert!(names.contains(&"name"));
520        assert!(names.contains(&"age"));
521
522        let age = profiles
523            .iter()
524            .find(|profile| profile.name == "age")
525            .unwrap();
526        assert_eq!(age.total_count, 2);
527        assert_eq!(age.null_count, 0);
528    }
529
530    #[test]
531    fn test_analyze_csv_with_nulls() {
532        let csv = write_csv("name,age,email\nAlice,25,a@b.com\nBob,,\nCharlie,30,c@d.com\n");
533        let config = CsvParserConfig::default();
534        let report = analyze_csv_file(csv.path(), &config).unwrap();
535        let profiles = &report.column_profiles;
536
537        let age = profiles
538            .iter()
539            .find(|profile| profile.name == "age")
540            .unwrap();
541        assert_eq!(age.total_count, 3);
542        assert_eq!(age.null_count, 1);
543
544        let email = profiles
545            .iter()
546            .find(|profile| profile.name == "email")
547            .unwrap();
548        assert_eq!(email.null_count, 1);
549    }
550
551    #[test]
552    fn test_analyze_csv_file_returns_quality_report_legacy() {
553        let csv = write_csv("x,y\n1,a\n2,b\n3,c\n");
554        let config = CsvParserConfig::default();
555        let report = analyze_csv_file(csv.path(), &config).unwrap();
556
557        assert_eq!(report.column_profiles.len(), 2);
558        assert_eq!(report.execution.rows_processed, 3);
559        assert_eq!(report.execution.columns_detected, 2);
560        assert!(report.quality_score().unwrap() >= 0.0);
561        assert!(report.quality_score().unwrap() <= 100.0);
562    }
563
564    #[test]
565    fn test_analyze_csv_file_empty_file() {
566        let csv = write_csv("name,age\n");
567        let config = CsvParserConfig::default();
568        let report = analyze_csv_file(csv.path(), &config).unwrap();
569
570        assert_eq!(report.column_profiles.len(), 0);
571        assert_eq!(report.execution.rows_processed, 0);
572    }
573
574    #[test]
575    fn test_analyze_csv_with_max_rows_small_file() {
576        let csv = write_csv("val\n1\n2\n3\n4\n5\n");
577        let config = CsvParserConfig::default();
578        let report = analyze_csv_file(csv.path(), &config).unwrap();
579
580        assert_eq!(report.execution.rows_processed, 5);
581        assert_eq!(report.execution.columns_detected, 1);
582        assert!((report.execution.sampling_ratio.unwrap_or(1.0) - 1.0).abs() < 0.01);
583    }
584
585    #[test]
586    fn test_analyze_csv_numeric_types_detected() {
587        let csv = write_csv("int_col,float_col,str_col\n1,1.5,hello\n2,2.5,world\n3,3.5,foo\n");
588        let config = CsvParserConfig::default();
589        let report = analyze_csv_file(csv.path(), &config).unwrap();
590        let profiles = &report.column_profiles;
591
592        let int_col = profiles
593            .iter()
594            .find(|profile| profile.name == "int_col")
595            .unwrap();
596        assert!(matches!(int_col.data_type, DataType::Integer));
597
598        let float_col = profiles
599            .iter()
600            .find(|profile| profile.name == "float_col")
601            .unwrap();
602        assert!(matches!(float_col.data_type, DataType::Float));
603
604        let str_col = profiles
605            .iter()
606            .find(|profile| profile.name == "str_col")
607            .unwrap();
608        assert!(matches!(str_col.data_type, DataType::String));
609    }
610
611    #[test]
612    fn test_analyze_csv_strict_rejects_ragged_rows() {
613        let csv = write_csv("a,b\n1,2\n3,4,5\n");
614        let config = CsvParserConfig::strict();
615        assert!(analyze_csv_file(csv.path(), &config).is_err());
616    }
617
618    #[test]
619    fn test_analyze_csv_file_handles_ragged_rows() {
620        let csv = write_csv("a,b\n1,2\n3,4,5\n6\n");
621        let config = CsvParserConfig::default();
622        let report = analyze_csv_file(csv.path(), &config).unwrap();
623        assert!(!report.column_profiles.is_empty());
624    }
625
626    #[test]
627    fn test_analyze_csv_with_quality_metrics() {
628        let csv = write_csv(
629            "name,age\n\
630             Alice,25\nBob,\nCharlie,30\nDave,\nEve,28\n\
631             Frank,\nGrace,35\nHeidi,40\nIvan,\nJudy,22\n",
632        );
633        let config = CsvParserConfig::default();
634        let report = analyze_csv_file(csv.path(), &config).unwrap();
635
636        let age = report
637            .column_profiles
638            .iter()
639            .find(|profile| profile.name == "age")
640            .unwrap();
641        assert_eq!(age.null_count, 4);
642        assert_eq!(age.total_count, 10);
643        assert!(report.quality_score().unwrap() >= 0.0);
644    }
645
646    #[test]
647    fn test_detect_delimiter_comma() {
648        let data = b"name,age,city\nAlice,25,NYC\nBob,30,London\n";
649        assert_eq!(detect_delimiter(Cursor::new(data.as_ref())).unwrap(), b',');
650    }
651
652    #[test]
653    fn test_detect_delimiter_semicolon() {
654        let data = b"name;age;city\nAlice;25;NYC\nBob;30;London\n";
655        assert_eq!(detect_delimiter(Cursor::new(data.as_ref())).unwrap(), b';');
656    }
657
658    #[test]
659    fn test_detect_delimiter_pipe() {
660        let data = b"name|age|city\nAlice|25|NYC\nBob|30|London\n";
661        assert_eq!(detect_delimiter(Cursor::new(data.as_ref())).unwrap(), b'|');
662    }
663
664    #[test]
665    fn test_detect_delimiter_tab() {
666        let data = b"name\tage\tcity\nAlice\t25\tNYC\nBob\t30\tLondon\n";
667        assert_eq!(detect_delimiter(Cursor::new(data.as_ref())).unwrap(), b'\t');
668    }
669
670    #[test]
671    fn test_detect_delimiter_from_path_semicolon() {
672        let csv = write_csv("id;name;salary\n1;Alice;50000\n2;Bob;60000\n");
673        assert_eq!(detect_delimiter_from_path(csv.path()).unwrap(), b';');
674    }
675
676    #[test]
677    fn test_detect_delimiter_quoted_fields() {
678        let data = b"name;desc;val\n\"hello;world\";foo;1\nbar;baz;2\n";
679        assert_eq!(detect_delimiter(Cursor::new(data.as_ref())).unwrap(), b';');
680    }
681
682    #[test]
683    fn test_detect_delimiter_quoted_embedded_newline() {
684        // A quoted cell containing a newline must not break record splitting:
685        // before the quote-aware sniffer this file was detected as semicolon
686        // (zero occurrences, "perfectly consistent" single-field records) and
687        // profiled as a single column named "id,comment".
688        let data = b"id,comment\n1,\"line one\nline two\"\n2,\"normal\"\n3,\"has \"\"quotes\"\" inside\"\n";
689        assert_eq!(detect_delimiter(Cursor::new(data.as_ref())).unwrap(), b',');
690    }
691
692    #[test]
693    fn test_detect_delimiter_absent_candidate_never_beats_real_one() {
694        // One ragged record makes comma imperfect (consistency 3/4); an absent
695        // delimiter is uniformly single-field (4/4) and must still lose.
696        let data = b"a,b,c\n1,2,3\nragged line without delimiters\n4,5,6\n";
697        assert_eq!(detect_delimiter(Cursor::new(data.as_ref())).unwrap(), b',');
698    }
699
700    #[test]
701    fn test_detect_delimiter_single_column_falls_back_to_comma() {
702        let data = b"value\n1\n2\n3\n";
703        assert_eq!(detect_delimiter(Cursor::new(data.as_ref())).unwrap(), b',');
704    }
705
706    #[test]
707    fn test_detect_delimiter_two_column_semicolon_with_commas_in_text() {
708        // Free text containing commas must not outvote the true delimiter.
709        let data = b"id;note\n1;hello, world\n2;foo, bar, baz\n3;plain\n";
710        assert_eq!(detect_delimiter(Cursor::new(data.as_ref())).unwrap(), b';');
711    }
712
713    #[test]
714    fn test_split_sample_records_quote_aware() {
715        let (records, last_unterminated) = split_sample_records("a,b\r\n1,\"x\ny\"\n2,z\n");
716        assert_eq!(records, vec!["a,b", "1,\"x\ny\"", "2,z"]);
717        assert!(!last_unterminated);
718
719        let (records, last_unterminated) = split_sample_records("a,b\n1,\"cut mid-fie");
720        assert_eq!(records, vec!["a,b", "1,\"cut mid-fie"]);
721        assert!(last_unterminated);
722    }
723
724    #[test]
725    fn test_detect_delimiter_full_sample_ending_on_record_boundary() {
726        // Exactly 4096 bytes ending with a clean newline: the final record is
727        // complete and must be scored, not dropped as a fragment.
728        let mut data = Vec::from(&b"a;b;c\n"[..]);
729        while data.len() < 4090 {
730            data.extend_from_slice(b"1;2;3\n");
731        }
732        while data.len() < 4095 {
733            data.push(b'#');
734        }
735        data.push(b'\n');
736        assert_eq!(data.len(), 4096);
737        assert_eq!(
738            detect_delimiter(Cursor::new(data.as_slice())).unwrap(),
739            b';'
740        );
741    }
742}