Skip to main content

dataprof_csv/
lib.rs

1use csv::ReaderBuilder;
2use std::io::Read;
3use std::path::Path;
4
5use dataprof_core::{
6    AnalysisOptions, ColumnProfile, CsvSourceMetadata, DataProfilerError, DataSource,
7    DelimiterDetection, ExecutionMetadata, FileFormat, TruncationReason,
8};
9use dataprof_runtime::{
10    ProfileReport, ReportAssembler, StreamingColumnCollection, profile_builder,
11};
12
13mod encoding;
14mod memmap_reader;
15mod quote_state;
16#[doc(hidden)]
17pub mod row_count;
18
19pub use encoding::{EncodingDiagnostic, diagnose_non_utf8};
20pub use memmap_reader::MemoryMappedCsvReader;
21pub use quote_state::{QuoteOutcome, QuoteTrackingReader, unterminated_quote_error};
22
23/// Configuration for CSV parsing and analysis.
24///
25/// Replaces the 7 separate `analyze_csv_*` functions with a single config struct.
26/// Supports both file-based and reader-based (streaming) entry points.
27#[derive(Debug, Clone)]
28pub struct CsvParserConfig {
29    /// Allow ragged rows (flexible field counts).
30    pub flexible: bool,
31    /// Custom delimiter (None = comma).
32    pub delimiter: Option<u8>,
33    /// Quote character, used for delimiter detection and parsing.
34    pub quote_char: u8,
35    /// Trim whitespace from fields.
36    pub trim_whitespace: bool,
37    /// Whether the first row contains column headers.
38    /// When `false`, columns are named `column_0`, `column_1`, etc.
39    pub has_header: bool,
40    /// Maximum rows to process (None = all rows).
41    pub max_rows: Option<usize>,
42}
43
44impl Default for CsvParserConfig {
45    fn default() -> Self {
46        Self {
47            flexible: true,
48            delimiter: None,
49            quote_char: b'"',
50            trim_whitespace: false,
51            has_header: true,
52            max_rows: None,
53        }
54    }
55}
56
57impl CsvParserConfig {
58    /// Create a strict config that rejects ragged rows.
59    pub fn strict() -> Self {
60        Self {
61            flexible: false,
62            ..Default::default()
63        }
64    }
65
66    /// Set the delimiter.
67    pub fn with_delimiter(mut self, delimiter: u8) -> Self {
68        self.delimiter = Some(delimiter);
69        self
70    }
71
72    /// Set whether the first row contains column headers.
73    pub fn has_header(mut self, yes: bool) -> Self {
74        self.has_header = yes;
75        self
76    }
77
78    /// Set maximum number of rows to evaluate.
79    pub fn max_rows(mut self, max: Option<usize>) -> Self {
80        self.max_rows = max;
81        self
82    }
83}
84
85/// Count the fields in a record for a given delimiter, respecting quoted
86/// sections, and how many of them at the end are empty.
87///
88/// Returns `(fields, trailing_empty)`. A field is empty when it decodes to
89/// nothing, as the readers decode it: nothing stands between its delimiters,
90/// or it is a quoted empty string `""`. Judging the raw text instead counted
91/// `""` as content, and a row ending in `;""` disagreed with its header
92/// (#1102).
93///
94/// A quote opens a quoted section only at the start of a field, as in the
95/// readers and [`split_sample_records`]. Opening one at any quote hid every
96/// delimiter after `id"x`, and a `id"x;value` header was read as one column.
97fn count_fields(line: &[u8], delimiter: u8, quote: u8) -> (usize, usize) {
98    let decodes_empty = |field: &[u8]| field.is_empty() || field == [quote, quote];
99    let mut fields = 1;
100    let mut trailing_empty = 0;
101    let mut field_start = 0;
102    let mut in_quotes = false;
103    let mut bytes = line.iter().copied().enumerate().peekable();
104    while let Some((index, byte)) = bytes.next() {
105        if byte == quote && in_quotes {
106            if bytes.peek().map(|&(_, next)| next) == Some(quote) {
107                bytes.next();
108            } else {
109                in_quotes = false;
110            }
111        } else if byte == quote && index == field_start {
112            in_quotes = true;
113        } else if byte == delimiter && !in_quotes {
114            trailing_empty = if decodes_empty(&line[field_start..index]) {
115                trailing_empty + 1
116            } else {
117                0
118            };
119            fields += 1;
120            field_start = index + 1;
121        }
122    }
123    trailing_empty = if decodes_empty(&line[field_start..]) {
124        trailing_empty + 1
125    } else {
126        0
127    };
128    (fields, trailing_empty)
129}
130
131/// Whether a record reads at the header's width without losing or inventing a
132/// value: it has exactly that many fields, or more whose extras are all empty,
133/// so cutting it to the header drops nothing (#1096).
134fn agrees_with_header(fields: usize, trailing_empty: usize, header_fields: usize) -> bool {
135    fields == header_fields || (fields > header_fields && fields - trailing_empty <= header_fields)
136}
137
138/// Split raw CSV sample text into logical records, honoring quoted sections
139/// so an embedded newline inside quotes does not start a new record.
140///
141/// A record ends at LF, CRLF or a lone CR outside quotes, as in the readers.
142/// Splitting on LF alone read a CR-only file as one header record (#1097).
143///
144/// As in the readers, a quote opens a quoted section only at the start of a
145/// field, here a record start or right after `delimiter`; anywhere else it is
146/// a literal character. Opening one at any quote read a header such as
147/// `id;height"in;note` as never ending. Taking any candidate as a field
148/// boundary did the same to `id,height;"in` read with comma (#1172 review),
149/// so each candidate splits the text by its own rule.
150///
151/// The returned flag is true when the final record was not terminated by a
152/// newline outside quotes — i.e. the text ends mid-record (or exactly at a
153/// record's end with no trailing newline, which a caller sampling a fixed
154/// byte budget cannot distinguish from a cut).
155fn split_sample_records(bytes: &[u8], delimiter: u8, quote: u8) -> (Vec<&[u8]>, bool) {
156    let mut records = Vec::new();
157    let mut in_quotes = false;
158    let mut start = 0usize;
159    let mut i = 0usize;
160    while i < bytes.len() {
161        match bytes[i] {
162            byte if byte == quote && in_quotes => {
163                if bytes.get(i + 1) == Some(&quote) {
164                    i += 1; // escaped quote inside a quoted section
165                } else {
166                    in_quotes = false;
167                }
168            }
169            // The reader skips leading record terminators before opening a
170            // field, even if the configured quote is itself CR or LF.
171            byte if byte == quote
172                && (i == start || bytes[i - 1] == delimiter)
173                && !(i == start && matches!(byte, b'\r' | b'\n')) =>
174            {
175                in_quotes = true;
176            }
177            b'\n' if !in_quotes => {
178                let end = if i > start && bytes[i - 1] == b'\r' {
179                    i - 1
180                } else {
181                    i
182                };
183                records.push(&bytes[start..end]);
184                start = i + 1;
185            }
186            b'\r' if !in_quotes && bytes.get(i + 1) != Some(&b'\n') => {
187                records.push(&bytes[start..i]);
188                start = i + 1;
189            }
190            _ => {}
191        }
192        i += 1;
193    }
194    let last_unterminated = start < bytes.len();
195    if last_unterminated {
196        let mut end = bytes.len();
197        while end > start && bytes[end - 1] == b'\r' {
198            end -= 1;
199        }
200        records.push(&bytes[start..end]);
201    }
202    (records, last_unterminated)
203}
204
205/// Delimiters detection chooses between, in the order that breaks a tie.
206const CANDIDATES: [u8; 4] = *b",;\t|";
207
208/// Bytes delimiter detection reads when the header record ends within them.
209const DETECTION_SAMPLE_BYTES: usize = 4096;
210
211/// Most bytes delimiter detection reads to see the whole header record.
212const MAX_HEADER_SAMPLE_BYTES: usize = 1 << 20;
213
214/// The bytes of a detection head. A UTF-8 BOM marks the encoding, as in the
215/// readers, and is no part of the first record: left in, a BOM followed by a
216/// blank line was scored as a one-field header and outvoted the real one.
217/// Keep quote bytes intact: a configured non-ASCII byte may delimit fields
218/// whose decoded contents are valid UTF-8.
219fn head_bytes(head: &[u8]) -> &[u8] {
220    head.strip_prefix(b"\xEF\xBB\xBF").unwrap_or(head)
221}
222
223/// Preserve detection's whitespace rule without decoding its CSV syntax.
224fn record_is_blank(record: &[u8]) -> bool {
225    String::from_utf8_lossy(record).trim().is_empty()
226}
227
228/// Whether `head`, the start of a source, holds its header record up to the
229/// newline that ends it, as every candidate splits it. The readers take the
230/// first non-blank record as the header.
231fn header_is_complete(head: &[u8], quote: u8) -> bool {
232    let head = head_bytes(head);
233    CANDIDATES.iter().all(|&delimiter| {
234        let (records, last_unterminated) = split_sample_records(head, delimiter, quote);
235        records
236            .iter()
237            .position(|record| !record_is_blank(record))
238            .is_some_and(|index| index + 1 < records.len() || !last_unterminated)
239    })
240}
241
242/// Read the head of a source that delimiter detection scores: 4 KB, or more
243/// when the header record runs past that, doubling up to 1 MiB. Each read
244/// takes one byte past its target, so the returned flag says for certain
245/// whether the head was cut before the end of the source.
246///
247/// A fixed 4 KB window scored the start of a longer header as the whole of
248/// it; no candidate split the fragment, and the file was read as one column
249/// with no error (#1098).
250fn read_detection_head<R: Read>(reader: &mut R, quote: u8) -> std::io::Result<(Vec<u8>, bool)> {
251    let mut head = Vec::new();
252    let mut target = DETECTION_SAMPLE_BYTES;
253    loop {
254        let wanted = (target + 1 - head.len()) as u64;
255        reader.by_ref().take(wanted).read_to_end(&mut head)?;
256        let cut = head.len() > target;
257        if !cut || target == MAX_HEADER_SAMPLE_BYTES || header_is_complete(&head, quote) {
258            return Ok((head, cut));
259        }
260        target = (target * 2).min(MAX_HEADER_SAMPLE_BYTES);
261    }
262}
263
264const HEADER_TOO_LONG: &str = "The first 1 MiB of the CSV source holds no complete header record, so its delimiter could not be detected.";
265const HEADER_TOO_LONG_HINT: &str = "Set the delimiter explicitly (csv_delimiter on Profiler or dataprof.profile) to read this file.";
266
267/// Detect the delimiter of a source and return it with the bytes read to
268/// detect it, so a caller that cannot rewind the source can chain them back
269/// in front of the rest.
270///
271/// Detection reads 4 KB, or up to 1 MiB when the header record is longer. A
272/// source with no complete header in its first 1 MiB is an error, never a
273/// guess. Quoted fields use the default double-quote character.
274pub fn detect_delimiter_with_head<R: Read>(
275    reader: &mut R,
276) -> Result<(u8, Vec<u8>), DataProfilerError> {
277    let (detection, head) = detect_delimiter_detailed_with_head(reader)?;
278    Ok((detection_byte(&detection), head))
279}
280
281/// [`detect_delimiter_with_head`], returning what detection saw with the
282/// delimiter it chose: the header's field count and a candidate that read the
283/// sampled records as well, so a report can say when the choice was not
284/// clear (#1100).
285pub fn detect_delimiter_detailed_with_head<R: Read>(
286    reader: &mut R,
287) -> Result<(DelimiterDetection, Vec<u8>), DataProfilerError> {
288    detect_delimiter_with_quote(reader, b'"')
289}
290
291/// Use the reader's quote grammar throughout sampling and scoring.
292fn detect_delimiter_with_quote<R: Read>(
293    reader: &mut R,
294    quote: u8,
295) -> Result<(DelimiterDetection, Vec<u8>), DataProfilerError> {
296    let (head, cut) = read_detection_head(reader, quote)?;
297    match score_head(&head, cut, quote) {
298        Some(detection) => Ok((detection, head)),
299        None => Err(DataProfilerError::csv_structure(
300            HEADER_TOO_LONG,
301            HEADER_TOO_LONG_HINT,
302        )),
303    }
304}
305
306/// [`detect_delimiter`], returning what detection saw with the delimiter it
307/// chose (see [`detect_delimiter_detailed_with_head`]).
308pub fn detect_delimiter_detailed<R: Read>(mut reader: R) -> std::io::Result<DelimiterDetection> {
309    let (head, cut) = read_detection_head(&mut reader, b'"')?;
310    score_head(&head, cut, b'"').ok_or_else(|| {
311        std::io::Error::new(
312            std::io::ErrorKind::InvalidData,
313            format!("{HEADER_TOO_LONG} {HEADER_TOO_LONG_HINT}"),
314        )
315    })
316}
317
318/// The delimiter byte of a detection. Detection only ever chooses one of
319/// the ASCII candidates, so the recorded character is that byte.
320pub fn detection_byte(detection: &DelimiterDetection) -> u8 {
321    detection.delimiter.as_bytes()[0]
322}
323
324/// Detect the most likely CSV delimiter by sampling 4 KB of data, or more
325/// when the header record is longer (see [`detect_delimiter_with_head`]).
326///
327/// The sample is split into logical records with double-quote handling, so quoted
328/// fields containing embedded newlines or delimiter characters do not skew
329/// detection. Tests comma, semicolon, tab, and pipe over the first 5 non-blank
330/// records, measured against the first one, the header: it sets the column
331/// count, and the readers cut or pad a wider or narrower record to it, or
332/// reject it under strict parsing. A delimiter that splits the header always
333/// beats one that does not; then more records agreeing with the header win,
334/// then more records at exactly the header's width, then a wider header, and
335/// comma wins a tie. A record agrees when it reads at the header's width
336/// without losing a value: the same field count, or extra fields that are all
337/// empty, as an exporter's trailing delimiter leaves.
338///
339/// Judging by the data rows alone let a delimiter that never occurs in the
340/// header win, and the readers then cut every row back to its first field: a
341/// one-column file of `123|456` keys was profiled as `123` with fabricated
342/// duplicates (#1092).
343///
344/// A source with no complete header record in its first 1 MiB is an
345/// `InvalidData` error.
346pub fn detect_delimiter<R: Read>(reader: R) -> std::io::Result<u8> {
347    detect_delimiter_detailed(reader).map(|detection| detection_byte(&detection))
348}
349
350/// Score the candidate delimiters over a detection head, or `None` when the
351/// head was cut before any candidate saw a whole header record.
352///
353/// A runner-up is a candidate that also splits the header and has as many
354/// records agreeing with it. Exact width and header width still pick between
355/// the two, but they do not make the reading clear: `id;amount,EUR` with rows
356/// `1;10,50;` reads at two fields with either comma or semicolon (#1100).
357fn score_head(head: &[u8], cut: bool, quote: u8) -> Option<DelimiterDetection> {
358    let head = head_bytes(head);
359    // (splits the header into >1 field, records agreeing with the header,
360    //  records at exactly the header's width, header field count) — compared
361    //  lexicographically, first wins ties. Exact width only breaks a tie: a
362    //  candidate that reads every record as it stands beats one that needs
363    //  trailing empty fields cut to agree.
364    let mut scores: Vec<((bool, usize, usize, usize), u8)> = Vec::new();
365    // The scored records of each candidate, in the order of `scores`.
366    let mut sampled: Vec<Vec<&[u8]>> = Vec::new();
367
368    for delimiter in CANDIDATES {
369        let (mut records, last_unterminated) = split_sample_records(head, delimiter, quote);
370        // A head cut mid-record ends in a fragment; never score it. A head
371        // ending on a clean newline keeps all its records.
372        if cut && last_unterminated {
373            records.pop();
374        }
375        // Blank records carry no delimiter signal: a header followed by
376        // trailing blank lines must not outvote the true delimiter with
377        // single-field "records" (gh #559).
378        let scored: Vec<&[u8]> = records
379            .into_iter()
380            .filter(|record| !record_is_blank(record))
381            .take(5)
382            .collect();
383        let Some((header, rows)) = scored.split_first() else {
384            // A cut head with no whole header has nothing to score for this
385            // candidate: its header is the fragment, or lies past the head.
386            // A whole source with no header is blank to every candidate.
387            if cut {
388                continue;
389            }
390            return Some(DelimiterDetection::new(b',', 0, None));
391        };
392
393        let (header_fields, _) = count_fields(header, delimiter, quote);
394        let (mut agreeing, mut exact) = (1, 1);
395        for record in rows {
396            let (fields, trailing_empty) = count_fields(record, delimiter, quote);
397            if agrees_with_header(fields, trailing_empty, header_fields) {
398                agreeing += 1;
399            }
400            if fields == header_fields {
401                exact += 1;
402            }
403        }
404
405        scores.push((
406            (header_fields > 1, agreeing, exact, header_fields),
407            delimiter,
408        ));
409        sampled.push(scored);
410    }
411
412    // The highest score wins, and the earlier candidate a tie.
413    let pick = |candidates: &mut dyn Iterator<Item = ((bool, usize, usize, usize), u8)>| {
414        candidates.fold(
415            None,
416            |best: Option<((bool, usize, usize, usize), u8)>, entry| match best {
417                Some(held) if entry.0 <= held.0 => Some(held),
418                _ => Some(entry),
419            },
420        )
421    };
422    let (score, delimiter) = pick(&mut scores.iter().copied())?;
423    let (splits, agreeing, _, header_fields) = score;
424    let runner_up = pick(
425        &mut scores.iter().copied().filter(|&(other, other_delimiter)| {
426            splits && other_delimiter != delimiter && other.0 && other.1 == agreeing
427        }),
428    )
429    .map(|(_, other_delimiter)| other_delimiter);
430    let unlisted = if splits {
431        None
432    } else {
433        scores
434            .iter()
435            .position(|&(_, candidate)| candidate == delimiter)
436            .and_then(|index| unlisted_delimiter(&sampled[index], quote))
437    };
438    Some(
439        DelimiterDetection::new(delimiter, header_fields, runner_up)
440            .with_unlisted_delimiter(unlisted),
441    )
442}
443
444/// Punctuation that can delimit a file detection does not try: ASCII
445/// punctuation except the candidates and `.`, `-` and `_`, which
446/// sit inside single values (`first.last`, `2024-01-02`, `event_0`).
447/// The active quote character is excluded when scoring these bytes.
448const UNLISTED_DELIMITERS: &[u8] = b"!\"#$%&'()*+/:<=>?@[\\]^`{}~";
449
450/// A character outside the candidates that reads the sampled records as a
451/// table when no candidate splits the header: it occurs outside quotes in the
452/// header, and as often in every sampled record. `None` without a data record
453/// to compare, and for an ordinary one-column file, where nothing does.
454///
455/// `id^amount` over `1^10` is read as one column; this names `^` so the
456/// finding can say which delimiter to set (#1100).
457fn unlisted_delimiter(records: &[&[u8]], quote: u8) -> Option<u8> {
458    let (header, rows) = records.split_first()?;
459    if rows.is_empty() {
460        return None;
461    }
462    UNLISTED_DELIMITERS.iter().copied().find(|&byte| {
463        if byte == quote {
464            return false;
465        }
466        let occurrences = |record: &[u8]| count_fields(record, byte, quote).0 - 1;
467        let in_header = occurrences(header);
468        in_header > 0 && rows.iter().all(|row| occurrences(row) == in_header)
469    })
470}
471
472/// Detect the most likely CSV delimiter from a file path.
473pub fn detect_delimiter_from_path(path: &Path) -> std::io::Result<u8> {
474    let file = std::fs::File::open(path)?;
475    detect_delimiter(std::io::BufReader::new(file))
476}
477
478/// Analyze CSV data from any `Read` source using streaming statistics.
479///
480/// This is the primary generic entry point for both file-based and stream-based
481/// CSV profiling. Data is processed incrementally through [`StreamingColumnCollection`]
482/// with bounded memory, rather than accumulating all rows in a `HashMap`.
483///
484/// Returns `(column_profiles, streaming_stats, rows_read)`.
485pub fn analyze_csv_from_reader<R: Read>(
486    reader: R,
487    config: &CsvParserConfig,
488) -> Result<
489    (
490        Vec<ColumnProfile>,
491        StreamingColumnCollection,
492        usize,
493        Vec<String>,
494    ),
495    DataProfilerError,
496> {
497    let scan = scan_csv(reader, config)?;
498    let profiles = profile_builder::profiles_from_streaming(&scan.column_stats, false, false, None);
499
500    Ok((
501        profiles,
502        scan.column_stats,
503        scan.rows_read,
504        scan.header_names,
505    ))
506}
507
508/// What one pass over CSV input read, and how much of the input that was.
509struct CsvScan {
510    column_stats: StreamingColumnCollection,
511    header_names: Vec<String>,
512    rows_read: usize,
513    /// Records whose field count differs from the header's, padded or cut to
514    /// the header width before profiling.
515    ragged_rows: usize,
516    /// Set when `max_rows` stopped the pass with records left in the input.
517    truncation: Option<TruncationReason>,
518    /// The delimiter the pass split records on and how it was chosen.
519    csv_metadata: CsvSourceMetadata,
520}
521
522fn scan_csv<R: Read>(reader: R, config: &CsvParserConfig) -> Result<CsvScan, DataProfilerError> {
523    let mut csv_builder = ReaderBuilder::new();
524    csv_builder.has_headers(config.has_header);
525    csv_builder.flexible(config.flexible);
526    csv_builder.quote(config.quote_char);
527    if config.trim_whitespace {
528        csv_builder.trim(csv::Trim::All);
529    }
530    // The byte is kept as it was given: the metadata holds it as a character,
531    // and a configured byte above 0x7F would not read back as itself.
532    let (boxed_reader, csv_metadata, delimiter): (Box<dyn std::io::Read + '_>, _, _) = match config
533        .delimiter
534    {
535        Some(delimiter) => (
536            Box::new(reader),
537            CsvSourceMetadata::configured(delimiter),
538            delimiter,
539        ),
540        None => {
541            let mut reader = reader;
542            let (detection, head) = detect_delimiter_with_quote(&mut reader, config.quote_char)?;
543            let delimiter = detection_byte(&detection);
544            (
545                Box::new(std::io::Cursor::new(head).chain(reader)),
546                CsvSourceMetadata::detected(detection),
547                delimiter,
548            )
549        }
550    };
551    csv_builder.delimiter(delimiter);
552
553    let mut csv_reader = csv_builder.from_reader(boxed_reader);
554
555    let header_names: Vec<String> = if config.has_header {
556        let headers = csv_reader.headers()?;
557        let names: Vec<String> = headers.iter().map(|header| header.to_string()).collect();
558        // Reject duplicate headers before profiling: process_record keys on name,
559        // so a repeat would merge two source columns into one inflated profile.
560        dataprof_core::validate_unique_column_names(&names, "CSV header")?;
561        names
562    } else {
563        let headers = csv_reader.headers()?;
564        (0..headers.len())
565            .map(|index| format!("column_{index}"))
566            .collect()
567    };
568
569    let mut column_stats = StreamingColumnCollection::new();
570    // A header declares the columns even when no data records follow it.
571    column_stats.init_columns(&header_names);
572    let mut rows_read = 0;
573    let mut ragged_rows = 0;
574    let mut truncation = None;
575
576    for result in csv_reader.records() {
577        if let Some(max_rows) = config.max_rows
578            && rows_read >= max_rows
579        {
580            // A record past the cap means the cap, not the end of the input,
581            // stopped the pass. That record is neither profiled nor judged.
582            truncation = Some(TruncationReason::MaxRows(max_rows as u64));
583            break;
584        }
585
586        let record = result?;
587        let mut values: Vec<String> = record.iter().map(|value| value.to_string()).collect();
588
589        let header_len = header_names.len();
590        if values.len() != header_len {
591            ragged_rows += 1;
592        }
593        if values.len() < header_len {
594            values.resize(header_len, String::new());
595        } else if values.len() > header_len {
596            values.truncate(header_len);
597        }
598
599        column_stats.process_record(&header_names, values);
600        rows_read += 1;
601    }
602
603    Ok(CsvScan {
604        column_stats,
605        header_names,
606        rows_read,
607        ragged_rows,
608        truncation,
609        csv_metadata,
610    })
611}
612
613/// Analyze a CSV file, returning a full [`ProfileReport`].
614///
615/// Opens the file, reads it as [`analyze_csv_from_reader`] does, and wraps the
616/// result with file metadata, ISO 8000/25012 data quality metrics, and the
617/// scan's own record: how many rows were ragged, and whether `max_rows` left
618/// rows unread.
619pub fn analyze_csv_file(
620    file_path: &Path,
621    config: &CsvParserConfig,
622) -> Result<ProfileReport, DataProfilerError> {
623    let metadata = std::fs::metadata(file_path).map_err(|error| map_io_error(file_path, error))?;
624    dataprof_core::binary_input::reject_binary_input(file_path)?;
625    let start = std::time::Instant::now();
626
627    let file = std::fs::File::open(file_path).map_err(|error| map_io_error(file_path, error))?;
628    let buf_reader = std::io::BufReader::new(file);
629
630    let scan = scan_csv(buf_reader, config)?;
631    let column_stats = &scan.column_stats;
632    let column_profiles =
633        profile_builder::profiles_from_streaming(column_stats, false, false, None);
634
635    let file_source = DataSource::File {
636        path: file_path.display().to_string(),
637        format: FileFormat::Csv,
638        size_bytes: metadata.len(),
639        modified_at: None,
640        parquet_metadata: None,
641        csv_metadata: Some(scan.csv_metadata.clone()),
642    };
643
644    let sample_columns = profile_builder::quality_check_samples(column_stats);
645    let scan_time_ms = start.elapsed().as_millis();
646    let num_columns = column_profiles.len();
647
648    let mut execution = ExecutionMetadata::new(scan.rows_read, num_columns, scan_time_ms)
649        .with_engine("csv")
650        .with_ragged_row_count(scan.ragged_rows);
651    if let Some(reason) = scan.truncation {
652        execution = execution.with_truncation(reason);
653    }
654
655    // This path takes no analysis options: it profiles with the defaults,
656    // so it records them rather than leaving its settings unknown.
657    Ok(ReportAssembler::new(file_source, execution)
658        .with_analysis_options(&AnalysisOptions::default())
659        .columns(column_profiles)
660        .with_quality_data(sample_columns)
661        .with_row_duplicates(column_stats.row_duplicate_summary())
662        .with_row_completeness(column_stats.row_completeness_summary())
663        .build())
664}
665
666fn map_io_error(file_path: &Path, error: std::io::Error) -> DataProfilerError {
667    if error.kind() == std::io::ErrorKind::NotFound {
668        DataProfilerError::FileNotFound {
669            path: file_path.display().to_string(),
670        }
671    } else {
672        DataProfilerError::from(error)
673    }
674}
675
676#[cfg(test)]
677mod tests {
678    use super::*;
679    use dataprof_core::DataType;
680    use std::io::{Cursor, Write};
681    use tempfile::NamedTempFile;
682
683    fn write_csv(content: &str) -> NamedTempFile {
684        let mut file = NamedTempFile::new().unwrap();
685        write!(file, "{}", content).unwrap();
686        file.flush().unwrap();
687        file
688    }
689
690    #[test]
691    fn test_analyze_csv_from_reader_basic() {
692        let data = b"name,age\nAlice,25\nBob,30\n";
693        let cursor = Cursor::new(data.as_ref());
694        let config = CsvParserConfig::default();
695
696        let (profiles, _stats, rows, _) = analyze_csv_from_reader(cursor, &config).unwrap();
697        assert_eq!(rows, 2);
698        assert_eq!(profiles.len(), 2);
699
700        let age = profiles
701            .iter()
702            .find(|profile| profile.name == "age")
703            .unwrap();
704        assert_eq!(age.total_count, 2);
705        assert_eq!(age.data_type, DataType::Integer);
706    }
707
708    #[test]
709    fn test_analyze_csv_from_reader_with_max_rows() {
710        let data = b"val\n1\n2\n3\n4\n5\n";
711        let cursor = Cursor::new(data.as_ref());
712        let config = CsvParserConfig::default().max_rows(Some(3));
713
714        let (_profiles, _stats, rows, _) = analyze_csv_from_reader(cursor, &config).unwrap();
715        assert_eq!(rows, 3);
716    }
717
718    #[test]
719    fn test_analyze_csv_from_reader_strict_rejects_ragged() {
720        let data = b"a,b\n1,2\n3,4,5\n";
721        let cursor = Cursor::new(data.as_ref());
722        let config = CsvParserConfig::strict();
723
724        assert!(analyze_csv_from_reader(cursor, &config).is_err());
725    }
726
727    #[test]
728    fn test_analyze_csv_from_reader_flexible_handles_ragged() {
729        let data = b"a,b\n1,2\n3,4,5\n6\n";
730        let cursor = Cursor::new(data.as_ref());
731        let config = CsvParserConfig::default();
732
733        let (profiles, _stats, rows, _) = analyze_csv_from_reader(cursor, &config).unwrap();
734        assert_eq!(rows, 3);
735        assert!(!profiles.is_empty());
736    }
737
738    #[test]
739    fn test_analyze_csv_from_reader_custom_delimiter() {
740        let data = b"a;b\n1;2\n3;4\n";
741        let cursor = Cursor::new(data.as_ref());
742        let config = CsvParserConfig::default().with_delimiter(b';');
743
744        let (profiles, _stats, rows, _) = analyze_csv_from_reader(cursor, &config).unwrap();
745        assert_eq!(rows, 2);
746        assert_eq!(profiles.len(), 2);
747    }
748
749    #[test]
750    fn test_analyze_csv_file_returns_quality_report() {
751        let csv = write_csv("x,y\n1,a\n2,b\n3,c\n");
752        let config = CsvParserConfig::default();
753        let report = analyze_csv_file(csv.path(), &config).unwrap();
754
755        assert_eq!(report.column_profiles.len(), 2);
756        assert_eq!(report.execution.rows_processed, 3);
757        assert!(report.quality_score().unwrap() >= 0.0);
758        assert!(report.quality_score().unwrap() <= 100.0);
759    }
760
761    #[test]
762    fn test_csv_auto_detects_delimiter_from_reader() {
763        let data = b"name;age\nAlice;25\nBob;30\n";
764        let cursor = Cursor::new(data.as_ref());
765        let config = CsvParserConfig::default();
766
767        let (profiles, _stats, rows, _) = analyze_csv_from_reader(cursor, &config).unwrap();
768        assert_eq!(rows, 2);
769        assert_eq!(profiles.len(), 2);
770
771        let age = profiles
772            .iter()
773            .find(|profile| profile.name == "age")
774            .unwrap();
775        assert_eq!(age.total_count, 2);
776    }
777
778    #[test]
779    fn test_analyze_csv_basic() {
780        let csv = write_csv("name,age\nAlice,25\nBob,30\n");
781        let config = CsvParserConfig::default();
782        let report = analyze_csv_file(csv.path(), &config).unwrap();
783        let profiles = &report.column_profiles;
784        assert_eq!(profiles.len(), 2);
785
786        let names: Vec<&str> = profiles
787            .iter()
788            .map(|profile| profile.name.as_str())
789            .collect();
790        assert!(names.contains(&"name"));
791        assert!(names.contains(&"age"));
792
793        let age = profiles
794            .iter()
795            .find(|profile| profile.name == "age")
796            .unwrap();
797        assert_eq!(age.total_count, 2);
798        assert_eq!(age.null_count, 0);
799    }
800
801    #[test]
802    fn test_analyze_csv_with_nulls() {
803        let csv = write_csv("name,age,email\nAlice,25,a@b.com\nBob,,\nCharlie,30,c@d.com\n");
804        let config = CsvParserConfig::default();
805        let report = analyze_csv_file(csv.path(), &config).unwrap();
806        let profiles = &report.column_profiles;
807
808        let age = profiles
809            .iter()
810            .find(|profile| profile.name == "age")
811            .unwrap();
812        assert_eq!(age.total_count, 3);
813        assert_eq!(age.null_count, 1);
814
815        let email = profiles
816            .iter()
817            .find(|profile| profile.name == "email")
818            .unwrap();
819        assert_eq!(email.null_count, 1);
820    }
821
822    #[test]
823    fn test_analyze_csv_file_returns_quality_report_legacy() {
824        let csv = write_csv("x,y\n1,a\n2,b\n3,c\n");
825        let config = CsvParserConfig::default();
826        let report = analyze_csv_file(csv.path(), &config).unwrap();
827
828        assert_eq!(report.column_profiles.len(), 2);
829        assert_eq!(report.execution.rows_processed, 3);
830        assert_eq!(report.execution.columns_detected, 2);
831        assert!(report.quality_score().unwrap() >= 0.0);
832        assert!(report.quality_score().unwrap() <= 100.0);
833    }
834
835    #[test]
836    fn test_analyze_csv_file_header_only_preserves_declared_columns() {
837        let csv = write_csv("name,age\n");
838        let config = CsvParserConfig::default();
839        let report = analyze_csv_file(csv.path(), &config).unwrap();
840
841        assert_eq!(
842            report
843                .column_profiles
844                .iter()
845                .map(|column| (column.name.as_str(), column.data_type.clone()))
846                .collect::<Vec<_>>(),
847            vec![("name", DataType::String), ("age", DataType::String)]
848        );
849        assert_eq!(report.execution.rows_processed, 0);
850        assert!(report.quality.is_some());
851    }
852
853    #[test]
854    fn test_analyze_csv_file_empty_file_has_no_columns() {
855        let csv = write_csv("");
856        let config = CsvParserConfig::default();
857        let report = analyze_csv_file(csv.path(), &config).unwrap();
858
859        assert!(report.column_profiles.is_empty());
860        assert_eq!(report.execution.rows_processed, 0);
861        assert_eq!(report.execution.columns_detected, 0);
862        assert_eq!(
863            report.quality_status,
864            dataprof_runtime::QualityAnalysisStatus::Computed
865        );
866        assert!(report.quality.is_some());
867        assert!(report.quality_score().is_none());
868    }
869
870    #[test]
871    fn test_analyze_csv_with_max_rows_small_file() {
872        let csv = write_csv("val\n1\n2\n3\n4\n5\n");
873        let config = CsvParserConfig::default();
874        let report = analyze_csv_file(csv.path(), &config).unwrap();
875
876        assert_eq!(report.execution.rows_processed, 5);
877        assert_eq!(report.execution.columns_detected, 1);
878        assert!((report.execution.sampling_ratio.unwrap_or(1.0) - 1.0).abs() < 0.01);
879    }
880
881    #[test]
882    fn a_row_cap_that_leaves_rows_unread_is_a_truncation() {
883        let csv = write_csv("val\n1\n2\n3\n4\n5\n");
884        let config = CsvParserConfig::default().max_rows(Some(2));
885        let report = analyze_csv_file(csv.path(), &config).unwrap();
886
887        assert_eq!(report.execution.rows_processed, 2);
888        assert!(!report.execution.source_exhausted);
889        assert_eq!(
890            report.execution.truncation_reason,
891            Some(TruncationReason::MaxRows(2))
892        );
893    }
894
895    #[test]
896    fn a_row_cap_at_the_last_row_is_a_complete_scan() {
897        let csv = write_csv("val\n1\n2\n");
898        let config = CsvParserConfig::default().max_rows(Some(2));
899        let report = analyze_csv_file(csv.path(), &config).unwrap();
900
901        assert_eq!(report.execution.rows_processed, 2);
902        assert!(report.execution.source_exhausted);
903        assert_eq!(report.execution.truncation_reason, None);
904    }
905
906    #[test]
907    fn short_and_long_rows_are_counted_as_ragged() {
908        let csv = write_csv("a,b,c\n1,2,3\n4,5\n6,7,8,9\n10,11,12\n");
909        let report = analyze_csv_file(csv.path(), &CsvParserConfig::default()).unwrap();
910
911        assert_eq!(report.execution.rows_processed, 4);
912        assert_eq!(report.execution.ragged_row_count, 2);
913    }
914
915    #[test]
916    fn a_ragged_row_past_the_row_cap_is_not_counted() {
917        let csv = write_csv("a,b,c\n1,2,3\n4,5\n");
918        let config = CsvParserConfig::default().max_rows(Some(1));
919        let report = analyze_csv_file(csv.path(), &config).unwrap();
920
921        assert_eq!(report.execution.ragged_row_count, 0);
922        assert_eq!(
923            report.execution.truncation_reason,
924            Some(TruncationReason::MaxRows(1))
925        );
926    }
927
928    #[test]
929    fn test_analyze_csv_numeric_types_detected() {
930        let csv = write_csv("int_col,float_col,str_col\n1,1.5,hello\n2,2.5,world\n3,3.5,foo\n");
931        let config = CsvParserConfig::default();
932        let report = analyze_csv_file(csv.path(), &config).unwrap();
933        let profiles = &report.column_profiles;
934
935        let int_col = profiles
936            .iter()
937            .find(|profile| profile.name == "int_col")
938            .unwrap();
939        assert!(matches!(int_col.data_type, DataType::Integer));
940
941        let float_col = profiles
942            .iter()
943            .find(|profile| profile.name == "float_col")
944            .unwrap();
945        assert!(matches!(float_col.data_type, DataType::Float));
946
947        let str_col = profiles
948            .iter()
949            .find(|profile| profile.name == "str_col")
950            .unwrap();
951        assert!(matches!(str_col.data_type, DataType::String));
952    }
953
954    #[test]
955    fn test_analyze_csv_strict_rejects_ragged_rows() {
956        let csv = write_csv("a,b\n1,2\n3,4,5\n");
957        let config = CsvParserConfig::strict();
958        assert!(analyze_csv_file(csv.path(), &config).is_err());
959    }
960
961    #[test]
962    fn test_analyze_csv_file_handles_ragged_rows() {
963        let csv = write_csv("a,b\n1,2\n3,4,5\n6\n");
964        let config = CsvParserConfig::default();
965        let report = analyze_csv_file(csv.path(), &config).unwrap();
966        assert!(!report.column_profiles.is_empty());
967    }
968
969    #[test]
970    fn test_analyze_csv_with_quality_metrics() {
971        let csv = write_csv(
972            "name,age\n\
973             Alice,25\nBob,\nCharlie,30\nDave,\nEve,28\n\
974             Frank,\nGrace,35\nHeidi,40\nIvan,\nJudy,22\n",
975        );
976        let config = CsvParserConfig::default();
977        let report = analyze_csv_file(csv.path(), &config).unwrap();
978
979        let age = report
980            .column_profiles
981            .iter()
982            .find(|profile| profile.name == "age")
983            .unwrap();
984        assert_eq!(age.null_count, 4);
985        assert_eq!(age.total_count, 10);
986        assert!(report.quality_score().unwrap() >= 0.0);
987    }
988
989    /// What detection records beside the delimiter it picks (#1100): a
990    /// runner-up only on a structural tie, and an unlisted delimiter only when
991    /// it splits the header and every sampled record alike.
992    #[test]
993    fn detection_records_a_tie_and_an_unlisted_delimiter_only_when_present() {
994        // (case, source, delimiter, header fields, runner-up, unlisted)
995        type Case<'a> = (
996            &'a str,
997            &'a [u8],
998            &'a str,
999            usize,
1000            Option<&'a str>,
1001            Option<&'a str>,
1002        );
1003        let cases: [Case; 9] = [
1004            (
1005                "tie",
1006                b"id;amount,EUR\n1;10,50;\n2;20,75;\n",
1007                ",",
1008                2,
1009                Some(";"),
1010                None,
1011            ),
1012            ("clear comma", b"a,b,c\n1,2,3\n4,5,6\n", ",", 3, None, None),
1013            (
1014                "clear semicolon",
1015                b"a;b\n1,5;2\n3,5;4\n",
1016                ";",
1017                2,
1018                None,
1019                None,
1020            ),
1021            (
1022                "unlisted caret",
1023                b"id^amount\n1^10\n2^20\n",
1024                ",",
1025                1,
1026                None,
1027                Some("^"),
1028            ),
1029            ("one column", b"value\n1\n2\n", ",", 1, None, None),
1030            (
1031                "dot inside values",
1032                b"name.first\njohn.doe\njane.roe\n",
1033                ",",
1034                1,
1035                None,
1036                None,
1037            ),
1038            ("header lacks it", b"ratio\n1/2\n3/4\n", ",", 1, None, None),
1039            ("a record lacks it", b"a:b\n1:2\n3\n", ",", 1, None, None),
1040            ("header only", b"a^b\n", ",", 1, None, None),
1041        ];
1042        let mut wrong = Vec::new();
1043        for (case, data, delimiter, header_fields, runner_up, unlisted) in cases {
1044            let detection = detect_delimiter_detailed(Cursor::new(data)).unwrap();
1045            let seen = (
1046                detection.delimiter.as_str(),
1047                detection.header_fields,
1048                detection.runner_up.as_deref(),
1049                detection.unlisted_delimiter.as_deref(),
1050            );
1051            if seen != (delimiter, header_fields, runner_up, unlisted) {
1052                wrong.push(format!("{case}: {seen:?}"));
1053            }
1054        }
1055        assert!(wrong.is_empty(), "{}", wrong.join("\n"));
1056    }
1057
1058    #[test]
1059    fn test_detect_delimiter_comma() {
1060        let data = b"name,age,city\nAlice,25,NYC\nBob,30,London\n";
1061        assert_eq!(detect_delimiter(Cursor::new(data.as_ref())).unwrap(), b',');
1062    }
1063
1064    #[test]
1065    fn test_detect_delimiter_semicolon() {
1066        let data = b"name;age;city\nAlice;25;NYC\nBob;30;London\n";
1067        assert_eq!(detect_delimiter(Cursor::new(data.as_ref())).unwrap(), b';');
1068    }
1069
1070    #[test]
1071    fn test_detect_delimiter_pipe() {
1072        let data = b"name|age|city\nAlice|25|NYC\nBob|30|London\n";
1073        assert_eq!(detect_delimiter(Cursor::new(data.as_ref())).unwrap(), b'|');
1074    }
1075
1076    #[test]
1077    fn test_detect_delimiter_tab() {
1078        let data = b"name\tage\tcity\nAlice\t25\tNYC\nBob\t30\tLondon\n";
1079        assert_eq!(detect_delimiter(Cursor::new(data.as_ref())).unwrap(), b'\t');
1080    }
1081
1082    #[test]
1083    fn test_detect_delimiter_from_path_semicolon() {
1084        let csv = write_csv("id;name;salary\n1;Alice;50000\n2;Bob;60000\n");
1085        assert_eq!(detect_delimiter_from_path(csv.path()).unwrap(), b';');
1086    }
1087
1088    #[test]
1089    fn test_detect_delimiter_blank_records_do_not_outvote_header() {
1090        // A header-only file with trailing blank lines: the blank records are
1091        // single-field for every candidate, so counting them made `;` win on
1092        // perfect consistency and collapse the whole header into one column
1093        // (gh #559).
1094        let data = b"name,age\n\n\n";
1095        assert_eq!(detect_delimiter(Cursor::new(data.as_ref())).unwrap(), b',');
1096
1097        let semicolon = b"name;age\n\n";
1098        assert_eq!(
1099            detect_delimiter(Cursor::new(semicolon.as_ref())).unwrap(),
1100            b';'
1101        );
1102    }
1103
1104    #[test]
1105    fn test_detect_delimiter_quoted_fields() {
1106        let data = b"name;desc;val\n\"hello;world\";foo;1\nbar;baz;2\n";
1107        assert_eq!(detect_delimiter(Cursor::new(data.as_ref())).unwrap(), b';');
1108    }
1109
1110    #[test]
1111    fn test_detect_delimiter_quoted_embedded_newline() {
1112        // A quoted cell containing a newline must not break record splitting:
1113        // before the quote-aware sniffer this file was detected as semicolon
1114        // (zero occurrences, "perfectly consistent" single-field records) and
1115        // profiled as a single column named "id,comment".
1116        let data = b"id,comment\n1,\"line one\nline two\"\n2,\"normal\"\n3,\"has \"\"quotes\"\" inside\"\n";
1117        assert_eq!(detect_delimiter(Cursor::new(data.as_ref())).unwrap(), b',');
1118    }
1119
1120    #[test]
1121    fn test_detect_delimiter_absent_candidate_never_beats_real_one() {
1122        // One ragged record makes comma imperfect (consistency 3/4); an absent
1123        // delimiter is uniformly single-field (4/4) and must still lose.
1124        let data = b"a,b,c\n1,2,3\nragged line without delimiters\n4,5,6\n";
1125        assert_eq!(detect_delimiter(Cursor::new(data.as_ref())).unwrap(), b',');
1126    }
1127
1128    #[test]
1129    fn test_detect_delimiter_single_column_falls_back_to_comma() {
1130        let data = b"value\n1\n2\n3\n";
1131        assert_eq!(detect_delimiter(Cursor::new(data.as_ref())).unwrap(), b',');
1132    }
1133
1134    #[test]
1135    fn test_detect_delimiter_two_column_semicolon_with_commas_in_text() {
1136        // Free text containing commas must not outvote the true delimiter.
1137        let data = b"id;note\n1;hello, world\n2;foo, bar, baz\n3;plain\n";
1138        assert_eq!(detect_delimiter(Cursor::new(data.as_ref())).unwrap(), b';');
1139    }
1140
1141    #[test]
1142    fn test_detect_delimiter_never_splits_rows_the_header_does_not() {
1143        // Pipe splits both data rows and not the header; the readers cut rows
1144        // to the header's width, so choosing it kept only `123` (gh #1092).
1145        let data = b"key\n123|456\n123|789\n";
1146        assert_eq!(detect_delimiter(Cursor::new(data.as_ref())).unwrap(), b',');
1147    }
1148
1149    #[test]
1150    fn test_detect_delimiter_header_split_beats_uniform_rows() {
1151        // A trailing delimiter on every data row: comma splits the header,
1152        // which no other candidate does.
1153        let data = b"a,b\n1,2,\n3,4,\n";
1154        assert_eq!(detect_delimiter(Cursor::new(data.as_ref())).unwrap(), b',');
1155    }
1156
1157    #[test]
1158    fn test_detect_delimiter_trailing_empty_fields_still_agree() {
1159        // Semicolon rows carry a trailing empty field, comma occurs inside the
1160        // header name and a decimal. Counting the empty extras against
1161        // semicolon let comma win and invent a null column (gh #1096).
1162        let data = b"id;amount,EUR\n1;10,50;\n2;20;\n";
1163        assert_eq!(detect_delimiter(Cursor::new(data.as_ref())).unwrap(), b';');
1164    }
1165
1166    #[test]
1167    fn test_detect_delimiter_quoted_empty_trailing_field_still_agrees() {
1168        // `""` decodes to an empty field in every reader; judged on its raw
1169        // text it counted as content and comma won again (gh #1102).
1170        let data = b"id;amount,EUR\n1;10,50;\"\"\n2;20;\"\"\n";
1171        assert_eq!(detect_delimiter(Cursor::new(data.as_ref())).unwrap(), b';');
1172    }
1173
1174    #[test]
1175    fn test_detect_delimiter_exact_width_beats_cut_trailing_fields() {
1176        // Both candidates split the header and keep every value; semicolon
1177        // reads each row as it stands, comma only after cutting an empty
1178        // trailing field. Comma won the tie and renamed the columns.
1179        let data = b"id,part;value\nx;y,z,\nu;v,w,\n";
1180        assert_eq!(detect_delimiter(Cursor::new(data.as_ref())).unwrap(), b';');
1181    }
1182
1183    #[test]
1184    fn test_detect_delimiter_skips_a_utf8_bom() {
1185        // A BOM then a blank line: the BOM alone was scored as a one-field
1186        // header, and comma won over the real semicolon header.
1187        let data = b"\xEF\xBB\xBF\nid;value\n1;2\n3;4\n";
1188        assert_eq!(detect_delimiter(Cursor::new(data.as_ref())).unwrap(), b';');
1189        let no_blank = b"\xEF\xBB\xBFid;value\n1;2\n";
1190        assert_eq!(
1191            detect_delimiter(Cursor::new(no_blank.as_ref())).unwrap(),
1192            b';'
1193        );
1194    }
1195
1196    #[test]
1197    fn test_detect_delimiter_lone_cr_ends_a_record() {
1198        // CR-only line endings: read as one header record, pipe split it and
1199        // won, and the keys were cut to `123` (gh #1097).
1200        let data = b"key\r123|456\r123|789\r";
1201        assert_eq!(detect_delimiter(Cursor::new(data.as_ref())).unwrap(), b',');
1202        let semicolon = b"id;value\r1;10\r2;20\r";
1203        assert_eq!(
1204            detect_delimiter(Cursor::new(semicolon.as_ref())).unwrap(),
1205            b';'
1206        );
1207    }
1208
1209    #[test]
1210    fn test_count_fields_reports_trailing_empty_fields() {
1211        assert_eq!(count_fields(b"1;10,50;", b';', b'"'), (3, 1));
1212        assert_eq!(count_fields(b"a;;", b';', b'"'), (3, 2));
1213        assert_eq!(count_fields(b"a;\"\"", b';', b'"'), (2, 1));
1214        assert_eq!(count_fields(b"a;\"\"\"\"", b';', b'"'), (2, 0));
1215        assert_eq!(count_fields(b"a; ", b';', b'"'), (2, 0));
1216        assert_eq!(count_fields(b"a;b", b';', b'"'), (2, 0));
1217        assert_eq!(count_fields(b"\"x;y\";", b';', b'"'), (2, 1));
1218    }
1219
1220    #[test]
1221    fn test_split_sample_records_quote_aware() {
1222        let (records, last_unterminated) =
1223            split_sample_records(b"a,b\r\n1,\"x\ny\"\n2,z\n", b',', b'"');
1224        assert_eq!(records, ["a,b", "1,\"x\ny\"", "2,z"].map(str::as_bytes));
1225        assert!(!last_unterminated);
1226
1227        let (records, last_unterminated) =
1228            split_sample_records(b"a,b\n1,\"cut mid-fie", b',', b'"');
1229        assert_eq!(records, ["a,b", "1,\"cut mid-fie"].map(str::as_bytes));
1230        assert!(last_unterminated);
1231
1232        let (records, last_unterminated) =
1233            split_sample_records(b"a,b\r1,\"x\ry\"\r2,z\r", b',', b'"');
1234        assert_eq!(records, ["a,b", "1,\"x\ry\"", "2,z"].map(str::as_bytes));
1235        assert!(!last_unterminated);
1236
1237        let (records, _) = split_sample_records(b"a\rb\r\nc\n", b',', b'"');
1238        assert_eq!(records, ["a", "b", "c"].map(str::as_bytes));
1239    }
1240
1241    #[test]
1242    fn test_detect_delimiter_full_sample_ending_on_record_boundary() {
1243        // Exactly 4096 bytes ending with a clean newline: the final record is
1244        // complete and must be scored, not dropped as a fragment.
1245        let mut data = Vec::from(&b"a;b;c\n"[..]);
1246        while data.len() < 4090 {
1247            data.extend_from_slice(b"1;2;3\n");
1248        }
1249        while data.len() < 4095 {
1250            data.push(b'#');
1251        }
1252        data.push(b'\n');
1253        assert_eq!(data.len(), 4096);
1254        assert_eq!(
1255            detect_delimiter(Cursor::new(data.as_slice())).unwrap(),
1256            b';'
1257        );
1258    }
1259
1260    #[test]
1261    fn test_detect_delimiter_reads_past_the_sample_to_end_the_header() {
1262        // #1098: the first `;` is past byte 4096, so a 4 KB sample held a
1263        // fragment of the header that no candidate split.
1264        let issue = format!("\"{}\";value\n1;10\n2;20\n", "x".repeat(4094));
1265        assert_eq!(
1266            detect_delimiter(Cursor::new(issue.as_bytes())).unwrap(),
1267            b';'
1268        );
1269
1270        // A header record ending exactly at the 4 KB boundary needs no more
1271        // than the one byte read past it.
1272        let mut exact = "a".repeat(4093).into_bytes();
1273        exact.extend_from_slice(b";b\n1;2\n");
1274        let mut reader = Cursor::new(exact.as_slice());
1275        let (delimiter, head) = detect_delimiter_with_head(&mut reader).unwrap();
1276        assert_eq!((delimiter, head.len()), (b';', 4097));
1277    }
1278
1279    #[test]
1280    fn test_detect_delimiter_with_head_returns_every_byte_it_read() {
1281        let data = format!("{};value\n1;2\n", "a".repeat(20_000));
1282        let mut reader = Cursor::new(data.as_bytes());
1283        let (delimiter, head) = detect_delimiter_with_head(&mut reader).unwrap();
1284        assert_eq!(delimiter, b';');
1285        assert_eq!(head.len(), 32_768.min(data.len()));
1286        let mut rest = Vec::new();
1287        reader.read_to_end(&mut rest).unwrap();
1288        assert_eq!([head, rest].concat(), data.as_bytes());
1289    }
1290
1291    #[test]
1292    fn test_detect_delimiter_refuses_a_header_past_the_bound() {
1293        let data = format!("{};value\n1;2\n", "h".repeat(MAX_HEADER_SAMPLE_BYTES));
1294        let error = detect_delimiter(Cursor::new(data.as_bytes())).unwrap_err();
1295        assert_eq!(error.kind(), std::io::ErrorKind::InvalidData);
1296        assert!(error.to_string().contains("csv_delimiter"), "{error}");
1297        let error = detect_delimiter_with_head(&mut Cursor::new(data.as_bytes())).unwrap_err();
1298        assert!(matches!(error, DataProfilerError::CsvParsingError { .. }));
1299
1300        // A whole source no longer than the bound is never refused.
1301        let short = "h".repeat(MAX_HEADER_SAMPLE_BYTES);
1302        assert_eq!(
1303            detect_delimiter(Cursor::new(short.as_bytes())).unwrap(),
1304            b','
1305        );
1306    }
1307
1308    #[test]
1309    fn test_detect_delimiter_quote_rule_is_per_candidate() {
1310        // #1172 review: with any candidate taken as a field boundary, the quote
1311        // after `;` in a comma header (and after `,` in a semicolon one) opened
1312        // a quoted section, and a short header past a 1 MiB source was refused.
1313        for (header, row, expected) in [
1314            (&b"id,height;\"in\n"[..], &b"1,5\n"[..], b','),
1315            (&b"id;a;b,\"c\n"[..], &b"1;2;3\n"[..], b';'),
1316        ] {
1317            let mut data = header.to_vec();
1318            while data.len() <= MAX_HEADER_SAMPLE_BYTES {
1319                data.extend_from_slice(row);
1320            }
1321            assert_eq!(
1322                detect_delimiter(Cursor::new(data.as_slice())).unwrap(),
1323                expected,
1324                "{}",
1325                String::from_utf8_lossy(header)
1326            );
1327        }
1328    }
1329
1330    #[test]
1331    fn test_detect_delimiter_refuses_a_head_of_blank_records() {
1332        // #1172 review: a 1 MiB head of blank lines has not reached the header
1333        // yet, and guessing comma read `id;value` as one column.
1334        let mut data = vec![b'\n'; MAX_HEADER_SAMPLE_BYTES + 1];
1335        data.extend_from_slice(b"id;value\n1;2\n");
1336        assert!(detect_delimiter(Cursor::new(data.as_slice())).is_err());
1337
1338        // A whole source that is blank still reads as comma.
1339        let blank = vec![b'\n'; 10];
1340        assert_eq!(
1341            detect_delimiter(Cursor::new(blank.as_slice())).unwrap(),
1342            b','
1343        );
1344    }
1345
1346    #[test]
1347    fn test_detect_delimiter_mid_field_quote_is_literal() {
1348        // The readers take a quote inside a field as a literal, so the header
1349        // ends at its newline. Taken as opening a quote, it never ended, and a
1350        // source past the 1 MiB bound was refused.
1351        let mut data = b"id;height\"in;note\n".to_vec();
1352        while data.len() <= MAX_HEADER_SAMPLE_BYTES {
1353            data.extend_from_slice(b"1;5;x\n");
1354        }
1355        assert_eq!(
1356            detect_delimiter(Cursor::new(data.as_slice())).unwrap(),
1357            b';'
1358        );
1359
1360        let (records, last_unterminated) =
1361            split_sample_records(b"a;b\"c;d\n1;\"x\ny\";2\n", b';', b'"');
1362        assert_eq!(records, ["a;b\"c;d", "1;\"x\ny\";2"].map(str::as_bytes));
1363        assert!(!last_unterminated);
1364
1365        // Field counting follows the same rule: the readers split `id"x;value`
1366        // into two fields, and a header no candidate split fell back to comma.
1367        assert_eq!(count_fields(b"id\"x;value", b';', b'"'), (2, 0));
1368        assert_eq!(count_fields(b"\"a;\"\"b\";c", b';', b'"'), (2, 0));
1369        assert_eq!(
1370            detect_delimiter(Cursor::new(&b"id\"x;value\n1;2\n3;4\n"[..])).unwrap(),
1371            b';'
1372        );
1373    }
1374}