csv_nose/

sniffer.rs

//! Main Sniffer builder and sniff methods.
//!
//! This module provides the qsv-sniffer compatible API.

use std::borrow::Cow;
use std::fs::File;
use std::io::{Read, Seek};
use std::path::Path;

use crate::encoding::{detect_and_transcode, detect_encoding, skip_bom};
use crate::error::{Result, SnifferError};
use crate::field_type::Type;
use crate::metadata::{Dialect, Header, Metadata, Quote};
use crate::sample::{DatePreference, SampleSize};
use crate::tum::potential_dialects::{
    PotentialDialect, detect_line_terminator, generate_dialects_with_terminator,
};
use crate::tum::score::{DialectScore, find_best_dialect, score_all_dialects_with_best_table};
use crate::tum::table::{Table, parse_table};
use crate::tum::type_detection::infer_column_types;

/// Maximum buffer size for `SampleSize::Records` mode (100 MB).
const MAX_RECORDS_BYTES: usize = 100 * 1024 * 1024;

/// CSV dialect sniffer using the Table Uniformity Method.
///
/// # Example
///
/// ```no_run
/// use csv_nose::{Sniffer, SampleSize};
///
/// let mut sniffer = Sniffer::new();
/// sniffer.sample_size(SampleSize::Records(100));
///
/// let metadata = sniffer.sniff_path("data.csv").unwrap();
/// println!("Delimiter: {}", metadata.dialect.delimiter as char);
/// println!("Has header: {}", metadata.dialect.header.has_header_row);
/// ```
#[derive(Debug, Clone)]
pub struct Sniffer {
    /// Sample size for sniffing.
    sample_size: SampleSize,
    /// Date format preference for ambiguous dates.
    date_preference: DatePreference,
    /// Optional forced delimiter.
    forced_delimiter: Option<u8>,
    /// Optional forced quote character.
    forced_quote: Option<Quote>,
}

impl Default for Sniffer {
    fn default() -> Self {
        Self::new()
    }
}

impl Sniffer {
    /// Create a new Sniffer with default settings.
    pub const fn new() -> Self {
        Self {
            sample_size: SampleSize::Records(100),
            date_preference: DatePreference::MdyFormat,
            forced_delimiter: None,
            forced_quote: None,
        }
    }

    /// Set the sample size for sniffing.
    pub fn sample_size(&mut self, sample_size: SampleSize) -> &mut Self {
        self.sample_size = sample_size;
        self
    }

    /// Set the date preference for ambiguous date parsing.
    pub fn date_preference(&mut self, date_preference: DatePreference) -> &mut Self {
        self.date_preference = date_preference;
        self
    }

    /// Force a specific delimiter (skip delimiter detection).
    pub fn delimiter(&mut self, delimiter: u8) -> &mut Self {
        self.forced_delimiter = Some(delimiter);
        self
    }

    /// Force a specific quote character.
    pub fn quote(&mut self, quote: Quote) -> &mut Self {
        self.forced_quote = Some(quote);
        self
    }

    /// Sniff a CSV file at the given path.
    pub fn sniff_path<P: AsRef<Path>>(&mut self, path: P) -> Result<Metadata> {
        let file = File::open(path.as_ref())?;
        let mut reader = std::io::BufReader::new(file);
        self.sniff_reader(&mut reader)
    }

    /// Sniff CSV data from a reader.
    pub fn sniff_reader<R: Read + Seek>(&mut self, reader: R) -> Result<Metadata> {
        let data = self.read_sample(reader)?;

        if data.is_empty() {
            return Err(SnifferError::EmptyData);
        }

        self.sniff_bytes(&data)
    }

    /// Sniff CSV data from bytes.
    pub fn sniff_bytes(&self, data: &[u8]) -> Result<Metadata> {
        if data.is_empty() {
            return Err(SnifferError::EmptyData);
        }

        // Detect encoding and transcode to UTF-8 if necessary
        let (transcoded_data, was_transcoded) = detect_and_transcode(data);
        let data = &transcoded_data[..];

        // Detect encoding info (for metadata)
        let encoding_info = detect_encoding(data);
        let is_utf8 = !was_transcoded || encoding_info.is_utf8;

        // Skip BOM
        let data = skip_bom(data);

        // Skip comment/preamble lines (lines starting with #)
        let (comment_preamble_rows, data) = skip_preamble(data);

        // Detect line terminator first to reduce search space
        let line_terminator = detect_line_terminator(data);

        // Generate potential dialects
        let dialects = self.forced_delimiter.map_or_else(
            || generate_dialects_with_terminator(line_terminator),
            |delim| {
                // If delimiter is forced, only test that delimiter with different quotes
                let quotes = if let Some(q) = self.forced_quote {
                    vec![q]
                } else {
                    vec![Quote::Some(b'"'), Quote::Some(b'\''), Quote::None]
                };

                quotes
                    .into_iter()
                    .map(|q| PotentialDialect::new(delim, q, line_terminator))
                    .collect()
            },
        );
        // Determine max rows for scoring
        let max_rows = match self.sample_size {
            SampleSize::Records(n) => n,
            SampleSize::Bytes(_) | SampleSize::All => 0, // Already limited by read_sample
        };

        // Score all dialects and get the best table (avoids re-parsing)
        let (scores, best_table) = score_all_dialects_with_best_table(data, &dialects, max_rows);

        // Find the best dialect
        let best = find_best_dialect(&scores)
            .ok_or_else(|| SnifferError::NoDialectDetected("No valid dialect found".to_string()))?;

        // Detect structural preamble using the already-parsed table.
        //
        // `best_table` was parsed with the top-gamma dialect, but
        // `find_best_dialect` may deliberately select a different dialect when
        // scores are close (delimiter/quote tiebreakers). Reuse the cached table
        // only when it was parsed with the dialect we actually selected;
        // otherwise re-parse with the selected dialect so preamble detection,
        // field names, and type inference all run on the correct parse.
        let table_for_preamble = match best_table {
            Some((dialect, table)) if dialect == best.dialect => table,
            _ => parse_table(data, &best.dialect, max_rows),
        };
        let structural_preamble = detect_structural_preamble(&table_for_preamble);

        // Total preamble = comment rows + structural rows
        let total_preamble_rows = comment_preamble_rows + structural_preamble;

        // Build metadata from the best dialect, reusing the already-parsed table
        // Pass structural_preamble for table row indexing (since comment rows are already skipped from data)
        // Pass total_preamble_rows for Header metadata (to report true preamble count in original file)
        self.build_metadata(
            best,
            is_utf8,
            structural_preamble,
            total_preamble_rows,
            &table_for_preamble,
            data,
        )
    }

    /// Read a sample of data from the reader based on `sample_size` settings.
    fn read_sample<R: Read + Seek>(&self, mut reader: R) -> Result<Vec<u8>> {
        // `Read::read` may return fewer bytes than requested even when more data
        // is available (pipes, BufReader chunk boundaries). Loop until the buffer
        // is full or EOF is reached so the sample is not silently truncated.
        // Returns the number of bytes actually read.
        fn fill<R: Read>(reader: &mut R, buf: &mut [u8]) -> std::io::Result<usize> {
            let mut filled = 0;
            while filled < buf.len() {
                match reader.read(&mut buf[filled..]) {
                    Ok(0) => break,
                    Ok(n) => filled += n,
                    Err(ref e) if e.kind() == std::io::ErrorKind::Interrupted => {}
                    Err(e) => return Err(e),
                }
            }
            Ok(filled)
        }

        match self.sample_size {
            SampleSize::Bytes(n) => {
                let mut buffer = vec![0u8; n];
                let bytes_read = fill(&mut reader, &mut buffer)?;
                buffer.truncate(bytes_read);
                Ok(buffer)
            }
            SampleSize::All => {
                const MAX_BYTES: u64 = 1024 * 1024 * 1024; // 1 GB hard cap
                let mut buffer = Vec::new();
                (&mut reader).take(MAX_BYTES).read_to_end(&mut buffer)?;
                if buffer.len() as u64 == MAX_BYTES {
                    let mut probe = [0u8; 1];
                    if reader.read(&mut probe)? > 0 {
                        eprintln!(
                            "warning: input exceeds 1 GB; sniffing on truncated sample — results may be inaccurate"
                        );
                    }
                }
                Ok(buffer)
            }
            SampleSize::Records(n) => {
                // For records, we read enough to capture n records
                // Estimate ~1KB per record as a starting point, with a minimum
                let estimated_size = n.saturating_mul(1024).clamp(8192, MAX_RECORDS_BYTES);
                let mut buffer = vec![0u8; estimated_size];
                let bytes_read = fill(&mut reader, &mut buffer)?;
                buffer.truncate(bytes_read);

                // If we filled the estimate, we may not have enough records yet.
                if bytes_read == estimated_size {
                    // Count newlines to see if we have enough records
                    let newlines = bytecount::count(&buffer, b'\n');
                    if newlines < n {
                        // Read more data, but never let the total exceed the
                        // MAX_RECORDS_BYTES cap (cap against remaining capacity,
                        // not just per-read).
                        let remaining = MAX_RECORDS_BYTES.saturating_sub(buffer.len());
                        let additional = (n - newlines).saturating_mul(2048).min(remaining);
                        let mut more = vec![0u8; additional];
                        let more_read = fill(&mut reader, &mut more)?;
                        more.truncate(more_read);
                        buffer.extend(more);
                    }
                }

                if buffer.len() >= MAX_RECORDS_BYTES {
                    let mut probe = [0u8; 1];
                    if reader.read(&mut probe)? > 0 {
                        eprintln!(
                            "warning: Records sample capped at 100 MB; \
                             sniff result may be approximate for very large inputs"
                        );
                    }
                }

                Ok(buffer)
            }
        }
    }

    /// Build Metadata from the best scoring dialect.
    ///
    /// # Arguments
    /// * `structural_preamble` - Number of structural preamble rows in the table (for row indexing)
    /// * `total_preamble_rows` - Total preamble rows including comments (for Header metadata)
    /// * `table` - Pre-parsed table to avoid redundant parsing
    /// * `data` - Raw data bytes for accurate avg_record_len calculation
    fn build_metadata(
        &self,
        score: &DialectScore,
        is_utf8: bool,
        structural_preamble: usize,
        total_preamble_rows: usize,
        table: &Table,
        data: &[u8],
    ) -> Result<Metadata> {
        if table.is_empty() {
            return Err(SnifferError::EmptyData);
        }

        // Create a view of the table without structural preamble
        // (comment preamble rows are already stripped from data)
        // Use Cow to avoid cloning in the common no-preamble case
        let effective_table: Cow<'_, Table> =
            if structural_preamble > 0 && table.rows.len() > structural_preamble {
                let mut et = Table::new();
                et.rows = table.rows[structural_preamble..].to_vec();
                et.field_counts = table.field_counts[structural_preamble..].to_vec();
                et.update_modal_field_count();
                Cow::Owned(et)
            } else {
                Cow::Borrowed(table)
            };

        // Detect header on the effective table (pass total_preamble_rows for Header metadata)
        let header = detect_header(&effective_table, total_preamble_rows);

        // Get field names from the effective table (first row after structural preamble)
        let fields = if header.has_header_row && !effective_table.rows.is_empty() {
            effective_table.rows[0].clone()
        } else {
            // Generate field names
            (0..score.num_fields)
                .map(|i| format!("field_{}", i + 1))
                .collect()
        };

        // Skip header row for type inference if present
        let data_table = if header.has_header_row && effective_table.rows.len() > 1 {
            let mut dt = crate::tum::table::Table::new();
            dt.rows = effective_table.rows[1..].to_vec();
            dt.field_counts = effective_table.field_counts[1..].to_vec();
            dt.update_modal_field_count();
            dt
        } else {
            effective_table.into_owned()
        };

        // Infer types for each column
        let types = infer_column_types(&data_table);

        // Build dialect
        let dialect = Dialect {
            delimiter: score.dialect.delimiter,
            header,
            quote: score.dialect.quote,
            flexible: !score.is_uniform,
            is_utf8,
        };

        // Calculate average record length from the raw data
        let avg_record_len = calculate_avg_record_len(data, table.num_rows());

        Ok(Metadata {
            dialect,
            avg_record_len,
            num_fields: score.num_fields,
            fields,
            types,
        })
    }
}

/// Detect if the first row (after preamble) is likely a header row.
///
/// Optimized: Computes type counts in a single pass without allocating Vecs.
fn detect_header(table: &crate::tum::table::Table, preamble_rows: usize) -> Header {
    if table.rows.is_empty() {
        return Header::new(false, preamble_rows);
    }

    if table.rows.len() < 2 {
        // Can't determine header with only one row
        return Header::new(false, preamble_rows);
    }

    let first_row = &table.rows[0];
    let second_row = &table.rows[1];

    // Heuristics for header detection:
    // 1. First row has different types than subsequent rows
    // 2. First row values look like labels (text when data is numeric)
    // 3. First row has no duplicates (header columns should be unique)

    let mut header_score = 0.0;
    let mut checks = 0;

    // Check 1 & 2: Count types in a single pass for first row
    let (first_text_count, first_numeric_count) =
        first_row.iter().fold((0, 0), |(text, num), s| {
            let t = crate::tum::type_detection::detect_cell_type(s);
            (
                text + usize::from(t == Type::Text),
                num + usize::from(t.is_numeric()),
            )
        });

    // Count types in a single pass for second row
    let second_text_count = second_row
        .iter()
        .filter(|s| crate::tum::type_detection::detect_cell_type(s) == Type::Text)
        .count();

    if first_text_count > second_text_count {
        header_score += 1.0;
    }
    checks += 1;

    // Check 2: First row has more text than numeric
    if first_text_count > first_numeric_count {
        header_score += 0.5;
    }
    checks += 1;

    // Check 3: No duplicates in first row
    let unique_count = {
        let mut seen = std::collections::HashSet::new();
        first_row.iter().filter(|s| seen.insert(s.as_str())).count()
    };
    if unique_count == first_row.len() {
        header_score += 0.5;
    }
    checks += 1;

    // Check 4: First row values are shorter (headers tend to be concise)
    let avg_first_len: f64 = first_row
        .iter()
        .map(std::string::String::len)
        .sum::<usize>() as f64
        / first_row.len().max(1) as f64;
    let avg_second_len: f64 = second_row
        .iter()
        .map(std::string::String::len)
        .sum::<usize>() as f64
        / second_row.len().max(1) as f64;

    if avg_first_len <= avg_second_len {
        header_score += 0.3;
    }
    checks += 1;

    // Threshold for header detection
    let has_header = (header_score / checks as f64) > 0.4;

    Header::new(has_header, preamble_rows)
}

/// Calculate average record length from raw data.
///
/// Uses the byte length of the first `num_rows` rows for accurate results
/// that include quote characters and actual line terminators.
/// This handles the case where `data` contains more bytes than `num_rows` rows
/// (e.g., when `SampleSize::Records(n)` reads more data than needed).
fn calculate_avg_record_len(data: &[u8], num_rows: usize) -> usize {
    if num_rows == 0 || data.is_empty() {
        return 0;
    }

    // Find the byte offset where the num_rows-th row ends
    // by counting newlines (handling both \n and \r\n)
    let mut rows_seen = 0;
    let mut byte_offset = 0;

    for (i, &byte) in data.iter().enumerate() {
        if byte == b'\n' {
            rows_seen += 1;
            if rows_seen >= num_rows {
                byte_offset = i + 1; // Include the newline
                break;
            }
        }
    }

    // If we didn't find enough newlines, use the entire data length
    // (this handles files without trailing newlines or small files)
    if byte_offset == 0 {
        byte_offset = data.len();
    }

    byte_offset / num_rows
}

/// Skip preamble/comment lines at the start of data.
///
/// Detects lines starting with '#' at the beginning of the file and returns
/// the number of preamble rows and a slice starting after the preamble.
///
/// Blank lines interspersed within the leading comment block are tolerated:
/// they are buffered and only counted as preamble once a further comment line
/// is seen. A trailing run of blank lines that is followed by data (rather than
/// by another comment) is left in place, so `#c\n\n\ndata` reports one preamble
/// row, not three.
fn skip_preamble(data: &[u8]) -> (usize, &[u8]) {
    let mut preamble_rows = 0;
    let mut offset = 0;

    // Blank lines seen since the last comment line; only committed to the
    // preamble if another comment line follows.
    let mut pending_blank_rows = 0;
    let mut pending_blank_offset = offset;

    while offset < data.len() {
        // Find end of line (exclusive of the terminator).
        let mut line_end = offset;
        while line_end < data.len() && data[line_end] != b'\n' && data[line_end] != b'\r' {
            line_end += 1;
        }

        // Include the line terminator in the consumed span.
        let mut next_offset = line_end;
        if next_offset < data.len() && data[next_offset] == b'\r' {
            next_offset += 1;
        }
        if next_offset < data.len() && data[next_offset] == b'\n' {
            next_offset += 1;
        }

        // Skip leading whitespace to classify the line.
        let mut line_start = offset;
        while line_start < line_end && (data[line_start] == b' ' || data[line_start] == b'\t') {
            line_start += 1;
        }

        if line_start == line_end {
            // Blank (or whitespace-only) line: defer judgement.
            if next_offset == offset {
                break; // no progress possible
            }
            pending_blank_rows += 1;
            offset = next_offset;
        } else if data[line_start] == b'#' {
            // Comment line: any deferred blanks are part of the comment block.
            preamble_rows += pending_blank_rows + 1;
            pending_blank_rows = 0;
            offset = next_offset;
            pending_blank_offset = offset;
        } else {
            // Data line: deferred blanks belong to the data, not the preamble.
            break;
        }
    }

    if pending_blank_rows > 0 {
        offset = pending_blank_offset;
    }

    (preamble_rows, &data[offset..])
}

/// Detect structural preamble rows using field count consistency analysis.
///
/// Identifies rows at the start that don't match the predominant field count
/// pattern (metadata rows, empty rows, title rows with different structure).
fn detect_structural_preamble(table: &crate::tum::table::Table) -> usize {
    let n = table.field_counts.len();
    if n < 3 {
        return 0;
    }

    let modal_count = table.modal_field_count();

    // Pre-compute suffix counts: for each position i, how many rows from i to end match modal_count
    // This converts O(n²) scanning to O(n) preprocessing + O(1) lookups
    let mut matching_suffix = vec![0usize; n];
    let mut count = 0;
    for i in (0..n).rev() {
        if table.field_counts[i] == modal_count {
            count += 1;
        }
        matching_suffix[i] = count;
    }

    // Find first row where remaining data is 80%+ consistent with modal field count
    for (i, &field_count) in table.field_counts.iter().enumerate() {
        if field_count == modal_count {
            let remaining_len = n - i;
            let matching = matching_suffix[i];
            let consistency = matching as f64 / remaining_len as f64;

            if consistency >= 0.8 {
                return i;
            }
        }
    }

    0
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn test_sniffer_builder() {
        let mut sniffer = Sniffer::new();
        sniffer
            .sample_size(SampleSize::Records(50))
            .date_preference(DatePreference::DmyFormat)
            .delimiter(b',');

        assert_eq!(sniffer.sample_size, SampleSize::Records(50));
        assert_eq!(sniffer.date_preference, DatePreference::DmyFormat);
        assert_eq!(sniffer.forced_delimiter, Some(b','));
    }

    #[test]
    fn test_sniff_bytes() {
        let data = b"name,age,city\nAlice,30,NYC\nBob,25,LA\n";
        let sniffer = Sniffer::new();

        let metadata = sniffer.sniff_bytes(data).unwrap();

        assert_eq!(metadata.dialect.delimiter, b',');
        assert!(metadata.dialect.header.has_header_row);
        assert_eq!(metadata.num_fields, 3);
        assert_eq!(metadata.fields, vec!["name", "age", "city"]);
    }

    #[test]
    fn test_sniff_tsv() {
        let data = b"name\tage\tcity\nAlice\t30\tNYC\nBob\t25\tLA\n";
        let sniffer = Sniffer::new();

        let metadata = sniffer.sniff_bytes(data).unwrap();

        assert_eq!(metadata.dialect.delimiter, b'\t');
        assert!(metadata.dialect.header.has_header_row);
    }

    #[test]
    fn test_sniff_semicolon() {
        let data = b"name;age;city\nAlice;30;NYC\nBob;25;LA\n";
        let sniffer = Sniffer::new();

        let metadata = sniffer.sniff_bytes(data).unwrap();

        assert_eq!(metadata.dialect.delimiter, b';');
    }

    #[test]
    fn test_sniff_no_header() {
        let data = b"1,2,3\n4,5,6\n7,8,9\n";
        let sniffer = Sniffer::new();

        let metadata = sniffer.sniff_bytes(data).unwrap();

        assert_eq!(metadata.dialect.delimiter, b',');
        // All numeric data - should not detect header
        assert!(!metadata.dialect.header.has_header_row);
    }

    #[test]
    fn test_sniff_with_quotes() {
        let data = b"\"name\",\"value\"\n\"hello, world\",123\n\"test\",456\n";
        let sniffer = Sniffer::new();

        let metadata = sniffer.sniff_bytes(data).unwrap();

        assert_eq!(metadata.dialect.delimiter, b',');
        assert_eq!(metadata.dialect.quote, Quote::Some(b'"'));
    }

    #[test]
    fn test_sniff_empty() {
        let data = b"";
        let sniffer = Sniffer::new();

        let result = sniffer.sniff_bytes(data);
        assert!(result.is_err());
    }

    #[test]
    fn test_skip_preamble() {
        // Test with comment lines
        let data = b"# This is a comment\n# Another comment\nname,age\nAlice,30\n";
        let (preamble_rows, remaining) = skip_preamble(data);
        assert_eq!(preamble_rows, 2);
        assert_eq!(remaining, b"name,age\nAlice,30\n");

        // Test without comment lines
        let data = b"name,age\nAlice,30\n";
        let (preamble_rows, remaining) = skip_preamble(data);
        assert_eq!(preamble_rows, 0);
        assert_eq!(remaining, b"name,age\nAlice,30\n");

        // Test with whitespace before #
        let data = b"  # Indented comment\nname,age\n";
        let (preamble_rows, remaining) = skip_preamble(data);
        assert_eq!(preamble_rows, 1);
        assert_eq!(remaining, b"name,age\n");
    }

    #[test]
    fn test_skip_preamble_blank_lines_within_comment_block() {
        // A blank line between comments is part of the comment block.
        let data = b"#c\n\n#d\ndata\n";
        let (preamble_rows, remaining) = skip_preamble(data);
        assert_eq!(preamble_rows, 3);
        assert_eq!(remaining, b"data\n");

        // Trailing blanks followed by data belong to the data, not the preamble.
        let data = b"#c\n\n\ndata\n";
        let (preamble_rows, remaining) = skip_preamble(data);
        assert_eq!(preamble_rows, 1);
        assert_eq!(remaining, b"\n\ndata\n");

        // Several blank runs interleaved with comments, then a trailing run.
        let data = b"#a\n\n#b\n\n\n#c\n\ndata\n";
        let (preamble_rows, remaining) = skip_preamble(data);
        assert_eq!(preamble_rows, 6);
        assert_eq!(remaining, b"\ndata\n");

        // Leading blanks with no comment at all: nothing is preamble.
        let data = b"\n\nname,age\n";
        let (preamble_rows, remaining) = skip_preamble(data);
        assert_eq!(preamble_rows, 0);
        assert_eq!(remaining, b"\n\nname,age\n");

        // Whitespace-only lines count as blank, and CRLF is handled.
        let data = b"#c\r\n   \r\n#d\r\ndata\r\n";
        let (preamble_rows, remaining) = skip_preamble(data);
        assert_eq!(preamble_rows, 3);
        assert_eq!(remaining, b"data\r\n");

        // A comment block that runs to EOF with trailing blanks.
        let data = b"#c\n\n";
        let (preamble_rows, remaining) = skip_preamble(data);
        assert_eq!(preamble_rows, 1);
        assert_eq!(remaining, b"\n");
    }

    #[test]
    fn test_sniff_with_preamble() {
        let data = b"# LimeSurvey export\n# Generated 2024-01-01\nname,age,city\nAlice,30,NYC\nBob,25,LA\n";
        let sniffer = Sniffer::new();

        let metadata = sniffer.sniff_bytes(data).unwrap();

        assert_eq!(metadata.dialect.delimiter, b',');
        assert!(metadata.dialect.header.has_header_row);
        assert_eq!(metadata.num_fields, 3);
    }

    #[test]
    fn test_comment_preamble_propagated() {
        let data = b"# Comment 1\n# Comment 2\nname,age\nAlice,30\nBob,25\n";
        let metadata = Sniffer::new().sniff_bytes(data).unwrap();
        assert_eq!(metadata.dialect.header.num_preamble_rows, 2);
        assert!(metadata.dialect.header.has_header_row);
        assert_eq!(metadata.fields, vec!["name", "age"]);
    }

    #[test]
    fn test_structural_preamble_detection() {
        // TITLE row has 1 field, SUBTITLE has 2 fields, data has 5 fields
        let data = b"TITLE\nSUB,TITLE\nA,B,C,D,E\n1,2,3,4,5\n2,3,4,5,6\n3,4,5,6,7\n";
        let metadata = Sniffer::new().sniff_bytes(data).unwrap();
        assert_eq!(metadata.dialect.header.num_preamble_rows, 2);
        assert!(metadata.dialect.header.has_header_row);
        assert_eq!(metadata.fields, vec!["A", "B", "C", "D", "E"]);
    }

    #[test]
    fn test_mixed_preamble_detection() {
        // Both comment preamble and structural preamble
        // METADATA has 1 field, data has 3 fields
        let data =
            b"# File header\nMETADATA\nname,age,city\nAlice,30,NYC\nBob,25,LA\nCharlie,35,CHI\n";
        let metadata = Sniffer::new().sniff_bytes(data).unwrap();
        // 1 comment + 1 structural = 2 total
        assert_eq!(metadata.dialect.header.num_preamble_rows, 2);
        assert!(metadata.dialect.header.has_header_row);
        assert_eq!(metadata.fields, vec!["name", "age", "city"]);
    }

    #[test]
    fn test_no_preamble() {
        let data = b"a,b,c\n1,2,3\n4,5,6\n";
        let metadata = Sniffer::new().sniff_bytes(data).unwrap();
        assert_eq!(metadata.dialect.header.num_preamble_rows, 0);
    }

    #[test]
    fn test_detect_structural_preamble_function() {
        use crate::tum::table::Table;

        // Table with 2 preamble rows (different field counts)
        let mut table = Table::new();
        table.rows = vec![
            vec!["TITLE".to_string()],
            vec!["".to_string(), "".to_string()],
            vec!["A".to_string(), "B".to_string(), "C".to_string()],
            vec!["1".to_string(), "2".to_string(), "3".to_string()],
            vec!["4".to_string(), "5".to_string(), "6".to_string()],
        ];
        table.field_counts = vec![1, 2, 3, 3, 3];
        table.update_modal_field_count();
        assert_eq!(detect_structural_preamble(&table), 2);

        // Table with no preamble (uniform field counts)
        let mut table = Table::new();
        table.rows = vec![
            vec!["A".to_string(), "B".to_string(), "C".to_string()],
            vec!["1".to_string(), "2".to_string(), "3".to_string()],
        ];
        table.field_counts = vec![3, 3];
        table.update_modal_field_count();
        assert_eq!(detect_structural_preamble(&table), 0);

        // Table too small to determine preamble
        let mut table = Table::new();
        table.rows = vec![vec!["A".to_string()]];
        table.field_counts = vec![1];
        table.update_modal_field_count();
        assert_eq!(detect_structural_preamble(&table), 0);
    }

    #[test]
    fn test_avg_record_len_calculated_from_data() {
        // Test that avg_record_len uses raw bytes, not parsed content
        let short_data = b"a,b\n1,2\n3,4\n";
        let sniffer = Sniffer::new();
        let metadata = sniffer.sniff_bytes(short_data).unwrap();

        // Each row: "a,b\n" = 4 bytes, "1,2\n" = 4 bytes, "3,4\n" = 4 bytes
        // Average: 12 / 3 = 4 bytes
        assert_eq!(metadata.avg_record_len, 4);
    }

    #[test]
    fn test_avg_record_len_with_quoted_fields() {
        let quoted_data = b"\"hello\",\"world\"\n\"foo\",\"bar\"\n";
        let sniffer = Sniffer::new();
        let metadata = sniffer.sniff_bytes(quoted_data).unwrap();

        // Raw: 16 + 12 = 28 bytes for 2 rows = 14 bytes avg
        assert_eq!(metadata.avg_record_len, 14);
    }

    #[test]
    fn test_records_mode_cap_boundary_ok() {
        // Verify that a reader with more than MAX_RECORDS_BYTES of valid CSV is handled
        // gracefully and returns Ok. Uses a cycling pattern to avoid a large literal in
        // test source; the runtime still materializes ~100 MB via collect().
        // We supply MAX_RECORDS_BYTES + one extra row so the probe-read finds data and
        // the truncation warning fires, but the sniff still succeeds.
        let row = b"col1,col2,col3\n1,2,3\n"; // 21 bytes, valid CSV pair
        let total = MAX_RECORDS_BYTES + row.len();
        let data: Vec<u8> = row.iter().copied().cycle().take(total).collect();
        // Confirm the test data actually exceeds the cap so the probe path is exercised.
        assert!(
            data.len() > MAX_RECORDS_BYTES,
            "test data must exceed MAX_RECORDS_BYTES to exercise probe-read path"
        );
        let cursor = std::io::Cursor::new(data);
        let mut sniffer = Sniffer::new();
        // Records(200_000): estimated_size = 200_000 * 1024 > MAX_RECORDS_BYTES (100 MB), clamped to MAX_RECORDS_BYTES.
        sniffer.sample_size(SampleSize::Records(200_000));
        let result = sniffer.sniff_reader(cursor);
        assert!(
            result.is_ok(),
            "sniff should succeed at cap boundary: {result:?}"
        );
        // Note: the eprintln! truncation warning cannot be captured in a standard Rust
        // unit test without process-level stderr redirection. The probe-read path is
        // exercised by virtue of data.len() > MAX_RECORDS_BYTES (asserted above).
    }
}