use csv::ReaderBuilder;
use std::io::Read;
use std::path::Path;
use dataprof_core::{
AnalysisOptions, ColumnProfile, CsvSourceMetadata, DataProfilerError, DataSource,
DelimiterDetection, ExecutionMetadata, FileFormat, TruncationReason,
};
use dataprof_runtime::{
ProfileReport, ReportAssembler, StreamingColumnCollection, profile_builder,
};
mod encoding;
mod memmap_reader;
mod quote_state;
#[doc(hidden)]
pub mod row_count;
pub use encoding::{EncodingDiagnostic, diagnose_non_utf8};
pub use memmap_reader::MemoryMappedCsvReader;
pub use quote_state::{QuoteOutcome, QuoteTrackingReader, unterminated_quote_error};
#[derive(Debug, Clone)]
pub struct CsvParserConfig {
pub flexible: bool,
pub delimiter: Option<u8>,
pub quote_char: u8,
pub trim_whitespace: bool,
pub has_header: bool,
pub max_rows: Option<usize>,
}
impl Default for CsvParserConfig {
fn default() -> Self {
Self {
flexible: true,
delimiter: None,
quote_char: b'"',
trim_whitespace: false,
has_header: true,
max_rows: None,
}
}
}
impl CsvParserConfig {
pub fn strict() -> Self {
Self {
flexible: false,
..Default::default()
}
}
pub fn with_delimiter(mut self, delimiter: u8) -> Self {
self.delimiter = Some(delimiter);
self
}
pub fn has_header(mut self, yes: bool) -> Self {
self.has_header = yes;
self
}
pub fn max_rows(mut self, max: Option<usize>) -> Self {
self.max_rows = max;
self
}
}
fn count_fields(line: &[u8], delimiter: u8, quote: u8) -> (usize, usize) {
let decodes_empty = |field: &[u8]| field.is_empty() || field == [quote, quote];
let mut fields = 1;
let mut trailing_empty = 0;
let mut field_start = 0;
let mut in_quotes = false;
let mut bytes = line.iter().copied().enumerate().peekable();
while let Some((index, byte)) = bytes.next() {
if byte == quote && in_quotes {
if bytes.peek().map(|&(_, next)| next) == Some(quote) {
bytes.next();
} else {
in_quotes = false;
}
} else if byte == quote && index == field_start {
in_quotes = true;
} else if byte == delimiter && !in_quotes {
trailing_empty = if decodes_empty(&line[field_start..index]) {
trailing_empty + 1
} else {
0
};
fields += 1;
field_start = index + 1;
}
}
trailing_empty = if decodes_empty(&line[field_start..]) {
trailing_empty + 1
} else {
0
};
(fields, trailing_empty)
}
fn agrees_with_header(fields: usize, trailing_empty: usize, header_fields: usize) -> bool {
fields == header_fields || (fields > header_fields && fields - trailing_empty <= header_fields)
}
fn split_sample_records(bytes: &[u8], delimiter: u8, quote: u8) -> (Vec<&[u8]>, bool) {
let mut records = Vec::new();
let mut in_quotes = false;
let mut start = 0usize;
let mut i = 0usize;
while i < bytes.len() {
match bytes[i] {
byte if byte == quote && in_quotes => {
if bytes.get(i + 1) == Some("e) {
i += 1; } else {
in_quotes = false;
}
}
byte if byte == quote
&& (i == start || bytes[i - 1] == delimiter)
&& !(i == start && matches!(byte, b'\r' | b'\n')) =>
{
in_quotes = true;
}
b'\n' if !in_quotes => {
let end = if i > start && bytes[i - 1] == b'\r' {
i - 1
} else {
i
};
records.push(&bytes[start..end]);
start = i + 1;
}
b'\r' if !in_quotes && bytes.get(i + 1) != Some(&b'\n') => {
records.push(&bytes[start..i]);
start = i + 1;
}
_ => {}
}
i += 1;
}
let last_unterminated = start < bytes.len();
if last_unterminated {
let mut end = bytes.len();
while end > start && bytes[end - 1] == b'\r' {
end -= 1;
}
records.push(&bytes[start..end]);
}
(records, last_unterminated)
}
const CANDIDATES: [u8; 4] = *b",;\t|";
const DETECTION_SAMPLE_BYTES: usize = 4096;
const MAX_HEADER_SAMPLE_BYTES: usize = 1 << 20;
fn head_bytes(head: &[u8]) -> &[u8] {
head.strip_prefix(b"\xEF\xBB\xBF").unwrap_or(head)
}
fn record_is_blank(record: &[u8]) -> bool {
String::from_utf8_lossy(record).trim().is_empty()
}
fn header_is_complete(head: &[u8], quote: u8) -> bool {
let head = head_bytes(head);
CANDIDATES.iter().all(|&delimiter| {
let (records, last_unterminated) = split_sample_records(head, delimiter, quote);
records
.iter()
.position(|record| !record_is_blank(record))
.is_some_and(|index| index + 1 < records.len() || !last_unterminated)
})
}
fn read_detection_head<R: Read>(reader: &mut R, quote: u8) -> std::io::Result<(Vec<u8>, bool)> {
let mut head = Vec::new();
let mut target = DETECTION_SAMPLE_BYTES;
loop {
let wanted = (target + 1 - head.len()) as u64;
reader.by_ref().take(wanted).read_to_end(&mut head)?;
let cut = head.len() > target;
if !cut || target == MAX_HEADER_SAMPLE_BYTES || header_is_complete(&head, quote) {
return Ok((head, cut));
}
target = (target * 2).min(MAX_HEADER_SAMPLE_BYTES);
}
}
const HEADER_TOO_LONG: &str = "The first 1 MiB of the CSV source holds no complete header record, so its delimiter could not be detected.";
const HEADER_TOO_LONG_HINT: &str = "Set the delimiter explicitly (csv_delimiter on Profiler or dataprof.profile) to read this file.";
pub fn detect_delimiter_with_head<R: Read>(
reader: &mut R,
) -> Result<(u8, Vec<u8>), DataProfilerError> {
let (detection, head) = detect_delimiter_detailed_with_head(reader)?;
Ok((detection_byte(&detection), head))
}
pub fn detect_delimiter_detailed_with_head<R: Read>(
reader: &mut R,
) -> Result<(DelimiterDetection, Vec<u8>), DataProfilerError> {
detect_delimiter_with_quote(reader, b'"')
}
fn detect_delimiter_with_quote<R: Read>(
reader: &mut R,
quote: u8,
) -> Result<(DelimiterDetection, Vec<u8>), DataProfilerError> {
let (head, cut) = read_detection_head(reader, quote)?;
match score_head(&head, cut, quote) {
Some(detection) => Ok((detection, head)),
None => Err(DataProfilerError::csv_structure(
HEADER_TOO_LONG,
HEADER_TOO_LONG_HINT,
)),
}
}
pub fn detect_delimiter_detailed<R: Read>(mut reader: R) -> std::io::Result<DelimiterDetection> {
let (head, cut) = read_detection_head(&mut reader, b'"')?;
score_head(&head, cut, b'"').ok_or_else(|| {
std::io::Error::new(
std::io::ErrorKind::InvalidData,
format!("{HEADER_TOO_LONG} {HEADER_TOO_LONG_HINT}"),
)
})
}
pub fn detection_byte(detection: &DelimiterDetection) -> u8 {
detection.delimiter.as_bytes()[0]
}
pub fn detect_delimiter<R: Read>(reader: R) -> std::io::Result<u8> {
detect_delimiter_detailed(reader).map(|detection| detection_byte(&detection))
}
fn score_head(head: &[u8], cut: bool, quote: u8) -> Option<DelimiterDetection> {
let head = head_bytes(head);
let mut scores: Vec<((bool, usize, usize, usize), u8)> = Vec::new();
let mut sampled: Vec<Vec<&[u8]>> = Vec::new();
for delimiter in CANDIDATES {
let (mut records, last_unterminated) = split_sample_records(head, delimiter, quote);
if cut && last_unterminated {
records.pop();
}
let scored: Vec<&[u8]> = records
.into_iter()
.filter(|record| !record_is_blank(record))
.take(5)
.collect();
let Some((header, rows)) = scored.split_first() else {
if cut {
continue;
}
return Some(DelimiterDetection::new(b',', 0, None));
};
let (header_fields, _) = count_fields(header, delimiter, quote);
let (mut agreeing, mut exact) = (1, 1);
for record in rows {
let (fields, trailing_empty) = count_fields(record, delimiter, quote);
if agrees_with_header(fields, trailing_empty, header_fields) {
agreeing += 1;
}
if fields == header_fields {
exact += 1;
}
}
scores.push((
(header_fields > 1, agreeing, exact, header_fields),
delimiter,
));
sampled.push(scored);
}
let pick = |candidates: &mut dyn Iterator<Item = ((bool, usize, usize, usize), u8)>| {
candidates.fold(
None,
|best: Option<((bool, usize, usize, usize), u8)>, entry| match best {
Some(held) if entry.0 <= held.0 => Some(held),
_ => Some(entry),
},
)
};
let (score, delimiter) = pick(&mut scores.iter().copied())?;
let (splits, agreeing, _, header_fields) = score;
let runner_up = pick(
&mut scores.iter().copied().filter(|&(other, other_delimiter)| {
splits && other_delimiter != delimiter && other.0 && other.1 == agreeing
}),
)
.map(|(_, other_delimiter)| other_delimiter);
let unlisted = if splits {
None
} else {
scores
.iter()
.position(|&(_, candidate)| candidate == delimiter)
.and_then(|index| unlisted_delimiter(&sampled[index], quote))
};
Some(
DelimiterDetection::new(delimiter, header_fields, runner_up)
.with_unlisted_delimiter(unlisted),
)
}
const UNLISTED_DELIMITERS: &[u8] = b"!\"#$%&'()*+/:<=>?@[\\]^`{}~";
fn unlisted_delimiter(records: &[&[u8]], quote: u8) -> Option<u8> {
let (header, rows) = records.split_first()?;
if rows.is_empty() {
return None;
}
UNLISTED_DELIMITERS.iter().copied().find(|&byte| {
if byte == quote {
return false;
}
let occurrences = |record: &[u8]| count_fields(record, byte, quote).0 - 1;
let in_header = occurrences(header);
in_header > 0 && rows.iter().all(|row| occurrences(row) == in_header)
})
}
pub fn detect_delimiter_from_path(path: &Path) -> std::io::Result<u8> {
let file = std::fs::File::open(path)?;
detect_delimiter(std::io::BufReader::new(file))
}
pub fn analyze_csv_from_reader<R: Read>(
reader: R,
config: &CsvParserConfig,
) -> Result<
(
Vec<ColumnProfile>,
StreamingColumnCollection,
usize,
Vec<String>,
),
DataProfilerError,
> {
let scan = scan_csv(reader, config)?;
let profiles = profile_builder::profiles_from_streaming(&scan.column_stats, false, false, None);
Ok((
profiles,
scan.column_stats,
scan.rows_read,
scan.header_names,
))
}
struct CsvScan {
column_stats: StreamingColumnCollection,
header_names: Vec<String>,
rows_read: usize,
ragged_rows: usize,
truncation: Option<TruncationReason>,
csv_metadata: CsvSourceMetadata,
}
fn scan_csv<R: Read>(reader: R, config: &CsvParserConfig) -> Result<CsvScan, DataProfilerError> {
let mut csv_builder = ReaderBuilder::new();
csv_builder.has_headers(config.has_header);
csv_builder.flexible(config.flexible);
csv_builder.quote(config.quote_char);
if config.trim_whitespace {
csv_builder.trim(csv::Trim::All);
}
let (boxed_reader, csv_metadata, delimiter): (Box<dyn std::io::Read + '_>, _, _) = match config
.delimiter
{
Some(delimiter) => (
Box::new(reader),
CsvSourceMetadata::configured(delimiter),
delimiter,
),
None => {
let mut reader = reader;
let (detection, head) = detect_delimiter_with_quote(&mut reader, config.quote_char)?;
let delimiter = detection_byte(&detection);
(
Box::new(std::io::Cursor::new(head).chain(reader)),
CsvSourceMetadata::detected(detection),
delimiter,
)
}
};
csv_builder.delimiter(delimiter);
let mut csv_reader = csv_builder.from_reader(boxed_reader);
let header_names: Vec<String> = if config.has_header {
let headers = csv_reader.headers()?;
let names: Vec<String> = headers.iter().map(|header| header.to_string()).collect();
dataprof_core::validate_unique_column_names(&names, "CSV header")?;
names
} else {
let headers = csv_reader.headers()?;
(0..headers.len())
.map(|index| format!("column_{index}"))
.collect()
};
let mut column_stats = StreamingColumnCollection::new();
column_stats.init_columns(&header_names);
let mut rows_read = 0;
let mut ragged_rows = 0;
let mut truncation = None;
for result in csv_reader.records() {
if let Some(max_rows) = config.max_rows
&& rows_read >= max_rows
{
truncation = Some(TruncationReason::MaxRows(max_rows as u64));
break;
}
let record = result?;
let mut values: Vec<String> = record.iter().map(|value| value.to_string()).collect();
let header_len = header_names.len();
if values.len() != header_len {
ragged_rows += 1;
}
if values.len() < header_len {
values.resize(header_len, String::new());
} else if values.len() > header_len {
values.truncate(header_len);
}
column_stats.process_record(&header_names, values);
rows_read += 1;
}
Ok(CsvScan {
column_stats,
header_names,
rows_read,
ragged_rows,
truncation,
csv_metadata,
})
}
pub fn analyze_csv_file(
file_path: &Path,
config: &CsvParserConfig,
) -> Result<ProfileReport, DataProfilerError> {
let metadata = std::fs::metadata(file_path).map_err(|error| map_io_error(file_path, error))?;
dataprof_core::binary_input::reject_binary_input(file_path)?;
let start = std::time::Instant::now();
let file = std::fs::File::open(file_path).map_err(|error| map_io_error(file_path, error))?;
let buf_reader = std::io::BufReader::new(file);
let scan = scan_csv(buf_reader, config)?;
let column_stats = &scan.column_stats;
let column_profiles =
profile_builder::profiles_from_streaming(column_stats, false, false, None);
let file_source = DataSource::File {
path: file_path.display().to_string(),
format: FileFormat::Csv,
size_bytes: metadata.len(),
modified_at: None,
parquet_metadata: None,
csv_metadata: Some(scan.csv_metadata.clone()),
};
let sample_columns = profile_builder::quality_check_samples(column_stats);
let scan_time_ms = start.elapsed().as_millis();
let num_columns = column_profiles.len();
let mut execution = ExecutionMetadata::new(scan.rows_read, num_columns, scan_time_ms)
.with_engine("csv")
.with_ragged_row_count(scan.ragged_rows);
if let Some(reason) = scan.truncation {
execution = execution.with_truncation(reason);
}
Ok(ReportAssembler::new(file_source, execution)
.with_analysis_options(&AnalysisOptions::default())
.columns(column_profiles)
.with_quality_data(sample_columns)
.with_row_duplicates(column_stats.row_duplicate_summary())
.with_row_completeness(column_stats.row_completeness_summary())
.build())
}
fn map_io_error(file_path: &Path, error: std::io::Error) -> DataProfilerError {
if error.kind() == std::io::ErrorKind::NotFound {
DataProfilerError::FileNotFound {
path: file_path.display().to_string(),
}
} else {
DataProfilerError::from(error)
}
}
#[cfg(test)]
mod tests {
use super::*;
use dataprof_core::DataType;
use std::io::{Cursor, Write};
use tempfile::NamedTempFile;
fn write_csv(content: &str) -> NamedTempFile {
let mut file = NamedTempFile::new().unwrap();
write!(file, "{}", content).unwrap();
file.flush().unwrap();
file
}
#[test]
fn test_analyze_csv_from_reader_basic() {
let data = b"name,age\nAlice,25\nBob,30\n";
let cursor = Cursor::new(data.as_ref());
let config = CsvParserConfig::default();
let (profiles, _stats, rows, _) = analyze_csv_from_reader(cursor, &config).unwrap();
assert_eq!(rows, 2);
assert_eq!(profiles.len(), 2);
let age = profiles
.iter()
.find(|profile| profile.name == "age")
.unwrap();
assert_eq!(age.total_count, 2);
assert_eq!(age.data_type, DataType::Integer);
}
#[test]
fn test_analyze_csv_from_reader_with_max_rows() {
let data = b"val\n1\n2\n3\n4\n5\n";
let cursor = Cursor::new(data.as_ref());
let config = CsvParserConfig::default().max_rows(Some(3));
let (_profiles, _stats, rows, _) = analyze_csv_from_reader(cursor, &config).unwrap();
assert_eq!(rows, 3);
}
#[test]
fn test_analyze_csv_from_reader_strict_rejects_ragged() {
let data = b"a,b\n1,2\n3,4,5\n";
let cursor = Cursor::new(data.as_ref());
let config = CsvParserConfig::strict();
assert!(analyze_csv_from_reader(cursor, &config).is_err());
}
#[test]
fn test_analyze_csv_from_reader_flexible_handles_ragged() {
let data = b"a,b\n1,2\n3,4,5\n6\n";
let cursor = Cursor::new(data.as_ref());
let config = CsvParserConfig::default();
let (profiles, _stats, rows, _) = analyze_csv_from_reader(cursor, &config).unwrap();
assert_eq!(rows, 3);
assert!(!profiles.is_empty());
}
#[test]
fn test_analyze_csv_from_reader_custom_delimiter() {
let data = b"a;b\n1;2\n3;4\n";
let cursor = Cursor::new(data.as_ref());
let config = CsvParserConfig::default().with_delimiter(b';');
let (profiles, _stats, rows, _) = analyze_csv_from_reader(cursor, &config).unwrap();
assert_eq!(rows, 2);
assert_eq!(profiles.len(), 2);
}
#[test]
fn test_analyze_csv_file_returns_quality_report() {
let csv = write_csv("x,y\n1,a\n2,b\n3,c\n");
let config = CsvParserConfig::default();
let report = analyze_csv_file(csv.path(), &config).unwrap();
assert_eq!(report.column_profiles.len(), 2);
assert_eq!(report.execution.rows_processed, 3);
assert!(report.quality_score().unwrap() >= 0.0);
assert!(report.quality_score().unwrap() <= 100.0);
}
#[test]
fn test_csv_auto_detects_delimiter_from_reader() {
let data = b"name;age\nAlice;25\nBob;30\n";
let cursor = Cursor::new(data.as_ref());
let config = CsvParserConfig::default();
let (profiles, _stats, rows, _) = analyze_csv_from_reader(cursor, &config).unwrap();
assert_eq!(rows, 2);
assert_eq!(profiles.len(), 2);
let age = profiles
.iter()
.find(|profile| profile.name == "age")
.unwrap();
assert_eq!(age.total_count, 2);
}
#[test]
fn test_analyze_csv_basic() {
let csv = write_csv("name,age\nAlice,25\nBob,30\n");
let config = CsvParserConfig::default();
let report = analyze_csv_file(csv.path(), &config).unwrap();
let profiles = &report.column_profiles;
assert_eq!(profiles.len(), 2);
let names: Vec<&str> = profiles
.iter()
.map(|profile| profile.name.as_str())
.collect();
assert!(names.contains(&"name"));
assert!(names.contains(&"age"));
let age = profiles
.iter()
.find(|profile| profile.name == "age")
.unwrap();
assert_eq!(age.total_count, 2);
assert_eq!(age.null_count, 0);
}
#[test]
fn test_analyze_csv_with_nulls() {
let csv = write_csv("name,age,email\nAlice,25,a@b.com\nBob,,\nCharlie,30,c@d.com\n");
let config = CsvParserConfig::default();
let report = analyze_csv_file(csv.path(), &config).unwrap();
let profiles = &report.column_profiles;
let age = profiles
.iter()
.find(|profile| profile.name == "age")
.unwrap();
assert_eq!(age.total_count, 3);
assert_eq!(age.null_count, 1);
let email = profiles
.iter()
.find(|profile| profile.name == "email")
.unwrap();
assert_eq!(email.null_count, 1);
}
#[test]
fn test_analyze_csv_file_returns_quality_report_legacy() {
let csv = write_csv("x,y\n1,a\n2,b\n3,c\n");
let config = CsvParserConfig::default();
let report = analyze_csv_file(csv.path(), &config).unwrap();
assert_eq!(report.column_profiles.len(), 2);
assert_eq!(report.execution.rows_processed, 3);
assert_eq!(report.execution.columns_detected, 2);
assert!(report.quality_score().unwrap() >= 0.0);
assert!(report.quality_score().unwrap() <= 100.0);
}
#[test]
fn test_analyze_csv_file_header_only_preserves_declared_columns() {
let csv = write_csv("name,age\n");
let config = CsvParserConfig::default();
let report = analyze_csv_file(csv.path(), &config).unwrap();
assert_eq!(
report
.column_profiles
.iter()
.map(|column| (column.name.as_str(), column.data_type.clone()))
.collect::<Vec<_>>(),
vec![("name", DataType::String), ("age", DataType::String)]
);
assert_eq!(report.execution.rows_processed, 0);
assert!(report.quality.is_some());
}
#[test]
fn test_analyze_csv_file_empty_file_has_no_columns() {
let csv = write_csv("");
let config = CsvParserConfig::default();
let report = analyze_csv_file(csv.path(), &config).unwrap();
assert!(report.column_profiles.is_empty());
assert_eq!(report.execution.rows_processed, 0);
assert_eq!(report.execution.columns_detected, 0);
assert_eq!(
report.quality_status,
dataprof_runtime::QualityAnalysisStatus::Computed
);
assert!(report.quality.is_some());
assert!(report.quality_score().is_none());
}
#[test]
fn test_analyze_csv_with_max_rows_small_file() {
let csv = write_csv("val\n1\n2\n3\n4\n5\n");
let config = CsvParserConfig::default();
let report = analyze_csv_file(csv.path(), &config).unwrap();
assert_eq!(report.execution.rows_processed, 5);
assert_eq!(report.execution.columns_detected, 1);
assert!((report.execution.sampling_ratio.unwrap_or(1.0) - 1.0).abs() < 0.01);
}
#[test]
fn a_row_cap_that_leaves_rows_unread_is_a_truncation() {
let csv = write_csv("val\n1\n2\n3\n4\n5\n");
let config = CsvParserConfig::default().max_rows(Some(2));
let report = analyze_csv_file(csv.path(), &config).unwrap();
assert_eq!(report.execution.rows_processed, 2);
assert!(!report.execution.source_exhausted);
assert_eq!(
report.execution.truncation_reason,
Some(TruncationReason::MaxRows(2))
);
}
#[test]
fn a_row_cap_at_the_last_row_is_a_complete_scan() {
let csv = write_csv("val\n1\n2\n");
let config = CsvParserConfig::default().max_rows(Some(2));
let report = analyze_csv_file(csv.path(), &config).unwrap();
assert_eq!(report.execution.rows_processed, 2);
assert!(report.execution.source_exhausted);
assert_eq!(report.execution.truncation_reason, None);
}
#[test]
fn short_and_long_rows_are_counted_as_ragged() {
let csv = write_csv("a,b,c\n1,2,3\n4,5\n6,7,8,9\n10,11,12\n");
let report = analyze_csv_file(csv.path(), &CsvParserConfig::default()).unwrap();
assert_eq!(report.execution.rows_processed, 4);
assert_eq!(report.execution.ragged_row_count, 2);
}
#[test]
fn a_ragged_row_past_the_row_cap_is_not_counted() {
let csv = write_csv("a,b,c\n1,2,3\n4,5\n");
let config = CsvParserConfig::default().max_rows(Some(1));
let report = analyze_csv_file(csv.path(), &config).unwrap();
assert_eq!(report.execution.ragged_row_count, 0);
assert_eq!(
report.execution.truncation_reason,
Some(TruncationReason::MaxRows(1))
);
}
#[test]
fn test_analyze_csv_numeric_types_detected() {
let csv = write_csv("int_col,float_col,str_col\n1,1.5,hello\n2,2.5,world\n3,3.5,foo\n");
let config = CsvParserConfig::default();
let report = analyze_csv_file(csv.path(), &config).unwrap();
let profiles = &report.column_profiles;
let int_col = profiles
.iter()
.find(|profile| profile.name == "int_col")
.unwrap();
assert!(matches!(int_col.data_type, DataType::Integer));
let float_col = profiles
.iter()
.find(|profile| profile.name == "float_col")
.unwrap();
assert!(matches!(float_col.data_type, DataType::Float));
let str_col = profiles
.iter()
.find(|profile| profile.name == "str_col")
.unwrap();
assert!(matches!(str_col.data_type, DataType::String));
}
#[test]
fn test_analyze_csv_strict_rejects_ragged_rows() {
let csv = write_csv("a,b\n1,2\n3,4,5\n");
let config = CsvParserConfig::strict();
assert!(analyze_csv_file(csv.path(), &config).is_err());
}
#[test]
fn test_analyze_csv_file_handles_ragged_rows() {
let csv = write_csv("a,b\n1,2\n3,4,5\n6\n");
let config = CsvParserConfig::default();
let report = analyze_csv_file(csv.path(), &config).unwrap();
assert!(!report.column_profiles.is_empty());
}
#[test]
fn test_analyze_csv_with_quality_metrics() {
let csv = write_csv(
"name,age\n\
Alice,25\nBob,\nCharlie,30\nDave,\nEve,28\n\
Frank,\nGrace,35\nHeidi,40\nIvan,\nJudy,22\n",
);
let config = CsvParserConfig::default();
let report = analyze_csv_file(csv.path(), &config).unwrap();
let age = report
.column_profiles
.iter()
.find(|profile| profile.name == "age")
.unwrap();
assert_eq!(age.null_count, 4);
assert_eq!(age.total_count, 10);
assert!(report.quality_score().unwrap() >= 0.0);
}
#[test]
fn detection_records_a_tie_and_an_unlisted_delimiter_only_when_present() {
type Case<'a> = (
&'a str,
&'a [u8],
&'a str,
usize,
Option<&'a str>,
Option<&'a str>,
);
let cases: [Case; 9] = [
(
"tie",
b"id;amount,EUR\n1;10,50;\n2;20,75;\n",
",",
2,
Some(";"),
None,
),
("clear comma", b"a,b,c\n1,2,3\n4,5,6\n", ",", 3, None, None),
(
"clear semicolon",
b"a;b\n1,5;2\n3,5;4\n",
";",
2,
None,
None,
),
(
"unlisted caret",
b"id^amount\n1^10\n2^20\n",
",",
1,
None,
Some("^"),
),
("one column", b"value\n1\n2\n", ",", 1, None, None),
(
"dot inside values",
b"name.first\njohn.doe\njane.roe\n",
",",
1,
None,
None,
),
("header lacks it", b"ratio\n1/2\n3/4\n", ",", 1, None, None),
("a record lacks it", b"a:b\n1:2\n3\n", ",", 1, None, None),
("header only", b"a^b\n", ",", 1, None, None),
];
let mut wrong = Vec::new();
for (case, data, delimiter, header_fields, runner_up, unlisted) in cases {
let detection = detect_delimiter_detailed(Cursor::new(data)).unwrap();
let seen = (
detection.delimiter.as_str(),
detection.header_fields,
detection.runner_up.as_deref(),
detection.unlisted_delimiter.as_deref(),
);
if seen != (delimiter, header_fields, runner_up, unlisted) {
wrong.push(format!("{case}: {seen:?}"));
}
}
assert!(wrong.is_empty(), "{}", wrong.join("\n"));
}
#[test]
fn test_detect_delimiter_comma() {
let data = b"name,age,city\nAlice,25,NYC\nBob,30,London\n";
assert_eq!(detect_delimiter(Cursor::new(data.as_ref())).unwrap(), b',');
}
#[test]
fn test_detect_delimiter_semicolon() {
let data = b"name;age;city\nAlice;25;NYC\nBob;30;London\n";
assert_eq!(detect_delimiter(Cursor::new(data.as_ref())).unwrap(), b';');
}
#[test]
fn test_detect_delimiter_pipe() {
let data = b"name|age|city\nAlice|25|NYC\nBob|30|London\n";
assert_eq!(detect_delimiter(Cursor::new(data.as_ref())).unwrap(), b'|');
}
#[test]
fn test_detect_delimiter_tab() {
let data = b"name\tage\tcity\nAlice\t25\tNYC\nBob\t30\tLondon\n";
assert_eq!(detect_delimiter(Cursor::new(data.as_ref())).unwrap(), b'\t');
}
#[test]
fn test_detect_delimiter_from_path_semicolon() {
let csv = write_csv("id;name;salary\n1;Alice;50000\n2;Bob;60000\n");
assert_eq!(detect_delimiter_from_path(csv.path()).unwrap(), b';');
}
#[test]
fn test_detect_delimiter_blank_records_do_not_outvote_header() {
let data = b"name,age\n\n\n";
assert_eq!(detect_delimiter(Cursor::new(data.as_ref())).unwrap(), b',');
let semicolon = b"name;age\n\n";
assert_eq!(
detect_delimiter(Cursor::new(semicolon.as_ref())).unwrap(),
b';'
);
}
#[test]
fn test_detect_delimiter_quoted_fields() {
let data = b"name;desc;val\n\"hello;world\";foo;1\nbar;baz;2\n";
assert_eq!(detect_delimiter(Cursor::new(data.as_ref())).unwrap(), b';');
}
#[test]
fn test_detect_delimiter_quoted_embedded_newline() {
let data = b"id,comment\n1,\"line one\nline two\"\n2,\"normal\"\n3,\"has \"\"quotes\"\" inside\"\n";
assert_eq!(detect_delimiter(Cursor::new(data.as_ref())).unwrap(), b',');
}
#[test]
fn test_detect_delimiter_absent_candidate_never_beats_real_one() {
let data = b"a,b,c\n1,2,3\nragged line without delimiters\n4,5,6\n";
assert_eq!(detect_delimiter(Cursor::new(data.as_ref())).unwrap(), b',');
}
#[test]
fn test_detect_delimiter_single_column_falls_back_to_comma() {
let data = b"value\n1\n2\n3\n";
assert_eq!(detect_delimiter(Cursor::new(data.as_ref())).unwrap(), b',');
}
#[test]
fn test_detect_delimiter_two_column_semicolon_with_commas_in_text() {
let data = b"id;note\n1;hello, world\n2;foo, bar, baz\n3;plain\n";
assert_eq!(detect_delimiter(Cursor::new(data.as_ref())).unwrap(), b';');
}
#[test]
fn test_detect_delimiter_never_splits_rows_the_header_does_not() {
let data = b"key\n123|456\n123|789\n";
assert_eq!(detect_delimiter(Cursor::new(data.as_ref())).unwrap(), b',');
}
#[test]
fn test_detect_delimiter_header_split_beats_uniform_rows() {
let data = b"a,b\n1,2,\n3,4,\n";
assert_eq!(detect_delimiter(Cursor::new(data.as_ref())).unwrap(), b',');
}
#[test]
fn test_detect_delimiter_trailing_empty_fields_still_agree() {
let data = b"id;amount,EUR\n1;10,50;\n2;20;\n";
assert_eq!(detect_delimiter(Cursor::new(data.as_ref())).unwrap(), b';');
}
#[test]
fn test_detect_delimiter_quoted_empty_trailing_field_still_agrees() {
let data = b"id;amount,EUR\n1;10,50;\"\"\n2;20;\"\"\n";
assert_eq!(detect_delimiter(Cursor::new(data.as_ref())).unwrap(), b';');
}
#[test]
fn test_detect_delimiter_exact_width_beats_cut_trailing_fields() {
let data = b"id,part;value\nx;y,z,\nu;v,w,\n";
assert_eq!(detect_delimiter(Cursor::new(data.as_ref())).unwrap(), b';');
}
#[test]
fn test_detect_delimiter_skips_a_utf8_bom() {
let data = b"\xEF\xBB\xBF\nid;value\n1;2\n3;4\n";
assert_eq!(detect_delimiter(Cursor::new(data.as_ref())).unwrap(), b';');
let no_blank = b"\xEF\xBB\xBFid;value\n1;2\n";
assert_eq!(
detect_delimiter(Cursor::new(no_blank.as_ref())).unwrap(),
b';'
);
}
#[test]
fn test_detect_delimiter_lone_cr_ends_a_record() {
let data = b"key\r123|456\r123|789\r";
assert_eq!(detect_delimiter(Cursor::new(data.as_ref())).unwrap(), b',');
let semicolon = b"id;value\r1;10\r2;20\r";
assert_eq!(
detect_delimiter(Cursor::new(semicolon.as_ref())).unwrap(),
b';'
);
}
#[test]
fn test_count_fields_reports_trailing_empty_fields() {
assert_eq!(count_fields(b"1;10,50;", b';', b'"'), (3, 1));
assert_eq!(count_fields(b"a;;", b';', b'"'), (3, 2));
assert_eq!(count_fields(b"a;\"\"", b';', b'"'), (2, 1));
assert_eq!(count_fields(b"a;\"\"\"\"", b';', b'"'), (2, 0));
assert_eq!(count_fields(b"a; ", b';', b'"'), (2, 0));
assert_eq!(count_fields(b"a;b", b';', b'"'), (2, 0));
assert_eq!(count_fields(b"\"x;y\";", b';', b'"'), (2, 1));
}
#[test]
fn test_split_sample_records_quote_aware() {
let (records, last_unterminated) =
split_sample_records(b"a,b\r\n1,\"x\ny\"\n2,z\n", b',', b'"');
assert_eq!(records, ["a,b", "1,\"x\ny\"", "2,z"].map(str::as_bytes));
assert!(!last_unterminated);
let (records, last_unterminated) =
split_sample_records(b"a,b\n1,\"cut mid-fie", b',', b'"');
assert_eq!(records, ["a,b", "1,\"cut mid-fie"].map(str::as_bytes));
assert!(last_unterminated);
let (records, last_unterminated) =
split_sample_records(b"a,b\r1,\"x\ry\"\r2,z\r", b',', b'"');
assert_eq!(records, ["a,b", "1,\"x\ry\"", "2,z"].map(str::as_bytes));
assert!(!last_unterminated);
let (records, _) = split_sample_records(b"a\rb\r\nc\n", b',', b'"');
assert_eq!(records, ["a", "b", "c"].map(str::as_bytes));
}
#[test]
fn test_detect_delimiter_full_sample_ending_on_record_boundary() {
let mut data = Vec::from(&b"a;b;c\n"[..]);
while data.len() < 4090 {
data.extend_from_slice(b"1;2;3\n");
}
while data.len() < 4095 {
data.push(b'#');
}
data.push(b'\n');
assert_eq!(data.len(), 4096);
assert_eq!(
detect_delimiter(Cursor::new(data.as_slice())).unwrap(),
b';'
);
}
#[test]
fn test_detect_delimiter_reads_past_the_sample_to_end_the_header() {
let issue = format!("\"{}\";value\n1;10\n2;20\n", "x".repeat(4094));
assert_eq!(
detect_delimiter(Cursor::new(issue.as_bytes())).unwrap(),
b';'
);
let mut exact = "a".repeat(4093).into_bytes();
exact.extend_from_slice(b";b\n1;2\n");
let mut reader = Cursor::new(exact.as_slice());
let (delimiter, head) = detect_delimiter_with_head(&mut reader).unwrap();
assert_eq!((delimiter, head.len()), (b';', 4097));
}
#[test]
fn test_detect_delimiter_with_head_returns_every_byte_it_read() {
let data = format!("{};value\n1;2\n", "a".repeat(20_000));
let mut reader = Cursor::new(data.as_bytes());
let (delimiter, head) = detect_delimiter_with_head(&mut reader).unwrap();
assert_eq!(delimiter, b';');
assert_eq!(head.len(), 32_768.min(data.len()));
let mut rest = Vec::new();
reader.read_to_end(&mut rest).unwrap();
assert_eq!([head, rest].concat(), data.as_bytes());
}
#[test]
fn test_detect_delimiter_refuses_a_header_past_the_bound() {
let data = format!("{};value\n1;2\n", "h".repeat(MAX_HEADER_SAMPLE_BYTES));
let error = detect_delimiter(Cursor::new(data.as_bytes())).unwrap_err();
assert_eq!(error.kind(), std::io::ErrorKind::InvalidData);
assert!(error.to_string().contains("csv_delimiter"), "{error}");
let error = detect_delimiter_with_head(&mut Cursor::new(data.as_bytes())).unwrap_err();
assert!(matches!(error, DataProfilerError::CsvParsingError { .. }));
let short = "h".repeat(MAX_HEADER_SAMPLE_BYTES);
assert_eq!(
detect_delimiter(Cursor::new(short.as_bytes())).unwrap(),
b','
);
}
#[test]
fn test_detect_delimiter_quote_rule_is_per_candidate() {
for (header, row, expected) in [
(&b"id,height;\"in\n"[..], &b"1,5\n"[..], b','),
(&b"id;a;b,\"c\n"[..], &b"1;2;3\n"[..], b';'),
] {
let mut data = header.to_vec();
while data.len() <= MAX_HEADER_SAMPLE_BYTES {
data.extend_from_slice(row);
}
assert_eq!(
detect_delimiter(Cursor::new(data.as_slice())).unwrap(),
expected,
"{}",
String::from_utf8_lossy(header)
);
}
}
#[test]
fn test_detect_delimiter_refuses_a_head_of_blank_records() {
let mut data = vec![b'\n'; MAX_HEADER_SAMPLE_BYTES + 1];
data.extend_from_slice(b"id;value\n1;2\n");
assert!(detect_delimiter(Cursor::new(data.as_slice())).is_err());
let blank = vec![b'\n'; 10];
assert_eq!(
detect_delimiter(Cursor::new(blank.as_slice())).unwrap(),
b','
);
}
#[test]
fn test_detect_delimiter_mid_field_quote_is_literal() {
let mut data = b"id;height\"in;note\n".to_vec();
while data.len() <= MAX_HEADER_SAMPLE_BYTES {
data.extend_from_slice(b"1;5;x\n");
}
assert_eq!(
detect_delimiter(Cursor::new(data.as_slice())).unwrap(),
b';'
);
let (records, last_unterminated) =
split_sample_records(b"a;b\"c;d\n1;\"x\ny\";2\n", b';', b'"');
assert_eq!(records, ["a;b\"c;d", "1;\"x\ny\";2"].map(str::as_bytes));
assert!(!last_unterminated);
assert_eq!(count_fields(b"id\"x;value", b';', b'"'), (2, 0));
assert_eq!(count_fields(b"\"a;\"\"b\";c", b';', b'"'), (2, 0));
assert_eq!(
detect_delimiter(Cursor::new(&b"id\"x;value\n1;2\n3;4\n"[..])).unwrap(),
b';'
);
}
}