use csv::ReaderBuilder;
use std::io::Read;
use std::path::Path;
use dataprof_core::{
ColumnProfile, DataProfilerError, DataSource, ExecutionMetadata, FileFormat, QualityDimension,
SemanticHints,
};
use dataprof_runtime::{
ProfileReport, ReportAssembler, StreamingColumnCollection, profile_builder,
};
mod memmap_reader;
mod quote_state;
mod robust_csv;
pub use memmap_reader::MemoryMappedCsvReader;
pub use quote_state::{QuoteOutcome, QuoteTrackingReader, unterminated_quote_error};
pub use robust_csv::{
CsvDiagnostics, CsvParseOutput, EncodingDiagnostic, RobustCsvParser, RobustParseResult,
diagnose_non_utf8,
};
#[derive(Debug, Clone)]
pub struct CsvParserConfig {
pub flexible: bool,
pub delimiter: Option<u8>,
pub quote_char: u8,
pub trim_whitespace: bool,
pub has_header: bool,
pub max_rows: Option<usize>,
}
impl Default for CsvParserConfig {
fn default() -> Self {
Self {
flexible: true,
delimiter: None,
quote_char: b'"',
trim_whitespace: false,
has_header: true,
max_rows: None,
}
}
}
impl CsvParserConfig {
pub fn strict() -> Self {
Self {
flexible: false,
..Default::default()
}
}
pub fn with_delimiter(mut self, delimiter: u8) -> Self {
self.delimiter = Some(delimiter);
self
}
pub fn has_header(mut self, yes: bool) -> Self {
self.has_header = yes;
self
}
pub fn max_rows(mut self, max: Option<usize>) -> Self {
self.max_rows = max;
self
}
}
pub(crate) fn count_fields(line: &str, delimiter: char) -> usize {
let mut count = 1;
let mut in_quotes = false;
let mut chars = line.chars().peekable();
while let Some(ch) = chars.next() {
if ch == '"' {
if chars.peek() == Some(&'"') {
chars.next();
} else {
in_quotes = !in_quotes;
}
} else if ch == delimiter && !in_quotes {
count += 1;
}
}
count
}
fn split_sample_records(text: &str) -> (Vec<&str>, bool) {
let bytes = text.as_bytes();
let mut records = Vec::new();
let mut in_quotes = false;
let mut start = 0usize;
let mut i = 0usize;
while i < bytes.len() {
match bytes[i] {
b'"' => {
if in_quotes && bytes.get(i + 1) == Some(&b'"') {
i += 1; } else {
in_quotes = !in_quotes;
}
}
b'\n' if !in_quotes => {
let end = if i > start && bytes[i - 1] == b'\r' {
i - 1
} else {
i
};
records.push(&text[start..end]);
start = i + 1;
}
_ => {}
}
i += 1;
}
let last_unterminated = start < text.len();
if last_unterminated {
records.push(text[start..].trim_end_matches('\r'));
}
(records, last_unterminated)
}
pub fn detect_delimiter<R: Read>(reader: R) -> std::io::Result<u8> {
const SAMPLE_BYTES: u64 = 4096;
let mut preamble = Vec::new();
let mut take = reader.take(SAMPLE_BYTES);
take.read_to_end(&mut preamble)?;
let truncated = preamble.len() as u64 == SAMPLE_BYTES;
let preamble_str = String::from_utf8_lossy(&preamble);
let (mut records, last_unterminated) = split_sample_records(&preamble_str);
if truncated && last_unterminated && records.len() > 1 {
records.pop();
}
let delimiters = *b",;\t|";
let mut best_delimiter = b',';
let mut best_score = (false, 0usize, 0usize);
for &delimiter in &delimiters {
let mut counts = std::collections::HashMap::new();
let scored = records
.iter()
.filter(|record| !record.trim().is_empty())
.take(5);
for record in scored {
let fields = count_fields(record, delimiter as char);
*counts.entry(fields).or_insert(0usize) += 1;
}
let (modal_fields, consistency) = counts
.into_iter()
.max_by_key(|&(fields, count)| (count, fields))
.unwrap_or((0, 0));
let score = (modal_fields > 1, consistency, modal_fields);
if score > best_score {
best_score = score;
best_delimiter = delimiter;
}
}
Ok(best_delimiter)
}
pub fn detect_delimiter_from_path(path: &Path) -> std::io::Result<u8> {
let file = std::fs::File::open(path)?;
detect_delimiter(std::io::BufReader::new(file))
}
pub fn analyze_csv_from_reader<R: Read>(
reader: R,
config: &CsvParserConfig,
) -> Result<
(
Vec<ColumnProfile>,
StreamingColumnCollection,
usize,
Vec<String>,
),
DataProfilerError,
> {
analyze_csv_from_reader_with_hints(reader, config, &SemanticHints::default())
}
pub fn analyze_csv_from_reader_with_hints<R: Read>(
reader: R,
config: &CsvParserConfig,
semantic_hints: &SemanticHints,
) -> Result<
(
Vec<ColumnProfile>,
StreamingColumnCollection,
usize,
Vec<String>,
),
DataProfilerError,
> {
let mut csv_builder = ReaderBuilder::new();
csv_builder.has_headers(config.has_header);
csv_builder.flexible(config.flexible);
csv_builder.quote(config.quote_char);
if config.trim_whitespace {
csv_builder.trim(csv::Trim::All);
}
let mut actual_delimiter = config.delimiter;
let boxed_reader: Box<dyn std::io::Read + '_> = if config.delimiter.is_none() {
let mut preamble = Vec::new();
let mut take = reader.take(4096);
take.read_to_end(&mut preamble)?;
let detected = detect_delimiter(std::io::Cursor::new(&preamble)).unwrap_or(b',');
actual_delimiter = Some(detected);
Box::new(std::io::Cursor::new(preamble).chain(take.into_inner()))
} else {
Box::new(reader)
};
if let Some(delimiter) = actual_delimiter {
csv_builder.delimiter(delimiter);
}
let mut csv_reader = csv_builder.from_reader(boxed_reader);
let header_names: Vec<String> = if config.has_header {
let headers = csv_reader.headers()?;
let names: Vec<String> = headers.iter().map(|header| header.to_string()).collect();
dataprof_core::validate_unique_column_names(&names, "CSV header")?;
names
} else {
let headers = csv_reader.headers()?;
(0..headers.len())
.map(|index| format!("column_{index}"))
.collect()
};
let mut column_stats = StreamingColumnCollection::new().with_semantic_hints(semantic_hints);
column_stats.init_columns(&header_names);
let mut rows_read = 0;
for result in csv_reader.records() {
if let Some(max_rows) = config.max_rows
&& rows_read >= max_rows
{
break;
}
let record = result?;
let mut values: Vec<String> = record.iter().map(|value| value.to_string()).collect();
let header_len = header_names.len();
if values.len() < header_len {
values.resize(header_len, String::new());
} else if values.len() > header_len {
values.truncate(header_len);
}
column_stats.process_record(&header_names, values);
rows_read += 1;
}
let profiles = profile_builder::profiles_from_streaming_with_hints(
&column_stats,
false,
false,
None,
semantic_hints,
);
Ok((profiles, column_stats, rows_read, header_names))
}
pub fn analyze_csv_file(
file_path: &Path,
config: &CsvParserConfig,
) -> Result<ProfileReport, DataProfilerError> {
analyze_csv_file_with_dimensions(file_path, config, None)
}
pub fn analyze_csv_file_with_dimensions(
file_path: &Path,
config: &CsvParserConfig,
quality_dimensions: Option<&[QualityDimension]>,
) -> Result<ProfileReport, DataProfilerError> {
analyze_csv_file_with_dimensions_and_hints(
file_path,
config,
quality_dimensions,
&SemanticHints::default(),
)
}
pub fn analyze_csv_file_with_dimensions_and_hints(
file_path: &Path,
config: &CsvParserConfig,
quality_dimensions: Option<&[QualityDimension]>,
semantic_hints: &SemanticHints,
) -> Result<ProfileReport, DataProfilerError> {
let metadata = std::fs::metadata(file_path).map_err(|error| map_io_error(file_path, error))?;
let start = std::time::Instant::now();
let file = std::fs::File::open(file_path).map_err(|error| map_io_error(file_path, error))?;
let buf_reader = std::io::BufReader::new(file);
let (column_profiles, column_stats, rows_read, _header_names) =
analyze_csv_from_reader_with_hints(buf_reader, config, semantic_hints)?;
let file_source = DataSource::File {
path: file_path.display().to_string(),
format: FileFormat::Csv,
size_bytes: metadata.len(),
modified_at: None,
parquet_metadata: None,
};
let sample_columns = profile_builder::quality_check_samples(&column_stats);
let scan_time_ms = start.elapsed().as_millis();
let num_columns = column_profiles.len();
let mut assembler = ReportAssembler::new(
file_source,
ExecutionMetadata::new(rows_read, num_columns, scan_time_ms).with_engine("csv"),
)
.columns(column_profiles)
.with_quality_data(sample_columns)
.with_row_duplicates(column_stats.row_duplicate_summary())
.with_row_completeness(column_stats.row_completeness_summary())
.with_exact_value_hint_bindings(column_stats.semantic_hint_bindings())
.with_semantic_hints(semantic_hints.clone());
if let Some(dimensions) = quality_dimensions {
assembler = assembler.with_requested_dimensions(dimensions.to_vec());
}
Ok(assembler.build())
}
fn map_io_error(file_path: &Path, error: std::io::Error) -> DataProfilerError {
if error.kind() == std::io::ErrorKind::NotFound {
DataProfilerError::FileNotFound {
path: file_path.display().to_string(),
}
} else {
DataProfilerError::from(error)
}
}
#[cfg(test)]
mod tests {
use super::*;
use dataprof_core::DataType;
use std::io::{Cursor, Write};
use tempfile::NamedTempFile;
fn write_csv(content: &str) -> NamedTempFile {
let mut file = NamedTempFile::new().unwrap();
write!(file, "{}", content).unwrap();
file.flush().unwrap();
file
}
#[test]
fn test_analyze_csv_from_reader_basic() {
let data = b"name,age\nAlice,25\nBob,30\n";
let cursor = Cursor::new(data.as_ref());
let config = CsvParserConfig::default();
let (profiles, _stats, rows, _) = analyze_csv_from_reader(cursor, &config).unwrap();
assert_eq!(rows, 2);
assert_eq!(profiles.len(), 2);
let age = profiles
.iter()
.find(|profile| profile.name == "age")
.unwrap();
assert_eq!(age.total_count, 2);
assert_eq!(age.data_type, DataType::Integer);
}
#[test]
fn test_analyze_csv_from_reader_with_max_rows() {
let data = b"val\n1\n2\n3\n4\n5\n";
let cursor = Cursor::new(data.as_ref());
let config = CsvParserConfig::default().max_rows(Some(3));
let (_profiles, _stats, rows, _) = analyze_csv_from_reader(cursor, &config).unwrap();
assert_eq!(rows, 3);
}
#[test]
fn test_analyze_csv_from_reader_strict_rejects_ragged() {
let data = b"a,b\n1,2\n3,4,5\n";
let cursor = Cursor::new(data.as_ref());
let config = CsvParserConfig::strict();
assert!(analyze_csv_from_reader(cursor, &config).is_err());
}
#[test]
fn test_analyze_csv_from_reader_flexible_handles_ragged() {
let data = b"a,b\n1,2\n3,4,5\n6\n";
let cursor = Cursor::new(data.as_ref());
let config = CsvParserConfig::default();
let (profiles, _stats, rows, _) = analyze_csv_from_reader(cursor, &config).unwrap();
assert_eq!(rows, 3);
assert!(!profiles.is_empty());
}
#[test]
fn test_analyze_csv_from_reader_custom_delimiter() {
let data = b"a;b\n1;2\n3;4\n";
let cursor = Cursor::new(data.as_ref());
let config = CsvParserConfig::default().with_delimiter(b';');
let (profiles, _stats, rows, _) = analyze_csv_from_reader(cursor, &config).unwrap();
assert_eq!(rows, 2);
assert_eq!(profiles.len(), 2);
}
#[test]
fn test_analyze_csv_file_returns_quality_report() {
let csv = write_csv("x,y\n1,a\n2,b\n3,c\n");
let config = CsvParserConfig::default();
let report = analyze_csv_file(csv.path(), &config).unwrap();
assert_eq!(report.column_profiles.len(), 2);
assert_eq!(report.execution.rows_processed, 3);
assert!(report.quality_score().unwrap() >= 0.0);
assert!(report.quality_score().unwrap() <= 100.0);
}
#[test]
fn test_csv_auto_detects_delimiter_from_reader() {
let data = b"name;age\nAlice;25\nBob;30\n";
let cursor = Cursor::new(data.as_ref());
let config = CsvParserConfig::default();
let (profiles, _stats, rows, _) = analyze_csv_from_reader(cursor, &config).unwrap();
assert_eq!(rows, 2);
assert_eq!(profiles.len(), 2);
let age = profiles
.iter()
.find(|profile| profile.name == "age")
.unwrap();
assert_eq!(age.total_count, 2);
}
#[test]
fn test_analyze_csv_basic() {
let csv = write_csv("name,age\nAlice,25\nBob,30\n");
let config = CsvParserConfig::default();
let report = analyze_csv_file(csv.path(), &config).unwrap();
let profiles = &report.column_profiles;
assert_eq!(profiles.len(), 2);
let names: Vec<&str> = profiles
.iter()
.map(|profile| profile.name.as_str())
.collect();
assert!(names.contains(&"name"));
assert!(names.contains(&"age"));
let age = profiles
.iter()
.find(|profile| profile.name == "age")
.unwrap();
assert_eq!(age.total_count, 2);
assert_eq!(age.null_count, 0);
}
#[test]
fn test_analyze_csv_with_nulls() {
let csv = write_csv("name,age,email\nAlice,25,a@b.com\nBob,,\nCharlie,30,c@d.com\n");
let config = CsvParserConfig::default();
let report = analyze_csv_file(csv.path(), &config).unwrap();
let profiles = &report.column_profiles;
let age = profiles
.iter()
.find(|profile| profile.name == "age")
.unwrap();
assert_eq!(age.total_count, 3);
assert_eq!(age.null_count, 1);
let email = profiles
.iter()
.find(|profile| profile.name == "email")
.unwrap();
assert_eq!(email.null_count, 1);
}
#[test]
fn test_analyze_csv_file_returns_quality_report_legacy() {
let csv = write_csv("x,y\n1,a\n2,b\n3,c\n");
let config = CsvParserConfig::default();
let report = analyze_csv_file(csv.path(), &config).unwrap();
assert_eq!(report.column_profiles.len(), 2);
assert_eq!(report.execution.rows_processed, 3);
assert_eq!(report.execution.columns_detected, 2);
assert!(report.quality_score().unwrap() >= 0.0);
assert!(report.quality_score().unwrap() <= 100.0);
}
#[test]
fn test_analyze_csv_file_header_only_preserves_declared_columns() {
let csv = write_csv("name,age\n");
let config = CsvParserConfig::default();
let report = analyze_csv_file(csv.path(), &config).unwrap();
assert_eq!(
report
.column_profiles
.iter()
.map(|column| (column.name.as_str(), column.data_type.clone()))
.collect::<Vec<_>>(),
vec![("name", DataType::String), ("age", DataType::String)]
);
assert_eq!(report.execution.rows_processed, 0);
assert!(report.quality.is_some());
}
#[test]
fn test_analyze_csv_file_empty_file_has_no_columns() {
let csv = write_csv("");
let config = CsvParserConfig::default();
let report = analyze_csv_file(csv.path(), &config).unwrap();
assert!(report.column_profiles.is_empty());
assert_eq!(report.execution.rows_processed, 0);
assert_eq!(report.execution.columns_detected, 0);
assert_eq!(
report.quality_status,
dataprof_runtime::QualityAnalysisStatus::Computed
);
assert!(report.quality.is_some());
assert!(report.quality_score().is_none());
}
#[test]
fn test_analyze_csv_with_max_rows_small_file() {
let csv = write_csv("val\n1\n2\n3\n4\n5\n");
let config = CsvParserConfig::default();
let report = analyze_csv_file(csv.path(), &config).unwrap();
assert_eq!(report.execution.rows_processed, 5);
assert_eq!(report.execution.columns_detected, 1);
assert!((report.execution.sampling_ratio.unwrap_or(1.0) - 1.0).abs() < 0.01);
}
#[test]
fn test_analyze_csv_numeric_types_detected() {
let csv = write_csv("int_col,float_col,str_col\n1,1.5,hello\n2,2.5,world\n3,3.5,foo\n");
let config = CsvParserConfig::default();
let report = analyze_csv_file(csv.path(), &config).unwrap();
let profiles = &report.column_profiles;
let int_col = profiles
.iter()
.find(|profile| profile.name == "int_col")
.unwrap();
assert!(matches!(int_col.data_type, DataType::Integer));
let float_col = profiles
.iter()
.find(|profile| profile.name == "float_col")
.unwrap();
assert!(matches!(float_col.data_type, DataType::Float));
let str_col = profiles
.iter()
.find(|profile| profile.name == "str_col")
.unwrap();
assert!(matches!(str_col.data_type, DataType::String));
}
#[test]
fn test_analyze_csv_strict_rejects_ragged_rows() {
let csv = write_csv("a,b\n1,2\n3,4,5\n");
let config = CsvParserConfig::strict();
assert!(analyze_csv_file(csv.path(), &config).is_err());
}
#[test]
fn test_analyze_csv_file_handles_ragged_rows() {
let csv = write_csv("a,b\n1,2\n3,4,5\n6\n");
let config = CsvParserConfig::default();
let report = analyze_csv_file(csv.path(), &config).unwrap();
assert!(!report.column_profiles.is_empty());
}
#[test]
fn test_analyze_csv_with_quality_metrics() {
let csv = write_csv(
"name,age\n\
Alice,25\nBob,\nCharlie,30\nDave,\nEve,28\n\
Frank,\nGrace,35\nHeidi,40\nIvan,\nJudy,22\n",
);
let config = CsvParserConfig::default();
let report = analyze_csv_file(csv.path(), &config).unwrap();
let age = report
.column_profiles
.iter()
.find(|profile| profile.name == "age")
.unwrap();
assert_eq!(age.null_count, 4);
assert_eq!(age.total_count, 10);
assert!(report.quality_score().unwrap() >= 0.0);
}
#[test]
fn test_detect_delimiter_comma() {
let data = b"name,age,city\nAlice,25,NYC\nBob,30,London\n";
assert_eq!(detect_delimiter(Cursor::new(data.as_ref())).unwrap(), b',');
}
#[test]
fn test_detect_delimiter_semicolon() {
let data = b"name;age;city\nAlice;25;NYC\nBob;30;London\n";
assert_eq!(detect_delimiter(Cursor::new(data.as_ref())).unwrap(), b';');
}
#[test]
fn test_detect_delimiter_pipe() {
let data = b"name|age|city\nAlice|25|NYC\nBob|30|London\n";
assert_eq!(detect_delimiter(Cursor::new(data.as_ref())).unwrap(), b'|');
}
#[test]
fn test_detect_delimiter_tab() {
let data = b"name\tage\tcity\nAlice\t25\tNYC\nBob\t30\tLondon\n";
assert_eq!(detect_delimiter(Cursor::new(data.as_ref())).unwrap(), b'\t');
}
#[test]
fn test_detect_delimiter_from_path_semicolon() {
let csv = write_csv("id;name;salary\n1;Alice;50000\n2;Bob;60000\n");
assert_eq!(detect_delimiter_from_path(csv.path()).unwrap(), b';');
}
#[test]
fn test_detect_delimiter_blank_records_do_not_outvote_header() {
let data = b"name,age\n\n\n";
assert_eq!(detect_delimiter(Cursor::new(data.as_ref())).unwrap(), b',');
let semicolon = b"name;age\n\n";
assert_eq!(
detect_delimiter(Cursor::new(semicolon.as_ref())).unwrap(),
b';'
);
}
#[test]
fn test_detect_delimiter_quoted_fields() {
let data = b"name;desc;val\n\"hello;world\";foo;1\nbar;baz;2\n";
assert_eq!(detect_delimiter(Cursor::new(data.as_ref())).unwrap(), b';');
}
#[test]
fn test_detect_delimiter_quoted_embedded_newline() {
let data = b"id,comment\n1,\"line one\nline two\"\n2,\"normal\"\n3,\"has \"\"quotes\"\" inside\"\n";
assert_eq!(detect_delimiter(Cursor::new(data.as_ref())).unwrap(), b',');
}
#[test]
fn test_detect_delimiter_absent_candidate_never_beats_real_one() {
let data = b"a,b,c\n1,2,3\nragged line without delimiters\n4,5,6\n";
assert_eq!(detect_delimiter(Cursor::new(data.as_ref())).unwrap(), b',');
}
#[test]
fn test_detect_delimiter_single_column_falls_back_to_comma() {
let data = b"value\n1\n2\n3\n";
assert_eq!(detect_delimiter(Cursor::new(data.as_ref())).unwrap(), b',');
}
#[test]
fn test_detect_delimiter_two_column_semicolon_with_commas_in_text() {
let data = b"id;note\n1;hello, world\n2;foo, bar, baz\n3;plain\n";
assert_eq!(detect_delimiter(Cursor::new(data.as_ref())).unwrap(), b';');
}
#[test]
fn test_split_sample_records_quote_aware() {
let (records, last_unterminated) = split_sample_records("a,b\r\n1,\"x\ny\"\n2,z\n");
assert_eq!(records, vec!["a,b", "1,\"x\ny\"", "2,z"]);
assert!(!last_unterminated);
let (records, last_unterminated) = split_sample_records("a,b\n1,\"cut mid-fie");
assert_eq!(records, vec!["a,b", "1,\"cut mid-fie"]);
assert!(last_unterminated);
}
#[test]
fn test_detect_delimiter_full_sample_ending_on_record_boundary() {
let mut data = Vec::from(&b"a;b;c\n"[..]);
while data.len() < 4090 {
data.extend_from_slice(b"1;2;3\n");
}
while data.len() < 4095 {
data.push(b'#');
}
data.push(b'\n');
assert_eq!(data.len(), 4096);
assert_eq!(
detect_delimiter(Cursor::new(data.as_slice())).unwrap(),
b';'
);
}
}