Skip to main content

dataprof_csv/
lib.rs

1use csv::ReaderBuilder;
2use std::io::Read;
3use std::path::Path;
4
5use dataprof_core::{
6    ColumnProfile, DataProfilerError, DataSource, ExecutionMetadata, FileFormat, QualityDimension,
7    SemanticHints,
8};
9use dataprof_runtime::{
10    ProfileReport, ReportAssembler, StreamingColumnCollection, profile_builder,
11};
12
13mod memmap_reader;
14mod robust_csv;
15
16pub use memmap_reader::MemoryMappedCsvReader;
17pub use robust_csv::{CsvDiagnostics, RobustCsvParser, RobustParseResult};
18
19/// Configuration for CSV parsing and analysis.
20///
21/// Replaces the 7 separate `analyze_csv_*` functions with a single config struct.
22/// Supports both file-based and reader-based (streaming) entry points.
23#[derive(Debug, Clone)]
24pub struct CsvParserConfig {
25    /// Allow ragged rows (flexible field counts).
26    pub flexible: bool,
27    /// Custom delimiter (None = comma).
28    pub delimiter: Option<u8>,
29    /// Quote character.
30    pub quote_char: u8,
31    /// Trim whitespace from fields.
32    pub trim_whitespace: bool,
33    /// Whether the first row contains column headers.
34    /// When `false`, columns are named `column_0`, `column_1`, etc.
35    pub has_header: bool,
36    /// Maximum rows to process (None = all rows).
37    pub max_rows: Option<usize>,
38}
39
40impl Default for CsvParserConfig {
41    fn default() -> Self {
42        Self {
43            flexible: true,
44            delimiter: None,
45            quote_char: b'"',
46            trim_whitespace: false,
47            has_header: true,
48            max_rows: None,
49        }
50    }
51}
52
53impl CsvParserConfig {
54    /// Create a strict config that rejects ragged rows.
55    pub fn strict() -> Self {
56        Self {
57            flexible: false,
58            ..Default::default()
59        }
60    }
61
62    /// Set the delimiter.
63    pub fn with_delimiter(mut self, delimiter: u8) -> Self {
64        self.delimiter = Some(delimiter);
65        self
66    }
67
68    /// Set whether the first row contains column headers.
69    pub fn has_header(mut self, yes: bool) -> Self {
70        self.has_header = yes;
71        self
72    }
73
74    /// Set maximum number of rows to evaluate.
75    pub fn max_rows(mut self, max: Option<usize>) -> Self {
76        self.max_rows = max;
77        self
78    }
79}
80
81/// Count fields in a line for a given delimiter, respecting quoted sections.
82fn count_fields(line: &str, delimiter: char) -> usize {
83    let mut count = 1;
84    let mut in_quotes = false;
85    let mut chars = line.chars().peekable();
86    while let Some(ch) = chars.next() {
87        if ch == '"' {
88            if chars.peek() == Some(&'"') {
89                chars.next();
90            } else {
91                in_quotes = !in_quotes;
92            }
93        } else if ch == delimiter && !in_quotes {
94            count += 1;
95        }
96    }
97    count
98}
99
100/// Detect the most likely CSV delimiter by sampling up to 4 KB of data.
101///
102/// Tests comma, semicolon, tab, and pipe. Returns the delimiter that produces
103/// the most consistent field counts across the first 5 sample lines, preferring
104/// delimiters that yield more than one field.
105///
106/// Falls back to comma if no clear winner is found.
107pub fn detect_delimiter<R: Read>(reader: R) -> std::io::Result<u8> {
108    let mut preamble = Vec::new();
109    let mut take = reader.take(4096);
110    take.read_to_end(&mut preamble)?;
111
112    let preamble_str = String::from_utf8_lossy(&preamble);
113    let lines: Vec<String> = preamble_str.lines().map(|line| line.to_string()).collect();
114
115    let delimiters = [b',', b';', b'\t', b'|'];
116    let mut best_delimiter = b',';
117    let mut max_consistency = 0;
118    let mut max_fields = 0;
119
120    for &delimiter in &delimiters {
121        let delimiter_char = delimiter as char;
122        let mut counts = std::collections::HashMap::new();
123        let mut total_fields = 0;
124
125        let sample_lines = std::cmp::min(lines.len(), 5);
126        for line in lines.iter().take(sample_lines) {
127            let count = count_fields(line, delimiter_char);
128            *counts.entry(count).or_insert(0) += 1;
129            total_fields += count;
130        }
131
132        let consistency = counts.values().max().copied().unwrap_or(0);
133        let avg_fields = total_fields.checked_div(sample_lines).unwrap_or(0);
134
135        let should_update = if avg_fields > 1 && max_fields <= 1 {
136            true
137        } else if avg_fields <= 1 && max_fields > 1 {
138            false
139        } else {
140            consistency > max_consistency
141                || (consistency == max_consistency && avg_fields > max_fields)
142        };
143
144        if should_update {
145            max_consistency = consistency;
146            max_fields = avg_fields;
147            best_delimiter = delimiter;
148        }
149    }
150
151    Ok(best_delimiter)
152}
153
154/// Detect the most likely CSV delimiter from a file path.
155pub fn detect_delimiter_from_path(path: &Path) -> std::io::Result<u8> {
156    let file = std::fs::File::open(path)?;
157    detect_delimiter(std::io::BufReader::new(file))
158}
159
160/// Analyze CSV data from any `Read` source using streaming statistics.
161///
162/// This is the primary generic entry point for both file-based and stream-based
163/// CSV profiling. Data is processed incrementally through [`StreamingColumnCollection`]
164/// with bounded memory, rather than accumulating all rows in a `HashMap`.
165///
166/// Returns `(column_profiles, streaming_stats, rows_read)`.
167pub fn analyze_csv_from_reader<R: Read>(
168    reader: R,
169    config: &CsvParserConfig,
170) -> Result<
171    (
172        Vec<ColumnProfile>,
173        StreamingColumnCollection,
174        usize,
175        Vec<String>,
176    ),
177    DataProfilerError,
178> {
179    analyze_csv_from_reader_with_hints(reader, config, &SemanticHints::default())
180}
181
182/// Analyze CSV data from any `Read` source while applying semantic hints.
183pub fn analyze_csv_from_reader_with_hints<R: Read>(
184    reader: R,
185    config: &CsvParserConfig,
186    semantic_hints: &SemanticHints,
187) -> Result<
188    (
189        Vec<ColumnProfile>,
190        StreamingColumnCollection,
191        usize,
192        Vec<String>,
193    ),
194    DataProfilerError,
195> {
196    let mut csv_builder = ReaderBuilder::new();
197    csv_builder.has_headers(config.has_header);
198    csv_builder.flexible(config.flexible);
199    csv_builder.quote(config.quote_char);
200    if config.trim_whitespace {
201        csv_builder.trim(csv::Trim::All);
202    }
203    let mut actual_delimiter = config.delimiter;
204
205    let boxed_reader: Box<dyn std::io::Read + '_> = if config.delimiter.is_none() {
206        let mut preamble = Vec::new();
207        let mut take = reader.take(4096);
208        take.read_to_end(&mut preamble)?;
209
210        let detected = detect_delimiter(std::io::Cursor::new(&preamble)).unwrap_or(b',');
211        actual_delimiter = Some(detected);
212
213        Box::new(std::io::Cursor::new(preamble).chain(take.into_inner()))
214    } else {
215        Box::new(reader)
216    };
217
218    if let Some(delimiter) = actual_delimiter {
219        csv_builder.delimiter(delimiter);
220    }
221
222    let mut csv_reader = csv_builder.from_reader(boxed_reader);
223
224    let header_names: Vec<String> = if config.has_header {
225        let headers = csv_reader.headers()?;
226        headers.iter().map(|header| header.to_string()).collect()
227    } else {
228        let headers = csv_reader.headers()?;
229        (0..headers.len())
230            .map(|index| format!("column_{index}"))
231            .collect()
232    };
233
234    let mut column_stats = StreamingColumnCollection::new();
235    let mut rows_read = 0;
236
237    for result in csv_reader.records() {
238        if let Some(max_rows) = config.max_rows
239            && rows_read >= max_rows
240        {
241            break;
242        }
243
244        let record = result?;
245        let mut values: Vec<String> = record.iter().map(|value| value.to_string()).collect();
246
247        let header_len = header_names.len();
248        if values.len() < header_len {
249            values.resize(header_len, String::new());
250        } else if values.len() > header_len {
251            values.truncate(header_len);
252        }
253
254        column_stats.process_record(&header_names, values);
255        rows_read += 1;
256    }
257
258    let profiles = profile_builder::profiles_from_streaming_with_hints(
259        &column_stats,
260        false,
261        false,
262        None,
263        semantic_hints,
264    );
265
266    Ok((profiles, column_stats, rows_read, header_names))
267}
268
269/// Analyze a CSV file, returning a full [`ProfileReport`].
270///
271/// Opens the file, delegates to [`analyze_csv_from_reader`], and wraps the
272/// result with file metadata and ISO 8000/25012 data quality metrics.
273pub fn analyze_csv_file(
274    file_path: &Path,
275    config: &CsvParserConfig,
276) -> Result<ProfileReport, DataProfilerError> {
277    analyze_csv_file_with_dimensions(file_path, config, None)
278}
279
280/// Like [`analyze_csv_file`] but only computes the requested quality dimensions.
281pub fn analyze_csv_file_with_dimensions(
282    file_path: &Path,
283    config: &CsvParserConfig,
284    quality_dimensions: Option<&[QualityDimension]>,
285) -> Result<ProfileReport, DataProfilerError> {
286    analyze_csv_file_with_dimensions_and_hints(
287        file_path,
288        config,
289        quality_dimensions,
290        &SemanticHints::default(),
291    )
292}
293
294pub fn analyze_csv_file_with_dimensions_and_hints(
295    file_path: &Path,
296    config: &CsvParserConfig,
297    quality_dimensions: Option<&[QualityDimension]>,
298    semantic_hints: &SemanticHints,
299) -> Result<ProfileReport, DataProfilerError> {
300    let metadata = std::fs::metadata(file_path).map_err(|error| map_io_error(file_path, error))?;
301    let start = std::time::Instant::now();
302
303    let file = std::fs::File::open(file_path).map_err(|error| map_io_error(file_path, error))?;
304    let buf_reader = std::io::BufReader::new(file);
305
306    let (column_profiles, column_stats, rows_read, header_names) =
307        analyze_csv_from_reader_with_hints(buf_reader, config, semantic_hints)?;
308
309    let file_source = DataSource::File {
310        path: file_path.display().to_string(),
311        format: FileFormat::Csv,
312        size_bytes: metadata.len(),
313        modified_at: None,
314        parquet_metadata: None,
315    };
316
317    if column_profiles.is_empty() && rows_read == 0 {
318        return Ok(ReportAssembler::new(
319            file_source,
320            ExecutionMetadata::new(0, header_names.len(), start.elapsed().as_millis()),
321        )
322        .skip_quality()
323        .build());
324    }
325
326    let sample_columns = profile_builder::quality_check_samples(&column_stats);
327    let scan_time_ms = start.elapsed().as_millis();
328    let num_columns = column_profiles.len();
329
330    let mut assembler = ReportAssembler::new(
331        file_source,
332        ExecutionMetadata::new(rows_read, num_columns, scan_time_ms),
333    )
334    .columns(column_profiles)
335    .with_quality_data(sample_columns)
336    .with_semantic_hints(semantic_hints.clone());
337    if let Some(dimensions) = quality_dimensions {
338        assembler = assembler.with_requested_dimensions(dimensions.to_vec());
339    }
340    Ok(assembler.build())
341}
342
343fn map_io_error(file_path: &Path, error: std::io::Error) -> DataProfilerError {
344    if error.kind() == std::io::ErrorKind::NotFound {
345        DataProfilerError::FileNotFound {
346            path: file_path.display().to_string(),
347        }
348    } else {
349        DataProfilerError::from(error)
350    }
351}
352
353#[cfg(test)]
354mod tests {
355    use super::*;
356    use dataprof_core::DataType;
357    use std::io::{Cursor, Write};
358    use tempfile::NamedTempFile;
359
360    fn write_csv(content: &str) -> NamedTempFile {
361        let mut file = NamedTempFile::new().unwrap();
362        write!(file, "{}", content).unwrap();
363        file.flush().unwrap();
364        file
365    }
366
367    #[test]
368    fn test_analyze_csv_from_reader_basic() {
369        let data = b"name,age\nAlice,25\nBob,30\n";
370        let cursor = Cursor::new(data.as_ref());
371        let config = CsvParserConfig::default();
372
373        let (profiles, _stats, rows, _) = analyze_csv_from_reader(cursor, &config).unwrap();
374        assert_eq!(rows, 2);
375        assert_eq!(profiles.len(), 2);
376
377        let age = profiles
378            .iter()
379            .find(|profile| profile.name == "age")
380            .unwrap();
381        assert_eq!(age.total_count, 2);
382        assert_eq!(age.data_type, DataType::Integer);
383    }
384
385    #[test]
386    fn test_analyze_csv_from_reader_with_max_rows() {
387        let data = b"val\n1\n2\n3\n4\n5\n";
388        let cursor = Cursor::new(data.as_ref());
389        let config = CsvParserConfig::default().max_rows(Some(3));
390
391        let (_profiles, _stats, rows, _) = analyze_csv_from_reader(cursor, &config).unwrap();
392        assert_eq!(rows, 3);
393    }
394
395    #[test]
396    fn test_analyze_csv_from_reader_strict_rejects_ragged() {
397        let data = b"a,b\n1,2\n3,4,5\n";
398        let cursor = Cursor::new(data.as_ref());
399        let config = CsvParserConfig::strict();
400
401        assert!(analyze_csv_from_reader(cursor, &config).is_err());
402    }
403
404    #[test]
405    fn test_analyze_csv_from_reader_flexible_handles_ragged() {
406        let data = b"a,b\n1,2\n3,4,5\n6\n";
407        let cursor = Cursor::new(data.as_ref());
408        let config = CsvParserConfig::default();
409
410        let (profiles, _stats, rows, _) = analyze_csv_from_reader(cursor, &config).unwrap();
411        assert_eq!(rows, 3);
412        assert!(!profiles.is_empty());
413    }
414
415    #[test]
416    fn test_analyze_csv_from_reader_custom_delimiter() {
417        let data = b"a;b\n1;2\n3;4\n";
418        let cursor = Cursor::new(data.as_ref());
419        let config = CsvParserConfig::default().with_delimiter(b';');
420
421        let (profiles, _stats, rows, _) = analyze_csv_from_reader(cursor, &config).unwrap();
422        assert_eq!(rows, 2);
423        assert_eq!(profiles.len(), 2);
424    }
425
426    #[test]
427    fn test_analyze_csv_file_returns_quality_report() {
428        let csv = write_csv("x,y\n1,a\n2,b\n3,c\n");
429        let config = CsvParserConfig::default();
430        let report = analyze_csv_file(csv.path(), &config).unwrap();
431
432        assert_eq!(report.column_profiles.len(), 2);
433        assert_eq!(report.execution.rows_processed, 3);
434        assert!(report.quality_score().unwrap() >= 0.0);
435        assert!(report.quality_score().unwrap() <= 100.0);
436    }
437
438    #[test]
439    fn test_csv_auto_detects_delimiter_from_reader() {
440        let data = b"name;age\nAlice;25\nBob;30\n";
441        let cursor = Cursor::new(data.as_ref());
442        let config = CsvParserConfig::default();
443
444        let (profiles, _stats, rows, _) = analyze_csv_from_reader(cursor, &config).unwrap();
445        assert_eq!(rows, 2);
446        assert_eq!(profiles.len(), 2);
447
448        let age = profiles
449            .iter()
450            .find(|profile| profile.name == "age")
451            .unwrap();
452        assert_eq!(age.total_count, 2);
453    }
454
455    #[test]
456    fn test_analyze_csv_basic() {
457        let csv = write_csv("name,age\nAlice,25\nBob,30\n");
458        let config = CsvParserConfig::default();
459        let report = analyze_csv_file(csv.path(), &config).unwrap();
460        let profiles = &report.column_profiles;
461        assert_eq!(profiles.len(), 2);
462
463        let names: Vec<&str> = profiles
464            .iter()
465            .map(|profile| profile.name.as_str())
466            .collect();
467        assert!(names.contains(&"name"));
468        assert!(names.contains(&"age"));
469
470        let age = profiles
471            .iter()
472            .find(|profile| profile.name == "age")
473            .unwrap();
474        assert_eq!(age.total_count, 2);
475        assert_eq!(age.null_count, 0);
476    }
477
478    #[test]
479    fn test_analyze_csv_with_nulls() {
480        let csv = write_csv("name,age,email\nAlice,25,a@b.com\nBob,,\nCharlie,30,c@d.com\n");
481        let config = CsvParserConfig::default();
482        let report = analyze_csv_file(csv.path(), &config).unwrap();
483        let profiles = &report.column_profiles;
484
485        let age = profiles
486            .iter()
487            .find(|profile| profile.name == "age")
488            .unwrap();
489        assert_eq!(age.total_count, 3);
490        assert_eq!(age.null_count, 1);
491
492        let email = profiles
493            .iter()
494            .find(|profile| profile.name == "email")
495            .unwrap();
496        assert_eq!(email.null_count, 1);
497    }
498
499    #[test]
500    fn test_analyze_csv_file_returns_quality_report_legacy() {
501        let csv = write_csv("x,y\n1,a\n2,b\n3,c\n");
502        let config = CsvParserConfig::default();
503        let report = analyze_csv_file(csv.path(), &config).unwrap();
504
505        assert_eq!(report.column_profiles.len(), 2);
506        assert_eq!(report.execution.rows_processed, 3);
507        assert_eq!(report.execution.columns_detected, 2);
508        assert!(report.quality_score().unwrap() >= 0.0);
509        assert!(report.quality_score().unwrap() <= 100.0);
510    }
511
512    #[test]
513    fn test_analyze_csv_file_empty_file() {
514        let csv = write_csv("name,age\n");
515        let config = CsvParserConfig::default();
516        let report = analyze_csv_file(csv.path(), &config).unwrap();
517
518        assert_eq!(report.column_profiles.len(), 0);
519        assert_eq!(report.execution.rows_processed, 0);
520    }
521
522    #[test]
523    fn test_analyze_csv_with_max_rows_small_file() {
524        let csv = write_csv("val\n1\n2\n3\n4\n5\n");
525        let config = CsvParserConfig::default();
526        let report = analyze_csv_file(csv.path(), &config).unwrap();
527
528        assert_eq!(report.execution.rows_processed, 5);
529        assert_eq!(report.execution.columns_detected, 1);
530        assert!((report.execution.sampling_ratio.unwrap_or(1.0) - 1.0).abs() < 0.01);
531    }
532
533    #[test]
534    fn test_analyze_csv_numeric_types_detected() {
535        let csv = write_csv("int_col,float_col,str_col\n1,1.5,hello\n2,2.5,world\n3,3.5,foo\n");
536        let config = CsvParserConfig::default();
537        let report = analyze_csv_file(csv.path(), &config).unwrap();
538        let profiles = &report.column_profiles;
539
540        let int_col = profiles
541            .iter()
542            .find(|profile| profile.name == "int_col")
543            .unwrap();
544        assert!(matches!(int_col.data_type, DataType::Integer));
545
546        let float_col = profiles
547            .iter()
548            .find(|profile| profile.name == "float_col")
549            .unwrap();
550        assert!(matches!(float_col.data_type, DataType::Float));
551
552        let str_col = profiles
553            .iter()
554            .find(|profile| profile.name == "str_col")
555            .unwrap();
556        assert!(matches!(str_col.data_type, DataType::String));
557    }
558
559    #[test]
560    fn test_analyze_csv_strict_rejects_ragged_rows() {
561        let csv = write_csv("a,b\n1,2\n3,4,5\n");
562        let config = CsvParserConfig::strict();
563        assert!(analyze_csv_file(csv.path(), &config).is_err());
564    }
565
566    #[test]
567    fn test_analyze_csv_file_handles_ragged_rows() {
568        let csv = write_csv("a,b\n1,2\n3,4,5\n6\n");
569        let config = CsvParserConfig::default();
570        let report = analyze_csv_file(csv.path(), &config).unwrap();
571        assert!(!report.column_profiles.is_empty());
572    }
573
574    #[test]
575    fn test_analyze_csv_with_quality_metrics() {
576        let csv = write_csv(
577            "name,age\n\
578             Alice,25\nBob,\nCharlie,30\nDave,\nEve,28\n\
579             Frank,\nGrace,35\nHeidi,40\nIvan,\nJudy,22\n",
580        );
581        let config = CsvParserConfig::default();
582        let report = analyze_csv_file(csv.path(), &config).unwrap();
583
584        let age = report
585            .column_profiles
586            .iter()
587            .find(|profile| profile.name == "age")
588            .unwrap();
589        assert_eq!(age.null_count, 4);
590        assert_eq!(age.total_count, 10);
591        assert!(report.quality_score().unwrap() >= 0.0);
592    }
593
594    #[test]
595    fn test_detect_delimiter_comma() {
596        let data = b"name,age,city\nAlice,25,NYC\nBob,30,London\n";
597        assert_eq!(detect_delimiter(Cursor::new(data.as_ref())).unwrap(), b',');
598    }
599
600    #[test]
601    fn test_detect_delimiter_semicolon() {
602        let data = b"name;age;city\nAlice;25;NYC\nBob;30;London\n";
603        assert_eq!(detect_delimiter(Cursor::new(data.as_ref())).unwrap(), b';');
604    }
605
606    #[test]
607    fn test_detect_delimiter_pipe() {
608        let data = b"name|age|city\nAlice|25|NYC\nBob|30|London\n";
609        assert_eq!(detect_delimiter(Cursor::new(data.as_ref())).unwrap(), b'|');
610    }
611
612    #[test]
613    fn test_detect_delimiter_tab() {
614        let data = b"name\tage\tcity\nAlice\t25\tNYC\nBob\t30\tLondon\n";
615        assert_eq!(detect_delimiter(Cursor::new(data.as_ref())).unwrap(), b'\t');
616    }
617
618    #[test]
619    fn test_detect_delimiter_from_path_semicolon() {
620        let csv = write_csv("id;name;salary\n1;Alice;50000\n2;Bob;60000\n");
621        assert_eq!(detect_delimiter_from_path(csv.path()).unwrap(), b';');
622    }
623
624    #[test]
625    fn test_detect_delimiter_quoted_fields() {
626        let data = b"name;desc;val\n\"hello;world\";foo;1\nbar;baz;2\n";
627        assert_eq!(detect_delimiter(Cursor::new(data.as_ref())).unwrap(), b';');
628    }
629}