1use csv::ReaderBuilder;
2use std::io::Read;
3use std::path::Path;
4
5use dataprof_core::{
6 ColumnProfile, DataProfilerError, DataSource, ExecutionMetadata, FileFormat, QualityDimension,
7 SemanticHints,
8};
9use dataprof_runtime::{
10 ProfileReport, ReportAssembler, StreamingColumnCollection, profile_builder,
11};
12
13mod memmap_reader;
14mod robust_csv;
15
16pub use memmap_reader::MemoryMappedCsvReader;
17pub use robust_csv::{CsvDiagnostics, RobustCsvParser, RobustParseResult};
18
19#[derive(Debug, Clone)]
24pub struct CsvParserConfig {
25 pub flexible: bool,
27 pub delimiter: Option<u8>,
29 pub quote_char: u8,
31 pub trim_whitespace: bool,
33 pub has_header: bool,
36 pub max_rows: Option<usize>,
38}
39
40impl Default for CsvParserConfig {
41 fn default() -> Self {
42 Self {
43 flexible: true,
44 delimiter: None,
45 quote_char: b'"',
46 trim_whitespace: false,
47 has_header: true,
48 max_rows: None,
49 }
50 }
51}
52
53impl CsvParserConfig {
54 pub fn strict() -> Self {
56 Self {
57 flexible: false,
58 ..Default::default()
59 }
60 }
61
62 pub fn with_delimiter(mut self, delimiter: u8) -> Self {
64 self.delimiter = Some(delimiter);
65 self
66 }
67
68 pub fn has_header(mut self, yes: bool) -> Self {
70 self.has_header = yes;
71 self
72 }
73
74 pub fn max_rows(mut self, max: Option<usize>) -> Self {
76 self.max_rows = max;
77 self
78 }
79}
80
81fn count_fields(line: &str, delimiter: char) -> usize {
83 let mut count = 1;
84 let mut in_quotes = false;
85 let mut chars = line.chars().peekable();
86 while let Some(ch) = chars.next() {
87 if ch == '"' {
88 if chars.peek() == Some(&'"') {
89 chars.next();
90 } else {
91 in_quotes = !in_quotes;
92 }
93 } else if ch == delimiter && !in_quotes {
94 count += 1;
95 }
96 }
97 count
98}
99
100pub fn detect_delimiter<R: Read>(reader: R) -> std::io::Result<u8> {
108 let mut preamble = Vec::new();
109 let mut take = reader.take(4096);
110 take.read_to_end(&mut preamble)?;
111
112 let preamble_str = String::from_utf8_lossy(&preamble);
113 let lines: Vec<String> = preamble_str.lines().map(|line| line.to_string()).collect();
114
115 let delimiters = *b",;\t|";
116 let mut best_delimiter = b',';
117 let mut max_consistency = 0;
118 let mut max_fields = 0;
119
120 for &delimiter in &delimiters {
121 let delimiter_char = delimiter as char;
122 let mut counts = std::collections::HashMap::new();
123 let mut total_fields = 0;
124
125 let sample_lines = std::cmp::min(lines.len(), 5);
126 for line in lines.iter().take(sample_lines) {
127 let count = count_fields(line, delimiter_char);
128 *counts.entry(count).or_insert(0) += 1;
129 total_fields += count;
130 }
131
132 let consistency = counts.values().max().copied().unwrap_or(0);
133 let avg_fields = total_fields.checked_div(sample_lines).unwrap_or(0);
134
135 let should_update = if avg_fields > 1 && max_fields <= 1 {
136 true
137 } else if avg_fields <= 1 && max_fields > 1 {
138 false
139 } else {
140 consistency > max_consistency
141 || (consistency == max_consistency && avg_fields > max_fields)
142 };
143
144 if should_update {
145 max_consistency = consistency;
146 max_fields = avg_fields;
147 best_delimiter = delimiter;
148 }
149 }
150
151 Ok(best_delimiter)
152}
153
154pub fn detect_delimiter_from_path(path: &Path) -> std::io::Result<u8> {
156 let file = std::fs::File::open(path)?;
157 detect_delimiter(std::io::BufReader::new(file))
158}
159
160pub fn analyze_csv_from_reader<R: Read>(
168 reader: R,
169 config: &CsvParserConfig,
170) -> Result<
171 (
172 Vec<ColumnProfile>,
173 StreamingColumnCollection,
174 usize,
175 Vec<String>,
176 ),
177 DataProfilerError,
178> {
179 analyze_csv_from_reader_with_hints(reader, config, &SemanticHints::default())
180}
181
182pub fn analyze_csv_from_reader_with_hints<R: Read>(
184 reader: R,
185 config: &CsvParserConfig,
186 semantic_hints: &SemanticHints,
187) -> Result<
188 (
189 Vec<ColumnProfile>,
190 StreamingColumnCollection,
191 usize,
192 Vec<String>,
193 ),
194 DataProfilerError,
195> {
196 let mut csv_builder = ReaderBuilder::new();
197 csv_builder.has_headers(config.has_header);
198 csv_builder.flexible(config.flexible);
199 csv_builder.quote(config.quote_char);
200 if config.trim_whitespace {
201 csv_builder.trim(csv::Trim::All);
202 }
203 let mut actual_delimiter = config.delimiter;
204
205 let boxed_reader: Box<dyn std::io::Read + '_> = if config.delimiter.is_none() {
206 let mut preamble = Vec::new();
207 let mut take = reader.take(4096);
208 take.read_to_end(&mut preamble)?;
209
210 let detected = detect_delimiter(std::io::Cursor::new(&preamble)).unwrap_or(b',');
211 actual_delimiter = Some(detected);
212
213 Box::new(std::io::Cursor::new(preamble).chain(take.into_inner()))
214 } else {
215 Box::new(reader)
216 };
217
218 if let Some(delimiter) = actual_delimiter {
219 csv_builder.delimiter(delimiter);
220 }
221
222 let mut csv_reader = csv_builder.from_reader(boxed_reader);
223
224 let header_names: Vec<String> = if config.has_header {
225 let headers = csv_reader.headers()?;
226 headers.iter().map(|header| header.to_string()).collect()
227 } else {
228 let headers = csv_reader.headers()?;
229 (0..headers.len())
230 .map(|index| format!("column_{index}"))
231 .collect()
232 };
233
234 let mut column_stats = StreamingColumnCollection::new();
235 let mut rows_read = 0;
236
237 for result in csv_reader.records() {
238 if let Some(max_rows) = config.max_rows
239 && rows_read >= max_rows
240 {
241 break;
242 }
243
244 let record = result?;
245 let mut values: Vec<String> = record.iter().map(|value| value.to_string()).collect();
246
247 let header_len = header_names.len();
248 if values.len() < header_len {
249 values.resize(header_len, String::new());
250 } else if values.len() > header_len {
251 values.truncate(header_len);
252 }
253
254 column_stats.process_record(&header_names, values);
255 rows_read += 1;
256 }
257
258 let profiles = profile_builder::profiles_from_streaming_with_hints(
259 &column_stats,
260 false,
261 false,
262 None,
263 semantic_hints,
264 );
265
266 Ok((profiles, column_stats, rows_read, header_names))
267}
268
269pub fn analyze_csv_file(
274 file_path: &Path,
275 config: &CsvParserConfig,
276) -> Result<ProfileReport, DataProfilerError> {
277 analyze_csv_file_with_dimensions(file_path, config, None)
278}
279
280pub fn analyze_csv_file_with_dimensions(
282 file_path: &Path,
283 config: &CsvParserConfig,
284 quality_dimensions: Option<&[QualityDimension]>,
285) -> Result<ProfileReport, DataProfilerError> {
286 analyze_csv_file_with_dimensions_and_hints(
287 file_path,
288 config,
289 quality_dimensions,
290 &SemanticHints::default(),
291 )
292}
293
294pub fn analyze_csv_file_with_dimensions_and_hints(
295 file_path: &Path,
296 config: &CsvParserConfig,
297 quality_dimensions: Option<&[QualityDimension]>,
298 semantic_hints: &SemanticHints,
299) -> Result<ProfileReport, DataProfilerError> {
300 let metadata = std::fs::metadata(file_path).map_err(|error| map_io_error(file_path, error))?;
301 let start = std::time::Instant::now();
302
303 let file = std::fs::File::open(file_path).map_err(|error| map_io_error(file_path, error))?;
304 let buf_reader = std::io::BufReader::new(file);
305
306 let (column_profiles, column_stats, rows_read, header_names) =
307 analyze_csv_from_reader_with_hints(buf_reader, config, semantic_hints)?;
308
309 let file_source = DataSource::File {
310 path: file_path.display().to_string(),
311 format: FileFormat::Csv,
312 size_bytes: metadata.len(),
313 modified_at: None,
314 parquet_metadata: None,
315 };
316
317 if column_profiles.is_empty() && rows_read == 0 {
318 return Ok(ReportAssembler::new(
319 file_source,
320 ExecutionMetadata::new(0, header_names.len(), start.elapsed().as_millis()),
321 )
322 .skip_quality()
323 .build());
324 }
325
326 let sample_columns = profile_builder::quality_check_samples(&column_stats);
327 let scan_time_ms = start.elapsed().as_millis();
328 let num_columns = column_profiles.len();
329
330 let mut assembler = ReportAssembler::new(
331 file_source,
332 ExecutionMetadata::new(rows_read, num_columns, scan_time_ms),
333 )
334 .columns(column_profiles)
335 .with_quality_data(sample_columns)
336 .with_semantic_hints(semantic_hints.clone());
337 if let Some(dimensions) = quality_dimensions {
338 assembler = assembler.with_requested_dimensions(dimensions.to_vec());
339 }
340 Ok(assembler.build())
341}
342
343fn map_io_error(file_path: &Path, error: std::io::Error) -> DataProfilerError {
344 if error.kind() == std::io::ErrorKind::NotFound {
345 DataProfilerError::FileNotFound {
346 path: file_path.display().to_string(),
347 }
348 } else {
349 DataProfilerError::from(error)
350 }
351}
352
353#[cfg(test)]
354mod tests {
355 use super::*;
356 use dataprof_core::DataType;
357 use std::io::{Cursor, Write};
358 use tempfile::NamedTempFile;
359
360 fn write_csv(content: &str) -> NamedTempFile {
361 let mut file = NamedTempFile::new().unwrap();
362 write!(file, "{}", content).unwrap();
363 file.flush().unwrap();
364 file
365 }
366
367 #[test]
368 fn test_analyze_csv_from_reader_basic() {
369 let data = b"name,age\nAlice,25\nBob,30\n";
370 let cursor = Cursor::new(data.as_ref());
371 let config = CsvParserConfig::default();
372
373 let (profiles, _stats, rows, _) = analyze_csv_from_reader(cursor, &config).unwrap();
374 assert_eq!(rows, 2);
375 assert_eq!(profiles.len(), 2);
376
377 let age = profiles
378 .iter()
379 .find(|profile| profile.name == "age")
380 .unwrap();
381 assert_eq!(age.total_count, 2);
382 assert_eq!(age.data_type, DataType::Integer);
383 }
384
385 #[test]
386 fn test_analyze_csv_from_reader_with_max_rows() {
387 let data = b"val\n1\n2\n3\n4\n5\n";
388 let cursor = Cursor::new(data.as_ref());
389 let config = CsvParserConfig::default().max_rows(Some(3));
390
391 let (_profiles, _stats, rows, _) = analyze_csv_from_reader(cursor, &config).unwrap();
392 assert_eq!(rows, 3);
393 }
394
395 #[test]
396 fn test_analyze_csv_from_reader_strict_rejects_ragged() {
397 let data = b"a,b\n1,2\n3,4,5\n";
398 let cursor = Cursor::new(data.as_ref());
399 let config = CsvParserConfig::strict();
400
401 assert!(analyze_csv_from_reader(cursor, &config).is_err());
402 }
403
404 #[test]
405 fn test_analyze_csv_from_reader_flexible_handles_ragged() {
406 let data = b"a,b\n1,2\n3,4,5\n6\n";
407 let cursor = Cursor::new(data.as_ref());
408 let config = CsvParserConfig::default();
409
410 let (profiles, _stats, rows, _) = analyze_csv_from_reader(cursor, &config).unwrap();
411 assert_eq!(rows, 3);
412 assert!(!profiles.is_empty());
413 }
414
415 #[test]
416 fn test_analyze_csv_from_reader_custom_delimiter() {
417 let data = b"a;b\n1;2\n3;4\n";
418 let cursor = Cursor::new(data.as_ref());
419 let config = CsvParserConfig::default().with_delimiter(b';');
420
421 let (profiles, _stats, rows, _) = analyze_csv_from_reader(cursor, &config).unwrap();
422 assert_eq!(rows, 2);
423 assert_eq!(profiles.len(), 2);
424 }
425
426 #[test]
427 fn test_analyze_csv_file_returns_quality_report() {
428 let csv = write_csv("x,y\n1,a\n2,b\n3,c\n");
429 let config = CsvParserConfig::default();
430 let report = analyze_csv_file(csv.path(), &config).unwrap();
431
432 assert_eq!(report.column_profiles.len(), 2);
433 assert_eq!(report.execution.rows_processed, 3);
434 assert!(report.quality_score().unwrap() >= 0.0);
435 assert!(report.quality_score().unwrap() <= 100.0);
436 }
437
438 #[test]
439 fn test_csv_auto_detects_delimiter_from_reader() {
440 let data = b"name;age\nAlice;25\nBob;30\n";
441 let cursor = Cursor::new(data.as_ref());
442 let config = CsvParserConfig::default();
443
444 let (profiles, _stats, rows, _) = analyze_csv_from_reader(cursor, &config).unwrap();
445 assert_eq!(rows, 2);
446 assert_eq!(profiles.len(), 2);
447
448 let age = profiles
449 .iter()
450 .find(|profile| profile.name == "age")
451 .unwrap();
452 assert_eq!(age.total_count, 2);
453 }
454
455 #[test]
456 fn test_analyze_csv_basic() {
457 let csv = write_csv("name,age\nAlice,25\nBob,30\n");
458 let config = CsvParserConfig::default();
459 let report = analyze_csv_file(csv.path(), &config).unwrap();
460 let profiles = &report.column_profiles;
461 assert_eq!(profiles.len(), 2);
462
463 let names: Vec<&str> = profiles
464 .iter()
465 .map(|profile| profile.name.as_str())
466 .collect();
467 assert!(names.contains(&"name"));
468 assert!(names.contains(&"age"));
469
470 let age = profiles
471 .iter()
472 .find(|profile| profile.name == "age")
473 .unwrap();
474 assert_eq!(age.total_count, 2);
475 assert_eq!(age.null_count, 0);
476 }
477
478 #[test]
479 fn test_analyze_csv_with_nulls() {
480 let csv = write_csv("name,age,email\nAlice,25,a@b.com\nBob,,\nCharlie,30,c@d.com\n");
481 let config = CsvParserConfig::default();
482 let report = analyze_csv_file(csv.path(), &config).unwrap();
483 let profiles = &report.column_profiles;
484
485 let age = profiles
486 .iter()
487 .find(|profile| profile.name == "age")
488 .unwrap();
489 assert_eq!(age.total_count, 3);
490 assert_eq!(age.null_count, 1);
491
492 let email = profiles
493 .iter()
494 .find(|profile| profile.name == "email")
495 .unwrap();
496 assert_eq!(email.null_count, 1);
497 }
498
499 #[test]
500 fn test_analyze_csv_file_returns_quality_report_legacy() {
501 let csv = write_csv("x,y\n1,a\n2,b\n3,c\n");
502 let config = CsvParserConfig::default();
503 let report = analyze_csv_file(csv.path(), &config).unwrap();
504
505 assert_eq!(report.column_profiles.len(), 2);
506 assert_eq!(report.execution.rows_processed, 3);
507 assert_eq!(report.execution.columns_detected, 2);
508 assert!(report.quality_score().unwrap() >= 0.0);
509 assert!(report.quality_score().unwrap() <= 100.0);
510 }
511
512 #[test]
513 fn test_analyze_csv_file_empty_file() {
514 let csv = write_csv("name,age\n");
515 let config = CsvParserConfig::default();
516 let report = analyze_csv_file(csv.path(), &config).unwrap();
517
518 assert_eq!(report.column_profiles.len(), 0);
519 assert_eq!(report.execution.rows_processed, 0);
520 }
521
522 #[test]
523 fn test_analyze_csv_with_max_rows_small_file() {
524 let csv = write_csv("val\n1\n2\n3\n4\n5\n");
525 let config = CsvParserConfig::default();
526 let report = analyze_csv_file(csv.path(), &config).unwrap();
527
528 assert_eq!(report.execution.rows_processed, 5);
529 assert_eq!(report.execution.columns_detected, 1);
530 assert!((report.execution.sampling_ratio.unwrap_or(1.0) - 1.0).abs() < 0.01);
531 }
532
533 #[test]
534 fn test_analyze_csv_numeric_types_detected() {
535 let csv = write_csv("int_col,float_col,str_col\n1,1.5,hello\n2,2.5,world\n3,3.5,foo\n");
536 let config = CsvParserConfig::default();
537 let report = analyze_csv_file(csv.path(), &config).unwrap();
538 let profiles = &report.column_profiles;
539
540 let int_col = profiles
541 .iter()
542 .find(|profile| profile.name == "int_col")
543 .unwrap();
544 assert!(matches!(int_col.data_type, DataType::Integer));
545
546 let float_col = profiles
547 .iter()
548 .find(|profile| profile.name == "float_col")
549 .unwrap();
550 assert!(matches!(float_col.data_type, DataType::Float));
551
552 let str_col = profiles
553 .iter()
554 .find(|profile| profile.name == "str_col")
555 .unwrap();
556 assert!(matches!(str_col.data_type, DataType::String));
557 }
558
559 #[test]
560 fn test_analyze_csv_strict_rejects_ragged_rows() {
561 let csv = write_csv("a,b\n1,2\n3,4,5\n");
562 let config = CsvParserConfig::strict();
563 assert!(analyze_csv_file(csv.path(), &config).is_err());
564 }
565
566 #[test]
567 fn test_analyze_csv_file_handles_ragged_rows() {
568 let csv = write_csv("a,b\n1,2\n3,4,5\n6\n");
569 let config = CsvParserConfig::default();
570 let report = analyze_csv_file(csv.path(), &config).unwrap();
571 assert!(!report.column_profiles.is_empty());
572 }
573
574 #[test]
575 fn test_analyze_csv_with_quality_metrics() {
576 let csv = write_csv(
577 "name,age\n\
578 Alice,25\nBob,\nCharlie,30\nDave,\nEve,28\n\
579 Frank,\nGrace,35\nHeidi,40\nIvan,\nJudy,22\n",
580 );
581 let config = CsvParserConfig::default();
582 let report = analyze_csv_file(csv.path(), &config).unwrap();
583
584 let age = report
585 .column_profiles
586 .iter()
587 .find(|profile| profile.name == "age")
588 .unwrap();
589 assert_eq!(age.null_count, 4);
590 assert_eq!(age.total_count, 10);
591 assert!(report.quality_score().unwrap() >= 0.0);
592 }
593
594 #[test]
595 fn test_detect_delimiter_comma() {
596 let data = b"name,age,city\nAlice,25,NYC\nBob,30,London\n";
597 assert_eq!(detect_delimiter(Cursor::new(data.as_ref())).unwrap(), b',');
598 }
599
600 #[test]
601 fn test_detect_delimiter_semicolon() {
602 let data = b"name;age;city\nAlice;25;NYC\nBob;30;London\n";
603 assert_eq!(detect_delimiter(Cursor::new(data.as_ref())).unwrap(), b';');
604 }
605
606 #[test]
607 fn test_detect_delimiter_pipe() {
608 let data = b"name|age|city\nAlice|25|NYC\nBob|30|London\n";
609 assert_eq!(detect_delimiter(Cursor::new(data.as_ref())).unwrap(), b'|');
610 }
611
612 #[test]
613 fn test_detect_delimiter_tab() {
614 let data = b"name\tage\tcity\nAlice\t25\tNYC\nBob\t30\tLondon\n";
615 assert_eq!(detect_delimiter(Cursor::new(data.as_ref())).unwrap(), b'\t');
616 }
617
618 #[test]
619 fn test_detect_delimiter_from_path_semicolon() {
620 let csv = write_csv("id;name;salary\n1;Alice;50000\n2;Bob;60000\n");
621 assert_eq!(detect_delimiter_from_path(csv.path()).unwrap(), b';');
622 }
623
624 #[test]
625 fn test_detect_delimiter_quoted_fields() {
626 let data = b"name;desc;val\n\"hello;world\";foo;1\nbar;baz;2\n";
627 assert_eq!(detect_delimiter(Cursor::new(data.as_ref())).unwrap(), b';');
628 }
629}