dataprof_core/
validation.rs

1use crate::errors::DataProfilerError;
2use anyhow::Result;
3use std::collections::HashSet;
4use std::path::{Path, PathBuf};
5
6/// Reject repeated names in an ordered column-name list, before any per-column
7/// state is built.
8///
9/// Every engine keys its per-column accumulators by name, so a repeated name is
10/// otherwise silently merged (file/Arrow paths) or shadowed at mapping access
11/// (columnar path), yielding a report where `total_count` can exceed the row
12/// count or a column simply disappears. Rejecting up front is the only behavior
13/// all engines can share.
14///
15/// `names` is the final list each engine uses as profile keys, so the check runs
16/// on already-normalized names (the caller's stringification is the normalization
17/// step). `source` is a short transport label — e.g. `"CSV header"`, `"CSV bytes"`,
18/// `"Arrow/Parquet schema"` — surfaced to the user. Duplicates are reported in
19/// first-seen order, deduped; no cell values are ever embedded.
20pub fn validate_unique_column_names(
21    names: &[String],
22    source: &str,
23) -> Result<(), DataProfilerError> {
24    let mut seen: HashSet<&str> = HashSet::with_capacity(names.len());
25    let mut reported: HashSet<&str> = HashSet::new();
26    let mut duplicates: Vec<String> = Vec::new();
27    for name in names {
28        if !seen.insert(name.as_str()) && reported.insert(name.as_str()) {
29            duplicates.push(name.clone());
30        }
31    }
32    if duplicates.is_empty() {
33        Ok(())
34    } else {
35        Err(DataProfilerError::duplicate_column_name(
36            &duplicates,
37            source,
38        ))
39    }
40}
41
42/// Enhanced input validation with helpful error messages and suggestions
43pub struct InputValidator;
44
45#[derive(Debug)]
46pub struct ValidationError {
47    pub message: String,
48    pub suggestion: String,
49    pub error_code: i32,
50}
51
52impl std::fmt::Display for ValidationError {
53    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
54        write!(f, "{}\n{}", self.message, self.suggestion)
55    }
56}
57
58impl std::error::Error for ValidationError {}
59
60impl InputValidator {
61    /// Validate file input with helpful suggestions
62    pub fn validate_file_input(file_path: &Path) -> Result<(), ValidationError> {
63        // Check if file exists
64        if !file_path.exists() {
65            return Err(ValidationError {
66                message: format!("File not found: {}", file_path.display()),
67                suggestion: Self::generate_file_suggestions(file_path),
68                error_code: 2, // ENOENT
69            });
70        }
71
72        // Check if it's actually a file (not directory)
73        if file_path.is_dir() {
74            return Err(ValidationError {
75                message: format!("Path is a directory, not a file: {}", file_path.display()),
76                suggestion: "Use --recursive flag to process directories, or specify a file path"
77                    .to_string(),
78                error_code: 21, // EISDIR
79            });
80        }
81
82        // Check file extension
83        Self::validate_file_extension(file_path)?;
84
85        // Check file permissions
86        Self::validate_file_permissions(file_path)?;
87
88        // Check file size (warn for very large files)
89        Self::validate_file_size(file_path)?;
90
91        Ok(())
92    }
93
94    /// Validate configuration file
95    pub fn validate_config_file(config_path: &Path) -> Result<(), ValidationError> {
96        if !config_path.exists() {
97            return Err(ValidationError {
98                message: format!("Configuration file not found: {}", config_path.display()),
99                suggestion: format!(
100                    "Create a config file at {} or use 'dataprof --help' to see configuration options",
101                    config_path.display()
102                ),
103                error_code: 2,
104            });
105        }
106
107        // Check if it's a TOML file
108        if let Some(ext) = config_path.extension()
109            && ext != "toml"
110        {
111            return Err(ValidationError {
112                message: "Configuration file must have .toml extension".to_string(),
113                suggestion:
114                    "Rename your config file to have .toml extension (e.g., .dataprof.toml)"
115                        .to_string(),
116                error_code: 22, // EINVAL
117            });
118        }
119
120        Ok(())
121    }
122
123    /// Validate chunk size parameter
124    pub fn validate_chunk_size(chunk_size: usize) -> Result<(), ValidationError> {
125        if chunk_size == 0 {
126            return Err(ValidationError {
127                message: "Chunk size cannot be zero".to_string(),
128                suggestion: "Use a positive chunk size (e.g. `ChunkSize::Fixed(1000)`) or `ChunkSize::Adaptive` for automatic sizing".to_string(),
129                error_code: 22,
130            });
131        }
132
133        if chunk_size < 10 {
134            return Err(ValidationError {
135                message: format!("Chunk size too small: {}", chunk_size),
136                suggestion: "Use at least 10 rows per chunk for efficient processing".to_string(),
137                error_code: 22,
138            });
139        }
140
141        if chunk_size > 10_000_000 {
142            return Err(ValidationError {
143                message: format!("Chunk size very large: {}", chunk_size),
144                suggestion: "Consider using smaller chunks (< 10M rows) to avoid memory issues"
145                    .to_string(),
146                error_code: 22,
147            });
148        }
149
150        Ok(())
151    }
152
153    /// Validate sample size parameter
154    pub fn validate_sample_size(sample_size: usize) -> Result<(), ValidationError> {
155        if sample_size == 0 {
156            return Err(ValidationError {
157                message: "Sample size cannot be zero".to_string(),
158                suggestion:
159                    "Use a positive sample size (e.g., --sample 10000) or omit for full analysis"
160                        .to_string(),
161                error_code: 22,
162            });
163        }
164
165        if sample_size < 100 {
166            return Err(ValidationError {
167                message: format!("Sample size very small: {}", sample_size),
168                suggestion: "Use at least 100 samples for meaningful statistical analysis"
169                    .to_string(),
170                error_code: 22,
171            });
172        }
173
174        Ok(())
175    }
176
177    /// Validate conflicting arguments
178    pub fn validate_argument_combinations(
179        streaming: bool,
180        sample: Option<usize>,
181        progress: bool,
182        benchmark: bool,
183    ) -> Result<(), ValidationError> {
184        // Progress requires streaming
185        if progress && !streaming {
186            return Err(ValidationError {
187                message: "Progress display requires streaming mode".to_string(),
188                suggestion: "Add --streaming flag when using --progress".to_string(),
189                error_code: 22,
190            });
191        }
192
193        // Benchmark conflicts with other modes
194        if benchmark && streaming {
195            return Err(ValidationError {
196                message: "Benchmark mode conflicts with streaming".to_string(),
197                suggestion: "Use either --benchmark OR --streaming, not both".to_string(),
198                error_code: 22,
199            });
200        }
201
202        if benchmark && sample.is_some() {
203            return Err(ValidationError {
204                message: "Benchmark mode conflicts with sampling".to_string(),
205                suggestion: "Use either --benchmark OR --sample, not both".to_string(),
206                error_code: 22,
207            });
208        }
209
210        Ok(())
211    }
212
213    /// Generate helpful file suggestions
214    fn generate_file_suggestions(file_path: &Path) -> String {
215        let mut suggestions = Vec::new();
216
217        // Check parent directory
218        if let Some(parent) = file_path.parent() {
219            if parent.exists() {
220                // Look for similar files
221                if let Ok(entries) = std::fs::read_dir(parent) {
222                    let similar_files: Vec<PathBuf> = entries
223                        .filter_map(|entry| entry.ok())
224                        .filter(|entry| {
225                            if let Some(ext) = entry.path().extension() {
226                                matches!(ext.to_str(), Some("csv") | Some("json") | Some("jsonl"))
227                            } else {
228                                false
229                            }
230                        })
231                        .take(3)
232                        .map(|entry| entry.path())
233                        .collect();
234
235                    if !similar_files.is_empty() {
236                        suggestions.push(format!("Similar files found in {}:", parent.display()));
237                        for file in similar_files {
238                            suggestions.push(format!("  • {}", file.display()));
239                        }
240                    }
241                }
242            } else {
243                suggestions.push(format!(
244                    "Parent directory does not exist: {}",
245                    parent.display()
246                ));
247            }
248        }
249
250        // Check current directory
251        if file_path.is_relative() {
252            suggestions
253                .push("Try using an absolute path or check your current directory".to_string());
254        }
255
256        if suggestions.is_empty() {
257            "Check the file path and make sure the file exists".to_string()
258        } else {
259            suggestions.join("\n")
260        }
261    }
262
263    /// Validate file extension
264    fn validate_file_extension(file_path: &Path) -> Result<(), ValidationError> {
265        if let Some(ext) = file_path.extension().and_then(|e| e.to_str()) {
266            match ext.to_lowercase().as_str() {
267                "csv" | "json" | "jsonl" => Ok(()),
268                _ => Err(ValidationError {
269                    message: format!("Unsupported file format: .{}", ext),
270                    suggestion: "Supported formats: .csv, .json, .jsonl".to_string(),
271                    error_code: 22,
272                }),
273            }
274        } else {
275            Err(ValidationError {
276                message: "File has no extension or unrecognizable format".to_string(),
277                suggestion: "Use files with extensions: .csv, .json, or .jsonl".to_string(),
278                error_code: 22,
279            })
280        }
281    }
282
283    /// Validate file permissions
284    fn validate_file_permissions(file_path: &Path) -> Result<(), ValidationError> {
285        match std::fs::metadata(file_path) {
286            Ok(metadata) => {
287                if metadata.permissions().readonly() {
288                    // This is actually OK for reading, but warn if they might want to write
289                    log::debug!("File is read-only: {}", file_path.display());
290                }
291                Ok(())
292            }
293            Err(e) => Err(ValidationError {
294                message: format!("Cannot access file metadata: {}", e),
295                suggestion: "Check file permissions and try again".to_string(),
296                error_code: 13,
297            }),
298        }
299    }
300
301    /// Validate file size and provide warnings
302    fn validate_file_size(file_path: &Path) -> Result<(), ValidationError> {
303        match std::fs::metadata(file_path) {
304            Ok(metadata) => {
305                let size_mb = metadata.len() as f64 / 1_048_576.0;
306
307                if size_mb > 1000.0 {
308                    // Large file warning, not error
309                    log::warn!(
310                        "Large file detected ({:.1} MB). Consider using --streaming for better performance",
311                        size_mb
312                    );
313                }
314
315                if size_mb > 10_000.0 {
316                    return Err(ValidationError {
317                        message: format!("File very large: {:.1} GB", size_mb / 1024.0),
318                        suggestion: "Use --streaming --sample for very large files, or ensure sufficient memory".to_string(),
319                        error_code: 27, // EFBIG
320                    });
321                }
322
323                Ok(())
324            }
325            Err(e) => Err(ValidationError {
326                message: format!("Cannot check file size: {}", e),
327                suggestion: "Ensure file is accessible and try again".to_string(),
328                error_code: 13,
329            }),
330        }
331    }
332
333    /// Validate database connection string format
334    #[cfg(feature = "database")]
335    pub fn validate_database_connection(connection_string: &str) -> Result<(), ValidationError> {
336        if connection_string.is_empty() {
337            return Err(ValidationError {
338                message: "Database connection string is empty".to_string(),
339                suggestion:
340                    "Provide a valid connection string (e.g., postgresql://user:pass@host/db)"
341                        .to_string(),
342                error_code: 22,
343            });
344        }
345
346        // Basic format validation
347        if !connection_string.contains("://") {
348            return Err(ValidationError {
349                message: "Invalid connection string format".to_string(),
350                suggestion: "Use format: protocol://[user:password@]host[:port]/database"
351                    .to_string(),
352                error_code: 22,
353            });
354        }
355
356        // Check for supported protocols
357        let supported_protocols = ["postgresql", "postgres", "mysql", "sqlite"];
358        let protocol = connection_string.split("://").next().unwrap_or("");
359
360        if !supported_protocols.contains(&protocol) {
361            return Err(ValidationError {
362                message: format!("Unsupported database protocol: {}", protocol),
363                suggestion: format!("Supported protocols: {}", supported_protocols.join(", ")),
364                error_code: 22,
365            });
366        }
367
368        Ok(())
369    }
370
371    /// Validate glob pattern
372    pub fn validate_glob_pattern(pattern: &str) -> Result<(), ValidationError> {
373        if pattern.is_empty() {
374            return Err(ValidationError {
375                message: "Glob pattern is empty".to_string(),
376                suggestion: "Provide a valid glob pattern (e.g., \"data/**/*.csv\")".to_string(),
377                error_code: 22,
378            });
379        }
380
381        // Test if pattern compiles
382        match glob::Pattern::new(pattern) {
383            Ok(_) => Ok(()),
384            Err(e) => Err(ValidationError {
385                message: format!("Invalid glob pattern: {}", e),
386                suggestion: "Use valid glob syntax with *, **, ?, [abc], etc.".to_string(),
387                error_code: 22,
388            }),
389        }
390    }
391
392    /// Get appropriate exit code for validation error
393    pub fn get_exit_code(error: &ValidationError) -> i32 {
394        error.error_code
395    }
396}
397
398/// Exit codes following Unix conventions
399pub mod exit_codes {
400    pub const SUCCESS: i32 = 0;
401    pub const GENERAL_ERROR: i32 = 1;
402    pub const FILE_NOT_FOUND: i32 = 2;
403    pub const PERMISSION_DENIED: i32 = 13;
404    pub const INVALID_ARGUMENT: i32 = 22;
405    pub const FILE_TOO_LARGE: i32 = 27;
406    pub const NO_SPACE_LEFT: i32 = 28;
407    pub const BROKEN_PIPE: i32 = 32;
408
409    // Custom application codes
410    pub const INVALID_DATA_FORMAT: i32 = 65;
411    pub const PROCESSING_ERROR: i32 = 66;
412    pub const CONFIG_ERROR: i32 = 67;
413    pub const DATABASE_ERROR: i32 = 68;
414    pub const NETWORK_ERROR: i32 = 69;
415}
416
417#[cfg(test)]
418mod tests {
419    use super::*;
420    use std::io::Write;
421    use tempfile::NamedTempFile;
422
423    // -- chunk size validation --
424
425    #[test]
426    fn test_chunk_size_zero_rejected() {
427        assert!(InputValidator::validate_chunk_size(0).is_err());
428    }
429
430    #[test]
431    fn test_chunk_size_too_small_rejected() {
432        assert!(InputValidator::validate_chunk_size(5).is_err());
433    }
434
435    #[test]
436    fn test_chunk_size_too_large_rejected() {
437        assert!(InputValidator::validate_chunk_size(20_000_000).is_err());
438    }
439
440    #[test]
441    fn test_chunk_size_valid() {
442        assert!(InputValidator::validate_chunk_size(1000).is_ok());
443        assert!(InputValidator::validate_chunk_size(10).is_ok());
444        assert!(InputValidator::validate_chunk_size(10_000_000).is_ok());
445    }
446
447    // -- sample size validation --
448
449    #[test]
450    fn test_sample_size_zero_rejected() {
451        assert!(InputValidator::validate_sample_size(0).is_err());
452    }
453
454    #[test]
455    fn test_sample_size_too_small_rejected() {
456        assert!(InputValidator::validate_sample_size(50).is_err());
457    }
458
459    #[test]
460    fn test_sample_size_valid() {
461        assert!(InputValidator::validate_sample_size(100).is_ok());
462        assert!(InputValidator::validate_sample_size(10_000).is_ok());
463    }
464
465    // -- argument combinations --
466
467    #[test]
468    fn test_progress_without_streaming_rejected() {
469        let result = InputValidator::validate_argument_combinations(false, None, true, false);
470        assert!(result.is_err());
471    }
472
473    #[test]
474    fn test_benchmark_with_streaming_rejected() {
475        let result = InputValidator::validate_argument_combinations(true, None, false, true);
476        assert!(result.is_err());
477    }
478
479    #[test]
480    fn test_benchmark_with_sample_rejected() {
481        let result = InputValidator::validate_argument_combinations(false, Some(1000), false, true);
482        assert!(result.is_err());
483    }
484
485    #[test]
486    fn test_valid_argument_combinations() {
487        // streaming + progress: OK
488        assert!(InputValidator::validate_argument_combinations(true, None, true, false).is_ok());
489        // no flags: OK
490        assert!(InputValidator::validate_argument_combinations(false, None, false, false).is_ok());
491        // benchmark alone: OK
492        assert!(InputValidator::validate_argument_combinations(false, None, false, true).is_ok());
493    }
494
495    // -- file validation --
496
497    #[test]
498    fn test_validate_file_nonexistent() {
499        let result = InputValidator::validate_file_input(Path::new("/nonexistent/file.csv"));
500        assert!(result.is_err());
501        let err = result.unwrap_err();
502        assert_eq!(err.error_code, 2); // ENOENT
503    }
504
505    #[test]
506    fn test_validate_file_directory_rejected() {
507        let dir = tempfile::tempdir().unwrap();
508        let result = InputValidator::validate_file_input(dir.path());
509        assert!(result.is_err());
510        let err = result.unwrap_err();
511        assert_eq!(err.error_code, 21); // EISDIR
512    }
513
514    #[test]
515    fn test_validate_file_unsupported_extension() {
516        let mut f = NamedTempFile::with_suffix(".xlsx").unwrap();
517        write!(f, "data").unwrap();
518        f.flush().unwrap();
519        let result = InputValidator::validate_file_input(f.path());
520        assert!(result.is_err());
521        assert_eq!(result.unwrap_err().error_code, 22); // EINVAL
522    }
523
524    #[test]
525    fn test_validate_file_valid_csv() {
526        let mut f = NamedTempFile::with_suffix(".csv").unwrap();
527        write!(f, "a,b\n1,2\n").unwrap();
528        f.flush().unwrap();
529        assert!(InputValidator::validate_file_input(f.path()).is_ok());
530    }
531
532    #[test]
533    fn test_validate_file_valid_json() {
534        let mut f = NamedTempFile::with_suffix(".json").unwrap();
535        write!(f, "[]").unwrap();
536        f.flush().unwrap();
537        assert!(InputValidator::validate_file_input(f.path()).is_ok());
538    }
539
540    // -- glob pattern validation --
541
542    #[test]
543    fn test_glob_pattern_empty_rejected() {
544        assert!(InputValidator::validate_glob_pattern("").is_err());
545    }
546
547    #[test]
548    fn test_glob_pattern_valid() {
549        assert!(InputValidator::validate_glob_pattern("*.csv").is_ok());
550        assert!(InputValidator::validate_glob_pattern("data/**/*.json").is_ok());
551    }
552
553    // -- config file validation --
554
555    #[test]
556    fn test_config_file_nonexistent() {
557        assert!(InputValidator::validate_config_file(Path::new("/no/config.toml")).is_err());
558    }
559
560    #[test]
561    fn test_config_file_wrong_extension() {
562        let mut f = NamedTempFile::with_suffix(".yaml").unwrap();
563        write!(f, "key: value").unwrap();
564        f.flush().unwrap();
565        assert!(InputValidator::validate_config_file(f.path()).is_err());
566    }
567
568    #[test]
569    fn test_config_file_valid_toml() {
570        let mut f = NamedTempFile::with_suffix(".toml").unwrap();
571        write!(f, "[settings]").unwrap();
572        f.flush().unwrap();
573        assert!(InputValidator::validate_config_file(f.path()).is_ok());
574    }
575
576    // -- unique column names --
577
578    fn names(v: &[&str]) -> Vec<String> {
579        v.iter().map(|s| s.to_string()).collect()
580    }
581
582    #[test]
583    fn unique_column_names_pass() {
584        assert!(validate_unique_column_names(&names(&["x", "y", "z"]), "CSV header").is_ok());
585        assert!(validate_unique_column_names(&[], "CSV header").is_ok());
586    }
587
588    #[test]
589    fn duplicate_first_middle_last_rejected() {
590        for cols in [
591            names(&["x", "x", "y"]), // first
592            names(&["a", "x", "x"]), // last
593            names(&["a", "x", "b", "x", "c"]),
594        ] {
595            let err = validate_unique_column_names(&cols, "CSV header").unwrap_err();
596            assert!(matches!(err, DataProfilerError::DuplicateColumnName { .. }));
597        }
598    }
599
600    #[test]
601    fn duplicate_error_names_offender_and_source_only() {
602        let cols = names(&["id", "amount", "id", "amount", "note"]);
603        let err = validate_unique_column_names(&cols, "CSV bytes").unwrap_err();
604        let msg = err.to_string();
605        // Each offender named once, source present, no cell values.
606        assert!(msg.contains("'id'"));
607        assert!(msg.contains("'amount'"));
608        assert_eq!(msg.matches("'id'").count(), 1);
609        assert!(msg.contains("CSV bytes"));
610        assert!(msg.contains("Duplicate column names:")); // plural "s"
611    }
612}
dataprof_core/validation.rs

dataprof_core/
validation.rs
