Skip to main content

trustformers_models/developer_tools/
validation_tools.rs

1//! Validation Tools
2//!
3//! Tools for validating model implementations, configurations, and outputs.
4
5use anyhow::Result;
6use scirs2_core::ndarray::ArrayD; // SciRS2 Integration Policy
7use serde::{Deserialize, Serialize};
8use std::collections::HashMap;
9use trustformers_core::tensor::Tensor;
10use trustformers_core::traits::{Config, Model};
11
12/// Current process resident-set size in MB, or `None` when unavailable.
13fn current_resident_mb() -> Option<f64> {
14    crate::memory_profiling::MemoryProfiler::get_process_memory_info()
15        .ok()
16        .map(|info| info.resident_mb)
17}
18
19/// Validation configuration
20#[derive(Debug, Clone, Serialize, Deserialize)]
21pub struct ValidationConfig {
22    /// Numerical tolerance for comparisons
23    pub numerical_tolerance: f64,
24    /// Whether to check for NaN values
25    pub check_nan: bool,
26    /// Whether to check for infinite values
27    pub check_infinite: bool,
28    /// Expected output shapes
29    pub expected_shapes: HashMap<String, Vec<usize>>,
30    /// Value range constraints
31    pub value_ranges: HashMap<String, (f64, f64)>,
32    /// Performance thresholds
33    pub performance_thresholds: PerformanceThresholds,
34}
35
36/// Performance validation thresholds
37#[derive(Debug, Clone, Serialize, Deserialize)]
38pub struct PerformanceThresholds {
39    /// Maximum forward pass time in milliseconds
40    pub max_forward_time_ms: f64,
41    /// Maximum memory usage in MB
42    pub max_memory_mb: f64,
43    /// Minimum throughput (samples/second)
44    pub min_throughput: f64,
45}
46
47/// Validation result
48#[derive(Debug, Clone, Serialize, Deserialize)]
49pub struct ValidationResult {
50    /// Whether validation passed
51    pub passed: bool,
52    /// Validation errors
53    pub errors: Vec<String>,
54    /// Validation warnings
55    pub warnings: Vec<String>,
56    /// Performance metrics
57    pub performance_metrics: Option<PerformanceMetrics>,
58}
59
60/// Performance metrics measured by [`ModelValidator`].
61///
62/// Fields that were not measured are `None` — the validator never reports a
63/// placeholder as if it had been observed.
64#[derive(Debug, Clone, Serialize, Deserialize)]
65pub struct PerformanceMetrics {
66    /// Mean measured forward-pass time in milliseconds
67    pub forward_time_ms: f64,
68    /// Resident-set growth observed across the timed runs, in MB.
69    ///
70    /// `None` when the platform cannot report process memory.
71    pub memory_usage_mb: Option<f64>,
72    /// Throughput in samples/second, derived from the measured time
73    pub throughput: f64,
74    /// Parameter count, when the caller supplied a model to introspect
75    pub parameter_count: Option<usize>,
76}
77
78/// Model validator
79pub struct ModelValidator {
80    config: ValidationConfig,
81}
82
83impl ModelValidator {
84    /// Create a new model validator
85    pub fn new(config: ValidationConfig) -> Self {
86        Self { config }
87    }
88
89    /// Validate tensor output
90    pub fn validate_tensor(&self, name: &str, tensor: &Tensor) -> ValidationResult {
91        let mut result = ValidationResult {
92            passed: true,
93            errors: Vec::new(),
94            warnings: Vec::new(),
95            performance_metrics: None,
96        };
97
98        // Validate tensor shape
99        if let Some(expected_shape) = self.config.expected_shapes.get(name) {
100            let actual_shape = match tensor {
101                Tensor::F32(arr) => arr.shape().to_vec(),
102                Tensor::F64(arr) => arr.shape().to_vec(),
103                Tensor::I64(arr) => arr.shape().to_vec(),
104                _ => {
105                    result.errors.push("Unsupported tensor type for shape validation".to_string());
106                    result.passed = false;
107                    return result;
108                },
109            };
110
111            if actual_shape != *expected_shape {
112                result.errors.push(format!(
113                    "Shape mismatch for '{}': expected {:?}, got {:?}",
114                    name, expected_shape, actual_shape
115                ));
116                result.passed = false;
117            }
118        }
119
120        // Validate numerical values
121        match tensor {
122            Tensor::F32(arr) => {
123                self.validate_f32_values(name, arr, &mut result);
124            },
125            Tensor::F64(arr) => {
126                self.validate_f64_values(name, arr, &mut result);
127            },
128            Tensor::I64(_) => {
129                // Integer tensors are generally safe from NaN/Inf issues
130            },
131            _ => {
132                // Other tensor types (F16, BF16, Complex) not supported for validation yet
133            },
134        }
135
136        result
137    }
138
139    /// Validate F32 tensor values
140    fn validate_f32_values(&self, name: &str, arr: &ArrayD<f32>, result: &mut ValidationResult) {
141        // Check for NaN values
142        if self.config.check_nan {
143            let nan_count = arr.iter().filter(|&&x| x.is_nan()).count();
144            if nan_count > 0 {
145                result.errors.push(format!(
146                    "Found {} NaN values in tensor '{}'",
147                    nan_count, name
148                ));
149                result.passed = false;
150            }
151        }
152
153        // Check for infinite values
154        if self.config.check_infinite {
155            let inf_count = arr.iter().filter(|&&x| x.is_infinite()).count();
156            if inf_count > 0 {
157                result.errors.push(format!(
158                    "Found {} infinite values in tensor '{}'",
159                    inf_count, name
160                ));
161                result.passed = false;
162            }
163        }
164
165        // Check value ranges
166        if let Some((min_val, max_val)) = self.config.value_ranges.get(name) {
167            let actual_min = arr.iter().cloned().fold(f32::INFINITY, f32::min) as f64;
168            let actual_max = arr.iter().cloned().fold(f32::NEG_INFINITY, f32::max) as f64;
169
170            if actual_min < *min_val {
171                result.warnings.push(format!(
172                    "Values in '{}' below expected minimum: {} < {}",
173                    name, actual_min, min_val
174                ));
175            }
176
177            if actual_max > *max_val {
178                result.warnings.push(format!(
179                    "Values in '{}' above expected maximum: {} > {}",
180                    name, actual_max, max_val
181                ));
182            }
183        }
184    }
185
186    /// Validate F64 tensor values
187    fn validate_f64_values(&self, name: &str, arr: &ArrayD<f64>, result: &mut ValidationResult) {
188        // Check for NaN values
189        if self.config.check_nan {
190            let nan_count = arr.iter().filter(|&&x| x.is_nan()).count();
191            if nan_count > 0 {
192                result.errors.push(format!(
193                    "Found {} NaN values in tensor '{}'",
194                    nan_count, name
195                ));
196                result.passed = false;
197            }
198        }
199
200        // Check for infinite values
201        if self.config.check_infinite {
202            let inf_count = arr.iter().filter(|&&x| x.is_infinite()).count();
203            if inf_count > 0 {
204                result.errors.push(format!(
205                    "Found {} infinite values in tensor '{}'",
206                    inf_count, name
207                ));
208                result.passed = false;
209            }
210        }
211
212        // Check value ranges
213        if let Some((min_val, max_val)) = self.config.value_ranges.get(name) {
214            let actual_min = arr.iter().cloned().fold(f64::INFINITY, f64::min);
215            let actual_max = arr.iter().cloned().fold(f64::NEG_INFINITY, f64::max);
216
217            if actual_min < *min_val {
218                result.warnings.push(format!(
219                    "Values in '{}' below expected minimum: {} < {}",
220                    name, actual_min, min_val
221                ));
222            }
223
224            if actual_max > *max_val {
225                result.warnings.push(format!(
226                    "Values in '{}' above expected maximum: {} > {}",
227                    name, actual_max, max_val
228                ));
229            }
230        }
231    }
232
233    /// Validate a model configuration by delegating to the configuration's own
234    /// [`Config::validate`] implementation.
235    ///
236    /// The validator has no way to reason about an arbitrary struct's invariants;
237    /// the configuration does. Anything that does not implement [`Config`] cannot
238    /// be validated here — which is why the bound is `Config` and not `Debug`.
239    pub fn validate_config<C>(&self, config: &C) -> ValidationResult
240    where
241        C: Config,
242    {
243        let mut result = ValidationResult {
244            passed: true,
245            errors: Vec::new(),
246            warnings: Vec::new(),
247            performance_metrics: None,
248        };
249
250        if let Err(error) = config.validate() {
251            result.passed = false;
252            result.errors.push(format!(
253                "configuration for architecture `{}` is invalid: {error}",
254                config.architecture()
255            ));
256        }
257
258        if config.architecture().trim().is_empty() {
259            result.passed = false;
260            result
261                .errors
262                .push("the configuration reports an empty architecture name".to_string());
263        }
264
265        result
266    }
267
268    /// Validate model performance
269    pub fn validate_performance<F>(&self, mut model_fn: F) -> ValidationResult
270    where
271        F: FnMut() -> Result<Tensor>,
272    {
273        let mut result = ValidationResult {
274            passed: true,
275            errors: Vec::new(),
276            warnings: Vec::new(),
277            performance_metrics: None,
278        };
279
280        let memory_before = current_resident_mb();
281
282        // Warmup runs
283        for _ in 0..3 {
284            if let Err(e) = model_fn() {
285                result.errors.push(format!("Warmup run failed: {}", e));
286                result.passed = false;
287                return result;
288            }
289        }
290
291        // Timed runs
292        let mut durations = Vec::new();
293        let num_runs = 10;
294
295        for _ in 0..num_runs {
296            let start = std::time::Instant::now();
297            if let Err(e) = model_fn() {
298                result.errors.push(format!("Timed run failed: {}", e));
299                result.passed = false;
300                return result;
301            }
302            durations.push(start.elapsed().as_millis() as f64);
303        }
304
305        // Calculate performance metrics
306        let avg_time_ms = durations.iter().sum::<f64>() / durations.len() as f64;
307        let throughput = if avg_time_ms > 0.0 { 1000.0 / avg_time_ms } else { 0.0 };
308
309        let performance_metrics = PerformanceMetrics {
310            forward_time_ms: avg_time_ms,
311            memory_usage_mb: match (memory_before, current_resident_mb()) {
312                (Some(before), Some(after)) => Some(after - before),
313                _ => None,
314            },
315            throughput,
316            // The closure hides the model, so there is nothing to introspect here;
317            // `validate_model_performance` fills this in.
318            parameter_count: None,
319        };
320
321        // Check against thresholds
322        if avg_time_ms > self.config.performance_thresholds.max_forward_time_ms {
323            result.errors.push(format!(
324                "Forward pass too slow: {:.2}ms > {:.2}ms",
325                avg_time_ms, self.config.performance_thresholds.max_forward_time_ms
326            ));
327            result.passed = false;
328        }
329
330        if throughput < self.config.performance_thresholds.min_throughput {
331            result.warnings.push(format!(
332                "Throughput below threshold: {:.2} < {:.2} samples/sec",
333                throughput, self.config.performance_thresholds.min_throughput
334            ));
335        }
336
337        result.performance_metrics = Some(performance_metrics);
338        result
339    }
340
341    /// Validate a model's forward-pass performance and record its real parameter
342    /// count.
343    ///
344    /// Runs the same warm-up / timed-run protocol as
345    /// [`ModelValidator::validate_performance`], but because it owns the model it
346    /// can report `parameter_count` from [`Model::num_parameters`] instead of
347    /// leaving it unmeasured.
348    pub fn validate_model_performance<M>(&self, model: &M, input: &Tensor) -> ValidationResult
349    where
350        M: Model<Input = Tensor, Output = Tensor>,
351    {
352        let mut result = self.validate_performance(|| {
353            model.forward(input.clone()).map_err(|e| anyhow::anyhow!("{}", e))
354        });
355
356        if let Some(metrics) = result.performance_metrics.as_mut() {
357            metrics.parameter_count = Some(model.num_parameters());
358        }
359
360        result
361    }
362
363    /// Compare two tensors for numerical equality
364    pub fn compare_tensors(
365        &self,
366        name: &str,
367        expected: &Tensor,
368        actual: &Tensor,
369    ) -> ValidationResult {
370        let mut result = ValidationResult {
371            passed: true,
372            errors: Vec::new(),
373            warnings: Vec::new(),
374            performance_metrics: None,
375        };
376
377        match (expected, actual) {
378            (Tensor::F32(exp_arr), Tensor::F32(act_arr)) => {
379                if exp_arr.shape() != act_arr.shape() {
380                    result.errors.push(format!(
381                        "Shape mismatch for '{}': expected {:?}, got {:?}",
382                        name,
383                        exp_arr.shape(),
384                        act_arr.shape()
385                    ));
386                    result.passed = false;
387                    return result;
388                }
389
390                let mut max_diff = 0.0f64;
391                let mut total_diff = 0.0f64;
392                let mut count = 0;
393
394                for (exp_val, act_val) in exp_arr.iter().zip(act_arr.iter()) {
395                    let diff = (*exp_val - *act_val).abs() as f64;
396                    max_diff = max_diff.max(diff);
397                    total_diff += diff;
398                    count += 1;
399
400                    if diff > self.config.numerical_tolerance {
401                        result.errors.push(format!(
402                            "Value difference exceeds tolerance for '{}': {} vs {} (diff: {})",
403                            name, exp_val, act_val, diff
404                        ));
405                        result.passed = false;
406                    }
407                }
408
409                let avg_diff = total_diff / count as f64;
410                if avg_diff > self.config.numerical_tolerance * 0.1 {
411                    result.warnings.push(format!(
412                        "Average difference for '{}' is high: {:.6}",
413                        name, avg_diff
414                    ));
415                }
416            },
417            (Tensor::I64(exp_arr), Tensor::I64(act_arr)) => {
418                if exp_arr.shape() != act_arr.shape() {
419                    result.errors.push(format!(
420                        "Shape mismatch for '{}': expected {:?}, got {:?}",
421                        name,
422                        exp_arr.shape(),
423                        act_arr.shape()
424                    ));
425                    result.passed = false;
426                    return result;
427                }
428
429                for (exp_val, act_val) in exp_arr.iter().zip(act_arr.iter()) {
430                    if exp_val != act_val {
431                        result.errors.push(format!(
432                            "Integer value mismatch for '{}': {} vs {}",
433                            name, exp_val, act_val
434                        ));
435                        result.passed = false;
436                    }
437                }
438            },
439            _ => {
440                result.errors.push(format!(
441                    "Tensor type mismatch for '{}': different tensor types",
442                    name
443                ));
444                result.passed = false;
445            },
446        }
447
448        result
449    }
450
451    /// Validate model against reference implementation
452    pub fn validate_against_reference<M, R>(
453        &self,
454        model: M,
455        reference: R,
456        test_inputs: Vec<Tensor>,
457    ) -> ValidationResult
458    where
459        M: Fn(&Tensor) -> Result<Tensor>,
460        R: Fn(&Tensor) -> Result<Tensor>,
461    {
462        let mut result = ValidationResult {
463            passed: true,
464            errors: Vec::new(),
465            warnings: Vec::new(),
466            performance_metrics: None,
467        };
468
469        for (i, input) in test_inputs.iter().enumerate() {
470            let model_output = match model(input) {
471                Ok(output) => output,
472                Err(e) => {
473                    result.errors.push(format!("Model failed on input {}: {}", i, e));
474                    result.passed = false;
475                    continue;
476                },
477            };
478
479            let reference_output = match reference(input) {
480                Ok(output) => output,
481                Err(e) => {
482                    result.errors.push(format!("Reference failed on input {}: {}", i, e));
483                    result.passed = false;
484                    continue;
485                },
486            };
487
488            let comparison_result =
489                self.compare_tensors(&format!("output_{}", i), &reference_output, &model_output);
490
491            if !comparison_result.passed {
492                result.errors.extend(comparison_result.errors);
493                result.passed = false;
494            }
495            result.warnings.extend(comparison_result.warnings);
496        }
497
498        result
499    }
500}
501
502impl Default for ValidationConfig {
503    fn default() -> Self {
504        Self {
505            numerical_tolerance: 1e-5,
506            check_nan: true,
507            check_infinite: true,
508            expected_shapes: HashMap::new(),
509            value_ranges: HashMap::new(),
510            performance_thresholds: PerformanceThresholds {
511                max_forward_time_ms: 1000.0,
512                max_memory_mb: 1024.0,
513                min_throughput: 10.0,
514            },
515        }
516    }
517}
518
519/// Validation utilities
520pub struct ValidationUtils;
521
522impl ValidationUtils {
523    /// Create a strict validation config
524    pub fn strict_config() -> ValidationConfig {
525        ValidationConfig {
526            numerical_tolerance: 1e-6,
527            check_nan: true,
528            check_infinite: true,
529            expected_shapes: HashMap::new(),
530            value_ranges: HashMap::new(),
531            performance_thresholds: PerformanceThresholds {
532                max_forward_time_ms: 500.0,
533                max_memory_mb: 512.0,
534                min_throughput: 50.0,
535            },
536        }
537    }
538
539    /// Create a lenient validation config
540    pub fn lenient_config() -> ValidationConfig {
541        ValidationConfig {
542            numerical_tolerance: 1e-3,
543            check_nan: true,
544            check_infinite: true,
545            expected_shapes: HashMap::new(),
546            value_ranges: HashMap::new(),
547            performance_thresholds: PerformanceThresholds {
548                max_forward_time_ms: 5000.0,
549                max_memory_mb: 4096.0,
550                min_throughput: 1.0,
551            },
552        }
553    }
554
555    /// Create test tensors for validation
556    pub fn create_test_tensors() -> Result<Vec<Tensor>> {
557        Ok(vec![
558            Tensor::zeros(&[1, 10])?,
559            Tensor::ones(&[2, 20])?,
560            Tensor::randn(&[4, 32])?,
561            Tensor::randn(&[8, 64])?, // Using randn instead of rand
562        ])
563    }
564
565    /// Generate validation report
566    pub fn generate_report(results: &[ValidationResult]) -> String {
567        let mut report = String::new();
568        report.push_str("# Validation Report\n\n");
569
570        let total_tests = results.len();
571        let passed_tests = results.iter().filter(|r| r.passed).count();
572        let failed_tests = total_tests - passed_tests;
573
574        report.push_str("## Summary\n\n");
575        report.push_str(&format!("- Total tests: {}\n", total_tests));
576        report.push_str(&format!("- Passed: {}\n", passed_tests));
577        report.push_str(&format!("- Failed: {}\n\n", failed_tests));
578
579        if failed_tests > 0 {
580            report.push_str("## Errors\n\n");
581            for (i, result) in results.iter().enumerate() {
582                if !result.passed {
583                    report.push_str(&format!("### Test {}\n\n", i + 1));
584                    for error in &result.errors {
585                        report.push_str(&format!("- ❌ {}\n", error));
586                    }
587                    report.push('\n');
588                }
589            }
590        }
591
592        let total_warnings: usize = results.iter().map(|r| r.warnings.len()).sum();
593        if total_warnings > 0 {
594            report.push_str("## Warnings\n\n");
595            for (i, result) in results.iter().enumerate() {
596                if !result.warnings.is_empty() {
597                    report.push_str(&format!("### Test {}\n\n", i + 1));
598                    for warning in &result.warnings {
599                        report.push_str(&format!("- ⚠️ {}\n", warning));
600                    }
601                    report.push('\n');
602                }
603            }
604        }
605
606        // Performance metrics
607        let performance_results: Vec<_> =
608            results.iter().filter_map(|r| r.performance_metrics.as_ref()).collect();
609
610        if !performance_results.is_empty() {
611            report.push_str("## Performance Metrics\n\n");
612            let avg_time: f64 = performance_results.iter().map(|p| p.forward_time_ms).sum::<f64>()
613                / performance_results.len() as f64;
614            let avg_throughput: f64 = performance_results.iter().map(|p| p.throughput).sum::<f64>()
615                / performance_results.len() as f64;
616
617            report.push_str(&format!("- Average forward time: {:.2} ms\n", avg_time));
618            report.push_str(&format!(
619                "- Average throughput: {:.2} samples/sec\n",
620                avg_throughput
621            ));
622        }
623
624        report
625    }
626}
627
628/// Batch validator for running multiple validations
629pub struct BatchValidator {
630    validators: Vec<ModelValidator>,
631}
632
633impl BatchValidator {
634    /// Create a new batch validator
635    pub fn new() -> Self {
636        Self {
637            validators: Vec::new(),
638        }
639    }
640
641    /// Add a validator to the batch
642    pub fn add_validator(&mut self, validator: ModelValidator) {
643        self.validators.push(validator);
644    }
645
646    /// Run all validators on a tensor
647    pub fn validate_tensor_batch(&self, name: &str, tensor: &Tensor) -> Vec<ValidationResult> {
648        self.validators
649            .iter()
650            .map(|validator| validator.validate_tensor(name, tensor))
651            .collect()
652    }
653
654    /// Generate combined report
655    pub fn generate_batch_report(&self, results: &[Vec<ValidationResult>]) -> String {
656        let mut report = String::new();
657        report.push_str("# Batch Validation Report\n\n");
658
659        for (validator_idx, validator_results) in results.iter().enumerate() {
660            report.push_str(&format!("## Validator {}\n\n", validator_idx + 1));
661            report.push_str(&ValidationUtils::generate_report(validator_results));
662            report.push('\n');
663        }
664
665        report
666    }
667}
668
669impl Default for BatchValidator {
670    fn default() -> Self {
671        Self::new()
672    }
673}