1use anyhow::Result;
6use scirs2_core::ndarray::ArrayD; use serde::{Deserialize, Serialize};
8use std::collections::HashMap;
9use trustformers_core::tensor::Tensor;
10use trustformers_core::traits::{Config, Model};
11
12fn current_resident_mb() -> Option<f64> {
14 crate::memory_profiling::MemoryProfiler::get_process_memory_info()
15 .ok()
16 .map(|info| info.resident_mb)
17}
18
19#[derive(Debug, Clone, Serialize, Deserialize)]
21pub struct ValidationConfig {
22 pub numerical_tolerance: f64,
24 pub check_nan: bool,
26 pub check_infinite: bool,
28 pub expected_shapes: HashMap<String, Vec<usize>>,
30 pub value_ranges: HashMap<String, (f64, f64)>,
32 pub performance_thresholds: PerformanceThresholds,
34}
35
36#[derive(Debug, Clone, Serialize, Deserialize)]
38pub struct PerformanceThresholds {
39 pub max_forward_time_ms: f64,
41 pub max_memory_mb: f64,
43 pub min_throughput: f64,
45}
46
47#[derive(Debug, Clone, Serialize, Deserialize)]
49pub struct ValidationResult {
50 pub passed: bool,
52 pub errors: Vec<String>,
54 pub warnings: Vec<String>,
56 pub performance_metrics: Option<PerformanceMetrics>,
58}
59
60#[derive(Debug, Clone, Serialize, Deserialize)]
65pub struct PerformanceMetrics {
66 pub forward_time_ms: f64,
68 pub memory_usage_mb: Option<f64>,
72 pub throughput: f64,
74 pub parameter_count: Option<usize>,
76}
77
78pub struct ModelValidator {
80 config: ValidationConfig,
81}
82
83impl ModelValidator {
84 pub fn new(config: ValidationConfig) -> Self {
86 Self { config }
87 }
88
89 pub fn validate_tensor(&self, name: &str, tensor: &Tensor) -> ValidationResult {
91 let mut result = ValidationResult {
92 passed: true,
93 errors: Vec::new(),
94 warnings: Vec::new(),
95 performance_metrics: None,
96 };
97
98 if let Some(expected_shape) = self.config.expected_shapes.get(name) {
100 let actual_shape = match tensor {
101 Tensor::F32(arr) => arr.shape().to_vec(),
102 Tensor::F64(arr) => arr.shape().to_vec(),
103 Tensor::I64(arr) => arr.shape().to_vec(),
104 _ => {
105 result.errors.push("Unsupported tensor type for shape validation".to_string());
106 result.passed = false;
107 return result;
108 },
109 };
110
111 if actual_shape != *expected_shape {
112 result.errors.push(format!(
113 "Shape mismatch for '{}': expected {:?}, got {:?}",
114 name, expected_shape, actual_shape
115 ));
116 result.passed = false;
117 }
118 }
119
120 match tensor {
122 Tensor::F32(arr) => {
123 self.validate_f32_values(name, arr, &mut result);
124 },
125 Tensor::F64(arr) => {
126 self.validate_f64_values(name, arr, &mut result);
127 },
128 Tensor::I64(_) => {
129 },
131 _ => {
132 },
134 }
135
136 result
137 }
138
139 fn validate_f32_values(&self, name: &str, arr: &ArrayD<f32>, result: &mut ValidationResult) {
141 if self.config.check_nan {
143 let nan_count = arr.iter().filter(|&&x| x.is_nan()).count();
144 if nan_count > 0 {
145 result.errors.push(format!(
146 "Found {} NaN values in tensor '{}'",
147 nan_count, name
148 ));
149 result.passed = false;
150 }
151 }
152
153 if self.config.check_infinite {
155 let inf_count = arr.iter().filter(|&&x| x.is_infinite()).count();
156 if inf_count > 0 {
157 result.errors.push(format!(
158 "Found {} infinite values in tensor '{}'",
159 inf_count, name
160 ));
161 result.passed = false;
162 }
163 }
164
165 if let Some((min_val, max_val)) = self.config.value_ranges.get(name) {
167 let actual_min = arr.iter().cloned().fold(f32::INFINITY, f32::min) as f64;
168 let actual_max = arr.iter().cloned().fold(f32::NEG_INFINITY, f32::max) as f64;
169
170 if actual_min < *min_val {
171 result.warnings.push(format!(
172 "Values in '{}' below expected minimum: {} < {}",
173 name, actual_min, min_val
174 ));
175 }
176
177 if actual_max > *max_val {
178 result.warnings.push(format!(
179 "Values in '{}' above expected maximum: {} > {}",
180 name, actual_max, max_val
181 ));
182 }
183 }
184 }
185
186 fn validate_f64_values(&self, name: &str, arr: &ArrayD<f64>, result: &mut ValidationResult) {
188 if self.config.check_nan {
190 let nan_count = arr.iter().filter(|&&x| x.is_nan()).count();
191 if nan_count > 0 {
192 result.errors.push(format!(
193 "Found {} NaN values in tensor '{}'",
194 nan_count, name
195 ));
196 result.passed = false;
197 }
198 }
199
200 if self.config.check_infinite {
202 let inf_count = arr.iter().filter(|&&x| x.is_infinite()).count();
203 if inf_count > 0 {
204 result.errors.push(format!(
205 "Found {} infinite values in tensor '{}'",
206 inf_count, name
207 ));
208 result.passed = false;
209 }
210 }
211
212 if let Some((min_val, max_val)) = self.config.value_ranges.get(name) {
214 let actual_min = arr.iter().cloned().fold(f64::INFINITY, f64::min);
215 let actual_max = arr.iter().cloned().fold(f64::NEG_INFINITY, f64::max);
216
217 if actual_min < *min_val {
218 result.warnings.push(format!(
219 "Values in '{}' below expected minimum: {} < {}",
220 name, actual_min, min_val
221 ));
222 }
223
224 if actual_max > *max_val {
225 result.warnings.push(format!(
226 "Values in '{}' above expected maximum: {} > {}",
227 name, actual_max, max_val
228 ));
229 }
230 }
231 }
232
233 pub fn validate_config<C>(&self, config: &C) -> ValidationResult
240 where
241 C: Config,
242 {
243 let mut result = ValidationResult {
244 passed: true,
245 errors: Vec::new(),
246 warnings: Vec::new(),
247 performance_metrics: None,
248 };
249
250 if let Err(error) = config.validate() {
251 result.passed = false;
252 result.errors.push(format!(
253 "configuration for architecture `{}` is invalid: {error}",
254 config.architecture()
255 ));
256 }
257
258 if config.architecture().trim().is_empty() {
259 result.passed = false;
260 result
261 .errors
262 .push("the configuration reports an empty architecture name".to_string());
263 }
264
265 result
266 }
267
268 pub fn validate_performance<F>(&self, mut model_fn: F) -> ValidationResult
270 where
271 F: FnMut() -> Result<Tensor>,
272 {
273 let mut result = ValidationResult {
274 passed: true,
275 errors: Vec::new(),
276 warnings: Vec::new(),
277 performance_metrics: None,
278 };
279
280 let memory_before = current_resident_mb();
281
282 for _ in 0..3 {
284 if let Err(e) = model_fn() {
285 result.errors.push(format!("Warmup run failed: {}", e));
286 result.passed = false;
287 return result;
288 }
289 }
290
291 let mut durations = Vec::new();
293 let num_runs = 10;
294
295 for _ in 0..num_runs {
296 let start = std::time::Instant::now();
297 if let Err(e) = model_fn() {
298 result.errors.push(format!("Timed run failed: {}", e));
299 result.passed = false;
300 return result;
301 }
302 durations.push(start.elapsed().as_millis() as f64);
303 }
304
305 let avg_time_ms = durations.iter().sum::<f64>() / durations.len() as f64;
307 let throughput = if avg_time_ms > 0.0 { 1000.0 / avg_time_ms } else { 0.0 };
308
309 let performance_metrics = PerformanceMetrics {
310 forward_time_ms: avg_time_ms,
311 memory_usage_mb: match (memory_before, current_resident_mb()) {
312 (Some(before), Some(after)) => Some(after - before),
313 _ => None,
314 },
315 throughput,
316 parameter_count: None,
319 };
320
321 if avg_time_ms > self.config.performance_thresholds.max_forward_time_ms {
323 result.errors.push(format!(
324 "Forward pass too slow: {:.2}ms > {:.2}ms",
325 avg_time_ms, self.config.performance_thresholds.max_forward_time_ms
326 ));
327 result.passed = false;
328 }
329
330 if throughput < self.config.performance_thresholds.min_throughput {
331 result.warnings.push(format!(
332 "Throughput below threshold: {:.2} < {:.2} samples/sec",
333 throughput, self.config.performance_thresholds.min_throughput
334 ));
335 }
336
337 result.performance_metrics = Some(performance_metrics);
338 result
339 }
340
341 pub fn validate_model_performance<M>(&self, model: &M, input: &Tensor) -> ValidationResult
349 where
350 M: Model<Input = Tensor, Output = Tensor>,
351 {
352 let mut result = self.validate_performance(|| {
353 model.forward(input.clone()).map_err(|e| anyhow::anyhow!("{}", e))
354 });
355
356 if let Some(metrics) = result.performance_metrics.as_mut() {
357 metrics.parameter_count = Some(model.num_parameters());
358 }
359
360 result
361 }
362
363 pub fn compare_tensors(
365 &self,
366 name: &str,
367 expected: &Tensor,
368 actual: &Tensor,
369 ) -> ValidationResult {
370 let mut result = ValidationResult {
371 passed: true,
372 errors: Vec::new(),
373 warnings: Vec::new(),
374 performance_metrics: None,
375 };
376
377 match (expected, actual) {
378 (Tensor::F32(exp_arr), Tensor::F32(act_arr)) => {
379 if exp_arr.shape() != act_arr.shape() {
380 result.errors.push(format!(
381 "Shape mismatch for '{}': expected {:?}, got {:?}",
382 name,
383 exp_arr.shape(),
384 act_arr.shape()
385 ));
386 result.passed = false;
387 return result;
388 }
389
390 let mut max_diff = 0.0f64;
391 let mut total_diff = 0.0f64;
392 let mut count = 0;
393
394 for (exp_val, act_val) in exp_arr.iter().zip(act_arr.iter()) {
395 let diff = (*exp_val - *act_val).abs() as f64;
396 max_diff = max_diff.max(diff);
397 total_diff += diff;
398 count += 1;
399
400 if diff > self.config.numerical_tolerance {
401 result.errors.push(format!(
402 "Value difference exceeds tolerance for '{}': {} vs {} (diff: {})",
403 name, exp_val, act_val, diff
404 ));
405 result.passed = false;
406 }
407 }
408
409 let avg_diff = total_diff / count as f64;
410 if avg_diff > self.config.numerical_tolerance * 0.1 {
411 result.warnings.push(format!(
412 "Average difference for '{}' is high: {:.6}",
413 name, avg_diff
414 ));
415 }
416 },
417 (Tensor::I64(exp_arr), Tensor::I64(act_arr)) => {
418 if exp_arr.shape() != act_arr.shape() {
419 result.errors.push(format!(
420 "Shape mismatch for '{}': expected {:?}, got {:?}",
421 name,
422 exp_arr.shape(),
423 act_arr.shape()
424 ));
425 result.passed = false;
426 return result;
427 }
428
429 for (exp_val, act_val) in exp_arr.iter().zip(act_arr.iter()) {
430 if exp_val != act_val {
431 result.errors.push(format!(
432 "Integer value mismatch for '{}': {} vs {}",
433 name, exp_val, act_val
434 ));
435 result.passed = false;
436 }
437 }
438 },
439 _ => {
440 result.errors.push(format!(
441 "Tensor type mismatch for '{}': different tensor types",
442 name
443 ));
444 result.passed = false;
445 },
446 }
447
448 result
449 }
450
451 pub fn validate_against_reference<M, R>(
453 &self,
454 model: M,
455 reference: R,
456 test_inputs: Vec<Tensor>,
457 ) -> ValidationResult
458 where
459 M: Fn(&Tensor) -> Result<Tensor>,
460 R: Fn(&Tensor) -> Result<Tensor>,
461 {
462 let mut result = ValidationResult {
463 passed: true,
464 errors: Vec::new(),
465 warnings: Vec::new(),
466 performance_metrics: None,
467 };
468
469 for (i, input) in test_inputs.iter().enumerate() {
470 let model_output = match model(input) {
471 Ok(output) => output,
472 Err(e) => {
473 result.errors.push(format!("Model failed on input {}: {}", i, e));
474 result.passed = false;
475 continue;
476 },
477 };
478
479 let reference_output = match reference(input) {
480 Ok(output) => output,
481 Err(e) => {
482 result.errors.push(format!("Reference failed on input {}: {}", i, e));
483 result.passed = false;
484 continue;
485 },
486 };
487
488 let comparison_result =
489 self.compare_tensors(&format!("output_{}", i), &reference_output, &model_output);
490
491 if !comparison_result.passed {
492 result.errors.extend(comparison_result.errors);
493 result.passed = false;
494 }
495 result.warnings.extend(comparison_result.warnings);
496 }
497
498 result
499 }
500}
501
502impl Default for ValidationConfig {
503 fn default() -> Self {
504 Self {
505 numerical_tolerance: 1e-5,
506 check_nan: true,
507 check_infinite: true,
508 expected_shapes: HashMap::new(),
509 value_ranges: HashMap::new(),
510 performance_thresholds: PerformanceThresholds {
511 max_forward_time_ms: 1000.0,
512 max_memory_mb: 1024.0,
513 min_throughput: 10.0,
514 },
515 }
516 }
517}
518
519pub struct ValidationUtils;
521
522impl ValidationUtils {
523 pub fn strict_config() -> ValidationConfig {
525 ValidationConfig {
526 numerical_tolerance: 1e-6,
527 check_nan: true,
528 check_infinite: true,
529 expected_shapes: HashMap::new(),
530 value_ranges: HashMap::new(),
531 performance_thresholds: PerformanceThresholds {
532 max_forward_time_ms: 500.0,
533 max_memory_mb: 512.0,
534 min_throughput: 50.0,
535 },
536 }
537 }
538
539 pub fn lenient_config() -> ValidationConfig {
541 ValidationConfig {
542 numerical_tolerance: 1e-3,
543 check_nan: true,
544 check_infinite: true,
545 expected_shapes: HashMap::new(),
546 value_ranges: HashMap::new(),
547 performance_thresholds: PerformanceThresholds {
548 max_forward_time_ms: 5000.0,
549 max_memory_mb: 4096.0,
550 min_throughput: 1.0,
551 },
552 }
553 }
554
555 pub fn create_test_tensors() -> Result<Vec<Tensor>> {
557 Ok(vec![
558 Tensor::zeros(&[1, 10])?,
559 Tensor::ones(&[2, 20])?,
560 Tensor::randn(&[4, 32])?,
561 Tensor::randn(&[8, 64])?, ])
563 }
564
565 pub fn generate_report(results: &[ValidationResult]) -> String {
567 let mut report = String::new();
568 report.push_str("# Validation Report\n\n");
569
570 let total_tests = results.len();
571 let passed_tests = results.iter().filter(|r| r.passed).count();
572 let failed_tests = total_tests - passed_tests;
573
574 report.push_str("## Summary\n\n");
575 report.push_str(&format!("- Total tests: {}\n", total_tests));
576 report.push_str(&format!("- Passed: {}\n", passed_tests));
577 report.push_str(&format!("- Failed: {}\n\n", failed_tests));
578
579 if failed_tests > 0 {
580 report.push_str("## Errors\n\n");
581 for (i, result) in results.iter().enumerate() {
582 if !result.passed {
583 report.push_str(&format!("### Test {}\n\n", i + 1));
584 for error in &result.errors {
585 report.push_str(&format!("- ❌ {}\n", error));
586 }
587 report.push('\n');
588 }
589 }
590 }
591
592 let total_warnings: usize = results.iter().map(|r| r.warnings.len()).sum();
593 if total_warnings > 0 {
594 report.push_str("## Warnings\n\n");
595 for (i, result) in results.iter().enumerate() {
596 if !result.warnings.is_empty() {
597 report.push_str(&format!("### Test {}\n\n", i + 1));
598 for warning in &result.warnings {
599 report.push_str(&format!("- ⚠️ {}\n", warning));
600 }
601 report.push('\n');
602 }
603 }
604 }
605
606 let performance_results: Vec<_> =
608 results.iter().filter_map(|r| r.performance_metrics.as_ref()).collect();
609
610 if !performance_results.is_empty() {
611 report.push_str("## Performance Metrics\n\n");
612 let avg_time: f64 = performance_results.iter().map(|p| p.forward_time_ms).sum::<f64>()
613 / performance_results.len() as f64;
614 let avg_throughput: f64 = performance_results.iter().map(|p| p.throughput).sum::<f64>()
615 / performance_results.len() as f64;
616
617 report.push_str(&format!("- Average forward time: {:.2} ms\n", avg_time));
618 report.push_str(&format!(
619 "- Average throughput: {:.2} samples/sec\n",
620 avg_throughput
621 ));
622 }
623
624 report
625 }
626}
627
628pub struct BatchValidator {
630 validators: Vec<ModelValidator>,
631}
632
633impl BatchValidator {
634 pub fn new() -> Self {
636 Self {
637 validators: Vec::new(),
638 }
639 }
640
641 pub fn add_validator(&mut self, validator: ModelValidator) {
643 self.validators.push(validator);
644 }
645
646 pub fn validate_tensor_batch(&self, name: &str, tensor: &Tensor) -> Vec<ValidationResult> {
648 self.validators
649 .iter()
650 .map(|validator| validator.validate_tensor(name, tensor))
651 .collect()
652 }
653
654 pub fn generate_batch_report(&self, results: &[Vec<ValidationResult>]) -> String {
656 let mut report = String::new();
657 report.push_str("# Batch Validation Report\n\n");
658
659 for (validator_idx, validator_results) in results.iter().enumerate() {
660 report.push_str(&format!("## Validator {}\n\n", validator_idx + 1));
661 report.push_str(&ValidationUtils::generate_report(validator_results));
662 report.push('\n');
663 }
664
665 report
666 }
667}
668
669impl Default for BatchValidator {
670 fn default() -> Self {
671 Self::new()
672 }
673}