1pub mod regression_detector;
11
12use crate::RecognitionError;
13use std::collections::HashMap;
14use std::time::{Duration, Instant};
15use voirs_sdk::AudioBuffer;
16
17#[derive(Debug, Clone)]
19pub struct PerformanceRequirements {
20 pub max_rtf: f32,
22 pub max_memory_usage: u64,
24 pub max_startup_time_ms: u64,
26 pub max_streaming_latency_ms: u64,
28}
29
30impl Default for PerformanceRequirements {
31 fn default() -> Self {
32 Self {
33 max_rtf: 0.3,
34 max_memory_usage: 2 * 1024 * 1024 * 1024, max_startup_time_ms: 5000, max_streaming_latency_ms: 200, }
38 }
39}
40
41#[derive(Debug, Clone)]
43pub struct PerformanceMetrics {
44 pub rtf: f32,
46 pub memory_usage: u64,
48 pub startup_time_ms: u64,
50 pub streaming_latency_ms: u64,
52 pub throughput_samples_per_sec: f64,
54 pub cpu_utilization: f32,
56}
57
58#[derive(Debug, Clone)]
60pub struct ValidationResult {
61 pub passed: bool,
63 pub metrics: PerformanceMetrics,
65 pub requirements: PerformanceRequirements,
67 pub test_results: HashMap<String, bool>,
69 pub notes: Vec<String>,
71}
72
73pub struct PerformanceValidator {
75 requirements: PerformanceRequirements,
76 verbose: bool,
77}
78
79impl PerformanceValidator {
80 #[must_use]
82 pub fn new() -> Self {
83 Self {
84 requirements: PerformanceRequirements::default(),
85 verbose: false,
86 }
87 }
88
89 #[must_use]
91 pub fn with_requirements(requirements: PerformanceRequirements) -> Self {
92 Self {
93 requirements,
94 verbose: false,
95 }
96 }
97
98 #[must_use]
100 pub fn with_verbose(mut self, verbose: bool) -> Self {
101 self.verbose = verbose;
102 self
103 }
104
105 #[must_use]
107 pub fn requirements(&self) -> &PerformanceRequirements {
108 &self.requirements
109 }
110
111 #[must_use]
113 pub fn validate_rtf(&self, audio: &AudioBuffer, processing_time: Duration) -> (f32, bool) {
114 let audio_duration_seconds = audio.duration();
115 let processing_seconds = processing_time.as_secs_f32();
116 let rtf = processing_seconds / audio_duration_seconds;
117
118 let passed = rtf <= self.requirements.max_rtf;
119
120 if self.verbose {
121 println!(
122 "RTF Validation: {:.3} (target: ≤{:.3}) - {}",
123 rtf,
124 self.requirements.max_rtf,
125 if passed { "PASS" } else { "FAIL" }
126 );
127 }
128
129 (rtf, passed)
130 }
131
132 pub fn estimate_memory_usage(&self) -> Result<(u64, bool), RecognitionError> {
134 let memory_usage = get_memory_usage()?;
135 let passed = memory_usage <= self.requirements.max_memory_usage;
136
137 if self.verbose {
138 println!(
139 "Memory Usage: {:.2} MB (target: ≤{:.2} MB) - {}",
140 memory_usage as f64 / (1024.0 * 1024.0),
141 self.requirements.max_memory_usage as f64 / (1024.0 * 1024.0),
142 if passed { "PASS" } else { "FAIL" }
143 );
144 }
145
146 Ok((memory_usage, passed))
147 }
148
149 pub async fn measure_startup_time<F, Fut>(
151 &self,
152 startup_fn: F,
153 ) -> Result<(u64, bool), RecognitionError>
154 where
155 F: FnOnce() -> Fut,
156 Fut: std::future::Future<Output = Result<(), RecognitionError>>,
157 {
158 let start = Instant::now();
159 startup_fn().await?;
160 let startup_time = start.elapsed();
161
162 let startup_ms = startup_time.as_millis() as u64;
163 let passed = startup_ms <= self.requirements.max_startup_time_ms;
164
165 if self.verbose {
166 println!(
167 "Startup Time: {}ms (target: ≤{}ms) - {}",
168 startup_ms,
169 self.requirements.max_startup_time_ms,
170 if passed { "PASS" } else { "FAIL" }
171 );
172 }
173
174 Ok((startup_ms, passed))
175 }
176
177 #[must_use]
179 pub fn validate_streaming_latency(&self, latency: Duration) -> (u64, bool) {
180 let latency_ms = latency.as_millis() as u64;
181 let passed = latency_ms <= self.requirements.max_streaming_latency_ms;
182
183 if self.verbose {
184 println!(
185 "Streaming Latency: {}ms (target: ≤{}ms) - {}",
186 latency_ms,
187 self.requirements.max_streaming_latency_ms,
188 if passed { "PASS" } else { "FAIL" }
189 );
190 }
191
192 (latency_ms, passed)
193 }
194
195 #[must_use]
197 pub fn calculate_throughput(&self, samples_processed: usize, processing_time: Duration) -> f64 {
198 let processing_seconds = processing_time.as_secs_f64();
199 if processing_seconds > 0.0 {
200 samples_processed as f64 / processing_seconds
201 } else {
202 0.0
203 }
204 }
205
206 pub async fn validate_comprehensive<F, Fut>(
208 &self,
209 audio: &AudioBuffer,
210 startup_fn: F,
211 processing_time: Duration,
212 streaming_latency: Option<Duration>,
213 ) -> Result<ValidationResult, RecognitionError>
214 where
215 F: FnOnce() -> Fut,
216 Fut: std::future::Future<Output = Result<(), RecognitionError>>,
217 {
218 let mut test_results = HashMap::new();
219 let mut notes = Vec::new();
220
221 let (rtf, rtf_passed) = self.validate_rtf(audio, processing_time);
223 test_results.insert("rtf".to_string(), rtf_passed);
224
225 let (memory_usage, memory_passed) = self.estimate_memory_usage()?;
227 test_results.insert("memory".to_string(), memory_passed);
228
229 let (startup_time_ms, startup_passed) = self.measure_startup_time(startup_fn).await?;
231 test_results.insert("startup".to_string(), startup_passed);
232
233 let streaming_latency_ms = if let Some(latency) = streaming_latency {
235 let (latency_ms, latency_passed) = self.validate_streaming_latency(latency);
236 test_results.insert("streaming_latency".to_string(), latency_passed);
237 latency_ms
238 } else {
239 notes.push("Streaming latency not measured".to_string());
240 0
241 };
242
243 let throughput_samples_per_sec =
245 self.calculate_throughput(audio.samples().len(), processing_time);
246
247 let audio_duration = Duration::from_secs_f32(audio.duration());
249 let cpu_utilization = estimate_cpu_utilization(processing_time, audio_duration);
250
251 let metrics = PerformanceMetrics {
252 rtf,
253 memory_usage,
254 startup_time_ms,
255 streaming_latency_ms,
256 throughput_samples_per_sec,
257 cpu_utilization,
258 };
259
260 let passed = test_results.values().all(|&result| result);
262
263 if self.verbose {
264 println!("\n=== Performance Validation Summary ===");
265 println!("Overall Result: {}", if passed { "PASS" } else { "FAIL" });
266 println!(
267 "RTF: {:.3} ({})",
268 rtf,
269 if test_results["rtf"] { "PASS" } else { "FAIL" }
270 );
271 println!(
272 "Memory: {:.1} MB ({})",
273 memory_usage as f64 / (1024.0 * 1024.0),
274 if test_results["memory"] {
275 "PASS"
276 } else {
277 "FAIL"
278 }
279 );
280 println!(
281 "Startup: {}ms ({})",
282 startup_time_ms,
283 if test_results["startup"] {
284 "PASS"
285 } else {
286 "FAIL"
287 }
288 );
289 if streaming_latency.is_some() {
290 println!(
291 "Streaming Latency: {}ms ({})",
292 streaming_latency_ms,
293 if *test_results.get("streaming_latency").unwrap_or(&false) {
294 "PASS"
295 } else {
296 "FAIL"
297 }
298 );
299 }
300 println!("Throughput: {throughput_samples_per_sec:.0} samples/sec");
301 println!("CPU Utilization: {cpu_utilization:.1}%");
302 }
303
304 Ok(ValidationResult {
305 passed,
306 metrics,
307 requirements: self.requirements.clone(),
308 test_results,
309 notes,
310 })
311 }
312}
313
314impl Default for PerformanceValidator {
315 fn default() -> Self {
316 Self::new()
317 }
318}
319
320fn get_memory_usage() -> Result<u64, RecognitionError> {
322 #[cfg(target_os = "linux")]
323 {
324 use std::fs;
325 let status = fs::read_to_string("/proc/self/status").map_err(|e| {
326 RecognitionError::ResourceError {
327 message: format!("Failed to read memory info: {e}"),
328 source: Some(Box::new(e)),
329 }
330 })?;
331
332 for line in status.lines() {
333 if line.starts_with("VmRSS:") {
334 let parts: Vec<&str> = line.split_whitespace().collect();
335 if parts.len() >= 2 {
336 let kb: u64 = parts[1].parse().unwrap_or(0);
337 return Ok(kb * 1024); }
339 }
340 }
341 Ok(0)
342 }
343
344 #[cfg(target_os = "macos")]
345 {
346 use std::process::Command;
347 let output = Command::new("ps")
348 .args(["-o", "rss=", "-p", &std::process::id().to_string()])
349 .output()
350 .map_err(|e| RecognitionError::ResourceError {
351 message: format!("Failed to get memory info: {e}"),
352 source: Some(Box::new(e)),
353 })?;
354
355 let output_str = String::from_utf8_lossy(&output.stdout);
356 let kb: u64 = output_str.trim().parse().unwrap_or(0);
357 Ok(kb * 1024) }
359
360 #[cfg(target_os = "windows")]
361 {
362 if let Ok(output) = std::process::Command::new("powershell")
366 .args([
367 "-Command",
368 &format!("(Get-Process -Id {}).WorkingSet64", std::process::id()),
369 ])
370 .output()
371 {
372 if let Ok(output_str) = String::from_utf8(output.stdout) {
373 if let Ok(bytes) = output_str.trim().parse::<u64>() {
374 tracing::debug!(
375 "Windows memory usage detected via PowerShell: {} MB",
376 bytes / 1024 / 1024
377 );
378 return Ok(bytes);
379 }
380 }
381 }
382
383 if let Ok(output) = std::process::Command::new("wmic")
385 .args([
386 "process",
387 "where",
388 &format!("ProcessId={}", std::process::id()),
389 "get",
390 "WorkingSetSize",
391 "/value",
392 ])
393 .output()
394 {
395 if let Ok(output_str) = String::from_utf8(output.stdout) {
396 for line in output_str.lines() {
397 if line.starts_with("WorkingSetSize=") {
398 if let Ok(bytes) = line
399 .strip_prefix("WorkingSetSize=")
400 .unwrap_or("")
401 .parse::<u64>()
402 {
403 tracing::debug!(
404 "Windows memory usage detected via WMIC: {} MB",
405 bytes / 1024 / 1024
406 );
407 return Ok(bytes);
408 }
409 }
410 }
411 }
412 }
413
414 if let Ok(output) = std::process::Command::new("tasklist")
416 .args([
417 "/FI",
418 &format!("PID eq {}", std::process::id()),
419 "/FO",
420 "CSV",
421 ])
422 .output()
423 {
424 if let Ok(output_str) = String::from_utf8(output.stdout) {
425 for line in output_str.lines().skip(1) {
427 let fields: Vec<&str> = line.split(',').collect();
428 if fields.len() >= 5 {
429 let memory_str = fields[4].trim_matches('"').replace([',', ' '], "");
430 if let Ok(kb) = memory_str.replace('K', "").parse::<u64>() {
431 let bytes = kb * 1024;
432 tracing::debug!(
433 "Windows memory usage detected via tasklist: {} MB",
434 bytes / 1024 / 1024
435 );
436 return Ok(bytes);
437 }
438 }
439 }
440 }
441 }
442
443 if let Ok(output) = std::process::Command::new("powershell")
445 .args([
446 "-Command",
447 &format!(
448 "Get-Process -Id {} | Select-Object -ExpandProperty WorkingSet",
449 std::process::id()
450 ),
451 ])
452 .output()
453 {
454 if let Ok(output_str) = String::from_utf8(output.stdout) {
455 if let Ok(bytes) = output_str.trim().parse::<u64>() {
456 tracing::debug!(
457 "Windows memory usage detected via PowerShell fallback: {} MB",
458 bytes / 1024 / 1024
459 );
460 return Ok(bytes);
461 }
462 }
463 }
464
465 let estimated_usage = estimate_process_memory_usage();
467 tracing::warn!(
468 "Could not detect actual Windows memory usage, using intelligent estimation: {} MB",
469 estimated_usage / 1024 / 1024
470 );
471 Ok(estimated_usage)
472 }
473
474 #[cfg(not(any(target_os = "linux", target_os = "macos", target_os = "windows")))]
475 {
476 let estimated_usage = estimate_process_memory_usage();
478 tracing::warn!(
479 "Platform-specific memory measurement not available, using estimation: {} MB",
480 estimated_usage / 1024 / 1024
481 );
482 Ok(estimated_usage)
483 }
484}
485
486fn estimate_process_memory_usage() -> u64 {
488 let mut estimated_memory = 50 * 1024 * 1024; if let Ok(total_memory) = get_total_system_memory() {
493 let memory_percentage = if total_memory > 32 * 1024 * 1024 * 1024 {
495 0.010 } else if total_memory > 16 * 1024 * 1024 * 1024 {
498 0.008 } else if total_memory > 8 * 1024 * 1024 * 1024 {
501 0.006 } else if total_memory > 4 * 1024 * 1024 * 1024 {
504 0.005 } else {
507 0.004 };
510
511 estimated_memory = ((total_memory as f64 * memory_percentage) as u64)
512 .max(estimated_memory)
513 .min(800 * 1024 * 1024); }
515
516 estimated_memory += detect_model_memory_footprint();
518
519 tracing::debug!(
520 "Estimated process memory usage: {:.1} MB (based on system memory detection)",
521 estimated_memory as f64 / (1024.0 * 1024.0)
522 );
523
524 estimated_memory
525}
526
527fn detect_model_memory_footprint() -> u64 {
529 let mut model_memory = 30 * 1024 * 1024; model_memory += 120 * 1024 * 1024; if std::env::var("VOIRS_LARGE_MODELS").is_ok() {
539 model_memory += 200 * 1024 * 1024; }
541
542 model_memory += 16 * 1024 * 1024; model_memory += 32 * 1024 * 1024; model_memory
549}
550
551fn get_total_system_memory() -> Result<u64, ()> {
553 #[cfg(target_os = "linux")]
554 {
555 if let Ok(meminfo) = std::fs::read_to_string("/proc/meminfo") {
556 for line in meminfo.lines() {
557 if line.starts_with("MemTotal:") {
558 let parts: Vec<&str> = line.split_whitespace().collect();
559 if parts.len() >= 2 {
560 if let Ok(kb) = parts[1].parse::<u64>() {
561 return Ok(kb * 1024); }
563 }
564 }
565 }
566 }
567 }
568
569 #[cfg(target_os = "macos")]
570 {
571 if let Ok(output) = std::process::Command::new("sysctl")
572 .args(["-n", "hw.memsize"])
573 .output()
574 {
575 if let Ok(output_str) = String::from_utf8(output.stdout) {
576 if let Ok(bytes) = output_str.trim().parse::<u64>() {
577 return Ok(bytes);
578 }
579 }
580 }
581 }
582
583 #[cfg(target_os = "windows")]
584 {
585 if let Ok(output) = std::process::Command::new("wmic")
587 .args(["computersystem", "get", "TotalPhysicalMemory", "/value"])
588 .output()
589 {
590 if let Ok(output_str) = String::from_utf8(output.stdout) {
591 for line in output_str.lines() {
592 if line.starts_with("TotalPhysicalMemory=") {
593 if let Ok(bytes) = line
594 .strip_prefix("TotalPhysicalMemory=")
595 .unwrap_or("")
596 .parse::<u64>()
597 {
598 return Ok(bytes);
599 }
600 }
601 }
602 }
603 }
604
605 if let Ok(output) = std::process::Command::new("powershell")
607 .args([
608 "-Command",
609 "(Get-WmiObject -Class Win32_ComputerSystem).TotalPhysicalMemory",
610 ])
611 .output()
612 {
613 if let Ok(output_str) = String::from_utf8(output.stdout) {
614 if let Ok(bytes) = output_str.trim().parse::<u64>() {
615 return Ok(bytes);
616 }
617 }
618 }
619
620 if let Ok(output) = std::process::Command::new("systeminfo").output() {
622 if let Ok(output_str) = String::from_utf8(output.stdout) {
623 for line in output_str.lines() {
624 if line.contains("Total Physical Memory:") {
625 if let Some(memory_part) = line.split(':').nth(1) {
627 let cleaned = memory_part
628 .replace(',', "")
629 .replace(" MB", "")
630 .trim()
631 .to_string();
632 if let Ok(mb) = cleaned.parse::<u64>() {
633 return Ok(mb * 1024 * 1024); }
635 }
636 }
637 }
638 }
639 }
640
641 tracing::warn!("Could not detect Windows system memory, using conservative estimate");
643 Ok(8 * 1024 * 1024 * 1024) }
645
646 Ok(8 * 1024 * 1024 * 1024) }
649
650fn estimate_cpu_utilization(processing_time: Duration, audio_duration: Duration) -> f32 {
652 if audio_duration.as_secs_f32() > 0.0 {
653 let utilization = (processing_time.as_secs_f32() / audio_duration.as_secs_f32()) * 100.0;
654 utilization.min(100.0) } else {
656 0.0
657 }
658}
659
660#[cfg(test)]
661mod tests {
662 use super::*;
663 use std::time::Duration;
664
665 #[test]
666 fn test_performance_requirements_default() {
667 let req = PerformanceRequirements::default();
668 assert_eq!(req.max_rtf, 0.3);
669 assert_eq!(req.max_memory_usage, 2 * 1024 * 1024 * 1024);
670 assert_eq!(req.max_startup_time_ms, 5000);
671 assert_eq!(req.max_streaming_latency_ms, 200);
672 }
673
674 #[test]
675 fn test_validator_creation() {
676 let validator = PerformanceValidator::new();
677 assert_eq!(validator.requirements.max_rtf, 0.3);
678 assert!(!validator.verbose);
679
680 let validator = PerformanceValidator::new().with_verbose(true);
681 assert!(validator.verbose);
682 }
683
684 #[test]
685 fn test_rtf_validation() {
686 let validator = PerformanceValidator::new();
687 let audio = AudioBuffer::mono(vec![0.0; 16000], 16000); let processing_time = Duration::from_millis(200); let (rtf, passed) = validator.validate_rtf(&audio, processing_time);
692 assert_eq!(rtf, 0.2);
693 assert!(passed);
694
695 let processing_time = Duration::from_millis(500); let (rtf, passed) = validator.validate_rtf(&audio, processing_time);
698 assert_eq!(rtf, 0.5);
699 assert!(!passed);
700 }
701
702 #[test]
703 fn test_streaming_latency_validation() {
704 let validator = PerformanceValidator::new();
705
706 let latency = Duration::from_millis(150);
708 let (latency_ms, passed) = validator.validate_streaming_latency(latency);
709 assert_eq!(latency_ms, 150);
710 assert!(passed);
711
712 let latency = Duration::from_millis(300);
714 let (latency_ms, passed) = validator.validate_streaming_latency(latency);
715 assert_eq!(latency_ms, 300);
716 assert!(!passed);
717 }
718
719 #[test]
720 fn test_throughput_calculation() {
721 let validator = PerformanceValidator::new();
722 let processing_time = Duration::from_millis(100);
723 let throughput = validator.calculate_throughput(1600, processing_time);
724 assert_eq!(throughput, 16000.0); }
726
727 #[test]
728 fn test_cpu_utilization_estimation() {
729 let processing_time = Duration::from_millis(200);
730 let audio_duration = Duration::from_secs(1);
731 let utilization = estimate_cpu_utilization(processing_time, audio_duration);
732 assert_eq!(utilization, 20.0); }
734
735 #[test]
736 fn test_memory_usage_estimation() {
737 let usage = estimate_process_memory_usage();
738
739 assert!(usage >= 50 * 1024 * 1024);
741
742 assert!(usage <= 2 * 1024 * 1024 * 1024);
744
745 assert!(usage >= 150 * 1024 * 1024); }
748
749 #[test]
750 fn test_model_memory_footprint_detection() {
751 let footprint = detect_model_memory_footprint();
752
753 assert!(footprint >= 30 * 1024 * 1024);
755
756 assert!(footprint >= 150 * 1024 * 1024);
758
759 assert!(footprint <= 1024 * 1024 * 1024); }
762
763 #[test]
764 fn test_system_memory_detection() {
765 match get_total_system_memory() {
767 Ok(memory) => {
768 assert!(memory >= 1024 * 1024 * 1024);
770 assert!(memory <= 1024 * 1024 * 1024 * 1024);
772 }
773 Err(_) => {
774 println!("System memory detection not supported on this platform");
776 }
777 }
778 }
779
780 #[test]
781 fn test_memory_percentage_calculations() {
782 let test_cases = [
784 (2u64 * 1024 * 1024 * 1024, 0.004), (8u64 * 1024 * 1024 * 1024, 0.005), (9u64 * 1024 * 1024 * 1024, 0.006), (16u64 * 1024 * 1024 * 1024, 0.006), (17u64 * 1024 * 1024 * 1024, 0.008), (32u64 * 1024 * 1024 * 1024, 0.008), (33u64 * 1024 * 1024 * 1024, 0.010), ];
792
793 for (total_memory, expected_percentage) in test_cases {
794 let percentage = if total_memory > 32u64 * 1024 * 1024 * 1024 {
795 0.010
796 } else if total_memory > 16u64 * 1024 * 1024 * 1024 {
797 0.008
798 } else if total_memory > 8u64 * 1024 * 1024 * 1024 {
799 0.006
800 } else if total_memory > 4u64 * 1024 * 1024 * 1024 {
801 0.005
802 } else {
803 0.004
804 };
805
806 assert_eq!(percentage, expected_percentage);
807 }
808 }
809
810 #[tokio::test]
811 async fn test_startup_time_measurement() {
812 let validator = PerformanceValidator::new();
813
814 let startup_fn = || async {
815 tokio::time::sleep(Duration::from_millis(100)).await;
816 Ok(())
817 };
818
819 let result = validator.measure_startup_time(startup_fn).await;
820 assert!(result.is_ok());
821
822 let (startup_ms, passed) = result.unwrap();
823 assert!(startup_ms >= 100);
824 assert!(startup_ms < 5000); assert!(passed);
826 }
827}