use async_trait::async_trait;
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
use std::time::Duration;
use voirs_recognizer::traits::PhonemeAlignment;
use voirs_sdk::{AudioBuffer, LanguageCode, VoirsError};
pub type EvaluationResult<T> = Result<T, VoirsError>;
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct QualityScore {
pub overall_score: f32,
pub component_scores: HashMap<String, f32>,
pub recommendations: Vec<String>,
pub confidence: f32,
pub processing_time: Option<Duration>,
}
#[derive(Debug, Clone, PartialEq)]
pub struct PronunciationScore {
pub overall_score: f32,
pub phoneme_scores: Vec<PhonemeAccuracyScore>,
pub word_scores: Vec<WordPronunciationScore>,
pub fluency_score: f32,
pub rhythm_score: f32,
pub stress_accuracy: f32,
pub intonation_accuracy: f32,
pub feedback: Vec<PronunciationFeedback>,
pub confidence: f32,
}
#[derive(Debug, Clone, PartialEq)]
pub struct PhonemeAccuracyScore {
pub expected_phoneme: String,
pub actual_phoneme: Option<String>,
pub accuracy: f32,
pub duration_accuracy: f32,
pub position: usize,
pub start_time: f32,
pub end_time: f32,
}
#[derive(Debug, Clone, PartialEq)]
pub struct WordPronunciationScore {
pub word: String,
pub accuracy: f32,
pub stress_accuracy: f32,
pub syllable_accuracy: f32,
pub phoneme_scores: Vec<PhonemeAccuracyScore>,
pub position: usize,
}
#[derive(Debug, Clone, PartialEq)]
pub struct PronunciationFeedback {
pub position: usize,
pub feedback_type: FeedbackType,
pub severity: f32,
pub message: String,
pub suggestion: Option<String>,
}
#[derive(Debug, Clone, PartialEq)]
pub enum FeedbackType {
PhonemeSubstitution,
PhonemeDeletion,
PhonemeInsertion,
StressError,
DurationError,
RhythmError,
IntonationError,
QualityIssue,
}
#[derive(Debug, Clone, PartialEq)]
pub struct ComparisonResult {
pub system_a: String,
pub system_b: String,
pub preference_score: f32,
pub metric_comparisons: HashMap<String, MetricComparison>,
pub statistical_significance: HashMap<String, f32>,
pub analysis: String,
pub confidence: f32,
}
#[derive(Debug, Clone, PartialEq)]
pub struct MetricComparison {
pub metric: String,
pub score_a: f32,
pub score_b: f32,
pub difference: f32,
pub relative_improvement: f32,
pub p_value: f32,
}
#[derive(Debug, Clone, PartialEq)]
pub struct SelfEvaluationResult {
pub quality_score: QualityScore,
pub improvement_areas: Vec<ImprovementArea>,
pub suggested_changes: Vec<ConfigurationSuggestion>,
pub performance_metrics: HashMap<String, f32>,
pub system_health: f32,
}
#[derive(Debug, Clone, PartialEq)]
pub struct ImprovementArea {
pub area: String,
pub current_performance: f32,
pub target_performance: f32,
pub priority: f32,
pub suggestions: Vec<ImprovementSuggestion>,
}
#[derive(Debug, Clone, PartialEq)]
pub struct ImprovementSuggestion {
pub suggestion_type: SuggestionType,
pub description: String,
pub expected_impact: f32,
pub difficulty: f32,
pub priority: f32,
}
#[derive(Debug, Clone, PartialEq)]
pub struct ConfigurationSuggestion {
pub parameter: String,
pub current_value: String,
pub suggested_value: String,
pub rationale: String,
pub expected_improvement: f32,
}
#[derive(Debug, Clone, PartialEq)]
pub enum SuggestionType {
ParameterTuning,
DataImprovement,
ArchitectureChange,
PostProcessing,
Preprocessing,
HardwareOptimization,
}
#[derive(Debug, Clone, PartialEq)]
pub struct QualityEvaluationConfig {
pub objective_metrics: bool,
pub subjective_metrics: bool,
pub perceptual_metrics: bool,
pub metrics: Vec<QualityMetric>,
pub require_reference: bool,
pub detailed_analysis: bool,
pub confidence_threshold: f32,
pub use_deep_learning_mos: bool,
pub demographic_adaptation: Option<DemographicProfile>,
pub cultural_adaptation: Option<CulturalProfile>,
pub simulate_listening_tests: bool,
pub real_time_mode: bool,
}
impl Default for QualityEvaluationConfig {
fn default() -> Self {
Self {
objective_metrics: true,
subjective_metrics: true,
perceptual_metrics: false,
metrics: vec![
QualityMetric::MOS,
QualityMetric::SpectralDistortion,
QualityMetric::Naturalness,
QualityMetric::Intelligibility,
QualityMetric::ArtifactDetection,
],
require_reference: false,
detailed_analysis: true,
confidence_threshold: 0.7,
use_deep_learning_mos: false,
demographic_adaptation: None,
cultural_adaptation: None,
simulate_listening_tests: false,
real_time_mode: false,
}
}
}
#[derive(Debug, Clone)]
pub struct PronunciationEvaluationConfig {
pub language: LanguageCode,
pub phoneme_level_scoring: bool,
pub word_level_scoring: bool,
pub prosody_assessment: bool,
pub metrics: Vec<PronunciationMetric>,
pub strictness: f32,
pub native_reference: Option<AudioBuffer>,
pub custom_dictionary: Option<HashMap<String, Vec<String>>>,
}
impl Default for PronunciationEvaluationConfig {
fn default() -> Self {
Self {
language: LanguageCode::EnUs,
phoneme_level_scoring: true,
word_level_scoring: true,
prosody_assessment: true,
metrics: vec![
PronunciationMetric::PhonemeAccuracy,
PronunciationMetric::WordAccuracy,
PronunciationMetric::Fluency,
PronunciationMetric::Rhythm,
PronunciationMetric::StressAccuracy,
],
strictness: 0.7,
native_reference: None,
custom_dictionary: None,
}
}
}
#[derive(Debug, Clone, PartialEq)]
pub struct ComparisonConfig {
pub metrics: Vec<ComparisonMetric>,
pub enable_statistical_analysis: bool,
pub significance_threshold: f32,
pub bootstrap_samples: usize,
pub predict_human_preference: bool,
pub detailed_breakdown: bool,
}
impl Default for ComparisonConfig {
fn default() -> Self {
Self {
metrics: vec![
ComparisonMetric::OverallQuality,
ComparisonMetric::Naturalness,
ComparisonMetric::Intelligibility,
ComparisonMetric::PronunciationAccuracy,
],
enable_statistical_analysis: true,
significance_threshold: 0.05,
bootstrap_samples: 1000,
predict_human_preference: true,
detailed_breakdown: true,
}
}
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub enum QualityMetric {
MOS,
PESQ,
STOI,
MCD,
SpectralDistortion,
Naturalness,
Intelligibility,
SpeakerSimilarity,
ProsodyQuality,
ArtifactDetection,
}
#[derive(Debug, Clone, PartialEq)]
pub enum PronunciationMetric {
PhonemeAccuracy,
WordAccuracy,
SentenceAccuracy,
Fluency,
Rhythm,
StressAccuracy,
IntonationAccuracy,
SpeakingRate,
PausePattern,
Comprehensibility,
}
#[derive(Debug, Clone, PartialEq)]
pub enum ComparisonMetric {
OverallQuality,
Naturalness,
Intelligibility,
PronunciationAccuracy,
Prosody,
SpeakerConsistency,
Artifacts,
Efficiency,
}
#[async_trait]
pub trait QualityEvaluator: Send + Sync {
async fn evaluate_quality(
&self,
generated: &AudioBuffer,
reference: Option<&AudioBuffer>,
config: Option<&QualityEvaluationConfig>,
) -> EvaluationResult<QualityScore>;
async fn evaluate_quality_batch(
&self,
samples: &[(AudioBuffer, Option<AudioBuffer>)],
config: Option<&QualityEvaluationConfig>,
) -> EvaluationResult<Vec<QualityScore>>;
fn supported_metrics(&self) -> Vec<QualityMetric>;
fn requires_reference(&self, metric: &QualityMetric) -> bool;
fn metadata(&self) -> QualityEvaluatorMetadata;
}
#[async_trait]
pub trait PronunciationEvaluator: Send + Sync {
async fn evaluate_pronunciation(
&self,
audio: &AudioBuffer,
text: &str,
config: Option<&PronunciationEvaluationConfig>,
) -> EvaluationResult<PronunciationScore>;
async fn evaluate_pronunciation_with_alignment(
&self,
audio: &AudioBuffer,
alignment: &PhonemeAlignment,
config: Option<&PronunciationEvaluationConfig>,
) -> EvaluationResult<PronunciationScore>;
async fn evaluate_pronunciation_batch(
&self,
samples: &[(AudioBuffer, String)],
config: Option<&PronunciationEvaluationConfig>,
) -> EvaluationResult<Vec<PronunciationScore>>;
fn supported_metrics(&self) -> Vec<PronunciationMetric>;
fn supported_languages(&self) -> Vec<LanguageCode>;
fn metadata(&self) -> PronunciationEvaluatorMetadata;
}
#[async_trait]
pub trait ComparativeEvaluator: Send + Sync {
async fn compare_samples(
&self,
sample_a: &AudioBuffer,
sample_b: &AudioBuffer,
config: Option<&ComparisonConfig>,
) -> EvaluationResult<ComparisonResult>;
async fn compare_systems(
&self,
system_a_samples: &[AudioBuffer],
system_b_samples: &[AudioBuffer],
config: Option<&ComparisonConfig>,
) -> EvaluationResult<ComparisonResult>;
async fn compare_multiple_systems(
&self,
systems: &HashMap<String, Vec<AudioBuffer>>,
config: Option<&ComparisonConfig>,
) -> EvaluationResult<HashMap<(String, String), ComparisonResult>>;
fn supported_metrics(&self) -> Vec<ComparisonMetric>;
fn metadata(&self) -> ComparativeEvaluatorMetadata;
}
#[async_trait]
pub trait SelfEvaluator: Send + Sync {
async fn self_evaluate(
&self,
pipeline: &voirs_sdk::pipeline::VoirsPipeline,
test_texts: &[String],
config: Option<&QualityEvaluationConfig>,
) -> EvaluationResult<SelfEvaluationResult>;
async fn suggest_improvements(
&self,
evaluation: &SelfEvaluationResult,
) -> EvaluationResult<Vec<ImprovementSuggestion>>;
async fn monitor_performance(
&self,
pipeline: &voirs_sdk::pipeline::VoirsPipeline,
monitoring_texts: &[String],
) -> EvaluationResult<PerformanceReport>;
fn metadata(&self) -> SelfEvaluatorMetadata;
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct QualityEvaluatorMetadata {
pub name: String,
pub version: String,
pub description: String,
pub supported_metrics: Vec<QualityMetric>,
pub supported_languages: Vec<LanguageCode>,
pub requires_reference: bool,
pub processing_speed: f32,
}
#[derive(Debug, Clone, PartialEq)]
pub struct PronunciationEvaluatorMetadata {
pub name: String,
pub version: String,
pub description: String,
pub supported_metrics: Vec<PronunciationMetric>,
pub supported_languages: Vec<LanguageCode>,
pub accuracy_benchmarks: HashMap<LanguageCode, f32>,
pub processing_speed: f32,
}
#[derive(Debug, Clone, PartialEq)]
pub struct ComparativeEvaluatorMetadata {
pub name: String,
pub version: String,
pub description: String,
pub supported_metrics: Vec<ComparisonMetric>,
pub statistical_methods: Vec<String>,
pub processing_speed: f32,
}
#[derive(Debug, Clone, PartialEq)]
pub struct SelfEvaluatorMetadata {
pub name: String,
pub version: String,
pub description: String,
pub capabilities: Vec<String>,
pub improvement_types: Vec<SuggestionType>,
}
#[derive(Debug, Clone, PartialEq)]
pub struct PerformanceReport {
pub timestamp: chrono::DateTime<chrono::Utc>,
pub overall_score: f32,
pub metric_trends: HashMap<String, Vec<(chrono::DateTime<chrono::Utc>, f32)>>,
pub alerts: Vec<PerformanceAlert>,
pub recommendations: Vec<String>,
}
#[derive(Debug, Clone, PartialEq)]
pub struct PerformanceAlert {
pub severity: AlertSeverity,
pub message: String,
pub metric: String,
pub current_value: f32,
pub expected_value: f32,
pub suggested_action: Option<String>,
}
#[derive(Debug, Clone, PartialEq)]
pub enum AlertSeverity {
Info,
Warning,
Critical,
}
#[derive(Debug, Clone, PartialEq)]
pub struct DemographicProfile {
pub age_group: AgeGroup,
pub gender: Gender,
pub education_level: EducationLevel,
pub audio_expertise: AudioExpertise,
pub language_proficiency: LanguageProficiency,
pub hearing_ability: HearingAbility,
}
#[derive(Debug, Clone, PartialEq)]
pub struct CulturalProfile {
pub cultural_background: CulturalBackground,
pub regional_preferences: RegionalPreferences,
pub language_group: LanguageGroup,
pub speech_style_preferences: SpeechStylePreferences,
pub perceptual_biases: Vec<PerceptualBias>,
}
#[derive(Debug, Clone, PartialEq)]
pub enum AgeGroup {
Young,
MiddleAge,
Mature,
Senior,
}
#[derive(Debug, Clone, PartialEq)]
pub enum Gender {
Male,
Female,
NonBinary,
PreferNotToSay,
}
#[derive(Debug, Clone, PartialEq)]
pub enum EducationLevel {
HighSchool,
SomeCollege,
Bachelor,
Graduate,
Professional,
}
#[derive(Debug, Clone, PartialEq)]
pub enum AudioExpertise {
Novice,
Intermediate,
Expert,
Researcher,
}
#[derive(Debug, Clone, PartialEq)]
pub enum LanguageProficiency {
Native,
Fluent,
Intermediate,
Basic,
Learning,
}
#[derive(Debug, Clone, PartialEq)]
pub enum HearingAbility {
Normal,
Mild,
Moderate,
Severe,
HearingAid,
}
#[derive(Debug, Clone, PartialEq)]
pub enum CulturalBackground {
Western,
EastAsian,
SouthAsian,
MiddleEastern,
African,
LatinAmerican,
Mixed,
}
#[derive(Debug, Clone, PartialEq)]
pub enum RegionalPreferences {
AmericanEnglish,
BritishEnglish,
AustralianEnglish,
CanadianEnglish,
Other(String),
}
#[derive(Debug, Clone, PartialEq)]
pub enum LanguageGroup {
Germanic,
Romance,
Slavic,
SinoTibetan,
Japonic,
Austronesian,
Other(String),
}
#[derive(Debug, Clone, PartialEq)]
pub enum SpeechStylePreferences {
Formal,
Casual,
Expressive,
Neutral,
RegionalAccent,
}
#[derive(Debug, Clone, PartialEq)]
pub struct PerceptualBias {
pub bias_type: BiasType,
pub strength: f32,
pub description: String,
}
#[derive(Debug, Clone, PartialEq)]
pub enum BiasType {
PitchPreference,
SpeedPreference,
FormalityPreference,
AccentFamiliarity,
GenderPreference,
AgePreference,
CulturalPatternPreference,
}