use crate::perceptual::cross_cultural::{
CrossCulturalAdaptation, CrossCulturalConfig, CrossCulturalPerceptualModel,
};
use crate::perceptual::{CulturalProfile, DemographicProfile};
use crate::quality::universal_phoneme_mapping::{
PhonemeConverageAnalysis, UniversalPhonemeMapper, UniversalPhonemeMappingConfig,
};
use crate::traits::{EvaluationResult, QualityScore};
use crate::EvaluationError;
use async_trait::async_trait;
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
use std::time::Duration;
use voirs_recognizer::traits::PhonemeAlignment;
use voirs_sdk::{AudioBuffer, LanguageCode, Phoneme};
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct CrossLanguageIntelligibilityConfig {
pub enable_phonetic_distance: bool,
pub enable_acoustic_similarity: bool,
pub enable_perceptual_adaptation: bool,
pub enable_listener_proficiency: bool,
pub enable_context_dependency: bool,
pub phonetic_distance_weight: f32,
pub acoustic_similarity_weight: f32,
pub perceptual_adaptation_weight: f32,
pub listener_proficiency_weight: f32,
pub context_dependency_weight: f32,
pub min_intelligibility_threshold: f32,
pub max_phonetic_distance: f32,
}
impl Default for CrossLanguageIntelligibilityConfig {
fn default() -> Self {
Self {
enable_phonetic_distance: true,
enable_acoustic_similarity: true,
enable_perceptual_adaptation: true,
enable_listener_proficiency: true,
enable_context_dependency: true,
phonetic_distance_weight: 0.3,
acoustic_similarity_weight: 0.25,
perceptual_adaptation_weight: 0.2,
listener_proficiency_weight: 0.15,
context_dependency_weight: 0.1,
min_intelligibility_threshold: 0.1,
max_phonetic_distance: 2.0,
}
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct CrossLanguageIntelligibilityResult {
pub source_language: LanguageCode,
pub target_language: LanguageCode,
pub overall_intelligibility: f32,
pub phonetic_distance_score: f32,
pub acoustic_similarity_score: f32,
pub perceptual_adaptation_score: f32,
pub listener_proficiency_score: f32,
pub context_dependency_score: f32,
pub word_intelligibility: Vec<WordIntelligibilityScore>,
pub phoneme_intelligibility: Vec<PhonemeIntelligibilityScore>,
pub problematic_regions: Vec<ProblematicRegion>,
pub prediction_confidence: f32,
pub processing_time: Duration,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct WordIntelligibilityScore {
pub word: String,
pub intelligibility_score: f32,
pub phonetic_complexity: f32,
pub acoustic_clarity: f32,
pub contextual_support: f32,
pub listener_familiarity: f32,
pub start_time: f32,
pub end_time: f32,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct PhonemeIntelligibilityScore {
pub source_phoneme: String,
pub perceived_phoneme: Option<String>,
pub intelligibility_score: f32,
pub phonetic_distance: f32,
pub acoustic_similarity: f32,
pub perceptual_confusability: f32,
pub position: usize,
pub start_time: f32,
pub end_time: f32,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ProblematicRegion {
pub start_time: f32,
pub end_time: f32,
pub severity: f32,
pub problem_type: IntelligibilityProblemType,
pub description: String,
pub suggestions: Vec<String>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub enum IntelligibilityProblemType {
PhoneticMismatch,
AcousticDistortion,
ProsodicInterference,
LexicalUnfamiliarity,
CulturalAdaptationFailure,
ContextDependencyIssue,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ListenerProficiencyProfile {
pub target_language_proficiency: ProficiencyLevel,
pub source_language_familiarity: ProficiencyLevel,
pub accent_exposure: Vec<AccentExposure>,
pub listening_experience: ListeningExperience,
pub adaptation_capability: f32,
pub attention_capacity: f32,
}
#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]
pub enum ProficiencyLevel {
Beginner,
Elementary,
Intermediate,
UpperIntermediate,
Advanced,
Native,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct AccentExposure {
pub source_accent: String,
pub exposure_duration: f32,
pub exposure_recency: f32,
pub exposure_quality: f32,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ListeningExperience {
pub total_listening_hours: f32,
pub cross_linguistic_hours: f32,
pub synthetic_speech_familiarity: f32,
pub accent_tolerance: f32,
pub cognitive_load_capacity: f32,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ContextDependencyFactors {
pub semantic_predictability: f32,
pub syntactic_complexity: f32,
pub lexical_frequency: f32,
pub discourse_coherence: f32,
pub visual_context_available: bool,
pub background_noise_level: f32,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct AcousticAnalysisResult {
pub formant_clarity: f32,
pub spectral_envelope_similarity: f32,
pub temporal_envelope_preservation: f32,
pub voice_quality_metrics: VoiceQualityMetrics,
pub prosodic_features: ProsodicFeatureAnalysis,
pub signal_to_noise_ratio: f32,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct VoiceQualityMetrics {
pub jitter: f32,
pub shimmer: f32,
pub harmonic_to_noise_ratio: f32,
pub spectral_tilt: f32,
pub formant_bandwidth: Vec<f32>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ProsodicFeatureAnalysis {
pub f0_contour_appropriateness: f32,
pub stress_pattern_accuracy: f32,
pub rhythm_regularity: f32,
pub intonation_naturalness: f32,
pub pause_pattern_appropriateness: f32,
}
pub struct CrossLanguageIntelligibilityEvaluator {
config: CrossLanguageIntelligibilityConfig,
phoneme_mapper: UniversalPhonemeMapper,
cultural_model: CrossCulturalPerceptualModel,
intelligibility_cache: HashMap<(LanguageCode, LanguageCode), f32>,
phonetic_distance_matrices: HashMap<(LanguageCode, LanguageCode), Vec<Vec<f32>>>,
}
impl CrossLanguageIntelligibilityEvaluator {
pub fn new(config: CrossLanguageIntelligibilityConfig) -> Self {
let phoneme_mapper = UniversalPhonemeMapper::new(UniversalPhonemeMappingConfig::default());
let cultural_model = CrossCulturalPerceptualModel::new(CrossCulturalConfig::default());
let mut evaluator = Self {
config,
phoneme_mapper,
cultural_model,
intelligibility_cache: HashMap::new(),
phonetic_distance_matrices: HashMap::new(),
};
evaluator.precompute_phonetic_distance_matrices();
evaluator
}
fn precompute_phonetic_distance_matrices(&mut self) {
let supported_languages = self.phoneme_mapper.get_supported_languages();
for &lang1 in &supported_languages {
for &lang2 in &supported_languages {
if lang1 != lang2 {
let distance_matrix = self.compute_phonetic_distance_matrix(lang1, lang2);
self.phonetic_distance_matrices
.insert((lang1, lang2), distance_matrix);
}
}
}
}
fn compute_phonetic_distance_matrix(
&self,
lang1: LanguageCode,
lang2: LanguageCode,
) -> Vec<Vec<f32>> {
let mut distance_matrix = Vec::new();
if let (Some(inventory1), Some(inventory2)) = (
self.phoneme_mapper.get_language_inventory(lang1),
self.phoneme_mapper.get_language_inventory(lang2),
) {
for phoneme1 in inventory1 {
let mut row = Vec::new();
for phoneme2 in inventory2 {
let similarity = self
.phoneme_mapper
.calculate_similarity_score(phoneme1, phoneme2, lang1, lang2);
let distance = 1.0 - similarity;
row.push(distance);
}
distance_matrix.push(row);
}
}
distance_matrix
}
pub async fn evaluate_intelligibility(
&self,
audio: &AudioBuffer,
source_language: LanguageCode,
target_language: LanguageCode,
phoneme_alignment: Option<&PhonemeAlignment>,
listener_profile: Option<&ListenerProficiencyProfile>,
context_factors: Option<&ContextDependencyFactors>,
) -> EvaluationResult<CrossLanguageIntelligibilityResult> {
let start_time = std::time::Instant::now();
let phonetic_distance_score = if self.config.enable_phonetic_distance {
self.calculate_phonetic_distance_score(
source_language,
target_language,
phoneme_alignment,
)?
} else {
0.5
};
let acoustic_similarity_score = if self.config.enable_acoustic_similarity {
self.calculate_acoustic_similarity_score(audio, source_language, target_language)
.await?
} else {
0.5
};
let perceptual_adaptation_score = if self.config.enable_perceptual_adaptation {
self.calculate_perceptual_adaptation_score(audio, source_language, target_language)
.await?
} else {
0.5
};
let listener_proficiency_score = if self.config.enable_listener_proficiency {
self.calculate_listener_proficiency_score(
listener_profile,
source_language,
target_language,
)?
} else {
0.5
};
let context_dependency_score = if self.config.enable_context_dependency {
self.calculate_context_dependency_score(context_factors, phoneme_alignment)?
} else {
0.5
};
let overall_intelligibility = self.calculate_overall_intelligibility(
phonetic_distance_score,
acoustic_similarity_score,
perceptual_adaptation_score,
listener_proficiency_score,
context_dependency_score,
);
let word_intelligibility = self.calculate_word_intelligibility(
phoneme_alignment,
source_language,
target_language,
overall_intelligibility,
)?;
let phoneme_intelligibility = self.calculate_phoneme_intelligibility(
phoneme_alignment,
source_language,
target_language,
)?;
let problematic_regions = self.identify_problematic_regions(
&phoneme_intelligibility,
&word_intelligibility,
phoneme_alignment,
);
let prediction_confidence = self.calculate_prediction_confidence(
phonetic_distance_score,
acoustic_similarity_score,
perceptual_adaptation_score,
listener_proficiency_score,
context_dependency_score,
);
let processing_time = start_time.elapsed();
Ok(CrossLanguageIntelligibilityResult {
source_language,
target_language,
overall_intelligibility,
phonetic_distance_score,
acoustic_similarity_score,
perceptual_adaptation_score,
listener_proficiency_score,
context_dependency_score,
word_intelligibility,
phoneme_intelligibility,
problematic_regions,
prediction_confidence,
processing_time,
})
}
fn calculate_phonetic_distance_score(
&self,
source_language: LanguageCode,
target_language: LanguageCode,
phoneme_alignment: Option<&PhonemeAlignment>,
) -> EvaluationResult<f32> {
if let Some(alignment) = phoneme_alignment {
let mut total_distance = 0.0;
let mut phoneme_count = 0;
for aligned_phoneme in &alignment.phonemes {
if let Some(mapping) = self.phoneme_mapper.map_phoneme(
&aligned_phoneme.phoneme.symbol,
source_language,
target_language,
) {
total_distance += 1.0 - mapping.similarity_score;
phoneme_count += 1;
}
}
if phoneme_count > 0 {
let average_distance = total_distance / phoneme_count as f32;
Ok(1.0
- average_distance.min(self.config.max_phonetic_distance)
/ self.config.max_phonetic_distance)
} else {
Ok(0.5)
}
} else {
if let Some(coverage) = self
.phoneme_mapper
.analyze_phoneme_coverage(source_language, target_language)
.ok()
{
Ok(coverage.average_mapping_quality)
} else {
Ok(0.5)
}
}
}
async fn calculate_acoustic_similarity_score(
&self,
audio: &AudioBuffer,
source_language: LanguageCode,
target_language: LanguageCode,
) -> EvaluationResult<f32> {
let acoustic_analysis = self.analyze_acoustic_features(audio).await?;
let formant_score =
self.evaluate_formant_clarity(&acoustic_analysis, source_language, target_language);
let spectral_score = self.evaluate_spectral_envelope_similarity(
&acoustic_analysis,
source_language,
target_language,
);
let temporal_score = self.evaluate_temporal_envelope_preservation(
&acoustic_analysis,
source_language,
target_language,
);
let voice_quality_score = self.evaluate_voice_quality_metrics(
&acoustic_analysis,
source_language,
target_language,
);
let prosodic_score =
self.evaluate_prosodic_features(&acoustic_analysis, source_language, target_language);
let overall_score = formant_score * 0.25
+ spectral_score * 0.25
+ temporal_score * 0.2
+ voice_quality_score * 0.15
+ prosodic_score * 0.15;
Ok(overall_score)
}
async fn analyze_acoustic_features(
&self,
audio: &AudioBuffer,
) -> EvaluationResult<AcousticAnalysisResult> {
let samples = audio.samples();
let voice_quality_metrics = self.calculate_voice_quality_metrics(samples);
let prosodic_features = self.calculate_prosodic_features(samples);
let signal_to_noise_ratio = self.calculate_snr(samples);
Ok(AcousticAnalysisResult {
formant_clarity: 0.8, spectral_envelope_similarity: 0.75, temporal_envelope_preservation: 0.85, voice_quality_metrics,
prosodic_features,
signal_to_noise_ratio,
})
}
fn calculate_voice_quality_metrics(&self, samples: &[f32]) -> VoiceQualityMetrics {
let rms = (samples.iter().map(|&x| x * x).sum::<f32>() / samples.len() as f32).sqrt();
let peak = samples.iter().map(|&x| x.abs()).fold(0.0f32, f32::max);
VoiceQualityMetrics {
jitter: 0.02, shimmer: 0.03, harmonic_to_noise_ratio: if rms > 0.0 {
20.0 * (peak / rms).log10()
} else {
0.0
},
spectral_tilt: -6.0, formant_bandwidth: vec![50.0, 70.0, 90.0], }
}
fn calculate_prosodic_features(&self, samples: &[f32]) -> ProsodicFeatureAnalysis {
let energy_variance = self.calculate_energy_variance(samples);
ProsodicFeatureAnalysis {
f0_contour_appropriateness: 0.8, stress_pattern_accuracy: 0.75, rhythm_regularity: 1.0 - energy_variance.min(1.0), intonation_naturalness: 0.7, pause_pattern_appropriateness: 0.8, }
}
fn calculate_energy_variance(&self, samples: &[f32]) -> f32 {
let chunk_size = 1600; let mut energy_values = Vec::new();
for chunk in samples.chunks(chunk_size) {
let energy = chunk.iter().map(|&x| x * x).sum::<f32>() / chunk.len() as f32;
energy_values.push(energy);
}
if energy_values.len() < 2 {
return 0.0;
}
let mean = energy_values.iter().sum::<f32>() / energy_values.len() as f32;
let variance = energy_values
.iter()
.map(|&x| (x - mean).powi(2))
.sum::<f32>()
/ energy_values.len() as f32;
variance.sqrt() / mean.max(0.001)
}
fn calculate_snr(&self, samples: &[f32]) -> f32 {
let signal_power = samples.iter().map(|&x| x * x).sum::<f32>() / samples.len() as f32;
let noise_floor = 0.001;
if signal_power > noise_floor {
10.0 * (signal_power / noise_floor).log10()
} else {
0.0
}
}
fn evaluate_formant_clarity(
&self,
acoustic_analysis: &AcousticAnalysisResult,
_source_language: LanguageCode,
_target_language: LanguageCode,
) -> f32 {
acoustic_analysis.formant_clarity
}
fn evaluate_spectral_envelope_similarity(
&self,
acoustic_analysis: &AcousticAnalysisResult,
_source_language: LanguageCode,
_target_language: LanguageCode,
) -> f32 {
acoustic_analysis.spectral_envelope_similarity
}
fn evaluate_temporal_envelope_preservation(
&self,
acoustic_analysis: &AcousticAnalysisResult,
_source_language: LanguageCode,
_target_language: LanguageCode,
) -> f32 {
acoustic_analysis.temporal_envelope_preservation
}
fn evaluate_voice_quality_metrics(
&self,
acoustic_analysis: &AcousticAnalysisResult,
_source_language: LanguageCode,
_target_language: LanguageCode,
) -> f32 {
let voice_quality = &acoustic_analysis.voice_quality_metrics;
let jitter_score = 1.0 - (voice_quality.jitter / 0.1).min(1.0);
let shimmer_score = 1.0 - (voice_quality.shimmer / 0.1).min(1.0);
let hnr_score = (voice_quality.harmonic_to_noise_ratio / 20.0)
.min(1.0)
.max(0.0);
(jitter_score + shimmer_score + hnr_score) / 3.0
}
fn evaluate_prosodic_features(
&self,
acoustic_analysis: &AcousticAnalysisResult,
_source_language: LanguageCode,
_target_language: LanguageCode,
) -> f32 {
let prosodic = &acoustic_analysis.prosodic_features;
(prosodic.f0_contour_appropriateness
+ prosodic.stress_pattern_accuracy
+ prosodic.rhythm_regularity
+ prosodic.intonation_naturalness
+ prosodic.pause_pattern_appropriateness)
/ 5.0
}
async fn calculate_perceptual_adaptation_score(
&self,
audio: &AudioBuffer,
source_language: LanguageCode,
target_language: LanguageCode,
) -> EvaluationResult<f32> {
let get_language_code = |lang: LanguageCode| -> String {
match lang {
LanguageCode::EnUs | LanguageCode::EnGb => "en".to_string(),
LanguageCode::EsEs | LanguageCode::EsMx | LanguageCode::Es => "es".to_string(),
LanguageCode::FrFr | LanguageCode::Fr => "fr".to_string(),
LanguageCode::DeDe | LanguageCode::De => "de".to_string(),
LanguageCode::JaJp | LanguageCode::Ja => "ja".to_string(),
LanguageCode::ZhCn => "zh".to_string(),
LanguageCode::PtBr | LanguageCode::Pt => "pt".to_string(),
LanguageCode::RuRu | LanguageCode::Ru => "ru".to_string(),
LanguageCode::ItIt | LanguageCode::It => "it".to_string(),
LanguageCode::KoKr | LanguageCode::Ko => "ko".to_string(),
LanguageCode::Ar => "ar".to_string(),
LanguageCode::Hi => "hi".to_string(),
_ => "en".to_string(), }
};
let target_lang_code = get_language_code(target_language);
let source_lang_code = get_language_code(source_language);
let cultural_profile = CulturalProfile {
region: crate::perceptual::CulturalRegion::NorthAmerica,
language_familiarity: vec![target_lang_code.clone()],
musical_training: false,
accent_tolerance: 0.7,
};
let demographic_profile = DemographicProfile {
age_group: crate::perceptual::AgeGroup::MiddleAged,
gender: crate::perceptual::Gender::Other,
education_level: crate::perceptual::EducationLevel::Bachelor,
native_language: target_lang_code,
audio_experience: crate::perceptual::ExperienceLevel::Intermediate,
};
let adaptation_factors = self.cultural_model.calculate_adaptation_factors(
&cultural_profile,
&demographic_profile,
audio,
&source_lang_code,
)?;
let overall_adaptation = adaptation_factors.phonetic_distance_factor * 0.3
+ adaptation_factors.prosodic_mismatch_factor * 0.3
+ adaptation_factors.accent_familiarity_factor * 0.2
+ adaptation_factors.communication_style_factor * 0.1
+ adaptation_factors.linguistic_distance_factor * 0.1;
Ok(overall_adaptation)
}
fn calculate_listener_proficiency_score(
&self,
listener_profile: Option<&ListenerProficiencyProfile>,
_source_language: LanguageCode,
_target_language: LanguageCode,
) -> EvaluationResult<f32> {
if let Some(profile) = listener_profile {
let proficiency_score =
self.proficiency_level_to_score(&profile.target_language_proficiency);
let familiarity_score =
self.proficiency_level_to_score(&profile.source_language_familiarity);
let experience_score =
self.listening_experience_to_score(&profile.listening_experience);
let overall_score = proficiency_score * 0.4
+ familiarity_score * 0.3
+ experience_score * 0.2
+ profile.adaptation_capability * 0.1;
Ok(overall_score)
} else {
Ok(0.6)
}
}
fn proficiency_level_to_score(&self, level: &ProficiencyLevel) -> f32 {
match level {
ProficiencyLevel::Beginner => 0.2,
ProficiencyLevel::Elementary => 0.35,
ProficiencyLevel::Intermediate => 0.5,
ProficiencyLevel::UpperIntermediate => 0.65,
ProficiencyLevel::Advanced => 0.8,
ProficiencyLevel::Native => 1.0,
}
}
fn listening_experience_to_score(&self, experience: &ListeningExperience) -> f32 {
let total_hours_score = (experience.total_listening_hours / 1000.0).min(1.0);
let cross_linguistic_score = (experience.cross_linguistic_hours / 500.0).min(1.0);
let synthetic_familiarity_score = experience.synthetic_speech_familiarity;
let accent_tolerance_score = experience.accent_tolerance;
(total_hours_score
+ cross_linguistic_score
+ synthetic_familiarity_score
+ accent_tolerance_score)
/ 4.0
}
fn calculate_context_dependency_score(
&self,
context_factors: Option<&ContextDependencyFactors>,
_phoneme_alignment: Option<&PhonemeAlignment>,
) -> EvaluationResult<f32> {
if let Some(factors) = context_factors {
let semantic_score = factors.semantic_predictability;
let syntactic_score = 1.0 - (factors.syntactic_complexity / 2.0).min(1.0);
let lexical_score = factors.lexical_frequency;
let discourse_score = factors.discourse_coherence;
let visual_score = if factors.visual_context_available {
1.0
} else {
0.7
};
let noise_score = 1.0 - (factors.background_noise_level / 2.0).min(1.0);
let overall_score = semantic_score * 0.25
+ syntactic_score * 0.2
+ lexical_score * 0.2
+ discourse_score * 0.15
+ visual_score * 0.1
+ noise_score * 0.1;
Ok(overall_score)
} else {
Ok(0.5)
}
}
fn calculate_overall_intelligibility(
&self,
phonetic_distance_score: f32,
acoustic_similarity_score: f32,
perceptual_adaptation_score: f32,
listener_proficiency_score: f32,
context_dependency_score: f32,
) -> f32 {
let weighted_score = phonetic_distance_score * self.config.phonetic_distance_weight
+ acoustic_similarity_score * self.config.acoustic_similarity_weight
+ perceptual_adaptation_score * self.config.perceptual_adaptation_weight
+ listener_proficiency_score * self.config.listener_proficiency_weight
+ context_dependency_score * self.config.context_dependency_weight;
weighted_score
.max(self.config.min_intelligibility_threshold)
.min(1.0)
}
fn calculate_word_intelligibility(
&self,
phoneme_alignment: Option<&PhonemeAlignment>,
source_language: LanguageCode,
target_language: LanguageCode,
base_intelligibility: f32,
) -> EvaluationResult<Vec<WordIntelligibilityScore>> {
let mut word_scores = Vec::new();
if let Some(alignment) = phoneme_alignment {
for word_alignment in &alignment.word_alignments {
let phonetic_complexity = self.calculate_word_phonetic_complexity(
&word_alignment.phonemes,
source_language,
target_language,
);
let acoustic_clarity = word_alignment.confidence;
let contextual_support = 0.7; let listener_familiarity = 0.6;
let word_intelligibility = base_intelligibility
* phonetic_complexity
* acoustic_clarity
* contextual_support
* listener_familiarity;
word_scores.push(WordIntelligibilityScore {
word: word_alignment.word.clone(),
intelligibility_score: word_intelligibility,
phonetic_complexity,
acoustic_clarity,
contextual_support,
listener_familiarity,
start_time: word_alignment.start_time,
end_time: word_alignment.end_time,
});
}
}
Ok(word_scores)
}
fn calculate_word_phonetic_complexity(
&self,
phonemes: &[voirs_recognizer::traits::AlignedPhoneme],
source_language: LanguageCode,
target_language: LanguageCode,
) -> f32 {
if phonemes.is_empty() {
return 0.5;
}
let mut total_complexity = 0.0;
for phoneme in phonemes {
if let Some(mapping) = self.phoneme_mapper.map_phoneme(
&phoneme.phoneme.symbol,
source_language,
target_language,
) {
total_complexity += 1.0 - mapping.similarity_score;
}
}
1.0 - (total_complexity / phonemes.len() as f32).min(1.0)
}
fn calculate_phoneme_intelligibility(
&self,
phoneme_alignment: Option<&PhonemeAlignment>,
source_language: LanguageCode,
target_language: LanguageCode,
) -> EvaluationResult<Vec<PhonemeIntelligibilityScore>> {
let mut phoneme_scores = Vec::new();
if let Some(alignment) = phoneme_alignment {
for (position, aligned_phoneme) in alignment.phonemes.iter().enumerate() {
if let Some(mapping) = self.phoneme_mapper.map_phoneme(
&aligned_phoneme.phoneme.symbol,
source_language,
target_language,
) {
let phonetic_distance = 1.0 - mapping.similarity_score;
let acoustic_similarity = mapping.similarity_score;
let perceptual_confusability = 0.3;
let intelligibility_score = mapping.similarity_score
* aligned_phoneme.confidence
* (1.0 - perceptual_confusability);
phoneme_scores.push(PhonemeIntelligibilityScore {
source_phoneme: aligned_phoneme.phoneme.symbol.clone(),
perceived_phoneme: Some(mapping.target_phoneme),
intelligibility_score,
phonetic_distance,
acoustic_similarity,
perceptual_confusability,
position,
start_time: aligned_phoneme.start_time,
end_time: aligned_phoneme.end_time,
});
} else {
phoneme_scores.push(PhonemeIntelligibilityScore {
source_phoneme: aligned_phoneme.phoneme.symbol.clone(),
perceived_phoneme: None,
intelligibility_score: 0.0,
phonetic_distance: 2.0,
acoustic_similarity: 0.0,
perceptual_confusability: 1.0,
position,
start_time: aligned_phoneme.start_time,
end_time: aligned_phoneme.end_time,
});
}
}
}
Ok(phoneme_scores)
}
fn identify_problematic_regions(
&self,
phoneme_intelligibility: &[PhonemeIntelligibilityScore],
word_intelligibility: &[WordIntelligibilityScore],
_phoneme_alignment: Option<&PhonemeAlignment>,
) -> Vec<ProblematicRegion> {
let mut problematic_regions = Vec::new();
for phoneme_score in phoneme_intelligibility {
if phoneme_score.intelligibility_score < 0.3 {
let problem_type = if phoneme_score.phonetic_distance > 1.5 {
IntelligibilityProblemType::PhoneticMismatch
} else if phoneme_score.acoustic_similarity < 0.3 {
IntelligibilityProblemType::AcousticDistortion
} else {
IntelligibilityProblemType::ProsodicInterference
};
problematic_regions.push(ProblematicRegion {
start_time: phoneme_score.start_time,
end_time: phoneme_score.end_time,
severity: 1.0 - phoneme_score.intelligibility_score,
problem_type,
description: format!(
"Phoneme '{}' has low intelligibility",
phoneme_score.source_phoneme
),
suggestions: vec![
"Consider acoustic model adaptation".to_string(),
"Improve phonetic clarity".to_string(),
],
});
}
}
for word_score in word_intelligibility {
if word_score.intelligibility_score < 0.4 {
let problem_type = if word_score.phonetic_complexity < 0.5 {
IntelligibilityProblemType::PhoneticMismatch
} else if word_score.acoustic_clarity < 0.5 {
IntelligibilityProblemType::AcousticDistortion
} else {
IntelligibilityProblemType::LexicalUnfamiliarity
};
problematic_regions.push(ProblematicRegion {
start_time: word_score.start_time,
end_time: word_score.end_time,
severity: 1.0 - word_score.intelligibility_score,
problem_type,
description: format!("Word '{}' has low intelligibility", word_score.word),
suggestions: vec![
"Consider lexical substitution".to_string(),
"Improve pronunciation clarity".to_string(),
],
});
}
}
problematic_regions
}
fn calculate_prediction_confidence(
&self,
phonetic_distance_score: f32,
acoustic_similarity_score: f32,
perceptual_adaptation_score: f32,
listener_proficiency_score: f32,
context_dependency_score: f32,
) -> f32 {
let scores = vec![
phonetic_distance_score,
acoustic_similarity_score,
perceptual_adaptation_score,
listener_proficiency_score,
context_dependency_score,
];
let mean_score = scores.iter().sum::<f32>() / scores.len() as f32;
let variance = scores
.iter()
.map(|&x| (x - mean_score).powi(2))
.sum::<f32>()
/ scores.len() as f32;
let consistency = 1.0 - variance.sqrt();
let absolute_confidence = mean_score;
(consistency * 0.6 + absolute_confidence * 0.4)
.max(0.1)
.min(1.0)
}
pub fn get_supported_language_pairs(&self) -> Vec<(LanguageCode, LanguageCode)> {
let languages = self.phoneme_mapper.get_supported_languages();
let mut pairs = Vec::new();
for &lang1 in &languages {
for &lang2 in &languages {
if lang1 != lang2 {
pairs.push((lang1, lang2));
}
}
}
pairs
}
pub fn predict_intelligibility(
&self,
source_language: LanguageCode,
target_language: LanguageCode,
listener_proficiency: Option<ProficiencyLevel>,
) -> f32 {
if let Some(&cached_score) = self
.intelligibility_cache
.get(&(source_language, target_language))
{
return cached_score;
}
let phoneme_coverage = self
.phoneme_mapper
.analyze_phoneme_coverage(source_language, target_language)
.map(|coverage| coverage.average_mapping_quality)
.unwrap_or(0.5);
let proficiency_adjustment = if let Some(proficiency) = listener_proficiency {
self.proficiency_level_to_score(&proficiency)
} else {
0.6
};
let linguistic_distance = self.cultural_model.calculate_linguistic_distance_factor(
&format!("{:?}", source_language).to_lowercase(),
&format!("{:?}", target_language).to_lowercase(),
);
let predicted_intelligibility =
phoneme_coverage * 0.5 + proficiency_adjustment * 0.3 + linguistic_distance * 0.2;
predicted_intelligibility.max(0.1).min(1.0)
}
}
#[async_trait]
pub trait CrossLanguageIntelligibilityEvaluationTrait {
async fn evaluate_cross_language_intelligibility(
&self,
audio: &AudioBuffer,
source_language: LanguageCode,
target_language: LanguageCode,
phoneme_alignment: Option<&PhonemeAlignment>,
listener_profile: Option<&ListenerProficiencyProfile>,
context_factors: Option<&ContextDependencyFactors>,
) -> EvaluationResult<CrossLanguageIntelligibilityResult>;
fn predict_intelligibility(
&self,
source_language: LanguageCode,
target_language: LanguageCode,
listener_proficiency: Option<ProficiencyLevel>,
) -> f32;
fn get_supported_language_pairs(&self) -> Vec<(LanguageCode, LanguageCode)>;
}
#[cfg(test)]
mod tests {
use super::*;
#[tokio::test]
async fn test_cross_language_intelligibility_evaluator_creation() {
let config = CrossLanguageIntelligibilityConfig::default();
let evaluator = CrossLanguageIntelligibilityEvaluator::new(config);
assert!(!evaluator.get_supported_language_pairs().is_empty());
}
#[tokio::test]
async fn test_intelligibility_prediction() {
let config = CrossLanguageIntelligibilityConfig::default();
let evaluator = CrossLanguageIntelligibilityEvaluator::new(config);
let score = evaluator.predict_intelligibility(
LanguageCode::EnUs,
LanguageCode::EsEs,
Some(ProficiencyLevel::Intermediate),
);
assert!(score >= 0.1 && score <= 1.0);
}
#[tokio::test]
async fn test_intelligibility_evaluation() {
let config = CrossLanguageIntelligibilityConfig::default();
let evaluator = CrossLanguageIntelligibilityEvaluator::new(config);
let audio = AudioBuffer::new(vec![0.1; 16000], 16000, 1);
let result = evaluator
.evaluate_intelligibility(
&audio,
LanguageCode::EnUs,
LanguageCode::EsEs,
None,
None,
None,
)
.await
.unwrap();
assert_eq!(result.source_language, LanguageCode::EnUs);
assert_eq!(result.target_language, LanguageCode::EsEs);
assert!(result.overall_intelligibility >= 0.0 && result.overall_intelligibility <= 1.0);
assert!(result.prediction_confidence >= 0.0 && result.prediction_confidence <= 1.0);
}
#[test]
fn test_proficiency_level_conversion() {
let config = CrossLanguageIntelligibilityConfig::default();
let evaluator = CrossLanguageIntelligibilityEvaluator::new(config);
assert_eq!(
evaluator.proficiency_level_to_score(&ProficiencyLevel::Beginner),
0.2
);
assert_eq!(
evaluator.proficiency_level_to_score(&ProficiencyLevel::Native),
1.0
);
assert_eq!(
evaluator.proficiency_level_to_score(&ProficiencyLevel::Intermediate),
0.5
);
}
#[test]
fn test_supported_language_pairs() {
let config = CrossLanguageIntelligibilityConfig::default();
let evaluator = CrossLanguageIntelligibilityEvaluator::new(config);
let pairs = evaluator.get_supported_language_pairs();
assert!(!pairs.is_empty());
assert!(pairs.contains(&(LanguageCode::EnUs, LanguageCode::EsEs)));
assert!(pairs.contains(&(LanguageCode::EsEs, LanguageCode::EnUs)));
}
}