use crate::oxyde_game::emotion::EmotionalState;
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
use std::sync::Arc;
use tokio::sync::RwLock;
pub mod audio_cache;
pub mod emotion;
pub mod providers;
pub mod voice_profiles;
pub use audio_cache::*;
pub use providers::*;
pub use voice_profiles::*;
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct AudioData {
pub format: AudioFormat,
pub data: Vec<u8>,
pub sample_rate: u32,
pub channels: u8,
pub duration_ms: u32,
}
impl AudioData {
pub fn size_bytes(&self) -> usize {
self.data.len()
}
}
#[derive(Debug, Clone)]
pub struct TTSService {
provider: TTSProvider,
pub cache: Arc<RwLock<AudioCache>>,
voice_profiles: Arc<RwLock<HashMap<String, VoiceProfile>>>,
config: TTSConfig,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub enum TTSProvider {
ElevenLabs,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct TTSConfig {
pub default_provider: TTSProvider,
pub cache_enabled: bool,
pub cache_max_size_mb: usize,
pub voice_speed: f32,
pub voice_pitch: f32,
pub enable_ssml: bool,
pub output_format: AudioFormat,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub enum AudioFormat {
MP3,
}
impl TTSService {
pub fn new(provider: TTSProvider, config: TTSConfig) -> Self {
Self {
provider,
cache: Arc::new(RwLock::new(AudioCache::new(config.cache_max_size_mb))),
voice_profiles: Arc::new(RwLock::new(HashMap::new())),
config,
}
}
pub async fn synthesize_npc_speech(
&self,
npc_name: &str,
text: &str,
emotional_state: &EmotionalState, urgency: f32,
) -> Result<AudioData, TTSError> {
let cache_key = self.generate_cache_key(npc_name, text, emotional_state);
if self.config.cache_enabled {
let mut cache = self.cache.write().await;
if let Some(cached_audio) = cache.get(&cache_key) {
return Ok(cached_audio);
}
}
let voice_profile = self.get_voice_profile(npc_name).await;
let voice_settings =
self.modulate_voice_for_emotion(&voice_profile, emotional_state, urgency);
let enhanced_text = if self.config.enable_ssml {
self.add_emotional_ssml(text, emotional_state, urgency)
} else {
text.to_string()
};
let audio_data = match self.provider {
TTSProvider::ElevenLabs => {
self.elevenlabs_synthesize(&enhanced_text, &voice_settings)
.await?
}
};
if self.config.cache_enabled {
let mut cache = self.cache.write().await;
cache.insert(cache_key, audio_data.clone());
}
Ok(audio_data)
}
pub async fn create_voice_profile_for_npc(
&self,
npc_name: &str,
personality: &str,
) -> VoiceProfile {
let voice_profile = VoiceProfile {
npc_name: npc_name.to_string(),
base_voice: BaseVoice {
voice_id: "default".to_string(),
base_pitch: 0.5,
base_rate: 0.5,
base_volume: 0.7,
},
emotional_range: EmotionalVoiceRange::from_personality(personality),
};
let mut profiles = self.voice_profiles.write().await;
profiles.insert(npc_name.to_string(), voice_profile.clone());
voice_profile
}
fn modulate_voice_for_emotion(
&self,
base_profile: &VoiceProfile,
e: &EmotionalState,
_urgency: f32, ) -> VoiceSettings {
let mut settings = VoiceSettings::from_profile(base_profile);
let joy = (e.joy + 1.0) * 0.5;
let anger = (e.anger + 1.0) * 0.5;
let fear = (e.fear + 1.0) * 0.5;
let trust = (e.trust + 1.0) * 0.5;
let surprise = (e.surprise + 1.0) * 0.5;
let sadness = (e.sadness + 1.0) * 0.5;
let disgust = (e.disgust + 1.0) * 0.5;
let anticipation = (e.anticipation + 1.0) * 0.5;
settings.style_exaggeration += 0.25 * joy;
settings.stability += 0.05 * joy;
settings.stability -= 0.3 * anger;
settings.style_exaggeration += 0.1 * anger;
settings.stability -= 0.2 * fear;
settings.similarity_boost -= 0.1 * fear;
settings.stability += 0.2 * sadness;
settings.style_exaggeration -= 0.1 * sadness;
settings.style_exaggeration += 0.2 * surprise;
settings.stability -= 0.15 * disgust;
settings.style_exaggeration += 0.1 * anticipation;
settings
}
fn add_emotional_ssml(
&self,
text: &str,
emotions: &EmotionalState, urgency: f32,
) -> String {
let mut ssml = String::from("<speak>");
let mut prosody_attrs = Vec::new();
if emotions.joy > 0.6 {
prosody_attrs.push(format!("rate=\"{:.0}%\"", 100.0 + (emotions.joy * 20.0)));
prosody_attrs.push(format!("pitch=\"+{:.0}Hz\"", emotions.joy * 30.0));
}
if emotions.anger > 0.5 {
prosody_attrs.push(format!("rate=\"{:.0}%\"", 100.0 + (emotions.anger * 25.0)));
prosody_attrs.push(format!(
"volume=\"{:.0}%\"",
100.0 + (emotions.anger * 15.0)
));
}
if emotions.fear > 0.5 {
prosody_attrs.push(format!("pitch=\"+{:.0}Hz\"", emotions.fear * 40.0));
prosody_attrs.push(format!("rate=\"{:.0}%\"", 100.0 - (emotions.fear * 10.0)));
}
if urgency > 0.5 {
prosody_attrs.push(format!("rate=\"{:.0}%\"", 100.0 + (urgency * 30.0)));
}
if !prosody_attrs.is_empty() {
ssml.push_str(&format!("<prosody {}>", prosody_attrs.join(" ")));
ssml.push_str(text);
ssml.push_str("</prosody>");
} else {
ssml.push_str(text);
}
ssml.push_str("</speak>");
ssml
}
async fn elevenlabs_synthesize(
&self,
text: &str,
settings: &VoiceSettings,
) -> Result<AudioData, TTSError> {
let client = reqwest::Client::new();
let api_key = std::env::var("ELEVENLABS_API_KEY")
.map_err(|_| TTSError::MissingApiKey("ElevenLabs"))?;
let voice_id = if settings.voice_id == "default" {
"21m00Tcm4TlvDq8ikWAM" } else {
&settings.voice_id
};
let request_body = serde_json::json!({
"text": text,
"model_id": "eleven_monolingual_v1",
"voice_settings": {
"stability": settings.stability,
"similarity_boost": settings.similarity_boost,
"style": settings.style_exaggeration,
"use_speaker_boost": true
}
});
let url = format!("https://api.elevenlabs.io/v1/text-to-speech/{}", voice_id);
let response = client
.post(&url)
.header("Accept", "audio/mpeg")
.header("xi-api-key", api_key)
.header("Content-Type", "application/json")
.json(&request_body)
.send()
.await
.map_err(|e| TTSError::Network(e))?;
let status = response.status();
let headers = response.headers().clone();
let audio_bytes = response.bytes().await.map_err(|e| TTSError::Network(e))?;
if !status.is_success() {
let error_text = String::from_utf8_lossy(&audio_bytes);
return Err(TTSError::ApiError(format!(
"ElevenLabs API error ({}): {}",
status, error_text
)));
}
if let Some(content_type) = headers.get("content-type") {
let content_type_str = content_type.to_str().unwrap_or("");
if !content_type_str.starts_with("audio/") {
let error_text = String::from_utf8_lossy(&audio_bytes);
return Err(TTSError::ApiError(format!(
"Expected audio content but received '{}': {}",
content_type_str, error_text
)));
}
}
if audio_bytes.len() < 100 {
let text_content = String::from_utf8_lossy(&audio_bytes);
return Err(TTSError::ApiError(format!(
"Response too small ({} bytes), likely an error: {}",
audio_bytes.len(),
text_content
)));
}
if audio_bytes.len() >= 3 {
let has_id3 = &audio_bytes[0..3] == b"ID3";
let has_mp3_sync = audio_bytes.len() >= 2
&& (audio_bytes[0] == 0xFF && (audio_bytes[1] & 0xE0) == 0xE0);
if !has_id3 && !has_mp3_sync {
let text_content =
String::from_utf8_lossy(&audio_bytes[0..100.min(audio_bytes.len())]);
if text_content.contains("error") || text_content.contains("detail") {
return Err(TTSError::ApiError(format!(
"Received error response instead of audio: {}",
text_content
)));
}
log::warn!("Audio data doesn't have standard MP3 headers but proceeding");
}
}
Ok(AudioData {
format: AudioFormat::MP3,
data: audio_bytes.to_vec(),
sample_rate: 22050,
channels: 1,
duration_ms: self.estimate_duration(text),
})
}
fn estimate_duration(&self, text: &str) -> u32 {
let word_count = text.split_whitespace().count();
let minutes = word_count as f32 / 150.0;
(minutes * 60.0 * 1000.0) as u32 }
async fn get_voice_profile(&self, npc_name: &str) -> VoiceProfile {
let profiles = self.voice_profiles.read().await;
profiles
.get(npc_name)
.cloned()
.unwrap_or_else(|| VoiceProfile::default_for_npc(npc_name))
}
fn generate_cache_key(
&self,
npc_name: &str,
text: &str,
emotions: &EmotionalState, ) -> String {
use std::collections::hash_map::DefaultHasher;
use std::hash::{Hash, Hasher};
let mut hasher = DefaultHasher::new();
npc_name.hash(&mut hasher);
text.hash(&mut hasher);
let rounded_emotions = (
(emotions.joy * 10.0).round() as i32,
(emotions.anger * 10.0).round() as i32,
(emotions.fear * 10.0).round() as i32,
);
rounded_emotions.hash(&mut hasher);
format!("tts_{:x}", hasher.finish())
}
}