microclaw_engine/tools/
text_to_speech.rs

1use async_trait::async_trait;
2use serde_json::{json, Value};
3use std::path::PathBuf;
4use std::sync::Arc;
5
6use crate::internal::channels::channel::deliver_and_store_bot_message;
7use crate::internal::channels::channel_adapter::ChannelRegistry;
8use crate::internal::storage::db::Database;
9use crate::internal::tool_runtime::media_client::{persist_output, MediaClient};
10use crate::internal::tool_runtime::runtime::auth_context_from_input;
11use microclaw_core::llm_types::ToolDefinition;
12
13use super::{schema_object, Tool, ToolResult};
14use crate::config::{Config, MediaConfig, TtsConfig};
15
16/// Trim and reject blank optional strings (e.g. an explicitly-empty config key).
17fn nonempty(s: Option<&str>) -> Option<String> {
18    s.map(str::trim)
19        .filter(|s| !s.is_empty())
20        .map(str::to_string)
21}
22
23const ALLOWED_FORMATS: &[&str] = &["mp3", "opus", "aac", "flac", "wav", "pcm"];
24const ALLOWED_VOICES: &[&str] = &[
25    "alloy", "ash", "ballad", "coral", "echo", "fable", "onyx", "nova", "sage", "shimmer", "verse",
26];
27
28/// OpenAI-compatible text-to-speech tool.
29///
30/// Calls `/v1/audio/speech`. Saves the generated audio under
31/// `<data_dir>/media/audio/<uuid>.<fmt>` and, when the channel supports
32/// attachments, also delivers it inline.
33pub struct TextToSpeechTool {
34    data_dir: PathBuf,
35    channels: Arc<ChannelRegistry>,
36    db: Arc<Database>,
37    cfg: TtsConfig,
38    media: MediaConfig,
39    openai_api_key: Option<String>,
40    openai_base_url: Option<String>,
41    timeout_secs: u64,
42}
43
44impl TextToSpeechTool {
45    pub fn new(config: &Config, channels: Arc<ChannelRegistry>, db: Arc<Database>) -> Self {
46        Self {
47            data_dir: PathBuf::from(&config.data_dir),
48            channels,
49            db,
50            cfg: config.media.tts.clone(),
51            media: config.media.clone(),
52            openai_api_key: config.openai_api_key.clone(),
53            openai_base_url: config.openai_base_url.clone(),
54            timeout_secs: config.tool_timeout_secs("text_to_speech", 60),
55        }
56    }
57
58    fn client(&self) -> Result<MediaClient, String> {
59        // TTS-specific override first, then the shared media credentials.
60        let key = nonempty(self.cfg.api_key.as_deref())
61            .or_else(|| self.media.resolve_api_key(self.openai_api_key.as_deref()))
62            .ok_or_else(|| {
63                "text_to_speech requires an API key (media.tts.api_key, \
64                 media.api_key, MICROCLAW_OPENAI_API_KEY, OPENAI_API_KEY, or \
65                 top-level openai_api_key)."
66                    .to_string()
67            })?;
68        let base = nonempty(self.cfg.base_url.as_deref())
69            .map(|s| s.trim_end_matches('/').to_string())
70            .unwrap_or_else(|| self.media.resolve_base_url(self.openai_base_url.as_deref()));
71        MediaClient::new(base, key, self.timeout_secs)
72    }
73}
74
75#[async_trait]
76impl Tool for TextToSpeechTool {
77    fn name(&self) -> &str {
78        "text_to_speech"
79    }
80
81    fn definition(&self) -> ToolDefinition {
82        ToolDefinition {
83            name: self.name().into(),
84            description: "Synthesize speech from text via an OpenAI-compatible \
85                /audio/speech endpoint. Saves the audio under the bot's data \
86                directory and — when the active channel supports attachments — \
87                also sends it back to the user inline. Disabled by default; \
88                requires operator opt-in via `media.tts.enabled`."
89                .into(),
90            input_schema: schema_object(
91                json!({
92                    "text": {
93                        "type": "string",
94                        "description": "What to say (max ~4096 chars for most providers)."
95                    },
96                    "voice": {
97                        "type": "string",
98                        "enum": ALLOWED_VOICES,
99                        "description": "Voice preset. Defaults to media.tts.default_voice."
100                    },
101                    "format": {
102                        "type": "string",
103                        "enum": ALLOWED_FORMATS,
104                        "description": "Audio format. Defaults to media.tts.default_format."
105                    },
106                    "model": {
107                        "type": "string",
108                        "description": "Optional model override (e.g. tts-1, tts-1-hd, gpt-4o-mini-tts)."
109                    },
110                    "deliver": {
111                        "type": "boolean",
112                        "description": "Whether to attempt channel delivery (default true)."
113                    }
114                }),
115                &["text"],
116            ),
117        }
118    }
119
120    async fn execute(&self, input: Value) -> ToolResult {
121        if !self.cfg.enabled {
122            return ToolResult::error(
123                "text_to_speech is disabled. Set media.tts.enabled=true to enable.".into(),
124            );
125        }
126        let text = match input.get("text").and_then(|v| v.as_str()) {
127            Some(t) if !t.trim().is_empty() => t.trim().to_string(),
128            _ => return ToolResult::error("Missing parameter: text".into()),
129        };
130        if text.chars().count() > 4096 {
131            return ToolResult::error(
132                "text exceeds 4096 characters — split into smaller chunks".into(),
133            );
134        }
135        let voice = input
136            .get("voice")
137            .and_then(|v| v.as_str())
138            .map(|s| s.to_string())
139            .unwrap_or_else(|| self.cfg.default_voice.clone());
140        if !ALLOWED_VOICES.iter().any(|v| v == &voice.as_str()) {
141            return ToolResult::error(format!(
142                "invalid voice '{voice}'. Allowed: {}",
143                ALLOWED_VOICES.join(", ")
144            ));
145        }
146        let format = input
147            .get("format")
148            .and_then(|v| v.as_str())
149            .map(|s| s.to_string())
150            .unwrap_or_else(|| self.cfg.default_format.clone());
151        if !ALLOWED_FORMATS.iter().any(|v| v == &format.as_str()) {
152            return ToolResult::error(format!(
153                "invalid format '{format}'. Allowed: {}",
154                ALLOWED_FORMATS.join(", ")
155            ));
156        }
157        let model = input
158            .get("model")
159            .and_then(|v| v.as_str())
160            .map(|s| s.to_string())
161            .unwrap_or_else(|| self.cfg.model.clone());
162        let deliver = input
163            .get("deliver")
164            .and_then(|v| v.as_bool())
165            .unwrap_or(true);
166
167        let client = match self.client() {
168            Ok(c) => c,
169            Err(e) => return ToolResult::error(e),
170        };
171
172        let body = json!({
173            "model": model,
174            "voice": voice,
175            "input": text,
176            "response_format": format,
177        });
178
179        let resp = match client.post_json("audio/speech").json(&body).send().await {
180            Ok(r) => r,
181            Err(e) => return ToolResult::error(format!("audio/speech request failed: {e}")),
182        };
183        if !resp.status().is_success() {
184            let status = resp.status();
185            let txt = resp.text().await.unwrap_or_default();
186            return ToolResult::error(format!("audio/speech HTTP {status}: {txt}"));
187        }
188        let bytes = match resp.bytes().await {
189            Ok(b) => b.to_vec(),
190            Err(e) => return ToolResult::error(format!("audio body read failed: {e}")),
191        };
192
193        let saved = match persist_output(&self.data_dir, "audio", &format, &bytes) {
194            Ok(p) => p,
195            Err(e) => return ToolResult::error(format!("failed to save audio: {e}")),
196        };
197
198        let mut summary = format!(
199            "TTS {} voice={} format={} ({} bytes) -> {}",
200            model,
201            voice,
202            format,
203            bytes.len(),
204            saved.display()
205        );
206        if deliver {
207            if let Some(auth) = auth_context_from_input(&input) {
208                match deliver_attachment(
209                    &self.channels,
210                    self.db.clone(),
211                    auth.caller_chat_id,
212                    &saved,
213                    &text,
214                )
215                .await
216                {
217                    Ok(msg) => summary.push_str(&format!("; {msg}")),
218                    Err(e) => summary.push_str(&format!("; delivery skipped: {e}")),
219                }
220            }
221        }
222        ToolResult::success(summary).with_metadata(json!({
223            "path": saved.to_string_lossy(),
224            "voice": voice,
225            "format": format,
226            "model": model,
227        }))
228    }
229}
230
231async fn deliver_attachment(
232    channels: &ChannelRegistry,
233    db: Arc<Database>,
234    chat_id: i64,
235    file: &std::path::Path,
236    caption: &str,
237) -> Result<String, String> {
238    let routing = match crate::internal::channels::channel::get_required_chat_routing(
239        channels,
240        db.clone(),
241        chat_id,
242    )
243    .await
244    {
245        Ok(r) => r,
246        Err(e) => return Err(e),
247    };
248    let Some(adapter) = channels.get(&routing.channel_name) else {
249        return Err(format!("no adapter for channel '{}'", routing.channel_name));
250    };
251    if adapter.is_local_only() {
252        return Ok(format!(
253            "channel '{}' is local-only, path retained at: {}",
254            routing.channel_name,
255            file.display()
256        ));
257    }
258    let external_chat_id = crate::internal::storage::db::call_blocking(db.clone(), move |d| {
259        d.get_chat_external_id(chat_id)
260    })
261    .await
262    .map_err(|e| e.to_string())?
263    .unwrap_or_else(|| chat_id.to_string());
264    let caption_short = caption.chars().take(120).collect::<String>();
265    match adapter
266        .send_attachment(&external_chat_id, file, Some(&caption_short))
267        .await
268    {
269        Ok(_) => {
270            let _ = deliver_and_store_bot_message(
271                channels,
272                db,
273                "bot",
274                chat_id,
275                &format!("[audio attached: {}]", file.display()),
276            )
277            .await;
278            Ok(format!("delivered via channel '{}'", routing.channel_name))
279        }
280        Err(e) => Err(e),
281    }
282}
283
284#[cfg(test)]
285mod tests {
286    use super::*;
287
288    #[test]
289    fn voice_and_format_constants_wellformed() {
290        assert!(ALLOWED_VOICES.contains(&"alloy"));
291        assert!(ALLOWED_FORMATS.contains(&"mp3"));
292    }
293}
microclaw_engine/tools/text_to_speech.rs

microclaw_engine/tools/
text_to_speech.rs
