microclaw_engine/tools/
transcribe_audio.rs

1use async_trait::async_trait;
2use reqwest::multipart;
3use serde_json::{json, Value};
4use std::path::PathBuf;
5
6use crate::internal::tool_runtime::media_client::{load_bytes_from_location, MediaClient};
7use microclaw_core::llm_types::ToolDefinition;
8
9use super::{schema_object, Tool, ToolResult};
10use crate::config::{Config, MediaConfig, SttConfig};
11
12/// OpenAI-compatible speech-to-text tool.
13///
14/// Calls `/v1/audio/transcriptions` (multipart/form-data). Accepts a local
15/// audio path (inside working_dir), a public URL, or a `data:` URI.
16/// Disabled by default.
17pub struct TranscribeAudioTool {
18    working_dir: PathBuf,
19    allowed_read_dirs: Vec<PathBuf>,
20    cfg: SttConfig,
21    media: MediaConfig,
22    openai_api_key: Option<String>,
23    openai_base_url: Option<String>,
24    timeout_secs: u64,
25}
26
27impl TranscribeAudioTool {
28    pub fn new(config: &Config) -> Self {
29        Self {
30            working_dir: PathBuf::from(&config.working_dir),
31            allowed_read_dirs: config
32                .media
33                .allowed_read_dirs
34                .iter()
35                .map(PathBuf::from)
36                .collect(),
37            cfg: config.media.stt.clone(),
38            media: config.media.clone(),
39            openai_api_key: config.openai_api_key.clone(),
40            openai_base_url: config.openai_base_url.clone(),
41            timeout_secs: config.tool_timeout_secs("transcribe_audio", 120),
42        }
43    }
44
45    fn client(&self) -> Result<MediaClient, String> {
46        let key = self
47            .media
48            .resolve_api_key(self.openai_api_key.as_deref())
49            .ok_or_else(|| {
50                "transcribe_audio requires an API key (media.api_key, \
51                 MICROCLAW_OPENAI_API_KEY, OPENAI_API_KEY, or top-level \
52                 openai_api_key)."
53                    .to_string()
54            })?;
55        let base = self.media.resolve_base_url(self.openai_base_url.as_deref());
56        MediaClient::new(base, key, self.timeout_secs)
57    }
58}
59
60fn filename_for_mime(mime: Option<&str>) -> (&'static str, &'static str) {
61    match mime.unwrap_or("") {
62        "audio/mpeg" => ("audio.mp3", "audio/mpeg"),
63        "audio/mp4" => ("audio.m4a", "audio/mp4"),
64        "audio/ogg" | "audio/opus" => ("audio.ogg", "audio/ogg"),
65        "audio/wav" | "audio/wave" | "audio/x-wav" => ("audio.wav", "audio/wav"),
66        "audio/flac" => ("audio.flac", "audio/flac"),
67        "audio/webm" => ("audio.webm", "audio/webm"),
68        _ => ("audio.ogg", "audio/ogg"),
69    }
70}
71
72#[async_trait]
73impl Tool for TranscribeAudioTool {
74    fn name(&self) -> &str {
75        "transcribe_audio"
76    }
77
78    fn definition(&self) -> ToolDefinition {
79        ToolDefinition {
80            name: self.name().into(),
81            description: "Transcribe an audio file to text via an OpenAI-compatible \
82                /audio/transcriptions endpoint (Whisper-style). Accepts a file path \
83                under the working directory, a public URL, or a `data:` URI. \
84                Returns the transcript. Disabled by default; requires operator \
85                opt-in via `media.stt.enabled`."
86                .into(),
87            input_schema: schema_object(
88                json!({
89                    "audio": {
90                        "type": "string",
91                        "description": "Audio source: local file path (inside working_dir), https:// URL, or data: URI."
92                    },
93                    "language": {
94                        "type": "string",
95                        "description": "Optional ISO-639-1 hint (e.g. 'en', 'zh'). Defaults to media.stt.language (auto if unset)."
96                    },
97                    "prompt": {
98                        "type": "string",
99                        "description": "Optional biasing prompt (e.g. glossary of expected names/terms)."
100                    },
101                    "model": {
102                        "type": "string",
103                        "description": "Optional model override (e.g. whisper-1, gpt-4o-transcribe)."
104                    }
105                }),
106                &["audio"],
107            ),
108        }
109    }
110
111    async fn execute(&self, input: Value) -> ToolResult {
112        if !self.cfg.enabled {
113            return ToolResult::error(
114                "transcribe_audio is disabled. Set media.stt.enabled=true to enable.".into(),
115            );
116        }
117        let audio = match input.get("audio").and_then(|v| v.as_str()) {
118            Some(v) if !v.trim().is_empty() => v.trim().to_string(),
119            _ => return ToolResult::error("Missing parameter: audio".into()),
120        };
121        let language = input
122            .get("language")
123            .and_then(|v| v.as_str())
124            .map(|s| s.to_string())
125            .or_else(|| self.cfg.language.clone());
126        let prompt = input
127            .get("prompt")
128            .and_then(|v| v.as_str())
129            .map(|s| s.to_string());
130        let model = input
131            .get("model")
132            .and_then(|v| v.as_str())
133            .map(|s| s.to_string())
134            .unwrap_or_else(|| self.cfg.model.clone());
135
136        let client = match self.client() {
137            Ok(c) => c,
138            Err(e) => return ToolResult::error(e),
139        };
140
141        let (bytes, mime) = match load_bytes_from_location(
142            client.client(),
143            &audio,
144            &self.working_dir,
145            &self.allowed_read_dirs,
146        )
147        .await
148        {
149            Ok(v) => v,
150            Err(e) => return ToolResult::error(format!("audio fetch failed: {e}")),
151        };
152        let (file_name, mime_str) = filename_for_mime(mime.as_deref());
153
154        let part = match multipart::Part::bytes(bytes)
155            .file_name(file_name)
156            .mime_str(mime_str)
157        {
158            Ok(p) => p,
159            Err(e) => return ToolResult::error(format!("invalid audio mime '{mime_str}': {e}")),
160        };
161        let mut form = multipart::Form::new()
162            .text("model", model.clone())
163            .part("file", part);
164        if let Some(lang) = language.as_deref().filter(|s| !s.is_empty()) {
165            form = form.text("language", lang.to_string());
166        }
167        if let Some(p) = prompt.as_deref().filter(|s| !s.is_empty()) {
168            form = form.text("prompt", p.to_string());
169        }
170
171        let resp = match client
172            .post_multipart("audio/transcriptions", form)
173            .send()
174            .await
175        {
176            Ok(r) => r,
177            Err(e) => return ToolResult::error(format!("transcriptions request failed: {e}")),
178        };
179        if !resp.status().is_success() {
180            let status = resp.status();
181            let text = resp.text().await.unwrap_or_default();
182            return ToolResult::error(format!("transcriptions HTTP {status}: {text}"));
183        }
184        let parsed: Value = match resp.json().await {
185            Ok(v) => v,
186            Err(e) => return ToolResult::error(format!("invalid JSON from transcriptions: {e}")),
187        };
188        match parsed.get("text").and_then(|v| v.as_str()) {
189            Some(t) => ToolResult::success(t.to_string()),
190            None => ToolResult::error("transcriptions response missing 'text' field".into()),
191        }
192    }
193}
194
195#[cfg(test)]
196mod tests {
197    use super::*;
198
199    #[test]
200    fn mime_mapping() {
201        assert_eq!(filename_for_mime(Some("audio/mpeg")).1, "audio/mpeg");
202        assert_eq!(filename_for_mime(Some("audio/wav")).1, "audio/wav");
203        assert_eq!(filename_for_mime(None).1, "audio/ogg");
204    }
205}
microclaw_engine/tools/transcribe_audio.rs

microclaw_engine/tools/
transcribe_audio.rs
