Skip to main content

robius_speech/
lib.rs

1//! Native streaming speech-to-text input and microphone capture for Rust apps.
2//!
3//! Recognition keeps going across as many utterances as the user speaks, until you
4//! stop it. An "utterance" is just a chunk of words spoken by the user: partial
5//! transcripts replace the current one, whereas a final transcript commits it.
6//!
7//! Note that speech always stays as plain text here. This crate never interprets
8//! spoken works as commands or synthesizes key events from them,
9//! so saying something like "enter" will just type the word "enter".
10//!
11//! [`Dictation`] turns those transcripts into edits of a text field, so an app
12//! only has to apply the replacements it hands back.
13//!
14//! Apple builds need the Xcode command line tools (`xcode-select --install`), and
15//! support macOS 11+ and iOS 13+. For macOS builds, set the `MACOSX_DEPLOYMENT_TARGET`
16//! env var to 11.0 or newer for the whole Cargo invocation, including the final app.
17//! This mostly just matters on Intel x86 macs, but it doesn't hurt to always set it.
18
19use std::collections::HashMap;
20use std::sync::{Arc, Mutex, OnceLock};
21use std::sync::atomic::{AtomicU64, Ordering};
22
23pub mod dictation;
24pub use dictation::{Dictation, Replacement};
25
26#[cfg(any(target_os = "macos", target_os = "ios"))]
27mod apple;
28#[cfg(all(target_os = "android", native_speech_android))]
29mod android;
30#[cfg(target_os = "windows")]
31mod windows;
32
33#[cfg(any(target_os = "macos", target_os = "ios"))]
34use apple as backend;
35#[cfg(all(target_os = "android", native_speech_android))]
36use android as backend;
37#[cfg(target_os = "windows")]
38use windows as backend;
39
40#[derive(Clone, Debug)]
41pub struct NativeSpeechOptions {
42    /// A BCP-47 locale (e.g., `"en-US"`), or the OS's default speech language.
43    pub locale: Option<String>,
44    /// Prefer on-device recognition when the native service supports it.
45    /// Otherwise the system service may need a network connection.
46    pub prefer_on_device: bool,
47}
48
49impl Default for NativeSpeechOptions {
50    fn default() -> Self {
51        Self { locale: None, prefer_on_device: true }
52    }
53}
54
55/// The cause of a speech failure, so you can handle each one properly
56/// instead of matching on message text.
57#[derive(Clone, Copy, Debug, Eq, PartialEq, Hash)]
58#[non_exhaustive]
59pub enum SpeechErrorKind {
60    /// The user denied permission, or your app is missing the privacy
61    /// declaration it needs in order to even ask.
62    /// Usually you'll want to tell the user to enable audio and/or speech
63    /// permissions in their system settings.
64    PermissionDenied,
65    /// The audio input or speech recognition service just doesn't exist here,
66    /// so retrying will never work.
67    Unavailable,
68    /// The recognizer doesn't support the language you asked for.
69    Language,
70    /// The microphone is missing, busy, or changed mid-session.
71    /// Starting things again will likely work.
72    Audio,
73    /// Another recording is already active; only one session can be active at a time.
74    Busy,
75    /// Anything else that the OS reported.
76    Other,
77}
78
79/// A [`SpeechErrorKind`] that you can match on, plus the message the OS gave us.
80/// `Display` writes just that message, which is already worded for end users.
81#[derive(Clone, Debug, Eq, PartialEq)]
82pub struct SpeechError {
83    kind: SpeechErrorKind,
84    message: String,
85}
86
87impl SpeechError {
88    pub fn new(kind: SpeechErrorKind, message: impl Into<String>) -> Self {
89        Self { kind, message: message.into() }
90    }
91
92    pub fn kind(&self) -> SpeechErrorKind {
93        self.kind
94    }
95
96    pub fn message(&self) -> &str {
97        &self.message
98    }
99}
100
101impl std::fmt::Display for SpeechError {
102    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
103        f.write_str(&self.message)
104    }
105}
106
107impl std::error::Error for SpeechError {}
108
109#[derive(Clone, Debug, PartialEq)]
110pub enum NativeSpeechEvent {
111    /// Permissions were granted and the microphone is actually recording.
112    Started,
113    /// Recognized text is available.
114    ///
115    /// If `is_final` is `false`, the text is a partial transcript that may be updated/changed later.
116    /// If `is_final` is `true`, the text is a final transcript that is fully "committed".
117    Transcript {
118        text: String,
119        is_final: bool,
120    },
121    /// Microphone amplitude, normalized between 0 and 1, which you can use to
122    /// display a level meter or similar sound wave animation.
123    AudioLevel(f32),
124    /// Capture and final recognition are done. This is a final event.
125    Stopped,
126    /// A final error, e.g., denied permissions or unavailable hardware.
127    Error(SpeechError),
128}
129
130/// Event codes shared with the Swift and Java bridges. The error kinds are just
131/// extra values of the same `kind` argument, so classifying them needs no ABI change.
132#[cfg(any(target_os = "macos", target_os = "ios", all(target_os = "android", native_speech_android)))]
133#[allow(dead_code)] // Not every backend uses every code.
134pub(crate) mod codes {
135    pub(crate) const STARTED: i32 = 0;
136    pub(crate) const PARTIAL: i32 = 1;
137    pub(crate) const FINAL: i32 = 2;
138    pub(crate) const LEVEL: i32 = 3;
139    pub(crate) const STOPPED: i32 = 4;
140    pub(crate) const ERROR: i32 = 5;
141    pub(crate) const ERROR_PERMISSION: i32 = 6;
142    pub(crate) const ERROR_UNAVAILABLE: i32 = 7;
143    pub(crate) const ERROR_LANGUAGE: i32 = 8;
144    pub(crate) const ERROR_AUDIO: i32 = 9;
145}
146
147/// Unknown codes fall back to `Other`, so a newer bridge still reports its message.
148#[cfg(any(target_os = "macos", target_os = "ios", all(target_os = "android", native_speech_android)))]
149pub(crate) fn error_kind_from_code(code: i32) -> SpeechErrorKind {
150    match code {
151        codes::ERROR_PERMISSION => SpeechErrorKind::PermissionDenied,
152        codes::ERROR_UNAVAILABLE => SpeechErrorKind::Unavailable,
153        codes::ERROR_LANGUAGE => SpeechErrorKind::Language,
154        codes::ERROR_AUDIO => SpeechErrorKind::Audio,
155        _ => SpeechErrorKind::Other,
156    }
157}
158
159type Callback = Arc<dyn Fn(NativeSpeechEvent) + Send + Sync>;
160static SESSIONS: OnceLock<Mutex<HashMap<u64, Callback>>> = OnceLock::new();
161static NEXT_SESSION: AtomicU64 = AtomicU64::new(1);
162
163fn sessions() -> &'static Mutex<HashMap<u64, Callback>> {
164    SESSIONS.get_or_init(|| Mutex::new(HashMap::new()))
165}
166
167/// Owns the system microphone and recognition task; dropping it cancels both.
168///
169/// Importantly, only one session can be active at a time.
170///
171/// Callbacks arrive on platform-dependent native OS threads, so you'll want to
172/// forward them to your main UI thread or event loop. Note that it's possible for
173/// an in-progress callback to continue executing after you cancel the session.
174pub struct NativeSpeechSession {
175    id: u64,
176}
177
178/// The short, stable name of the recognizer this build uses, e.g., `"apple-sfspeech"`.
179/// Handy for logs and bug reports.
180pub fn engine_name() -> &'static str {
181    backend::ENGINE
182}
183
184impl NativeSpeechSession {
185    /// Whether this platform offers a native speech recognizer at all.
186    ///
187    /// This is false on Linux and the web, so you'll usually want to hide any
188    /// dictation button entirely rather than let it fail when pressed.
189    pub fn is_supported() -> bool {
190        backend::is_supported()
191    }
192
193    /// Starts dictating, requesting microphone (and on Apple, speech recognition)
194    /// permission if needed, so you don't need any platform-specific permission code.
195    ///
196    /// This returns immediately rather than blocking on the permission prompt.
197    /// The `Started` event tells you when the microphone is actually recording,
198    /// and a refusal arrives as [`SpeechErrorKind::PermissionDenied`].
199    pub fn start(
200        options: NativeSpeechOptions,
201        callback: impl Fn(NativeSpeechEvent) + Send + Sync + 'static,
202    ) -> Result<Self, SpeechError> {
203        if !Self::is_supported() {
204            return Err(SpeechError::new(
205                SpeechErrorKind::Unavailable,
206                "Native speech recognition is not available on this platform.",
207            ));
208        }
209        if options.locale.as_ref().is_some_and(|locale| locale.contains('\0')) {
210            return Err(SpeechError::new(
211                SpeechErrorKind::Language,
212                "The speech recognition language is invalid.",
213            ));
214        }
215        let id = NEXT_SESSION.fetch_add(1, Ordering::Relaxed);
216        {
217            let mut sessions = sessions().lock().unwrap();
218            if !sessions.is_empty() {
219                return Err(SpeechError::new(
220                    SpeechErrorKind::Busy,
221                    "Another speech recording is already active.",
222                ));
223            }
224            sessions.insert(id, Arc::new(callback));
225        }
226        if let Err(error) = backend::start(id, &options) {
227            sessions().lock().unwrap().remove(&id);
228            return Err(error);
229        }
230        Ok(Self { id })
231    }
232
233    /// Closes the microphone, but still lets the final recognition result arrive.
234    pub fn stop(&self) {
235        backend::stop(self.id, false);
236    }
237
238    /// Stops immediately and discards anything that's still pending.
239    pub fn cancel(&self) {
240        sessions().lock().unwrap().remove(&self.id);
241        backend::stop(self.id, true);
242    }
243}
244
245impl Drop for NativeSpeechSession {
246    fn drop(&mut self) {
247        self.cancel();
248    }
249}
250
251/// Cancels every active session, for things like suspending or quitting the app.
252pub fn cancel_all() {
253    let ids: Vec<_> = sessions().lock().unwrap().drain().map(|(id, _)| id).collect();
254    for id in ids {
255        backend::stop(id, true);
256    }
257}
258
259#[cfg(any(test, target_os = "macos", target_os = "ios", all(target_os = "android", native_speech_android), target_os = "windows"))]
260pub(crate) fn emit(id: u64, event: NativeSpeechEvent) {
261    let callback = {
262        let mut sessions = sessions().lock().unwrap();
263        if matches!(event, NativeSpeechEvent::Stopped | NativeSpeechEvent::Error(_)) {
264            sessions.remove(&id)
265        } else {
266            sessions.get(&id).cloned()
267        }
268    };
269    if let Some(callback) = callback {
270        // Never unwind through a native callback boundary.
271        let _ = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| callback(event)));
272    }
273}
274
275#[cfg(not(any(target_os = "macos", target_os = "ios", all(target_os = "android", native_speech_android), target_os = "windows")))]
276mod backend {
277    use super::{NativeSpeechOptions, SpeechError, SpeechErrorKind};
278    pub(super) const ENGINE: &str = "none";
279    pub(super) fn is_supported() -> bool { false }
280    pub(super) fn start(_: u64, _: &NativeSpeechOptions) -> Result<(), SpeechError> {
281        Err(SpeechError::new(
282            SpeechErrorKind::Unavailable,
283            "Native speech recognition is not available on this platform.",
284        ))
285    }
286    pub(super) fn stop(_: u64, _: bool) {}
287}
288
289#[cfg(test)]
290mod tests {
291    use super::*;
292
293    #[test]
294    fn terminal_and_cancelled_sessions_discard_late_native_results() {
295        let received = Arc::new(Mutex::new(Vec::new()));
296        let output = received.clone();
297        sessions().lock().unwrap().insert(1001, Arc::new(move |event| {
298            output.lock().unwrap().push(event);
299        }));
300        emit(1001, NativeSpeechEvent::Transcript { text: "draft".into(), is_final: false });
301        emit(1001, NativeSpeechEvent::Stopped);
302        emit(1001, NativeSpeechEvent::Transcript { text: "stale".into(), is_final: true });
303        assert_eq!(*received.lock().unwrap(), vec![
304            NativeSpeechEvent::Transcript { text: "draft".into(), is_final: false },
305            NativeSpeechEvent::Stopped,
306        ]);
307
308        let output = received.clone();
309        sessions().lock().unwrap().insert(1002, Arc::new(move |event| {
310            output.lock().unwrap().push(event);
311        }));
312        // A previous session's delayed callback must not be delivered to its
313        // successor, even when recognition is restarted immediately.
314        emit(1001, NativeSpeechEvent::Error(SpeechError::new(SpeechErrorKind::Other, "late failure")));
315        NativeSpeechSession { id: 1002 }.cancel();
316        emit(1002, NativeSpeechEvent::Started);
317        assert_eq!(received.lock().unwrap().len(), 2);
318        let sessions = sessions().lock().unwrap();
319        assert!(!sessions.contains_key(&1001));
320        assert!(!sessions.contains_key(&1002));
321    }
322}