audio_code/speech/
mod.rs

1//! Screen reader and text-to-speech output. Available as `ctx.speech`.
2//!
3//! The engine tries these outputs in order and uses the first one that works:
4//!
5//! 1. **[prism](https://github.com/ethindp/prism)** (feature `prism`, on by default). It talks
6//!    to the running screen reader (NVDA, JAWS, VoiceOver, Orca, ...) or falls back to the
7//!    system voice (SAPI / OneCore, AVSpeech, speech-dispatcher, ...).
8//! 2. **VoiceOver or the system voice** (macOS, feature `macos`). Speaks through VoiceOver
9//!    announcements when VoiceOver is on, otherwise through `AVSpeechSynthesizer`.
10//! 3. **The console.** Everything is printed, so the game always keeps working.
11//!
12//! Speech runs on its own thread, so speaking never blocks your game loop.
13//!
14//! [`Speech::voices`] lists the voices of the system voice output and [`Speech::set_voice`]
15//! picks one. Screen readers keep the voice the player chose in the screen reader.
16
17#[cfg(all(target_os = "macos", feature = "macos"))]
18mod macos;
19
20use std::sync::atomic::{AtomicBool, Ordering};
21use std::sync::mpsc::{Receiver, Sender, channel};
22use std::sync::{Arc, Condvar, Mutex};
23use std::time::Duration;
24
25const HISTORY_LEN: usize = 100;
26/// How long [`Speech::voices`] waits for the output to start.
27const VOICES_TIMEOUT: Duration = Duration::from_secs(3);
28
29// Only prism uses every command (macOS ignores braille).
30#[cfg_attr(not(feature = "prism"), allow(dead_code))]
31enum Command {
32    Speak { text: String, interrupt: bool },
33    Braille(String),
34    Stop,
35    Rate(f32),
36    Volume(f32),
37    /// A [`Voice::id`] from the list the output published, or `None` for the default voice.
38    Voice(Option<String>),
39}
40
41/// A voice that [`Speech::set_voice`] can switch to. Get the list with [`Speech::voices`].
42#[derive(Debug, Clone, PartialEq, Eq)]
43pub struct Voice {
44    /// Identifies the voice for [`Speech::set_voice`]. Its format depends on the output, for
45    /// example `"com.apple.voice.compact.de-DE.Anna"` on macOS.
46    pub id: String,
47    /// Display name, e.g. "Anna". Show this to players.
48    pub name: String,
49    /// Language tag such as "de-DE" or "en-US", if the output reports one.
50    pub language: Option<String>,
51}
52
53impl std::fmt::Display for Voice {
54    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
55        match &self.language {
56            Some(language) => write!(f, "{} ({language})", self.name),
57            None => f.write_str(&self.name),
58        }
59    }
60}
61
62/// State shared between [`Speech`] and its output thread.
63#[derive(Debug, Default)]
64struct Shared {
65    backend: Mutex<Option<String>>,
66    echo: AtomicBool,
67    /// `None` until the output has started and listed its voices.
68    voices: Mutex<Option<Vec<Voice>>>,
69    voices_ready: Condvar,
70}
71
72impl Shared {
73    #[cfg_attr(not(any(feature = "prism", all(target_os = "macos", feature = "macos"))), allow(dead_code))]
74    fn set_backend(&self, name: String) {
75        *self.backend.lock().unwrap_or_else(|e| e.into_inner()) = Some(name);
76    }
77
78    /// Publishes the voice list (empty if the output has no voices to choose from).
79    fn set_voices(&self, voices: Vec<Voice>) {
80        *self.voices.lock().unwrap_or_else(|e| e.into_inner()) = Some(voices);
81        self.voices_ready.notify_all();
82    }
83}
84
85/// Speech and braille output. Reach it through `ctx.speech`.
86#[derive(Debug)]
87pub struct Speech {
88    tx: Option<Sender<Command>>,
89    shared: Arc<Shared>,
90    muted: bool,
91    history: Vec<String>,
92    voice: Option<Voice>,
93}
94
95impl Speech {
96    pub(crate) fn new(echo: bool) -> Self {
97        let shared = Arc::new(Shared::default());
98        shared.echo.store(echo, Ordering::Relaxed);
99        let tx = spawn(shared.clone());
100        if tx.is_none() {
101            shared.set_voices(Vec::new());
102        }
103        Self { tx, shared, muted: false, history: Vec::new(), voice: None }
104    }
105
106    /// Speech that only prints to the console. Useful for tests and headless tools.
107    pub fn console() -> Self {
108        let shared = Arc::new(Shared::default());
109        shared.echo.store(true, Ordering::Relaxed);
110        shared.set_voices(Vec::new());
111        Self { tx: None, shared, muted: false, history: Vec::new(), voice: None }
112    }
113
114    fn echo(&self) -> bool {
115        self.shared.echo.load(Ordering::Relaxed)
116    }
117
118    fn send(&mut self, cmd: Command) -> bool {
119        match &self.tx {
120            Some(tx) => {
121                if tx.send(cmd).is_err() {
122                    // The speech thread gave up (no output found); fall back to the console.
123                    self.tx = None;
124                    self.shared.echo.store(true, Ordering::Relaxed);
125                    false
126                } else {
127                    true
128                }
129            }
130            None => false,
131        }
132    }
133
134    fn say(&mut self, text: &str, interrupt: bool) {
135        if text.is_empty() {
136            return;
137        }
138        if self.history.len() >= HISTORY_LEN {
139            self.history.remove(0);
140        }
141        self.history.push(text.to_string());
142        if self.muted {
143            return;
144        }
145        let sent = self.send(Command::Speak { text: text.to_string(), interrupt });
146        if self.echo() || !sent {
147            println!("{text}");
148        }
149    }
150
151    /// Speaks text, interrupting whatever is being said. This is what you want most of the time.
152    pub fn speak(&mut self, text: impl AsRef<str>) {
153        self.say(text.as_ref(), true);
154    }
155
156    /// Speaks text after the current speech has finished.
157    pub fn speak_queued(&mut self, text: impl AsRef<str>) {
158        self.say(text.as_ref(), false);
159    }
160
161    /// Shows text on a braille display only.
162    pub fn braille(&mut self, text: impl AsRef<str>) {
163        self.send(Command::Braille(text.as_ref().to_string()));
164    }
165
166    /// Silences speech immediately.
167    pub fn stop(&mut self) {
168        self.send(Command::Stop);
169    }
170
171    /// Speaks the last message again. Bind this to a key so players can catch up.
172    pub fn repeat_last(&mut self) {
173        if let Some(last) = self.history.last().cloned() {
174            self.history.pop();
175            self.speak(last);
176        }
177    }
178
179    /// The most recently spoken message.
180    pub fn last(&self) -> Option<&str> {
181        self.history.last().map(String::as_str)
182    }
183
184    /// Recently spoken messages, oldest first (up to 100).
185    pub fn history(&self) -> &[String] {
186        &self.history
187    }
188
189    /// Sets the speech rate if the output supports it, from 0.0 (slowest) to 1.0
190    /// (fastest), passed on to the voice as is.
191    /// Screen readers usually ignore this and keep the user's own settings.
192    pub fn set_rate(&mut self, rate: f32) {
193        self.send(Command::Rate(rate.clamp(0.0, 1.0)));
194    }
195
196    /// Sets the speech volume if the output supports it (0.0 to 1.0).
197    pub fn set_volume(&mut self, volume: f32) {
198        self.send(Command::Volume(volume.clamp(0.0, 1.0)));
199    }
200
201    /// The voices of the system voice output, sorted by language and name, for example to
202    /// build a voice menu. Empty when the output has no choice of voices: screen readers
203    /// (NVDA, VoiceOver, ...) speak with the voice the player set up there, and the console has
204    /// none.
205    ///
206    /// Right after the game starts this waits until the output is ready (at most a few
207    /// seconds); after that it returns immediately.
208    ///
209    /// ```no_run
210    /// # use audio_code::prelude::*;
211    /// # fn f(ctx: &mut Context) {
212    /// for voice in ctx.speech.voices() {
213    ///     println!("{} {}", voice.name, voice.language.as_deref().unwrap_or("?"));
214    /// }
215    /// # }
216    /// ```
217    pub fn voices(&self) -> Vec<Voice> {
218        let voices = self.shared.voices.lock().unwrap_or_else(|e| e.into_inner());
219        let (voices, _) = self
220            .shared
221            .voices_ready
222            .wait_timeout_while(voices, VOICES_TIMEOUT, |v| v.is_none())
223            .unwrap_or_else(|e| e.into_inner());
224        voices.clone().unwrap_or_default()
225    }
226
227    /// Switches the system voice. `voice` is a [`Voice::id`] or a voice name (any case),
228    /// e.g. `"Anna"`. Returns `false` and keeps the current voice if no such voice exists.
229    /// Screen readers ignore this, see [`Speech::voices`].
230    pub fn set_voice(&mut self, voice: impl AsRef<str>) -> bool {
231        let wanted = voice.as_ref();
232        let voices = self.voices();
233        let found = voices
234            .iter()
235            .find(|v| v.id == wanted)
236            .or_else(|| voices.iter().find(|v| v.name.eq_ignore_ascii_case(wanted)))
237            .cloned();
238        match found {
239            Some(voice) => {
240                self.send(Command::Voice(Some(voice.id.clone())));
241                self.voice = Some(voice);
242                true
243            }
244            None => {
245                eprintln!("audio-code: no speech voice called \"{wanted}\"");
246                false
247            }
248        }
249    }
250
251    /// Goes back to the default voice, the one the game started with.
252    pub fn reset_voice(&mut self) {
253        if self.voice.take().is_some() {
254            self.send(Command::Voice(None));
255        }
256    }
257
258    /// The voice chosen with [`Speech::set_voice`], or `None` for the default voice.
259    pub fn voice(&self) -> Option<&Voice> {
260        self.voice.as_ref()
261    }
262
263    /// Also print everything spoken to the console. Handy while developing.
264    pub fn set_echo(&mut self, echo: bool) {
265        self.shared.echo.store(echo, Ordering::Relaxed);
266    }
267
268    /// Mutes speech output (messages still go to the history).
269    pub fn set_muted(&mut self, muted: bool) {
270        self.muted = muted;
271    }
272
273    /// Name of the active output, e.g. "NVDA", "SAPI", "VoiceOver" or "AVSpeechSynthesizer".
274    /// `None` while starting up or when printing to the console.
275    pub fn backend_name(&self) -> Option<String> {
276        self.shared.backend.lock().unwrap_or_else(|e| e.into_inner()).clone()
277    }
278}
279
280/// Starts the output thread, or returns `None` if no output is compiled in.
281fn spawn(shared: Arc<Shared>) -> Option<Sender<Command>> {
282    if !cfg!(any(feature = "prism", all(target_os = "macos", feature = "macos"))) {
283        return None;
284    }
285    let (tx, rx) = channel::<Command>();
286    let spawned = std::thread::Builder::new().name("audio-code speech".into()).spawn(move || {
287        #[cfg_attr(not(any(feature = "prism", all(target_os = "macos", feature = "macos"))), allow(unused_mut))]
288        let mut reasons: Vec<String> = Vec::new();
289
290        #[cfg(feature = "prism")]
291        match run_prism(&rx, &shared) {
292            Ok(()) => return,
293            Err(e) => reasons.push(format!("prism: {e}")),
294        }
295
296        #[cfg(all(target_os = "macos", feature = "macos"))]
297        match macos::run(&rx, &shared) {
298            Ok(()) => return,
299            Err(e) => reasons.push(format!("macOS speech: {e}")),
300        }
301
302        eprintln!("audio-code: no speech output found ({}); printing to console", reasons.join("; "));
303        shared.set_voices(Vec::new());
304        // Dropping the receiver makes Speech switch to console output.
305        drop(rx);
306    });
307    spawned.ok().map(|_| tx)
308}
309
310/// Sorts voices by language, then name, so a voice menu groups languages together.
311#[cfg_attr(not(any(feature = "prism", all(target_os = "macos", feature = "macos"))), allow(dead_code))]
312fn sort_voices(voices: &mut [Voice]) {
313    voices.sort_by(|a, b| a.language.cmp(&b.language).then_with(|| a.name.cmp(&b.name)));
314}
315
316/// Runs prism until the channel closes. Returns an error if prism has no usable output.
317#[cfg(feature = "prism")]
318fn run_prism(rx: &Receiver<Command>, shared: &Shared) -> Result<(), String> {
319    // Declared before the backend, so the backend is dropped first.
320    let prism = prismer::Prism::new().map_err(|e| e.to_string())?;
321    let backend = prism.acquire_best().map_err(|e| e.to_string())?;
322    shared.set_backend(backend.name());
323    let count = backend.voice_count().unwrap_or(0);
324    let mut voices: Vec<Voice> = (0..count)
325        .filter_map(|i| {
326            let name = backend.voice_name(i).ok()?;
327            let language = backend.voice_language(i).ok().filter(|l| !l.is_empty());
328            Some(Voice { id: i.to_string(), name, language })
329        })
330        .collect();
331    sort_voices(&mut voices);
332    shared.set_voices(voices);
333    let default_voice = backend.voice().ok();
334    for cmd in rx {
335        // Errors such as "not implemented" for one feature are not worth crashing for.
336        let _ = match cmd {
337            Command::Speak { text, interrupt } => {
338                backend.output(&text, interrupt).or_else(|_| backend.speak(&text, interrupt))
339            }
340            Command::Braille(text) => backend.braille(&text),
341            Command::Stop => backend.stop(),
342            Command::Rate(r) => backend.set_rate(r),
343            Command::Volume(v) => backend.set_volume(v),
344            Command::Voice(id) => match id.map_or(default_voice, |id| id.parse().ok()) {
345                Some(index) => backend.set_voice(index),
346                None => Ok(()),
347            },
348        };
349    }
350    Ok(())
351}
352
353#[cfg(not(feature = "prism"))]
354#[allow(dead_code)]
355fn run_prism(_: &Receiver<Command>, _: &Shared) -> Result<(), String> {
356    Err("not compiled in".into())
357}
358
359#[cfg(test)]
360mod tests {
361    use super::*;
362
363    #[test]
364    fn console_has_no_voices() {
365        let mut speech = Speech::console();
366        assert!(speech.voices().is_empty());
367        assert!(!speech.set_voice("Anna"));
368        assert_eq!(speech.voice(), None);
369        speech.reset_voice();
370    }
371
372    #[test]
373    fn voices_sort_by_language_then_name() {
374        let voice = |name: &str, language: &str| Voice { id: name.into(), name: name.into(), language: Some(language.into()) };
375        let mut voices = vec![voice("Zoe", "de-DE"), voice("Alex", "en-US"), voice("Anna", "de-DE")];
376        sort_voices(&mut voices);
377        let names: Vec<&str> = voices.iter().map(|v| v.name.as_str()).collect();
378        assert_eq!(names, ["Anna", "Zoe", "Alex"]);
379        assert_eq!(voices[0].to_string(), "Anna (de-DE)");
380    }
381}
audio_code/speech/mod.rs

audio_code/speech/
mod.rs
