Skip to main content

rigspark_runtime/
voice.rs

1//! Local voice: text-to-speech ("speak") and speech-to-text ("listen").
2//!
3//! RigSpark starts whisper.cpp's `whisper-server` with a pinned, SHA-256 verified ggml
4//! file, or `mlx_audio.server` on Apple Silicon, always bound to 127.0.0.1. Any other
5//! loopback server that speaks OpenAI's `/v1/audio/speech` and `/v1/audio/transcriptions`
6//! can be attached instead. Requests and replies are bounded; audio is WAV only.
7use crate::acquire::{Acquisition, Artifact, HfTransport, SizePolicy};
8use rigspark_core::{
9    sizing::Hardware,
10    voice::{VoiceCatalog, VoiceEngine, VoiceKind, VoiceModel},
11};
12use serde::Serialize;
13use serde_json::{Value, json};
14use std::{
15    collections::BTreeMap,
16    path::{Path, PathBuf},
17    sync::{Arc, Mutex},
18    time::Duration,
19};
20use tokio_util::sync::CancellationToken;
21
22pub const WHISPER_SERVER_ENV: &str = "RIGSPARK_WHISPER_SERVER";
23pub const MLX_AUDIO_SERVER_ENV: &str = "RIGSPARK_MLX_AUDIO_SERVER";
24/// Longest text one speak request accepts.
25pub const MAX_SPEAK_CHARS: usize = 8000;
26/// Largest WAV a speak reply or a transcription upload may be.
27pub const MAX_AUDIO_BYTES: usize = 48 * 1024 * 1024;
28/// Longest transcript accepted back from an engine.
29pub const MAX_TRANSCRIPT_BYTES: usize = 256 * 1024;
30
31#[derive(Debug, Clone, PartialEq, Eq)]
32pub struct VoiceError(pub String);
33impl std::fmt::Display for VoiceError {
34    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
35        formatter.write_str(&self.0)
36    }
37}
38impl std::error::Error for VoiceError {}
39fn fail<T>(message: impl Into<String>) -> Result<T, VoiceError> {
40    Err(VoiceError(message.into()))
41}
42
43/// Where an engine's server binary was found, if anywhere.
44#[derive(Debug, Clone, Serialize)]
45#[serde(rename_all = "camelCase")]
46pub struct EngineProbe {
47    pub engine: &'static str,
48    pub label: &'static str,
49    pub installed: bool,
50    pub supported: bool,
51    #[serde(skip)]
52    pub path: Option<PathBuf>,
53    pub install: &'static str,
54}
55
56fn binary_from(variable: Option<std::ffi::OsString>, name: &str) -> Option<PathBuf> {
57    match variable {
58        Some(value) => {
59            let path = PathBuf::from(value);
60            (path.is_absolute() && path.is_file())
61                .then(|| path.canonicalize().ok())
62                .flatten()
63        }
64        None => crate::process_control::resolve_binary(name).ok(),
65    }
66}
67
68pub fn probe_engine(engine: VoiceEngine, hardware: &Hardware) -> EngineProbe {
69    let (label, variable, binary, install) = match engine {
70        VoiceEngine::WhisperCpp => (
71            "whisper.cpp",
72            WHISPER_SERVER_ENV,
73            "whisper-server",
74            "brew install whisper-cpp (or build whisper.cpp and set RIGSPARK_WHISPER_SERVER)",
75        ),
76        VoiceEngine::MlxAudio => (
77            "mlx-audio",
78            MLX_AUDIO_SERVER_ENV,
79            "mlx_audio.server",
80            "pip install -U mlx-audio (Apple Silicon only)",
81        ),
82    };
83    let path = binary_from(std::env::var_os(variable), binary);
84    EngineProbe {
85        engine: engine.name(),
86        label,
87        installed: path.is_some(),
88        supported: engine.supports(hardware),
89        path,
90        install,
91    }
92}
93
94#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
95#[serde(rename_all = "lowercase")]
96pub enum SlotState {
97    Starting,
98    Running,
99    Failed,
100}
101
102/// How requests reach a running voice server.
103#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
104#[serde(rename_all = "kebab-case")]
105pub enum Protocol {
106    /// whisper.cpp's `/inference` multipart endpoint.
107    WhisperCpp,
108    /// OpenAI's `/v1/audio/speech` and `/v1/audio/transcriptions`.
109    OpenAi,
110}
111
112struct Slot {
113    model: String,
114    engine: &'static str,
115    state: SlotState,
116    detail: String,
117    progress: Option<(u64, u64)>,
118    endpoint: Option<String>,
119    /// The `model` value sent to an OpenAI-style server.
120    request_model: String,
121    protocol: Protocol,
122    owned: bool,
123    child: Option<tokio::process::Child>,
124    cancel: CancellationToken,
125    generation: u64,
126}
127
128#[derive(Debug, Clone, Serialize)]
129#[serde(rename_all = "camelCase")]
130pub struct SlotView {
131    pub kind: &'static str,
132    pub model: String,
133    pub engine: &'static str,
134    pub state: SlotState,
135    pub detail: String,
136    pub progress: Option<(u64, u64)>,
137    pub endpoint: Option<String>,
138    pub owned: bool,
139}
140
141/// One speak and one listen server at a time, each owned or attached.
142pub struct VoiceService {
143    cache: PathBuf,
144    slots: Mutex<BTreeMap<&'static str, Slot>>,
145    client: reqwest::Client,
146    generation: std::sync::atomic::AtomicU64,
147}
148
149#[derive(Debug, Clone)]
150pub struct SpeakRequest {
151    pub text: String,
152    pub voice: Option<String>,
153    pub speed: Option<f64>,
154}
155
156fn kind_key(kind: VoiceKind) -> &'static str {
157    kind.name()
158}
159
160fn token(value: &str, maximum: usize) -> bool {
161    !value.is_empty()
162        && value.len() <= maximum
163        && value
164            .bytes()
165            .all(|byte| byte.is_ascii_alphanumeric() || b"._-".contains(&byte))
166}
167
168/// A RIFF/WAVE header; every audio body in and out must start with one.
169pub fn is_wav(bytes: &[u8]) -> bool {
170    bytes.len() >= 12 && &bytes[..4] == b"RIFF" && &bytes[8..12] == b"WAVE"
171}
172
173fn free_port() -> Result<u16, VoiceError> {
174    let listener = std::net::TcpListener::bind((std::net::Ipv4Addr::LOCALHOST, 0))
175        .map_err(|_| VoiceError("no free local port".into()))?;
176    listener
177        .local_addr()
178        .map(|address| address.port())
179        .map_err(|_| VoiceError("no free local port".into()))
180}
181
182fn multipart(boundary: &str, fields: &[(&str, &str)], audio: &[u8]) -> Vec<u8> {
183    let mut body = Vec::with_capacity(audio.len() + 512);
184    for (name, value) in fields {
185        body.extend_from_slice(
186            format!(
187                "--{boundary}\r\nContent-Disposition: form-data; name=\"{name}\"\r\n\r\n{value}\r\n"
188            )
189            .as_bytes(),
190        );
191    }
192    body.extend_from_slice(
193        format!(
194            "--{boundary}\r\nContent-Disposition: form-data; name=\"file\"; filename=\"audio.wav\"\r\nContent-Type: audio/wav\r\n\r\n"
195        )
196        .as_bytes(),
197    );
198    body.extend_from_slice(audio);
199    body.extend_from_slice(format!("\r\n--{boundary}--\r\n").as_bytes());
200    body
201}
202
203async fn bounded(response: reqwest::Response, maximum: usize) -> Result<Vec<u8>, VoiceError> {
204    if !response.status().is_success() {
205        return fail(format!(
206            "voice server answered {}",
207            response.status().as_u16()
208        ));
209    }
210    if response
211        .content_length()
212        .is_some_and(|length| length > maximum as u64)
213    {
214        return fail("voice server reply exceeds the size limit");
215    }
216    let mut response = response;
217    let mut bytes = Vec::new();
218    while let Some(chunk) = response
219        .chunk()
220        .await
221        .map_err(|_| VoiceError("voice server reply was interrupted".into()))?
222    {
223        if chunk.len() > maximum - bytes.len() {
224            return fail("voice server reply exceeds the size limit");
225        }
226        bytes.extend_from_slice(&chunk);
227    }
228    Ok(bytes)
229}
230
231impl VoiceService {
232    pub fn new(home: &Path) -> Result<Self, VoiceError> {
233        let client = reqwest::Client::builder()
234            .no_proxy()
235            .redirect(reqwest::redirect::Policy::none())
236            .connect_timeout(Duration::from_secs(5))
237            .build()
238            .map_err(|_| VoiceError("voice client setup failed".into()))?;
239        Ok(Self {
240            cache: home.join("cache"),
241            slots: Mutex::new(BTreeMap::new()),
242            client,
243            generation: std::sync::atomic::AtomicU64::new(0),
244        })
245    }
246
247    fn lock(&self) -> Result<std::sync::MutexGuard<'_, BTreeMap<&'static str, Slot>>, VoiceError> {
248        self.slots
249            .lock()
250            .map_err(|_| VoiceError("voice state unavailable".into()))
251    }
252
253    pub fn status(&self) -> Vec<SlotView> {
254        let Ok(slots) = self.lock() else {
255            return Vec::new();
256        };
257        slots
258            .iter()
259            .map(|(kind, slot)| SlotView {
260                kind,
261                model: slot.model.clone(),
262                engine: slot.engine,
263                state: slot.state,
264                detail: slot.detail.clone(),
265                progress: slot.progress,
266                endpoint: slot.endpoint.clone(),
267                owned: slot.owned,
268            })
269            .collect()
270    }
271
272    fn update(&self, kind: &'static str, generation: u64, change: impl FnOnce(&mut Slot)) {
273        if let Ok(mut slots) = self.lock()
274            && let Some(slot) = slots
275                .get_mut(kind)
276                .filter(|slot| slot.generation == generation)
277        {
278            change(slot);
279        }
280    }
281
282    /// Replaces the slot for `kind`, stopping whatever ran there before.
283    fn claim(&self, kind: VoiceKind, slot: Slot) -> Result<(), VoiceError> {
284        let previous = self.lock()?.insert(kind_key(kind), slot);
285        if let Some(mut previous) = previous {
286            previous.cancel.cancel();
287            if let Some(child) = previous.child.as_mut() {
288                let _ = child.start_kill();
289            }
290        }
291        Ok(())
292    }
293
294    pub fn stop(&self, kind: VoiceKind) -> bool {
295        let Ok(mut slots) = self.lock() else {
296            return false;
297        };
298        match slots.remove(kind_key(kind)) {
299            Some(mut slot) => {
300                slot.cancel.cancel();
301                if let Some(child) = slot.child.as_mut() {
302                    let _ = child.start_kill();
303                }
304                true
305            }
306            None => false,
307        }
308    }
309
310    pub fn shutdown(&self) {
311        for kind in [VoiceKind::Speak, VoiceKind::Listen] {
312            self.stop(kind);
313        }
314    }
315
316    /// Uses a voice server someone else runs on loopback, speaking OpenAI's audio API.
317    pub async fn attach(
318        &self,
319        kind: VoiceKind,
320        endpoint: &str,
321        model: &str,
322    ) -> Result<(), VoiceError> {
323        let url = crate::state::loopback(endpoint).map_err(|_| {
324            VoiceError("endpoint must be http://127.0.0.1:<port> or localhost".into())
325        })?;
326        if url.query().is_some() || url.fragment().is_some() || endpoint.len() > 256 {
327            return fail("endpoint must be a plain loopback base URL");
328        }
329        if model.len() > 200
330            || !model.split('/').all(|part| token(part, 100))
331            || model.split('/').count() > 2
332        {
333            return fail("model must be a repository id or a short name");
334        }
335        let base = endpoint.trim_end_matches('/').to_owned();
336        let response = tokio::time::timeout(
337            Duration::from_secs(5),
338            self.client.get(format!("{base}/v1/models")).send(),
339        )
340        .await
341        .map_err(|_| VoiceError("voice server did not answer".into()))?
342        .map_err(|_| VoiceError("voice server is not reachable".into()))?;
343        if !response.status().is_success() {
344            return fail("voice server does not offer /v1/models");
345        }
346        let generation = self.next_generation();
347        self.claim(
348            kind,
349            Slot {
350                model: model.into(),
351                engine: "attached",
352                state: SlotState::Running,
353                detail: "Attached server".into(),
354                progress: None,
355                endpoint: Some(base),
356                request_model: model.into(),
357                protocol: Protocol::OpenAi,
358                owned: false,
359                child: None,
360                cancel: CancellationToken::new(),
361                generation,
362            },
363        )
364    }
365
366    fn next_generation(&self) -> u64 {
367        self.generation
368            .fetch_add(1, std::sync::atomic::Ordering::Relaxed)
369            + 1
370    }
371
372    /// Starts `model`'s engine in the background; `status` reports its progress.
373    pub fn start(
374        self: &Arc<Self>,
375        model: &VoiceModel,
376        hardware: &Hardware,
377        parent: &CancellationToken,
378    ) -> Result<(), VoiceError> {
379        let probe = probe_engine(model.engine, hardware);
380        if !probe.supported {
381            return fail(format!("{} does not run on this machine", probe.label));
382        }
383        let Some(binary) = probe.path.clone() else {
384            return fail(format!(
385                "{} is not installed: {}",
386                probe.label, probe.install
387            ));
388        };
389        self.begin(model, binary, parent)
390    }
391
392    /// Claims `model`'s slot and launches `binary` for it in the background.
393    fn begin(
394        self: &Arc<Self>,
395        model: &VoiceModel,
396        binary: PathBuf,
397        parent: &CancellationToken,
398    ) -> Result<(), VoiceError> {
399        let kind = kind_key(model.kind);
400        let generation = self.next_generation();
401        let cancel = parent.child_token();
402        self.claim(
403            model.kind,
404            Slot {
405                model: model.id.clone(),
406                engine: model.engine.name(),
407                state: SlotState::Starting,
408                detail: "Starting".into(),
409                progress: None,
410                endpoint: None,
411                request_model: model
412                    .engine_model
413                    .clone()
414                    .unwrap_or_else(|| model.id.clone()),
415                protocol: match model.engine {
416                    VoiceEngine::WhisperCpp => Protocol::WhisperCpp,
417                    VoiceEngine::MlxAudio => Protocol::OpenAi,
418                },
419                owned: true,
420                child: None,
421                cancel: cancel.clone(),
422                generation,
423            },
424        )?;
425        let service = self.clone();
426        let model = model.clone();
427        tokio::spawn(async move {
428            let outcome = service
429                .launch(&model, &binary, kind, generation, &cancel)
430                .await;
431            service.update(kind, generation, |slot| match outcome {
432                Ok(()) => {}
433                Err(error) => {
434                    slot.state = SlotState::Failed;
435                    slot.detail = error.0;
436                    if let Some(child) = slot.child.as_mut() {
437                        let _ = child.start_kill();
438                    }
439                    slot.child = None;
440                }
441            });
442        });
443        Ok(())
444    }
445
446    async fn launch(
447        self: &Arc<Self>,
448        model: &VoiceModel,
449        binary: &Path,
450        kind: &'static str,
451        generation: u64,
452        cancel: &CancellationToken,
453    ) -> Result<(), VoiceError> {
454        let port = free_port()?;
455        let base = format!("http://127.0.0.1:{port}");
456        let mut command = tokio::process::Command::new(binary);
457        match model.engine {
458            VoiceEngine::WhisperCpp => {
459                let file = model
460                    .file
461                    .as_ref()
462                    .ok_or_else(|| VoiceError("missing pinned file".into()))?;
463                self.update(kind, generation, |slot| {
464                    slot.detail = format!("Downloading {}", file.file);
465                });
466                let artifact = Artifact {
467                    backend: "whispercpp".into(),
468                    repo: file.repo.clone(),
469                    revision: file.revision.clone(),
470                    file: file.file.clone(),
471                    sha256: file.sha256.clone(),
472                    bytes: file.bytes,
473                };
474                let progress = {
475                    let service = self.clone();
476                    move |done: u64, total: u64, _: &str| {
477                        service
478                            .update(kind, generation, |slot| slot.progress = Some((done, total)));
479                    }
480                };
481                let acquisition = Acquisition::new(&self.cache)
482                    .map_err(VoiceError)?
483                    .with_progress(progress);
484                let transport = HfTransport::new().map_err(VoiceError)?;
485                let size = if file.exact_size {
486                    SizePolicy::Exact
487                } else {
488                    SizePolicy::Ceiling
489                };
490                let acquired = acquisition
491                    .acquire_sized(&artifact, size, &transport, cancel)
492                    .await
493                    .map_err(|error| VoiceError(format!("download failed: {error}")))?;
494                command.arg("-m").arg(&acquired.path).args([
495                    "--host",
496                    "127.0.0.1",
497                    "--port",
498                    &port.to_string(),
499                ]);
500            }
501            VoiceEngine::MlxAudio => {
502                command.args(["--host", "127.0.0.1", "--port", &port.to_string()]);
503            }
504        }
505        command
506            .stdin(std::process::Stdio::null())
507            .stdout(std::process::Stdio::null())
508            .stderr(std::process::Stdio::null())
509            .kill_on_drop(true);
510        let child = command
511            .spawn()
512            .map_err(|_| VoiceError("voice server failed to start".into()))?;
513        self.update(kind, generation, |slot| {
514            slot.child = Some(child);
515            slot.progress = None;
516            slot.detail = "Loading the model".into();
517        });
518        let probe = match model.engine {
519            VoiceEngine::WhisperCpp => format!("{base}/"),
520            VoiceEngine::MlxAudio => format!("{base}/v1/models"),
521        };
522        self.wait_ready(&probe, kind, generation, cancel).await?;
523        if model.engine == VoiceEngine::MlxAudio {
524            self.update(kind, generation, |slot| {
525                slot.detail = "Fetching and loading weights (mlx-audio)".into();
526            });
527            let repository = model.engine_model.clone().unwrap_or_default();
528            let response = tokio::select! {
529                biased;
530                _ = cancel.cancelled() => return fail("cancelled"),
531                response = self
532                    .client
533                    .post(format!("{base}/v1/models"))
534                    .query(&[("model_name", repository.as_str())])
535                    .timeout(Duration::from_secs(1800))
536                    .send() => response,
537            };
538            match response {
539                Ok(response) if response.status().is_success() => {}
540                _ => return fail("mlx-audio could not load the model"),
541            }
542        }
543        self.update(kind, generation, |slot| {
544            slot.state = SlotState::Running;
545            slot.detail = "Ready".into();
546            slot.endpoint = Some(base.clone());
547        });
548        Ok(())
549    }
550
551    async fn wait_ready(
552        &self,
553        url: &str,
554        kind: &'static str,
555        generation: u64,
556        cancel: &CancellationToken,
557    ) -> Result<(), VoiceError> {
558        let deadline = tokio::time::Instant::now() + Duration::from_secs(180);
559        loop {
560            if cancel.is_cancelled() {
561                return fail("cancelled");
562            }
563            let exited = {
564                let mut slots = self.lock()?;
565                match slots
566                    .get_mut(kind)
567                    .filter(|slot| slot.generation == generation)
568                {
569                    Some(slot) => slot
570                        .child
571                        .as_mut()
572                        .is_some_and(|child| matches!(child.try_wait(), Ok(Some(_)))),
573                    None => return fail("replaced"),
574                }
575            };
576            if exited {
577                return fail("voice server exited while starting");
578            }
579            if let Ok(Ok(response)) =
580                tokio::time::timeout(Duration::from_secs(2), self.client.get(url).send()).await
581                && response.status().is_success()
582            {
583                return Ok(());
584            }
585            if tokio::time::Instant::now() > deadline {
586                return fail("voice server did not become ready");
587            }
588            tokio::time::sleep(Duration::from_millis(400)).await;
589        }
590    }
591
592    fn running(
593        &self,
594        kind: VoiceKind,
595    ) -> Result<(String, String, Protocol, &'static str), VoiceError> {
596        let slots = self.lock()?;
597        let slot = slots
598            .get(kind_key(kind))
599            .filter(|slot| slot.state == SlotState::Running)
600            .ok_or_else(|| {
601                VoiceError(format!(
602                    "no {} model is running; start one from Models › Voice",
603                    kind.name()
604                ))
605            })?;
606        Ok((
607            slot.endpoint.clone().unwrap_or_default(),
608            slot.request_model.clone(),
609            slot.protocol,
610            slot.engine,
611        ))
612    }
613
614    /// Text to WAV with the running speak model.
615    pub async fn speak(
616        &self,
617        request: &SpeakRequest,
618        catalog: &VoiceCatalog,
619        cancel: &CancellationToken,
620    ) -> Result<Vec<u8>, VoiceError> {
621        let text = request.text.trim();
622        if text.is_empty() || text.chars().count() > MAX_SPEAK_CHARS {
623            return fail(format!("text must be 1 to {MAX_SPEAK_CHARS} characters"));
624        }
625        if request
626            .speed
627            .is_some_and(|speed| !(0.5..=2.0).contains(&speed) || !speed.is_finite())
628        {
629            return fail("speed must be between 0.5 and 2.0");
630        }
631        if request
632            .voice
633            .as_deref()
634            .is_some_and(|voice| !token(voice, 64))
635        {
636            return fail("invalid voice");
637        }
638        let (endpoint, model, _, engine) = self.running(VoiceKind::Speak)?;
639        let mut body = json!({
640            "model": model,
641            "input": text,
642            "response_format": "wav",
643            "speed": request.speed.unwrap_or(1.0),
644        });
645        if let Some(voice) = &request.voice {
646            body["voice"] = json!(voice);
647            // mlx-audio picks Kokoro's language from lang_code, which its voice ids encode.
648            if engine == "mlx-audio"
649                && let Some(entry) = catalog
650                    .models
651                    .iter()
652                    .find(|entry| entry.engine_model.as_deref() == Some(model.as_str()))
653                    .and_then(|entry| entry.voice(voice))
654            {
655                body["lang_code"] = json!(entry.lang);
656            }
657        }
658        let send = self
659            .client
660            .post(format!("{endpoint}/v1/audio/speech"))
661            .json(&body)
662            .timeout(Duration::from_secs(600))
663            .send();
664        let response = tokio::select! {
665            biased;
666            _ = cancel.cancelled() => return fail("cancelled"),
667            response = send => response.map_err(|_| VoiceError("speech request failed".into()))?,
668        };
669        let audio = bounded(response, MAX_AUDIO_BYTES).await?;
670        if !is_wav(&audio) {
671            return fail("voice server did not return WAV audio");
672        }
673        Ok(audio)
674    }
675
676    /// WAV to text with the running listen model.
677    pub async fn transcribe(
678        &self,
679        audio: &[u8],
680        language: Option<&str>,
681        cancel: &CancellationToken,
682    ) -> Result<String, VoiceError> {
683        if !is_wav(audio) || audio.len() > MAX_AUDIO_BYTES {
684            return fail("audio must be a WAV file under 48 MiB");
685        }
686        if language.is_some_and(|language| !token(language, 16)) {
687            return fail("invalid language");
688        }
689        let (endpoint, model, protocol, _) = self.running(VoiceKind::Listen)?;
690        let boundary = format!("rigspark{}", uuid::Uuid::new_v4().simple());
691        let mut fields: Vec<(&str, &str)> = Vec::new();
692        let url = match protocol {
693            Protocol::WhisperCpp => {
694                fields.extend([("response_format", "json"), ("temperature", "0.0")]);
695                format!("{endpoint}/inference")
696            }
697            Protocol::OpenAi => {
698                fields.extend([("model", model.as_str()), ("response_format", "json")]);
699                format!("{endpoint}/v1/audio/transcriptions")
700            }
701        };
702        if let Some(language) = language {
703            fields.push(("language", language));
704        }
705        let body = multipart(&boundary, &fields, audio);
706        let send = self
707            .client
708            .post(url)
709            .header(
710                reqwest::header::CONTENT_TYPE,
711                format!("multipart/form-data; boundary={boundary}"),
712            )
713            .body(body)
714            .timeout(Duration::from_secs(600))
715            .send();
716        let response = tokio::select! {
717            biased;
718            _ = cancel.cancelled() => return fail("cancelled"),
719            response = send => response.map_err(|_| VoiceError("transcription request failed".into()))?,
720        };
721        let raw = bounded(response, MAX_TRANSCRIPT_BYTES).await?;
722        let value: Value = serde_json::from_slice(&raw)
723            .map_err(|_| VoiceError("voice server returned an invalid transcript".into()))?;
724        let text = value
725            .get("text")
726            .and_then(Value::as_str)
727            .ok_or_else(|| VoiceError("voice server returned no text".into()))?;
728        Ok(rigspark_core::reports::strip_control(text.trim()))
729    }
730}
731
732impl Drop for VoiceService {
733    fn drop(&mut self) {
734        self.shutdown();
735    }
736}
737
738#[cfg(test)]
739mod tests {
740    use super::*;
741    use tokio::io::{AsyncReadExt, AsyncWriteExt};
742
743    fn wav(samples: usize) -> Vec<u8> {
744        let data = samples * 2;
745        let mut bytes = Vec::new();
746        bytes.extend_from_slice(b"RIFF");
747        bytes.extend_from_slice(&(36 + data as u32).to_le_bytes());
748        bytes.extend_from_slice(b"WAVEfmt ");
749        bytes.extend_from_slice(&16u32.to_le_bytes());
750        bytes.extend_from_slice(&1u16.to_le_bytes());
751        bytes.extend_from_slice(&1u16.to_le_bytes());
752        bytes.extend_from_slice(&16000u32.to_le_bytes());
753        bytes.extend_from_slice(&32000u32.to_le_bytes());
754        bytes.extend_from_slice(&2u16.to_le_bytes());
755        bytes.extend_from_slice(&16u16.to_le_bytes());
756        bytes.extend_from_slice(b"data");
757        bytes.extend_from_slice(&(data as u32).to_le_bytes());
758        bytes.resize(44 + data, 0);
759        bytes
760    }
761
762    /// A loopback server that answers every request with `reply` and records requests.
763    async fn server(
764        reply: Vec<u8>,
765        content_type: &'static str,
766    ) -> (String, Arc<Mutex<Vec<Vec<u8>>>>) {
767        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
768        let address = listener.local_addr().unwrap();
769        let seen = Arc::new(Mutex::new(Vec::new()));
770        let record = seen.clone();
771        tokio::spawn(async move {
772            loop {
773                let Ok((mut socket, _)) = listener.accept().await else {
774                    return;
775                };
776                let reply = reply.clone();
777                let record = record.clone();
778                tokio::spawn(async move {
779                    let mut request = Vec::new();
780                    let mut buffer = vec![0u8; 65536];
781                    loop {
782                        let read = socket.read(&mut buffer).await.unwrap_or(0);
783                        if read == 0 {
784                            break;
785                        }
786                        request.extend_from_slice(&buffer[..read]);
787                        let text = String::from_utf8_lossy(&request);
788                        if let Some(end) = text.find("\r\n\r\n") {
789                            let length = text[..end]
790                                .lines()
791                                .find_map(|line| {
792                                    line.to_ascii_lowercase()
793                                        .strip_prefix("content-length:")
794                                        .map(|value| value.trim().parse::<usize>().unwrap_or(0))
795                                })
796                                .unwrap_or(0);
797                            if request.len() >= end + 4 + length {
798                                break;
799                            }
800                        }
801                    }
802                    record.lock().unwrap().push(request);
803                    let head = format!(
804                        "HTTP/1.1 200 OK\r\nContent-Type: {content_type}\r\nContent-Length: {}\r\nConnection: close\r\n\r\n",
805                        reply.len()
806                    );
807                    let _ = socket.write_all(head.as_bytes()).await;
808                    let _ = socket.write_all(&reply).await;
809                });
810            }
811        });
812        (format!("http://{address}"), seen)
813    }
814
815    fn catalog() -> VoiceCatalog {
816        VoiceCatalog::bundled().unwrap()
817    }
818    fn request(text: &str) -> SpeakRequest {
819        SpeakRequest {
820            text: text.into(),
821            voice: Some("af_heart".into()),
822            speed: Some(1.0),
823        }
824    }
825
826    #[test]
827    fn wav_headers_and_multipart_bodies_are_exact() {
828        assert!(is_wav(&wav(4)));
829        assert!(!is_wav(b"RIFF\0\0\0\0WAVX"));
830        assert!(!is_wav(b"RIFF"));
831        let body = multipart("b", &[("model", "m")], b"AUDIO");
832        assert_eq!(
833            String::from_utf8(body).unwrap(),
834            "--b\r\nContent-Disposition: form-data; name=\"model\"\r\n\r\nm\r\n--b\r\nContent-Disposition: form-data; name=\"file\"; filename=\"audio.wav\"\r\nContent-Type: audio/wav\r\n\r\nAUDIO\r\n--b--\r\n"
835        );
836    }
837
838    #[tokio::test]
839    async fn nothing_runs_until_a_model_is_started_or_attached() {
840        let home = tempfile::tempdir().unwrap();
841        let service = VoiceService::new(home.path()).unwrap();
842        assert!(service.status().is_empty());
843        let error = service
844            .speak(&request("hi"), &catalog(), &CancellationToken::new())
845            .await
846            .unwrap_err();
847        assert!(error.0.contains("no speak model is running"), "{error}");
848        assert!(
849            service
850                .transcribe(&wav(4), None, &CancellationToken::new())
851                .await
852                .unwrap_err()
853                .0
854                .contains("no listen model")
855        );
856        assert!(!service.stop(VoiceKind::Speak));
857    }
858
859    #[tokio::test]
860    async fn attached_servers_must_be_plain_loopback_openai_audio() {
861        let home = tempfile::tempdir().unwrap();
862        let service = VoiceService::new(home.path()).unwrap();
863        for endpoint in [
864            "https://127.0.0.1:1",
865            "http://example.com:8000",
866            "http://user@127.0.0.1:1",
867            "http://127.0.0.1:1/?x=1",
868        ] {
869            assert!(
870                service
871                    .attach(VoiceKind::Speak, endpoint, "kokoro")
872                    .await
873                    .is_err(),
874                "{endpoint}"
875            );
876        }
877        let (base, _) = server(b"{\"data\":[]}".to_vec(), "application/json").await;
878        for model in ["", "a/b/c", "bad model", &"m".repeat(101)] {
879            assert!(
880                service
881                    .attach(VoiceKind::Speak, &base, model)
882                    .await
883                    .is_err(),
884                "{model:?}"
885            );
886        }
887        service
888            .attach(
889                VoiceKind::Speak,
890                &format!("{base}/"),
891                "mlx-community/Kokoro-82M-bf16",
892            )
893            .await
894            .unwrap();
895        let status = service.status();
896        assert_eq!(status.len(), 1);
897        assert_eq!(status[0].state, SlotState::Running);
898        assert_eq!(status[0].endpoint.as_deref(), Some(base.as_str()));
899        assert!(!status[0].owned);
900        assert!(service.stop(VoiceKind::Speak));
901        assert!(service.status().is_empty());
902    }
903
904    #[tokio::test]
905    async fn speech_requests_are_bounded_and_replies_must_be_wav() {
906        let home = tempfile::tempdir().unwrap();
907        let service = VoiceService::new(home.path()).unwrap();
908        let (base, seen) = server(wav(8), "audio/wav").await;
909        service
910            .attach(VoiceKind::Speak, &base, "mlx-community/Kokoro-82M-bf16")
911            .await
912            .unwrap();
913        let cancel = CancellationToken::new();
914        let audio = service
915            .speak(&request("Hello there"), &catalog(), &cancel)
916            .await
917            .unwrap();
918        assert_eq!(audio, wav(8));
919        let sent =
920            String::from_utf8_lossy(&seen.lock().unwrap().last().unwrap().clone()).into_owned();
921        assert!(sent.starts_with("POST /v1/audio/speech"), "{sent}");
922        assert!(sent.contains("\"input\":\"Hello there\""), "{sent}");
923        assert!(sent.contains("\"voice\":\"af_heart\""), "{sent}");
924        assert!(sent.contains("\"response_format\":\"wav\""), "{sent}");
925        // Attached servers get no mlx-audio language hint.
926        assert!(!sent.contains("lang_code"), "{sent}");
927        for (text, voice, speed) in [
928            ("", None, None),
929            ("   ", None, None),
930            (&"x".repeat(MAX_SPEAK_CHARS + 1)[..], None, None),
931            ("hi", Some("bad voice"), None),
932            ("hi", None, Some(0.4)),
933            ("hi", None, Some(2.1)),
934            ("hi", None, Some(f64::NAN)),
935        ] {
936            let request = SpeakRequest {
937                text: text.into(),
938                voice: voice.map(Into::into),
939                speed,
940            };
941            assert!(
942                service.speak(&request, &catalog(), &cancel).await.is_err(),
943                "{request:?}"
944            );
945        }
946        assert!(
947            service
948                .speak(&request(&"x".repeat(MAX_SPEAK_CHARS)), &catalog(), &cancel)
949                .await
950                .is_ok()
951        );
952        let (base, _) = server(b"not audio".to_vec(), "audio/wav").await;
953        service
954            .attach(VoiceKind::Speak, &base, "kokoro")
955            .await
956            .unwrap();
957        assert_eq!(
958            service
959                .speak(&request("hi"), &catalog(), &cancel)
960                .await
961                .unwrap_err()
962                .0,
963            "voice server did not return WAV audio"
964        );
965    }
966
967    #[tokio::test]
968    async fn transcription_posts_multipart_wav_and_reads_text() {
969        let home = tempfile::tempdir().unwrap();
970        let service = VoiceService::new(home.path()).unwrap();
971        let (base, seen) = server(
972            b"{\"text\":\"  hello \\u001b[31mworld \"}".to_vec(),
973            "application/json",
974        )
975        .await;
976        service
977            .attach(VoiceKind::Listen, &base, "whisper-large-v3-turbo")
978            .await
979            .unwrap();
980        let cancel = CancellationToken::new();
981        let text = service
982            .transcribe(&wav(16), Some("en"), &cancel)
983            .await
984            .unwrap();
985        assert_eq!(text, "hello world");
986        let sent = seen.lock().unwrap().last().unwrap().clone();
987        let head = String::from_utf8_lossy(&sent);
988        assert!(head.starts_with("POST /v1/audio/transcriptions"), "{head}");
989        assert!(
990            head.contains("multipart/form-data; boundary=rigspark"),
991            "{head}"
992        );
993        assert!(
994            head.contains("name=\"model\"\r\n\r\nwhisper-large-v3-turbo"),
995            "{head}"
996        );
997        assert!(head.contains("name=\"language\"\r\n\r\nen"), "{head}");
998        assert!(sent.windows(wav(16).len()).any(|window| window == wav(16)));
999        assert!(
1000            service
1001                .transcribe(b"not audio", None, &cancel)
1002                .await
1003                .is_err()
1004        );
1005        assert!(
1006            service
1007                .transcribe(&wav(4), Some("e n"), &cancel)
1008                .await
1009                .is_err()
1010        );
1011        let (base, _) = server(b"{\"segments\":[]}".to_vec(), "application/json").await;
1012        service
1013            .attach(VoiceKind::Listen, &base, "whisper")
1014            .await
1015            .unwrap();
1016        assert_eq!(
1017            service
1018                .transcribe(&wav(4), None, &cancel)
1019                .await
1020                .unwrap_err()
1021                .0,
1022            "voice server returned no text"
1023        );
1024    }
1025
1026    #[tokio::test]
1027    async fn starting_needs_an_installed_supported_engine() {
1028        let home = tempfile::tempdir().unwrap();
1029        let service = Arc::new(VoiceService::new(home.path()).unwrap());
1030        let linux = Hardware {
1031            arch: rigspark_core::sizing::CpuArch::X64,
1032            platform: rigspark_core::sizing::Platform::Linux,
1033            total_ram_bytes: 16e9,
1034            free_ram_bytes: 16e9,
1035            free_disk_bytes: 1e12,
1036            gpu: Vec::new(),
1037            unified_memory: None,
1038        };
1039        let kokoro = catalog().resolve("speak").unwrap().clone();
1040        let error = service
1041            .start(&kokoro, &linux, &CancellationToken::new())
1042            .unwrap_err();
1043        assert_eq!(error.0, "mlx-audio does not run on this machine");
1044        assert!(service.status().is_empty());
1045        let probe = probe_engine(VoiceEngine::MlxAudio, &linux);
1046        assert!(!probe.supported);
1047        assert!(probe.install.contains("mlx-audio"));
1048    }
1049
1050    #[test]
1051    fn an_explicit_engine_path_must_be_an_absolute_file() {
1052        let directory = tempfile::tempdir().unwrap();
1053        let binary = directory.path().join("whisper-server");
1054        std::fs::write(&binary, "").unwrap();
1055        assert_eq!(
1056            binary_from(Some(binary.clone().into()), "unused"),
1057            Some(binary.canonicalize().unwrap())
1058        );
1059        assert_eq!(binary_from(Some("whisper-server".into()), "unused"), None);
1060        assert_eq!(binary_from(Some(directory.path().into()), "unused"), None);
1061    }
1062
1063    fn slot(kind: VoiceKind, engine: &'static str, endpoint: &str, generation: u64) -> Slot {
1064        Slot {
1065            model: kind.name().into(),
1066            engine,
1067            state: SlotState::Running,
1068            detail: "Ready".into(),
1069            progress: None,
1070            endpoint: Some(endpoint.into()),
1071            request_model: "mlx-community/Kokoro-82M-bf16".into(),
1072            protocol: Protocol::OpenAi,
1073            owned: false,
1074            child: None,
1075            cancel: CancellationToken::new(),
1076            generation,
1077        }
1078    }
1079
1080    #[test]
1081    fn limits_errors_ports_and_generations_are_exact() {
1082        assert_eq!(MAX_AUDIO_BYTES, 50_331_648);
1083        assert_eq!(MAX_TRANSCRIPT_BYTES, 262_144);
1084        assert_eq!(VoiceError("boom".into()).to_string(), "boom");
1085        let port = free_port().unwrap();
1086        assert!(port > 1);
1087        std::net::TcpListener::bind((std::net::Ipv4Addr::LOCALHOST, port)).unwrap();
1088        let home = tempfile::tempdir().unwrap();
1089        let service = VoiceService::new(home.path()).unwrap();
1090        assert_eq!(service.next_generation(), 1);
1091        assert_eq!(service.next_generation(), 2);
1092        assert_eq!(service.next_generation(), 3);
1093    }
1094
1095    #[test]
1096    fn updates_reach_only_the_current_generation() {
1097        let home = tempfile::tempdir().unwrap();
1098        let service = VoiceService::new(home.path()).unwrap();
1099        service
1100            .claim(
1101                VoiceKind::Listen,
1102                slot(VoiceKind::Listen, "attached", "http://127.0.0.1:9", 7),
1103            )
1104            .unwrap();
1105        service.update("listen", 6, |slot| slot.detail = "stale".into());
1106        assert_eq!(service.status()[0].detail, "Ready");
1107        service.update("listen", 7, |slot| slot.detail = "fresh".into());
1108        let status = service.status();
1109        assert_eq!(status[0].detail, "fresh");
1110        assert_eq!(status[0].kind, "listen");
1111        service
1112            .claim(
1113                VoiceKind::Speak,
1114                slot(VoiceKind::Speak, "attached", "http://127.0.0.1:9", 8),
1115            )
1116            .unwrap();
1117        assert_eq!(
1118            service
1119                .status()
1120                .iter()
1121                .map(|view| view.kind)
1122                .collect::<Vec<_>>(),
1123            ["listen", "speak"]
1124        );
1125        service.shutdown();
1126        assert!(service.status().is_empty());
1127    }
1128
1129    fn response(chunks: Vec<&'static [u8]>, length: bool) -> reqwest::Response {
1130        let total: usize = chunks.iter().map(|chunk| chunk.len()).sum();
1131        let body = reqwest::Body::wrap_stream(futures_util::stream::iter(
1132            chunks
1133                .into_iter()
1134                .map(|chunk| Ok::<_, std::io::Error>(bytes::Bytes::from_static(chunk))),
1135        ));
1136        let mut builder = http::Response::builder().status(200);
1137        if length {
1138            builder = builder.header("content-length", total.to_string());
1139        }
1140        reqwest::Response::from(builder.body(body).unwrap())
1141    }
1142
1143    #[tokio::test]
1144    async fn replies_are_bounded_by_length_and_by_streamed_bytes() {
1145        assert_eq!(
1146            bounded(response(vec![b"abcd"], true), 4).await.unwrap(),
1147            b"abcd"
1148        );
1149        assert!(bounded(response(vec![b"abcde"], true), 4).await.is_err());
1150        assert_eq!(
1151            bounded(response(vec![b"ab", b"cd"], false), 4)
1152                .await
1153                .unwrap(),
1154            b"abcd"
1155        );
1156        assert!(
1157            bounded(response(vec![b"ab", b"cde"], false), 4)
1158                .await
1159                .is_err()
1160        );
1161        assert!(
1162            bounded(response(vec![b"abc", b"de"], false), 4)
1163                .await
1164                .is_err()
1165        );
1166    }
1167
1168    /// Sends `raw` verbatim to the first connection, then closes it.
1169    async fn raw_reply(raw: &'static [u8]) -> reqwest::Response {
1170        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
1171        let address = listener.local_addr().unwrap();
1172        tokio::spawn(async move {
1173            let (mut socket, _) = listener.accept().await.unwrap();
1174            let mut buffer = vec![0u8; 4096];
1175            let _ = socket.read(&mut buffer).await;
1176            let _ = socket.write_all(raw).await;
1177        });
1178        reqwest::get(format!("http://{address}/")).await.unwrap()
1179    }
1180
1181    #[tokio::test]
1182    async fn a_declared_length_is_checked_before_reading() {
1183        let exact =
1184            raw_reply(b"HTTP/1.1 200 OK\r\nContent-Length: 4\r\nConnection: close\r\n\r\nabcd")
1185                .await;
1186        assert_eq!(bounded(exact, 4).await.unwrap(), b"abcd");
1187        // Declares 5 bytes but sends 2: refused on the header, not on the cut-off body.
1188        let declared =
1189            raw_reply(b"HTTP/1.1 200 OK\r\nContent-Length: 5\r\nConnection: close\r\n\r\nab").await;
1190        assert_eq!(
1191            bounded(declared, 4).await.unwrap_err().0,
1192            "voice server reply exceeds the size limit"
1193        );
1194    }
1195
1196    #[tokio::test]
1197    async fn attach_refuses_fragments_queries_long_endpoints_and_long_models() {
1198        let home = tempfile::tempdir().unwrap();
1199        let service = VoiceService::new(home.path()).unwrap();
1200        let (base, _) = server(b"{\"data\":[]}".to_vec(), "application/json").await;
1201        for endpoint in [format!("{base}/?x=1"), format!("{base}/#top")] {
1202            assert_eq!(
1203                service
1204                    .attach(VoiceKind::Speak, &endpoint, "kokoro")
1205                    .await
1206                    .unwrap_err()
1207                    .0,
1208                "endpoint must be a plain loopback base URL",
1209                "{endpoint}"
1210            );
1211        }
1212        let longest = format!("{base}{}", "/".repeat(256 - base.len()));
1213        service
1214            .attach(VoiceKind::Speak, &longest, "kokoro")
1215            .await
1216            .unwrap();
1217        assert!(
1218            service
1219                .attach(VoiceKind::Speak, &format!("{longest}/"), "kokoro")
1220                .await
1221                .is_err()
1222        );
1223        let widest = format!("{}/{}", "a".repeat(100), "b".repeat(99));
1224        service
1225            .attach(VoiceKind::Speak, &base, &widest)
1226            .await
1227            .unwrap();
1228        assert!(
1229            service
1230                .attach(VoiceKind::Speak, &base, &format!("{widest}b"))
1231                .await
1232                .is_err()
1233        );
1234    }
1235
1236    #[tokio::test]
1237    async fn mlx_audio_requests_carry_the_voice_language() {
1238        let home = tempfile::tempdir().unwrap();
1239        let service = VoiceService::new(home.path()).unwrap();
1240        let (base, seen) = server(wav(8), "audio/wav").await;
1241        service
1242            .claim(
1243                VoiceKind::Speak,
1244                slot(VoiceKind::Speak, "mlx-audio", &base, 1),
1245            )
1246            .unwrap();
1247        service
1248            .speak(&request("Hello"), &catalog(), &CancellationToken::new())
1249            .await
1250            .unwrap();
1251        let sent =
1252            String::from_utf8_lossy(&seen.lock().unwrap().last().unwrap().clone()).into_owned();
1253        assert!(sent.contains("\"lang_code\":\"a\""), "{sent}");
1254    }
1255
1256    /// A fake `mlx_audio.server`: waits `delay` seconds, then answers GET with 200
1257    /// and POST with `post_status`; or exits at once when `delay` is negative.
1258    #[cfg(unix)]
1259    fn fake_engine(directory: &Path, delay: i32, post_status: u16) -> PathBuf {
1260        use std::os::unix::fs::PermissionsExt;
1261        let script = directory.join(format!("engine-{delay}-{post_status}"));
1262        let body = if delay < 0 {
1263            "#!/bin/sh\nexit 0\n".to_owned()
1264        } else {
1265            format!(
1266                r#"#!/bin/sh
1267exec python3 -c '
1268import sys, time, http.server
1269port = int(sys.argv[sys.argv.index("--port") + 1])
1270time.sleep({delay})
1271class H(http.server.BaseHTTPRequestHandler):
1272    def reply(self, code):
1273        self.send_response(code); self.send_header("Content-Length", "2"); self.end_headers(); self.wfile.write(b"{{}}")
1274    def do_GET(self): self.reply(200)
1275    def do_POST(self): self.reply({post_status})
1276    def log_message(self, *a): pass
1277http.server.HTTPServer(("127.0.0.1", port), H).serve_forever()
1278' x "$@"
1279"#
1280            )
1281        };
1282        std::fs::write(&script, body).unwrap();
1283        std::fs::set_permissions(&script, std::fs::Permissions::from_mode(0o755)).unwrap();
1284        script
1285    }
1286
1287    #[cfg(unix)]
1288    async fn settle(service: &VoiceService) -> SlotView {
1289        for _ in 0..150 {
1290            let view = service.status().remove(0);
1291            if view.state != SlotState::Starting {
1292                return view;
1293            }
1294            tokio::time::sleep(Duration::from_millis(100)).await;
1295        }
1296        panic!("the engine never settled");
1297    }
1298
1299    #[cfg(unix)]
1300    #[tokio::test]
1301    async fn launched_engines_become_ready_load_the_model_or_fail_clearly() {
1302        let home = tempfile::tempdir().unwrap();
1303        let kokoro = catalog().resolve("speak").unwrap().clone();
1304        let cancel = CancellationToken::new();
1305        for (delay, post, state, detail) in [
1306            (1, 200, SlotState::Running, "Ready"),
1307            (
1308                0,
1309                500,
1310                SlotState::Failed,
1311                "mlx-audio could not load the model",
1312            ),
1313            (
1314                -1,
1315                200,
1316                SlotState::Failed,
1317                "voice server exited while starting",
1318            ),
1319        ] {
1320            let service = Arc::new(VoiceService::new(home.path()).unwrap());
1321            let binary = fake_engine(home.path(), delay, post);
1322            service.begin(&kokoro, binary, &cancel).unwrap();
1323            let view = settle(&service).await;
1324            assert_eq!(
1325                (view.state, view.detail.as_str()),
1326                (state, detail),
1327                "{delay}"
1328            );
1329            assert!(view.owned);
1330            assert_eq!(view.engine, "mlx-audio");
1331            assert_eq!(view.endpoint.is_some(), state == SlotState::Running);
1332            if state == SlotState::Running {
1333                let endpoint = view.endpoint.unwrap();
1334                let port: u16 = endpoint.rsplit(':').next().unwrap().parse().unwrap();
1335                assert!(port > 1);
1336            }
1337            service.shutdown();
1338        }
1339    }
1340}