Skip to main content

rigspark_runtime/
voice.rs

1//! Local voice: text-to-speech ("speak") and speech-to-text ("listen").
2//!
3//! RigSpark starts whisper.cpp's `whisper-server` with a pinned, SHA-256 verified ggml
4//! file, or `mlx_audio.server` on Apple Silicon, always bound to 127.0.0.1. Any other
5//! loopback server that speaks OpenAI's `/v1/audio/speech` and `/v1/audio/transcriptions`
6//! can be attached instead. Requests and replies are bounded; audio is WAV only.
7use crate::acquire::{Acquisition, Artifact, HfTransport, SizePolicy};
8use rigspark_core::{
9    sizing::Hardware,
10    voice::{VoiceCatalog, VoiceEngine, VoiceKind, VoiceModel},
11};
12use serde::Serialize;
13use serde_json::{Value, json};
14use std::{
15    collections::BTreeMap,
16    path::{Path, PathBuf},
17    sync::{Arc, Mutex},
18    time::Duration,
19};
20use tokio_util::sync::CancellationToken;
21
22pub const WHISPER_SERVER_ENV: &str = "RIGSPARK_WHISPER_SERVER";
23pub const MLX_AUDIO_SERVER_ENV: &str = "RIGSPARK_MLX_AUDIO_SERVER";
24/// Longest text one speak request accepts.
25pub const MAX_SPEAK_CHARS: usize = 8000;
26/// Largest WAV a speak reply or a transcription upload may be.
27pub const MAX_AUDIO_BYTES: usize = 48 * 1024 * 1024;
28/// Longest transcript accepted back from an engine.
29pub const MAX_TRANSCRIPT_BYTES: usize = 256 * 1024;
30
31#[derive(Debug, Clone, PartialEq, Eq)]
32pub struct VoiceError(pub String);
33impl std::fmt::Display for VoiceError {
34    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
35        formatter.write_str(&self.0)
36    }
37}
38impl std::error::Error for VoiceError {}
39fn fail<T>(message: impl Into<String>) -> Result<T, VoiceError> {
40    Err(VoiceError(message.into()))
41}
42
43/// Where an engine's server binary was found, if anywhere.
44#[derive(Debug, Clone, Serialize)]
45#[serde(rename_all = "camelCase")]
46pub struct EngineProbe {
47    pub engine: &'static str,
48    pub label: &'static str,
49    pub installed: bool,
50    pub supported: bool,
51    #[serde(skip)]
52    pub path: Option<PathBuf>,
53    pub install: &'static str,
54}
55
56fn binary_from(variable: Option<std::ffi::OsString>, name: &str) -> Option<PathBuf> {
57    match variable {
58        Some(value) => {
59            let path = PathBuf::from(value);
60            (path.is_absolute() && path.is_file())
61                .then(|| path.canonicalize().ok())
62                .flatten()
63        }
64        None => crate::process_control::resolve_binary(name).ok(),
65    }
66}
67
68pub fn probe_engine(engine: VoiceEngine, hardware: &Hardware) -> EngineProbe {
69    probe_engine_in(engine, hardware, None)
70}
71
72/// Like [`probe_engine`], also finding the mlx-audio RigSpark installed under `engines_root`
73/// (see `crate::engines`) when none is on PATH.
74pub fn probe_engine_in(
75    engine: VoiceEngine,
76    hardware: &Hardware,
77    engines_root: Option<&Path>,
78) -> EngineProbe {
79    let (label, variable, binary, install) = match engine {
80        VoiceEngine::WhisperCpp => (
81            "whisper.cpp",
82            WHISPER_SERVER_ENV,
83            "whisper-server",
84            "brew install whisper-cpp (or build whisper.cpp and set RIGSPARK_WHISPER_SERVER)",
85        ),
86        VoiceEngine::MlxAudio => (
87            "mlx-audio",
88            MLX_AUDIO_SERVER_ENV,
89            "mlx_audio.server",
90            "choose Set up in Models › Voice, or pip install -U mlx-audio (Apple Silicon only)",
91        ),
92    };
93    let path = binary_from(std::env::var_os(variable), binary).or_else(|| {
94        (engine == VoiceEngine::MlxAudio)
95            .then(|| engines_root.and_then(crate::engines::managed_mlx_audio_server_in))
96            .flatten()
97    });
98    EngineProbe {
99        engine: engine.name(),
100        label,
101        installed: path.is_some(),
102        supported: engine.supports(hardware),
103        path,
104        install,
105    }
106}
107
108#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
109#[serde(rename_all = "lowercase")]
110pub enum SlotState {
111    Starting,
112    Running,
113    Failed,
114}
115
116/// How requests reach a running voice server.
117#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
118#[serde(rename_all = "kebab-case")]
119pub enum Protocol {
120    /// whisper.cpp's `/inference` multipart endpoint.
121    WhisperCpp,
122    /// OpenAI's `/v1/audio/speech` and `/v1/audio/transcriptions`.
123    OpenAi,
124}
125
126struct Slot {
127    model: String,
128    engine: &'static str,
129    state: SlotState,
130    detail: String,
131    progress: Option<(u64, u64)>,
132    endpoint: Option<String>,
133    /// The `model` value sent to an OpenAI-style server.
134    request_model: String,
135    protocol: Protocol,
136    owned: bool,
137    child: Option<tokio::process::Child>,
138    cancel: CancellationToken,
139    generation: u64,
140}
141
142#[derive(Debug, Clone, Serialize)]
143#[serde(rename_all = "camelCase")]
144pub struct SlotView {
145    pub kind: &'static str,
146    pub model: String,
147    pub engine: &'static str,
148    pub state: SlotState,
149    pub detail: String,
150    pub progress: Option<(u64, u64)>,
151    pub endpoint: Option<String>,
152    pub owned: bool,
153}
154
155/// One speak and one listen server at a time, each owned or attached.
156pub struct VoiceService {
157    cache: PathBuf,
158    /// Where managed engines live (`crate::engines`).
159    engines: PathBuf,
160    slots: Mutex<BTreeMap<&'static str, Slot>>,
161    client: reqwest::Client,
162    generation: std::sync::atomic::AtomicU64,
163}
164
165#[derive(Debug, Clone)]
166pub struct SpeakRequest {
167    pub text: String,
168    pub voice: Option<String>,
169    pub speed: Option<f64>,
170}
171
172fn kind_key(kind: VoiceKind) -> &'static str {
173    kind.name()
174}
175
176fn token(value: &str, maximum: usize) -> bool {
177    !value.is_empty()
178        && value.len() <= maximum
179        && value
180            .bytes()
181            .all(|byte| byte.is_ascii_alphanumeric() || b"._-".contains(&byte))
182}
183
184/// A RIFF/WAVE header; every audio body in and out must start with one.
185pub fn is_wav(bytes: &[u8]) -> bool {
186    bytes.len() >= 12 && &bytes[..4] == b"RIFF" && &bytes[8..12] == b"WAVE"
187}
188
189fn free_port() -> Result<u16, VoiceError> {
190    let listener = std::net::TcpListener::bind((std::net::Ipv4Addr::LOCALHOST, 0))
191        .map_err(|_| VoiceError("no free local port".into()))?;
192    listener
193        .local_addr()
194        .map(|address| address.port())
195        .map_err(|_| VoiceError("no free local port".into()))
196}
197
198fn multipart(boundary: &str, fields: &[(&str, &str)], audio: &[u8]) -> Vec<u8> {
199    let mut body = Vec::with_capacity(audio.len() + 512);
200    for (name, value) in fields {
201        body.extend_from_slice(
202            format!(
203                "--{boundary}\r\nContent-Disposition: form-data; name=\"{name}\"\r\n\r\n{value}\r\n"
204            )
205            .as_bytes(),
206        );
207    }
208    body.extend_from_slice(
209        format!(
210            "--{boundary}\r\nContent-Disposition: form-data; name=\"file\"; filename=\"audio.wav\"\r\nContent-Type: audio/wav\r\n\r\n"
211        )
212        .as_bytes(),
213    );
214    body.extend_from_slice(audio);
215    body.extend_from_slice(format!("\r\n--{boundary}--\r\n").as_bytes());
216    body
217}
218
219async fn bounded(response: reqwest::Response, maximum: usize) -> Result<Vec<u8>, VoiceError> {
220    if !response.status().is_success() {
221        return fail(format!(
222            "voice server answered {}",
223            response.status().as_u16()
224        ));
225    }
226    if response
227        .content_length()
228        .is_some_and(|length| length > maximum as u64)
229    {
230        return fail("voice server reply exceeds the size limit");
231    }
232    let mut response = response;
233    let mut bytes = Vec::new();
234    while let Some(chunk) = response
235        .chunk()
236        .await
237        .map_err(|_| VoiceError("voice server reply was interrupted".into()))?
238    {
239        if chunk.len() > maximum - bytes.len() {
240            return fail("voice server reply exceeds the size limit");
241        }
242        bytes.extend_from_slice(&chunk);
243    }
244    Ok(bytes)
245}
246
247impl VoiceService {
248    pub fn new(home: &Path) -> Result<Self, VoiceError> {
249        let client = reqwest::Client::builder()
250            .no_proxy()
251            .redirect(reqwest::redirect::Policy::none())
252            .connect_timeout(Duration::from_secs(5))
253            .build()
254            .map_err(|_| VoiceError("voice client setup failed".into()))?;
255        Ok(Self {
256            cache: home.join("cache"),
257            engines: home.join("engines"),
258            slots: Mutex::new(BTreeMap::new()),
259            client,
260            generation: std::sync::atomic::AtomicU64::new(0),
261        })
262    }
263
264    fn lock(&self) -> Result<std::sync::MutexGuard<'_, BTreeMap<&'static str, Slot>>, VoiceError> {
265        self.slots
266            .lock()
267            .map_err(|_| VoiceError("voice state unavailable".into()))
268    }
269
270    pub fn status(&self) -> Vec<SlotView> {
271        let Ok(slots) = self.lock() else {
272            return Vec::new();
273        };
274        slots
275            .iter()
276            .map(|(kind, slot)| SlotView {
277                kind,
278                model: slot.model.clone(),
279                engine: slot.engine,
280                state: slot.state,
281                detail: slot.detail.clone(),
282                progress: slot.progress,
283                endpoint: slot.endpoint.clone(),
284                owned: slot.owned,
285            })
286            .collect()
287    }
288
289    fn update(&self, kind: &'static str, generation: u64, change: impl FnOnce(&mut Slot)) {
290        if let Ok(mut slots) = self.lock()
291            && let Some(slot) = slots
292                .get_mut(kind)
293                .filter(|slot| slot.generation == generation)
294        {
295            change(slot);
296        }
297    }
298
299    /// Replaces the slot for `kind`, stopping whatever ran there before.
300    fn claim(&self, kind: VoiceKind, slot: Slot) -> Result<(), VoiceError> {
301        let previous = self.lock()?.insert(kind_key(kind), slot);
302        if let Some(mut previous) = previous {
303            previous.cancel.cancel();
304            if let Some(child) = previous.child.as_mut() {
305                let _ = child.start_kill();
306            }
307        }
308        Ok(())
309    }
310
311    pub fn stop(&self, kind: VoiceKind) -> bool {
312        let Ok(mut slots) = self.lock() else {
313            return false;
314        };
315        match slots.remove(kind_key(kind)) {
316            Some(mut slot) => {
317                slot.cancel.cancel();
318                if let Some(child) = slot.child.as_mut() {
319                    let _ = child.start_kill();
320                }
321                true
322            }
323            None => false,
324        }
325    }
326
327    pub fn shutdown(&self) {
328        for kind in [VoiceKind::Speak, VoiceKind::Listen] {
329            self.stop(kind);
330        }
331    }
332
333    /// Uses a voice server someone else runs on loopback, speaking OpenAI's audio API.
334    pub async fn attach(
335        &self,
336        kind: VoiceKind,
337        endpoint: &str,
338        model: &str,
339    ) -> Result<(), VoiceError> {
340        let url = crate::state::loopback(endpoint).map_err(|_| {
341            VoiceError("endpoint must be http://127.0.0.1:<port> or localhost".into())
342        })?;
343        if url.query().is_some() || url.fragment().is_some() || endpoint.len() > 256 {
344            return fail("endpoint must be a plain loopback base URL");
345        }
346        if model.len() > 200
347            || !model.split('/').all(|part| token(part, 100))
348            || model.split('/').count() > 2
349        {
350            return fail("model must be a repository id or a short name");
351        }
352        let base = endpoint.trim_end_matches('/').to_owned();
353        let response = tokio::time::timeout(
354            Duration::from_secs(5),
355            self.client.get(format!("{base}/v1/models")).send(),
356        )
357        .await
358        .map_err(|_| VoiceError("voice server did not answer".into()))?
359        .map_err(|_| VoiceError("voice server is not reachable".into()))?;
360        if !response.status().is_success() {
361            return fail("voice server does not offer /v1/models");
362        }
363        let generation = self.next_generation();
364        self.claim(
365            kind,
366            Slot {
367                model: model.into(),
368                engine: "attached",
369                state: SlotState::Running,
370                detail: "Attached server".into(),
371                progress: None,
372                endpoint: Some(base),
373                request_model: model.into(),
374                protocol: Protocol::OpenAi,
375                owned: false,
376                child: None,
377                cancel: CancellationToken::new(),
378                generation,
379            },
380        )
381    }
382
383    fn next_generation(&self) -> u64 {
384        self.generation
385            .fetch_add(1, std::sync::atomic::Ordering::Relaxed)
386            + 1
387    }
388
389    /// Starts `model`'s engine in the background; `status` reports its progress.
390    pub fn start(
391        self: &Arc<Self>,
392        model: &VoiceModel,
393        hardware: &Hardware,
394        parent: &CancellationToken,
395    ) -> Result<(), VoiceError> {
396        let probe = probe_engine_in(model.engine, hardware, Some(&self.engines));
397        if !probe.supported {
398            return fail(format!("{} does not run on this machine", probe.label));
399        }
400        let Some(binary) = probe.path.clone() else {
401            return fail(format!(
402                "{} is not installed: {}",
403                probe.label, probe.install
404            ));
405        };
406        self.begin(model, binary, parent)
407    }
408
409    /// Claims `model`'s slot and launches `binary` for it in the background.
410    fn begin(
411        self: &Arc<Self>,
412        model: &VoiceModel,
413        binary: PathBuf,
414        parent: &CancellationToken,
415    ) -> Result<(), VoiceError> {
416        let kind = kind_key(model.kind);
417        let generation = self.next_generation();
418        let cancel = parent.child_token();
419        self.claim(
420            model.kind,
421            Slot {
422                model: model.id.clone(),
423                engine: model.engine.name(),
424                state: SlotState::Starting,
425                detail: "Starting".into(),
426                progress: None,
427                endpoint: None,
428                request_model: model
429                    .engine_model
430                    .clone()
431                    .unwrap_or_else(|| model.id.clone()),
432                protocol: match model.engine {
433                    VoiceEngine::WhisperCpp => Protocol::WhisperCpp,
434                    VoiceEngine::MlxAudio => Protocol::OpenAi,
435                },
436                owned: true,
437                child: None,
438                cancel: cancel.clone(),
439                generation,
440            },
441        )?;
442        let service = self.clone();
443        let model = model.clone();
444        tokio::spawn(async move {
445            let outcome = service
446                .launch(&model, &binary, kind, generation, &cancel)
447                .await;
448            service.update(kind, generation, |slot| match outcome {
449                Ok(()) => {}
450                Err(error) => {
451                    slot.state = SlotState::Failed;
452                    slot.detail = error.0;
453                    if let Some(child) = slot.child.as_mut() {
454                        let _ = child.start_kill();
455                    }
456                    slot.child = None;
457                }
458            });
459        });
460        Ok(())
461    }
462
463    async fn launch(
464        self: &Arc<Self>,
465        model: &VoiceModel,
466        binary: &Path,
467        kind: &'static str,
468        generation: u64,
469        cancel: &CancellationToken,
470    ) -> Result<(), VoiceError> {
471        let port = free_port()?;
472        let base = format!("http://127.0.0.1:{port}");
473        let mut command = tokio::process::Command::new(binary);
474        match model.engine {
475            VoiceEngine::WhisperCpp => {
476                let file = model
477                    .file
478                    .as_ref()
479                    .ok_or_else(|| VoiceError("missing pinned file".into()))?;
480                self.update(kind, generation, |slot| {
481                    slot.detail = format!("Downloading {}", file.file);
482                });
483                let artifact = Artifact {
484                    backend: "whispercpp".into(),
485                    repo: file.repo.clone(),
486                    revision: file.revision.clone(),
487                    file: file.file.clone(),
488                    sha256: file.sha256.clone(),
489                    bytes: file.bytes,
490                };
491                let progress = {
492                    let service = self.clone();
493                    move |done: u64, total: u64, _: &str| {
494                        service
495                            .update(kind, generation, |slot| slot.progress = Some((done, total)));
496                    }
497                };
498                let acquisition = Acquisition::new(&self.cache)
499                    .map_err(VoiceError)?
500                    .with_progress(progress);
501                let transport = HfTransport::new().map_err(VoiceError)?;
502                let size = if file.exact_size {
503                    SizePolicy::Exact
504                } else {
505                    SizePolicy::Ceiling
506                };
507                let acquired = acquisition
508                    .acquire_sized(&artifact, size, &transport, cancel)
509                    .await
510                    .map_err(|error| VoiceError(format!("download failed: {error}")))?;
511                command.arg("-m").arg(&acquired.path).args([
512                    "--host",
513                    "127.0.0.1",
514                    "--port",
515                    &port.to_string(),
516                ]);
517            }
518            VoiceEngine::MlxAudio => {
519                command.args(["--host", "127.0.0.1", "--port", &port.to_string()]);
520            }
521        }
522        command
523            .stdin(std::process::Stdio::null())
524            .stdout(std::process::Stdio::null())
525            .stderr(std::process::Stdio::null())
526            .kill_on_drop(true);
527        let child = command
528            .spawn()
529            .map_err(|_| VoiceError("voice server failed to start".into()))?;
530        self.update(kind, generation, |slot| {
531            slot.child = Some(child);
532            slot.progress = None;
533            slot.detail = "Loading the model".into();
534        });
535        let probe = match model.engine {
536            VoiceEngine::WhisperCpp => format!("{base}/"),
537            VoiceEngine::MlxAudio => format!("{base}/v1/models"),
538        };
539        self.wait_ready(&probe, kind, generation, cancel).await?;
540        if model.engine == VoiceEngine::MlxAudio {
541            self.update(kind, generation, |slot| {
542                slot.detail = "Fetching and loading weights (mlx-audio)".into();
543            });
544            let repository = model.engine_model.clone().unwrap_or_default();
545            let response = tokio::select! {
546                biased;
547                _ = cancel.cancelled() => return fail("cancelled"),
548                response = self
549                    .client
550                    .post(format!("{base}/v1/models"))
551                    .query(&[("model_name", repository.as_str())])
552                    .timeout(Duration::from_secs(1800))
553                    .send() => response,
554            };
555            match response {
556                Ok(response) if response.status().is_success() => {}
557                _ => return fail("mlx-audio could not load the model"),
558            }
559        }
560        self.update(kind, generation, |slot| {
561            slot.state = SlotState::Running;
562            slot.detail = "Ready".into();
563            slot.endpoint = Some(base.clone());
564        });
565        Ok(())
566    }
567
568    async fn wait_ready(
569        &self,
570        url: &str,
571        kind: &'static str,
572        generation: u64,
573        cancel: &CancellationToken,
574    ) -> Result<(), VoiceError> {
575        let deadline = tokio::time::Instant::now() + Duration::from_secs(180);
576        loop {
577            if cancel.is_cancelled() {
578                return fail("cancelled");
579            }
580            let exited = {
581                let mut slots = self.lock()?;
582                match slots
583                    .get_mut(kind)
584                    .filter(|slot| slot.generation == generation)
585                {
586                    Some(slot) => slot
587                        .child
588                        .as_mut()
589                        .is_some_and(|child| matches!(child.try_wait(), Ok(Some(_)))),
590                    None => return fail("replaced"),
591                }
592            };
593            if exited {
594                return fail("voice server exited while starting");
595            }
596            if let Ok(Ok(response)) =
597                tokio::time::timeout(Duration::from_secs(2), self.client.get(url).send()).await
598                && response.status().is_success()
599            {
600                return Ok(());
601            }
602            if tokio::time::Instant::now() > deadline {
603                return fail("voice server did not become ready");
604            }
605            tokio::time::sleep(Duration::from_millis(400)).await;
606        }
607    }
608
609    fn running(
610        &self,
611        kind: VoiceKind,
612    ) -> Result<(String, String, Protocol, &'static str), VoiceError> {
613        let slots = self.lock()?;
614        let slot = slots
615            .get(kind_key(kind))
616            .filter(|slot| slot.state == SlotState::Running)
617            .ok_or_else(|| {
618                VoiceError(format!(
619                    "no {} model is running; start one from Models › Voice",
620                    kind.name()
621                ))
622            })?;
623        Ok((
624            slot.endpoint.clone().unwrap_or_default(),
625            slot.request_model.clone(),
626            slot.protocol,
627            slot.engine,
628        ))
629    }
630
631    /// Text to WAV with the running speak model.
632    pub async fn speak(
633        &self,
634        request: &SpeakRequest,
635        catalog: &VoiceCatalog,
636        cancel: &CancellationToken,
637    ) -> Result<Vec<u8>, VoiceError> {
638        let text = request.text.trim();
639        if text.is_empty() || text.chars().count() > MAX_SPEAK_CHARS {
640            return fail(format!("text must be 1 to {MAX_SPEAK_CHARS} characters"));
641        }
642        if request
643            .speed
644            .is_some_and(|speed| !(0.5..=2.0).contains(&speed) || !speed.is_finite())
645        {
646            return fail("speed must be between 0.5 and 2.0");
647        }
648        if request
649            .voice
650            .as_deref()
651            .is_some_and(|voice| !token(voice, 64))
652        {
653            return fail("invalid voice");
654        }
655        let (endpoint, model, _, engine) = self.running(VoiceKind::Speak)?;
656        let mut body = json!({
657            "model": model,
658            "input": text,
659            "response_format": "wav",
660            "speed": request.speed.unwrap_or(1.0),
661        });
662        if let Some(voice) = &request.voice {
663            body["voice"] = json!(voice);
664            // mlx-audio picks Kokoro's language from lang_code, which its voice ids encode.
665            if engine == "mlx-audio"
666                && let Some(entry) = catalog
667                    .models
668                    .iter()
669                    .find(|entry| entry.engine_model.as_deref() == Some(model.as_str()))
670                    .and_then(|entry| entry.voice(voice))
671            {
672                body["lang_code"] = json!(entry.lang);
673            }
674        }
675        let send = self
676            .client
677            .post(format!("{endpoint}/v1/audio/speech"))
678            .json(&body)
679            .timeout(Duration::from_secs(600))
680            .send();
681        let response = tokio::select! {
682            biased;
683            _ = cancel.cancelled() => return fail("cancelled"),
684            response = send => response.map_err(|_| VoiceError("speech request failed".into()))?,
685        };
686        let audio = bounded(response, MAX_AUDIO_BYTES).await?;
687        if !is_wav(&audio) {
688            return fail("voice server did not return WAV audio");
689        }
690        Ok(audio)
691    }
692
693    /// WAV to text with the running listen model.
694    pub async fn transcribe(
695        &self,
696        audio: &[u8],
697        language: Option<&str>,
698        cancel: &CancellationToken,
699    ) -> Result<String, VoiceError> {
700        if !is_wav(audio) || audio.len() > MAX_AUDIO_BYTES {
701            return fail("audio must be a WAV file under 48 MiB");
702        }
703        if language.is_some_and(|language| !token(language, 16)) {
704            return fail("invalid language");
705        }
706        let (endpoint, model, protocol, _) = self.running(VoiceKind::Listen)?;
707        let boundary = format!("rigspark{}", uuid::Uuid::new_v4().simple());
708        let mut fields: Vec<(&str, &str)> = Vec::new();
709        let url = match protocol {
710            Protocol::WhisperCpp => {
711                fields.extend([("response_format", "json"), ("temperature", "0.0")]);
712                format!("{endpoint}/inference")
713            }
714            Protocol::OpenAi => {
715                fields.extend([("model", model.as_str()), ("response_format", "json")]);
716                format!("{endpoint}/v1/audio/transcriptions")
717            }
718        };
719        if let Some(language) = language {
720            fields.push(("language", language));
721        }
722        let body = multipart(&boundary, &fields, audio);
723        let send = self
724            .client
725            .post(url)
726            .header(
727                reqwest::header::CONTENT_TYPE,
728                format!("multipart/form-data; boundary={boundary}"),
729            )
730            .body(body)
731            .timeout(Duration::from_secs(600))
732            .send();
733        let response = tokio::select! {
734            biased;
735            _ = cancel.cancelled() => return fail("cancelled"),
736            response = send => response.map_err(|_| VoiceError("transcription request failed".into()))?,
737        };
738        let raw = bounded(response, MAX_TRANSCRIPT_BYTES).await?;
739        let value: Value = serde_json::from_slice(&raw)
740            .map_err(|_| VoiceError("voice server returned an invalid transcript".into()))?;
741        let text = value
742            .get("text")
743            .and_then(Value::as_str)
744            .ok_or_else(|| VoiceError("voice server returned no text".into()))?;
745        Ok(rigspark_core::reports::strip_control(text.trim()))
746    }
747}
748
749impl Drop for VoiceService {
750    fn drop(&mut self) {
751        self.shutdown();
752    }
753}
754
755#[cfg(test)]
756mod tests {
757    use super::*;
758    use tokio::io::{AsyncReadExt, AsyncWriteExt};
759
760    fn wav(samples: usize) -> Vec<u8> {
761        let data = samples * 2;
762        let mut bytes = Vec::new();
763        bytes.extend_from_slice(b"RIFF");
764        bytes.extend_from_slice(&(36 + data as u32).to_le_bytes());
765        bytes.extend_from_slice(b"WAVEfmt ");
766        bytes.extend_from_slice(&16u32.to_le_bytes());
767        bytes.extend_from_slice(&1u16.to_le_bytes());
768        bytes.extend_from_slice(&1u16.to_le_bytes());
769        bytes.extend_from_slice(&16000u32.to_le_bytes());
770        bytes.extend_from_slice(&32000u32.to_le_bytes());
771        bytes.extend_from_slice(&2u16.to_le_bytes());
772        bytes.extend_from_slice(&16u16.to_le_bytes());
773        bytes.extend_from_slice(b"data");
774        bytes.extend_from_slice(&(data as u32).to_le_bytes());
775        bytes.resize(44 + data, 0);
776        bytes
777    }
778
779    /// A loopback server that answers every request with `reply` and records requests.
780    async fn server(
781        reply: Vec<u8>,
782        content_type: &'static str,
783    ) -> (String, Arc<Mutex<Vec<Vec<u8>>>>) {
784        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
785        let address = listener.local_addr().unwrap();
786        let seen = Arc::new(Mutex::new(Vec::new()));
787        let record = seen.clone();
788        tokio::spawn(async move {
789            loop {
790                let Ok((mut socket, _)) = listener.accept().await else {
791                    return;
792                };
793                let reply = reply.clone();
794                let record = record.clone();
795                tokio::spawn(async move {
796                    let mut request = Vec::new();
797                    let mut buffer = vec![0u8; 65536];
798                    loop {
799                        let read = socket.read(&mut buffer).await.unwrap_or(0);
800                        if read == 0 {
801                            break;
802                        }
803                        request.extend_from_slice(&buffer[..read]);
804                        let text = String::from_utf8_lossy(&request);
805                        if let Some(end) = text.find("\r\n\r\n") {
806                            let length = text[..end]
807                                .lines()
808                                .find_map(|line| {
809                                    line.to_ascii_lowercase()
810                                        .strip_prefix("content-length:")
811                                        .map(|value| value.trim().parse::<usize>().unwrap_or(0))
812                                })
813                                .unwrap_or(0);
814                            if request.len() >= end + 4 + length {
815                                break;
816                            }
817                        }
818                    }
819                    record.lock().unwrap().push(request);
820                    let head = format!(
821                        "HTTP/1.1 200 OK\r\nContent-Type: {content_type}\r\nContent-Length: {}\r\nConnection: close\r\n\r\n",
822                        reply.len()
823                    );
824                    let _ = socket.write_all(head.as_bytes()).await;
825                    let _ = socket.write_all(&reply).await;
826                });
827            }
828        });
829        (format!("http://{address}"), seen)
830    }
831
832    fn catalog() -> VoiceCatalog {
833        VoiceCatalog::bundled().unwrap()
834    }
835    fn request(text: &str) -> SpeakRequest {
836        SpeakRequest {
837            text: text.into(),
838            voice: Some("af_heart".into()),
839            speed: Some(1.0),
840        }
841    }
842
843    #[test]
844    fn wav_headers_and_multipart_bodies_are_exact() {
845        assert!(is_wav(&wav(4)));
846        assert!(!is_wav(b"RIFF\0\0\0\0WAVX"));
847        assert!(!is_wav(b"RIFF"));
848        let body = multipart("b", &[("model", "m")], b"AUDIO");
849        assert_eq!(
850            String::from_utf8(body).unwrap(),
851            "--b\r\nContent-Disposition: form-data; name=\"model\"\r\n\r\nm\r\n--b\r\nContent-Disposition: form-data; name=\"file\"; filename=\"audio.wav\"\r\nContent-Type: audio/wav\r\n\r\nAUDIO\r\n--b--\r\n"
852        );
853    }
854
855    #[tokio::test]
856    async fn nothing_runs_until_a_model_is_started_or_attached() {
857        let home = tempfile::tempdir().unwrap();
858        let service = VoiceService::new(home.path()).unwrap();
859        assert!(service.status().is_empty());
860        let error = service
861            .speak(&request("hi"), &catalog(), &CancellationToken::new())
862            .await
863            .unwrap_err();
864        assert!(error.0.contains("no speak model is running"), "{error}");
865        assert!(
866            service
867                .transcribe(&wav(4), None, &CancellationToken::new())
868                .await
869                .unwrap_err()
870                .0
871                .contains("no listen model")
872        );
873        assert!(!service.stop(VoiceKind::Speak));
874    }
875
876    #[tokio::test]
877    async fn attached_servers_must_be_plain_loopback_openai_audio() {
878        let home = tempfile::tempdir().unwrap();
879        let service = VoiceService::new(home.path()).unwrap();
880        for endpoint in [
881            "https://127.0.0.1:1",
882            "http://example.com:8000",
883            "http://user@127.0.0.1:1",
884            "http://127.0.0.1:1/?x=1",
885        ] {
886            assert!(
887                service
888                    .attach(VoiceKind::Speak, endpoint, "kokoro")
889                    .await
890                    .is_err(),
891                "{endpoint}"
892            );
893        }
894        let (base, _) = server(b"{\"data\":[]}".to_vec(), "application/json").await;
895        for model in ["", "a/b/c", "bad model", &"m".repeat(101)] {
896            assert!(
897                service
898                    .attach(VoiceKind::Speak, &base, model)
899                    .await
900                    .is_err(),
901                "{model:?}"
902            );
903        }
904        service
905            .attach(
906                VoiceKind::Speak,
907                &format!("{base}/"),
908                "mlx-community/Kokoro-82M-bf16",
909            )
910            .await
911            .unwrap();
912        let status = service.status();
913        assert_eq!(status.len(), 1);
914        assert_eq!(status[0].state, SlotState::Running);
915        assert_eq!(status[0].endpoint.as_deref(), Some(base.as_str()));
916        assert!(!status[0].owned);
917        assert!(service.stop(VoiceKind::Speak));
918        assert!(service.status().is_empty());
919    }
920
921    #[tokio::test]
922    async fn speech_requests_are_bounded_and_replies_must_be_wav() {
923        let home = tempfile::tempdir().unwrap();
924        let service = VoiceService::new(home.path()).unwrap();
925        let (base, seen) = server(wav(8), "audio/wav").await;
926        service
927            .attach(VoiceKind::Speak, &base, "mlx-community/Kokoro-82M-bf16")
928            .await
929            .unwrap();
930        let cancel = CancellationToken::new();
931        let audio = service
932            .speak(&request("Hello there"), &catalog(), &cancel)
933            .await
934            .unwrap();
935        assert_eq!(audio, wav(8));
936        let sent =
937            String::from_utf8_lossy(&seen.lock().unwrap().last().unwrap().clone()).into_owned();
938        assert!(sent.starts_with("POST /v1/audio/speech"), "{sent}");
939        assert!(sent.contains("\"input\":\"Hello there\""), "{sent}");
940        assert!(sent.contains("\"voice\":\"af_heart\""), "{sent}");
941        assert!(sent.contains("\"response_format\":\"wav\""), "{sent}");
942        // Attached servers get no mlx-audio language hint.
943        assert!(!sent.contains("lang_code"), "{sent}");
944        for (text, voice, speed) in [
945            ("", None, None),
946            ("   ", None, None),
947            (&"x".repeat(MAX_SPEAK_CHARS + 1)[..], None, None),
948            ("hi", Some("bad voice"), None),
949            ("hi", None, Some(0.4)),
950            ("hi", None, Some(2.1)),
951            ("hi", None, Some(f64::NAN)),
952        ] {
953            let request = SpeakRequest {
954                text: text.into(),
955                voice: voice.map(Into::into),
956                speed,
957            };
958            assert!(
959                service.speak(&request, &catalog(), &cancel).await.is_err(),
960                "{request:?}"
961            );
962        }
963        assert!(
964            service
965                .speak(&request(&"x".repeat(MAX_SPEAK_CHARS)), &catalog(), &cancel)
966                .await
967                .is_ok()
968        );
969        let (base, _) = server(b"not audio".to_vec(), "audio/wav").await;
970        service
971            .attach(VoiceKind::Speak, &base, "kokoro")
972            .await
973            .unwrap();
974        assert_eq!(
975            service
976                .speak(&request("hi"), &catalog(), &cancel)
977                .await
978                .unwrap_err()
979                .0,
980            "voice server did not return WAV audio"
981        );
982    }
983
984    #[tokio::test]
985    async fn transcription_posts_multipart_wav_and_reads_text() {
986        let home = tempfile::tempdir().unwrap();
987        let service = VoiceService::new(home.path()).unwrap();
988        let (base, seen) = server(
989            b"{\"text\":\"  hello \\u001b[31mworld \"}".to_vec(),
990            "application/json",
991        )
992        .await;
993        service
994            .attach(VoiceKind::Listen, &base, "whisper-large-v3-turbo")
995            .await
996            .unwrap();
997        let cancel = CancellationToken::new();
998        let text = service
999            .transcribe(&wav(16), Some("en"), &cancel)
1000            .await
1001            .unwrap();
1002        assert_eq!(text, "hello world");
1003        let sent = seen.lock().unwrap().last().unwrap().clone();
1004        let head = String::from_utf8_lossy(&sent);
1005        assert!(head.starts_with("POST /v1/audio/transcriptions"), "{head}");
1006        assert!(
1007            head.contains("multipart/form-data; boundary=rigspark"),
1008            "{head}"
1009        );
1010        assert!(
1011            head.contains("name=\"model\"\r\n\r\nwhisper-large-v3-turbo"),
1012            "{head}"
1013        );
1014        assert!(head.contains("name=\"language\"\r\n\r\nen"), "{head}");
1015        assert!(sent.windows(wav(16).len()).any(|window| window == wav(16)));
1016        assert!(
1017            service
1018                .transcribe(b"not audio", None, &cancel)
1019                .await
1020                .is_err()
1021        );
1022        assert!(
1023            service
1024                .transcribe(&wav(4), Some("e n"), &cancel)
1025                .await
1026                .is_err()
1027        );
1028        let (base, _) = server(b"{\"segments\":[]}".to_vec(), "application/json").await;
1029        service
1030            .attach(VoiceKind::Listen, &base, "whisper")
1031            .await
1032            .unwrap();
1033        assert_eq!(
1034            service
1035                .transcribe(&wav(4), None, &cancel)
1036                .await
1037                .unwrap_err()
1038                .0,
1039            "voice server returned no text"
1040        );
1041    }
1042
1043    #[tokio::test]
1044    async fn starting_needs_an_installed_supported_engine() {
1045        let home = tempfile::tempdir().unwrap();
1046        let service = Arc::new(VoiceService::new(home.path()).unwrap());
1047        let linux = Hardware {
1048            arch: rigspark_core::sizing::CpuArch::X64,
1049            platform: rigspark_core::sizing::Platform::Linux,
1050            total_ram_bytes: 16e9,
1051            free_ram_bytes: 16e9,
1052            free_disk_bytes: 1e12,
1053            gpu: Vec::new(),
1054            unified_memory: None,
1055        };
1056        let kokoro = catalog().resolve("speak").unwrap().clone();
1057        let error = service
1058            .start(&kokoro, &linux, &CancellationToken::new())
1059            .unwrap_err();
1060        assert_eq!(error.0, "mlx-audio does not run on this machine");
1061        assert!(service.status().is_empty());
1062        let probe = probe_engine(VoiceEngine::MlxAudio, &linux);
1063        assert!(!probe.supported);
1064        assert!(probe.install.contains("mlx-audio"));
1065    }
1066
1067    #[test]
1068    fn an_explicit_engine_path_must_be_an_absolute_file() {
1069        let directory = tempfile::tempdir().unwrap();
1070        let binary = directory.path().join("whisper-server");
1071        std::fs::write(&binary, "").unwrap();
1072        assert_eq!(
1073            binary_from(Some(binary.clone().into()), "unused"),
1074            Some(binary.canonicalize().unwrap())
1075        );
1076        assert_eq!(binary_from(Some("whisper-server".into()), "unused"), None);
1077        assert_eq!(binary_from(Some(directory.path().into()), "unused"), None);
1078    }
1079
1080    fn slot(kind: VoiceKind, engine: &'static str, endpoint: &str, generation: u64) -> Slot {
1081        Slot {
1082            model: kind.name().into(),
1083            engine,
1084            state: SlotState::Running,
1085            detail: "Ready".into(),
1086            progress: None,
1087            endpoint: Some(endpoint.into()),
1088            request_model: "mlx-community/Kokoro-82M-bf16".into(),
1089            protocol: Protocol::OpenAi,
1090            owned: false,
1091            child: None,
1092            cancel: CancellationToken::new(),
1093            generation,
1094        }
1095    }
1096
1097    #[test]
1098    fn limits_errors_ports_and_generations_are_exact() {
1099        assert_eq!(MAX_AUDIO_BYTES, 50_331_648);
1100        assert_eq!(MAX_TRANSCRIPT_BYTES, 262_144);
1101        assert_eq!(VoiceError("boom".into()).to_string(), "boom");
1102        let port = free_port().unwrap();
1103        assert!(port > 1);
1104        std::net::TcpListener::bind((std::net::Ipv4Addr::LOCALHOST, port)).unwrap();
1105        let home = tempfile::tempdir().unwrap();
1106        let service = VoiceService::new(home.path()).unwrap();
1107        assert_eq!(service.next_generation(), 1);
1108        assert_eq!(service.next_generation(), 2);
1109        assert_eq!(service.next_generation(), 3);
1110    }
1111
1112    #[test]
1113    fn updates_reach_only_the_current_generation() {
1114        let home = tempfile::tempdir().unwrap();
1115        let service = VoiceService::new(home.path()).unwrap();
1116        service
1117            .claim(
1118                VoiceKind::Listen,
1119                slot(VoiceKind::Listen, "attached", "http://127.0.0.1:9", 7),
1120            )
1121            .unwrap();
1122        service.update("listen", 6, |slot| slot.detail = "stale".into());
1123        assert_eq!(service.status()[0].detail, "Ready");
1124        service.update("listen", 7, |slot| slot.detail = "fresh".into());
1125        let status = service.status();
1126        assert_eq!(status[0].detail, "fresh");
1127        assert_eq!(status[0].kind, "listen");
1128        service
1129            .claim(
1130                VoiceKind::Speak,
1131                slot(VoiceKind::Speak, "attached", "http://127.0.0.1:9", 8),
1132            )
1133            .unwrap();
1134        assert_eq!(
1135            service
1136                .status()
1137                .iter()
1138                .map(|view| view.kind)
1139                .collect::<Vec<_>>(),
1140            ["listen", "speak"]
1141        );
1142        service.shutdown();
1143        assert!(service.status().is_empty());
1144    }
1145
1146    fn response(chunks: Vec<&'static [u8]>, length: bool) -> reqwest::Response {
1147        let total: usize = chunks.iter().map(|chunk| chunk.len()).sum();
1148        let body = reqwest::Body::wrap_stream(futures_util::stream::iter(
1149            chunks
1150                .into_iter()
1151                .map(|chunk| Ok::<_, std::io::Error>(bytes::Bytes::from_static(chunk))),
1152        ));
1153        let mut builder = http::Response::builder().status(200);
1154        if length {
1155            builder = builder.header("content-length", total.to_string());
1156        }
1157        reqwest::Response::from(builder.body(body).unwrap())
1158    }
1159
1160    #[tokio::test]
1161    async fn replies_are_bounded_by_length_and_by_streamed_bytes() {
1162        assert_eq!(
1163            bounded(response(vec![b"abcd"], true), 4).await.unwrap(),
1164            b"abcd"
1165        );
1166        assert!(bounded(response(vec![b"abcde"], true), 4).await.is_err());
1167        assert_eq!(
1168            bounded(response(vec![b"ab", b"cd"], false), 4)
1169                .await
1170                .unwrap(),
1171            b"abcd"
1172        );
1173        assert!(
1174            bounded(response(vec![b"ab", b"cde"], false), 4)
1175                .await
1176                .is_err()
1177        );
1178        assert!(
1179            bounded(response(vec![b"abc", b"de"], false), 4)
1180                .await
1181                .is_err()
1182        );
1183    }
1184
1185    /// Sends `raw` verbatim to the first connection, then closes it.
1186    async fn raw_reply(raw: &'static [u8]) -> reqwest::Response {
1187        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
1188        let address = listener.local_addr().unwrap();
1189        tokio::spawn(async move {
1190            let (mut socket, _) = listener.accept().await.unwrap();
1191            let mut buffer = vec![0u8; 4096];
1192            let _ = socket.read(&mut buffer).await;
1193            let _ = socket.write_all(raw).await;
1194        });
1195        reqwest::get(format!("http://{address}/")).await.unwrap()
1196    }
1197
1198    #[tokio::test]
1199    async fn a_declared_length_is_checked_before_reading() {
1200        let exact =
1201            raw_reply(b"HTTP/1.1 200 OK\r\nContent-Length: 4\r\nConnection: close\r\n\r\nabcd")
1202                .await;
1203        assert_eq!(bounded(exact, 4).await.unwrap(), b"abcd");
1204        // Declares 5 bytes but sends 2: refused on the header, not on the cut-off body.
1205        let declared =
1206            raw_reply(b"HTTP/1.1 200 OK\r\nContent-Length: 5\r\nConnection: close\r\n\r\nab").await;
1207        assert_eq!(
1208            bounded(declared, 4).await.unwrap_err().0,
1209            "voice server reply exceeds the size limit"
1210        );
1211    }
1212
1213    #[tokio::test]
1214    async fn attach_refuses_fragments_queries_long_endpoints_and_long_models() {
1215        let home = tempfile::tempdir().unwrap();
1216        let service = VoiceService::new(home.path()).unwrap();
1217        let (base, _) = server(b"{\"data\":[]}".to_vec(), "application/json").await;
1218        for endpoint in [format!("{base}/?x=1"), format!("{base}/#top")] {
1219            assert_eq!(
1220                service
1221                    .attach(VoiceKind::Speak, &endpoint, "kokoro")
1222                    .await
1223                    .unwrap_err()
1224                    .0,
1225                "endpoint must be a plain loopback base URL",
1226                "{endpoint}"
1227            );
1228        }
1229        let longest = format!("{base}{}", "/".repeat(256 - base.len()));
1230        service
1231            .attach(VoiceKind::Speak, &longest, "kokoro")
1232            .await
1233            .unwrap();
1234        assert!(
1235            service
1236                .attach(VoiceKind::Speak, &format!("{longest}/"), "kokoro")
1237                .await
1238                .is_err()
1239        );
1240        let widest = format!("{}/{}", "a".repeat(100), "b".repeat(99));
1241        service
1242            .attach(VoiceKind::Speak, &base, &widest)
1243            .await
1244            .unwrap();
1245        assert!(
1246            service
1247                .attach(VoiceKind::Speak, &base, &format!("{widest}b"))
1248                .await
1249                .is_err()
1250        );
1251    }
1252
1253    #[tokio::test]
1254    async fn mlx_audio_requests_carry_the_voice_language() {
1255        let home = tempfile::tempdir().unwrap();
1256        let service = VoiceService::new(home.path()).unwrap();
1257        let (base, seen) = server(wav(8), "audio/wav").await;
1258        service
1259            .claim(
1260                VoiceKind::Speak,
1261                slot(VoiceKind::Speak, "mlx-audio", &base, 1),
1262            )
1263            .unwrap();
1264        service
1265            .speak(&request("Hello"), &catalog(), &CancellationToken::new())
1266            .await
1267            .unwrap();
1268        let sent =
1269            String::from_utf8_lossy(&seen.lock().unwrap().last().unwrap().clone()).into_owned();
1270        assert!(sent.contains("\"lang_code\":\"a\""), "{sent}");
1271    }
1272
1273    /// A fake `mlx_audio.server`: waits `delay` seconds, then answers GET with 200
1274    /// and POST with `post_status`; or exits at once when `delay` is negative.
1275    #[cfg(unix)]
1276    fn fake_engine(directory: &Path, delay: i32, post_status: u16) -> PathBuf {
1277        use std::os::unix::fs::PermissionsExt;
1278        let script = directory.join(format!("engine-{delay}-{post_status}"));
1279        let body = if delay < 0 {
1280            "#!/bin/sh\nexit 0\n".to_owned()
1281        } else {
1282            format!(
1283                r#"#!/bin/sh
1284exec python3 -c '
1285import sys, time, http.server
1286port = int(sys.argv[sys.argv.index("--port") + 1])
1287time.sleep({delay})
1288class H(http.server.BaseHTTPRequestHandler):
1289    def reply(self, code):
1290        self.send_response(code); self.send_header("Content-Length", "2"); self.end_headers(); self.wfile.write(b"{{}}")
1291    def do_GET(self): self.reply(200)
1292    def do_POST(self): self.reply({post_status})
1293    def log_message(self, *a): pass
1294http.server.HTTPServer(("127.0.0.1", port), H).serve_forever()
1295' x "$@"
1296"#
1297            )
1298        };
1299        std::fs::write(&script, body).unwrap();
1300        std::fs::set_permissions(&script, std::fs::Permissions::from_mode(0o755)).unwrap();
1301        script
1302    }
1303
1304    #[cfg(unix)]
1305    async fn settle(service: &VoiceService) -> SlotView {
1306        for _ in 0..150 {
1307            let view = service.status().remove(0);
1308            if view.state != SlotState::Starting {
1309                return view;
1310            }
1311            tokio::time::sleep(Duration::from_millis(100)).await;
1312        }
1313        panic!("the engine never settled");
1314    }
1315
1316    #[cfg(unix)]
1317    #[tokio::test]
1318    async fn launched_engines_become_ready_load_the_model_or_fail_clearly() {
1319        let home = tempfile::tempdir().unwrap();
1320        let kokoro = catalog().resolve("speak").unwrap().clone();
1321        let cancel = CancellationToken::new();
1322        for (delay, post, state, detail) in [
1323            (1, 200, SlotState::Running, "Ready"),
1324            (
1325                0,
1326                500,
1327                SlotState::Failed,
1328                "mlx-audio could not load the model",
1329            ),
1330            (
1331                -1,
1332                200,
1333                SlotState::Failed,
1334                "voice server exited while starting",
1335            ),
1336        ] {
1337            let service = Arc::new(VoiceService::new(home.path()).unwrap());
1338            let binary = fake_engine(home.path(), delay, post);
1339            service.begin(&kokoro, binary, &cancel).unwrap();
1340            let view = settle(&service).await;
1341            assert_eq!(
1342                (view.state, view.detail.as_str()),
1343                (state, detail),
1344                "{delay}"
1345            );
1346            assert!(view.owned);
1347            assert_eq!(view.engine, "mlx-audio");
1348            assert_eq!(view.endpoint.is_some(), state == SlotState::Running);
1349            if state == SlotState::Running {
1350                let endpoint = view.endpoint.unwrap();
1351                let port: u16 = endpoint.rsplit(':').next().unwrap().parse().unwrap();
1352                assert!(port > 1);
1353            }
1354            service.shutdown();
1355        }
1356    }
1357}