1use crate::acquire::{Acquisition, Artifact, HfTransport, SizePolicy};
8use rigspark_core::{
9 sizing::Hardware,
10 voice::{VoiceCatalog, VoiceEngine, VoiceKind, VoiceModel},
11};
12use serde::Serialize;
13use serde_json::{Value, json};
14use std::{
15 collections::BTreeMap,
16 path::{Path, PathBuf},
17 sync::{Arc, Mutex},
18 time::Duration,
19};
20use tokio_util::sync::CancellationToken;
21
22pub const WHISPER_SERVER_ENV: &str = "RIGSPARK_WHISPER_SERVER";
23pub const MLX_AUDIO_SERVER_ENV: &str = "RIGSPARK_MLX_AUDIO_SERVER";
24pub const MAX_SPEAK_CHARS: usize = 8000;
26pub const MAX_AUDIO_BYTES: usize = 48 * 1024 * 1024;
28pub const MAX_TRANSCRIPT_BYTES: usize = 256 * 1024;
30
31#[derive(Debug, Clone, PartialEq, Eq)]
32pub struct VoiceError(pub String);
33impl std::fmt::Display for VoiceError {
34 fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
35 formatter.write_str(&self.0)
36 }
37}
38impl std::error::Error for VoiceError {}
39fn fail<T>(message: impl Into<String>) -> Result<T, VoiceError> {
40 Err(VoiceError(message.into()))
41}
42
43#[derive(Debug, Clone, Serialize)]
45#[serde(rename_all = "camelCase")]
46pub struct EngineProbe {
47 pub engine: &'static str,
48 pub label: &'static str,
49 pub installed: bool,
50 pub supported: bool,
51 #[serde(skip)]
52 pub path: Option<PathBuf>,
53 pub install: &'static str,
54}
55
56fn binary_from(variable: Option<std::ffi::OsString>, name: &str) -> Option<PathBuf> {
57 match variable {
58 Some(value) => {
59 let path = PathBuf::from(value);
60 (path.is_absolute() && path.is_file())
61 .then(|| path.canonicalize().ok())
62 .flatten()
63 }
64 None => crate::process_control::resolve_binary(name).ok(),
65 }
66}
67
68pub fn probe_engine(engine: VoiceEngine, hardware: &Hardware) -> EngineProbe {
69 probe_engine_in(engine, hardware, None)
70}
71
72pub fn probe_engine_in(
75 engine: VoiceEngine,
76 hardware: &Hardware,
77 engines_root: Option<&Path>,
78) -> EngineProbe {
79 let (label, variable, binary, install) = match engine {
80 VoiceEngine::WhisperCpp => (
81 "whisper.cpp",
82 WHISPER_SERVER_ENV,
83 "whisper-server",
84 "brew install whisper-cpp (or build whisper.cpp and set RIGSPARK_WHISPER_SERVER)",
85 ),
86 VoiceEngine::MlxAudio => (
87 "mlx-audio",
88 MLX_AUDIO_SERVER_ENV,
89 "mlx_audio.server",
90 "choose Set up in Models › Voice, or pip install -U mlx-audio (Apple Silicon only)",
91 ),
92 };
93 let path = binary_from(std::env::var_os(variable), binary).or_else(|| {
94 (engine == VoiceEngine::MlxAudio)
95 .then(|| engines_root.and_then(crate::engines::managed_mlx_audio_server_in))
96 .flatten()
97 });
98 EngineProbe {
99 engine: engine.name(),
100 label,
101 installed: path.is_some(),
102 supported: engine.supports(hardware),
103 path,
104 install,
105 }
106}
107
108#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
109#[serde(rename_all = "lowercase")]
110pub enum SlotState {
111 Starting,
112 Running,
113 Failed,
114}
115
116#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
118#[serde(rename_all = "kebab-case")]
119pub enum Protocol {
120 WhisperCpp,
122 OpenAi,
124}
125
126struct Slot {
127 model: String,
128 engine: &'static str,
129 state: SlotState,
130 detail: String,
131 progress: Option<(u64, u64)>,
132 endpoint: Option<String>,
133 request_model: String,
135 protocol: Protocol,
136 owned: bool,
137 child: Option<tokio::process::Child>,
138 cancel: CancellationToken,
139 generation: u64,
140}
141
142#[derive(Debug, Clone, Serialize)]
143#[serde(rename_all = "camelCase")]
144pub struct SlotView {
145 pub kind: &'static str,
146 pub model: String,
147 pub engine: &'static str,
148 pub state: SlotState,
149 pub detail: String,
150 pub progress: Option<(u64, u64)>,
151 pub endpoint: Option<String>,
152 pub owned: bool,
153}
154
155pub struct VoiceService {
157 cache: PathBuf,
158 engines: PathBuf,
160 slots: Mutex<BTreeMap<&'static str, Slot>>,
161 client: reqwest::Client,
162 generation: std::sync::atomic::AtomicU64,
163}
164
165#[derive(Debug, Clone)]
166pub struct SpeakRequest {
167 pub text: String,
168 pub voice: Option<String>,
169 pub speed: Option<f64>,
170}
171
172fn kind_key(kind: VoiceKind) -> &'static str {
173 kind.name()
174}
175
176fn token(value: &str, maximum: usize) -> bool {
177 !value.is_empty()
178 && value.len() <= maximum
179 && value
180 .bytes()
181 .all(|byte| byte.is_ascii_alphanumeric() || b"._-".contains(&byte))
182}
183
184pub fn is_wav(bytes: &[u8]) -> bool {
186 bytes.len() >= 12 && &bytes[..4] == b"RIFF" && &bytes[8..12] == b"WAVE"
187}
188
189fn free_port() -> Result<u16, VoiceError> {
190 let listener = std::net::TcpListener::bind((std::net::Ipv4Addr::LOCALHOST, 0))
191 .map_err(|_| VoiceError("no free local port".into()))?;
192 listener
193 .local_addr()
194 .map(|address| address.port())
195 .map_err(|_| VoiceError("no free local port".into()))
196}
197
198fn multipart(boundary: &str, fields: &[(&str, &str)], audio: &[u8]) -> Vec<u8> {
199 let mut body = Vec::with_capacity(audio.len() + 512);
200 for (name, value) in fields {
201 body.extend_from_slice(
202 format!(
203 "--{boundary}\r\nContent-Disposition: form-data; name=\"{name}\"\r\n\r\n{value}\r\n"
204 )
205 .as_bytes(),
206 );
207 }
208 body.extend_from_slice(
209 format!(
210 "--{boundary}\r\nContent-Disposition: form-data; name=\"file\"; filename=\"audio.wav\"\r\nContent-Type: audio/wav\r\n\r\n"
211 )
212 .as_bytes(),
213 );
214 body.extend_from_slice(audio);
215 body.extend_from_slice(format!("\r\n--{boundary}--\r\n").as_bytes());
216 body
217}
218
219async fn bounded(response: reqwest::Response, maximum: usize) -> Result<Vec<u8>, VoiceError> {
220 if !response.status().is_success() {
221 return fail(format!(
222 "voice server answered {}",
223 response.status().as_u16()
224 ));
225 }
226 if response
227 .content_length()
228 .is_some_and(|length| length > maximum as u64)
229 {
230 return fail("voice server reply exceeds the size limit");
231 }
232 let mut response = response;
233 let mut bytes = Vec::new();
234 while let Some(chunk) = response
235 .chunk()
236 .await
237 .map_err(|_| VoiceError("voice server reply was interrupted".into()))?
238 {
239 if chunk.len() > maximum - bytes.len() {
240 return fail("voice server reply exceeds the size limit");
241 }
242 bytes.extend_from_slice(&chunk);
243 }
244 Ok(bytes)
245}
246
247impl VoiceService {
248 pub fn new(home: &Path) -> Result<Self, VoiceError> {
249 let client = reqwest::Client::builder()
250 .no_proxy()
251 .redirect(reqwest::redirect::Policy::none())
252 .connect_timeout(Duration::from_secs(5))
253 .build()
254 .map_err(|_| VoiceError("voice client setup failed".into()))?;
255 Ok(Self {
256 cache: home.join("cache"),
257 engines: home.join("engines"),
258 slots: Mutex::new(BTreeMap::new()),
259 client,
260 generation: std::sync::atomic::AtomicU64::new(0),
261 })
262 }
263
264 fn lock(&self) -> Result<std::sync::MutexGuard<'_, BTreeMap<&'static str, Slot>>, VoiceError> {
265 self.slots
266 .lock()
267 .map_err(|_| VoiceError("voice state unavailable".into()))
268 }
269
270 pub fn status(&self) -> Vec<SlotView> {
271 let Ok(slots) = self.lock() else {
272 return Vec::new();
273 };
274 slots
275 .iter()
276 .map(|(kind, slot)| SlotView {
277 kind,
278 model: slot.model.clone(),
279 engine: slot.engine,
280 state: slot.state,
281 detail: slot.detail.clone(),
282 progress: slot.progress,
283 endpoint: slot.endpoint.clone(),
284 owned: slot.owned,
285 })
286 .collect()
287 }
288
289 fn update(&self, kind: &'static str, generation: u64, change: impl FnOnce(&mut Slot)) {
290 if let Ok(mut slots) = self.lock()
291 && let Some(slot) = slots
292 .get_mut(kind)
293 .filter(|slot| slot.generation == generation)
294 {
295 change(slot);
296 }
297 }
298
299 fn claim(&self, kind: VoiceKind, slot: Slot) -> Result<(), VoiceError> {
301 let previous = self.lock()?.insert(kind_key(kind), slot);
302 if let Some(mut previous) = previous {
303 previous.cancel.cancel();
304 if let Some(child) = previous.child.as_mut() {
305 let _ = child.start_kill();
306 }
307 }
308 Ok(())
309 }
310
311 pub fn stop(&self, kind: VoiceKind) -> bool {
312 let Ok(mut slots) = self.lock() else {
313 return false;
314 };
315 match slots.remove(kind_key(kind)) {
316 Some(mut slot) => {
317 slot.cancel.cancel();
318 if let Some(child) = slot.child.as_mut() {
319 let _ = child.start_kill();
320 }
321 true
322 }
323 None => false,
324 }
325 }
326
327 pub fn shutdown(&self) {
328 for kind in [VoiceKind::Speak, VoiceKind::Listen] {
329 self.stop(kind);
330 }
331 }
332
333 pub async fn attach(
335 &self,
336 kind: VoiceKind,
337 endpoint: &str,
338 model: &str,
339 ) -> Result<(), VoiceError> {
340 let url = crate::state::loopback(endpoint).map_err(|_| {
341 VoiceError("endpoint must be http://127.0.0.1:<port> or localhost".into())
342 })?;
343 if url.query().is_some() || url.fragment().is_some() || endpoint.len() > 256 {
344 return fail("endpoint must be a plain loopback base URL");
345 }
346 if model.len() > 200
347 || !model.split('/').all(|part| token(part, 100))
348 || model.split('/').count() > 2
349 {
350 return fail("model must be a repository id or a short name");
351 }
352 let base = endpoint.trim_end_matches('/').to_owned();
353 let response = tokio::time::timeout(
354 Duration::from_secs(5),
355 self.client.get(format!("{base}/v1/models")).send(),
356 )
357 .await
358 .map_err(|_| VoiceError("voice server did not answer".into()))?
359 .map_err(|_| VoiceError("voice server is not reachable".into()))?;
360 if !response.status().is_success() {
361 return fail("voice server does not offer /v1/models");
362 }
363 let generation = self.next_generation();
364 self.claim(
365 kind,
366 Slot {
367 model: model.into(),
368 engine: "attached",
369 state: SlotState::Running,
370 detail: "Attached server".into(),
371 progress: None,
372 endpoint: Some(base),
373 request_model: model.into(),
374 protocol: Protocol::OpenAi,
375 owned: false,
376 child: None,
377 cancel: CancellationToken::new(),
378 generation,
379 },
380 )
381 }
382
383 fn next_generation(&self) -> u64 {
384 self.generation
385 .fetch_add(1, std::sync::atomic::Ordering::Relaxed)
386 + 1
387 }
388
389 pub fn start(
391 self: &Arc<Self>,
392 model: &VoiceModel,
393 hardware: &Hardware,
394 parent: &CancellationToken,
395 ) -> Result<(), VoiceError> {
396 let probe = probe_engine_in(model.engine, hardware, Some(&self.engines));
397 if !probe.supported {
398 return fail(format!("{} does not run on this machine", probe.label));
399 }
400 let Some(binary) = probe.path.clone() else {
401 return fail(format!(
402 "{} is not installed: {}",
403 probe.label, probe.install
404 ));
405 };
406 self.begin(model, binary, parent)
407 }
408
409 fn begin(
411 self: &Arc<Self>,
412 model: &VoiceModel,
413 binary: PathBuf,
414 parent: &CancellationToken,
415 ) -> Result<(), VoiceError> {
416 let kind = kind_key(model.kind);
417 let generation = self.next_generation();
418 let cancel = parent.child_token();
419 self.claim(
420 model.kind,
421 Slot {
422 model: model.id.clone(),
423 engine: model.engine.name(),
424 state: SlotState::Starting,
425 detail: "Starting".into(),
426 progress: None,
427 endpoint: None,
428 request_model: model
429 .engine_model
430 .clone()
431 .unwrap_or_else(|| model.id.clone()),
432 protocol: match model.engine {
433 VoiceEngine::WhisperCpp => Protocol::WhisperCpp,
434 VoiceEngine::MlxAudio => Protocol::OpenAi,
435 },
436 owned: true,
437 child: None,
438 cancel: cancel.clone(),
439 generation,
440 },
441 )?;
442 let service = self.clone();
443 let model = model.clone();
444 tokio::spawn(async move {
445 let outcome = service
446 .launch(&model, &binary, kind, generation, &cancel)
447 .await;
448 service.update(kind, generation, |slot| match outcome {
449 Ok(()) => {}
450 Err(error) => {
451 slot.state = SlotState::Failed;
452 slot.detail = error.0;
453 if let Some(child) = slot.child.as_mut() {
454 let _ = child.start_kill();
455 }
456 slot.child = None;
457 }
458 });
459 });
460 Ok(())
461 }
462
463 async fn launch(
464 self: &Arc<Self>,
465 model: &VoiceModel,
466 binary: &Path,
467 kind: &'static str,
468 generation: u64,
469 cancel: &CancellationToken,
470 ) -> Result<(), VoiceError> {
471 let port = free_port()?;
472 let base = format!("http://127.0.0.1:{port}");
473 let mut command = tokio::process::Command::new(binary);
474 match model.engine {
475 VoiceEngine::WhisperCpp => {
476 let file = model
477 .file
478 .as_ref()
479 .ok_or_else(|| VoiceError("missing pinned file".into()))?;
480 self.update(kind, generation, |slot| {
481 slot.detail = format!("Downloading {}", file.file);
482 });
483 let artifact = Artifact {
484 backend: "whispercpp".into(),
485 repo: file.repo.clone(),
486 revision: file.revision.clone(),
487 file: file.file.clone(),
488 sha256: file.sha256.clone(),
489 bytes: file.bytes,
490 };
491 let progress = {
492 let service = self.clone();
493 move |done: u64, total: u64, _: &str| {
494 service
495 .update(kind, generation, |slot| slot.progress = Some((done, total)));
496 }
497 };
498 let acquisition = Acquisition::new(&self.cache)
499 .map_err(VoiceError)?
500 .with_progress(progress);
501 let transport = HfTransport::new().map_err(VoiceError)?;
502 let size = if file.exact_size {
503 SizePolicy::Exact
504 } else {
505 SizePolicy::Ceiling
506 };
507 let acquired = acquisition
508 .acquire_sized(&artifact, size, &transport, cancel)
509 .await
510 .map_err(|error| VoiceError(format!("download failed: {error}")))?;
511 command.arg("-m").arg(&acquired.path).args([
512 "--host",
513 "127.0.0.1",
514 "--port",
515 &port.to_string(),
516 ]);
517 }
518 VoiceEngine::MlxAudio => {
519 command.args(["--host", "127.0.0.1", "--port", &port.to_string()]);
520 }
521 }
522 command
523 .stdin(std::process::Stdio::null())
524 .stdout(std::process::Stdio::null())
525 .stderr(std::process::Stdio::null())
526 .kill_on_drop(true);
527 let child = command
528 .spawn()
529 .map_err(|_| VoiceError("voice server failed to start".into()))?;
530 self.update(kind, generation, |slot| {
531 slot.child = Some(child);
532 slot.progress = None;
533 slot.detail = "Loading the model".into();
534 });
535 let probe = match model.engine {
536 VoiceEngine::WhisperCpp => format!("{base}/"),
537 VoiceEngine::MlxAudio => format!("{base}/v1/models"),
538 };
539 self.wait_ready(&probe, kind, generation, cancel).await?;
540 if model.engine == VoiceEngine::MlxAudio {
541 self.update(kind, generation, |slot| {
542 slot.detail = "Fetching and loading weights (mlx-audio)".into();
543 });
544 let repository = model.engine_model.clone().unwrap_or_default();
545 let response = tokio::select! {
546 biased;
547 _ = cancel.cancelled() => return fail("cancelled"),
548 response = self
549 .client
550 .post(format!("{base}/v1/models"))
551 .query(&[("model_name", repository.as_str())])
552 .timeout(Duration::from_secs(1800))
553 .send() => response,
554 };
555 match response {
556 Ok(response) if response.status().is_success() => {}
557 _ => return fail("mlx-audio could not load the model"),
558 }
559 }
560 self.update(kind, generation, |slot| {
561 slot.state = SlotState::Running;
562 slot.detail = "Ready".into();
563 slot.endpoint = Some(base.clone());
564 });
565 Ok(())
566 }
567
568 async fn wait_ready(
569 &self,
570 url: &str,
571 kind: &'static str,
572 generation: u64,
573 cancel: &CancellationToken,
574 ) -> Result<(), VoiceError> {
575 let deadline = tokio::time::Instant::now() + Duration::from_secs(180);
576 loop {
577 if cancel.is_cancelled() {
578 return fail("cancelled");
579 }
580 let exited = {
581 let mut slots = self.lock()?;
582 match slots
583 .get_mut(kind)
584 .filter(|slot| slot.generation == generation)
585 {
586 Some(slot) => slot
587 .child
588 .as_mut()
589 .is_some_and(|child| matches!(child.try_wait(), Ok(Some(_)))),
590 None => return fail("replaced"),
591 }
592 };
593 if exited {
594 return fail("voice server exited while starting");
595 }
596 if let Ok(Ok(response)) =
597 tokio::time::timeout(Duration::from_secs(2), self.client.get(url).send()).await
598 && response.status().is_success()
599 {
600 return Ok(());
601 }
602 if tokio::time::Instant::now() > deadline {
603 return fail("voice server did not become ready");
604 }
605 tokio::time::sleep(Duration::from_millis(400)).await;
606 }
607 }
608
609 fn running(
610 &self,
611 kind: VoiceKind,
612 ) -> Result<(String, String, Protocol, &'static str), VoiceError> {
613 let slots = self.lock()?;
614 let slot = slots
615 .get(kind_key(kind))
616 .filter(|slot| slot.state == SlotState::Running)
617 .ok_or_else(|| {
618 VoiceError(format!(
619 "no {} model is running; start one from Models › Voice",
620 kind.name()
621 ))
622 })?;
623 Ok((
624 slot.endpoint.clone().unwrap_or_default(),
625 slot.request_model.clone(),
626 slot.protocol,
627 slot.engine,
628 ))
629 }
630
631 pub async fn speak(
633 &self,
634 request: &SpeakRequest,
635 catalog: &VoiceCatalog,
636 cancel: &CancellationToken,
637 ) -> Result<Vec<u8>, VoiceError> {
638 let text = request.text.trim();
639 if text.is_empty() || text.chars().count() > MAX_SPEAK_CHARS {
640 return fail(format!("text must be 1 to {MAX_SPEAK_CHARS} characters"));
641 }
642 if request
643 .speed
644 .is_some_and(|speed| !(0.5..=2.0).contains(&speed) || !speed.is_finite())
645 {
646 return fail("speed must be between 0.5 and 2.0");
647 }
648 if request
649 .voice
650 .as_deref()
651 .is_some_and(|voice| !token(voice, 64))
652 {
653 return fail("invalid voice");
654 }
655 let (endpoint, model, _, engine) = self.running(VoiceKind::Speak)?;
656 let mut body = json!({
657 "model": model,
658 "input": text,
659 "response_format": "wav",
660 "speed": request.speed.unwrap_or(1.0),
661 });
662 if let Some(voice) = &request.voice {
663 body["voice"] = json!(voice);
664 if engine == "mlx-audio"
666 && let Some(entry) = catalog
667 .models
668 .iter()
669 .find(|entry| entry.engine_model.as_deref() == Some(model.as_str()))
670 .and_then(|entry| entry.voice(voice))
671 {
672 body["lang_code"] = json!(entry.lang);
673 }
674 }
675 let send = self
676 .client
677 .post(format!("{endpoint}/v1/audio/speech"))
678 .json(&body)
679 .timeout(Duration::from_secs(600))
680 .send();
681 let response = tokio::select! {
682 biased;
683 _ = cancel.cancelled() => return fail("cancelled"),
684 response = send => response.map_err(|_| VoiceError("speech request failed".into()))?,
685 };
686 let audio = bounded(response, MAX_AUDIO_BYTES).await?;
687 if !is_wav(&audio) {
688 return fail("voice server did not return WAV audio");
689 }
690 Ok(audio)
691 }
692
693 pub async fn transcribe(
695 &self,
696 audio: &[u8],
697 language: Option<&str>,
698 cancel: &CancellationToken,
699 ) -> Result<String, VoiceError> {
700 if !is_wav(audio) || audio.len() > MAX_AUDIO_BYTES {
701 return fail("audio must be a WAV file under 48 MiB");
702 }
703 if language.is_some_and(|language| !token(language, 16)) {
704 return fail("invalid language");
705 }
706 let (endpoint, model, protocol, _) = self.running(VoiceKind::Listen)?;
707 let boundary = format!("rigspark{}", uuid::Uuid::new_v4().simple());
708 let mut fields: Vec<(&str, &str)> = Vec::new();
709 let url = match protocol {
710 Protocol::WhisperCpp => {
711 fields.extend([("response_format", "json"), ("temperature", "0.0")]);
712 format!("{endpoint}/inference")
713 }
714 Protocol::OpenAi => {
715 fields.extend([("model", model.as_str()), ("response_format", "json")]);
716 format!("{endpoint}/v1/audio/transcriptions")
717 }
718 };
719 if let Some(language) = language {
720 fields.push(("language", language));
721 }
722 let body = multipart(&boundary, &fields, audio);
723 let send = self
724 .client
725 .post(url)
726 .header(
727 reqwest::header::CONTENT_TYPE,
728 format!("multipart/form-data; boundary={boundary}"),
729 )
730 .body(body)
731 .timeout(Duration::from_secs(600))
732 .send();
733 let response = tokio::select! {
734 biased;
735 _ = cancel.cancelled() => return fail("cancelled"),
736 response = send => response.map_err(|_| VoiceError("transcription request failed".into()))?,
737 };
738 let raw = bounded(response, MAX_TRANSCRIPT_BYTES).await?;
739 let value: Value = serde_json::from_slice(&raw)
740 .map_err(|_| VoiceError("voice server returned an invalid transcript".into()))?;
741 let text = value
742 .get("text")
743 .and_then(Value::as_str)
744 .ok_or_else(|| VoiceError("voice server returned no text".into()))?;
745 Ok(rigspark_core::reports::strip_control(text.trim()))
746 }
747}
748
749impl Drop for VoiceService {
750 fn drop(&mut self) {
751 self.shutdown();
752 }
753}
754
755#[cfg(test)]
756mod tests {
757 use super::*;
758 use tokio::io::{AsyncReadExt, AsyncWriteExt};
759
760 fn wav(samples: usize) -> Vec<u8> {
761 let data = samples * 2;
762 let mut bytes = Vec::new();
763 bytes.extend_from_slice(b"RIFF");
764 bytes.extend_from_slice(&(36 + data as u32).to_le_bytes());
765 bytes.extend_from_slice(b"WAVEfmt ");
766 bytes.extend_from_slice(&16u32.to_le_bytes());
767 bytes.extend_from_slice(&1u16.to_le_bytes());
768 bytes.extend_from_slice(&1u16.to_le_bytes());
769 bytes.extend_from_slice(&16000u32.to_le_bytes());
770 bytes.extend_from_slice(&32000u32.to_le_bytes());
771 bytes.extend_from_slice(&2u16.to_le_bytes());
772 bytes.extend_from_slice(&16u16.to_le_bytes());
773 bytes.extend_from_slice(b"data");
774 bytes.extend_from_slice(&(data as u32).to_le_bytes());
775 bytes.resize(44 + data, 0);
776 bytes
777 }
778
779 async fn server(
781 reply: Vec<u8>,
782 content_type: &'static str,
783 ) -> (String, Arc<Mutex<Vec<Vec<u8>>>>) {
784 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
785 let address = listener.local_addr().unwrap();
786 let seen = Arc::new(Mutex::new(Vec::new()));
787 let record = seen.clone();
788 tokio::spawn(async move {
789 loop {
790 let Ok((mut socket, _)) = listener.accept().await else {
791 return;
792 };
793 let reply = reply.clone();
794 let record = record.clone();
795 tokio::spawn(async move {
796 let mut request = Vec::new();
797 let mut buffer = vec![0u8; 65536];
798 loop {
799 let read = socket.read(&mut buffer).await.unwrap_or(0);
800 if read == 0 {
801 break;
802 }
803 request.extend_from_slice(&buffer[..read]);
804 let text = String::from_utf8_lossy(&request);
805 if let Some(end) = text.find("\r\n\r\n") {
806 let length = text[..end]
807 .lines()
808 .find_map(|line| {
809 line.to_ascii_lowercase()
810 .strip_prefix("content-length:")
811 .map(|value| value.trim().parse::<usize>().unwrap_or(0))
812 })
813 .unwrap_or(0);
814 if request.len() >= end + 4 + length {
815 break;
816 }
817 }
818 }
819 record.lock().unwrap().push(request);
820 let head = format!(
821 "HTTP/1.1 200 OK\r\nContent-Type: {content_type}\r\nContent-Length: {}\r\nConnection: close\r\n\r\n",
822 reply.len()
823 );
824 let _ = socket.write_all(head.as_bytes()).await;
825 let _ = socket.write_all(&reply).await;
826 });
827 }
828 });
829 (format!("http://{address}"), seen)
830 }
831
832 fn catalog() -> VoiceCatalog {
833 VoiceCatalog::bundled().unwrap()
834 }
835 fn request(text: &str) -> SpeakRequest {
836 SpeakRequest {
837 text: text.into(),
838 voice: Some("af_heart".into()),
839 speed: Some(1.0),
840 }
841 }
842
843 #[test]
844 fn wav_headers_and_multipart_bodies_are_exact() {
845 assert!(is_wav(&wav(4)));
846 assert!(!is_wav(b"RIFF\0\0\0\0WAVX"));
847 assert!(!is_wav(b"RIFF"));
848 let body = multipart("b", &[("model", "m")], b"AUDIO");
849 assert_eq!(
850 String::from_utf8(body).unwrap(),
851 "--b\r\nContent-Disposition: form-data; name=\"model\"\r\n\r\nm\r\n--b\r\nContent-Disposition: form-data; name=\"file\"; filename=\"audio.wav\"\r\nContent-Type: audio/wav\r\n\r\nAUDIO\r\n--b--\r\n"
852 );
853 }
854
855 #[tokio::test]
856 async fn nothing_runs_until_a_model_is_started_or_attached() {
857 let home = tempfile::tempdir().unwrap();
858 let service = VoiceService::new(home.path()).unwrap();
859 assert!(service.status().is_empty());
860 let error = service
861 .speak(&request("hi"), &catalog(), &CancellationToken::new())
862 .await
863 .unwrap_err();
864 assert!(error.0.contains("no speak model is running"), "{error}");
865 assert!(
866 service
867 .transcribe(&wav(4), None, &CancellationToken::new())
868 .await
869 .unwrap_err()
870 .0
871 .contains("no listen model")
872 );
873 assert!(!service.stop(VoiceKind::Speak));
874 }
875
876 #[tokio::test]
877 async fn attached_servers_must_be_plain_loopback_openai_audio() {
878 let home = tempfile::tempdir().unwrap();
879 let service = VoiceService::new(home.path()).unwrap();
880 for endpoint in [
881 "https://127.0.0.1:1",
882 "http://example.com:8000",
883 "http://user@127.0.0.1:1",
884 "http://127.0.0.1:1/?x=1",
885 ] {
886 assert!(
887 service
888 .attach(VoiceKind::Speak, endpoint, "kokoro")
889 .await
890 .is_err(),
891 "{endpoint}"
892 );
893 }
894 let (base, _) = server(b"{\"data\":[]}".to_vec(), "application/json").await;
895 for model in ["", "a/b/c", "bad model", &"m".repeat(101)] {
896 assert!(
897 service
898 .attach(VoiceKind::Speak, &base, model)
899 .await
900 .is_err(),
901 "{model:?}"
902 );
903 }
904 service
905 .attach(
906 VoiceKind::Speak,
907 &format!("{base}/"),
908 "mlx-community/Kokoro-82M-bf16",
909 )
910 .await
911 .unwrap();
912 let status = service.status();
913 assert_eq!(status.len(), 1);
914 assert_eq!(status[0].state, SlotState::Running);
915 assert_eq!(status[0].endpoint.as_deref(), Some(base.as_str()));
916 assert!(!status[0].owned);
917 assert!(service.stop(VoiceKind::Speak));
918 assert!(service.status().is_empty());
919 }
920
921 #[tokio::test]
922 async fn speech_requests_are_bounded_and_replies_must_be_wav() {
923 let home = tempfile::tempdir().unwrap();
924 let service = VoiceService::new(home.path()).unwrap();
925 let (base, seen) = server(wav(8), "audio/wav").await;
926 service
927 .attach(VoiceKind::Speak, &base, "mlx-community/Kokoro-82M-bf16")
928 .await
929 .unwrap();
930 let cancel = CancellationToken::new();
931 let audio = service
932 .speak(&request("Hello there"), &catalog(), &cancel)
933 .await
934 .unwrap();
935 assert_eq!(audio, wav(8));
936 let sent =
937 String::from_utf8_lossy(&seen.lock().unwrap().last().unwrap().clone()).into_owned();
938 assert!(sent.starts_with("POST /v1/audio/speech"), "{sent}");
939 assert!(sent.contains("\"input\":\"Hello there\""), "{sent}");
940 assert!(sent.contains("\"voice\":\"af_heart\""), "{sent}");
941 assert!(sent.contains("\"response_format\":\"wav\""), "{sent}");
942 assert!(!sent.contains("lang_code"), "{sent}");
944 for (text, voice, speed) in [
945 ("", None, None),
946 (" ", None, None),
947 (&"x".repeat(MAX_SPEAK_CHARS + 1)[..], None, None),
948 ("hi", Some("bad voice"), None),
949 ("hi", None, Some(0.4)),
950 ("hi", None, Some(2.1)),
951 ("hi", None, Some(f64::NAN)),
952 ] {
953 let request = SpeakRequest {
954 text: text.into(),
955 voice: voice.map(Into::into),
956 speed,
957 };
958 assert!(
959 service.speak(&request, &catalog(), &cancel).await.is_err(),
960 "{request:?}"
961 );
962 }
963 assert!(
964 service
965 .speak(&request(&"x".repeat(MAX_SPEAK_CHARS)), &catalog(), &cancel)
966 .await
967 .is_ok()
968 );
969 let (base, _) = server(b"not audio".to_vec(), "audio/wav").await;
970 service
971 .attach(VoiceKind::Speak, &base, "kokoro")
972 .await
973 .unwrap();
974 assert_eq!(
975 service
976 .speak(&request("hi"), &catalog(), &cancel)
977 .await
978 .unwrap_err()
979 .0,
980 "voice server did not return WAV audio"
981 );
982 }
983
984 #[tokio::test]
985 async fn transcription_posts_multipart_wav_and_reads_text() {
986 let home = tempfile::tempdir().unwrap();
987 let service = VoiceService::new(home.path()).unwrap();
988 let (base, seen) = server(
989 b"{\"text\":\" hello \\u001b[31mworld \"}".to_vec(),
990 "application/json",
991 )
992 .await;
993 service
994 .attach(VoiceKind::Listen, &base, "whisper-large-v3-turbo")
995 .await
996 .unwrap();
997 let cancel = CancellationToken::new();
998 let text = service
999 .transcribe(&wav(16), Some("en"), &cancel)
1000 .await
1001 .unwrap();
1002 assert_eq!(text, "hello world");
1003 let sent = seen.lock().unwrap().last().unwrap().clone();
1004 let head = String::from_utf8_lossy(&sent);
1005 assert!(head.starts_with("POST /v1/audio/transcriptions"), "{head}");
1006 assert!(
1007 head.contains("multipart/form-data; boundary=rigspark"),
1008 "{head}"
1009 );
1010 assert!(
1011 head.contains("name=\"model\"\r\n\r\nwhisper-large-v3-turbo"),
1012 "{head}"
1013 );
1014 assert!(head.contains("name=\"language\"\r\n\r\nen"), "{head}");
1015 assert!(sent.windows(wav(16).len()).any(|window| window == wav(16)));
1016 assert!(
1017 service
1018 .transcribe(b"not audio", None, &cancel)
1019 .await
1020 .is_err()
1021 );
1022 assert!(
1023 service
1024 .transcribe(&wav(4), Some("e n"), &cancel)
1025 .await
1026 .is_err()
1027 );
1028 let (base, _) = server(b"{\"segments\":[]}".to_vec(), "application/json").await;
1029 service
1030 .attach(VoiceKind::Listen, &base, "whisper")
1031 .await
1032 .unwrap();
1033 assert_eq!(
1034 service
1035 .transcribe(&wav(4), None, &cancel)
1036 .await
1037 .unwrap_err()
1038 .0,
1039 "voice server returned no text"
1040 );
1041 }
1042
1043 #[tokio::test]
1044 async fn starting_needs_an_installed_supported_engine() {
1045 let home = tempfile::tempdir().unwrap();
1046 let service = Arc::new(VoiceService::new(home.path()).unwrap());
1047 let linux = Hardware {
1048 arch: rigspark_core::sizing::CpuArch::X64,
1049 platform: rigspark_core::sizing::Platform::Linux,
1050 total_ram_bytes: 16e9,
1051 free_ram_bytes: 16e9,
1052 free_disk_bytes: 1e12,
1053 gpu: Vec::new(),
1054 unified_memory: None,
1055 };
1056 let kokoro = catalog().resolve("speak").unwrap().clone();
1057 let error = service
1058 .start(&kokoro, &linux, &CancellationToken::new())
1059 .unwrap_err();
1060 assert_eq!(error.0, "mlx-audio does not run on this machine");
1061 assert!(service.status().is_empty());
1062 let probe = probe_engine(VoiceEngine::MlxAudio, &linux);
1063 assert!(!probe.supported);
1064 assert!(probe.install.contains("mlx-audio"));
1065 }
1066
1067 #[test]
1068 fn an_explicit_engine_path_must_be_an_absolute_file() {
1069 let directory = tempfile::tempdir().unwrap();
1070 let binary = directory.path().join("whisper-server");
1071 std::fs::write(&binary, "").unwrap();
1072 assert_eq!(
1073 binary_from(Some(binary.clone().into()), "unused"),
1074 Some(binary.canonicalize().unwrap())
1075 );
1076 assert_eq!(binary_from(Some("whisper-server".into()), "unused"), None);
1077 assert_eq!(binary_from(Some(directory.path().into()), "unused"), None);
1078 }
1079
1080 fn slot(kind: VoiceKind, engine: &'static str, endpoint: &str, generation: u64) -> Slot {
1081 Slot {
1082 model: kind.name().into(),
1083 engine,
1084 state: SlotState::Running,
1085 detail: "Ready".into(),
1086 progress: None,
1087 endpoint: Some(endpoint.into()),
1088 request_model: "mlx-community/Kokoro-82M-bf16".into(),
1089 protocol: Protocol::OpenAi,
1090 owned: false,
1091 child: None,
1092 cancel: CancellationToken::new(),
1093 generation,
1094 }
1095 }
1096
1097 #[test]
1098 fn limits_errors_ports_and_generations_are_exact() {
1099 assert_eq!(MAX_AUDIO_BYTES, 50_331_648);
1100 assert_eq!(MAX_TRANSCRIPT_BYTES, 262_144);
1101 assert_eq!(VoiceError("boom".into()).to_string(), "boom");
1102 let port = free_port().unwrap();
1103 assert!(port > 1);
1104 std::net::TcpListener::bind((std::net::Ipv4Addr::LOCALHOST, port)).unwrap();
1105 let home = tempfile::tempdir().unwrap();
1106 let service = VoiceService::new(home.path()).unwrap();
1107 assert_eq!(service.next_generation(), 1);
1108 assert_eq!(service.next_generation(), 2);
1109 assert_eq!(service.next_generation(), 3);
1110 }
1111
1112 #[test]
1113 fn updates_reach_only_the_current_generation() {
1114 let home = tempfile::tempdir().unwrap();
1115 let service = VoiceService::new(home.path()).unwrap();
1116 service
1117 .claim(
1118 VoiceKind::Listen,
1119 slot(VoiceKind::Listen, "attached", "http://127.0.0.1:9", 7),
1120 )
1121 .unwrap();
1122 service.update("listen", 6, |slot| slot.detail = "stale".into());
1123 assert_eq!(service.status()[0].detail, "Ready");
1124 service.update("listen", 7, |slot| slot.detail = "fresh".into());
1125 let status = service.status();
1126 assert_eq!(status[0].detail, "fresh");
1127 assert_eq!(status[0].kind, "listen");
1128 service
1129 .claim(
1130 VoiceKind::Speak,
1131 slot(VoiceKind::Speak, "attached", "http://127.0.0.1:9", 8),
1132 )
1133 .unwrap();
1134 assert_eq!(
1135 service
1136 .status()
1137 .iter()
1138 .map(|view| view.kind)
1139 .collect::<Vec<_>>(),
1140 ["listen", "speak"]
1141 );
1142 service.shutdown();
1143 assert!(service.status().is_empty());
1144 }
1145
1146 fn response(chunks: Vec<&'static [u8]>, length: bool) -> reqwest::Response {
1147 let total: usize = chunks.iter().map(|chunk| chunk.len()).sum();
1148 let body = reqwest::Body::wrap_stream(futures_util::stream::iter(
1149 chunks
1150 .into_iter()
1151 .map(|chunk| Ok::<_, std::io::Error>(bytes::Bytes::from_static(chunk))),
1152 ));
1153 let mut builder = http::Response::builder().status(200);
1154 if length {
1155 builder = builder.header("content-length", total.to_string());
1156 }
1157 reqwest::Response::from(builder.body(body).unwrap())
1158 }
1159
1160 #[tokio::test]
1161 async fn replies_are_bounded_by_length_and_by_streamed_bytes() {
1162 assert_eq!(
1163 bounded(response(vec![b"abcd"], true), 4).await.unwrap(),
1164 b"abcd"
1165 );
1166 assert!(bounded(response(vec![b"abcde"], true), 4).await.is_err());
1167 assert_eq!(
1168 bounded(response(vec![b"ab", b"cd"], false), 4)
1169 .await
1170 .unwrap(),
1171 b"abcd"
1172 );
1173 assert!(
1174 bounded(response(vec![b"ab", b"cde"], false), 4)
1175 .await
1176 .is_err()
1177 );
1178 assert!(
1179 bounded(response(vec![b"abc", b"de"], false), 4)
1180 .await
1181 .is_err()
1182 );
1183 }
1184
1185 async fn raw_reply(raw: &'static [u8]) -> reqwest::Response {
1187 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
1188 let address = listener.local_addr().unwrap();
1189 tokio::spawn(async move {
1190 let (mut socket, _) = listener.accept().await.unwrap();
1191 let mut buffer = vec![0u8; 4096];
1192 let _ = socket.read(&mut buffer).await;
1193 let _ = socket.write_all(raw).await;
1194 });
1195 reqwest::get(format!("http://{address}/")).await.unwrap()
1196 }
1197
1198 #[tokio::test]
1199 async fn a_declared_length_is_checked_before_reading() {
1200 let exact =
1201 raw_reply(b"HTTP/1.1 200 OK\r\nContent-Length: 4\r\nConnection: close\r\n\r\nabcd")
1202 .await;
1203 assert_eq!(bounded(exact, 4).await.unwrap(), b"abcd");
1204 let declared =
1206 raw_reply(b"HTTP/1.1 200 OK\r\nContent-Length: 5\r\nConnection: close\r\n\r\nab").await;
1207 assert_eq!(
1208 bounded(declared, 4).await.unwrap_err().0,
1209 "voice server reply exceeds the size limit"
1210 );
1211 }
1212
1213 #[tokio::test]
1214 async fn attach_refuses_fragments_queries_long_endpoints_and_long_models() {
1215 let home = tempfile::tempdir().unwrap();
1216 let service = VoiceService::new(home.path()).unwrap();
1217 let (base, _) = server(b"{\"data\":[]}".to_vec(), "application/json").await;
1218 for endpoint in [format!("{base}/?x=1"), format!("{base}/#top")] {
1219 assert_eq!(
1220 service
1221 .attach(VoiceKind::Speak, &endpoint, "kokoro")
1222 .await
1223 .unwrap_err()
1224 .0,
1225 "endpoint must be a plain loopback base URL",
1226 "{endpoint}"
1227 );
1228 }
1229 let longest = format!("{base}{}", "/".repeat(256 - base.len()));
1230 service
1231 .attach(VoiceKind::Speak, &longest, "kokoro")
1232 .await
1233 .unwrap();
1234 assert!(
1235 service
1236 .attach(VoiceKind::Speak, &format!("{longest}/"), "kokoro")
1237 .await
1238 .is_err()
1239 );
1240 let widest = format!("{}/{}", "a".repeat(100), "b".repeat(99));
1241 service
1242 .attach(VoiceKind::Speak, &base, &widest)
1243 .await
1244 .unwrap();
1245 assert!(
1246 service
1247 .attach(VoiceKind::Speak, &base, &format!("{widest}b"))
1248 .await
1249 .is_err()
1250 );
1251 }
1252
1253 #[tokio::test]
1254 async fn mlx_audio_requests_carry_the_voice_language() {
1255 let home = tempfile::tempdir().unwrap();
1256 let service = VoiceService::new(home.path()).unwrap();
1257 let (base, seen) = server(wav(8), "audio/wav").await;
1258 service
1259 .claim(
1260 VoiceKind::Speak,
1261 slot(VoiceKind::Speak, "mlx-audio", &base, 1),
1262 )
1263 .unwrap();
1264 service
1265 .speak(&request("Hello"), &catalog(), &CancellationToken::new())
1266 .await
1267 .unwrap();
1268 let sent =
1269 String::from_utf8_lossy(&seen.lock().unwrap().last().unwrap().clone()).into_owned();
1270 assert!(sent.contains("\"lang_code\":\"a\""), "{sent}");
1271 }
1272
1273 #[cfg(unix)]
1276 fn fake_engine(directory: &Path, delay: i32, post_status: u16) -> PathBuf {
1277 use std::os::unix::fs::PermissionsExt;
1278 let script = directory.join(format!("engine-{delay}-{post_status}"));
1279 let body = if delay < 0 {
1280 "#!/bin/sh\nexit 0\n".to_owned()
1281 } else {
1282 format!(
1283 r#"#!/bin/sh
1284exec python3 -c '
1285import sys, time, http.server
1286port = int(sys.argv[sys.argv.index("--port") + 1])
1287time.sleep({delay})
1288class H(http.server.BaseHTTPRequestHandler):
1289 def reply(self, code):
1290 self.send_response(code); self.send_header("Content-Length", "2"); self.end_headers(); self.wfile.write(b"{{}}")
1291 def do_GET(self): self.reply(200)
1292 def do_POST(self): self.reply({post_status})
1293 def log_message(self, *a): pass
1294http.server.HTTPServer(("127.0.0.1", port), H).serve_forever()
1295' x "$@"
1296"#
1297 )
1298 };
1299 std::fs::write(&script, body).unwrap();
1300 std::fs::set_permissions(&script, std::fs::Permissions::from_mode(0o755)).unwrap();
1301 script
1302 }
1303
1304 #[cfg(unix)]
1305 async fn settle(service: &VoiceService) -> SlotView {
1306 for _ in 0..150 {
1307 let view = service.status().remove(0);
1308 if view.state != SlotState::Starting {
1309 return view;
1310 }
1311 tokio::time::sleep(Duration::from_millis(100)).await;
1312 }
1313 panic!("the engine never settled");
1314 }
1315
1316 #[cfg(unix)]
1317 #[tokio::test]
1318 async fn launched_engines_become_ready_load_the_model_or_fail_clearly() {
1319 let home = tempfile::tempdir().unwrap();
1320 let kokoro = catalog().resolve("speak").unwrap().clone();
1321 let cancel = CancellationToken::new();
1322 for (delay, post, state, detail) in [
1323 (1, 200, SlotState::Running, "Ready"),
1324 (
1325 0,
1326 500,
1327 SlotState::Failed,
1328 "mlx-audio could not load the model",
1329 ),
1330 (
1331 -1,
1332 200,
1333 SlotState::Failed,
1334 "voice server exited while starting",
1335 ),
1336 ] {
1337 let service = Arc::new(VoiceService::new(home.path()).unwrap());
1338 let binary = fake_engine(home.path(), delay, post);
1339 service.begin(&kokoro, binary, &cancel).unwrap();
1340 let view = settle(&service).await;
1341 assert_eq!(
1342 (view.state, view.detail.as_str()),
1343 (state, detail),
1344 "{delay}"
1345 );
1346 assert!(view.owned);
1347 assert_eq!(view.engine, "mlx-audio");
1348 assert_eq!(view.endpoint.is_some(), state == SlotState::Running);
1349 if state == SlotState::Running {
1350 let endpoint = view.endpoint.unwrap();
1351 let port: u16 = endpoint.rsplit(':').next().unwrap().parse().unwrap();
1352 assert!(port > 1);
1353 }
1354 service.shutdown();
1355 }
1356 }
1357}