ryu_hardware/
codec.rs

1//! Audio bridging for the hardware session: Opus <-> PCM and PCM <-> WAV.
2//!
3//! The rest of Core speaks WAV/PCM (whisper transcribe, OuteTTS synth), while the
4//! device link speaks Opus (PROTOCOL.md §2):
5//!   - **Uplink** mic: Opus, mono, 16 kHz, 60 ms frames (960 samples/frame).
6//!   - **Downlink** TTS: Opus, mono, 24 kHz, 60 ms frames (1440 samples/frame).
7//!
8//! This module owns the codec edges so [`super::session`] only ever deals in
9//! decoded PCM / WAV bytes. Opus runs through `audiopus` (which vendors libopus
10//! via `audiopus_sys`, so no system libopus is required); WAV read/write runs
11//! through `hound`.
12
13use anyhow::{Context, Result};
14use opus::{Application, Channels, Decoder as OpusDecoder, Encoder as OpusEncoder};
15
16/// Mic uplink sample rate (Hz).
17pub const UPLINK_RATE: u32 = 16_000;
18/// TTS downlink sample rate (Hz).
19pub const DOWNLINK_RATE: u32 = 24_000;
20/// Frame duration in milliseconds (both directions).
21pub const FRAME_MS: u32 = 60;
22
23/// Samples per 60 ms downlink frame at 24 kHz (`24000 * 60 / 1000`).
24const DOWNLINK_FRAME_SAMPLES: usize = (DOWNLINK_RATE as usize * FRAME_MS as usize) / 1000;
25/// A decoded uplink Opus packet never exceeds 120 ms of 16 kHz mono audio; size
26/// the scratch buffer for that worst case so any conformant frame fits.
27const UPLINK_DECODE_CAP: usize = (UPLINK_RATE as usize * 120) / 1000;
28
29/// Opus decoder for the 16 kHz mono mic uplink. One per session (Opus decoder
30/// state is stateful across frames).
31pub struct UplinkDecoder {
32    decoder: OpusDecoder,
33}
34
35impl UplinkDecoder {
36    pub fn new() -> Result<Self> {
37        let decoder = OpusDecoder::new(UPLINK_RATE, Channels::Mono)
38            .context("creating 16 kHz Opus decoder")?;
39        Ok(Self { decoder })
40    }
41
42    /// Decode one uplink Opus packet to 16 kHz mono PCM (i16) samples.
43    pub fn decode(&mut self, packet: &[u8]) -> Result<Vec<i16>> {
44        let mut out = vec![0i16; UPLINK_DECODE_CAP];
45        let decoded = self
46            .decoder
47            .decode(packet, &mut out[..], false)
48            .context("decoding uplink Opus packet")?;
49        out.truncate(decoded);
50        Ok(out)
51    }
52}
53
54/// Opus encoder for the 24 kHz mono TTS downlink. One per TTS stream.
55pub struct DownlinkEncoder {
56    encoder: OpusEncoder,
57}
58
59impl DownlinkEncoder {
60    pub fn new() -> Result<Self> {
61        let encoder = OpusEncoder::new(DOWNLINK_RATE, Channels::Mono, Application::Voip)
62            .context("creating 24 kHz Opus encoder")?;
63        Ok(Self { encoder })
64    }
65
66    /// Encode 24 kHz mono PCM into a sequence of 60 ms Opus packets. The tail is
67    /// zero-padded to a full frame so the last words are never clipped.
68    pub fn encode_stream(&mut self, pcm: &[i16]) -> Result<Vec<Vec<u8>>> {
69        let mut packets = Vec::new();
70        let mut offset = 0;
71        // Reusable per-packet output buffer (Opus packets are well under 4 KiB at
72        // these bitrates; 4000 is the conventional max).
73        let mut scratch = [0u8; 4000];
74        while offset < pcm.len() {
75            let end = (offset + DOWNLINK_FRAME_SAMPLES).min(pcm.len());
76            let mut frame: Vec<i16> = pcm[offset..end].to_vec();
77            if frame.len() < DOWNLINK_FRAME_SAMPLES {
78                frame.resize(DOWNLINK_FRAME_SAMPLES, 0);
79            }
80            let n = self
81                .encoder
82                .encode(&frame, &mut scratch)
83                .context("encoding downlink Opus frame")?;
84            packets.push(scratch[..n].to_vec());
85            offset += DOWNLINK_FRAME_SAMPLES;
86        }
87        Ok(packets)
88    }
89}
90
91/// Wrap 16 kHz mono PCM (i16) as a RIFF/WAV byte blob for the whisper/meetings
92/// transcribe path, which takes WAV `file` bytes.
93pub fn pcm16_to_wav(pcm: &[i16], sample_rate: u32) -> Result<Vec<u8>> {
94    let spec = hound::WavSpec {
95        channels: 1,
96        sample_rate,
97        bits_per_sample: 16,
98        sample_format: hound::SampleFormat::Int,
99    };
100    let mut cursor = std::io::Cursor::new(Vec::<u8>::new());
101    {
102        let mut writer = hound::WavWriter::new(&mut cursor, spec).context("creating WAV writer")?;
103        for &sample in pcm {
104            writer.write_sample(sample).context("writing WAV sample")?;
105        }
106        writer.finalize().context("finalizing WAV")?;
107    }
108    Ok(cursor.into_inner())
109}
110
111/// Parsed PCM from a WAV blob (mono i16 at its native rate). Returned by
112/// [`wav_to_pcm16`] so the TTS WAV produced by OuteTTS can be re-encoded as Opus.
113pub struct DecodedWav {
114    pub samples: Vec<i16>,
115    pub sample_rate: u32,
116}
117
118/// Decode a WAV blob to mono i16 PCM. Down-mixes any multi-channel input and
119/// converts float samples to i16. The sample rate is read from the header so the
120/// caller can resample to the 24 kHz Opus downlink as needed.
121pub fn wav_to_pcm16(wav: &[u8]) -> Result<DecodedWav> {
122    let cursor = std::io::Cursor::new(wav);
123    let mut reader = hound::WavReader::new(cursor).context("parsing WAV header")?;
124    let spec = reader.spec();
125    let channels = spec.channels.max(1) as usize;
126
127    let interleaved: Vec<i32> = match spec.sample_format {
128        hound::SampleFormat::Int => reader
129            .samples::<i32>()
130            .collect::<std::result::Result<Vec<_>, _>>()
131            .context("reading int WAV samples")?,
132        hound::SampleFormat::Float => reader
133            .samples::<f32>()
134            .map(|s| s.map(|v| (v.clamp(-1.0, 1.0) * i16::MAX as f32) as i32))
135            .collect::<std::result::Result<Vec<_>, _>>()
136            .context("reading float WAV samples")?,
137    };
138
139    // Scale integer PCM down to 16-bit if the source was wider (e.g. 24/32-bit).
140    let shift = match spec.bits_per_sample {
141        0..=16 => 0,
142        bits => bits - 16,
143    };
144
145    let mut mono = Vec::with_capacity(interleaved.len() / channels.max(1));
146    for frame in interleaved.chunks(channels) {
147        let sum: i64 = frame.iter().map(|&s| (s >> shift) as i64).sum();
148        let avg = (sum / channels as i64).clamp(i16::MIN as i64, i16::MAX as i64);
149        mono.push(avg as i16);
150    }
151
152    Ok(DecodedWav {
153        samples: mono,
154        sample_rate: spec.sample_rate,
155    })
156}
157
158/// Linear-interpolation resampler to the 24 kHz Opus downlink rate. OuteTTS
159/// already emits 24 kHz mono, so this is a near-no-op fast path when rates match;
160/// it exists so a different TTS engine (a `?engine=` sidecar at another rate)
161/// still produces a correctly-pitched downlink rather than a chipmunk artifact.
162pub fn resample_to(pcm: &[i16], from_rate: u32, to_rate: u32) -> Vec<i16> {
163    if from_rate == to_rate || pcm.is_empty() {
164        return pcm.to_vec();
165    }
166    let ratio = to_rate as f64 / from_rate as f64;
167    let out_len = ((pcm.len() as f64) * ratio).round() as usize;
168    let mut out = Vec::with_capacity(out_len);
169    for i in 0..out_len {
170        let src_pos = i as f64 / ratio;
171        let idx = src_pos.floor() as usize;
172        let frac = src_pos - idx as f64;
173        let a = pcm.get(idx).copied().unwrap_or(0) as f64;
174        let b = pcm.get(idx + 1).copied().unwrap_or(a as i16) as f64;
175        out.push((a + (b - a) * frac).round() as i16);
176    }
177    out
178}
179
180#[cfg(test)]
181mod tests {
182    use super::*;
183
184    #[test]
185    fn wav_roundtrip_preserves_samples() {
186        let pcm: Vec<i16> = (0..480).map(|i| ((i * 137) % 1000 - 500) as i16).collect();
187        let wav = pcm16_to_wav(&pcm, UPLINK_RATE).unwrap();
188        let decoded = wav_to_pcm16(&wav).unwrap();
189        assert_eq!(decoded.sample_rate, UPLINK_RATE);
190        assert_eq!(decoded.samples, pcm);
191    }
192
193    #[test]
194    fn resample_is_noop_when_rates_match() {
195        let pcm = vec![1i16, 2, 3, 4];
196        assert_eq!(resample_to(&pcm, 24_000, 24_000), pcm);
197    }
198
199    #[test]
200    fn resample_doubles_length_when_rate_doubles() {
201        let pcm = vec![0i16, 100, 0, 100];
202        let out = resample_to(&pcm, 12_000, 24_000);
203        assert_eq!(out.len(), 8);
204    }
205
206    #[test]
207    fn opus_encode_then_decode_roundtrips_length() {
208        // 24 kHz: encode a 60 ms tone, decode it back through a 24 kHz decoder.
209        let mut enc = DownlinkEncoder::new().unwrap();
210        let tone: Vec<i16> = (0..DOWNLINK_FRAME_SAMPLES)
211            .map(|i| ((i as f64 * 0.2).sin() * 8000.0) as i16)
212            .collect();
213        let packets = enc.encode_stream(&tone).unwrap();
214        assert_eq!(packets.len(), 1);
215        assert!(!packets[0].is_empty());
216
217        let mut dec = OpusDecoder::new(DOWNLINK_RATE, Channels::Mono).unwrap();
218        let mut out = vec![0i16; DOWNLINK_FRAME_SAMPLES];
219        let n = dec.decode(&packets[0], &mut out[..], false).unwrap();
220        assert_eq!(n, DOWNLINK_FRAME_SAMPLES);
221    }
222}
ryu_hardware/codec.rs

ryu_hardware/
codec.rs
