crabcamera 0.9.2

Advanced cross-platform camera integration for Tauri applications
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
//! Opus audio encoder
//!
//! Encodes PCM audio into Opus packets suitable for MP4 muxing
//! with proper frame buffering and flush semantics.
//!
//! ## Properties
//!
//! - Accepts interleaved f32 PCM samples
//! - Outputs valid Opus packets (RFC 6716)
//! - Flush operation emits remaining packets
//! - Operates at 48kHz (Opus standard)
//! - No hidden resampling
//! - Maintains channel count

use super::capture::AudioFrame;
use crate::constants::{OPUS_APPLICATION_AUDIO, OPUS_FRAME_SAMPLES, OPUS_SAMPLE_RATE};
use crate::errors::CameraError;

/// Encoded Opus audio packet
#[derive(Debug, Clone)]
pub struct EncodedAudio {
    /// Raw Opus packet data
    pub data: Vec<u8>,
    /// Presentation timestamp in seconds
    pub timestamp: f64,
    /// Duration of this packet in seconds
    pub duration: f64,
}

/// Opus encoder for PCM to Opus conversion
///
/// # Thread Safety
/// This type implements `Send` to allow moving the encoder to a dedicated audio thread.
/// The underlying `libopus` encoder is NOT thread-safe for concurrent access, but IS safe
/// to use from a single thread after being moved there.
///
/// **Invariant:** Once created, an `OpusEncoder` must only be accessed from one thread
/// at a time. The current architecture enforces this by:
/// 1. Creating the encoder in `start_audio_capture()`
/// 2. Moving it into a dedicated audio thread via `std::thread::spawn(move || ...)`
/// 3. The encoder never escapes that thread until dropped
///
/// Do NOT implement `Clone` or `Sync` for this type.
pub struct OpusEncoder {
    encoder: *mut libopus_sys::OpusEncoder,
    channels: u16,
    sample_rate: u32,
    /// Buffer for accumulating samples until we have a full frame
    sample_buffer: Vec<f32>,
    /// Timestamp of the first sample in the buffer (set once, never updated)
    buffer_start_pts: Option<f64>,
    /// Total samples encoded (for PTS calculation)
    samples_encoded: u64,
    /// Running PTS accumulator in samples, stored as `f64` to avoid an
    /// `u64 -> f64` precision-loss cast when computing presentation timestamps.
    samples_encoded_f64: f64,
}

// SAFETY: OpusEncoder can be sent to another thread because:
// 1. The raw pointer `encoder` points to memory allocated by libopus
// 2. libopus encoders are safe to use from any single thread
// 3. We do NOT implement Sync, preventing concurrent access
// 4. The ownership model ensures only one thread accesses the encoder at a time
unsafe impl Send for OpusEncoder {}

impl OpusEncoder {
    /// Create a new Opus encoder
    ///
    /// # Arguments
    /// * `sample_rate` - Must be 48000 (Opus requirement)
    /// * `channels` - 1 for mono, 2 for stereo
    /// * `bitrate` - Target bitrate in bits per second (e.g., 128000)
    ///
    /// # Errors
    /// Returns a [`CameraError::AudioError`] if `sample_rate` is not the
    /// required Opus rate, if `channels` is not `1` or `2`, or if the
    /// underlying Opus encoder cannot be created.
    pub fn new(sample_rate: u32, channels: u16, bitrate: u32) -> Result<Self, CameraError> {
        if sample_rate != OPUS_SAMPLE_RATE {
            return Err(CameraError::AudioError(format!(
                "Opus requires {OPUS_SAMPLE_RATE} Hz sample rate"
            )));
        }

        if channels != 1 && channels != 2 {
            return Err(CameraError::AudioError(
                "Opus supports only mono (1) or stereo (2) channels".to_string(),
            ));
        }

        let sample_rate_i32 = i32::try_from(sample_rate)
            .map_err(|_| CameraError::AudioError("sample rate exceeds i32 range".to_string()))?;

        let mut error: i32 = 0;
        let encoder = unsafe {
            libopus_sys::opus_encoder_create(
                sample_rate_i32,
                i32::from(channels),
                OPUS_APPLICATION_AUDIO,
                &raw mut error,
            )
        };

        if encoder.is_null() || error != 0 {
            return Err(CameraError::AudioError(format!(
                "Failed to create Opus encoder: error code {error}"
            )));
        }

        // Set bitrate
        let bitrate_i32 = i32::try_from(bitrate)
            .map_err(|_| CameraError::AudioError("bitrate exceeds i32 range".to_string()))?;
        let bitrate_request =
            i32::try_from(libopus_sys::OPUS_SET_BITRATE_REQUEST).map_err(|_| {
                CameraError::AudioError("OPUS_SET_BITRATE_REQUEST exceeds i32".to_string())
            })?;
        let result =
            unsafe { libopus_sys::opus_encoder_ctl(encoder, bitrate_request, bitrate_i32) };

        if result != 0 {
            unsafe { libopus_sys::opus_encoder_destroy(encoder) };
            return Err(CameraError::AudioError(format!(
                "Failed to set bitrate: error code {result}"
            )));
        }

        Ok(Self {
            encoder,
            channels,
            sample_rate,
            sample_buffer: Vec::with_capacity(OPUS_FRAME_SAMPLES * channels as usize * 2),
            buffer_start_pts: None,
            samples_encoded: 0,
            samples_encoded_f64: 0.0,
        })
    }

    /// Encode an audio frame.
    ///
    /// May return empty vec if not enough samples accumulated for a full Opus frame.
    /// May return multiple packets if input contains multiple frames worth of samples.
    ///
    /// # Errors
    ///
    /// * `CameraError::AudioError`: If sample rate/channels don't match or encoding fails.
    pub fn encode(&mut self, frame: &AudioFrame) -> Result<Vec<EncodedAudio>, CameraError> {
        // Validate input
        if frame.sample_rate != self.sample_rate {
            return Err(CameraError::AudioError(format!(
                "Sample rate mismatch: expected {}, got {}",
                self.sample_rate, frame.sample_rate
            )));
        }

        if frame.channels != self.channels {
            return Err(CameraError::AudioError(format!(
                "Channel count mismatch: expected {}, got {}",
                self.channels, frame.channels
            )));
        }

        // Track PTS of first sample in buffer
        if self.buffer_start_pts.is_none() && !frame.samples.is_empty() {
            self.buffer_start_pts = Some(frame.timestamp);
        }

        // Add samples to buffer
        self.sample_buffer.extend_from_slice(&frame.samples);

        // Encode complete frames
        let mut encoded_packets = Vec::new();
        let samples_per_frame = OPUS_FRAME_SAMPLES * self.channels as usize;

        // Use f64::from for safe lossless casting where possible
        let sample_rate_f64 = f64::from(self.sample_rate);
        let opus_samples_f64 =
            f64::from(u32::try_from(OPUS_FRAME_SAMPLES).map_err(|_| {
                CameraError::AudioError("OPUS_FRAME_SAMPLES exceeds u32".to_string())
            })?);
        let frame_duration = opus_samples_f64 / sample_rate_f64;

        while self.sample_buffer.len() >= samples_per_frame {
            let frame_samples: Vec<f32> = self.sample_buffer.drain(..samples_per_frame).collect();

            // Calculate PTS for this frame
            let pts = self.samples_encoded_f64 / sample_rate_f64;

            // Encode to Opus
            let mut output = vec![0u8; 4000]; // Max Opus packet size
            let frame_samples_i32 = i32::try_from(OPUS_FRAME_SAMPLES).map_err(|_| {
                CameraError::AudioError("OPUS_FRAME_SAMPLES exceeds i32".to_string())
            })?;
            let max_bytes = i32::try_from(output.len()).map_err(|_| {
                CameraError::AudioError("output buffer length exceeds i32".to_string())
            })?;
            let len = unsafe {
                libopus_sys::opus_encode_float(
                    self.encoder,
                    frame_samples.as_ptr(),
                    frame_samples_i32,
                    output.as_mut_ptr(),
                    max_bytes,
                )
            };

            if len < 0 {
                return Err(CameraError::AudioError(format!(
                    "Opus encoding failed: error code {len}"
                )));
            }

            output.truncate(usize::try_from(len).unwrap_or(0));

            encoded_packets.push(EncodedAudio {
                data: output,
                timestamp: self.buffer_start_pts.unwrap_or(0.0) + pts,
                duration: frame_duration,
            });

            self.samples_encoded += u64::try_from(OPUS_FRAME_SAMPLES).map_err(|_| {
                CameraError::AudioError("OPUS_FRAME_SAMPLES exceeds u64".to_string())
            })?;
            self.samples_encoded_f64 +=
                f64::from(u32::try_from(OPUS_FRAME_SAMPLES).map_err(|_| {
                    CameraError::AudioError("OPUS_FRAME_SAMPLES exceeds u32".to_string())
                })?);
        }

        // NOTE: Do NOT update buffer_start_pts here. The samples_encoded counter
        // already tracks absolute position from recording start. Updating
        // buffer_start_pts would cause double-counting of timestamps.

        Ok(encoded_packets)
    }

    /// Flush remaining samples.
    ///
    /// Call this when recording ends to encode any remaining buffered samples.
    ///
    /// # Errors
    ///
    /// * `CameraError::AudioError`: If encoding fails.
    pub fn flush(&mut self) -> Result<Vec<EncodedAudio>, CameraError> {
        if self.sample_buffer.is_empty() {
            return Ok(Vec::new());
        }

        // Pad to full frame size
        let samples_per_frame = OPUS_FRAME_SAMPLES * self.channels as usize;
        let padding_needed = samples_per_frame - (self.sample_buffer.len() % samples_per_frame);
        if padding_needed < samples_per_frame {
            self.sample_buffer.extend(vec![0.0f32; padding_needed]);
        }

        // Encode remaining
        let mut encoded_packets = Vec::new();
        // Use f64::from for safe lossless casting where possible
        let sample_rate_f64 = f64::from(self.sample_rate);
        let opus_samples_f64 =
            f64::from(u32::try_from(OPUS_FRAME_SAMPLES).map_err(|_| {
                CameraError::AudioError("OPUS_FRAME_SAMPLES exceeds u32".to_string())
            })?);
        let frame_duration = opus_samples_f64 / sample_rate_f64;

        while self.sample_buffer.len() >= samples_per_frame {
            let frame_samples: Vec<f32> = self.sample_buffer.drain(..samples_per_frame).collect();
            // Calculate PTS for this frame
            let pts = self.samples_encoded_f64 / sample_rate_f64;

            let mut output = vec![0u8; 4000];
            let frame_samples_i32 = i32::try_from(OPUS_FRAME_SAMPLES).map_err(|_| {
                CameraError::AudioError("OPUS_FRAME_SAMPLES exceeds i32".to_string())
            })?;
            let max_bytes = i32::try_from(output.len()).map_err(|_| {
                CameraError::AudioError("output buffer length exceeds i32".to_string())
            })?;
            let len = unsafe {
                libopus_sys::opus_encode_float(
                    self.encoder,
                    frame_samples.as_ptr(),
                    frame_samples_i32,
                    output.as_mut_ptr(),
                    max_bytes,
                )
            };

            if len < 0 {
                return Err(CameraError::AudioError(format!(
                    "Opus flush failed: error code {len}"
                )));
            }

            output.truncate(usize::try_from(len).unwrap_or(0));

            encoded_packets.push(EncodedAudio {
                data: output,
                timestamp: self.buffer_start_pts.unwrap_or(0.0) + pts,
                duration: frame_duration,
            });

            self.samples_encoded += u64::try_from(OPUS_FRAME_SAMPLES).map_err(|_| {
                CameraError::AudioError("OPUS_FRAME_SAMPLES exceeds u64".to_string())
            })?;
            self.samples_encoded_f64 +=
                f64::from(u32::try_from(OPUS_FRAME_SAMPLES).map_err(|_| {
                    CameraError::AudioError("OPUS_FRAME_SAMPLES exceeds u32".to_string())
                })?);
        }

        Ok(encoded_packets)
    }

    /// Get the configured sample rate
    pub fn sample_rate(&self) -> u32 {
        self.sample_rate
    }

    /// Get the configured channel count
    pub fn channels(&self) -> u16 {
        self.channels
    }
}

impl Drop for OpusEncoder {
    fn drop(&mut self) {
        if !self.encoder.is_null() {
            unsafe {
                libopus_sys::opus_encoder_destroy(self.encoder);
            }
        }
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn test_encoder_creation() {
        let encoder = OpusEncoder::new(48000, 2, 128_000);
        assert!(encoder.is_ok());
    }

    #[test]
    fn test_encoder_rejects_wrong_sample_rate() {
        let encoder = OpusEncoder::new(44100, 2, 128_000);
        assert!(encoder.is_err());
    }

    #[test]
    fn test_encoder_rejects_wrong_channels() {
        let encoder = OpusEncoder::new(48000, 5, 128_000);
        assert!(encoder.is_err());
    }

    #[test]
    fn test_encode_full_frame() {
        let mut encoder = OpusEncoder::new(48000, 2, 128_000).expect("create Opus encoder");

        // Create a full frame worth of stereo samples (960 samples * 2 channels)
        let frame = AudioFrame {
            samples: vec![0.0f32; OPUS_FRAME_SAMPLES * 2],
            sample_rate: 48000,
            channels: 2,
            timestamp: 0.0,
        };

        let encoded_packets = encoder.encode(&frame).expect("encode full frame");
        assert_eq!(encoded_packets.len(), 1);
        assert!(!encoded_packets[0].data.is_empty());
    }

    #[test]
    fn test_encode_partial_frame() {
        let mut encoder = OpusEncoder::new(48000, 2, 128_000).expect("create Opus encoder");

        // Less than a full frame
        let frame = AudioFrame {
            samples: vec![0.0f32; 100],
            sample_rate: 48000,
            channels: 2,
            timestamp: 0.0,
        };

        let encoded_packets = encoder.encode(&frame).expect("encode partial frame");
        assert!(
            encoded_packets.is_empty(),
            "Partial frame should not produce output"
        );
    }

    #[test]
    fn test_flush_remaining() {
        let mut encoder = OpusEncoder::new(48000, 2, 128_000).expect("create Opus encoder");

        // Add partial frame
        let frame = AudioFrame {
            samples: vec![0.0f32; 100],
            sample_rate: 48000,
            channels: 2,
            timestamp: 0.0,
        };
        encoder.encode(&frame).expect("encode partial frame");

        // Flush should produce output
        let flushed = encoder.flush().expect("flush encoder");
        assert_eq!(flushed.len(), 1);
    }
}