Skip to main content

videocall_cli/producers/
camera.rs

1/*
2 * Copyright 2025 Security Union LLC
3 *
4 * Licensed under either of
5 *
6 * * Apache License, Version 2.0
7 *   (http://www.apache.org/licenses/LICENSE-2.0)
8 * * MIT license
9 *   (http://opensource.org/licenses/MIT)
10 *
11 * at your option.
12 *
13 * Unless you explicitly state otherwise, any contribution intentionally
14 * submitted for inclusion in the work by you, as defined in the Apache-2.0
15 * license, shall be dual licensed as above, without any additional terms or
16 * conditions.
17 */
18
19use crate::cli_args::IndexKind;
20use crate::video_encoder::Frame;
21use anyhow::Result;
22use protobuf::Message;
23use std::sync::atomic::AtomicBool;
24use std::sync::Arc;
25use std::thread::JoinHandle;
26use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
27use tokio::sync::mpsc::{self, Sender};
28use tracing::{debug, error, info};
29use videocall_nokhwa::pixel_format::I420Format;
30use videocall_nokhwa::utils::RequestedFormat;
31use videocall_nokhwa::utils::RequestedFormatType;
32use videocall_nokhwa::{
33    utils::{ApiBackend, CameraFormat, CameraIndex, FrameFormat},
34    Camera,
35};
36
37use videocall_types::protos::media_packet::media_packet::MediaType;
38use videocall_types::protos::media_packet::{MediaPacket, VideoCodec, VideoMetadata};
39use videocall_types::protos::packet_wrapper::{packet_wrapper::PacketType, PacketWrapper};
40
41use super::encoder_thread::encoder_thread;
42use super::producer::Producer;
43
44pub struct CameraPacket {
45    pub data: Vec<u8>,
46    pub _format: FrameFormat,
47    pub age: u128,
48}
49
50impl CameraPacket {
51    pub fn new(data: Vec<u8>, format: FrameFormat, age: u128) -> CameraPacket {
52        CameraPacket {
53            data,
54            _format: format,
55            age,
56        }
57    }
58}
59
60pub fn transform_video_chunk(frame: &Frame, user_id: &str) -> PacketWrapper {
61    let frame_type = if frame.key {
62        "key".to_string()
63    } else {
64        "delta".to_string()
65    };
66    let user_id_bytes = user_id.as_bytes();
67    let media_packet: MediaPacket = MediaPacket {
68        data: frame.data.to_vec(),
69        frame_type,
70        user_id: user_id_bytes.to_vec(),
71        media_type: MediaType::VIDEO.into(),
72        timestamp: since_the_epoch().as_micros() as f64,
73        video_metadata: Some(VideoMetadata {
74            sequence: frame.pts as u64,
75            codec: VideoCodec::VP9_PROFILE0_LEVEL10_8BIT.into(),
76            ..Default::default()
77        })
78        .into(),
79        ..Default::default()
80    };
81    let data = media_packet.write_to_bytes().unwrap();
82    PacketWrapper {
83        data,
84        user_id: user_id_bytes.to_vec(),
85        packet_type: PacketType::MEDIA.into(),
86        ..Default::default()
87    }
88}
89
90pub static THRESHOLD_MILLIS: u128 = 1000;
91
92pub fn since_the_epoch() -> Duration {
93    SystemTime::now().duration_since(UNIX_EPOCH).unwrap()
94}
95
96#[derive(Clone, Debug)]
97pub struct CameraConfig {
98    pub width: u32,
99    pub height: u32,
100    pub framerate: u32,
101    pub video_device_index: IndexKind,
102    pub frame_format: FrameFormat,
103    pub bitrate_kbps: u32,
104    pub cpu_used: u8,
105}
106
107pub struct CameraDaemon {
108    config: CameraConfig,
109    user_id: String,
110    cam_rx: Option<mpsc::Receiver<Option<CameraPacket>>>,
111    cam_tx: Arc<mpsc::Sender<Option<CameraPacket>>>,
112    wt_tx: Sender<Vec<u8>>,
113    quit: Arc<AtomicBool>,
114    handles: Vec<JoinHandle<()>>,
115}
116
117impl Producer for CameraDaemon {
118    fn start(&mut self) -> Result<()> {
119        self.handles.push(self.camera_thread()?);
120        let encoder = encoder_thread(
121            self.cam_rx.take().unwrap(),
122            self.wt_tx.clone(),
123            self.quit.clone(),
124            self.config.clone(),
125            self.user_id.clone(),
126        );
127        self.handles.push(encoder);
128        Ok(())
129    }
130
131    fn stop(&mut self) -> Result<()> {
132        self.quit.store(true, std::sync::atomic::Ordering::Relaxed);
133        for handle in self.handles.drain(..) {
134            handle.join().unwrap();
135        }
136        Ok(())
137    }
138}
139
140impl CameraDaemon {
141    pub fn from_config(
142        config: CameraConfig,
143        user_id: String,
144        wt_tx: Sender<Vec<u8>>,
145    ) -> CameraDaemon {
146        let (cam_tx, cam_rx) = mpsc::channel(100);
147        CameraDaemon {
148            config,
149            user_id,
150            cam_rx: Some(cam_rx),
151            cam_tx: Arc::new(cam_tx),
152            quit: Arc::new(AtomicBool::new(false)),
153            handles: vec![],
154            wt_tx,
155        }
156    }
157
158    fn camera_thread(&self) -> Result<JoinHandle<()>> {
159        let devices = videocall_nokhwa::query(ApiBackend::Auto)?;
160        for (i, camera_info) in devices.iter().enumerate() {
161            info!("AVAILABLE CAMERA DEVICE INDEX {}: {:?}", i, camera_info);
162        }
163        let cam_tx = self.cam_tx.clone();
164        let width = self.config.width;
165        let height = self.config.height;
166        let framerate = self.config.framerate;
167        let frame_format = self.config.frame_format;
168        let video_device_index = match &self.config.video_device_index {
169            IndexKind::String(s) => CameraIndex::String(s.clone()),
170            IndexKind::Index(i) => CameraIndex::Index(*i),
171        };
172        let quit = self.quit.clone();
173        Ok(std::thread::spawn(move || {
174            debug!("Camera opened... waiting for frames");
175            let mut camera = match Camera::new(
176                video_device_index,
177                RequestedFormat::new::<I420Format>(RequestedFormatType::Exact(
178                    CameraFormat::new_from(width, height, frame_format, framerate),
179                )),
180            ) {
181                Ok(camera) => camera,
182                Err(e) => camera_setup_failed("Failed to open camera", e),
183            };
184            let actual_resolution = camera.resolution();
185            if let Err(e) = camera.open_stream() {
186                camera_setup_failed("Failed to start camera stream", e);
187            }
188            println!("Requested format: {frame_format:?}");
189            println!("Actual stream format: {:?}", camera.frame_format());
190
191            // Allocate buffer for raw data based on actual format
192            let mut i420_image_buffer = vec![
193                0u8;
194                buffer_size_i420(actual_resolution.width(), actual_resolution.height())
195                    as usize
196            ];
197
198            let frame_time = Duration::from_millis(1000u64 / framerate as u64);
199            let mut last_frame_time = Instant::now();
200            // A single decode failure can be a transient/malformed frame and
201            // shouldn't kill the call; a sustained run means the stream is
202            // unusable (e.g. a format/size mismatch), so bail out cleanly.
203            let mut consecutive_decode_failures = 0u32;
204            const MAX_CONSECUTIVE_DECODE_FAILURES: u32 = 30;
205            loop {
206                // use last_frame_time to calculate if we should skip this frame
207                let elapsed = last_frame_time.elapsed();
208                if elapsed < frame_time {
209                    continue;
210                }
211                last_frame_time = Instant::now();
212                let frame = camera.frame().unwrap();
213                if let Err(e) = frame.decode_image_to_buffer::<I420Format>(&mut i420_image_buffer) {
214                    consecutive_decode_failures += 1;
215                    error!(
216                        "Failed to decode camera frame (failure {consecutive_decode_failures}/{MAX_CONSECUTIVE_DECODE_FAILURES}): {e}"
217                    );
218                    if consecutive_decode_failures >= MAX_CONSECUTIVE_DECODE_FAILURES {
219                        camera_setup_failed("Camera frame decoding failed repeatedly", e);
220                    }
221                    continue;
222                }
223                consecutive_decode_failures = 0;
224                // Check if we should quit
225                if quit.load(std::sync::atomic::Ordering::Relaxed) {
226                    info!("Quit signal received, exiting frame loop.");
227                    return;
228                }
229
230                // Try sending the frame over the channel
231                if let Err(e) = cam_tx.try_send(Some(CameraPacket::new(
232                    i420_image_buffer.clone(),
233                    frame_format,
234                    since_the_epoch().as_millis(),
235                ))) {
236                    error!("Error sending image: {}", e);
237                }
238            }
239        }))
240    }
241}
242
243pub fn buffer_size_i420(width: u32, height: u32) -> u32 {
244    width
245        .checked_mul(height)
246        .and_then(|y_size| y_size.checked_add(y_size / 2)) // Total size = Y + U + V
247        .expect("Buffer size calculation overflowed")
248}
249
250/// Report a fatal camera-setup failure and terminate the process.
251///
252/// Camera setup runs on a dedicated thread, so a panic here would only unwind
253/// that thread and leave the process half-alive. Instead we print the (already
254/// user-facing) error plus a pointer to the `info` command and exit non-zero so
255/// the failure is visible and the process stops cleanly.
256fn camera_setup_failed(context: &str, error: impl std::fmt::Display) -> ! {
257    eprintln!("{context}: {error}");
258    eprintln!(
259        "Run `videocall-cli info --list-formats <camera-index>` to see the resolutions, \
260         frame formats, and frame rates this camera supports."
261    );
262    std::process::exit(1);
263}